Jarian Cottingham
1271f0b21b
- Remove agent/agent.md (dev-time agent context dumps), .DS_Store, committed venv configs (pyvenv.cfg), 0-byte runtime cache - Remove hardcoded /home/userpath from cron_scraper feed lookup - Replace ad-hoc test_implementation.py with pytest tests/test_scraper_cache.py - ruff clean (33 fixes: bare excepts, unused Config, whitespace) - Root pyproject.toml (activates shared Gitea CI), MIT LICENSE, README Tests
95 lines
2.7 KiB
Python
95 lines
2.7 KiB
Python
"""
|
|
Main entry point for the AI Processor.
|
|
Handles the orchestration of article processing and fact extraction.
|
|
"""
|
|
|
|
import logging
|
|
import sys
|
|
import os
|
|
|
|
from article_processor import ArticleProcessor
|
|
from metrics_collector import metrics_collector
|
|
from config import CACHE_FILE, LOG_FILE, LOG_LEVEL
|
|
|
|
# Setup logging
|
|
logging.basicConfig(
|
|
level=getattr(logging, LOG_LEVEL.upper(), logging.INFO),
|
|
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
|
|
handlers=[
|
|
logging.FileHandler(LOG_FILE),
|
|
logging.StreamHandler()
|
|
]
|
|
)
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
def setup_logging():
|
|
"""Setup logging configuration."""
|
|
# Ensure log directory exists
|
|
log_dir = os.path.dirname(LOG_FILE)
|
|
if log_dir:
|
|
os.makedirs(log_dir, exist_ok=True)
|
|
|
|
# Ensure output directory exists for cache files
|
|
output_dir = os.path.dirname(CACHE_FILE)
|
|
if output_dir:
|
|
os.makedirs(output_dir, exist_ok=True)
|
|
|
|
def main():
|
|
"""Main function to run the AI processor."""
|
|
logger.info("Starting AI Processor")
|
|
|
|
try:
|
|
# Setup logging
|
|
setup_logging()
|
|
|
|
# Create processor instance
|
|
processor = ArticleProcessor()
|
|
|
|
# Process all articles
|
|
logger.info("Starting article processing...")
|
|
stats = processor.process_all_articles()
|
|
|
|
# Log final statistics
|
|
logger.info("Processing completed")
|
|
logger.info(f"Total processed: {stats['total_processed']}")
|
|
logger.info(f"Total failed: {stats['total_failed']}")
|
|
logger.info(f"Duration: {stats['duration']:.2f} seconds")
|
|
if 'cache_stats' in stats:
|
|
logger.info(f"Cache stats: {stats['cache_stats']}")
|
|
else:
|
|
logger.info("No cache stats available")
|
|
|
|
# Print metrics summary
|
|
metrics_summary = metrics_collector.get_metrics_summary()
|
|
logger.info(f"Metrics summary: {metrics_summary}")
|
|
|
|
logger.info("AI Processor completed successfully")
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error in main function: {e}")
|
|
raise
|
|
|
|
def process_new_articles():
|
|
"""Process only new articles (for real-time processing)."""
|
|
logger.info("Starting real-time processing of new articles")
|
|
|
|
try:
|
|
processor = ArticleProcessor()
|
|
stats = processor.process_new_articles()
|
|
|
|
logger.info("Real-time processing completed")
|
|
logger.info(f"Total processed: {stats['total_processed']}")
|
|
logger.info(f"Total failed: {stats['total_failed']}")
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error in real-time processing: {e}")
|
|
raise
|
|
|
|
if __name__ == "__main__":
|
|
# Check if we're running with specific arguments
|
|
if len(sys.argv) > 1 and sys.argv[1] == "new":
|
|
process_new_articles()
|
|
else:
|
|
main()
|