Jarian Cottingham
0fa94d3ee3
Added comprehensive logging to diagnose 'Expecting value: line 1 column 1 (char 0)' errors in AI service responses. The changes include detailed logging of AI requests/responses, cache operations, and article processing steps to better identify when the AI service returns empty or invalid responses.
97 lines
2.9 KiB
Python
97 lines
2.9 KiB
Python
"""
|
|
Main entry point for the AI Processor.
|
|
Handles the orchestration of article processing and fact extraction.
|
|
"""
|
|
|
|
import logging
|
|
import sys
|
|
import os
|
|
from datetime import datetime
|
|
|
|
from article_processor import ArticleProcessor
|
|
from metrics_collector import metrics_collector
|
|
from cache_manager import CacheManager
|
|
from config import CACHE_FILE, LOG_FILE, LOG_LEVEL
|
|
|
|
# Setup logging
|
|
logging.basicConfig(
|
|
level=getattr(logging, LOG_LEVEL.upper(), logging.INFO),
|
|
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
|
|
handlers=[
|
|
logging.FileHandler(LOG_FILE),
|
|
logging.StreamHandler()
|
|
]
|
|
)
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
def setup_logging():
|
|
"""Setup logging configuration."""
|
|
# Ensure log directory exists
|
|
log_dir = os.path.dirname(LOG_FILE)
|
|
if log_dir:
|
|
os.makedirs(log_dir, exist_ok=True)
|
|
|
|
# Ensure output directory exists for cache files
|
|
output_dir = os.path.dirname(CACHE_FILE)
|
|
if output_dir:
|
|
os.makedirs(output_dir, exist_ok=True)
|
|
|
|
def main():
|
|
"""Main function to run the AI processor."""
|
|
logger.info("Starting AI Processor")
|
|
|
|
try:
|
|
# Setup logging
|
|
setup_logging()
|
|
|
|
# Create processor instance
|
|
processor = ArticleProcessor()
|
|
|
|
# Process all articles
|
|
logger.info("Starting article processing...")
|
|
stats = processor.process_all_articles()
|
|
|
|
# Log final statistics
|
|
logger.info("Processing completed")
|
|
logger.info(f"Total processed: {stats['total_processed']}")
|
|
logger.info(f"Total failed: {stats['total_failed']}")
|
|
logger.info(f"Duration: {stats['duration']:.2f} seconds")
|
|
if 'cache_stats' in stats:
|
|
logger.info(f"Cache stats: {stats['cache_stats']}")
|
|
else:
|
|
logger.info("No cache stats available")
|
|
|
|
# Print metrics summary
|
|
metrics_summary = metrics_collector.get_metrics_summary()
|
|
logger.info(f"Metrics summary: {metrics_summary}")
|
|
|
|
logger.info("AI Processor completed successfully")
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error in main function: {e}")
|
|
raise
|
|
|
|
def process_new_articles():
|
|
"""Process only new articles (for real-time processing)."""
|
|
logger.info("Starting real-time processing of new articles")
|
|
|
|
try:
|
|
processor = ArticleProcessor()
|
|
stats = processor.process_new_articles()
|
|
|
|
logger.info("Real-time processing completed")
|
|
logger.info(f"Total processed: {stats['total_processed']}")
|
|
logger.info(f"Total failed: {stats['total_failed']}")
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error in real-time processing: {e}")
|
|
raise
|
|
|
|
if __name__ == "__main__":
|
|
# Check if we're running with specific arguments
|
|
if len(sys.argv) > 1 and sys.argv[1] == "new":
|
|
process_new_articles()
|
|
else:
|
|
main()
|