Fix cache manager to properly handle empty cache files and add diagnostic logging

ai_processor/article_processor.py

@@ -3,20 +3,21 @@ Article processor for handling the processing of articles from the scraper direc
 Manages batching, processing, and integration with the fact extraction system.
 """
 
-import os
 import json
 import logging
+import os
 import time
 from datetime import datetime
 from typing import List, Tuple
 
-from fact_extractor import FactExtractor
 from cache_manager import CacheManager
-from metrics_collector import metrics_collector
 from config import BATCH_SIZE, CACHE_FILE
+from fact_extractor import FactExtractor
+from metrics_collector import metrics_collector
 
 logger = logging.getLogger(__name__)
 
+
 class ArticleProcessor:
     """Processes articles from the scraper directory and extracts facts."""
 
@@ -25,7 +26,9 @@ class ArticleProcessor:
         self.cache_manager = CacheManager(CACHE_FILE)
         self.batch_size = BATCH_SIZE
 
-    def find_unprocessed_articles(self, scraper_dir: str = "../scraper/articles") -> List[Tuple[str, str]]:
+    def find_unprocessed_articles(
+        self, scraper_dir: str = "../scraper/articles"
+    ) -> List[Tuple[str, str]]:
         """
         Find all unprocessed articles in the scraper directory.
 
@@ -47,33 +50,78 @@ class ArticleProcessor:
                     "../scraper/articles",
                     "/scraper/articles",
                     "/app/articles",
-                    "/articles"
+                    "/articles",
                 ]
                 for alt_path in alternative_paths:
                     if os.path.exists(alt_path):
-                        logger.info(f"Found articles directory at alternative path: {alt_path}")
+                        logger.info(
+                            f"Found articles directory at alternative path: {alt_path}"
+                        )
                         scraper_dir = alt_path
                         break
                 else:
                     return []
 
+            # Log cache state before scanning
+            cache_stats = self.cache_manager.get_cache_stats()
+            logger.info(
+                f"Cache state before scanning: {cache_stats['processed_files']} files marked as processed"
+            )
+
             logger.info(f"Directory exists, walking through files...")
             file_count = 0
+            article_file_count = 0
+            already_processed_count = 0
+            extension_counts = {}
+
             for root, dirs, files in os.walk(scraper_dir):
                 for file in files:
                     file_count += 1
-                    # Check for JSON files (expected format)
-                    if file.endswith('.json'):
-                        file_path = os.path.join(root, file)
-                        if not self.cache_manager.is_processed(file_path):
-                            unprocessed_articles.append((file_path, file))
-                    # Also check for text files (fallback for different formats)
-                    elif file.endswith(('.txt', '.md')):
-                        file_path = os.path.join(root, file)
-                        if not self.cache_manager.is_processed(file_path):
-                            unprocessed_articles.append((file_path, file))
 
-            logger.info(f"Scanned {file_count} files, found {len(unprocessed_articles)} unprocessed articles")
+                    # Track file extensions for debugging
+                    ext = (
+                        os.path.splitext(file)[1].lower()
+                        if "." in file
+                        else "(no extension)"
+                    )
+                    extension_counts[ext] = extension_counts.get(ext, 0) + 1
+
+                    # Check for JSON files (expected format)
+                    if file.endswith(".json"):
+                        article_file_count += 1
+                        file_path = os.path.join(root, file)
+                        if not self.cache_manager.is_processed(file_path):
+                            unprocessed_articles.append((file_path, file))
+                        else:
+                            already_processed_count += 1
+                    # Also check for text files (fallback for different formats)
+                    elif file.endswith((".txt", ".md")):
+                        article_file_count += 1
+                        file_path = os.path.join(root, file)
+                        if not self.cache_manager.is_processed(file_path):
+                            unprocessed_articles.append((file_path, file))
+                        else:
+                            already_processed_count += 1
+
+            # Log detailed breakdown
+            logger.info(f"File extension breakdown: {extension_counts}")
+            logger.info(
+                f"Scanned {file_count} total files, {article_file_count} are article files (.json/.txt/.md)"
+            )
+            logger.info(
+                f"Already processed (in cache): {already_processed_count}, Unprocessed: {len(unprocessed_articles)}"
+            )
+
+            if (
+                article_file_count > 0
+                and len(unprocessed_articles) == 0
+                and already_processed_count == article_file_count
+            ):
+                logger.warning(
+                    f"All {article_file_count} article files are marked as processed in cache. "
+                    f"If cache should be empty, check cache file: {self.cache_manager.cache_file}"
+                )
+
             return unprocessed_articles
 
         except Exception as e:
@@ -92,20 +140,20 @@ class ArticleProcessor:
             Dictionary containing the extracted facts or None if failed
         """
         try:
-            with open(file_path, 'r', encoding='utf-8') as f:
+            with open(file_path, "r", encoding="utf-8") as f:
                 article_data = json.load(f)
 
             # Extract facts from the article
             facts = self.fact_extractor.extract_facts_from_article(
-                article_data.get('original_content', ''), 
+                article_data.get("original_content", ""),
-                article_data.get('title', filename)
+                article_data.get("title", filename),
             )
 
             # Add metadata
-            facts['source'] = article_data.get('source', 'Unknown')
+            facts["source"] = article_data.get("source", "Unknown")
-            facts['published'] = article_data.get('published', 'Unknown')
+            facts["published"] = article_data.get("published", "Unknown")
-            facts['filename'] = filename
+            facts["filename"] = filename
-            facts['processed_at'] = datetime.now().isoformat()
+            facts["processed_at"] = datetime.now().isoformat()
 
             # Mark as processed in cache
             self.cache_manager.mark_processed(file_path)
@@ -174,17 +222,19 @@ class ArticleProcessor:
                 "total_processed": 0,
                 "total_failed": 0,
                 "duration": 0,
-                "status": "no_new_articles"
+                "status": "no_new_articles",
             }
 
-        logger.info(f"Starting to process {len(unprocessed_articles)} articles in batches of {self.batch_size}")
+        logger.info(
+            f"Starting to process {len(unprocessed_articles)} articles in batches of {self.batch_size}"
+        )
 
         total_processed = 0
         total_failed = 0
 
         # Process articles in batches
         for i in range(0, len(unprocessed_articles), self.batch_size):
-            batch = unprocessed_articles[i:i + self.batch_size]
+            batch = unprocessed_articles[i : i + self.batch_size]
             successful, failed = self.process_batch(batch)
             total_processed += successful
             total_failed += failed
@@ -205,7 +255,7 @@ class ArticleProcessor:
             "duration": duration,
             "batch_size": self.batch_size,
             "cache_stats": self.cache_manager.get_cache_stats(),
-            "status": "completed"
+            "status": "completed",
         }
 
         logger.info(f"Processing completed in {duration:.2f} seconds")

ai_processor/cache_manager.py

@@ -3,8 +3,8 @@ Cache manager for tracking processed articles to avoid reprocessing.
 """
 
 import json
-import os
 import logging
+import os
 from datetime import datetime
 from typing import Dict, List, Optional
 
@@ -12,6 +12,7 @@ from config import CACHE_FILE
 
 logger = logging.getLogger(__name__)
 
+
 class CacheManager:
     """Manages caching of processed articles to prevent duplicate processing."""
 
@@ -21,34 +22,74 @@ class CacheManager:
 
     def _load_cache(self) -> Dict:
         """Load cache from file."""
+        empty_cache = {
+            "cache_version": "1.0",
+            "created": datetime.now().isoformat(),
+            "processed_files": {},
+        }
+
         try:
-            if os.path.exists(self.cache_file):
+            if not os.path.exists(self.cache_file):
-                with open(self.cache_file, 'r', encoding='utf-8') as f:
+                logger.info(
-                    cache_data = json.load(f)
+                    f"Cache file does not exist: {self.cache_file}. Creating new empty cache."
-                    # Ensure the cache has the correct structure
+                )
-                    if "processed_files" not in cache_data:
+                return empty_cache
-                        cache_data["processed_files"] = {}
+
-                    return cache_data
+            # Check if the file is empty (0 bytes)
-            else:
+            file_size = os.path.getsize(self.cache_file)
-                # Create empty cache file if it doesn't exist
+            if file_size == 0:
-                cache_data = {
+                logger.warning(
-                    "cache_version": "1.0",
+                    f"Cache file is empty (0 bytes): {self.cache_file}. Treating as fresh cache."
-                    "created": datetime.now().isoformat(),
+                )
-                    "processed_files": {}
+                return empty_cache
-                }
+
-                # Set the cache attribute directly
+            with open(self.cache_file, "r", encoding="utf-8") as f:
-                self.cache = cache_data
+                content = f.read().strip()
-                self._save_cache()
+
-                return cache_data
+            # Check if content is empty or just whitespace
+            if not content:
+                logger.warning(
+                    f"Cache file contains only whitespace: {self.cache_file}. Treating as fresh cache."
+                )
+                return empty_cache
+
+            cache_data = json.loads(content)
+
+            # Handle case where JSON loaded as None, list, or other non-dict type
+            if not isinstance(cache_data, dict):
+                logger.warning(
+                    f"Cache file contains invalid data type ({type(cache_data).__name__}). Treating as fresh cache."
+                )
+                return empty_cache
+
+            # Ensure the cache has the correct structure
+            if (
+                "processed_files" not in cache_data
+                or cache_data["processed_files"] is None
+            ):
+                cache_data["processed_files"] = {}
+
+            # Validate that processed_files is a dict
+            if not isinstance(cache_data["processed_files"], dict):
+                logger.warning(
+                    f"processed_files is not a dict ({type(cache_data['processed_files']).__name__}). Resetting to empty."
+                )
+                cache_data["processed_files"] = {}
+
+            logger.info(
+                f"Loaded cache with {len(cache_data['processed_files'])} processed files"
+            )
+            return cache_data
+
+        except json.JSONDecodeError as e:
+            logger.warning(
+                f"Cache file contains invalid JSON: {e}. Treating as fresh cache."
+            )
+            return empty_cache
         except Exception as e:
             logger.error(f"Error loading cache file {self.cache_file}: {e}")
             logger.info("Creating new empty cache due to load error")
-            # Return empty cache on error
+            return empty_cache
-            return {
-                "cache_version": "1.0",
-                "created": datetime.now().isoformat(),
-                "processed_files": {}
-            }
 
     def _save_cache(self) -> None:
         """Save cache to file."""
@@ -56,14 +97,23 @@ class CacheManager:
             # Create directory if it doesn't exist
             os.makedirs(os.path.dirname(self.cache_file), exist_ok=True)
 
-            with open(self.cache_file, 'w', encoding='utf-8') as f:
+            with open(self.cache_file, "w", encoding="utf-8") as f:
                 json.dump(self.cache, f, indent=2, ensure_ascii=False)
         except Exception as e:
             logger.error(f"Error saving cache file {self.cache_file}: {e}")
 
     def is_processed(self, file_path: str) -> bool:
         """Check if a file has been processed."""
-        return file_path in self.cache.get("processed_files", {})
+        processed_files = self.cache.get("processed_files")
+
+        # Handle case where processed_files is None or not a dict
+        if not isinstance(processed_files, dict):
+            logger.warning(
+                f"processed_files is not a valid dict (got {type(processed_files).__name__}). Returning False."
+            )
+            return False
+
+        return file_path in processed_files
 
     def mark_processed(self, file_path: str, status: str = "processed") -> None:
         """Mark a file as processed."""
@@ -73,7 +123,7 @@ class CacheManager:
         self.cache["processed_files"][file_path] = {
             "processed_date": datetime.now().isoformat(),
             "status": status,
-            "last_updated": datetime.now().isoformat()
+            "last_updated": datetime.now().isoformat(),
         }
         self._save_cache()
 
@@ -86,7 +136,7 @@ class CacheManager:
         return {
             "total_files": len(self.cache.get("processed_files", {})),
             "processed_files": len(self.cache.get("processed_files", {})),
-            "cache_file": self.cache_file
+            "cache_file": self.cache_file,
         }
 
     def clear_cache(self) -> None:

ai_processor/processed_articles_cache.json

@@ -1,5 +0,0 @@
-{
-  "cache_version": "1.0",
-  "created": "2026-02-02T10:02:48.000Z",
-  "processed_files": {}
-}