""" Cache manager for tracking processed articles to avoid reprocessing. """ import json import logging import os from datetime import datetime from typing import Dict, List from config import CACHE_FILE logger = logging.getLogger(__name__) class CacheManager: """Manages caching of processed articles to prevent duplicate processing.""" def __init__(self, cache_file: str = CACHE_FILE): self.cache_file = cache_file self.cache = self._load_cache() logger.info(f"Cache manager initialized with cache file: {self.cache_file}") def _load_cache(self) -> Dict: """Load cache from file.""" empty_cache = { "cache_version": "1.0", "created": datetime.now().isoformat(), "processed_files": {}, } try: logger.debug(f"Attempting to load cache from: {self.cache_file}") if not os.path.exists(self.cache_file): logger.info( f"Cache file does not exist: {self.cache_file}. Creating new empty cache." ) return empty_cache # Check if the file is empty (0 bytes) file_size = os.path.getsize(self.cache_file) if file_size == 0: logger.warning( f"Cache file is empty (0 bytes): {self.cache_file}. Treating as fresh cache." ) return empty_cache with open(self.cache_file, "r", encoding="utf-8") as f: content = f.read().strip() # Check if content is empty or just whitespace if not content: logger.warning( f"Cache file contains only whitespace: {self.cache_file}. Treating as fresh cache." ) return empty_cache cache_data = json.loads(content) # Handle case where JSON loaded as None, list, or other non-dict type if not isinstance(cache_data, dict): logger.warning( f"Cache file contains invalid data type ({type(cache_data).__name__}). Treating as fresh cache." ) return empty_cache # Ensure the cache has the correct structure if ( "processed_files" not in cache_data or cache_data["processed_files"] is None ): cache_data["processed_files"] = {} # Validate that processed_files is a dict if not isinstance(cache_data["processed_files"], dict): logger.warning( f"processed_files is not a dict ({type(cache_data['processed_files']).__name__}). Resetting to empty." ) cache_data["processed_files"] = {} logger.info( f"Loaded cache with {len(cache_data['processed_files'])} processed files from {self.cache_file}" ) logger.debug(f"Cache file size: {file_size} bytes") return cache_data except json.JSONDecodeError as e: logger.error( f"Cache file contains invalid JSON: {e}. Creating fresh cache." ) return empty_cache except Exception as e: logger.error(f"Error loading cache file {self.cache_file}: {e}") logger.info("Creating new empty cache due to load error") return empty_cache def _save_cache(self) -> None: """Save cache to file.""" try: logger.debug(f"Saving cache to file: {self.cache_file}") # Create directory if it doesn't exist os.makedirs(os.path.dirname(self.cache_file), exist_ok=True) with open(self.cache_file, "w", encoding="utf-8") as f: json.dump(self.cache, f, indent=2, ensure_ascii=False) logger.debug(f"Successfully saved cache to {self.cache_file}") except Exception as e: logger.error(f"Error saving cache file {self.cache_file}: {e}") def is_processed(self, file_path: str) -> bool: """Check if a file has been processed.""" logger.debug(f"Checking if file is processed: {file_path}") processed_files = self.cache.get("processed_files") # Handle case where processed_files is None or not a dict if not isinstance(processed_files, dict): logger.warning( f"processed_files is not a valid dict (got {type(processed_files).__name__}). Returning False." ) return False result = file_path in processed_files logger.debug(f"File {file_path} processed status: {result}") return result def mark_processed(self, file_path: str, status: str = "processed") -> None: """Mark a file as processed.""" logger.debug(f"Marking file as processed: {file_path}") # Ensure we're using the correct cache structure if "processed_files" not in self.cache: self.cache["processed_files"] = {} self.cache["processed_files"][file_path] = { "processed_date": datetime.now().isoformat(), "status": status, "last_updated": datetime.now().isoformat(), } self._save_cache() logger.info(f"Successfully marked file as processed: {file_path}") def get_processed_files(self) -> List[str]: """Get list of all processed files.""" files = list(self.cache.get("processed_files", {}).keys()) logger.debug(f"Retrieved {len(files)} processed files from cache") return files def get_cache_stats(self) -> Dict: """Get cache statistics.""" stats = { "total_files": len(self.cache.get("processed_files", {})), "processed_files": len(self.cache.get("processed_files", {})), "cache_file": self.cache_file, } logger.debug(f"Cache stats: {stats}") return stats def clear_cache(self) -> None: """Clear the entire cache.""" logger.info("Clearing entire cache") self.cache = {} self._save_cache() logger.info("Cache cleared successfully")