diff --git a/ai_processor/article_processor.py b/ai_processor/article_processor.py index eacf16b..dd474e4 100644 --- a/ai_processor/article_processor.py +++ b/ai_processor/article_processor.py @@ -86,27 +86,18 @@ class ArticleProcessor: ) extension_counts[ext] = extension_counts.get(ext, 0) + 1 - # Check for JSON files (expected format) - if file.endswith(".json"): - article_file_count += 1 - file_path = os.path.join(root, file) - if not self.cache_manager.is_processed(file_path): - unprocessed_articles.append((file_path, file)) - else: - already_processed_count += 1 - # Also check for text files (fallback for different formats) - elif file.endswith((".txt", ".md")): - article_file_count += 1 - file_path = os.path.join(root, file) - if not self.cache_manager.is_processed(file_path): - unprocessed_articles.append((file_path, file)) - else: - already_processed_count += 1 + # All files in this directory are guaranteed to be article files + article_file_count += 1 + file_path = os.path.join(root, file) + if not self.cache_manager.is_processed(file_path): + unprocessed_articles.append((file_path, file)) + else: + already_processed_count += 1 # Log detailed breakdown logger.info(f"File extension breakdown: {extension_counts}") logger.info( - f"Scanned {file_count} total files, {article_file_count} are article files (.json/.txt/.md)" + f"Scanned {file_count} total files, {article_file_count} are article files" ) logger.info( f"Already processed (in cache): {already_processed_count}, Unprocessed: {len(unprocessed_articles)}"