From e29e5dbc76c5126f73dfa0e1688f2ca9682f805f Mon Sep 17 00:00:00 2001 From: Jarian Cottingham Date: Mon, 2 Feb 2026 10:38:35 -0600 Subject: [PATCH] Remove file extension filtering - all files in articles directory are valid articles --- ai_processor/article_processor.py | 25 ++++++++----------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/ai_processor/article_processor.py b/ai_processor/article_processor.py index eacf16b..dd474e4 100644 --- a/ai_processor/article_processor.py +++ b/ai_processor/article_processor.py @@ -86,27 +86,18 @@ class ArticleProcessor: ) extension_counts[ext] = extension_counts.get(ext, 0) + 1 - # Check for JSON files (expected format) - if file.endswith(".json"): - article_file_count += 1 - file_path = os.path.join(root, file) - if not self.cache_manager.is_processed(file_path): - unprocessed_articles.append((file_path, file)) - else: - already_processed_count += 1 - # Also check for text files (fallback for different formats) - elif file.endswith((".txt", ".md")): - article_file_count += 1 - file_path = os.path.join(root, file) - if not self.cache_manager.is_processed(file_path): - unprocessed_articles.append((file_path, file)) - else: - already_processed_count += 1 + # All files in this directory are guaranteed to be article files + article_file_count += 1 + file_path = os.path.join(root, file) + if not self.cache_manager.is_processed(file_path): + unprocessed_articles.append((file_path, file)) + else: + already_processed_count += 1 # Log detailed breakdown logger.info(f"File extension breakdown: {extension_counts}") logger.info( - f"Scanned {file_count} total files, {article_file_count} are article files (.json/.txt/.md)" + f"Scanned {file_count} total files, {article_file_count} are article files" ) logger.info( f"Already processed (in cache): {already_processed_count}, Unprocessed: {len(unprocessed_articles)}"