From abb63db4b7f7b170bbed9d55765b4fdc2fdf5ede Mon Sep 17 00:00:00 2001 From: Jarian Cottingham Date: Mon, 2 Feb 2026 09:31:14 -0600 Subject: [PATCH] enhancement: Make article processor more robust with multiple path support and fallback file format detection --- ai_processor/article_processor.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/ai_processor/article_processor.py b/ai_processor/article_processor.py index 37604ce..e7ebdd0 100644 --- a/ai_processor/article_processor.py +++ b/ai_processor/article_processor.py @@ -42,17 +42,37 @@ class ArticleProcessor: logger.info(f"Checking for articles in: {scraper_dir}") if not os.path.exists(scraper_dir): logger.warning(f"Scraper directory does not exist: {scraper_dir}") - return [] + # Try alternative paths + alternative_paths = [ + "/scraper/articles", + "/app/articles", + "/articles" + ] + for alt_path in alternative_paths: + if os.path.exists(alt_path): + logger.info(f"Found articles directory at alternative path: {alt_path}") + scraper_dir = alt_path + break + else: + return [] logger.info(f"Directory exists, walking through files...") + file_count = 0 for root, dirs, files in os.walk(scraper_dir): for file in files: + file_count += 1 + # Check for JSON files (expected format) if file.endswith('.json'): file_path = os.path.join(root, file) if not self.cache_manager.is_processed(file_path): unprocessed_articles.append((file_path, file)) + # Also check for text files (fallback for different formats) + elif file.endswith(('.txt', '.md')): + file_path = os.path.join(root, file) + if not self.cache_manager.is_processed(file_path): + unprocessed_articles.append((file_path, file)) - logger.info(f"Found {len(unprocessed_articles)} unprocessed articles") + logger.info(f"Scanned {file_count} files, found {len(unprocessed_articles)} unprocessed articles") return unprocessed_articles except Exception as e: