jarianc/StockDocs
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

enhancement: Make article processor more robust with multiple path support and fallback file format detection

Browse Source
This commit is contained in:
Jarian Cottingham 2026-02-02 09:31:14 -06:00
parent ba8f24fcbd
commit abb63db4b7
1 changed files with 22 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
ai_processor/article_processor.py
View File

@ -42,17 +42,37 @@ class ArticleProcessor:
logger.info(f"Checking for articles in: {scraper_dir}") logger.info(f"Checking for articles in: {scraper_dir}")
if not os.path.exists(scraper_dir): if not os.path.exists(scraper_dir):
logger.warning(f"Scraper directory does not exist: {scraper_dir}") logger.warning(f"Scraper directory does not exist: {scraper_dir}")
# Try alternative paths
alternative_paths = [
"/scraper/articles",
"/app/articles",
"/articles"
]
for alt_path in alternative_paths:
if os.path.exists(alt_path):
logger.info(f"Found articles directory at alternative path: {alt_path}")
scraper_dir = alt_path
break
else:
return [] return []
logger.info(f"Directory exists, walking through files...") logger.info(f"Directory exists, walking through files...")
file_count = 0
for root, dirs, files in os.walk(scraper_dir): for root, dirs, files in os.walk(scraper_dir):
for file in files: for file in files:
file_count += 1
# Check for JSON files (expected format)
if file.endswith('.json'): if file.endswith('.json'):
file_path = os.path.join(root, file) file_path = os.path.join(root, file)
if not self.cache_manager.is_processed(file_path): if not self.cache_manager.is_processed(file_path):
unprocessed_articles.append((file_path, file)) unprocessed_articles.append((file_path, file))
# Also check for text files (fallback for different formats)
elif file.endswith(('.txt', '.md')):
file_path = os.path.join(root, file)
if not self.cache_manager.is_processed(file_path):
unprocessed_articles.append((file_path, file))
logger.info(f"Found {len(unprocessed_articles)} unprocessed articles") logger.info(f"Scanned {file_count} files, found {len(unprocessed_articles)} unprocessed articles")
return unprocessed_articles return unprocessed_articles
except Exception as e: except Exception as e: