jarianc/StockDocs
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

fix: Update scraper directory path for Docker container compatibility and add debugging

Browse Source
This commit is contained in:
Jarian Cottingham 2026-02-02 09:27:02 -06:00
parent 0b82bb13c4
commit ba8f24fcbd
1 changed files with 8 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
ai_processor/article_processor.py
View File

@ -25,7 +25,7 @@ class ArticleProcessor:
self.cache_manager = CacheManager(CACHE_FILE) self.cache_manager = CacheManager(CACHE_FILE)
self.batch_size = BATCH_SIZE self.batch_size = BATCH_SIZE
def find_unprocessed_articles(self, scraper_dir: str = "/scraper/articles") -> List[Tuple[str, str]]: def find_unprocessed_articles(self, scraper_dir: str = "/app/articles") -> List[Tuple[str, str]]:
""" """
Find all unprocessed articles in the scraper directory. Find all unprocessed articles in the scraper directory.
@ -38,6 +38,13 @@ class ArticleProcessor:
unprocessed_articles = [] unprocessed_articles = []
try: try:
# Debug: Check if directory exists
logger.info(f"Checking for articles in: {scraper_dir}")
if not os.path.exists(scraper_dir):
logger.warning(f"Scraper directory does not exist: {scraper_dir}")
return []
logger.info(f"Directory exists, walking through files...")
for root, dirs, files in os.walk(scraper_dir): for root, dirs, files in os.walk(scraper_dir):
for file in files: for file in files:
if file.endswith('.json'): if file.endswith('.json'):