jarianc/StockDocs
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Remove file extension filtering - all files in articles directory are valid articles

Browse Source
This commit is contained in:
Jarian Cottingham 2026-02-02 10:38:35 -06:00
parent 492b30a713
commit e29e5dbc76
1 changed files with 8 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
ai_processor/article_processor.py
View File

@ -86,27 +86,18 @@ class ArticleProcessor:
)
extension_counts[ext] = extension_counts.get(ext, 0) + 1
# Check for JSON files (expected format)
if file.endswith(".json"):
article_file_count += 1
file_path = os.path.join(root, file)
if not self.cache_manager.is_processed(file_path):
unprocessed_articles.append((file_path, file))
else:
already_processed_count += 1
# Also check for text files (fallback for different formats)
elif file.endswith((".txt", ".md")):
article_file_count += 1
file_path = os.path.join(root, file)
if not self.cache_manager.is_processed(file_path):
unprocessed_articles.append((file_path, file))
else:
already_processed_count += 1
# All files in this directory are guaranteed to be article files
article_file_count += 1
file_path = os.path.join(root, file)
if not self.cache_manager.is_processed(file_path):
unprocessed_articles.append((file_path, file))
else:
already_processed_count += 1
# Log detailed breakdown
logger.info(f"File extension breakdown: {extension_counts}")
logger.info(
f"Scanned {file_count} total files, {article_file_count} are article files (.json/.txt/.md)"
f"Scanned {file_count} total files, {article_file_count} are article files"
)
logger.info(
f"Already processed (in cache): {already_processed_count}, Unprocessed: {len(unprocessed_articles)}"