jarianc/StockDocs
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Remove file extension filtering - all files in articles directory are valid articles

Browse Source
This commit is contained in:
Jarian Cottingham 2026-02-02 10:38:35 -06:00
parent 492b30a713
commit e29e5dbc76
1 changed files with 8 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
ai_processor/article_processor.py
View File

@ -86,16 +86,7 @@ class ArticleProcessor:
) )
extension_counts[ext] = extension_counts.get(ext, 0) + 1 extension_counts[ext] = extension_counts.get(ext, 0) + 1
# Check for JSON files (expected format) # All files in this directory are guaranteed to be article files
if file.endswith(".json"):
article_file_count += 1
file_path = os.path.join(root, file)
if not self.cache_manager.is_processed(file_path):
unprocessed_articles.append((file_path, file))
else:
already_processed_count += 1
# Also check for text files (fallback for different formats)
elif file.endswith((".txt", ".md")):
article_file_count += 1 article_file_count += 1
file_path = os.path.join(root, file) file_path = os.path.join(root, file)
if not self.cache_manager.is_processed(file_path): if not self.cache_manager.is_processed(file_path):
@ -106,7 +97,7 @@ class ArticleProcessor:
# Log detailed breakdown # Log detailed breakdown
logger.info(f"File extension breakdown: {extension_counts}") logger.info(f"File extension breakdown: {extension_counts}")
logger.info( logger.info(
f"Scanned {file_count} total files, {article_file_count} are article files (.json/.txt/.md)" f"Scanned {file_count} total files, {article_file_count} are article files"
) )
logger.info( logger.info(
f"Already processed (in cache): {already_processed_count}, Unprocessed: {len(unprocessed_articles)}" f"Already processed (in cache): {already_processed_count}, Unprocessed: {len(unprocessed_articles)}"