jarianc/StockDocs
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Implement article filtering to avoid reprocessing already downloaded articles

Browse Source
This commit is contained in:
Jarian Cottingham 2026-01-31 12:06:37 -06:00
parent 1cbeb55463
commit cc192cf566
1 changed files with 34 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
scraper/scraper.py
View File

@ -344,6 +344,19 @@ def pull_article(link, source, title=None, save_to_file=True):
return text
def is_article_downloaded(source, title, link):
"""
Check if an article is already downloaded by checking if the file exists.
"""
# Generate the same filename that would be used for saving
filename = title if title else link
safe_filename = generate_filename_from_url(filename)
# Check if file exists in the articles directory
file_path = os.path.join("articles", source, safe_filename)
return os.path.exists(file_path)
def safe_pull_articles(article_list):
"""
Safely pull articles with improved error handling and increased parallelism.
@ -351,13 +364,30 @@ def safe_pull_articles(article_list):
if not article_list:
return [], []
# Filter out articles that are already downloaded
filtered_article_list = []
total_articles = len(article_list)
for source, title, link in article_list:
if not is_article_downloaded(source, title, link):
filtered_article_list.append((source, title, link))
else:
logger.info(f"Skipping already downloaded article: {title[:50]}... from {source}")
logger.info(f"Filtered out {total_articles - len(filtered_article_list)} articles that were already downloaded")
logger.info(f"Processing {len(filtered_article_list)} remaining articles")
if not filtered_article_list:
logger.info("No new articles to process")
return [], []
results = []
errors = []
# Use ThreadPoolExecutor for parallel article pulling with higher concurrency
# Use configurable worker setting
max_workers = min(MAX_ARTICLE_WORKERS, len(article_list)) # Cap at configured workers, but don't exceed article count
batch_size = max(1, min(20, len(article_list) // 4)) # Dynamic batch size
max_workers = min(MAX_ARTICLE_WORKERS, len(filtered_article_list)) # Cap at configured workers, but don't exceed article count
batch_size = max(1, min(20, len(filtered_article_list) // 4)) # Dynamic batch size
logger.info(f"Starting parallel article pulling with {max_workers} workers and batch size {batch_size}")
@ -366,7 +396,7 @@ def safe_pull_articles(article_list):
# Submit all tasks at once for maximum parallelism
futures = [
executor.submit(pull_article, link, source, title)
for source, title, link in article_list
for source, title, link in filtered_article_list
]
# Collect results as they complete
@ -377,7 +407,7 @@ def safe_pull_articles(article_list):
results.append(result)
# Log progress every 100 articles with proper batch information
if (i + 1) % 100 == 0:
logger.info(f"Processed {i + 1} articles out of {len(article_list)}")
logger.info(f"Processed {i + 1} articles out of {len(filtered_article_list)}")
except Exception as e:
logger.error(f"Error processing article: {e}")
errors.append(e)