From cc192cf5667038bf7f219f28e5af5e05f1bb7dee Mon Sep 17 00:00:00 2001 From: Jarian Cottingham Date: Sat, 31 Jan 2026 12:06:37 -0600 Subject: [PATCH] Implement article filtering to avoid reprocessing already downloaded articles --- scraper/scraper.py | 38 ++++++++++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/scraper/scraper.py b/scraper/scraper.py index d268a30..5a6eb47 100644 --- a/scraper/scraper.py +++ b/scraper/scraper.py @@ -344,6 +344,19 @@ def pull_article(link, source, title=None, save_to_file=True): return text +def is_article_downloaded(source, title, link): + """ + Check if an article is already downloaded by checking if the file exists. + """ + # Generate the same filename that would be used for saving + filename = title if title else link + safe_filename = generate_filename_from_url(filename) + + # Check if file exists in the articles directory + file_path = os.path.join("articles", source, safe_filename) + return os.path.exists(file_path) + + def safe_pull_articles(article_list): """ Safely pull articles with improved error handling and increased parallelism. @@ -351,13 +364,30 @@ def safe_pull_articles(article_list): if not article_list: return [], [] + # Filter out articles that are already downloaded + filtered_article_list = [] + total_articles = len(article_list) + + for source, title, link in article_list: + if not is_article_downloaded(source, title, link): + filtered_article_list.append((source, title, link)) + else: + logger.info(f"Skipping already downloaded article: {title[:50]}... from {source}") + + logger.info(f"Filtered out {total_articles - len(filtered_article_list)} articles that were already downloaded") + logger.info(f"Processing {len(filtered_article_list)} remaining articles") + + if not filtered_article_list: + logger.info("No new articles to process") + return [], [] + results = [] errors = [] # Use ThreadPoolExecutor for parallel article pulling with higher concurrency # Use configurable worker setting - max_workers = min(MAX_ARTICLE_WORKERS, len(article_list)) # Cap at configured workers, but don't exceed article count - batch_size = max(1, min(20, len(article_list) // 4)) # Dynamic batch size + max_workers = min(MAX_ARTICLE_WORKERS, len(filtered_article_list)) # Cap at configured workers, but don't exceed article count + batch_size = max(1, min(20, len(filtered_article_list) // 4)) # Dynamic batch size logger.info(f"Starting parallel article pulling with {max_workers} workers and batch size {batch_size}") @@ -366,7 +396,7 @@ def safe_pull_articles(article_list): # Submit all tasks at once for maximum parallelism futures = [ executor.submit(pull_article, link, source, title) - for source, title, link in article_list + for source, title, link in filtered_article_list ] # Collect results as they complete @@ -377,7 +407,7 @@ def safe_pull_articles(article_list): results.append(result) # Log progress every 100 articles with proper batch information if (i + 1) % 100 == 0: - logger.info(f"Processed {i + 1} articles out of {len(article_list)}") + logger.info(f"Processed {i + 1} articles out of {len(filtered_article_list)}") except Exception as e: logger.error(f"Error processing article: {e}") errors.append(e)