Implement article filtering to avoid reprocessing already downloaded articles

scraper/scraper.py

@@ -344,6 +344,19 @@ def pull_article(link, source, title=None, save_to_file=True):
     return text
 
 
+def is_article_downloaded(source, title, link):
+    """
+    Check if an article is already downloaded by checking if the file exists.
+    """
+    # Generate the same filename that would be used for saving
+    filename = title if title else link
+    safe_filename = generate_filename_from_url(filename)
+    
+    # Check if file exists in the articles directory
+    file_path = os.path.join("articles", source, safe_filename)
+    return os.path.exists(file_path)
+
+
 def safe_pull_articles(article_list):
     """
     Safely pull articles with improved error handling and increased parallelism.
@@ -351,13 +364,30 @@ def safe_pull_articles(article_list):
     if not article_list:
         return [], []
 
+    # Filter out articles that are already downloaded
+    filtered_article_list = []
+    total_articles = len(article_list)
+    
+    for source, title, link in article_list:
+        if not is_article_downloaded(source, title, link):
+            filtered_article_list.append((source, title, link))
+        else:
+            logger.info(f"Skipping already downloaded article: {title[:50]}... from {source}")
+    
+    logger.info(f"Filtered out {total_articles - len(filtered_article_list)} articles that were already downloaded")
+    logger.info(f"Processing {len(filtered_article_list)} remaining articles")
+    
+    if not filtered_article_list:
+        logger.info("No new articles to process")
+        return [], []
+
     results = []
     errors = []
 
     # Use ThreadPoolExecutor for parallel article pulling with higher concurrency
     # Use configurable worker setting
-    max_workers = min(MAX_ARTICLE_WORKERS, len(article_list))  # Cap at configured workers, but don't exceed article count
+    max_workers = min(MAX_ARTICLE_WORKERS, len(filtered_article_list))  # Cap at configured workers, but don't exceed article count
-    batch_size = max(1, min(20, len(article_list) // 4))  # Dynamic batch size
+    batch_size = max(1, min(20, len(filtered_article_list) // 4))  # Dynamic batch size
     
     logger.info(f"Starting parallel article pulling with {max_workers} workers and batch size {batch_size}")
 
@@ -366,7 +396,7 @@ def safe_pull_articles(article_list):
         # Submit all tasks at once for maximum parallelism
         futures = [
             executor.submit(pull_article, link, source, title)
-            for source, title, link in article_list
+            for source, title, link in filtered_article_list
         ]
 
         # Collect results as they complete
@@ -377,7 +407,7 @@ def safe_pull_articles(article_list):
                     results.append(result)
                 # Log progress every 100 articles with proper batch information
                 if (i + 1) % 100 == 0:
-                    logger.info(f"Processed {i + 1} articles out of {len(article_list)}")
+                    logger.info(f"Processed {i + 1} articles out of {len(filtered_article_list)}")
             except Exception as e:
                 logger.error(f"Error processing article: {e}")
                 errors.append(e)