diff --git a/scraper/scraper.py b/scraper/scraper.py index 071c451..5116e67 100644 --- a/scraper/scraper.py +++ b/scraper/scraper.py @@ -102,8 +102,18 @@ def mine_all_articles(rss_feed_sources, limit=None): logger.error(f"Timeout parsing RSS feed: {site} Error: {str(e)}") return [] except Exception as e: - logger.error(f"Error parsing RSS feed: {site} Error: {str(e)}") - return [] + error_str = str(e).lower() + # Handle specific network connection issues + if "remote end closed connection" in error_str or "connection closed" in error_str: + logger.warning(f"Network connection closed by remote end for feed: {site} - {str(e)}") + logger.info(f"Skipping problematic feed: {site}") + return [] + elif "timeout" in error_str: + logger.error(f"Timeout parsing RSS feed: {site} Error: {str(e)}") + return [] + else: + logger.error(f"Error parsing RSS feed: {site} Error: {str(e)}") + return [] # Use ThreadPoolExecutor for parallel RSS feed parsing from concurrent.futures import ThreadPoolExecutor, as_completed