From e23b4c2ac9341cf125ea9207ab441dd803f31d87 Mon Sep 17 00:00:00 2001 From: Jarian Cottingham Date: Sat, 31 Jan 2026 11:44:08 -0600 Subject: [PATCH] Add enhanced error handling for 'Remote end closed connection' errors in RSS feed parsing --- scraper/scraper.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/scraper/scraper.py b/scraper/scraper.py index 071c451..5116e67 100644 --- a/scraper/scraper.py +++ b/scraper/scraper.py @@ -102,8 +102,18 @@ def mine_all_articles(rss_feed_sources, limit=None): logger.error(f"Timeout parsing RSS feed: {site} Error: {str(e)}") return [] except Exception as e: - logger.error(f"Error parsing RSS feed: {site} Error: {str(e)}") - return [] + error_str = str(e).lower() + # Handle specific network connection issues + if "remote end closed connection" in error_str or "connection closed" in error_str: + logger.warning(f"Network connection closed by remote end for feed: {site} - {str(e)}") + logger.info(f"Skipping problematic feed: {site}") + return [] + elif "timeout" in error_str: + logger.error(f"Timeout parsing RSS feed: {site} Error: {str(e)}") + return [] + else: + logger.error(f"Error parsing RSS feed: {site} Error: {str(e)}") + return [] # Use ThreadPoolExecutor for parallel RSS feed parsing from concurrent.futures import ThreadPoolExecutor, as_completed