jarianc/StockDocs
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Add enhanced error handling for 'Remote end closed connection' errors in RSS feed parsing

Browse Source
This commit is contained in:
Jarian Cottingham 2026-01-31 11:44:08 -06:00
parent e975350701
commit e23b4c2ac9
1 changed files with 12 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
scraper/scraper.py
View File

@ -102,8 +102,18 @@ def mine_all_articles(rss_feed_sources, limit=None):
logger.error(f"Timeout parsing RSS feed: {site} Error: {str(e)}")
return []
except Exception as e:
logger.error(f"Error parsing RSS feed: {site} Error: {str(e)}")
return []
error_str = str(e).lower()
# Handle specific network connection issues
if "remote end closed connection" in error_str or "connection closed" in error_str:
logger.warning(f"Network connection closed by remote end for feed: {site} - {str(e)}")
logger.info(f"Skipping problematic feed: {site}")
return []
elif "timeout" in error_str:
logger.error(f"Timeout parsing RSS feed: {site} Error: {str(e)}")
return []
else:
logger.error(f"Error parsing RSS feed: {site} Error: {str(e)}")
return []
# Use ThreadPoolExecutor for parallel RSS feed parsing
from concurrent.futures import ThreadPoolExecutor, as_completed