From b8a24a9fb1e822f544fe2ceffa630c6df575b12f Mon Sep 17 00:00:00 2001 From: Jarian Cottingham Date: Sat, 31 Jan 2026 00:03:06 -0600 Subject: [PATCH] feat(scraper): Add cron_scraper.py to fix freezing issues --- scraper/cron_scraper.py | 347 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 347 insertions(+) create mode 100644 scraper/cron_scraper.py diff --git a/scraper/cron_scraper.py b/scraper/cron_scraper.py new file mode 100644 index 0000000..2c65f9c --- /dev/null +++ b/scraper/cron_scraper.py @@ -0,0 +1,347 @@ +#!/usr/bin/env python3 +""" +Cron-based scraper for downloading articles from RSS feeds. +This version replaces the infinite while loop with a single execution +that can be scheduled via cron job. +""" + +import newspaper +import json +import feedparser +import time +import os +from selenium import webdriver +from selenium.webdriver.firefox.options import Options as FirefoxOptions +from selenium.webdriver.common.by import By +from selenium.webdriver.support.ui import WebDriverWait +from selenium.webdriver.support import expected_conditions as EC +from concurrent.futures import ThreadPoolExecutor, as_completed +import nltk +from nltk.downloader import Downloader + +FEED_FILE = os.getenv("FEED_FILE", "./rss_feeds.json") + +# Ensure necessary NLTK resources are downloaded +d = Downloader() +if not d.is_installed("punkt_tab"): + nltk.download("punkt_tab") + +articles = [] + + +def load_rss_feed_sources(feed_file=FEED_FILE): + """ + Loads the RSS feed sources from a JSON file. + """ + print("Loading RSS feed sources from rss_feeds.json...") + + try: + with open(FEED_FILE, "r", encoding="utf-8") as f: + return json.load(f) + except FileNotFoundError: + print(FEED_FILE + " not found, returning empty list.") + return [] + except json.JSONDecodeError: + print("Error decoding " + FEED_FILE + " , returning empty list.") + return [] + + +def mine_all_articles(rss_feed_sources, limit=None): + """ + Mines all articles from the given RSS feed sources. + Returns a list of (site, title, link) tuples. + """ + all_links = [] + sources = rss_feed_sources["rss_feeds"] + + for site, data in sources.items(): + print(f"Parsing RSS feed: {data['rss_url']}") + try: + feed = feedparser.parse(data["rss_url"]) + feed_entries = feed.entries[:limit] if limit else feed.entries + + for entry in feed_entries: + if "link" in entry and "title" in entry: + all_links.append((site, entry.title, entry.link)) + except Exception as e: + print(f"Error parsing RSS feed: {site} Error: {str(e)}") + return all_links + + +def generate_filename_from_url(url): + """ + Generates a filename from the given URL by replacing slashes with underscores. + """ + # Use only the last part of the URL or replace slashes + return url.replace("https://", "").replace("http://", "").replace("/", "_") + + +def generate_safe_filename(name): + # Remove/replace characters not allowed in filenames + import re + + safe = re.sub(r'[\\/*?:"<>|]', "_", name) + return safe + + +def save_article_to_file(article, filename, source="Unfiltered"): + """ + Saves the given article text to a file with the specified filename. + """ + # articles dir should already be there + # os.makedirs("articles", exist_ok=True) + + outputDir = "articles/" + source + os.makedirs(outputDir, exist_ok=True) if source else None + + # Sanitize filename: use only the last part of the URL or replace slashes + safe_filename = generate_filename_from_url(filename) + file_path = os.path.join(outputDir, safe_filename) + # Save the source as the first line in the file for later retrieval + with open(file_path, "w", encoding="utf-8") as f: + f.write(f"SOURCE:{source}\n") + f.write(article) + print(f"Article saved to {file_path}") + + +def get_article_with_selenium(url): + """ + Gets article text using Selenium Firefox driver with proper error handling + and cleanup. + """ + driver = None + try: + # Configure Firefox options + options = FirefoxOptions() + options.add_argument("--headless") + options.set_preference("dom.ipc.processCount", 1) # Reduce process count + + # Initialize driver with timeout + driver = webdriver.Firefox(options=options) + driver.set_page_load_timeout(30) # 30 seconds timeout + + # Navigate to URL + driver.get(url) + + # Wait for page to load (explicit wait instead of sleep) + try: + WebDriverWait(driver, 15).until( + EC.presence_of_element_located((By.TAG_NAME, "body")) + ) + except: + pass # Continue even if wait times out + + time.sleep(2) # Brief additional wait + + html = driver.page_source + + # Parse with Newspaper4k + article = newspaper.article(url, input_html=html, language="en") + article.nlp() + return article.text + + except Exception as e: + print(f"Selenium failed for {url}: {str(e)}") + return "" + finally: + # Always quit the driver + if driver: + try: + driver.quit() + except: + pass # Ignore errors in cleanup + + +def get_article_with_playwright(url): + """ + Gets article text using Playwright with proper error handling. + """ + try: + from playwright.sync_api import sync_playwright + + with sync_playwright() as p: + # Use Chromium instead of Firefox for better compatibility + browser = p.chromium.launch(headless=True, timeout=30000) + page = browser.new_page() + + # Set user agent to avoid bot detection + page.set_extra_http_headers( + { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36" + } + ) + + page.goto(url, wait_until="load") + + # Wait for content to load + time.sleep(3) + + html = page.content() + browser.close() + + # Parse with Newspaper4k + article = newspaper.article(url, input_html=html, language="en") + article.nlp() + return article.text + + except Exception as e: + print(f"Playwright failed for {url}: {str(e)}") + return "" + + +def pull_article(link, source, title=None, save_to_file=True): + """ + Pulls an article from a given link with fallback mechanisms. + """ + filename = title if title else link + safe_filename = generate_filename_from_url(filename) + + # Check if already cached + if os.path.exists(os.path.join("articles", source, safe_filename)): + print(f"Article already cached: {filename}") + with open( + os.path.join("articles", source, safe_filename), "r", encoding="utf-8" + ) as f: + return f.read() + + text = "" + + try: + # Try newspaper4k first + article = newspaper.article(link) + article.download() + article.parse() + text = article.text + + if not text or len(text) < 200: + raise ValueError( + "\tArticle text too short, falling back to Playwright/Selenium." + ) + print(f"\tSuccessfully pulled article with newspaper4k from {link}") + + except Exception as e: + print( + f"\tnewspaper4k extraction failed for {link}: {e}, falling back to Playwright." + ) + + try: + text = get_article_with_playwright(link) + print(f"\t\tSuccessfully pulled article from {link} with Playwright") + + if not text or len(text) < 200: + print(f"\t\tPlaywright article too short, falling back to Selenium.") + # Fallback to Selenium with better error handling + text = get_article_with_selenium(link) + print(f"\t\t\tSuccessfully pulled article from {link} with Selenium") + except Exception as e: + print(f"\t\tPlaywright failed for {link}: {e}") + + # Fallback to Selenium + try: + text = get_article_with_selenium(link) + print(f"\t\tSuccessfully pulled article from {link} with Selenium") + except Exception as e: + print(f"\t\tSelenium failed for {link}: {e}") + return "" + + if save_to_file: + save_article_to_file(text, filename, source) + + return text + + +def safe_pull_articles(article_list): + """ + Safely pull articles with improved error handling and reduced parallelism. + """ + results = [] + errors = [] + + # Process in smaller batches to reduce resource strain + batch_size = 5 + + for i in range(0, len(article_list), batch_size): + batch = article_list[i : i + batch_size] + print(f"Processing batch {i // batch_size + 1} with {len(batch)} articles") + + # Use ThreadPoolExecutor instead of ProcessPoolExecutor to avoid + # process termination issues with browser automation + with ThreadPoolExecutor(max_workers=3) as executor: # Reduced workers + futures = [ + executor.submit(pull_article, link, source, title) + for source, title, link in batch + ] + + for future in as_completed(futures): + try: + result = future.result(timeout=120) # 2 minute timeout + results.append(result) + except Exception as e: + print(f"Error in pull_article: {e}") + errors.append(e) + + # Add a small delay between batches to reduce system load + time.sleep(5) + + return results, errors + + +def main(): + """ + Main scraping function for cron execution. + This replaces the infinite while loop with a single execution. + """ + print("=========================================") + print("Starting new scraping iteration...") + + try: + # Pull the RSS feed sources from the JSON file + rss_feed_sources = load_rss_feed_sources() + + # Mine all articles from the RSS feed sources + rss_feed_links = mine_all_articles(rss_feed_sources) + + # Randomize the order of the links to help with load balancing + import random + + random.shuffle(rss_feed_links) + + print(f"Found {len(rss_feed_links)} articles to process") + + if not rss_feed_links: + print("No articles found") + return + + # Process articles with better error handling and resource management + results, errors = safe_pull_articles(rss_feed_links) + + print(f"Attempted to Pull {len(results)} articles in parallel.") + print( + f"Encountered {len(errors)} errors during article pulling. " + + "Outputting errors to a local file." + ) + + # Output errors to a local file + if errors: + with open("errors.txt", "w", encoding="utf-8") as f: + for error in errors: + f.write(str(error) + "\n") + print(f"Errors logged to errors.txt") + + # Print all results to a log file + with open("results.txt", "w", encoding="utf-8") as f: + for result in results: + f.write(result + "\n") + print("All articles pulled successfully.") + + except Exception as e: + print(f"Major error in main execution: {e}") + import traceback + traceback.print_exc() + raise # Re-raise to ensure the script exits with error code + + print("Scraping completed successfully.") + + +if __name__ == "__main__": + main() \ No newline at end of file