feat(scraper): Add cron_scraper.py to fix freezing issues

scraper/cron_scraper.py

@@ -0,0 +1,347 @@
+#!/usr/bin/env python3
+"""
+Cron-based scraper for downloading articles from RSS feeds.
+This version replaces the infinite while loop with a single execution
+that can be scheduled via cron job.
+"""
+
+import newspaper
+import json
+import feedparser
+import time
+import os
+from selenium import webdriver
+from selenium.webdriver.firefox.options import Options as FirefoxOptions
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+from concurrent.futures import ThreadPoolExecutor, as_completed
+import nltk
+from nltk.downloader import Downloader
+
+FEED_FILE = os.getenv("FEED_FILE", "./rss_feeds.json")
+
+# Ensure necessary NLTK resources are downloaded
+d = Downloader()
+if not d.is_installed("punkt_tab"):
+    nltk.download("punkt_tab")
+
+articles = []
+
+
+def load_rss_feed_sources(feed_file=FEED_FILE):
+    """
+    Loads the RSS feed sources from a JSON file.
+    """
+    print("Loading RSS feed sources from rss_feeds.json...")
+
+    try:
+        with open(FEED_FILE, "r", encoding="utf-8") as f:
+            return json.load(f)
+    except FileNotFoundError:
+        print(FEED_FILE + " not found, returning empty list.")
+        return []
+    except json.JSONDecodeError:
+        print("Error decoding " + FEED_FILE + " , returning empty list.")
+        return []
+
+
+def mine_all_articles(rss_feed_sources, limit=None):
+    """
+    Mines all articles from the given RSS feed sources.
+    Returns a list of (site, title, link) tuples.
+    """
+    all_links = []
+    sources = rss_feed_sources["rss_feeds"]
+
+    for site, data in sources.items():
+        print(f"Parsing RSS feed: {data['rss_url']}")
+        try:
+            feed = feedparser.parse(data["rss_url"])
+            feed_entries = feed.entries[:limit] if limit else feed.entries
+
+            for entry in feed_entries:
+                if "link" in entry and "title" in entry:
+                    all_links.append((site, entry.title, entry.link))
+        except Exception as e:
+            print(f"Error parsing RSS feed: {site} Error: {str(e)}")
+    return all_links
+
+
+def generate_filename_from_url(url):
+    """
+    Generates a filename from the given URL by replacing slashes with underscores.
+    """
+    # Use only the last part of the URL or replace slashes
+    return url.replace("https://", "").replace("http://", "").replace("/", "_")
+
+
+def generate_safe_filename(name):
+    # Remove/replace characters not allowed in filenames
+    import re
+
+    safe = re.sub(r'[\\/*?:"<>|]', "_", name)
+    return safe
+
+
+def save_article_to_file(article, filename, source="Unfiltered"):
+    """
+    Saves the given article text to a file with the specified filename.
+    """
+    # articles dir should already be there
+    # os.makedirs("articles", exist_ok=True)
+
+    outputDir = "articles/" + source
+    os.makedirs(outputDir, exist_ok=True) if source else None
+
+    # Sanitize filename: use only the last part of the URL or replace slashes
+    safe_filename = generate_filename_from_url(filename)
+    file_path = os.path.join(outputDir, safe_filename)
+    # Save the source as the first line in the file for later retrieval
+    with open(file_path, "w", encoding="utf-8") as f:
+        f.write(f"SOURCE:{source}\n")
+        f.write(article)
+    print(f"Article saved to {file_path}")
+
+
+def get_article_with_selenium(url):
+    """
+    Gets article text using Selenium Firefox driver with proper error handling
+    and cleanup.
+    """
+    driver = None
+    try:
+        # Configure Firefox options
+        options = FirefoxOptions()
+        options.add_argument("--headless")
+        options.set_preference("dom.ipc.processCount", 1)  # Reduce process count
+
+        # Initialize driver with timeout
+        driver = webdriver.Firefox(options=options)
+        driver.set_page_load_timeout(30)  # 30 seconds timeout
+
+        # Navigate to URL
+        driver.get(url)
+
+        # Wait for page to load (explicit wait instead of sleep)
+        try:
+            WebDriverWait(driver, 15).until(
+                EC.presence_of_element_located((By.TAG_NAME, "body"))
+            )
+        except:
+            pass  # Continue even if wait times out
+
+        time.sleep(2)  # Brief additional wait
+
+        html = driver.page_source
+
+        # Parse with Newspaper4k
+        article = newspaper.article(url, input_html=html, language="en")
+        article.nlp()
+        return article.text
+
+    except Exception as e:
+        print(f"Selenium failed for {url}: {str(e)}")
+        return ""
+    finally:
+        # Always quit the driver
+        if driver:
+            try:
+                driver.quit()
+            except:
+                pass  # Ignore errors in cleanup
+
+
+def get_article_with_playwright(url):
+    """
+    Gets article text using Playwright with proper error handling.
+    """
+    try:
+        from playwright.sync_api import sync_playwright
+
+        with sync_playwright() as p:
+            # Use Chromium instead of Firefox for better compatibility
+            browser = p.chromium.launch(headless=True, timeout=30000)
+            page = browser.new_page()
+
+            # Set user agent to avoid bot detection
+            page.set_extra_http_headers(
+                {
+                    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
+                }
+            )
+
+            page.goto(url, wait_until="load")
+
+            # Wait for content to load
+            time.sleep(3)
+
+            html = page.content()
+            browser.close()
+
+        # Parse with Newspaper4k
+        article = newspaper.article(url, input_html=html, language="en")
+        article.nlp()
+        return article.text
+
+    except Exception as e:
+        print(f"Playwright failed for {url}: {str(e)}")
+        return ""
+
+
+def pull_article(link, source, title=None, save_to_file=True):
+    """
+    Pulls an article from a given link with fallback mechanisms.
+    """
+    filename = title if title else link
+    safe_filename = generate_filename_from_url(filename)
+
+    # Check if already cached
+    if os.path.exists(os.path.join("articles", source, safe_filename)):
+        print(f"Article already cached: {filename}")
+        with open(
+            os.path.join("articles", source, safe_filename), "r", encoding="utf-8"
+        ) as f:
+            return f.read()
+
+    text = ""
+
+    try:
+        # Try newspaper4k first
+        article = newspaper.article(link)
+        article.download()
+        article.parse()
+        text = article.text
+
+        if not text or len(text) < 200:
+            raise ValueError(
+                "\tArticle text too short, falling back to Playwright/Selenium."
+            )
+        print(f"\tSuccessfully pulled article with newspaper4k from {link}")
+
+    except Exception as e:
+        print(
+            f"\tnewspaper4k extraction failed for {link}: {e}, falling back to Playwright."
+        )
+
+        try:
+            text = get_article_with_playwright(link)
+            print(f"\t\tSuccessfully pulled article from {link} with Playwright")
+
+            if not text or len(text) < 200:
+                print(f"\t\tPlaywright article too short, falling back to Selenium.")
+                # Fallback to Selenium with better error handling
+                text = get_article_with_selenium(link)
+                print(f"\t\t\tSuccessfully pulled article from {link} with Selenium")
+        except Exception as e:
+            print(f"\t\tPlaywright failed for {link}: {e}")
+
+            # Fallback to Selenium
+            try:
+                text = get_article_with_selenium(link)
+                print(f"\t\tSuccessfully pulled article from {link} with Selenium")
+            except Exception as e:
+                print(f"\t\tSelenium failed for {link}: {e}")
+                return ""
+
+    if save_to_file:
+        save_article_to_file(text, filename, source)
+
+    return text
+
+
+def safe_pull_articles(article_list):
+    """
+    Safely pull articles with improved error handling and reduced parallelism.
+    """
+    results = []
+    errors = []
+
+    # Process in smaller batches to reduce resource strain
+    batch_size = 5
+
+    for i in range(0, len(article_list), batch_size):
+        batch = article_list[i : i + batch_size]
+        print(f"Processing batch {i // batch_size + 1} with {len(batch)} articles")
+
+        # Use ThreadPoolExecutor instead of ProcessPoolExecutor to avoid
+        # process termination issues with browser automation
+        with ThreadPoolExecutor(max_workers=3) as executor:  # Reduced workers
+            futures = [
+                executor.submit(pull_article, link, source, title)
+                for source, title, link in batch
+            ]
+
+            for future in as_completed(futures):
+                try:
+                    result = future.result(timeout=120)  # 2 minute timeout
+                    results.append(result)
+                except Exception as e:
+                    print(f"Error in pull_article: {e}")
+                    errors.append(e)
+
+        # Add a small delay between batches to reduce system load
+        time.sleep(5)
+
+    return results, errors
+
+
+def main():
+    """
+    Main scraping function for cron execution.
+    This replaces the infinite while loop with a single execution.
+    """
+    print("=========================================")
+    print("Starting new scraping iteration...")
+
+    try:
+        # Pull the RSS feed sources from the JSON file
+        rss_feed_sources = load_rss_feed_sources()
+
+        # Mine all articles from the RSS feed sources
+        rss_feed_links = mine_all_articles(rss_feed_sources)
+
+        # Randomize the order of the links to help with load balancing
+        import random
+
+        random.shuffle(rss_feed_links)
+
+        print(f"Found {len(rss_feed_links)} articles to process")
+
+        if not rss_feed_links:
+            print("No articles found")
+            return
+
+        # Process articles with better error handling and resource management
+        results, errors = safe_pull_articles(rss_feed_links)
+
+        print(f"Attempted to Pull {len(results)} articles in parallel.")
+        print(
+            f"Encountered {len(errors)} errors during article pulling. "
+            + "Outputting errors to a local file."
+        )
+
+        # Output errors to a local file
+        if errors:
+            with open("errors.txt", "w", encoding="utf-8") as f:
+                for error in errors:
+                    f.write(str(error) + "\n")
+            print(f"Errors logged to errors.txt")
+
+        # Print all results to a log file
+        with open("results.txt", "w", encoding="utf-8") as f:
+            for result in results:
+                f.write(result + "\n")
+        print("All articles pulled successfully.")
+
+    except Exception as e:
+        print(f"Major error in main execution: {e}")
+        import traceback
+        traceback.print_exc()
+        raise  # Re-raise to ensure the script exits with error code
+
+    print("Scraping completed successfully.")
+
+
+if __name__ == "__main__":
+    main()