StockDocs/scraper/scraper.py

import newspaper
import json
import feedparser
import time
import os
from selenium import webdriver
from selenium.webdriver.firefox.options import Options as FirefoxOptions
from newspaper.google_news import GoogleNewsSource
from concurrent.futures import ProcessPoolExecutor
import nltk
from nltk.downloader import Downloader

FEED_FILE = os.getenv("FEED_FILE")

# Ensure necessary NLTK resources are downloaded / Needed for selenium + newspaper4k article parsing
d = Downloader()
if not d.is_installed('punkt_tab'):
    nltk.download('punkt_tab')

articles = []

def load_rss_feed_sources(feed_file=FEED_FILE):
    """
    Loads the RSS feed sources from a JSON file.
    """
    print("Loading RSS feed sources from rss_feeds.json...")

    try:
        with open(FEED_FILE, "r", encoding="utf-8") as f:
            return json.load(f)
    except FileNotFoundError:
        print(FEED_FILE + " not found, returning empty list.")
        return []
    except json.JSONDecodeError:
        print("Error decoding " + FEED_FILE + " , returning empty list.")
        return []

def mine_all_articles(rss_feed_sources, limit=None):
    """
    Mines all articles from the given RSS feed sources.
    Returns a list of (site, title, link) tuples.
    """
    all_links = []
    sources = rss_feed_sources['rss_feeds']

    for site, data in sources.items():
        print(f"Parsing RSS feed: {data['rss_url']}")
        try:
            feed = feedparser.parse(data["rss_url"])
            feed_entries = feed.entries[:limit] if limit else feed.entries

            for entry in feed_entries:
                if "link" in entry and "title" in entry:
                    all_links.append((site, entry.title, entry.link))
        except Exception as e:
            print(f"Error parsing RSS feed: {site} Error: {str(e)}")
    return all_links

def generate_filename_from_url(url):
    """
    Generates a filename from the given URL by replacing slashes with underscores.
    """
    # Use only the last part of the URL or replace slashes
    return url.replace("https://", "").replace("http://", "").replace("/", "_")

def generate_safe_filename(name):
    # Remove/replace characters not allowed in filenames
    import re
    safe = re.sub(r'[\\/*?:"<>|]', "_", name)
    return safe

def save_article_to_file(article, filename, source="Unfiltered"):
    """
    Saves the given article text to a file with the specified filename.
    """
    # articles dir should already be there
    # os.makedirs("articles", exist_ok=True)

    outputDir = "articles/"+source
    os.makedirs(outputDir, exist_ok=True) if source else None

    # Sanitize filename: use only the last part of the URL or replace slashes
    safe_filename = generate_filename_from_url(filename)
    file_path = os.path.join(outputDir, safe_filename)
    # Save the source as the first line in the file for later retrieval
    with open(file_path, "w", encoding="utf-8") as f:
        f.write(f"SOURCE:{source}\n")
        f.write(article)
    print(f"Article saved to {file_path}")

def get_article_with_selenium(url):
    options = FirefoxOptions()
    options.add_argument("--headless")
    driver = webdriver.Firefox(options=options)
    driver.set_page_load_timeout(30)  # 30 seconds timeout


    #driver = webdriver.Remote(
    #command_executor='http://localhost:4444/wd/hub',
    #options=options)
    try:
        driver.get(url)
        time.sleep(5)  # Wait for JS to load
        html = driver.page_source

        # Parse with Newspaper4k
        article = newspaper.article(url, input_html=html, language='en')
        article.nlp()
        return article.text
    finally:
        driver.quit()

def get_article_with_playwright(url):
    import asyncio
    from playwright.sync_api import sync_playwright
    from newspaper import Article

    with sync_playwright() as p:
        browser = p.chromium.launch(headless=True)
        page = browser.new_page()
        page.goto(url)
        
        # Optional: wait for specific content to load
        time.sleep(5)  # Adjust as needed for the page to load completely

        html = page.content()

        browser.close()

    # Parse with Newspaper4k
    article = newspaper.article(url, input_html=html, language='en')
    article.nlp()
    return article.text


def pull_article(link, source, title=None, save_to_file=True):
    filename = title if title else link
    safe_filename = generate_filename_from_url(filename)
    if os.path.exists(os.path.join("articles", source, safe_filename)):
        print(f"Article already cached: {filename}")
        with open(os.path.join("articles", source, safe_filename), "r", encoding="utf-8") as f:
            return f.read()
        
    text = ""
    
    time.sleep(5) # Since we spawn lots of processes, we need to sleep at the start

    try:
        article = newspaper.article(link)
        article.download()
        article.parse()
        text = article.text
        if not text or len(text) < 200:
            raise ValueError("\tArticle text too short, falling back to Playwright/Selenium.")
        print(f"\tSuccessfully pulled article with newspaper4k from {link}")

    except Exception as e:
        print(f"\tnewspaper4k extraction failed for {link}: {e}, falling back to Playwright.")
        try:
            text = get_article_with_playwright(link)
            print(f"\t\tSuccessfully pulled article from {link} with Playwright")
            if not text or len(text) < 200:
                print(f"\t\tPlaywright article too short, falling back to Selenium.")
                try:
                    text = get_article_with_selenium(link)
                    print(f"\t\t\tSuccessfully pulled article from {link} with Selenium")
                except Exception as e:
                    print(f"\t\t\tSelenium failed for {link}: {e}")
                    return ""
        except Exception as e:
            print(f"\t\tPlaywright failed for {link}: {e}")
            try:
                text = get_article_with_selenium(link)
                print(f"\t\tSuccessfully pulled article from {link} with Selenium")
            except Exception as e:
                print(f"\t\tSelenium failed for {link}: {e}")
                return ""

    if save_to_file:
        save_article_to_file(text, filename, source)
    return text

while True:
    print("=========================================")
    print("Starting new scraping iteration...")
    # Pull the RSS feed sources from the JSON file
    rss_feed_sources = load_rss_feed_sources()

    # Mine all articles from the RSS feed sources
    rss_feed_links = mine_all_articles(rss_feed_sources)

    # Randomize the order of the links to help with load balancing
    import random
    random.shuffle(rss_feed_links)

    link_list = [link for _, title, link in rss_feed_links] 
    source_list = [source for source, _, _ in rss_feed_links]
    title_list = [title for _, title, link in rss_feed_links]

    with ProcessPoolExecutor() as executor:
        futures = [executor.submit(pull_article, link, source, title) for link, source, title in zip(link_list, source_list, title_list)]
        results = []
        errors = []
        for future in futures:
            try:
                results.append(future.result(timeout=60))  # seconds
            except Exception as e:
                print(f"Error in pull_article: {e}")
                errors.append(e)
        print(f"Attempted to Pull {len(results)} articles in parallel.")
        print(f"Encountered {len(errors)} errors during article pulling. " +\
            "Outputting errors to a local file.")
        
        # Ouput errors to a local file
        if errors:
            with open("errors.txt", "w", encoding="utf-8") as f:
                for error in errors:
                    f.write(str(error) + "\n")
            print(f"Errors logged to errors.txt")

        # Print all results to a log file
        with open("results.txt", "w", encoding="utf-8") as f:
            for result in results:
                f.write(result + "\n")
        print("All articles pulled successfully.")

    # Sleep for a while before the next iteration
    print("Sleeping for 15 minutes before the next iteration...")
    time.sleep(15 * 60)  # Sleep for 15 minutes