Initial commit: StockDocs RSS scraper

Carved out from the StockDocs monorepo. Collects financial news
from 60+ outlets via RSS into a structured article corpus.

README.md

@@ -0,0 +1,90 @@
+# StockDocs — Scraper
+
+RSS scraper for the StockDocs financial news platform: collects financial news from 60+ outlets (Reuters, Bloomberg, Forbes, FT, WSJ...) and stores articles in a structured directory hierarchy for the processing pipeline.
+
+Part of the StockDocs project family:
+
+| Repo | What it is |
+|------|------------|
+| [StockDocs](https://git.jarianc.com/jarianc/StockDocs) | Processing core — article server, NLP analysis, embeddings |
+| [stockdocs-mcp](https://git.jarianc.com/jarianc/stockdocs-mcp) | MCP server exposing the processed data to LLM clients |
+
+## Project Structure
+
+```
+.
+├── rss_feeds.json         # Configuration file with RSS feed URLs
+├── scraper.py             # Main scraping logic
+├── cron_scraper.py        # Scheduled scraping entry point
+├── run_scraper.sh         # Run helper
+├── requirements.txt       # Python dependencies
+├── dockerfile             # Docker configuration
+├── dockerfile-selenium    # Dockerfile for selenium-based scraping
+├── articles/              # Directory where scraped articles are stored
+│   ├── Reuters – Business News/
+│   │   ├── article1.txt
+│   │   └── ...
+│   └── ...
+└── tests/                 # Unit tests (article-processing cache)
+```
+
+## Features
+
+- **RSS Feed Integration**: Supports multiple financial news sources through RSS feeds
+- **Automated Scraping**: Regularly fetches and processes new articles from configured feeds  
+- **Structured Storage**: Organizes articles in a directory structure by news outlet
+- **Duplicate Detection**: Prevents re-processing of already collected articles
+- **Caching Mechanism**: Maintains a cache of processed articles to optimize performance
+
+## RSS Feed Sources
+
+The scraper supports 60+ news outlets including:
+
+- Reuters – Business News
+- Associated Press – Business  
+- Financial Times
+- Forbes – Real-Time
+- Wall Street Journal – U.S. Business
+- Bloomberg – Surveillance Podcast
+- CNN Money
+- BBC News – Business
+- And many more...
+
+## Usage
+
+### Running the Scraper
+
+```bash
+python scraper.py
+```
+
+### Configuration
+
+The scraper can be configured by modifying `rss_feeds.json` to:
+- Add new news sources
+- Update existing RSS feed URLs  
+- Remove sources that are no longer active
+
+### Article Storage
+
+Articles are stored in `articles/` directory with the following structure:
+
+```
+articles/
+└── <News Outlet Name>/
+    ├── article1.txt
+    ├── article2.txt
+    └── ...
+```
+
+Where each article file contains the full text content of that news article.
+
+## Requirements
+
+- Python 3.6+
+- Selenium WebDriver (for certain scraping operations)
+- Additional dependencies listed in `requirements.txt`
+
+## License
+
+This project is licensed under the MIT License.

cron_scraper.py

@@ -0,0 +1,488 @@
+#!/usr/bin/env python3
+"""
+Cron-based scraper for downloading articles from RSS feeds.
+This version replaces the infinite while loop with a single execution
+that can be scheduled via cron job.
+"""
+
+import newspaper
+import json
+import feedparser
+import time
+import os
+import requests
+import logging
+import random
+from selenium import webdriver
+from selenium.webdriver.firefox.options import Options as FirefoxOptions
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+from concurrent.futures import ThreadPoolExecutor, as_completed
+import nltk
+from nltk.downloader import Downloader
+
+# Setup logging with timestamps
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(levelname)s - %(message)s',
+    datefmt='%Y-%m-%d %H:%M:%S'
+)
+logger = logging.getLogger(__name__)
+
+# Rotating User-Agents to bypass bot detection (Reuters, etc.)
+USER_AGENTS = [
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Safari/605.1.15",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:134.0) Gecko/20100101 Firefox/134.0",
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
+]
+
+def get_random_ua():
+    return random.choice(USER_AGENTS)
+
+# Robust file path handling - try multiple locations
+def get_feed_file_path():
+    """Get the RSS feed file path, trying multiple locations."""
+    possible_paths = [
+        "./rss_feeds.json",           # Current directory
+        "../rss_feeds.json",          # Parent directory
+        "/app/rss_feeds.json",        # Docker path
+        "./scraper/rss_feeds.json"    # Scraper subdirectory
+    ]
+
+    for path in possible_paths:
+        if os.path.exists(path):
+            print(f"Found feed file at: {path}")
+            return path
+
+    # If no file found, exit the program
+    print("Error: RSS feed file not found in any expected location")
+    print("Exiting program...")
+    exit(1)
+
+# Get the feed file path
+FEED_FILE = get_feed_file_path()
+
+# Ensure necessary NLTK resources are downloaded
+d = Downloader()
+if not d.is_installed("punkt_tab"):
+    nltk.download("punkt_tab")
+
+articles = []
+
+
+def load_rss_feed_sources(feed_file=FEED_FILE):
+    """
+    Loads the RSS feed sources from a JSON file.
+    """
+    logger.info(f"Loading RSS feed sources from {feed_file}...")
+
+    # Debug: Print current working directory
+    logger.debug(f"Current working directory: {os.getcwd()}")
+
+    try:
+        with open(feed_file, "r", encoding="utf-8") as f:
+            data = json.load(f)
+            logger.info(f"Successfully loaded {feed_file}")
+            logger.debug(f"Data type: {type(data)}")
+            if isinstance(data, dict) and "rss_feeds" in data:
+                logger.info(f"Found rss_feeds section with {len(data['rss_feeds'])} sources")
+                return data
+            else:
+                logger.warning(f"Unexpected data structure. Data keys: {list(data.keys()) if isinstance(data, dict) else 'Not a dict'}")
+                return {}
+    except FileNotFoundError:
+        logger.error(f"{feed_file} not found, returning empty dict.")
+        return {}
+    except json.JSONDecodeError as e:
+        logger.error(f"Error decoding {feed_file}: {e}, returning empty dict.")
+        return {}
+    except Exception as e:
+        logger.error(f"Unexpected error loading {feed_file}: {e}")
+        return {}
+
+
+def mine_all_articles(rss_feed_sources, limit=None):
+    """
+    Mines all articles from the given RSS feed sources.
+    Returns a list of (site, title, link) tuples.
+    """
+    all_links = []
+
+    # Check if rss_feed_sources is a valid dict with rss_feeds key
+    if not isinstance(rss_feed_sources, dict):
+        logger.warning(f"rss_feed_sources is not a dict, it's {type(rss_feed_sources)}")
+        return all_links
+
+    if "rss_feeds" not in rss_feed_sources:
+        logger.warning("rss_feeds key not found in rss_feed_sources")
+        return all_links
+
+    sources = rss_feed_sources["rss_feeds"]
+
+    for site, data in sources.items():
+        logger.info(f"Parsing RSS feed: {data['rss_url']}")
+        try:
+            feed = feedparser.parse(data["rss_url"])
+            feed_entries = feed.entries[:limit] if limit else feed.entries
+
+            for entry in feed_entries:
+                if "link" in entry and "title" in entry:
+                    all_links.append((site, entry.title, entry.link))
+        except Exception as e:
+            logger.error(f"Error parsing RSS feed: {site} Error: {str(e)}")
+    return all_links
+
+
+def generate_filename_from_url(url):
+    """
+    Generates a filename from the given URL by replacing slashes with underscores.
+    """
+    # Use only the last part of the URL or replace slashes
+    return url.replace("https://", "").replace("http://", "").replace("/", "_")
+
+
+def generate_safe_filename(name):
+    # Remove/replace characters not allowed in filenames
+    import re
+
+    safe = re.sub(r'[\\/*?:"<>|]', "_", name)
+    return safe
+
+
+def save_article_to_file(article, filename, source="Unfiltered"):
+    """
+    Saves the given article text to a file with the specified filename.
+    """
+    # articles dir should already be there
+    # os.makedirs("articles", exist_ok=True)
+
+    outputDir = "articles/" + source
+    os.makedirs(outputDir, exist_ok=True) if source else None
+
+    # Sanitize filename: use only the last part of the URL or replace slashes
+    safe_filename = generate_filename_from_url(filename)
+    file_path = os.path.join(outputDir, safe_filename)
+    # Save the source as the first line in the file for later retrieval
+    with open(file_path, "w", encoding="utf-8") as f:
+        f.write(f"SOURCE:{source}\n")
+        f.write(article)
+
+    # Only log when a new file is actually created (not cached)
+    logger.info(f"New article saved: {safe_filename} from {source}")
+
+
+def get_article_with_selenium(url):
+    """
+    Gets article text using Selenium Firefox driver with proper error handling,
+    cleanup, and bot-detection evasion.
+    """
+    driver = None
+    try:
+        # Configure Firefox options with bot-detection evasion
+        options = FirefoxOptions()
+        options.add_argument("--headless")
+        options.set_preference("dom.ipc.processCount", 1)
+        options.set_preference("general.useragent.override", get_random_ua())
+        options.set_preference("permissions.default.image", 2)
+        options.set_preference("dom.webnotifications.enabled", False)
+
+        # Initialize driver with timeout
+        driver = webdriver.Firefox(options=options)
+        driver.set_page_load_timeout(30)
+
+        # Navigate to URL
+        driver.get(url)
+
+        # Wait for page to load (explicit wait instead of sleep)
+        try:
+            WebDriverWait(driver, 15).until(
+                EC.presence_of_element_located((By.TAG_NAME, "body"))
+            )
+        except Exception:
+            pass
+
+        time.sleep(random.uniform(1, 3))
+
+        html = driver.page_source
+
+        # Parse with Newspaper4k
+        article = newspaper.article(url, input_html=html, language="en")
+        article.nlp()
+        logger.info(f"Successfully extracted article with Selenium from {url}")
+        return article.text
+
+    except Exception as e:
+        logger.error(f"Selenium failed for {url}: {str(e)}")
+        return ""
+    finally:
+        if driver:
+            try:
+                driver.quit()
+            except Exception:
+                pass
+
+
+def get_article_with_playwright(url):
+    """
+    Gets article text using Playwright with proper bot-detection evasion.
+    """
+    try:
+        from playwright.sync_api import sync_playwright
+
+        with sync_playwright() as p:
+            browser = p.chromium.launch(headless=True, timeout=30000)
+            context = browser.new_context(
+                user_agent=get_random_ua(),
+                viewport={"width": 1920, "height": 1080},
+                locale="en-US",
+                timezone_id="America/New_York",
+            )
+            page = context.new_page()
+
+            page.set_extra_http_headers({
+                "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+                "Accept-Language": "en-US,en;q=0.9",
+                "Accept-Encoding": "gzip, deflate, br",
+                "Connection": "keep-alive",
+                "Upgrade-Insecure-Requests": "1",
+            })
+
+            page.goto(url, wait_until="domcontentloaded", timeout=30000)
+            time.sleep(random.uniform(2, 4))
+
+            html = page.content()
+            context.close()
+            browser.close()
+
+        article = newspaper.article(url, input_html=html, language="en")
+        article.nlp()
+        logger.info(f"Successfully extracted article with Playwright from {url}")
+        return article.text
+
+    except Exception as e:
+        logger.error(f"Playwright failed for {url}: {str(e)}")
+        return ""
+
+
+def pull_article(link, source, title=None, save_to_file=True):
+    """
+    Pulls an article from a given link with fallback mechanisms.
+    """
+    filename = title if title else link
+    safe_filename = generate_filename_from_url(filename)
+
+    # Check if already cached
+    if os.path.exists(os.path.join("articles", source, safe_filename)):
+        logger.info(f"Article already cached: {filename}")
+        with open(
+            os.path.join("articles", source, safe_filename), "r", encoding="utf-8"
+        ) as f:
+            return f.read()
+
+    # Random delay before fetching to avoid rate-limiting / bot detection
+    time.sleep(random.uniform(0.5, 2))
+
+    text = ""
+
+    try:
+        # Try newspaper4k first with proper User-Agent to bypass bot detection
+        ua = get_random_ua()
+        article = newspaper.article(link, browser_user_agent=ua)
+        article.download()
+        article.parse()
+        text = article.text
+
+        if not text or len(text) < 200:
+            raise ValueError(
+                "\tArticle text too short, falling back to Playwright/Selenium."
+            )
+        logger.info(f"Successfully pulled article with newspaper4k from {link}")
+
+    except Exception as e:
+        logger.warning(
+            f"newspaper4k extraction failed for {link}: {e}, falling back to Playwright."
+        )
+
+        try:
+            text = get_article_with_playwright(link)
+            logger.info(f"Successfully pulled article from {link} with Playwright")
+
+            if not text or len(text) < 200:
+                logger.warning("Playwright article too short, falling back to Selenium.")
+                # Fallback to Selenium with better error handling
+                text = get_article_with_selenium(link)
+                logger.info(f"Successfully pulled article from {link} with Selenium")
+        except Exception as e:
+            logger.error(f"Playwright failed for {link}: {e}")
+
+            # Fallback to Selenium
+            try:
+                text = get_article_with_selenium(link)
+                logger.info(f"Successfully pulled article from {link} with Selenium")
+            except Exception as e:
+                logger.error(f"Selenium failed for {link}: {e}")
+                return ""
+
+    if save_to_file:
+        save_article_to_file(text, filename, source)
+
+    return text
+
+
+def safe_pull_articles(article_list):
+    """
+    Safely pull articles with improved error handling and reduced parallelism.
+    """
+    results = []
+    errors = []
+
+    # Process in smaller batches to reduce resource strain
+    batch_size = 5
+
+    for i in range(0, len(article_list), batch_size):
+        batch = article_list[i : i + batch_size]
+        logger.info(f"Processing batch {i // batch_size + 1} with {len(batch)} articles")
+
+        # Use ThreadPoolExecutor instead of ProcessPoolExecutor to avoid
+        # process termination issues with browser automation
+        with ThreadPoolExecutor(max_workers=3) as executor:  # Reduced workers
+            futures = [
+                executor.submit(pull_article, link, source, title)
+                for source, title, link in batch
+            ]
+
+            for future in as_completed(futures):
+                try:
+                    result = future.result(timeout=120)  # 2 minute timeout
+                    results.append(result)
+                except Exception as e:
+                    logger.error(f"Error in pull_article: {e}")
+                    errors.append(e)
+
+        # Add a small delay between batches to reduce system load
+        time.sleep(5)
+
+    return results, errors
+
+
+def gather_new_articles():
+    """
+    Gather list of all newly downloaded articles and format them for webhook.
+    """
+    new_articles = []
+
+    # Walk through all article directories
+    for root, dirs, files in os.walk("articles"):
+        for file in files:
+            if file != "processed_articles_cache.json":  # Skip cache file
+                # Get the full file path
+                file_path = os.path.join(root, file)
+
+                # Get the outlet name from the directory path
+                outlet = os.path.basename(root)
+
+                # Create the relative path for the article
+                relative_path = os.path.relpath(file_path, "scraper")
+
+                # Create article data structure
+                article_data = {
+                    "created_at": time.strftime("%Y-%m-%dT%H:%M:%S.%f", time.localtime(os.path.getctime(file_path))),
+                    "name": file,
+                    "outlet": outlet,
+                    "path": f"../{relative_path}"
+                }
+
+                new_articles.append(article_data)
+
+    return new_articles
+
+
+def send_to_webhook(articles):
+    """
+    Send list of articles to the webhook URL.
+    """
+    webhook_url = "http://agents.example.com/webhook/49c5b169-c68c-4f8c-90c2-0fcca6e2d387"
+    headers = {
+        "StockDocsN8NAuthToken": "ganvT4gsgRjWpGE8FMw9uCzFjZrTx8RZCoVm2Dh7skbZecov"
+    }
+
+    try:
+        response = requests.post(webhook_url, json=articles, headers=headers, timeout=30)
+        if response.status_code == 200:
+            print(f"Successfully sent {len(articles)} articles to webhook")
+        else:
+            print(f"Webhook request failed with status code: {response.status_code}")
+            print(f"Response: {response.text}")
+    except Exception as e:
+        print(f"Error sending to webhook: {e}")
+
+
+def main():
+    """
+    Main scraping function for cron execution.
+    This replaces the infinite while loop with a single execution.
+    """
+    logger.info("=========================================")
+    logger.info("Starting new scraping iteration...")
+
+    try:
+        # Pull the RSS feed sources from the JSON file
+        rss_feed_sources = load_rss_feed_sources()
+
+        # Mine all articles from the RSS feed sources
+        rss_feed_links = mine_all_articles(rss_feed_sources)
+
+        # Randomize the order of the links to help with load balancing
+        import random
+
+        random.shuffle(rss_feed_links)
+
+        logger.info(f"Found {len(rss_feed_links)} articles to process")
+
+        if not rss_feed_links:
+            logger.info("No articles found")
+            return
+
+        # Process articles with better error handling and resource management
+        results, errors = safe_pull_articles(rss_feed_links)
+
+        logger.info(f"Attempted to Pull {len(results)} articles in parallel.")
+        logger.info(
+            f"Encountered {len(errors)} errors during article pulling. "
+            + "Outputting errors to a local file."
+        )
+
+        # Output errors to a local file
+        if errors:
+            with open("errors.txt", "w", encoding="utf-8") as f:
+                for error in errors:
+                    f.write(str(error) + "\n")
+            logger.info("Errors logged to errors.txt")
+
+        # Print all results to a log file
+        with open("results.txt", "w", encoding="utf-8") as f:
+            for result in results:
+                f.write(result + "\n")
+        logger.info("All articles pulled successfully.")
+
+        # Gather and send new articles to webhook
+        new_articles = gather_new_articles()
+        if new_articles:
+            send_to_webhook(new_articles)
+        else:
+            logger.info("No new articles to send to webhook")
+
+    except Exception as e:
+        logger.error(f"Major error in main execution: {e}")
+        import traceback
+        traceback.print_exc()
+        raise  # Re-raise to ensure the script exits with error code
+
+    logger.info("Scraping completed successfully.")
+
+
+if __name__ == "__main__":
+    main()

dockerfile

@@ -0,0 +1,20 @@
+# --- Dockerfile.local ---
+FROM python:3.13.5 
+
+# Firefox + GeckoDriver
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        firefox-esr wget ca-certificates gnupg2 \
+ && GECKO=v0.36.0 && \
+    wget -qO- "https://github.com/mozilla/geckodriver/releases/download/${GECKO}/geckodriver-${GECKO}-linux64.tar.gz" \
+    | tar -xz -C /usr/local/bin geckodriver \
+ && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+RUN playwright install
+RUN playwright install-deps
+RUN python3 -m nltk.downloader punkt_tab # Download NLTK data and bake into image
+COPY . .
+
+CMD ["python3", "scraper.py"]

dockerfile-selenium

@@ -0,0 +1,22 @@
+# Use an official Selenium Firefox standalone as a base image
+FROM selenium/standalone-firefox:latest
+
+USER root
+# Set the working directory in the container
+WORKDIR /app
+
+# Install Python dependencies
+COPY requirements.txt /app/
+RUN pip install --break-system-packages --no-cache-dir -r requirements.txt && \
+    playwright install
+
+# Copy the current directory contents into the container at /app
+COPY . /app
+RUN mkdir -p /app/articles && \
+    chown seluser:seluser /app/articles && \
+    chmod 755 /app/articles
+
+USER seluser                            
+# Run the Flask app
+CMD ["python3", "scraper.py"]
+

requirements.txt

@@ -0,0 +1,43 @@
+attrs
+beautifulsoup4
+certifi
+charset-normalizer
+click
+dnspython
+feedparser
+filelock
+gnews
+greenlet
+h11
+idna
+joblib
+lxml
+lxml-html-clean
+newspaper4k
+nltk
+numpy
+outcome
+pandas
+pillow
+playwright
+pyee
+pysocks
+python-dateutil
+pytz
+pyyaml
+regex
+requests
+requests-file
+selenium
+sgmllib3k
+six
+sniffio
+sortedcontainers
+soupsieve
+tldextract
+tqdm
+trio
+trio-websocket
+tzdata
+websocket-client
+wsproto

requirements_clean.txt

@@ -0,0 +1,43 @@
+attrs
+beautifulsoup4
+certifi
+charset-normalizer
+click
+dnspython
+feedparser
+filelock
+gnews
+greenlet
+h11
+idna
+joblib
+lxml
+lxml-html-clean
+newspaper4k
+nltk
+numpy
+outcome
+pandas
+pillow
+playwright
+pyee
+pysocks
+python-dateutil
+pytz
+pyyaml
+regex
+requests
+requests-file
+selenium
+sgmllib3k
+six
+sniffio
+sortedcontainers
+soupsieve
+tldextract
+tqdm
+trio
+trio-websocket
+tzdata
+websocket-client
+wsproto

rss_feeds.json

@@ -0,0 +1,196 @@
+{
+  "rss_feeds": {
+    "Reuters – Business News": {
+      "source_website": "reuters.com",
+      "rss_url": "https://www.reutersagency.com/feed/"
+    },
+    "Associated Press – Business": {
+      "source_website": "apnews.com",
+      "rss_url": "https://rsshub.app/apnews/topics/apf-topnews"
+    },
+    "Financial Times": {
+      "source_website": "ft.com",
+      "rss_url": "https://www.ft.com/rss/home"
+    },
+    "Fortune – Top Stories": {
+      "source_website": "fortune.com",
+      "rss_url": "https://fortune.com/feed/fortune-feeds/?id=3230629"
+    },
+    "Seeking Alpha – Market News": {
+      "source_website": "seekingalpha.com",
+      "rss_url": "https://seekingalpha.com/feed.xml"
+    },
+    "The Motley Fool – Stock News & Analysis": {
+      "source_website": "fool.com",
+      "rss_url": "https://www.fool.com/a/feeds/partner/googlechromefollow?apikey=5e092c1f-c5f9-4428-9219-908a47d2e2de"
+    },
+    "Business Standard – Latest News": {
+      "source_website": "business-standard.com",
+      "rss_url": "https://www.business-standard.com/rss/latest.rss"
+    },
+    "TheStreet – Full Articles": {
+      "source_website": "thestreet.com",
+      "rss_url": "https://www.thestreet.com/.rss/full"
+    },
+    "Benzinga – Financial News": {
+      "source_website": "benzinga.com",
+      "rss_url": "https://feeds.benzinga.com/benzinga"
+    },
+    "MarketBeat – Market News": {
+      "source_website": "marketbeat.com",
+      "rss_url": "https://www.marketbeat.com/feed/"
+    },
+    "Money (Time) – Personal Finance": {
+      "source_website": "money.com",
+      "rss_url": "https://money.com/money/feed/"
+    },
+    "Global Finance Magazine": {
+      "source_website": "gfmag.com",
+      "rss_url": "https://www.gfmag.com/feed"
+    },
+    "Financial Samurai": {
+      "source_website": "financialsamurai.com",
+      "rss_url": "https://www.financialsamurai.com/feed/"
+    },
+    "MoneyWeek": {
+      "source_website": "moneyweek.com",
+      "rss_url": "https://moneyweek.com/feed/all"
+    },
+    "Finance Monthly": {
+      "source_website": "finance-monthly.com",
+      "rss_url": "https://www.finance-monthly.com/feed/"
+    },
+    "European Financial Review": {
+      "source_website": "europeanfinancialreview.com",
+      "rss_url": "https://www.europeanfinancialreview.com/feed"
+    },
+    "Money Morning": {
+      "source_website": "moneymorning.com",
+      "rss_url": "https://moneymorning.com/feed"
+    },
+    "Dealbreaker": {
+      "source_website": "dealbreaker.com",
+      "rss_url": "https://dealbreaker.com/.rss/full"
+    },
+    "World Finance": {
+      "source_website": "worldfinance.com",
+      "rss_url": "https://www.worldfinance.com/feed"
+    },
+    "Fox Business – Headlines": {
+      "source_website": "foxbusiness.com",
+      "rss_url": "https://moxie.foxbusiness.com/google-publisher/latest.xml"
+    },
+    "FinanceAsia": {
+      "source_website": "financeasia.com",
+      "rss_url": "https://www.financeasia.com/rss/latest"
+    },
+    "CNBC – Business": {
+      "source_website": "cnbc.com",
+      "rss_url": "https://www.cnbc.com/id/100003114/device/rss/rss.html"
+    },
+    "CNN Money": {
+      "source_website": "cnn.com",
+      "rss_url": "http://rss.cnn.com/rss/money_topstories.rss"
+    },
+    "Markets Insider": {
+      "source_website": "markets.businessinsider.com",
+      "rss_url": "https://markets.businessinsider.com/rss/news"
+    },
+    "The Economist – Business & Finance": {
+      "source_website": "economist.com",
+      "rss_url": "https://www.economist.com/business/rss.xml"
+    },
+    "Barchart News": {
+      "source_website": "barchart.com",
+      "rss_url": "http://feeds.feedburner.com/BarchartNews"
+    },
+    "The Guardian – Business": {
+      "source_website": "theguardian.com",
+      "rss_url": "http://feeds.theguardian.com/theguardian/uk/business/rss"
+    },
+    "Economy Watch": {
+      "source_website": "economywatch.com",
+      "rss_url": "https://www.economywatch.com/feed"
+    },
+    "CFI.co": {
+      "source_website": "cfi.co",
+      "rss_url": "https://cfi.co/feed"
+    },
+    "BBC News – Business": {
+      "source_website": "bbc.co.uk",
+      "rss_url": "http://feeds.bbci.co.uk/news/business/rss.xml"
+    },
+    "Investor’s Business Daily": {
+      "source_website": "investors.com",
+      "rss_url": "https://www.investors.com/feed/"
+    },
+    "Forbes – Real-Time": {
+      "source_website": "forbes.com",
+      "rss_url": "https://www.forbes.com/real-time/feed2/"
+    },
+    "The Financial Express": {
+      "source_website": "financialexpress.com",
+      "rss_url": "https://www.financialexpress.com/feed/"
+    },
+    "MarketWatch – Top Stories": {
+      "source_website": "marketwatch.com",
+      "rss_url": "http://feeds.marketwatch.com/marketwatch/topstories/"
+    },
+    "Wall Street Journal – U.S. Business": {
+      "source_website": "wsj.com",
+      "rss_url": "https://feeds.a.dj.com/rss/WSJcomUSBusiness.xml"
+    },
+    "Sky News – Business": {
+      "source_website": "news.sky.com",
+      "rss_url": "http://news.sky.com/feeds/rss/business.xml"
+    },
+    "Bloomberg – Surveillance Podcast": {
+      "source_website": "bloomberg.com",
+      "rss_url": "https://www.bloomberg.com/feed/podcast/bloomberg-surveillance.xml"
+    },
+    "Barron’s – Markets": {
+      "source_website": "barrons.com",
+      "rss_url": "https://www.barrons.com/xml/rss/markets.xml"
+    },
+    "Yahoo Finance": {
+      "source_website": "finance.yahoo.com",
+      "rss_url": "https://www.yahoo.com/news/rss/finance"
+    },
+    "Investing.com – News": {
+      "source_website": "investing.com",
+      "rss_url": "https://www.investing.com/rss/news.rss"
+    },
+    "Investopedia – Headlines": {
+      "source_website": "investopedia.com",
+      "rss_url": "https://www.investopedia.com/feedbuilder/feed/getfeed/?feedName=rss_headline"
+    },
+    "NerdWallet – Finance": {
+      "source_website": "nerdwallet.com",
+      "rss_url": "https://www.nerdwallet.com/news/finance/feed"
+    },
+    "Newsmax Finance": {
+      "source_website": "newsmax.com",
+      "rss_url": "https://www.newsmax.com/rss/finance"
+    },
+    "Bankrate – News": {
+      "source_website": "bankrate.com",
+      "rss_url": "https://www.bankrate.com/rss/"
+    },
+    "Morningstar – Articles": {
+      "source_website": "morningstar.com",
+      "rss_url": "https://www.morningstar.com/articles.rss"
+    },
+    "Kiplinger": {
+      "source_website": "kiplinger.com",
+      "rss_url": "https://www.kiplinger.com/kiplinger.rss"
+    },
+    "International Business Times": {
+      "source_website": "ibtimes.com",
+      "rss_url": "https://www.ibtimes.com/rss"
+    },
+    "Policygenius – News": {
+      "source_website": "policygenius.com",
+      "rss_url": "https://www.policygenius.com/news/feed/"
+    }
+  }
+}

rss_short_feed.json

@@ -0,0 +1,8 @@
+{
+  "rss_feeds": {
+    "Associated Press – Business": {
+      "source_website": "apnews.com",
+      "rss_url": "https://news.google.com/rss/search?q=site:apnews.com&hl=en-US&gl=US&ceid=US:en"
+    }
+  }
+}

run_scraper.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+# ------------------------------------------------------------------
+# run_scraper.sh
+#
+# Activates the venv *implicitly* by calling the venv’s Python binary.
+# ------------------------------------------------------------------
+# Absolute path to the venv – change only if you move the venv.
+VENV_DIR="/home/user/StockDocs/scraper/venv"
+
+# Absolute path to the script you want to run.
+SCRIPT="/home/user/StockDocs/scraper/cron_scraper.py"
+
+# Invoke the venv’s Python directly.
+"${VENV_DIR}/bin/python" "${SCRIPT}"  # output is redirected by cron

scraper.py

@@ -0,0 +1,568 @@
+import newspaper
+import json
+import feedparser
+import time
+import os
+import logging
+import random
+from datetime import datetime
+from selenium import webdriver
+from selenium.webdriver.firefox.options import Options as FirefoxOptions
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+from concurrent.futures import ThreadPoolExecutor, as_completed, TimeoutError
+import nltk
+from nltk.downloader import Downloader
+
+# Setup logging with timestamps
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(levelname)s - %(message)s',
+    datefmt='%Y-%m-%d %H:%M:%S'
+)
+logger = logging.getLogger(__name__)
+
+FEED_FILE = os.getenv("FEED_FILE", "./rss_short_feed.json")
+MAX_FEED_WORKERS = int(os.getenv("MAX_FEED_WORKERS", "10"))
+MAX_ARTICLE_WORKERS = int(os.getenv("MAX_ARTICLE_WORKERS", "10"))
+BATCH_SIZE = int(os.getenv("BATCH_SIZE", "50"))  # Batch processing size
+
+# Rotating User-Agents to bypass bot detection (Reuters, etc.)
+USER_AGENTS = [
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Safari/605.1.15",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:134.0) Gecko/20100101 Firefox/134.0",
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
+]
+
+def get_random_ua():
+    return random.choice(USER_AGENTS)
+
+# Ensure necessary NLTK resources are downloaded
+d = Downloader()
+if not d.is_installed("punkt_tab"):
+    nltk.download("punkt_tab")
+
+articles = []
+
+
+# Enhanced cache system
+def load_processed_cache():
+    """Load the processed articles cache with enhanced tracking"""
+    cache_path = "articles/processed_articles_cache.json"
+    try:
+        if os.path.exists(cache_path):
+            with open(cache_path, 'r', encoding='utf-8') as f:
+                return json.load(f)
+        else:
+            return {}
+    except Exception as e:
+        logger.error(f"Error loading cache: {e}")
+        return {}
+
+def save_processed_cache(cache_data):
+    """Save the processed articles cache with enhanced tracking"""
+    cache_path = "articles/processed_articles_cache.json"
+    try:
+        with open(cache_path, 'w', encoding='utf-8') as f:
+            json.dump(cache_data, f, indent=2, ensure_ascii=False)
+        logger.info(f"Cache saved with {len(cache_data)} entries")
+    except Exception as e:
+        logger.error(f"Error saving cache: {e}")
+
+def is_article_processed(article_path, cache_data):
+    """Check if an article has been processed"""
+    return article_path in cache_data
+
+def mark_article_processed(article_path, status="completed", embedding_status="pending"):
+    """Mark an article as processed with detailed status tracking"""
+    cache_data = load_processed_cache()
+    cache_data[article_path] = {
+        "processed_date": datetime.now().isoformat(),
+        "status": status,
+        "embedding_status": embedding_status,
+        "last_updated": datetime.now().isoformat()
+    }
+    save_processed_cache(cache_data)
+
+def get_processing_progress():
+    """Get overall processing progress"""
+    cache_data = load_processed_cache()
+    total_articles = len(cache_data)
+    completed_articles = sum(1 for data in cache_data.values() if data.get('status') == 'completed')
+    embedded_articles = sum(1 for data in cache_data.values() if data.get('embedding_status') == 'completed')
+
+    return {
+        "total_articles": total_articles,
+        "completed_articles": completed_articles,
+        "embedded_articles": embedded_articles,
+        "completion_rate": (completed_articles / total_articles * 100) if total_articles > 0 else 0
+    }
+
+
+def load_rss_feed_sources(feed_file=FEED_FILE):
+    """
+    Loads the RSS feed sources from a JSON file.
+    """
+    logger.info(f"Loading RSS feed sources from {feed_file}...")
+
+    try:
+        with open(feed_file, "r", encoding="utf-8") as f:
+            data = json.load(f)
+            logger.info(f"Successfully loaded {feed_file}")
+            return data
+    except FileNotFoundError:
+        logger.error(f"{feed_file} not found, returning empty list.")
+        return []
+    except json.JSONDecodeError:
+        logger.error(f"Error decoding {feed_file} , returning empty list.")
+        return []
+
+
+def mine_all_articles(rss_feed_sources, limit=None):
+    """
+    Mines all articles from the given RSS feed sources.
+    Returns a list of (site, title, link) tuples.
+    """
+    all_links = []
+
+    # Check if rss_feed_sources is a valid dict with rss_feeds key
+    if not isinstance(rss_feed_sources, dict):
+        logger.warning(f"rss_feed_sources is not a dict, it's {type(rss_feed_sources)}")
+        return all_links
+
+    if "rss_feeds" not in rss_feed_sources:
+        logger.warning("rss_feeds key not found in rss_feed_sources")
+        return all_links
+
+    sources = rss_feed_sources["rss_feeds"]
+
+    # Parse RSS feeds in parallel for better performance
+    def parse_single_feed(site, data):
+        """Parse a single RSS feed with timeout and error handling"""
+        try:
+            logger.info(f"Parsing RSS feed: {data['rss_url']}")
+            # Add more aggressive timeout settings with fallback
+            # Use a wrapper to ensure we don't hang indefinitely
+            import signal
+
+            def timeout_handler(signum, frame):
+                raise TimeoutError(f"Timeout parsing feed: {site}")
+
+            # Set up signal-based timeout (this is a fallback for truly hanging requests)
+            old_handler = signal.signal(signal.SIGALRM, timeout_handler)
+            signal.alarm(10)  # 10 second alarm
+
+            feed = feedparser.parse(data["rss_url"], timeout=8)  # 8 second timeout
+            signal.alarm(0)  # Cancel the alarm
+            signal.signal(signal.SIGALRM, old_handler)
+
+            feed_entries = feed.entries[:limit] if limit else feed.entries
+
+            entries = []
+            for entry in feed_entries:
+                if "link" in entry and "title" in entry:
+                    entries.append((site, entry.title, entry.link))
+            return entries
+        except TimeoutError as e:
+            logger.error(f"Timeout parsing RSS feed: {site} Error: {str(e)}")
+            return []
+        except Exception as e:
+            error_str = str(e).lower()
+            # Handle specific network connection issues
+            if "remote end closed connection" in error_str or "connection closed" in error_str:
+                logger.warning(f"Network connection closed by remote end for feed: {site} - {str(e)}")
+                logger.info(f"Skipping problematic feed: {site}")
+                return []
+            elif "timeout" in error_str:
+                logger.error(f"Timeout parsing RSS feed: {site} Error: {str(e)}")
+                return []
+            else:
+                logger.error(f"Error parsing RSS feed: {site} Error: {str(e)}")
+                return []
+
+    # Use ThreadPoolExecutor for parallel RSS feed parsing
+    from concurrent.futures import ThreadPoolExecutor, as_completed
+    max_workers = min(10, len(sources))  # Limit concurrent workers
+
+    with ThreadPoolExecutor(max_workers=max_workers) as executor:
+        # Submit all feed parsing tasks
+        future_to_site = {
+            executor.submit(parse_single_feed, site, data): site
+            for site, data in sources.items()
+        }
+
+        # Collect results as they complete
+        for future in as_completed(future_to_site, timeout=30):  # 30 second overall timeout
+            try:
+                entries = future.result()
+                all_links.extend(entries)
+            except Exception as e:
+                site = future_to_site[future]
+                logger.error(f"Error processing feed for {site}: {str(e)}")
+
+    return all_links
+
+
+def generate_filename_from_url(url):
+    """
+    Generates a filename from the given URL by replacing slashes with underscores.
+    """
+    # Use only the last part of the URL or replace slashes
+    filename = url.replace("https://", "").replace("http://", "").replace("/", "_")
+    # Sanitize filename to remove/replace invalid characters
+    import re
+    filename = re.sub(r'[\\/*?:"<>|]', "_", filename)
+    return filename
+
+
+def generate_safe_filename(name):
+    # Remove/replace characters not allowed in filenames
+    import re
+
+    safe = re.sub(r'[\\/*?:"<>|]', "_", name)
+    return safe
+
+
+def save_article_to_file(article, filename, source="Unfiltered"):
+    """
+    Saves the given article text to a file with the specified filename.
+    """
+    # articles dir should already be there
+    # os.makedirs("articles", exist_ok=True)
+
+    outputDir = "articles/" + source
+    os.makedirs(outputDir, exist_ok=True) if source else None
+
+    # Sanitize filename: use only the last part of the URL or replace slashes
+    safe_filename = generate_filename_from_url(filename)
+    file_path = os.path.join(outputDir, safe_filename)
+    # Save the source as the first line in the file for later retrieval
+    with open(file_path, "w", encoding="utf-8") as f:
+        f.write(f"SOURCE:{source}\n")
+        f.write(article)
+
+    # Only log when a new file is actually created (not cached)
+    logger.info(f"New article saved: {safe_filename} from {source}")
+
+
+def get_article_with_selenium(url):
+    """
+    Gets article text using Selenium Firefox driver with proper error handling,
+    cleanup, and bot-detection evasion.
+    """
+    driver = None
+    try:
+        # Configure Firefox options with bot-detection evasion
+        options = FirefoxOptions()
+        options.add_argument("--headless")
+        options.set_preference("dom.ipc.processCount", 1)
+        options.set_preference("general.useragent.override", get_random_ua())
+        options.set_preference("permissions.default.image", 2)  # Block images for speed
+        options.set_preference("dom.webnotifications.enabled", False)
+
+        # Try to initialize driver with explicit path to Firefox
+        try:
+            driver = webdriver.Firefox(options=options)
+        except Exception as e:
+            # If that fails, try with explicit Firefox path
+            if "binary is not a firefox executable" in str(e).lower():
+                logger.info("Attempting to use Firefox at /usr/bin/firefox")
+                options.binary_location = "/usr/bin/firefox"
+                driver = webdriver.Firefox(options=options)
+            else:
+                raise e
+
+        driver.set_page_load_timeout(30)  # 30 seconds timeout
+
+        # Navigate to URL
+        driver.get(url)
+
+        # Wait for page to load (explicit wait instead of sleep)
+        try:
+            WebDriverWait(driver, 15).until(
+                EC.presence_of_element_located((By.TAG_NAME, "body"))
+            )
+        except Exception:
+            pass  # Continue even if wait times out
+
+        time.sleep(random.uniform(1, 3))  # Random wait to mimic human behavior
+
+        html = driver.page_source
+
+        # Parse with Newspaper4k
+        article = newspaper.article(url, input_html=html, language="en")
+        article.nlp()
+        logger.info(f"Successfully extracted article with Selenium from {url}")
+        return article.text
+
+    except Exception as e:
+        # Check if this is a Firefox binary not found error
+        error_str = str(e).lower()
+        if "binary is not a firefox executable" in error_str or "firefox" in error_str:
+            logger.error(f"Firefox not found or not properly configured for {url}: {str(e)}")
+            logger.error("Firefox is installed at /usr/bin/firefox but may not be accessible. Check PATH or permissions.")
+        else:
+            logger.error(f"Selenium failed for {url}: {str(e)}")
+        return ""
+    finally:
+        # Always quit the driver
+        if driver:
+            try:
+                driver.quit()
+            except Exception:
+                pass  # Ignore errors in cleanup
+
+
+def get_article_with_playwright(url):
+    """
+    Gets article text using Playwright with proper bot-detection evasion.
+    """
+    try:
+        from playwright.sync_api import sync_playwright
+
+        with sync_playwright() as p:
+            # Use Chromium with full browser context for UA spoofing
+            browser = p.chromium.launch(headless=True, timeout=30000)
+            context = browser.new_context(
+                user_agent=get_random_ua(),
+                viewport={"width": 1920, "height": 1080},
+                locale="en-US",
+                timezone_id="America/New_York",
+            )
+            page = context.new_page()
+
+            # Additional headers for legitimacy
+            page.set_extra_http_headers({
+                "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+                "Accept-Language": "en-US,en;q=0.9",
+                "Accept-Encoding": "gzip, deflate, br",
+                "Connection": "keep-alive",
+                "Upgrade-Insecure-Requests": "1",
+            })
+
+            page.goto(url, wait_until="domcontentloaded", timeout=30000)
+
+            # Wait for content to load
+            time.sleep(random.uniform(2, 4))
+
+            html = page.content()
+            context.close()
+            browser.close()
+
+        # Parse with Newspaper4k
+        article = newspaper.article(url, input_html=html, language="en")
+        article.nlp()
+        logger.info(f"Successfully extracted article with Playwright from {url}")
+        return article.text
+
+    except Exception as e:
+        logger.error(f"Playwright failed for {url}: {str(e)}")
+        return ""
+
+
+def pull_article(link, source, title=None, save_to_file=True):
+    """
+    Pulls an article from a given link with fallback mechanisms.
+    """
+    filename = title if title else link
+    safe_filename = generate_filename_from_url(filename)
+
+    # Check if already cached
+    if os.path.exists(os.path.join("articles", source, safe_filename)):
+        logger.info(f"Article already cached: {filename}")
+        with open(
+            os.path.join("articles", source, safe_filename), "r", encoding="utf-8"
+        ) as f:
+            return f.read()
+
+    # Random delay before fetching to avoid rate-limiting / bot detection
+    time.sleep(random.uniform(0.5, 2))
+
+    text = ""
+
+    try:
+        # Try newspaper4k first with proper User-Agent to bypass bot detection
+        ua = get_random_ua()
+        article = newspaper.article(link, browser_user_agent=ua)
+        article.download()
+        article.parse()
+        text = article.text
+
+        if not text or len(text) < 200:
+            raise ValueError(
+                "\tArticle text too short, falling back to Playwright/Selenium."
+            )
+        logger.info(f"Successfully pulled article with newspaper4k from {link}")
+
+    except Exception as e:
+        logger.warning(
+            f"newspaper4k extraction failed for {link}: {e}, falling back to Playwright."
+        )
+
+        try:
+            text = get_article_with_playwright(link)
+            logger.info(f"Successfully pulled article from {link} with Playwright")
+
+            if not text or len(text) < 200:
+                logger.warning("Playwright article too short, falling back to Selenium.")
+                # Fallback to Selenium with better error handling
+                text = get_article_with_selenium(link)
+                logger.info(f"Successfully pulled article from {link} with Selenium")
+        except Exception as e:
+            logger.error(f"Playwright failed for {link}: {e}")
+
+            # Fallback to Selenium
+            try:
+                text = get_article_with_selenium(link)
+                logger.info(f"Successfully pulled article from {link} with Selenium")
+            except Exception as e:
+                logger.error(f"Selenium failed for {link}: {e}")
+                return ""
+
+    if save_to_file:
+        save_article_to_file(text, filename, source)
+
+    return text
+
+
+def is_article_downloaded(source, title, link):
+    """
+    Check if an article is already downloaded by checking if the file exists.
+    """
+    # Generate the same filename that would be used for saving
+    filename = title if title else link
+    safe_filename = generate_filename_from_url(filename)
+
+    # Check if file exists in the articles directory
+    file_path = os.path.join("articles", source, safe_filename)
+    return os.path.exists(file_path)
+
+
+def safe_pull_articles(article_list):
+    """
+    Safely pull articles with improved error handling and increased parallelism.
+    """
+    if not article_list:
+        return [], []
+
+    # Filter out articles that are already downloaded
+    filtered_article_list = []
+    total_articles = len(article_list)
+
+    for source, title, link in article_list:
+        if not is_article_downloaded(source, title, link):
+            filtered_article_list.append((source, title, link))
+        else:
+            logger.info(f"Skipping already downloaded article: {title[:50]}... from {source}")
+
+    logger.info(f"Filtered out {total_articles - len(filtered_article_list)} articles that were already downloaded")
+    logger.info(f"Processing {len(filtered_article_list)} remaining articles")
+
+    if not filtered_article_list:
+        logger.info("No new articles to process")
+        return [], []
+
+    results = []
+    errors = []
+
+    # Use ThreadPoolExecutor for parallel article pulling with higher concurrency
+    # Use configurable worker setting
+    max_workers = min(MAX_ARTICLE_WORKERS, len(filtered_article_list))  # Cap at configured workers, but don't exceed article count
+    batch_size = max(1, min(20, len(filtered_article_list) // 4))  # Dynamic batch size
+
+    logger.info(f"Starting parallel article pulling with {max_workers} workers and batch size {batch_size}")
+
+    # Process all articles in parallel with proper error handling
+    with ThreadPoolExecutor(max_workers=max_workers) as executor:
+        # Submit all tasks at once for maximum parallelism
+        futures = [
+            executor.submit(pull_article, link, source, title)
+            for source, title, link in filtered_article_list
+        ]
+
+        # Collect results as they complete
+        for i, future in enumerate(as_completed(futures, timeout=300)):  # 5 minute timeout total
+            try:
+                result = future.result(timeout=120)  # 2 minute timeout per article
+                if result:  # Only count non-empty results
+                    results.append(result)
+                # Log progress every 100 articles with proper batch information
+                if (i + 1) % 100 == 0:
+                    logger.info(f"Processed {i + 1} articles out of {len(filtered_article_list)}")
+            except Exception as e:
+                logger.error(f"Error processing article: {e}")
+                errors.append(e)
+
+    return results, errors
+
+
+def main():
+    """
+    Main scraping loop.
+    """
+    while True:
+        logger.info("=========================================")
+        logger.info("Starting new scraping iteration...")
+
+        try:
+            # Pull the RSS feed sources from the JSON file
+            rss_feed_sources = load_rss_feed_sources()
+
+            # Mine all articles from the RSS feed sources
+            rss_feed_links = mine_all_articles(rss_feed_sources)
+
+            # Randomize the order of the links to help with load balancing
+            import random
+
+            random.shuffle(rss_feed_links)
+
+            logger.info(f"Found {len(rss_feed_links)} articles to process")
+
+            if not rss_feed_links:
+                logger.info("No articles found, sleeping for 15 minutes")
+                time.sleep(15 * 60)
+                continue
+
+            # Process articles with better error handling and resource management
+            results, errors = safe_pull_articles(rss_feed_links)
+
+            logger.info(f"Attempted to Pull {len(results)} articles in parallel.")
+            logger.info(
+                f"Encountered {len(errors)} errors during article pulling. "
+                + "Outputting errors to a local file."
+            )
+
+            # Output errors to a local file
+            if errors:
+                with open("errors.txt", "w", encoding="utf-8") as f:
+                    for error in errors:
+                        f.write(str(error) + "\n")
+                logger.info("Errors logged to errors.txt")
+
+            # Print all results to a log file
+            with open("results.txt", "w", encoding="utf-8") as f:
+                for result in results:
+                    f.write(result + "\n")
+            logger.info("All articles pulled successfully.")
+
+            # Log processing progress
+            progress = get_processing_progress()
+            logger.info(f"Processing progress - Total: {progress['total_articles']}, "
+                       f"Completed: {progress['completed_articles']}, "
+                       f"Embedded: {progress['embedded_articles']}, "
+                       f"Completion rate: {progress['completion_rate']:.1f}%")
+
+        except Exception as e:
+            logger.error(f"Major error in main loop: {e}")
+            # Continue to next iteration even if there's a major error
+
+        # Sleep for a while before the next iteration
+        logger.info("Sleeping for 15 minutes before the next iteration...")
+        time.sleep(15 * 60)  # Sleep for 15 minutes
+
+
+if __name__ == "__main__":
+    main()

setup_scraper.sh

@@ -0,0 +1,91 @@
+#!/usr/bin/env bash
+# ------------------------------------------------------------------
+# setup_scraper.sh
+#
+# Installs a virtual‑env, a tiny wrapper, optional dependencies
+# and an hourly cron job – *always using the venv’s python/pip*.
+# ------------------------------------------------------------------
+set -euo pipefail
+
+# ---------- Configuration ----------
+SCRAPER_DIR="/home/user/StockDocs/scraper"
+VENV_DIR="${SCRAPER_DIR}/venv"
+SCRIPT_PATH="${SCRAPER_DIR}/cron_scraper.py"
+WRAPPER_PATH="${SCRAPER_DIR}/run_scraper.sh"
+LOG_PATH="${SCRAPER_DIR}/cron.log"
+REQUIREMENTS="${SCRAPER_DIR}/requirements.txt"
+
+# Cron line – this will call the wrapper, which in turn calls the
+# venv’s Python interpreter.
+CRON_LINE="0 * * * * ${WRAPPER_PATH} >> ${LOG_PATH} 2>&1"
+
+# ---------- Helper functions ----------
+log() { printf '[setup_scraper] %s\n' "$*"; }
+error_exit() { printf '[setup_scraper] ERROR: %s\n' "$*" >&2; exit 1; }
+
+# ---------- 1️⃣  Create the venv if it’s missing ----------
+if [[ ! -d "$VENV_DIR" ]]; then
+    log "Creating virtual‑environment at ${VENV_DIR}"
+    python3 -m venv "$VENV_DIR" || error_exit "Failed to create venv"
+else
+    log "Virtual‑environment already exists at ${VENV_DIR}"
+fi
+
+# ---------- 2️⃣  Activate the venv *for the rest of this script* ----------
+# This changes PATH for the current shell only – it does NOT touch the
+# system Python.  The next line will confirm that we’re really in the venv.
+# shellcheck source=/dev/null
+source "${VENV_DIR}/bin/activate"
+
+# Quick sanity‑check: make sure we’re using the venv’s python and pip.
+log "Current python: $(python -c 'import sys;print(sys.executable)')"
+log "pip version: $(pip --version)"
+
+# ---------- 3️⃣  Install dependencies (optional) ----------
+if [[ -f "$REQUIREMENTS" ]]; then
+    log "Installing Python packages from ${REQUIREMENTS}"
+    # Use the *venv’s* pip explicitly – this guarantees no system installs.
+    "${VENV_DIR}/bin/pip" install --upgrade pip
+    "${VENV_DIR}/bin/pip" install -r "$REQUIREMENTS" || error_exit "pip install failed"
+else
+    log "No requirements.txt found – skipping dependency install."
+fi
+
+# ---------- 4️⃣  Create the wrapper script ----------
+# The wrapper *does NOT* source the venv any more – it calls the venv’s
+# Python binary directly.  This eliminates the subtle “activate” pitfall.
+if [[ ! -f "$WRAPPER_PATH" ]]; then
+    log "Creating wrapper script at ${WRAPPER_PATH}"
+    cat > "$WRAPPER_PATH" <<'EOF'
+#!/usr/bin/env bash
+# ------------------------------------------------------------------
+# run_scraper.sh
+#
+# Activates the venv *implicitly* by calling the venv’s Python binary.
+# ------------------------------------------------------------------
+# Absolute path to the venv – change only if you move the venv.
+VENV_DIR="/home/user/StockDocs/scraper/venv"
+
+# Absolute path to the script you want to run.
+SCRIPT="/home/user/StockDocs/scraper/cron_scraper.py"
+
+# Invoke the venv’s Python directly.
+"${VENV_DIR}/bin/python" "${SCRIPT}"  # output is redirected by cron
+EOF
+    chmod +x "$WRAPPER_PATH"
+else
+    log "Wrapper script already exists at ${WRAPPER_PATH}"
+fi
+
+# ---------- 5️⃣  Install the cron job ----------
+CURRENT_CRON=$(crontab -l 2>/dev/null || true)
+
+if echo "$CURRENT_CRON" | grep -Fqx "$CRON_LINE"; then
+    log "Crontab entry already present – nothing to do."
+else
+    log "Adding new cron entry."
+    # Append the new line and reinstall the crontab.
+    (printf '%s\n' "$CURRENT_CRON" ; printf '%s\n' "$CRON_LINE") | crontab -
+fi
+
+log "✅  Setup complete!  ${SCRIPT_PATH} will run every hour via the venv."

tests/test_scraper_cache.py

@@ -0,0 +1,61 @@
+"""Tests for the scraper article-processing cache."""
+
+import sys
+from pathlib import Path
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+from scraper.scraper import (  # noqa: E402
+    get_processing_progress,
+    load_processed_cache,
+    mark_article_processed,
+    save_processed_cache,
+)
+
+
+@pytest.fixture()
+def cache_dir(tmp_path, monkeypatch):
+    monkeypatch.chdir(tmp_path)
+    (tmp_path / "articles").mkdir()
+    return tmp_path
+
+
+def test_load_missing_cache_returns_empty(cache_dir):
+    assert load_processed_cache() == {}
+
+
+def test_save_and_load_roundtrip(cache_dir):
+    save_processed_cache({"a.html": {"status": "completed"}})
+    cache = load_processed_cache()
+    assert cache["a.html"]["status"] == "completed"
+
+
+def test_mark_article_processed(cache_dir):
+    mark_article_processed("articles/x/article.html")
+    cache = load_processed_cache()
+    entry = cache["articles/x/article.html"]
+    assert entry["status"] == "completed"
+    assert entry["embedding_status"] == "pending"
+    assert "processed_date" in entry
+    assert "last_updated" in entry
+
+
+def test_processing_progress_empty(cache_dir):
+    progress = get_processing_progress()
+    assert progress["total_articles"] == 0
+    assert progress["completed_articles"] == 0
+    assert progress["embedded_articles"] == 0
+    assert progress["completion_rate"] == 0
+
+
+def test_processing_progress_mixed(cache_dir):
+    mark_article_processed("a.html")
+    mark_article_processed("b.html", status="failed")
+    mark_article_processed("c.html", embedding_status="completed")
+    progress = get_processing_progress()
+    assert progress["total_articles"] == 3
+    assert progress["completed_articles"] == 2
+    assert progress["embedded_articles"] == 1
+    assert progress["completion_rate"] == pytest.approx(2 / 3 * 100)