From 724be90e7cd4849958fb5578b1f293ddd2271ed5 Mon Sep 17 00:00:00 2001 From: Jarian Cottingham Date: Fri, 21 Aug 2026 18:33:59 +0000 Subject: [PATCH] Initial commit: StockDocs RSS scraper Carved out from the StockDocs monorepo. Collects financial news from 60+ outlets via RSS into a structured article corpus. --- README.md | 90 ++++++ cron_scraper.py | 488 +++++++++++++++++++++++++++++++ dockerfile | 20 ++ dockerfile-selenium | 22 ++ requirements.txt | 43 +++ requirements_clean.txt | 43 +++ rss_feeds.json | 196 +++++++++++++ rss_short_feed.json | 8 + run_scraper.sh | 14 + scraper.py | 568 ++++++++++++++++++++++++++++++++++++ setup_scraper.sh | 91 ++++++ tests/test_scraper_cache.py | 61 ++++ 12 files changed, 1644 insertions(+) create mode 100644 README.md create mode 100644 cron_scraper.py create mode 100644 dockerfile create mode 100644 dockerfile-selenium create mode 100644 requirements.txt create mode 100644 requirements_clean.txt create mode 100644 rss_feeds.json create mode 100644 rss_short_feed.json create mode 100755 run_scraper.sh create mode 100644 scraper.py create mode 100755 setup_scraper.sh create mode 100644 tests/test_scraper_cache.py diff --git a/README.md b/README.md new file mode 100644 index 0000000..337c78f --- /dev/null +++ b/README.md @@ -0,0 +1,90 @@ +# StockDocs — Scraper + +RSS scraper for the StockDocs financial news platform: collects financial news from 60+ outlets (Reuters, Bloomberg, Forbes, FT, WSJ...) and stores articles in a structured directory hierarchy for the processing pipeline. + +Part of the StockDocs project family: + +| Repo | What it is | +|------|------------| +| [StockDocs](https://git.jarianc.com/jarianc/StockDocs) | Processing core — article server, NLP analysis, embeddings | +| [stockdocs-mcp](https://git.jarianc.com/jarianc/stockdocs-mcp) | MCP server exposing the processed data to LLM clients | + +## Project Structure + +``` +. +├── rss_feeds.json # Configuration file with RSS feed URLs +├── scraper.py # Main scraping logic +├── cron_scraper.py # Scheduled scraping entry point +├── run_scraper.sh # Run helper +├── requirements.txt # Python dependencies +├── dockerfile # Docker configuration +├── dockerfile-selenium # Dockerfile for selenium-based scraping +├── articles/ # Directory where scraped articles are stored +│ ├── Reuters – Business News/ +│ │ ├── article1.txt +│ │ └── ... +│ └── ... +└── tests/ # Unit tests (article-processing cache) +``` + +## Features + +- **RSS Feed Integration**: Supports multiple financial news sources through RSS feeds +- **Automated Scraping**: Regularly fetches and processes new articles from configured feeds +- **Structured Storage**: Organizes articles in a directory structure by news outlet +- **Duplicate Detection**: Prevents re-processing of already collected articles +- **Caching Mechanism**: Maintains a cache of processed articles to optimize performance + +## RSS Feed Sources + +The scraper supports 60+ news outlets including: + +- Reuters – Business News +- Associated Press – Business +- Financial Times +- Forbes – Real-Time +- Wall Street Journal – U.S. Business +- Bloomberg – Surveillance Podcast +- CNN Money +- BBC News – Business +- And many more... + +## Usage + +### Running the Scraper + +```bash +python scraper.py +``` + +### Configuration + +The scraper can be configured by modifying `rss_feeds.json` to: +- Add new news sources +- Update existing RSS feed URLs +- Remove sources that are no longer active + +### Article Storage + +Articles are stored in `articles/` directory with the following structure: + +``` +articles/ +└── / + ├── article1.txt + ├── article2.txt + └── ... +``` + +Where each article file contains the full text content of that news article. + +## Requirements + +- Python 3.6+ +- Selenium WebDriver (for certain scraping operations) +- Additional dependencies listed in `requirements.txt` + +## License + +This project is licensed under the MIT License. \ No newline at end of file diff --git a/cron_scraper.py b/cron_scraper.py new file mode 100644 index 0000000..4b7645b --- /dev/null +++ b/cron_scraper.py @@ -0,0 +1,488 @@ +#!/usr/bin/env python3 +""" +Cron-based scraper for downloading articles from RSS feeds. +This version replaces the infinite while loop with a single execution +that can be scheduled via cron job. +""" + +import newspaper +import json +import feedparser +import time +import os +import requests +import logging +import random +from selenium import webdriver +from selenium.webdriver.firefox.options import Options as FirefoxOptions +from selenium.webdriver.common.by import By +from selenium.webdriver.support.ui import WebDriverWait +from selenium.webdriver.support import expected_conditions as EC +from concurrent.futures import ThreadPoolExecutor, as_completed +import nltk +from nltk.downloader import Downloader + +# Setup logging with timestamps +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(levelname)s - %(message)s', + datefmt='%Y-%m-%d %H:%M:%S' +) +logger = logging.getLogger(__name__) + +# Rotating User-Agents to bypass bot detection (Reuters, etc.) +USER_AGENTS = [ + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Safari/605.1.15", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:134.0) Gecko/20100101 Firefox/134.0", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36", +] + +def get_random_ua(): + return random.choice(USER_AGENTS) + +# Robust file path handling - try multiple locations +def get_feed_file_path(): + """Get the RSS feed file path, trying multiple locations.""" + possible_paths = [ + "./rss_feeds.json", # Current directory + "../rss_feeds.json", # Parent directory + "/app/rss_feeds.json", # Docker path + "./scraper/rss_feeds.json" # Scraper subdirectory + ] + + for path in possible_paths: + if os.path.exists(path): + print(f"Found feed file at: {path}") + return path + + # If no file found, exit the program + print("Error: RSS feed file not found in any expected location") + print("Exiting program...") + exit(1) + +# Get the feed file path +FEED_FILE = get_feed_file_path() + +# Ensure necessary NLTK resources are downloaded +d = Downloader() +if not d.is_installed("punkt_tab"): + nltk.download("punkt_tab") + +articles = [] + + +def load_rss_feed_sources(feed_file=FEED_FILE): + """ + Loads the RSS feed sources from a JSON file. + """ + logger.info(f"Loading RSS feed sources from {feed_file}...") + + # Debug: Print current working directory + logger.debug(f"Current working directory: {os.getcwd()}") + + try: + with open(feed_file, "r", encoding="utf-8") as f: + data = json.load(f) + logger.info(f"Successfully loaded {feed_file}") + logger.debug(f"Data type: {type(data)}") + if isinstance(data, dict) and "rss_feeds" in data: + logger.info(f"Found rss_feeds section with {len(data['rss_feeds'])} sources") + return data + else: + logger.warning(f"Unexpected data structure. Data keys: {list(data.keys()) if isinstance(data, dict) else 'Not a dict'}") + return {} + except FileNotFoundError: + logger.error(f"{feed_file} not found, returning empty dict.") + return {} + except json.JSONDecodeError as e: + logger.error(f"Error decoding {feed_file}: {e}, returning empty dict.") + return {} + except Exception as e: + logger.error(f"Unexpected error loading {feed_file}: {e}") + return {} + + +def mine_all_articles(rss_feed_sources, limit=None): + """ + Mines all articles from the given RSS feed sources. + Returns a list of (site, title, link) tuples. + """ + all_links = [] + + # Check if rss_feed_sources is a valid dict with rss_feeds key + if not isinstance(rss_feed_sources, dict): + logger.warning(f"rss_feed_sources is not a dict, it's {type(rss_feed_sources)}") + return all_links + + if "rss_feeds" not in rss_feed_sources: + logger.warning("rss_feeds key not found in rss_feed_sources") + return all_links + + sources = rss_feed_sources["rss_feeds"] + + for site, data in sources.items(): + logger.info(f"Parsing RSS feed: {data['rss_url']}") + try: + feed = feedparser.parse(data["rss_url"]) + feed_entries = feed.entries[:limit] if limit else feed.entries + + for entry in feed_entries: + if "link" in entry and "title" in entry: + all_links.append((site, entry.title, entry.link)) + except Exception as e: + logger.error(f"Error parsing RSS feed: {site} Error: {str(e)}") + return all_links + + +def generate_filename_from_url(url): + """ + Generates a filename from the given URL by replacing slashes with underscores. + """ + # Use only the last part of the URL or replace slashes + return url.replace("https://", "").replace("http://", "").replace("/", "_") + + +def generate_safe_filename(name): + # Remove/replace characters not allowed in filenames + import re + + safe = re.sub(r'[\\/*?:"<>|]', "_", name) + return safe + + +def save_article_to_file(article, filename, source="Unfiltered"): + """ + Saves the given article text to a file with the specified filename. + """ + # articles dir should already be there + # os.makedirs("articles", exist_ok=True) + + outputDir = "articles/" + source + os.makedirs(outputDir, exist_ok=True) if source else None + + # Sanitize filename: use only the last part of the URL or replace slashes + safe_filename = generate_filename_from_url(filename) + file_path = os.path.join(outputDir, safe_filename) + # Save the source as the first line in the file for later retrieval + with open(file_path, "w", encoding="utf-8") as f: + f.write(f"SOURCE:{source}\n") + f.write(article) + + # Only log when a new file is actually created (not cached) + logger.info(f"New article saved: {safe_filename} from {source}") + + +def get_article_with_selenium(url): + """ + Gets article text using Selenium Firefox driver with proper error handling, + cleanup, and bot-detection evasion. + """ + driver = None + try: + # Configure Firefox options with bot-detection evasion + options = FirefoxOptions() + options.add_argument("--headless") + options.set_preference("dom.ipc.processCount", 1) + options.set_preference("general.useragent.override", get_random_ua()) + options.set_preference("permissions.default.image", 2) + options.set_preference("dom.webnotifications.enabled", False) + + # Initialize driver with timeout + driver = webdriver.Firefox(options=options) + driver.set_page_load_timeout(30) + + # Navigate to URL + driver.get(url) + + # Wait for page to load (explicit wait instead of sleep) + try: + WebDriverWait(driver, 15).until( + EC.presence_of_element_located((By.TAG_NAME, "body")) + ) + except Exception: + pass + + time.sleep(random.uniform(1, 3)) + + html = driver.page_source + + # Parse with Newspaper4k + article = newspaper.article(url, input_html=html, language="en") + article.nlp() + logger.info(f"Successfully extracted article with Selenium from {url}") + return article.text + + except Exception as e: + logger.error(f"Selenium failed for {url}: {str(e)}") + return "" + finally: + if driver: + try: + driver.quit() + except Exception: + pass + + +def get_article_with_playwright(url): + """ + Gets article text using Playwright with proper bot-detection evasion. + """ + try: + from playwright.sync_api import sync_playwright + + with sync_playwright() as p: + browser = p.chromium.launch(headless=True, timeout=30000) + context = browser.new_context( + user_agent=get_random_ua(), + viewport={"width": 1920, "height": 1080}, + locale="en-US", + timezone_id="America/New_York", + ) + page = context.new_page() + + page.set_extra_http_headers({ + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.9", + "Accept-Encoding": "gzip, deflate, br", + "Connection": "keep-alive", + "Upgrade-Insecure-Requests": "1", + }) + + page.goto(url, wait_until="domcontentloaded", timeout=30000) + time.sleep(random.uniform(2, 4)) + + html = page.content() + context.close() + browser.close() + + article = newspaper.article(url, input_html=html, language="en") + article.nlp() + logger.info(f"Successfully extracted article with Playwright from {url}") + return article.text + + except Exception as e: + logger.error(f"Playwright failed for {url}: {str(e)}") + return "" + + +def pull_article(link, source, title=None, save_to_file=True): + """ + Pulls an article from a given link with fallback mechanisms. + """ + filename = title if title else link + safe_filename = generate_filename_from_url(filename) + + # Check if already cached + if os.path.exists(os.path.join("articles", source, safe_filename)): + logger.info(f"Article already cached: {filename}") + with open( + os.path.join("articles", source, safe_filename), "r", encoding="utf-8" + ) as f: + return f.read() + + # Random delay before fetching to avoid rate-limiting / bot detection + time.sleep(random.uniform(0.5, 2)) + + text = "" + + try: + # Try newspaper4k first with proper User-Agent to bypass bot detection + ua = get_random_ua() + article = newspaper.article(link, browser_user_agent=ua) + article.download() + article.parse() + text = article.text + + if not text or len(text) < 200: + raise ValueError( + "\tArticle text too short, falling back to Playwright/Selenium." + ) + logger.info(f"Successfully pulled article with newspaper4k from {link}") + + except Exception as e: + logger.warning( + f"newspaper4k extraction failed for {link}: {e}, falling back to Playwright." + ) + + try: + text = get_article_with_playwright(link) + logger.info(f"Successfully pulled article from {link} with Playwright") + + if not text or len(text) < 200: + logger.warning("Playwright article too short, falling back to Selenium.") + # Fallback to Selenium with better error handling + text = get_article_with_selenium(link) + logger.info(f"Successfully pulled article from {link} with Selenium") + except Exception as e: + logger.error(f"Playwright failed for {link}: {e}") + + # Fallback to Selenium + try: + text = get_article_with_selenium(link) + logger.info(f"Successfully pulled article from {link} with Selenium") + except Exception as e: + logger.error(f"Selenium failed for {link}: {e}") + return "" + + if save_to_file: + save_article_to_file(text, filename, source) + + return text + + +def safe_pull_articles(article_list): + """ + Safely pull articles with improved error handling and reduced parallelism. + """ + results = [] + errors = [] + + # Process in smaller batches to reduce resource strain + batch_size = 5 + + for i in range(0, len(article_list), batch_size): + batch = article_list[i : i + batch_size] + logger.info(f"Processing batch {i // batch_size + 1} with {len(batch)} articles") + + # Use ThreadPoolExecutor instead of ProcessPoolExecutor to avoid + # process termination issues with browser automation + with ThreadPoolExecutor(max_workers=3) as executor: # Reduced workers + futures = [ + executor.submit(pull_article, link, source, title) + for source, title, link in batch + ] + + for future in as_completed(futures): + try: + result = future.result(timeout=120) # 2 minute timeout + results.append(result) + except Exception as e: + logger.error(f"Error in pull_article: {e}") + errors.append(e) + + # Add a small delay between batches to reduce system load + time.sleep(5) + + return results, errors + + +def gather_new_articles(): + """ + Gather list of all newly downloaded articles and format them for webhook. + """ + new_articles = [] + + # Walk through all article directories + for root, dirs, files in os.walk("articles"): + for file in files: + if file != "processed_articles_cache.json": # Skip cache file + # Get the full file path + file_path = os.path.join(root, file) + + # Get the outlet name from the directory path + outlet = os.path.basename(root) + + # Create the relative path for the article + relative_path = os.path.relpath(file_path, "scraper") + + # Create article data structure + article_data = { + "created_at": time.strftime("%Y-%m-%dT%H:%M:%S.%f", time.localtime(os.path.getctime(file_path))), + "name": file, + "outlet": outlet, + "path": f"../{relative_path}" + } + + new_articles.append(article_data) + + return new_articles + + +def send_to_webhook(articles): + """ + Send list of articles to the webhook URL. + """ + webhook_url = "http://agents.example.com/webhook/49c5b169-c68c-4f8c-90c2-0fcca6e2d387" + headers = { + "StockDocsN8NAuthToken": "ganvT4gsgRjWpGE8FMw9uCzFjZrTx8RZCoVm2Dh7skbZecov" + } + + try: + response = requests.post(webhook_url, json=articles, headers=headers, timeout=30) + if response.status_code == 200: + print(f"Successfully sent {len(articles)} articles to webhook") + else: + print(f"Webhook request failed with status code: {response.status_code}") + print(f"Response: {response.text}") + except Exception as e: + print(f"Error sending to webhook: {e}") + + +def main(): + """ + Main scraping function for cron execution. + This replaces the infinite while loop with a single execution. + """ + logger.info("=========================================") + logger.info("Starting new scraping iteration...") + + try: + # Pull the RSS feed sources from the JSON file + rss_feed_sources = load_rss_feed_sources() + + # Mine all articles from the RSS feed sources + rss_feed_links = mine_all_articles(rss_feed_sources) + + # Randomize the order of the links to help with load balancing + import random + + random.shuffle(rss_feed_links) + + logger.info(f"Found {len(rss_feed_links)} articles to process") + + if not rss_feed_links: + logger.info("No articles found") + return + + # Process articles with better error handling and resource management + results, errors = safe_pull_articles(rss_feed_links) + + logger.info(f"Attempted to Pull {len(results)} articles in parallel.") + logger.info( + f"Encountered {len(errors)} errors during article pulling. " + + "Outputting errors to a local file." + ) + + # Output errors to a local file + if errors: + with open("errors.txt", "w", encoding="utf-8") as f: + for error in errors: + f.write(str(error) + "\n") + logger.info("Errors logged to errors.txt") + + # Print all results to a log file + with open("results.txt", "w", encoding="utf-8") as f: + for result in results: + f.write(result + "\n") + logger.info("All articles pulled successfully.") + + # Gather and send new articles to webhook + new_articles = gather_new_articles() + if new_articles: + send_to_webhook(new_articles) + else: + logger.info("No new articles to send to webhook") + + except Exception as e: + logger.error(f"Major error in main execution: {e}") + import traceback + traceback.print_exc() + raise # Re-raise to ensure the script exits with error code + + logger.info("Scraping completed successfully.") + + +if __name__ == "__main__": + main() diff --git a/dockerfile b/dockerfile new file mode 100644 index 0000000..37dd4fb --- /dev/null +++ b/dockerfile @@ -0,0 +1,20 @@ +# --- Dockerfile.local --- +FROM python:3.13.5 + +# Firefox + GeckoDriver +RUN apt-get update && apt-get install -y --no-install-recommends \ + firefox-esr wget ca-certificates gnupg2 \ + && GECKO=v0.36.0 && \ + wget -qO- "https://github.com/mozilla/geckodriver/releases/download/${GECKO}/geckodriver-${GECKO}-linux64.tar.gz" \ + | tar -xz -C /usr/local/bin geckodriver \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt +RUN playwright install +RUN playwright install-deps +RUN python3 -m nltk.downloader punkt_tab # Download NLTK data and bake into image +COPY . . + +CMD ["python3", "scraper.py"] diff --git a/dockerfile-selenium b/dockerfile-selenium new file mode 100644 index 0000000..afe4a12 --- /dev/null +++ b/dockerfile-selenium @@ -0,0 +1,22 @@ +# Use an official Selenium Firefox standalone as a base image +FROM selenium/standalone-firefox:latest + +USER root +# Set the working directory in the container +WORKDIR /app + +# Install Python dependencies +COPY requirements.txt /app/ +RUN pip install --break-system-packages --no-cache-dir -r requirements.txt && \ + playwright install + +# Copy the current directory contents into the container at /app +COPY . /app +RUN mkdir -p /app/articles && \ + chown seluser:seluser /app/articles && \ + chmod 755 /app/articles + +USER seluser +# Run the Flask app +CMD ["python3", "scraper.py"] + diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..70f6507 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,43 @@ +attrs +beautifulsoup4 +certifi +charset-normalizer +click +dnspython +feedparser +filelock +gnews +greenlet +h11 +idna +joblib +lxml +lxml-html-clean +newspaper4k +nltk +numpy +outcome +pandas +pillow +playwright +pyee +pysocks +python-dateutil +pytz +pyyaml +regex +requests +requests-file +selenium +sgmllib3k +six +sniffio +sortedcontainers +soupsieve +tldextract +tqdm +trio +trio-websocket +tzdata +websocket-client +wsproto diff --git a/requirements_clean.txt b/requirements_clean.txt new file mode 100644 index 0000000..70f6507 --- /dev/null +++ b/requirements_clean.txt @@ -0,0 +1,43 @@ +attrs +beautifulsoup4 +certifi +charset-normalizer +click +dnspython +feedparser +filelock +gnews +greenlet +h11 +idna +joblib +lxml +lxml-html-clean +newspaper4k +nltk +numpy +outcome +pandas +pillow +playwright +pyee +pysocks +python-dateutil +pytz +pyyaml +regex +requests +requests-file +selenium +sgmllib3k +six +sniffio +sortedcontainers +soupsieve +tldextract +tqdm +trio +trio-websocket +tzdata +websocket-client +wsproto diff --git a/rss_feeds.json b/rss_feeds.json new file mode 100644 index 0000000..d6ad9e8 --- /dev/null +++ b/rss_feeds.json @@ -0,0 +1,196 @@ +{ + "rss_feeds": { + "Reuters – Business News": { + "source_website": "reuters.com", + "rss_url": "https://www.reutersagency.com/feed/" + }, + "Associated Press – Business": { + "source_website": "apnews.com", + "rss_url": "https://rsshub.app/apnews/topics/apf-topnews" + }, + "Financial Times": { + "source_website": "ft.com", + "rss_url": "https://www.ft.com/rss/home" + }, + "Fortune – Top Stories": { + "source_website": "fortune.com", + "rss_url": "https://fortune.com/feed/fortune-feeds/?id=3230629" + }, + "Seeking Alpha – Market News": { + "source_website": "seekingalpha.com", + "rss_url": "https://seekingalpha.com/feed.xml" + }, + "The Motley Fool – Stock News & Analysis": { + "source_website": "fool.com", + "rss_url": "https://www.fool.com/a/feeds/partner/googlechromefollow?apikey=5e092c1f-c5f9-4428-9219-908a47d2e2de" + }, + "Business Standard – Latest News": { + "source_website": "business-standard.com", + "rss_url": "https://www.business-standard.com/rss/latest.rss" + }, + "TheStreet – Full Articles": { + "source_website": "thestreet.com", + "rss_url": "https://www.thestreet.com/.rss/full" + }, + "Benzinga – Financial News": { + "source_website": "benzinga.com", + "rss_url": "https://feeds.benzinga.com/benzinga" + }, + "MarketBeat – Market News": { + "source_website": "marketbeat.com", + "rss_url": "https://www.marketbeat.com/feed/" + }, + "Money (Time) – Personal Finance": { + "source_website": "money.com", + "rss_url": "https://money.com/money/feed/" + }, + "Global Finance Magazine": { + "source_website": "gfmag.com", + "rss_url": "https://www.gfmag.com/feed" + }, + "Financial Samurai": { + "source_website": "financialsamurai.com", + "rss_url": "https://www.financialsamurai.com/feed/" + }, + "MoneyWeek": { + "source_website": "moneyweek.com", + "rss_url": "https://moneyweek.com/feed/all" + }, + "Finance Monthly": { + "source_website": "finance-monthly.com", + "rss_url": "https://www.finance-monthly.com/feed/" + }, + "European Financial Review": { + "source_website": "europeanfinancialreview.com", + "rss_url": "https://www.europeanfinancialreview.com/feed" + }, + "Money Morning": { + "source_website": "moneymorning.com", + "rss_url": "https://moneymorning.com/feed" + }, + "Dealbreaker": { + "source_website": "dealbreaker.com", + "rss_url": "https://dealbreaker.com/.rss/full" + }, + "World Finance": { + "source_website": "worldfinance.com", + "rss_url": "https://www.worldfinance.com/feed" + }, + "Fox Business – Headlines": { + "source_website": "foxbusiness.com", + "rss_url": "https://moxie.foxbusiness.com/google-publisher/latest.xml" + }, + "FinanceAsia": { + "source_website": "financeasia.com", + "rss_url": "https://www.financeasia.com/rss/latest" + }, + "CNBC – Business": { + "source_website": "cnbc.com", + "rss_url": "https://www.cnbc.com/id/100003114/device/rss/rss.html" + }, + "CNN Money": { + "source_website": "cnn.com", + "rss_url": "http://rss.cnn.com/rss/money_topstories.rss" + }, + "Markets Insider": { + "source_website": "markets.businessinsider.com", + "rss_url": "https://markets.businessinsider.com/rss/news" + }, + "The Economist – Business & Finance": { + "source_website": "economist.com", + "rss_url": "https://www.economist.com/business/rss.xml" + }, + "Barchart News": { + "source_website": "barchart.com", + "rss_url": "http://feeds.feedburner.com/BarchartNews" + }, + "The Guardian – Business": { + "source_website": "theguardian.com", + "rss_url": "http://feeds.theguardian.com/theguardian/uk/business/rss" + }, + "Economy Watch": { + "source_website": "economywatch.com", + "rss_url": "https://www.economywatch.com/feed" + }, + "CFI.co": { + "source_website": "cfi.co", + "rss_url": "https://cfi.co/feed" + }, + "BBC News – Business": { + "source_website": "bbc.co.uk", + "rss_url": "http://feeds.bbci.co.uk/news/business/rss.xml" + }, + "Investor’s Business Daily": { + "source_website": "investors.com", + "rss_url": "https://www.investors.com/feed/" + }, + "Forbes – Real-Time": { + "source_website": "forbes.com", + "rss_url": "https://www.forbes.com/real-time/feed2/" + }, + "The Financial Express": { + "source_website": "financialexpress.com", + "rss_url": "https://www.financialexpress.com/feed/" + }, + "MarketWatch – Top Stories": { + "source_website": "marketwatch.com", + "rss_url": "http://feeds.marketwatch.com/marketwatch/topstories/" + }, + "Wall Street Journal – U.S. Business": { + "source_website": "wsj.com", + "rss_url": "https://feeds.a.dj.com/rss/WSJcomUSBusiness.xml" + }, + "Sky News – Business": { + "source_website": "news.sky.com", + "rss_url": "http://news.sky.com/feeds/rss/business.xml" + }, + "Bloomberg – Surveillance Podcast": { + "source_website": "bloomberg.com", + "rss_url": "https://www.bloomberg.com/feed/podcast/bloomberg-surveillance.xml" + }, + "Barron’s – Markets": { + "source_website": "barrons.com", + "rss_url": "https://www.barrons.com/xml/rss/markets.xml" + }, + "Yahoo Finance": { + "source_website": "finance.yahoo.com", + "rss_url": "https://www.yahoo.com/news/rss/finance" + }, + "Investing.com – News": { + "source_website": "investing.com", + "rss_url": "https://www.investing.com/rss/news.rss" + }, + "Investopedia – Headlines": { + "source_website": "investopedia.com", + "rss_url": "https://www.investopedia.com/feedbuilder/feed/getfeed/?feedName=rss_headline" + }, + "NerdWallet – Finance": { + "source_website": "nerdwallet.com", + "rss_url": "https://www.nerdwallet.com/news/finance/feed" + }, + "Newsmax Finance": { + "source_website": "newsmax.com", + "rss_url": "https://www.newsmax.com/rss/finance" + }, + "Bankrate – News": { + "source_website": "bankrate.com", + "rss_url": "https://www.bankrate.com/rss/" + }, + "Morningstar – Articles": { + "source_website": "morningstar.com", + "rss_url": "https://www.morningstar.com/articles.rss" + }, + "Kiplinger": { + "source_website": "kiplinger.com", + "rss_url": "https://www.kiplinger.com/kiplinger.rss" + }, + "International Business Times": { + "source_website": "ibtimes.com", + "rss_url": "https://www.ibtimes.com/rss" + }, + "Policygenius – News": { + "source_website": "policygenius.com", + "rss_url": "https://www.policygenius.com/news/feed/" + } + } +} diff --git a/rss_short_feed.json b/rss_short_feed.json new file mode 100644 index 0000000..ec924ad --- /dev/null +++ b/rss_short_feed.json @@ -0,0 +1,8 @@ +{ + "rss_feeds": { + "Associated Press – Business": { + "source_website": "apnews.com", + "rss_url": "https://news.google.com/rss/search?q=site:apnews.com&hl=en-US&gl=US&ceid=US:en" + } + } +} diff --git a/run_scraper.sh b/run_scraper.sh new file mode 100755 index 0000000..7871c1a --- /dev/null +++ b/run_scraper.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# ------------------------------------------------------------------ +# run_scraper.sh +# +# Activates the venv *implicitly* by calling the venv’s Python binary. +# ------------------------------------------------------------------ +# Absolute path to the venv – change only if you move the venv. +VENV_DIR="/home/user/StockDocs/scraper/venv" + +# Absolute path to the script you want to run. +SCRIPT="/home/user/StockDocs/scraper/cron_scraper.py" + +# Invoke the venv’s Python directly. +"${VENV_DIR}/bin/python" "${SCRIPT}" # output is redirected by cron diff --git a/scraper.py b/scraper.py new file mode 100644 index 0000000..cc4f1ce --- /dev/null +++ b/scraper.py @@ -0,0 +1,568 @@ +import newspaper +import json +import feedparser +import time +import os +import logging +import random +from datetime import datetime +from selenium import webdriver +from selenium.webdriver.firefox.options import Options as FirefoxOptions +from selenium.webdriver.common.by import By +from selenium.webdriver.support.ui import WebDriverWait +from selenium.webdriver.support import expected_conditions as EC +from concurrent.futures import ThreadPoolExecutor, as_completed, TimeoutError +import nltk +from nltk.downloader import Downloader + +# Setup logging with timestamps +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(levelname)s - %(message)s', + datefmt='%Y-%m-%d %H:%M:%S' +) +logger = logging.getLogger(__name__) + +FEED_FILE = os.getenv("FEED_FILE", "./rss_short_feed.json") +MAX_FEED_WORKERS = int(os.getenv("MAX_FEED_WORKERS", "10")) +MAX_ARTICLE_WORKERS = int(os.getenv("MAX_ARTICLE_WORKERS", "10")) +BATCH_SIZE = int(os.getenv("BATCH_SIZE", "50")) # Batch processing size + +# Rotating User-Agents to bypass bot detection (Reuters, etc.) +USER_AGENTS = [ + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Safari/605.1.15", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:134.0) Gecko/20100101 Firefox/134.0", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36", +] + +def get_random_ua(): + return random.choice(USER_AGENTS) + +# Ensure necessary NLTK resources are downloaded +d = Downloader() +if not d.is_installed("punkt_tab"): + nltk.download("punkt_tab") + +articles = [] + + +# Enhanced cache system +def load_processed_cache(): + """Load the processed articles cache with enhanced tracking""" + cache_path = "articles/processed_articles_cache.json" + try: + if os.path.exists(cache_path): + with open(cache_path, 'r', encoding='utf-8') as f: + return json.load(f) + else: + return {} + except Exception as e: + logger.error(f"Error loading cache: {e}") + return {} + +def save_processed_cache(cache_data): + """Save the processed articles cache with enhanced tracking""" + cache_path = "articles/processed_articles_cache.json" + try: + with open(cache_path, 'w', encoding='utf-8') as f: + json.dump(cache_data, f, indent=2, ensure_ascii=False) + logger.info(f"Cache saved with {len(cache_data)} entries") + except Exception as e: + logger.error(f"Error saving cache: {e}") + +def is_article_processed(article_path, cache_data): + """Check if an article has been processed""" + return article_path in cache_data + +def mark_article_processed(article_path, status="completed", embedding_status="pending"): + """Mark an article as processed with detailed status tracking""" + cache_data = load_processed_cache() + cache_data[article_path] = { + "processed_date": datetime.now().isoformat(), + "status": status, + "embedding_status": embedding_status, + "last_updated": datetime.now().isoformat() + } + save_processed_cache(cache_data) + +def get_processing_progress(): + """Get overall processing progress""" + cache_data = load_processed_cache() + total_articles = len(cache_data) + completed_articles = sum(1 for data in cache_data.values() if data.get('status') == 'completed') + embedded_articles = sum(1 for data in cache_data.values() if data.get('embedding_status') == 'completed') + + return { + "total_articles": total_articles, + "completed_articles": completed_articles, + "embedded_articles": embedded_articles, + "completion_rate": (completed_articles / total_articles * 100) if total_articles > 0 else 0 + } + + +def load_rss_feed_sources(feed_file=FEED_FILE): + """ + Loads the RSS feed sources from a JSON file. + """ + logger.info(f"Loading RSS feed sources from {feed_file}...") + + try: + with open(feed_file, "r", encoding="utf-8") as f: + data = json.load(f) + logger.info(f"Successfully loaded {feed_file}") + return data + except FileNotFoundError: + logger.error(f"{feed_file} not found, returning empty list.") + return [] + except json.JSONDecodeError: + logger.error(f"Error decoding {feed_file} , returning empty list.") + return [] + + +def mine_all_articles(rss_feed_sources, limit=None): + """ + Mines all articles from the given RSS feed sources. + Returns a list of (site, title, link) tuples. + """ + all_links = [] + + # Check if rss_feed_sources is a valid dict with rss_feeds key + if not isinstance(rss_feed_sources, dict): + logger.warning(f"rss_feed_sources is not a dict, it's {type(rss_feed_sources)}") + return all_links + + if "rss_feeds" not in rss_feed_sources: + logger.warning("rss_feeds key not found in rss_feed_sources") + return all_links + + sources = rss_feed_sources["rss_feeds"] + + # Parse RSS feeds in parallel for better performance + def parse_single_feed(site, data): + """Parse a single RSS feed with timeout and error handling""" + try: + logger.info(f"Parsing RSS feed: {data['rss_url']}") + # Add more aggressive timeout settings with fallback + # Use a wrapper to ensure we don't hang indefinitely + import signal + + def timeout_handler(signum, frame): + raise TimeoutError(f"Timeout parsing feed: {site}") + + # Set up signal-based timeout (this is a fallback for truly hanging requests) + old_handler = signal.signal(signal.SIGALRM, timeout_handler) + signal.alarm(10) # 10 second alarm + + feed = feedparser.parse(data["rss_url"], timeout=8) # 8 second timeout + signal.alarm(0) # Cancel the alarm + signal.signal(signal.SIGALRM, old_handler) + + feed_entries = feed.entries[:limit] if limit else feed.entries + + entries = [] + for entry in feed_entries: + if "link" in entry and "title" in entry: + entries.append((site, entry.title, entry.link)) + return entries + except TimeoutError as e: + logger.error(f"Timeout parsing RSS feed: {site} Error: {str(e)}") + return [] + except Exception as e: + error_str = str(e).lower() + # Handle specific network connection issues + if "remote end closed connection" in error_str or "connection closed" in error_str: + logger.warning(f"Network connection closed by remote end for feed: {site} - {str(e)}") + logger.info(f"Skipping problematic feed: {site}") + return [] + elif "timeout" in error_str: + logger.error(f"Timeout parsing RSS feed: {site} Error: {str(e)}") + return [] + else: + logger.error(f"Error parsing RSS feed: {site} Error: {str(e)}") + return [] + + # Use ThreadPoolExecutor for parallel RSS feed parsing + from concurrent.futures import ThreadPoolExecutor, as_completed + max_workers = min(10, len(sources)) # Limit concurrent workers + + with ThreadPoolExecutor(max_workers=max_workers) as executor: + # Submit all feed parsing tasks + future_to_site = { + executor.submit(parse_single_feed, site, data): site + for site, data in sources.items() + } + + # Collect results as they complete + for future in as_completed(future_to_site, timeout=30): # 30 second overall timeout + try: + entries = future.result() + all_links.extend(entries) + except Exception as e: + site = future_to_site[future] + logger.error(f"Error processing feed for {site}: {str(e)}") + + return all_links + + +def generate_filename_from_url(url): + """ + Generates a filename from the given URL by replacing slashes with underscores. + """ + # Use only the last part of the URL or replace slashes + filename = url.replace("https://", "").replace("http://", "").replace("/", "_") + # Sanitize filename to remove/replace invalid characters + import re + filename = re.sub(r'[\\/*?:"<>|]', "_", filename) + return filename + + +def generate_safe_filename(name): + # Remove/replace characters not allowed in filenames + import re + + safe = re.sub(r'[\\/*?:"<>|]', "_", name) + return safe + + +def save_article_to_file(article, filename, source="Unfiltered"): + """ + Saves the given article text to a file with the specified filename. + """ + # articles dir should already be there + # os.makedirs("articles", exist_ok=True) + + outputDir = "articles/" + source + os.makedirs(outputDir, exist_ok=True) if source else None + + # Sanitize filename: use only the last part of the URL or replace slashes + safe_filename = generate_filename_from_url(filename) + file_path = os.path.join(outputDir, safe_filename) + # Save the source as the first line in the file for later retrieval + with open(file_path, "w", encoding="utf-8") as f: + f.write(f"SOURCE:{source}\n") + f.write(article) + + # Only log when a new file is actually created (not cached) + logger.info(f"New article saved: {safe_filename} from {source}") + + +def get_article_with_selenium(url): + """ + Gets article text using Selenium Firefox driver with proper error handling, + cleanup, and bot-detection evasion. + """ + driver = None + try: + # Configure Firefox options with bot-detection evasion + options = FirefoxOptions() + options.add_argument("--headless") + options.set_preference("dom.ipc.processCount", 1) + options.set_preference("general.useragent.override", get_random_ua()) + options.set_preference("permissions.default.image", 2) # Block images for speed + options.set_preference("dom.webnotifications.enabled", False) + + # Try to initialize driver with explicit path to Firefox + try: + driver = webdriver.Firefox(options=options) + except Exception as e: + # If that fails, try with explicit Firefox path + if "binary is not a firefox executable" in str(e).lower(): + logger.info("Attempting to use Firefox at /usr/bin/firefox") + options.binary_location = "/usr/bin/firefox" + driver = webdriver.Firefox(options=options) + else: + raise e + + driver.set_page_load_timeout(30) # 30 seconds timeout + + # Navigate to URL + driver.get(url) + + # Wait for page to load (explicit wait instead of sleep) + try: + WebDriverWait(driver, 15).until( + EC.presence_of_element_located((By.TAG_NAME, "body")) + ) + except Exception: + pass # Continue even if wait times out + + time.sleep(random.uniform(1, 3)) # Random wait to mimic human behavior + + html = driver.page_source + + # Parse with Newspaper4k + article = newspaper.article(url, input_html=html, language="en") + article.nlp() + logger.info(f"Successfully extracted article with Selenium from {url}") + return article.text + + except Exception as e: + # Check if this is a Firefox binary not found error + error_str = str(e).lower() + if "binary is not a firefox executable" in error_str or "firefox" in error_str: + logger.error(f"Firefox not found or not properly configured for {url}: {str(e)}") + logger.error("Firefox is installed at /usr/bin/firefox but may not be accessible. Check PATH or permissions.") + else: + logger.error(f"Selenium failed for {url}: {str(e)}") + return "" + finally: + # Always quit the driver + if driver: + try: + driver.quit() + except Exception: + pass # Ignore errors in cleanup + + +def get_article_with_playwright(url): + """ + Gets article text using Playwright with proper bot-detection evasion. + """ + try: + from playwright.sync_api import sync_playwright + + with sync_playwright() as p: + # Use Chromium with full browser context for UA spoofing + browser = p.chromium.launch(headless=True, timeout=30000) + context = browser.new_context( + user_agent=get_random_ua(), + viewport={"width": 1920, "height": 1080}, + locale="en-US", + timezone_id="America/New_York", + ) + page = context.new_page() + + # Additional headers for legitimacy + page.set_extra_http_headers({ + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.9", + "Accept-Encoding": "gzip, deflate, br", + "Connection": "keep-alive", + "Upgrade-Insecure-Requests": "1", + }) + + page.goto(url, wait_until="domcontentloaded", timeout=30000) + + # Wait for content to load + time.sleep(random.uniform(2, 4)) + + html = page.content() + context.close() + browser.close() + + # Parse with Newspaper4k + article = newspaper.article(url, input_html=html, language="en") + article.nlp() + logger.info(f"Successfully extracted article with Playwright from {url}") + return article.text + + except Exception as e: + logger.error(f"Playwright failed for {url}: {str(e)}") + return "" + + +def pull_article(link, source, title=None, save_to_file=True): + """ + Pulls an article from a given link with fallback mechanisms. + """ + filename = title if title else link + safe_filename = generate_filename_from_url(filename) + + # Check if already cached + if os.path.exists(os.path.join("articles", source, safe_filename)): + logger.info(f"Article already cached: {filename}") + with open( + os.path.join("articles", source, safe_filename), "r", encoding="utf-8" + ) as f: + return f.read() + + # Random delay before fetching to avoid rate-limiting / bot detection + time.sleep(random.uniform(0.5, 2)) + + text = "" + + try: + # Try newspaper4k first with proper User-Agent to bypass bot detection + ua = get_random_ua() + article = newspaper.article(link, browser_user_agent=ua) + article.download() + article.parse() + text = article.text + + if not text or len(text) < 200: + raise ValueError( + "\tArticle text too short, falling back to Playwright/Selenium." + ) + logger.info(f"Successfully pulled article with newspaper4k from {link}") + + except Exception as e: + logger.warning( + f"newspaper4k extraction failed for {link}: {e}, falling back to Playwright." + ) + + try: + text = get_article_with_playwright(link) + logger.info(f"Successfully pulled article from {link} with Playwright") + + if not text or len(text) < 200: + logger.warning("Playwright article too short, falling back to Selenium.") + # Fallback to Selenium with better error handling + text = get_article_with_selenium(link) + logger.info(f"Successfully pulled article from {link} with Selenium") + except Exception as e: + logger.error(f"Playwright failed for {link}: {e}") + + # Fallback to Selenium + try: + text = get_article_with_selenium(link) + logger.info(f"Successfully pulled article from {link} with Selenium") + except Exception as e: + logger.error(f"Selenium failed for {link}: {e}") + return "" + + if save_to_file: + save_article_to_file(text, filename, source) + + return text + + +def is_article_downloaded(source, title, link): + """ + Check if an article is already downloaded by checking if the file exists. + """ + # Generate the same filename that would be used for saving + filename = title if title else link + safe_filename = generate_filename_from_url(filename) + + # Check if file exists in the articles directory + file_path = os.path.join("articles", source, safe_filename) + return os.path.exists(file_path) + + +def safe_pull_articles(article_list): + """ + Safely pull articles with improved error handling and increased parallelism. + """ + if not article_list: + return [], [] + + # Filter out articles that are already downloaded + filtered_article_list = [] + total_articles = len(article_list) + + for source, title, link in article_list: + if not is_article_downloaded(source, title, link): + filtered_article_list.append((source, title, link)) + else: + logger.info(f"Skipping already downloaded article: {title[:50]}... from {source}") + + logger.info(f"Filtered out {total_articles - len(filtered_article_list)} articles that were already downloaded") + logger.info(f"Processing {len(filtered_article_list)} remaining articles") + + if not filtered_article_list: + logger.info("No new articles to process") + return [], [] + + results = [] + errors = [] + + # Use ThreadPoolExecutor for parallel article pulling with higher concurrency + # Use configurable worker setting + max_workers = min(MAX_ARTICLE_WORKERS, len(filtered_article_list)) # Cap at configured workers, but don't exceed article count + batch_size = max(1, min(20, len(filtered_article_list) // 4)) # Dynamic batch size + + logger.info(f"Starting parallel article pulling with {max_workers} workers and batch size {batch_size}") + + # Process all articles in parallel with proper error handling + with ThreadPoolExecutor(max_workers=max_workers) as executor: + # Submit all tasks at once for maximum parallelism + futures = [ + executor.submit(pull_article, link, source, title) + for source, title, link in filtered_article_list + ] + + # Collect results as they complete + for i, future in enumerate(as_completed(futures, timeout=300)): # 5 minute timeout total + try: + result = future.result(timeout=120) # 2 minute timeout per article + if result: # Only count non-empty results + results.append(result) + # Log progress every 100 articles with proper batch information + if (i + 1) % 100 == 0: + logger.info(f"Processed {i + 1} articles out of {len(filtered_article_list)}") + except Exception as e: + logger.error(f"Error processing article: {e}") + errors.append(e) + + return results, errors + + +def main(): + """ + Main scraping loop. + """ + while True: + logger.info("=========================================") + logger.info("Starting new scraping iteration...") + + try: + # Pull the RSS feed sources from the JSON file + rss_feed_sources = load_rss_feed_sources() + + # Mine all articles from the RSS feed sources + rss_feed_links = mine_all_articles(rss_feed_sources) + + # Randomize the order of the links to help with load balancing + import random + + random.shuffle(rss_feed_links) + + logger.info(f"Found {len(rss_feed_links)} articles to process") + + if not rss_feed_links: + logger.info("No articles found, sleeping for 15 minutes") + time.sleep(15 * 60) + continue + + # Process articles with better error handling and resource management + results, errors = safe_pull_articles(rss_feed_links) + + logger.info(f"Attempted to Pull {len(results)} articles in parallel.") + logger.info( + f"Encountered {len(errors)} errors during article pulling. " + + "Outputting errors to a local file." + ) + + # Output errors to a local file + if errors: + with open("errors.txt", "w", encoding="utf-8") as f: + for error in errors: + f.write(str(error) + "\n") + logger.info("Errors logged to errors.txt") + + # Print all results to a log file + with open("results.txt", "w", encoding="utf-8") as f: + for result in results: + f.write(result + "\n") + logger.info("All articles pulled successfully.") + + # Log processing progress + progress = get_processing_progress() + logger.info(f"Processing progress - Total: {progress['total_articles']}, " + f"Completed: {progress['completed_articles']}, " + f"Embedded: {progress['embedded_articles']}, " + f"Completion rate: {progress['completion_rate']:.1f}%") + + except Exception as e: + logger.error(f"Major error in main loop: {e}") + # Continue to next iteration even if there's a major error + + # Sleep for a while before the next iteration + logger.info("Sleeping for 15 minutes before the next iteration...") + time.sleep(15 * 60) # Sleep for 15 minutes + + +if __name__ == "__main__": + main() diff --git a/setup_scraper.sh b/setup_scraper.sh new file mode 100755 index 0000000..eb847b6 --- /dev/null +++ b/setup_scraper.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +# ------------------------------------------------------------------ +# setup_scraper.sh +# +# Installs a virtual‑env, a tiny wrapper, optional dependencies +# and an hourly cron job – *always using the venv’s python/pip*. +# ------------------------------------------------------------------ +set -euo pipefail + +# ---------- Configuration ---------- +SCRAPER_DIR="/home/user/StockDocs/scraper" +VENV_DIR="${SCRAPER_DIR}/venv" +SCRIPT_PATH="${SCRAPER_DIR}/cron_scraper.py" +WRAPPER_PATH="${SCRAPER_DIR}/run_scraper.sh" +LOG_PATH="${SCRAPER_DIR}/cron.log" +REQUIREMENTS="${SCRAPER_DIR}/requirements.txt" + +# Cron line – this will call the wrapper, which in turn calls the +# venv’s Python interpreter. +CRON_LINE="0 * * * * ${WRAPPER_PATH} >> ${LOG_PATH} 2>&1" + +# ---------- Helper functions ---------- +log() { printf '[setup_scraper] %s\n' "$*"; } +error_exit() { printf '[setup_scraper] ERROR: %s\n' "$*" >&2; exit 1; } + +# ---------- 1️⃣ Create the venv if it’s missing ---------- +if [[ ! -d "$VENV_DIR" ]]; then + log "Creating virtual‑environment at ${VENV_DIR}" + python3 -m venv "$VENV_DIR" || error_exit "Failed to create venv" +else + log "Virtual‑environment already exists at ${VENV_DIR}" +fi + +# ---------- 2️⃣ Activate the venv *for the rest of this script* ---------- +# This changes PATH for the current shell only – it does NOT touch the +# system Python. The next line will confirm that we’re really in the venv. +# shellcheck source=/dev/null +source "${VENV_DIR}/bin/activate" + +# Quick sanity‑check: make sure we’re using the venv’s python and pip. +log "Current python: $(python -c 'import sys;print(sys.executable)')" +log "pip version: $(pip --version)" + +# ---------- 3️⃣ Install dependencies (optional) ---------- +if [[ -f "$REQUIREMENTS" ]]; then + log "Installing Python packages from ${REQUIREMENTS}" + # Use the *venv’s* pip explicitly – this guarantees no system installs. + "${VENV_DIR}/bin/pip" install --upgrade pip + "${VENV_DIR}/bin/pip" install -r "$REQUIREMENTS" || error_exit "pip install failed" +else + log "No requirements.txt found – skipping dependency install." +fi + +# ---------- 4️⃣ Create the wrapper script ---------- +# The wrapper *does NOT* source the venv any more – it calls the venv’s +# Python binary directly. This eliminates the subtle “activate” pitfall. +if [[ ! -f "$WRAPPER_PATH" ]]; then + log "Creating wrapper script at ${WRAPPER_PATH}" + cat > "$WRAPPER_PATH" <<'EOF' +#!/usr/bin/env bash +# ------------------------------------------------------------------ +# run_scraper.sh +# +# Activates the venv *implicitly* by calling the venv’s Python binary. +# ------------------------------------------------------------------ +# Absolute path to the venv – change only if you move the venv. +VENV_DIR="/home/user/StockDocs/scraper/venv" + +# Absolute path to the script you want to run. +SCRIPT="/home/user/StockDocs/scraper/cron_scraper.py" + +# Invoke the venv’s Python directly. +"${VENV_DIR}/bin/python" "${SCRIPT}" # output is redirected by cron +EOF + chmod +x "$WRAPPER_PATH" +else + log "Wrapper script already exists at ${WRAPPER_PATH}" +fi + +# ---------- 5️⃣ Install the cron job ---------- +CURRENT_CRON=$(crontab -l 2>/dev/null || true) + +if echo "$CURRENT_CRON" | grep -Fqx "$CRON_LINE"; then + log "Crontab entry already present – nothing to do." +else + log "Adding new cron entry." + # Append the new line and reinstall the crontab. + (printf '%s\n' "$CURRENT_CRON" ; printf '%s\n' "$CRON_LINE") | crontab - +fi + +log "✅ Setup complete! ${SCRIPT_PATH} will run every hour via the venv." diff --git a/tests/test_scraper_cache.py b/tests/test_scraper_cache.py new file mode 100644 index 0000000..6db65c5 --- /dev/null +++ b/tests/test_scraper_cache.py @@ -0,0 +1,61 @@ +"""Tests for the scraper article-processing cache.""" + +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from scraper.scraper import ( # noqa: E402 + get_processing_progress, + load_processed_cache, + mark_article_processed, + save_processed_cache, +) + + +@pytest.fixture() +def cache_dir(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + (tmp_path / "articles").mkdir() + return tmp_path + + +def test_load_missing_cache_returns_empty(cache_dir): + assert load_processed_cache() == {} + + +def test_save_and_load_roundtrip(cache_dir): + save_processed_cache({"a.html": {"status": "completed"}}) + cache = load_processed_cache() + assert cache["a.html"]["status"] == "completed" + + +def test_mark_article_processed(cache_dir): + mark_article_processed("articles/x/article.html") + cache = load_processed_cache() + entry = cache["articles/x/article.html"] + assert entry["status"] == "completed" + assert entry["embedding_status"] == "pending" + assert "processed_date" in entry + assert "last_updated" in entry + + +def test_processing_progress_empty(cache_dir): + progress = get_processing_progress() + assert progress["total_articles"] == 0 + assert progress["completed_articles"] == 0 + assert progress["embedded_articles"] == 0 + assert progress["completion_rate"] == 0 + + +def test_processing_progress_mixed(cache_dir): + mark_article_processed("a.html") + mark_article_processed("b.html", status="failed") + mark_article_processed("c.html", embedding_status="completed") + progress = get_processing_progress() + assert progress["total_articles"] == 3 + assert progress["completed_articles"] == 2 + assert progress["embedded_articles"] == 1 + assert progress["completion_rate"] == pytest.approx(2 / 3 * 100)