From 874c64c9d16390bcc9fe0980ecfb8e86d9c5ad9f Mon Sep 17 00:00:00 2001
From: Jarian Cottingham <jarianc@proton.me>
Date: Sat, 31 Jan 2026 11:00:14 -0600
Subject: [PATCH] Fix file path handling and add robust error handling for cron
 job execution

---
 scraper/cron_scraper.py | 65 ++++++++++++++++++++++++++++++++++-------
 1 file changed, 55 insertions(+), 10 deletions(-)

diff --git a/scraper/cron_scraper.py b/scraper/cron_scraper.py
index 8efa717..9e75f1f 100644
--- a/scraper/cron_scraper.py
+++ b/scraper/cron_scraper.py
@@ -20,7 +20,28 @@ from concurrent.futures import ThreadPoolExecutor, as_completed
 import nltk
 from nltk.downloader import Downloader
 
-FEED_FILE = os.getenv("FEED_FILE", "./rss_feeds.json")
+# Robust file path handling - try multiple locations
+def get_feed_file_path():
+    """Get the RSS feed file path, trying multiple locations."""
+    possible_paths = [
+        "./rss_feeds.json",           # Current directory
+        "../rss_feeds.json",          # Parent directory
+        "/home/user/StockDocs/scraper/rss_feeds.json",  # Explicit path
+        "/app/rss_feeds.json",        # Docker path
+        "./scraper/rss_feeds.json"    # Scraper subdirectory
+    ]
+    
+    for path in possible_paths:
+        if os.path.exists(path):
+            print(f"Found feed file at: {path}")
+            return path
+    
+    # If no file found, return default and let it fail gracefully
+    print("Warning: RSS feed file not found in any expected location")
+    return "./rss_feeds.json"
+
+# Get the feed file path
+FEED_FILE = get_feed_file_path()
 
 # Ensure necessary NLTK resources are downloaded
 d = Downloader()
@@ -34,17 +55,31 @@ def load_rss_feed_sources(feed_file=FEED_FILE):
     """
     Loads the RSS feed sources from a JSON file.
     """
-    print("Loading RSS feed sources from rss_feeds.json...")
-
+    print(f"Loading RSS feed sources from {feed_file}...")
+    
+    # Debug: Print current working directory
+    print(f"Current working directory: {os.getcwd()}")
+    
     try:
-        with open(FEED_FILE, "r", encoding="utf-8") as f:
-            return json.load(f)
+        with open(feed_file, "r", encoding="utf-8") as f:
+            data = json.load(f)
+            print(f"Successfully loaded {feed_file}")
+            print(f"Data type: {type(data)}")
+            if isinstance(data, dict) and "rss_feeds" in data:
+                print(f"Found rss_feeds section with {len(data['rss_feeds'])} sources")
+                return data
+            else:
+                print(f"Warning: Unexpected data structure. Data keys: {list(data.keys()) if isinstance(data, dict) else 'Not a dict'}")
+                return {}
     except FileNotFoundError:
-        print(FEED_FILE + " not found, returning empty list.")
-        return []
-    except json.JSONDecodeError:
-        print("Error decoding " + FEED_FILE + " , returning empty list.")
-        return []
+        print(f"{feed_file} not found, returning empty dict.")
+        return {}
+    except json.JSONDecodeError as e:
+        print(f"Error decoding {feed_file}: {e}, returning empty dict.")
+        return {}
+    except Exception as e:
+        print(f"Unexpected error loading {feed_file}: {e}")
+        return {}
 
 
 def mine_all_articles(rss_feed_sources, limit=None):
@@ -53,6 +88,16 @@ def mine_all_articles(rss_feed_sources, limit=None):
     Returns a list of (site, title, link) tuples.
     """
     all_links = []
+    
+    # Check if rss_feed_sources is a valid dict with rss_feeds key
+    if not isinstance(rss_feed_sources, dict):
+        print(f"Warning: rss_feed_sources is not a dict, it's {type(rss_feed_sources)}")
+        return all_links
+        
+    if "rss_feeds" not in rss_feed_sources:
+        print("Warning: rss_feeds key not found in rss_feed_sources")
+        return all_links
+
     sources = rss_feed_sources["rss_feeds"]
 
     for site, data in sources.items():