Fix file path handling and add robust error handling for cron job execution

scraper/cron_scraper.py

@@ -20,7 +20,28 @@ from concurrent.futures import ThreadPoolExecutor, as_completed
 import nltk
 from nltk.downloader import Downloader
 
-FEED_FILE = os.getenv("FEED_FILE", "./rss_feeds.json")
+# Robust file path handling - try multiple locations
+def get_feed_file_path():
+    """Get the RSS feed file path, trying multiple locations."""
+    possible_paths = [
+        "./rss_feeds.json",           # Current directory
+        "../rss_feeds.json",          # Parent directory
+        "/home/user/StockDocs/scraper/rss_feeds.json",  # Explicit path
+        "/app/rss_feeds.json",        # Docker path
+        "./scraper/rss_feeds.json"    # Scraper subdirectory
+    ]
+    
+    for path in possible_paths:
+        if os.path.exists(path):
+            print(f"Found feed file at: {path}")
+            return path
+    
+    # If no file found, return default and let it fail gracefully
+    print("Warning: RSS feed file not found in any expected location")
+    return "./rss_feeds.json"
+
+# Get the feed file path
+FEED_FILE = get_feed_file_path()
 
 # Ensure necessary NLTK resources are downloaded
 d = Downloader()
@@ -34,17 +55,31 @@ def load_rss_feed_sources(feed_file=FEED_FILE):
     """
     Loads the RSS feed sources from a JSON file.
     """
-    print("Loading RSS feed sources from rss_feeds.json...")
+    print(f"Loading RSS feed sources from {feed_file}...")
+    
+    # Debug: Print current working directory
+    print(f"Current working directory: {os.getcwd()}")
     
     try:
-        with open(FEED_FILE, "r", encoding="utf-8") as f:
+        with open(feed_file, "r", encoding="utf-8") as f:
-            return json.load(f)
+            data = json.load(f)
+            print(f"Successfully loaded {feed_file}")
+            print(f"Data type: {type(data)}")
+            if isinstance(data, dict) and "rss_feeds" in data:
+                print(f"Found rss_feeds section with {len(data['rss_feeds'])} sources")
+                return data
+            else:
+                print(f"Warning: Unexpected data structure. Data keys: {list(data.keys()) if isinstance(data, dict) else 'Not a dict'}")
+                return {}
     except FileNotFoundError:
-        print(FEED_FILE + " not found, returning empty list.")
+        print(f"{feed_file} not found, returning empty dict.")
-        return []
+        return {}
-    except json.JSONDecodeError:
+    except json.JSONDecodeError as e:
-        print("Error decoding " + FEED_FILE + " , returning empty list.")
+        print(f"Error decoding {feed_file}: {e}, returning empty dict.")
-        return []
+        return {}
+    except Exception as e:
+        print(f"Unexpected error loading {feed_file}: {e}")
+        return {}
 
 
 def mine_all_articles(rss_feed_sources, limit=None):
@@ -53,6 +88,16 @@ def mine_all_articles(rss_feed_sources, limit=None):
     Returns a list of (site, title, link) tuples.
     """
     all_links = []
+    
+    # Check if rss_feed_sources is a valid dict with rss_feeds key
+    if not isinstance(rss_feed_sources, dict):
+        print(f"Warning: rss_feed_sources is not a dict, it's {type(rss_feed_sources)}")
+        return all_links
+        
+    if "rss_feeds" not in rss_feed_sources:
+        print("Warning: rss_feeds key not found in rss_feed_sources")
+        return all_links
+
     sources = rss_feed_sources["rss_feeds"]
 
     for site, data in sources.items():