diff --git a/scraper/cron_scraper.py b/scraper/cron_scraper.py index 8c92503..5546f19 100644 --- a/scraper/cron_scraper.py +++ b/scraper/cron_scraper.py @@ -11,6 +11,8 @@ import feedparser import time import os import requests +import logging +from datetime import datetime from selenium import webdriver from selenium.webdriver.firefox.options import Options as FirefoxOptions from selenium.webdriver.common.by import By @@ -20,6 +22,14 @@ from concurrent.futures import ThreadPoolExecutor, as_completed import nltk from nltk.downloader import Downloader +# Setup logging with timestamps +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(levelname)s - %(message)s', + datefmt='%Y-%m-%d %H:%M:%S' +) +logger = logging.getLogger(__name__) + # Robust file path handling - try multiple locations def get_feed_file_path(): """Get the RSS feed file path, trying multiple locations.""" @@ -56,30 +66,30 @@ def load_rss_feed_sources(feed_file=FEED_FILE): """ Loads the RSS feed sources from a JSON file. """ - print(f"Loading RSS feed sources from {feed_file}...") + logger.info(f"Loading RSS feed sources from {feed_file}...") # Debug: Print current working directory - print(f"Current working directory: {os.getcwd()}") + logger.debug(f"Current working directory: {os.getcwd()}") try: with open(feed_file, "r", encoding="utf-8") as f: data = json.load(f) - print(f"Successfully loaded {feed_file}") - print(f"Data type: {type(data)}") + logger.info(f"Successfully loaded {feed_file}") + logger.debug(f"Data type: {type(data)}") if isinstance(data, dict) and "rss_feeds" in data: - print(f"Found rss_feeds section with {len(data['rss_feeds'])} sources") + logger.info(f"Found rss_feeds section with {len(data['rss_feeds'])} sources") return data else: - print(f"Warning: Unexpected data structure. Data keys: {list(data.keys()) if isinstance(data, dict) else 'Not a dict'}") + logger.warning(f"Unexpected data structure. Data keys: {list(data.keys()) if isinstance(data, dict) else 'Not a dict'}") return {} except FileNotFoundError: - print(f"{feed_file} not found, returning empty dict.") + logger.error(f"{feed_file} not found, returning empty dict.") return {} except json.JSONDecodeError as e: - print(f"Error decoding {feed_file}: {e}, returning empty dict.") + logger.error(f"Error decoding {feed_file}: {e}, returning empty dict.") return {} except Exception as e: - print(f"Unexpected error loading {feed_file}: {e}") + logger.error(f"Unexpected error loading {feed_file}: {e}") return {} @@ -92,17 +102,17 @@ def mine_all_articles(rss_feed_sources, limit=None): # Check if rss_feed_sources is a valid dict with rss_feeds key if not isinstance(rss_feed_sources, dict): - print(f"Warning: rss_feed_sources is not a dict, it's {type(rss_feed_sources)}") + logger.warning(f"rss_feed_sources is not a dict, it's {type(rss_feed_sources)}") return all_links if "rss_feeds" not in rss_feed_sources: - print("Warning: rss_feeds key not found in rss_feed_sources") + logger.warning("rss_feeds key not found in rss_feed_sources") return all_links sources = rss_feed_sources["rss_feeds"] for site, data in sources.items(): - print(f"Parsing RSS feed: {data['rss_url']}") + logger.info(f"Parsing RSS feed: {data['rss_url']}") try: feed = feedparser.parse(data["rss_url"]) feed_entries = feed.entries[:limit] if limit else feed.entries @@ -111,7 +121,7 @@ def mine_all_articles(rss_feed_sources, limit=None): if "link" in entry and "title" in entry: all_links.append((site, entry.title, entry.link)) except Exception as e: - print(f"Error parsing RSS feed: {site} Error: {str(e)}") + logger.error(f"Error parsing RSS feed: {site} Error: {str(e)}") return all_links @@ -148,7 +158,9 @@ def save_article_to_file(article, filename, source="Unfiltered"): with open(file_path, "w", encoding="utf-8") as f: f.write(f"SOURCE:{source}\n") f.write(article) - print(f"Article saved to {file_path}") + + # Only log when a new file is actually created (not cached) + logger.info(f"New article saved: {safe_filename} from {source}") def get_article_with_selenium(url): @@ -185,10 +197,11 @@ def get_article_with_selenium(url): # Parse with Newspaper4k article = newspaper.article(url, input_html=html, language="en") article.nlp() + logger.info(f"Successfully extracted article with Selenium from {url}") return article.text except Exception as e: - print(f"Selenium failed for {url}: {str(e)}") + logger.error(f"Selenium failed for {url}: {str(e)}") return "" finally: # Always quit the driver @@ -229,10 +242,11 @@ def get_article_with_playwright(url): # Parse with Newspaper4k article = newspaper.article(url, input_html=html, language="en") article.nlp() + logger.info(f"Successfully extracted article with Playwright from {url}") return article.text except Exception as e: - print(f"Playwright failed for {url}: {str(e)}") + logger.error(f"Playwright failed for {url}: {str(e)}") return "" @@ -245,7 +259,7 @@ def pull_article(link, source, title=None, save_to_file=True): # Check if already cached if os.path.exists(os.path.join("articles", source, safe_filename)): - print(f"Article already cached: {filename}") + logger.info(f"Article already cached: {filename}") with open( os.path.join("articles", source, safe_filename), "r", encoding="utf-8" ) as f: @@ -264,31 +278,31 @@ def pull_article(link, source, title=None, save_to_file=True): raise ValueError( "\tArticle text too short, falling back to Playwright/Selenium." ) - print(f"\tSuccessfully pulled article with newspaper4k from {link}") + logger.info(f"Successfully pulled article with newspaper4k from {link}") except Exception as e: - print( - f"\tnewspaper4k extraction failed for {link}: {e}, falling back to Playwright." + logger.warning( + f"newspaper4k extraction failed for {link}: {e}, falling back to Playwright." ) try: text = get_article_with_playwright(link) - print(f"\t\tSuccessfully pulled article from {link} with Playwright") + logger.info(f"Successfully pulled article from {link} with Playwright") if not text or len(text) < 200: - print(f"\t\tPlaywright article too short, falling back to Selenium.") + logger.warning(f"Playwright article too short, falling back to Selenium.") # Fallback to Selenium with better error handling text = get_article_with_selenium(link) - print(f"\t\t\tSuccessfully pulled article from {link} with Selenium") + logger.info(f"Successfully pulled article from {link} with Selenium") except Exception as e: - print(f"\t\tPlaywright failed for {link}: {e}") + logger.error(f"Playwright failed for {link}: {e}") # Fallback to Selenium try: text = get_article_with_selenium(link) - print(f"\t\tSuccessfully pulled article from {link} with Selenium") + logger.info(f"Successfully pulled article from {link} with Selenium") except Exception as e: - print(f"\t\tSelenium failed for {link}: {e}") + logger.error(f"Selenium failed for {link}: {e}") return "" if save_to_file: @@ -309,7 +323,7 @@ def safe_pull_articles(article_list): for i in range(0, len(article_list), batch_size): batch = article_list[i : i + batch_size] - print(f"Processing batch {i // batch_size + 1} with {len(batch)} articles") + logger.info(f"Processing batch {i // batch_size + 1} with {len(batch)} articles") # Use ThreadPoolExecutor instead of ProcessPoolExecutor to avoid # process termination issues with browser automation @@ -324,7 +338,7 @@ def safe_pull_articles(article_list): result = future.result(timeout=120) # 2 minute timeout results.append(result) except Exception as e: - print(f"Error in pull_article: {e}") + logger.error(f"Error in pull_article: {e}") errors.append(e) # Add a small delay between batches to reduce system load @@ -390,8 +404,8 @@ def main(): Main scraping function for cron execution. This replaces the infinite while loop with a single execution. """ - print("=========================================") - print("Starting new scraping iteration...") + logger.info("=========================================") + logger.info("Starting new scraping iteration...") try: # Pull the RSS feed sources from the JSON file @@ -405,17 +419,17 @@ def main(): random.shuffle(rss_feed_links) - print(f"Found {len(rss_feed_links)} articles to process") + logger.info(f"Found {len(rss_feed_links)} articles to process") if not rss_feed_links: - print("No articles found") + logger.info("No articles found") return # Process articles with better error handling and resource management results, errors = safe_pull_articles(rss_feed_links) - print(f"Attempted to Pull {len(results)} articles in parallel.") - print( + logger.info(f"Attempted to Pull {len(results)} articles in parallel.") + logger.info( f"Encountered {len(errors)} errors during article pulling. " + "Outputting errors to a local file." ) @@ -425,28 +439,28 @@ def main(): with open("errors.txt", "w", encoding="utf-8") as f: for error in errors: f.write(str(error) + "\n") - print(f"Errors logged to errors.txt") + logger.info(f"Errors logged to errors.txt") # Print all results to a log file with open("results.txt", "w", encoding="utf-8") as f: for result in results: f.write(result + "\n") - print("All articles pulled successfully.") + logger.info("All articles pulled successfully.") # Gather and send new articles to webhook new_articles = gather_new_articles() if new_articles: send_to_webhook(new_articles) else: - print("No new articles to send to webhook") + logger.info("No new articles to send to webhook") except Exception as e: - print(f"Major error in main execution: {e}") + logger.error(f"Major error in main execution: {e}") import traceback traceback.print_exc() raise # Re-raise to ensure the script exits with error code - print("Scraping completed successfully.") + logger.info("Scraping completed successfully.") if __name__ == "__main__": diff --git a/scraper/scraper.py b/scraper/scraper.py index 1d6abf6..e9acc47 100644 --- a/scraper/scraper.py +++ b/scraper/scraper.py @@ -3,6 +3,8 @@ import json import feedparser import time import os +import logging +from datetime import datetime from selenium import webdriver from selenium.webdriver.firefox.options import Options as FirefoxOptions from selenium.webdriver.common.by import By @@ -12,6 +14,14 @@ from concurrent.futures import ThreadPoolExecutor, as_completed import nltk from nltk.downloader import Downloader +# Setup logging with timestamps +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(levelname)s - %(message)s', + datefmt='%Y-%m-%d %H:%M:%S' +) +logger = logging.getLogger(__name__) + FEED_FILE = os.getenv("FEED_FILE", "./rss_short_feed.json") # Ensure necessary NLTK resources are downloaded @@ -26,16 +36,18 @@ def load_rss_feed_sources(feed_file=FEED_FILE): """ Loads the RSS feed sources from a JSON file. """ - print("Loading RSS feed sources from rss_feeds.json...") + logger.info(f"Loading RSS feed sources from {feed_file}...") try: - with open(FEED_FILE, "r", encoding="utf-8") as f: - return json.load(f) + with open(feed_file, "r", encoding="utf-8") as f: + data = json.load(f) + logger.info(f"Successfully loaded {feed_file}") + return data except FileNotFoundError: - print(FEED_FILE + " not found, returning empty list.") + logger.error(f"{feed_file} not found, returning empty list.") return [] except json.JSONDecodeError: - print("Error decoding " + FEED_FILE + " , returning empty list.") + logger.error(f"Error decoding {feed_file} , returning empty list.") return [] @@ -45,10 +57,20 @@ def mine_all_articles(rss_feed_sources, limit=None): Returns a list of (site, title, link) tuples. """ all_links = [] + + # Check if rss_feed_sources is a valid dict with rss_feeds key + if not isinstance(rss_feed_sources, dict): + logger.warning(f"rss_feed_sources is not a dict, it's {type(rss_feed_sources)}") + return all_links + + if "rss_feeds" not in rss_feed_sources: + logger.warning("rss_feeds key not found in rss_feed_sources") + return all_links + sources = rss_feed_sources["rss_feeds"] for site, data in sources.items(): - print(f"Parsing RSS feed: {data['rss_url']}") + logger.info(f"Parsing RSS feed: {data['rss_url']}") try: feed = feedparser.parse(data["rss_url"]) feed_entries = feed.entries[:limit] if limit else feed.entries @@ -57,7 +79,7 @@ def mine_all_articles(rss_feed_sources, limit=None): if "link" in entry and "title" in entry: all_links.append((site, entry.title, entry.link)) except Exception as e: - print(f"Error parsing RSS feed: {site} Error: {str(e)}") + logger.error(f"Error parsing RSS feed: {site} Error: {str(e)}") return all_links @@ -94,7 +116,9 @@ def save_article_to_file(article, filename, source="Unfiltered"): with open(file_path, "w", encoding="utf-8") as f: f.write(f"SOURCE:{source}\n") f.write(article) - print(f"Article saved to {file_path}") + + # Only log when a new file is actually created (not cached) + logger.info(f"New article saved: {safe_filename} from {source}") def get_article_with_selenium(url): @@ -131,10 +155,11 @@ def get_article_with_selenium(url): # Parse with Newspaper4k article = newspaper.article(url, input_html=html, language="en") article.nlp() + logger.info(f"Successfully extracted article with Selenium from {url}") return article.text except Exception as e: - print(f"Selenium failed for {url}: {str(e)}") + logger.error(f"Selenium failed for {url}: {str(e)}") return "" finally: # Always quit the driver @@ -175,10 +200,11 @@ def get_article_with_playwright(url): # Parse with Newspaper4k article = newspaper.article(url, input_html=html, language="en") article.nlp() + logger.info(f"Successfully extracted article with Playwright from {url}") return article.text except Exception as e: - print(f"Playwright failed for {url}: {str(e)}") + logger.error(f"Playwright failed for {url}: {str(e)}") return "" @@ -191,7 +217,7 @@ def pull_article(link, source, title=None, save_to_file=True): # Check if already cached if os.path.exists(os.path.join("articles", source, safe_filename)): - print(f"Article already cached: {filename}") + logger.info(f"Article already cached: {filename}") with open( os.path.join("articles", source, safe_filename), "r", encoding="utf-8" ) as f: @@ -210,31 +236,31 @@ def pull_article(link, source, title=None, save_to_file=True): raise ValueError( "\tArticle text too short, falling back to Playwright/Selenium." ) - print(f"\tSuccessfully pulled article with newspaper4k from {link}") + logger.info(f"Successfully pulled article with newspaper4k from {link}") except Exception as e: - print( - f"\tnewspaper4k extraction failed for {link}: {e}, falling back to Playwright." + logger.warning( + f"newspaper4k extraction failed for {link}: {e}, falling back to Playwright." ) try: text = get_article_with_playwright(link) - print(f"\t\tSuccessfully pulled article from {link} with Playwright") + logger.info(f"Successfully pulled article from {link} with Playwright") if not text or len(text) < 200: - print(f"\t\tPlaywright article too short, falling back to Selenium.") + logger.warning(f"Playwright article too short, falling back to Selenium.") # Fallback to Selenium with better error handling text = get_article_with_selenium(link) - print(f"\t\t\tSuccessfully pulled article from {link} with Selenium") + logger.info(f"Successfully pulled article from {link} with Selenium") except Exception as e: - print(f"\t\tPlaywright failed for {link}: {e}") + logger.error(f"Playwright failed for {link}: {e}") # Fallback to Selenium try: text = get_article_with_selenium(link) - print(f"\t\tSuccessfully pulled article from {link} with Selenium") + logger.info(f"Successfully pulled article from {link} with Selenium") except Exception as e: - print(f"\t\tSelenium failed for {link}: {e}") + logger.error(f"Selenium failed for {link}: {e}") return "" if save_to_file: @@ -255,7 +281,7 @@ def safe_pull_articles(article_list): for i in range(0, len(article_list), batch_size): batch = article_list[i : i + batch_size] - print(f"Processing batch {i // batch_size + 1} with {len(batch)} articles") + logger.info(f"Processing batch {i // batch_size + 1} with {len(batch)} articles") # Use ThreadPoolExecutor instead of ProcessPoolExecutor to avoid # process termination issues with browser automation @@ -270,7 +296,7 @@ def safe_pull_articles(article_list): result = future.result(timeout=120) # 2 minute timeout results.append(result) except Exception as e: - print(f"Error in pull_article: {e}") + logger.error(f"Error in pull_article: {e}") errors.append(e) # Add a small delay between batches to reduce system load @@ -284,8 +310,8 @@ def main(): Main scraping loop. """ while True: - print("=========================================") - print("Starting new scraping iteration...") + logger.info("=========================================") + logger.info("Starting new scraping iteration...") try: # Pull the RSS feed sources from the JSON file @@ -299,18 +325,18 @@ def main(): random.shuffle(rss_feed_links) - print(f"Found {len(rss_feed_links)} articles to process") + logger.info(f"Found {len(rss_feed_links)} articles to process") if not rss_feed_links: - print("No articles found, sleeping for 15 minutes") + logger.info("No articles found, sleeping for 15 minutes") time.sleep(15 * 60) continue # Process articles with better error handling and resource management results, errors = safe_pull_articles(rss_feed_links) - print(f"Attempted to Pull {len(results)} articles in parallel.") - print( + logger.info(f"Attempted to Pull {len(results)} articles in parallel.") + logger.info( f"Encountered {len(errors)} errors during article pulling. " + "Outputting errors to a local file." ) @@ -320,20 +346,20 @@ def main(): with open("errors.txt", "w", encoding="utf-8") as f: for error in errors: f.write(str(error) + "\n") - print(f"Errors logged to errors.txt") + logger.info(f"Errors logged to errors.txt") # Print all results to a log file with open("results.txt", "w", encoding="utf-8") as f: for result in results: f.write(result + "\n") - print("All articles pulled successfully.") + logger.info("All articles pulled successfully.") except Exception as e: - print(f"Major error in main loop: {e}") + logger.error(f"Major error in main loop: {e}") # Continue to next iteration even if there's a major error # Sleep for a while before the next iteration - print("Sleeping for 15 minutes before the next iteration...") + logger.info("Sleeping for 15 minutes before the next iteration...") time.sleep(15 * 60) # Sleep for 15 minutes