diff --git a/scraper/scraper.py b/scraper/scraper.py index 8b9a57e..071c451 100644 --- a/scraper/scraper.py +++ b/scraper/scraper.py @@ -178,8 +178,18 @@ def get_article_with_selenium(url): options.add_argument("--headless") options.set_preference("dom.ipc.processCount", 1) # Reduce process count - # Initialize driver - driver = webdriver.Firefox(options=options) + # Try to initialize driver with explicit path to Firefox + try: + driver = webdriver.Firefox(options=options) + except Exception as e: + # If that fails, try with explicit Firefox path + if "binary is not a firefox executable" in str(e).lower(): + logger.info("Attempting to use Firefox at /usr/bin/firefox") + options.binary_location = "/usr/bin/firefox" + driver = webdriver.Firefox(options=options) + else: + raise e + driver.set_page_load_timeout(30) # 30 seconds timeout # Navigate to URL @@ -208,7 +218,7 @@ def get_article_with_selenium(url): error_str = str(e).lower() if "binary is not a firefox executable" in error_str or "firefox" in error_str: logger.error(f"Firefox not found or not properly configured for {url}: {str(e)}") - logger.error("Please ensure Firefox is installed and properly configured in PATH") + logger.error("Firefox is installed at /usr/bin/firefox but may not be accessible. Check PATH or permissions.") else: logger.error(f"Selenium failed for {url}: {str(e)}") return ""