#!/usr/bin/env python3
"""
Cron-based scraper for downloading articles from RSS feeds.
This version replaces the infinite while loop with a single execution
that can be scheduled via cron job.
"""

import newspaper
import json
import feedparser
import time
import os
from selenium import webdriver
from selenium.webdriver.firefox.options import Options as FirefoxOptions
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from concurrent.futures import ThreadPoolExecutor, as_completed
import nltk
from nltk.downloader import Downloader

FEED_FILE = os.getenv("FEED_FILE", "./rss_feeds.json")

# Ensure necessary NLTK resources are downloaded
d = Downloader()
if not d.is_installed("punkt_tab"):
    nltk.download("punkt_tab")

articles = []


def load_rss_feed_sources(feed_file=FEED_FILE):
    """
    Loads the RSS feed sources from a JSON file.
    """
    print("Loading RSS feed sources from rss_feeds.json...")

    try:
        with open(FEED_FILE, "r", encoding="utf-8") as f:
            return json.load(f)
    except FileNotFoundError:
        print(FEED_FILE + " not found, returning empty list.")
        return []
    except json.JSONDecodeError:
        print("Error decoding " + FEED_FILE + " , returning empty list.")
        return []


def mine_all_articles(rss_feed_sources, limit=None):
    """
    Mines all articles from the given RSS feed sources.
    Returns a list of (site, title, link) tuples.
    """
    all_links = []
    sources = rss_feed_sources["rss_feeds"]

    for site, data in sources.items():
        print(f"Parsing RSS feed: {data['rss_url']}")
        try:
            feed = feedparser.parse(data["rss_url"])
            feed_entries = feed.entries[:limit] if limit else feed.entries

            for entry in feed_entries:
                if "link" in entry and "title" in entry:
                    all_links.append((site, entry.title, entry.link))
        except Exception as e:
            print(f"Error parsing RSS feed: {site} Error: {str(e)}")
    return all_links


def generate_filename_from_url(url):
    """
    Generates a filename from the given URL by replacing slashes with underscores.
    """
    # Use only the last part of the URL or replace slashes
    return url.replace("https://", "").replace("http://", "").replace("/", "_")


def generate_safe_filename(name):
    # Remove/replace characters not allowed in filenames
    import re

    safe = re.sub(r'[\\/*?:"<>|]', "_", name)
    return safe


def save_article_to_file(article, filename, source="Unfiltered"):
    """
    Saves the given article text to a file with the specified filename.
    """
    # articles dir should already be there
    # os.makedirs("articles", exist_ok=True)

    outputDir = "articles/" + source
    os.makedirs(outputDir, exist_ok=True) if source else None

    # Sanitize filename: use only the last part of the URL or replace slashes
    safe_filename = generate_filename_from_url(filename)
    file_path = os.path.join(outputDir, safe_filename)
    # Save the source as the first line in the file for later retrieval
    with open(file_path, "w", encoding="utf-8") as f:
        f.write(f"SOURCE:{source}\n")
        f.write(article)
    print(f"Article saved to {file_path}")


def get_article_with_selenium(url):
    """
    Gets article text using Selenium Firefox driver with proper error handling
    and cleanup.
    """
    driver = None
    try:
        # Configure Firefox options
        options = FirefoxOptions()
        options.add_argument("--headless")
        options.set_preference("dom.ipc.processCount", 1)  # Reduce process count

        # Initialize driver with timeout
        driver = webdriver.Firefox(options=options)
        driver.set_page_load_timeout(30)  # 30 seconds timeout

        # Navigate to URL
        driver.get(url)

        # Wait for page to load (explicit wait instead of sleep)
        try:
            WebDriverWait(driver, 15).until(
                EC.presence_of_element_located((By.TAG_NAME, "body"))
            )
        except:
            pass  # Continue even if wait times out

        time.sleep(2)  # Brief additional wait

        html = driver.page_source

        # Parse with Newspaper4k
        article = newspaper.article(url, input_html=html, language="en")
        article.nlp()
        return article.text

    except Exception as e:
        print(f"Selenium failed for {url}: {str(e)}")
        return ""
    finally:
        # Always quit the driver
        if driver:
            try:
                driver.quit()
            except:
                pass  # Ignore errors in cleanup


def get_article_with_playwright(url):
    """
    Gets article text using Playwright with proper error handling.
    """
    try:
        from playwright.sync_api import sync_playwright

        with sync_playwright() as p:
            # Use Chromium instead of Firefox for better compatibility
            browser = p.chromium.launch(headless=True, timeout=30000)
            page = browser.new_page()

            # Set user agent to avoid bot detection
            page.set_extra_http_headers(
                {
                    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
                }
            )

            page.goto(url, wait_until="load")

            # Wait for content to load
            time.sleep(3)

            html = page.content()
            browser.close()

        # Parse with Newspaper4k
        article = newspaper.article(url, input_html=html, language="en")
        article.nlp()
        return article.text

    except Exception as e:
        print(f"Playwright failed for {url}: {str(e)}")
        return ""


def pull_article(link, source, title=None, save_to_file=True):
    """
    Pulls an article from a given link with fallback mechanisms.
    """
    filename = title if title else link
    safe_filename = generate_filename_from_url(filename)

    # Check if already cached
    if os.path.exists(os.path.join("articles", source, safe_filename)):
        print(f"Article already cached: {filename}")
        with open(
            os.path.join("articles", source, safe_filename), "r", encoding="utf-8"
        ) as f:
            return f.read()

    text = ""

    try:
        # Try newspaper4k first
        article = newspaper.article(link)
        article.download()
        article.parse()
        text = article.text

        if not text or len(text) < 200:
            raise ValueError(
                "\tArticle text too short, falling back to Playwright/Selenium."
            )
        print(f"\tSuccessfully pulled article with newspaper4k from {link}")

    except Exception as e:
        print(
            f"\tnewspaper4k extraction failed for {link}: {e}, falling back to Playwright."
        )

        try:
            text = get_article_with_playwright(link)
            print(f"\t\tSuccessfully pulled article from {link} with Playwright")

            if not text or len(text) < 200:
                print(f"\t\tPlaywright article too short, falling back to Selenium.")
                # Fallback to Selenium with better error handling
                text = get_article_with_selenium(link)
                print(f"\t\t\tSuccessfully pulled article from {link} with Selenium")
        except Exception as e:
            print(f"\t\tPlaywright failed for {link}: {e}")

            # Fallback to Selenium
            try:
                text = get_article_with_selenium(link)
                print(f"\t\tSuccessfully pulled article from {link} with Selenium")
            except Exception as e:
                print(f"\t\tSelenium failed for {link}: {e}")
                return ""

    if save_to_file:
        save_article_to_file(text, filename, source)

    return text


def safe_pull_articles(article_list):
    """
    Safely pull articles with improved error handling and reduced parallelism.
    """
    results = []
    errors = []

    # Process in smaller batches to reduce resource strain
    batch_size = 5

    for i in range(0, len(article_list), batch_size):
        batch = article_list[i : i + batch_size]
        print(f"Processing batch {i // batch_size + 1} with {len(batch)} articles")

        # Use ThreadPoolExecutor instead of ProcessPoolExecutor to avoid
        # process termination issues with browser automation
        with ThreadPoolExecutor(max_workers=3) as executor:  # Reduced workers
            futures = [
                executor.submit(pull_article, link, source, title)
                for source, title, link in batch
            ]

            for future in as_completed(futures):
                try:
                    result = future.result(timeout=120)  # 2 minute timeout
                    results.append(result)
                except Exception as e:
                    print(f"Error in pull_article: {e}")
                    errors.append(e)

        # Add a small delay between batches to reduce system load
        time.sleep(5)

    return results, errors


def main():
    """
    Main scraping function for cron execution.
    This replaces the infinite while loop with a single execution.
    """
    print("=========================================")
    print("Starting new scraping iteration...")

    try:
        # Pull the RSS feed sources from the JSON file
        rss_feed_sources = load_rss_feed_sources()

        # Mine all articles from the RSS feed sources
        rss_feed_links = mine_all_articles(rss_feed_sources)

        # Randomize the order of the links to help with load balancing
        import random

        random.shuffle(rss_feed_links)

        print(f"Found {len(rss_feed_links)} articles to process")

        if not rss_feed_links:
            print("No articles found")
            return

        # Process articles with better error handling and resource management
        results, errors = safe_pull_articles(rss_feed_links)

        print(f"Attempted to Pull {len(results)} articles in parallel.")
        print(
            f"Encountered {len(errors)} errors during article pulling. "
            + "Outputting errors to a local file."
        )

        # Output errors to a local file
        if errors:
            with open("errors.txt", "w", encoding="utf-8") as f:
                for error in errors:
                    f.write(str(error) + "\n")
            print(f"Errors logged to errors.txt")

        # Print all results to a log file
        with open("results.txt", "w", encoding="utf-8") as f:
            for result in results:
                f.write(result + "\n")
        print("All articles pulled successfully.")

    except Exception as e:
        print(f"Major error in main execution: {e}")
        import traceback
        traceback.print_exc()
        raise  # Re-raise to ensure the script exits with error code

    print("Scraping completed successfully.")


if __name__ == "__main__":
    main()