feat(scraper): Add cron_scraper.py to fix freezing issues
This commit is contained in:
parent
15862cf32a
commit
b8a24a9fb1
347
scraper/cron_scraper.py
Normal file
347
scraper/cron_scraper.py
Normal file
@ -0,0 +1,347 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Cron-based scraper for downloading articles from RSS feeds.
|
||||
This version replaces the infinite while loop with a single execution
|
||||
that can be scheduled via cron job.
|
||||
"""
|
||||
|
||||
import newspaper
|
||||
import json
|
||||
import feedparser
|
||||
import time
|
||||
import os
|
||||
from selenium import webdriver
|
||||
from selenium.webdriver.firefox.options import Options as FirefoxOptions
|
||||
from selenium.webdriver.common.by import By
|
||||
from selenium.webdriver.support.ui import WebDriverWait
|
||||
from selenium.webdriver.support import expected_conditions as EC
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
import nltk
|
||||
from nltk.downloader import Downloader
|
||||
|
||||
FEED_FILE = os.getenv("FEED_FILE", "./rss_feeds.json")
|
||||
|
||||
# Ensure necessary NLTK resources are downloaded
|
||||
d = Downloader()
|
||||
if not d.is_installed("punkt_tab"):
|
||||
nltk.download("punkt_tab")
|
||||
|
||||
articles = []
|
||||
|
||||
|
||||
def load_rss_feed_sources(feed_file=FEED_FILE):
|
||||
"""
|
||||
Loads the RSS feed sources from a JSON file.
|
||||
"""
|
||||
print("Loading RSS feed sources from rss_feeds.json...")
|
||||
|
||||
try:
|
||||
with open(FEED_FILE, "r", encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
except FileNotFoundError:
|
||||
print(FEED_FILE + " not found, returning empty list.")
|
||||
return []
|
||||
except json.JSONDecodeError:
|
||||
print("Error decoding " + FEED_FILE + " , returning empty list.")
|
||||
return []
|
||||
|
||||
|
||||
def mine_all_articles(rss_feed_sources, limit=None):
|
||||
"""
|
||||
Mines all articles from the given RSS feed sources.
|
||||
Returns a list of (site, title, link) tuples.
|
||||
"""
|
||||
all_links = []
|
||||
sources = rss_feed_sources["rss_feeds"]
|
||||
|
||||
for site, data in sources.items():
|
||||
print(f"Parsing RSS feed: {data['rss_url']}")
|
||||
try:
|
||||
feed = feedparser.parse(data["rss_url"])
|
||||
feed_entries = feed.entries[:limit] if limit else feed.entries
|
||||
|
||||
for entry in feed_entries:
|
||||
if "link" in entry and "title" in entry:
|
||||
all_links.append((site, entry.title, entry.link))
|
||||
except Exception as e:
|
||||
print(f"Error parsing RSS feed: {site} Error: {str(e)}")
|
||||
return all_links
|
||||
|
||||
|
||||
def generate_filename_from_url(url):
|
||||
"""
|
||||
Generates a filename from the given URL by replacing slashes with underscores.
|
||||
"""
|
||||
# Use only the last part of the URL or replace slashes
|
||||
return url.replace("https://", "").replace("http://", "").replace("/", "_")
|
||||
|
||||
|
||||
def generate_safe_filename(name):
|
||||
# Remove/replace characters not allowed in filenames
|
||||
import re
|
||||
|
||||
safe = re.sub(r'[\\/*?:"<>|]', "_", name)
|
||||
return safe
|
||||
|
||||
|
||||
def save_article_to_file(article, filename, source="Unfiltered"):
|
||||
"""
|
||||
Saves the given article text to a file with the specified filename.
|
||||
"""
|
||||
# articles dir should already be there
|
||||
# os.makedirs("articles", exist_ok=True)
|
||||
|
||||
outputDir = "articles/" + source
|
||||
os.makedirs(outputDir, exist_ok=True) if source else None
|
||||
|
||||
# Sanitize filename: use only the last part of the URL or replace slashes
|
||||
safe_filename = generate_filename_from_url(filename)
|
||||
file_path = os.path.join(outputDir, safe_filename)
|
||||
# Save the source as the first line in the file for later retrieval
|
||||
with open(file_path, "w", encoding="utf-8") as f:
|
||||
f.write(f"SOURCE:{source}\n")
|
||||
f.write(article)
|
||||
print(f"Article saved to {file_path}")
|
||||
|
||||
|
||||
def get_article_with_selenium(url):
|
||||
"""
|
||||
Gets article text using Selenium Firefox driver with proper error handling
|
||||
and cleanup.
|
||||
"""
|
||||
driver = None
|
||||
try:
|
||||
# Configure Firefox options
|
||||
options = FirefoxOptions()
|
||||
options.add_argument("--headless")
|
||||
options.set_preference("dom.ipc.processCount", 1) # Reduce process count
|
||||
|
||||
# Initialize driver with timeout
|
||||
driver = webdriver.Firefox(options=options)
|
||||
driver.set_page_load_timeout(30) # 30 seconds timeout
|
||||
|
||||
# Navigate to URL
|
||||
driver.get(url)
|
||||
|
||||
# Wait for page to load (explicit wait instead of sleep)
|
||||
try:
|
||||
WebDriverWait(driver, 15).until(
|
||||
EC.presence_of_element_located((By.TAG_NAME, "body"))
|
||||
)
|
||||
except:
|
||||
pass # Continue even if wait times out
|
||||
|
||||
time.sleep(2) # Brief additional wait
|
||||
|
||||
html = driver.page_source
|
||||
|
||||
# Parse with Newspaper4k
|
||||
article = newspaper.article(url, input_html=html, language="en")
|
||||
article.nlp()
|
||||
return article.text
|
||||
|
||||
except Exception as e:
|
||||
print(f"Selenium failed for {url}: {str(e)}")
|
||||
return ""
|
||||
finally:
|
||||
# Always quit the driver
|
||||
if driver:
|
||||
try:
|
||||
driver.quit()
|
||||
except:
|
||||
pass # Ignore errors in cleanup
|
||||
|
||||
|
||||
def get_article_with_playwright(url):
|
||||
"""
|
||||
Gets article text using Playwright with proper error handling.
|
||||
"""
|
||||
try:
|
||||
from playwright.sync_api import sync_playwright
|
||||
|
||||
with sync_playwright() as p:
|
||||
# Use Chromium instead of Firefox for better compatibility
|
||||
browser = p.chromium.launch(headless=True, timeout=30000)
|
||||
page = browser.new_page()
|
||||
|
||||
# Set user agent to avoid bot detection
|
||||
page.set_extra_http_headers(
|
||||
{
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
|
||||
}
|
||||
)
|
||||
|
||||
page.goto(url, wait_until="load")
|
||||
|
||||
# Wait for content to load
|
||||
time.sleep(3)
|
||||
|
||||
html = page.content()
|
||||
browser.close()
|
||||
|
||||
# Parse with Newspaper4k
|
||||
article = newspaper.article(url, input_html=html, language="en")
|
||||
article.nlp()
|
||||
return article.text
|
||||
|
||||
except Exception as e:
|
||||
print(f"Playwright failed for {url}: {str(e)}")
|
||||
return ""
|
||||
|
||||
|
||||
def pull_article(link, source, title=None, save_to_file=True):
|
||||
"""
|
||||
Pulls an article from a given link with fallback mechanisms.
|
||||
"""
|
||||
filename = title if title else link
|
||||
safe_filename = generate_filename_from_url(filename)
|
||||
|
||||
# Check if already cached
|
||||
if os.path.exists(os.path.join("articles", source, safe_filename)):
|
||||
print(f"Article already cached: {filename}")
|
||||
with open(
|
||||
os.path.join("articles", source, safe_filename), "r", encoding="utf-8"
|
||||
) as f:
|
||||
return f.read()
|
||||
|
||||
text = ""
|
||||
|
||||
try:
|
||||
# Try newspaper4k first
|
||||
article = newspaper.article(link)
|
||||
article.download()
|
||||
article.parse()
|
||||
text = article.text
|
||||
|
||||
if not text or len(text) < 200:
|
||||
raise ValueError(
|
||||
"\tArticle text too short, falling back to Playwright/Selenium."
|
||||
)
|
||||
print(f"\tSuccessfully pulled article with newspaper4k from {link}")
|
||||
|
||||
except Exception as e:
|
||||
print(
|
||||
f"\tnewspaper4k extraction failed for {link}: {e}, falling back to Playwright."
|
||||
)
|
||||
|
||||
try:
|
||||
text = get_article_with_playwright(link)
|
||||
print(f"\t\tSuccessfully pulled article from {link} with Playwright")
|
||||
|
||||
if not text or len(text) < 200:
|
||||
print(f"\t\tPlaywright article too short, falling back to Selenium.")
|
||||
# Fallback to Selenium with better error handling
|
||||
text = get_article_with_selenium(link)
|
||||
print(f"\t\t\tSuccessfully pulled article from {link} with Selenium")
|
||||
except Exception as e:
|
||||
print(f"\t\tPlaywright failed for {link}: {e}")
|
||||
|
||||
# Fallback to Selenium
|
||||
try:
|
||||
text = get_article_with_selenium(link)
|
||||
print(f"\t\tSuccessfully pulled article from {link} with Selenium")
|
||||
except Exception as e:
|
||||
print(f"\t\tSelenium failed for {link}: {e}")
|
||||
return ""
|
||||
|
||||
if save_to_file:
|
||||
save_article_to_file(text, filename, source)
|
||||
|
||||
return text
|
||||
|
||||
|
||||
def safe_pull_articles(article_list):
|
||||
"""
|
||||
Safely pull articles with improved error handling and reduced parallelism.
|
||||
"""
|
||||
results = []
|
||||
errors = []
|
||||
|
||||
# Process in smaller batches to reduce resource strain
|
||||
batch_size = 5
|
||||
|
||||
for i in range(0, len(article_list), batch_size):
|
||||
batch = article_list[i : i + batch_size]
|
||||
print(f"Processing batch {i // batch_size + 1} with {len(batch)} articles")
|
||||
|
||||
# Use ThreadPoolExecutor instead of ProcessPoolExecutor to avoid
|
||||
# process termination issues with browser automation
|
||||
with ThreadPoolExecutor(max_workers=3) as executor: # Reduced workers
|
||||
futures = [
|
||||
executor.submit(pull_article, link, source, title)
|
||||
for source, title, link in batch
|
||||
]
|
||||
|
||||
for future in as_completed(futures):
|
||||
try:
|
||||
result = future.result(timeout=120) # 2 minute timeout
|
||||
results.append(result)
|
||||
except Exception as e:
|
||||
print(f"Error in pull_article: {e}")
|
||||
errors.append(e)
|
||||
|
||||
# Add a small delay between batches to reduce system load
|
||||
time.sleep(5)
|
||||
|
||||
return results, errors
|
||||
|
||||
|
||||
def main():
|
||||
"""
|
||||
Main scraping function for cron execution.
|
||||
This replaces the infinite while loop with a single execution.
|
||||
"""
|
||||
print("=========================================")
|
||||
print("Starting new scraping iteration...")
|
||||
|
||||
try:
|
||||
# Pull the RSS feed sources from the JSON file
|
||||
rss_feed_sources = load_rss_feed_sources()
|
||||
|
||||
# Mine all articles from the RSS feed sources
|
||||
rss_feed_links = mine_all_articles(rss_feed_sources)
|
||||
|
||||
# Randomize the order of the links to help with load balancing
|
||||
import random
|
||||
|
||||
random.shuffle(rss_feed_links)
|
||||
|
||||
print(f"Found {len(rss_feed_links)} articles to process")
|
||||
|
||||
if not rss_feed_links:
|
||||
print("No articles found")
|
||||
return
|
||||
|
||||
# Process articles with better error handling and resource management
|
||||
results, errors = safe_pull_articles(rss_feed_links)
|
||||
|
||||
print(f"Attempted to Pull {len(results)} articles in parallel.")
|
||||
print(
|
||||
f"Encountered {len(errors)} errors during article pulling. "
|
||||
+ "Outputting errors to a local file."
|
||||
)
|
||||
|
||||
# Output errors to a local file
|
||||
if errors:
|
||||
with open("errors.txt", "w", encoding="utf-8") as f:
|
||||
for error in errors:
|
||||
f.write(str(error) + "\n")
|
||||
print(f"Errors logged to errors.txt")
|
||||
|
||||
# Print all results to a log file
|
||||
with open("results.txt", "w", encoding="utf-8") as f:
|
||||
for result in results:
|
||||
f.write(result + "\n")
|
||||
print("All articles pulled successfully.")
|
||||
|
||||
except Exception as e:
|
||||
print(f"Major error in main execution: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
raise # Re-raise to ensure the script exits with error code
|
||||
|
||||
print("Scraping completed successfully.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
x
Reference in New Issue
Block a user