342 lines
11 KiB
Python
342 lines
11 KiB
Python
import newspaper
|
|
import json
|
|
import feedparser
|
|
import time
|
|
import os
|
|
from selenium import webdriver
|
|
from selenium.webdriver.firefox.options import Options as FirefoxOptions
|
|
from selenium.webdriver.common.by import By
|
|
from selenium.webdriver.support.ui import WebDriverWait
|
|
from selenium.webdriver.support import expected_conditions as EC
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
import nltk
|
|
from nltk.downloader import Downloader
|
|
|
|
FEED_FILE = os.getenv("FEED_FILE", "./rss_short_feed.json")
|
|
|
|
# Ensure necessary NLTK resources are downloaded
|
|
d = Downloader()
|
|
if not d.is_installed("punkt_tab"):
|
|
nltk.download("punkt_tab")
|
|
|
|
articles = []
|
|
|
|
|
|
def load_rss_feed_sources(feed_file=FEED_FILE):
|
|
"""
|
|
Loads the RSS feed sources from a JSON file.
|
|
"""
|
|
print("Loading RSS feed sources from rss_feeds.json...")
|
|
|
|
try:
|
|
with open(FEED_FILE, "r", encoding="utf-8") as f:
|
|
return json.load(f)
|
|
except FileNotFoundError:
|
|
print(FEED_FILE + " not found, returning empty list.")
|
|
return []
|
|
except json.JSONDecodeError:
|
|
print("Error decoding " + FEED_FILE + " , returning empty list.")
|
|
return []
|
|
|
|
|
|
def mine_all_articles(rss_feed_sources, limit=None):
|
|
"""
|
|
Mines all articles from the given RSS feed sources.
|
|
Returns a list of (site, title, link) tuples.
|
|
"""
|
|
all_links = []
|
|
sources = rss_feed_sources["rss_feeds"]
|
|
|
|
for site, data in sources.items():
|
|
print(f"Parsing RSS feed: {data['rss_url']}")
|
|
try:
|
|
feed = feedparser.parse(data["rss_url"])
|
|
feed_entries = feed.entries[:limit] if limit else feed.entries
|
|
|
|
for entry in feed_entries:
|
|
if "link" in entry and "title" in entry:
|
|
all_links.append((site, entry.title, entry.link))
|
|
except Exception as e:
|
|
print(f"Error parsing RSS feed: {site} Error: {str(e)}")
|
|
return all_links
|
|
|
|
|
|
def generate_filename_from_url(url):
|
|
"""
|
|
Generates a filename from the given URL by replacing slashes with underscores.
|
|
"""
|
|
# Use only the last part of the URL or replace slashes
|
|
return url.replace("https://", "").replace("http://", "").replace("/", "_")
|
|
|
|
|
|
def generate_safe_filename(name):
|
|
# Remove/replace characters not allowed in filenames
|
|
import re
|
|
|
|
safe = re.sub(r'[\\/*?:"<>|]', "_", name)
|
|
return safe
|
|
|
|
|
|
def save_article_to_file(article, filename, source="Unfiltered"):
|
|
"""
|
|
Saves the given article text to a file with the specified filename.
|
|
"""
|
|
# articles dir should already be there
|
|
# os.makedirs("articles", exist_ok=True)
|
|
|
|
outputDir = "articles/" + source
|
|
os.makedirs(outputDir, exist_ok=True) if source else None
|
|
|
|
# Sanitize filename: use only the last part of the URL or replace slashes
|
|
safe_filename = generate_filename_from_url(filename)
|
|
file_path = os.path.join(outputDir, safe_filename)
|
|
# Save the source as the first line in the file for later retrieval
|
|
with open(file_path, "w", encoding="utf-8") as f:
|
|
f.write(f"SOURCE:{source}\n")
|
|
f.write(article)
|
|
print(f"Article saved to {file_path}")
|
|
|
|
|
|
def get_article_with_selenium(url):
|
|
"""
|
|
Gets article text using Selenium Firefox driver with proper error handling
|
|
and cleanup.
|
|
"""
|
|
driver = None
|
|
try:
|
|
# Configure Firefox options
|
|
options = FirefoxOptions()
|
|
options.add_argument("--headless")
|
|
options.set_preference("dom.ipc.processCount", 1) # Reduce process count
|
|
|
|
# Initialize driver
|
|
driver = webdriver.Firefox(options=options)
|
|
driver.set_page_load_timeout(30) # 30 seconds timeout
|
|
|
|
# Navigate to URL
|
|
driver.get(url)
|
|
|
|
# Wait for page to load (explicit wait instead of sleep)
|
|
try:
|
|
WebDriverWait(driver, 15).until(
|
|
EC.presence_of_element_located((By.TAG_NAME, "body"))
|
|
)
|
|
except:
|
|
pass # Continue even if wait times out
|
|
|
|
time.sleep(2) # Brief additional wait
|
|
|
|
html = driver.page_source
|
|
|
|
# Parse with Newspaper4k
|
|
article = newspaper.article(url, input_html=html, language="en")
|
|
article.nlp()
|
|
return article.text
|
|
|
|
except Exception as e:
|
|
print(f"Selenium failed for {url}: {str(e)}")
|
|
return ""
|
|
finally:
|
|
# Always quit the driver
|
|
if driver:
|
|
try:
|
|
driver.quit()
|
|
except:
|
|
pass # Ignore errors in cleanup
|
|
|
|
|
|
def get_article_with_playwright(url):
|
|
"""
|
|
Gets article text using Playwright with proper error handling.
|
|
"""
|
|
try:
|
|
from playwright.sync_api import sync_playwright
|
|
|
|
with sync_playwright() as p:
|
|
# Use Chromium instead of Firefox for better compatibility
|
|
browser = p.chromium.launch(headless=True, timeout=30000)
|
|
page = browser.new_page()
|
|
|
|
# Set user agent to avoid bot detection
|
|
page.set_extra_http_headers(
|
|
{
|
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
|
|
}
|
|
)
|
|
|
|
page.goto(url, wait_until="load")
|
|
|
|
# Wait for content to load
|
|
time.sleep(3)
|
|
|
|
html = page.content()
|
|
browser.close()
|
|
|
|
# Parse with Newspaper4k
|
|
article = newspaper.article(url, input_html=html, language="en")
|
|
article.nlp()
|
|
return article.text
|
|
|
|
except Exception as e:
|
|
print(f"Playwright failed for {url}: {str(e)}")
|
|
return ""
|
|
|
|
|
|
def pull_article(link, source, title=None, save_to_file=True):
|
|
"""
|
|
Pulls an article from a given link with fallback mechanisms.
|
|
"""
|
|
filename = title if title else link
|
|
safe_filename = generate_filename_from_url(filename)
|
|
|
|
# Check if already cached
|
|
if os.path.exists(os.path.join("articles", source, safe_filename)):
|
|
print(f"Article already cached: {filename}")
|
|
with open(
|
|
os.path.join("articles", source, safe_filename), "r", encoding="utf-8"
|
|
) as f:
|
|
return f.read()
|
|
|
|
text = ""
|
|
|
|
try:
|
|
# Try newspaper4k first
|
|
article = newspaper.article(link)
|
|
article.download()
|
|
article.parse()
|
|
text = article.text
|
|
|
|
if not text or len(text) < 200:
|
|
raise ValueError(
|
|
"\tArticle text too short, falling back to Playwright/Selenium."
|
|
)
|
|
print(f"\tSuccessfully pulled article with newspaper4k from {link}")
|
|
|
|
except Exception as e:
|
|
print(
|
|
f"\tnewspaper4k extraction failed for {link}: {e}, falling back to Playwright."
|
|
)
|
|
|
|
try:
|
|
text = get_article_with_playwright(link)
|
|
print(f"\t\tSuccessfully pulled article from {link} with Playwright")
|
|
|
|
if not text or len(text) < 200:
|
|
print(f"\t\tPlaywright article too short, falling back to Selenium.")
|
|
# Fallback to Selenium with better error handling
|
|
text = get_article_with_selenium(link)
|
|
print(f"\t\t\tSuccessfully pulled article from {link} with Selenium")
|
|
except Exception as e:
|
|
print(f"\t\tPlaywright failed for {link}: {e}")
|
|
|
|
# Fallback to Selenium
|
|
try:
|
|
text = get_article_with_selenium(link)
|
|
print(f"\t\tSuccessfully pulled article from {link} with Selenium")
|
|
except Exception as e:
|
|
print(f"\t\tSelenium failed for {link}: {e}")
|
|
return ""
|
|
|
|
if save_to_file:
|
|
save_article_to_file(text, filename, source)
|
|
|
|
return text
|
|
|
|
|
|
def safe_pull_articles(article_list):
|
|
"""
|
|
Safely pull articles with improved error handling and reduced parallelism.
|
|
"""
|
|
results = []
|
|
errors = []
|
|
|
|
# Process in smaller batches to reduce resource strain
|
|
batch_size = 5
|
|
|
|
for i in range(0, len(article_list), batch_size):
|
|
batch = article_list[i : i + batch_size]
|
|
print(f"Processing batch {i // batch_size + 1} with {len(batch)} articles")
|
|
|
|
# Use ThreadPoolExecutor instead of ProcessPoolExecutor to avoid
|
|
# process termination issues with browser automation
|
|
with ThreadPoolExecutor(max_workers=3) as executor: # Reduced workers
|
|
futures = [
|
|
executor.submit(pull_article, link, source, title)
|
|
for source, title, link in batch
|
|
]
|
|
|
|
for future in as_completed(futures):
|
|
try:
|
|
result = future.result(timeout=120) # 2 minute timeout
|
|
results.append(result)
|
|
except Exception as e:
|
|
print(f"Error in pull_article: {e}")
|
|
errors.append(e)
|
|
|
|
# Add a small delay between batches to reduce system load
|
|
time.sleep(5)
|
|
|
|
return results, errors
|
|
|
|
|
|
def main():
|
|
"""
|
|
Main scraping loop.
|
|
"""
|
|
while True:
|
|
print("=========================================")
|
|
print("Starting new scraping iteration...")
|
|
|
|
try:
|
|
# Pull the RSS feed sources from the JSON file
|
|
rss_feed_sources = load_rss_feed_sources()
|
|
|
|
# Mine all articles from the RSS feed sources
|
|
rss_feed_links = mine_all_articles(rss_feed_sources)
|
|
|
|
# Randomize the order of the links to help with load balancing
|
|
import random
|
|
|
|
random.shuffle(rss_feed_links)
|
|
|
|
print(f"Found {len(rss_feed_links)} articles to process")
|
|
|
|
if not rss_feed_links:
|
|
print("No articles found, sleeping for 15 minutes")
|
|
time.sleep(15 * 60)
|
|
continue
|
|
|
|
# Process articles with better error handling and resource management
|
|
results, errors = safe_pull_articles(rss_feed_links)
|
|
|
|
print(f"Attempted to Pull {len(results)} articles in parallel.")
|
|
print(
|
|
f"Encountered {len(errors)} errors during article pulling. "
|
|
+ "Outputting errors to a local file."
|
|
)
|
|
|
|
# Output errors to a local file
|
|
if errors:
|
|
with open("errors.txt", "w", encoding="utf-8") as f:
|
|
for error in errors:
|
|
f.write(str(error) + "\n")
|
|
print(f"Errors logged to errors.txt")
|
|
|
|
# Print all results to a log file
|
|
with open("results.txt", "w", encoding="utf-8") as f:
|
|
for result in results:
|
|
f.write(result + "\n")
|
|
print("All articles pulled successfully.")
|
|
|
|
except Exception as e:
|
|
print(f"Major error in main loop: {e}")
|
|
# Continue to next iteration even if there's a major error
|
|
|
|
# Sleep for a while before the next iteration
|
|
print("Sleeping for 15 minutes before the next iteration...")
|
|
time.sleep(15 * 60) # Sleep for 15 minutes
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|