220 lines
7.9 KiB
Python
220 lines
7.9 KiB
Python
import newspaper
|
|
import json
|
|
import feedparser
|
|
import time
|
|
import os
|
|
from selenium import webdriver
|
|
from selenium.webdriver.firefox.options import Options as FirefoxOptions
|
|
from newspaper.google_news import GoogleNewsSource
|
|
from concurrent.futures import ProcessPoolExecutor
|
|
import nltk
|
|
|
|
# Ensure necessary NLTK resources are downloaded / Needed for selenium + newspaper4k article parsing
|
|
nltk.download('punkt_tab')
|
|
|
|
articles = []
|
|
|
|
def load_rss_feed_sources():
|
|
"""
|
|
Loads the RSS feed sources from a JSON file.
|
|
"""
|
|
print("Loading RSS feed sources from rss_feeds.json...")
|
|
|
|
try:
|
|
with open("rss_short_feed.json", "r", encoding="utf-8") as f:
|
|
return json.load(f)
|
|
except FileNotFoundError:
|
|
print("rss_feeds.json not found, returning empty list.")
|
|
return []
|
|
except json.JSONDecodeError:
|
|
print("Error decoding rss_feeds.json, returning empty list.")
|
|
return []
|
|
|
|
def mine_all_articles(rss_feed_sources, limit=None):
|
|
"""
|
|
Mines all articles from the given RSS feed sources.
|
|
Returns a list of (site, title, link) tuples.
|
|
"""
|
|
all_links = []
|
|
sources = rss_feed_sources['rss_feeds']
|
|
|
|
for site, data in sources.items():
|
|
print(f"Parsing RSS feed: {data['rss_url']}")
|
|
try:
|
|
feed = feedparser.parse(data["rss_url"])
|
|
feed_entries = feed.entries[:limit] if limit else feed.entries
|
|
|
|
for entry in feed_entries:
|
|
if "link" in entry and "title" in entry:
|
|
all_links.append((site, entry.title, entry.link))
|
|
except Exception as e:
|
|
print(f"Error parsing RSS feed: {site} Error: {str(e)}")
|
|
return all_links
|
|
|
|
def generate_filename_from_url(url):
|
|
"""
|
|
Generates a filename from the given URL by replacing slashes with underscores.
|
|
"""
|
|
# Use only the last part of the URL or replace slashes
|
|
return url.replace("https://", "").replace("http://", "").replace("/", "_")
|
|
|
|
def generate_safe_filename(name):
|
|
# Remove/replace characters not allowed in filenames
|
|
import re
|
|
safe = re.sub(r'[\\/*?:"<>|]', "_", name)
|
|
return safe
|
|
|
|
def save_article_to_file(article, filename, source="Unfiltered"):
|
|
"""
|
|
Saves the given article text to a file with the specified filename.
|
|
"""
|
|
os.makedirs("articles", exist_ok=True)
|
|
|
|
outputDir = "articles/"+source if source else "articles"
|
|
os.makedirs(outputDir, exist_ok=True) if source else None
|
|
|
|
# Sanitize filename: use only the last part of the URL or replace slashes
|
|
safe_filename = generate_filename_from_url(filename)
|
|
file_path = os.path.join(outputDir, safe_filename)
|
|
# Save the source as the first line in the file for later retrieval
|
|
with open(file_path, "w", encoding="utf-8") as f:
|
|
f.write(f"SOURCE:{source}\n")
|
|
f.write(article)
|
|
print(f"Article saved to {file_path}")
|
|
|
|
def get_article_with_selenium(url):
|
|
options = FirefoxOptions()
|
|
options.add_argument("--headless")
|
|
driver = webdriver.Firefox(options=options)
|
|
try:
|
|
driver.get(url)
|
|
time.sleep(5) # Wait for JS to load
|
|
html = driver.page_source
|
|
|
|
# Parse with Newspaper4k
|
|
article = newspaper.article(url, input_html=html, language='en')
|
|
article.nlp()
|
|
return article.text
|
|
finally:
|
|
driver.quit()
|
|
|
|
def get_article_with_playwright(url):
|
|
import asyncio
|
|
from playwright.sync_api import sync_playwright
|
|
from newspaper import Article
|
|
|
|
with sync_playwright() as p:
|
|
browser = p.chromium.launch(headless=True)
|
|
page = browser.new_page()
|
|
page.goto(url)
|
|
|
|
# Optional: wait for specific content to load
|
|
time.sleep(5) # Adjust as needed for the page to load completely
|
|
|
|
html = page.content()
|
|
'''
|
|
# Check for iframes and use the first one's content if present
|
|
frames = page.frames
|
|
main_frame = page.main_frame
|
|
for frame in frames:
|
|
if frame != main_frame:
|
|
try:
|
|
frame.wait_for_load_state("domcontentloaded", timeout=5000)
|
|
html = frame.content()
|
|
print("Extracted content from iframe.")
|
|
break
|
|
except Exception as e:
|
|
print(f"Could not extract iframe content: {e}")
|
|
'''
|
|
|
|
browser.close()
|
|
|
|
# Parse with Newspaper4k
|
|
article = newspaper.article(url, input_html=html, language='en')
|
|
article.nlp()
|
|
return article.text
|
|
|
|
|
|
def pull_articles(link, source, title=None, save_to_file=True):
|
|
filename = title if title else link
|
|
safe_filename = generate_filename_from_url(filename)
|
|
if os.path.exists(os.path.join("articles", safe_filename)):
|
|
print(f"Article already cached: {filename}")
|
|
with open(os.path.join("articles", safe_filename), "r", encoding="utf-8") as f:
|
|
return f.read()
|
|
|
|
text = ""
|
|
|
|
time.sleep(5) # Since we spawn lots of processes, we need to sleep at the start
|
|
|
|
try:
|
|
# --- Google News Source handling temporarily disabled ---
|
|
# if "news.google.com" in link:
|
|
# gn = GoogleNewsSource(link)
|
|
# gn.build()
|
|
# # Pull the first article from the Google News cluster
|
|
# if gn.articles:
|
|
# article = gn.articles[0]
|
|
# article.download()
|
|
# article.parse()
|
|
# text = article.text
|
|
# if not text or len(text) < 200:
|
|
# raise ValueError("\tGoogle News article text too short, falling back to Selenium.")
|
|
# print(f"\tSuccessfully pulled Google News article from {link}")
|
|
# else:
|
|
article = newspaper.article(link)
|
|
article.download()
|
|
article.parse()
|
|
text = article.text
|
|
if not text or len(text) < 200:
|
|
raise ValueError("\tArticle text too short, falling back to Playwright/Selenium.")
|
|
print(f"\tSuccessfully pulled article with newspaper4k from {link}")
|
|
|
|
except Exception as e:
|
|
print(f"\tnewspaper4k extraction failed for {link}: {e}, falling back to Playwright.")
|
|
try:
|
|
text = get_article_with_playwright(link)
|
|
print(f"\t\tSuccessfully pulled article from {link} with Playwright")
|
|
if not text or len(text) < 200:
|
|
print(f"\t\tPlaywright article too short, falling back to Selenium.")
|
|
try:
|
|
text = get_article_with_selenium(link)
|
|
print(f"\t\t\tSuccessfully pulled article from {link} with Selenium")
|
|
except Exception as e:
|
|
print(f"\t\t\tSelenium failed for {link}: {e}")
|
|
return ""
|
|
except Exception as e:
|
|
print(f"\t\tPlaywright failed for {link}: {e}")
|
|
try:
|
|
text = get_article_with_selenium(link)
|
|
print(f"\t\tSuccessfully pulled article from {link} with Selenium")
|
|
except Exception as e:
|
|
print(f"\t\tSelenium failed for {link}: {e}")
|
|
return ""
|
|
|
|
if save_to_file:
|
|
save_article_to_file(text, filename, source)
|
|
return text
|
|
|
|
# Pull the RSS feed sources from the JSON file
|
|
rss_feed_sources = load_rss_feed_sources()
|
|
|
|
# Mine all articles from the RSS feed sources
|
|
rss_feed_links = mine_all_articles(rss_feed_sources)
|
|
|
|
# Randomize the order of the links to help with load balancing
|
|
import random
|
|
random.shuffle(rss_feed_links)
|
|
|
|
# Pull the latest article from a variety of news sources
|
|
#[pull_articles(link, source, title) for source, title, link in rss_feed_links]
|
|
|
|
link_list = [link for _, title, link in rss_feed_links]
|
|
source_list = [source for source, _, _ in rss_feed_links]
|
|
title_list = [title for _, title, link in rss_feed_links]
|
|
|
|
with ProcessPoolExecutor() as executor:
|
|
results = list(executor.map(pull_articles, link_list, source_list, title_list))
|
|
print(f"Pulled {len(results)} articles in parallel.")
|
|
print("All articles pulled successfully.")
|