diff --git a/.gitignore b/.gitignore index 9d16dee..28a18d0 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,6 @@ scraper/articles/* # Anything from AI Process ai_processor/output/* + +# Anything from running processes in the background +nohup.out diff --git a/MCPServer/docker-compose.yml b/MCPServer/docker-compose.yml deleted file mode 100644 index 8f1630c..0000000 --- a/MCPServer/docker-compose.yml +++ /dev/null @@ -1,20 +0,0 @@ -services: - flask-app: - build: . - container_name: stockdocs-mcp - networks: - - ainetwork - ports: - - "5005:5005" - environment: - - FLASK_ENV=development - deploy: - replicas: 1 # You can scale the number of instances (replicas) as needed - resources: - limits: - cpus: '1' - memory: 1024M -networks: - ainetwork: - external: true - name: ainetwork diff --git a/ai_processor/ai_processor.py b/ai_processor/ai_processor.py index 8e0e173..b9ed7bb 100644 --- a/ai_processor/ai_processor.py +++ b/ai_processor/ai_processor.py @@ -1,8 +1,10 @@ import os import requests import json +import datetime +import time -LOCAL_AI_SERVICE_URL = "http://192.168.8.124:11434" +LOCAL_AI_SERVICE_URL = os.getenv("AI_SERVICE_URL") class ArticleFile: def __init__(self, filename, content, source="Unfiltered"): @@ -18,9 +20,21 @@ def load_articles_from_folder(folder_path): Loads all articles from text files in the specified folder and returns them as a list of ArticleFile objects. """ articles = [] + # Skip non-directory files in the root folder for newspaper in os.listdir(folder_path): - for filename in os.listdir(os.path.join(folder_path, newspaper)): - file_path = os.path.join(folder_path, newspaper, filename) + newspaper_path = os.path.join(folder_path, newspaper) + if not os.path.isdir(newspaper_path): + continue + + for filename in os.listdir(newspaper_path): + file_path = os.path.join(newspaper_path, filename) + output_path = os.path.join("output", f"{filename}.json") + + # Skip if already processed + if os.path.isfile(output_path): + print(f"Skipping already processed article: {filename}") + continue + if os.path.isfile(file_path): with open(file_path, 'r', encoding='utf-8') as f: first_line = f.readline() @@ -32,7 +46,7 @@ def load_articles_from_folder(folder_path): content = first_line + f.read() articles.append(ArticleFile(filename, content, source)) - print(f"Loaded {len(articles)} articles from folder {folder_path}.") + print(f"Loaded {len(articles)} new articles from folder {folder_path}.") return articles def call_local_ai_service(article): @@ -54,8 +68,11 @@ def call_local_ai_service(article): payload = { "model": "llama3.2:latest", "prompt": f"{prompt}\n\nArticle:\n{article}", - "stream": False - # TODO: In the future, we should force the response to be json and adjust the temperature + "stream": False, + "format": "json", + "options": { + "temperature": 0.1 + } } print(f"Sending request to local AI service ...") @@ -85,7 +102,7 @@ def process_article(article_file): processed_article = None # Check if we already processed this article - # TODO : protentially expensive, consider using a database or cache + # TODO : potentially expensive, consider using a database or cache if os.path.exists(f"output/{article_file.filename}.json"): print(f"Article {article_file.filename} already processed, skipping.") @@ -96,6 +113,10 @@ def process_article(article_file): for attempt in range(3): try: + if not article_file.content.strip() or len(article_file.content) < 30: + print(f"Skipping empty article or article with insufficient content: {article_file.filename}") + return None + # Pass the article content to the AI service processed_article = call_local_ai_service(article_file.content) print(f"Processed article: {processed_article['summary'][:100]}...") @@ -114,19 +135,35 @@ def process_article(article_file): return processed_article -# Retrieve the current archive of pulled articles -articles_folder = os.path.join(os.path.dirname(__file__), "../scraper/articles") -print("Loading articles from folder " + articles_folder + " ...") -articles = load_articles_from_folder(articles_folder) +while True: + print("=========================================") -# Process the articles retrieved -results = [] -for article in articles: - try: - result = process_article(article) + # Retrieve the current archive of pulled articles + articles_folder = os.path.join("/app/articles") + if not os.path.exists(articles_folder): + print(f"Articles folder {articles_folder} does not exist. Please check the path.") + exit(1) + print("Loading articles from folder " + articles_folder + " ...") + articles = load_articles_from_folder(articles_folder) - # Save the processed result to a JSON file - with open("output/" + result['filename'] + ".json", "w", encoding="utf-8") as f: - json.dump(result, f, ensure_ascii=False, indent=2) - except Exception as e: - print(f"Error processing or saving article {article.filename}: {e}") + # Process the articles retrieved + results = [] + failedProcessCount = 0 + for article in articles: + try: + result = process_article(article) + + if result is None: + failedProcessCount += 1 + #print(f"Skipping article {article.filename} due to processing error.") + continue + + # Save the processed result to a JSON file + with open("output/" + result['filename'] + ".json", "w", encoding="utf-8") as f: + json.dump(result, f, ensure_ascii=False, indent=2) + except Exception as e: + print(f"Error processing or saving article {article.filename}: {e}") + + print(f"Processed {len(articles) - failedProcessCount} articles successfully, {failedProcessCount} failed to process.") + + time.delay(60 * 5) # Wait for 5 minutes before the next iteration \ No newline at end of file diff --git a/ai_processor/dockerfile b/ai_processor/dockerfile new file mode 100644 index 0000000..0d3d3bc --- /dev/null +++ b/ai_processor/dockerfile @@ -0,0 +1,16 @@ +FROM python:3.11-slim + +WORKDIR /app + +# Copy requirements first for better caching +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +# Copy the rest of the application +COPY . . + +# Create output directory +RUN mkdir -p output + +# Command to run the processor +CMD ["python", "ai_processor.py"] \ No newline at end of file diff --git a/ai_processor/pyvenv.cfg b/ai_processor/pyvenv.cfg index 1536dac..4760c1f 100644 --- a/ai_processor/pyvenv.cfg +++ b/ai_processor/pyvenv.cfg @@ -1,5 +1,3 @@ -home = /usr/bin +home = /Library/Developer/CommandLineTools/usr/bin include-system-site-packages = false -version = 3.12.3 -executable = /usr/bin/python3.12 -command = /usr/bin/python3 -m venv /home/user/embedding +version = 3.9.6 diff --git a/ai_processor/requirements.txt b/ai_processor/requirements.txt new file mode 100644 index 0000000..08623cd --- /dev/null +++ b/ai_processor/requirements.txt @@ -0,0 +1,5 @@ +certifi==2025.7.14 +charset-normalizer==3.4.2 +idna==3.10 +requests==2.32.4 +urllib3==2.5.0 diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..155eb2b --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,49 @@ +services: + scraper: + build: ./scraper + platform: linux/amd64 + container_name: stockdocs-scraper + networks: + - ainetwork + volumes: + - ./scraper/articles:/app/articles + environment: + - FEED_FILE=rss_feeds.json + flask-app: + build: ./MCPServer + platform: linux/amd64 + container_name: stockdocs-mcp + networks: + - ainetwork + ports: + - "5005:5005" + environment: + - FLASK_ENV=development + ai_processor: + build: ./ai_processor + platform: linux/amd64 + container_name: stockdocs-ai-processor + networks: + - ainetwork + volumes: + - ./scraper/articles:/app/articles + - ./ai_processor/output:/app/output + environment: + - AI_SERVICE_URL=http://192.168.8.124:11434 # Local AI service IP + embedder: + build: ./embedding + platform: linux/amd64 + container_name: stockdocs-embedder + networks: + - ainetwork + volumes: + - ./ai_processor/output:/app/output + environment: + - CHROMADB_HOST=chromadb + - CHROMADB_PORT=8000 + + +networks: + ainetwork: + external: true + name: ainetwork diff --git a/embedding/dockerfile b/embedding/dockerfile new file mode 100644 index 0000000..6831f64 --- /dev/null +++ b/embedding/dockerfile @@ -0,0 +1,22 @@ +FROM python:3.11-slim + +WORKDIR /app + +# Install system dependencies +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + build-essential \ + && rm -rf /var/lib/apt/lists/* + +# Copy requirements first for better caching +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +# Create directories +RUN mkdir -p input output + +# Copy the rest of the application +COPY . . + +# Command to run the embedder +CMD ["python", "embedder.py"] \ No newline at end of file diff --git a/embedding/embedder.py b/embedding/embedder.py index 63480c2..bd80418 100644 --- a/embedding/embedder.py +++ b/embedding/embedder.py @@ -2,9 +2,13 @@ import os import json import chromadb import uuid - +import time +import datetime from sentence_transformers import SentenceTransformer +CHROMADB_HOST = os.getenv("CHROMADB_HOST") +CHROMADB_PORT = os.getenv("CHROMADB_PORT") + model = SentenceTransformer( "Snowflake/snowflake-arctic-embed-m-long", device="cpu", # <-- the only line that changes @@ -41,50 +45,91 @@ def embed_text(text, specific_context, max_tokens=2048, overlap=256): return (chunks, model.encode(chunks, normalize_embeddings=True)) -client = chromadb.HttpClient(host="localhost", port=8000) +client = chromadb.HttpClient(host=CHROMADB_HOST, port=CHROMADB_PORT) # client.delete_collection("news") # Replace "news" with your collection name collection = client.get_or_create_collection("news") +# Load cache of previously embedded articles +cache_file = "embedded_articles_cache.json" + # Load all the processed articles from the output folder -output_folder = os.path.join(os.path.dirname(__file__), "../ai_processor/output") -output_articles = [] -# When embedding and upserting, use the source from the processed result -for filename in os.listdir(output_folder): - if filename.endswith(".json"): - with open(os.path.join(output_folder, filename), 'r', encoding='utf-8') as f: - prev_proc = json.load(f) - output_articles.append(prev_proc) +output_folder = "/app/output" - def makeTickerMovement(ticker, movement, percent_change): - """ - Helper function to create a ticker movement entry. - """ - ticker = "" if ticker is None else ticker - movement = "" if movement is None else movement - percent_change = "" if percent_change is None else percent_change - return [ticker, movement, percent_change] +while True: + print(f"Starting embedding run at {datetime.datetime.now()}") + + embedded_cache = {} + if os.path.exists(cache_file): + with open(cache_file, 'r', encoding='utf-8') as f: + embedded_cache = json.load(f) - # Embed the article content - try: - specific_context = prev_proc['summary'] + ",".join([' '.join(makeTickerMovement(*x)) for x in prev_proc['tickersAndMovements']]) - chunks, embedded_content = embed_text(prev_proc['original_content'], prev_proc['summary']) - - print(f"Embedded content for {filename}: \n {embedded_content[:10]}...") # Print first 10 values for preview - - collection.upsert( - ids = [ str(uuid.uuid4()) for d in chunks ], # Generate unique IDs for each embedded content - documents = chunks, # optional but nice for debugging - embeddings = embedded_content.tolist(), # Chroma expects List[List[float]] - metadatas =[{ - "source": prev_proc.get('source', 'Unknown'), # Use the source from the processed result - "published": prev_proc.get('published', 'Unknown'), - "filename": prev_proc.get('filename', 'Unknown') # Add filename for traceability - } for d in chunks], # Metadata for each embedded content - ) - - except Exception as e: - print(f"Error embedding content for {filename}: {e}") + output_articles = [] + processed_articles = [] + # When embedding and upserting, use the source from the processed result + for filename in os.listdir(output_folder): + if filename.endswith(".json"): + # Check if already embedded + if filename in embedded_cache: + print(f"Article {filename} already embedded, skipping.") continue + + with open(os.path.join(output_folder, filename), 'r', encoding='utf-8') as f: + prev_proc = json.load(f) + output_articles.append(prev_proc) + + def makeTickerMovement(ticker, movement, percent_change): + """ + Helper function to create a ticker movement entry. + """ + ticker = "" if ticker is None else ticker + movement = "" if movement is None else movement + percent_change = "" if percent_change is None else percent_change + return [ticker, movement, percent_change] + + # Embed the article content + try: + specific_context = prev_proc['summary'] + ",".join([' '.join(makeTickerMovement(*x)) for x in prev_proc['tickersAndMovements']]) + chunks, embedded_content = embed_text(prev_proc['original_content'], prev_proc['summary']) + + print(f"Embedded content for {filename}: \n {embedded_content[:10]}...") # Print first 10 values for preview + + collection.upsert( + ids = [ str(uuid.uuid4()) for d in chunks ], # Generate unique IDs for each embedded content + documents = chunks, # optional but nice for debugging + embeddings = embedded_content.tolist(), # Chroma expects List[List[float]] + metadatas =[{ + "source": prev_proc.get('source', 'Unknown'), # Use the source from the processed result + "published": prev_proc.get('published', 'Unknown'), + "filename": prev_proc.get('filename', 'Unknown') # Add filename for traceability + } for d in chunks], # Metadata for each embedded content + ) + + # Add to processed list for cache update + processed_articles.append(filename) + print(f"Successfully embedded {filename}") + + except Exception as e: + print(f"Error embedding content for {filename}: {e}") + continue + + # Update cache with newly processed articles + for filename in processed_articles: + embedded_cache[filename] = { + "embedded_date": str(datetime.datetime.now()), + "status": "completed" + } + + # Save updated cache + try: + with open(cache_file, 'w', encoding='utf-8') as f: + json.dump(embedded_cache, f, indent=2) + print(f"Updated cache with {len(processed_articles)} newly embedded articles.") + except Exception as e: + print(f"Error saving cache file: {e}") + + print(f"Embedding process completed. Total articles in cache: {len(embedded_cache)}") + print("Waiting 5 minutes before next run...") + time.sleep(300) # Wait 5 minutes (300 seconds) diff --git a/embedding/requirements.txt b/embedding/requirements.txt index 15c98f7..c32505a 100644 --- a/embedding/requirements.txt +++ b/embedding/requirements.txt @@ -1,82 +1,4 @@ -attrs==23.2.0 -Automat==22.10.0 -Babel==2.10.3 -bcc==0.29.1 -bcrypt==3.2.2 -blinker==1.7.0 -boto3==1.34.46 -botocore==1.34.46 -certifi==2023.11.17 -chardet==5.2.0 -click==8.1.6 -cloud-init==25.1.2 -colorama==0.4.6 -command-not-found==0.3 -configobj==5.0.8 -constantly==23.10.4 -cryptography==41.0.7 -dbus-python==1.3.2 -distro==1.9.0 -distro-info==1.7+build1 -gpg==1.18.0 -h11==0.14.0 -httplib2==0.20.4 -hyperlink==21.0.0 -idna==3.6 -incremental==22.10.0 -Jinja2==3.1.2 -jmespath==1.0.1 -jsonpatch==1.32 -jsonpointer==2.0 -jsonschema==4.10.3 -launchpadlib==1.11.0 -lazr.restfulclient==0.14.6 -lazr.uri==1.0.6 -Markdown==3.5.2 -markdown-it-py==3.0.0 -MarkupSafe==2.1.5 -mdurl==0.1.2 -netaddr==0.8.0 -netifaces==0.11.0 -oauthlib==3.2.2 -packaging==24.0 -pexpect==4.9.0 -ptyprocess==0.7.0 -pyasn1==0.4.8 -pyasn1-modules==0.2.8 -Pygments==2.17.2 -PyGObject==3.48.2 -PyHamcrest==2.1.0 -PyJWT==2.7.0 -pyOpenSSL==23.2.0 -pyparsing==3.1.1 -pyrsistent==0.20.0 -pyserial==3.5 -python-apt==2.7.7+ubuntu4 -python-dateutil==2.8.2 -python-debian==0.1.49+ubuntu2 -python-magic==0.4.27 -pytz==2024.1 -PyYAML==6.0.1 +chromadb==1.0.13 +einops==0.8.1 requests==2.31.0 -rich==13.7.1 -s3transfer==0.10.1 -service-identity==24.1.0 -setuptools==68.1.2 -six==1.16.0 -sos==4.7.2 -ssh-import-id==5.11 -systemd-python==235 -Twisted==24.3.0 -ubuntu-drivers-common==0.0.0 -ubuntu-pro-client==8001 -ufw==0.36.2 -unattended-upgrades==0.1 -urllib3==2.0.7 -uvicorn==0.27.1 -uvloop==0.19.0 -wadllib==1.3.6 -wheel==0.42.0 -wsproto==1.2.0 -xkit==0.0.0 -zope.interface==6.1 +sentence-transformers==5.0.0 \ No newline at end of file diff --git a/scraper/dockerfile b/scraper/dockerfile new file mode 100644 index 0000000..37dd4fb --- /dev/null +++ b/scraper/dockerfile @@ -0,0 +1,20 @@ +# --- Dockerfile.local --- +FROM python:3.13.5 + +# Firefox + GeckoDriver +RUN apt-get update && apt-get install -y --no-install-recommends \ + firefox-esr wget ca-certificates gnupg2 \ + && GECKO=v0.36.0 && \ + wget -qO- "https://github.com/mozilla/geckodriver/releases/download/${GECKO}/geckodriver-${GECKO}-linux64.tar.gz" \ + | tar -xz -C /usr/local/bin geckodriver \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt +RUN playwright install +RUN playwright install-deps +RUN python3 -m nltk.downloader punkt_tab # Download NLTK data and bake into image +COPY . . + +CMD ["python3", "scraper.py"] diff --git a/scraper/dockerfile-selenium b/scraper/dockerfile-selenium new file mode 100644 index 0000000..afe4a12 --- /dev/null +++ b/scraper/dockerfile-selenium @@ -0,0 +1,22 @@ +# Use an official Selenium Firefox standalone as a base image +FROM selenium/standalone-firefox:latest + +USER root +# Set the working directory in the container +WORKDIR /app + +# Install Python dependencies +COPY requirements.txt /app/ +RUN pip install --break-system-packages --no-cache-dir -r requirements.txt && \ + playwright install + +# Copy the current directory contents into the container at /app +COPY . /app +RUN mkdir -p /app/articles && \ + chown seluser:seluser /app/articles && \ + chmod 755 /app/articles + +USER seluser +# Run the Flask app +CMD ["python3", "scraper.py"] + diff --git a/scraper/requirements.txt b/scraper/requirements.txt index 21c339e..4e5d262 100644 --- a/scraper/requirements.txt +++ b/scraper/requirements.txt @@ -38,9 +38,7 @@ tldextract==5.3.0 tqdm==4.67.1 trio==0.30.0 trio-websocket==0.12.2 -typing-extensions==4.14.0 tzdata==2025.2 -urllib3==2.5.0 websocket-client==1.8.0 wsproto==1.2.0 diff --git a/scraper/scraper.py b/scraper/scraper.py index 1b4b222..30a7a85 100644 --- a/scraper/scraper.py +++ b/scraper/scraper.py @@ -8,26 +8,31 @@ from selenium.webdriver.firefox.options import Options as FirefoxOptions from newspaper.google_news import GoogleNewsSource from concurrent.futures import ProcessPoolExecutor import nltk +from nltk.downloader import Downloader + +FEED_FILE = os.getenv("FEED_FILE") # Ensure necessary NLTK resources are downloaded / Needed for selenium + newspaper4k article parsing -nltk.download('punkt_tab') +d = Downloader() +if not d.is_installed('punkt_tab'): + nltk.download('punkt_tab') articles = [] -def load_rss_feed_sources(): +def load_rss_feed_sources(feed_file=FEED_FILE): """ Loads the RSS feed sources from a JSON file. """ print("Loading RSS feed sources from rss_feeds.json...") try: - with open("rss_short_feed.json", "r", encoding="utf-8") as f: + with open(FEED_FILE, "r", encoding="utf-8") as f: return json.load(f) except FileNotFoundError: - print("rss_feeds.json not found, returning empty list.") + print(FEED_FILE + " not found, returning empty list.") return [] except json.JSONDecodeError: - print("Error decoding rss_feeds.json, returning empty list.") + print("Error decoding " + FEED_FILE + " , returning empty list.") return [] def mine_all_articles(rss_feed_sources, limit=None): @@ -68,9 +73,10 @@ def save_article_to_file(article, filename, source="Unfiltered"): """ Saves the given article text to a file with the specified filename. """ - os.makedirs("articles", exist_ok=True) + # articles dir should already be there + # os.makedirs("articles", exist_ok=True) - outputDir = "articles/"+source if source else "articles" + outputDir = "articles/"+source os.makedirs(outputDir, exist_ok=True) if source else None # Sanitize filename: use only the last part of the URL or replace slashes @@ -86,6 +92,12 @@ def get_article_with_selenium(url): options = FirefoxOptions() options.add_argument("--headless") driver = webdriver.Firefox(options=options) + driver.set_page_load_timeout(30) # 30 seconds timeout + + + #driver = webdriver.Remote( + #command_executor='http://localhost:4444/wd/hub', + #options=options) try: driver.get(url) time.sleep(5) # Wait for JS to load @@ -112,20 +124,6 @@ def get_article_with_playwright(url): time.sleep(5) # Adjust as needed for the page to load completely html = page.content() - ''' - # Check for iframes and use the first one's content if present - frames = page.frames - main_frame = page.main_frame - for frame in frames: - if frame != main_frame: - try: - frame.wait_for_load_state("domcontentloaded", timeout=5000) - html = frame.content() - print("Extracted content from iframe.") - break - except Exception as e: - print(f"Could not extract iframe content: {e}") - ''' browser.close() @@ -135,12 +133,12 @@ def get_article_with_playwright(url): return article.text -def pull_articles(link, source, title=None, save_to_file=True): +def pull_article(link, source, title=None, save_to_file=True): filename = title if title else link safe_filename = generate_filename_from_url(filename) - if os.path.exists(os.path.join("articles", safe_filename)): + if os.path.exists(os.path.join("articles", source, safe_filename)): print(f"Article already cached: {filename}") - with open(os.path.join("articles", safe_filename), "r", encoding="utf-8") as f: + with open(os.path.join("articles", source, safe_filename), "r", encoding="utf-8") as f: return f.read() text = "" @@ -148,20 +146,6 @@ def pull_articles(link, source, title=None, save_to_file=True): time.sleep(5) # Since we spawn lots of processes, we need to sleep at the start try: - # --- Google News Source handling temporarily disabled --- - # if "news.google.com" in link: - # gn = GoogleNewsSource(link) - # gn.build() - # # Pull the first article from the Google News cluster - # if gn.articles: - # article = gn.articles[0] - # article.download() - # article.parse() - # text = article.text - # if not text or len(text) < 200: - # raise ValueError("\tGoogle News article text too short, falling back to Selenium.") - # print(f"\tSuccessfully pulled Google News article from {link}") - # else: article = newspaper.article(link) article.download() article.parse() @@ -196,24 +180,50 @@ def pull_articles(link, source, title=None, save_to_file=True): save_article_to_file(text, filename, source) return text -# Pull the RSS feed sources from the JSON file -rss_feed_sources = load_rss_feed_sources() +while True: + print("=========================================") + print("Starting new scraping iteration...") + # Pull the RSS feed sources from the JSON file + rss_feed_sources = load_rss_feed_sources() -# Mine all articles from the RSS feed sources -rss_feed_links = mine_all_articles(rss_feed_sources) + # Mine all articles from the RSS feed sources + rss_feed_links = mine_all_articles(rss_feed_sources) -# Randomize the order of the links to help with load balancing -import random -random.shuffle(rss_feed_links) + # Randomize the order of the links to help with load balancing + import random + random.shuffle(rss_feed_links) -# Pull the latest article from a variety of news sources -#[pull_articles(link, source, title) for source, title, link in rss_feed_links] + link_list = [link for _, title, link in rss_feed_links] + source_list = [source for source, _, _ in rss_feed_links] + title_list = [title for _, title, link in rss_feed_links] -link_list = [link for _, title, link in rss_feed_links] -source_list = [source for source, _, _ in rss_feed_links] -title_list = [title for _, title, link in rss_feed_links] + with ProcessPoolExecutor() as executor: + futures = [executor.submit(pull_article, link, source, title) for link, source, title in zip(link_list, source_list, title_list)] + results = [] + errors = [] + for future in futures: + try: + results.append(future.result(timeout=60)) # seconds + except Exception as e: + print(f"Error in pull_article: {e}") + errors.append(e) + print(f"Attempted to Pull {len(results)} articles in parallel.") + print(f"Encountered {len(errors)} errors during article pulling. " +\ + "Outputting errors to a local file.") + + # Ouput errors to a local file + if errors: + with open("errors.txt", "w", encoding="utf-8") as f: + for error in errors: + f.write(str(error) + "\n") + print(f"Errors logged to errors.txt") -with ProcessPoolExecutor() as executor: - results = list(executor.map(pull_articles, link_list, source_list, title_list)) - print(f"Pulled {len(results)} articles in parallel.") - print("All articles pulled successfully.") + # Print all results to a log file + with open("results.txt", "w", encoding="utf-8") as f: + for result in results: + f.write(result + "\n") + print("All articles pulled successfully.") + + # Sleep for a while before the next iteration + print("Sleeping for 15 minutes before the next iteration...") + time.sleep(15 * 60) # Sleep for 15 minutes