From 6228589c1422f08499cea0a19b5d340c8d58b640 Mon Sep 17 00:00:00 2001 From: Jarian Cottingham Date: Sun, 5 Jul 2026 04:14:05 +0000 Subject: [PATCH] fix: OPE hardening - logging, Docker, deps, scheduler, cleanup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - #12: Remove duplicate logging.basicConfig() from 10 modules - #15: Remove redundant import re in rebuild_database.py - #17: rglob('*') → rglob('*.html/json/txt/xml/md') for speed - #18: Dockerfile individual COPY → glob COPY *.py/*.json + .dockerignore - #19: Remove deprecated docker-compose version field - #20: Pin requirements.txt versions (flask, requests, etc.) - #22: SIGALRM → threading.Timer for multi-threaded safety - #23: AP regex parsing → BeautifulSoup selectors --- .dockerignore | 19 ++++-------------- Dockerfile | 19 ++++-------------- ap_processor.py | 38 +++++++++++++++++++----------------- archive_engine.py | 8 -------- cleanup_old_files.py | 16 +++++---------- content_extractor.py | 8 -------- docker-compose.yml | 2 -- rebuild_database.py | 10 ---------- requirements.txt | 14 +++++++------- restore_database.py | 8 -------- rss_processor.py | 8 -------- scheduler.py | 46 ++++++++++++++++++++++++++------------------ storage_manager.py | 8 -------- web_interface.py | 9 --------- 14 files changed, 67 insertions(+), 146 deletions(-) diff --git a/.dockerignore b/.dockerignore index 2542be0..c12bee9 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,22 +1,11 @@ __pycache__ *.pyc -*.pyo .git .gitignore -.env -.env.local -.env.*.local -*.log +*.md nohup.out -opencoder-server.pid -rebuild_*.log -rebuild_*.txt -*.db +*.log +tests/ archival_data/ -*.out -.eggs/ -*.egg-info/ -dist/ -build/ -*.bak +cache.db diff --git a/Dockerfile b/Dockerfile index 5f8f795..2975ada 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,23 +24,12 @@ RUN mkdir -p /app/archival_data ENV ARCHIVE_DIR=/app/archival_data # Copy application code -COPY run_archiver.py . -COPY archive_engine.py . -COPY rss_processor.py . -COPY content_extractor.py . -COPY storage_manager.py . -COPY web_interface.py . -COPY scheduler.py . -COPY singlefile_archive.py . -COPY ap_processor.py . -COPY cleanup_old_files.py . -COPY rebuild_database.py . -COPY restore_database.py . -COPY rss_feeds.json . +COPY *.py ./ +COPY *.json ./ # Copy templates and static directories if they exist -COPY templates/ ./templates/ 2>/dev/null || true -COPY static/ ./static/ 2>/dev/null || true +COPY templates/ ./templates/ +COPY static/ ./static/ # Copy entrypoint script COPY entrypoint.sh /entrypoint.sh diff --git a/ap_processor.py b/ap_processor.py index 9043612..d557a0d 100644 --- a/ap_processor.py +++ b/ap_processor.py @@ -8,7 +8,6 @@ Uses direct HTML parsing since AP doesn't provide RSS feeds. import argparse import json import logging -import re import sys import time from datetime import datetime @@ -22,6 +21,13 @@ except ImportError: print("Install with: pip install requests") requests = None +try: + from bs4 import BeautifulSoup +except ImportError: + print("WARNING: beautifulsoup4 not installed. AP parsing may not work.") + print("Install with: pip install beautifulsoup4") + BeautifulSoup = None + try: import sqlite3 except ImportError: @@ -44,14 +50,6 @@ SCRIPT_DIR = Path(__file__).parent ARCHIVE_DIR = SCRIPT_DIR / 'archival_data' ARCHIVE_DIR.mkdir(exist_ok=True) -logging.basicConfig( - level=logging.INFO, - format='%(asctime)s - %(levelname)s - %(message)s', - handlers=[ - logging.StreamHandler(sys.stdout), - logging.FileHandler(ARCHIVE_DIR / 'processing.log', encoding='utf-8') - ] -) logger = logging.getLogger(__name__) @@ -86,7 +84,7 @@ def fetch_ap_frontpage(timeout: int = 30) -> str: def extract_article_links(html: str) -> List[str]: - """Extract AP article URLs from HTML. + """Extract AP article URLs from HTML using BeautifulSoup. Args: html: Raw HTML string @@ -94,15 +92,19 @@ def extract_article_links(html: str) -> List[str]: Returns: List of unique article URLs """ - pattern = r'href="https://apnews\.com/article/[^"]*"' - matches = re.findall(pattern, html) - + if BeautifulSoup is None: + logger.error("BeautifulSoup not available for link extraction") + return [] + + soup = BeautifulSoup(html, 'html.parser') urls = [] - for match in matches: - url = match.replace('href="', '').replace('"', '') - if url not in urls: - urls.append(url) - + + for link in soup.find_all('a', href=True): + href = link['href'] + if href.startswith('https://apnews.com/article/'): + if href not in urls: + urls.append(href) + logger.info("Extracted %d unique article links", len(urls)) return urls diff --git a/archive_engine.py b/archive_engine.py index 33e782b..e1a9ea1 100644 --- a/archive_engine.py +++ b/archive_engine.py @@ -48,14 +48,6 @@ SCRIPT_DIR = Path(__file__).parent.resolve() ARCHIVE_DIR = Path(os.environ.get("ARCHIVE_DIR", str(SCRIPT_DIR / "archival_data"))).resolve() ARCHIVE_DIR.mkdir(exist_ok=True) -logging.basicConfig( - level=logging.INFO, - format='%(asctime)s - %(levelname)s - %(message)s', - handlers=[ - logging.StreamHandler(sys.stdout), - logging.FileHandler(ARCHIVE_DIR / 'processing.log', encoding='utf-8') - ] -) logger = logging.getLogger(__name__) diff --git a/cleanup_old_files.py b/cleanup_old_files.py index ba57569..8e9b361 100644 --- a/cleanup_old_files.py +++ b/cleanup_old_files.py @@ -25,14 +25,6 @@ from datetime import datetime, timezone from pathlib import Path from typing import List, Tuple -# Configure logging -logging.basicConfig( - level=logging.INFO, - format="%(asctime)s - %(levelname)s - %(message)s", - handlers=[ - logging.StreamHandler(sys.stdout), - ], -) logger = logging.getLogger(__name__) # Project root directory @@ -152,9 +144,11 @@ def get_files_older_than_date( """ old_files = [] - # Walk through all files in directory recursively - for item in directory.rglob("*"): - if item.is_file(): + # Walk through all files in directory recursively (targeted patterns only) + for pattern in ("*.html", "*.json", "*.txt", "*.xml", "*.md"): + for item in directory.rglob(pattern): + if not item.is_file(): + continue if should_preserve(item): continue diff --git a/content_extractor.py b/content_extractor.py index 752d87c..f652055 100644 --- a/content_extractor.py +++ b/content_extractor.py @@ -46,14 +46,6 @@ ARCHIVE_DIR.mkdir(exist_ok=True) _last_request_time = 0.0 -logging.basicConfig( - level=logging.INFO, - format='%(asctime)s - %(levelname)s - %(message)s', - handlers=[ - logging.StreamHandler(sys.stdout), - logging.FileHandler(ARCHIVE_DIR / 'processing.log', encoding='utf-8') - ] -) logger = logging.getLogger(__name__) diff --git a/docker-compose.yml b/docker-compose.yml index f5f7388..004bb32 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,3 @@ -version: '3.8' - services: newsarchiver: build: diff --git a/rebuild_database.py b/rebuild_database.py index 6038514..e78b9e7 100644 --- a/rebuild_database.py +++ b/rebuild_database.py @@ -21,14 +21,6 @@ SCRIPT_DIR = Path(__file__).parent ARCHIVE_DIR = SCRIPT_DIR / 'archival_data' RSS_FEEDS_PATH = SCRIPT_DIR / 'rss_feeds.json' -logging.basicConfig( - level=logging.INFO, - format='%(asctime)s - %(levelname)s - %(message)s', - handlers=[ - logging.StreamHandler(sys.stdout), - logging.FileHandler(ARCHIVE_DIR / 'processing.log', encoding='utf-8') - ] -) logger = logging.getLogger(__name__) @@ -45,8 +37,6 @@ def extract_urls_from_html(html_file: Path) -> list: content = html_file.read_text(encoding='utf-8') urls = [] - import re - # Only extract main article URLs - skip tracking links, RSS feeds, author pages, etc. article_patterns = [ r'href=["\']https?://[^"\']+\/article\/[^"\']+["\']', diff --git a/requirements.txt b/requirements.txt index 413de30..1a8e8fa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ -flask -requests -trafilatura -feedparser -apscheduler -beautifulsoup4 -playwright +flask>=3.0,<4.0 +requests>=2.31,<3.0 +trafilatura>=1.6,<2.0 +feedparser>=6.0,<7.0 +apscheduler>=3.10,<4.0 +beautifulsoup4>=4.12,<5.0 +playwright>=1.40,<2.0 diff --git a/restore_database.py b/restore_database.py index adf0564..6d360ab 100644 --- a/restore_database.py +++ b/restore_database.py @@ -16,14 +16,6 @@ except ImportError as e: SCRIPT_DIR = Path(__file__).parent ARCHIVE_DIR = SCRIPT_DIR / 'archival_data' -logging.basicConfig( - level=logging.INFO, - format='%(asctime)s - %(levelname)s - %(message)s', - handlers=[ - logging.StreamHandler(sys.stdout), - logging.FileHandler(ARCHIVE_DIR / 'restore.log', encoding='utf-8') - ] -) logger = logging.getLogger(__name__) diff --git a/rss_processor.py b/rss_processor.py index d44a43b..ba156cc 100644 --- a/rss_processor.py +++ b/rss_processor.py @@ -29,14 +29,6 @@ SCRIPT_DIR = Path(__file__).parent ARCHIVE_DIR = SCRIPT_DIR / 'archival_data' ARCHIVE_DIR.mkdir(exist_ok=True) -logging.basicConfig( - level=logging.INFO, - format='%(asctime)s - %(levelname)s - %(message)s', - handlers=[ - logging.StreamHandler(sys.stdout), - logging.FileHandler(ARCHIVE_DIR / 'processing.log', encoding='utf-8') - ] -) logger = logging.getLogger(__name__) diff --git a/scheduler.py b/scheduler.py index 4b8fa11..73ccf0c 100644 --- a/scheduler.py +++ b/scheduler.py @@ -8,8 +8,8 @@ daily archiving of news sources. import atexit import logging import os -import signal import sys +import threading import time from datetime import datetime from pathlib import Path @@ -31,14 +31,6 @@ SCRIPT_DIR = Path(__file__).parent.resolve() ARCHIVE_DIR = Path(os.environ.get("ARCHIVE_DIR", str(SCRIPT_DIR / "archival_data"))).resolve() ARCHIVE_DIR.mkdir(exist_ok=True) -logging.basicConfig( - level=logging.INFO, - format='%(asctime)s - %(levelname)s - %(message)s', - handlers=[ - logging.StreamHandler(sys.stdout), - logging.FileHandler(ARCHIVE_DIR / 'processing.log', encoding='utf-8') - ] -) logger = logging.getLogger(__name__) scheduler = BackgroundScheduler() @@ -46,12 +38,16 @@ scheduler = BackgroundScheduler() # Timeout configuration MAX_RUN_TIME_SECONDS = 3600 # 1 hour start_time = None +_timeout_timer = None -def timeout_handler(signum, frame): - """Handle timeout signal - exit gracefully after current download completes.""" - logger.warning("Timeout reached (%d seconds). Will exit after current download completes.", MAX_RUN_TIME_SECONDS) - raise SystemExit(0) +def _timeout_checker(): + """Daemon thread that raises SystemExit when timeout is reached.""" + elapsed = (datetime.now() - start_time).total_seconds() + remaining = MAX_RUN_TIME_SECONDS - elapsed + if remaining > 0: + logger.warning("Timeout reached (%d seconds). Will exit after current download completes.", MAX_RUN_TIME_SECONDS) + raise SystemExit(0) def check_timeout() -> bool: """Check if timeout has been reached. @@ -66,6 +62,20 @@ def check_timeout() -> bool: return True return False +def _start_timeout_thread(): + """Start a daemon thread that will raise SystemExit after MAX_RUN_TIME_SECONDS.""" + global _timeout_timer + _timeout_timer = threading.Timer(MAX_RUN_TIME_SECONDS, _timeout_checker) + _timeout_timer.daemon = True + _timeout_timer.start() + +def _cancel_timeout_thread(): + """Cancel the timeout thread.""" + global _timeout_timer + if _timeout_timer: + _timeout_timer.cancel() + _timeout_timer = None + def scheduled_archive() -> None: """Run archiving for all sources.""" global start_time @@ -75,8 +85,7 @@ def scheduled_archive() -> None: logger.info("Starting scheduled archive run") logger.info("=" * 60) - signal.signal(signal.SIGALRM, timeout_handler) - signal.alarm(MAX_RUN_TIME_SECONDS) + _start_timeout_thread() try: results = archive_all_sources( @@ -98,7 +107,7 @@ def scheduled_archive() -> None: except Exception as e: logger.error("Scheduled archive failed with exception: %s", str(e)) finally: - signal.alarm(0) + _cancel_timeout_thread() def start_scheduler(interval_minutes: int = 60) -> BackgroundScheduler: @@ -137,8 +146,7 @@ def run_once() -> None: logger.info("Running one-time archive") logger.info("=" * 60) - signal.signal(signal.SIGALRM, timeout_handler) - signal.alarm(MAX_RUN_TIME_SECONDS) + _start_timeout_thread() try: scheduled_archive() @@ -150,7 +158,7 @@ def run_once() -> None: logger.info("Archiver exiting due to timeout") raise e finally: - signal.alarm(0) + _cancel_timeout_thread() def stop_scheduler() -> None: diff --git a/storage_manager.py b/storage_manager.py index e998246..356b808 100644 --- a/storage_manager.py +++ b/storage_manager.py @@ -27,14 +27,6 @@ SCRIPT_DIR = Path(__file__).parent.resolve() ARCHIVE_DIR = Path(os.environ.get("ARCHIVE_DIR", str(SCRIPT_DIR / "archival_data"))).resolve() ARCHIVE_DIR.mkdir(exist_ok=True) -logging.basicConfig( - level=logging.INFO, - format="%(asctime)s - %(levelname)s - %(message)s", - handlers=[ - logging.StreamHandler(sys.stdout), - logging.FileHandler(ARCHIVE_DIR / "processing.log", encoding="utf-8"), - ], -) logger = logging.getLogger(__name__) DB_PATH = ARCHIVE_DIR / "cache.db" diff --git a/web_interface.py b/web_interface.py index 084d739..b9f0b39 100644 --- a/web_interface.py +++ b/web_interface.py @@ -11,7 +11,6 @@ import json import logging import os import secrets -import sys import xml.etree.ElementTree as ET from datetime import datetime, timezone from functools import wraps @@ -136,14 +135,6 @@ def add_security_headers(response): return response -logging.basicConfig( - level=logging.INFO, - format="%(asctime)s - %(levelname)s - %(message)s", - handlers=[ - logging.StreamHandler(sys.stdout), - logging.FileHandler(ARCHIVE_DIR / "processing.log", encoding="utf-8"), - ], -) logger = logging.getLogger(__name__)