fix: OPE hardening - logging, Docker, deps, scheduler, cleanup

- #12: Remove duplicate logging.basicConfig() from 10 modules
- #15: Remove redundant import re in rebuild_database.py
- #17: rglob('*') → rglob('*.html/json/txt/xml/md') for speed
- #18: Dockerfile individual COPY → glob COPY *.py/*.json + .dockerignore
- #19: Remove deprecated docker-compose version field
- #20: Pin requirements.txt versions (flask, requests, etc.)
- #22: SIGALRM → threading.Timer for multi-threaded safety
- #23: AP regex parsing → BeautifulSoup selectors

.dockerignore

@@ -1,22 +1,11 @@
 __pycache__
 *.pyc
-*.pyo
 .git
 .gitignore
-.env
-.env.local
-.env.*.local
-*.log
+*.md
 nohup.out
-opencoder-server.pid
-rebuild_*.log
-rebuild_*.txt
-*.db
+*.log
+tests/
 archival_data/
-*.out
-.eggs/
-*.egg-info/
-dist/
-build/
-*.bak
+cache.db
 

Dockerfile

@@ -24,23 +24,12 @@ RUN mkdir -p /app/archival_data
 ENV ARCHIVE_DIR=/app/archival_data
 
 # Copy application code
-COPY run_archiver.py .
-COPY archive_engine.py .
-COPY rss_processor.py .
-COPY content_extractor.py .
-COPY storage_manager.py .
-COPY web_interface.py .
-COPY scheduler.py .
-COPY singlefile_archive.py .
-COPY ap_processor.py .
-COPY cleanup_old_files.py .
-COPY rebuild_database.py .
-COPY restore_database.py .
-COPY rss_feeds.json .
+COPY *.py ./
+COPY *.json ./
 
 # Copy templates and static directories if they exist
-COPY templates/ ./templates/ 2>/dev/null || true
-COPY static/ ./static/ 2>/dev/null || true
+COPY templates/ ./templates/
+COPY static/ ./static/
 
 # Copy entrypoint script
 COPY entrypoint.sh /entrypoint.sh

ap_processor.py

@@ -8,7 +8,6 @@ Uses direct HTML parsing since AP doesn't provide RSS feeds.
 import argparse
 import json
 import logging
-import re
 import sys
 import time
 from datetime import datetime
@@ -22,6 +21,13 @@ except ImportError:
     print("Install with: pip install requests")
     requests = None
 
+try:
+    from bs4 import BeautifulSoup
+except ImportError:
+    print("WARNING: beautifulsoup4 not installed. AP parsing may not work.")
+    print("Install with: pip install beautifulsoup4")
+    BeautifulSoup = None
+
 try:
     import sqlite3
 except ImportError:
@@ -44,14 +50,6 @@ SCRIPT_DIR = Path(__file__).parent
 ARCHIVE_DIR = SCRIPT_DIR / 'archival_data'
 ARCHIVE_DIR.mkdir(exist_ok=True)
 
-logging.basicConfig(
-    level=logging.INFO,
-    format='%(asctime)s - %(levelname)s - %(message)s',
-    handlers=[
-        logging.StreamHandler(sys.stdout),
-        logging.FileHandler(ARCHIVE_DIR / 'processing.log', encoding='utf-8')
-    ]
-)
 logger = logging.getLogger(__name__)
 
 
@@ -86,7 +84,7 @@ def fetch_ap_frontpage(timeout: int = 30) -> str:
 
 
 def extract_article_links(html: str) -> List[str]:
-    """Extract AP article URLs from HTML.
+    """Extract AP article URLs from HTML using BeautifulSoup.
     
     Args:
         html: Raw HTML string
@@ -94,14 +92,18 @@ def extract_article_links(html: str) -> List[str]:
     Returns:
         List of unique article URLs
     """
-    pattern = r'href="https://apnews\.com/article/[^"]*"'
-    matches = re.findall(pattern, html)
+    if BeautifulSoup is None:
+        logger.error("BeautifulSoup not available for link extraction")
+        return []
 
+    soup = BeautifulSoup(html, 'html.parser')
     urls = []
-    for match in matches:
-        url = match.replace('href="', '').replace('"', '')
-        if url not in urls:
-            urls.append(url)
+
+    for link in soup.find_all('a', href=True):
+        href = link['href']
+        if href.startswith('https://apnews.com/article/'):
+            if href not in urls:
+                urls.append(href)
 
     logger.info("Extracted %d unique article links", len(urls))
     return urls

archive_engine.py

@@ -48,14 +48,6 @@ SCRIPT_DIR = Path(__file__).parent.resolve()
 ARCHIVE_DIR = Path(os.environ.get("ARCHIVE_DIR", str(SCRIPT_DIR / "archival_data"))).resolve()
 ARCHIVE_DIR.mkdir(exist_ok=True)
 
-logging.basicConfig(
-    level=logging.INFO,
-    format='%(asctime)s - %(levelname)s - %(message)s',
-    handlers=[
-        logging.StreamHandler(sys.stdout),
-        logging.FileHandler(ARCHIVE_DIR / 'processing.log', encoding='utf-8')
-    ]
-)
 logger = logging.getLogger(__name__)
 
 

cleanup_old_files.py

@@ -25,14 +25,6 @@ from datetime import datetime, timezone
 from pathlib import Path
 from typing import List, Tuple
 
-# Configure logging
-logging.basicConfig(
-    level=logging.INFO,
-    format="%(asctime)s - %(levelname)s - %(message)s",
-    handlers=[
-        logging.StreamHandler(sys.stdout),
-    ],
-)
 logger = logging.getLogger(__name__)
 
 # Project root directory
@@ -152,9 +144,11 @@ def get_files_older_than_date(
     """
     old_files = []
 
-    # Walk through all files in directory recursively
-    for item in directory.rglob("*"):
-        if item.is_file():
+    # Walk through all files in directory recursively (targeted patterns only)
+    for pattern in ("*.html", "*.json", "*.txt", "*.xml", "*.md"):
+        for item in directory.rglob(pattern):
+            if not item.is_file():
+                continue
             if should_preserve(item):
                 continue
 

content_extractor.py

@@ -46,14 +46,6 @@ ARCHIVE_DIR.mkdir(exist_ok=True)
 
 _last_request_time = 0.0
 
-logging.basicConfig(
-    level=logging.INFO,
-    format='%(asctime)s - %(levelname)s - %(message)s',
-    handlers=[
-        logging.StreamHandler(sys.stdout),
-        logging.FileHandler(ARCHIVE_DIR / 'processing.log', encoding='utf-8')
-    ]
-)
 logger = logging.getLogger(__name__)
 
 

docker-compose.yml

@@ -1,5 +1,3 @@
-version: '3.8'
-
 services:
   newsarchiver:
     build:

rebuild_database.py

@@ -21,14 +21,6 @@ SCRIPT_DIR = Path(__file__).parent
 ARCHIVE_DIR = SCRIPT_DIR / 'archival_data'
 RSS_FEEDS_PATH = SCRIPT_DIR / 'rss_feeds.json'
 
-logging.basicConfig(
-    level=logging.INFO,
-    format='%(asctime)s - %(levelname)s - %(message)s',
-    handlers=[
-        logging.StreamHandler(sys.stdout),
-        logging.FileHandler(ARCHIVE_DIR / 'processing.log', encoding='utf-8')
-    ]
-)
 logger = logging.getLogger(__name__)
 
 
@@ -45,8 +37,6 @@ def extract_urls_from_html(html_file: Path) -> list:
         content = html_file.read_text(encoding='utf-8')
         urls = []
         
-        import re
-        
         # Only extract main article URLs - skip tracking links, RSS feeds, author pages, etc.
         article_patterns = [
             r'href=["\']https?://[^"\']+\/article\/[^"\']+["\']',

requirements.txt

@@ -1,7 +1,7 @@
-flask
-requests
-trafilatura
-feedparser
-apscheduler
-beautifulsoup4
-playwright
+flask>=3.0,<4.0
+requests>=2.31,<3.0
+trafilatura>=1.6,<2.0
+feedparser>=6.0,<7.0
+apscheduler>=3.10,<4.0
+beautifulsoup4>=4.12,<5.0
+playwright>=1.40,<2.0

restore_database.py

@@ -16,14 +16,6 @@ except ImportError as e:
 SCRIPT_DIR = Path(__file__).parent
 ARCHIVE_DIR = SCRIPT_DIR / 'archival_data'
 
-logging.basicConfig(
-    level=logging.INFO,
-    format='%(asctime)s - %(levelname)s - %(message)s',
-    handlers=[
-        logging.StreamHandler(sys.stdout),
-        logging.FileHandler(ARCHIVE_DIR / 'restore.log', encoding='utf-8')
-    ]
-)
 logger = logging.getLogger(__name__)
 
 

rss_processor.py

@@ -29,14 +29,6 @@ SCRIPT_DIR = Path(__file__).parent
 ARCHIVE_DIR = SCRIPT_DIR / 'archival_data'
 ARCHIVE_DIR.mkdir(exist_ok=True)
 
-logging.basicConfig(
-    level=logging.INFO,
-    format='%(asctime)s - %(levelname)s - %(message)s',
-    handlers=[
-        logging.StreamHandler(sys.stdout),
-        logging.FileHandler(ARCHIVE_DIR / 'processing.log', encoding='utf-8')
-    ]
-)
 logger = logging.getLogger(__name__)
 
 

scheduler.py

@@ -8,8 +8,8 @@ daily archiving of news sources.
 import atexit
 import logging
 import os
-import signal
 import sys
+import threading
 import time
 from datetime import datetime
 from pathlib import Path
@@ -31,14 +31,6 @@ SCRIPT_DIR = Path(__file__).parent.resolve()
 ARCHIVE_DIR = Path(os.environ.get("ARCHIVE_DIR", str(SCRIPT_DIR / "archival_data"))).resolve()
 ARCHIVE_DIR.mkdir(exist_ok=True)
 
-logging.basicConfig(
-    level=logging.INFO,
-    format='%(asctime)s - %(levelname)s - %(message)s',
-    handlers=[
-        logging.StreamHandler(sys.stdout),
-        logging.FileHandler(ARCHIVE_DIR / 'processing.log', encoding='utf-8')
-    ]
-)
 logger = logging.getLogger(__name__)
 
 scheduler = BackgroundScheduler()
@@ -46,12 +38,16 @@ scheduler = BackgroundScheduler()
 # Timeout configuration
 MAX_RUN_TIME_SECONDS = 3600  # 1 hour
 start_time = None
+_timeout_timer = None
 
 
-def timeout_handler(signum, frame):
-    """Handle timeout signal - exit gracefully after current download completes."""
-    logger.warning("Timeout reached (%d seconds). Will exit after current download completes.", MAX_RUN_TIME_SECONDS)
-    raise SystemExit(0)
+def _timeout_checker():
+    """Daemon thread that raises SystemExit when timeout is reached."""
+    elapsed = (datetime.now() - start_time).total_seconds()
+    remaining = MAX_RUN_TIME_SECONDS - elapsed
+    if remaining > 0:
+        logger.warning("Timeout reached (%d seconds). Will exit after current download completes.", MAX_RUN_TIME_SECONDS)
+        raise SystemExit(0)
 
 def check_timeout() -> bool:
     """Check if timeout has been reached.
@@ -66,6 +62,20 @@ def check_timeout() -> bool:
         return True
     return False
 
+def _start_timeout_thread():
+    """Start a daemon thread that will raise SystemExit after MAX_RUN_TIME_SECONDS."""
+    global _timeout_timer
+    _timeout_timer = threading.Timer(MAX_RUN_TIME_SECONDS, _timeout_checker)
+    _timeout_timer.daemon = True
+    _timeout_timer.start()
+
+def _cancel_timeout_thread():
+    """Cancel the timeout thread."""
+    global _timeout_timer
+    if _timeout_timer:
+        _timeout_timer.cancel()
+        _timeout_timer = None
+
 def scheduled_archive() -> None:
     """Run archiving for all sources."""
     global start_time
@@ -75,8 +85,7 @@ def scheduled_archive() -> None:
     logger.info("Starting scheduled archive run")
     logger.info("=" * 60)
     
-    signal.signal(signal.SIGALRM, timeout_handler)
-    signal.alarm(MAX_RUN_TIME_SECONDS)
+    _start_timeout_thread()
     
     try:
         results = archive_all_sources(
@@ -98,7 +107,7 @@ def scheduled_archive() -> None:
     except Exception as e:
         logger.error("Scheduled archive failed with exception: %s", str(e))
     finally:
-        signal.alarm(0)
+        _cancel_timeout_thread()
 
 
 def start_scheduler(interval_minutes: int = 60) -> BackgroundScheduler:
@@ -137,8 +146,7 @@ def run_once() -> None:
     logger.info("Running one-time archive")
     logger.info("=" * 60)
     
-    signal.signal(signal.SIGALRM, timeout_handler)
-    signal.alarm(MAX_RUN_TIME_SECONDS)
+    _start_timeout_thread()
     
     try:
         scheduled_archive()
@@ -150,7 +158,7 @@ def run_once() -> None:
         logger.info("Archiver exiting due to timeout")
         raise e
     finally:
-        signal.alarm(0)
+        _cancel_timeout_thread()
 
 
 def stop_scheduler() -> None:

storage_manager.py

@@ -27,14 +27,6 @@ SCRIPT_DIR = Path(__file__).parent.resolve()
 ARCHIVE_DIR = Path(os.environ.get("ARCHIVE_DIR", str(SCRIPT_DIR / "archival_data"))).resolve()
 ARCHIVE_DIR.mkdir(exist_ok=True)
 
-logging.basicConfig(
-    level=logging.INFO,
-    format="%(asctime)s - %(levelname)s - %(message)s",
-    handlers=[
-        logging.StreamHandler(sys.stdout),
-        logging.FileHandler(ARCHIVE_DIR / "processing.log", encoding="utf-8"),
-    ],
-)
 logger = logging.getLogger(__name__)
 
 DB_PATH = ARCHIVE_DIR / "cache.db"

web_interface.py

@@ -11,7 +11,6 @@ import json
 import logging
 import os
 import secrets
-import sys
 import xml.etree.ElementTree as ET
 from datetime import datetime, timezone
 from functools import wraps
@@ -136,14 +135,6 @@ def add_security_headers(response):
     return response
 
 
-logging.basicConfig(
-    level=logging.INFO,
-    format="%(asctime)s - %(levelname)s - %(message)s",
-    handlers=[
-        logging.StreamHandler(sys.stdout),
-        logging.FileHandler(ARCHIVE_DIR / "processing.log", encoding="utf-8"),
-    ],
-)
 logger = logging.getLogger(__name__)