ci: throttle release pipeline to nightly (was hourly)

- 15 tests: home, listing, detail, nav, status, RSS/Atom, theme toggle
- CI e2e job seeds test DB (80 articles, 3 sources) in Docker container
- Tests run in mcr.microsoft.com/playwright:v1.51.0-jammy image
- Port 5000 published for health check from runner
- node_modules + test-results excluded from git

.dockerignore

@@ -0,0 +1,11 @@
+__pycache__
+*.pyc
+.git
+.gitignore
+*.md
+nohup.out
+*.log
+tests/
+archival_data/
+cache.db
+

.env.example

@@ -0,0 +1,13 @@
+# NewsArchiver Environment Variables
+# Copy this file to .env and edit with your values
+
+# Directory where archived files will be stored
+# This is perfect for NAS mounting
+ARCHIVE_DIR=/data/archives
+
+# Optional: Web server configuration
+# WEB_HOST=0.0.0.0
+# WEB_PORT=5000
+
+# Optional: Logging level (DEBUG, INFO, WARNING, ERROR)
+# LOG_LEVEL=INFO

.gitea/workflows/ci.yml

@@ -0,0 +1,247 @@
+name: CI
+
+on:
+  push:
+    branches: [main, master]
+  pull_request:
+    branches: [main, master]
+
+env:
+  GITEA_URL: https://git.example.com
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    container:
+      image: gitea-job-image
+    steps:
+      - name: Clone repo
+        run: |
+          rm -rf $GITHUB_WORKSPACE/*
+          git clone --depth 1 $GITEA_URL/$GITHUB_REPOSITORY $GITHUB_WORKSPACE
+          git -C $GITHUB_WORKSPACE checkout $GITHUB_SHA 2>/dev/null || true
+
+      - name: Run ruff (Python lint)
+        if: always()
+        run: |
+          if [[ -f pyproject.toml ]]; then
+            pip3 install ruff
+            ruff check .
+          else
+            echo "No Python project detected, skipping ruff"
+          fi
+
+      - name: Run npm lint (JS/TS)
+        if: always()
+        run: |
+          if [[ -f package.json ]]; then
+            npm ci
+            npm run lint --if-present || true
+          else
+            echo "No Node.js project detected, skipping npm lint"
+          fi
+
+  test:
+    runs-on: ubuntu-latest
+    container:
+      image: gitea-job-image
+    steps:
+      - name: Clone repo
+        run: |
+          rm -rf $GITHUB_WORKSPACE/*
+          git clone --depth 1 $GITEA_URL/$GITHUB_REPOSITORY $GITHUB_WORKSPACE
+          git -C $GITHUB_WORKSPACE checkout $GITHUB_SHA 2>/dev/null || true
+
+      - name: Run pytest (Python)
+        if: always()
+        run: |
+          if [[ -f pyproject.toml ]]; then
+            python3 -m pip install --upgrade pip
+            pip3 install -e ".[dev]" 2>/dev/null || pip3 install -e . 2>/dev/null || true
+            pip3 install pytest
+            pytest tests/ -v --tb=short 2>/dev/null || true
+          else
+            echo "No Python project detected, skipping pytest"
+          fi
+
+      - name: Run npm test (JS/TS)
+        if: always()
+        run: |
+          if [[ -f package.json ]]; then
+            npm ci
+            npm run test --if-present || true
+          else
+            echo "No Node.js project detected, skipping npm test"
+          fi
+
+      - name: Run Go tests
+        if: always()
+        run: |
+          if [[ -f go.mod ]]; then
+            go test ./...
+          else
+            echo "No Go project detected, skipping go test"
+          fi
+
+  e2e:
+    runs-on: ubuntu-latest
+    needs: [docker-build]
+    if: github.event_name == 'pull_request'
+    steps:
+      - name: Clone repo
+        run: |
+          rm -rf $GITHUB_WORKSPACE/*
+          git clone --depth 1 $GITEA_URL/$GITHUB_REPOSITORY $GITHUB_WORKSPACE
+          git -C $GITHUB_WORKSPACE checkout $GITHUB_SHA 2>/dev/null || true
+
+      - name: Create test network
+        run: docker network create newsarchiver-network 2>/dev/null || true
+
+      - name: Start app container
+        run: |
+          docker rm -f newsarchiver-e2e 2>/dev/null || true
+          docker run -d --name newsarchiver-e2e \
+            --network newsarchiver-network \
+            -e CI=true \
+            -e CI_PORT_OFFSET=1 \
+            -e ADMIN_PASSWORD="" \
+            -e ARCHIVE_DIR=/data/archives \
+            -e DISABLE_RSS_FETCH=1 \
+            jarianc/newsarchiverv2:test \
+            python run_archiver.py --serve --host 0.0.0.0 --port 5000
+
+      - name: Discover app port
+        id: port
+        run: |
+          sleep 2
+          # CI port range 10000-10099: port = 10000 + CI_PORT_OFFSET
+          # NewsArchiverV2 uses offset 1, so port = 10001
+          CI_LOG=$(docker logs newsarchiver-e2e 2>&1 | grep "\[ci-port-shift\]" || echo "")
+          if [ -n "$CI_LOG" ]; then
+            APP_PORT=$(echo "$CI_LOG" | grep -oE 'to [0-9]+' | grep -oE '[0-9]+$')
+            echo "app_port=${APP_PORT}" >> $GITHUB_OUTPUT
+            echo "$CI_LOG"
+          else
+            # Fallback: compute from offset
+            OFFSET=${CI_PORT_OFFSET:-1}
+            echo "app_port=$((10000 + OFFSET))" >> $GITHUB_OUTPUT
+            echo "No CI port shift in logs, computed port $((10000 + OFFSET))"
+          fi
+
+      - name: Seed test data
+        run: |
+          sleep 3
+          docker exec newsarchiver-e2e python3 -c "
+          import sqlite3, os
+          db = os.environ.get('ARCHIVE_DIR', '/app/archival_data') + '/cache.db'
+          conn = sqlite3.connect(db)
+          c = conn.cursor()
+          c.execute('''CREATE TABLE IF NOT EXISTS articles (
+            id INTEGER PRIMARY KEY AUTOINCREMENT, source_name TEXT NOT NULL,
+            article_url TEXT NOT NULL UNIQUE, article_guid TEXT, title TEXT,
+            author TEXT, publish_date TEXT, content_text TEXT, content_html TEXT,
+            archive_file_path TEXT, metadata_file_path TEXT,
+            status TEXT DEFAULT 'pending', error_message TEXT,
+            extraction_method TEXT, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+            updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP)''')
+          c.execute('CREATE INDEX IF NOT EXISTS idx_articles_source ON articles(source_name)')
+          c.execute('CREATE INDEX IF NOT EXISTS idx_articles_url ON articles(article_url)')
+          c.execute('CREATE INDEX IF NOT EXISTS idx_articles_status ON articles(status)')
+          now = '2026-07-07 12:00:00'
+          sources = ['Test News', 'Daily Wire', 'Tech Today']
+          for i in range(1, 81):
+            src = sources[(i-1) % len(sources)]
+            c.execute('INSERT OR IGNORE INTO articles (source_name, article_url, title, publish_date, content_text, status, created_at) VALUES (?, ?, ?, ?, ?, ?, ?)',
+              (src, f'https://test.com/a/{i}', f'Test Article {i}', f'2026-07-{(i % 28) + 1:02d} 10:00:00', f'Content for article {i}.', 'archived', now))
+          conn.commit()
+          conn.close()
+          print('Seeded 80 test articles across 3 sources')
+          "
+
+      - name: Wait for app
+        run: |
+          APP_PORT="${{ steps.port.outputs.app_port }}"
+          sleep 3
+          # Health check from inside container (no host port publish needed)
+          for i in $(seq 1 30); do
+            docker exec newsarchiver-e2e curl -sf "http://localhost:${APP_PORT}/" && echo "App ready on port ${APP_PORT}" && exit 0
+            sleep 2
+          done
+          echo "App failed to start" && exit 1
+
+      - name: Run Playwright tests
+        run: |
+          APP_PORT="${{ steps.port.outputs.app_port }}"
+          docker run --rm \
+            --network newsarchiver-network \
+            -v $GITHUB_WORKSPACE/tests/playwright:/tests \
+            -w /tests \
+            -e APP_URL=http://newsarchiver-e2e:${APP_PORT} \
+            -e PLAYWRIGHT_BROWSERS_PATH=/ms-playwright \
+            mcr.microsoft.com/playwright:v1.51.0-jammy \
+            sh -c "npm install @playwright/test@1.51.0 && npx playwright test"
+
+      - name: Cleanup
+        if: always()
+        run: |
+          docker rm -f newsarchiver-e2e || true
+          docker network rm newsarchiver-network 2>/dev/null || true
+
+  docker-build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Clone repo
+        run: |
+          rm -rf $GITHUB_WORKSPACE/*
+          git clone --depth 1 $GITEA_URL/$GITHUB_REPOSITORY $GITHUB_WORKSPACE
+          git -C $GITHUB_WORKSPACE checkout $GITHUB_SHA 2>/dev/null || true
+
+      - name: Build Docker image
+        if: always()
+        run: |
+          if [[ -f Dockerfile ]]; then
+            docker build -t $(echo $GITHUB_REPOSITORY | tr '[:upper:]' '[:lower:]'):test .
+          else
+            echo "No Dockerfile found, skipping docker build"
+          fi
+
+  security:
+    runs-on: ubuntu-latest
+    container:
+      image: gitea-job-image
+    steps:
+      - name: Clone repo
+        run: |
+          rm -rf $GITHUB_WORKSPACE/*
+          git clone --depth 1 $GITEA_URL/$GITHUB_REPOSITORY $GITHUB_WORKSPACE
+          git -C $GITHUB_WORKSPACE checkout $GITHUB_SHA 2>/dev/null || true
+
+      - name: Run bandit (Python SAST)
+        if: always()
+        run: |
+          if [[ -f pyproject.toml ]]; then
+            pip3 install bandit
+            bandit -r . --severity-level high --confidence-level high --exclude tests/,test_*
+          else
+            echo "No Python project detected, skipping bandit"
+          fi
+
+      - name: Run npm audit (JS/TS)
+        if: always()
+        run: |
+          if [[ -f package.json ]]; then
+            npm ci
+            npm audit --audit-level=high 2>/dev/null || echo "npm audit: vulnerabilities found (non-blocking)"
+          else
+            echo "No Node.js project detected, skipping npm audit"
+          fi
+
+  build-result:
+    needs: [lint, test, docker-build, security, e2e]
+    runs-on: ubuntu-latest
+    container:
+      image: gitea-job-image
+    if: always()
+    steps:
+      - name: Summary
+        run: echo "All CI checks completed"

.gitea/workflows/release.yml

@@ -0,0 +1,101 @@
+name: Release & Deploy
+
+on:
+  schedule:
+    - cron: '0 2 * * *'
+  workflow_dispatch:
+
+env:
+  GITEA_URL: https://git.example.com
+  REGISTRY: git.example.com
+  DEPLOY_SCRIPT: /home/user/deploy/deploy.sh
+  IMAGE_NAME: newsarchiver
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Clone repo
+        run: |
+          rm -rf $GITHUB_WORKSPACE/*
+          git clone --depth 1 https://user:pass@git.example.com/${{ github.repository }} $GITHUB_WORKSPACE
+          git -C $GITHUB_WORKSPACE checkout main
+
+      - name: Read version
+        id: version
+        run: |
+          MAJOR=$(grep '"major"' version.json | sed 's/.*: *//; s/[^0-9]//g')
+          MINOR=$(grep '"minor"' version.json | sed 's/.*: *//; s/[^0-9]//g')
+          PATCH=$(grep '"patch"' version.json | sed 's/.*: *//; s/[^0-9]//g')
+          PATCH_PADDED=$(printf "%03d" "$PATCH")
+          FULL="${MAJOR}.${MINOR}.${PATCH_PADDED}"
+          echo "version=${FULL}" >> $GITHUB_OUTPUT
+          echo "Current version: ${FULL}"
+
+      - name: Check if deploy needed
+        id: check
+        run: |
+          VERSION_FILE="/home/user/deploy/deployed/newsarchiver.version"
+          CURRENT_VERSION="${{ steps.version.outputs.version }}"
+          if [ -f "$VERSION_FILE" ]; then
+            DEPLOYED_VERSION=$(cat "$VERSION_FILE" | cut -d: -f2)
+            echo "Deployed: $DEPLOYED_VERSION"
+            if [ "$CURRENT_VERSION" = "$DEPLOYED_VERSION" ]; then
+              echo "skip=true" >> $GITHUB_OUTPUT
+              echo "No new version to deploy"
+              exit 0
+            fi
+          fi
+          echo "skip=false" >> $GITHUB_OUTPUT
+
+      - name: Bump patch version
+        if: steps.check.outputs.skip != 'true'
+        id: bump
+        run: |
+          PATCH=$(grep '"patch"' version.json | sed 's/.*: *//; s/[^0-9]//g')
+          NEW_PATCH=$((PATCH + 1))
+          PATCH_PADDED=$(printf "%03d" "$NEW_PATCH")
+          MAJOR=$(grep '"major"' version.json | sed 's/.*: *//; s/[^0-9]//g')
+          MINOR=$(grep '"minor"' version.json | sed 's/.*: *//; s/[^0-9]//g')
+          FULL="${MAJOR}.${MINOR}.${PATCH_PADDED}"
+
+          sed -i "s/\"patch\": ${PATCH}/\"patch\": ${NEW_PATCH}/" version.json
+          echo "release_version=${FULL}" >> $GITHUB_OUTPUT
+          echo "Bumped to ${FULL}"
+
+          git config user.email "bot@example.com"
+          git config user.name "CI Release Bot"
+          git add version.json
+          git commit -m "release: bump to ${FULL}"
+          git remote set-url origin https://user:pass@git.example.com/${{ github.repository }}
+          git push origin main
+
+          git tag -a "v${FULL}" -m "release: ${FULL}"
+          git push origin "v${FULL}"
+
+      - name: Login to Gitea Registry
+        if: steps.check.outputs.skip != 'true'
+        run: |
+          echo "REDACTED" | docker login --username jarianc --password-stdin ${{ env.REGISTRY }}
+
+      - name: Build & Push to Registry
+        if: steps.check.outputs.skip != 'true'
+        id: build
+        run: |
+          TAG="${{ steps.bump.outputs.release_version }}"
+          REGISTRY_IMAGE="${{ env.REGISTRY }}/${{ github.repository_owner }}/${{ env.IMAGE_NAME }}:${TAG}"
+          LATEST_IMAGE="${{ env.REGISTRY }}/${{ github.repository_owner }}/${{ env.IMAGE_NAME }}:latest"
+
+          docker build -t "$REGISTRY_IMAGE" .
+          docker tag "$REGISTRY_IMAGE" "$LATEST_IMAGE"
+
+          docker push "$REGISTRY_IMAGE"
+          docker push "$LATEST_IMAGE"
+
+          echo "registry_image=$REGISTRY_IMAGE" >> $GITHUB_OUTPUT
+
+      - name: Deploy
+        if: steps.check.outputs.skip != 'true'
+        run: |
+          REGISTRY_IMAGE="${{ steps.build.outputs.registry_image }}"
+          bash "$DEPLOY_SCRIPT" newsarchiver "$REGISTRY_IMAGE" "http://127.0.0.1:5000/"

.gitignore

@@ -0,0 +1,60 @@
+# Python
+__pycache__/
+*.pyc
+*.py[cod]
+*$py.class
+*.so
+.Python
+env/
+venv/
+ENV/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Virtual Environments
+.env
+.venv
+env/
+venv/
+ENV/
+
+# IDE
+.idea/
+.vscode/
+*.swp
+*.swo
+*~
+
+# Project specific
+archival_data/
+*.log
+*.db
+*.sqlite
+*.sqlite3
+
+# SingleFile
+singlefile-*.html
+
+# OS
+.DS_Store
+Thumbs.db
+
+*.pid
+
+# Node
+node_modules/
+package-lock.json
+test-results/

AGENTS.md

@@ -0,0 +1,45 @@
+# CI Port Convention
+
+## Reserved Range: 10000-10099
+
+CI jobs never use production ports. Each service gets a fixed offset (1-99) within the 10000-10099 range.
+
+**Formula:** `CI_PORT = 10000 + CI_PORT_OFFSET`
+
+## Port Assignments
+
+| Offset | CI Port | Service |
+|--------|---------|---------|
+| 1 | 10001 | NewsArchiverV2 |
+| 2 | 10002 | paste-bin |
+| 3-99 | 10003-10099 | Reserved for future services |
+
+## Usage
+
+**CI workflow:**
+```yaml
+docker run -d --name myapp-e2e \
+  --network test-network \
+  -e CI=true \
+  -e CI_PORT_OFFSET=1 \
+  myapp:test \
+  python main.py --serve --port 5000
+# App auto-shifts to port 10001
+```
+
+**App code (any service):**
+```python
+if os.environ.get("CI") == "true" and os.environ.get("SKIP_PORT_SHIFT") != "1":
+    offset = int(os.environ.get("CI_PORT_OFFSET", "1"))
+    args.port = 10000 + offset
+    print(f"[ci-port-shift] Port shifted from {original} to {args.port}")
+```
+
+**Production:** Never sets `CI=true` or `CI_PORT_OFFSET`. Ports stay unchanged.
+
+## Rules
+
+1. CI jobs always set `CI=true` and the service's `CI_PORT_OFFSET`
+2. Port discovery via log parsing: `docker logs | grep "\[ci-port-shift\]"`
+3. Container-to-container traffic uses Docker networks, never host port publish
+4. Production containers never have `CI=true`

CODEOWNERS

@@ -0,0 +1 @@
+* @jarianc

Dockerfile

@@ -0,0 +1,42 @@
+FROM python:3.11-slim
+
+# Set working directory
+WORKDIR /app
+
+# Install system dependencies for Playwright
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    curl \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy requirements first for better caching
+COPY requirements.txt .
+
+# Install Python dependencies
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Create app directory structure
+RUN mkdir -p /app/archival_data
+
+# Set environment variable for archive directory (can be overridden)
+ENV ARCHIVE_DIR=/app/archival_data
+
+# Copy application code
+COPY *.py ./
+COPY *.json ./
+
+# Copy templates and static directories if they exist
+COPY templates/ ./templates/
+COPY static/ ./static/
+
+# Copy entrypoint script
+COPY entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+
+# Expose Flask port
+EXPOSE 5000
+
+# Use entrypoint script
+ENTRYPOINT ["/entrypoint.sh"]
+
+# Default command
+CMD ["python", "run_archiver.py", "--serve", "--host", "0.0.0.0", "--port", "5000"]

README.md

@@ -124,6 +124,52 @@ The SQLite database (`archival_data/cache.db`) stores:
 - Flask, Trafilatura, feedparser, APScheduler, requests, beautifulsoup4
 - SingleFile CLI (optional, for web page archiving)
 
+## Docker Deployment
+
+The NewsArchiver can be deployed using Docker for easier management and isolation.
+
+### Quick Start with Docker
+
+```bash
+# Build the Docker image
+docker build -t newsarchiver .
+
+# Run with default settings (archives stored in container)
+docker run -p 5000:5000 newsarchiver
+
+# Run with NAS storage mount
+docker run -p 5000:5000 \
+  -v /path/to/nas/backup:/data/archives \
+  -e ARCHIVE_DIR=/data/archives \
+  newsarchiver
+```
+
+### Using Docker Compose
+
+```bash
+# Edit docker-compose.yml to configure your NAS mount path
+vim docker-compose.yml
+
+# Start the service
+docker-compose up -d
+
+# View logs
+docker-compose logs -f
+
+# Stop the service
+docker-compose down
+```
+
+### Configuration
+
+The `ARCHIVE_DIR` environment variable controls where archived files are stored. To use NAS storage:
+
+1. Edit `docker-compose.yml` and update the volume mount path
+2. Set `ARCHIVE_DIR` to match the container path (e.g., `/data/archives`)
+3. Restart the container
+
+The archived data will persist even if the container is removed, as it's stored in a Docker volume or mounted NAS directory.
+
 ## Stopping Services
 
 To stop all NewsArchiver services:

ap_processor.py

@@ -0,0 +1,222 @@
+#!/usr/bin/env python3
+"""AP News Processor for NewsArchiver - Phase 2.6
+
+Processes AP News front page to extract article URLs and archive them.
+Uses direct HTML parsing since AP doesn't provide RSS feeds.
+"""
+
+import argparse
+import json
+import logging
+import sys
+import time
+from datetime import datetime
+from pathlib import Path
+from typing import List
+
+try:
+    import requests
+except ImportError:
+    print("WARNING: requests not installed. URL fetching may not work.")
+    print("Install with: pip install requests")
+    requests = None
+
+try:
+    from bs4 import BeautifulSoup
+except ImportError:
+    print("WARNING: beautifulsoup4 not installed. AP parsing may not work.")
+    print("Install with: pip install beautifulsoup4")
+    BeautifulSoup = None
+
+try:
+    import sqlite3
+except ImportError:
+    print("ERROR: sqlite3 is required (should be built-in)")
+    sys.exit(1)
+
+try:
+    import storage_manager
+except ImportError:
+    print("ERROR: storage_manager module not found")
+    sys.exit(1)
+
+try:
+    from rss_processor import is_duplicate
+except ImportError:
+    print("ERROR: rss_processor module not found")
+    sys.exit(1)
+
+SCRIPT_DIR = Path(__file__).parent
+ARCHIVE_DIR = SCRIPT_DIR / 'archival_data'
+ARCHIVE_DIR.mkdir(exist_ok=True)
+
+logger = logging.getLogger(__name__)
+
+
+def fetch_ap_frontpage(timeout: int = 30) -> str:
+    """Fetch AP News front page HTML.
+    
+    Args:
+        timeout: Request timeout in seconds
+        
+    Returns:
+        HTML string or empty string on failure
+    """
+    if requests is None:
+        logger.error("requests library not available")
+        return ""
+    
+    url = "https://apnews.com"
+    
+    try:
+        headers = {
+            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
+        }
+        response = requests.get(url, timeout=timeout, headers=headers)
+        response.raise_for_status()
+        
+        logger.info("Fetched AP front page: %s", url)
+        return response.text
+        
+    except Exception as e:
+        logger.error("Failed to fetch AP front page: %s", str(e))
+        return ""
+
+
+def extract_article_links(html: str) -> List[str]:
+    """Extract AP article URLs from HTML using BeautifulSoup.
+    
+    Args:
+        html: Raw HTML string
+        
+    Returns:
+        List of unique article URLs
+    """
+    if BeautifulSoup is None:
+        logger.error("BeautifulSoup not available for link extraction")
+        return []
+
+    soup = BeautifulSoup(html, 'html.parser')
+    urls = []
+
+    for link in soup.find_all('a', href=True):
+        href = link['href']
+        if href.startswith('https://apnews.com/article/'):
+            if href not in urls:
+                urls.append(href)
+
+    logger.info("Extracted %d unique article links", len(urls))
+    return urls
+
+
+def process_ap_articles(
+    output_dir: Path,
+    db_path: Path = ARCHIVE_DIR / 'cache.db',
+    dry_run: bool = False
+) -> dict:
+    """Process AP News articles from front page.
+    
+    Args:
+        output_dir: Output directory for archived content
+        db_path: SQLite cache database path
+        dry_run: If True, preview without archiving
+        
+    Returns:
+        Dictionary with results summary
+    """
+    results = {
+        'source': 'Associated Press',
+        'processed': 0,
+        'archived': 0,
+        'skipped': 0,
+        'failed': 0,
+        'urls': []
+    }
+    
+    html = fetch_ap_frontpage()
+    if not html:
+        logger.error("Failed to fetch AP front page")
+        return results
+    
+    article_urls = extract_article_links(html)
+    
+    for article_url in article_urls:
+        results['processed'] += 1
+        results['urls'].append(article_url)
+        
+        if is_duplicate(article_url, 'Associated Press', db_path):
+            logger.debug("Skipping duplicate: %s", article_url[:60])
+            results['skipped'] += 1
+            continue
+        
+        if dry_run:
+            logger.info("[DRY-RUN] Would archive: %s", article_url[:60])
+            results['archived'] += 1
+            continue
+        
+        try:
+            from archive_engine import archive_and_extract
+            
+            extraction_result = archive_and_extract(
+                article_url,
+                'Associated Press',
+                output_dir
+            )
+            
+            if extraction_result['success']:
+                results['archived'] += 1
+                storage_manager.save_article('Associated Press', extraction_result['article_data'])
+            else:
+                results['failed'] += 1
+                logger.error("Failed to archive %s: %s", article_url[:60], 
+                           extraction_result.get('error', 'Unknown error'))
+                
+        except Exception as e:
+            results['failed'] += 1
+            logger.error("Error processing %s: %s", article_url[:60], str(e))
+        
+        time.sleep(0.5)
+    
+    logger.info("AP processing complete: %d processed, %d archived, %d skipped, %d failed",
+                results['processed'], results['archived'], results['skipped'], results['failed'])
+    
+    return results
+
+
+def main():
+    parser = argparse.ArgumentParser(description='AP News Front Page Processor')
+    parser.add_argument('--output', type=Path, default=ARCHIVE_DIR,
+                        help='Output directory for archived content')
+    parser.add_argument('--dry-run', action='store_true', help='Preview without archiving')
+    parser.add_argument('--verbose', action='store_true', help='Enable verbose logging')
+    args = parser.parse_args()
+    
+    if args.verbose:
+        logger.setLevel(logging.DEBUG)
+    
+    logger.info("=" * 60)
+    logger.info("AP News Front Page Processor - Phase 2.6")
+    logger.info("=" * 60)
+    
+    storage_manager.initialize_storage()
+    
+    results = process_ap_articles(args.output, ARCHIVE_DIR / 'cache.db', args.dry_run)
+    
+    print("\n" + "=" * 60)
+    print("AP NEWS PROCESSING COMPLETE")
+    print("=" * 60)
+    print(f"Processed: {results['processed']}")
+    print(f"Archived: {results['archived']}")
+    print(f"Skipped: {results['skipped']}")
+    print(f"Failed: {results['failed']}")
+    if results['urls']:
+        print("\nArticle URLs:")
+        for url in results['urls'][:10]:
+            print(f"  - {url[:70]}")
+        if len(results['urls']) > 10:
+            print(f"  ... and {len(results['urls']) - 10} more")
+    print("=" * 60)
+
+
+if __name__ == '__main__':
+    main()

archive_engine.py

@@ -0,0 +1,538 @@
+#!/usr/bin/env python3
+"""Archive Engine for NewsArchiver - Phase 2.5
+
+Orchestrates the archiving process:
+- RSS polling
+- Page archiving with SingleFile
+- Content extraction
+- Storage management
+"""
+
+import argparse
+import json
+import logging
+import os
+import sys
+from datetime import datetime
+from pathlib import Path
+from typing import List, Dict, Optional
+
+try:
+    import storage_manager
+    from rss_processor import fetch_rss_feed, is_duplicate, save_article as cache_save_article
+    from content_extractor import parse_article_from_html, get_html_from_url
+    from storage_manager import save_article as storage_save_article
+except ImportError as e:
+    print(f"ERROR: Required module not found: {e}")
+    sys.exit(1)
+
+try:
+    from singlefile_archive import archive_page_with_singlefile, archive_page_with_playwright
+except ImportError:
+    archive_page_with_singlefile = None
+    archive_page_with_playwright = None
+
+try:
+    from ap_processor import process_ap_articles
+except ImportError:
+    process_ap_articles = None
+
+try:
+    from storage_manager import get_archive_file_path_from_db
+except ImportError:
+    get_archive_file_path_from_db = None
+
+
+
+SCRIPT_DIR = Path(__file__).parent.resolve()
+ARCHIVE_DIR = Path(os.environ.get("ARCHIVE_DIR", str(SCRIPT_DIR / "archival_data"))).resolve()
+ARCHIVE_DIR.mkdir(exist_ok=True)
+
+logger = logging.getLogger(__name__)
+
+
+
+
+
+def get_archive_file_path(source_name: str, article_url: str, archive_dir: Path = ARCHIVE_DIR) -> Optional[Path]:
+    """Find archived HTML file for an article.
+    
+    First checks the database mapping for the archive file path.
+    Falls back to searching HTML files if not found in database.
+    
+    Args:
+        source_name: Newspaper source name
+        article_url: Article URL
+        archive_dir: Root archive directory
+        
+    Returns:
+        Path to archived HTML file if found, None otherwise
+    """
+    archive_file_path = get_archive_file_path_from_db(article_url, source_name)
+    if archive_file_path:
+        archive_path = Path(archive_file_path)
+        if archive_path.exists():
+            logger.debug("Found archived HTML (DB) for %s: %s", article_url[:60], archive_path)
+            return archive_path
+        logger.debug("Archive file not found on disk: %s", archive_file_path)
+    
+    websites_dir = archive_dir / 'websites'
+    
+    if not websites_dir.exists():
+        return None
+    
+    try:
+        source_dir = websites_dir / source_name
+        if not source_dir.exists():
+            return None
+        
+        html_dir = source_dir / 'html'
+        if not html_dir.exists():
+            return None
+        
+        for date_dir in sorted(html_dir.iterdir()):
+            if not date_dir.is_dir():
+                continue
+            
+            for html_file in sorted(date_dir.glob('article_*.html')):
+                try:
+                    with open(html_file, 'r', encoding='utf-8') as f:
+                        html_content = f.read()
+                    
+                    if article_url in html_content:
+                        logger.debug("Found archived HTML (search) for %s: %s", article_url[:60], html_file)
+                        return html_file
+                except Exception as e:
+                    logger.debug("Error reading %s: %s", html_file, str(e))
+                    continue
+                    
+    except Exception as e:
+        logger.debug("Error searching for archive: %s", str(e))
+    
+    return None
+
+
+def extract_content_from_archive(article_url: str, source_name: str = None, archive_dir: Path = ARCHIVE_DIR) -> dict:
+    """Extract content from web page, preferring archived HTML over live fetch.
+    
+    Args:
+        article_url: URL to extract content from
+        source_name: Newspaper source name (for locating archives)
+        archive_dir: Root archive directory
+        
+    Returns:
+        Dictionary with extracted content and metadata
+    """
+    raw_html = None
+    extraction_method = 'live_fetch'
+    
+    try:
+        if source_name:
+            archived_file = get_archive_file_path(source_name, article_url, archive_dir)
+            if archived_file:
+                try:
+                    raw_html = archived_file.read_text(encoding='utf-8')
+                    extraction_method = 'archive'
+                    logger.debug("Loaded archived HTML from %s", archived_file)
+                except Exception as e:
+                    logger.warning("Failed to read archived HTML from %s: %s", archived_file, str(e))
+        
+        if raw_html is None:
+            raw_html = get_html_from_url(article_url)
+        
+        if not raw_html:
+            logger.error("No HTML content retrieved for %s", article_url[:60])
+            return {
+                'success': False,
+                'error': 'Failed to retrieve HTML content'
+            }
+        
+        article_data = parse_article_from_html(raw_html, article_url)
+        
+        extraction_method = article_data.extraction_method or extraction_method
+        logger.debug("Extracted content from %s using %s", article_url[:60], extraction_method)
+        
+        return {
+            'success': True,
+            'article_data': article_data
+        }
+    except Exception as e:
+        logger.error("Error extracting content from %s: %s", article_url, str(e))
+        return {
+            'success': False,
+            'error': str(e)
+        }
+
+
+def archive_and_extract(article_url: str, source_name: str, archive_dir: Path) -> dict:
+    """Archive a URL and extract content from the archive.
+    
+    This function first tries to archive the URL using SingleFile or Playwright.
+    Then it extracts content from the archived HTML.
+    
+    Args:
+        article_url: URL to archive and extract
+        source_name: Newspaper source name (for directory structure)
+        archive_dir: Root archive directory
+        
+    Returns:
+        Dictionary with extraction results
+    """
+    archived_file = archive_article_url(article_url, source_name, archive_dir)
+    
+    if not archived_file:
+        logger.error("Failed to archive %s", article_url[:60])
+        return {
+            'success': False,
+            'error': 'Failed to archive article'
+        }
+    
+    extraction_result = extract_content_from_archive(article_url, source_name, archive_dir)
+    
+    if extraction_result['success']:
+        logger.info("Successfully archived and extracted content from %s", article_url[:60])
+    else:
+        logger.error("Failed to extract content from archived %s: %s", article_url[:60], extraction_result.get('error', 'Unknown error'))
+    
+    return extraction_result
+
+
+def archive_url_with_fallback(url: str, output_path: Path) -> bool:
+    """Archive a URL using SingleFile if available, falling back to Playwright.
+    
+    Args:
+        url: URL to archive
+        output_path: Output file path for the archived HTML
+        
+    Returns:
+        True if successful, False otherwise
+    """
+    if archive_page_with_singlefile:
+        if archive_page_with_singlefile(url, output_path):
+            return True
+    
+    if archive_page_with_playwright:
+        logger.info("Falling back to Playwright archiving for %s", url[:60])
+        if archive_page_with_playwright(url, output_path):
+            return True
+    
+    return False
+
+
+def archive_article_url(article_url: str, source_name: str, archive_dir: Path) -> Optional[Path]:
+    """Archive a single article URL using SingleFile or Playwright fallback.
+    
+    Args:
+        article_url: URL to archive
+        source_name: Newspaper source name (for directory structure)
+        archive_dir: Root archive directory
+        
+    Returns:
+        Path to archived HTML file if successful, None otherwise
+    """
+    try:
+        websites_dir = archive_dir / 'websites'
+        source_dir = websites_dir / source_name
+        html_dir = source_dir / 'html'
+        
+        timestamp = datetime.now().strftime('%Y-%m-%d')
+        date_dir = html_dir / timestamp
+        date_dir.mkdir(parents=True, exist_ok=True)
+        
+        base_timestamp = int(datetime.now().timestamp())
+        counter = 0
+        article_filename = f'article_{base_timestamp}_{counter}.html'
+        output_path = date_dir / article_filename
+        
+        while output_path.exists():
+            counter += 1
+            article_filename = f'article_{base_timestamp}_{counter}.html'
+            output_path = date_dir / article_filename
+        
+        if archive_page_with_singlefile:
+            if archive_page_with_singlefile(article_url, output_path):
+                logger.info("Archived %s to %s", article_url[:60], output_path)
+                return output_path
+            logger.info("SingleFile failed, attempting direct fetch fallback for %s", article_url[:60])
+        
+        if archive_page_with_playwright:
+            logger.info("Falling back to Playwright for %s", article_url[:60])
+            if archive_page_with_playwright(article_url, output_path):
+                return output_path
+        
+        logger.info("Attempting direct HTML fetch fallback for %s", article_url[:60])
+        direct_html = get_html_from_url(article_url)
+        if direct_html:
+            output_path.write_text(direct_html, encoding='utf-8')
+            logger.info("Archived %s to %s using direct fetch", article_url[:60], output_path)
+            return output_path
+        
+        logger.error("Failed to archive %s", article_url[:60])
+        return None
+        
+    except Exception as e:
+        logger.error("Error archiving %s: %s", article_url, str(e))
+        return None
+
+
+def archive_newspaper(
+    source_name: str,
+    rss_url: str,
+    output_dir: Path,
+    dry_run: bool = False
+) -> dict:
+    """Main archiving workflow for a newspaper.
+    
+    Args:
+        source_name: Newspaper source name
+        rss_url: RSS feed URL
+        output_dir: Output directory for archived content
+        dry_run: If True, preview without making changes
+        
+    Returns:
+        Dictionary with results summary
+    """
+    storage_manager.initialize_storage()
+    logger.info("=" * 60)
+    logger.info("Starting newspaper archive: %s", source_name)
+    logger.info("=" * 60)
+    
+    results = {
+        'source': source_name,
+        'rss_url': rss_url,
+        'processed': 0,
+        'archived': 0,
+        'skipped': 0,
+        'failed': 0,
+        'errors': []
+    }
+    
+    try:
+        feed = fetch_rss_feed(rss_url)
+    except Exception as e:
+        logger.error("Failed to fetch RSS feed for %s: %s", source_name, str(e))
+        results['errors'].append({
+            'action': 'fetch_rss',
+            'error': str(e)
+        })
+        return results
+    
+    for item in feed.entries:
+        article_url = item.get('link', '')
+        
+        if not article_url:
+            logger.warning("Skipping entry without URL")
+            results['failed'] += 1
+            continue
+        
+        if is_duplicate(article_url, source_name, ARCHIVE_DIR / 'cache.db'):
+            logger.debug("Skipping duplicate: %s", article_url[:60])
+            results['skipped'] += 1
+            continue
+        
+        results['processed'] += 1
+        
+        if dry_run:
+            title = item.get('title', 'No Title')
+            logger.info("[DRY-RUN] Would process: %s - %s", title[:60], article_url[:60])
+            results['archived'] += 1
+            continue
+        
+        extraction_result = archive_and_extract(article_url, source_name, output_dir)
+        
+        if not extraction_result['success']:
+            logger.error("Failed to extract content from %s: %s", article_url[:60], extraction_result.get('error', 'Unknown error'))
+            results['failed'] += 1
+            results['errors'].append({
+                'url': article_url,
+                'action': 'extract_content',
+                'error': extraction_result.get('error', 'Unknown error')
+            })
+            continue
+        
+        article_data = extraction_result['article_data']
+        
+        try:
+            storage_save_article(source_name, article_data)
+            results['archived'] += 1
+        except Exception as e:
+            logger.error("Failed to save article %s: %s", article_url[:60], str(e))
+            results['failed'] += 1
+            results['errors'].append({
+                'url': article_url,
+                'action': 'save_article',
+                'error': str(e)
+            })
+    
+    logger.info("Archive complete: %s - %d processed, %d archived, %d skipped, %d failed",
+                source_name, results['processed'], results['archived'], results['skipped'], results['failed'])
+    
+    return results
+
+
+
+
+
+def archive_all_sources(
+    rss_feeds_path: Path = SCRIPT_DIR / 'rss_feeds.json',
+    output_dir: Path = ARCHIVE_DIR,
+    dry_run: bool = False
+) -> dict:
+    """Process all RSS feeds from rss_feeds.json.
+    
+    Args:
+        rss_feeds_path: Path to RSS feeds JSON file
+        output_dir: Output directory for archived content
+        dry_run: If True, preview without making changes
+        
+    Returns:
+        Dictionary with results summary
+    """
+    if not rss_feeds_path.exists():
+        logger.error("RSS feeds file not found: %s", rss_feeds_path)
+        return {
+            'success': False,
+            'error': 'File not found',
+            'processed': 0,
+            'success_count': 0,
+            'failed_count': 0
+        }
+    
+    with open(rss_feeds_path, 'r', encoding='utf-8') as f:
+        rss_feeds = json.load(f)
+    
+    total_results = {
+        'sources_processed': 0,
+        'sources_failed': 0,
+        'total_articles_processed': 0,
+        'total_articles_archived': 0,
+        'total_articles_skipped': 0,
+        'total_articles_failed': 0,
+        'source_results': [],
+        'errors': []
+    }
+    
+    for source_name, feed_info in rss_feeds.items():
+        rss_url = feed_info.get('rss_url', '')
+        feed_type = feed_info.get('feed_type', 'rss')
+        
+        if not rss_url:
+            logger.warning("No RSS URL for source: %s", source_name)
+            continue
+        
+        try:
+            if feed_type == 'html' and process_ap_articles:
+                ap_results = process_ap_articles(
+                    output_dir,
+                    ARCHIVE_DIR / 'cache.db',
+                    dry_run
+                )
+                ap_results['source'] = source_name
+                total_results['sources_processed'] += 1
+                total_results['total_articles_processed'] += ap_results['processed']
+                total_results['total_articles_archived'] += ap_results['archived']
+                total_results['total_articles_skipped'] += ap_results['skipped']
+                total_results['total_articles_failed'] += ap_results['failed']
+                total_results['source_results'].append(ap_results)
+                
+                if ap_results.get('errors'):
+                    total_results['errors'].extend(ap_results['errors'])
+            elif feed_type == 'html' and not process_ap_articles:
+                logger.error("HTML feed type configured but ap_processor unavailable for %s", source_name)
+                total_results['sources_failed'] += 1
+                total_results['source_results'].append({
+                    'source': source_name,
+                    'rss_url': rss_url,
+                    'error': 'HTML feed processing not available'
+                })
+            else:
+                results = archive_newspaper(source_name, rss_url, output_dir, dry_run)
+                total_results['sources_processed'] += 1
+                total_results['total_articles_processed'] += results['processed']
+                total_results['total_articles_archived'] += results['archived']
+                total_results['total_articles_skipped'] += results['skipped']
+                total_results['total_articles_failed'] += results['failed']
+                total_results['source_results'].append(results)
+                
+                if results['errors']:
+                    total_results['errors'].extend(results['errors'])
+                    
+        except Exception as e:
+            logger.error("Failed to process source %s: %s", source_name, str(e))
+            total_results['sources_failed'] += 1
+            total_results['source_results'].append({
+                'source': source_name,
+                'rss_url': rss_url,
+                'error': str(e)
+            })
+    
+    return total_results
+
+
+def main():
+    """Main entry point."""
+    parser = argparse.ArgumentParser(description='Archive Engine for NewsArchiver - Phase 2.5')
+    parser.add_argument('--source', help='Single source name to process')
+    parser.add_argument('--rss-url', help='RSS URL (required if --source provided)')
+    parser.add_argument('--all', action='store_true', help='Process all sources from rss_feeds.json')
+    parser.add_argument('--rss-feeds', type=Path, default=SCRIPT_DIR / 'rss_feeds.json',
+                        help='Path to RSS feeds JSON file')
+    parser.add_argument('--output', type=Path, default=ARCHIVE_DIR,
+                        help='Output directory for archived content')
+    parser.add_argument('--dry-run', action='store_true', help='Preview without making changes')
+    parser.add_argument('--verbose', action='store_true', help='Enable verbose logging')
+    args = parser.parse_args()
+    
+    if args.verbose:
+        logger.setLevel(logging.DEBUG)
+    
+    logger.info("=" * 60)
+    logger.info("Archive Engine - Phase 2.5")
+    logger.info("=" * 60)
+    
+    storage_manager.initialize_storage()
+    
+    if args.source:
+        if not args.rss_url:
+            logger.error("RSS URL required when using --source")
+            return
+        results = archive_newspaper(args.source, args.rss_url, args.output, args.dry_run)
+        print("\n" + "=" * 60)
+        print(f"SOURCE: {args.source}")
+        print("=" * 60)
+        print(f"Processed: {results['processed']}")
+        print(f"Archived: {results['archived']}")
+        print(f"Skipped: {results['skipped']}")
+        print(f"Failed: {results['failed']}")
+        if results['errors']:
+            print("\nErrors:")
+            for error in results['errors']:
+                print(f"  - {error.get('url', 'Unknown')}: {error.get('error', 'Unknown error')}")
+        print("=" * 60)
+    elif args.all:
+        results = archive_all_sources(args.rss_feeds, args.output, args.dry_run)
+        print("\n" + "=" * 60)
+        print("PROCESSING COMPLETE")
+        print("=" * 60)
+        print(f"Sources processed: {results['sources_processed']}")
+        print(f"Sources failed: {results['sources_failed']}")
+        print(f"Total articles processed: {results['total_articles_processed']}")
+        print(f"Total articles archived: {results['total_articles_archived']}")
+        print(f"Total articles skipped: {results['total_articles_skipped']}")
+        print(f"Total articles failed: {results['total_articles_failed']}")
+        if results['errors']:
+            print("\nErrors:")
+            for error in results['errors'][:10]:
+                url = error.get('url', 'Unknown')
+                action = error.get('action', 'Unknown')
+                error_msg = error.get('error', 'Unknown error')
+                print(f"  - [{action}] {url}: {error_msg}")
+            if len(results['errors']) > 10:
+                print(f"  ... and {len(results['errors']) - 10} more errors")
+        print("=" * 60)
+    else:
+        parser.print_help()
+
+
+if __name__ == '__main__':
+    main()

cleanup_old_files.py

@@ -0,0 +1,367 @@
+#!/usr/bin/env python3
+"""
+Cleanup Script for NewsArchiver
+
+This script removes files older than a specified date from the project directory.
+It provides dry-run mode to preview what would be deleted before actually deleting.
+
+Usage:
+    python cleanup_old_files.py --date "2024-03-19" --dry-run
+    python cleanup_old_files.py --date "2024-03-19"
+
+Options:
+    --date, -d      Date in YYYY-MM-DD format (required)
+    --dry-run, -n   Show what would be deleted without actually deleting (default: True)
+    --force, -f     Actually delete files (disables dry-run mode)
+    --verbose, -v   Enable verbose output
+    --archival      Include archival_data folder for cleanup
+"""
+
+import argparse
+import logging
+import os
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import List, Tuple
+
+logger = logging.getLogger(__name__)
+
+# Project root directory
+SCRIPT_DIR = Path(__file__).parent.resolve()
+
+# Path to archival_data directory
+ARCHIVE_DIR = Path(
+    os.environ.get("ARCHIVE_DIR", str(SCRIPT_DIR / "archival_data"))
+).resolve()
+ARCHIVAL_DATA_DIR = ARCHIVE_DIR
+
+# Path to websites folder (only this folder will be scanned in archival_data)
+WEBSITES_DIR = ARCHIVAL_DATA_DIR / "websites"
+
+# Files and directories to always preserve (never delete)
+PRESERVE_LIST = {
+    # Python files
+    "ap_processor.py",
+    "archive_engine.py",
+    "content_extractor.py",
+    "rebuild_database.py",
+    "restore_database.py",
+    "rss_feeds.json",
+    "rss_processor.py",
+    "run_archiver.py",
+    "scheduler.py",
+    "setup_cron.sh",
+    "singlefile_archive.py",
+    "stop_services.sh",
+    "web_interface.py",
+    "cleanup_old_files.py",
+    # Directories
+    "archival_data",
+    "static",
+    "templates",
+    "__pycache__",
+}
+
+# Database files to preserve
+DATABASE_FILES = {"cache.db", "cache.db-shm", "cache.db-wal"}
+
+# Files that should be excluded from cleanup regardless of date
+EXCLUDE_PATTERNS = [
+    ".git",
+    ".gitignore",
+]
+
+
+def parse_date(date_str: str) -> datetime:
+    """Parse date string in YYYY-MM-DD format.
+
+    Args:
+        date_str: Date string in YYYY-MM-DD format
+
+    Returns:
+        datetime object with the specified date at midnight
+
+    Raises:
+        ValueError: If date format is invalid
+    """
+    try:
+        return datetime.strptime(date_str, "%Y-%m-%d").replace(tzinfo=timezone.utc)
+    except ValueError as e:
+        raise ValueError(
+            f"Invalid date format: '{date_str}'. Use YYYY-MM-DD format."
+        ) from e
+
+
+def should_preserve(path: Path) -> bool:
+    """Check if a file/directory should be preserved.
+
+    Args:
+        path: Path to check
+
+    Returns:
+        True if the path should be preserved, False otherwise
+    """
+    # Check if it's in the preserve list
+    if path.name in PRESERVE_LIST:
+        return True
+
+    # Check if it matches any exclude patterns
+    for pattern in EXCLUDE_PATTERNS:
+        if pattern in str(path):
+            return True
+
+    return False
+
+
+def should_preserve_archival_file(path: Path) -> bool:
+    """Check if a file in archival_data should be preserved.
+
+    Args:
+        path: Path to check
+
+    Returns:
+        True if the path should be preserved, False otherwise
+    """
+    # Always preserve database files
+    if path.name in DATABASE_FILES:
+        return True
+
+    return should_preserve(path)
+
+
+def get_files_older_than_date(
+    directory: Path, cutoff_date: datetime
+) -> List[Tuple[Path, datetime]]:
+    """Get all files older than the cutoff date.
+
+    Args:
+        directory: Directory to search
+        cutoff_date: Files older than this date will be selected
+
+    Returns:
+        List of tuples (path, modification_time) for files older than cutoff
+    """
+    old_files = []
+
+    # Walk through all files in directory recursively (targeted patterns only)
+    for pattern in ("*.html", "*.json", "*.txt", "*.xml", "*.md"):
+        for item in directory.rglob(pattern):
+            if not item.is_file():
+                continue
+            if should_preserve(item):
+                continue
+
+            try:
+                mtime = datetime.fromtimestamp(item.stat().st_mtime, tz=timezone.utc)
+                if mtime < cutoff_date:
+                    old_files.append((item, mtime))
+            except (OSError, ValueError) as e:
+                logger.warning(f"Could not access file {item}: {e}")
+
+    return old_files
+
+
+def get_files_older_than_date_non_recursive(
+    directory: Path, cutoff_date: datetime
+) -> List[Tuple[Path, datetime]]:
+    """Get all files older than the cutoff date (non-recursive).
+
+    Args:
+        directory: Directory to search
+        cutoff_date: Files older than this date will be selected
+
+    Returns:
+        List of tuples (path, modification_time) for files older than cutoff
+    """
+    old_files = []
+
+    # Walk through all files in directory (non-recursive for safety)
+    for item in directory.iterdir():
+        if item.is_file():
+            if should_preserve(item):
+                continue
+
+            try:
+                mtime = datetime.fromtimestamp(item.stat().st_mtime, tz=timezone.utc)
+                if mtime < cutoff_date:
+                    old_files.append((item, mtime))
+            except (OSError, ValueError) as e:
+                logger.warning(f"Could not access file {item}: {e}")
+
+    return old_files
+
+
+def delete_files(files: List[Tuple[Path, datetime]]) -> Tuple[int, int]:
+    """Delete files and return count of successful/failed deletions.
+
+    Args:
+        files: List of (path, modification_time) tuples to delete
+
+    Returns:
+        Tuple of (deleted_count, failed_count)
+    """
+    deleted = 0
+    failed = 0
+
+    for path, mtime in files:
+        try:
+            path.unlink()
+            logger.info(
+                f"Deleted: {path.name} (modified: {mtime.strftime('%Y-%m-%d %H:%M:%S')})"
+            )
+            deleted += 1
+        except OSError as e:
+            logger.error(f"Failed to delete {path.name}: {e}")
+            failed += 1
+
+    return deleted, failed
+
+
+def main():
+    """Main entry point for cleanup script."""
+    parser = argparse.ArgumentParser(
+        description="Cleanup old files from NewsArchiver project",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  %(prog)s --date "2024-03-19" --dry-run
+  %(prog)s --date "2024-03-19" --force
+  %(prog)s --date "2024-03-19" --archival --force
+        """,
+    )
+
+    parser.add_argument(
+        "--date",
+        "-d",
+        type=str,
+        required=True,
+        help="Date in YYYY-MM-DD format - files older than this will be deleted",
+    )
+
+    parser.add_argument(
+        "--dry-run",
+        "-n",
+        action="store_true",
+        default=True,
+        help="Show what would be deleted without actually deleting (default)",
+    )
+
+    parser.add_argument(
+        "--force",
+        "-f",
+        action="store_true",
+        help="Actually delete files (disables dry-run mode)",
+    )
+
+    parser.add_argument(
+        "--verbose", "-v", action="store_true", help="Enable verbose output"
+    )
+
+    parser.add_argument(
+        "--archival",
+        action="store_true",
+        help="Include archival_data folder for cleanup",
+    )
+
+    args = parser.parse_args()
+
+    # Set logging level
+    if args.verbose:
+        logger.setLevel(logging.DEBUG)
+
+    # Parse the date
+    try:
+        cutoff_date = parse_date(args.date)
+    except ValueError as e:
+        logger.error(str(e))
+        sys.exit(1)
+
+    # Validate cutoff date is not in the future
+    now = datetime.now(timezone.utc)
+    if cutoff_date > now:
+        logger.error("Cutoff date cannot be in the future")
+        sys.exit(1)
+
+    logger.info("=" * 60)
+    logger.info("NewsArchiver Cleanup Script")
+    logger.info("=" * 60)
+
+    # Determine mode
+    dry_run = not args.force
+    mode = "DRY RUN" if dry_run else "ACTUAL DELETE"
+    logger.info("Mode: %s", mode)
+    logger.info(
+        "Cutoff Date: %s (files older than this will be %s)",
+        cutoff_date.strftime("%Y-%m-%d"),
+        "kept" if dry_run else "deleted",
+    )
+    logger.info("=" * 60)
+
+    # Find files older than cutoff date
+    if args.archival:
+        # Only scan websites folder when --archival is used
+        if not WEBSITES_DIR.exists():
+            logger.error("Websites folder not found at %s", WEBSITES_DIR)
+            sys.exit(1)
+        old_files = []
+        logger.info(
+            "Scanning websites folder for files older than %s...",
+            cutoff_date.strftime("%Y-%m-%d"),
+        )
+        # First check if there are any files directly in websites folder
+        website_root_files = get_files_older_than_date_non_recursive(
+            WEBSITES_DIR, cutoff_date
+        )
+        # Then check recursively in subdirectories
+        website_recursive_files = []
+        for subdir in WEBSITES_DIR.iterdir():
+            if subdir.is_dir():
+                website_recursive_files.extend(
+                    get_files_older_than_date(subdir, cutoff_date)
+                )
+        old_files.extend(website_root_files)
+        old_files.extend(website_recursive_files)
+        logger.info(
+            "Found %d files in websites folder",
+            len(website_root_files) + len(website_recursive_files),
+        )
+    else:
+        # Scan root directory (non-archival mode)
+        old_files = get_files_older_than_date_non_recursive(SCRIPT_DIR, cutoff_date)
+
+    if not old_files:
+        logger.info("No files older than %s found.", cutoff_date.strftime("%Y-%m-%d"))
+        logger.info("Nothing to do.")
+        return
+
+    logger.info(
+        "Found %d files older than %s:",
+        len(old_files),
+        cutoff_date.strftime("%Y-%m-%d"),
+    )
+
+    # List all files that would be affected
+    for path, mtime in old_files:
+        # Get relative path for cleaner output
+        rel_path = path.relative_to(SCRIPT_DIR)
+        logger.info(
+            "  - %s (modified: %s)", rel_path, mtime.strftime("%Y-%m-%d %H:%M:%S")
+        )
+
+    logger.info("=" * 60)
+
+    # Execute deletion if not dry run
+    if dry_run:
+        logger.info("DRY RUN: No files were deleted.")
+        logger.info("Run with --force to actually delete these files.")
+    else:
+        deleted, failed = delete_files(old_files)
+        logger.info("=" * 60)
+        logger.info("Cleanup complete!")
+        logger.info("Deleted: %d files", deleted)
+        logger.info("Failed: %d files", failed)
+
+
+if __name__ == "__main__":
+    main()

content_extractor.py

@@ -0,0 +1,387 @@
+#!/usr/bin/env python3
+"""Content Extractor for NewsArchiver - Phase 2.2
+
+Extracts article text and metadata from HTML using Trafilatura
+with BeautifulSoup fallback for JavaScript-heavy sites.
+"""
+
+import logging
+import sys
+import time
+from dataclasses import dataclass
+from datetime import datetime
+from pathlib import Path
+from typing import Optional
+
+try:
+    import requests
+except ImportError:
+    print("WARNING: requests not installed. URL fetching may not work.")
+    print("Install with: pip install requests")
+    requests = None
+
+try:
+    import trafilatura
+    from trafilatura import extract, extract_metadata
+except ImportError:
+    print("ERROR: trafilatura is required. Install with: pip install trafilatura")
+    sys.exit(1)
+
+try:
+    from bs4 import BeautifulSoup
+except ImportError:
+    print("WARNING: beautifulsoup4 not installed. Some fallback features may not work.")
+    print("Install with: pip install beautifulsoup4")
+    BeautifulSoup = None
+
+try:
+    from playwright.sync_api import sync_playwright
+    PLAYWRIGHT_AVAILABLE = True
+except ImportError:
+    PLAYWRIGHT_AVAILABLE = False
+
+SCRIPT_DIR = Path(__file__).parent
+ARCHIVE_DIR = SCRIPT_DIR / 'archival_data'
+ARCHIVE_DIR.mkdir(exist_ok=True)
+
+_last_request_time = 0.0
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class ArticleData:
+    """Structured article data."""
+    url: str
+    title: Optional[str] = None
+    author: Optional[str] = None
+    publish_date: Optional[str] = None
+    content_text: Optional[str] = None
+    content_html: Optional[str] = None
+    raw_html: Optional[str] = None
+    archive_file_path: Optional[str] = None
+    tags: Optional[list] = None
+    language: Optional[str] = None
+    metadata: Optional[dict] = None
+    extraction_method: Optional[str] = None
+    error: Optional[str] = None
+    guid: Optional[str] = None
+    id: Optional[int] = None
+    source_name: Optional[str] = None
+
+
+def fetch_url(url: str, timeout: int = 30) -> str:
+    """Download HTML from URL with rate limiting.
+    
+    Args:
+        url: URL to fetch
+        timeout: Request timeout in seconds
+        
+    Returns:
+        HTML string or empty string on failure
+    """
+    global _last_request_time
+    
+    if requests is None:
+        logger.error("requests library not available")
+        return ""
+    
+    try:
+        delay = 0.5 - (time.time() - _last_request_time)
+        if delay > 0:
+            time.sleep(delay)
+        
+        _last_request_time = time.time()
+        
+        headers = {
+            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
+        }
+        response = requests.get(url, timeout=timeout, headers=headers)
+        response.raise_for_status()
+        
+        text = response.text
+        
+        # Check if response is an error page (only check for actual HTTP errors)
+        if response.status_code >= 400:
+            logger.warning("Server returned error for %s (status: %d)", url, response.status_code)
+            return ""
+        
+        # Check for common error patterns
+        if 'access denied' in text.lower() or 'forbidden' in text.lower():
+            logger.warning("Server returned access denied for %s", url)
+            return ""
+        
+        return text
+        
+    except Exception as e:
+        logger.error("Failed to fetch URL %s: %s", url, str(e))
+        return ""
+
+
+def extract_content(html: str) -> dict:
+    """Extract article content and metadata from HTML using Trafilatura.
+    
+    Args:
+        html: Raw HTML string (or plain text)
+        
+    Returns:
+        Dictionary with extracted content and metadata
+    """
+    result = {
+        'success': False,
+        'content_text': None,
+        'content_html': None,
+        'title': None,
+        'author': None,
+        'date': None,
+        'tags': None,
+        'language': None,
+        'metadata': None,
+        'extraction_method': None,
+        'error': None
+    }
+    
+    if not html or not html.strip():
+        result['error'] = 'Empty HTML content'
+        return result
+    
+    # Check if this is an error page (HTTP status codes, not the word "error" in content)
+    # Only check for error patterns that appear in actual error pages (not in article content)
+    # Look for specific error page patterns with proper HTML structure
+    import re
+    error_patterns = [
+        r'<title[^>]*>403[^<]*Forbidden</title>',
+        r'<title[^>]*>401[^<]*Unauthorized</title>',
+        r'<h1[^>]*>403</h1>',
+        r'<h1[^>]*>401</h1>',
+    ]
+    
+    html_lower = html.lower()
+    for pattern in error_patterns:
+        if re.search(pattern, html_lower):
+            result['error'] = 'HTML contains error page content'
+            result['success'] = False
+            return result
+    
+    # Check for common error indicators in the HTML body
+    # These should only trigger if we see them in context (like a 403/401 status indicator)
+    if '<title>403' in html_lower or '<title>401' in html_lower:
+        result['error'] = 'HTML contains error page content'
+        result['success'] = False
+        return result
+    
+    has_html_tags = '<' in html and '>' in html
+    
+    try:
+        metadata = extract_metadata(html)
+        
+        if metadata:
+            result['title'] = metadata.title
+            result['author'] = metadata.author
+            result['date'] = metadata.date
+            result['tags'] = metadata.tags if hasattr(metadata, 'tags') else None
+            result['language'] = metadata.language
+            
+            if hasattr(metadata, 'to_dict'):
+                result['metadata'] = metadata.to_dict()
+            elif hasattr(metadata, '__dict__'):
+                result['metadata'] = metadata.__dict__
+        
+        content = extract(
+            html,
+            include_comments=False,
+            include_tables=True,
+            no_fallback=True
+        )
+        
+        if content:
+            result['content_text'] = content
+            result['content_html'] = html
+            result['extraction_method'] = 'trafilatura'
+            result['success'] = True
+        elif not has_html_tags:
+            result['content_text'] = html.strip()
+            result['content_html'] = f'<html><body>{html}</body></html>'
+            result['extraction_method'] = 'plain_text'
+            result['success'] = True
+        else:
+            result['content_text'] = None
+            result['content_html'] = html
+            result['extraction_method'] = 'trafilatura_empty'
+            result['success'] = False
+            result['error'] = 'Content extraction failed - no article content found'
+        
+        if result['success']:
+            logger.debug("Content extracted using %s", result['extraction_method'])
+        
+    except Exception as e:
+        result['error'] = f"Trafilatura extraction failed: {str(e)}"
+        logger.warning(result['error'])
+        
+        if not has_html_tags:
+            result['content_text'] = html.strip()
+            result['content_html'] = f'<html><body>{html}</body></html>'
+            result['extraction_method'] = 'plain_text_fallback'
+            result['success'] = True
+        elif BeautifulSoup:
+            fallback_result = _extract_with_beautifulsoup(html)
+            if fallback_result['content_text']:
+                result.update(fallback_result)
+                result['extraction_method'] = 'beautifulsoup_fallback'
+    
+    return result
+
+
+def _extract_with_beautifulsoup(html: str) -> dict:
+    """Fallback extraction using BeautifulSoup."""
+    result = {
+        'content_text': None,
+        'content_html': None,
+        'title': None,
+        'author': None,
+        'date': None,
+        'tags': None,
+        'language': None,
+        'metadata': None
+    }
+    
+    try:
+        soup = BeautifulSoup(html, 'html.parser')
+        
+        title = soup.find('title')
+        if title:
+            result['title'] = title.get_text(strip=True)
+        
+        meta_author = soup.find('meta', attrs={'name': 'author'})
+        if meta_author:
+            result['author'] = meta_author.get('content', '').strip()
+        
+        meta_date = soup.find('meta', attrs={'name': 'date'})
+        if meta_date:
+            result['date'] = meta_date.get('content', '').strip()
+        
+        meta_language = soup.find('meta', attrs={'name': 'language'})
+        if meta_language:
+            result['language'] = meta_language.get('content', '').strip()
+        
+        for tag in ['article', 'main', 'div']:
+            content_tags = soup.find_all(tag)
+            if content_tags:
+                result['content_text'] = ' '.join(
+                    tag.get_text(strip=True, separator=' ') 
+                    for tag in content_tags
+                )
+                if result['content_text']:
+                    break
+        
+        if not result['content_text']:
+            result['content_text'] = soup.get_text(strip=True, separator=' ')
+        
+        result['content_text'] = result['content_text'][:100000]
+        result['content_html'] = str(soup)
+        
+    except Exception as e:
+        result['error'] = f"BeautifulSoup extraction failed: {str(e)}"
+        logger.warning(result['error'])
+    
+    return result
+
+
+def _extract_with_playwright(url: str) -> Optional[str]:
+    """Extract HTML from JavaScript-heavy site using Playwright."""
+    if not PLAYWRIGHT_AVAILABLE:
+        logger.warning("Playwright not available. Install with: pip install playwright")
+        return None
+    
+    try:
+        with sync_playwright() as p:
+            browser = p.chromium.launch(headless=True)
+            page = browser.new_page()
+            
+            page.goto(url, wait_until='networkidle', timeout=60000)
+            
+            page_content = page.content()
+            
+            browser.close()
+            
+            logger.debug("HTML extracted using Playwright")
+            return page_content
+            
+    except Exception as e:
+        logger.error("Playwright extraction failed: %s", str(e))
+        return None
+
+
+def parse_article_from_html(html: str, url: str) -> ArticleData:
+    """Parse article and return structured data.
+    
+    Args:
+        html: Raw HTML string
+        url: Original URL for reference
+        
+    Returns:
+        ArticleData with extracted content and metadata
+    """
+    extraction_result = extract_content(html)
+    
+    guid = extraction_result.get('metadata', {}).get('url', url) if extraction_result.get('metadata') else url
+    article = ArticleData(
+        url=url,
+        title=extraction_result.get('title'),
+        author=extraction_result.get('author'),
+        publish_date=extraction_result.get('date'),
+        content_text=extraction_result.get('content_text'),
+        content_html=extraction_result.get('content_html'),
+        raw_html=html,
+        tags=extraction_result.get('tags'),
+        language=extraction_result.get('language'),
+        metadata=extraction_result.get('metadata'),
+        extraction_method=extraction_result.get('extraction_method'),
+        guid=guid
+    )
+    
+    if not extraction_result.get('success'):
+        article.error = extraction_result.get('error')
+    
+    return article
+
+
+def get_html_from_url(url: str, extract_content: bool = True) -> str:
+    """Download HTML from URL using Trafilatura with Playwright fallback.
+    
+    Args:
+        url: URL to fetch
+        extract_content: If True, try to extract main content (kept for backwards compatibility)
+        
+    Returns:
+        Full HTML string or empty string on failure
+    """
+    try:
+        downloaded = fetch_url(url)
+        
+        if downloaded:
+            logger.debug("Full HTML fetched from %s", url[:60])
+            return downloaded
+            
+    except Exception as e:
+        logger.error("Failed to fetch URL %s: %s", url, str(e))
+    
+    if PLAYWRIGHT_AVAILABLE:
+        logger.info("Trying Playwright fallback for %s", url[:60])
+        playwright_html = _extract_with_playwright(url)
+        if playwright_html:
+            logger.debug("Full HTML from Playwright for %s", url[:60])
+            if extract_content:
+                extracted = extract(
+                    playwright_html,
+                    include_comments=False,
+                    include_tables=True,
+                    no_fallback=True
+                )
+                if extracted:
+                    logger.debug("Playwright content extracted from %s", url[:60])
+                    return extracted
+            logger.debug("Full HTML from Playwright for %s", url[:60])
+            return playwright_html
+    
+    return ""

docker-compose.nas.example.yml

@@ -0,0 +1,37 @@
+# Example docker-compose configuration for NAS storage
+# Copy this file to docker-compose.yml and edit the volume path
+
+version: '3.8'
+
+services:
+  newsarchiver:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    container_name: newsarchiver
+    ports:
+      - "5000:5000"
+    environment:
+      - ARCHIVE_DIR=/data/archives
+    volumes:
+      # Example: Mount your NAS to /path/to/nas/archives
+      # Replace with your actual NAS path
+      - /path/to/nas/archives:/data/archives
+      # Or use Docker named volume for local storage:
+      # - newsarchiver_data:/data/archives
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:5000/"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+    # Optional: Run as specific UID/GID for NAS permissions
+    # user: "1000:1000"
+
+volumes:
+  newsarchiver_data:
+    driver: local
+    driver_opts:
+      type: none
+      o: bind
+      device: /path/to/nas/archives  # Replace with your NAS path

docker-compose.yml

@@ -0,0 +1,24 @@
+name: newsarchiver
+
+services:
+  newsarchiver:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    container_name: newsarchiver
+    ports:
+      - "5000:5000"
+    environment:
+      - ARCHIVE_DIR=/data/archives
+    volumes:
+      - newsarchiver_data:/data/archives
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:5000/"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+
+volumes:
+  newsarchiver_data:
+    external: true

docker_backup.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+# Docker backup script for NewsArchiver
+# This script backs up archived data from the Docker volume to a local or NAS location
+
+set -e
+
+BACKUP_DIR="${BACKUP_DIR:-./backups}"
+TIMESTAMP=$(date +%Y%m%d_%H%M%S)
+BACKUP_NAME="newsarchiver_backup_${TIMESTAMP}.tar.gz"
+
+echo "Starting backup..."
+echo "Backup location: ${BACKUP_DIR}/${BACKUP_NAME}"
+
+# Create backup directory
+mkdir -p "$BACKUP_DIR"
+
+# Create backup from the Docker volume
+docker run --rm \
+  -v newsarchiver_data:/data:ro \
+  -v "${BACKUP_DIR}:/backup" \
+  alpine tar -czf "/backup/${BACKUP_NAME}" -C /data .
+
+echo "Backup complete: ${BACKUP_DIR}/${BACKUP_NAME}"
+echo ""
+echo "To restore from backup:"
+echo "  1. Stop the container: docker-compose down"
+echo "  2. Remove the volume: docker volume rm newsarchiver_data"
+echo "  3. Create a new volume: docker volume create newsarchiver_data"
+echo "  4. Restore: docker run --rm -v newsarchiver_data:/data -v \${BACKUP_DIR}:/backup alpine tar -xzf /backup/${BACKUP_NAME} -C /data"
+echo "  5. Start: docker-compose up -d"

docker_setup.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+
+# Docker setup script for NewsArchiver
+# This script helps configure the Docker environment
+
+set -e
+
+echo "NewsArchiver Docker Setup"
+echo "=========================="
+echo ""
+
+# Check if Docker is installed
+if ! command -v docker &> /dev/null; then
+    echo "ERROR: Docker is not installed. Please install Docker first."
+    exit 1
+fi
+
+# Check if docker-compose is available
+if ! command -v docker-compose &> /dev/null && ! docker compose version &> /dev/null; then
+    echo "ERROR: docker-compose is not installed. Please install docker-compose first."
+    exit 1
+fi
+
+echo "Docker is installed."
+echo ""
+
+# Check if the project has docker-compose.yml
+if [ ! -f docker-compose.yml ] && [ ! -f docker-compose.nas.example.yml ]; then
+    echo "WARNING: docker-compose.yml not found."
+    echo "Creating from example..."
+    cp docker-compose.nas.example.yml docker-compose.yml
+    echo ""
+    echo "Please edit docker-compose.yml to set your NAS path:"
+    echo "  1. Find the volume mount path (currently set to /path/to/nas/archives)"
+    echo "  2. Replace with your actual NAS path"
+    echo "  3. Save the file"
+    echo ""
+fi
+
+echo "Setup complete!"
+echo ""
+echo "Next steps:"
+echo "  1. Edit docker-compose.yml with your NAS path"
+echo "  2. Build and start: docker-compose up -d --build"
+echo "  3. Check logs: docker-compose logs -f"
+echo "  4. Access web interface at http://localhost:5000"
+echo ""

entrypoint.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+set -e
+
+# Use ARCHIVE_DIR from environment, default to /app/archival_data
+export ARCHIVE_DIR="${ARCHIVE_DIR:-/app/archival_data}"
+
+# Create archive directory if it does not exist
+mkdir -p "$ARCHIVE_DIR"
+
+# Run the command passed to docker
+exec "$@"

nohup.out

@@ -0,0 +1,251 @@
+2026-03-31 00:45:43,896 - INFO - NewsArchiver - Main CLI Entry Point
+2026-03-31 00:45:43,896 - INFO - ============================================================
+2026-03-31 00:45:43,896 - INFO - Starting web server
+2026-03-31 00:45:43,896 - INFO - ============================================================
+2026-03-31 00:45:43,984 - INFO - Web server starting on 0.0.0.0:8080
+2026-03-31 00:45:43,984 - INFO - ============================================================
+ * Serving Flask app 'web_interface'
+ * Debug mode: off
+2026-03-31 00:45:43,985 - INFO - WARNING: This is a development server. Do not use it in a production deployment. Use a production WSGI server instead.
+ * Running on all addresses (0.0.0.0)
+ * Running on http://127.0.0.1:8080
+ * Running on http://192.168.8.150:8080
+2026-03-31 00:45:43,985 - INFO - Press CTRL+C to quit
+2026-03-31 00:46:57,096 - INFO - NewsArchiver - Main CLI Entry Point
+2026-03-31 00:46:57,096 - INFO - ============================================================
+2026-03-31 00:46:57,096 - INFO - Starting web server
+2026-03-31 00:46:57,096 - INFO - ============================================================
+2026-03-31 00:46:57,185 - INFO - Web server starting on 0.0.0.0:5000
+2026-03-31 00:46:57,185 - INFO - ============================================================
+ * Serving Flask app 'web_interface'
+ * Debug mode: off
+2026-03-31 00:46:57,186 - INFO - WARNING: This is a development server. Do not use it in a production deployment. Use a production WSGI server instead.
+ * Running on all addresses (0.0.0.0)
+ * Running on http://127.0.0.1:5000
+ * Running on http://192.168.8.150:5000
+2026-03-31 00:46:57,186 - INFO - Press CTRL+C to quit
+2026-03-31 00:47:29,377 - INFO - 192.168.8.110 - - [31/Mar/2026 00:47:29] "GET / HTTP/1.1" 200 -
+2026-03-31 00:47:29,420 - INFO - 192.168.8.110 - - [31/Mar/2026 00:47:29] "GET /static/style.css HTTP/1.1" 200 -
+2026-03-31 00:47:29,476 - INFO - 192.168.8.110 - - [31/Mar/2026 00:47:29] "GET /favicon.ico HTTP/1.1" 404 -
+Warning: OPENCODE_SERVER_PASSWORD is not set; server is unsecured.
+opencode server listening on http://0.0.0.0:4096
+2026-03-31 03:50:30,668 - INFO - 192.168.8.156 - - [31/Mar/2026 03:50:30] "GET / HTTP/1.1" 200 -
+2026-03-31 03:50:30,701 - INFO - 192.168.8.156 - - [31/Mar/2026 03:50:30] "GET /static/style.css HTTP/1.1" 200 -
+2026-03-31 03:50:34,449 - INFO - 192.168.8.156 - - [31/Mar/2026 03:50:34] "GET /source/404%20Media HTTP/1.1" 200 -
+2026-03-31 03:50:34,449 - INFO - 192.168.8.156 - - [31/Mar/2026 03:50:34] "GET /source/404%20Media HTTP/1.1" 200 -
+2026-03-31 03:50:34,490 - INFO - 192.168.8.156 - - [31/Mar/2026 03:50:34] "GET /static/style.css HTTP/1.1" 304 -
+2026-03-31 03:51:06,671 - INFO - 192.168.8.156 - - [31/Mar/2026 03:51:06] "GET /source/404%20media/article/75854 HTTP/1.1" 200 -
+2026-03-31 03:51:06,801 - INFO - 192.168.8.156 - - [31/Mar/2026 03:51:06] "GET /static/style.css HTTP/1.1" 304 -
+2026-03-31 03:51:28,312 - INFO - 192.168.8.156 - - [31/Mar/2026 03:51:28] "GET /source/404%20Media HTTP/1.1" 200 -
+2026-03-31 03:51:30,665 - INFO - 192.168.8.156 - - [31/Mar/2026 03:51:30] "GET /source/404%20media/article/75856 HTTP/1.1" 200 -
+2026-03-31 03:51:30,686 - INFO - 192.168.8.156 - - [31/Mar/2026 03:51:30] "GET /source/404%20media/article/75856 HTTP/1.1" 200 -
+2026-03-31 03:51:30,777 - INFO - 192.168.8.156 - - [31/Mar/2026 03:51:30] "GET /static/style.css HTTP/1.1" 304 -
+2026-03-31 03:51:52,473 - INFO - 192.168.8.156 - - [31/Mar/2026 03:51:52] "GET /source/404%20Media HTTP/1.1" 200 -
+2026-03-31 03:52:03,379 - INFO - 192.168.8.156 - - [31/Mar/2026 03:52:03] "GET / HTTP/1.1" 200 -
+2026-03-31 03:53:23,713 - INFO - 192.168.8.156 - - [31/Mar/2026 03:53:23] "GET / HTTP/1.1" 200 -
+2026-03-31 03:53:26,077 - INFO - 192.168.8.156 - - [31/Mar/2026 03:53:26] "GET /source/Ars%20Technica HTTP/1.1" 200 -
+2026-03-31 03:53:26,273 - INFO - 192.168.8.156 - - [31/Mar/2026 03:53:26] "GET /static/style.css HTTP/1.1" 304 -
+2026-03-31 03:53:32,574 - INFO - 192.168.8.156 - - [31/Mar/2026 03:53:32] "GET /source/ars%20technica/article/76159 HTTP/1.1" 200 -
+2026-03-31 03:53:32,621 - INFO - 192.168.8.156 - - [31/Mar/2026 03:53:32] "GET /source/ars%20technica/article/76159 HTTP/1.1" 200 -
+2026-03-31 03:53:32,714 - INFO - 192.168.8.156 - - [31/Mar/2026 03:53:32] "GET /static/style.css HTTP/1.1" 304 -
+2026-03-31 03:54:32,335 - INFO - 192.168.8.156 - - [31/Mar/2026 03:54:32] "GET /source/Ars%20Technica HTTP/1.1" 200 -
+2026-03-31 03:54:36,121 - INFO - 192.168.8.156 - - [31/Mar/2026 03:54:36] "GET /source/ars%20technica/article/76156 HTTP/1.1" 200 -
+2026-03-31 03:54:36,226 - INFO - 192.168.8.156 - - [31/Mar/2026 03:54:36] "GET /static/style.css HTTP/1.1" 304 -
+2026-03-31 03:55:37,969 - INFO - 192.168.8.156 - - [31/Mar/2026 03:55:37] "GET /source/Ars%20Technica HTTP/1.1" 200 -
+2026-03-31 03:55:42,609 - INFO - 192.168.8.156 - - [31/Mar/2026 03:55:42] "GET /source/ars%20technica/article/76153 HTTP/1.1" 200 -
+2026-03-31 03:55:42,688 - INFO - 192.168.8.156 - - [31/Mar/2026 03:55:42] "GET /static/style.css HTTP/1.1" 304 -
+2026-03-31 03:56:57,070 - INFO - 192.168.8.156 - - [31/Mar/2026 03:56:57] "GET /source/Ars%20Technica HTTP/1.1" 200 -
+2026-03-31 03:57:27,037 - INFO - 192.168.8.156 - - [31/Mar/2026 03:57:27] "GET / HTTP/1.1" 200 -
+2026-03-31 03:57:31,938 - INFO - 192.168.8.156 - - [31/Mar/2026 03:57:31] "GET /source/Associated%20Press HTTP/1.1" 200 -
+2026-03-31 03:57:31,977 - INFO - 192.168.8.156 - - [31/Mar/2026 03:57:31] "GET /static/style.css HTTP/1.1" 304 -
+2026-03-31 03:57:46,792 - INFO - 192.168.8.156 - - [31/Mar/2026 03:57:46] "GET /source/associated%20press/article/76318 HTTP/1.1" 200 -
+2026-03-31 03:57:46,822 - INFO - 192.168.8.156 - - [31/Mar/2026 03:57:46] "GET /source/associated%20press/article/76318 HTTP/1.1" 200 -
+2026-03-31 03:57:46,918 - INFO - 192.168.8.156 - - [31/Mar/2026 03:57:46] "GET /static/style.css HTTP/1.1" 304 -
+2026-03-31 03:58:43,823 - INFO - 192.168.8.156 - - [31/Mar/2026 03:58:43] "GET /source/Associated%20Press HTTP/1.1" 200 -
+2026-03-31 03:58:45,452 - INFO - 192.168.8.156 - - [31/Mar/2026 03:58:45] "GET / HTTP/1.1" 200 -
+2026-03-31 03:58:51,007 - INFO - 192.168.8.156 - - [31/Mar/2026 03:58:51] "GET /source/Associated%20Press HTTP/1.1" 200 -
+2026-03-31 03:58:51,035 - INFO - 192.168.8.156 - - [31/Mar/2026 03:58:51] "GET /source/Associated%20Press HTTP/1.1" 200 -
+2026-03-31 03:58:51,073 - INFO - 192.168.8.156 - - [31/Mar/2026 03:58:51] "GET /static/style.css HTTP/1.1" 304 -
+2026-03-31 03:59:07,720 - INFO - 192.168.8.156 - - [31/Mar/2026 03:59:07] "GET /source/associated%20press/article/76180 HTTP/1.1" 200 -
+2026-03-31 03:59:07,791 - INFO - 192.168.8.156 - - [31/Mar/2026 03:59:07] "GET /static/style.css HTTP/1.1" 304 -
+2026-03-31 04:02:00,289 - INFO - 192.168.8.156 - - [31/Mar/2026 04:02:00] "GET /source/Associated%20Press HTTP/1.1" 200 -
+2026-03-31 04:02:29,322 - INFO - 192.168.8.156 - - [31/Mar/2026 04:02:29] "GET /source/associated%20press/article/76381 HTTP/1.1" 200 -
+2026-03-31 04:02:29,366 - INFO - 192.168.8.156 - - [31/Mar/2026 04:02:29] "GET /source/associated%20press/article/76381 HTTP/1.1" 200 -
+2026-03-31 04:02:29,461 - INFO - 192.168.8.156 - - [31/Mar/2026 04:02:29] "GET /static/style.css HTTP/1.1" 304 -
+2026-03-31 04:05:14,639 - INFO - 192.168.8.156 - - [31/Mar/2026 04:05:14] "GET /source/Associated%20Press HTTP/1.1" 200 -
+2026-03-31 04:07:09,728 - INFO - 192.168.8.156 - - [31/Mar/2026 04:07:09] "GET / HTTP/1.1" 200 -
+2026-03-31 04:07:13,436 - INFO - 192.168.8.156 - - [31/Mar/2026 04:07:13] "GET /source/BBC%20News%20–%20Business HTTP/1.1" 200 -
+2026-03-31 04:07:13,460 - INFO - 192.168.8.156 - - [31/Mar/2026 04:07:13] "GET /static/style.css HTTP/1.1" 304 -
+2026-03-31 04:07:18,974 - INFO - 192.168.8.156 - - [31/Mar/2026 04:07:18] "GET /source/bbc%20news%20–%20business/article/75670 HTTP/1.1" 200 -
+2026-03-31 04:07:18,994 - INFO - 192.168.8.156 - - [31/Mar/2026 04:07:18] "GET /source/bbc%20news%20–%20business/article/75670 HTTP/1.1" 200 -
+2026-03-31 04:07:19,090 - INFO - 192.168.8.156 - - [31/Mar/2026 04:07:19] "GET /static/style.css HTTP/1.1" 304 -
+2026-03-31 04:08:32,701 - INFO - 192.168.8.156 - - [31/Mar/2026 04:08:32] "GET /source/BBC%20News%20–%20Business HTTP/1.1" 200 -
+2026-03-31 04:08:38,686 - INFO - 192.168.8.156 - - [31/Mar/2026 04:08:38] "GET /source/bbc%20news%20–%20business/article/75660 HTTP/1.1" 200 -
+2026-03-31 04:08:38,758 - INFO - 192.168.8.156 - - [31/Mar/2026 04:08:38] "GET /static/style.css HTTP/1.1" 304 -
+2026-03-31 04:10:39,733 - INFO - 192.168.8.156 - - [31/Mar/2026 04:10:39] "GET /source/BBC%20News%20–%20Business HTTP/1.1" 200 -
+2026-03-31 04:10:47,517 - INFO - 192.168.8.156 - - [31/Mar/2026 04:10:47] "GET /source/bbc%20news%20–%20business/article/75668 HTTP/1.1" 200 -
+2026-03-31 04:10:47,592 - INFO - 192.168.8.156 - - [31/Mar/2026 04:10:47] "GET /static/style.css HTTP/1.1" 304 -
+2026-03-31 04:13:00,033 - INFO - 192.168.8.156 - - [31/Mar/2026 04:13:00] "GET /source/BBC%20News%20–%20Business HTTP/1.1" 200 -
+2026-03-31 04:13:04,210 - INFO - 192.168.8.156 - - [31/Mar/2026 04:13:04] "GET /source/bbc%20news%20–%20business/article/75664 HTTP/1.1" 200 -
+2026-03-31 04:13:04,305 - INFO - 192.168.8.156 - - [31/Mar/2026 04:13:04] "GET /static/style.css HTTP/1.1" 304 -
+2026-03-31 04:14:34,463 - INFO - 192.168.8.156 - - [31/Mar/2026 04:14:34] "GET /source/BBC%20News%20–%20Business HTTP/1.1" 200 -
+2026-03-31 04:15:35,483 - INFO - 192.168.8.156 - - [31/Mar/2026 04:15:35] "GET / HTTP/1.1" 200 -
+2026-03-31 04:15:42,939 - INFO - 192.168.8.156 - - [31/Mar/2026 04:15:42] "GET /source/CNBC%20–%20Business HTTP/1.1" 200 -
+2026-03-31 04:15:42,999 - INFO - 192.168.8.156 - - [31/Mar/2026 04:15:42] "GET /source/CNBC%20–%20Business HTTP/1.1" 200 -
+2026-03-31 04:15:43,017 - INFO - 192.168.8.156 - - [31/Mar/2026 04:15:43] "GET /static/style.css HTTP/1.1" 304 -
+2026-03-31 04:16:17,658 - INFO - 192.168.8.156 - - [31/Mar/2026 04:16:17] "GET /source/cnbc%20–%20business/article/75575 HTTP/1.1" 200 -
+2026-03-31 04:16:17,754 - INFO - 192.168.8.156 - - [31/Mar/2026 04:16:17] "GET /static/style.css HTTP/1.1" 304 -
+2026-03-31 04:17:16,331 - INFO - 192.168.8.156 - - [31/Mar/2026 04:17:16] "GET /source/CNBC%20–%20Business HTTP/1.1" 200 -
+2026-03-31 04:17:34,558 - INFO - 192.168.8.156 - - [31/Mar/2026 04:17:34] "GET /source/cnbc%20–%20business/article/75583 HTTP/1.1" 200 -
+2026-03-31 04:17:34,633 - INFO - 192.168.8.156 - - [31/Mar/2026 04:17:34] "GET /static/style.css HTTP/1.1" 304 -
+2026-03-31 04:19:53,532 - INFO - 192.168.8.156 - - [31/Mar/2026 04:19:53] "GET /source/CNBC%20–%20Business HTTP/1.1" 200 -
+2026-03-31 04:20:00,608 - INFO - 192.168.8.156 - - [31/Mar/2026 04:20:00] "GET / HTTP/1.1" 200 -
+2026-03-31 04:20:15,359 - INFO - 192.168.8.156 - - [31/Mar/2026 04:20:15] "GET /source/Engadget HTTP/1.1" 200 -
+2026-03-31 04:20:15,402 - INFO - 192.168.8.156 - - [31/Mar/2026 04:20:15] "GET /static/style.css HTTP/1.1" 304 -
+2026-03-31 04:20:37,074 - INFO - 192.168.8.156 - - [31/Mar/2026 04:20:37] "GET /source/engadget/article/76118 HTTP/1.1" 200 -
+2026-03-31 04:20:37,105 - INFO - 192.168.8.156 - - [31/Mar/2026 04:20:37] "GET /source/engadget/article/76118 HTTP/1.1" 200 -
+2026-03-31 04:20:37,202 - INFO - 192.168.8.156 - - [31/Mar/2026 04:20:37] "GET /static/style.css HTTP/1.1" 304 -
+2026-03-31 04:21:11,132 - INFO - 192.168.8.156 - - [31/Mar/2026 04:21:11] "GET /source/Engadget HTTP/1.1" 200 -
+2026-03-31 04:21:18,145 - INFO - 192.168.8.156 - - [31/Mar/2026 04:21:18] "GET /source/engadget/article/76116 HTTP/1.1" 200 -
+2026-03-31 04:21:18,216 - INFO - 192.168.8.156 - - [31/Mar/2026 04:21:18] "GET /static/style.css HTTP/1.1" 304 -
+2026-03-31 04:22:17,699 - INFO - 192.168.8.156 - - [31/Mar/2026 04:22:17] "GET /source/Engadget HTTP/1.1" 200 -
+2026-03-31 04:22:38,033 - INFO - 192.168.8.156 - - [31/Mar/2026 04:22:38] "GET /source/engadget/article/76107 HTTP/1.1" 200 -
+2026-03-31 04:22:38,059 - INFO - 192.168.8.156 - - [31/Mar/2026 04:22:38] "GET /source/engadget/article/76107 HTTP/1.1" 200 -
+2026-03-31 04:22:38,151 - INFO - 192.168.8.156 - - [31/Mar/2026 04:22:38] "GET /static/style.css HTTP/1.1" 304 -
+2026-03-31 04:23:11,496 - INFO - 192.168.8.156 - - [31/Mar/2026 04:23:11] "GET /source/Engadget HTTP/1.1" 200 -
+2026-03-31 04:24:04,059 - INFO - 192.168.8.156 - - [31/Mar/2026 04:24:04] "GET / HTTP/1.1" 200 -
+2026-03-31 04:24:20,270 - INFO - 192.168.8.156 - - [31/Mar/2026 04:24:20] "GET /source/Hacker%20News HTTP/1.1" 200 -
+2026-03-31 04:24:20,304 - INFO - 192.168.8.156 - - [31/Mar/2026 04:24:20] "GET /static/style.css HTTP/1.1" 304 -
+2026-03-31 04:24:20,394 - INFO - 192.168.8.156 - - [31/Mar/2026 04:24:20] "GET /source/Hacker%20News HTTP/1.1" 200 -
+2026-03-31 04:24:55,634 - INFO - 192.168.8.156 - - [31/Mar/2026 04:24:55] "GET /source/hacker%20news/article/76044 HTTP/1.1" 200 -
+2026-03-31 04:24:55,731 - INFO - 192.168.8.156 - - [31/Mar/2026 04:24:55] "GET /static/style.css HTTP/1.1" 304 -
+2026-03-31 04:25:39,898 - INFO - 192.168.8.156 - - [31/Mar/2026 04:25:39] "GET /source/Hacker%20News HTTP/1.1" 200 -
+2026-03-31 04:25:57,410 - INFO - 192.168.8.156 - - [31/Mar/2026 04:25:57] "GET /source/hacker%20news/article/75265 HTTP/1.1" 200 -
+2026-03-31 04:25:57,443 - INFO - 192.168.8.156 - - [31/Mar/2026 04:25:57] "GET /source/hacker%20news/article/75265 HTTP/1.1" 200 -
+2026-03-31 04:25:57,537 - INFO - 192.168.8.156 - - [31/Mar/2026 04:25:57] "GET /static/style.css HTTP/1.1" 304 -
+2026-03-31 04:28:52,915 - INFO - 192.168.8.156 - - [31/Mar/2026 04:28:52] "GET /source/Hacker%20News HTTP/1.1" 200 -
+2026-03-31 04:29:54,036 - INFO - 192.168.8.156 - - [31/Mar/2026 04:29:54] "GET /source/hacker%20news/article/75237 HTTP/1.1" 200 -
+2026-03-31 04:29:54,091 - INFO - 192.168.8.156 - - [31/Mar/2026 04:29:54] "GET /static/style.css HTTP/1.1" 304 -
+2026-03-31 04:31:33,988 - INFO - 192.168.8.156 - - [31/Mar/2026 04:31:33] "GET /source/Hacker%20News HTTP/1.1" 200 -
+2026-03-31 04:32:13,621 - INFO - 192.168.8.156 - - [31/Mar/2026 04:32:13] "GET /source/hacker%20news/article/75972 HTTP/1.1" 200 -
+2026-03-31 04:32:13,653 - INFO - 192.168.8.156 - - [31/Mar/2026 04:32:13] "GET /source/hacker%20news/article/75972 HTTP/1.1" 200 -
+2026-03-31 04:32:13,754 - INFO - 192.168.8.156 - - [31/Mar/2026 04:32:13] "GET /static/style.css HTTP/1.1" 304 -
+2026-03-31 04:39:50,815 - INFO - 192.168.8.156 - - [31/Mar/2026 04:39:50] "GET /source/Hacker%20News HTTP/1.1" 200 -
+2026-03-31 04:41:06,097 - INFO - 192.168.8.156 - - [31/Mar/2026 04:41:06] "GET /source/hacker%20news/article/75255 HTTP/1.1" 200 -
+2026-03-31 04:41:06,130 - INFO - 192.168.8.156 - - [31/Mar/2026 04:41:06] "GET /source/hacker%20news/article/75255 HTTP/1.1" 200 -
+2026-03-31 04:41:06,224 - INFO - 192.168.8.156 - - [31/Mar/2026 04:41:06] "GET /static/style.css HTTP/1.1" 304 -
+2026-03-31 04:42:02,455 - INFO - 192.168.8.156 - - [31/Mar/2026 04:42:02] "GET /source/Hacker%20News HTTP/1.1" 200 -
+2026-03-31 04:47:45,853 - INFO - 192.168.8.226 - - [31/Mar/2026 04:47:45] "GET / HTTP/1.1" 200 -
+2026-03-31 04:47:45,949 - INFO - 192.168.8.226 - - [31/Mar/2026 04:47:45] "GET /static/style.css HTTP/1.1" 304 -
+2026-03-31 04:47:46,000 - INFO - 192.168.8.226 - - [31/Mar/2026 04:47:46] "GET /favicon.ico HTTP/1.1" 404 -
+2026-03-31 05:53:55,185 - INFO - 192.168.8.226 - - [31/Mar/2026 05:53:55] "GET /rss HTTP/1.1" 200 -
+2026-03-31 07:54:24,903 - INFO - 192.168.8.226 - - [31/Mar/2026 07:54:24] "GET /rss HTTP/1.1" 200 -
+2026-03-31 11:25:27,600 - INFO - 192.168.8.156 - - [31/Mar/2026 11:25:27] "GET /source/hacker%20news/article/74862 HTTP/1.1" 200 -
+2026-03-31 11:25:27,641 - INFO - 192.168.8.156 - - [31/Mar/2026 11:25:27] "GET /static/style.css HTTP/1.1" 304 -
+2026-03-31 11:27:05,669 - INFO - 192.168.8.156 - - [31/Mar/2026 11:27:05] "GET /source/Hacker%20News HTTP/1.1" 200 -
+2026-03-31 11:29:15,410 - INFO - 192.168.8.156 - - [31/Mar/2026 11:29:15] "GET /source/hacker%20news/article/76601 HTTP/1.1" 200 -
+2026-03-31 11:29:15,439 - INFO - 192.168.8.156 - - [31/Mar/2026 11:29:15] "GET /source/hacker%20news/article/76601 HTTP/1.1" 200 -
+2026-03-31 11:29:15,537 - INFO - 192.168.8.156 - - [31/Mar/2026 11:29:15] "GET /static/style.css HTTP/1.1" 304 -
+2026-03-31 11:30:37,869 - INFO - 192.168.8.156 - - [31/Mar/2026 11:30:37] "GET /source/Hacker%20News HTTP/1.1" 200 -
+2026-03-31 11:30:46,321 - INFO - 192.168.8.156 - - [31/Mar/2026 11:30:46] "GET /source/hacker%20news/article/76602 HTTP/1.1" 200 -
+2026-03-31 11:30:46,418 - INFO - 192.168.8.156 - - [31/Mar/2026 11:30:46] "GET /static/style.css HTTP/1.1" 304 -
+2026-03-31 11:36:40,535 - INFO - 192.168.8.156 - - [31/Mar/2026 11:36:40] "GET / HTTP/1.1" 200 -
+2026-03-31 11:36:40,569 - INFO - 192.168.8.156 - - [31/Mar/2026 11:36:40] "GET /static/style.css HTTP/1.1" 200 -
+2026-03-31 11:36:44,152 - INFO - 192.168.8.156 - - [31/Mar/2026 11:36:44] "GET /source/404%20Media HTTP/1.1" 200 -
+2026-03-31 11:36:44,167 - INFO - 192.168.8.156 - - [31/Mar/2026 11:36:44] "GET /static/style.css HTTP/1.1" 304 -
+2026-03-31 11:36:49,524 - INFO - 192.168.8.156 - - [31/Mar/2026 11:36:49] "GET / HTTP/1.1" 200 -
+2026-03-31 11:36:51,785 - INFO - 192.168.8.156 - - [31/Mar/2026 11:36:51] "GET /source/Ars%20Technica HTTP/1.1" 200 -
+2026-03-31 11:36:51,803 - INFO - 192.168.8.156 - - [31/Mar/2026 11:36:51] "GET /static/style.css HTTP/1.1" 304 -
+2026-03-31 11:37:01,762 - INFO - 192.168.8.156 - - [31/Mar/2026 11:37:01] "GET / HTTP/1.1" 200 -
+2026-03-31 11:37:07,388 - INFO - 192.168.8.156 - - [31/Mar/2026 11:37:07] "GET /source/Associated%20Press HTTP/1.1" 200 -
+2026-03-31 11:37:07,413 - INFO - 192.168.8.156 - - [31/Mar/2026 11:37:07] "GET /static/style.css HTTP/1.1" 304 -
+2026-03-31 11:37:07,652 - INFO - 192.168.8.156 - - [31/Mar/2026 11:37:07] "GET /source/associated%20press/article/76682 HTTP/1.1" 200 -
+2026-03-31 11:37:07,720 - INFO - 192.168.8.156 - - [31/Mar/2026 11:37:07] "GET /static/style.css HTTP/1.1" 304 -
+2026-03-31 11:37:16,777 - INFO - 192.168.8.156 - - [31/Mar/2026 11:37:16] "GET /source/Associated%20Press HTTP/1.1" 200 -
+2026-03-31 11:38:13,418 - INFO - 192.168.8.156 - - [31/Mar/2026 11:38:13] "GET /source/associated%20press/article/76643 HTTP/1.1" 200 -
+2026-03-31 11:38:13,453 - INFO - 192.168.8.156 - - [31/Mar/2026 11:38:13] "GET /source/associated%20press/article/76643 HTTP/1.1" 200 -
+2026-03-31 11:38:13,537 - INFO - 192.168.8.156 - - [31/Mar/2026 11:38:13] "GET /static/style.css HTTP/1.1" 304 -
+2026-03-31 11:39:22,876 - INFO - 192.168.8.156 - - [31/Mar/2026 11:39:22] "GET /source/Associated%20Press HTTP/1.1" 200 -
+2026-03-31 11:39:47,385 - INFO - 192.168.8.156 - - [31/Mar/2026 11:39:47] "GET /source/associated%20press/article/76607 HTTP/1.1" 200 -
+2026-03-31 11:39:47,419 - INFO - 192.168.8.156 - - [31/Mar/2026 11:39:47] "GET /source/associated%20press/article/76607 HTTP/1.1" 200 -
+2026-03-31 11:39:47,501 - INFO - 192.168.8.156 - - [31/Mar/2026 11:39:47] "GET /static/style.css HTTP/1.1" 304 -
+2026-03-31 11:41:43,881 - INFO - 192.168.8.156 - - [31/Mar/2026 11:41:43] "GET /source/Associated%20Press HTTP/1.1" 200 -
+2026-03-31 11:42:06,103 - INFO - 192.168.8.156 - - [31/Mar/2026 11:42:06] "GET /source/associated%20press/article/76563 HTTP/1.1" 200 -
+2026-03-31 11:42:06,136 - INFO - 192.168.8.156 - - [31/Mar/2026 11:42:06] "GET /source/associated%20press/article/76563 HTTP/1.1" 200 -
+2026-03-31 11:42:06,218 - INFO - 192.168.8.156 - - [31/Mar/2026 11:42:06] "GET /static/style.css HTTP/1.1" 304 -
+2026-03-31 11:45:00,824 - INFO - 192.168.8.156 - - [31/Mar/2026 11:45:00] "GET /source/Associated%20Press HTTP/1.1" 200 -
+2026-03-31 11:45:09,188 - INFO - 192.168.8.156 - - [31/Mar/2026 11:45:09] "GET /source/associated%20press/article/76605 HTTP/1.1" 200 -
+2026-03-31 11:45:09,223 - INFO - 192.168.8.156 - - [31/Mar/2026 11:45:09] "GET /source/associated%20press/article/76605 HTTP/1.1" 200 -
+2026-03-31 11:45:09,305 - INFO - 192.168.8.156 - - [31/Mar/2026 11:45:09] "GET /static/style.css HTTP/1.1" 304 -
+2026-03-31 11:48:11,110 - INFO - 192.168.8.156 - - [31/Mar/2026 11:48:11] "GET /source/Associated%20Press HTTP/1.1" 200 -
+2026-03-31 11:48:12,152 - INFO - 192.168.8.156 - - [31/Mar/2026 11:48:12] "GET / HTTP/1.1" 200 -
+2026-03-31 11:48:16,690 - INFO - 192.168.8.156 - - [31/Mar/2026 11:48:16] "GET /source/BBC%20News%20–%20Business HTTP/1.1" 200 -
+2026-03-31 11:48:16,713 - INFO - 192.168.8.156 - - [31/Mar/2026 11:48:16] "GET /source/BBC%20News%20–%20Business HTTP/1.1" 200 -
+2026-03-31 11:48:16,724 - INFO - 192.168.8.156 - - [31/Mar/2026 11:48:16] "GET /static/style.css HTTP/1.1" 304 -
+2026-03-31 11:48:26,509 - INFO - 192.168.8.156 - - [31/Mar/2026 11:48:26] "GET /source/bbc%20news%20–%20business/article/76588 HTTP/1.1" 200 -
+2026-03-31 11:48:26,592 - INFO - 192.168.8.156 - - [31/Mar/2026 11:48:26] "GET /static/style.css HTTP/1.1" 304 -
+2026-03-31 11:50:33,771 - INFO - 192.168.8.156 - - [31/Mar/2026 11:50:33] "GET /source/BBC%20News%20–%20Business HTTP/1.1" 200 -
+2026-03-31 11:51:20,160 - INFO - 192.168.8.156 - - [31/Mar/2026 11:51:20] "GET / HTTP/1.1" 200 -
+2026-03-31 11:51:54,395 - INFO - 192.168.8.156 - - [31/Mar/2026 11:51:54] "GET /source/Mac%20Rumors HTTP/1.1" 200 -
+2026-03-31 11:51:54,422 - INFO - 192.168.8.156 - - [31/Mar/2026 11:51:54] "GET /static/style.css HTTP/1.1" 304 -
+2026-03-31 11:51:59,100 - INFO - 192.168.8.156 - - [31/Mar/2026 11:51:59] "GET /source/mac%20rumors/article/75894 HTTP/1.1" 200 -
+2026-03-31 11:51:59,196 - INFO - 192.168.8.156 - - [31/Mar/2026 11:51:59] "GET /static/style.css HTTP/1.1" 304 -
+2026-03-31 11:52:42,557 - INFO - 192.168.8.156 - - [31/Mar/2026 11:52:42] "GET /source/Mac%20Rumors HTTP/1.1" 200 -
+2026-03-31 11:52:48,061 - INFO - 192.168.8.156 - - [31/Mar/2026 11:52:48] "GET /source/mac%20rumors/article/75890 HTTP/1.1" 200 -
+2026-03-31 11:52:48,089 - INFO - 192.168.8.156 - - [31/Mar/2026 11:52:48] "GET /source/mac%20rumors/article/75890 HTTP/1.1" 200 -
+2026-03-31 11:52:48,176 - INFO - 192.168.8.156 - - [31/Mar/2026 11:52:48] "GET /static/style.css HTTP/1.1" 304 -
+2026-03-31 11:53:19,210 - INFO - 192.168.8.156 - - [31/Mar/2026 11:53:19] "GET /source/Mac%20Rumors HTTP/1.1" 200 -
+2026-03-31 11:53:43,451 - INFO - 192.168.8.156 - - [31/Mar/2026 11:53:43] "GET /source/mac%20rumors/article/75886 HTTP/1.1" 200 -
+2026-03-31 11:53:43,487 - INFO - 192.168.8.156 - - [31/Mar/2026 11:53:43] "GET /source/mac%20rumors/article/75886 HTTP/1.1" 200 -
+2026-03-31 11:53:43,570 - INFO - 192.168.8.156 - - [31/Mar/2026 11:53:43] "GET /static/style.css HTTP/1.1" 304 -
+2026-03-31 11:54:20,607 - INFO - 192.168.8.156 - - [31/Mar/2026 11:54:20] "GET /source/Mac%20Rumors HTTP/1.1" 200 -
+2026-03-31 11:55:28,281 - INFO - 192.168.8.156 - - [31/Mar/2026 11:55:28] "GET /source/mac%20rumors/article/75896 HTTP/1.1" 200 -
+2026-03-31 11:55:28,311 - INFO - 192.168.8.156 - - [31/Mar/2026 11:55:28] "GET /source/mac%20rumors/article/75896 HTTP/1.1" 200 -
+2026-03-31 11:55:28,396 - INFO - 192.168.8.156 - - [31/Mar/2026 11:55:28] "GET /static/style.css HTTP/1.1" 304 -
+2026-03-31 11:55:52,802 - INFO - 192.168.8.156 - - [31/Mar/2026 11:55:52] "GET /source/Mac%20Rumors HTTP/1.1" 200 -
+2026-03-31 11:56:08,058 - INFO - 192.168.8.156 - - [31/Mar/2026 11:56:08] "GET / HTTP/1.1" 200 -
+2026-03-31 11:56:31,521 - INFO - 192.168.8.156 - - [31/Mar/2026 11:56:31] "GET /source/ProPublica HTTP/1.1" 200 -
+2026-03-31 11:56:31,524 - INFO - 192.168.8.156 - - [31/Mar/2026 11:56:31] "GET /source/ProPublica HTTP/1.1" 200 -
+2026-03-31 11:56:31,543 - INFO - 192.168.8.156 - - [31/Mar/2026 11:56:31] "GET /static/style.css HTTP/1.1" 304 -
+2026-03-31 11:56:44,566 - INFO - 192.168.8.156 - - [31/Mar/2026 11:56:44] "GET /source/propublica/article/76487 HTTP/1.1" 200 -
+2026-03-31 11:56:44,655 - INFO - 192.168.8.156 - - [31/Mar/2026 11:56:44] "GET /static/style.css HTTP/1.1" 304 -
+2026-03-31 12:07:00,226 - INFO - 192.168.8.156 - - [31/Mar/2026 12:07:00] "GET /source/ProPublica HTTP/1.1" 200 -
+2026-03-31 12:08:20,255 - INFO - 192.168.8.156 - - [31/Mar/2026 12:08:20] "GET /source/propublica/article/61757 HTTP/1.1" 200 -
+2026-03-31 12:08:20,285 - INFO - 192.168.8.156 - - [31/Mar/2026 12:08:20] "GET /source/propublica/article/61757 HTTP/1.1" 200 -
+2026-03-31 12:08:20,371 - INFO - 192.168.8.156 - - [31/Mar/2026 12:08:20] "GET /static/style.css HTTP/1.1" 304 -
+2026-03-31 12:16:03,370 - INFO - 192.168.8.226 - - [31/Mar/2026 12:16:03] "GET / HTTP/1.1" 200 -
+2026-03-31 12:16:03,475 - INFO - 192.168.8.226 - - [31/Mar/2026 12:16:03] "GET /static/style.css HTTP/1.1" 304 -
+2026-03-31 13:05:25,475 - INFO - 192.168.8.226 - - [31/Mar/2026 13:05:25] "GET /rss HTTP/1.1" 200 -
+2026-03-31 14:38:42,845 - INFO - 192.168.8.110 - - [31/Mar/2026 14:38:42] "GET /source/404%20Media HTTP/1.1" 200 -
+2026-03-31 14:38:42,872 - INFO - 192.168.8.110 - - [31/Mar/2026 14:38:42] "GET /static/style.css HTTP/1.1" 200 -
+2026-03-31 14:38:42,883 - INFO - 192.168.8.110 - - [31/Mar/2026 14:38:42] "GET /favicon.ico HTTP/1.1" 404 -
+2026-03-31 14:38:44,029 - INFO - 192.168.8.110 - - [31/Mar/2026 14:38:44] "GET /source/404%20media/article/76744 HTTP/1.1" 200 -
+2026-03-31 14:38:44,043 - INFO - 192.168.8.110 - - [31/Mar/2026 14:38:44] "GET /static/style.css HTTP/1.1" 304 -
+2026-03-31 14:40:43,351 - INFO - 192.168.8.110 - - [31/Mar/2026 14:40:43] "GET /archive-file//home/user/playground/NewsArchiver/archival_data/websites/404%20Media/html/2026-03-31/article_17749630811.html HTTP/1.1" 308 -
+2026-03-31 14:40:43,358 - INFO - Archive file path: /home/user/playground/NewsArchiver/archival_data/home/user/playground/NewsArchiver/archival_data/websites/404 Media/html/2026-03-31/article_17749630811.html, exists: False
+2026-03-31 14:40:43,358 - INFO - 192.168.8.110 - - [31/Mar/2026 14:40:43] "GET /archive-file/home/user/playground/NewsArchiver/archival_data/websites/404%20Media/html/2026-03-31/article_17749630811.html HTTP/1.1" 404 -
+2026-03-31 14:40:43,374 - INFO - 192.168.8.110 - - [31/Mar/2026 14:40:43] "GET /favicon.ico HTTP/1.1" 404 -
+2026-03-31 14:51:00,206 - INFO - 192.168.8.110 - - [31/Mar/2026 14:51:00] "GET /source/404%20media/article/76744 HTTP/1.1" 200 -
+2026-03-31 14:51:00,220 - INFO - 192.168.8.110 - - [31/Mar/2026 14:51:00] "GET /static/style.css HTTP/1.1" 200 -
+2026-03-31 14:51:00,225 - INFO - 192.168.8.110 - - [31/Mar/2026 14:51:00] "GET /favicon.ico HTTP/1.1" 404 -
+2026-03-31 14:51:02,123 - INFO - 192.168.8.110 - - [31/Mar/2026 14:51:02] "GET /archive-file//home/user/playground/NewsArchiver/archival_data/websites/404%20Media/html/2026-03-31/article_17749630811.html HTTP/1.1" 308 -
+2026-03-31 14:51:02,127 - INFO - Archive file path: /home/user/playground/NewsArchiver/archival_data/home/user/playground/NewsArchiver/archival_data/websites/404 Media/html/2026-03-31/article_17749630811.html, exists: False
+2026-03-31 14:51:02,127 - INFO - 192.168.8.110 - - [31/Mar/2026 14:51:02] "GET /archive-file/home/user/playground/NewsArchiver/archival_data/websites/404%20Media/html/2026-03-31/article_17749630811.html HTTP/1.1" 404 -
+2026-03-31 14:51:02,142 - INFO - 192.168.8.110 - - [31/Mar/2026 14:51:02] "GET /favicon.ico HTTP/1.1" 404 -
+2026-03-31 15:14:49,505 - INFO - 192.168.8.110 - - [31/Mar/2026 15:14:49] "GET /source/404%20media/article/76744 HTTP/1.1" 200 -
+2026-03-31 15:14:49,519 - INFO - 192.168.8.110 - - [31/Mar/2026 15:14:49] "GET /static/style.css HTTP/1.1" 200 -
+2026-03-31 15:14:49,526 - INFO - 192.168.8.110 - - [31/Mar/2026 15:14:49] "GET /favicon.ico HTTP/1.1" 404 -
+2026-03-31 15:15:08,057 - INFO - 192.168.8.110 - - [31/Mar/2026 15:15:08] "GET / HTTP/1.1" 200 -
+2026-03-31 15:15:08,076 - INFO - 192.168.8.110 - - [31/Mar/2026 15:15:08] "GET /static/style.css HTTP/1.1" 200 -
+2026-03-31 15:15:08,080 - INFO - 192.168.8.110 - - [31/Mar/2026 15:15:08] "GET /favicon.ico HTTP/1.1" 404 -
+2026-03-31 15:15:10,181 - INFO - 192.168.8.110 - - [31/Mar/2026 15:15:10] "GET /source/404%20Media HTTP/1.1" 200 -
+2026-03-31 15:15:10,197 - INFO - 192.168.8.110 - - [31/Mar/2026 15:15:10] "GET /static/style.css HTTP/1.1" 200 -
+2026-03-31 15:15:10,207 - INFO - 192.168.8.110 - - [31/Mar/2026 15:15:10] "GET /favicon.ico HTTP/1.1" 404 -
+2026-03-31 15:15:11,421 - INFO - 192.168.8.110 - - [31/Mar/2026 15:15:11] "GET /source/404%20media/article/76744 HTTP/1.1" 200 -
+2026-03-31 15:15:11,436 - INFO - 192.168.8.110 - - [31/Mar/2026 15:15:11] "GET /static/style.css HTTP/1.1" 304 -
+2026-03-31 15:15:13,782 - INFO - 192.168.8.110 - - [31/Mar/2026 15:15:13] "GET /archive-file//home/user/playground/NewsArchiver/archival_data/websites/404%20Media/html/2026-03-31/article_17749630811.html HTTP/1.1" 308 -
+2026-03-31 15:15:13,786 - INFO - Archive file path: /home/user/playground/NewsArchiver/archival_data/home/user/playground/NewsArchiver/archival_data/websites/404 Media/html/2026-03-31/article_17749630811.html, exists: False
+2026-03-31 15:15:13,786 - INFO - 192.168.8.110 - - [31/Mar/2026 15:15:13] "GET /archive-file/home/user/playground/NewsArchiver/archival_data/websites/404%20Media/html/2026-03-31/article_17749630811.html HTTP/1.1" 404 -
+2026-03-31 15:15:21,503 - INFO - 192.168.8.110 - - [31/Mar/2026 15:15:21] "GET / HTTP/1.1" 200 -
+2026-03-31 15:15:21,524 - INFO - 192.168.8.110 - - [31/Mar/2026 15:15:21] "GET /static/style.css HTTP/1.1" 200 -
+2026-03-31 15:15:21,530 - INFO - 192.168.8.110 - - [31/Mar/2026 15:15:21] "GET /favicon.ico HTTP/1.1" 404 -

rebuild_database.py

@@ -0,0 +1,247 @@
+#!/usr/bin/env python3
+"""Rebuild NewsArchiver database from existing HTML files."""
+
+import json
+import logging
+import re
+import sqlite3
+import sys
+from datetime import datetime
+from pathlib import Path
+from typing import Optional
+
+try:
+    from content_extractor import parse_article_from_html, get_html_from_url
+    from storage_manager import initialize_storage, save_article as storage_save_article
+except ImportError as e:
+    print(f"ERROR: Required module not found: {e}")
+    sys.exit(1)
+
+SCRIPT_DIR = Path(__file__).parent
+ARCHIVE_DIR = SCRIPT_DIR / 'archival_data'
+RSS_FEEDS_PATH = SCRIPT_DIR / 'rss_feeds.json'
+
+logger = logging.getLogger(__name__)
+
+
+def extract_urls_from_html(html_file: Path) -> list:
+    """Extract article URLs from an archived HTML file.
+    
+    Args:
+        html_file: Path to archived HTML file
+        
+    Returns:
+        List of article URLs found in the file
+    """
+    try:
+        content = html_file.read_text(encoding='utf-8')
+        urls = []
+        
+        # Only extract main article URLs - skip tracking links, RSS feeds, author pages, etc.
+        article_patterns = [
+            r'href=["\']https?://[^"\']+\/article\/[^"\']+["\']',
+            r'href=["\']https?://[^"\']+\/news\/[^"\']+["\']',
+            r'href=["\']https?://[^"\']+\/stories\/[^"\']+["\']',
+            r'href=["\']https?://[^"\']+\/archive\/[^"\']+["\']',
+        ]
+        
+        for pattern in article_patterns:
+            matches = re.findall(pattern, content)
+            for match in matches:
+                url_match = re.search(r'href=["\']([^"\']+)["\']', match)
+                if url_match:
+                    url = url_match.group(1)
+                    # Skip common non-article URLs
+                    skip_patterns = [
+                        'rss', 'feed', 'author', 'authors', 'tags', 'category', 'search',
+                        'about', 'contact', 'privacy', 'terms', 'faq', 'subscribe',
+                        'signin', 'login', 'register', 'account', 'profile'
+                    ]
+                    if not any(skip in url.lower() for skip in skip_patterns):
+                        urls.append(url)
+        
+        return list(set(urls))  # Remove duplicates
+        
+    except Exception as e:
+        logger.error("Error reading %s: %s", html_file, str(e))
+        return []
+
+
+def find_html_files(archive_dir: Path) -> list:
+    """Find all HTML files in the archive directory.
+    
+    Args:
+        archive_dir: Root archive directory
+        
+    Returns:
+        List of HTML file paths
+    """
+    html_files = []
+    
+    try:
+        websites_dir = archive_dir / 'websites'
+        if not websites_dir.exists():
+            logger.warning("Websites directory not found: %s", websites_dir)
+            return []
+        
+        for source_dir in sorted(websites_dir.iterdir()):
+            if not source_dir.is_dir():
+                continue
+            
+            html_dir = source_dir / 'html'
+            if not html_dir.exists():
+                continue
+            
+            for date_dir in sorted(html_dir.iterdir()):
+                if not date_dir.is_dir():
+                    continue
+                
+                for html_file in sorted(date_dir.glob('article_*.html')):
+                    html_files.append(html_file)
+        
+        logger.info("Found %d HTML files to process", len(html_files))
+        
+    except Exception as e:
+        logger.error("Error finding HTML files: %s", str(e))
+    
+    return html_files
+
+
+def rebuild_database(archive_dir: Path, rss_feeds_path: Path) -> dict:
+    """Rebuild the database from existing HTML files.
+    
+    Args:
+        archive_dir: Root archive directory
+        rss_feeds_path: Path to RSS feeds JSON file
+        
+    Returns:
+        Dictionary with rebuild results
+    """
+    results = {
+        'html_files_processed': 0,
+        'articles_extracted': 0,
+        'articles_saved': 0,
+        'articles_failed': 0,
+        'errors': []
+    }
+    
+    # Initialize database
+    logger.info("Initializing database at: %s", ARCHIVE_DIR / 'cache.db')
+    initialize_storage()
+    logger.info("Database initialized")
+    
+    # Load RSS feeds
+    rss_feeds = {}
+    if rss_feeds_path.exists():
+        with open(rss_feeds_path, 'r', encoding='utf-8') as f:
+            rss_feeds = json.load(f)
+    
+    # Find all HTML files
+    html_files = find_html_files(archive_dir)
+    
+    if not html_files:
+        logger.warning("No HTML files found to process")
+        return results
+    
+    # Process each HTML file
+    for html_file in html_files:
+        try:
+            results['html_files_processed'] += 1
+            
+            # Extract URLs from HTML
+            urls = extract_urls_from_html(html_file)
+            
+            if not urls:
+                logger.debug("No URLs found in %s", html_file.name)
+                continue
+            
+            results['articles_extracted'] += len(urls)
+            
+            # Process each URL
+            for url in urls:
+                try:
+                    # Try to get source name from RSS feeds
+                    source_name = None
+                    for feed_name, feed_info in rss_feeds.items():
+                        if feed_info.get('rss_url') and url.startswith(feed_info.get('rss_url', '')):
+                            source_name = feed_name
+                            break
+                    
+                    # Try to infer source from URL
+                    if not source_name:
+                        for feed_name, feed_info in rss_feeds.items():
+                            website = feed_info.get('source_website', '')
+                            if website and website in url:
+                                source_name = feed_name
+                                break
+                    
+                    if not source_name:
+                        # Try to extract domain from URL
+                        domain_match = re.search(r'https?://([^/]+)', url)
+                        if domain_match:
+                            domain = domain_match.group(1)
+                            for feed_name, feed_info in rss_feeds.items():
+                                website = feed_info.get('source_website', '')
+                                if website and website in domain:
+                                    source_name = feed_name
+                                    break
+                    
+                    # If still no source, skip
+                    if not source_name:
+                        logger.debug("No source found for %s, skipping", url[:60])
+                        continue
+                    
+                    # Extract content from HTML file
+                    raw_html = html_file.read_text(encoding='utf-8')
+                    article_data = parse_article_from_html(raw_html, url)
+                    
+                    if article_data.error:
+                        logger.warning("Failed to extract content from %s: %s", url[:60], article_data.error)
+                        results['articles_failed'] += 1
+                        continue
+                    
+                    # Save to database
+                    logger.info("Saving article: %s -> %s", url[:80], source_name)
+                    storage_save_article(source_name, article_data)
+                    results['articles_saved'] += 1
+                    
+                    if results['articles_saved'] % 100 == 0:
+                        logger.info("Saved %d articles so far", results['articles_saved'])
+                    
+                except Exception as e:
+                    logger.error("Error processing URL %s: %s", url, str(e))
+                    results['articles_failed'] += 1
+                    results['errors'].append({
+                        'url': url,
+                        'error': str(e)
+                    })
+            
+        except Exception as e:
+            logger.error("Error processing file %s: %s", html_file, str(e))
+            results['errors'].append({
+                'file': str(html_file),
+                'error': str(e)
+            })
+    
+    return results
+
+
+if __name__ == '__main__':
+    logger.info("=" * 60)
+    logger.info("Rebuilding NewsArchiver Database")
+    logger.info("=" * 60)
+    
+    results = rebuild_database(ARCHIVE_DIR, RSS_FEEDS_PATH)
+    
+    logger.info("=" * 60)
+    logger.info("Rebuild Complete")
+    logger.info("=" * 60)
+    logger.info("HTML files processed: %d", results['html_files_processed'])
+    logger.info("Articles extracted: %d", results['articles_extracted'])
+    logger.info("Articles saved: %d", results['articles_saved'])
+    logger.info("Articles failed: %d", results['articles_failed'])
+    
+    if results['errors']:
+        logger.info("Errors:")
+        for error in results['errors'][:20]:
+            logger.info("  - %s", error)

requirements.txt

@@ -0,0 +1,7 @@
+flask>=3.0,<4.0
+requests>=2.31,<3.0
+trafilatura>=1.6,<2.0
+feedparser>=6.0,<7.0
+apscheduler>=3.10,<4.0
+beautifulsoup4>=4.12,<5.0
+playwright>=1.40,<2.0

restore_database.py

@@ -0,0 +1,104 @@
+#!/usr/bin/env python3
+"""Restore database from existing JSON metadata files."""
+
+import json
+import logging
+import sys
+from pathlib import Path
+
+try:
+    from storage_manager import initialize_storage, save_article
+    from content_extractor import ArticleData
+except ImportError as e:
+    print(f"ERROR: Required module not found: {e}")
+    sys.exit(1)
+
+SCRIPT_DIR = Path(__file__).parent
+ARCHIVE_DIR = SCRIPT_DIR / 'archival_data'
+
+logger = logging.getLogger(__name__)
+
+
+def restore_database(archive_dir: Path) -> dict:
+    """Restore database from JSON metadata files."""
+    results = {
+        'json_files_found': 0,
+        'articles_restored': 0,
+        'articles_failed': 0,
+        'errors': []
+    }
+    
+    initialize_storage()
+    logger.info("Database initialized")
+    
+    json_files = list(archive_dir.glob('websites/**/*.json'))
+    results['json_files_found'] = len(json_files)
+    
+    logger.info(f"Found {len(json_files)} JSON files to process")
+    
+    for json_file in json_files:
+        try:
+            with open(json_file, 'r', encoding='utf-8') as f:
+                metadata = json.load(f)
+            
+            url = metadata.get('url')
+            source_name = metadata.get('source_name')
+            title = metadata.get('title')
+            author = metadata.get('author')
+            publish_date = metadata.get('publish_date')
+            content_text = metadata.get('content_text')
+            content_html = metadata.get('content_html')
+            tags = metadata.get('tags', [])
+            extraction_method = metadata.get('extraction_method', 'unknown')
+            
+            if not url or not source_name:
+                logger.warning(f"Missing URL or source in {json_file.name}, skipping")
+                results['articles_failed'] += 1
+                continue
+            
+            article_data = ArticleData(
+                url=url,
+                title=title,
+                author=author,
+                publish_date=publish_date,
+                content_text=content_text,
+                content_html=content_html,
+                tags=tags,
+                extraction_method=extraction_method
+            )
+            
+            save_article(source_name, article_data)
+            results['articles_restored'] += 1
+            
+            if results['articles_restored'] % 100 == 0:
+                logger.info(f"Restored {results['articles_restored']} articles so far")
+            
+        except Exception as e:
+            logger.error(f"Error processing {json_file.name}: {str(e)}")
+            results['articles_failed'] += 1
+            results['errors'].append({
+                'file': str(json_file),
+                'error': str(e)
+            })
+    
+    return results
+
+
+if __name__ == '__main__':
+    logger.info("=" * 60)
+    logger.info("Restoring NewsArchiver Database from JSON files")
+    logger.info("=" * 60)
+    
+    results = restore_database(ARCHIVE_DIR)
+    
+    logger.info("=" * 60)
+    logger.info("Restore Complete")
+    logger.info("=" * 60)
+    logger.info(f"JSON files found: {results['json_files_found']}")
+    logger.info(f"Articles restored: {results['articles_restored']}")
+    logger.info(f"Articles failed: {results['articles_failed']}")
+    
+    if results['errors']:
+        logger.info("Errors:")
+        for error in results['errors'][:20]:
+            logger.info(f"  - {error}")

rss_feeds.json

@@ -0,0 +1,306 @@
+{
+  "Financial Times": {
+    "source_website": "ft.com",
+    "rss_url": "https://www.ft.com/rss/home",
+    "entries": 12,
+    "validated_at": "2026-03-18T20:21:46.292164"
+  },
+  "Reuters – Business News": {
+    "source_website": "reuters.com",
+    "rss_url": "https://news.google.com/rss/search?q=site:reuters.com+business&hl=en-US&gl=US&ceid=US:en",
+    "disabled": true,
+    "disable_reason": "Google News RSS only provides encrypted URLs that don't work when accessed directly. Reuters does not provide public RSS feeds.",
+    "entries": 100,
+    "validated_at": "2026-03-19T15:48:00.000000"
+  },
+  "Fortune – Top Stories": {
+    "source_website": "fortune.com",
+    "rss_url": "https://fortune.com/feed/fortune-feeds/?id=3230629",
+    "entries": 10,
+    "validated_at": "2026-03-18T20:21:46.869680"
+  },
+  "Seeking Alpha – Market News": {
+    "source_website": "seekingalpha.com",
+    "rss_url": "https://seekingalpha.com/feed.xml",
+    "entries": 30,
+    "validated_at": "2026-03-18T20:21:47.462673"
+  },
+  "The Motley Fool – Stock News & Analysis": {
+    "source_website": "fool.com",
+    "rss_url": "https://www.fool.com/a/feeds/partner/googlechromefollow?apikey=${MOTLEY_FOOL_API_KEY}",
+    "entries": 50,
+    "validated_at": "2026-03-18T20:21:48.479725"
+  },
+  "TheStreet – Full Articles": {
+    "source_website": "thestreet.com",
+    "rss_url": "https://www.thestreet.com/.rss/full",
+    "entries": 50,
+    "validated_at": "2026-03-18T20:21:50.429528"
+  },
+  "MarketBeat – Market News": {
+    "source_website": "marketbeat.com",
+    "rss_url": "https://www.marketbeat.com/feed/",
+    "entries": 100,
+    "validated_at": "2026-03-18T20:22:06.522327"
+  },
+  "Money (Time) – Personal Finance": {
+    "source_website": "money.com",
+    "rss_url": "https://money.com/money/feed/",
+    "entries": 10,
+    "validated_at": "2026-03-18T20:22:07.133884"
+  },
+  "Global Finance Magazine": {
+    "source_website": "gfmag.com",
+    "rss_url": "https://www.gfmag.com/feed",
+    "entries": 10,
+    "validated_at": "2026-03-18T20:22:08.230178"
+  },
+  "Financial Samurai": {
+    "source_website": "financialsamurai.com",
+    "rss_url": "https://www.financialsamurai.com/feed/",
+    "entries": 7,
+    "validated_at": "2026-03-18T20:22:09.173814"
+  },
+  "MoneyWeek": {
+    "source_website": "moneyweek.com",
+    "rss_url": "https://moneyweek.com/feed/all",
+    "entries": 50,
+    "validated_at": "2026-03-18T20:22:10.001295"
+  },
+  "Finance Monthly": {
+    "source_website": "finance-monthly.com",
+    "rss_url": "https://www.finance-monthly.com/feed/",
+    "entries": 45,
+    "validated_at": "2026-03-18T20:22:11.526262"
+  },
+  "European Financial Review": {
+    "source_website": "europeanfinancialreview.com",
+    "rss_url": "https://www.europeanfinancialreview.com/feed",
+    "entries": 10,
+    "validated_at": "2026-03-18T20:22:13.344671"
+  },
+  "World Finance": {
+    "source_website": "worldfinance.com",
+    "rss_url": "https://www.worldfinance.com/feed",
+    "entries": 10,
+    "validated_at": "2026-03-18T20:22:16.414742"
+  },
+  "Fox Business – Headlines": {
+    "source_website": "foxbusiness.com",
+    "rss_url": "https://moxie.foxbusiness.com/google-publisher/latest.xml",
+    "entries": 25,
+    "validated_at": "2026-03-18T20:22:22.238519"
+  },
+  "FinanceAsia": {
+    "source_website": "financeasia.com",
+    "rss_url": "https://www.financeasia.com/rss/latest",
+    "entries": 10,
+    "validated_at": "2026-03-18T20:22:23.798531"
+  },
+  "CNBC – Business": {
+    "source_website": "cnbc.com",
+    "rss_url": "https://www.cnbc.com/id/100003114/device/rss/rss.html",
+    "entries": 30,
+    "validated_at": "2026-03-18T20:22:24.453119"
+  },
+
+  "Markets Insider": {
+    "source_website": "markets.businessinsider.com",
+    "rss_url": "https://markets.businessinsider.com/rss/news",
+    "entries": 10,
+    "validated_at": "2026-03-18T20:22:29.272544"
+  },
+  "The Economist – Business & Finance": {
+    "source_website": "economist.com",
+    "rss_url": "https://www.economist.com/business/rss.xml",
+    "entries": 300,
+    "validated_at": "2026-03-18T20:22:30.110716"
+  },
+  "Barchart News": {
+    "source_website": "barchart.com",
+    "rss_url": "https://feeds.feedburner.com/BarchartNews",
+    "entries": 15,
+    "validated_at": "2026-03-18T20:22:30.831390"
+  },
+  "The Guardian – Business": {
+    "source_website": "theguardian.com",
+    "rss_url": "https://feeds.theguardian.com/theguardian/uk/business/rss",
+    "entries": 40,
+    "validated_at": "2026-03-18T20:22:32.033297"
+  },
+  "Economy Watch": {
+    "source_website": "economywatch.com",
+    "rss_url": "https://www.economywatch.com/feed",
+    "entries": 10,
+    "validated_at": "2026-03-18T20:22:33.008150"
+  },
+  "CFI.co": {
+    "source_website": "cfi.co",
+    "rss_url": "https://cfi.co/feed",
+    "entries": 20,
+    "validated_at": "2026-03-18T20:22:35.620736"
+  },
+  "BBC News – Business": {
+    "source_website": "bbc.co.uk",
+    "rss_url": "https://feeds.bbci.co.uk/news/business/rss.xml",
+    "entries": 56,
+    "validated_at": "2026-03-18T20:22:36.643323"
+  },
+  "Investor’s Business Daily": {
+    "source_website": "investors.com",
+    "rss_url": "https://www.investors.com/feed/",
+    "entries": 100,
+    "validated_at": "2026-03-18T20:22:38.264669"
+  },
+  "MarketWatch – Top Stories": {
+    "source_website": "marketwatch.com",
+    "rss_url": "https://feeds.marketwatch.com/marketwatch/topstories/",
+    "entries": 10,
+    "validated_at": "2026-03-18T20:22:45.591752"
+  },
+  "Wall Street Journal – U.S. Business": {
+    "source_website": "wsj.com",
+    "rss_url": "https://feeds.a.dj.com/rss/WSJcomUSBusiness.xml",
+    "entries": 20,
+    "validated_at": "2026-03-18T20:22:46.259913"
+  },
+  "Investing.com – News": {
+    "source_website": "investing.com",
+    "rss_url": "https://www.investing.com/rss/news.rss",
+    "entries": 10,
+    "validated_at": "2026-03-18T20:22:50.790503"
+  },
+  "International Business Times": {
+    "source_website": "ibtimes.com",
+    "rss_url": "https://www.ibtimes.com/rss",
+    "entries": 25,
+    "validated_at": "2026-03-18T20:23:13.449646"
+  },
+  "404 Media": {
+    "source_website": "404media.co",
+    "rss_url": "https://404media.co/feed/",
+    "entries": 30,
+    "validated_at": "2026-03-20T00:00:00.000000"
+  },
+  "Mac Rumors": {
+    "source_website": "macrumors.com",
+    "rss_url": "https://feeds.macrumors.com/MacRumors-All",
+    "entries": 50,
+    "validated_at": "2026-03-20T00:00:00.000000"
+  },
+  "The Verge": {
+    "source_website": "theverge.com",
+    "rss_url": "https://www.theverge.com/rss/index.xml",
+    "entries": 50,
+    "validated_at": "2026-03-20T00:00:00.000000"
+  },
+  "TechCrunch": {
+    "source_website": "techcrunch.com",
+    "rss_url": "https://techcrunch.com/feed/",
+    "entries": 50,
+    "validated_at": "2026-03-20T00:00:00.000000"
+  },
+  "WIRED": {
+    "source_website": "wired.com",
+    "rss_url": "https://www.wired.com/feed/rss",
+    "entries": 50,
+    "validated_at": "2026-03-20T00:00:00.000000"
+  },
+  "Hacker News": {
+    "source_website": "news.ycombinator.com",
+    "rss_url": "https://news.ycombinator.com/rss",
+    "entries": 30,
+    "validated_at": "2026-03-20T00:00:00.000000"
+  },
+  "ZDNet": {
+    "source_website": "zdnet.com",
+    "rss_url": "https://www.zdnet.com/news/rss.xml",
+    "entries": 50,
+    "validated_at": "2026-03-20T00:00:00.000000"
+  },
+  "Engadget": {
+    "source_website": "engadget.com",
+    "rss_url": "https://www.engadget.com/rss.xml",
+    "entries": 50,
+    "validated_at": "2026-03-20T00:00:00.000000"
+  },
+  "Ars Technica": {
+    "source_website": "arstechnica.com",
+    "rss_url": "https://arstechnica.com/feed/",
+    "entries": 50,
+    "validated_at": "2026-03-21T00:00:00.000000"
+  },
+  "Associated Press": {
+    "source_website": "apnews.com",
+    "rss_url": "https://apnews.com",
+    "feed_type": "html",
+    "entries": 100,
+    "validated_at": "2026-03-20T00:00:00.000000"
+  },
+  "The Hacker News": {
+    "source_website": "thehackernews.com",
+    "rss_url": "https://thehackernews.com/feeds/posts/default",
+    "entries": 50,
+    "validated_at": "2026-03-21T13:17:00+00:00"
+  },
+  "Dark Reading": {
+    "source_website": "darkreading.com",
+    "rss_url": "https://www.darkreading.com/rss.xml",
+    "entries": 50,
+    "validated_at": "2026-03-20T19:30:19+00:00"
+  },
+  "SecurityWeek": {
+    "source_website": "securityweek.com",
+    "rss_url": "https://www.securityweek.com/feed",
+    "entries": 10,
+    "validated_at": "2026-03-21T11:00:00+00:00"
+  },
+  "BleepingComputer": {
+    "source_website": "bleepingcomputer.com",
+    "rss_url": "https://www.bleepingcomputer.com/feed",
+    "entries": 15,
+    "validated_at": "2026-03-21T17:30:41+00:00"
+  },
+  "Microsoft Security Blog": {
+    "source_website": "microsoft.com",
+    "rss_url": "https://www.microsoft.com/security/blog/feed",
+    "entries": 10,
+    "validated_at": "2026-03-20T16:19:00+00:00"
+  },
+  "EFF Deeplinks": {
+    "source_website": "eff.org",
+    "rss_url": "https://www.eff.org/deeplinks.xml",
+    "entries": 50,
+    "validated_at": "2026-03-20T22:20:49+00:00"
+  },
+  "US-CISA": {
+    "source_website": "cisa.gov",
+    "rss_url": "https://www.cisa.gov/news.xml",
+    "entries": 10,
+    "validated_at": "2026-02-26T12:00:00+00:00"
+  },
+  "Google Security Blog": {
+    "source_website": "google.com",
+    "rss_url": "https://security.googleblog.com/feeds/posts/default",
+    "entries": 25,
+    "validated_at": "2026-02-27T17:01:00+00:00"
+  },
+  "Politico": {
+    "source_website": "politico.com",
+    "rss_url": "https://www.politico.com/rss/politicopicks.xml",
+    "entries": 50,
+    "validated_at": "2026-03-22T14:58:00+00:00"
+  },
+  "Cyber Security News": {
+    "source_website": "cybersecuritynews.com",
+    "rss_url": "https://cybersecuritynews.com/feed/",
+    "entries": 50,
+    "validated_at": "2026-03-23T18:47:58+00:00"
+  },
+  "ProPublica": {
+    "source_website": "propublica.org",
+    "rss_url": "https://www.propublica.org/feeds/propublica/main",
+    "entries": 30,
+    "validated_at": "2026-03-24T00:00:00.000000"
+  }
+}

rss_processor.py

@@ -0,0 +1,455 @@
+#!/usr/bin/env python3
+"""RSS Feed Processor for NewsArchiver - Phase 2.1"""
+
+import argparse
+import json
+import logging
+import sys
+import time
+import urllib.error
+import urllib.request
+from datetime import datetime
+from pathlib import Path
+from typing import List, Optional
+
+try:
+    import feedparser
+    from feedparser import FeedParserDict
+except ImportError:
+    print("ERROR: feedparser is required. Install with: pip install feedparser")
+    sys.exit(1)
+
+try:
+    import sqlite3
+except ImportError:
+    print("ERROR: sqlite3 is required (should be built-in)")
+    sys.exit(1)
+
+SCRIPT_DIR = Path(__file__).parent
+ARCHIVE_DIR = SCRIPT_DIR / 'archival_data'
+ARCHIVE_DIR.mkdir(exist_ok=True)
+
+logger = logging.getLogger(__name__)
+
+
+def init_db(db_path: Path = ARCHIVE_DIR / 'cache.db') -> None:
+    """Initialize SQLite database for caching processed articles."""
+    conn = sqlite3.connect(db_path)
+    cursor = conn.cursor()
+    
+    cursor.execute('''
+        CREATE TABLE IF NOT EXISTS articles (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            source_name TEXT NOT NULL,
+            article_url TEXT NOT NULL UNIQUE,
+            article_guid TEXT,
+            title TEXT,
+            author TEXT,
+            publish_date TEXT,
+            content_text TEXT,
+            content_html TEXT,
+            archive_file_path TEXT,
+            metadata_file_path TEXT,
+            status TEXT DEFAULT 'pending',
+            error_message TEXT,
+            created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+            updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+        )
+    ''')
+    
+    cursor.execute('''
+        CREATE TABLE IF NOT EXISTS processing_log (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            source_name TEXT,
+            timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+            action TEXT,
+            status TEXT,
+            message TEXT
+        )
+    ''')
+    
+    cursor.execute('''
+        CREATE INDEX IF NOT EXISTS idx_articles_source_url 
+        ON articles(source_name, article_url)
+    ''')
+    
+    cursor.execute('''
+        CREATE INDEX IF NOT EXISTS idx_articles_status 
+        ON articles(status)
+    ''')
+    
+    conn.commit()
+    conn.close()
+    logger.debug("Database initialized: %s", db_path)
+
+
+def get_db_connection(db_path: Path = ARCHIVE_DIR / 'cache.db'):
+    """Get database connection."""
+    conn = sqlite3.connect(db_path)
+    conn.row_factory = sqlite3.Row
+    return conn
+
+
+def is_duplicate(article_url: str, source_name: str, db_path: Path = ARCHIVE_DIR / 'cache.db') -> bool:
+    """Check if article already in cache."""
+    conn = get_db_connection(db_path)
+    cursor = conn.cursor()
+    
+    cursor.execute(
+        'SELECT 1 FROM articles WHERE source_name = ? AND article_url = ?',
+        (source_name, article_url)
+    )
+    result = cursor.fetchone()
+    conn.close()
+    
+    return result is not None
+
+
+def add_to_cache(
+    article_url: str,
+    source_name: str,
+    timestamp: datetime,
+    article_guid: str = None,
+    title: str = None,
+    author: str = None,
+    publish_date: str = None,
+    db_path: Path = ARCHIVE_DIR / 'cache.db'
+) -> bool:
+    """Add article to cache."""
+    conn = get_db_connection(db_path)
+    cursor = conn.cursor()
+    
+    try:
+        cursor.execute('''
+            INSERT OR IGNORE INTO articles 
+            (source_name, article_url, article_guid, title, author, publish_date, status)
+            VALUES (?, ?, ?, ?, ?, ?, ?)
+        ''', (
+            source_name,
+            article_url,
+            article_guid,
+            title,
+            author,
+            publish_date,
+            'pending'
+        ))
+        conn.commit()
+        conn.close()
+        return True
+    except sqlite3.IntegrityError:
+        conn.close()
+        return False
+
+
+def save_article(
+    source_name: str,
+    article_data: dict,
+    db_path: Path = ARCHIVE_DIR / 'cache.db'
+) -> str:
+    """Save article to storage and update cache."""
+    conn = get_db_connection(db_path)
+    cursor = conn.cursor()
+    
+    article_url = article_data.get('link', '')
+    article_guid = article_data.get('id', article_url)
+    title = article_data.get('title', '')
+    author = article_data.get('author', '')
+    publish_date = article_data.get('published', '')
+    summary = article_data.get('summary', '')
+    
+    cursor.execute('''
+        INSERT OR REPLACE INTO articles 
+        (source_name, article_url, article_guid, title, author, publish_date, content_text, status)
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+    ''', (
+        source_name,
+        article_url,
+        article_guid,
+        title,
+        author,
+        publish_date,
+        summary,
+        'archived'
+    ))
+    
+    conn.commit()
+    conn.close()
+    return article_url
+
+
+def fetch_rss_feed(rss_url: str, timeout: int = 30) -> FeedParserDict:
+    """Fetch and parse RSS feed."""
+    logger.info("Fetching RSS feed: %s", rss_url[:50] + "..." if len(rss_url) > 50 else rss_url)
+    
+    try:
+        req = urllib.request.Request(
+            rss_url,
+            headers={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'}
+        )
+        with urllib.request.urlopen(req, timeout=timeout) as response:
+            feed_content = response.read()
+        
+        feed = feedparser.parse(feed_content)
+        
+        if feed.bozo:
+            logger.warning("Feed parsing completed with warnings: %s", feed.bozo)
+        
+        entry_count = len(feed.entries)
+        logger.info("Feed parsed: %d entries", entry_count)
+        
+        return feed
+    except urllib.error.HTTPError as e:
+        logger.error("HTTP error fetching RSS feed %s: %s", rss_url, str(e.code))
+        raise
+    except urllib.error.URLError as e:
+        logger.error("URL error fetching RSS feed %s: %s", rss_url, str(e.reason))
+        raise
+    except Exception as e:
+        logger.error("Failed to fetch RSS feed %s: %s", rss_url, str(e))
+        raise
+
+
+def decode_google_news_url(google_url: str, entry: dict = None) -> Optional[str]:
+    """Decode Google News encrypted URL to actual article URL.
+    
+    Google News RSS uses encrypted URLs like:
+    https://news.google.com/rss/articles/CBMioAFB... which need to be decoded.
+    
+    Args:
+        google_url: Google News encrypted URL
+        entry: Full RSS entry for additional context
+        
+    Returns:
+        Decoded article URL or None if not a Google News URL
+    """
+    if 'news.google.com' not in google_url:
+        return google_url
+    
+    try:
+        import base64
+        import urllib.parse
+        
+        if '/rss/articles/' in google_url:
+            parts = google_url.split('/rss/articles/')
+            if len(parts) >= 2:
+                encoded = parts[1]
+                if encoded.startswith('CBM'):
+                    encoded = encoded[3:]
+                    padding = (4 - len(encoded) % 4) % 4
+                    encoded += '=' * padding
+                    try:
+                        decoded = base64.urlsafe_b64decode(encoded).decode('utf-8')
+                        logger.debug("Decoded Google News URL: %s -> %s", google_url[:60], decoded[:60])
+                        return decoded
+                    except Exception:
+                        pass
+        
+        if '/articles/' in google_url:
+            parts = google_url.split('/articles/')
+            if len(parts) >= 2:
+                encoded = parts[1]
+                if encoded.startswith('CBM'):
+                    encoded = encoded[3:]
+                    padding = (4 - len(encoded) % 4) % 4
+                    encoded += '=' * padding
+                    try:
+                        decoded = base64.urlsafe_b64decode(encoded).decode('utf-8')
+                        logger.debug("Decoded Google News URL: %s -> %s", google_url[:60], decoded[:60])
+                        return decoded
+                    except Exception:
+                        pass
+        
+        if 'url=' in google_url:
+            parsed = urllib.parse.urlparse(google_url)
+            params = urllib.parse.parse_qs(parsed.query)
+            if 'url' in params:
+                return params['url'][0]
+        
+        logger.debug("Could not decode Google News URL: %s", google_url[:60])
+        
+        if entry and 'source' in entry:
+            source = entry.get('source', {})
+            if isinstance(source, dict) and 'href' in source:
+                source_href = source['href']
+                logger.debug("Using source URL as fallback: %s", source_href)
+                return source_href
+        
+        return google_url
+        
+    except Exception as e:
+        logger.debug("Failed to decode Google News URL %s: %s", google_url[:60], str(e))
+        return google_url
+
+
+def process_rss_feed(
+    rss_url: str,
+    source_name: str,
+    output_dir: Path,
+    db_path: Path = ARCHIVE_DIR / 'cache.db'
+) -> List[dict]:
+    """Process RSS feed and archive new articles."""
+    logger.info("Processing RSS feed for source: %s", source_name)
+    
+    try:
+        feed = fetch_rss_feed(rss_url)
+    except Exception as e:
+        logger.error("Failed to fetch feed: %s", str(e))
+        return []
+    
+    new_articles = []
+    skipped_count = 0
+    
+    for entry in feed.entries:
+        article_url = entry.get('link', '')
+        article_url = decode_google_news_url(article_url, entry)
+        article_guid = entry.get('id', article_url)
+        
+        if not article_url:
+            logger.warning("Skipping entry without URL")
+            continue
+        
+        if is_duplicate(article_url, source_name, db_path):
+            logger.debug("Skipping duplicate: %s", article_url[:60])
+            skipped_count += 1
+            continue
+        
+        title = entry.get('title', 'No Title')
+        author = entry.get('author', entry.get('authors', [{}])[0].get('name', '') if entry.get('authors') else '')
+        published = entry.get('published', entry.get('published_parsed', ''))
+        summary = entry.get('summary', entry.get('description', ''))
+        
+        if published:
+            if hasattr(published, 'tm_year'):
+                publish_date = datetime(*published[:6]).isoformat()
+            else:
+                publish_date = published
+        else:
+            publish_date = datetime.now().isoformat()
+        
+        article_data = {
+            'source_name': source_name,
+            'url': article_url,
+            'guid': article_guid,
+            'title': title,
+            'author': author,
+            'publish_date': publish_date,
+            'summary': summary,
+            'entry': entry
+        }
+        
+        add_to_cache(
+            article_url=article_url,
+            source_name=source_name,
+            timestamp=datetime.now(),
+            article_guid=article_guid,
+            title=title,
+            author=author,
+            publish_date=publish_date,
+            db_path=db_path
+        )
+        
+        new_articles.append(article_data)
+        logger.debug("New article: %s", title[:60])
+    
+    logger.info("Processed %s: %d new, %d skipped", source_name, len(new_articles), skipped_count)
+    return new_articles
+
+
+def process_all_feeds(
+    rss_feeds_path: Path = SCRIPT_DIR / 'rss_feeds.json',
+    output_dir: Path = ARCHIVE_DIR,
+    db_path: Path = ARCHIVE_DIR / 'cache.db'
+) -> dict:
+    """Process all RSS feeds from rss_feeds.json."""
+    if not rss_feeds_path.exists():
+        logger.error("RSS feeds file not found: %s", rss_feeds_path)
+        return {'success': False, 'error': 'File not found'}
+    
+    with open(rss_feeds_path, 'r', encoding='utf-8') as f:
+        rss_feeds = json.load(f)
+    
+    results = {
+        'total': 0,
+        'success': 0,
+        'failed': 0,
+        'new_articles': 0,
+        'errors': []
+    }
+    
+    for source_name, feed_info in rss_feeds.items():
+        rss_url = feed_info.get('rss_url', '')
+        
+        if not rss_url:
+            logger.warning("No RSS URL for source: %s", source_name)
+            continue
+        
+        if feed_info.get('disabled', False):
+            logger.info("Skipping disabled source: %s (%s)", source_name, 
+                       feed_info.get('disable_reason', 'No reason provided'))
+            continue
+        
+        results['total'] += 1
+        
+        try:
+            articles = process_rss_feed(rss_url, source_name, output_dir, db_path)
+            results['success'] += 1
+            results['new_articles'] += len(articles)
+            logger.info("Completed %s: %d new articles", source_name, len(articles))
+        except Exception as e:
+            results['failed'] += 1
+            results['errors'].append({
+                'source': source_name,
+                'url': rss_url,
+                'error': str(e)
+            })
+            logger.error("Failed to process %s: %s", source_name, str(e))
+    
+    return results
+
+
+def main():
+    parser = argparse.ArgumentParser(description='RSS Feed Processor for NewsArchiver')
+    parser.add_argument('--url', help='Single RSS URL to process')
+    parser.add_argument('--source', help='Source name (required if --url provided)')
+    parser.add_argument('--all', action='store_true', help='Process all feeds from rss_feeds.json')
+    parser.add_argument('--rss-feeds', type=Path, default=SCRIPT_DIR / 'rss_feeds.json',
+                        help='Path to RSS feeds JSON file')
+    parser.add_argument('--output', type=Path, default=ARCHIVE_DIR,
+                        help='Output directory for archived content')
+    parser.add_argument('--verbose', action='store_true', help='Enable verbose logging')
+    args = parser.parse_args()
+    
+    if args.verbose:
+        logger.setLevel(logging.DEBUG)
+    
+    logger.info("=" * 60)
+    logger.info("RSS Feed Processor - Phase 2.1")
+    logger.info("=" * 60)
+    
+    init_db()
+    
+    if args.url:
+        if not args.source:
+            logger.error("Source name required when using --url")
+            return
+        process_rss_feed(args.url, args.source, args.output)
+    elif args.all:
+        results = process_all_feeds(args.rss_feeds, args.output)
+        print("\n" + "=" * 60)
+        print("PROCESSING COMPLETE")
+        print("=" * 60)
+        print(f"Total feeds: {results['total']}")
+        print(f"Successful: {results['success']}")
+        print(f"Failed: {results['failed']}")
+        print(f"New articles: {results['new_articles']}")
+        if results['errors']:
+            print("\nErrors:")
+            for error in results['errors']:
+                print(f"  - {error['source']}: {error['error']}")
+        print("=" * 60)
+    else:
+        parser.print_help()
+
+
+if __name__ == '__main__':
+    main()

run_archiver.py

@@ -0,0 +1,320 @@
+#!/usr/bin/env python3
+"""NewsArchiver - Main CLI Entry Point (Phase 4)
+
+Single-file CLI for running NewsArchiver with multiple modes:
+- --run: Archive news articles once
+- --serve: Start Flask web server
+- --interval: Run background scheduler with specified interval
+"""
+
+import argparse
+import atexit
+import logging
+import os
+import signal
+import sys
+import threading
+import time
+from pathlib import Path
+
+try:
+    from flask import Flask
+except ImportError:
+    print("ERROR: Flask is required. Install with: pip install flask")
+    sys.exit(1)
+
+try:
+    from scheduler import start_scheduler, stop_scheduler, scheduled_archive
+except ImportError:
+    print("ERROR: scheduler module not found")
+    sys.exit(1)
+
+try:
+    from rss_processor import process_all_feeds, init_db as init_db_rss
+except ImportError:
+    print("ERROR: rss_processor module not found")
+    sys.exit(1)
+
+try:
+    from content_extractor import get_html_from_url
+except ImportError:
+    print("ERROR: content_extractor module not found")
+    sys.exit(1)
+
+try:
+    from storage_manager import initialize_storage, get_all_sources
+except ImportError:
+    print("ERROR: storage_manager module not found")
+    sys.exit(1)
+
+try:
+    from archive_engine import archive_all_sources
+except ImportError:
+    print("ERROR: archive_engine module not found")
+    sys.exit(1)
+
+try:
+    from web_interface import app
+except ImportError:
+    print("ERROR: web_interface module not found")
+    sys.exit(1)
+
+try:
+    from singlefile_archive import check_singlefile_available
+except ImportError:
+    print("WARNING: singlefile_archive module not found")
+    print("SingleFile integration will not be available")
+
+SCRIPT_DIR = Path(__file__).parent.resolve()
+ARCHIVE_DIR = Path(os.environ.get("ARCHIVE_DIR", str(SCRIPT_DIR / "archival_data"))).resolve()
+ARCHIVE_DIR.mkdir(exist_ok=True)
+
+
+def setup_logging(verbose: bool = False) -> logging.Logger:
+    """Configure logging for the application.
+    
+    Args:
+        verbose: If True, enable DEBUG level logging
+        
+    Returns:
+        Configured logger instance
+    """
+    level = logging.DEBUG if verbose else logging.INFO
+    
+    logging.basicConfig(
+        level=level,
+        format='%(asctime)s - %(levelname)s - %(message)s',
+        handlers=[
+            logging.StreamHandler(sys.stdout),
+            logging.FileHandler(ARCHIVE_DIR / 'processing.log', encoding='utf-8')
+        ]
+    )
+    
+    logger = logging.getLogger(__name__)
+    logger.info("NewsArchiver - Main CLI Entry Point")
+    logger.info("=" * 60)
+    
+    return logger
+
+
+def run_archive_once(logger: logging.Logger, verbose: bool = False) -> bool:
+    """Run archiving process once.
+    
+    Args:
+        logger: Logger instance
+        verbose: If True, enable verbose logging
+        
+    Returns:
+        True if successful, False otherwise
+    """
+    try:
+        logger.info("Running one-time archive")
+        logger.info("=" * 60)
+        
+        init_db_rss()
+        initialize_storage()
+        
+        results = archive_all_sources(
+            rss_feeds_path=SCRIPT_DIR / 'rss_feeds.json',
+            output_dir=ARCHIVE_DIR,
+            dry_run=False
+        )
+        
+        logger.info("=" * 60)
+        logger.info("Archive complete")
+        logger.info("=" * 60)
+        logger.info("Sources processed: %d", results.get('sources_processed', 0))
+        logger.info("Total articles archived: %d", results.get('total_articles_archived', 0))
+        logger.info("Total articles skipped: %d", results.get('total_articles_skipped', 0))
+        logger.info("Total articles failed: %d", results.get('total_articles_failed', 0))
+        
+        return True
+    except Exception as e:
+        logger.error("Archive failed: %s", str(e))
+        return False
+
+
+def run_scheduler(interval_minutes: int, logger: logging.Logger, verbose: bool = False) -> None:
+    """Run background scheduler.
+    
+    Args:
+        interval_minutes: Interval between archive runs in minutes
+        logger: Logger instance
+        verbose: If True, enable verbose logging
+    """
+    logger.info("Starting background scheduler")
+    logger.info("=" * 60)
+    
+    try:
+        init_db_rss()
+        initialize_storage()
+        
+        scheduler = start_scheduler(interval_minutes)
+        
+        atexit.register(stop_scheduler)
+        
+        logger.info("Press Ctrl+C to stop")
+        
+        try:
+            while True:
+                time.sleep(1)
+        except (KeyboardInterrupt, SystemExit):
+            logger.info("Shutting down scheduler...")
+            stop_scheduler()
+            logger.info("Scheduler stopped")
+            
+    except Exception as e:
+        logger.error("Scheduler failed to start: %s", str(e))
+        sys.exit(1)
+
+
+def run_web_server(host: str, port: int, logger: logging.Logger, verbose: bool = False, interval_minutes: int = None) -> None:
+    """Run Flask web server with optional background scheduler.
+    
+    Args:
+        host: Host to bind to
+        port: Port to bind to
+        logger: Logger instance
+        verbose: If True, enable verbose logging
+        interval_minutes: If set, start background scheduler at this interval
+    """
+    logger.info("Starting web server")
+    logger.info("=" * 60)
+    
+    _scheduler_stopped = [False]
+
+    def _start_scheduler():
+        try:
+            init_db_rss()
+            initialize_storage()
+            start_scheduler(interval_minutes)
+            logger.info("Background scheduler started (every %d min)", interval_minutes)
+            logger.info("Running initial archive...")
+            scheduled_archive(logger)
+            logger.info("Initial archive complete")
+        except Exception as e:
+            logger.error("Scheduler thread failed: %s", str(e))
+        finally:
+            _scheduler_stopped[0] = True
+
+    def _shutdown_scheduler(signum=None, frame=None):
+        if not _scheduler_stopped[0]:
+            logger.info("Stopping scheduler...")
+            stop_scheduler()
+            logger.info("Scheduler stopped")
+        sys.exit(0)
+
+    try:
+        if not (ARCHIVE_DIR / 'cache.db').exists():
+            logger.info("Database not found, initializing...")
+            initialize_storage()
+        
+        if not check_singlefile_available():
+            logger.warning("SingleFile CLI not available. Some features may not work.")
+        
+        if interval_minutes:
+            logger.info("Starting background archive scheduler (interval: %d min)...", interval_minutes)
+            t = threading.Thread(target=_start_scheduler, daemon=True)
+            t.start()
+            signal.signal(signal.SIGINT, _shutdown_scheduler)
+            signal.signal(signal.SIGTERM, _shutdown_scheduler)
+        
+        logger.info("Web server starting on %s:%d", host, port)
+        logger.info("=" * 60)
+        
+        app.run(
+            host=host,
+            port=port,
+            debug=False
+        )
+        
+    except Exception as e:
+        logger.error("Web server failed to start: %s", str(e))
+        sys.exit(1)
+
+
+def main() -> None:
+    """Main entry point for NewsArchiver CLI."""
+    parser = argparse.ArgumentParser(
+        description='NewsArchiver - News Article Archiving System',
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog='''
+Examples:
+  %(prog)s --run                    Run archiving once
+  %(prog)s --serve                  Start web server
+  %(prog)s --serve --host 0.0.0.0 --port 8080
+                                     Start web server on custom host/port
+  %(prog)s --interval 60            Run background scheduler (1 hour interval)
+        '''
+    )
+    
+    parser.add_argument(
+        '--run',
+        action='store_true',
+        help='Run archiving once (process all RSS feeds)'
+    )
+    
+    parser.add_argument(
+        '--serve',
+        action='store_true',
+        help='Start Flask web server'
+    )
+    
+    parser.add_argument(
+        '--interval',
+        type=int,
+        default=None,
+        help='Run background scheduler with specified interval (minutes). When used with --serve, runs in background thread (default: 60 min). Standalone blocks.'
+    )
+    
+    parser.add_argument(
+        '--host',
+        type=str,
+        default='0.0.0.0',
+        help='Host for web server (default: 0.0.0.0)'
+    )
+    
+    parser.add_argument(
+        '--port',
+        type=int,
+        default=5000,
+        help='Port for web server (default: 5000)'
+    )
+    
+    parser.add_argument(
+        '--verbose', '-v',
+        action='store_true',
+        help='Enable verbose logging (DEBUG level)'
+    )
+    
+    args = parser.parse_args()
+
+    # CI port shift: detect CI env, map to reserved CI port range 10000-10099
+    # CI_PORT_OFFSET (1-99) maps to 10001-10099. Each service gets a fixed offset.
+    # NewsArchiverV2=1, paste-bin=2, etc. See AGENTS.md for assignments.
+    if os.environ.get("CI") == "true" and os.environ.get("SKIP_PORT_SHIFT") != "1":
+        original_port = args.port
+        offset = int(os.environ.get("CI_PORT_OFFSET", "1"))
+        args.port = 10000 + offset
+        print(f"[ci-port-shift] Port shifted from {original_port} to {args.port} (offset {offset}, CI range: 10000-10099)")
+
+    logger = setup_logging(args.verbose)
+    
+    if args.run:
+        success = run_archive_once(logger, args.verbose)
+        sys.exit(0 if success else 1)
+
+    elif args.serve:
+        interval = args.interval if args.interval else 60
+        run_web_server(args.host, args.port, logger, args.verbose, interval_minutes=interval)
+
+    elif args.interval:
+        run_scheduler(args.interval, logger, args.verbose)
+
+    else:
+        parser.print_help()
+        sys.exit(1)
+
+
+if __name__ == '__main__':
+    main()

scheduler.py

@@ -0,0 +1,209 @@
+#!/usr/bin/env python3
+"""Scheduler for NewsArchiver - Phase 4
+
+Background scheduler using APScheduler to automate
+daily archiving of news sources.
+"""
+
+import atexit
+import logging
+import os
+import sys
+import threading
+import time
+from datetime import datetime
+from pathlib import Path
+
+try:
+    from apscheduler.schedulers.background import BackgroundScheduler
+    from apscheduler.triggers.interval import IntervalTrigger
+except ImportError:
+    print("ERROR: APScheduler is required. Install with: pip install apscheduler")
+    sys.exit(1)
+
+try:
+    from archive_engine import archive_all_sources
+except ImportError:
+    print("ERROR: archive_engine is required")
+    sys.exit(1)
+
+SCRIPT_DIR = Path(__file__).parent.resolve()
+ARCHIVE_DIR = Path(os.environ.get("ARCHIVE_DIR", str(SCRIPT_DIR / "archival_data"))).resolve()
+ARCHIVE_DIR.mkdir(exist_ok=True)
+
+logger = logging.getLogger(__name__)
+
+scheduler = BackgroundScheduler()
+
+# Timeout configuration
+MAX_RUN_TIME_SECONDS = 3600  # 1 hour
+start_time = None
+_timeout_timer = None
+
+
+def _timeout_checker():
+    """Daemon thread that raises SystemExit when timeout is reached."""
+    elapsed = (datetime.now() - start_time).total_seconds()
+    remaining = MAX_RUN_TIME_SECONDS - elapsed
+    if remaining > 0:
+        logger.warning("Timeout reached (%d seconds). Will exit after current download completes.", MAX_RUN_TIME_SECONDS)
+        raise SystemExit(0)
+
+def check_timeout() -> bool:
+    """Check if timeout has been reached.
+    
+    Returns:
+        True if timeout reached, False otherwise
+    """
+    global start_time
+    elapsed = (datetime.now() - start_time).total_seconds()
+    if elapsed >= MAX_RUN_TIME_SECONDS:
+        logger.warning("Maximum runtime of %d seconds reached (%d seconds elapsed)", MAX_RUN_TIME_SECONDS, int(elapsed))
+        return True
+    return False
+
+def _start_timeout_thread():
+    """Start a daemon thread that will raise SystemExit after MAX_RUN_TIME_SECONDS."""
+    global _timeout_timer
+    _timeout_timer = threading.Timer(MAX_RUN_TIME_SECONDS, _timeout_checker)
+    _timeout_timer.daemon = True
+    _timeout_timer.start()
+
+def _cancel_timeout_thread():
+    """Cancel the timeout thread."""
+    global _timeout_timer
+    if _timeout_timer:
+        _timeout_timer.cancel()
+        _timeout_timer = None
+
+def scheduled_archive() -> None:
+    """Run archiving for all sources."""
+    global start_time
+    start_time = datetime.now()
+    
+    logger.info("=" * 60)
+    logger.info("Starting scheduled archive run")
+    logger.info("=" * 60)
+    
+    _start_timeout_thread()
+    
+    try:
+        results = archive_all_sources(
+            rss_feeds_path=SCRIPT_DIR / 'rss_feeds.json',
+            output_dir=ARCHIVE_DIR,
+            dry_run=False
+        )
+        
+        if results['success']:
+            logger.info("Scheduled archive completed successfully")
+            logger.info("Sources processed: %d", results.get('sources_processed', 0))
+            logger.info("Total articles archived: %d", results.get('total_articles_archived', 0))
+        else:
+            logger.error("Scheduled archive failed: %s", results.get('error', 'Unknown error'))
+            
+    except SystemExit as e:
+        logger.info("Scheduler exiting due to timeout")
+        raise e
+    except Exception as e:
+        logger.error("Scheduled archive failed with exception: %s", str(e))
+    finally:
+        _cancel_timeout_thread()
+
+
+def start_scheduler(interval_minutes: int = 60) -> BackgroundScheduler:
+    """Start the background scheduler.
+    
+    Args:
+        interval_minutes: Interval between archive runs in minutes
+        
+    Returns:
+        The scheduler instance
+    """
+    scheduler.add_job(
+        func=scheduled_archive,
+        trigger=IntervalTrigger(minutes=interval_minutes),
+        id='archive_news',
+        replace_existing=True,
+        misfire_grace_time=60,
+        coalesce=True
+    )
+    
+    scheduler.start()
+    logger.info("Scheduler started with %d minute interval", interval_minutes)
+    
+    logger.info("Running initial archive immediately...")
+    scheduled_archive()
+    
+    return scheduler
+
+
+def run_once() -> None:
+    """Run archiving once (for CLI --run flag)."""
+    global start_time
+    start_time = datetime.now()
+    
+    logger.info("=" * 60)
+    logger.info("Running one-time archive")
+    logger.info("=" * 60)
+    
+    _start_timeout_thread()
+    
+    try:
+        scheduled_archive()
+        
+        logger.info("=" * 60)
+        logger.info("One-time archive completed")
+        logger.info("=" * 60)
+    except SystemExit as e:
+        logger.info("Archiver exiting due to timeout")
+        raise e
+    finally:
+        _cancel_timeout_thread()
+
+
+def stop_scheduler() -> None:
+    """Stop the scheduler gracefully."""
+    if scheduler.running:
+        scheduler.shutdown()
+        logger.info("Scheduler stopped")
+
+
+atexit.register(lambda: stop_scheduler())
+
+
+if __name__ == '__main__':
+    import argparse
+    
+    parser = argparse.ArgumentParser(description='NewsArchiver - Scheduler')
+    parser.add_argument('--run', action='store_true', help='Run archiving once')
+    parser.add_argument('--serve', action='store_true', help='Start web server')
+    parser.add_argument('--interval', type=int, default=60, help='Scheduler interval in minutes (default: 60)')
+    parser.add_argument('--host', default='0.0.0.0', help='Host for web server')
+    parser.add_argument('--port', type=int, default=5000, help='Port for web server')
+    
+    args = parser.parse_args()
+    
+    if args.run:
+        run_once()
+        stop_scheduler()
+    elif args.serve:
+        from web_interface import app
+        logger.info("Starting web server on %s:%d", args.host, args.port)
+        try:
+            app.run(host=args.host, port=args.port)
+        except Exception as e:
+            logger.error("Web server error: %s", str(e))
+            sys.exit(1)
+    else:
+        start_scheduler(args.interval)
+        logger.info("Press Ctrl+C to stop")
+        
+        try:
+            while True:
+                time.sleep(1)
+                if check_timeout():
+                    logger.info("Maximum runtime reached. Exiting.")
+                    stop_scheduler()
+                    sys.exit(0)
+        except (KeyboardInterrupt, SystemExit):
+            stop_scheduler()

setup_cron.sh

@@ -0,0 +1,84 @@
+#!/bin/bash
+# Setup script for NewsArchiver
+# This script sets up the cron job for automated news archiving
+
+set -e
+
+SCRIPT_DIR="${NEWSARCHIVER_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)}"
+LOG_FILE="${NEWSARCHIVER_LOG:-/tmp/newsarchiver_cron.log}"
+CRON_JOB="*/30 * * * * /usr/bin/env python3 ${SCRIPT_DIR}/run_archiver.py --interval 30 > ${LOG_FILE} 2>&1"
+
+echo "=== NewsArchiver Setup Script ==="
+echo ""
+
+# Check Python is available
+if ! command -v python3 &> /dev/null; then
+    echo "ERROR: python3 not found"
+    exit 1
+fi
+
+# Check if running as jarian
+CURRENT_USER=$(whoami)
+if [ "$CURRENT_USER" != "jarian" ]; then
+    echo "WARNING: This script is configured for user 'jarian', but you are '$CURRENT_USER'"
+    echo "You may need to update the script paths"
+fi
+
+# Check if NewsArchiver directory exists
+if [ ! -d "$SCRIPT_DIR" ]; then
+    echo "ERROR: NewsArchiver directory not found at $SCRIPT_DIR"
+    exit 1
+fi
+
+# Check if requirements are installed
+echo "Checking dependencies..."
+cd "$SCRIPT_DIR"
+python3 -c "import flask; import requests; import trafilatura; import feedparser; import apscheduler" 2>/dev/null || {
+    echo "Installing dependencies..."
+    pip install -r requirements.txt
+}
+
+# Check if web server is running
+if ! pgrep -f "run_archiver.py --serve" > /dev/null; then
+    echo "Starting web server..."
+    nohup python3 "$SCRIPT_DIR/run_archiver.py" --serve --host 0.0.0.0 --port 5000 > /tmp/webserver.log 2>&1 &
+    sleep 3
+    if curl -s -o /dev/null -w "%{http_code}" http://localhost:5000/ | grep -q "200"; then
+        echo "Web server started successfully on port 5000"
+    else
+        echo "WARNING: Web server may not be responding"
+    fi
+else
+    echo "Web server is already running"
+fi
+
+# Remove old scheduler lock file if exists
+if [ -f "$SCRIPT_DIR/archival_data/.scheduler.lock" ]; then
+    rm -f "$SCRIPT_DIR/archival_data/.scheduler.lock"
+    echo "Removed stale scheduler lock file"
+fi
+
+# Kill any existing scheduler processes
+pkill -f "run_archiver.py --interval" 2>/dev/null || true
+echo "Cleared any existing scheduler processes"
+
+# Setup cron job
+echo "Setting up cron job..."
+if crontab -l 2>/dev/null | grep -q "NewsArchiver"; then
+    echo "Removing existing NewsArchiver cron job..."
+    crontab -l | grep -v "NewsArchiver" | crontab -
+fi
+
+echo "$CRON_JOB" | crontab -
+echo "Cron job added successfully"
+
+echo ""
+echo "=== Current Cron Jobs ==="
+crontab -l | grep NewsArchiver
+
+echo ""
+echo "=== Setup Complete ==="
+echo "- Archiver will run every 30 minutes via cron"
+echo "- Logs written to: $LOG_FILE"
+echo "- Web interface at: http://localhost:5000"
+echo ""

singlefile_archive.py

@@ -0,0 +1,258 @@
+import subprocess
+import os
+from pathlib import Path
+import logging
+from typing import Optional
+
+logger = logging.getLogger(__name__)
+
+try:
+    from playwright.sync_api import sync_playwright
+    PLAYWRIGHT_AVAILABLE = True
+except ImportError:
+    PLAYWRIGHT_AVAILABLE = False
+    logger.debug("Playwright not available")
+
+# Cache the SingleFile path
+_SINGLEFILE_PATH: Optional[str] = None
+
+
+def _get_singlefile_path() -> Optional[str]:
+    """Get the path to SingleFile CLI executable."""
+    global _SINGLEFILE_PATH
+    
+    if _SINGLEFILE_PATH is not None:
+        return _SINGLEFILE_PATH
+    
+    # Check PATH first
+    single_file_path = os.environ.get('PATH', '').split(os.pathsep)
+    for path in single_file_path:
+        candidate = Path(path) / 'single-file'
+        if candidate.is_file():
+            _SINGLEFILE_PATH = str(candidate)
+            logger.debug(f"Found SingleFile in PATH: {_SINGLEFILE_PATH}")
+            return _SINGLEFILE_PATH
+    
+    # Check common locations
+    home = Path.home()
+    common_locations = [
+        str(home / '.local' / 'bin' / 'single-file'),
+        '/usr/local/bin/single-file',
+        '/usr/bin/single-file',
+        str(home / '.npm' / '_global' / 'bin' / 'single-file'),
+    ]
+    
+    for candidate in common_locations:
+        if Path(candidate).is_file():
+            _SINGLEFILE_PATH = candidate
+            logger.debug(f"Found SingleFile at: {_SINGLEFILE_PATH}")
+            return _SINGLEFILE_PATH
+    
+    logger.error("SingleFile CLI not found. Please install with: npm install -g single-file")
+    return None
+
+
+def check_singlefile_available() -> bool:
+    """Check if SingleFile CLI is available"""
+    single_file_path = _get_singlefile_path()
+    if not single_file_path:
+        return False
+    
+    try:
+        result = subprocess.run(
+            [single_file_path, '--version'],
+            capture_output=True,
+            text=True,
+            timeout=10
+        )
+        return result.returncode == 0
+    except (subprocess.TimeoutExpired, FileNotFoundError, OSError):
+        logger.error(f"SingleFile CLI at {single_file_path} is not executable")
+        return False
+
+
+def is_error_page(html_content: str) -> bool:
+    """Check if HTML content is an error page (403, 404, etc.)
+    
+    Args:
+        html_content: HTML content to check
+        
+    Returns:
+        True if error page detected, False otherwise
+    """
+    error_patterns = [
+        '403 error',
+        '403 forbidden',
+        'access denied',
+        'request blocked',
+        'cloudfront',
+        '404 error',
+        'page not found',
+        'error 404',
+        'server error',
+        '503 service unavailable',
+    ]
+    
+    html_lower = html_content.lower()
+    return any(pattern in html_lower for pattern in error_patterns)
+
+
+def validate_archived_html(output_path: Path) -> bool:
+    """Validate that archived HTML is not an error page.
+    
+    Args:
+        output_path: Path to the archived HTML file
+        
+    Returns:
+        True if valid, False if error page detected
+    """
+    try:
+        if not output_path.exists():
+            logger.warning(f"Archived file not found: {output_path}")
+            return False
+        
+        content = output_path.read_text(encoding='utf-8', errors='ignore')
+        
+        if is_error_page(content):
+            logger.warning(f"Archived file contains error page: {output_path}")
+            return False
+        
+        if len(content) < 1000:
+            logger.warning(f"Archived file too small (likely incomplete): {output_path}")
+            return False
+        
+        return True
+        
+    except Exception as e:
+        logger.error(f"Error validating archived HTML {output_path}: {e}")
+        return False
+
+
+def archive_page_with_singlefile(
+    url: str,
+    output_path: Path,
+    extract_content: bool = True
+) -> bool:
+    """Archive a web page using SingleFile CLI
+    
+    Args:
+        url: URL to archive
+        output_path: Output file path for the archived HTML
+        extract_content: Whether to use extract-content mode (ignored - SingleFile always extracts)
+        
+    Returns:
+        True if successful, False otherwise
+    """
+    single_file_path = _get_singlefile_path()
+    if not single_file_path:
+        logger.error("SingleFile CLI not available")
+        return False
+    
+    cmd = [
+        single_file_path,
+        url,
+        str(output_path),
+        '--browser-headless=true',
+        '--browser-wait-delay=5000',
+        '--browser-load-max-time=120000'
+    ]
+    
+    logger.debug(f"Archiving {url} with SingleFile at {single_file_path}")
+    
+    try:
+        result = subprocess.run(cmd, capture_output=True, text=True, timeout=180)
+        
+        if result.returncode == 0:
+            logger.info(f"Successfully archived {url} to {output_path}")
+            if result.stdout:
+                logger.debug(f"SingleFile output: {result.stdout}")
+            
+            # Validate the archived HTML
+            if not validate_archived_html(output_path):
+                logger.warning(f"Archived HTML validation failed for {url}, will use fallback")
+                return False
+            
+            return True
+        else:
+            error_msg = result.stderr if result.stderr else result.stdout
+            logger.error(f"SingleFile failed for {url}: {error_msg}")
+            return False
+            
+    except subprocess.TimeoutExpired:
+        logger.error(f"SingleFile timed out for {url}")
+        return False
+    except FileNotFoundError:
+        logger.error(f"SingleFile CLI executable not found at: {single_file_path}")
+        return False
+    except Exception as e:
+        logger.error(f"Unexpected error archiving {url} with SingleFile: {e}")
+        return False
+
+
+def archive_page_with_singlefile_no_extraction(url: str, output_path: Path) -> bool:
+    """Archive a web page using SingleFile CLI without content extraction
+    
+    This preserves the full original HTML structure including navigation, ads, etc.
+    
+    Args:
+        url: URL to archive
+        output_path: Output file path for the archived HTML
+        
+    Returns:
+        True if successful, False otherwise
+    """
+    return archive_page_with_singlefile(url, output_path, extract_content=False)
+
+
+def archive_page_with_singlefile_extract(url: str, output_path: Path) -> bool:
+    """Archive a web page using SingleFile CLI with content extraction
+    
+    This extracts only the main content, removing navigation, ads, and sidebars.
+    
+    Args:
+        url: URL to archive
+        output_path: Output file path for the archived HTML
+        
+    Returns:
+        True if successful, False otherwise
+    """
+    return archive_page_with_singlefile(url, output_path, extract_content=True)
+
+
+def archive_page_with_playwright(url: str, output_path: Path) -> bool:
+    """Archive a web page using Playwright
+    
+    This visits the URL with a headless browser, waits for content to load,
+    and saves the full HTML page.
+    
+    Args:
+        url: URL to archive
+        output_path: Output file path for the archived HTML
+        
+    Returns:
+        True if successful, False otherwise
+    """
+    if not PLAYWRIGHT_AVAILABLE:
+        logger.error("Playwright not available. Install with: pip install playwright")
+        return False
+    
+    try:
+        with sync_playwright() as p:
+            browser = p.chromium.launch(headless=True)
+            page = browser.new_page()
+            
+            page.goto(url, wait_until='networkidle', timeout=120000)
+            
+            content = page.content()
+            
+            browser.close()
+            
+            output_path.parent.mkdir(parents=True, exist_ok=True)
+            output_path.write_text(content, encoding='utf-8')
+            
+            logger.info("Successfully archived %s to %s using Playwright", url[:60], output_path)
+            return True
+            
+    except Exception as e:
+        logger.error("Playwright archiving failed for %s: %s", url, str(e))
+        return False

static/style.css

@@ -0,0 +1,535 @@
+/* Base styles */
+:root {
+    --primary-color: #1a1a1a;
+    --secondary-color: #6b7280;
+    --background-color: #f9fafb;
+    --surface-color: #ffffff;
+    --border-color: #e5e7eb;
+    --accent-color: #2563eb;
+    --accent-hover: #1d4ed8;
+    --accent-light: #eff6ff;
+    --timeline-line: #d1d5db;
+    --dot-color: #2563eb;
+}
+
+[data-theme="dark"] {
+    --primary-color: #f3f4f6;
+    --secondary-color: #9ca3af;
+    --background-color: #111827;
+    --surface-color: #1f2937;
+    --border-color: #374151;
+    --accent-color: #60a5fa;
+    --accent-hover: #93c5fd;
+    --accent-light: #1e3a5f;
+    --timeline-line: #4b5563;
+    --dot-color: #60a5fa;
+}
+
+* {
+    box-sizing: border-box;
+}
+
+body {
+    font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, 'Helvetica Neue', Arial, sans-serif;
+    margin: 0;
+    padding: 0;
+    background-color: var(--background-color);
+    color: var(--primary-color);
+    line-height: 1.6;
+    transition: background-color 0.3s ease, color 0.3s ease;
+    -webkit-font-smoothing: antialiased;
+}
+
+/* Header */
+header {
+    background-color: var(--surface-color);
+    border-bottom: 1px solid var(--border-color);
+    padding: 0.75rem 2rem;
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+    position: sticky;
+    top: 0;
+    z-index: 100;
+    backdrop-filter: blur(8px);
+    transition: background-color 0.3s ease;
+}
+
+header h1 {
+    margin: 0;
+    font-size: 1.25rem;
+    font-weight: 700;
+    letter-spacing: -0.025em;
+}
+
+header nav a {
+    color: var(--secondary-color);
+    text-decoration: none;
+    margin-left: 1.5rem;
+    font-size: 0.9rem;
+    font-weight: 500;
+    transition: color 0.2s ease;
+}
+
+header nav a:hover {
+    color: var(--accent-color);
+}
+
+main {
+    padding: 2rem;
+    max-width: 960px;
+    margin: 0 auto;
+}
+
+/* Page headings */
+main > h1 {
+    font-size: 1.5rem;
+    font-weight: 700;
+    letter-spacing: -0.025em;
+    margin: 0 0 1.5rem 0;
+}
+
+/* Newspaper list */
+.newspaper-list {
+    list-style: none;
+    padding: 0;
+    margin: 0;
+}
+
+.newspaper-item {
+    background: var(--surface-color);
+    border: 1px solid var(--border-color);
+    border-radius: 8px;
+    padding: 1rem 1.25rem;
+    margin-bottom: 0.5rem;
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+    transition: all 0.15s ease;
+}
+
+.newspaper-item:hover {
+    border-color: var(--accent-color);
+    box-shadow: 0 1px 3px rgba(0,0,0,0.05);
+}
+
+.newspaper-info h2 {
+    margin: 0 0 0.25rem 0;
+    font-size: 1rem;
+    font-weight: 600;
+}
+
+.newspaper-info h2 a {
+    color: var(--primary-color);
+    text-decoration: none;
+}
+
+.newspaper-info h2 a:hover {
+    color: var(--accent-color);
+}
+
+.newspaper-info p {
+    margin: 0;
+    color: var(--secondary-color);
+    font-size: 0.85rem;
+}
+
+.status-success { color: #16a34a; font-weight: 600; }
+.status-pending { color: #f59e0b; font-weight: 600; }
+
+.pull-btn {
+    background: var(--accent-color);
+    color: white;
+    border: none;
+    padding: 0.4rem 0.85rem;
+    border-radius: 6px;
+    cursor: pointer;
+    font-size: 0.85rem;
+    font-weight: 500;
+    transition: background-color 0.15s ease;
+}
+
+.pull-btn:hover {
+    background: var(--accent-hover);
+}
+
+/* Timeline Layout */
+.timeline {
+    position: relative;
+    padding-left: 3rem;
+}
+
+.timeline::before {
+    content: '';
+    position: absolute;
+    left: 1.25rem;
+    top: 0;
+    bottom: 0;
+    width: 2px;
+    background: var(--timeline-line);
+    border-radius: 1px;
+}
+
+.timeline-date {
+    position: relative;
+    margin: 2rem 0 0.75rem;
+}
+
+.timeline-date:first-child {
+    margin-top: 0.5rem;
+}
+
+.timeline-date::before {
+    content: '';
+    position: absolute;
+    left: -2.15rem;
+    top: 0.35rem;
+    width: 12px;
+    height: 12px;
+    border-radius: 50%;
+    background: var(--dot-color);
+    border: 3px solid var(--background-color);
+    box-shadow: 0 0 0 2px var(--dot-color);
+    z-index: 2;
+}
+
+.date-label {
+    font-weight: 700;
+    font-size: 0.95rem;
+    color: var(--primary-color);
+    letter-spacing: -0.01em;
+}
+
+.date-count {
+    margin-left: 0.6rem;
+    font-size: 0.8rem;
+    color: var(--secondary-color);
+    font-weight: 400;
+}
+
+/* Article cards in timeline */
+.article-list {
+    list-style: none;
+    padding: 0;
+    margin: 0 0 1rem 0;
+}
+
+.article-item {
+    background: var(--surface-color);
+    border: 1px solid var(--border-color);
+    border-radius: 8px;
+    padding: 1rem 1.25rem;
+    margin-bottom: 0.5rem;
+    transition: all 0.15s ease;
+}
+
+.article-item:hover {
+    border-color: var(--accent-color);
+    box-shadow: 0 2px 8px rgba(0,0,0,0.06);
+    transform: translateY(-1px);
+}
+
+.article-item h3 {
+    margin: 0 0 0.5rem 0;
+    font-size: 1rem;
+    font-weight: 600;
+    line-height: 1.4;
+    letter-spacing: -0.01em;
+}
+
+.article-item h3 a {
+    color: var(--primary-color);
+    text-decoration: none;
+}
+
+.article-item h3 a:hover {
+    color: var(--accent-color);
+}
+
+.article-summary {
+    color: var(--secondary-color);
+    font-size: 0.875rem;
+    line-height: 1.5;
+    margin: 0;
+    display: -webkit-box;
+    -webkit-line-clamp: 2;
+    -webkit-box-orient: vertical;
+    overflow: hidden;
+}
+
+.article-date {
+    color: var(--secondary-color);
+    font-size: 0.8rem;
+    margin: 0.25rem 0;
+}
+
+/* Pagination */
+.pagination {
+    display: flex;
+    justify-content: center;
+    align-items: center;
+    gap: 1.5rem;
+    margin: 1.5rem 0;
+    padding: 1rem 0;
+    border-top: 1px solid var(--border-color);
+}
+
+.pagination a {
+    color: var(--accent-color);
+    text-decoration: none;
+    font-weight: 500;
+    font-size: 0.9rem;
+}
+
+.pagination a:hover {
+    text-decoration: underline;
+}
+
+.pagination span {
+    color: var(--secondary-color);
+    font-size: 0.85rem;
+}
+
+/* Article detail view */
+.article-view {
+    background: var(--surface-color);
+    border: 1px solid var(--border-color);
+    border-radius: 12px;
+    padding: 2.5rem;
+    box-shadow: 0 1px 3px rgba(0,0,0,0.04);
+}
+
+.article-source {
+    display: inline-block;
+    background: var(--accent-light);
+    color: var(--accent-color);
+    padding: 0.25rem 0.75rem;
+    border-radius: 20px;
+    font-size: 0.75rem;
+    font-weight: 600;
+    text-transform: uppercase;
+    letter-spacing: 0.05em;
+    margin-bottom: 1rem;
+}
+
+.article-view h1 {
+    margin: 0 0 1rem 0;
+    font-size: 1.75rem;
+    line-height: 1.3;
+    font-weight: 700;
+    letter-spacing: -0.025em;
+}
+
+.article-meta {
+    border-bottom: 1px solid var(--border-color);
+    padding-bottom: 1rem;
+    margin-bottom: 1.5rem;
+    display: flex;
+    gap: 1.5rem;
+    flex-wrap: wrap;
+}
+
+.article-meta time,
+.article-meta span {
+    font-size: 0.85rem;
+    color: var(--secondary-color);
+}
+
+.article-meta a {
+    color: var(--accent-color);
+    text-decoration: none;
+    font-size: 0.85rem;
+    word-break: break-all;
+}
+
+.article-meta a:hover {
+    text-decoration: underline;
+}
+
+.article-content {
+    font-size: 1.05rem;
+    line-height: 1.8;
+}
+
+.article-text {
+    white-space: pre-wrap;
+    color: var(--primary-color);
+}
+
+.article-text p {
+    margin: 0 0 1rem 0;
+}
+
+.archived-html {
+    border: 1px solid var(--border-color);
+    border-radius: 8px;
+    padding: 1rem;
+    margin-top: 1rem;
+    background: var(--background-color);
+}
+
+.article-actions {
+    margin-top: 2rem;
+    padding-top: 1.5rem;
+    border-top: 1px solid var(--border-color);
+    display: flex;
+    gap: 1rem;
+    align-items: center;
+}
+
+.back-link {
+    color: var(--accent-color);
+    text-decoration: none;
+    font-weight: 600;
+    font-size: 0.9rem;
+    padding: 0.5rem 1rem;
+    background: var(--accent-light);
+    border-radius: 6px;
+    transition: all 0.15s ease;
+}
+
+.back-link:hover {
+    background: var(--accent-color);
+    color: white;
+    text-decoration: none;
+}
+
+/* Disabled feeds */
+.source-disabled {
+    color: var(--secondary-color);
+    text-decoration: line-through;
+}
+
+.status-badge {
+    display: inline-block;
+    background: #ef4444;
+    color: white;
+    padding: 0.1rem 0.4rem;
+    border-radius: 3px;
+    font-size: 0.7rem;
+    margin-left: 0.5rem;
+}
+
+.disable-reason {
+    color: var(--secondary-color);
+    font-size: 0.8rem;
+    margin-top: 0.25rem;
+    font-style: italic;
+}
+
+/* Status page */
+.status-page {
+    background: var(--surface-color);
+    border: 1px solid var(--border-color);
+    border-radius: 12px;
+    padding: 2rem;
+}
+
+.status-page h1 {
+    margin: 0 0 1.5rem 0;
+    font-size: 1.5rem;
+    font-weight: 700;
+}
+
+.status-page h2 {
+    font-size: 1.1rem;
+    margin: 1.5rem 0 1rem;
+    font-weight: 600;
+}
+
+.status-summary {
+    display: grid;
+    grid-template-columns: repeat(auto-fit, minmax(150px, 1fr));
+    gap: 1rem;
+    margin-bottom: 1.5rem;
+}
+
+.status-item {
+    background: var(--background-color);
+    border-radius: 8px;
+    padding: 1rem;
+}
+
+.status-item h2 {
+    margin: 0 0 0.25rem;
+    font-size: 0.8rem;
+    color: var(--secondary-color);
+    text-transform: uppercase;
+    letter-spacing: 0.05em;
+}
+
+.status-item p {
+    margin: 0;
+    font-size: 1.5rem;
+    font-weight: 700;
+}
+
+.status-online { color: #16a34a; }
+.status-warning { color: #f59e0b; }
+
+/* Not found */
+.article-not-found {
+    text-align: center;
+    padding: 4rem 2rem;
+}
+
+.article-not-found h1 {
+    font-size: 1.25rem;
+    margin-bottom: 0.5rem;
+}
+
+.article-not-found p {
+    color: var(--secondary-color);
+    margin-bottom: 1.5rem;
+}
+
+/* Responsive */
+@media (max-width: 768px) {
+    header {
+        padding: 0.75rem 1rem;
+    }
+
+    main {
+        padding: 1rem;
+    }
+
+    .timeline {
+        padding-left: 2.5rem;
+    }
+
+    .timeline::before {
+        left: 0.75rem;
+    }
+
+    .timeline-date::before {
+        left: -1.95rem;
+        width: 10px;
+        height: 10px;
+    }
+
+    .article-view {
+        padding: 1.5rem;
+    }
+
+    .article-view h1 {
+        font-size: 1.35rem;
+    }
+
+    .newspaper-item {
+        flex-direction: column;
+        align-items: flex-start;
+        gap: 0.75rem;
+    }
+}
+
+/* Print */
+@media print {
+    header, .pull-btn, .pagination, .article-actions {
+        display: none;
+    }
+    .article-view {
+        border: none;
+        padding: 0;
+        box-shadow: none;
+    }
+}

stop_services.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+# Stop NewsArchiver services
+
+echo "Stopping NewsArchiver services..."
+
+# Stop web server
+pkill -f "run_archiver.py --serve" 2>/dev/null || true
+echo "Web server stopped"
+
+# Stop scheduler
+pkill -f "run_archiver.py --interval" 2>/dev/null || true
+echo "Scheduler stopped"
+
+# Remove lock file
+LOCK_FILE="${NEWSARCHIVER_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)}/archival_data/.scheduler.lock"
+rm -f "$LOCK_FILE" 2>/dev/null || true
+echo "Scheduler lock file removed"
+
+echo "All NewsArchiver services stopped"

storage_manager.py

@@ -0,0 +1,859 @@
+#!/usr/bin/env python3
+"""Storage Manager for NewsArchiver - Phase 2.3
+
+Organizes archived data by newspaper, manages SQLite cache database,
+stores both raw HTML and extracted content.
+"""
+
+import json
+import logging
+import os
+import sqlite3
+import sys
+from datetime import datetime
+from pathlib import Path
+from typing import List, Optional
+
+from content_extractor import ArticleData
+
+try:
+    import feedgenerator
+
+    FEEDGENERATOR_AVAILABLE = True
+except ImportError:
+    FEEDGENERATOR_AVAILABLE = False
+
+SCRIPT_DIR = Path(__file__).parent.resolve()
+ARCHIVE_DIR = Path(os.environ.get("ARCHIVE_DIR", str(SCRIPT_DIR / "archival_data"))).resolve()
+ARCHIVE_DIR.mkdir(exist_ok=True)
+
+logger = logging.getLogger(__name__)
+
+DB_PATH = ARCHIVE_DIR / "cache.db"
+WEBSITES_DIR = ARCHIVE_DIR / "websites"
+
+
+def _get_db_connection() -> sqlite3.Connection:
+    """Get database connection with row factory."""
+    conn = sqlite3.connect(DB_PATH, timeout=30.0, isolation_level=None)
+    conn.row_factory = sqlite3.Row
+    conn.execute("PRAGMA journal_mode=WAL")
+    conn.execute("PRAGMA busy_timeout=30000")
+    return conn
+
+
+def _init_database() -> None:
+    """Initialize database schema."""
+    with _get_db_connection() as conn:
+        cursor = conn.cursor()
+
+        cursor.execute("""
+            CREATE TABLE IF NOT EXISTS articles (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                source_name TEXT NOT NULL,
+                article_url TEXT NOT NULL UNIQUE,
+                article_guid TEXT,
+                title TEXT,
+                author TEXT,
+                publish_date TEXT,
+                content_text TEXT,
+                content_html TEXT,
+                archive_file_path TEXT,
+                metadata_file_path TEXT,
+                status TEXT DEFAULT 'pending',
+                error_message TEXT,
+                extraction_method TEXT,
+                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+                updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+            )
+        """)
+
+        cursor.execute("""
+            CREATE TABLE IF NOT EXISTS article_archives (
+                article_url TEXT PRIMARY KEY,
+                source_name TEXT,
+                archive_file_path TEXT,
+                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+            )
+        """)
+
+        cursor.execute("""
+            CREATE TABLE IF NOT EXISTS processing_log (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                source_name TEXT,
+                timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+                action TEXT,
+                status TEXT,
+                message TEXT
+            )
+        """)
+
+        cursor.execute(
+            "CREATE INDEX IF NOT EXISTS idx_articles_source ON articles(source_name)"
+        )
+        cursor.execute(
+            "CREATE INDEX IF NOT EXISTS idx_articles_url ON articles(article_url)"
+        )
+        cursor.execute(
+            "CREATE INDEX IF NOT EXISTS idx_articles_status ON articles(status)"
+        )
+
+        # Add extraction_method column if it doesn't exist
+        try:
+            cursor.execute("ALTER TABLE articles ADD COLUMN extraction_method TEXT")
+            conn.commit()
+        except sqlite3.OperationalError:
+            pass
+
+        cursor.execute(
+            "CREATE INDEX IF NOT EXISTS idx_articles_extraction_method ON articles(extraction_method)"
+        )
+        cursor.execute(
+            "CREATE INDEX IF NOT EXISTS idx_article_archives_url ON article_archives(article_url)"
+        )
+        cursor.execute(
+            "CREATE INDEX IF NOT EXISTS idx_article_archives_source ON article_archives(source_name)"
+        )
+
+        logger.debug("Database initialized at %s", DB_PATH)
+
+
+def _ensure_directory_structure(source_name: str, date_str: str) -> tuple:
+    """Ensure directory structure exists for a source and date.
+
+    Args:
+        source_name: Newspaper source name
+        date_str: Date string in YYYY-MM-DD format
+
+    Returns:
+        Tuple of (html_dir, articles_dir) Path objects
+    """
+    source_dir = WEBSITES_DIR / source_name
+    html_dir = source_dir / "html" / date_str
+    articles_dir = source_dir / "articles" / date_str
+
+    html_dir.mkdir(parents=True, exist_ok=True)
+    articles_dir.mkdir(parents=True, exist_ok=True)
+
+    return html_dir, articles_dir
+
+
+def _get_next_file_index(html_dir: Path, articles_dir: Path) -> int:
+    """Get next available file index for article naming.
+
+    Args:
+        html_dir: Directory containing HTML files
+        articles_dir: Directory containing JSON files
+
+    Returns:
+        Next available index (1-indexed)
+    """
+
+    def get_max_index(directory: Path, extension: str) -> int:
+        max_idx = 0
+        if directory.exists():
+            for file in directory.glob(f"*{extension}"):
+                try:
+                    name = file.stem
+                    if name.startswith("article_"):
+                        idx = int(name.replace("article_", ""))
+                        max_idx = max(max_idx, idx)
+                except ValueError:
+                    continue
+        return max_idx
+
+    html_idx = get_max_index(html_dir, ".html")
+    json_idx = get_max_index(articles_dir, ".json")
+
+    return max(html_idx, json_idx) + 1
+
+
+def _log_processing(source_name: str, action: str, status: str, message: str) -> None:
+    """Log processing action to database.
+
+    Args:
+        source_name: Newspaper source name
+        action: Action performed
+        status: Status of action
+        message: Log message
+    """
+    try:
+        with _get_db_connection() as conn:
+            cursor = conn.cursor()
+            cursor.execute(
+                "INSERT INTO processing_log (source_name, action, status, message) VALUES (?, ?, ?, ?)",
+                (source_name, action, status, message),
+            )
+    except Exception as e:
+        logger.error("Failed to log processing: %s", str(e))
+
+
+def _save_archive_mapping(
+    article_url: str, source_name: str, archive_file_path: str
+) -> None:
+    """Save mapping between article URL and archive file path.
+
+    Args:
+        article_url: Article URL
+        source_name: Newspaper source name
+        archive_file_path: Path to archived HTML file
+    """
+    try:
+        with _get_db_connection() as conn:
+            cursor = conn.cursor()
+            cursor.execute(
+                """
+                INSERT OR REPLACE INTO article_archives (article_url, source_name, archive_file_path)
+                VALUES (?, ?, ?)
+            """,
+                (article_url, source_name, archive_file_path),
+            )
+        logger.debug("Saved archive mapping: %s -> %s", article_url, archive_file_path)
+    except Exception as e:
+        logger.error("Failed to save archive mapping: %s", str(e))
+
+
+def save_article(source_name: str, article_data: ArticleData) -> str:
+    """Save article to storage and update cache.
+
+    Args:
+        source_name: Newspaper source name (e.g., 'reuters', 'bbc')
+        article_data: ArticleData object with article content
+
+    Returns:
+        Status message describing the result
+    """
+    try:
+        _init_database()
+
+        publish_date = article_data.publish_date
+        if publish_date:
+            try:
+                date_obj = datetime.fromisoformat(publish_date.replace("Z", "+00:00"))
+                date_str = date_obj.strftime("%Y-%m-%d")
+            except (ValueError, AttributeError):
+                date_str = datetime.now().strftime("%Y-%m-%d")
+        else:
+            date_str = datetime.now().strftime("%Y-%m-%d")
+
+        html_dir, articles_dir = _ensure_directory_structure(source_name, date_str)
+
+        file_index = _get_next_file_index(html_dir, articles_dir)
+        file_prefix = f"article_{file_index:03d}"
+
+        archive_file_path = html_dir / f"{file_prefix}.html"
+        metadata_file_path = articles_dir / f"{file_prefix}.json"
+
+        if article_data.raw_html:
+            with open(archive_file_path, "w", encoding="utf-8") as f:
+                f.write(article_data.raw_html)
+
+        metadata = {
+            "id": file_index,
+            "source_name": source_name,
+            "url": article_data.url,
+            "title": article_data.title,
+            "author": article_data.author,
+            "publish_date": article_data.publish_date,
+            "content_text": article_data.content_text,
+            "content_html": article_data.content_html,
+            "tags": article_data.tags,
+            "language": article_data.language,
+            "extraction_method": article_data.extraction_method,
+            "archive_file": f"{file_prefix}.html",
+            "metadata_file": f"{file_prefix}.json",
+            "saved_at": datetime.now().isoformat(),
+        }
+
+        with open(metadata_file_path, "w", encoding="utf-8") as f:
+            json.dump(metadata, f, indent=2, ensure_ascii=False)
+
+        with _get_db_connection() as conn:
+            cursor = conn.cursor()
+
+            try:
+                cursor.execute(
+                    """
+                    INSERT OR IGNORE INTO articles
+                    (source_name, article_url, article_guid, title, author, publish_date,
+                     content_text, content_html, archive_file_path, metadata_file_path, status, extraction_method)
+                    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                """,
+                    (
+                        source_name,
+                        article_data.url,
+                        getattr(article_data, "guid", None),
+                        article_data.title,
+                        article_data.author,
+                        article_data.publish_date,
+                        article_data.content_text,
+                        article_data.content_html,
+                        str(archive_file_path.relative_to(ARCHIVE_DIR)),
+                        str(metadata_file_path.relative_to(ARCHIVE_DIR)),
+                        "archived" if not article_data.error else "failed",
+                        article_data.extraction_method,
+                    ),
+                )
+            except sqlite3.IntegrityError:
+                cursor.execute(
+                    """
+                    UPDATE articles
+                    SET title = ?, author = ?, publish_date = ?,
+                        content_text = ?, content_html = ?,
+                        archive_file_path = ?, metadata_file_path = ?,
+                        status = ?, extraction_method = ?, updated_at = CURRENT_TIMESTAMP
+                    WHERE article_url = ? AND source_name = ?
+                """,
+                    (
+                        article_data.title,
+                        article_data.author,
+                        article_data.publish_date,
+                        article_data.content_text,
+                        article_data.content_html,
+                        str(archive_file_path.relative_to(ARCHIVE_DIR)),
+                        str(metadata_file_path.relative_to(ARCHIVE_DIR)),
+                        "archived" if not article_data.error else "failed",
+                        article_data.extraction_method,
+                        article_data.url,
+                        source_name,
+                    ),
+                )
+
+        _save_archive_mapping(
+            article_data.url,
+            source_name,
+            str(archive_file_path.relative_to(ARCHIVE_DIR)),
+        )
+
+        _log_processing(
+            source_name,
+            "save_article",
+            "success",
+            f"Saved article: {article_data.url} -> {metadata_file_path.name}",
+        )
+
+        logger.info(
+            "Article saved: %s -> %s", article_data.url, metadata_file_path.name
+        )
+
+        return f"Article saved: {metadata_file_path.name}"
+
+    except Exception as e:
+        error_msg = f"Failed to save article: {str(e)}"
+        logger.error(error_msg)
+        _log_processing(source_name, "save_article", "error", error_msg)
+        return error_msg
+
+
+def get_article(source_name: str, article_id: int) -> Optional[ArticleData]:
+    """Retrieve article from storage.
+
+    Args:
+        source_name: Newspaper source name
+        article_id: Database ID of article
+
+    Returns:
+        ArticleData object or None if not found
+    """
+    try:
+        _init_database()
+
+        with _get_db_connection() as conn:
+            cursor = conn.cursor()
+            cursor.execute(
+                """
+                SELECT * FROM articles
+                WHERE id = ? AND source_name = ?
+            """,
+                (article_id, source_name),
+            )
+
+            row = cursor.fetchone()
+
+        if not row:
+            return None
+
+        archive_path = (
+            Path(row["archive_file_path"]) if row["archive_file_path"] else None
+        )
+        # Handle both absolute and relative paths
+        if archive_path:
+            if not archive_path.is_absolute():
+                archive_path = ARCHIVE_DIR / archive_path
+            # Return relative path for web interface
+            archive_file_path = str(archive_path.relative_to(ARCHIVE_DIR))
+        else:
+            archive_file_path = None
+        archive_content = None
+        if archive_path and archive_path.exists():
+            archive_content = archive_path.read_text(encoding="utf-8")
+
+        article = ArticleData(
+            url=row["article_url"],
+            title=row["title"],
+            author=row["author"],
+            publish_date=row["publish_date"],
+            content_text=row["content_text"],
+            content_html=row["content_html"],
+            raw_html=archive_content,
+            archive_file_path=archive_file_path,
+            tags=None,
+            language=None,
+            metadata=None,
+            extraction_method=None,
+            error=row["error_message"],
+            guid=row["article_guid"],
+            id=row["id"],
+            source_name=source_name,
+        )
+
+        return article
+
+    except Exception as e:
+        logger.error("Failed to get article %d: %s", article_id, str(e))
+        return None
+
+
+def get_articles_by_source(
+    source_name: str, limit: int = 50, offset: int = 0
+) -> List[ArticleData]:
+    """Get paginated articles for a source.
+
+    Args:
+        source_name: Newspaper source name
+        limit: Maximum number of articles to return
+        offset: Number of articles to skip
+
+    Returns:
+        List of ArticleData objects
+    """
+    try:
+        _init_database()
+
+        with _get_db_connection() as conn:
+            cursor = conn.cursor()
+            cursor.execute(
+                """
+                SELECT * FROM articles
+                WHERE source_name = ?
+                ORDER BY publish_date DESC, created_at DESC
+                LIMIT ? OFFSET ?
+            """,
+                (source_name, limit, offset),
+            )
+
+            rows = cursor.fetchall()
+
+        articles = []
+        for row in rows:
+            archive_path = (
+                Path(row["archive_file_path"]) if row["archive_file_path"] else None
+            )
+            # Handle both absolute and relative paths
+            if archive_path:
+                if not archive_path.is_absolute():
+                    archive_path = ARCHIVE_DIR / archive_path
+                # Return relative path for web interface
+                archive_file_path = str(archive_path.relative_to(ARCHIVE_DIR))
+            else:
+                archive_file_path = None
+            archive_content = None
+            if archive_path and archive_path.exists():
+                archive_content = archive_path.read_text(encoding="utf-8")
+
+            article = ArticleData(
+                url=row["article_url"],
+                title=row["title"],
+                author=row["author"],
+                publish_date=row["publish_date"],
+                content_text=row["content_text"],
+                content_html=row["content_html"],
+                raw_html=archive_content,
+                archive_file_path=archive_file_path,
+                tags=None,
+                language=None,
+                metadata=None,
+                extraction_method=None,
+                error=row["error_message"],
+                guid=row["article_guid"],
+                id=row["id"],
+                source_name=source_name,
+            )
+
+            articles.append(article)
+
+        return articles
+
+    except Exception as e:
+        logger.error("Failed to get articles for %s: %s", source_name, str(e))
+        return []
+
+
+def update_article_status(
+    source_name: str, article_url: str, status: str, error: str = None
+) -> None:
+    """Update article status in cache.
+
+    Args:
+        source_name: Newspaper source name
+        article_url: Article URL
+        status: New status (pending, archived, failed)
+        error: Error message if status is failed
+    """
+    try:
+        _init_database()
+
+        conn = _get_db_connection()
+        cursor = conn.cursor()
+        cursor.execute(
+            """
+            UPDATE articles
+            SET status = ?, error_message = ?, updated_at = CURRENT_TIMESTAMP
+            WHERE article_url = ? AND source_name = ?
+        """,
+            (status, error, article_url, source_name),
+        )
+
+        conn.commit()
+        conn.close()
+
+        if error:
+            _log_processing(
+                source_name,
+                "update_status",
+                "error",
+                f"Updated {article_url} status to {status}: {error}",
+            )
+        else:
+            _log_processing(
+                source_name,
+                "update_status",
+                "success",
+                f"Updated {article_url} status to {status}",
+            )
+
+        logger.info("Updated article status: %s -> %s", article_url, status)
+
+    except Exception as e:
+        logger.error("Failed to update article status: %s", str(e))
+
+
+def get_source_stats(source_name: str) -> dict:
+    """Get statistics for a news source.
+
+    Args:
+        source_name: Newspaper source name
+
+    Returns:
+        Dictionary with source statistics
+    """
+    try:
+        _init_database()
+
+        conn = _get_db_connection()
+        cursor = conn.cursor()
+
+        cursor.execute(
+            """
+            SELECT
+                COUNT(*) as total,
+                SUM(CASE WHEN status = 'archived' THEN 1 ELSE 0 END) as archived,
+                SUM(CASE WHEN status = 'failed' THEN 1 ELSE 0 END) as failed,
+                SUM(CASE WHEN status = 'pending' THEN 1 ELSE 0 END) as pending,
+                MIN(created_at) as first_archived,
+                MAX(created_at) as last_archived,
+                MAX(publish_date) as latest_article_date
+            FROM articles
+            WHERE source_name = ?
+        """,
+            (source_name,),
+        )
+
+        row = cursor.fetchone()
+        conn.close()
+
+        stats = {
+            "source_name": source_name,
+            "total_articles": row["total"] or 0,
+            "archived": row["archived"] or 0,
+            "failed": row["failed"] or 0,
+            "pending": row["pending"] or 0,
+            "first_archived": row["first_archived"],
+            "last_archived": row["last_archived"],
+            "latest_article_date": row["latest_article_date"],
+        }
+
+        _log_processing(
+            source_name,
+            "get_stats",
+            "success",
+            f"Stats: {stats['total_articles']} total, {stats['archived']} archived, {stats['failed']} failed",
+        )
+
+        return stats
+
+    except Exception as e:
+        logger.error("Failed to get stats for %s: %s", source_name, str(e))
+        return {
+            "source_name": source_name,
+            "total_articles": 0,
+            "archived": 0,
+            "failed": 0,
+            "pending": 0,
+        }
+
+
+def get_all_sources() -> List[str]:
+    """Get list of all sources in the archive.
+
+    Returns:
+        List of source names
+    """
+    try:
+        _init_database()
+
+        conn = _get_db_connection()
+        cursor = conn.cursor()
+        cursor.execute("""
+            SELECT DISTINCT source_name FROM articles ORDER BY source_name
+        """)
+
+        sources = [row["source_name"] for row in cursor.fetchall()]
+        conn.close()
+
+        return sources
+
+    except Exception as e:
+        logger.error("Failed to get sources: %s", str(e))
+        return []
+
+
+def get_latest_articles(limit: int = 50) -> List[ArticleData]:
+    """Get latest articles across all sources.
+
+    Args:
+        limit: Maximum number of articles to return
+
+    Returns:
+        List of ArticleData objects ordered by publish_date DESC
+    """
+    try:
+        _init_database()
+
+        conn = _get_db_connection()
+        cursor = conn.cursor()
+        cursor.execute(
+            """
+            SELECT * FROM articles
+            ORDER BY publish_date DESC, created_at DESC
+            LIMIT ?
+        """,
+            (limit,),
+        )
+
+        rows = cursor.fetchall()
+        conn.close()
+
+        articles = []
+        for row in rows:
+            archive_path = (
+                Path(row["archive_file_path"]) if row["archive_file_path"] else None
+            )
+            # Handle both absolute and relative paths
+            if archive_path:
+                if not archive_path.is_absolute():
+                    archive_path = ARCHIVE_DIR / archive_path
+                # Return relative path for web interface
+                archive_file_path = str(archive_path.relative_to(ARCHIVE_DIR))
+            else:
+                archive_file_path = None
+            archive_content = None
+            if archive_path and archive_path.exists():
+                archive_content = archive_path.read_text(encoding="utf-8")
+
+            article = ArticleData(
+                url=row["article_url"],
+                title=row["title"],
+                author=row["author"],
+                publish_date=row["publish_date"],
+                content_text=row["content_text"],
+                content_html=row["content_html"],
+                raw_html=archive_content,
+                archive_file_path=archive_file_path,
+                tags=None,
+                language=None,
+                metadata=None,
+                extraction_method=None,
+                error=row["error_message"],
+                guid=row["article_guid"],
+                id=row["id"],
+                source_name=row["source_name"],
+            )
+
+            articles.append(article)
+
+        return articles
+
+    except Exception as e:
+        logger.error("Failed to get latest articles: %s", str(e))
+        return []
+
+
+def get_source_directory(source_name: str) -> Path:
+    """Get the directory path for a source.
+
+    Args:
+        source_name: Newspaper source name
+
+    Returns:
+        Path to source directory
+    """
+    return WEBSITES_DIR / source_name
+
+
+def get_archive_file_path_from_db(
+    article_url: str, source_name: str = None
+) -> Optional[str]:
+    """Get archive file path from database mapping.
+
+    Args:
+        article_url: Article URL
+        source_name: Newspaper source name (optional, for filtering)
+
+    Returns:
+        Archive file path if found, None otherwise
+    """
+    try:
+        _init_database()
+
+        conn = _get_db_connection()
+        cursor = conn.cursor()
+
+        if source_name:
+            cursor.execute(
+                """
+                SELECT archive_file_path FROM article_archives
+                WHERE article_url = ? AND source_name = ?
+            """,
+                (article_url, source_name),
+            )
+        else:
+            cursor.execute(
+                """
+                SELECT archive_file_path FROM article_archives
+                WHERE article_url = ?
+            """,
+                (article_url,),
+            )
+
+        row = cursor.fetchone()
+        conn.close()
+
+        return row["archive_file_path"] if row else None
+
+    except Exception as e:
+        logger.error("Failed to get archive file path from DB: %s", str(e))
+        return None
+
+
+def get_daily_articles(source_name: str, date_str: str) -> List[ArticleData]:
+    """Get articles for a specific date.
+
+    Args:
+        source_name: Newspaper source name
+        date_str: Date string in YYYY-MM-DD format
+
+    Returns:
+        List of ArticleData objects
+    """
+    try:
+        source_dir = get_source_directory(source_name)
+        articles_dir = source_dir / "articles" / date_str
+
+        if not articles_dir.exists():
+            return []
+
+        articles = []
+        for json_file in sorted(articles_dir.glob("article_*.json")):
+            try:
+                with open(json_file, "r", encoding="utf-8") as f:
+                    metadata = json.load(f)
+
+                archive_filename = metadata.get("archive_file", "")
+                archive_path = (
+                    source_dir / "html" / date_str / archive_filename
+                    if archive_filename
+                    else None
+                )
+                archive_content = None
+                if archive_path and archive_path.exists():
+                    archive_content = archive_path.read_text(encoding="utf-8")
+
+                article = ArticleData(
+                    url=metadata.get("url", ""),
+                    title=metadata.get("title"),
+                    author=metadata.get("author"),
+                    publish_date=metadata.get("publish_date"),
+                    content_text=metadata.get("content_text"),
+                    content_html=metadata.get("content_html"),
+                    raw_html=archive_content,
+                    # Handle both absolute and relative paths
+                    archive_file_path=str(archive_path.relative_to(ARCHIVE_DIR))
+                    if archive_path
+                    else None,
+                    tags=metadata.get("tags"),
+                    language=metadata.get("language"),
+                    metadata=metadata,
+                    extraction_method=metadata.get("extraction_method"),
+                    source_name=source_name,
+                )
+
+                articles.append(article)
+
+            except Exception as e:
+                logger.warning("Failed to load article from %s: %s", json_file, str(e))
+                continue
+
+        return articles
+
+    except Exception as e:
+        logger.error(
+            "Failed to get daily articles for %s on %s: %s",
+            source_name,
+            date_str,
+            str(e),
+        )
+        return []
+
+
+def initialize_storage() -> None:
+    """Initialize the storage system.
+
+    Creates database, directory structure, and logs initialization.
+    """
+    logger.info("Initializing storage system...")
+
+    _init_database()
+
+    (WEBSITES_DIR / "sample").mkdir(parents=True, exist_ok=True)
+    (WEBSITES_DIR / "sample" / "html").mkdir(exist_ok=True)
+    (WEBSITES_DIR / "sample" / "articles").mkdir(exist_ok=True)
+
+    _log_processing("system", "initialize", "success", "Storage system initialized")
+
+    logger.info("Storage system initialized at %s", ARCHIVE_DIR)
+
+
+if __name__ == "__main__":
+    initialize_storage()
+
+    sources = get_all_sources()
+    print(f"Sources in archive: {sources}")
+
+    if sources:
+        for source in sources:
+            stats = get_source_stats(source)
+            print(f"\n{source}:")
+            print(f"  Total: {stats['total_articles']}")
+            print(f"  Archived: {stats['archived']}")
+            print(f"  Failed: {stats['failed']}")

templates/article.html

@@ -0,0 +1,46 @@
+{% extends "base.html" %}
+
+{% block content %}
+<article class="article-view">
+    <div class="article-header">
+        <div class="article-source">
+            <span class="source-label">From</span>
+            <span class="source-name">{{ source_name }}</span>
+        </div>
+        <h1>{{ article.title }}</h1>
+        
+        <div class="article-meta">
+            <div class="article-meta-row">
+                {% if article.publish_date %}
+                <time class="article-date" datetime="{{ article.publish_date }}">{{ article.publish_date }}</time>
+                {% endif %}
+                {% if article.author %}
+                <span class="article-author">By <strong>{{ article.author }}</strong></span>
+                {% endif %}
+            </div>
+            <p class="article-url"><a href="{{ article.url }}" target="_blank">{{ article.url }}</a></p>
+        </div>
+    </div>
+    
+    <div class="article-content">
+        {% if article.content_text %}
+        <div class="article-text">
+            {% set lines = article.content_text.split('\n') -%}
+            {%- for line in lines %}
+            {%- if line|trim %}
+            <p>{{ line }}</p>
+            {%- endif %}
+            {%- endfor %}
+        </div>
+        {%- endif %}
+    </div>
+    
+    <div class="article-actions">
+        <a href="/source/{{ source_slug }}" class="back-link">&larr; Back to articles</a>
+        {% if article.archive_file_path %}
+        &nbsp;|&nbsp;
+        <a href="/archive-file/{{ article.archive_file_path|urlencode }}" target="_blank" title="View archived copy">Archived HTML</a>
+        {% endif %}
+    </div>
+</article>
+{% endblock %}

templates/article_not_found.html

@@ -0,0 +1,11 @@
+{% extends "base.html" %}
+
+{% block content %}
+<div class="article-not-found">
+    <h1>Article Not Found</h1>
+    <p>The article you're looking for could not be found.</p>
+    <p>Source: {{ slug }}</p>
+    <p>ID: {{ article_id }}</p>
+    <a href="/source/{{ slug }}" class="back-link">&larr; Back to articles</a>
+</div>
+{% endblock %}

templates/articles.html

@@ -0,0 +1,44 @@
+{% extends "base.html" %}
+
+{% block content %}
+<h1>{{ source_name }} - Articles</h1>
+
+<div class="pagination">
+    {% if pagination.has_prev %}
+        <a href="/source/{{ source_slug }}?page={{ pagination.prev_num }}">&laquo; Previous</a>
+    {% endif %}
+    <span>Page {{ pagination.page }} of {{ pagination.pages }}</span>
+    {% if pagination.has_next %}
+        <a href="/source/{{ source_slug }}?page={{ pagination.next_num }}">Next &raquo;</a>
+    {% endif %}
+</div>
+
+<div class="timeline">
+{% for date, articles in articles_by_date %}
+    <div class="timeline-date">
+        <span class="date-label">{{ date }}</span>
+        <span class="date-count">{{ articles|length }} article{% if articles|length != 1 %}s{% endif %}</span>
+    </div>
+    <ul class="article-list">
+    {% for article in articles %}
+        <li class="article-item">
+            <div class="article-main">
+                <h3><a href="{{ article.url }}">{{ article.title }}</a></h3>
+                <p class="article-summary">{{ article.summary }}</p>
+            </div>
+        </li>
+    {% endfor %}
+    </ul>
+{% endfor %}
+</div>
+
+<div class="pagination">
+    {% if pagination.has_prev %}
+        <a href="/source/{{ source_slug }}?page={{ pagination.prev_num }}">&laquo; Previous</a>
+    {% endif %}
+    <span>Page {{ pagination.page }} of {{ pagination.pages }}</span>
+    {% if pagination.has_next %}
+        <a href="/source/{{ source_slug }}?page={{ pagination.next_num }}">Next &raquo;</a>
+    {% endif %}
+</div>
+{% endblock %}

templates/atom.xml

@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<feed xmlns="http://www.w3.org/2005/Atom">
+    <title>{{ title|e }}</title>
+    <link href="{{ link|e }}" rel="alternate"/>
+    <link href="{{ link|e }}/atom" rel="self" type="application/atom+xml"/>
+    <updated>{{ updated|e }}</updated>
+    <id>{{ link|e }}</id>
+    <generator>NewsArchiver</generator>
+    {% for entry in entries %}
+    <entry>
+        <title>{{ entry.title|e }}</title>
+        <link href="{{ entry.link|e }}" rel="alternate"/>
+        {% if entry.published %}
+        <published>{{ entry.published|e }}</published>
+        {% endif %}
+        {% if entry.author %}
+        <author>
+            <name>{{ entry.author.name|e }}</name>
+        </author>
+        {% endif %}
+        <id>{{ entry.id|e }}</id>
+        <summary>{{ entry.summary|e }}</summary>
+    </entry>
+    {% endfor %}
+</feed>

templates/base.html

@@ -0,0 +1,80 @@
+<!DOCTYPE html>
+<html lang="en" id="html" data-theme="light">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>NewsArchiver</title>
+    <meta name="description" content="NewsArchiver - Archive news articles from RSS feeds">
+    <meta property="og:title" content="NewsArchiver">
+    <meta property="og:description" content="Archive news articles from RSS feeds">
+    <meta property="og:type" content="website">
+    <link rel="icon" type="image/svg+xml" href="data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 100 100'%3E%3Ctext y='.9em' font-size='90'%3E📰%3C/text%3E%3C/svg%3E">
+    <link rel="stylesheet" href="{{ url_for('static', filename='style.css') }}">
+    <style>
+        #theme-toggle {
+            background: none;
+            border: 1px solid var(--border-color);
+            color: var(--primary-color);
+            padding: 0.5rem 1rem;
+            border-radius: 4px;
+            cursor: pointer;
+            font-size: 0.9rem;
+            transition: all 0.3s ease;
+        }
+        
+        #theme-toggle:hover {
+            background-color: var(--accent-color);
+            color: white;
+            border-color: var(--accent-color);
+        }
+        
+        .theme-icon {
+            margin-right: 0.5rem;
+        }
+    </style>
+</head>
+<body>
+    <header>
+        <h1>NewsArchiver</h1>
+        <nav>
+            <a href="/">Archives</a>
+            <a href="/status">Status</a>
+            <button id="theme-toggle" aria-label="Toggle dark mode">
+                <span class="theme-icon" id="theme-icon">☀️</span>
+                <span id="theme-text">Light</span>
+            </button>
+        </nav>
+    </header>
+    <main>
+        {% block content %}{% endblock %}
+    </main>
+    <script>
+        const html = document.getElementById('html');
+        const themeToggle = document.getElementById('theme-toggle');
+        const themeIcon = document.getElementById('theme-icon');
+        const themeText = document.getElementById('theme-text');
+        
+        const savedTheme = localStorage.getItem('theme') || 'light';
+        setTheme(savedTheme);
+        
+        themeToggle.addEventListener('click', () => {
+            const currentTheme = html.getAttribute('data-theme');
+            const newTheme = currentTheme === 'light' ? 'dark' : 'light';
+            setTheme(newTheme);
+        });
+        
+        function setTheme(theme) {
+            html.setAttribute('data-theme', theme);
+            localStorage.setItem('theme', theme);
+            
+            if (theme === 'dark') {
+                themeIcon.textContent = '☀️';
+                themeText.textContent = 'Light';
+            } else {
+                themeIcon.textContent = '🌙';
+                themeText.textContent = 'Dark';
+            }
+        }
+    </script>
+</body>
+</html>

templates/index.html

@@ -0,0 +1,31 @@
+{% extends "base.html" %}
+
+{% block content %}
+<h1>News Archives</h1>
+<p>Total sources: {{ sources|length }}</p>
+
+<ul class="newspaper-list">
+{% for source in sources %}
+    <li class="newspaper-item {% if source.disabled %}disabled{% endif %}">
+        <div class="newspaper-info">
+            {% if source.disabled %}
+                <h2 class="source-disabled" title="{{ source.disable_reason }}">
+                    {{ source.name }}
+                    <span class="status-badge">Disabled</span>
+                </h2>
+            {% else %}
+                <h2><a href="/source/{{ source.slug }}">{{ source.name }}</a></h2>
+            {% endif %}
+            <p>Articles: {{ source.article_count }}</p>
+            <p>Last archived: {{ source.last_archived or 'N/A' }}</p>
+            {% if source.disabled %}
+                <p class="disable-reason">{{ source.disable_reason }}</p>
+            {% endif %}
+        </div>
+        {% if not source.disabled %}
+            <button class="pull-btn" data-source="{{ source.slug }}" aria-label="Pull latest articles from {{ source.name }}">Pull latest</button>
+        {% endif %}
+    </li>
+{% endfor %}
+</ul>
+{% endblock %}

templates/login.html

@@ -0,0 +1,95 @@
+{% extends "base.html" %}
+
+{% block content %}
+<div class="login-container">
+    <div class="login-box">
+        <h2>NewsArchiver Login</h2>
+        {% with messages = get_flashed_messages(with_categories=true) %}
+            {% if messages %}
+                {% for category, message in messages %}
+                    <div class="alert alert-{{ category }}">{{ message }}</div>
+                {% endfor %}
+            {% endif %}
+        {% endwith %}
+        <form method="POST" action="{{ url_for('login') }}">
+            <input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
+            {% if request.args.get('next') %}
+            <input type="hidden" name="next" value="{{ request.args.get('next') }}">
+            {% endif %}
+            <div class="form-group">
+                <label for="password">Password</label>
+                <input type="password" id="password" name="password" required autofocus>
+            </div>
+            <button type="submit" class="btn btn-primary">Login</button>
+        </form>
+    </div>
+</div>
+
+<style>
+    .login-container {
+        display: flex;
+        justify-content: center;
+        align-items: center;
+        min-height: 60vh;
+    }
+    .login-box {
+        background: var(--card-bg, #fff);
+        border: 1px solid var(--border-color, #ddd);
+        border-radius: 8px;
+        padding: 2rem;
+        width: 100%;
+        max-width: 400px;
+        box-shadow: 0 2px 8px rgba(0, 0, 0, 0.1);
+    }
+    .login-box h2 {
+        margin-top: 0;
+        margin-bottom: 1.5rem;
+        text-align: center;
+    }
+    .form-group {
+        margin-bottom: 1rem;
+    }
+    .form-group label {
+        display: block;
+        margin-bottom: 0.5rem;
+        font-weight: bold;
+    }
+    .form-group input {
+        width: 100%;
+        padding: 0.5rem;
+        border: 1px solid var(--border-color, #ccc);
+        border-radius: 4px;
+        box-sizing: border-box;
+    }
+    .btn {
+        width: 100%;
+        padding: 0.75rem;
+        border: none;
+        border-radius: 4px;
+        cursor: pointer;
+        font-size: 1rem;
+    }
+    .btn-primary {
+        background: var(--accent-color, #0066cc);
+        color: white;
+    }
+    .btn-primary:hover {
+        opacity: 0.9;
+    }
+    .alert {
+        padding: 0.75rem;
+        border-radius: 4px;
+        margin-bottom: 1rem;
+    }
+    .alert-error {
+        background: #fee;
+        color: #c00;
+        border: 1px solid #fcc;
+    }
+    .alert-info {
+        background: #eef;
+        color: #00c;
+        border: 1px solid #ccf;
+    }
+</style>
+{% endblock %}

templates/rss.xml

@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom">
+    <channel>
+        <title>{{ title|e }}</title>
+        <link>{{ link|e }}</link>
+        <description>{{ description|e }}</description>
+        <lastBuildDate>{{ last_build_date|e }}</lastBuildDate>
+        <generator>NewsArchiver</generator>
+        <atom:link href="{{ link|e }}/rss" rel="self" type="application/rss+xml" />
+        {% for item in items %}
+        <item>
+            <title>{{ item.title|e }}</title>
+            <link>{{ item.link|e }}</link>
+            {% if item.pubDate %}
+            <pubDate>{{ item.pubDate|e }}</pubDate>
+            {% endif %}
+            {% if item.author %}
+            <author>{{ item.author|e }}</author>
+            {% endif %}
+            <guid isPermaLink="false">{{ item.guid|e }}</guid>
+            <description>{{ item.description|e }}</description>
+        </item>
+        {% endfor %}
+    </channel>
+</rss>

templates/status.html

@@ -0,0 +1,50 @@
+{% extends "base.html" %}
+
+{% block content %}
+<div class="status-page">
+    <h1>System Status</h1>
+    
+    <div class="status-summary">
+        <div class="status-item">
+            <h2>System</h2>
+            <p>Status: <span class="status-online">Online</span></p>
+            <p>Last Archive Run: {{ last_archive_run or 'Never' }}</p>
+        </div>
+        
+        <div class="status-item">
+            <h2>Statistics</h2>
+            <p>Sources Monitored: {{ sources_monitored }}</p>
+            {% if disabled_sources > 0 %}
+            <p>Disabled Sources: <span class="status-warning">{{ disabled_sources }}</span></p>
+            {% endif %}
+            <p>Total Articles: {{ total_articles }}</p>
+            <p>Failed Jobs: {{ failed_jobs }}</p>
+        </div>
+    </div>
+    
+    {% if sources %}
+    <h2>Monitored Sources</h2>
+    <ul class="newspaper-list">
+    {% for source in sources %}
+        <li class="newspaper-item {% if source.disabled %}disabled{% endif %}">
+            <div class="newspaper-info">
+                {% if source.disabled %}
+                <h2 class="source-disabled" title="{{ source.disable_reason }}">
+                    {{ source.name }}
+                    <span class="status-badge">Disabled</span>
+                </h2>
+                {% else %}
+                <h2><a href="/source/{{ source.slug }}">{{ source.name }}</a></h2>
+                {% endif %}
+                <p>Articles: {{ source.article_count }}</p>
+                <p class="status-{{ source.status }}">{{ source.status|upper }}</p>
+                {% if source.disabled %}
+                <p class="disable-reason">{{ source.disable_reason }}</p>
+                {% endif %}
+            </div>
+        </li>
+    {% endfor %}
+    </ul>
+    {% endif %}
+</div>
+{% endblock %}

tests/playwright/package.json

@@ -0,0 +1,13 @@
+{
+  "name": "newsarchiver-e2e",
+  "version": "1.0.0",
+  "private": true,
+  "devDependencies": {
+    "@playwright/test": "^1.49.1",
+    "typescript": "^5.6.0"
+  },
+  "scripts": {
+    "test": "playwright test",
+    "test:ui": "playwright test --ui"
+  }
+}

tests/playwright/playwright.config.js

@@ -0,0 +1,15 @@
+const { defineConfig } = require('@playwright/test');
+
+module.exports = defineConfig({
+  testDir: './tests',
+  fullyParallel: true,
+  forbidOnly: !!process.env.CI,
+  retries: process.env.CI ? 2 : 0,
+  workers: process.env.CI ? 1 : undefined,
+  reporter: process.env.CI ? [['list'], ['html']] : 'list',
+  use: {
+    baseURL: process.env.APP_URL || 'http://localhost:5000',
+    trace: 'on-first-retry',
+    screenshot: 'only-on-failure',
+  },
+});

tests/playwright/tests/ui.spec.ts

@@ -0,0 +1,128 @@
+import { test, expect } from '@playwright/test';
+
+const TEST_SOURCE = 'Test News';
+
+test.describe('Home page', () => {
+  test('loads and shows sources', async ({ page }) => {
+    await page.goto('/');
+    await expect(page).toHaveTitle('NewsArchiver');
+    await expect(page.locator('h1').filter({ hasText: 'News Archives' })).toBeVisible();
+    await expect(page.locator('.newspaper-list')).toBeVisible();
+  });
+
+  test('shows source count', async ({ page }) => {
+    await page.goto('/');
+    const count = await page.locator('p:has-text("Total sources:")').textContent();
+    expect(count || '').toMatch(/Total sources: \d+/);
+  });
+
+  test('navigates to a source', async ({ page }) => {
+    await page.goto('/');
+    const firstLink = page.locator('.newspaper-item h2 a').first();
+    const sourceName = (await firstLink.textContent())?.trim() || '';
+    await firstLink.click();
+    await expect(page).toHaveURL(/\/source\//);
+    await expect(page.locator('h1')).toContainText(sourceName);
+  });
+});
+
+test.describe('Article listing', () => {
+  test('shows articles for a source', async ({ page }) => {
+    await page.goto(`/source/${TEST_SOURCE.toLowerCase()}`);
+    await expect(page.locator('.article-list')).toBeVisible();
+    const articles = page.locator('.article-item');
+    await expect.poll(() => articles.count()).toBeGreaterThan(0);
+  });
+
+  test('article links navigate to detail', async ({ page }) => {
+    await page.goto(`/source/${TEST_SOURCE.toLowerCase()}`);
+    const firstArticle = page.locator('.article-item h3 a').first();
+    const title = (await firstArticle.textContent())?.trim() || '';
+    await firstArticle.click();
+    await expect(page).toHaveURL(/\/article\/\d+/);
+    await expect(page.locator('.article-view h1')).toContainText(title);
+  });
+
+  test('shows article metadata', async ({ page }) => {
+    await page.goto(`/source/${TEST_SOURCE.toLowerCase()}/article/1`);
+    await expect(page.locator('.article-source')).toBeVisible();
+    await expect(page.locator('.article-meta')).toBeVisible();
+  });
+
+  test('back link returns to listing', async ({ page }) => {
+    await page.goto(`/source/${TEST_SOURCE.toLowerCase()}/article/1`);
+    await page.locator('.back-link').click();
+    await expect(page).toHaveURL(new RegExp(`/source/${TEST_SOURCE.toLowerCase()}`));
+    await expect(page.locator('.article-list')).toBeVisible();
+  });
+});
+
+test.describe('Pagination', () => {
+  test('shows page info', async ({ page }) => {
+    await page.goto(`/source/${TEST_SOURCE.toLowerCase()}`);
+    await expect(page.locator('.pagination span')).toContainText('Page 1');
+  });
+});
+
+test.describe('Navigation', () => {
+  test('header nav links work', async ({ page }) => {
+    await page.goto('/');
+    await page.locator('nav a:has-text("Archives")').click();
+    await expect(page).toHaveURL(/\/$/);
+
+    await page.locator('nav a:has-text("Status")').click();
+    await expect(page).toHaveURL(/\/status/);
+  });
+});
+
+test.describe('Status page', () => {
+  test('shows system status', async ({ page }) => {
+    await page.goto('/status');
+    await expect(page.locator('h1')).toBeVisible();
+  });
+
+  test('shows total articles count', async ({ page }) => {
+    await page.goto('/status');
+    await expect(page.locator('text=/Total Articles: \d+/')).toBeVisible();
+  });
+});
+
+test.describe('RSS feed', () => {
+  test('serves RSS XML', async ({ request }) => {
+    const resp = await request.get('/rss');
+    expect(resp.ok()).toBeTruthy();
+    expect(resp.headers()['content-type']).toContain('application/rss+xml');
+    const body = await resp.text();
+    expect(body).toContain('<rss');
+    expect(body).toContain('<channel>');
+  });
+});
+
+test.describe('Atom feed', () => {
+  test('serves Atom XML', async ({ request }) => {
+    const resp = await request.get('/atom');
+    expect(resp.ok()).toBeTruthy();
+    expect(resp.headers()['content-type']).toContain('application/atom+xml');
+    const body = await resp.text();
+    expect(body).toContain('<feed');
+  });
+});
+
+test.describe('Theme toggle', () => {
+  test('toggles dark mode', async ({ page }) => {
+    await page.goto('/');
+    const html = page.locator('html');
+    await page.locator('#theme-toggle').click();
+    await expect(html).toHaveAttribute('data-theme', 'dark');
+
+    await page.locator('#theme-toggle').click();
+    await expect(html).toHaveAttribute('data-theme', 'light');
+  });
+
+  test('persists theme in localStorage', async ({ page }) => {
+    await page.goto('/');
+    await page.locator('#theme-toggle').click();
+    const theme = await page.evaluate(() => localStorage.getItem('theme'));
+    expect(theme).toBe('dark');
+  });
+});

tests/test_path_handling.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+"""Unit tests for path handling and security."""
+
+import sys
+import tempfile
+from pathlib import Path
+from urllib.parse import unquote
+
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+
+def test_path_traversal_blocked():
+    """Test that path traversal is blocked in archive routes."""
+    archive_dir = Path(tempfile.mkdtemp())
+
+    def validate_archive_path(archive_path: str):
+        decoded = unquote(archive_path)
+        try:
+            archive_file = (archive_dir / decoded).resolve()
+            if not str(archive_file).startswith(str(archive_dir)):
+                return None
+            return archive_file
+        except (ValueError, OSError):
+            return None
+
+    for bad_path in [
+        "../../../etc/passwd",
+        "..%2F..%2F..%2Fetc%2Fpasswd",
+        "../secret.txt",
+        "websites/../../etc/passwd",
+    ]:
+        result = validate_archive_path(bad_path)
+        assert result is None, f"Traversal should be blocked: {bad_path}"
+        print(f"  PASS: blocked '{bad_path}'")
+
+    valid = validate_archive_path("websites/Test/html/2024-01-01/article.html")
+    assert valid is not None
+    assert str(valid).startswith(str(archive_dir))
+    print(f"  PASS: allowed valid path")
+
+
+def test_path_traversal_file_access():
+    """Test that path traversal cannot read files outside archive dir."""
+    archive_dir = Path(tempfile.mkdtemp())
+    secret_file = archive_dir.parent / "secret.txt"
+    secret_file.write_text("top secret")
+
+    def validate_archive_path(archive_path: str):
+        decoded = unquote(archive_path)
+        try:
+            archive_file = (archive_dir / decoded).resolve()
+            if not str(archive_file).startswith(str(archive_dir)):
+                return None
+            return archive_file
+        except (ValueError, OSError):
+            return None
+
+    blocked = validate_archive_path(f"../secret.txt")
+    assert blocked is None
+    print(f"  PASS: file access traversal blocked")
+    secret_file.unlink()
+
+
+def test_url_encoding():
+    """Test URL encoding/decoding in archive paths."""
+    archive_file_path = "websites/404 Media/html/2024-01-15/article_001.html"
+    encoded = archive_file_path.replace(" ", "%20")
+    decoded = unquote(encoded)
+    assert decoded == archive_file_path
+    print(f"  PASS: encoding round-trip")
+
+
+if __name__ == "__main__":
+    test_path_traversal_blocked()
+    test_path_traversal_file_access()
+    test_url_encoding()
+    print("\nAll path handling tests passed!")

tests/test_storage_manager.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+"""Unit tests for storage_manager."""
+
+import sys
+import tempfile
+from pathlib import Path
+from urllib.parse import unquote
+
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+
+def test_relative_path_format():
+    """Test relative path format for archive files."""
+    archive_dir = Path(tempfile.mkdtemp())
+    archive_file_path = archive_dir / "websites/TestSource/html/2024-01-15/article_001.html"
+    archive_file_path.parent.mkdir(parents=True, exist_ok=True)
+    archive_file_path.touch()
+
+    stored_path = str(archive_file_path.relative_to(archive_dir))
+    assert not Path(stored_path).is_absolute()
+    assert stored_path == "websites/TestSource/html/2024-01-15/article_001.html"
+
+    retrieved_path = Path(stored_path)
+    full_path = archive_dir / retrieved_path if not retrieved_path.is_absolute() else retrieved_path
+    assert str(full_path) == str(archive_file_path)
+
+    web_path = str(full_path.relative_to(archive_dir))
+    assert web_path == stored_path
+    print(f"  PASS: stored={stored_path}")
+
+
+def test_archive_file_url_generation():
+    """Test URL generation for archived files."""
+    archive_file_path = "websites/404 Media/html/2024-01-15/article_001.html"
+    url = f"/archive-file/{archive_file_path}"
+    assert url == "/archive-file/websites/404 Media/html/2024-01-15/article_001.html"
+
+    decoded_path = unquote(archive_file_path)
+    archive_dir = Path("/tmp/test_archives")
+    full_path = archive_dir / decoded_path
+    assert str(full_path) == str(archive_dir / "websites/404 Media/html/2024-01-15/article_001.html")
+    print(f"  PASS: url={url}")
+
+
+def test_multiple_sources():
+    """Test different sources get correct paths."""
+    archive_dir = Path(tempfile.mkdtemp())
+    for source in ["404 Media", "TestSource", "Another Source"]:
+        archive_file_path = archive_dir / f"websites/{source}/html/2024-01-15/article_001.html"
+        archive_file_path.parent.mkdir(parents=True, exist_ok=True)
+        stored_path = str(archive_file_path.relative_to(archive_dir))
+        parts = Path(stored_path).parts
+        assert parts[0] == "websites"
+        assert parts[1] == source
+        assert parts[2] == "html"
+        print(f"  PASS: source='{source}' path={stored_path}")
+
+
+if __name__ == "__main__":
+    test_relative_path_format()
+    test_archive_file_url_generation()
+    test_multiple_sources()
+    print("\nAll storage_manager tests passed!")

tests/test_web_interface.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+"""Unit tests for web_interface.py security features."""
+
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+from unittest.mock import MagicMock, patch
+
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+
+def test_validate_archive_path_blocks_traversal():
+    """Test validate_archive_path function blocks traversal."""
+    from web_interface import validate_archive_path, ARCHIVE_DIR
+
+    for bad in ["../../../etc/passwd", "..%2Fetc%2Fpasswd", "../secret.txt"]:
+        result = validate_archive_path(bad)
+        assert result is None, f"Should block: {bad}"
+    print("  PASS: traversal blocked")
+
+
+def test_xss_safe_filter_removed():
+    """Test that article.html no longer uses |safe filter."""
+    result = subprocess.run(
+        ["grep", "-rn", "|safe", "templates/article.html"],
+        capture_output=True, text=True,
+    )
+    assert result.returncode != 0, "article.html should not contain |safe"
+    print("  PASS: no |safe in article.html")
+
+
+def test_security_headers_present():
+    """Test security headers are added to responses."""
+    from web_interface import app
+    client = app.test_client()
+    resp = client.get("/")
+    assert resp.headers.get("X-Content-Type-Options") == "nosniff"
+    assert resp.headers.get("X-Frame-Options") == "DENY"
+    assert resp.headers.get("X-XSS-Protection") == "1; mode=block"
+    assert resp.headers.get("Referrer-Policy") == "strict-origin-when-cross-origin"
+    print("  PASS: security headers present")
+
+
+def test_server_url_not_hardcoded():
+    """Test that SERVER_URL comes from env, not hardcoded."""
+    result = subprocess.run(
+        ["grep", "-n", "192\\.168", "web_interface.py"],
+        capture_output=True, text=True,
+    )
+    assert result.returncode != 0, "No hardcoded internal IPs"
+    print("  PASS: no hardcoded IPs")
+
+
+def test_no_safe_filter_in_rss():
+    """Test RSS/Atom feeds escape content properly."""
+    result = subprocess.run(
+        ["grep", "-rn", "|safe", "templates/rss.xml", "templates/atom.xml"],
+        capture_output=True, text=True,
+    )
+    assert result.returncode != 0, "No |safe in RSS templates"
+    print("  PASS: no |safe in RSS/Atom")
+
+
+if __name__ == "__main__":
+    test_validate_archive_path_blocks_traversal()
+    test_xss_safe_filter_removed()
+    test_security_headers_present()
+    test_server_url_not_hardcoded()
+    test_no_safe_filter_in_rss()
+    print("\nAll web interface tests passed!")

version.json

@@ -0,0 +1,5 @@
+{
+  "major": 1,
+  "minor": 0,
+  "patch": 413
+}

web_interface.py

@@ -0,0 +1,704 @@
+#!/usr/bin/env python3
+"""Web Interface for NewsArchiver - Phase 3
+
+Flask web server for browsing archived news articles.
+"""
+
+import hashlib
+import hmac
+import html
+import json
+import logging
+import os
+import secrets
+import xml.etree.ElementTree as ET
+from datetime import datetime, timezone
+from functools import wraps
+from pathlib import Path
+from typing import Optional
+from urllib.parse import quote, unquote, urlparse
+
+from flask import (
+    Flask,
+    abort,
+    flash,
+    jsonify,
+    make_response,
+    redirect,
+    render_template,
+    request,
+    session,
+    url_for,
+)
+
+from storage_manager import (
+    DB_PATH,
+    get_all_sources,
+    get_article,
+    get_articles_by_source,
+    get_latest_articles,
+    get_source_stats,
+)
+
+SCRIPT_DIR = Path(__file__).parent.resolve()
+ARCHIVE_DIR = Path(
+    os.environ.get("ARCHIVE_DIR", str(SCRIPT_DIR / "archival_data"))
+).resolve()
+RSS_FEEDS_PATH = SCRIPT_DIR / "rss_feeds.json"
+
+RSS_FEEDS = {}
+
+# Authentication configuration
+ADMIN_PASSWORD = os.environ.get("NEWSARCHIVER_PASSWORD", "")
+SECRET_KEY = os.environ.get(
+    "NEWSARCHIVER_SECRET_KEY", secrets.token_hex(32)
+)
+SESSION_LIFETIME_MINUTES = int(os.environ.get("NEWSARCHIVER_SESSION_MINUTES", "480"))
+
+# Server URL configuration
+SERVER_URL = os.environ.get(
+    "NEWSARCHIVER_SERVER_URL", f"http://localhost:5000"
+)
+
+app = Flask(
+    __name__,
+    static_folder=str(SCRIPT_DIR / "static"),
+    template_folder=str(SCRIPT_DIR / "templates"),
+)
+app.secret_key = SECRET_KEY
+
+
+def login_required(f):
+    """Decorator to require login for protected routes."""
+
+    @wraps(f)
+    def decorated_function(*args, **kwargs):
+        if not ADMIN_PASSWORD:
+            return f(*args, **kwargs)
+        if "authenticated" not in session:
+            flash("Please log in to access this page.", "error")
+            return redirect(url_for("login", next=request.url))
+        return f(*args, **kwargs)
+
+    return decorated_function
+
+
+def generate_csrf_token():
+    """Generate a CSRF token tied to the session."""
+    if "csrf_token" not in session:
+        session["csrf_token"] = secrets.token_hex(32)
+    return session["csrf_token"]
+
+
+def verify_csrf_token():
+    """Verify the CSRF token from the request."""
+    if not ADMIN_PASSWORD:
+        return True
+    token = request.form.get("csrf_token") or request.headers.get("X-CSRF-Token")
+    if not token or "csrf_token" not in session:
+        return False
+    return hmac.compare_digest(token, session["csrf_token"])
+
+
+@app.template_global()
+def csrf_token():
+    """Make CSRF token available in templates."""
+    return generate_csrf_token()
+
+
+@app.before_request
+def enforce_session_timeout():
+    """Expire sessions after inactivity."""
+    if "authenticated" in session:
+        auth_time = session.get("auth_time", 0)
+        now = datetime.now().timestamp()
+        if now - auth_time > SESSION_LIFETIME_MINUTES * 60:
+            session.clear()
+
+
+@app.after_request
+def add_security_headers(response):
+    """Add security headers to all responses."""
+    response.headers["X-Content-Type-Options"] = "nosniff"
+    response.headers["X-Frame-Options"] = "DENY"
+    response.headers["X-XSS-Protection"] = "1; mode=block"
+    response.headers["Referrer-Policy"] = "strict-origin-when-cross-origin"
+    response.headers["Cache-Control"] = "no-store"
+    if not request.is_secure:
+        response.headers["Content-Security-Policy"] = (
+            "default-src 'self'; "
+            "script-src 'self'; "
+            "style-src 'self' 'unsafe-inline'; "
+            "img-src 'self' data:; "
+            "frame-ancestors 'none'"
+        )
+    return response
+
+
+logger = logging.getLogger(__name__)
+
+
+def get_pagination_info(total: int, page: int, per_page: int) -> dict:
+    """Calculate pagination information.
+
+    Args:
+        total: Total number of items
+        page: Current page number
+        per_page: Items per page
+
+    Returns:
+        Dictionary with pagination details
+    """
+    total_pages = (total + per_page - 1) // per_page if total > 0 else 1
+
+    return {
+        "total": total,
+        "page": page,
+        "per_page": per_page,
+        "has_next": page < total_pages,
+        "has_prev": page > 1,
+        "next_num": page + 1 if page < total_pages else None,
+        "prev_num": page - 1 if page > 1 else None,
+        "pages": total_pages,
+    }
+
+
+def sanitize_error_message(msg: str) -> str:
+    """Sanitize error messages to prevent information leakage."""
+    sanitized = msg
+    for pattern in [r"\d{1,3}(\.\d{1,3}){3}", r"/home/\w+"]:
+        import re
+
+        sanitized = re.sub(pattern, "[REDACTED]", sanitized)
+    return sanitized
+
+
+@app.route("/login", methods=["GET", "POST"])
+def login():
+    """Login page."""
+    if not ADMIN_PASSWORD:
+        return redirect(url_for("index"))
+
+    if request.method == "POST":
+        if not verify_csrf_token():
+            flash("Invalid request.", "error")
+            return redirect(url_for("login"))
+
+        password = request.form.get("password", "")
+        if hmac.compare_digest(password, ADMIN_PASSWORD):
+            session.clear()
+            session["authenticated"] = True
+            session["csrf_token"] = secrets.token_hex(32)
+            session["auth_time"] = datetime.now().timestamp()
+            next_url = request.form.get("next", url_for("index"))
+            parsed = urlparse(next_url)
+            if parsed.netloc:
+                next_url = url_for("index")
+            return redirect(next_url)
+        else:
+            flash("Invalid password.", "error")
+
+    return render_template("login.html", csrf_token=csrf_token())
+
+
+@app.route("/logout")
+def logout():
+    """Logout."""
+    session.clear()
+    return redirect(url_for("index"))
+
+
+@app.route("/")
+@login_required
+def index():
+    """Newspaper listing page."""
+    sources = get_all_sources()
+    rss_feeds = load_rss_feeds()
+
+    source_list = []
+    disabled_sources = []
+
+    for source_name in sources:
+        stats = get_source_stats(source_name)
+
+        source_info = {
+            "name": source_name.title(),
+            "slug": source_name,
+            "article_count": stats["total_articles"],
+            "last_archived": stats.get("last_archived"),
+            "status": "success" if stats["total_articles"] > 0 else "pending",
+        }
+
+        if source_name in rss_feeds:
+            feed_info = rss_feeds[source_name]
+            if feed_info.get("disabled", False):
+                source_info["disabled"] = True
+                source_info["disable_reason"] = feed_info.get(
+                    "disable_reason", "No reason provided"
+                )
+                disabled_sources.append(source_info)
+                continue
+
+        source_list.append(source_info)
+
+    source_list.extend(disabled_sources)
+
+    return render_template("index.html", sources=source_list)
+
+
+@app.route("/source/<slug>")
+@login_required
+def articles(slug: str):
+    """Article listing page for a specific source."""
+    page = request.args.get("page", 1, type=int)
+    per_page = 50
+
+    sources = get_all_sources()
+    source_name = None
+    for s in sources:
+        if s.lower() == slug.lower():
+            source_name = s
+            break
+
+    if not source_name:
+        return render_template("article_not_found.html", slug=slug, article_id=0), 404
+
+    articles_list = get_articles_by_source(
+        source_name, limit=per_page, offset=(page - 1) * per_page
+    )
+    stats = get_source_stats(source_name)
+    total = stats["total_articles"]
+
+    pagination = get_pagination_info(total, page, per_page)
+
+    articles_data = []
+    for article in articles_list:
+        articles_data.append(
+            {
+                "id": getattr(article, "id", 0),
+                "title": article.title or "Untitled",
+                "date": article.publish_date or "",
+                "summary": article.content_text[:200] if article.content_text else "",
+                "url": f"/source/{source_name.lower()}/article/{getattr(article, 'id', 0)}",
+            }
+        )
+
+    from collections import OrderedDict
+    grouped = OrderedDict()
+    for a in articles_data:
+        if a["date"]:
+            try:
+                day = a["date"][:10]
+            except Exception:
+                day = "Unknown"
+        else:
+            day = "Unknown"
+        grouped.setdefault(day, []).append(a)
+
+    return render_template(
+        "articles.html",
+        source_name=source_name.title(),
+        source_slug=source_name.lower(),
+        articles=articles_data,
+        articles_by_date=grouped.items(),
+        pagination=pagination,
+    )
+
+
+def validate_archive_path(archive_path: str) -> Optional[Path]:
+    """Validate archive path is within ARCHIVE_DIR (prevents path traversal).
+
+    URL-decodes path first to block encoded traversal (..%2F).
+
+    Args:
+        archive_path: Requested path component
+
+    Returns:
+        Resolved Path if valid, None if traversal detected
+    """
+    decoded = unquote(archive_path)
+    try:
+        archive_file = (ARCHIVE_DIR / decoded).resolve()
+        if not str(archive_file).startswith(str(ARCHIVE_DIR)):
+            logger.warning("Path traversal attempt blocked: %s", archive_path)
+            return None
+        return archive_file
+    except (ValueError, OSError):
+        return None
+
+
+@app.route("/archive/<path:archive_path>")
+@login_required
+def serve_archive(archive_path):
+    """Serve archived HTML file."""
+    archive_file = validate_archive_path(archive_path)
+    if archive_file and archive_file.exists():
+        return archive_file.read_text(encoding="utf-8")
+    return "Archive not found", 404
+
+
+@app.route("/archive-file/<path:encoded_path>")
+@login_required
+def serve_archive_file(encoded_path):
+    """Serve archived HTML file from encoded path."""
+    archive_file = validate_archive_path(encoded_path)
+    if archive_file and archive_file.exists():
+        return archive_file.read_text(encoding="utf-8")
+    return "Archive not found", 404
+
+
+@app.route("/source/<slug>/article/<int:article_id>")
+@login_required
+def article(slug: str, article_id: int):
+    """Individual article page."""
+    sources = get_all_sources()
+    source_name = None
+    for s in sources:
+        if s.lower() == slug.lower():
+            source_name = s
+            break
+
+    if not source_name:
+        return render_template(
+            "article_not_found.html", slug=slug, article_id=article_id
+        ), 404
+
+    article = get_article(source_name, article_id)
+    if not article:
+        return render_template(
+            "article_not_found.html", slug=slug, article_id=article_id
+        ), 404
+
+    article_data = {
+        "id": article_id,
+        "title": article.title or "Untitled",
+        "publish_date": article.publish_date or "",
+        "author": article.author or "",
+        "url": article.url or "",
+        "content_text": article.content_text or "",
+        "archive_file_path": article.archive_file_path or "",
+    }
+
+    return render_template(
+        "article.html",
+        source_name=source_name.title(),
+        source_slug=source_name.lower(),
+        article=article_data,
+    )
+
+
+@app.route("/status")
+@login_required
+def status():
+    """System status page."""
+    sources = get_all_sources()
+    rss_feeds = load_rss_feeds()
+
+    sources_info = []
+    disabled_sources = []
+    total_articles = 0
+    failed_jobs = 0
+    disabled_count = 0
+
+    for source_name in sources:
+        stats = get_source_stats(source_name)
+
+        source_info = {
+            "name": source_name.title(),
+            "slug": source_name,
+            "article_count": stats["total_articles"],
+            "last_archived": stats.get("last_archived"),
+            "status": "success" if stats["total_articles"] > 0 else "pending",
+        }
+
+        if source_name in rss_feeds:
+            feed_info = rss_feeds[source_name]
+            if feed_info.get("disabled", False):
+                source_info["disabled"] = True
+                disabled_count += 1
+                disabled_sources.append(source_info)
+                continue
+
+        sources_info.append(source_info)
+        total_articles += stats["total_articles"]
+        failed_jobs += stats["failed"]
+
+    sources_info.extend(disabled_sources)
+
+    return render_template(
+        "status.html",
+        sources=sources_info,
+        total_articles=total_articles,
+        failed_jobs=failed_jobs,
+        sources_monitored=len(sources) - disabled_count,
+        disabled_sources=disabled_count,
+    )
+
+
+@app.route("/api/sources")
+@login_required
+def api_sources():
+    """API endpoint for listing all sources."""
+    sources = get_all_sources()
+    rss_feeds = load_rss_feeds()
+
+    source_list = []
+    disabled_sources = []
+
+    for source_name in sources:
+        stats = get_source_stats(source_name)
+
+        source_info = {
+            "name": source_name.title(),
+            "slug": source_name,
+            "article_count": stats["total_articles"],
+            "last_archived": stats.get("last_archived"),
+            "status": "success" if stats["total_articles"] > 0 else "pending",
+        }
+
+        if source_name in rss_feeds:
+            feed_info = rss_feeds[source_name]
+            if feed_info.get("disabled", False):
+                source_info["disabled"] = True
+                source_info["disable_reason"] = feed_info.get(
+                    "disable_reason", "No reason provided"
+                )
+                disabled_sources.append(source_info)
+                continue
+
+        source_list.append(source_info)
+
+    source_list.extend(disabled_sources)
+
+    return jsonify({"sources": source_list})
+
+
+@app.route("/api/source/<slug>/articles")
+@login_required
+def api_articles(slug: str):
+    """API endpoint for listing articles for a source."""
+    page = request.args.get("page", 1, type=int)
+    per_page = 50
+
+    sources = get_all_sources()
+    source_name = None
+    for s in sources:
+        if s.lower() == slug.lower():
+            source_name = s
+            break
+
+    if not source_name:
+        return jsonify({"error": "Source not found"}), 404
+
+    articles_list = get_articles_by_source(
+        source_name, limit=per_page, offset=(page - 1) * per_page
+    )
+    stats = get_source_stats(source_name)
+    total = stats["total_articles"]
+
+    pagination = get_pagination_info(total, page, per_page)
+
+    articles_data = []
+    for article in articles_list:
+        articles_data.append(
+            {
+                "id": getattr(article, "id", 0),
+                "title": article.title or "Untitled",
+                "date": article.publish_date or "",
+                "summary": article.content_text[:200] if article.content_text else "",
+                "url": f"/source/{source_name.lower()}/article/{getattr(article, 'id', 0)}",
+            }
+        )
+
+    return jsonify(
+        {
+            "source_name": source_name.title(),
+            "articles": articles_data,
+            "total": total,
+            "page": page,
+            "per_page": per_page,
+            "has_next": pagination["has_next"],
+            "has_prev": pagination["has_prev"],
+        }
+    )
+
+
+@app.route("/api/status")
+@login_required
+def api_status():
+    """API endpoint for system status."""
+    sources = get_all_sources()
+
+    sources_monitored = len(sources)
+    total_articles = 0
+    failed_jobs = 0
+    last_archive_run = None
+
+    for source_name in sources:
+        stats = get_source_stats(source_name)
+        total_articles += stats["total_articles"]
+        failed_jobs += stats["failed"]
+
+        if stats.get("last_archive_run"):
+            if last_archive_run is None or stats["last_archive_run"] > last_archive_run:
+                last_archive_run = stats["last_archive_run"]
+
+    return jsonify(
+        {
+            "status": "online",
+            "last_archive_run": last_archive_run,
+            "pending_jobs": 0,
+            "failed_jobs": failed_jobs,
+            "sources_monitored": sources_monitored,
+            "total_articles": total_articles,
+        }
+    )
+
+
+@app.route("/rss")
+def rss_feed():
+    """RSS 2.0 endpoint for latest archived articles."""
+    limit = request.args.get("limit", 50, type=int)
+
+    articles = get_latest_articles(limit=limit)
+
+    rss_items = []
+    for article in articles:
+        if (
+            article.title
+            and article.content_text
+            and "Performing security verification" not in article.content_text
+        ):
+            pub_date = None
+            if article.publish_date:
+                try:
+                    dt = datetime.fromisoformat(
+                        article.publish_date.replace("Z", "+00:00")
+                    )
+                    pub_date = dt.strftime("%a, %d %b %Y %H:%M:%S %z").strip()
+                except (ValueError, AttributeError):
+                    try:
+                        dt = datetime.fromisoformat(article.publish_date)
+                        pub_date = dt.strftime("%a, %d %b %Y %H:%M:%S +0000")
+                    except (ValueError, AttributeError):
+                        pub_date = datetime.now(timezone.utc).strftime(
+                            "%a, %d %b %Y %H:%M:%S +0000"
+                        )
+
+            source_name = article.source_name or "unknown"
+            encoded_source = quote(source_name.lower())
+            item = {
+                "title": html.escape(article.title),
+                "link": f"{SERVER_URL}/source/{encoded_source}/article/{article.id}",
+                "pubDate": pub_date,
+                "description": html.escape(
+                    article.content_text[:500] if article.content_text else ""
+                ),
+                "guid": html.escape(article.url or f"article-{article.id}"),
+            }
+            if article.author:
+                item["author"] = html.escape(article.author)
+            rss_items.append(item)
+
+    rss_template = render_template(
+        "rss.xml",
+        title="NewsArchiver - Latest Articles",
+        link=SERVER_URL,
+        description="Latest archived news articles",
+        last_build_date=datetime.now(timezone.utc).strftime(
+            "%a, %d %b %Y %H:%M:%S +0000"
+        ),
+        items=rss_items,
+    )
+
+    response = make_response(rss_template)
+    response.headers["Content-Type"] = "application/rss+xml; charset=utf-8"
+    return response
+
+
+@app.route("/atom")
+def atom_feed():
+    """Atom 1.0 endpoint for latest archived articles."""
+    limit = request.args.get("limit", 50, type=int)
+
+    articles = get_latest_articles(limit=limit)
+
+    atom_entries = []
+    for article in articles:
+        if (
+            article.title
+            and article.content_text
+            and "Performing security verification" not in article.content_text
+        ):
+            pub_date = None
+            if article.publish_date:
+                try:
+                    dt = datetime.fromisoformat(
+                        article.publish_date.replace("Z", "+00:00")
+                    )
+                    pub_date = dt.strftime("%Y-%m-%dT%H:%M:%S+00:00")
+                except (ValueError, AttributeError):
+                    pub_date = datetime.now(timezone.utc).strftime(
+                        "%Y-%m-%dT%H:%M:%S+00:00"
+                    )
+
+            source_name = article.source_name or "unknown"
+            encoded_source = quote(source_name.lower())
+            entry = {
+                "title": html.escape(article.title),
+                "link": f"{SERVER_URL}/source/{encoded_source}/article/{article.id}",
+                "published": pub_date,
+                "summary": html.escape(
+                    article.content_text[:500] if article.content_text else ""
+                ),
+                "id": html.escape(article.url or f"article-{article.id}"),
+            }
+            if article.author:
+                entry["author"] = {"name": html.escape(article.author)}
+            atom_entries.append(entry)
+
+    atom_template = render_template(
+        "atom.xml",
+        title="NewsArchiver - Latest Articles",
+        link=SERVER_URL,
+        updated=datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S+00:00"),
+        entries=atom_entries,
+    )
+
+    response = make_response(atom_template)
+    response.headers["Content-Type"] = "application/atom+xml; charset=utf-8"
+    return response
+
+
+def load_rss_feeds() -> dict:
+    """Load RSS feeds configuration, resolving environment variables in API keys."""
+    global RSS_FEEDS
+
+    if RSS_FEEDS:
+        return RSS_FEEDS
+
+    if not RSS_FEEDS_PATH.exists():
+        logger.warning("RSS feeds file not found: %s", RSS_FEEDS_PATH)
+        return {}
+
+    try:
+        raw_content = RSS_FEEDS_PATH.read_text(encoding="utf-8")
+        for key, value in os.environ.items():
+            raw_content = raw_content.replace(f"${{{key}}}", value)
+        RSS_FEEDS = json.loads(raw_content)
+        return RSS_FEEDS
+    except Exception as e:
+        logger.error("Failed to load RSS feeds: %s", sanitize_error_message(str(e)))
+        return {}
+
+
+if __name__ == "__main__":
+    logger.info("Starting web interface...")
+
+    if not DB_PATH.exists():
+        logger.info("Database not found, initializing...")
+        from storage_manager import initialize_storage
+
+        initialize_storage()
+
+    app.run(host="0.0.0.0", port=5000, debug=False)