From aa51dc766b365a30b2603a338d555edf143d8893 Mon Sep 17 00:00:00 2001 From: Jarian Cottingham Date: Tue, 31 Mar 2026 10:15:50 -0500 Subject: [PATCH] added docker support --- ._.DS_Store | Bin 4096 -> 0 bytes ._.git | Bin 4096 -> 0 bytes ._.gitignore | Bin 4096 -> 0 bytes ._README.md | Bin 4096 -> 0 bytes ._cleanup_old_files.py | Bin 4096 -> 0 bytes .dockerignore | 22 ++ .env.example | 13 + Dockerfile | 56 ++++ README.md | 46 +++ archive_engine.py | 4 +- cleanup_old_files.py | 6 +- docker-compose.nas.example.yml | 37 +++ docker-compose.yml | 23 ++ docker_backup.sh | 31 ++ docker_setup.sh | 47 ++++ entrypoint.sh | 11 + nohup.out | 16 ++ requirements.txt | 7 + run_archiver.py | 4 +- scheduler.py | 4 +- storage_manager.py | 4 +- web_interface.py | 501 ++++++++++++++++++--------------- 22 files changed, 596 insertions(+), 236 deletions(-) delete mode 100644 ._.DS_Store delete mode 100644 ._.git delete mode 100644 ._.gitignore delete mode 100644 ._README.md delete mode 100644 ._cleanup_old_files.py create mode 100644 .dockerignore create mode 100644 .env.example create mode 100644 Dockerfile create mode 100644 docker-compose.nas.example.yml create mode 100644 docker-compose.yml create mode 100644 docker_backup.sh create mode 100644 docker_setup.sh create mode 100644 entrypoint.sh create mode 100644 requirements.txt diff --git a/._.DS_Store b/._.DS_Store deleted file mode 100644 index 28c42fb20a1f27e695fb64323501fc6476578b13..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4096 zcmZQz6=P>$Vqox1Ojhs@R)|o50+1L3ClDJkFz{^v(m+1nBL)UWIhYCu0iY;W;207T z1d#ygV5q>VXjE`C1V%$(Gz3ONU^E0qLtr!nMnhmU1V%$(Gz3ONU^E1%90H6$^FSC3 z$Vqox1Ojhs@R)|o50+1L3ClDJkFz{^v(m+1nBL)UWIUt(=a103vf+zv$ zV3+~K+-O=D5#plB`MG+D1qC^&dId%KWvO|IdC92^j7$vP0^!v=CR@9sX&vQ`hQMeD zjE2By2#kinXb6mkz-S1JhQMeDjE2By2#kgR_7DJdHbEE+$Vqox1Ojhs@R)|o50+1L3ClDJkFz{^v(m+1nBL)UWIUt(=a103vf+zv$ zV3+~K+-O=D5#plB`MG+D1qC^&dId%KWvO|IdC92^j7$vP0^!v=CR@9sX&vQ`hQMeD zjE2By2#kinXb6mkz-S1JhQMeDjE2By2#kgR_7DJdHbEE+$Vqox1Ojhs@R)|o50+1L3ClDJkFz{^v(m+1nBL)UWIUt(=a103vf+zv$ zV3+~K+-O=D5#plB`MG+D1qC^&dId%KWvO|IdC92^j7$vP0^!v=CR@9sX&vQ`hQMeD zjE2By2#kinXb6mkz-S1JhQMeDjE2By2#kgR_7DJdHbEE+/dev/null || true +COPY static/ ./static/ 2>/dev/null || true + +# Copy entrypoint script +COPY entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + +# Expose Flask port +EXPOSE 5000 + +# Use entrypoint script +ENTRYPOINT ["/entrypoint.sh"] + +# Default command +CMD ["python", "run_archiver.py", "--serve", "--host", "0.0.0.0", "--port", "5000"] diff --git a/README.md b/README.md index 9d89e8e..36632d4 100644 --- a/README.md +++ b/README.md @@ -124,6 +124,52 @@ The SQLite database (`archival_data/cache.db`) stores: - Flask, Trafilatura, feedparser, APScheduler, requests, beautifulsoup4 - SingleFile CLI (optional, for web page archiving) +## Docker Deployment + +The NewsArchiver can be deployed using Docker for easier management and isolation. + +### Quick Start with Docker + +```bash +# Build the Docker image +docker build -t newsarchiver . + +# Run with default settings (archives stored in container) +docker run -p 5000:5000 newsarchiver + +# Run with NAS storage mount +docker run -p 5000:5000 \ + -v /path/to/nas/backup:/data/archives \ + -e ARCHIVE_DIR=/data/archives \ + newsarchiver +``` + +### Using Docker Compose + +```bash +# Edit docker-compose.yml to configure your NAS mount path +vim docker-compose.yml + +# Start the service +docker-compose up -d + +# View logs +docker-compose logs -f + +# Stop the service +docker-compose down +``` + +### Configuration + +The `ARCHIVE_DIR` environment variable controls where archived files are stored. To use NAS storage: + +1. Edit `docker-compose.yml` and update the volume mount path +2. Set `ARCHIVE_DIR` to match the container path (e.g., `/data/archives`) +3. Restart the container + +The archived data will persist even if the container is removed, as it's stored in a Docker volume or mounted NAS directory. + ## Stopping Services To stop all NewsArchiver services: diff --git a/archive_engine.py b/archive_engine.py index afda93c..97098a5 100644 --- a/archive_engine.py +++ b/archive_engine.py @@ -43,8 +43,8 @@ except ImportError: -SCRIPT_DIR = Path(__file__).parent -ARCHIVE_DIR = SCRIPT_DIR / 'archival_data' +SCRIPT_DIR = Path(__file__).parent.resolve() +ARCHIVE_DIR = Path(os.environ.get("ARCHIVE_DIR", str(SCRIPT_DIR / "archival_data"))).resolve() ARCHIVE_DIR.mkdir(exist_ok=True) logging.basicConfig( diff --git a/cleanup_old_files.py b/cleanup_old_files.py index f3fa6b4..ba57569 100644 --- a/cleanup_old_files.py +++ b/cleanup_old_files.py @@ -19,6 +19,7 @@ Options: import argparse import logging +import os import sys from datetime import datetime, timezone from pathlib import Path @@ -38,7 +39,10 @@ logger = logging.getLogger(__name__) SCRIPT_DIR = Path(__file__).parent.resolve() # Path to archival_data directory -ARCHIVAL_DATA_DIR = SCRIPT_DIR / "archival_data" +ARCHIVE_DIR = Path( + os.environ.get("ARCHIVE_DIR", str(SCRIPT_DIR / "archival_data")) +).resolve() +ARCHIVAL_DATA_DIR = ARCHIVE_DIR # Path to websites folder (only this folder will be scanned in archival_data) WEBSITES_DIR = ARCHIVAL_DATA_DIR / "websites" diff --git a/docker-compose.nas.example.yml b/docker-compose.nas.example.yml new file mode 100644 index 0000000..6deb495 --- /dev/null +++ b/docker-compose.nas.example.yml @@ -0,0 +1,37 @@ +# Example docker-compose configuration for NAS storage +# Copy this file to docker-compose.yml and edit the volume path + +version: '3.8' + +services: + newsarchiver: + build: + context: . + dockerfile: Dockerfile + container_name: newsarchiver + ports: + - "5000:5000" + environment: + - ARCHIVE_DIR=/data/archives + volumes: + # Example: Mount your NAS to /path/to/nas/archives + # Replace with your actual NAS path + - /path/to/nas/archives:/data/archives + # Or use Docker named volume for local storage: + # - newsarchiver_data:/data/archives + restart: unless-stopped + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:5000/"] + interval: 30s + timeout: 10s + retries: 3 + # Optional: Run as specific UID/GID for NAS permissions + # user: "1000:1000" + +volumes: + newsarchiver_data: + driver: local + driver_opts: + type: none + o: bind + device: /path/to/nas/archives # Replace with your NAS path diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..f5f7388 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,23 @@ +version: '3.8' + +services: + newsarchiver: + build: + context: . + dockerfile: Dockerfile + container_name: newsarchiver + ports: + - "5000:5000" + environment: + - ARCHIVE_DIR=/data/archives + volumes: + - newsarchiver_data:/data/archives + restart: unless-stopped + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:5000/"] + interval: 30s + timeout: 10s + retries: 3 + +volumes: + newsarchiver_data: diff --git a/docker_backup.sh b/docker_backup.sh new file mode 100644 index 0000000..a5ba3c1 --- /dev/null +++ b/docker_backup.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# Docker backup script for NewsArchiver +# This script backs up archived data from the Docker volume to a local or NAS location + +set -e + +BACKUP_DIR="${BACKUP_DIR:-./backups}" +TIMESTAMP=$(date +%Y%m%d_%H%M%S) +BACKUP_NAME="newsarchiver_backup_${TIMESTAMP}.tar.gz" + +echo "Starting backup..." +echo "Backup location: ${BACKUP_DIR}/${BACKUP_NAME}" + +# Create backup directory +mkdir -p "$BACKUP_DIR" + +# Create backup from the Docker volume +docker run --rm \ + -v newsarchiver_data:/data:ro \ + -v "${BACKUP_DIR}:/backup" \ + alpine tar -czf "/backup/${BACKUP_NAME}" -C /data . + +echo "Backup complete: ${BACKUP_DIR}/${BACKUP_NAME}" +echo "" +echo "To restore from backup:" +echo " 1. Stop the container: docker-compose down" +echo " 2. Remove the volume: docker volume rm newsarchiver_data" +echo " 3. Create a new volume: docker volume create newsarchiver_data" +echo " 4. Restore: docker run --rm -v newsarchiver_data:/data -v \${BACKUP_DIR}:/backup alpine tar -xzf /backup/${BACKUP_NAME} -C /data" +echo " 5. Start: docker-compose up -d" diff --git a/docker_setup.sh b/docker_setup.sh new file mode 100644 index 0000000..9bfb70c --- /dev/null +++ b/docker_setup.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +# Docker setup script for NewsArchiver +# This script helps configure the Docker environment + +set -e + +echo "NewsArchiver Docker Setup" +echo "==========================" +echo "" + +# Check if Docker is installed +if ! command -v docker &> /dev/null; then + echo "ERROR: Docker is not installed. Please install Docker first." + exit 1 +fi + +# Check if docker-compose is available +if ! command -v docker-compose &> /dev/null && ! docker compose version &> /dev/null; then + echo "ERROR: docker-compose is not installed. Please install docker-compose first." + exit 1 +fi + +echo "Docker is installed." +echo "" + +# Check if the project has docker-compose.yml +if [ ! -f docker-compose.yml ] && [ ! -f docker-compose.nas.example.yml ]; then + echo "WARNING: docker-compose.yml not found." + echo "Creating from example..." + cp docker-compose.nas.example.yml docker-compose.yml + echo "" + echo "Please edit docker-compose.yml to set your NAS path:" + echo " 1. Find the volume mount path (currently set to /path/to/nas/archives)" + echo " 2. Replace with your actual NAS path" + echo " 3. Save the file" + echo "" +fi + +echo "Setup complete!" +echo "" +echo "Next steps:" +echo " 1. Edit docker-compose.yml with your NAS path" +echo " 2. Build and start: docker-compose up -d --build" +echo " 3. Check logs: docker-compose logs -f" +echo " 4. Access web interface at http://localhost:5000" +echo "" diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100644 index 0000000..46f36a0 --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -e + +# Use ARCHIVE_DIR from environment, default to /app/archival_data +export ARCHIVE_DIR="${ARCHIVE_DIR:-/app/archival_data}" + +# Create archive directory if it does not exist +mkdir -p "$ARCHIVE_DIR" + +# Run the command passed to docker +exec "$@" diff --git a/nohup.out b/nohup.out index a77cdd5..722db3f 100644 --- a/nohup.out +++ b/nohup.out @@ -216,3 +216,19 @@ opencode server listening on http://0.0.0.0:4096 2026-03-31 12:16:03,370 - INFO - 192.168.8.226 - - [31/Mar/2026 12:16:03] "GET / HTTP/1.1" 200 - 2026-03-31 12:16:03,475 - INFO - 192.168.8.226 - - [31/Mar/2026 12:16:03] "GET /static/style.css HTTP/1.1" 304 - 2026-03-31 13:05:25,475 - INFO - 192.168.8.226 - - [31/Mar/2026 13:05:25] "GET /rss HTTP/1.1" 200 - +2026-03-31 14:38:42,845 - INFO - 192.168.8.110 - - [31/Mar/2026 14:38:42] "GET /source/404%20Media HTTP/1.1" 200 - +2026-03-31 14:38:42,872 - INFO - 192.168.8.110 - - [31/Mar/2026 14:38:42] "GET /static/style.css HTTP/1.1" 200 - +2026-03-31 14:38:42,883 - INFO - 192.168.8.110 - - [31/Mar/2026 14:38:42] "GET /favicon.ico HTTP/1.1" 404 - +2026-03-31 14:38:44,029 - INFO - 192.168.8.110 - - [31/Mar/2026 14:38:44] "GET /source/404%20media/article/76744 HTTP/1.1" 200 - +2026-03-31 14:38:44,043 - INFO - 192.168.8.110 - - [31/Mar/2026 14:38:44] "GET /static/style.css HTTP/1.1" 304 - +2026-03-31 14:40:43,351 - INFO - 192.168.8.110 - - [31/Mar/2026 14:40:43] "GET /archive-file//home/user/playground/NewsArchiver/archival_data/websites/404%20Media/html/2026-03-31/article_17749630811.html HTTP/1.1" 308 - +2026-03-31 14:40:43,358 - INFO - Archive file path: /home/user/playground/NewsArchiver/archival_data/home/user/playground/NewsArchiver/archival_data/websites/404 Media/html/2026-03-31/article_17749630811.html, exists: False +2026-03-31 14:40:43,358 - INFO - 192.168.8.110 - - [31/Mar/2026 14:40:43] "GET /archive-file/home/user/playground/NewsArchiver/archival_data/websites/404%20Media/html/2026-03-31/article_17749630811.html HTTP/1.1" 404 - +2026-03-31 14:40:43,374 - INFO - 192.168.8.110 - - [31/Mar/2026 14:40:43] "GET /favicon.ico HTTP/1.1" 404 - +2026-03-31 14:51:00,206 - INFO - 192.168.8.110 - - [31/Mar/2026 14:51:00] "GET /source/404%20media/article/76744 HTTP/1.1" 200 - +2026-03-31 14:51:00,220 - INFO - 192.168.8.110 - - [31/Mar/2026 14:51:00] "GET /static/style.css HTTP/1.1" 200 - +2026-03-31 14:51:00,225 - INFO - 192.168.8.110 - - [31/Mar/2026 14:51:00] "GET /favicon.ico HTTP/1.1" 404 - +2026-03-31 14:51:02,123 - INFO - 192.168.8.110 - - [31/Mar/2026 14:51:02] "GET /archive-file//home/user/playground/NewsArchiver/archival_data/websites/404%20Media/html/2026-03-31/article_17749630811.html HTTP/1.1" 308 - +2026-03-31 14:51:02,127 - INFO - Archive file path: /home/user/playground/NewsArchiver/archival_data/home/user/playground/NewsArchiver/archival_data/websites/404 Media/html/2026-03-31/article_17749630811.html, exists: False +2026-03-31 14:51:02,127 - INFO - 192.168.8.110 - - [31/Mar/2026 14:51:02] "GET /archive-file/home/user/playground/NewsArchiver/archival_data/websites/404%20Media/html/2026-03-31/article_17749630811.html HTTP/1.1" 404 - +2026-03-31 14:51:02,142 - INFO - 192.168.8.110 - - [31/Mar/2026 14:51:02] "GET /favicon.ico HTTP/1.1" 404 - diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..413de30 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,7 @@ +flask +requests +trafilatura +feedparser +apscheduler +beautifulsoup4 +playwright diff --git a/run_archiver.py b/run_archiver.py index 2da0a92..28219d5 100644 --- a/run_archiver.py +++ b/run_archiver.py @@ -62,8 +62,8 @@ except ImportError: print("WARNING: singlefile_archive module not found") print("SingleFile integration will not be available") -SCRIPT_DIR = Path(__file__).parent -ARCHIVE_DIR = SCRIPT_DIR / 'archival_data' +SCRIPT_DIR = Path(__file__).parent.resolve() +ARCHIVE_DIR = Path(os.environ.get("ARCHIVE_DIR", str(SCRIPT_DIR / "archival_data"))).resolve() ARCHIVE_DIR.mkdir(exist_ok=True) diff --git a/scheduler.py b/scheduler.py index 6221738..82f87f8 100644 --- a/scheduler.py +++ b/scheduler.py @@ -26,8 +26,8 @@ except ImportError: print("ERROR: archive_engine is required") sys.exit(1) -SCRIPT_DIR = Path(__file__).parent -ARCHIVE_DIR = SCRIPT_DIR / 'archival_data' +SCRIPT_DIR = Path(__file__).parent.resolve() +ARCHIVE_DIR = Path(os.environ.get("ARCHIVE_DIR", str(SCRIPT_DIR / "archival_data"))).resolve() ARCHIVE_DIR.mkdir(exist_ok=True) logging.basicConfig( diff --git a/storage_manager.py b/storage_manager.py index 88d6100..e998246 100644 --- a/storage_manager.py +++ b/storage_manager.py @@ -23,8 +23,8 @@ try: except ImportError: FEEDGENERATOR_AVAILABLE = False -SCRIPT_DIR = Path(__file__).parent -ARCHIVE_DIR = SCRIPT_DIR / "archival_data" +SCRIPT_DIR = Path(__file__).parent.resolve() +ARCHIVE_DIR = Path(os.environ.get("ARCHIVE_DIR", str(SCRIPT_DIR / "archival_data"))).resolve() ARCHIVE_DIR.mkdir(exist_ok=True) logging.basicConfig( diff --git a/web_interface.py b/web_interface.py index 300ebf8..612104c 100644 --- a/web_interface.py +++ b/web_interface.py @@ -6,28 +6,30 @@ Flask web server for browsing archived news articles. import json import logging +import os import sys -from datetime import datetime +import xml.etree.ElementTree as ET +from datetime import datetime, timezone from pathlib import Path from typing import Optional from urllib.parse import quote -from flask import Flask, jsonify, request, render_template, make_response -import xml.etree.ElementTree as ET -from datetime import datetime, timezone +from flask import Flask, jsonify, make_response, render_template, request from storage_manager import ( + DB_PATH, get_all_sources, - get_source_stats, - get_articles_by_source, get_article, + get_articles_by_source, get_latest_articles, - DB_PATH + get_source_stats, ) -SCRIPT_DIR = Path(__file__).parent -ARCHIVE_DIR = SCRIPT_DIR / 'archival_data' -RSS_FEEDS_PATH = SCRIPT_DIR / 'rss_feeds.json' +SCRIPT_DIR = Path(__file__).parent.resolve() +ARCHIVE_DIR = Path( + os.environ.get("ARCHIVE_DIR", str(SCRIPT_DIR / "archival_data")) +).resolve() +RSS_FEEDS_PATH = SCRIPT_DIR / "rss_feeds.json" RSS_FEEDS = {} @@ -35,163 +37,173 @@ RSS_FEEDS = {} def load_rss_feeds() -> dict: """Load RSS feeds configuration.""" global RSS_FEEDS - + if RSS_FEEDS: return RSS_FEEDS - + if not RSS_FEEDS_PATH.exists(): logger.warning("RSS feeds file not found: %s", RSS_FEEDS_PATH) return {} - + try: - with open(RSS_FEEDS_PATH, 'r', encoding='utf-8') as f: + with open(RSS_FEEDS_PATH, "r", encoding="utf-8") as f: RSS_FEEDS = json.load(f) return RSS_FEEDS except Exception as e: logger.error("Failed to load RSS feeds: %s", str(e)) return {} + app = Flask( __name__, - static_folder=str(SCRIPT_DIR / 'static'), - template_folder=str(SCRIPT_DIR / 'templates') + static_folder=str(SCRIPT_DIR / "static"), + template_folder=str(SCRIPT_DIR / "templates"), ) logging.basicConfig( level=logging.INFO, - format='%(asctime)s - %(levelname)s - %(message)s', + format="%(asctime)s - %(levelname)s - %(message)s", handlers=[ logging.StreamHandler(sys.stdout), - logging.FileHandler(ARCHIVE_DIR / 'processing.log', encoding='utf-8') - ] + logging.FileHandler(ARCHIVE_DIR / "processing.log", encoding="utf-8"), + ], ) logger = logging.getLogger(__name__) def get_pagination_info(total: int, page: int, per_page: int) -> dict: """Calculate pagination information. - + Args: total: Total number of items page: Current page number per_page: Items per page - + Returns: Dictionary with pagination details """ total_pages = (total + per_page - 1) // per_page if total > 0 else 1 - + return { - 'total': total, - 'page': page, - 'per_page': per_page, - 'has_next': page < total_pages, - 'has_prev': page > 1, - 'next_num': page + 1 if page < total_pages else None, - 'prev_num': page - 1 if page > 1 else None, - 'pages': total_pages + "total": total, + "page": page, + "per_page": per_page, + "has_next": page < total_pages, + "has_prev": page > 1, + "next_num": page + 1 if page < total_pages else None, + "prev_num": page - 1 if page > 1 else None, + "pages": total_pages, } -@app.route('/') +@app.route("/") def index(): """Newspaper listing page.""" sources = get_all_sources() rss_feeds = load_rss_feeds() - + source_list = [] disabled_sources = [] - + for source_name in sources: stats = get_source_stats(source_name) - + source_info = { - 'name': source_name.title(), - 'slug': source_name, - 'article_count': stats['total_articles'], - 'last_archived': stats.get('last_archived'), - 'status': 'success' if stats['total_articles'] > 0 else 'pending' + "name": source_name.title(), + "slug": source_name, + "article_count": stats["total_articles"], + "last_archived": stats.get("last_archived"), + "status": "success" if stats["total_articles"] > 0 else "pending", } - + if source_name in rss_feeds: feed_info = rss_feeds[source_name] - if feed_info.get('disabled', False): - source_info['disabled'] = True - source_info['disable_reason'] = feed_info.get('disable_reason', 'No reason provided') + if feed_info.get("disabled", False): + source_info["disabled"] = True + source_info["disable_reason"] = feed_info.get( + "disable_reason", "No reason provided" + ) disabled_sources.append(source_info) continue - + source_list.append(source_info) - + source_list.extend(disabled_sources) - - return render_template('index.html', sources=source_list) + + return render_template("index.html", sources=source_list) -@app.route('/source/') +@app.route("/source/") def articles(slug: str): """Article listing page for a specific source.""" - page = request.args.get('page', 1, type=int) + page = request.args.get("page", 1, type=int) per_page = 50 - + sources = get_all_sources() source_name = None for s in sources: if s.lower() == slug.lower(): source_name = s break - + if not source_name: - return render_template('article_not_found.html', slug=slug, article_id=0), 404 - - articles_list = get_articles_by_source(source_name, limit=per_page, offset=(page - 1) * per_page) + return render_template("article_not_found.html", slug=slug, article_id=0), 404 + + articles_list = get_articles_by_source( + source_name, limit=per_page, offset=(page - 1) * per_page + ) stats = get_source_stats(source_name) - total = stats['total_articles'] - + total = stats["total_articles"] + pagination = get_pagination_info(total, page, per_page) - + articles_data = [] for article in articles_list: - articles_data.append({ - 'id': getattr(article, 'id', 0), - 'title': article.title or 'Untitled', - 'date': article.publish_date or '', - 'summary': article.content_text[:200] if article.content_text else '', - 'url': f'/source/{source_name.lower()}/article/{getattr(article, "id", 0)}' - }) - + articles_data.append( + { + "id": getattr(article, "id", 0), + "title": article.title or "Untitled", + "date": article.publish_date or "", + "summary": article.content_text[:200] if article.content_text else "", + "url": f"/source/{source_name.lower()}/article/{getattr(article, 'id', 0)}", + } + ) + return render_template( - 'articles.html', + "articles.html", source_name=source_name.title(), source_slug=source_name.lower(), articles=articles_data, - pagination=pagination + pagination=pagination, ) -@app.route('/archive/') +@app.route("/archive/") def serve_archive(archive_path): """Serve archived HTML file.""" archive_file = ARCHIVE_DIR / archive_path if archive_file.exists(): - return archive_file.read_text(encoding='utf-8') - return 'Archive not found', 404 + return archive_file.read_text(encoding="utf-8") + return "Archive not found", 404 -@app.route('/archive-file/') +@app.route("/archive-file/") def serve_archive_file(encoded_path): """Serve archived HTML file from encoded path.""" import urllib.parse from pathlib import Path + archive_path = urllib.parse.unquote(encoded_path) archive_file = ARCHIVE_DIR / archive_path - logger.info("Archive file path: %s, exists: %s", str(archive_file), archive_file.exists()) + logger.info( + "Archive file path: %s, exists: %s", str(archive_file), archive_file.exists() + ) if archive_file.exists(): - return archive_file.read_text(encoding='utf-8') - return 'Archive not found', 404 + return archive_file.read_text(encoding="utf-8") + return "Archive not found", 404 -@app.route('/source//article/') +@app.route("/source//article/") def article(slug: str, article_id: int): """Individual article page.""" sources = get_all_sources() @@ -200,289 +212,324 @@ def article(slug: str, article_id: int): if s.lower() == slug.lower(): source_name = s break - + if not source_name: - return render_template('article_not_found.html', slug=slug, article_id=article_id), 404 - + return render_template( + "article_not_found.html", slug=slug, article_id=article_id + ), 404 + article = get_article(source_name, article_id) if not article: - return render_template('article_not_found.html', slug=slug, article_id=article_id), 404 - + return render_template( + "article_not_found.html", slug=slug, article_id=article_id + ), 404 + article_data = { - 'id': article_id, - 'title': article.title or 'Untitled', - 'publish_date': article.publish_date or '', - 'author': article.author or '', - 'url': article.url or '', - 'content_text': article.content_text or '', - 'archive_file_path': article.archive_file_path or '' + "id": article_id, + "title": article.title or "Untitled", + "publish_date": article.publish_date or "", + "author": article.author or "", + "url": article.url or "", + "content_text": article.content_text or "", + "archive_file_path": article.archive_file_path or "", } - + return render_template( - 'article.html', + "article.html", source_name=source_name.title(), source_slug=source_name.lower(), - article=article_data + article=article_data, ) -@app.route('/status') +@app.route("/status") def status(): """System status page.""" sources = get_all_sources() rss_feeds = load_rss_feeds() - + sources_info = [] disabled_sources = [] total_articles = 0 failed_jobs = 0 disabled_count = 0 - + for source_name in sources: stats = get_source_stats(source_name) - + source_info = { - 'name': source_name.title(), - 'slug': source_name, - 'article_count': stats['total_articles'], - 'last_archived': stats.get('last_archived'), - 'status': 'success' if stats['total_articles'] > 0 else 'pending' + "name": source_name.title(), + "slug": source_name, + "article_count": stats["total_articles"], + "last_archived": stats.get("last_archived"), + "status": "success" if stats["total_articles"] > 0 else "pending", } - + if source_name in rss_feeds: feed_info = rss_feeds[source_name] - if feed_info.get('disabled', False): - source_info['disabled'] = True + if feed_info.get("disabled", False): + source_info["disabled"] = True disabled_count += 1 disabled_sources.append(source_info) continue - + sources_info.append(source_info) - total_articles += stats['total_articles'] - failed_jobs += stats['failed'] - + total_articles += stats["total_articles"] + failed_jobs += stats["failed"] + sources_info.extend(disabled_sources) - + return render_template( - 'status.html', + "status.html", sources=sources_info, total_articles=total_articles, failed_jobs=failed_jobs, sources_monitored=len(sources) - disabled_count, - disabled_sources=disabled_count + disabled_sources=disabled_count, ) -@app.route('/api/sources') +@app.route("/api/sources") def api_sources(): """API endpoint for listing all sources.""" sources = get_all_sources() rss_feeds = load_rss_feeds() - + source_list = [] disabled_sources = [] - + for source_name in sources: stats = get_source_stats(source_name) - + source_info = { - 'name': source_name.title(), - 'slug': source_name, - 'article_count': stats['total_articles'], - 'last_archived': stats.get('last_archived'), - 'status': 'success' if stats['total_articles'] > 0 else 'pending' + "name": source_name.title(), + "slug": source_name, + "article_count": stats["total_articles"], + "last_archived": stats.get("last_archived"), + "status": "success" if stats["total_articles"] > 0 else "pending", } - + if source_name in rss_feeds: feed_info = rss_feeds[source_name] - if feed_info.get('disabled', False): - source_info['disabled'] = True - source_info['disable_reason'] = feed_info.get('disable_reason', 'No reason provided') + if feed_info.get("disabled", False): + source_info["disabled"] = True + source_info["disable_reason"] = feed_info.get( + "disable_reason", "No reason provided" + ) disabled_sources.append(source_info) continue - + source_list.append(source_info) - + source_list.extend(disabled_sources) - - return jsonify({'sources': source_list}) + + return jsonify({"sources": source_list}) -@app.route('/api/source//articles') +@app.route("/api/source//articles") def api_articles(slug: str): """API endpoint for listing articles for a source.""" - page = request.args.get('page', 1, type=int) + page = request.args.get("page", 1, type=int) per_page = 50 - + sources = get_all_sources() source_name = None for s in sources: if s.lower() == slug.lower(): source_name = s break - + if not source_name: - return jsonify({'error': 'Source not found'}), 404 - - articles_list = get_articles_by_source(source_name, limit=per_page, offset=(page - 1) * per_page) + return jsonify({"error": "Source not found"}), 404 + + articles_list = get_articles_by_source( + source_name, limit=per_page, offset=(page - 1) * per_page + ) stats = get_source_stats(source_name) - total = stats['total_articles'] - + total = stats["total_articles"] + pagination = get_pagination_info(total, page, per_page) - + articles_data = [] for article in articles_list: - articles_data.append({ - 'id': getattr(article, 'id', 0), - 'title': article.title or 'Untitled', - 'date': article.publish_date or '', - 'summary': article.content_text[:200] if article.content_text else '', - 'url': f'/source/{source_name.lower()}/article/{getattr(article, "id", 0)}' - }) - - return jsonify({ - 'source_name': source_name.title(), - 'articles': articles_data, - 'total': total, - 'page': page, - 'per_page': per_page, - 'has_next': pagination['has_next'], - 'has_prev': pagination['has_prev'] - }) + articles_data.append( + { + "id": getattr(article, "id", 0), + "title": article.title or "Untitled", + "date": article.publish_date or "", + "summary": article.content_text[:200] if article.content_text else "", + "url": f"/source/{source_name.lower()}/article/{getattr(article, 'id', 0)}", + } + ) + + return jsonify( + { + "source_name": source_name.title(), + "articles": articles_data, + "total": total, + "page": page, + "per_page": per_page, + "has_next": pagination["has_next"], + "has_prev": pagination["has_prev"], + } + ) -@app.route('/api/status') +@app.route("/api/status") def api_status(): """API endpoint for system status.""" sources = get_all_sources() - + sources_monitored = len(sources) total_articles = 0 failed_jobs = 0 last_archive_run = None - + for source_name in sources: stats = get_source_stats(source_name) - total_articles += stats['total_articles'] - failed_jobs += stats['failed'] - - if stats.get('last_archive_run'): - if last_archive_run is None or stats['last_archive_run'] > last_archive_run: - last_archive_run = stats['last_archive_run'] - - return jsonify({ - 'status': 'online', - 'last_archive_run': last_archive_run, - 'pending_jobs': 0, - 'failed_jobs': failed_jobs, - 'sources_monitored': sources_monitored, - 'total_articles': total_articles - }) + total_articles += stats["total_articles"] + failed_jobs += stats["failed"] + + if stats.get("last_archive_run"): + if last_archive_run is None or stats["last_archive_run"] > last_archive_run: + last_archive_run = stats["last_archive_run"] + + return jsonify( + { + "status": "online", + "last_archive_run": last_archive_run, + "pending_jobs": 0, + "failed_jobs": failed_jobs, + "sources_monitored": sources_monitored, + "total_articles": total_articles, + } + ) -@app.route('/rss') +@app.route("/rss") def rss_feed(): """RSS 2.0 endpoint for latest archived articles.""" - limit = request.args.get('limit', 50, type=int) - + limit = request.args.get("limit", 50, type=int) + articles = get_latest_articles(limit=limit) - - server_url = f'http://192.168.8.150:5000' - + + server_url = f"http://192.168.8.150:5000" + rss_items = [] for article in articles: - if article.title and article.content_text and 'Performing security verification' not in article.content_text: + if ( + article.title + and article.content_text + and "Performing security verification" not in article.content_text + ): pub_date = None if article.publish_date: try: - dt = datetime.fromisoformat(article.publish_date.replace('Z', '+00:00')) - pub_date = dt.strftime('%a, %d %b %Y %H:%M:%S %z').strip() + dt = datetime.fromisoformat( + article.publish_date.replace("Z", "+00:00") + ) + pub_date = dt.strftime("%a, %d %b %Y %H:%M:%S %z").strip() except (ValueError, AttributeError): try: dt = datetime.fromisoformat(article.publish_date) - pub_date = dt.strftime('%a, %d %b %Y %H:%M:%S +0000') + pub_date = dt.strftime("%a, %d %b %Y %H:%M:%S +0000") except (ValueError, AttributeError): - pub_date = datetime.now(timezone.utc).strftime('%a, %d %b %Y %H:%M:%S +0000') - - source_name = article.source_name or 'unknown' + pub_date = datetime.now(timezone.utc).strftime( + "%a, %d %b %Y %H:%M:%S +0000" + ) + + source_name = article.source_name or "unknown" encoded_source = quote(source_name.lower()) item = { - 'title': article.title, - 'link': f'{server_url}/source/{encoded_source}/article/{article.id}', - 'pubDate': pub_date, - 'description': article.content_text[:500] if article.content_text else '', - 'guid': article.url or f'article-{article.id}' + "title": article.title, + "link": f"{server_url}/source/{encoded_source}/article/{article.id}", + "pubDate": pub_date, + "description": article.content_text[:500] + if article.content_text + else "", + "guid": article.url or f"article-{article.id}", } if article.author: - item['author'] = article.author + item["author"] = article.author rss_items.append(item) - + rss_template = render_template( - 'rss.xml', - title='NewsArchiver - Latest Articles', + "rss.xml", + title="NewsArchiver - Latest Articles", link=server_url, - description='Latest archived news articles', - last_build_date=datetime.now(timezone.utc).strftime('%a, %d %b %Y %H:%M:%S +0000'), - items=rss_items + description="Latest archived news articles", + last_build_date=datetime.now(timezone.utc).strftime( + "%a, %d %b %Y %H:%M:%S +0000" + ), + items=rss_items, ) - + response = make_response(rss_template) - response.headers['Content-Type'] = 'application/rss+xml; charset=utf-8' + response.headers["Content-Type"] = "application/rss+xml; charset=utf-8" return response -@app.route('/atom') +@app.route("/atom") def atom_feed(): """Atom 1.0 endpoint for latest archived articles.""" - limit = request.args.get('limit', 50, type=int) - + limit = request.args.get("limit", 50, type=int) + articles = get_latest_articles(limit=limit) - - server_url = f'http://192.168.8.150:5000' - + + server_url = f"http://192.168.8.150:5000" + atom_entries = [] for article in articles: - if article.title and article.content_text and 'Performing security verification' not in article.content_text: + if ( + article.title + and article.content_text + and "Performing security verification" not in article.content_text + ): pub_date = None if article.publish_date: try: - dt = datetime.fromisoformat(article.publish_date.replace('Z', '+00:00')) - pub_date = dt.strftime('%Y-%m-%dT%H:%M:%S+00:00') + dt = datetime.fromisoformat( + article.publish_date.replace("Z", "+00:00") + ) + pub_date = dt.strftime("%Y-%m-%dT%H:%M:%S+00:00") except (ValueError, AttributeError): - pub_date = datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%S+00:00') - - source_name = article.source_name or 'unknown' + pub_date = datetime.now(timezone.utc).strftime( + "%Y-%m-%dT%H:%M:%S+00:00" + ) + + source_name = article.source_name or "unknown" encoded_source = quote(source_name.lower()) entry = { - 'title': article.title, - 'link': f'{server_url}/source/{encoded_source}/article/{article.id}', - 'published': pub_date, - 'summary': article.content_text[:500] if article.content_text else '', - 'id': article.url or f'article-{article.id}' + "title": article.title, + "link": f"{server_url}/source/{encoded_source}/article/{article.id}", + "published": pub_date, + "summary": article.content_text[:500] if article.content_text else "", + "id": article.url or f"article-{article.id}", } if article.author: - entry['author'] = {'name': article.author} + entry["author"] = {"name": article.author} atom_entries.append(entry) - + atom_template = render_template( - 'atom.xml', - title='NewsArchiver - Latest Articles', + "atom.xml", + title="NewsArchiver - Latest Articles", link=server_url, - updated=datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%S+00:00'), - entries=atom_entries + updated=datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S+00:00"), + entries=atom_entries, ) - + response = make_response(atom_template) - response.headers['Content-Type'] = 'application/atom+xml; charset=utf-8' + response.headers["Content-Type"] = "application/atom+xml; charset=utf-8" return response -if __name__ == '__main__': +if __name__ == "__main__": logger.info("Starting web interface...") - + if not DB_PATH.exists(): logger.info("Database not found, initializing...") from storage_manager import initialize_storage + initialize_storage() - - app.run(host='0.0.0.0', port=5000, debug=True) \ No newline at end of file + + app.run(host="0.0.0.0", port=5000, debug=True)