#!/usr/bin/env python3 """Web Interface for NewsArchiver - Phase 3 Flask web server for browsing archived news articles. """ import json import logging import os import sys import xml.etree.ElementTree as ET from datetime import datetime, timezone from pathlib import Path from typing import Optional from urllib.parse import quote from flask import Flask, jsonify, make_response, render_template, request from storage_manager import ( DB_PATH, get_all_sources, get_article, get_articles_by_source, get_latest_articles, get_source_stats, ) SCRIPT_DIR = Path(__file__).parent.resolve() ARCHIVE_DIR = Path( os.environ.get("ARCHIVE_DIR", str(SCRIPT_DIR / "archival_data")) ).resolve() RSS_FEEDS_PATH = SCRIPT_DIR / "rss_feeds.json" RSS_FEEDS = {} def load_rss_feeds() -> dict: """Load RSS feeds configuration.""" global RSS_FEEDS if RSS_FEEDS: return RSS_FEEDS if not RSS_FEEDS_PATH.exists(): logger.warning("RSS feeds file not found: %s", RSS_FEEDS_PATH) return {} try: with open(RSS_FEEDS_PATH, "r", encoding="utf-8") as f: RSS_FEEDS = json.load(f) return RSS_FEEDS except Exception as e: logger.error("Failed to load RSS feeds: %s", str(e)) return {} app = Flask( __name__, static_folder=str(SCRIPT_DIR / "static"), template_folder=str(SCRIPT_DIR / "templates"), ) logging.basicConfig( level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s", handlers=[ logging.StreamHandler(sys.stdout), logging.FileHandler(ARCHIVE_DIR / "processing.log", encoding="utf-8"), ], ) logger = logging.getLogger(__name__) def get_pagination_info(total: int, page: int, per_page: int) -> dict: """Calculate pagination information. Args: total: Total number of items page: Current page number per_page: Items per page Returns: Dictionary with pagination details """ total_pages = (total + per_page - 1) // per_page if total > 0 else 1 return { "total": total, "page": page, "per_page": per_page, "has_next": page < total_pages, "has_prev": page > 1, "next_num": page + 1 if page < total_pages else None, "prev_num": page - 1 if page > 1 else None, "pages": total_pages, } @app.route("/") def index(): """Newspaper listing page.""" sources = get_all_sources() rss_feeds = load_rss_feeds() source_list = [] disabled_sources = [] for source_name in sources: stats = get_source_stats(source_name) source_info = { "name": source_name.title(), "slug": source_name, "article_count": stats["total_articles"], "last_archived": stats.get("last_archived"), "status": "success" if stats["total_articles"] > 0 else "pending", } if source_name in rss_feeds: feed_info = rss_feeds[source_name] if feed_info.get("disabled", False): source_info["disabled"] = True source_info["disable_reason"] = feed_info.get( "disable_reason", "No reason provided" ) disabled_sources.append(source_info) continue source_list.append(source_info) source_list.extend(disabled_sources) return render_template("index.html", sources=source_list) @app.route("/source/") def articles(slug: str): """Article listing page for a specific source.""" page = request.args.get("page", 1, type=int) per_page = 50 sources = get_all_sources() source_name = None for s in sources: if s.lower() == slug.lower(): source_name = s break if not source_name: return render_template("article_not_found.html", slug=slug, article_id=0), 404 articles_list = get_articles_by_source( source_name, limit=per_page, offset=(page - 1) * per_page ) stats = get_source_stats(source_name) total = stats["total_articles"] pagination = get_pagination_info(total, page, per_page) articles_data = [] for article in articles_list: articles_data.append( { "id": getattr(article, "id", 0), "title": article.title or "Untitled", "date": article.publish_date or "", "summary": article.content_text[:200] if article.content_text else "", "url": f"/source/{source_name.lower()}/article/{getattr(article, 'id', 0)}", } ) return render_template( "articles.html", source_name=source_name.title(), source_slug=source_name.lower(), articles=articles_data, pagination=pagination, ) @app.route("/archive/") def serve_archive(archive_path): """Serve archived HTML file.""" archive_file = ARCHIVE_DIR / archive_path if archive_file.exists(): return archive_file.read_text(encoding="utf-8") return "Archive not found", 404 @app.route("/archive-file/") def serve_archive_file(encoded_path): """Serve archived HTML file from encoded path.""" import urllib.parse from pathlib import Path archive_path = urllib.parse.unquote(encoded_path) archive_file = ARCHIVE_DIR / archive_path logger.info( "Archive file path: %s, exists: %s", str(archive_file), archive_file.exists() ) if archive_file.exists(): return archive_file.read_text(encoding="utf-8") return "Archive not found", 404 @app.route("/source//article/") def article(slug: str, article_id: int): """Individual article page.""" sources = get_all_sources() source_name = None for s in sources: if s.lower() == slug.lower(): source_name = s break if not source_name: return render_template( "article_not_found.html", slug=slug, article_id=article_id ), 404 article = get_article(source_name, article_id) if not article: return render_template( "article_not_found.html", slug=slug, article_id=article_id ), 404 article_data = { "id": article_id, "title": article.title or "Untitled", "publish_date": article.publish_date or "", "author": article.author or "", "url": article.url or "", "content_text": article.content_text or "", "archive_file_path": article.archive_file_path or "", } return render_template( "article.html", source_name=source_name.title(), source_slug=source_name.lower(), article=article_data, ) @app.route("/status") def status(): """System status page.""" sources = get_all_sources() rss_feeds = load_rss_feeds() sources_info = [] disabled_sources = [] total_articles = 0 failed_jobs = 0 disabled_count = 0 for source_name in sources: stats = get_source_stats(source_name) source_info = { "name": source_name.title(), "slug": source_name, "article_count": stats["total_articles"], "last_archived": stats.get("last_archived"), "status": "success" if stats["total_articles"] > 0 else "pending", } if source_name in rss_feeds: feed_info = rss_feeds[source_name] if feed_info.get("disabled", False): source_info["disabled"] = True disabled_count += 1 disabled_sources.append(source_info) continue sources_info.append(source_info) total_articles += stats["total_articles"] failed_jobs += stats["failed"] sources_info.extend(disabled_sources) return render_template( "status.html", sources=sources_info, total_articles=total_articles, failed_jobs=failed_jobs, sources_monitored=len(sources) - disabled_count, disabled_sources=disabled_count, ) @app.route("/api/sources") def api_sources(): """API endpoint for listing all sources.""" sources = get_all_sources() rss_feeds = load_rss_feeds() source_list = [] disabled_sources = [] for source_name in sources: stats = get_source_stats(source_name) source_info = { "name": source_name.title(), "slug": source_name, "article_count": stats["total_articles"], "last_archived": stats.get("last_archived"), "status": "success" if stats["total_articles"] > 0 else "pending", } if source_name in rss_feeds: feed_info = rss_feeds[source_name] if feed_info.get("disabled", False): source_info["disabled"] = True source_info["disable_reason"] = feed_info.get( "disable_reason", "No reason provided" ) disabled_sources.append(source_info) continue source_list.append(source_info) source_list.extend(disabled_sources) return jsonify({"sources": source_list}) @app.route("/api/source//articles") def api_articles(slug: str): """API endpoint for listing articles for a source.""" page = request.args.get("page", 1, type=int) per_page = 50 sources = get_all_sources() source_name = None for s in sources: if s.lower() == slug.lower(): source_name = s break if not source_name: return jsonify({"error": "Source not found"}), 404 articles_list = get_articles_by_source( source_name, limit=per_page, offset=(page - 1) * per_page ) stats = get_source_stats(source_name) total = stats["total_articles"] pagination = get_pagination_info(total, page, per_page) articles_data = [] for article in articles_list: articles_data.append( { "id": getattr(article, "id", 0), "title": article.title or "Untitled", "date": article.publish_date or "", "summary": article.content_text[:200] if article.content_text else "", "url": f"/source/{source_name.lower()}/article/{getattr(article, 'id', 0)}", } ) return jsonify( { "source_name": source_name.title(), "articles": articles_data, "total": total, "page": page, "per_page": per_page, "has_next": pagination["has_next"], "has_prev": pagination["has_prev"], } ) @app.route("/api/status") def api_status(): """API endpoint for system status.""" sources = get_all_sources() sources_monitored = len(sources) total_articles = 0 failed_jobs = 0 last_archive_run = None for source_name in sources: stats = get_source_stats(source_name) total_articles += stats["total_articles"] failed_jobs += stats["failed"] if stats.get("last_archive_run"): if last_archive_run is None or stats["last_archive_run"] > last_archive_run: last_archive_run = stats["last_archive_run"] return jsonify( { "status": "online", "last_archive_run": last_archive_run, "pending_jobs": 0, "failed_jobs": failed_jobs, "sources_monitored": sources_monitored, "total_articles": total_articles, } ) @app.route("/rss") def rss_feed(): """RSS 2.0 endpoint for latest archived articles.""" limit = request.args.get("limit", 50, type=int) articles = get_latest_articles(limit=limit) server_url = f"http://192.168.8.150:5000" rss_items = [] for article in articles: if ( article.title and article.content_text and "Performing security verification" not in article.content_text ): pub_date = None if article.publish_date: try: dt = datetime.fromisoformat( article.publish_date.replace("Z", "+00:00") ) pub_date = dt.strftime("%a, %d %b %Y %H:%M:%S %z").strip() except (ValueError, AttributeError): try: dt = datetime.fromisoformat(article.publish_date) pub_date = dt.strftime("%a, %d %b %Y %H:%M:%S +0000") except (ValueError, AttributeError): pub_date = datetime.now(timezone.utc).strftime( "%a, %d %b %Y %H:%M:%S +0000" ) source_name = article.source_name or "unknown" encoded_source = quote(source_name.lower()) item = { "title": article.title, "link": f"{server_url}/source/{encoded_source}/article/{article.id}", "pubDate": pub_date, "description": article.content_text[:500] if article.content_text else "", "guid": article.url or f"article-{article.id}", } if article.author: item["author"] = article.author rss_items.append(item) rss_template = render_template( "rss.xml", title="NewsArchiver - Latest Articles", link=server_url, description="Latest archived news articles", last_build_date=datetime.now(timezone.utc).strftime( "%a, %d %b %Y %H:%M:%S +0000" ), items=rss_items, ) response = make_response(rss_template) response.headers["Content-Type"] = "application/rss+xml; charset=utf-8" return response @app.route("/atom") def atom_feed(): """Atom 1.0 endpoint for latest archived articles.""" limit = request.args.get("limit", 50, type=int) articles = get_latest_articles(limit=limit) server_url = f"http://192.168.8.150:5000" atom_entries = [] for article in articles: if ( article.title and article.content_text and "Performing security verification" not in article.content_text ): pub_date = None if article.publish_date: try: dt = datetime.fromisoformat( article.publish_date.replace("Z", "+00:00") ) pub_date = dt.strftime("%Y-%m-%dT%H:%M:%S+00:00") except (ValueError, AttributeError): pub_date = datetime.now(timezone.utc).strftime( "%Y-%m-%dT%H:%M:%S+00:00" ) source_name = article.source_name or "unknown" encoded_source = quote(source_name.lower()) entry = { "title": article.title, "link": f"{server_url}/source/{encoded_source}/article/{article.id}", "published": pub_date, "summary": article.content_text[:500] if article.content_text else "", "id": article.url or f"article-{article.id}", } if article.author: entry["author"] = {"name": article.author} atom_entries.append(entry) atom_template = render_template( "atom.xml", title="NewsArchiver - Latest Articles", link=server_url, updated=datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S+00:00"), entries=atom_entries, ) response = make_response(atom_template) response.headers["Content-Type"] = "application/atom+xml; charset=utf-8" return response if __name__ == "__main__": logger.info("Starting web interface...") if not DB_PATH.exists(): logger.info("Database not found, initializing...") from storage_manager import initialize_storage initialize_storage() app.run(host="0.0.0.0", port=5000, debug=True)