536 lines
16 KiB
Python
536 lines
16 KiB
Python
#!/usr/bin/env python3
|
|
"""Web Interface for NewsArchiver - Phase 3
|
|
|
|
Flask web server for browsing archived news articles.
|
|
"""
|
|
|
|
import json
|
|
import logging
|
|
import os
|
|
import sys
|
|
import xml.etree.ElementTree as ET
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Optional
|
|
from urllib.parse import quote
|
|
|
|
from flask import Flask, jsonify, make_response, render_template, request
|
|
|
|
from storage_manager import (
|
|
DB_PATH,
|
|
get_all_sources,
|
|
get_article,
|
|
get_articles_by_source,
|
|
get_latest_articles,
|
|
get_source_stats,
|
|
)
|
|
|
|
SCRIPT_DIR = Path(__file__).parent.resolve()
|
|
ARCHIVE_DIR = Path(
|
|
os.environ.get("ARCHIVE_DIR", str(SCRIPT_DIR / "archival_data"))
|
|
).resolve()
|
|
RSS_FEEDS_PATH = SCRIPT_DIR / "rss_feeds.json"
|
|
|
|
RSS_FEEDS = {}
|
|
|
|
|
|
def load_rss_feeds() -> dict:
|
|
"""Load RSS feeds configuration."""
|
|
global RSS_FEEDS
|
|
|
|
if RSS_FEEDS:
|
|
return RSS_FEEDS
|
|
|
|
if not RSS_FEEDS_PATH.exists():
|
|
logger.warning("RSS feeds file not found: %s", RSS_FEEDS_PATH)
|
|
return {}
|
|
|
|
try:
|
|
with open(RSS_FEEDS_PATH, "r", encoding="utf-8") as f:
|
|
RSS_FEEDS = json.load(f)
|
|
return RSS_FEEDS
|
|
except Exception as e:
|
|
logger.error("Failed to load RSS feeds: %s", str(e))
|
|
return {}
|
|
|
|
|
|
app = Flask(
|
|
__name__,
|
|
static_folder=str(SCRIPT_DIR / "static"),
|
|
template_folder=str(SCRIPT_DIR / "templates"),
|
|
)
|
|
|
|
logging.basicConfig(
|
|
level=logging.INFO,
|
|
format="%(asctime)s - %(levelname)s - %(message)s",
|
|
handlers=[
|
|
logging.StreamHandler(sys.stdout),
|
|
logging.FileHandler(ARCHIVE_DIR / "processing.log", encoding="utf-8"),
|
|
],
|
|
)
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def get_pagination_info(total: int, page: int, per_page: int) -> dict:
|
|
"""Calculate pagination information.
|
|
|
|
Args:
|
|
total: Total number of items
|
|
page: Current page number
|
|
per_page: Items per page
|
|
|
|
Returns:
|
|
Dictionary with pagination details
|
|
"""
|
|
total_pages = (total + per_page - 1) // per_page if total > 0 else 1
|
|
|
|
return {
|
|
"total": total,
|
|
"page": page,
|
|
"per_page": per_page,
|
|
"has_next": page < total_pages,
|
|
"has_prev": page > 1,
|
|
"next_num": page + 1 if page < total_pages else None,
|
|
"prev_num": page - 1 if page > 1 else None,
|
|
"pages": total_pages,
|
|
}
|
|
|
|
|
|
@app.route("/")
|
|
def index():
|
|
"""Newspaper listing page."""
|
|
sources = get_all_sources()
|
|
rss_feeds = load_rss_feeds()
|
|
|
|
source_list = []
|
|
disabled_sources = []
|
|
|
|
for source_name in sources:
|
|
stats = get_source_stats(source_name)
|
|
|
|
source_info = {
|
|
"name": source_name.title(),
|
|
"slug": source_name,
|
|
"article_count": stats["total_articles"],
|
|
"last_archived": stats.get("last_archived"),
|
|
"status": "success" if stats["total_articles"] > 0 else "pending",
|
|
}
|
|
|
|
if source_name in rss_feeds:
|
|
feed_info = rss_feeds[source_name]
|
|
if feed_info.get("disabled", False):
|
|
source_info["disabled"] = True
|
|
source_info["disable_reason"] = feed_info.get(
|
|
"disable_reason", "No reason provided"
|
|
)
|
|
disabled_sources.append(source_info)
|
|
continue
|
|
|
|
source_list.append(source_info)
|
|
|
|
source_list.extend(disabled_sources)
|
|
|
|
return render_template("index.html", sources=source_list)
|
|
|
|
|
|
@app.route("/source/<slug>")
|
|
def articles(slug: str):
|
|
"""Article listing page for a specific source."""
|
|
page = request.args.get("page", 1, type=int)
|
|
per_page = 50
|
|
|
|
sources = get_all_sources()
|
|
source_name = None
|
|
for s in sources:
|
|
if s.lower() == slug.lower():
|
|
source_name = s
|
|
break
|
|
|
|
if not source_name:
|
|
return render_template("article_not_found.html", slug=slug, article_id=0), 404
|
|
|
|
articles_list = get_articles_by_source(
|
|
source_name, limit=per_page, offset=(page - 1) * per_page
|
|
)
|
|
stats = get_source_stats(source_name)
|
|
total = stats["total_articles"]
|
|
|
|
pagination = get_pagination_info(total, page, per_page)
|
|
|
|
articles_data = []
|
|
for article in articles_list:
|
|
articles_data.append(
|
|
{
|
|
"id": getattr(article, "id", 0),
|
|
"title": article.title or "Untitled",
|
|
"date": article.publish_date or "",
|
|
"summary": article.content_text[:200] if article.content_text else "",
|
|
"url": f"/source/{source_name.lower()}/article/{getattr(article, 'id', 0)}",
|
|
}
|
|
)
|
|
|
|
return render_template(
|
|
"articles.html",
|
|
source_name=source_name.title(),
|
|
source_slug=source_name.lower(),
|
|
articles=articles_data,
|
|
pagination=pagination,
|
|
)
|
|
|
|
|
|
@app.route("/archive/<path:archive_path>")
|
|
def serve_archive(archive_path):
|
|
"""Serve archived HTML file."""
|
|
archive_file = ARCHIVE_DIR / archive_path
|
|
if archive_file.exists():
|
|
return archive_file.read_text(encoding="utf-8")
|
|
return "Archive not found", 404
|
|
|
|
|
|
@app.route("/archive-file/<path:encoded_path>")
|
|
def serve_archive_file(encoded_path):
|
|
"""Serve archived HTML file from encoded path."""
|
|
import urllib.parse
|
|
from pathlib import Path
|
|
|
|
archive_path = urllib.parse.unquote(encoded_path)
|
|
archive_file = ARCHIVE_DIR / archive_path
|
|
logger.info(
|
|
"Archive file path: %s, exists: %s", str(archive_file), archive_file.exists()
|
|
)
|
|
if archive_file.exists():
|
|
return archive_file.read_text(encoding="utf-8")
|
|
return "Archive not found", 404
|
|
|
|
|
|
@app.route("/source/<slug>/article/<int:article_id>")
|
|
def article(slug: str, article_id: int):
|
|
"""Individual article page."""
|
|
sources = get_all_sources()
|
|
source_name = None
|
|
for s in sources:
|
|
if s.lower() == slug.lower():
|
|
source_name = s
|
|
break
|
|
|
|
if not source_name:
|
|
return render_template(
|
|
"article_not_found.html", slug=slug, article_id=article_id
|
|
), 404
|
|
|
|
article = get_article(source_name, article_id)
|
|
if not article:
|
|
return render_template(
|
|
"article_not_found.html", slug=slug, article_id=article_id
|
|
), 404
|
|
|
|
article_data = {
|
|
"id": article_id,
|
|
"title": article.title or "Untitled",
|
|
"publish_date": article.publish_date or "",
|
|
"author": article.author or "",
|
|
"url": article.url or "",
|
|
"content_text": article.content_text or "",
|
|
"archive_file_path": article.archive_file_path or "",
|
|
}
|
|
|
|
return render_template(
|
|
"article.html",
|
|
source_name=source_name.title(),
|
|
source_slug=source_name.lower(),
|
|
article=article_data,
|
|
)
|
|
|
|
|
|
@app.route("/status")
|
|
def status():
|
|
"""System status page."""
|
|
sources = get_all_sources()
|
|
rss_feeds = load_rss_feeds()
|
|
|
|
sources_info = []
|
|
disabled_sources = []
|
|
total_articles = 0
|
|
failed_jobs = 0
|
|
disabled_count = 0
|
|
|
|
for source_name in sources:
|
|
stats = get_source_stats(source_name)
|
|
|
|
source_info = {
|
|
"name": source_name.title(),
|
|
"slug": source_name,
|
|
"article_count": stats["total_articles"],
|
|
"last_archived": stats.get("last_archived"),
|
|
"status": "success" if stats["total_articles"] > 0 else "pending",
|
|
}
|
|
|
|
if source_name in rss_feeds:
|
|
feed_info = rss_feeds[source_name]
|
|
if feed_info.get("disabled", False):
|
|
source_info["disabled"] = True
|
|
disabled_count += 1
|
|
disabled_sources.append(source_info)
|
|
continue
|
|
|
|
sources_info.append(source_info)
|
|
total_articles += stats["total_articles"]
|
|
failed_jobs += stats["failed"]
|
|
|
|
sources_info.extend(disabled_sources)
|
|
|
|
return render_template(
|
|
"status.html",
|
|
sources=sources_info,
|
|
total_articles=total_articles,
|
|
failed_jobs=failed_jobs,
|
|
sources_monitored=len(sources) - disabled_count,
|
|
disabled_sources=disabled_count,
|
|
)
|
|
|
|
|
|
@app.route("/api/sources")
|
|
def api_sources():
|
|
"""API endpoint for listing all sources."""
|
|
sources = get_all_sources()
|
|
rss_feeds = load_rss_feeds()
|
|
|
|
source_list = []
|
|
disabled_sources = []
|
|
|
|
for source_name in sources:
|
|
stats = get_source_stats(source_name)
|
|
|
|
source_info = {
|
|
"name": source_name.title(),
|
|
"slug": source_name,
|
|
"article_count": stats["total_articles"],
|
|
"last_archived": stats.get("last_archived"),
|
|
"status": "success" if stats["total_articles"] > 0 else "pending",
|
|
}
|
|
|
|
if source_name in rss_feeds:
|
|
feed_info = rss_feeds[source_name]
|
|
if feed_info.get("disabled", False):
|
|
source_info["disabled"] = True
|
|
source_info["disable_reason"] = feed_info.get(
|
|
"disable_reason", "No reason provided"
|
|
)
|
|
disabled_sources.append(source_info)
|
|
continue
|
|
|
|
source_list.append(source_info)
|
|
|
|
source_list.extend(disabled_sources)
|
|
|
|
return jsonify({"sources": source_list})
|
|
|
|
|
|
@app.route("/api/source/<slug>/articles")
|
|
def api_articles(slug: str):
|
|
"""API endpoint for listing articles for a source."""
|
|
page = request.args.get("page", 1, type=int)
|
|
per_page = 50
|
|
|
|
sources = get_all_sources()
|
|
source_name = None
|
|
for s in sources:
|
|
if s.lower() == slug.lower():
|
|
source_name = s
|
|
break
|
|
|
|
if not source_name:
|
|
return jsonify({"error": "Source not found"}), 404
|
|
|
|
articles_list = get_articles_by_source(
|
|
source_name, limit=per_page, offset=(page - 1) * per_page
|
|
)
|
|
stats = get_source_stats(source_name)
|
|
total = stats["total_articles"]
|
|
|
|
pagination = get_pagination_info(total, page, per_page)
|
|
|
|
articles_data = []
|
|
for article in articles_list:
|
|
articles_data.append(
|
|
{
|
|
"id": getattr(article, "id", 0),
|
|
"title": article.title or "Untitled",
|
|
"date": article.publish_date or "",
|
|
"summary": article.content_text[:200] if article.content_text else "",
|
|
"url": f"/source/{source_name.lower()}/article/{getattr(article, 'id', 0)}",
|
|
}
|
|
)
|
|
|
|
return jsonify(
|
|
{
|
|
"source_name": source_name.title(),
|
|
"articles": articles_data,
|
|
"total": total,
|
|
"page": page,
|
|
"per_page": per_page,
|
|
"has_next": pagination["has_next"],
|
|
"has_prev": pagination["has_prev"],
|
|
}
|
|
)
|
|
|
|
|
|
@app.route("/api/status")
|
|
def api_status():
|
|
"""API endpoint for system status."""
|
|
sources = get_all_sources()
|
|
|
|
sources_monitored = len(sources)
|
|
total_articles = 0
|
|
failed_jobs = 0
|
|
last_archive_run = None
|
|
|
|
for source_name in sources:
|
|
stats = get_source_stats(source_name)
|
|
total_articles += stats["total_articles"]
|
|
failed_jobs += stats["failed"]
|
|
|
|
if stats.get("last_archive_run"):
|
|
if last_archive_run is None or stats["last_archive_run"] > last_archive_run:
|
|
last_archive_run = stats["last_archive_run"]
|
|
|
|
return jsonify(
|
|
{
|
|
"status": "online",
|
|
"last_archive_run": last_archive_run,
|
|
"pending_jobs": 0,
|
|
"failed_jobs": failed_jobs,
|
|
"sources_monitored": sources_monitored,
|
|
"total_articles": total_articles,
|
|
}
|
|
)
|
|
|
|
|
|
@app.route("/rss")
|
|
def rss_feed():
|
|
"""RSS 2.0 endpoint for latest archived articles."""
|
|
limit = request.args.get("limit", 50, type=int)
|
|
|
|
articles = get_latest_articles(limit=limit)
|
|
|
|
server_url = f"http://192.168.8.150:5000"
|
|
|
|
rss_items = []
|
|
for article in articles:
|
|
if (
|
|
article.title
|
|
and article.content_text
|
|
and "Performing security verification" not in article.content_text
|
|
):
|
|
pub_date = None
|
|
if article.publish_date:
|
|
try:
|
|
dt = datetime.fromisoformat(
|
|
article.publish_date.replace("Z", "+00:00")
|
|
)
|
|
pub_date = dt.strftime("%a, %d %b %Y %H:%M:%S %z").strip()
|
|
except (ValueError, AttributeError):
|
|
try:
|
|
dt = datetime.fromisoformat(article.publish_date)
|
|
pub_date = dt.strftime("%a, %d %b %Y %H:%M:%S +0000")
|
|
except (ValueError, AttributeError):
|
|
pub_date = datetime.now(timezone.utc).strftime(
|
|
"%a, %d %b %Y %H:%M:%S +0000"
|
|
)
|
|
|
|
source_name = article.source_name or "unknown"
|
|
encoded_source = quote(source_name.lower())
|
|
item = {
|
|
"title": article.title,
|
|
"link": f"{server_url}/source/{encoded_source}/article/{article.id}",
|
|
"pubDate": pub_date,
|
|
"description": article.content_text[:500]
|
|
if article.content_text
|
|
else "",
|
|
"guid": article.url or f"article-{article.id}",
|
|
}
|
|
if article.author:
|
|
item["author"] = article.author
|
|
rss_items.append(item)
|
|
|
|
rss_template = render_template(
|
|
"rss.xml",
|
|
title="NewsArchiver - Latest Articles",
|
|
link=server_url,
|
|
description="Latest archived news articles",
|
|
last_build_date=datetime.now(timezone.utc).strftime(
|
|
"%a, %d %b %Y %H:%M:%S +0000"
|
|
),
|
|
items=rss_items,
|
|
)
|
|
|
|
response = make_response(rss_template)
|
|
response.headers["Content-Type"] = "application/rss+xml; charset=utf-8"
|
|
return response
|
|
|
|
|
|
@app.route("/atom")
|
|
def atom_feed():
|
|
"""Atom 1.0 endpoint for latest archived articles."""
|
|
limit = request.args.get("limit", 50, type=int)
|
|
|
|
articles = get_latest_articles(limit=limit)
|
|
|
|
server_url = f"http://192.168.8.150:5000"
|
|
|
|
atom_entries = []
|
|
for article in articles:
|
|
if (
|
|
article.title
|
|
and article.content_text
|
|
and "Performing security verification" not in article.content_text
|
|
):
|
|
pub_date = None
|
|
if article.publish_date:
|
|
try:
|
|
dt = datetime.fromisoformat(
|
|
article.publish_date.replace("Z", "+00:00")
|
|
)
|
|
pub_date = dt.strftime("%Y-%m-%dT%H:%M:%S+00:00")
|
|
except (ValueError, AttributeError):
|
|
pub_date = datetime.now(timezone.utc).strftime(
|
|
"%Y-%m-%dT%H:%M:%S+00:00"
|
|
)
|
|
|
|
source_name = article.source_name or "unknown"
|
|
encoded_source = quote(source_name.lower())
|
|
entry = {
|
|
"title": article.title,
|
|
"link": f"{server_url}/source/{encoded_source}/article/{article.id}",
|
|
"published": pub_date,
|
|
"summary": article.content_text[:500] if article.content_text else "",
|
|
"id": article.url or f"article-{article.id}",
|
|
}
|
|
if article.author:
|
|
entry["author"] = {"name": article.author}
|
|
atom_entries.append(entry)
|
|
|
|
atom_template = render_template(
|
|
"atom.xml",
|
|
title="NewsArchiver - Latest Articles",
|
|
link=server_url,
|
|
updated=datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S+00:00"),
|
|
entries=atom_entries,
|
|
)
|
|
|
|
response = make_response(atom_template)
|
|
response.headers["Content-Type"] = "application/atom+xml; charset=utf-8"
|
|
return response
|
|
|
|
|
|
if __name__ == "__main__":
|
|
logger.info("Starting web interface...")
|
|
|
|
if not DB_PATH.exists():
|
|
logger.info("Database not found, initializing...")
|
|
from storage_manager import initialize_storage
|
|
|
|
initialize_storage()
|
|
|
|
app.run(host="0.0.0.0", port=5000, debug=True)
|