488 lines
15 KiB
Python
488 lines
15 KiB
Python
#!/usr/bin/env python3
|
|
"""Web Interface for NewsArchiver - Phase 3
|
|
|
|
Flask web server for browsing archived news articles.
|
|
"""
|
|
|
|
import json
|
|
import logging
|
|
import sys
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
from typing import Optional
|
|
from urllib.parse import quote
|
|
|
|
from flask import Flask, jsonify, request, render_template, make_response
|
|
import xml.etree.ElementTree as ET
|
|
from datetime import datetime, timezone
|
|
|
|
from storage_manager import (
|
|
get_all_sources,
|
|
get_source_stats,
|
|
get_articles_by_source,
|
|
get_article,
|
|
get_latest_articles,
|
|
DB_PATH
|
|
)
|
|
|
|
SCRIPT_DIR = Path(__file__).parent
|
|
ARCHIVE_DIR = SCRIPT_DIR / 'archival_data'
|
|
RSS_FEEDS_PATH = SCRIPT_DIR / 'rss_feeds.json'
|
|
|
|
RSS_FEEDS = {}
|
|
|
|
|
|
def load_rss_feeds() -> dict:
|
|
"""Load RSS feeds configuration."""
|
|
global RSS_FEEDS
|
|
|
|
if RSS_FEEDS:
|
|
return RSS_FEEDS
|
|
|
|
if not RSS_FEEDS_PATH.exists():
|
|
logger.warning("RSS feeds file not found: %s", RSS_FEEDS_PATH)
|
|
return {}
|
|
|
|
try:
|
|
with open(RSS_FEEDS_PATH, 'r', encoding='utf-8') as f:
|
|
RSS_FEEDS = json.load(f)
|
|
return RSS_FEEDS
|
|
except Exception as e:
|
|
logger.error("Failed to load RSS feeds: %s", str(e))
|
|
return {}
|
|
|
|
app = Flask(
|
|
__name__,
|
|
static_folder=str(SCRIPT_DIR / 'static'),
|
|
template_folder=str(SCRIPT_DIR / 'templates')
|
|
)
|
|
|
|
logging.basicConfig(
|
|
level=logging.INFO,
|
|
format='%(asctime)s - %(levelname)s - %(message)s',
|
|
handlers=[
|
|
logging.StreamHandler(sys.stdout),
|
|
logging.FileHandler(ARCHIVE_DIR / 'processing.log', encoding='utf-8')
|
|
]
|
|
)
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def get_pagination_info(total: int, page: int, per_page: int) -> dict:
|
|
"""Calculate pagination information.
|
|
|
|
Args:
|
|
total: Total number of items
|
|
page: Current page number
|
|
per_page: Items per page
|
|
|
|
Returns:
|
|
Dictionary with pagination details
|
|
"""
|
|
total_pages = (total + per_page - 1) // per_page if total > 0 else 1
|
|
|
|
return {
|
|
'total': total,
|
|
'page': page,
|
|
'per_page': per_page,
|
|
'has_next': page < total_pages,
|
|
'has_prev': page > 1,
|
|
'next_num': page + 1 if page < total_pages else None,
|
|
'prev_num': page - 1 if page > 1 else None,
|
|
'pages': total_pages
|
|
}
|
|
|
|
|
|
@app.route('/')
|
|
def index():
|
|
"""Newspaper listing page."""
|
|
sources = get_all_sources()
|
|
rss_feeds = load_rss_feeds()
|
|
|
|
source_list = []
|
|
disabled_sources = []
|
|
|
|
for source_name in sources:
|
|
stats = get_source_stats(source_name)
|
|
|
|
source_info = {
|
|
'name': source_name.title(),
|
|
'slug': source_name,
|
|
'article_count': stats['total_articles'],
|
|
'last_archived': stats.get('last_archived'),
|
|
'status': 'success' if stats['total_articles'] > 0 else 'pending'
|
|
}
|
|
|
|
if source_name in rss_feeds:
|
|
feed_info = rss_feeds[source_name]
|
|
if feed_info.get('disabled', False):
|
|
source_info['disabled'] = True
|
|
source_info['disable_reason'] = feed_info.get('disable_reason', 'No reason provided')
|
|
disabled_sources.append(source_info)
|
|
continue
|
|
|
|
source_list.append(source_info)
|
|
|
|
source_list.extend(disabled_sources)
|
|
|
|
return render_template('index.html', sources=source_list)
|
|
|
|
|
|
@app.route('/source/<slug>')
|
|
def articles(slug: str):
|
|
"""Article listing page for a specific source."""
|
|
page = request.args.get('page', 1, type=int)
|
|
per_page = 50
|
|
|
|
sources = get_all_sources()
|
|
source_name = None
|
|
for s in sources:
|
|
if s.lower() == slug.lower():
|
|
source_name = s
|
|
break
|
|
|
|
if not source_name:
|
|
return render_template('article_not_found.html', slug=slug, article_id=0), 404
|
|
|
|
articles_list = get_articles_by_source(source_name, limit=per_page, offset=(page - 1) * per_page)
|
|
stats = get_source_stats(source_name)
|
|
total = stats['total_articles']
|
|
|
|
pagination = get_pagination_info(total, page, per_page)
|
|
|
|
articles_data = []
|
|
for article in articles_list:
|
|
articles_data.append({
|
|
'id': getattr(article, 'id', 0),
|
|
'title': article.title or 'Untitled',
|
|
'date': article.publish_date or '',
|
|
'summary': article.content_text[:200] if article.content_text else '',
|
|
'url': f'/source/{source_name.lower()}/article/{getattr(article, "id", 0)}'
|
|
})
|
|
|
|
return render_template(
|
|
'articles.html',
|
|
source_name=source_name.title(),
|
|
source_slug=source_name.lower(),
|
|
articles=articles_data,
|
|
pagination=pagination
|
|
)
|
|
|
|
|
|
@app.route('/archive/<path:archive_path>')
|
|
def serve_archive(archive_path):
|
|
"""Serve archived HTML file."""
|
|
archive_file = ARCHIVE_DIR / archive_path
|
|
if archive_file.exists():
|
|
return archive_file.read_text(encoding='utf-8')
|
|
return 'Archive not found', 404
|
|
|
|
|
|
@app.route('/archive-file/<path:encoded_path>')
|
|
def serve_archive_file(encoded_path):
|
|
"""Serve archived HTML file from encoded path."""
|
|
import urllib.parse
|
|
from pathlib import Path
|
|
archive_path = urllib.parse.unquote(encoded_path)
|
|
archive_file = ARCHIVE_DIR / archive_path
|
|
logger.info("Archive file path: %s, exists: %s", str(archive_file), archive_file.exists())
|
|
if archive_file.exists():
|
|
return archive_file.read_text(encoding='utf-8')
|
|
return 'Archive not found', 404
|
|
|
|
|
|
@app.route('/source/<slug>/article/<int:article_id>')
|
|
def article(slug: str, article_id: int):
|
|
"""Individual article page."""
|
|
sources = get_all_sources()
|
|
source_name = None
|
|
for s in sources:
|
|
if s.lower() == slug.lower():
|
|
source_name = s
|
|
break
|
|
|
|
if not source_name:
|
|
return render_template('article_not_found.html', slug=slug, article_id=article_id), 404
|
|
|
|
article = get_article(source_name, article_id)
|
|
if not article:
|
|
return render_template('article_not_found.html', slug=slug, article_id=article_id), 404
|
|
|
|
article_data = {
|
|
'id': article_id,
|
|
'title': article.title or 'Untitled',
|
|
'publish_date': article.publish_date or '',
|
|
'author': article.author or '',
|
|
'url': article.url or '',
|
|
'content_text': article.content_text or '',
|
|
'archive_file_path': article.archive_file_path or ''
|
|
}
|
|
|
|
return render_template(
|
|
'article.html',
|
|
source_name=source_name.title(),
|
|
source_slug=source_name.lower(),
|
|
article=article_data
|
|
)
|
|
|
|
|
|
@app.route('/status')
|
|
def status():
|
|
"""System status page."""
|
|
sources = get_all_sources()
|
|
rss_feeds = load_rss_feeds()
|
|
|
|
sources_info = []
|
|
disabled_sources = []
|
|
total_articles = 0
|
|
failed_jobs = 0
|
|
disabled_count = 0
|
|
|
|
for source_name in sources:
|
|
stats = get_source_stats(source_name)
|
|
|
|
source_info = {
|
|
'name': source_name.title(),
|
|
'slug': source_name,
|
|
'article_count': stats['total_articles'],
|
|
'last_archived': stats.get('last_archived'),
|
|
'status': 'success' if stats['total_articles'] > 0 else 'pending'
|
|
}
|
|
|
|
if source_name in rss_feeds:
|
|
feed_info = rss_feeds[source_name]
|
|
if feed_info.get('disabled', False):
|
|
source_info['disabled'] = True
|
|
disabled_count += 1
|
|
disabled_sources.append(source_info)
|
|
continue
|
|
|
|
sources_info.append(source_info)
|
|
total_articles += stats['total_articles']
|
|
failed_jobs += stats['failed']
|
|
|
|
sources_info.extend(disabled_sources)
|
|
|
|
return render_template(
|
|
'status.html',
|
|
sources=sources_info,
|
|
total_articles=total_articles,
|
|
failed_jobs=failed_jobs,
|
|
sources_monitored=len(sources) - disabled_count,
|
|
disabled_sources=disabled_count
|
|
)
|
|
|
|
|
|
@app.route('/api/sources')
|
|
def api_sources():
|
|
"""API endpoint for listing all sources."""
|
|
sources = get_all_sources()
|
|
rss_feeds = load_rss_feeds()
|
|
|
|
source_list = []
|
|
disabled_sources = []
|
|
|
|
for source_name in sources:
|
|
stats = get_source_stats(source_name)
|
|
|
|
source_info = {
|
|
'name': source_name.title(),
|
|
'slug': source_name,
|
|
'article_count': stats['total_articles'],
|
|
'last_archived': stats.get('last_archived'),
|
|
'status': 'success' if stats['total_articles'] > 0 else 'pending'
|
|
}
|
|
|
|
if source_name in rss_feeds:
|
|
feed_info = rss_feeds[source_name]
|
|
if feed_info.get('disabled', False):
|
|
source_info['disabled'] = True
|
|
source_info['disable_reason'] = feed_info.get('disable_reason', 'No reason provided')
|
|
disabled_sources.append(source_info)
|
|
continue
|
|
|
|
source_list.append(source_info)
|
|
|
|
source_list.extend(disabled_sources)
|
|
|
|
return jsonify({'sources': source_list})
|
|
|
|
|
|
@app.route('/api/source/<slug>/articles')
|
|
def api_articles(slug: str):
|
|
"""API endpoint for listing articles for a source."""
|
|
page = request.args.get('page', 1, type=int)
|
|
per_page = 50
|
|
|
|
sources = get_all_sources()
|
|
source_name = None
|
|
for s in sources:
|
|
if s.lower() == slug.lower():
|
|
source_name = s
|
|
break
|
|
|
|
if not source_name:
|
|
return jsonify({'error': 'Source not found'}), 404
|
|
|
|
articles_list = get_articles_by_source(source_name, limit=per_page, offset=(page - 1) * per_page)
|
|
stats = get_source_stats(source_name)
|
|
total = stats['total_articles']
|
|
|
|
pagination = get_pagination_info(total, page, per_page)
|
|
|
|
articles_data = []
|
|
for article in articles_list:
|
|
articles_data.append({
|
|
'id': getattr(article, 'id', 0),
|
|
'title': article.title or 'Untitled',
|
|
'date': article.publish_date or '',
|
|
'summary': article.content_text[:200] if article.content_text else '',
|
|
'url': f'/source/{source_name.lower()}/article/{getattr(article, "id", 0)}'
|
|
})
|
|
|
|
return jsonify({
|
|
'source_name': source_name.title(),
|
|
'articles': articles_data,
|
|
'total': total,
|
|
'page': page,
|
|
'per_page': per_page,
|
|
'has_next': pagination['has_next'],
|
|
'has_prev': pagination['has_prev']
|
|
})
|
|
|
|
|
|
@app.route('/api/status')
|
|
def api_status():
|
|
"""API endpoint for system status."""
|
|
sources = get_all_sources()
|
|
|
|
sources_monitored = len(sources)
|
|
total_articles = 0
|
|
failed_jobs = 0
|
|
last_archive_run = None
|
|
|
|
for source_name in sources:
|
|
stats = get_source_stats(source_name)
|
|
total_articles += stats['total_articles']
|
|
failed_jobs += stats['failed']
|
|
|
|
if stats.get('last_archive_run'):
|
|
if last_archive_run is None or stats['last_archive_run'] > last_archive_run:
|
|
last_archive_run = stats['last_archive_run']
|
|
|
|
return jsonify({
|
|
'status': 'online',
|
|
'last_archive_run': last_archive_run,
|
|
'pending_jobs': 0,
|
|
'failed_jobs': failed_jobs,
|
|
'sources_monitored': sources_monitored,
|
|
'total_articles': total_articles
|
|
})
|
|
|
|
|
|
@app.route('/rss')
|
|
def rss_feed():
|
|
"""RSS 2.0 endpoint for latest archived articles."""
|
|
limit = request.args.get('limit', 50, type=int)
|
|
|
|
articles = get_latest_articles(limit=limit)
|
|
|
|
server_url = f'http://192.168.8.150:5000'
|
|
|
|
rss_items = []
|
|
for article in articles:
|
|
if article.title and article.content_text and 'Performing security verification' not in article.content_text:
|
|
pub_date = None
|
|
if article.publish_date:
|
|
try:
|
|
dt = datetime.fromisoformat(article.publish_date.replace('Z', '+00:00'))
|
|
pub_date = dt.strftime('%a, %d %b %Y %H:%M:%S %z').strip()
|
|
except (ValueError, AttributeError):
|
|
try:
|
|
dt = datetime.fromisoformat(article.publish_date)
|
|
pub_date = dt.strftime('%a, %d %b %Y %H:%M:%S +0000')
|
|
except (ValueError, AttributeError):
|
|
pub_date = datetime.now(timezone.utc).strftime('%a, %d %b %Y %H:%M:%S +0000')
|
|
|
|
source_name = article.source_name or 'unknown'
|
|
encoded_source = quote(source_name.lower())
|
|
item = {
|
|
'title': article.title,
|
|
'link': f'{server_url}/source/{encoded_source}/article/{article.id}',
|
|
'pubDate': pub_date,
|
|
'description': article.content_text[:500] if article.content_text else '',
|
|
'guid': article.url or f'article-{article.id}'
|
|
}
|
|
if article.author:
|
|
item['author'] = article.author
|
|
rss_items.append(item)
|
|
|
|
rss_template = render_template(
|
|
'rss.xml',
|
|
title='NewsArchiver - Latest Articles',
|
|
link=server_url,
|
|
description='Latest archived news articles',
|
|
last_build_date=datetime.now(timezone.utc).strftime('%a, %d %b %Y %H:%M:%S +0000'),
|
|
items=rss_items
|
|
)
|
|
|
|
response = make_response(rss_template)
|
|
response.headers['Content-Type'] = 'application/rss+xml; charset=utf-8'
|
|
return response
|
|
|
|
|
|
@app.route('/atom')
|
|
def atom_feed():
|
|
"""Atom 1.0 endpoint for latest archived articles."""
|
|
limit = request.args.get('limit', 50, type=int)
|
|
|
|
articles = get_latest_articles(limit=limit)
|
|
|
|
server_url = f'http://192.168.8.150:5000'
|
|
|
|
atom_entries = []
|
|
for article in articles:
|
|
if article.title and article.content_text and 'Performing security verification' not in article.content_text:
|
|
pub_date = None
|
|
if article.publish_date:
|
|
try:
|
|
dt = datetime.fromisoformat(article.publish_date.replace('Z', '+00:00'))
|
|
pub_date = dt.strftime('%Y-%m-%dT%H:%M:%S+00:00')
|
|
except (ValueError, AttributeError):
|
|
pub_date = datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%S+00:00')
|
|
|
|
source_name = article.source_name or 'unknown'
|
|
encoded_source = quote(source_name.lower())
|
|
entry = {
|
|
'title': article.title,
|
|
'link': f'{server_url}/source/{encoded_source}/article/{article.id}',
|
|
'published': pub_date,
|
|
'summary': article.content_text[:500] if article.content_text else '',
|
|
'id': article.url or f'article-{article.id}'
|
|
}
|
|
if article.author:
|
|
entry['author'] = {'name': article.author}
|
|
atom_entries.append(entry)
|
|
|
|
atom_template = render_template(
|
|
'atom.xml',
|
|
title='NewsArchiver - Latest Articles',
|
|
link=server_url,
|
|
updated=datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%S+00:00'),
|
|
entries=atom_entries
|
|
)
|
|
|
|
response = make_response(atom_template)
|
|
response.headers['Content-Type'] = 'application/atom+xml; charset=utf-8'
|
|
return response
|
|
|
|
|
|
if __name__ == '__main__':
|
|
logger.info("Starting web interface...")
|
|
|
|
if not DB_PATH.exists():
|
|
logger.info("Database not found, initializing...")
|
|
from storage_manager import initialize_storage
|
|
initialize_storage()
|
|
|
|
app.run(host='0.0.0.0', port=5000, debug=True) |