320 lines
9.5 KiB
Python
320 lines
9.5 KiB
Python
#!/usr/bin/env python3
|
|
"""NewsArchiver - Main CLI Entry Point (Phase 4)
|
|
|
|
Single-file CLI for running NewsArchiver with multiple modes:
|
|
- --run: Archive news articles once
|
|
- --serve: Start Flask web server
|
|
- --interval: Run background scheduler with specified interval
|
|
"""
|
|
|
|
import argparse
|
|
import atexit
|
|
import logging
|
|
import os
|
|
import signal
|
|
import sys
|
|
import threading
|
|
import time
|
|
from pathlib import Path
|
|
|
|
try:
|
|
from flask import Flask
|
|
except ImportError:
|
|
print("ERROR: Flask is required. Install with: pip install flask")
|
|
sys.exit(1)
|
|
|
|
try:
|
|
from scheduler import start_scheduler, stop_scheduler, scheduled_archive
|
|
except ImportError:
|
|
print("ERROR: scheduler module not found")
|
|
sys.exit(1)
|
|
|
|
try:
|
|
from rss_processor import process_all_feeds, init_db as init_db_rss
|
|
except ImportError:
|
|
print("ERROR: rss_processor module not found")
|
|
sys.exit(1)
|
|
|
|
try:
|
|
from content_extractor import get_html_from_url
|
|
except ImportError:
|
|
print("ERROR: content_extractor module not found")
|
|
sys.exit(1)
|
|
|
|
try:
|
|
from storage_manager import initialize_storage, get_all_sources
|
|
except ImportError:
|
|
print("ERROR: storage_manager module not found")
|
|
sys.exit(1)
|
|
|
|
try:
|
|
from archive_engine import archive_all_sources
|
|
except ImportError:
|
|
print("ERROR: archive_engine module not found")
|
|
sys.exit(1)
|
|
|
|
try:
|
|
from web_interface import app
|
|
except ImportError:
|
|
print("ERROR: web_interface module not found")
|
|
sys.exit(1)
|
|
|
|
try:
|
|
from singlefile_archive import check_singlefile_available
|
|
except ImportError:
|
|
print("WARNING: singlefile_archive module not found")
|
|
print("SingleFile integration will not be available")
|
|
|
|
SCRIPT_DIR = Path(__file__).parent.resolve()
|
|
ARCHIVE_DIR = Path(os.environ.get("ARCHIVE_DIR", str(SCRIPT_DIR / "archival_data"))).resolve()
|
|
ARCHIVE_DIR.mkdir(exist_ok=True)
|
|
|
|
|
|
def setup_logging(verbose: bool = False) -> logging.Logger:
|
|
"""Configure logging for the application.
|
|
|
|
Args:
|
|
verbose: If True, enable DEBUG level logging
|
|
|
|
Returns:
|
|
Configured logger instance
|
|
"""
|
|
level = logging.DEBUG if verbose else logging.INFO
|
|
|
|
logging.basicConfig(
|
|
level=level,
|
|
format='%(asctime)s - %(levelname)s - %(message)s',
|
|
handlers=[
|
|
logging.StreamHandler(sys.stdout),
|
|
logging.FileHandler(ARCHIVE_DIR / 'processing.log', encoding='utf-8')
|
|
]
|
|
)
|
|
|
|
logger = logging.getLogger(__name__)
|
|
logger.info("NewsArchiver - Main CLI Entry Point")
|
|
logger.info("=" * 60)
|
|
|
|
return logger
|
|
|
|
|
|
def run_archive_once(logger: logging.Logger, verbose: bool = False) -> bool:
|
|
"""Run archiving process once.
|
|
|
|
Args:
|
|
logger: Logger instance
|
|
verbose: If True, enable verbose logging
|
|
|
|
Returns:
|
|
True if successful, False otherwise
|
|
"""
|
|
try:
|
|
logger.info("Running one-time archive")
|
|
logger.info("=" * 60)
|
|
|
|
init_db_rss()
|
|
initialize_storage()
|
|
|
|
results = archive_all_sources(
|
|
rss_feeds_path=SCRIPT_DIR / 'rss_feeds.json',
|
|
output_dir=ARCHIVE_DIR,
|
|
dry_run=False
|
|
)
|
|
|
|
logger.info("=" * 60)
|
|
logger.info("Archive complete")
|
|
logger.info("=" * 60)
|
|
logger.info("Sources processed: %d", results.get('sources_processed', 0))
|
|
logger.info("Total articles archived: %d", results.get('total_articles_archived', 0))
|
|
logger.info("Total articles skipped: %d", results.get('total_articles_skipped', 0))
|
|
logger.info("Total articles failed: %d", results.get('total_articles_failed', 0))
|
|
|
|
return True
|
|
except Exception as e:
|
|
logger.error("Archive failed: %s", str(e))
|
|
return False
|
|
|
|
|
|
def run_scheduler(interval_minutes: int, logger: logging.Logger, verbose: bool = False) -> None:
|
|
"""Run background scheduler.
|
|
|
|
Args:
|
|
interval_minutes: Interval between archive runs in minutes
|
|
logger: Logger instance
|
|
verbose: If True, enable verbose logging
|
|
"""
|
|
logger.info("Starting background scheduler")
|
|
logger.info("=" * 60)
|
|
|
|
try:
|
|
init_db_rss()
|
|
initialize_storage()
|
|
|
|
scheduler = start_scheduler(interval_minutes)
|
|
|
|
atexit.register(stop_scheduler)
|
|
|
|
logger.info("Press Ctrl+C to stop")
|
|
|
|
try:
|
|
while True:
|
|
time.sleep(1)
|
|
except (KeyboardInterrupt, SystemExit):
|
|
logger.info("Shutting down scheduler...")
|
|
stop_scheduler()
|
|
logger.info("Scheduler stopped")
|
|
|
|
except Exception as e:
|
|
logger.error("Scheduler failed to start: %s", str(e))
|
|
sys.exit(1)
|
|
|
|
|
|
def run_web_server(host: str, port: int, logger: logging.Logger, verbose: bool = False, interval_minutes: int = None) -> None:
|
|
"""Run Flask web server with optional background scheduler.
|
|
|
|
Args:
|
|
host: Host to bind to
|
|
port: Port to bind to
|
|
logger: Logger instance
|
|
verbose: If True, enable verbose logging
|
|
interval_minutes: If set, start background scheduler at this interval
|
|
"""
|
|
logger.info("Starting web server")
|
|
logger.info("=" * 60)
|
|
|
|
_scheduler_stopped = [False]
|
|
|
|
def _start_scheduler():
|
|
try:
|
|
init_db_rss()
|
|
initialize_storage()
|
|
start_scheduler(interval_minutes)
|
|
logger.info("Background scheduler started (every %d min)", interval_minutes)
|
|
logger.info("Running initial archive...")
|
|
scheduled_archive(logger)
|
|
logger.info("Initial archive complete")
|
|
except Exception as e:
|
|
logger.error("Scheduler thread failed: %s", str(e))
|
|
finally:
|
|
_scheduler_stopped[0] = True
|
|
|
|
def _shutdown_scheduler(signum=None, frame=None):
|
|
if not _scheduler_stopped[0]:
|
|
logger.info("Stopping scheduler...")
|
|
stop_scheduler()
|
|
logger.info("Scheduler stopped")
|
|
sys.exit(0)
|
|
|
|
try:
|
|
if not (ARCHIVE_DIR / 'cache.db').exists():
|
|
logger.info("Database not found, initializing...")
|
|
initialize_storage()
|
|
|
|
if not check_singlefile_available():
|
|
logger.warning("SingleFile CLI not available. Some features may not work.")
|
|
|
|
if interval_minutes:
|
|
logger.info("Starting background archive scheduler (interval: %d min)...", interval_minutes)
|
|
t = threading.Thread(target=_start_scheduler, daemon=True)
|
|
t.start()
|
|
signal.signal(signal.SIGINT, _shutdown_scheduler)
|
|
signal.signal(signal.SIGTERM, _shutdown_scheduler)
|
|
|
|
logger.info("Web server starting on %s:%d", host, port)
|
|
logger.info("=" * 60)
|
|
|
|
app.run(
|
|
host=host,
|
|
port=port,
|
|
debug=False
|
|
)
|
|
|
|
except Exception as e:
|
|
logger.error("Web server failed to start: %s", str(e))
|
|
sys.exit(1)
|
|
|
|
|
|
def main() -> None:
|
|
"""Main entry point for NewsArchiver CLI."""
|
|
parser = argparse.ArgumentParser(
|
|
description='NewsArchiver - News Article Archiving System',
|
|
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
epilog='''
|
|
Examples:
|
|
%(prog)s --run Run archiving once
|
|
%(prog)s --serve Start web server
|
|
%(prog)s --serve --host 0.0.0.0 --port 8080
|
|
Start web server on custom host/port
|
|
%(prog)s --interval 60 Run background scheduler (1 hour interval)
|
|
'''
|
|
)
|
|
|
|
parser.add_argument(
|
|
'--run',
|
|
action='store_true',
|
|
help='Run archiving once (process all RSS feeds)'
|
|
)
|
|
|
|
parser.add_argument(
|
|
'--serve',
|
|
action='store_true',
|
|
help='Start Flask web server'
|
|
)
|
|
|
|
parser.add_argument(
|
|
'--interval',
|
|
type=int,
|
|
default=None,
|
|
help='Run background scheduler with specified interval (minutes). When used with --serve, runs in background thread (default: 60 min). Standalone blocks.'
|
|
)
|
|
|
|
parser.add_argument(
|
|
'--host',
|
|
type=str,
|
|
default='0.0.0.0',
|
|
help='Host for web server (default: 0.0.0.0)'
|
|
)
|
|
|
|
parser.add_argument(
|
|
'--port',
|
|
type=int,
|
|
default=5000,
|
|
help='Port for web server (default: 5000)'
|
|
)
|
|
|
|
parser.add_argument(
|
|
'--verbose', '-v',
|
|
action='store_true',
|
|
help='Enable verbose logging (DEBUG level)'
|
|
)
|
|
|
|
args = parser.parse_args()
|
|
|
|
# CI port shift: detect CI env, map to reserved CI port range 10000-10099
|
|
# CI_PORT_OFFSET (1-99) maps to 10001-10099. Each service gets a fixed offset.
|
|
# NewsArchiverV2=1, paste-bin=2, etc. See AGENTS.md for assignments.
|
|
if os.environ.get("CI") == "true" and os.environ.get("SKIP_PORT_SHIFT") != "1":
|
|
original_port = args.port
|
|
offset = int(os.environ.get("CI_PORT_OFFSET", "1"))
|
|
args.port = 10000 + offset
|
|
print(f"[ci-port-shift] Port shifted from {original_port} to {args.port} (offset {offset}, CI range: 10000-10099)")
|
|
|
|
logger = setup_logging(args.verbose)
|
|
|
|
if args.run:
|
|
success = run_archive_once(logger, args.verbose)
|
|
sys.exit(0 if success else 1)
|
|
|
|
elif args.serve:
|
|
interval = args.interval if args.interval else 60
|
|
run_web_server(args.host, args.port, logger, args.verbose, interval_minutes=interval)
|
|
|
|
elif args.interval:
|
|
run_scheduler(args.interval, logger, args.verbose)
|
|
|
|
else:
|
|
parser.print_help()
|
|
sys.exit(1)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main() |