NewsArchiverV2/run_archiver.py

#!/usr/bin/env python3
"""NewsArchiver - Main CLI Entry Point (Phase 4)

Single-file CLI for running NewsArchiver with multiple modes:
- --run: Archive news articles once
- --serve: Start Flask web server
- --interval: Run background scheduler with specified interval
"""

import argparse
import atexit
import logging
import os
import sys
import time
from pathlib import Path

try:
    from flask import Flask
except ImportError:
    print("ERROR: Flask is required. Install with: pip install flask")
    sys.exit(1)

try:
    from scheduler import start_scheduler, stop_scheduler, scheduled_archive
except ImportError:
    print("ERROR: scheduler module not found")
    sys.exit(1)

try:
    from rss_processor import process_all_feeds, init_db as init_db_rss
except ImportError:
    print("ERROR: rss_processor module not found")
    sys.exit(1)

try:
    from content_extractor import get_html_from_url
except ImportError:
    print("ERROR: content_extractor module not found")
    sys.exit(1)

try:
    from storage_manager import initialize_storage, get_all_sources
except ImportError:
    print("ERROR: storage_manager module not found")
    sys.exit(1)

try:
    from archive_engine import archive_all_sources
except ImportError:
    print("ERROR: archive_engine module not found")
    sys.exit(1)

try:
    from web_interface import app
except ImportError:
    print("ERROR: web_interface module not found")
    sys.exit(1)

try:
    from singlefile_archive import check_singlefile_available
except ImportError:
    print("WARNING: singlefile_archive module not found")
    print("SingleFile integration will not be available")

SCRIPT_DIR = Path(__file__).parent.resolve()
ARCHIVE_DIR = Path(os.environ.get("ARCHIVE_DIR", str(SCRIPT_DIR / "archival_data"))).resolve()
ARCHIVE_DIR.mkdir(exist_ok=True)


def setup_logging(verbose: bool = False) -> logging.Logger:
    """Configure logging for the application.
    
    Args:
        verbose: If True, enable DEBUG level logging
        
    Returns:
        Configured logger instance
    """
    level = logging.DEBUG if verbose else logging.INFO
    
    logging.basicConfig(
        level=level,
        format='%(asctime)s - %(levelname)s - %(message)s',
        handlers=[
            logging.StreamHandler(sys.stdout),
            logging.FileHandler(ARCHIVE_DIR / 'processing.log', encoding='utf-8')
        ]
    )
    
    logger = logging.getLogger(__name__)
    logger.info("NewsArchiver - Main CLI Entry Point")
    logger.info("=" * 60)
    
    return logger


def run_archive_once(logger: logging.Logger, verbose: bool = False) -> bool:
    """Run archiving process once.
    
    Args:
        logger: Logger instance
        verbose: If True, enable verbose logging
        
    Returns:
        True if successful, False otherwise
    """
    try:
        logger.info("Running one-time archive")
        logger.info("=" * 60)
        
        init_db_rss()
        initialize_storage()
        
        results = archive_all_sources(
            rss_feeds_path=SCRIPT_DIR / 'rss_feeds.json',
            output_dir=ARCHIVE_DIR,
            dry_run=False
        )
        
        logger.info("=" * 60)
        logger.info("Archive complete")
        logger.info("=" * 60)
        logger.info("Sources processed: %d", results.get('sources_processed', 0))
        logger.info("Total articles archived: %d", results.get('total_articles_archived', 0))
        logger.info("Total articles skipped: %d", results.get('total_articles_skipped', 0))
        logger.info("Total articles failed: %d", results.get('total_articles_failed', 0))
        
        return True
    except Exception as e:
        logger.error("Archive failed: %s", str(e))
        return False


def run_scheduler(interval_minutes: int, logger: logging.Logger, verbose: bool = False) -> None:
    """Run background scheduler.
    
    Args:
        interval_minutes: Interval between archive runs in minutes
        logger: Logger instance
        verbose: If True, enable verbose logging
    """
    logger.info("Starting background scheduler")
    logger.info("=" * 60)
    
    try:
        init_db_rss()
        initialize_storage()
        
        scheduler = start_scheduler(interval_minutes)
        
        atexit.register(stop_scheduler)
        
        logger.info("Press Ctrl+C to stop")
        
        try:
            while True:
                time.sleep(1)
        except (KeyboardInterrupt, SystemExit):
            logger.info("Shutting down scheduler...")
            stop_scheduler()
            logger.info("Scheduler stopped")
            
    except Exception as e:
        logger.error("Scheduler failed to start: %s", str(e))
        sys.exit(1)


def run_web_server(host: str, port: int, logger: logging.Logger, verbose: bool = False) -> None:
    """Run Flask web server.
    
    Args:
        host: Host to bind to
        port: Port to bind to
        logger: Logger instance
        verbose: If True, enable verbose logging
    """
    logger.info("Starting web server")
    logger.info("=" * 60)
    
    try:
        if not (ARCHIVE_DIR / 'cache.db').exists():
            logger.info("Database not found, initializing...")
            initialize_storage()
        
        if not check_singlefile_available():
            logger.warning("SingleFile CLI not available. Some features may not work.")
        
        logger.info("Web server starting on %s:%d", host, port)
        logger.info("=" * 60)
        
        app.run(
            host=host,
            port=port,
            debug=False
        )
        
    except Exception as e:
        logger.error("Web server failed to start: %s", str(e))
        sys.exit(1)


def main() -> None:
    """Main entry point for NewsArchiver CLI."""
    parser = argparse.ArgumentParser(
        description='NewsArchiver - News Article Archiving System',
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog='''
Examples:
  %(prog)s --run                    Run archiving once
  %(prog)s --serve                  Start web server
  %(prog)s --serve --host 0.0.0.0 --port 8080
                                     Start web server on custom host/port
  %(prog)s --interval 60            Run background scheduler (1 hour interval)
        '''
    )
    
    parser.add_argument(
        '--run',
        action='store_true',
        help='Run archiving once (process all RSS feeds)'
    )
    
    parser.add_argument(
        '--serve',
        action='store_true',
        help='Start Flask web server'
    )
    
    parser.add_argument(
        '--interval',
        type=int,
        default=60,
        help='Run background scheduler with specified interval (minutes, default: 60)'
    )
    
    parser.add_argument(
        '--host',
        type=str,
        default='0.0.0.0',
        help='Host for web server (default: 0.0.0.0)'
    )
    
    parser.add_argument(
        '--port',
        type=int,
        default=5000,
        help='Port for web server (default: 5000)'
    )
    
    parser.add_argument(
        '--verbose', '-v',
        action='store_true',
        help='Enable verbose logging (DEBUG level)'
    )
    
    args = parser.parse_args()

    # CI port shift: detect CI env, shift port to avoid conflicts with production
    if os.environ.get("CI") == "true" and os.environ.get("SKIP_PORT_SHIFT") != "1":
        offset = int(os.environ.get("CI_PORT_OFFSET", "1"))
        original_port = args.port
        args.port = original_port + offset
        print(f"[ci-port-shift] Port shifted from {original_port} to {args.port} (offset: {offset})")

    logger = setup_logging(args.verbose)
    
    if args.run:
        success = run_archive_once(logger, args.verbose)
        sys.exit(0 if success else 1)
        
    elif args.serve:
        run_web_server(args.host, args.port, logger, args.verbose)
        
    elif args.interval:
        run_scheduler(args.interval, logger, args.verbose)
        
    else:
        parser.print_help()
        sys.exit(1)


if __name__ == '__main__':
    main()