NewsArchiverV2/cleanup_old_files.py

#!/usr/bin/env python3
"""
Cleanup Script for NewsArchiver

This script removes files older than a specified date from the project directory.
It provides dry-run mode to preview what would be deleted before actually deleting.

Usage:
    python cleanup_old_files.py --date "2024-03-19" --dry-run
    python cleanup_old_files.py --date "2024-03-19"

Options:
    --date, -d      Date in YYYY-MM-DD format (required)
    --dry-run, -n   Show what would be deleted without actually deleting (default: True)
    --force, -f     Actually delete files (disables dry-run mode)
    --verbose, -v   Enable verbose output
    --archival      Include archival_data folder for cleanup
"""

import argparse
import logging
import os
import sys
from datetime import datetime, timezone
from pathlib import Path
from typing import List, Tuple

logger = logging.getLogger(__name__)

# Project root directory
SCRIPT_DIR = Path(__file__).parent.resolve()

# Path to archival_data directory
ARCHIVE_DIR = Path(
    os.environ.get("ARCHIVE_DIR", str(SCRIPT_DIR / "archival_data"))
).resolve()
ARCHIVAL_DATA_DIR = ARCHIVE_DIR

# Path to websites folder (only this folder will be scanned in archival_data)
WEBSITES_DIR = ARCHIVAL_DATA_DIR / "websites"

# Files and directories to always preserve (never delete)
PRESERVE_LIST = {
    # Python files
    "ap_processor.py",
    "archive_engine.py",
    "content_extractor.py",
    "rebuild_database.py",
    "restore_database.py",
    "rss_feeds.json",
    "rss_processor.py",
    "run_archiver.py",
    "scheduler.py",
    "setup_cron.sh",
    "singlefile_archive.py",
    "stop_services.sh",
    "web_interface.py",
    "cleanup_old_files.py",
    # Directories
    "archival_data",
    "static",
    "templates",
    "__pycache__",
}

# Database files to preserve
DATABASE_FILES = {"cache.db", "cache.db-shm", "cache.db-wal"}

# Files that should be excluded from cleanup regardless of date
EXCLUDE_PATTERNS = [
    ".git",
    ".gitignore",
]


def parse_date(date_str: str) -> datetime:
    """Parse date string in YYYY-MM-DD format.

    Args:
        date_str: Date string in YYYY-MM-DD format

    Returns:
        datetime object with the specified date at midnight

    Raises:
        ValueError: If date format is invalid
    """
    try:
        return datetime.strptime(date_str, "%Y-%m-%d").replace(tzinfo=timezone.utc)
    except ValueError as e:
        raise ValueError(
            f"Invalid date format: '{date_str}'. Use YYYY-MM-DD format."
        ) from e


def should_preserve(path: Path) -> bool:
    """Check if a file/directory should be preserved.

    Args:
        path: Path to check

    Returns:
        True if the path should be preserved, False otherwise
    """
    # Check if it's in the preserve list
    if path.name in PRESERVE_LIST:
        return True

    # Check if it matches any exclude patterns
    for pattern in EXCLUDE_PATTERNS:
        if pattern in str(path):
            return True

    return False


def should_preserve_archival_file(path: Path) -> bool:
    """Check if a file in archival_data should be preserved.

    Args:
        path: Path to check

    Returns:
        True if the path should be preserved, False otherwise
    """
    # Always preserve database files
    if path.name in DATABASE_FILES:
        return True

    return should_preserve(path)


def get_files_older_than_date(
    directory: Path, cutoff_date: datetime
) -> List[Tuple[Path, datetime]]:
    """Get all files older than the cutoff date.

    Args:
        directory: Directory to search
        cutoff_date: Files older than this date will be selected

    Returns:
        List of tuples (path, modification_time) for files older than cutoff
    """
    old_files = []

    # Walk through all files in directory recursively (targeted patterns only)
    for pattern in ("*.html", "*.json", "*.txt", "*.xml", "*.md"):
        for item in directory.rglob(pattern):
            if not item.is_file():
                continue
            if should_preserve(item):
                continue

            try:
                mtime = datetime.fromtimestamp(item.stat().st_mtime, tz=timezone.utc)
                if mtime < cutoff_date:
                    old_files.append((item, mtime))
            except (OSError, ValueError) as e:
                logger.warning(f"Could not access file {item}: {e}")

    return old_files


def get_files_older_than_date_non_recursive(
    directory: Path, cutoff_date: datetime
) -> List[Tuple[Path, datetime]]:
    """Get all files older than the cutoff date (non-recursive).

    Args:
        directory: Directory to search
        cutoff_date: Files older than this date will be selected

    Returns:
        List of tuples (path, modification_time) for files older than cutoff
    """
    old_files = []

    # Walk through all files in directory (non-recursive for safety)
    for item in directory.iterdir():
        if item.is_file():
            if should_preserve(item):
                continue

            try:
                mtime = datetime.fromtimestamp(item.stat().st_mtime, tz=timezone.utc)
                if mtime < cutoff_date:
                    old_files.append((item, mtime))
            except (OSError, ValueError) as e:
                logger.warning(f"Could not access file {item}: {e}")

    return old_files


def delete_files(files: List[Tuple[Path, datetime]]) -> Tuple[int, int]:
    """Delete files and return count of successful/failed deletions.

    Args:
        files: List of (path, modification_time) tuples to delete

    Returns:
        Tuple of (deleted_count, failed_count)
    """
    deleted = 0
    failed = 0

    for path, mtime in files:
        try:
            path.unlink()
            logger.info(
                f"Deleted: {path.name} (modified: {mtime.strftime('%Y-%m-%d %H:%M:%S')})"
            )
            deleted += 1
        except OSError as e:
            logger.error(f"Failed to delete {path.name}: {e}")
            failed += 1

    return deleted, failed


def main():
    """Main entry point for cleanup script."""
    parser = argparse.ArgumentParser(
        description="Cleanup old files from NewsArchiver project",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog="""
Examples:
  %(prog)s --date "2024-03-19" --dry-run
  %(prog)s --date "2024-03-19" --force
  %(prog)s --date "2024-03-19" --archival --force
        """,
    )

    parser.add_argument(
        "--date",
        "-d",
        type=str,
        required=True,
        help="Date in YYYY-MM-DD format - files older than this will be deleted",
    )

    parser.add_argument(
        "--dry-run",
        "-n",
        action="store_true",
        default=True,
        help="Show what would be deleted without actually deleting (default)",
    )

    parser.add_argument(
        "--force",
        "-f",
        action="store_true",
        help="Actually delete files (disables dry-run mode)",
    )

    parser.add_argument(
        "--verbose", "-v", action="store_true", help="Enable verbose output"
    )

    parser.add_argument(
        "--archival",
        action="store_true",
        help="Include archival_data folder for cleanup",
    )

    args = parser.parse_args()

    # Set logging level
    if args.verbose:
        logger.setLevel(logging.DEBUG)

    # Parse the date
    try:
        cutoff_date = parse_date(args.date)
    except ValueError as e:
        logger.error(str(e))
        sys.exit(1)

    # Validate cutoff date is not in the future
    now = datetime.now(timezone.utc)
    if cutoff_date > now:
        logger.error("Cutoff date cannot be in the future")
        sys.exit(1)

    logger.info("=" * 60)
    logger.info("NewsArchiver Cleanup Script")
    logger.info("=" * 60)

    # Determine mode
    dry_run = not args.force
    mode = "DRY RUN" if dry_run else "ACTUAL DELETE"
    logger.info("Mode: %s", mode)
    logger.info(
        "Cutoff Date: %s (files older than this will be %s)",
        cutoff_date.strftime("%Y-%m-%d"),
        "kept" if dry_run else "deleted",
    )
    logger.info("=" * 60)

    # Find files older than cutoff date
    if args.archival:
        # Only scan websites folder when --archival is used
        if not WEBSITES_DIR.exists():
            logger.error("Websites folder not found at %s", WEBSITES_DIR)
            sys.exit(1)
        old_files = []
        logger.info(
            "Scanning websites folder for files older than %s...",
            cutoff_date.strftime("%Y-%m-%d"),
        )
        # First check if there are any files directly in websites folder
        website_root_files = get_files_older_than_date_non_recursive(
            WEBSITES_DIR, cutoff_date
        )
        # Then check recursively in subdirectories
        website_recursive_files = []
        for subdir in WEBSITES_DIR.iterdir():
            if subdir.is_dir():
                website_recursive_files.extend(
                    get_files_older_than_date(subdir, cutoff_date)
                )
        old_files.extend(website_root_files)
        old_files.extend(website_recursive_files)
        logger.info(
            "Found %d files in websites folder",
            len(website_root_files) + len(website_recursive_files),
        )
    else:
        # Scan root directory (non-archival mode)
        old_files = get_files_older_than_date_non_recursive(SCRIPT_DIR, cutoff_date)

    if not old_files:
        logger.info("No files older than %s found.", cutoff_date.strftime("%Y-%m-%d"))
        logger.info("Nothing to do.")
        return

    logger.info(
        "Found %d files older than %s:",
        len(old_files),
        cutoff_date.strftime("%Y-%m-%d"),
    )

    # List all files that would be affected
    for path, mtime in old_files:
        # Get relative path for cleaner output
        rel_path = path.relative_to(SCRIPT_DIR)
        logger.info(
            "  - %s (modified: %s)", rel_path, mtime.strftime("%Y-%m-%d %H:%M:%S")
        )

    logger.info("=" * 60)

    # Execute deletion if not dry run
    if dry_run:
        logger.info("DRY RUN: No files were deleted.")
        logger.info("Run with --force to actually delete these files.")
    else:
        deleted, failed = delete_files(old_files)
        logger.info("=" * 60)
        logger.info("Cleanup complete!")
        logger.info("Deleted: %d files", deleted)
        logger.info("Failed: %d files", failed)


if __name__ == "__main__":
    main()