jarianc/NewsArchiverV2
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Current state of NA

Browse Source
This commit is contained in:
Jarian Cottingham 2026-03-31 09:37:36 -05:00
parent 639f03fdb9
commit 0f1b2741db
33 changed files with 12856 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
._.DS_Store Normal file
View File

Binary file not shown.

BIN
._.git Normal file
View File

Binary file not shown.

BIN
._.gitignore Normal file
View File

Binary file not shown.

BIN
._README.md Normal file
View File

Binary file not shown.

BIN
._cleanup_old_files.py Normal file
View File

Binary file not shown.

55
.gitignore vendored Normal file
View File

@ -0,0 +1,55 @@
# Python
__pycache__/
*.pyc
*.py[cod]
*$py.class
*.so
.Python
env/
venv/
ENV/
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
*.egg-info/
.installed.cfg
*.egg
# Virtual Environments
.env
.venv
env/
venv/
ENV/
# IDE
.idea/
.vscode/
*.swp
*.swo
*~
# Project specific
archival_data/
*.log
*.db
*.sqlite
*.sqlite3
# SingleFile
singlefile-*.html
# OS
.DS_Store
Thumbs.db
*.pid

220
ap_processor.py Normal file
View File

@ -0,0 +1,220 @@
#!/usr/bin/env python3
"""AP News Processor for NewsArchiver - Phase 2.6
Processes AP News front page to extract article URLs and archive them.
Uses direct HTML parsing since AP doesn't provide RSS feeds.
"""
import argparse
import json
import logging
import re
import sys
import time
from datetime import datetime
from pathlib import Path
from typing import List
try:
import requests
except ImportError:
print("WARNING: requests not installed. URL fetching may not work.")
print("Install with: pip install requests")
requests = None
try:
import sqlite3
except ImportError:
print("ERROR: sqlite3 is required (should be built-in)")
sys.exit(1)
try:
import storage_manager
except ImportError:
print("ERROR: storage_manager module not found")
sys.exit(1)
try:
from rss_processor import is_duplicate
except ImportError:
print("ERROR: rss_processor module not found")
sys.exit(1)
SCRIPT_DIR = Path(__file__).parent
ARCHIVE_DIR = SCRIPT_DIR / 'archival_data'
ARCHIVE_DIR.mkdir(exist_ok=True)
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(levelname)s - %(message)s',
handlers=[
logging.StreamHandler(sys.stdout),
logging.FileHandler(ARCHIVE_DIR / 'processing.log', encoding='utf-8')
]
)
logger = logging.getLogger(__name__)
def fetch_ap_frontpage(timeout: int = 30) -> str:
"""Fetch AP News front page HTML.
Args:
timeout: Request timeout in seconds
Returns:
HTML string or empty string on failure
"""
if requests is None:
logger.error("requests library not available")
return ""
url = "https://apnews.com"
try:
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
}
response = requests.get(url, timeout=timeout, headers=headers)
response.raise_for_status()
logger.info("Fetched AP front page: %s", url)
return response.text
except Exception as e:
logger.error("Failed to fetch AP front page: %s", str(e))
return ""
def extract_article_links(html: str) -> List[str]:
"""Extract AP article URLs from HTML.
Args:
html: Raw HTML string
Returns:
List of unique article URLs
"""
pattern = r'href="https://apnews\.com/article/[^"]*"'
matches = re.findall(pattern, html)
urls = []
for match in matches:
url = match.replace('href="', '').replace('"', '')
if url not in urls:
urls.append(url)
logger.info("Extracted %d unique article links", len(urls))
return urls
def process_ap_articles(
output_dir: Path,
db_path: Path = ARCHIVE_DIR / 'cache.db',
dry_run: bool = False
) -> dict:
"""Process AP News articles from front page.
Args:
output_dir: Output directory for archived content
db_path: SQLite cache database path
dry_run: If True, preview without archiving
Returns:
Dictionary with results summary
"""
results = {
'source': 'Associated Press',
'processed': 0,
'archived': 0,
'skipped': 0,
'failed': 0,
'urls': []
}
html = fetch_ap_frontpage()
if not html:
logger.error("Failed to fetch AP front page")
return results
article_urls = extract_article_links(html)
for article_url in article_urls:
results['processed'] += 1
results['urls'].append(article_url)
if is_duplicate(article_url, 'Associated Press', db_path):
logger.debug("Skipping duplicate: %s", article_url[:60])
results['skipped'] += 1
continue
if dry_run:
logger.info("[DRY-RUN] Would archive: %s", article_url[:60])
results['archived'] += 1
continue
try:
from archive_engine import archive_and_extract
extraction_result = archive_and_extract(
article_url,
'Associated Press',
output_dir
)
if extraction_result['success']:
results['archived'] += 1
storage_manager.save_article('Associated Press', extraction_result['article_data'])
else:
results['failed'] += 1
logger.error("Failed to archive %s: %s", article_url[:60],
extraction_result.get('error', 'Unknown error'))
except Exception as e:
results['failed'] += 1
logger.error("Error processing %s: %s", article_url[:60], str(e))
time.sleep(0.5)
logger.info("AP processing complete: %d processed, %d archived, %d skipped, %d failed",
results['processed'], results['archived'], results['skipped'], results['failed'])
return results
def main():
parser = argparse.ArgumentParser(description='AP News Front Page Processor')
parser.add_argument('--output', type=Path, default=ARCHIVE_DIR,
help='Output directory for archived content')
parser.add_argument('--dry-run', action='store_true', help='Preview without archiving')
parser.add_argument('--verbose', action='store_true', help='Enable verbose logging')
args = parser.parse_args()
if args.verbose:
logger.setLevel(logging.DEBUG)
logger.info("=" * 60)
logger.info("AP News Front Page Processor - Phase 2.6")
logger.info("=" * 60)
storage_manager.initialize_storage()
results = process_ap_articles(args.output, ARCHIVE_DIR / 'cache.db', args.dry_run)
print("\n" + "=" * 60)
print("AP NEWS PROCESSING COMPLETE")
print("=" * 60)
print(f"Processed: {results['processed']}")
print(f"Archived: {results['archived']}")
print(f"Skipped: {results['skipped']}")
print(f"Failed: {results['failed']}")
if results['urls']:
print("\nArticle URLs:")
for url in results['urls'][:10]:
print(f" - {url[:70]}")
if len(results['urls']) > 10:
print(f" ... and {len(results['urls']) - 10} more")
print("=" * 60)
if __name__ == '__main__':
main()

545
archive_engine.py Normal file
View File

@ -0,0 +1,545 @@
#!/usr/bin/env python3
"""Archive Engine for NewsArchiver - Phase 2.5
Orchestrates the archiving process:
- RSS polling
- Page archiving with SingleFile
- Content extraction
- Storage management
"""
import argparse
import json
import logging
import sys
from datetime import datetime
from pathlib import Path
from typing import List, Dict, Optional
try:
import storage_manager
from rss_processor import fetch_rss_feed, is_duplicate, save_article as cache_save_article
from content_extractor import parse_article_from_html, get_html_from_url
from storage_manager import save_article as storage_save_article
except ImportError as e:
print(f"ERROR: Required module not found: {e}")
sys.exit(1)
try:
from singlefile_archive import archive_page_with_singlefile, archive_page_with_playwright
except ImportError:
archive_page_with_singlefile = None
archive_page_with_playwright = None
try:
from ap_processor import process_ap_articles
except ImportError:
process_ap_articles = None
try:
from storage_manager import get_archive_file_path_from_db
except ImportError:
get_archive_file_path_from_db = None
SCRIPT_DIR = Path(__file__).parent
ARCHIVE_DIR = SCRIPT_DIR / 'archival_data'
ARCHIVE_DIR.mkdir(exist_ok=True)
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(levelname)s - %(message)s',
handlers=[
logging.StreamHandler(sys.stdout),
logging.FileHandler(ARCHIVE_DIR / 'processing.log', encoding='utf-8')
]
)
logger = logging.getLogger(__name__)
def get_archive_file_path(source_name: str, article_url: str, archive_dir: Path = ARCHIVE_DIR) -> Optional[Path]:
"""Find archived HTML file for an article.
First checks the database mapping for the archive file path.
Falls back to searching HTML files if not found in database.
Args:
source_name: Newspaper source name
article_url: Article URL
archive_dir: Root archive directory
Returns:
Path to archived HTML file if found, None otherwise
"""
archive_file_path = get_archive_file_path_from_db(article_url, source_name)
if archive_file_path:
archive_path = Path(archive_file_path)
if archive_path.exists():
logger.debug("Found archived HTML (DB) for %s: %s", article_url[:60], archive_path)
return archive_path
logger.debug("Archive file not found on disk: %s", archive_file_path)
websites_dir = archive_dir / 'websites'
if not websites_dir.exists():
return None
try:
source_dir = websites_dir / source_name
if not source_dir.exists():
return None
html_dir = source_dir / 'html'
if not html_dir.exists():
return None
for date_dir in sorted(html_dir.iterdir()):
if not date_dir.is_dir():
continue
for html_file in sorted(date_dir.glob('article_*.html')):
try:
with open(html_file, 'r', encoding='utf-8') as f:
html_content = f.read()
if article_url in html_content:
logger.debug("Found archived HTML (search) for %s: %s", article_url[:60], html_file)
return html_file
except Exception as e:
logger.debug("Error reading %s: %s", html_file, str(e))
continue
except Exception as e:
logger.debug("Error searching for archive: %s", str(e))
return None
def extract_content_from_archive(article_url: str, source_name: str = None, archive_dir: Path = ARCHIVE_DIR) -> dict:
"""Extract content from web page, preferring archived HTML over live fetch.
Args:
article_url: URL to extract content from
source_name: Newspaper source name (for locating archives)
archive_dir: Root archive directory
Returns:
Dictionary with extracted content and metadata
"""
raw_html = None
extraction_method = 'live_fetch'
try:
if source_name:
archived_file = get_archive_file_path(source_name, article_url, archive_dir)
if archived_file:
try:
raw_html = archived_file.read_text(encoding='utf-8')
extraction_method = 'archive'
logger.debug("Loaded archived HTML from %s", archived_file)
except Exception as e:
logger.warning("Failed to read archived HTML from %s: %s", archived_file, str(e))
if raw_html is None:
raw_html = get_html_from_url(article_url)
if not raw_html:
logger.error("No HTML content retrieved for %s", article_url[:60])
return {
'success': False,
'error': 'Failed to retrieve HTML content'
}
article_data = parse_article_from_html(raw_html, article_url)
extraction_method = article_data.extraction_method or extraction_method
logger.debug("Extracted content from %s using %s", article_url[:60], extraction_method)
return {
'success': True,
'article_data': article_data
}
except Exception as e:
logger.error("Error extracting content from %s: %s", article_url, str(e))
return {
'success': False,
'error': str(e)
}
def archive_and_extract(article_url: str, source_name: str, archive_dir: Path) -> dict:
"""Archive a URL and extract content from the archive.
This function first tries to archive the URL using SingleFile or Playwright.
Then it extracts content from the archived HTML.
Args:
article_url: URL to archive and extract
source_name: Newspaper source name (for directory structure)
archive_dir: Root archive directory
Returns:
Dictionary with extraction results
"""
archived_file = archive_article_url(article_url, source_name, archive_dir)
if not archived_file:
logger.error("Failed to archive %s", article_url[:60])
return {
'success': False,
'error': 'Failed to archive article'
}
extraction_result = extract_content_from_archive(article_url, source_name, archive_dir)
if extraction_result['success']:
logger.info("Successfully archived and extracted content from %s", article_url[:60])
else:
logger.error("Failed to extract content from archived %s: %s", article_url[:60], extraction_result.get('error', 'Unknown error'))
return extraction_result
def archive_url_with_fallback(url: str, output_path: Path) -> bool:
"""Archive a URL using SingleFile if available, falling back to Playwright.
Args:
url: URL to archive
output_path: Output file path for the archived HTML
Returns:
True if successful, False otherwise
"""
if archive_page_with_singlefile:
if archive_page_with_singlefile(url, output_path):
return True
if archive_page_with_playwright:
logger.info("Falling back to Playwright archiving for %s", url[:60])
if archive_page_with_playwright(url, output_path):
return True
return False
def archive_article_url(article_url: str, source_name: str, archive_dir: Path) -> Optional[Path]:
"""Archive a single article URL using SingleFile or Playwright fallback.
Args:
article_url: URL to archive
source_name: Newspaper source name (for directory structure)
archive_dir: Root archive directory
Returns:
Path to archived HTML file if successful, None otherwise
"""
try:
websites_dir = archive_dir / 'websites'
source_dir = websites_dir / source_name
html_dir = source_dir / 'html'
timestamp = datetime.now().strftime('%Y-%m-%d')
date_dir = html_dir / timestamp
date_dir.mkdir(parents=True, exist_ok=True)
base_timestamp = int(datetime.now().timestamp())
counter = 0
article_filename = f'article_{base_timestamp}_{counter}.html'
output_path = date_dir / article_filename
while output_path.exists():
counter += 1
article_filename = f'article_{base_timestamp}_{counter}.html'
output_path = date_dir / article_filename
if archive_page_with_singlefile:
if archive_page_with_singlefile(article_url, output_path):
logger.info("Archived %s to %s", article_url[:60], output_path)
return output_path
logger.info("SingleFile failed, attempting direct fetch fallback for %s", article_url[:60])
if archive_page_with_playwright:
logger.info("Falling back to Playwright for %s", article_url[:60])
if archive_page_with_playwright(article_url, output_path):
return output_path
logger.info("Attempting direct HTML fetch fallback for %s", article_url[:60])
direct_html = get_html_from_url(article_url)
if direct_html:
output_path.write_text(direct_html, encoding='utf-8')
logger.info("Archived %s to %s using direct fetch", article_url[:60], output_path)
return output_path
logger.error("Failed to archive %s", article_url[:60])
return None
except Exception as e:
logger.error("Error archiving %s: %s", article_url, str(e))
return None
def archive_newspaper(
source_name: str,
rss_url: str,
output_dir: Path,
dry_run: bool = False
) -> dict:
"""Main archiving workflow for a newspaper.
Args:
source_name: Newspaper source name
rss_url: RSS feed URL
output_dir: Output directory for archived content
dry_run: If True, preview without making changes
Returns:
Dictionary with results summary
"""
storage_manager.initialize_storage()
logger.info("=" * 60)
logger.info("Starting newspaper archive: %s", source_name)
logger.info("=" * 60)
results = {
'source': source_name,
'rss_url': rss_url,
'processed': 0,
'archived': 0,
'skipped': 0,
'failed': 0,
'errors': []
}
try:
feed = fetch_rss_feed(rss_url)
except Exception as e:
logger.error("Failed to fetch RSS feed for %s: %s", source_name, str(e))
results['errors'].append({
'action': 'fetch_rss',
'error': str(e)
})
return results
for item in feed.entries:
article_url = item.get('link', '')
if not article_url:
logger.warning("Skipping entry without URL")
results['failed'] += 1
continue
if is_duplicate(article_url, source_name, ARCHIVE_DIR / 'cache.db'):
logger.debug("Skipping duplicate: %s", article_url[:60])
results['skipped'] += 1
continue
results['processed'] += 1
if dry_run:
title = item.get('title', 'No Title')
logger.info("[DRY-RUN] Would process: %s - %s", title[:60], article_url[:60])
results['archived'] += 1
continue
extraction_result = archive_and_extract(article_url, source_name, output_dir)
if not extraction_result['success']:
logger.error("Failed to extract content from %s: %s", article_url[:60], extraction_result.get('error', 'Unknown error'))
results['failed'] += 1
results['errors'].append({
'url': article_url,
'action': 'extract_content',
'error': extraction_result.get('error', 'Unknown error')
})
continue
article_data = extraction_result['article_data']
try:
storage_save_article(source_name, article_data)
results['archived'] += 1
except Exception as e:
logger.error("Failed to save article %s: %s", article_url[:60], str(e))
results['failed'] += 1
results['errors'].append({
'url': article_url,
'action': 'save_article',
'error': str(e)
})
logger.info("Archive complete: %s - %d processed, %d archived, %d skipped, %d failed",
source_name, results['processed'], results['archived'], results['skipped'], results['failed'])
return results
def archive_all_sources(
rss_feeds_path: Path = SCRIPT_DIR / 'rss_feeds.json',
output_dir: Path = ARCHIVE_DIR,
dry_run: bool = False
) -> dict:
"""Process all RSS feeds from rss_feeds.json.
Args:
rss_feeds_path: Path to RSS feeds JSON file
output_dir: Output directory for archived content
dry_run: If True, preview without making changes
Returns:
Dictionary with results summary
"""
if not rss_feeds_path.exists():
logger.error("RSS feeds file not found: %s", rss_feeds_path)
return {
'success': False,
'error': 'File not found',
'processed': 0,
'success_count': 0,
'failed_count': 0
}
with open(rss_feeds_path, 'r', encoding='utf-8') as f:
rss_feeds = json.load(f)
total_results = {
'sources_processed': 0,
'sources_failed': 0,
'total_articles_processed': 0,
'total_articles_archived': 0,
'total_articles_skipped': 0,
'total_articles_failed': 0,
'source_results': [],
'errors': []
}
for source_name, feed_info in rss_feeds.items():
rss_url = feed_info.get('rss_url', '')
feed_type = feed_info.get('feed_type', 'rss')
if not rss_url:
logger.warning("No RSS URL for source: %s", source_name)
continue
try:
if feed_type == 'html' and process_ap_articles:
ap_results = process_ap_articles(
output_dir,
ARCHIVE_DIR / 'cache.db',
dry_run
)
ap_results['source'] = source_name
total_results['sources_processed'] += 1
total_results['total_articles_processed'] += ap_results['processed']
total_results['total_articles_archived'] += ap_results['archived']
total_results['total_articles_skipped'] += ap_results['skipped']
total_results['total_articles_failed'] += ap_results['failed']
total_results['source_results'].append(ap_results)
if ap_results.get('errors'):
total_results['errors'].extend(ap_results['errors'])
elif feed_type == 'html' and not process_ap_articles:
logger.error("HTML feed type configured but ap_processor unavailable for %s", source_name)
total_results['sources_failed'] += 1
total_results['source_results'].append({
'source': source_name,
'rss_url': rss_url,
'error': 'HTML feed processing not available'
})
else:
results = archive_newspaper(source_name, rss_url, output_dir, dry_run)
total_results['sources_processed'] += 1
total_results['total_articles_processed'] += results['processed']
total_results['total_articles_archived'] += results['archived']
total_results['total_articles_skipped'] += results['skipped']
total_results['total_articles_failed'] += results['failed']
total_results['source_results'].append(results)
if results['errors']:
total_results['errors'].extend(results['errors'])
except Exception as e:
logger.error("Failed to process source %s: %s", source_name, str(e))
total_results['sources_failed'] += 1
total_results['source_results'].append({
'source': source_name,
'rss_url': rss_url,
'error': str(e)
})
return total_results
def main():
"""Main entry point."""
parser = argparse.ArgumentParser(description='Archive Engine for NewsArchiver - Phase 2.5')
parser.add_argument('--source', help='Single source name to process')
parser.add_argument('--rss-url', help='RSS URL (required if --source provided)')
parser.add_argument('--all', action='store_true', help='Process all sources from rss_feeds.json')
parser.add_argument('--rss-feeds', type=Path, default=SCRIPT_DIR / 'rss_feeds.json',
help='Path to RSS feeds JSON file')
parser.add_argument('--output', type=Path, default=ARCHIVE_DIR,
help='Output directory for archived content')
parser.add_argument('--dry-run', action='store_true', help='Preview without making changes')
parser.add_argument('--verbose', action='store_true', help='Enable verbose logging')
args = parser.parse_args()
if args.verbose:
logger.setLevel(logging.DEBUG)
logger.info("=" * 60)
logger.info("Archive Engine - Phase 2.5")
logger.info("=" * 60)
storage_manager.initialize_storage()
if args.source:
if not args.rss_url:
logger.error("RSS URL required when using --source")
return
results = archive_newspaper(args.source, args.rss_url, args.output, args.dry_run)
print("\n" + "=" * 60)
print(f"SOURCE: {args.source}")
print("=" * 60)
print(f"Processed: {results['processed']}")
print(f"Archived: {results['archived']}")
print(f"Skipped: {results['skipped']}")
print(f"Failed: {results['failed']}")
if results['errors']:
print("\nErrors:")
for error in results['errors']:
print(f" - {error.get('url', 'Unknown')}: {error.get('error', 'Unknown error')}")
print("=" * 60)
elif args.all:
results = archive_all_sources(args.rss_feeds, args.output, args.dry_run)
print("\n" + "=" * 60)
print("PROCESSING COMPLETE")
print("=" * 60)
print(f"Sources processed: {results['sources_processed']}")
print(f"Sources failed: {results['sources_failed']}")
print(f"Total articles processed: {results['total_articles_processed']}")
print(f"Total articles archived: {results['total_articles_archived']}")
print(f"Total articles skipped: {results['total_articles_skipped']}")
print(f"Total articles failed: {results['total_articles_failed']}")
if results['errors']:
print("\nErrors:")
for error in results['errors'][:10]:
url = error.get('url', 'Unknown')
action = error.get('action', 'Unknown')
error_msg = error.get('error', 'Unknown error')
print(f" - [{action}] {url}: {error_msg}")
if len(results['errors']) > 10:
print(f" ... and {len(results['errors']) - 10} more errors")
print("=" * 60)
else:
parser.print_help()
if __name__ == '__main__':
main()

369
cleanup_old_files.py Normal file
View File

@ -0,0 +1,369 @@
#!/usr/bin/env python3
"""
Cleanup Script for NewsArchiver
This script removes files older than a specified date from the project directory.
It provides dry-run mode to preview what would be deleted before actually deleting.
Usage:
python cleanup_old_files.py --date "2024-03-19" --dry-run
python cleanup_old_files.py --date "2024-03-19"
Options:
--date, -d Date in YYYY-MM-DD format (required)
--dry-run, -n Show what would be deleted without actually deleting (default: True)
--force, -f Actually delete files (disables dry-run mode)
--verbose, -v Enable verbose output
--archival Include archival_data folder for cleanup
"""
import argparse
import logging
import sys
from datetime import datetime, timezone
from pathlib import Path
from typing import List, Tuple
# Configure logging
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s - %(levelname)s - %(message)s",
handlers=[
logging.StreamHandler(sys.stdout),
],
)
logger = logging.getLogger(__name__)
# Project root directory
SCRIPT_DIR = Path(__file__).parent.resolve()
# Path to archival_data directory
ARCHIVAL_DATA_DIR = SCRIPT_DIR / "archival_data"
# Path to websites folder (only this folder will be scanned in archival_data)
WEBSITES_DIR = ARCHIVAL_DATA_DIR / "websites"
# Files and directories to always preserve (never delete)
PRESERVE_LIST = {
# Python files
"ap_processor.py",
"archive_engine.py",
"content_extractor.py",
"rebuild_database.py",
"restore_database.py",
"rss_feeds.json",
"rss_processor.py",
"run_archiver.py",
"scheduler.py",
"setup_cron.sh",
"singlefile_archive.py",
"stop_services.sh",
"web_interface.py",
"cleanup_old_files.py",
# Directories
"archival_data",
"static",
"templates",
"__pycache__",
}
# Database files to preserve
DATABASE_FILES = {"cache.db", "cache.db-shm", "cache.db-wal"}
# Files that should be excluded from cleanup regardless of date
EXCLUDE_PATTERNS = [
".git",
".gitignore",
]
def parse_date(date_str: str) -> datetime:
"""Parse date string in YYYY-MM-DD format.
Args:
date_str: Date string in YYYY-MM-DD format
Returns:
datetime object with the specified date at midnight
Raises:
ValueError: If date format is invalid
"""
try:
return datetime.strptime(date_str, "%Y-%m-%d").replace(tzinfo=timezone.utc)
except ValueError as e:
raise ValueError(
f"Invalid date format: '{date_str}'. Use YYYY-MM-DD format."
) from e
def should_preserve(path: Path) -> bool:
"""Check if a file/directory should be preserved.
Args:
path: Path to check
Returns:
True if the path should be preserved, False otherwise
"""
# Check if it's in the preserve list
if path.name in PRESERVE_LIST:
return True
# Check if it matches any exclude patterns
for pattern in EXCLUDE_PATTERNS:
if pattern in str(path):
return True
return False
def should_preserve_archival_file(path: Path) -> bool:
"""Check if a file in archival_data should be preserved.
Args:
path: Path to check
Returns:
True if the path should be preserved, False otherwise
"""
# Always preserve database files
if path.name in DATABASE_FILES:
return True
return should_preserve(path)
def get_files_older_than_date(
directory: Path, cutoff_date: datetime
) -> List[Tuple[Path, datetime]]:
"""Get all files older than the cutoff date.
Args:
directory: Directory to search
cutoff_date: Files older than this date will be selected
Returns:
List of tuples (path, modification_time) for files older than cutoff
"""
old_files = []
# Walk through all files in directory recursively
for item in directory.rglob("*"):
if item.is_file():
if should_preserve(item):
continue
try:
mtime = datetime.fromtimestamp(item.stat().st_mtime, tz=timezone.utc)
if mtime < cutoff_date:
old_files.append((item, mtime))
except (OSError, ValueError) as e:
logger.warning(f"Could not access file {item}: {e}")
return old_files
def get_files_older_than_date_non_recursive(
directory: Path, cutoff_date: datetime
) -> List[Tuple[Path, datetime]]:
"""Get all files older than the cutoff date (non-recursive).
Args:
directory: Directory to search
cutoff_date: Files older than this date will be selected
Returns:
List of tuples (path, modification_time) for files older than cutoff
"""
old_files = []
# Walk through all files in directory (non-recursive for safety)
for item in directory.iterdir():
if item.is_file():
if should_preserve(item):
continue
try:
mtime = datetime.fromtimestamp(item.stat().st_mtime, tz=timezone.utc)
if mtime < cutoff_date:
old_files.append((item, mtime))
except (OSError, ValueError) as e:
logger.warning(f"Could not access file {item}: {e}")
return old_files
def delete_files(files: List[Tuple[Path, datetime]]) -> Tuple[int, int]:
"""Delete files and return count of successful/failed deletions.
Args:
files: List of (path, modification_time) tuples to delete
Returns:
Tuple of (deleted_count, failed_count)
"""
deleted = 0
failed = 0
for path, mtime in files:
try:
path.unlink()
logger.info(
f"Deleted: {path.name} (modified: {mtime.strftime('%Y-%m-%d %H:%M:%S')})"
)
deleted += 1
except OSError as e:
logger.error(f"Failed to delete {path.name}: {e}")
failed += 1
return deleted, failed
def main():
"""Main entry point for cleanup script."""
parser = argparse.ArgumentParser(
description="Cleanup old files from NewsArchiver project",
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="""
Examples:
%(prog)s --date "2024-03-19" --dry-run
%(prog)s --date "2024-03-19" --force
%(prog)s --date "2024-03-19" --archival --force
""",
)
parser.add_argument(
"--date",
"-d",
type=str,
required=True,
help="Date in YYYY-MM-DD format - files older than this will be deleted",
)
parser.add_argument(
"--dry-run",
"-n",
action="store_true",
default=True,
help="Show what would be deleted without actually deleting (default)",
)
parser.add_argument(
"--force",
"-f",
action="store_true",
help="Actually delete files (disables dry-run mode)",
)
parser.add_argument(
"--verbose", "-v", action="store_true", help="Enable verbose output"
)
parser.add_argument(
"--archival",
action="store_true",
help="Include archival_data folder for cleanup",
)
args = parser.parse_args()
# Set logging level
if args.verbose:
logger.setLevel(logging.DEBUG)
# Parse the date
try:
cutoff_date = parse_date(args.date)
except ValueError as e:
logger.error(str(e))
sys.exit(1)
# Validate cutoff date is not in the future
now = datetime.now(timezone.utc)
if cutoff_date > now:
logger.error("Cutoff date cannot be in the future")
sys.exit(1)
logger.info("=" * 60)
logger.info("NewsArchiver Cleanup Script")
logger.info("=" * 60)
# Determine mode
dry_run = not args.force
mode = "DRY RUN" if dry_run else "ACTUAL DELETE"
logger.info("Mode: %s", mode)
logger.info(
"Cutoff Date: %s (files older than this will be %s)",
cutoff_date.strftime("%Y-%m-%d"),
"kept" if dry_run else "deleted",
)
logger.info("=" * 60)
# Find files older than cutoff date
if args.archival:
# Only scan websites folder when --archival is used
if not WEBSITES_DIR.exists():
logger.error("Websites folder not found at %s", WEBSITES_DIR)
sys.exit(1)
old_files = []
logger.info(
"Scanning websites folder for files older than %s...",
cutoff_date.strftime("%Y-%m-%d"),
)
# First check if there are any files directly in websites folder
website_root_files = get_files_older_than_date_non_recursive(
WEBSITES_DIR, cutoff_date
)
# Then check recursively in subdirectories
website_recursive_files = []
for subdir in WEBSITES_DIR.iterdir():
if subdir.is_dir():
website_recursive_files.extend(
get_files_older_than_date(subdir, cutoff_date)
)
old_files.extend(website_root_files)
old_files.extend(website_recursive_files)
logger.info(
"Found %d files in websites folder",
len(website_root_files) + len(website_recursive_files),
)
else:
# Scan root directory (non-archival mode)
old_files = get_files_older_than_date_non_recursive(SCRIPT_DIR, cutoff_date)
if not old_files:
logger.info("No files older than %s found.", cutoff_date.strftime("%Y-%m-%d"))
logger.info("Nothing to do.")
return
logger.info(
"Found %d files older than %s:",
len(old_files),
cutoff_date.strftime("%Y-%m-%d"),
)
# List all files that would be affected
for path, mtime in old_files:
# Get relative path for cleaner output
rel_path = path.relative_to(SCRIPT_DIR)
logger.info(
" - %s (modified: %s)", rel_path, mtime.strftime("%Y-%m-%d %H:%M:%S")
)
logger.info("=" * 60)
# Execute deletion if not dry run
if dry_run:
logger.info("DRY RUN: No files were deleted.")
logger.info("Run with --force to actually delete these files.")
else:
deleted, failed = delete_files(old_files)
logger.info("=" * 60)
logger.info("Cleanup complete!")
logger.info("Deleted: %d files", deleted)
logger.info("Failed: %d files", failed)
if __name__ == "__main__":
main()

395
content_extractor.py Normal file
View File

@ -0,0 +1,395 @@
#!/usr/bin/env python3
"""Content Extractor for NewsArchiver - Phase 2.2
Extracts article text and metadata from HTML using Trafilatura
with BeautifulSoup fallback for JavaScript-heavy sites.
"""
import logging
import sys
import time
from dataclasses import dataclass
from datetime import datetime
from pathlib import Path
from typing import Optional
try:
import requests
except ImportError:
print("WARNING: requests not installed. URL fetching may not work.")
print("Install with: pip install requests")
requests = None
try:
import trafilatura
from trafilatura import extract, extract_metadata
except ImportError:
print("ERROR: trafilatura is required. Install with: pip install trafilatura")
sys.exit(1)
try:
from bs4 import BeautifulSoup
except ImportError:
print("WARNING: beautifulsoup4 not installed. Some fallback features may not work.")
print("Install with: pip install beautifulsoup4")
BeautifulSoup = None
try:
from playwright.sync_api import sync_playwright
PLAYWRIGHT_AVAILABLE = True
except ImportError:
PLAYWRIGHT_AVAILABLE = False
SCRIPT_DIR = Path(__file__).parent
ARCHIVE_DIR = SCRIPT_DIR / 'archival_data'
ARCHIVE_DIR.mkdir(exist_ok=True)
_last_request_time = 0.0
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(levelname)s - %(message)s',
handlers=[
logging.StreamHandler(sys.stdout),
logging.FileHandler(ARCHIVE_DIR / 'processing.log', encoding='utf-8')
]
)
logger = logging.getLogger(__name__)
@dataclass
class ArticleData:
"""Structured article data."""
url: str
title: Optional[str] = None
author: Optional[str] = None
publish_date: Optional[str] = None
content_text: Optional[str] = None
content_html: Optional[str] = None
raw_html: Optional[str] = None
archive_file_path: Optional[str] = None
tags: Optional[list] = None
language: Optional[str] = None
metadata: Optional[dict] = None
extraction_method: Optional[str] = None
error: Optional[str] = None
guid: Optional[str] = None
id: Optional[int] = None
source_name: Optional[str] = None
def fetch_url(url: str, timeout: int = 30) -> str:
"""Download HTML from URL with rate limiting.
Args:
url: URL to fetch
timeout: Request timeout in seconds
Returns:
HTML string or empty string on failure
"""
global _last_request_time
if requests is None:
logger.error("requests library not available")
return ""
try:
delay = 0.5 - (time.time() - _last_request_time)
if delay > 0:
time.sleep(delay)
_last_request_time = time.time()
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
}
response = requests.get(url, timeout=timeout, headers=headers)
response.raise_for_status()
text = response.text
# Check if response is an error page (only check for actual HTTP errors)
if response.status_code >= 400:
logger.warning("Server returned error for %s (status: %d)", url, response.status_code)
return ""
# Check for common error patterns
if 'access denied' in text.lower() or 'forbidden' in text.lower():
logger.warning("Server returned access denied for %s", url)
return ""
return text
except Exception as e:
logger.error("Failed to fetch URL %s: %s", url, str(e))
return ""
def extract_content(html: str) -> dict:
"""Extract article content and metadata from HTML using Trafilatura.
Args:
html: Raw HTML string (or plain text)
Returns:
Dictionary with extracted content and metadata
"""
result = {
'success': False,
'content_text': None,
'content_html': None,
'title': None,
'author': None,
'date': None,
'tags': None,
'language': None,
'metadata': None,
'extraction_method': None,
'error': None
}
if not html or not html.strip():
result['error'] = 'Empty HTML content'
return result
# Check if this is an error page (HTTP status codes, not the word "error" in content)
# Only check for error patterns that appear in actual error pages (not in article content)
# Look for specific error page patterns with proper HTML structure
import re
error_patterns = [
r'<title[^>]*>403[^<]*Forbidden</title>',
r'<title[^>]*>401[^<]*Unauthorized</title>',
r'<h1[^>]*>403</h1>',
r'<h1[^>]*>401</h1>',
]
html_lower = html.lower()
for pattern in error_patterns:
if re.search(pattern, html_lower):
result['error'] = 'HTML contains error page content'
result['success'] = False
return result
# Check for common error indicators in the HTML body
# These should only trigger if we see them in context (like a 403/401 status indicator)
if '<title>403' in html_lower or '<title>401' in html_lower:
result['error'] = 'HTML contains error page content'
result['success'] = False
return result
has_html_tags = '<' in html and '>' in html
try:
metadata = extract_metadata(html)
if metadata:
result['title'] = metadata.title
result['author'] = metadata.author
result['date'] = metadata.date
result['tags'] = metadata.tags if hasattr(metadata, 'tags') else None
result['language'] = metadata.language
if hasattr(metadata, 'to_dict'):
result['metadata'] = metadata.to_dict()
elif hasattr(metadata, '__dict__'):
result['metadata'] = metadata.__dict__
content = extract(
html,
include_comments=False,
include_tables=True,
no_fallback=True
)
if content:
result['content_text'] = content
result['content_html'] = html
result['extraction_method'] = 'trafilatura'
result['success'] = True
elif not has_html_tags:
result['content_text'] = html.strip()
result['content_html'] = f'<html><body>{html}</body></html>'
result['extraction_method'] = 'plain_text'
result['success'] = True
else:
result['content_text'] = None
result['content_html'] = html
result['extraction_method'] = 'trafilatura_empty'
result['success'] = False
result['error'] = 'Content extraction failed - no article content found'
if result['success']:
logger.debug("Content extracted using %s", result['extraction_method'])
except Exception as e:
result['error'] = f"Trafilatura extraction failed: {str(e)}"
logger.warning(result['error'])
if not has_html_tags:
result['content_text'] = html.strip()
result['content_html'] = f'<html><body>{html}</body></html>'
result['extraction_method'] = 'plain_text_fallback'
result['success'] = True
elif BeautifulSoup:
fallback_result = _extract_with_beautifulsoup(html)
if fallback_result['content_text']:
result.update(fallback_result)
result['extraction_method'] = 'beautifulsoup_fallback'
return result
def _extract_with_beautifulsoup(html: str) -> dict:
"""Fallback extraction using BeautifulSoup."""
result = {
'content_text': None,
'content_html': None,
'title': None,
'author': None,
'date': None,
'tags': None,
'language': None,
'metadata': None
}
try:
soup = BeautifulSoup(html, 'html.parser')
title = soup.find('title')
if title:
result['title'] = title.get_text(strip=True)
meta_author = soup.find('meta', attrs={'name': 'author'})
if meta_author:
result['author'] = meta_author.get('content', '').strip()
meta_date = soup.find('meta', attrs={'name': 'date'})
if meta_date:
result['date'] = meta_date.get('content', '').strip()
meta_language = soup.find('meta', attrs={'name': 'language'})
if meta_language:
result['language'] = meta_language.get('content', '').strip()
for tag in ['article', 'main', 'div']:
content_tags = soup.find_all(tag)
if content_tags:
result['content_text'] = ' '.join(
tag.get_text(strip=True, separator=' ')
for tag in content_tags
)
if result['content_text']:
break
if not result['content_text']:
result['content_text'] = soup.get_text(strip=True, separator=' ')
result['content_text'] = result['content_text'][:100000]
result['content_html'] = str(soup)
except Exception as e:
result['error'] = f"BeautifulSoup extraction failed: {str(e)}"
logger.warning(result['error'])
return result
def _extract_with_playwright(url: str) -> Optional[str]:
"""Extract HTML from JavaScript-heavy site using Playwright."""
if not PLAYWRIGHT_AVAILABLE:
logger.warning("Playwright not available. Install with: pip install playwright")
return None
try:
with sync_playwright() as p:
browser = p.chromium.launch(headless=True)
page = browser.new_page()
page.goto(url, wait_until='networkidle', timeout=60000)
page_content = page.content()
browser.close()
logger.debug("HTML extracted using Playwright")
return page_content
except Exception as e:
logger.error("Playwright extraction failed: %s", str(e))
return None
def parse_article_from_html(html: str, url: str) -> ArticleData:
"""Parse article and return structured data.
Args:
html: Raw HTML string
url: Original URL for reference
Returns:
ArticleData with extracted content and metadata
"""
extraction_result = extract_content(html)
guid = extraction_result.get('metadata', {}).get('url', url) if extraction_result.get('metadata') else url
article = ArticleData(
url=url,
title=extraction_result.get('title'),
author=extraction_result.get('author'),
publish_date=extraction_result.get('date'),
content_text=extraction_result.get('content_text'),
content_html=extraction_result.get('content_html'),
raw_html=html,
tags=extraction_result.get('tags'),
language=extraction_result.get('language'),
metadata=extraction_result.get('metadata'),
extraction_method=extraction_result.get('extraction_method'),
guid=guid
)
if not extraction_result.get('success'):
article.error = extraction_result.get('error')
return article
def get_html_from_url(url: str, extract_content: bool = True) -> str:
"""Download HTML from URL using Trafilatura with Playwright fallback.
Args:
url: URL to fetch
extract_content: If True, try to extract main content (kept for backwards compatibility)
Returns:
Full HTML string or empty string on failure
"""
try:
downloaded = fetch_url(url)
if downloaded:
logger.debug("Full HTML fetched from %s", url[:60])
return downloaded
except Exception as e:
logger.error("Failed to fetch URL %s: %s", url, str(e))
if PLAYWRIGHT_AVAILABLE:
logger.info("Trying Playwright fallback for %s", url[:60])
playwright_html = _extract_with_playwright(url)
if playwright_html:
logger.debug("Full HTML from Playwright for %s", url[:60])
if extract_content:
extracted = extract(
playwright_html,
include_comments=False,
include_tables=True,
no_fallback=True
)
if extracted:
logger.debug("Playwright content extracted from %s", url[:60])
return extracted
logger.debug("Full HTML from Playwright for %s", url[:60])
return playwright_html
return ""

218
nohup.out Normal file
View File

@ -0,0 +1,218 @@
2026-03-31 00:45:43,896 - INFO - NewsArchiver - Main CLI Entry Point
2026-03-31 00:45:43,896 - INFO - ============================================================
2026-03-31 00:45:43,896 - INFO - Starting web server
2026-03-31 00:45:43,896 - INFO - ============================================================
2026-03-31 00:45:43,984 - INFO - Web server starting on 0.0.0.0:8080
2026-03-31 00:45:43,984 - INFO - ============================================================
* Serving Flask app 'web_interface'
* Debug mode: off
2026-03-31 00:45:43,985 - INFO - WARNING: This is a development server. Do not use it in a production deployment. Use a production WSGI server instead.
* Running on all addresses (0.0.0.0)
* Running on http://127.0.0.1:8080
* Running on http://192.168.8.150:8080
2026-03-31 00:45:43,985 - INFO - Press CTRL+C to quit
2026-03-31 00:46:57,096 - INFO - NewsArchiver - Main CLI Entry Point
2026-03-31 00:46:57,096 - INFO - ============================================================
2026-03-31 00:46:57,096 - INFO - Starting web server
2026-03-31 00:46:57,096 - INFO - ============================================================
2026-03-31 00:46:57,185 - INFO - Web server starting on 0.0.0.0:5000
2026-03-31 00:46:57,185 - INFO - ============================================================
* Serving Flask app 'web_interface'
* Debug mode: off
2026-03-31 00:46:57,186 - INFO - WARNING: This is a development server. Do not use it in a production deployment. Use a production WSGI server instead.
* Running on all addresses (0.0.0.0)
* Running on http://127.0.0.1:5000
* Running on http://192.168.8.150:5000
2026-03-31 00:46:57,186 - INFO - Press CTRL+C to quit
2026-03-31 00:47:29,377 - INFO - 192.168.8.110 - - [31/Mar/2026 00:47:29] "GET / HTTP/1.1" 200 -
2026-03-31 00:47:29,420 - INFO - 192.168.8.110 - - [31/Mar/2026 00:47:29] "GET /static/style.css HTTP/1.1" 200 -
2026-03-31 00:47:29,476 - INFO - 192.168.8.110 - - [31/Mar/2026 00:47:29] "GET /favicon.ico HTTP/1.1" 404 -
Warning: OPENCODE_SERVER_PASSWORD is not set; server is unsecured.
opencode server listening on http://0.0.0.0:4096
2026-03-31 03:50:30,668 - INFO - 192.168.8.156 - - [31/Mar/2026 03:50:30] "GET / HTTP/1.1" 200 -
2026-03-31 03:50:30,701 - INFO - 192.168.8.156 - - [31/Mar/2026 03:50:30] "GET /static/style.css HTTP/1.1" 200 -
2026-03-31 03:50:34,449 - INFO - 192.168.8.156 - - [31/Mar/2026 03:50:34] "GET /source/404%20Media HTTP/1.1" 200 -
2026-03-31 03:50:34,449 - INFO - 192.168.8.156 - - [31/Mar/2026 03:50:34] "GET /source/404%20Media HTTP/1.1" 200 -
2026-03-31 03:50:34,490 - INFO - 192.168.8.156 - - [31/Mar/2026 03:50:34] "GET /static/style.css HTTP/1.1" 304 -
2026-03-31 03:51:06,671 - INFO - 192.168.8.156 - - [31/Mar/2026 03:51:06] "GET /source/404%20media/article/75854 HTTP/1.1" 200 -
2026-03-31 03:51:06,801 - INFO - 192.168.8.156 - - [31/Mar/2026 03:51:06] "GET /static/style.css HTTP/1.1" 304 -
2026-03-31 03:51:28,312 - INFO - 192.168.8.156 - - [31/Mar/2026 03:51:28] "GET /source/404%20Media HTTP/1.1" 200 -
2026-03-31 03:51:30,665 - INFO - 192.168.8.156 - - [31/Mar/2026 03:51:30] "GET /source/404%20media/article/75856 HTTP/1.1" 200 -
2026-03-31 03:51:30,686 - INFO - 192.168.8.156 - - [31/Mar/2026 03:51:30] "GET /source/404%20media/article/75856 HTTP/1.1" 200 -
2026-03-31 03:51:30,777 - INFO - 192.168.8.156 - - [31/Mar/2026 03:51:30] "GET /static/style.css HTTP/1.1" 304 -
2026-03-31 03:51:52,473 - INFO - 192.168.8.156 - - [31/Mar/2026 03:51:52] "GET /source/404%20Media HTTP/1.1" 200 -
2026-03-31 03:52:03,379 - INFO - 192.168.8.156 - - [31/Mar/2026 03:52:03] "GET / HTTP/1.1" 200 -
2026-03-31 03:53:23,713 - INFO - 192.168.8.156 - - [31/Mar/2026 03:53:23] "GET / HTTP/1.1" 200 -
2026-03-31 03:53:26,077 - INFO - 192.168.8.156 - - [31/Mar/2026 03:53:26] "GET /source/Ars%20Technica HTTP/1.1" 200 -
2026-03-31 03:53:26,273 - INFO - 192.168.8.156 - - [31/Mar/2026 03:53:26] "GET /static/style.css HTTP/1.1" 304 -
2026-03-31 03:53:32,574 - INFO - 192.168.8.156 - - [31/Mar/2026 03:53:32] "GET /source/ars%20technica/article/76159 HTTP/1.1" 200 -
2026-03-31 03:53:32,621 - INFO - 192.168.8.156 - - [31/Mar/2026 03:53:32] "GET /source/ars%20technica/article/76159 HTTP/1.1" 200 -
2026-03-31 03:53:32,714 - INFO - 192.168.8.156 - - [31/Mar/2026 03:53:32] "GET /static/style.css HTTP/1.1" 304 -
2026-03-31 03:54:32,335 - INFO - 192.168.8.156 - - [31/Mar/2026 03:54:32] "GET /source/Ars%20Technica HTTP/1.1" 200 -
2026-03-31 03:54:36,121 - INFO - 192.168.8.156 - - [31/Mar/2026 03:54:36] "GET /source/ars%20technica/article/76156 HTTP/1.1" 200 -
2026-03-31 03:54:36,226 - INFO - 192.168.8.156 - - [31/Mar/2026 03:54:36] "GET /static/style.css HTTP/1.1" 304 -
2026-03-31 03:55:37,969 - INFO - 192.168.8.156 - - [31/Mar/2026 03:55:37] "GET /source/Ars%20Technica HTTP/1.1" 200 -
2026-03-31 03:55:42,609 - INFO - 192.168.8.156 - - [31/Mar/2026 03:55:42] "GET /source/ars%20technica/article/76153 HTTP/1.1" 200 -
2026-03-31 03:55:42,688 - INFO - 192.168.8.156 - - [31/Mar/2026 03:55:42] "GET /static/style.css HTTP/1.1" 304 -
2026-03-31 03:56:57,070 - INFO - 192.168.8.156 - - [31/Mar/2026 03:56:57] "GET /source/Ars%20Technica HTTP/1.1" 200 -
2026-03-31 03:57:27,037 - INFO - 192.168.8.156 - - [31/Mar/2026 03:57:27] "GET / HTTP/1.1" 200 -
2026-03-31 03:57:31,938 - INFO - 192.168.8.156 - - [31/Mar/2026 03:57:31] "GET /source/Associated%20Press HTTP/1.1" 200 -
2026-03-31 03:57:31,977 - INFO - 192.168.8.156 - - [31/Mar/2026 03:57:31] "GET /static/style.css HTTP/1.1" 304 -
2026-03-31 03:57:46,792 - INFO - 192.168.8.156 - - [31/Mar/2026 03:57:46] "GET /source/associated%20press/article/76318 HTTP/1.1" 200 -
2026-03-31 03:57:46,822 - INFO - 192.168.8.156 - - [31/Mar/2026 03:57:46] "GET /source/associated%20press/article/76318 HTTP/1.1" 200 -
2026-03-31 03:57:46,918 - INFO - 192.168.8.156 - - [31/Mar/2026 03:57:46] "GET /static/style.css HTTP/1.1" 304 -
2026-03-31 03:58:43,823 - INFO - 192.168.8.156 - - [31/Mar/2026 03:58:43] "GET /source/Associated%20Press HTTP/1.1" 200 -
2026-03-31 03:58:45,452 - INFO - 192.168.8.156 - - [31/Mar/2026 03:58:45] "GET / HTTP/1.1" 200 -
2026-03-31 03:58:51,007 - INFO - 192.168.8.156 - - [31/Mar/2026 03:58:51] "GET /source/Associated%20Press HTTP/1.1" 200 -
2026-03-31 03:58:51,035 - INFO - 192.168.8.156 - - [31/Mar/2026 03:58:51] "GET /source/Associated%20Press HTTP/1.1" 200 -
2026-03-31 03:58:51,073 - INFO - 192.168.8.156 - - [31/Mar/2026 03:58:51] "GET /static/style.css HTTP/1.1" 304 -
2026-03-31 03:59:07,720 - INFO - 192.168.8.156 - - [31/Mar/2026 03:59:07] "GET /source/associated%20press/article/76180 HTTP/1.1" 200 -
2026-03-31 03:59:07,791 - INFO - 192.168.8.156 - - [31/Mar/2026 03:59:07] "GET /static/style.css HTTP/1.1" 304 -
2026-03-31 04:02:00,289 - INFO - 192.168.8.156 - - [31/Mar/2026 04:02:00] "GET /source/Associated%20Press HTTP/1.1" 200 -
2026-03-31 04:02:29,322 - INFO - 192.168.8.156 - - [31/Mar/2026 04:02:29] "GET /source/associated%20press/article/76381 HTTP/1.1" 200 -
2026-03-31 04:02:29,366 - INFO - 192.168.8.156 - - [31/Mar/2026 04:02:29] "GET /source/associated%20press/article/76381 HTTP/1.1" 200 -
2026-03-31 04:02:29,461 - INFO - 192.168.8.156 - - [31/Mar/2026 04:02:29] "GET /static/style.css HTTP/1.1" 304 -
2026-03-31 04:05:14,639 - INFO - 192.168.8.156 - - [31/Mar/2026 04:05:14] "GET /source/Associated%20Press HTTP/1.1" 200 -
2026-03-31 04:07:09,728 - INFO - 192.168.8.156 - - [31/Mar/2026 04:07:09] "GET / HTTP/1.1" 200 -
2026-03-31 04:07:13,436 - INFO - 192.168.8.156 - - [31/Mar/2026 04:07:13] "GET /source/BBC%20News%20–%20Business HTTP/1.1" 200 -
2026-03-31 04:07:13,460 - INFO - 192.168.8.156 - - [31/Mar/2026 04:07:13] "GET /static/style.css HTTP/1.1" 304 -
2026-03-31 04:07:18,974 - INFO - 192.168.8.156 - - [31/Mar/2026 04:07:18] "GET /source/bbc%20news%20–%20business/article/75670 HTTP/1.1" 200 -
2026-03-31 04:07:18,994 - INFO - 192.168.8.156 - - [31/Mar/2026 04:07:18] "GET /source/bbc%20news%20–%20business/article/75670 HTTP/1.1" 200 -
2026-03-31 04:07:19,090 - INFO - 192.168.8.156 - - [31/Mar/2026 04:07:19] "GET /static/style.css HTTP/1.1" 304 -
2026-03-31 04:08:32,701 - INFO - 192.168.8.156 - - [31/Mar/2026 04:08:32] "GET /source/BBC%20News%20–%20Business HTTP/1.1" 200 -
2026-03-31 04:08:38,686 - INFO - 192.168.8.156 - - [31/Mar/2026 04:08:38] "GET /source/bbc%20news%20–%20business/article/75660 HTTP/1.1" 200 -
2026-03-31 04:08:38,758 - INFO - 192.168.8.156 - - [31/Mar/2026 04:08:38] "GET /static/style.css HTTP/1.1" 304 -
2026-03-31 04:10:39,733 - INFO - 192.168.8.156 - - [31/Mar/2026 04:10:39] "GET /source/BBC%20News%20–%20Business HTTP/1.1" 200 -
2026-03-31 04:10:47,517 - INFO - 192.168.8.156 - - [31/Mar/2026 04:10:47] "GET /source/bbc%20news%20–%20business/article/75668 HTTP/1.1" 200 -
2026-03-31 04:10:47,592 - INFO - 192.168.8.156 - - [31/Mar/2026 04:10:47] "GET /static/style.css HTTP/1.1" 304 -
2026-03-31 04:13:00,033 - INFO - 192.168.8.156 - - [31/Mar/2026 04:13:00] "GET /source/BBC%20News%20–%20Business HTTP/1.1" 200 -
2026-03-31 04:13:04,210 - INFO - 192.168.8.156 - - [31/Mar/2026 04:13:04] "GET /source/bbc%20news%20–%20business/article/75664 HTTP/1.1" 200 -
2026-03-31 04:13:04,305 - INFO - 192.168.8.156 - - [31/Mar/2026 04:13:04] "GET /static/style.css HTTP/1.1" 304 -
2026-03-31 04:14:34,463 - INFO - 192.168.8.156 - - [31/Mar/2026 04:14:34] "GET /source/BBC%20News%20–%20Business HTTP/1.1" 200 -
2026-03-31 04:15:35,483 - INFO - 192.168.8.156 - - [31/Mar/2026 04:15:35] "GET / HTTP/1.1" 200 -
2026-03-31 04:15:42,939 - INFO - 192.168.8.156 - - [31/Mar/2026 04:15:42] "GET /source/CNBC%20–%20Business HTTP/1.1" 200 -
2026-03-31 04:15:42,999 - INFO - 192.168.8.156 - - [31/Mar/2026 04:15:42] "GET /source/CNBC%20–%20Business HTTP/1.1" 200 -
2026-03-31 04:15:43,017 - INFO - 192.168.8.156 - - [31/Mar/2026 04:15:43] "GET /static/style.css HTTP/1.1" 304 -
2026-03-31 04:16:17,658 - INFO - 192.168.8.156 - - [31/Mar/2026 04:16:17] "GET /source/cnbc%20–%20business/article/75575 HTTP/1.1" 200 -
2026-03-31 04:16:17,754 - INFO - 192.168.8.156 - - [31/Mar/2026 04:16:17] "GET /static/style.css HTTP/1.1" 304 -
2026-03-31 04:17:16,331 - INFO - 192.168.8.156 - - [31/Mar/2026 04:17:16] "GET /source/CNBC%20–%20Business HTTP/1.1" 200 -
2026-03-31 04:17:34,558 - INFO - 192.168.8.156 - - [31/Mar/2026 04:17:34] "GET /source/cnbc%20–%20business/article/75583 HTTP/1.1" 200 -
2026-03-31 04:17:34,633 - INFO - 192.168.8.156 - - [31/Mar/2026 04:17:34] "GET /static/style.css HTTP/1.1" 304 -
2026-03-31 04:19:53,532 - INFO - 192.168.8.156 - - [31/Mar/2026 04:19:53] "GET /source/CNBC%20–%20Business HTTP/1.1" 200 -
2026-03-31 04:20:00,608 - INFO - 192.168.8.156 - - [31/Mar/2026 04:20:00] "GET / HTTP/1.1" 200 -
2026-03-31 04:20:15,359 - INFO - 192.168.8.156 - - [31/Mar/2026 04:20:15] "GET /source/Engadget HTTP/1.1" 200 -
2026-03-31 04:20:15,402 - INFO - 192.168.8.156 - - [31/Mar/2026 04:20:15] "GET /static/style.css HTTP/1.1" 304 -
2026-03-31 04:20:37,074 - INFO - 192.168.8.156 - - [31/Mar/2026 04:20:37] "GET /source/engadget/article/76118 HTTP/1.1" 200 -
2026-03-31 04:20:37,105 - INFO - 192.168.8.156 - - [31/Mar/2026 04:20:37] "GET /source/engadget/article/76118 HTTP/1.1" 200 -
2026-03-31 04:20:37,202 - INFO - 192.168.8.156 - - [31/Mar/2026 04:20:37] "GET /static/style.css HTTP/1.1" 304 -
2026-03-31 04:21:11,132 - INFO - 192.168.8.156 - - [31/Mar/2026 04:21:11] "GET /source/Engadget HTTP/1.1" 200 -
2026-03-31 04:21:18,145 - INFO - 192.168.8.156 - - [31/Mar/2026 04:21:18] "GET /source/engadget/article/76116 HTTP/1.1" 200 -
2026-03-31 04:21:18,216 - INFO - 192.168.8.156 - - [31/Mar/2026 04:21:18] "GET /static/style.css HTTP/1.1" 304 -
2026-03-31 04:22:17,699 - INFO - 192.168.8.156 - - [31/Mar/2026 04:22:17] "GET /source/Engadget HTTP/1.1" 200 -
2026-03-31 04:22:38,033 - INFO - 192.168.8.156 - - [31/Mar/2026 04:22:38] "GET /source/engadget/article/76107 HTTP/1.1" 200 -
2026-03-31 04:22:38,059 - INFO - 192.168.8.156 - - [31/Mar/2026 04:22:38] "GET /source/engadget/article/76107 HTTP/1.1" 200 -
2026-03-31 04:22:38,151 - INFO - 192.168.8.156 - - [31/Mar/2026 04:22:38] "GET /static/style.css HTTP/1.1" 304 -
2026-03-31 04:23:11,496 - INFO - 192.168.8.156 - - [31/Mar/2026 04:23:11] "GET /source/Engadget HTTP/1.1" 200 -
2026-03-31 04:24:04,059 - INFO - 192.168.8.156 - - [31/Mar/2026 04:24:04] "GET / HTTP/1.1" 200 -
2026-03-31 04:24:20,270 - INFO - 192.168.8.156 - - [31/Mar/2026 04:24:20] "GET /source/Hacker%20News HTTP/1.1" 200 -
2026-03-31 04:24:20,304 - INFO - 192.168.8.156 - - [31/Mar/2026 04:24:20] "GET /static/style.css HTTP/1.1" 304 -
2026-03-31 04:24:20,394 - INFO - 192.168.8.156 - - [31/Mar/2026 04:24:20] "GET /source/Hacker%20News HTTP/1.1" 200 -
2026-03-31 04:24:55,634 - INFO - 192.168.8.156 - - [31/Mar/2026 04:24:55] "GET /source/hacker%20news/article/76044 HTTP/1.1" 200 -
2026-03-31 04:24:55,731 - INFO - 192.168.8.156 - - [31/Mar/2026 04:24:55] "GET /static/style.css HTTP/1.1" 304 -
2026-03-31 04:25:39,898 - INFO - 192.168.8.156 - - [31/Mar/2026 04:25:39] "GET /source/Hacker%20News HTTP/1.1" 200 -
2026-03-31 04:25:57,410 - INFO - 192.168.8.156 - - [31/Mar/2026 04:25:57] "GET /source/hacker%20news/article/75265 HTTP/1.1" 200 -
2026-03-31 04:25:57,443 - INFO - 192.168.8.156 - - [31/Mar/2026 04:25:57] "GET /source/hacker%20news/article/75265 HTTP/1.1" 200 -
2026-03-31 04:25:57,537 - INFO - 192.168.8.156 - - [31/Mar/2026 04:25:57] "GET /static/style.css HTTP/1.1" 304 -
2026-03-31 04:28:52,915 - INFO - 192.168.8.156 - - [31/Mar/2026 04:28:52] "GET /source/Hacker%20News HTTP/1.1" 200 -
2026-03-31 04:29:54,036 - INFO - 192.168.8.156 - - [31/Mar/2026 04:29:54] "GET /source/hacker%20news/article/75237 HTTP/1.1" 200 -
2026-03-31 04:29:54,091 - INFO - 192.168.8.156 - - [31/Mar/2026 04:29:54] "GET /static/style.css HTTP/1.1" 304 -
2026-03-31 04:31:33,988 - INFO - 192.168.8.156 - - [31/Mar/2026 04:31:33] "GET /source/Hacker%20News HTTP/1.1" 200 -
2026-03-31 04:32:13,621 - INFO - 192.168.8.156 - - [31/Mar/2026 04:32:13] "GET /source/hacker%20news/article/75972 HTTP/1.1" 200 -
2026-03-31 04:32:13,653 - INFO - 192.168.8.156 - - [31/Mar/2026 04:32:13] "GET /source/hacker%20news/article/75972 HTTP/1.1" 200 -
2026-03-31 04:32:13,754 - INFO - 192.168.8.156 - - [31/Mar/2026 04:32:13] "GET /static/style.css HTTP/1.1" 304 -
2026-03-31 04:39:50,815 - INFO - 192.168.8.156 - - [31/Mar/2026 04:39:50] "GET /source/Hacker%20News HTTP/1.1" 200 -
2026-03-31 04:41:06,097 - INFO - 192.168.8.156 - - [31/Mar/2026 04:41:06] "GET /source/hacker%20news/article/75255 HTTP/1.1" 200 -
2026-03-31 04:41:06,130 - INFO - 192.168.8.156 - - [31/Mar/2026 04:41:06] "GET /source/hacker%20news/article/75255 HTTP/1.1" 200 -
2026-03-31 04:41:06,224 - INFO - 192.168.8.156 - - [31/Mar/2026 04:41:06] "GET /static/style.css HTTP/1.1" 304 -
2026-03-31 04:42:02,455 - INFO - 192.168.8.156 - - [31/Mar/2026 04:42:02] "GET /source/Hacker%20News HTTP/1.1" 200 -
2026-03-31 04:47:45,853 - INFO - 192.168.8.226 - - [31/Mar/2026 04:47:45] "GET / HTTP/1.1" 200 -
2026-03-31 04:47:45,949 - INFO - 192.168.8.226 - - [31/Mar/2026 04:47:45] "GET /static/style.css HTTP/1.1" 304 -
2026-03-31 04:47:46,000 - INFO - 192.168.8.226 - - [31/Mar/2026 04:47:46] "GET /favicon.ico HTTP/1.1" 404 -
2026-03-31 05:53:55,185 - INFO - 192.168.8.226 - - [31/Mar/2026 05:53:55] "GET /rss HTTP/1.1" 200 -
2026-03-31 07:54:24,903 - INFO - 192.168.8.226 - - [31/Mar/2026 07:54:24] "GET /rss HTTP/1.1" 200 -
2026-03-31 11:25:27,600 - INFO - 192.168.8.156 - - [31/Mar/2026 11:25:27] "GET /source/hacker%20news/article/74862 HTTP/1.1" 200 -
2026-03-31 11:25:27,641 - INFO - 192.168.8.156 - - [31/Mar/2026 11:25:27] "GET /static/style.css HTTP/1.1" 304 -
2026-03-31 11:27:05,669 - INFO - 192.168.8.156 - - [31/Mar/2026 11:27:05] "GET /source/Hacker%20News HTTP/1.1" 200 -
2026-03-31 11:29:15,410 - INFO - 192.168.8.156 - - [31/Mar/2026 11:29:15] "GET /source/hacker%20news/article/76601 HTTP/1.1" 200 -
2026-03-31 11:29:15,439 - INFO - 192.168.8.156 - - [31/Mar/2026 11:29:15] "GET /source/hacker%20news/article/76601 HTTP/1.1" 200 -
2026-03-31 11:29:15,537 - INFO - 192.168.8.156 - - [31/Mar/2026 11:29:15] "GET /static/style.css HTTP/1.1" 304 -
2026-03-31 11:30:37,869 - INFO - 192.168.8.156 - - [31/Mar/2026 11:30:37] "GET /source/Hacker%20News HTTP/1.1" 200 -
2026-03-31 11:30:46,321 - INFO - 192.168.8.156 - - [31/Mar/2026 11:30:46] "GET /source/hacker%20news/article/76602 HTTP/1.1" 200 -
2026-03-31 11:30:46,418 - INFO - 192.168.8.156 - - [31/Mar/2026 11:30:46] "GET /static/style.css HTTP/1.1" 304 -
2026-03-31 11:36:40,535 - INFO - 192.168.8.156 - - [31/Mar/2026 11:36:40] "GET / HTTP/1.1" 200 -
2026-03-31 11:36:40,569 - INFO - 192.168.8.156 - - [31/Mar/2026 11:36:40] "GET /static/style.css HTTP/1.1" 200 -
2026-03-31 11:36:44,152 - INFO - 192.168.8.156 - - [31/Mar/2026 11:36:44] "GET /source/404%20Media HTTP/1.1" 200 -
2026-03-31 11:36:44,167 - INFO - 192.168.8.156 - - [31/Mar/2026 11:36:44] "GET /static/style.css HTTP/1.1" 304 -
2026-03-31 11:36:49,524 - INFO - 192.168.8.156 - - [31/Mar/2026 11:36:49] "GET / HTTP/1.1" 200 -
2026-03-31 11:36:51,785 - INFO - 192.168.8.156 - - [31/Mar/2026 11:36:51] "GET /source/Ars%20Technica HTTP/1.1" 200 -
2026-03-31 11:36:51,803 - INFO - 192.168.8.156 - - [31/Mar/2026 11:36:51] "GET /static/style.css HTTP/1.1" 304 -
2026-03-31 11:37:01,762 - INFO - 192.168.8.156 - - [31/Mar/2026 11:37:01] "GET / HTTP/1.1" 200 -
2026-03-31 11:37:07,388 - INFO - 192.168.8.156 - - [31/Mar/2026 11:37:07] "GET /source/Associated%20Press HTTP/1.1" 200 -
2026-03-31 11:37:07,413 - INFO - 192.168.8.156 - - [31/Mar/2026 11:37:07] "GET /static/style.css HTTP/1.1" 304 -
2026-03-31 11:37:07,652 - INFO - 192.168.8.156 - - [31/Mar/2026 11:37:07] "GET /source/associated%20press/article/76682 HTTP/1.1" 200 -
2026-03-31 11:37:07,720 - INFO - 192.168.8.156 - - [31/Mar/2026 11:37:07] "GET /static/style.css HTTP/1.1" 304 -
2026-03-31 11:37:16,777 - INFO - 192.168.8.156 - - [31/Mar/2026 11:37:16] "GET /source/Associated%20Press HTTP/1.1" 200 -
2026-03-31 11:38:13,418 - INFO - 192.168.8.156 - - [31/Mar/2026 11:38:13] "GET /source/associated%20press/article/76643 HTTP/1.1" 200 -
2026-03-31 11:38:13,453 - INFO - 192.168.8.156 - - [31/Mar/2026 11:38:13] "GET /source/associated%20press/article/76643 HTTP/1.1" 200 -
2026-03-31 11:38:13,537 - INFO - 192.168.8.156 - - [31/Mar/2026 11:38:13] "GET /static/style.css HTTP/1.1" 304 -
2026-03-31 11:39:22,876 - INFO - 192.168.8.156 - - [31/Mar/2026 11:39:22] "GET /source/Associated%20Press HTTP/1.1" 200 -
2026-03-31 11:39:47,385 - INFO - 192.168.8.156 - - [31/Mar/2026 11:39:47] "GET /source/associated%20press/article/76607 HTTP/1.1" 200 -
2026-03-31 11:39:47,419 - INFO - 192.168.8.156 - - [31/Mar/2026 11:39:47] "GET /source/associated%20press/article/76607 HTTP/1.1" 200 -
2026-03-31 11:39:47,501 - INFO - 192.168.8.156 - - [31/Mar/2026 11:39:47] "GET /static/style.css HTTP/1.1" 304 -
2026-03-31 11:41:43,881 - INFO - 192.168.8.156 - - [31/Mar/2026 11:41:43] "GET /source/Associated%20Press HTTP/1.1" 200 -
2026-03-31 11:42:06,103 - INFO - 192.168.8.156 - - [31/Mar/2026 11:42:06] "GET /source/associated%20press/article/76563 HTTP/1.1" 200 -
2026-03-31 11:42:06,136 - INFO - 192.168.8.156 - - [31/Mar/2026 11:42:06] "GET /source/associated%20press/article/76563 HTTP/1.1" 200 -
2026-03-31 11:42:06,218 - INFO - 192.168.8.156 - - [31/Mar/2026 11:42:06] "GET /static/style.css HTTP/1.1" 304 -
2026-03-31 11:45:00,824 - INFO - 192.168.8.156 - - [31/Mar/2026 11:45:00] "GET /source/Associated%20Press HTTP/1.1" 200 -
2026-03-31 11:45:09,188 - INFO - 192.168.8.156 - - [31/Mar/2026 11:45:09] "GET /source/associated%20press/article/76605 HTTP/1.1" 200 -
2026-03-31 11:45:09,223 - INFO - 192.168.8.156 - - [31/Mar/2026 11:45:09] "GET /source/associated%20press/article/76605 HTTP/1.1" 200 -
2026-03-31 11:45:09,305 - INFO - 192.168.8.156 - - [31/Mar/2026 11:45:09] "GET /static/style.css HTTP/1.1" 304 -
2026-03-31 11:48:11,110 - INFO - 192.168.8.156 - - [31/Mar/2026 11:48:11] "GET /source/Associated%20Press HTTP/1.1" 200 -
2026-03-31 11:48:12,152 - INFO - 192.168.8.156 - - [31/Mar/2026 11:48:12] "GET / HTTP/1.1" 200 -
2026-03-31 11:48:16,690 - INFO - 192.168.8.156 - - [31/Mar/2026 11:48:16] "GET /source/BBC%20News%20–%20Business HTTP/1.1" 200 -
2026-03-31 11:48:16,713 - INFO - 192.168.8.156 - - [31/Mar/2026 11:48:16] "GET /source/BBC%20News%20–%20Business HTTP/1.1" 200 -
2026-03-31 11:48:16,724 - INFO - 192.168.8.156 - - [31/Mar/2026 11:48:16] "GET /static/style.css HTTP/1.1" 304 -
2026-03-31 11:48:26,509 - INFO - 192.168.8.156 - - [31/Mar/2026 11:48:26] "GET /source/bbc%20news%20–%20business/article/76588 HTTP/1.1" 200 -
2026-03-31 11:48:26,592 - INFO - 192.168.8.156 - - [31/Mar/2026 11:48:26] "GET /static/style.css HTTP/1.1" 304 -
2026-03-31 11:50:33,771 - INFO - 192.168.8.156 - - [31/Mar/2026 11:50:33] "GET /source/BBC%20News%20–%20Business HTTP/1.1" 200 -
2026-03-31 11:51:20,160 - INFO - 192.168.8.156 - - [31/Mar/2026 11:51:20] "GET / HTTP/1.1" 200 -
2026-03-31 11:51:54,395 - INFO - 192.168.8.156 - - [31/Mar/2026 11:51:54] "GET /source/Mac%20Rumors HTTP/1.1" 200 -
2026-03-31 11:51:54,422 - INFO - 192.168.8.156 - - [31/Mar/2026 11:51:54] "GET /static/style.css HTTP/1.1" 304 -
2026-03-31 11:51:59,100 - INFO - 192.168.8.156 - - [31/Mar/2026 11:51:59] "GET /source/mac%20rumors/article/75894 HTTP/1.1" 200 -
2026-03-31 11:51:59,196 - INFO - 192.168.8.156 - - [31/Mar/2026 11:51:59] "GET /static/style.css HTTP/1.1" 304 -
2026-03-31 11:52:42,557 - INFO - 192.168.8.156 - - [31/Mar/2026 11:52:42] "GET /source/Mac%20Rumors HTTP/1.1" 200 -
2026-03-31 11:52:48,061 - INFO - 192.168.8.156 - - [31/Mar/2026 11:52:48] "GET /source/mac%20rumors/article/75890 HTTP/1.1" 200 -
2026-03-31 11:52:48,089 - INFO - 192.168.8.156 - - [31/Mar/2026 11:52:48] "GET /source/mac%20rumors/article/75890 HTTP/1.1" 200 -
2026-03-31 11:52:48,176 - INFO - 192.168.8.156 - - [31/Mar/2026 11:52:48] "GET /static/style.css HTTP/1.1" 304 -
2026-03-31 11:53:19,210 - INFO - 192.168.8.156 - - [31/Mar/2026 11:53:19] "GET /source/Mac%20Rumors HTTP/1.1" 200 -
2026-03-31 11:53:43,451 - INFO - 192.168.8.156 - - [31/Mar/2026 11:53:43] "GET /source/mac%20rumors/article/75886 HTTP/1.1" 200 -
2026-03-31 11:53:43,487 - INFO - 192.168.8.156 - - [31/Mar/2026 11:53:43] "GET /source/mac%20rumors/article/75886 HTTP/1.1" 200 -
2026-03-31 11:53:43,570 - INFO - 192.168.8.156 - - [31/Mar/2026 11:53:43] "GET /static/style.css HTTP/1.1" 304 -
2026-03-31 11:54:20,607 - INFO - 192.168.8.156 - - [31/Mar/2026 11:54:20] "GET /source/Mac%20Rumors HTTP/1.1" 200 -
2026-03-31 11:55:28,281 - INFO - 192.168.8.156 - - [31/Mar/2026 11:55:28] "GET /source/mac%20rumors/article/75896 HTTP/1.1" 200 -
2026-03-31 11:55:28,311 - INFO - 192.168.8.156 - - [31/Mar/2026 11:55:28] "GET /source/mac%20rumors/article/75896 HTTP/1.1" 200 -
2026-03-31 11:55:28,396 - INFO - 192.168.8.156 - - [31/Mar/2026 11:55:28] "GET /static/style.css HTTP/1.1" 304 -
2026-03-31 11:55:52,802 - INFO - 192.168.8.156 - - [31/Mar/2026 11:55:52] "GET /source/Mac%20Rumors HTTP/1.1" 200 -
2026-03-31 11:56:08,058 - INFO - 192.168.8.156 - - [31/Mar/2026 11:56:08] "GET / HTTP/1.1" 200 -
2026-03-31 11:56:31,521 - INFO - 192.168.8.156 - - [31/Mar/2026 11:56:31] "GET /source/ProPublica HTTP/1.1" 200 -
2026-03-31 11:56:31,524 - INFO - 192.168.8.156 - - [31/Mar/2026 11:56:31] "GET /source/ProPublica HTTP/1.1" 200 -
2026-03-31 11:56:31,543 - INFO - 192.168.8.156 - - [31/Mar/2026 11:56:31] "GET /static/style.css HTTP/1.1" 304 -
2026-03-31 11:56:44,566 - INFO - 192.168.8.156 - - [31/Mar/2026 11:56:44] "GET /source/propublica/article/76487 HTTP/1.1" 200 -
2026-03-31 11:56:44,655 - INFO - 192.168.8.156 - - [31/Mar/2026 11:56:44] "GET /static/style.css HTTP/1.1" 304 -
2026-03-31 12:07:00,226 - INFO - 192.168.8.156 - - [31/Mar/2026 12:07:00] "GET /source/ProPublica HTTP/1.1" 200 -
2026-03-31 12:08:20,255 - INFO - 192.168.8.156 - - [31/Mar/2026 12:08:20] "GET /source/propublica/article/61757 HTTP/1.1" 200 -
2026-03-31 12:08:20,285 - INFO - 192.168.8.156 - - [31/Mar/2026 12:08:20] "GET /source/propublica/article/61757 HTTP/1.1" 200 -
2026-03-31 12:08:20,371 - INFO - 192.168.8.156 - - [31/Mar/2026 12:08:20] "GET /static/style.css HTTP/1.1" 304 -
2026-03-31 12:16:03,370 - INFO - 192.168.8.226 - - [31/Mar/2026 12:16:03] "GET / HTTP/1.1" 200 -
2026-03-31 12:16:03,475 - INFO - 192.168.8.226 - - [31/Mar/2026 12:16:03] "GET /static/style.css HTTP/1.1" 304 -
2026-03-31 13:05:25,475 - INFO - 192.168.8.226 - - [31/Mar/2026 13:05:25] "GET /rss HTTP/1.1" 200 -

257
rebuild_database.py Normal file
View File

@ -0,0 +1,257 @@
#!/usr/bin/env python3
"""Rebuild NewsArchiver database from existing HTML files."""
import json
import logging
import re
import sqlite3
import sys
from datetime import datetime
from pathlib import Path
from typing import Optional
try:
from content_extractor import parse_article_from_html, get_html_from_url
from storage_manager import initialize_storage, save_article as storage_save_article
except ImportError as e:
print(f"ERROR: Required module not found: {e}")
sys.exit(1)
SCRIPT_DIR = Path(__file__).parent
ARCHIVE_DIR = SCRIPT_DIR / 'archival_data'
RSS_FEEDS_PATH = SCRIPT_DIR / 'rss_feeds.json'
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(levelname)s - %(message)s',
handlers=[
logging.StreamHandler(sys.stdout),
logging.FileHandler(ARCHIVE_DIR / 'processing.log', encoding='utf-8')
]
)
logger = logging.getLogger(__name__)
def extract_urls_from_html(html_file: Path) -> list:
"""Extract article URLs from an archived HTML file.
Args:
html_file: Path to archived HTML file
Returns:
List of article URLs found in the file
"""
try:
content = html_file.read_text(encoding='utf-8')
urls = []
import re
# Only extract main article URLs - skip tracking links, RSS feeds, author pages, etc.
article_patterns = [
r'href=["\']https?://[^"\']+\/article\/[^"\']+["\']',
r'href=["\']https?://[^"\']+\/news\/[^"\']+["\']',
r'href=["\']https?://[^"\']+\/stories\/[^"\']+["\']',
r'href=["\']https?://[^"\']+\/archive\/[^"\']+["\']',
]
for pattern in article_patterns:
matches = re.findall(pattern, content)
for match in matches:
url_match = re.search(r'href=["\']([^"\']+)["\']', match)
if url_match:
url = url_match.group(1)
# Skip common non-article URLs
skip_patterns = [
'rss', 'feed', 'author', 'authors', 'tags', 'category', 'search',
'about', 'contact', 'privacy', 'terms', 'faq', 'subscribe',
'signin', 'login', 'register', 'account', 'profile'
]
if not any(skip in url.lower() for skip in skip_patterns):
urls.append(url)
return list(set(urls)) # Remove duplicates
except Exception as e:
logger.error("Error reading %s: %s", html_file, str(e))
return []
def find_html_files(archive_dir: Path) -> list:
"""Find all HTML files in the archive directory.
Args:
archive_dir: Root archive directory
Returns:
List of HTML file paths
"""
html_files = []
try:
websites_dir = archive_dir / 'websites'
if not websites_dir.exists():
logger.warning("Websites directory not found: %s", websites_dir)
return []
for source_dir in sorted(websites_dir.iterdir()):
if not source_dir.is_dir():
continue
html_dir = source_dir / 'html'
if not html_dir.exists():
continue
for date_dir in sorted(html_dir.iterdir()):
if not date_dir.is_dir():
continue
for html_file in sorted(date_dir.glob('article_*.html')):
html_files.append(html_file)
logger.info("Found %d HTML files to process", len(html_files))
except Exception as e:
logger.error("Error finding HTML files: %s", str(e))
return html_files
def rebuild_database(archive_dir: Path, rss_feeds_path: Path) -> dict:
"""Rebuild the database from existing HTML files.
Args:
archive_dir: Root archive directory
rss_feeds_path: Path to RSS feeds JSON file
Returns:
Dictionary with rebuild results
"""
results = {
'html_files_processed': 0,
'articles_extracted': 0,
'articles_saved': 0,
'articles_failed': 0,
'errors': []
}
# Initialize database
logger.info("Initializing database at: %s", ARCHIVE_DIR / 'cache.db')
initialize_storage()
logger.info("Database initialized")
# Load RSS feeds
rss_feeds = {}
if rss_feeds_path.exists():
with open(rss_feeds_path, 'r', encoding='utf-8') as f:
rss_feeds = json.load(f)
# Find all HTML files
html_files = find_html_files(archive_dir)
if not html_files:
logger.warning("No HTML files found to process")
return results
# Process each HTML file
for html_file in html_files:
try:
results['html_files_processed'] += 1
# Extract URLs from HTML
urls = extract_urls_from_html(html_file)
if not urls:
logger.debug("No URLs found in %s", html_file.name)
continue
results['articles_extracted'] += len(urls)
# Process each URL
for url in urls:
try:
# Try to get source name from RSS feeds
source_name = None
for feed_name, feed_info in rss_feeds.items():
if feed_info.get('rss_url') and url.startswith(feed_info.get('rss_url', '')):
source_name = feed_name
break
# Try to infer source from URL
if not source_name:
for feed_name, feed_info in rss_feeds.items():
website = feed_info.get('source_website', '')
if website and website in url:
source_name = feed_name
break
if not source_name:
# Try to extract domain from URL
domain_match = re.search(r'https?://([^/]+)', url)
if domain_match:
domain = domain_match.group(1)
for feed_name, feed_info in rss_feeds.items():
website = feed_info.get('source_website', '')
if website and website in domain:
source_name = feed_name
break
# If still no source, skip
if not source_name:
logger.debug("No source found for %s, skipping", url[:60])
continue
# Extract content from HTML file
raw_html = html_file.read_text(encoding='utf-8')
article_data = parse_article_from_html(raw_html, url)
if article_data.error:
logger.warning("Failed to extract content from %s: %s", url[:60], article_data.error)
results['articles_failed'] += 1
continue
# Save to database
logger.info("Saving article: %s -> %s", url[:80], source_name)
storage_save_article(source_name, article_data)
results['articles_saved'] += 1
if results['articles_saved'] % 100 == 0:
logger.info("Saved %d articles so far", results['articles_saved'])
except Exception as e:
logger.error("Error processing URL %s: %s", url, str(e))
results['articles_failed'] += 1
results['errors'].append({
'url': url,
'error': str(e)
})
except Exception as e:
logger.error("Error processing file %s: %s", html_file, str(e))
results['errors'].append({
'file': str(html_file),
'error': str(e)
})
return results
if __name__ == '__main__':
logger.info("=" * 60)
logger.info("Rebuilding NewsArchiver Database")
logger.info("=" * 60)
results = rebuild_database(ARCHIVE_DIR, RSS_FEEDS_PATH)
logger.info("=" * 60)
logger.info("Rebuild Complete")
logger.info("=" * 60)
logger.info("HTML files processed: %d", results['html_files_processed'])
logger.info("Articles extracted: %d", results['articles_extracted'])
logger.info("Articles saved: %d", results['articles_saved'])
logger.info("Articles failed: %d", results['articles_failed'])
if results['errors']:
logger.info("Errors:")
for error in results['errors'][:20]:
logger.info(" - %s", error)

2428
rebuild_log.txt Normal file
View File

File diff suppressed because it is too large Load Diff

4548
rebuild_log2.txt Normal file
View File

File diff suppressed because it is too large Load Diff

112
restore_database.py Normal file
View File

@ -0,0 +1,112 @@
#!/usr/bin/env python3
"""Restore database from existing JSON metadata files."""
import json
import logging
import sys
from pathlib import Path
try:
from storage_manager import initialize_storage, save_article
from content_extractor import ArticleData
except ImportError as e:
print(f"ERROR: Required module not found: {e}")
sys.exit(1)
SCRIPT_DIR = Path(__file__).parent
ARCHIVE_DIR = SCRIPT_DIR / 'archival_data'
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(levelname)s - %(message)s',
handlers=[
logging.StreamHandler(sys.stdout),
logging.FileHandler(ARCHIVE_DIR / 'restore.log', encoding='utf-8')
]
)
logger = logging.getLogger(__name__)
def restore_database(archive_dir: Path) -> dict:
"""Restore database from JSON metadata files."""
results = {
'json_files_found': 0,
'articles_restored': 0,
'articles_failed': 0,
'errors': []
}
initialize_storage()
logger.info("Database initialized")
json_files = list(archive_dir.glob('websites/**/*.json'))
results['json_files_found'] = len(json_files)
logger.info(f"Found {len(json_files)} JSON files to process")
for json_file in json_files:
try:
with open(json_file, 'r', encoding='utf-8') as f:
metadata = json.load(f)
url = metadata.get('url')
source_name = metadata.get('source_name')
title = metadata.get('title')
author = metadata.get('author')
publish_date = metadata.get('publish_date')
content_text = metadata.get('content_text')
content_html = metadata.get('content_html')
tags = metadata.get('tags', [])
extraction_method = metadata.get('extraction_method', 'unknown')
if not url or not source_name:
logger.warning(f"Missing URL or source in {json_file.name}, skipping")
results['articles_failed'] += 1
continue
article_data = ArticleData(
url=url,
title=title,
author=author,
publish_date=publish_date,
content_text=content_text,
content_html=content_html,
tags=tags,
extraction_method=extraction_method
)
save_article(source_name, article_data)
results['articles_restored'] += 1
if results['articles_restored'] % 100 == 0:
logger.info(f"Restored {results['articles_restored']} articles so far")
except Exception as e:
logger.error(f"Error processing {json_file.name}: {str(e)}")
results['articles_failed'] += 1
results['errors'].append({
'file': str(json_file),
'error': str(e)
})
return results
if __name__ == '__main__':
logger.info("=" * 60)
logger.info("Restoring NewsArchiver Database from JSON files")
logger.info("=" * 60)
results = restore_database(ARCHIVE_DIR)
logger.info("=" * 60)
logger.info("Restore Complete")
logger.info("=" * 60)
logger.info(f"JSON files found: {results['json_files_found']}")
logger.info(f"Articles restored: {results['articles_restored']}")
logger.info(f"Articles failed: {results['articles_failed']}")
if results['errors']:
logger.info("Errors:")
for error in results['errors'][:20]:
logger.info(f" - {error}")

306
rss_feeds.json Normal file
View File

@ -0,0 +1,306 @@
{
"Financial Times": {
"source_website": "ft.com",
"rss_url": "https://www.ft.com/rss/home",
"entries": 12,
"validated_at": "2026-03-18T20:21:46.292164"
},
"Reuters – Business News": {
"source_website": "reuters.com",
"rss_url": "https://news.google.com/rss/search?q=site:reuters.com+business&hl=en-US&gl=US&ceid=US:en",
"disabled": true,
"disable_reason": "Google News RSS only provides encrypted URLs that don't work when accessed directly. Reuters does not provide public RSS feeds.",
"entries": 100,
"validated_at": "2026-03-19T15:48:00.000000"
},
"Fortune – Top Stories": {
"source_website": "fortune.com",
"rss_url": "https://fortune.com/feed/fortune-feeds/?id=3230629",
"entries": 10,
"validated_at": "2026-03-18T20:21:46.869680"
},
"Seeking Alpha – Market News": {
"source_website": "seekingalpha.com",
"rss_url": "https://seekingalpha.com/feed.xml",
"entries": 30,
"validated_at": "2026-03-18T20:21:47.462673"
},
"The Motley Fool – Stock News & Analysis": {
"source_website": "fool.com",
"rss_url": "https://www.fool.com/a/feeds/partner/googlechromefollow?apikey=5e092c1f-c5f9-4428-9219-908a47d2e2de",
"entries": 50,
"validated_at": "2026-03-18T20:21:48.479725"
},
"TheStreet – Full Articles": {
"source_website": "thestreet.com",
"rss_url": "https://www.thestreet.com/.rss/full",
"entries": 50,
"validated_at": "2026-03-18T20:21:50.429528"
},
"MarketBeat – Market News": {
"source_website": "marketbeat.com",
"rss_url": "https://www.marketbeat.com/feed/",
"entries": 100,
"validated_at": "2026-03-18T20:22:06.522327"
},
"Money (Time) – Personal Finance": {
"source_website": "money.com",
"rss_url": "https://money.com/money/feed/",
"entries": 10,
"validated_at": "2026-03-18T20:22:07.133884"
},
"Global Finance Magazine": {
"source_website": "gfmag.com",
"rss_url": "https://www.gfmag.com/feed",
"entries": 10,
"validated_at": "2026-03-18T20:22:08.230178"
},
"Financial Samurai": {
"source_website": "financialsamurai.com",
"rss_url": "https://www.financialsamurai.com/feed/",
"entries": 7,
"validated_at": "2026-03-18T20:22:09.173814"
},
"MoneyWeek": {
"source_website": "moneyweek.com",
"rss_url": "https://moneyweek.com/feed/all",
"entries": 50,
"validated_at": "2026-03-18T20:22:10.001295"
},
"Finance Monthly": {
"source_website": "finance-monthly.com",
"rss_url": "https://www.finance-monthly.com/feed/",
"entries": 45,
"validated_at": "2026-03-18T20:22:11.526262"
},
"European Financial Review": {
"source_website": "europeanfinancialreview.com",
"rss_url": "https://www.europeanfinancialreview.com/feed",
"entries": 10,
"validated_at": "2026-03-18T20:22:13.344671"
},
"World Finance": {
"source_website": "worldfinance.com",
"rss_url": "https://www.worldfinance.com/feed",
"entries": 10,
"validated_at": "2026-03-18T20:22:16.414742"
},
"Fox Business – Headlines": {
"source_website": "foxbusiness.com",
"rss_url": "https://moxie.foxbusiness.com/google-publisher/latest.xml",
"entries": 25,
"validated_at": "2026-03-18T20:22:22.238519"
},
"FinanceAsia": {
"source_website": "financeasia.com",
"rss_url": "https://www.financeasia.com/rss/latest",
"entries": 10,
"validated_at": "2026-03-18T20:22:23.798531"
},
"CNBC – Business": {
"source_website": "cnbc.com",
"rss_url": "https://www.cnbc.com/id/100003114/device/rss/rss.html",
"entries": 30,
"validated_at": "2026-03-18T20:22:24.453119"
},
"Markets Insider": {
"source_website": "markets.businessinsider.com",
"rss_url": "https://markets.businessinsider.com/rss/news",
"entries": 10,
"validated_at": "2026-03-18T20:22:29.272544"
},
"The Economist – Business & Finance": {
"source_website": "economist.com",
"rss_url": "https://www.economist.com/business/rss.xml",
"entries": 300,
"validated_at": "2026-03-18T20:22:30.110716"
},
"Barchart News": {
"source_website": "barchart.com",
"rss_url": "http://feeds.feedburner.com/BarchartNews",
"entries": 15,
"validated_at": "2026-03-18T20:22:30.831390"
},
"The Guardian – Business": {
"source_website": "theguardian.com",
"rss_url": "http://feeds.theguardian.com/theguardian/uk/business/rss",
"entries": 40,
"validated_at": "2026-03-18T20:22:32.033297"
},
"Economy Watch": {
"source_website": "economywatch.com",
"rss_url": "https://www.economywatch.com/feed",
"entries": 10,
"validated_at": "2026-03-18T20:22:33.008150"
},
"CFI.co": {
"source_website": "cfi.co",
"rss_url": "https://cfi.co/feed",
"entries": 20,
"validated_at": "2026-03-18T20:22:35.620736"
},
"BBC News – Business": {
"source_website": "bbc.co.uk",
"rss_url": "http://feeds.bbci.co.uk/news/business/rss.xml",
"entries": 56,
"validated_at": "2026-03-18T20:22:36.643323"
},
"Investor’s Business Daily": {
"source_website": "investors.com",
"rss_url": "https://www.investors.com/feed/",
"entries": 100,
"validated_at": "2026-03-18T20:22:38.264669"
},
"MarketWatch – Top Stories": {
"source_website": "marketwatch.com",
"rss_url": "http://feeds.marketwatch.com/marketwatch/topstories/",
"entries": 10,
"validated_at": "2026-03-18T20:22:45.591752"
},
"Wall Street Journal – U.S. Business": {
"source_website": "wsj.com",
"rss_url": "https://feeds.a.dj.com/rss/WSJcomUSBusiness.xml",
"entries": 20,
"validated_at": "2026-03-18T20:22:46.259913"
},
"Investing.com – News": {
"source_website": "investing.com",
"rss_url": "https://www.investing.com/rss/news.rss",
"entries": 10,
"validated_at": "2026-03-18T20:22:50.790503"
},
"International Business Times": {
"source_website": "ibtimes.com",
"rss_url": "https://www.ibtimes.com/rss",
"entries": 25,
"validated_at": "2026-03-18T20:23:13.449646"
},
"404 Media": {
"source_website": "404media.co",
"rss_url": "https://404media.co/feed/",
"entries": 30,
"validated_at": "2026-03-20T00:00:00.000000"
},
"Mac Rumors": {
"source_website": "macrumors.com",
"rss_url": "https://feeds.macrumors.com/MacRumors-All",
"entries": 50,
"validated_at": "2026-03-20T00:00:00.000000"
},
"The Verge": {
"source_website": "theverge.com",
"rss_url": "https://www.theverge.com/rss/index.xml",
"entries": 50,
"validated_at": "2026-03-20T00:00:00.000000"
},
"TechCrunch": {
"source_website": "techcrunch.com",
"rss_url": "https://techcrunch.com/feed/",
"entries": 50,
"validated_at": "2026-03-20T00:00:00.000000"
},
"WIRED": {
"source_website": "wired.com",
"rss_url": "https://www.wired.com/feed/rss",
"entries": 50,
"validated_at": "2026-03-20T00:00:00.000000"
},
"Hacker News": {
"source_website": "news.ycombinator.com",
"rss_url": "https://news.ycombinator.com/rss",
"entries": 30,
"validated_at": "2026-03-20T00:00:00.000000"
},
"ZDNet": {
"source_website": "zdnet.com",
"rss_url": "https://www.zdnet.com/news/rss.xml",
"entries": 50,
"validated_at": "2026-03-20T00:00:00.000000"
},
"Engadget": {
"source_website": "engadget.com",
"rss_url": "https://www.engadget.com/rss.xml",
"entries": 50,
"validated_at": "2026-03-20T00:00:00.000000"
},
"Ars Technica": {
"source_website": "arstechnica.com",
"rss_url": "https://arstechnica.com/feed/",
"entries": 50,
"validated_at": "2026-03-21T00:00:00.000000"
},
"Associated Press": {
"source_website": "apnews.com",
"rss_url": "https://apnews.com",
"feed_type": "html",
"entries": 100,
"validated_at": "2026-03-20T00:00:00.000000"
},
"The Hacker News": {
"source_website": "thehackernews.com",
"rss_url": "https://thehackernews.com/feeds/posts/default",
"entries": 50,
"validated_at": "2026-03-21T13:17:00+00:00"
},
"Dark Reading": {
"source_website": "darkreading.com",
"rss_url": "https://www.darkreading.com/rss.xml",
"entries": 50,
"validated_at": "2026-03-20T19:30:19+00:00"
},
"SecurityWeek": {
"source_website": "securityweek.com",
"rss_url": "https://www.securityweek.com/feed",
"entries": 10,
"validated_at": "2026-03-21T11:00:00+00:00"
},
"BleepingComputer": {
"source_website": "bleepingcomputer.com",
"rss_url": "https://www.bleepingcomputer.com/feed",
"entries": 15,
"validated_at": "2026-03-21T17:30:41+00:00"
},
"Microsoft Security Blog": {
"source_website": "microsoft.com",
"rss_url": "https://www.microsoft.com/security/blog/feed",
"entries": 10,
"validated_at": "2026-03-20T16:19:00+00:00"
},
"EFF Deeplinks": {
"source_website": "eff.org",
"rss_url": "https://www.eff.org/deeplinks.xml",
"entries": 50,
"validated_at": "2026-03-20T22:20:49+00:00"
},
"US-CISA": {
"source_website": "cisa.gov",
"rss_url": "https://www.cisa.gov/news.xml",
"entries": 10,
"validated_at": "2026-02-26T12:00:00+00:00"
},
"Google Security Blog": {
"source_website": "google.com",
"rss_url": "https://security.googleblog.com/feeds/posts/default",
"entries": 25,
"validated_at": "2026-02-27T17:01:00+00:00"
},
"Politico": {
"source_website": "politico.com",
"rss_url": "https://www.politico.com/rss/politicopicks.xml",
"entries": 50,
"validated_at": "2026-03-22T14:58:00+00:00"
},
"Cyber Security News": {
"source_website": "cybersecuritynews.com",
"rss_url": "https://cybersecuritynews.com/feed/",
"entries": 50,
"validated_at": "2026-03-23T18:47:58+00:00"
},
"ProPublica": {
"source_website": "propublica.org",
"rss_url": "https://www.propublica.org/feeds/propublica/main",
"entries": 30,
"validated_at": "2026-03-24T00:00:00.000000"
}
}

463
rss_processor.py Normal file
View File

@ -0,0 +1,463 @@
#!/usr/bin/env python3
"""RSS Feed Processor for NewsArchiver - Phase 2.1"""
import argparse
import json
import logging
import sys
import time
import urllib.error
import urllib.request
from datetime import datetime
from pathlib import Path
from typing import List, Optional
try:
import feedparser
from feedparser import FeedParserDict
except ImportError:
print("ERROR: feedparser is required. Install with: pip install feedparser")
sys.exit(1)
try:
import sqlite3
except ImportError:
print("ERROR: sqlite3 is required (should be built-in)")
sys.exit(1)
SCRIPT_DIR = Path(__file__).parent
ARCHIVE_DIR = SCRIPT_DIR / 'archival_data'
ARCHIVE_DIR.mkdir(exist_ok=True)
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(levelname)s - %(message)s',
handlers=[
logging.StreamHandler(sys.stdout),
logging.FileHandler(ARCHIVE_DIR / 'processing.log', encoding='utf-8')
]
)
logger = logging.getLogger(__name__)
def init_db(db_path: Path = ARCHIVE_DIR / 'cache.db') -> None:
"""Initialize SQLite database for caching processed articles."""
conn = sqlite3.connect(db_path)
cursor = conn.cursor()
cursor.execute('''
CREATE TABLE IF NOT EXISTS articles (
id INTEGER PRIMARY KEY AUTOINCREMENT,
source_name TEXT NOT NULL,
article_url TEXT NOT NULL UNIQUE,
article_guid TEXT,
title TEXT,
author TEXT,
publish_date TEXT,
content_text TEXT,
content_html TEXT,
archive_file_path TEXT,
metadata_file_path TEXT,
status TEXT DEFAULT 'pending',
error_message TEXT,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)
''')
cursor.execute('''
CREATE TABLE IF NOT EXISTS processing_log (
id INTEGER PRIMARY KEY AUTOINCREMENT,
source_name TEXT,
timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
action TEXT,
status TEXT,
message TEXT
)
''')
cursor.execute('''
CREATE INDEX IF NOT EXISTS idx_articles_source_url
ON articles(source_name, article_url)
''')
cursor.execute('''
CREATE INDEX IF NOT EXISTS idx_articles_status
ON articles(status)
''')
conn.commit()
conn.close()
logger.debug("Database initialized: %s", db_path)
def get_db_connection(db_path: Path = ARCHIVE_DIR / 'cache.db'):
"""Get database connection."""
conn = sqlite3.connect(db_path)
conn.row_factory = sqlite3.Row
return conn
def is_duplicate(article_url: str, source_name: str, db_path: Path = ARCHIVE_DIR / 'cache.db') -> bool:
"""Check if article already in cache."""
conn = get_db_connection(db_path)
cursor = conn.cursor()
cursor.execute(
'SELECT 1 FROM articles WHERE source_name = ? AND article_url = ?',
(source_name, article_url)
)
result = cursor.fetchone()
conn.close()
return result is not None
def add_to_cache(
article_url: str,
source_name: str,
timestamp: datetime,
article_guid: str = None,
title: str = None,
author: str = None,
publish_date: str = None,
db_path: Path = ARCHIVE_DIR / 'cache.db'
) -> bool:
"""Add article to cache."""
conn = get_db_connection(db_path)
cursor = conn.cursor()
try:
cursor.execute('''
INSERT OR IGNORE INTO articles
(source_name, article_url, article_guid, title, author, publish_date, status)
VALUES (?, ?, ?, ?, ?, ?, ?)
''', (
source_name,
article_url,
article_guid,
title,
author,
publish_date,
'pending'
))
conn.commit()
conn.close()
return True
except sqlite3.IntegrityError:
conn.close()
return False
def save_article(
source_name: str,
article_data: dict,
db_path: Path = ARCHIVE_DIR / 'cache.db'
) -> str:
"""Save article to storage and update cache."""
conn = get_db_connection(db_path)
cursor = conn.cursor()
article_url = article_data.get('link', '')
article_guid = article_data.get('id', article_url)
title = article_data.get('title', '')
author = article_data.get('author', '')
publish_date = article_data.get('published', '')
summary = article_data.get('summary', '')
cursor.execute('''
INSERT OR REPLACE INTO articles
(source_name, article_url, article_guid, title, author, publish_date, content_text, status)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
''', (
source_name,
article_url,
article_guid,
title,
author,
publish_date,
summary,
'archived'
))
conn.commit()
conn.close()
return article_url
def fetch_rss_feed(rss_url: str, timeout: int = 30) -> FeedParserDict:
"""Fetch and parse RSS feed."""
logger.info("Fetching RSS feed: %s", rss_url[:50] + "..." if len(rss_url) > 50 else rss_url)
try:
req = urllib.request.Request(
rss_url,
headers={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'}
)
with urllib.request.urlopen(req, timeout=timeout) as response:
feed_content = response.read()
feed = feedparser.parse(feed_content)
if feed.bozo:
logger.warning("Feed parsing completed with warnings: %s", feed.bozo)
entry_count = len(feed.entries)
logger.info("Feed parsed: %d entries", entry_count)
return feed
except urllib.error.HTTPError as e:
logger.error("HTTP error fetching RSS feed %s: %s", rss_url, str(e.code))
raise
except urllib.error.URLError as e:
logger.error("URL error fetching RSS feed %s: %s", rss_url, str(e.reason))
raise
except Exception as e:
logger.error("Failed to fetch RSS feed %s: %s", rss_url, str(e))
raise
def decode_google_news_url(google_url: str, entry: dict = None) -> Optional[str]:
"""Decode Google News encrypted URL to actual article URL.
Google News RSS uses encrypted URLs like:
https://news.google.com/rss/articles/CBMioAFB... which need to be decoded.
Args:
google_url: Google News encrypted URL
entry: Full RSS entry for additional context
Returns:
Decoded article URL or None if not a Google News URL
"""
if 'news.google.com' not in google_url:
return google_url
try:
import base64
import urllib.parse
if '/rss/articles/' in google_url:
parts = google_url.split('/rss/articles/')
if len(parts) >= 2:
encoded = parts[1]
if encoded.startswith('CBM'):
encoded = encoded[3:]
padding = (4 - len(encoded) % 4) % 4
encoded += '=' * padding
try:
decoded = base64.urlsafe_b64decode(encoded).decode('utf-8')
logger.debug("Decoded Google News URL: %s -> %s", google_url[:60], decoded[:60])
return decoded
except Exception:
pass
if '/articles/' in google_url:
parts = google_url.split('/articles/')
if len(parts) >= 2:
encoded = parts[1]
if encoded.startswith('CBM'):
encoded = encoded[3:]
padding = (4 - len(encoded) % 4) % 4
encoded += '=' * padding
try:
decoded = base64.urlsafe_b64decode(encoded).decode('utf-8')
logger.debug("Decoded Google News URL: %s -> %s", google_url[:60], decoded[:60])
return decoded
except Exception:
pass
if 'url=' in google_url:
parsed = urllib.parse.urlparse(google_url)
params = urllib.parse.parse_qs(parsed.query)
if 'url' in params:
return params['url'][0]
logger.debug("Could not decode Google News URL: %s", google_url[:60])
if entry and 'source' in entry:
source = entry.get('source', {})
if isinstance(source, dict) and 'href' in source:
source_href = source['href']
logger.debug("Using source URL as fallback: %s", source_href)
return source_href
return google_url
except Exception as e:
logger.debug("Failed to decode Google News URL %s: %s", google_url[:60], str(e))
return google_url
def process_rss_feed(
rss_url: str,
source_name: str,
output_dir: Path,
db_path: Path = ARCHIVE_DIR / 'cache.db'
) -> List[dict]:
"""Process RSS feed and archive new articles."""
logger.info("Processing RSS feed for source: %s", source_name)
try:
feed = fetch_rss_feed(rss_url)
except Exception as e:
logger.error("Failed to fetch feed: %s", str(e))
return []
new_articles = []
skipped_count = 0
for entry in feed.entries:
article_url = entry.get('link', '')
article_url = decode_google_news_url(article_url, entry)
article_guid = entry.get('id', article_url)
if not article_url:
logger.warning("Skipping entry without URL")
continue
if is_duplicate(article_url, source_name, db_path):
logger.debug("Skipping duplicate: %s", article_url[:60])
skipped_count += 1
continue
title = entry.get('title', 'No Title')
author = entry.get('author', entry.get('authors', [{}])[0].get('name', '') if entry.get('authors') else '')
published = entry.get('published', entry.get('published_parsed', ''))
summary = entry.get('summary', entry.get('description', ''))
if published:
if hasattr(published, 'tm_year'):
publish_date = datetime(*published[:6]).isoformat()
else:
publish_date = published
else:
publish_date = datetime.now().isoformat()
article_data = {
'source_name': source_name,
'url': article_url,
'guid': article_guid,
'title': title,
'author': author,
'publish_date': publish_date,
'summary': summary,
'entry': entry
}
add_to_cache(
article_url=article_url,
source_name=source_name,
timestamp=datetime.now(),
article_guid=article_guid,
title=title,
author=author,
publish_date=publish_date,
db_path=db_path
)
new_articles.append(article_data)
logger.debug("New article: %s", title[:60])
logger.info("Processed %s: %d new, %d skipped", source_name, len(new_articles), skipped_count)
return new_articles
def process_all_feeds(
rss_feeds_path: Path = SCRIPT_DIR / 'rss_feeds.json',
output_dir: Path = ARCHIVE_DIR,
db_path: Path = ARCHIVE_DIR / 'cache.db'
) -> dict:
"""Process all RSS feeds from rss_feeds.json."""
if not rss_feeds_path.exists():
logger.error("RSS feeds file not found: %s", rss_feeds_path)
return {'success': False, 'error': 'File not found'}
with open(rss_feeds_path, 'r', encoding='utf-8') as f:
rss_feeds = json.load(f)
results = {
'total': 0,
'success': 0,
'failed': 0,
'new_articles': 0,
'errors': []
}
for source_name, feed_info in rss_feeds.items():
rss_url = feed_info.get('rss_url', '')
if not rss_url:
logger.warning("No RSS URL for source: %s", source_name)
continue
if feed_info.get('disabled', False):
logger.info("Skipping disabled source: %s (%s)", source_name,
feed_info.get('disable_reason', 'No reason provided'))
continue
results['total'] += 1
try:
articles = process_rss_feed(rss_url, source_name, output_dir, db_path)
results['success'] += 1
results['new_articles'] += len(articles)
logger.info("Completed %s: %d new articles", source_name, len(articles))
except Exception as e:
results['failed'] += 1
results['errors'].append({
'source': source_name,
'url': rss_url,
'error': str(e)
})
logger.error("Failed to process %s: %s", source_name, str(e))
return results
def main():
parser = argparse.ArgumentParser(description='RSS Feed Processor for NewsArchiver')
parser.add_argument('--url', help='Single RSS URL to process')
parser.add_argument('--source', help='Source name (required if --url provided)')
parser.add_argument('--all', action='store_true', help='Process all feeds from rss_feeds.json')
parser.add_argument('--rss-feeds', type=Path, default=SCRIPT_DIR / 'rss_feeds.json',
help='Path to RSS feeds JSON file')
parser.add_argument('--output', type=Path, default=ARCHIVE_DIR,
help='Output directory for archived content')
parser.add_argument('--verbose', action='store_true', help='Enable verbose logging')
args = parser.parse_args()
if args.verbose:
logger.setLevel(logging.DEBUG)
logger.info("=" * 60)
logger.info("RSS Feed Processor - Phase 2.1")
logger.info("=" * 60)
init_db()
if args.url:
if not args.source:
logger.error("Source name required when using --url")
return
process_rss_feed(args.url, args.source, args.output)
elif args.all:
results = process_all_feeds(args.rss_feeds, args.output)
print("\n" + "=" * 60)
print("PROCESSING COMPLETE")
print("=" * 60)
print(f"Total feeds: {results['total']}")
print(f"Successful: {results['success']}")
print(f"Failed: {results['failed']}")
print(f"New articles: {results['new_articles']}")
if results['errors']:
print("\nErrors:")
for error in results['errors']:
print(f" - {error['source']}: {error['error']}")
print("=" * 60)
else:
parser.print_help()
if __name__ == '__main__':
main()

276
run_archiver.py Normal file
View File

@ -0,0 +1,276 @@
#!/usr/bin/env python3
"""NewsArchiver - Main CLI Entry Point (Phase 4)
Single-file CLI for running NewsArchiver with multiple modes:
- --run: Archive news articles once
- --serve: Start Flask web server
- --interval: Run background scheduler with specified interval
"""
import argparse
import atexit
import logging
import sys
import time
from pathlib import Path
try:
from flask import Flask
except ImportError:
print("ERROR: Flask is required. Install with: pip install flask")
sys.exit(1)
try:
from scheduler import start_scheduler, stop_scheduler, scheduled_archive
except ImportError:
print("ERROR: scheduler module not found")
sys.exit(1)
try:
from rss_processor import process_all_feeds, init_db as init_db_rss
except ImportError:
print("ERROR: rss_processor module not found")
sys.exit(1)
try:
from content_extractor import get_html_from_url
except ImportError:
print("ERROR: content_extractor module not found")
sys.exit(1)
try:
from storage_manager import initialize_storage, get_all_sources
except ImportError:
print("ERROR: storage_manager module not found")
sys.exit(1)
try:
from archive_engine import archive_all_sources
except ImportError:
print("ERROR: archive_engine module not found")
sys.exit(1)
try:
from web_interface import app
except ImportError:
print("ERROR: web_interface module not found")
sys.exit(1)
try:
from singlefile_archive import check_singlefile_available
except ImportError:
print("WARNING: singlefile_archive module not found")
print("SingleFile integration will not be available")
SCRIPT_DIR = Path(__file__).parent
ARCHIVE_DIR = SCRIPT_DIR / 'archival_data'
ARCHIVE_DIR.mkdir(exist_ok=True)
def setup_logging(verbose: bool = False) -> logging.Logger:
"""Configure logging for the application.
Args:
verbose: If True, enable DEBUG level logging
Returns:
Configured logger instance
"""
level = logging.DEBUG if verbose else logging.INFO
logging.basicConfig(
level=level,
format='%(asctime)s - %(levelname)s - %(message)s',
handlers=[
logging.StreamHandler(sys.stdout),
logging.FileHandler(ARCHIVE_DIR / 'processing.log', encoding='utf-8')
]
)
logger = logging.getLogger(__name__)
logger.info("NewsArchiver - Main CLI Entry Point")
logger.info("=" * 60)
return logger
def run_archive_once(logger: logging.Logger, verbose: bool = False) -> bool:
"""Run archiving process once.
Args:
logger: Logger instance
verbose: If True, enable verbose logging
Returns:
True if successful, False otherwise
"""
try:
logger.info("Running one-time archive")
logger.info("=" * 60)
init_db_rss()
initialize_storage()
results = archive_all_sources(
rss_feeds_path=SCRIPT_DIR / 'rss_feeds.json',
output_dir=ARCHIVE_DIR,
dry_run=False
)
logger.info("=" * 60)
logger.info("Archive complete")
logger.info("=" * 60)
logger.info("Sources processed: %d", results.get('sources_processed', 0))
logger.info("Total articles archived: %d", results.get('total_articles_archived', 0))
logger.info("Total articles skipped: %d", results.get('total_articles_skipped', 0))
logger.info("Total articles failed: %d", results.get('total_articles_failed', 0))
return True
except Exception as e:
logger.error("Archive failed: %s", str(e))
return False
def run_scheduler(interval_minutes: int, logger: logging.Logger, verbose: bool = False) -> None:
"""Run background scheduler.
Args:
interval_minutes: Interval between archive runs in minutes
logger: Logger instance
verbose: If True, enable verbose logging
"""
logger.info("Starting background scheduler")
logger.info("=" * 60)
try:
init_db_rss()
initialize_storage()
scheduler = start_scheduler(interval_minutes)
atexit.register(stop_scheduler)
logger.info("Press Ctrl+C to stop")
try:
while True:
time.sleep(1)
except (KeyboardInterrupt, SystemExit):
logger.info("Shutting down scheduler...")
stop_scheduler()
logger.info("Scheduler stopped")
except Exception as e:
logger.error("Scheduler failed to start: %s", str(e))
sys.exit(1)
def run_web_server(host: str, port: int, logger: logging.Logger, verbose: bool = False) -> None:
"""Run Flask web server.
Args:
host: Host to bind to
port: Port to bind to
logger: Logger instance
verbose: If True, enable verbose logging
"""
logger.info("Starting web server")
logger.info("=" * 60)
try:
if not (ARCHIVE_DIR / 'cache.db').exists():
logger.info("Database not found, initializing...")
initialize_storage()
if not check_singlefile_available():
logger.warning("SingleFile CLI not available. Some features may not work.")
logger.info("Web server starting on %s:%d", host, port)
logger.info("=" * 60)
app.run(
host=host,
port=port,
debug=False
)
except Exception as e:
logger.error("Web server failed to start: %s", str(e))
sys.exit(1)
def main() -> None:
"""Main entry point for NewsArchiver CLI."""
parser = argparse.ArgumentParser(
description='NewsArchiver - News Article Archiving System',
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog='''
Examples:
%(prog)s --run Run archiving once
%(prog)s --serve Start web server
%(prog)s --serve --host 0.0.0.0 --port 8080
Start web server on custom host/port
%(prog)s --interval 60 Run background scheduler (1 hour interval)
'''
)
parser.add_argument(
'--run',
action='store_true',
help='Run archiving once (process all RSS feeds)'
)
parser.add_argument(
'--serve',
action='store_true',
help='Start Flask web server'
)
parser.add_argument(
'--interval',
type=int,
default=60,
help='Run background scheduler with specified interval (minutes, default: 60)'
)
parser.add_argument(
'--host',
type=str,
default='0.0.0.0',
help='Host for web server (default: 0.0.0.0)'
)
parser.add_argument(
'--port',
type=int,
default=5000,
help='Port for web server (default: 5000)'
)
parser.add_argument(
'--verbose', '-v',
action='store_true',
help='Enable verbose logging (DEBUG level)'
)
args = parser.parse_args()
logger = setup_logging(args.verbose)
if args.run:
success = run_archive_once(logger, args.verbose)
sys.exit(0 if success else 1)
elif args.serve:
run_web_server(args.host, args.port, logger, args.verbose)
elif args.interval:
run_scheduler(args.interval, logger, args.verbose)
else:
parser.print_help()
sys.exit(1)
if __name__ == '__main__':
main()

200
scheduler.py Normal file
View File

@ -0,0 +1,200 @@
#!/usr/bin/env python3
"""Scheduler for NewsArchiver - Phase 4
Background scheduler using APScheduler to automate
daily archiving of news sources.
"""
import atexit
import logging
import signal
import sys
import time
from datetime import datetime
from pathlib import Path
try:
from apscheduler.schedulers.background import BackgroundScheduler
from apscheduler.triggers.interval import IntervalTrigger
except ImportError:
print("ERROR: APScheduler is required. Install with: pip install apscheduler")
sys.exit(1)
try:
from archive_engine import archive_all_sources
except ImportError:
print("ERROR: archive_engine is required")
sys.exit(1)
SCRIPT_DIR = Path(__file__).parent
ARCHIVE_DIR = SCRIPT_DIR / 'archival_data'
ARCHIVE_DIR.mkdir(exist_ok=True)
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(levelname)s - %(message)s',
handlers=[
logging.StreamHandler(sys.stdout),
logging.FileHandler(ARCHIVE_DIR / 'processing.log', encoding='utf-8')
]
)
logger = logging.getLogger(__name__)
scheduler = BackgroundScheduler()
# Timeout configuration
MAX_RUN_TIME_SECONDS = 3600 # 1 hour
start_time = None
def timeout_handler(signum, frame):
"""Handle timeout signal - exit gracefully after current download completes."""
logger.warning("Timeout reached (%d seconds). Will exit after current download completes.", MAX_RUN_TIME_SECONDS)
raise SystemExit(0)
def check_timeout() -> bool:
"""Check if timeout has been reached.
Returns:
True if timeout reached, False otherwise
"""
global start_time
elapsed = (datetime.now() - start_time).total_seconds()
if elapsed >= MAX_RUN_TIME_SECONDS:
logger.warning("Maximum runtime of %d seconds reached (%d seconds elapsed)", MAX_RUN_TIME_SECONDS, int(elapsed))
return True
return False
def scheduled_archive() -> None:
"""Run archiving for all sources."""
global start_time
start_time = datetime.now()
logger.info("=" * 60)
logger.info("Starting scheduled archive run")
logger.info("=" * 60)
signal.signal(signal.SIGALRM, timeout_handler)
signal.alarm(MAX_RUN_TIME_SECONDS)
try:
results = archive_all_sources(
rss_feeds_path=SCRIPT_DIR / 'rss_feeds.json',
output_dir=ARCHIVE_DIR,
dry_run=False
)
if results['success']:
logger.info("Scheduled archive completed successfully")
logger.info("Sources processed: %d", results.get('sources_processed', 0))
logger.info("Total articles archived: %d", results.get('total_articles_archived', 0))
else:
logger.error("Scheduled archive failed: %s", results.get('error', 'Unknown error'))
except SystemExit as e:
logger.info("Scheduler exiting due to timeout")
raise e
except Exception as e:
logger.error("Scheduled archive failed with exception: %s", str(e))
finally:
signal.alarm(0)
def start_scheduler(interval_minutes: int = 60) -> BackgroundScheduler:
"""Start the background scheduler.
Args:
interval_minutes: Interval between archive runs in minutes
Returns:
The scheduler instance
"""
scheduler.add_job(
func=scheduled_archive,
trigger=IntervalTrigger(minutes=interval_minutes),
id='archive_news',
replace_existing=True,
misfire_grace_time=60,
coalesce=True
)
scheduler.start()
logger.info("Scheduler started with %d minute interval", interval_minutes)
logger.info("Running initial archive immediately...")
scheduled_archive()
return scheduler
def run_once() -> None:
"""Run archiving once (for CLI --run flag)."""
global start_time
start_time = datetime.now()
logger.info("=" * 60)
logger.info("Running one-time archive")
logger.info("=" * 60)
signal.signal(signal.SIGALRM, timeout_handler)
signal.alarm(MAX_RUN_TIME_SECONDS)
try:
scheduled_archive()
logger.info("=" * 60)
logger.info("One-time archive completed")
logger.info("=" * 60)
except SystemExit as e:
logger.info("Archiver exiting due to timeout")
raise e
finally:
signal.alarm(0)
def stop_scheduler() -> None:
"""Stop the scheduler gracefully."""
if scheduler.running:
scheduler.shutdown()
logger.info("Scheduler stopped")
atexit.register(lambda: stop_scheduler())
if __name__ == '__main__':
import argparse
parser = argparse.ArgumentParser(description='NewsArchiver - Scheduler')
parser.add_argument('--run', action='store_true', help='Run archiving once')
parser.add_argument('--serve', action='store_true', help='Start web server')
parser.add_argument('--interval', type=int, default=60, help='Scheduler interval in minutes (default: 60)')
parser.add_argument('--host', default='0.0.0.0', help='Host for web server')
parser.add_argument('--port', type=int, default=5000, help='Port for web server')
args = parser.parse_args()
if args.run:
run_once()
stop_scheduler()
elif args.serve:
from web_interface import app
logger.info("Starting web server on %s:%d", args.host, args.port)
try:
app.run(host=args.host, port=args.port)
except Exception as e:
logger.error("Web server error: %s", str(e))
sys.exit(1)
else:
start_scheduler(args.interval)
logger.info("Press Ctrl+C to stop")
try:
while True:
time.sleep(1)
if check_timeout():
logger.info("Maximum runtime reached. Exiting.")
stop_scheduler()
sys.exit(0)
except (KeyboardInterrupt, SystemExit):
stop_scheduler()

84
setup_cron.sh Normal file
View File

@ -0,0 +1,84 @@
#!/bin/bash
# Setup script for NewsArchiver
# This script sets up the cron job for automated news archiving
set -e
SCRIPT_DIR="/home/user/playground/NewsArchiver"
LOG_FILE="/tmp/newsarchiver_cron.log"
CRON_JOB="*/30 * * * * /usr/bin/env python3 ${SCRIPT_DIR}/run_archiver.py --interval 30 > ${LOG_FILE} 2>&1"
echo "=== NewsArchiver Setup Script ==="
echo ""
# Check Python is available
if ! command -v python3 &> /dev/null; then
echo "ERROR: python3 not found"
exit 1
fi
# Check if running as jarian
CURRENT_USER=$(whoami)
if [ "$CURRENT_USER" != "jarian" ]; then
echo "WARNING: This script is configured for user 'jarian', but you are '$CURRENT_USER'"
echo "You may need to update the script paths"
fi
# Check if NewsArchiver directory exists
if [ ! -d "$SCRIPT_DIR" ]; then
echo "ERROR: NewsArchiver directory not found at $SCRIPT_DIR"
exit 1
fi
# Check if requirements are installed
echo "Checking dependencies..."
cd "$SCRIPT_DIR"
python3 -c "import flask; import requests; import trafilatura; import feedparser; import apscheduler" 2>/dev/null || {
echo "Installing dependencies..."
pip install -r requirements.txt
}
# Check if web server is running
if ! pgrep -f "run_archiver.py --serve" > /dev/null; then
echo "Starting web server..."
nohup python3 "$SCRIPT_DIR/run_archiver.py" --serve --host 0.0.0.0 --port 5000 > /tmp/webserver.log 2>&1 &
sleep 3
if curl -s -o /dev/null -w "%{http_code}" http://localhost:5000/ | grep -q "200"; then
echo "Web server started successfully on port 5000"
else
echo "WARNING: Web server may not be responding"
fi
else
echo "Web server is already running"
fi
# Remove old scheduler lock file if exists
if [ -f "$SCRIPT_DIR/archival_data/.scheduler.lock" ]; then
rm -f "$SCRIPT_DIR/archival_data/.scheduler.lock"
echo "Removed stale scheduler lock file"
fi
# Kill any existing scheduler processes
pkill -f "run_archiver.py --interval" 2>/dev/null || true
echo "Cleared any existing scheduler processes"
# Setup cron job
echo "Setting up cron job..."
if crontab -l 2>/dev/null | grep -q "NewsArchiver"; then
echo "Removing existing NewsArchiver cron job..."
crontab -l | grep -v "NewsArchiver" | crontab -
fi
echo "$CRON_JOB" | crontab -
echo "Cron job added successfully"
echo ""
echo "=== Current Cron Jobs ==="
crontab -l | grep NewsArchiver
echo ""
echo "=== Setup Complete ==="
echo "- Archiver will run every 30 minutes via cron"
echo "- Logs written to: $LOG_FILE"
echo "- Web interface at: http://localhost:5000"
echo ""

257
singlefile_archive.py Normal file
View File

@ -0,0 +1,257 @@
import subprocess
import os
from pathlib import Path
import logging
from typing import Optional
logger = logging.getLogger(__name__)
try:
from playwright.sync_api import sync_playwright
PLAYWRIGHT_AVAILABLE = True
except ImportError:
PLAYWRIGHT_AVAILABLE = False
logger.debug("Playwright not available")
# Cache the SingleFile path
_SINGLEFILE_PATH: Optional[str] = None
def _get_singlefile_path() -> Optional[str]:
"""Get the path to SingleFile CLI executable."""
global _SINGLEFILE_PATH
if _SINGLEFILE_PATH is not None:
return _SINGLEFILE_PATH
# Check PATH first
single_file_path = os.environ.get('PATH', '').split(os.pathsep)
for path in single_file_path:
candidate = Path(path) / 'single-file'
if candidate.is_file():
_SINGLEFILE_PATH = str(candidate)
logger.debug(f"Found SingleFile in PATH: {_SINGLEFILE_PATH}")
return _SINGLEFILE_PATH
# Check common locations
common_locations = [
'/home/user/.local/bin/single-file',
'/usr/local/bin/single-file',
'/usr/bin/single-file',
'/home/user/.npm/_global/bin/single-file',
]
for candidate in common_locations:
if Path(candidate).is_file():
_SINGLEFILE_PATH = candidate
logger.debug(f"Found SingleFile at: {_SINGLEFILE_PATH}")
return _SINGLEFILE_PATH
logger.error("SingleFile CLI not found. Please install with: npm install -g single-file")
return None
def check_singlefile_available() -> bool:
"""Check if SingleFile CLI is available"""
single_file_path = _get_singlefile_path()
if not single_file_path:
return False
try:
result = subprocess.run(
[single_file_path, '--version'],
capture_output=True,
text=True,
timeout=10
)
return result.returncode == 0
except (subprocess.TimeoutExpired, FileNotFoundError, OSError):
logger.error(f"SingleFile CLI at {single_file_path} is not executable")
return False
def is_error_page(html_content: str) -> bool:
"""Check if HTML content is an error page (403, 404, etc.)
Args:
html_content: HTML content to check
Returns:
True if error page detected, False otherwise
"""
error_patterns = [
'403 error',
'403 forbidden',
'access denied',
'request blocked',
'cloudfront',
'404 error',
'page not found',
'error 404',
'server error',
'503 service unavailable',
]
html_lower = html_content.lower()
return any(pattern in html_lower for pattern in error_patterns)
def validate_archived_html(output_path: Path) -> bool:
"""Validate that archived HTML is not an error page.
Args:
output_path: Path to the archived HTML file
Returns:
True if valid, False if error page detected
"""
try:
if not output_path.exists():
logger.warning(f"Archived file not found: {output_path}")
return False
content = output_path.read_text(encoding='utf-8', errors='ignore')
if is_error_page(content):
logger.warning(f"Archived file contains error page: {output_path}")
return False
if len(content) < 1000:
logger.warning(f"Archived file too small (likely incomplete): {output_path}")
return False
return True
except Exception as e:
logger.error(f"Error validating archived HTML {output_path}: {e}")
return False
def archive_page_with_singlefile(
url: str,
output_path: Path,
extract_content: bool = True
) -> bool:
"""Archive a web page using SingleFile CLI
Args:
url: URL to archive
output_path: Output file path for the archived HTML
extract_content: Whether to use extract-content mode (ignored - SingleFile always extracts)
Returns:
True if successful, False otherwise
"""
single_file_path = _get_singlefile_path()
if not single_file_path:
logger.error("SingleFile CLI not available")
return False
cmd = [
single_file_path,
url,
str(output_path),
'--browser-headless=true',
'--browser-wait-delay=5000',
'--browser-load-max-time=120000'
]
logger.debug(f"Archiving {url} with SingleFile at {single_file_path}")
try:
result = subprocess.run(cmd, capture_output=True, text=True, timeout=180)
if result.returncode == 0:
logger.info(f"Successfully archived {url} to {output_path}")
if result.stdout:
logger.debug(f"SingleFile output: {result.stdout}")
# Validate the archived HTML
if not validate_archived_html(output_path):
logger.warning(f"Archived HTML validation failed for {url}, will use fallback")
return False
return True
else:
error_msg = result.stderr if result.stderr else result.stdout
logger.error(f"SingleFile failed for {url}: {error_msg}")
return False
except subprocess.TimeoutExpired:
logger.error(f"SingleFile timed out for {url}")
return False
except FileNotFoundError:
logger.error(f"SingleFile CLI executable not found at: {single_file_path}")
return False
except Exception as e:
logger.error(f"Unexpected error archiving {url} with SingleFile: {e}")
return False
def archive_page_with_singlefile_no_extraction(url: str, output_path: Path) -> bool:
"""Archive a web page using SingleFile CLI without content extraction
This preserves the full original HTML structure including navigation, ads, etc.
Args:
url: URL to archive
output_path: Output file path for the archived HTML
Returns:
True if successful, False otherwise
"""
return archive_page_with_singlefile(url, output_path, extract_content=False)
def archive_page_with_singlefile_extract(url: str, output_path: Path) -> bool:
"""Archive a web page using SingleFile CLI with content extraction
This extracts only the main content, removing navigation, ads, and sidebars.
Args:
url: URL to archive
output_path: Output file path for the archived HTML
Returns:
True if successful, False otherwise
"""
return archive_page_with_singlefile(url, output_path, extract_content=True)
def archive_page_with_playwright(url: str, output_path: Path) -> bool:
"""Archive a web page using Playwright
This visits the URL with a headless browser, waits for content to load,
and saves the full HTML page.
Args:
url: URL to archive
output_path: Output file path for the archived HTML
Returns:
True if successful, False otherwise
"""
if not PLAYWRIGHT_AVAILABLE:
logger.error("Playwright not available. Install with: pip install playwright")
return False
try:
with sync_playwright() as p:
browser = p.chromium.launch(headless=True)
page = browser.new_page()
page.goto(url, wait_until='networkidle', timeout=120000)
content = page.content()
browser.close()
output_path.parent.mkdir(parents=True, exist_ok=True)
output_path.write_text(content, encoding='utf-8')
logger.info("Successfully archived %s to %s using Playwright", url[:60], output_path)
return True
except Exception as e:
logger.error("Playwright archiving failed for %s: %s", url, str(e))
return False

548
static/style.css Normal file
View File

@ -0,0 +1,548 @@
/* Base styles */
:root {
--primary-color: #333;
--secondary-color: #666;
--background-color: #f5f5f5;
--border-color: #ddd;
--accent-color: #0066cc;
--accent-hover: #0055aa;
}
[data-theme="dark"] {
--primary-color: #e0e0e0;
--secondary-color: #a0a0a0;
--background-color: #1a1a1a;
--border-color: #444;
--accent-color: #4dabf7;
--accent-hover: #339af0;
}
body {
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, sans-serif;
margin: 0;
padding: 0;
background-color: var(--background-color);
color: var(--primary-color);
line-height: 1.6;
transition: background-color 0.3s ease, color 0.3s ease;
}
/* Header */
header {
background-color: #fff;
border-bottom: 1px solid var(--border-color);
padding: 1rem 2rem;
display: flex;
justify-content: space-between;
align-items: center;
transition: background-color 0.3s ease;
}
[data-theme="dark"] header {
background-color: #2d2d2d;
}
header h1 {
margin: 0;
font-size: 1.5rem;
transition: color 0.3s ease;
}
header nav a {
color: var(--primary-color);
text-decoration: none;
margin-left: 1rem;
transition: color 0.3s ease;
}
header nav a:hover {
color: var(--accent-color);
}
main {
padding: 1rem 2rem;
max-width: 800px;
margin: 0 auto;
transition: color 0.3s ease;
}
/* Newspaper list */
.newspaper-list {
list-style: none;
padding: 0;
margin: 1rem 0;
}
.newspaper-item {
background: #fff;
border: 1px solid var(--border-color);
border-radius: 4px;
padding: 1rem;
margin-bottom: 0.5rem;
display: flex;
justify-content: space-between;
align-items: center;
transition: background-color 0.3s ease, border-color 0.3s ease;
}
[data-theme="dark"] .newspaper-item {
background: #2d2d2d;
}
.newspaper-info h2 {
margin: 0 0 0.5rem 0;
font-size: 1.1rem;
transition: color 0.3s ease;
}
.newspaper-info h2 a {
color: var(--primary-color);
text-decoration: none;
}
.newspaper-info h2 a:hover {
color: var(--accent-color);
}
.newspaper-info p {
margin: 0.2rem 0;
color: var(--secondary-color);
font-size: 0.9rem;
transition: color 0.3s ease;
}
.status-success {
color: #4caf50;
font-weight: bold;
}
.status-pending {
color: #ff9800;
font-weight: bold;
}
.pull-btn {
background-color: var(--accent-color);
color: white;
border: none;
padding: 0.5rem 1rem;
border-radius: 4px;
cursor: pointer;
font-size: 0.9rem;
transition: background-color 0.3s ease;
}
.pull-btn:hover {
background-color: var(--accent-hover);
}
/* Article list */
.article-list {
list-style: none;
padding: 0;
margin: 1rem 0;
}
.article-item {
background: #fff;
border: 1px solid var(--border-color);
border-radius: 4px;
padding: 1rem;
margin-bottom: 0.5rem;
transition: background-color 0.3s ease, border-color 0.3s ease;
}
[data-theme="dark"] .article-item {
background: #2d2d2d;
}
.article-item h3 {
margin: 0 0 0.5rem 0;
font-size: 1rem;
transition: color 0.3s ease;
}
.article-item h3 a {
color: var(--primary-color);
text-decoration: none;
}
.article-item h3 a:hover {
color: var(--accent-color);
}
.article-date {
color: var(--secondary-color);
font-size: 0.85rem;
margin: 0.2rem 0;
transition: color 0.3s ease;
}
.article-summary {
color: var(--primary-color);
font-size: 0.9rem;
margin: 0.5rem 0 0 0;
display: -webkit-box;
-webkit-line-clamp: 3;
-webkit-box-orient: vertical;
overflow: hidden;
transition: color 0.3s ease;
}
/* Pagination */
.pagination {
display: flex;
justify-content: center;
align-items: center;
gap: 1rem;
margin: 1rem 0;
padding: 1rem 0;
border-top: 1px solid var(--border-color);
}
.pagination a {
color: var(--accent-color);
text-decoration: none;
}
.pagination a:hover {
text-decoration: underline;
}
/* Article view */
.article-view {
background: #fff;
border: 1px solid var(--border-color);
border-radius: 8px;
padding: 2rem;
box-shadow: 0 2px 8px rgba(0,0,0,0.08);
transition: background-color 0.3s ease, border-color 0.3s ease;
}
[data-theme="dark"] .article-view {
background: #2d2d2d;
}
.article-header {
margin-bottom: 1.5rem;
transition: color 0.3s ease;
}
.article-source {
display: inline-block;
background: #f0f7ff;
color: #0066cc;
padding: 0.35rem 0.75rem;
border-radius: 20px;
font-size: 0.8rem;
font-weight: 600;
text-transform: uppercase;
letter-spacing: 0.5px;
margin-bottom: 1rem;
transition: background-color 0.3s ease, color 0.3s ease;
}
[data-theme="dark"] .article-source {
background: #1a365d;
color: #60a5fa;
}
.source-label {
margin-right: 0.5rem;
color: #667599;
transition: color 0.3s ease;
}
.source-name {
font-weight: 700;
}
.article-view h1 {
margin-top: 0;
font-size: 2rem;
line-height: 1.3;
color: #222;
font-weight: 700;
transition: color 0.3s ease;
}
[data-theme="dark"] .article-view h1 {
color: #e0e0e0;
}
.article-meta {
border-bottom: 2px solid var(--border-color);
padding-bottom: 1.25rem;
margin-bottom: 1.5rem;
transition: border-color 0.3s ease;
}
.article-meta-row {
display: flex;
flex-wrap: wrap;
align-items: center;
gap: 1rem;
margin-bottom: 0.5rem;
}
.article-meta-row time,
.article-meta-row span {
font-size: 0.9rem;
color: var(--secondary-color);
transition: color 0.3s ease;
}
.article-meta a {
color: var(--accent-color);
text-decoration: none;
font-size: 0.85rem;
word-break: break-all;
transition: color 0.3s ease;
}
.article-meta a:hover {
text-decoration: underline;
}
.article-content {
margin: 1.5rem 0;
transition: color 0.3s ease;
}
.article-text {
white-space: pre-wrap;
font-size: 1.05rem;
line-height: 1.8;
color: #333;
transition: color 0.3s ease;
}
[data-theme="dark"] .article-text {
color: #e0e0e0;
}
.article-text p {
margin: 0;
text-indent: 2rem;
transition: color 0.3s ease;
}
.article-text p:first-child {
margin-top: 0;
text-indent: 0;
}
.archived-html {
border: 1px solid var(--border-color);
border-radius: 4px;
padding: 1rem;
margin-top: 1rem;
background: #fafafa;
transition: border-color 0.3s ease, background-color 0.3s ease;
}
[data-theme="dark"] .archived-html {
border-color: var(--border-color);
background: #2d2d2d;
}
.article-actions {
margin-top: 2rem;
padding-top: 1.25rem;
border-top: 2px solid var(--border-color);
display: flex;
flex-wrap: wrap;
gap: 1rem;
align-items: center;
transition: border-color 0.3s ease;
}
.back-link {
color: var(--accent-color);
text-decoration: none;
font-weight: 600;
font-size: 1rem;
padding: 0.5rem 1rem;
background: #f0f7ff;
border-radius: 6px;
transition: all 0.3s ease;
}
.back-link:hover {
text-decoration: none;
background: #e0efff;
transform: translateY(-1px);
}
[data-theme="dark"] .back-link {
background: #1a365d;
}
[data-theme="dark"] .back-link:hover {
background: #0d2a4c;
}
/* Disabled feeds */
.source-disabled {
color: var(--secondary-color);
text-decoration: line-through;
transition: color 0.3s ease;
}
.source-disabled:hover {
text-decoration: none;
cursor: not-allowed;
}
.status-badge {
display: inline-block;
background: #dc3545;
color: white;
padding: 0.15rem 0.5rem;
border-radius: 3px;
font-size: 0.75rem;
margin-left: 0.5rem;
transition: background-color 0.3s ease;
}
.disable-reason {
color: #666;
font-size: 0.85rem;
margin-top: 0.25rem;
font-style: italic;
transition: color 0.3s ease;
}
/* Status page */
.status-page {
background: #fff;
border: 1px solid var(--border-color);
border-radius: 4px;
padding: 1.5rem;
transition: background-color 0.3s ease, border-color 0.3s ease;
}
[data-theme="dark"] .status-page {
background: #2d2d2d;
}
.status-page h1 {
margin-top: 0;
transition: color 0.3s ease;
}
.status-page h2 {
font-size: 1.2rem;
margin-top: 1.5rem;
margin-bottom: 1rem;
transition: color 0.3s ease;
}
.status-summary {
display: grid;
grid-template-columns: 1fr 1fr;
gap: 1rem;
margin-bottom: 1.5rem;
transition: color 0.3s ease;
}
.status-item h2 {
margin: 0 0 0.5rem 0;
font-size: 1rem;
color: var(--secondary-color);
transition: color 0.3s ease;
}
.status-item p {
margin: 0.2rem 0;
transition: color 0.3s ease;
}
.status-online {
color: #28a745;
font-weight: bold;
}
.status-warning {
color: #ffc107;
font-weight: bold;
}
/* Responsive */
@media (max-width: 600px) {
header {
padding: 0.75rem 1rem;
flex-direction: column;
gap: 0.5rem;
}
header nav {
display: flex;
gap: 1rem;
}
main {
padding: 0.75rem 1rem;
}
.newspaper-item {
flex-direction: column;
align-items: flex-start;
}
.pull-btn {
margin-top: 0.75rem;
}
.status-summary {
grid-template-columns: 1fr;
}
.article-view {
padding: 1.25rem;
}
.article-view h1 {
font-size: 1.4rem;
}
.article-meta-row {
flex-direction: column;
gap: 0.25rem;
}
.article-text {
font-size: 0.95rem;
}
}
/* Kobo compatibility */
@media (max-width: 600px) {
body {
font-size: 14px;
}
h1 {
font-size: 1.25rem;
}
h2 {
font-size: 1.1rem;
}
}
/* Print styles */
@media print {
header, .pull-btn, .pagination, .article-actions, .back-link {
display: none;
}
.article-view {
border: none;
padding: 0;
}
.article-content {
margin: 0;
}
}

18
stop_services.sh Normal file
View File

@ -0,0 +1,18 @@
#!/bin/bash
# Stop NewsArchiver services
echo "Stopping NewsArchiver services..."
# Stop web server
pkill -f "run_archiver.py --serve" 2>/dev/null || true
echo "Web server stopped"
# Stop scheduler
pkill -f "run_archiver.py --interval" 2>/dev/null || true
echo "Scheduler stopped"
# Remove lock file
rm -f /home/user/playground/NewsArchiver/archival_data/.scheduler.lock 2>/dev/null || true
echo "Scheduler lock file removed"
echo "All NewsArchiver services stopped"

771
storage_manager.py Normal file
View File

@ -0,0 +1,771 @@
#!/usr/bin/env python3
"""Storage Manager for NewsArchiver - Phase 2.3
Organizes archived data by newspaper, manages SQLite cache database,
stores both raw HTML and extracted content.
"""
import json
import logging
import os
import sqlite3
import sys
from datetime import datetime
from pathlib import Path
from typing import List, Optional
from content_extractor import ArticleData
try:
import feedgenerator
FEEDGENERATOR_AVAILABLE = True
except ImportError:
FEEDGENERATOR_AVAILABLE = False
SCRIPT_DIR = Path(__file__).parent
ARCHIVE_DIR = SCRIPT_DIR / 'archival_data'
ARCHIVE_DIR.mkdir(exist_ok=True)
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(levelname)s - %(message)s',
handlers=[
logging.StreamHandler(sys.stdout),
logging.FileHandler(ARCHIVE_DIR / 'processing.log', encoding='utf-8')
]
)
logger = logging.getLogger(__name__)
DB_PATH = ARCHIVE_DIR / 'cache.db'
WEBSITES_DIR = ARCHIVE_DIR / 'websites'
def _get_db_connection() -> sqlite3.Connection:
"""Get database connection with row factory."""
conn = sqlite3.connect(
DB_PATH,
timeout=30.0,
isolation_level=None
)
conn.row_factory = sqlite3.Row
conn.execute('PRAGMA journal_mode=WAL')
conn.execute('PRAGMA busy_timeout=30000')
return conn
def _init_database() -> None:
"""Initialize database schema."""
with _get_db_connection() as conn:
cursor = conn.cursor()
cursor.execute('''
CREATE TABLE IF NOT EXISTS articles (
id INTEGER PRIMARY KEY AUTOINCREMENT,
source_name TEXT NOT NULL,
article_url TEXT NOT NULL UNIQUE,
article_guid TEXT,
title TEXT,
author TEXT,
publish_date TEXT,
content_text TEXT,
content_html TEXT,
archive_file_path TEXT,
metadata_file_path TEXT,
status TEXT DEFAULT 'pending',
error_message TEXT,
extraction_method TEXT,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)
''')
cursor.execute('''
CREATE TABLE IF NOT EXISTS article_archives (
article_url TEXT PRIMARY KEY,
source_name TEXT,
archive_file_path TEXT,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)
''')
cursor.execute('''
CREATE TABLE IF NOT EXISTS processing_log (
id INTEGER PRIMARY KEY AUTOINCREMENT,
source_name TEXT,
timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
action TEXT,
status TEXT,
message TEXT
)
''')
cursor.execute('CREATE INDEX IF NOT EXISTS idx_articles_source ON articles(source_name)')
cursor.execute('CREATE INDEX IF NOT EXISTS idx_articles_url ON articles(article_url)')
cursor.execute('CREATE INDEX IF NOT EXISTS idx_articles_status ON articles(status)')
# Add extraction_method column if it doesn't exist
try:
cursor.execute('ALTER TABLE articles ADD COLUMN extraction_method TEXT')
conn.commit()
except sqlite3.OperationalError:
pass
cursor.execute('CREATE INDEX IF NOT EXISTS idx_articles_extraction_method ON articles(extraction_method)')
cursor.execute('CREATE INDEX IF NOT EXISTS idx_article_archives_url ON article_archives(article_url)')
cursor.execute('CREATE INDEX IF NOT EXISTS idx_article_archives_source ON article_archives(source_name)')
logger.debug("Database initialized at %s", DB_PATH)
def _ensure_directory_structure(source_name: str, date_str: str) -> tuple:
"""Ensure directory structure exists for a source and date.
Args:
source_name: Newspaper source name
date_str: Date string in YYYY-MM-DD format
Returns:
Tuple of (html_dir, articles_dir) Path objects
"""
source_dir = WEBSITES_DIR / source_name
html_dir = source_dir / 'html' / date_str
articles_dir = source_dir / 'articles' / date_str
html_dir.mkdir(parents=True, exist_ok=True)
articles_dir.mkdir(parents=True, exist_ok=True)
return html_dir, articles_dir
def _get_next_file_index(html_dir: Path, articles_dir: Path) -> int:
"""Get next available file index for article naming.
Args:
html_dir: Directory containing HTML files
articles_dir: Directory containing JSON files
Returns:
Next available index (1-indexed)
"""
def get_max_index(directory: Path, extension: str) -> int:
max_idx = 0
if directory.exists():
for file in directory.glob(f'*{extension}'):
try:
name = file.stem
if name.startswith('article_'):
idx = int(name.replace('article_', ''))
max_idx = max(max_idx, idx)
except ValueError:
continue
return max_idx
html_idx = get_max_index(html_dir, '.html')
json_idx = get_max_index(articles_dir, '.json')
return max(html_idx, json_idx) + 1
def _log_processing(source_name: str, action: str, status: str, message: str) -> None:
"""Log processing action to database.
Args:
source_name: Newspaper source name
action: Action performed
status: Status of action
message: Log message
"""
try:
with _get_db_connection() as conn:
cursor = conn.cursor()
cursor.execute(
'INSERT INTO processing_log (source_name, action, status, message) VALUES (?, ?, ?, ?)',
(source_name, action, status, message)
)
except Exception as e:
logger.error("Failed to log processing: %s", str(e))
def _save_archive_mapping(article_url: str, source_name: str, archive_file_path: str) -> None:
"""Save mapping between article URL and archive file path.
Args:
article_url: Article URL
source_name: Newspaper source name
archive_file_path: Path to archived HTML file
"""
try:
with _get_db_connection() as conn:
cursor = conn.cursor()
cursor.execute('''
INSERT OR REPLACE INTO article_archives (article_url, source_name, archive_file_path)
VALUES (?, ?, ?)
''', (article_url, source_name, archive_file_path))
logger.debug("Saved archive mapping: %s -> %s", article_url, archive_file_path)
except Exception as e:
logger.error("Failed to save archive mapping: %s", str(e))
def save_article(source_name: str, article_data: ArticleData) -> str:
"""Save article to storage and update cache.
Args:
source_name: Newspaper source name (e.g., 'reuters', 'bbc')
article_data: ArticleData object with article content
Returns:
Status message describing the result
"""
try:
_init_database()
publish_date = article_data.publish_date
if publish_date:
try:
date_obj = datetime.fromisoformat(publish_date.replace('Z', '+00:00'))
date_str = date_obj.strftime('%Y-%m-%d')
except (ValueError, AttributeError):
date_str = datetime.now().strftime('%Y-%m-%d')
else:
date_str = datetime.now().strftime('%Y-%m-%d')
html_dir, articles_dir = _ensure_directory_structure(source_name, date_str)
file_index = _get_next_file_index(html_dir, articles_dir)
file_prefix = f'article_{file_index:03d}'
archive_file_path = html_dir / f'{file_prefix}.html'
metadata_file_path = articles_dir / f'{file_prefix}.json'
if article_data.raw_html:
with open(archive_file_path, 'w', encoding='utf-8') as f:
f.write(article_data.raw_html)
metadata = {
'id': file_index,
'source_name': source_name,
'url': article_data.url,
'title': article_data.title,
'author': article_data.author,
'publish_date': article_data.publish_date,
'content_text': article_data.content_text,
'content_html': article_data.content_html,
'tags': article_data.tags,
'language': article_data.language,
'extraction_method': article_data.extraction_method,
'archive_file': f'{file_prefix}.html',
'metadata_file': f'{file_prefix}.json',
'saved_at': datetime.now().isoformat()
}
with open(metadata_file_path, 'w', encoding='utf-8') as f:
json.dump(metadata, f, indent=2, ensure_ascii=False)
with _get_db_connection() as conn:
cursor = conn.cursor()
try:
cursor.execute('''
INSERT OR IGNORE INTO articles
(source_name, article_url, article_guid, title, author, publish_date,
content_text, content_html, archive_file_path, metadata_file_path, status, extraction_method)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
''', (
source_name,
article_data.url,
getattr(article_data, 'guid', None),
article_data.title,
article_data.author,
article_data.publish_date,
article_data.content_text,
article_data.content_html,
str(archive_file_path),
str(metadata_file_path),
'archived' if not article_data.error else 'failed',
article_data.extraction_method
))
except sqlite3.IntegrityError:
cursor.execute('''
UPDATE articles
SET title = ?, author = ?, publish_date = ?,
content_text = ?, content_html = ?,
archive_file_path = ?, metadata_file_path = ?,
status = ?, extraction_method = ?, updated_at = CURRENT_TIMESTAMP
WHERE article_url = ? AND source_name = ?
''', (
article_data.title,
article_data.author,
article_data.publish_date,
article_data.content_text,
article_data.content_html,
str(archive_file_path),
str(metadata_file_path),
'archived' if not article_data.error else 'failed',
article_data.extraction_method,
article_data.url,
source_name
))
_save_archive_mapping(article_data.url, source_name, str(archive_file_path))
_log_processing(
source_name,
'save_article',
'success',
f'Saved article: {article_data.url} -> {metadata_file_path.name}'
)
logger.info("Article saved: %s -> %s", article_data.url, metadata_file_path.name)
return f"Article saved: {metadata_file_path.name}"
except Exception as e:
error_msg = f"Failed to save article: {str(e)}"
logger.error(error_msg)
_log_processing(source_name, 'save_article', 'error', error_msg)
return error_msg
def get_article(source_name: str, article_id: int) -> Optional[ArticleData]:
"""Retrieve article from storage.
Args:
source_name: Newspaper source name
article_id: Database ID of article
Returns:
ArticleData object or None if not found
"""
try:
_init_database()
with _get_db_connection() as conn:
cursor = conn.cursor()
cursor.execute('''
SELECT * FROM articles
WHERE id = ? AND source_name = ?
''', (article_id, source_name))
row = cursor.fetchone()
if not row:
return None
archive_path = Path(row['archive_file_path']) if row['archive_file_path'] else None
archive_content = None
if archive_path and archive_path.exists():
archive_content = archive_path.read_text(encoding='utf-8')
article = ArticleData(
url=row['article_url'],
title=row['title'],
author=row['author'],
publish_date=row['publish_date'],
content_text=row['content_text'],
content_html=row['content_html'],
raw_html=archive_content,
archive_file_path=str(archive_path) if archive_path else None,
tags=None,
language=None,
metadata=None,
extraction_method=None,
error=row['error_message'],
guid=row['article_guid'],
id=row['id'],
source_name=source_name
)
return article
except Exception as e:
logger.error("Failed to get article %d: %s", article_id, str(e))
return None
def get_articles_by_source(source_name: str, limit: int = 50, offset: int = 0) -> List[ArticleData]:
"""Get paginated articles for a source.
Args:
source_name: Newspaper source name
limit: Maximum number of articles to return
offset: Number of articles to skip
Returns:
List of ArticleData objects
"""
try:
_init_database()
with _get_db_connection() as conn:
cursor = conn.cursor()
cursor.execute('''
SELECT * FROM articles
WHERE source_name = ?
ORDER BY publish_date DESC, created_at DESC
LIMIT ? OFFSET ?
''', (source_name, limit, offset))
rows = cursor.fetchall()
articles = []
for row in rows:
archive_path = Path(row['archive_file_path']) if row['archive_file_path'] else None
archive_content = None
if archive_path and archive_path.exists():
archive_content = archive_path.read_text(encoding='utf-8')
article = ArticleData(
url=row['article_url'],
title=row['title'],
author=row['author'],
publish_date=row['publish_date'],
content_text=row['content_text'],
content_html=row['content_html'],
raw_html=archive_content,
archive_file_path=str(archive_path) if archive_path else None,
tags=None,
language=None,
metadata=None,
extraction_method=None,
error=row['error_message'],
guid=row['article_guid'],
id=row['id'],
source_name=source_name
)
articles.append(article)
return articles
except Exception as e:
logger.error("Failed to get articles for %s: %s", source_name, str(e))
return []
def update_article_status(source_name: str, article_url: str, status: str, error: str = None) -> None:
"""Update article status in cache.
Args:
source_name: Newspaper source name
article_url: Article URL
status: New status (pending, archived, failed)
error: Error message if status is failed
"""
try:
_init_database()
conn = _get_db_connection()
cursor = conn.cursor()
cursor.execute('''
UPDATE articles
SET status = ?, error_message = ?, updated_at = CURRENT_TIMESTAMP
WHERE article_url = ? AND source_name = ?
''', (status, error, article_url, source_name))
conn.commit()
conn.close()
if error:
_log_processing(
source_name,
'update_status',
'error',
f'Updated {article_url} status to {status}: {error}'
)
else:
_log_processing(
source_name,
'update_status',
'success',
f'Updated {article_url} status to {status}'
)
logger.info("Updated article status: %s -> %s", article_url, status)
except Exception as e:
logger.error("Failed to update article status: %s", str(e))
def get_source_stats(source_name: str) -> dict:
"""Get statistics for a news source.
Args:
source_name: Newspaper source name
Returns:
Dictionary with source statistics
"""
try:
_init_database()
conn = _get_db_connection()
cursor = conn.cursor()
cursor.execute('''
SELECT
COUNT(*) as total,
SUM(CASE WHEN status = 'archived' THEN 1 ELSE 0 END) as archived,
SUM(CASE WHEN status = 'failed' THEN 1 ELSE 0 END) as failed,
SUM(CASE WHEN status = 'pending' THEN 1 ELSE 0 END) as pending,
MIN(created_at) as first_archived,
MAX(created_at) as last_archived,
MAX(publish_date) as latest_article_date
FROM articles
WHERE source_name = ?
''', (source_name,))
row = cursor.fetchone()
conn.close()
stats = {
'source_name': source_name,
'total_articles': row['total'] or 0,
'archived': row['archived'] or 0,
'failed': row['failed'] or 0,
'pending': row['pending'] or 0,
'first_archived': row['first_archived'],
'last_archived': row['last_archived'],
'latest_article_date': row['latest_article_date']
}
_log_processing(
source_name,
'get_stats',
'success',
f'Stats: {stats["total_articles"]} total, {stats["archived"]} archived, {stats["failed"]} failed'
)
return stats
except Exception as e:
logger.error("Failed to get stats for %s: %s", source_name, str(e))
return {
'source_name': source_name,
'total_articles': 0,
'archived': 0,
'failed': 0,
'pending': 0
}
def get_all_sources() -> List[str]:
"""Get list of all sources in the archive.
Returns:
List of source names
"""
try:
_init_database()
conn = _get_db_connection()
cursor = conn.cursor()
cursor.execute('''
SELECT DISTINCT source_name FROM articles ORDER BY source_name
''')
sources = [row['source_name'] for row in cursor.fetchall()]
conn.close()
return sources
except Exception as e:
logger.error("Failed to get sources: %s", str(e))
return []
def get_latest_articles(limit: int = 50) -> List[ArticleData]:
"""Get latest articles across all sources.
Args:
limit: Maximum number of articles to return
Returns:
List of ArticleData objects ordered by publish_date DESC
"""
try:
_init_database()
conn = _get_db_connection()
cursor = conn.cursor()
cursor.execute('''
SELECT * FROM articles
ORDER BY publish_date DESC, created_at DESC
LIMIT ?
''', (limit,))
rows = cursor.fetchall()
conn.close()
articles = []
for row in rows:
archive_path = Path(row['archive_file_path']) if row['archive_file_path'] else None
archive_content = None
if archive_path and archive_path.exists():
archive_content = archive_path.read_text(encoding='utf-8')
article = ArticleData(
url=row['article_url'],
title=row['title'],
author=row['author'],
publish_date=row['publish_date'],
content_text=row['content_text'],
content_html=row['content_html'],
raw_html=archive_content,
archive_file_path=str(archive_path) if archive_path else None,
tags=None,
language=None,
metadata=None,
extraction_method=None,
error=row['error_message'],
guid=row['article_guid'],
id=row['id'],
source_name=row['source_name']
)
articles.append(article)
return articles
except Exception as e:
logger.error("Failed to get latest articles: %s", str(e))
return []
def get_source_directory(source_name: str) -> Path:
"""Get the directory path for a source.
Args:
source_name: Newspaper source name
Returns:
Path to source directory
"""
return WEBSITES_DIR / source_name
def get_archive_file_path_from_db(article_url: str, source_name: str = None) -> Optional[str]:
"""Get archive file path from database mapping.
Args:
article_url: Article URL
source_name: Newspaper source name (optional, for filtering)
Returns:
Archive file path if found, None otherwise
"""
try:
_init_database()
conn = _get_db_connection()
cursor = conn.cursor()
if source_name:
cursor.execute('''
SELECT archive_file_path FROM article_archives
WHERE article_url = ? AND source_name = ?
''', (article_url, source_name))
else:
cursor.execute('''
SELECT archive_file_path FROM article_archives
WHERE article_url = ?
''', (article_url,))
row = cursor.fetchone()
conn.close()
return row['archive_file_path'] if row else None
except Exception as e:
logger.error("Failed to get archive file path from DB: %s", str(e))
return None
def get_daily_articles(source_name: str, date_str: str) -> List[ArticleData]:
"""Get articles for a specific date.
Args:
source_name: Newspaper source name
date_str: Date string in YYYY-MM-DD format
Returns:
List of ArticleData objects
"""
try:
source_dir = get_source_directory(source_name)
articles_dir = source_dir / 'articles' / date_str
if not articles_dir.exists():
return []
articles = []
for json_file in sorted(articles_dir.glob('article_*.json')):
try:
with open(json_file, 'r', encoding='utf-8') as f:
metadata = json.load(f)
archive_filename = metadata.get('archive_file', '')
archive_path = source_dir / 'html' / date_str / archive_filename if archive_filename else None
archive_content = None
if archive_path and archive_path.exists():
archive_content = archive_path.read_text(encoding='utf-8')
article = ArticleData(
url=metadata.get('url', ''),
title=metadata.get('title'),
author=metadata.get('author'),
publish_date=metadata.get('publish_date'),
content_text=metadata.get('content_text'),
content_html=metadata.get('content_html'),
raw_html=archive_content,
archive_file_path=str(archive_path) if archive_path else None,
tags=metadata.get('tags'),
language=metadata.get('language'),
metadata=metadata,
extraction_method=metadata.get('extraction_method'),
source_name=source_name
)
articles.append(article)
except Exception as e:
logger.warning("Failed to load article from %s: %s", json_file, str(e))
continue
return articles
except Exception as e:
logger.error("Failed to get daily articles for %s on %s: %s", source_name, date_str, str(e))
return []
def initialize_storage() -> None:
"""Initialize the storage system.
Creates database, directory structure, and logs initialization.
"""
logger.info("Initializing storage system...")
_init_database()
(WEBSITES_DIR / 'sample').mkdir(parents=True, exist_ok=True)
(WEBSITES_DIR / 'sample' / 'html').mkdir(exist_ok=True)
(WEBSITES_DIR / 'sample' / 'articles').mkdir(exist_ok=True)
_log_processing('system', 'initialize', 'success', 'Storage system initialized')
logger.info("Storage system initialized at %s", ARCHIVE_DIR)
if __name__ == '__main__':
initialize_storage()
sources = get_all_sources()
print(f"Sources in archive: {sources}")
if sources:
for source in sources:
stats = get_source_stats(source)
print(f"\n{source}:")
print(f" Total: {stats['total_articles']}")
print(f" Archived: {stats['archived']}")
print(f" Failed: {stats['failed']}")

46
templates/article.html Normal file
View File

@ -0,0 +1,46 @@
{% extends "base.html" %}
{% block content %}
<article class="article-view">
<div class="article-header">
<div class="article-source">
<span class="source-label">From</span>
<span class="source-name">{{ source_name }}</span>
</div>
<h1>{{ article.title }}</h1>
<div class="article-meta">
<div class="article-meta-row">
{% if article.publish_date %}
<time class="article-date" datetime="{{ article.publish_date }}">{{ article.publish_date }}</time>
{% endif %}
{% if article.author %}
<span class="article-author">By <strong>{{ article.author }}</strong></span>
{% endif %}
</div>
<p class="article-url"><a href="{{ article.url }}" target="_blank">{{ article.url }}</a></p>
</div>
</div>
<div class="article-content">
{% if article.content_text %}
<div class="article-text">
{% set lines = article.content_text.split('\n') -%}
{%- for line in lines %}
{%- if line|trim %}
<p>{{ line|safe }}</p>
{%- endif %}
{%- endfor %}
</div>
{%- endif %}
</div>
<div class="article-actions">
<a href="/source/{{ source_slug }}" class="back-link">&larr; Back to articles</a>
{% if article.archive_file_path %}
&nbsp;|&nbsp;
<a href="/archive-file/{{ article.archive_file_path|urlencode }}" target="_blank" title="View archived copy">Archived HTML</a>
{% endif %}
</div>
</article>
{% endblock %}

11
templates/article_not_found.html Normal file
View File

@ -0,0 +1,11 @@
{% extends "base.html" %}
{% block content %}
<div class="article-not-found">
<h1>Article Not Found</h1>
<p>The article you're looking for could not be found.</p>
<p>Source: {{ slug }}</p>
<p>ID: {{ article_id }}</p>
<a href="/source/{{ slug }}" class="back-link">&larr; Back to articles</a>
</div>
{% endblock %}

35
templates/articles.html Normal file
View File

@ -0,0 +1,35 @@
{% extends "base.html" %}
{% block content %}
<h1>{{ source_name }} - Articles</h1>
<div class="pagination">
{% if pagination.has_prev %}
<a href="/source/{{ source_slug }}?page={{ pagination.prev_num }}">&laquo; Previous</a>
{% endif %}
<span>Page {{ pagination.page }} of {{ pagination.pages }}</span>
{% if pagination.has_next %}
<a href="/source/{{ source_slug }}?page={{ pagination.next_num }}">Next &raquo;</a>
{% endif %}
</div>
<ul class="article-list">
{% for article in articles %}
<li class="article-item">
<h3><a href="/source/{{ source_slug }}/article/{{ article.id }}">{{ article.title }}</a></h3>
<p class="article-date">{{ article.date }}</p>
<p class="article-summary">{{ article.summary }}</p>
</li>
{% endfor %}
</ul>
<div class="pagination">
{% if pagination.has_prev %}
<a href="/source/{{ source_slug }}?page={{ pagination.prev_num }}">&laquo; Previous</a>
{% endif %}
<span>Page {{ pagination.page }} of {{ pagination.pages }}</span>
{% if pagination.has_next %}
<a href="/source/{{ source_slug }}?page={{ pagination.next_num }}">Next &raquo;</a>
{% endif %}
</div>
{% endblock %}

25
templates/atom.xml Normal file
View File

@ -0,0 +1,25 @@
<?xml version="1.0" encoding="UTF-8"?>
<feed xmlns="http://www.w3.org/2005/Atom">
<title>{{ title|e }}</title>
<link href="{{ link|e }}" rel="alternate"/>
<link href="{{ link|e }}/atom" rel="self" type="application/atom+xml"/>
<updated>{{ updated|e }}</updated>
<id>{{ link|e }}</id>
<generator>NewsArchiver</generator>
{% for entry in entries %}
<entry>
<title>{{ entry.title|e }}</title>
<link href="{{ entry.link|e }}" rel="alternate"/>
{% if entry.published %}
<published>{{ entry.published|e }}</published>
{% endif %}
{% if entry.author %}
<author>
<name>{{ entry.author.name|e }}</name>
</author>
{% endif %}
<id>{{ entry.id|e }}</id>
<summary>{{ entry.summary|e }}</summary>
</entry>
{% endfor %}
</feed>

75
templates/base.html Normal file
View File

@ -0,0 +1,75 @@
<!DOCTYPE html>
<html lang="en" id="html" data-theme="light">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>NewsArchiver</title>
<link rel="stylesheet" href="{{ url_for('static', filename='style.css') }}">
<style>
#theme-toggle {
background: none;
border: 1px solid var(--border-color);
color: var(--primary-color);
padding: 0.5rem 1rem;
border-radius: 4px;
cursor: pointer;
font-size: 0.9rem;
transition: all 0.3s ease;
}
#theme-toggle:hover {
background-color: var(--accent-color);
color: white;
border-color: var(--accent-color);
}
.theme-icon {
margin-right: 0.5rem;
}
</style>
</head>
<body>
<header>
<h1>NewsArchiver</h1>
<nav>
<a href="/">Archives</a>
<a href="/status">Status</a>
<button id="theme-toggle" aria-label="Toggle dark mode">
<span class="theme-icon" id="theme-icon">☀️</span>
<span id="theme-text">Light</span>
</button>
</nav>
</header>
<main>
{% block content %}{% endblock %}
</main>
<script>
const html = document.getElementById('html');
const themeToggle = document.getElementById('theme-toggle');
const themeIcon = document.getElementById('theme-icon');
const themeText = document.getElementById('theme-text');
const savedTheme = localStorage.getItem('theme') || 'light';
setTheme(savedTheme);
themeToggle.addEventListener('click', () => {
const currentTheme = html.getAttribute('data-theme');
const newTheme = currentTheme === 'light' ? 'dark' : 'light';
setTheme(newTheme);
});
function setTheme(theme) {
html.setAttribute('data-theme', theme);
localStorage.setItem('theme', theme);
if (theme === 'dark') {
themeIcon.textContent = '☀️';
themeText.textContent = 'Light';
} else {
themeIcon.textContent = '🌙';
themeText.textContent = 'Dark';
}
}
</script>
</body>
</html>

31
templates/index.html Normal file
View File

@ -0,0 +1,31 @@
{% extends "base.html" %}
{% block content %}
<h1>News Archives</h1>
<p>Total sources: {{ sources|length }}</p>
<ul class="newspaper-list">
{% for source in sources %}
<li class="newspaper-item {% if source.disabled %}disabled{% endif %}">
<div class="newspaper-info">
{% if source.disabled %}
<h2 class="source-disabled" title="{{ source.disable_reason }}">
{{ source.name }}
<span class="status-badge">Disabled</span>
</h2>
{% else %}
<h2><a href="/source/{{ source.slug }}">{{ source.name }}</a></h2>
{% endif %}
<p>Articles: {{ source.article_count }}</p>
<p>Last archived: {{ source.last_archived or 'N/A' }}</p>
{% if source.disabled %}
<p class="disable-reason">{{ source.disable_reason }}</p>
{% endif %}
</div>
{% if not source.disabled %}
<button class="pull-btn" data-source="{{ source.slug }}">Pull latest</button>
{% endif %}
</li>
{% endfor %}
</ul>
{% endblock %}

25
templates/rss.xml Normal file
View File

@ -0,0 +1,25 @@
<?xml version="1.0" encoding="UTF-8"?>
<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom">
<channel>
<title>{{ title|e }}</title>
<link>{{ link|e }}</link>
<description>{{ description|e }}</description>
<lastBuildDate>{{ last_build_date|e }}</lastBuildDate>
<generator>NewsArchiver</generator>
<atom:link href="{{ link|e }}/rss" rel="self" type="application/rss+xml" />
{% for item in items %}
<item>
<title>{{ item.title|e }}</title>
<link>{{ item.link|e }}</link>
{% if item.pubDate %}
<pubDate>{{ item.pubDate|e }}</pubDate>
{% endif %}
{% if item.author %}
<author>{{ item.author|e }}</author>
{% endif %}
<guid isPermaLink="false">{{ item.guid|e }}</guid>
<description>{{ item.description|e }}</description>
</item>
{% endfor %}
</channel>
</rss>

50
templates/status.html Normal file
View File

@ -0,0 +1,50 @@
{% extends "base.html" %}
{% block content %}
<div class="status-page">
<h1>System Status</h1>
<div class="status-summary">
<div class="status-item">
<h2>System</h2>
<p>Status: <span class="status-online">Online</span></p>
<p>Last Archive Run: {{ last_archive_run or 'Never' }}</p>
</div>
<div class="status-item">
<h2>Statistics</h2>
<p>Sources Monitored: {{ sources_monitored }}</p>
{% if disabled_sources > 0 %}
<p>Disabled Sources: <span class="status-warning">{{ disabled_sources }}</span></p>
{% endif %}
<p>Total Articles: {{ total_articles }}</p>
<p>Failed Jobs: {{ failed_jobs }}</p>
</div>
</div>
{% if sources %}
<h2>Monitored Sources</h2>
<ul class="newspaper-list">
{% for source in sources %}
<li class="newspaper-item {% if source.disabled %}disabled{% endif %}">
<div class="newspaper-info">
{% if source.disabled %}
<h2 class="source-disabled" title="{{ source.disable_reason }}">
{{ source.name }}
<span class="status-badge">Disabled</span>
</h2>
{% else %}
<h2><a href="/source/{{ source.slug }}">{{ source.name }}</a></h2>
{% endif %}
<p>Articles: {{ source.article_count }}</p>
<p class="status-{{ source.status }}">{{ source.status|upper }}</p>
{% if source.disabled %}
<p class="disable-reason">{{ source.disable_reason }}</p>
{% endif %}
</div>
</li>
{% endfor %}
</ul>
{% endif %}
</div>
{% endblock %}

488
web_interface.py Normal file
View File

@ -0,0 +1,488 @@
#!/usr/bin/env python3
"""Web Interface for NewsArchiver - Phase 3
Flask web server for browsing archived news articles.
"""
import json
import logging
import sys
from datetime import datetime
from pathlib import Path
from typing import Optional
from urllib.parse import quote
from flask import Flask, jsonify, request, render_template, make_response
import xml.etree.ElementTree as ET
from datetime import datetime, timezone
from storage_manager import (
get_all_sources,
get_source_stats,
get_articles_by_source,
get_article,
get_latest_articles,
DB_PATH
)
SCRIPT_DIR = Path(__file__).parent
ARCHIVE_DIR = SCRIPT_DIR / 'archival_data'
RSS_FEEDS_PATH = SCRIPT_DIR / 'rss_feeds.json'
RSS_FEEDS = {}
def load_rss_feeds() -> dict:
"""Load RSS feeds configuration."""
global RSS_FEEDS
if RSS_FEEDS:
return RSS_FEEDS
if not RSS_FEEDS_PATH.exists():
logger.warning("RSS feeds file not found: %s", RSS_FEEDS_PATH)
return {}
try:
with open(RSS_FEEDS_PATH, 'r', encoding='utf-8') as f:
RSS_FEEDS = json.load(f)
return RSS_FEEDS
except Exception as e:
logger.error("Failed to load RSS feeds: %s", str(e))
return {}
app = Flask(
__name__,
static_folder=str(SCRIPT_DIR / 'static'),
template_folder=str(SCRIPT_DIR / 'templates')
)
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(levelname)s - %(message)s',
handlers=[
logging.StreamHandler(sys.stdout),
logging.FileHandler(ARCHIVE_DIR / 'processing.log', encoding='utf-8')
]
)
logger = logging.getLogger(__name__)
def get_pagination_info(total: int, page: int, per_page: int) -> dict:
"""Calculate pagination information.
Args:
total: Total number of items
page: Current page number
per_page: Items per page
Returns:
Dictionary with pagination details
"""
total_pages = (total + per_page - 1) // per_page if total > 0 else 1
return {
'total': total,
'page': page,
'per_page': per_page,
'has_next': page < total_pages,
'has_prev': page > 1,
'next_num': page + 1 if page < total_pages else None,
'prev_num': page - 1 if page > 1 else None,
'pages': total_pages
}
@app.route('/')
def index():
"""Newspaper listing page."""
sources = get_all_sources()
rss_feeds = load_rss_feeds()
source_list = []
disabled_sources = []
for source_name in sources:
stats = get_source_stats(source_name)
source_info = {
'name': source_name.title(),
'slug': source_name,
'article_count': stats['total_articles'],
'last_archived': stats.get('last_archived'),
'status': 'success' if stats['total_articles'] > 0 else 'pending'
}
if source_name in rss_feeds:
feed_info = rss_feeds[source_name]
if feed_info.get('disabled', False):
source_info['disabled'] = True
source_info['disable_reason'] = feed_info.get('disable_reason', 'No reason provided')
disabled_sources.append(source_info)
continue
source_list.append(source_info)
source_list.extend(disabled_sources)
return render_template('index.html', sources=source_list)
@app.route('/source/<slug>')
def articles(slug: str):
"""Article listing page for a specific source."""
page = request.args.get('page', 1, type=int)
per_page = 50
sources = get_all_sources()
source_name = None
for s in sources:
if s.lower() == slug.lower():
source_name = s
break
if not source_name:
return render_template('article_not_found.html', slug=slug, article_id=0), 404
articles_list = get_articles_by_source(source_name, limit=per_page, offset=(page - 1) * per_page)
stats = get_source_stats(source_name)
total = stats['total_articles']
pagination = get_pagination_info(total, page, per_page)
articles_data = []
for article in articles_list:
articles_data.append({
'id': getattr(article, 'id', 0),
'title': article.title or 'Untitled',
'date': article.publish_date or '',
'summary': article.content_text[:200] if article.content_text else '',
'url': f'/source/{source_name.lower()}/article/{getattr(article, "id", 0)}'
})
return render_template(
'articles.html',
source_name=source_name.title(),
source_slug=source_name.lower(),
articles=articles_data,
pagination=pagination
)
@app.route('/archive/<path:archive_path>')
def serve_archive(archive_path):
"""Serve archived HTML file."""
archive_file = ARCHIVE_DIR / archive_path
if archive_file.exists():
return archive_file.read_text(encoding='utf-8')
return 'Archive not found', 404
@app.route('/archive-file/<path:encoded_path>')
def serve_archive_file(encoded_path):
"""Serve archived HTML file from encoded path."""
import urllib.parse
from pathlib import Path
archive_path = urllib.parse.unquote(encoded_path)
archive_file = ARCHIVE_DIR / archive_path
logger.info("Archive file path: %s, exists: %s", str(archive_file), archive_file.exists())
if archive_file.exists():
return archive_file.read_text(encoding='utf-8')
return 'Archive not found', 404
@app.route('/source/<slug>/article/<int:article_id>')
def article(slug: str, article_id: int):
"""Individual article page."""
sources = get_all_sources()
source_name = None
for s in sources:
if s.lower() == slug.lower():
source_name = s
break
if not source_name:
return render_template('article_not_found.html', slug=slug, article_id=article_id), 404
article = get_article(source_name, article_id)
if not article:
return render_template('article_not_found.html', slug=slug, article_id=article_id), 404
article_data = {
'id': article_id,
'title': article.title or 'Untitled',
'publish_date': article.publish_date or '',
'author': article.author or '',
'url': article.url or '',
'content_text': article.content_text or '',
'archive_file_path': article.archive_file_path or ''
}
return render_template(
'article.html',
source_name=source_name.title(),
source_slug=source_name.lower(),
article=article_data
)
@app.route('/status')
def status():
"""System status page."""
sources = get_all_sources()
rss_feeds = load_rss_feeds()
sources_info = []
disabled_sources = []
total_articles = 0
failed_jobs = 0
disabled_count = 0
for source_name in sources:
stats = get_source_stats(source_name)
source_info = {
'name': source_name.title(),
'slug': source_name,
'article_count': stats['total_articles'],
'last_archived': stats.get('last_archived'),
'status': 'success' if stats['total_articles'] > 0 else 'pending'
}
if source_name in rss_feeds:
feed_info = rss_feeds[source_name]
if feed_info.get('disabled', False):
source_info['disabled'] = True
disabled_count += 1
disabled_sources.append(source_info)
continue
sources_info.append(source_info)
total_articles += stats['total_articles']
failed_jobs += stats['failed']
sources_info.extend(disabled_sources)
return render_template(
'status.html',
sources=sources_info,
total_articles=total_articles,
failed_jobs=failed_jobs,
sources_monitored=len(sources) - disabled_count,
disabled_sources=disabled_count
)
@app.route('/api/sources')
def api_sources():
"""API endpoint for listing all sources."""
sources = get_all_sources()
rss_feeds = load_rss_feeds()
source_list = []
disabled_sources = []
for source_name in sources:
stats = get_source_stats(source_name)
source_info = {
'name': source_name.title(),
'slug': source_name,
'article_count': stats['total_articles'],
'last_archived': stats.get('last_archived'),
'status': 'success' if stats['total_articles'] > 0 else 'pending'
}
if source_name in rss_feeds:
feed_info = rss_feeds[source_name]
if feed_info.get('disabled', False):
source_info['disabled'] = True
source_info['disable_reason'] = feed_info.get('disable_reason', 'No reason provided')
disabled_sources.append(source_info)
continue
source_list.append(source_info)
source_list.extend(disabled_sources)
return jsonify({'sources': source_list})
@app.route('/api/source/<slug>/articles')
def api_articles(slug: str):
"""API endpoint for listing articles for a source."""
page = request.args.get('page', 1, type=int)
per_page = 50
sources = get_all_sources()
source_name = None
for s in sources:
if s.lower() == slug.lower():
source_name = s
break
if not source_name:
return jsonify({'error': 'Source not found'}), 404
articles_list = get_articles_by_source(source_name, limit=per_page, offset=(page - 1) * per_page)
stats = get_source_stats(source_name)
total = stats['total_articles']
pagination = get_pagination_info(total, page, per_page)
articles_data = []
for article in articles_list:
articles_data.append({
'id': getattr(article, 'id', 0),
'title': article.title or 'Untitled',
'date': article.publish_date or '',
'summary': article.content_text[:200] if article.content_text else '',
'url': f'/source/{source_name.lower()}/article/{getattr(article, "id", 0)}'
})
return jsonify({
'source_name': source_name.title(),
'articles': articles_data,
'total': total,
'page': page,
'per_page': per_page,
'has_next': pagination['has_next'],
'has_prev': pagination['has_prev']
})
@app.route('/api/status')
def api_status():
"""API endpoint for system status."""
sources = get_all_sources()
sources_monitored = len(sources)
total_articles = 0
failed_jobs = 0
last_archive_run = None
for source_name in sources:
stats = get_source_stats(source_name)
total_articles += stats['total_articles']
failed_jobs += stats['failed']
if stats.get('last_archive_run'):
if last_archive_run is None or stats['last_archive_run'] > last_archive_run:
last_archive_run = stats['last_archive_run']
return jsonify({
'status': 'online',
'last_archive_run': last_archive_run,
'pending_jobs': 0,
'failed_jobs': failed_jobs,
'sources_monitored': sources_monitored,
'total_articles': total_articles
})
@app.route('/rss')
def rss_feed():
"""RSS 2.0 endpoint for latest archived articles."""
limit = request.args.get('limit', 50, type=int)
articles = get_latest_articles(limit=limit)
server_url = f'http://192.168.8.150:5000'
rss_items = []
for article in articles:
if article.title and article.content_text and 'Performing security verification' not in article.content_text:
pub_date = None
if article.publish_date:
try:
dt = datetime.fromisoformat(article.publish_date.replace('Z', '+00:00'))
pub_date = dt.strftime('%a, %d %b %Y %H:%M:%S %z').strip()
except (ValueError, AttributeError):
try:
dt = datetime.fromisoformat(article.publish_date)
pub_date = dt.strftime('%a, %d %b %Y %H:%M:%S +0000')
except (ValueError, AttributeError):
pub_date = datetime.now(timezone.utc).strftime('%a, %d %b %Y %H:%M:%S +0000')
source_name = article.source_name or 'unknown'
encoded_source = quote(source_name.lower())
item = {
'title': article.title,
'link': f'{server_url}/source/{encoded_source}/article/{article.id}',
'pubDate': pub_date,
'description': article.content_text[:500] if article.content_text else '',
'guid': article.url or f'article-{article.id}'
}
if article.author:
item['author'] = article.author
rss_items.append(item)
rss_template = render_template(
'rss.xml',
title='NewsArchiver - Latest Articles',
link=server_url,
description='Latest archived news articles',
last_build_date=datetime.now(timezone.utc).strftime('%a, %d %b %Y %H:%M:%S +0000'),
items=rss_items
)
response = make_response(rss_template)
response.headers['Content-Type'] = 'application/rss+xml; charset=utf-8'
return response
@app.route('/atom')
def atom_feed():
"""Atom 1.0 endpoint for latest archived articles."""
limit = request.args.get('limit', 50, type=int)
articles = get_latest_articles(limit=limit)
server_url = f'http://192.168.8.150:5000'
atom_entries = []
for article in articles:
if article.title and article.content_text and 'Performing security verification' not in article.content_text:
pub_date = None
if article.publish_date:
try:
dt = datetime.fromisoformat(article.publish_date.replace('Z', '+00:00'))
pub_date = dt.strftime('%Y-%m-%dT%H:%M:%S+00:00')
except (ValueError, AttributeError):
pub_date = datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%S+00:00')
source_name = article.source_name or 'unknown'
encoded_source = quote(source_name.lower())
entry = {
'title': article.title,
'link': f'{server_url}/source/{encoded_source}/article/{article.id}',
'published': pub_date,
'summary': article.content_text[:500] if article.content_text else '',
'id': article.url or f'article-{article.id}'
}
if article.author:
entry['author'] = {'name': article.author}
atom_entries.append(entry)
atom_template = render_template(
'atom.xml',
title='NewsArchiver - Latest Articles',
link=server_url,
updated=datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%S+00:00'),
entries=atom_entries
)
response = make_response(atom_template)
response.headers['Content-Type'] = 'application/atom+xml; charset=utf-8'
return response
if __name__ == '__main__':
logger.info("Starting web interface...")
if not DB_PATH.exists():
logger.info("Database not found, initializing...")
from storage_manager import initialize_storage
initialize_storage()
app.run(host='0.0.0.0', port=5000, debug=True)