#!/usr/bin/env python3 """Rebuild NewsArchiver database from existing HTML files.""" import json import logging import re import sqlite3 import sys from datetime import datetime from pathlib import Path from typing import Optional try: from content_extractor import parse_article_from_html, get_html_from_url from storage_manager import initialize_storage, save_article as storage_save_article except ImportError as e: print(f"ERROR: Required module not found: {e}") sys.exit(1) SCRIPT_DIR = Path(__file__).parent ARCHIVE_DIR = SCRIPT_DIR / 'archival_data' RSS_FEEDS_PATH = SCRIPT_DIR / 'rss_feeds.json' logger = logging.getLogger(__name__) def extract_urls_from_html(html_file: Path) -> list: """Extract article URLs from an archived HTML file. Args: html_file: Path to archived HTML file Returns: List of article URLs found in the file """ try: content = html_file.read_text(encoding='utf-8') urls = [] # Only extract main article URLs - skip tracking links, RSS feeds, author pages, etc. article_patterns = [ r'href=["\']https?://[^"\']+\/article\/[^"\']+["\']', r'href=["\']https?://[^"\']+\/news\/[^"\']+["\']', r'href=["\']https?://[^"\']+\/stories\/[^"\']+["\']', r'href=["\']https?://[^"\']+\/archive\/[^"\']+["\']', ] for pattern in article_patterns: matches = re.findall(pattern, content) for match in matches: url_match = re.search(r'href=["\']([^"\']+)["\']', match) if url_match: url = url_match.group(1) # Skip common non-article URLs skip_patterns = [ 'rss', 'feed', 'author', 'authors', 'tags', 'category', 'search', 'about', 'contact', 'privacy', 'terms', 'faq', 'subscribe', 'signin', 'login', 'register', 'account', 'profile' ] if not any(skip in url.lower() for skip in skip_patterns): urls.append(url) return list(set(urls)) # Remove duplicates except Exception as e: logger.error("Error reading %s: %s", html_file, str(e)) return [] def find_html_files(archive_dir: Path) -> list: """Find all HTML files in the archive directory. Args: archive_dir: Root archive directory Returns: List of HTML file paths """ html_files = [] try: websites_dir = archive_dir / 'websites' if not websites_dir.exists(): logger.warning("Websites directory not found: %s", websites_dir) return [] for source_dir in sorted(websites_dir.iterdir()): if not source_dir.is_dir(): continue html_dir = source_dir / 'html' if not html_dir.exists(): continue for date_dir in sorted(html_dir.iterdir()): if not date_dir.is_dir(): continue for html_file in sorted(date_dir.glob('article_*.html')): html_files.append(html_file) logger.info("Found %d HTML files to process", len(html_files)) except Exception as e: logger.error("Error finding HTML files: %s", str(e)) return html_files def rebuild_database(archive_dir: Path, rss_feeds_path: Path) -> dict: """Rebuild the database from existing HTML files. Args: archive_dir: Root archive directory rss_feeds_path: Path to RSS feeds JSON file Returns: Dictionary with rebuild results """ results = { 'html_files_processed': 0, 'articles_extracted': 0, 'articles_saved': 0, 'articles_failed': 0, 'errors': [] } # Initialize database logger.info("Initializing database at: %s", ARCHIVE_DIR / 'cache.db') initialize_storage() logger.info("Database initialized") # Load RSS feeds rss_feeds = {} if rss_feeds_path.exists(): with open(rss_feeds_path, 'r', encoding='utf-8') as f: rss_feeds = json.load(f) # Find all HTML files html_files = find_html_files(archive_dir) if not html_files: logger.warning("No HTML files found to process") return results # Process each HTML file for html_file in html_files: try: results['html_files_processed'] += 1 # Extract URLs from HTML urls = extract_urls_from_html(html_file) if not urls: logger.debug("No URLs found in %s", html_file.name) continue results['articles_extracted'] += len(urls) # Process each URL for url in urls: try: # Try to get source name from RSS feeds source_name = None for feed_name, feed_info in rss_feeds.items(): if feed_info.get('rss_url') and url.startswith(feed_info.get('rss_url', '')): source_name = feed_name break # Try to infer source from URL if not source_name: for feed_name, feed_info in rss_feeds.items(): website = feed_info.get('source_website', '') if website and website in url: source_name = feed_name break if not source_name: # Try to extract domain from URL domain_match = re.search(r'https?://([^/]+)', url) if domain_match: domain = domain_match.group(1) for feed_name, feed_info in rss_feeds.items(): website = feed_info.get('source_website', '') if website and website in domain: source_name = feed_name break # If still no source, skip if not source_name: logger.debug("No source found for %s, skipping", url[:60]) continue # Extract content from HTML file raw_html = html_file.read_text(encoding='utf-8') article_data = parse_article_from_html(raw_html, url) if article_data.error: logger.warning("Failed to extract content from %s: %s", url[:60], article_data.error) results['articles_failed'] += 1 continue # Save to database logger.info("Saving article: %s -> %s", url[:80], source_name) storage_save_article(source_name, article_data) results['articles_saved'] += 1 if results['articles_saved'] % 100 == 0: logger.info("Saved %d articles so far", results['articles_saved']) except Exception as e: logger.error("Error processing URL %s: %s", url, str(e)) results['articles_failed'] += 1 results['errors'].append({ 'url': url, 'error': str(e) }) except Exception as e: logger.error("Error processing file %s: %s", html_file, str(e)) results['errors'].append({ 'file': str(html_file), 'error': str(e) }) return results if __name__ == '__main__': logger.info("=" * 60) logger.info("Rebuilding NewsArchiver Database") logger.info("=" * 60) results = rebuild_database(ARCHIVE_DIR, RSS_FEEDS_PATH) logger.info("=" * 60) logger.info("Rebuild Complete") logger.info("=" * 60) logger.info("HTML files processed: %d", results['html_files_processed']) logger.info("Articles extracted: %d", results['articles_extracted']) logger.info("Articles saved: %d", results['articles_saved']) logger.info("Articles failed: %d", results['articles_failed']) if results['errors']: logger.info("Errors:") for error in results['errors'][:20]: logger.info(" - %s", error)