113 lines
3.6 KiB
Python
113 lines
3.6 KiB
Python
#!/usr/bin/env python3
|
|
"""Restore database from existing JSON metadata files."""
|
|
|
|
import json
|
|
import logging
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
try:
|
|
from storage_manager import initialize_storage, save_article
|
|
from content_extractor import ArticleData
|
|
except ImportError as e:
|
|
print(f"ERROR: Required module not found: {e}")
|
|
sys.exit(1)
|
|
|
|
SCRIPT_DIR = Path(__file__).parent
|
|
ARCHIVE_DIR = SCRIPT_DIR / 'archival_data'
|
|
|
|
logging.basicConfig(
|
|
level=logging.INFO,
|
|
format='%(asctime)s - %(levelname)s - %(message)s',
|
|
handlers=[
|
|
logging.StreamHandler(sys.stdout),
|
|
logging.FileHandler(ARCHIVE_DIR / 'restore.log', encoding='utf-8')
|
|
]
|
|
)
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def restore_database(archive_dir: Path) -> dict:
|
|
"""Restore database from JSON metadata files."""
|
|
results = {
|
|
'json_files_found': 0,
|
|
'articles_restored': 0,
|
|
'articles_failed': 0,
|
|
'errors': []
|
|
}
|
|
|
|
initialize_storage()
|
|
logger.info("Database initialized")
|
|
|
|
json_files = list(archive_dir.glob('websites/**/*.json'))
|
|
results['json_files_found'] = len(json_files)
|
|
|
|
logger.info(f"Found {len(json_files)} JSON files to process")
|
|
|
|
for json_file in json_files:
|
|
try:
|
|
with open(json_file, 'r', encoding='utf-8') as f:
|
|
metadata = json.load(f)
|
|
|
|
url = metadata.get('url')
|
|
source_name = metadata.get('source_name')
|
|
title = metadata.get('title')
|
|
author = metadata.get('author')
|
|
publish_date = metadata.get('publish_date')
|
|
content_text = metadata.get('content_text')
|
|
content_html = metadata.get('content_html')
|
|
tags = metadata.get('tags', [])
|
|
extraction_method = metadata.get('extraction_method', 'unknown')
|
|
|
|
if not url or not source_name:
|
|
logger.warning(f"Missing URL or source in {json_file.name}, skipping")
|
|
results['articles_failed'] += 1
|
|
continue
|
|
|
|
article_data = ArticleData(
|
|
url=url,
|
|
title=title,
|
|
author=author,
|
|
publish_date=publish_date,
|
|
content_text=content_text,
|
|
content_html=content_html,
|
|
tags=tags,
|
|
extraction_method=extraction_method
|
|
)
|
|
|
|
save_article(source_name, article_data)
|
|
results['articles_restored'] += 1
|
|
|
|
if results['articles_restored'] % 100 == 0:
|
|
logger.info(f"Restored {results['articles_restored']} articles so far")
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error processing {json_file.name}: {str(e)}")
|
|
results['articles_failed'] += 1
|
|
results['errors'].append({
|
|
'file': str(json_file),
|
|
'error': str(e)
|
|
})
|
|
|
|
return results
|
|
|
|
|
|
if __name__ == '__main__':
|
|
logger.info("=" * 60)
|
|
logger.info("Restoring NewsArchiver Database from JSON files")
|
|
logger.info("=" * 60)
|
|
|
|
results = restore_database(ARCHIVE_DIR)
|
|
|
|
logger.info("=" * 60)
|
|
logger.info("Restore Complete")
|
|
logger.info("=" * 60)
|
|
logger.info(f"JSON files found: {results['json_files_found']}")
|
|
logger.info(f"Articles restored: {results['articles_restored']}")
|
|
logger.info(f"Articles failed: {results['articles_failed']}")
|
|
|
|
if results['errors']:
|
|
logger.info("Errors:")
|
|
for error in results['errors'][:20]:
|
|
logger.info(f" - {error}")
|