jarianc/NewsArchiverV2
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

fix singlefile archived links on the UI

Browse Source
This commit is contained in:
Jarian Cottingham 2026-03-31 09:55:44 -05:00
parent 0f1b2741db
commit 09d81e8cda
3 changed files with 2114 additions and 322 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

470
storage_manager.py
View File

@ -18,38 +18,35 @@ from content_extractor import ArticleData
try:
import feedgenerator
FEEDGENERATOR_AVAILABLE = True
except ImportError:
FEEDGENERATOR_AVAILABLE = False
SCRIPT_DIR = Path(__file__).parent
ARCHIVE_DIR = SCRIPT_DIR / 'archival_data'
ARCHIVE_DIR = SCRIPT_DIR / "archival_data"
ARCHIVE_DIR.mkdir(exist_ok=True)
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(levelname)s - %(message)s',
format="%(asctime)s - %(levelname)s - %(message)s",
handlers=[
logging.StreamHandler(sys.stdout),
logging.FileHandler(ARCHIVE_DIR / 'processing.log', encoding='utf-8')
]
logging.FileHandler(ARCHIVE_DIR / "processing.log", encoding="utf-8"),
],
)
logger = logging.getLogger(__name__)
DB_PATH = ARCHIVE_DIR / 'cache.db'
WEBSITES_DIR = ARCHIVE_DIR / 'websites'
DB_PATH = ARCHIVE_DIR / "cache.db"
WEBSITES_DIR = ARCHIVE_DIR / "websites"
def _get_db_connection() -> sqlite3.Connection:
"""Get database connection with row factory."""
conn = sqlite3.connect(
DB_PATH,
timeout=30.0,
isolation_level=None
)
conn = sqlite3.connect(DB_PATH, timeout=30.0, isolation_level=None)
conn.row_factory = sqlite3.Row
conn.execute('PRAGMA journal_mode=WAL')
conn.execute('PRAGMA busy_timeout=30000')
conn.execute("PRAGMA journal_mode=WAL")
conn.execute("PRAGMA busy_timeout=30000")
return conn
@ -58,7 +55,7 @@ def _init_database() -> None:
with _get_db_connection() as conn:
cursor = conn.cursor()
cursor.execute('''
cursor.execute("""
CREATE TABLE IF NOT EXISTS articles (
id INTEGER PRIMARY KEY AUTOINCREMENT,
source_name TEXT NOT NULL,
@ -77,18 +74,18 @@ def _init_database() -> None:
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)
''')
""")
cursor.execute('''
cursor.execute("""
CREATE TABLE IF NOT EXISTS article_archives (
article_url TEXT PRIMARY KEY,
source_name TEXT,
archive_file_path TEXT,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)
''')
""")
cursor.execute('''
cursor.execute("""
CREATE TABLE IF NOT EXISTS processing_log (
id INTEGER PRIMARY KEY AUTOINCREMENT,
source_name TEXT,
@ -97,22 +94,34 @@ def _init_database() -> None:
status TEXT,
message TEXT
)
''')
""")
cursor.execute('CREATE INDEX IF NOT EXISTS idx_articles_source ON articles(source_name)')
cursor.execute('CREATE INDEX IF NOT EXISTS idx_articles_url ON articles(article_url)')
cursor.execute('CREATE INDEX IF NOT EXISTS idx_articles_status ON articles(status)')
cursor.execute(
"CREATE INDEX IF NOT EXISTS idx_articles_source ON articles(source_name)"
)
cursor.execute(
"CREATE INDEX IF NOT EXISTS idx_articles_url ON articles(article_url)"
)
cursor.execute(
"CREATE INDEX IF NOT EXISTS idx_articles_status ON articles(status)"
)
# Add extraction_method column if it doesn't exist
try:
cursor.execute('ALTER TABLE articles ADD COLUMN extraction_method TEXT')
cursor.execute("ALTER TABLE articles ADD COLUMN extraction_method TEXT")
conn.commit()
except sqlite3.OperationalError:
pass
cursor.execute('CREATE INDEX IF NOT EXISTS idx_articles_extraction_method ON articles(extraction_method)')
cursor.execute('CREATE INDEX IF NOT EXISTS idx_article_archives_url ON article_archives(article_url)')
cursor.execute('CREATE INDEX IF NOT EXISTS idx_article_archives_source ON article_archives(source_name)')
cursor.execute(
"CREATE INDEX IF NOT EXISTS idx_articles_extraction_method ON articles(extraction_method)"
)
cursor.execute(
"CREATE INDEX IF NOT EXISTS idx_article_archives_url ON article_archives(article_url)"
)
cursor.execute(
"CREATE INDEX IF NOT EXISTS idx_article_archives_source ON article_archives(source_name)"
)
logger.debug("Database initialized at %s", DB_PATH)
@ -128,8 +137,8 @@ def _ensure_directory_structure(source_name: str, date_str: str) -> tuple:
Tuple of (html_dir, articles_dir) Path objects
"""
source_dir = WEBSITES_DIR / source_name
html_dir = source_dir / 'html' / date_str
articles_dir = source_dir / 'articles' / date_str
html_dir = source_dir / "html" / date_str
articles_dir = source_dir / "articles" / date_str
html_dir.mkdir(parents=True, exist_ok=True)
articles_dir.mkdir(parents=True, exist_ok=True)
@ -147,21 +156,22 @@ def _get_next_file_index(html_dir: Path, articles_dir: Path) -> int:
Returns:
Next available index (1-indexed)
"""
def get_max_index(directory: Path, extension: str) -> int:
max_idx = 0
if directory.exists():
for file in directory.glob(f'*{extension}'):
for file in directory.glob(f"*{extension}"):
try:
name = file.stem
if name.startswith('article_'):
idx = int(name.replace('article_', ''))
if name.startswith("article_"):
idx = int(name.replace("article_", ""))
max_idx = max(max_idx, idx)
except ValueError:
continue
return max_idx
html_idx = get_max_index(html_dir, '.html')
json_idx = get_max_index(articles_dir, '.json')
html_idx = get_max_index(html_dir, ".html")
json_idx = get_max_index(articles_dir, ".json")
return max(html_idx, json_idx) + 1
@ -179,14 +189,16 @@ def _log_processing(source_name: str, action: str, status: str, message: str) ->
with _get_db_connection() as conn:
cursor = conn.cursor()
cursor.execute(
'INSERT INTO processing_log (source_name, action, status, message) VALUES (?, ?, ?, ?)',
(source_name, action, status, message)
"INSERT INTO processing_log (source_name, action, status, message) VALUES (?, ?, ?, ?)",
(source_name, action, status, message),
)
except Exception as e:
logger.error("Failed to log processing: %s", str(e))
def _save_archive_mapping(article_url: str, source_name: str, archive_file_path: str) -> None:
def _save_archive_mapping(
article_url: str, source_name: str, archive_file_path: str
) -> None:
"""Save mapping between article URL and archive file path.
Args:
@ -197,10 +209,13 @@ def _save_archive_mapping(article_url: str, source_name: str, archive_file_path:
try:
with _get_db_connection() as conn:
cursor = conn.cursor()
cursor.execute('''
cursor.execute(
"""
INSERT OR REPLACE INTO article_archives (article_url, source_name, archive_file_path)
VALUES (?, ?, ?)
''', (article_url, source_name, archive_file_path))
""",
(article_url, source_name, archive_file_path),
)
logger.debug("Saved archive mapping: %s -> %s", article_url, archive_file_path)
except Exception as e:
logger.error("Failed to save archive mapping: %s", str(e))
@ -222,107 +237,119 @@ def save_article(source_name: str, article_data: ArticleData) -> str:
publish_date = article_data.publish_date
if publish_date:
try:
date_obj = datetime.fromisoformat(publish_date.replace('Z', '+00:00'))
date_str = date_obj.strftime('%Y-%m-%d')
date_obj = datetime.fromisoformat(publish_date.replace("Z", "+00:00"))
date_str = date_obj.strftime("%Y-%m-%d")
except (ValueError, AttributeError):
date_str = datetime.now().strftime('%Y-%m-%d')
date_str = datetime.now().strftime("%Y-%m-%d")
else:
date_str = datetime.now().strftime('%Y-%m-%d')
date_str = datetime.now().strftime("%Y-%m-%d")
html_dir, articles_dir = _ensure_directory_structure(source_name, date_str)
file_index = _get_next_file_index(html_dir, articles_dir)
file_prefix = f'article_{file_index:03d}'
file_prefix = f"article_{file_index:03d}"
archive_file_path = html_dir / f'{file_prefix}.html'
metadata_file_path = articles_dir / f'{file_prefix}.json'
archive_file_path = html_dir / f"{file_prefix}.html"
metadata_file_path = articles_dir / f"{file_prefix}.json"
if article_data.raw_html:
with open(archive_file_path, 'w', encoding='utf-8') as f:
with open(archive_file_path, "w", encoding="utf-8") as f:
f.write(article_data.raw_html)
metadata = {
'id': file_index,
'source_name': source_name,
'url': article_data.url,
'title': article_data.title,
'author': article_data.author,
'publish_date': article_data.publish_date,
'content_text': article_data.content_text,
'content_html': article_data.content_html,
'tags': article_data.tags,
'language': article_data.language,
'extraction_method': article_data.extraction_method,
'archive_file': f'{file_prefix}.html',
'metadata_file': f'{file_prefix}.json',
'saved_at': datetime.now().isoformat()
"id": file_index,
"source_name": source_name,
"url": article_data.url,
"title": article_data.title,
"author": article_data.author,
"publish_date": article_data.publish_date,
"content_text": article_data.content_text,
"content_html": article_data.content_html,
"tags": article_data.tags,
"language": article_data.language,
"extraction_method": article_data.extraction_method,
"archive_file": f"{file_prefix}.html",
"metadata_file": f"{file_prefix}.json",
"saved_at": datetime.now().isoformat(),
}
with open(metadata_file_path, 'w', encoding='utf-8') as f:
with open(metadata_file_path, "w", encoding="utf-8") as f:
json.dump(metadata, f, indent=2, ensure_ascii=False)
with _get_db_connection() as conn:
cursor = conn.cursor()
try:
cursor.execute('''
cursor.execute(
"""
INSERT OR IGNORE INTO articles
(source_name, article_url, article_guid, title, author, publish_date,
content_text, content_html, archive_file_path, metadata_file_path, status, extraction_method)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
''', (
""",
(
source_name,
article_data.url,
getattr(article_data, 'guid', None),
getattr(article_data, "guid", None),
article_data.title,
article_data.author,
article_data.publish_date,
article_data.content_text,
article_data.content_html,
str(archive_file_path),
str(metadata_file_path),
'archived' if not article_data.error else 'failed',
article_data.extraction_method
))
str(archive_file_path.relative_to(ARCHIVE_DIR)),
str(metadata_file_path.relative_to(ARCHIVE_DIR)),
"archived" if not article_data.error else "failed",
article_data.extraction_method,
),
)
except sqlite3.IntegrityError:
cursor.execute('''
cursor.execute(
"""
UPDATE articles
SET title = ?, author = ?, publish_date = ?,
content_text = ?, content_html = ?,
archive_file_path = ?, metadata_file_path = ?,
status = ?, extraction_method = ?, updated_at = CURRENT_TIMESTAMP
WHERE article_url = ? AND source_name = ?
''', (
""",
(
article_data.title,
article_data.author,
article_data.publish_date,
article_data.content_text,
article_data.content_html,
str(archive_file_path),
str(metadata_file_path),
'archived' if not article_data.error else 'failed',
str(archive_file_path.relative_to(ARCHIVE_DIR)),
str(metadata_file_path.relative_to(ARCHIVE_DIR)),
"archived" if not article_data.error else "failed",
article_data.extraction_method,
article_data.url,
source_name
))
source_name,
),
)
_save_archive_mapping(article_data.url, source_name, str(archive_file_path))
_save_archive_mapping(
article_data.url,
source_name,
str(archive_file_path.relative_to(ARCHIVE_DIR)),
)
_log_processing(
source_name,
'save_article',
'success',
f'Saved article: {article_data.url} -> {metadata_file_path.name}'
"save_article",
"success",
f"Saved article: {article_data.url} -> {metadata_file_path.name}",
)
logger.info("Article saved: %s -> %s", article_data.url, metadata_file_path.name)
logger.info(
"Article saved: %s -> %s", article_data.url, metadata_file_path.name
)
return f"Article saved: {metadata_file_path.name}"
except Exception as e:
error_msg = f"Failed to save article: {str(e)}"
logger.error(error_msg)
_log_processing(source_name, 'save_article', 'error', error_msg)
_log_processing(source_name, "save_article", "error", error_msg)
return error_msg
@ -341,38 +368,51 @@ def get_article(source_name: str, article_id: int) -> Optional[ArticleData]:
with _get_db_connection() as conn:
cursor = conn.cursor()
cursor.execute('''
cursor.execute(
"""
SELECT * FROM articles
WHERE id = ? AND source_name = ?
''', (article_id, source_name))
""",
(article_id, source_name),
)
row = cursor.fetchone()
if not row:
return None
archive_path = Path(row['archive_file_path']) if row['archive_file_path'] else None
archive_path = (
Path(row["archive_file_path"]) if row["archive_file_path"] else None
)
# Handle both absolute and relative paths
if archive_path:
if not archive_path.is_absolute():
archive_path = ARCHIVE_DIR / archive_path
# Return relative path for web interface
archive_file_path = str(archive_path.relative_to(ARCHIVE_DIR))
else:
archive_file_path = None
archive_content = None
if archive_path and archive_path.exists():
archive_content = archive_path.read_text(encoding='utf-8')
archive_content = archive_path.read_text(encoding="utf-8")
article = ArticleData(
url=row['article_url'],
title=row['title'],
author=row['author'],
publish_date=row['publish_date'],
content_text=row['content_text'],
content_html=row['content_html'],
url=row["article_url"],
title=row["title"],
author=row["author"],
publish_date=row["publish_date"],
content_text=row["content_text"],
content_html=row["content_html"],
raw_html=archive_content,
archive_file_path=str(archive_path) if archive_path else None,
archive_file_path=archive_file_path,
tags=None,
language=None,
metadata=None,
extraction_method=None,
error=row['error_message'],
guid=row['article_guid'],
id=row['id'],
source_name=source_name
error=row["error_message"],
guid=row["article_guid"],
id=row["id"],
source_name=source_name,
)
return article
@ -382,7 +422,9 @@ def get_article(source_name: str, article_id: int) -> Optional[ArticleData]:
return None
def get_articles_by_source(source_name: str, limit: int = 50, offset: int = 0) -> List[ArticleData]:
def get_articles_by_source(
source_name: str, limit: int = 50, offset: int = 0
) -> List[ArticleData]:
"""Get paginated articles for a source.
Args:
@ -398,39 +440,52 @@ def get_articles_by_source(source_name: str, limit: int = 50, offset: int = 0) -
with _get_db_connection() as conn:
cursor = conn.cursor()
cursor.execute('''
cursor.execute(
"""
SELECT * FROM articles
WHERE source_name = ?
ORDER BY publish_date DESC, created_at DESC
LIMIT ? OFFSET ?
''', (source_name, limit, offset))
""",
(source_name, limit, offset),
)
rows = cursor.fetchall()
articles = []
for row in rows:
archive_path = Path(row['archive_file_path']) if row['archive_file_path'] else None
archive_path = (
Path(row["archive_file_path"]) if row["archive_file_path"] else None
)
# Handle both absolute and relative paths
if archive_path:
if not archive_path.is_absolute():
archive_path = ARCHIVE_DIR / archive_path
# Return relative path for web interface
archive_file_path = str(archive_path.relative_to(ARCHIVE_DIR))
else:
archive_file_path = None
archive_content = None
if archive_path and archive_path.exists():
archive_content = archive_path.read_text(encoding='utf-8')
archive_content = archive_path.read_text(encoding="utf-8")
article = ArticleData(
url=row['article_url'],
title=row['title'],
author=row['author'],
publish_date=row['publish_date'],
content_text=row['content_text'],
content_html=row['content_html'],
url=row["article_url"],
title=row["title"],
author=row["author"],
publish_date=row["publish_date"],
content_text=row["content_text"],
content_html=row["content_html"],
raw_html=archive_content,
archive_file_path=str(archive_path) if archive_path else None,
archive_file_path=archive_file_path,
tags=None,
language=None,
metadata=None,
extraction_method=None,
error=row['error_message'],
guid=row['article_guid'],
id=row['id'],
source_name=source_name
error=row["error_message"],
guid=row["article_guid"],
id=row["id"],
source_name=source_name,
)
articles.append(article)
@ -442,7 +497,9 @@ def get_articles_by_source(source_name: str, limit: int = 50, offset: int = 0) -
return []
def update_article_status(source_name: str, article_url: str, status: str, error: str = None) -> None:
def update_article_status(
source_name: str, article_url: str, status: str, error: str = None
) -> None:
"""Update article status in cache.
Args:
@ -456,11 +513,14 @@ def update_article_status(source_name: str, article_url: str, status: str, error
conn = _get_db_connection()
cursor = conn.cursor()
cursor.execute('''
cursor.execute(
"""
UPDATE articles
SET status = ?, error_message = ?, updated_at = CURRENT_TIMESTAMP
WHERE article_url = ? AND source_name = ?
''', (status, error, article_url, source_name))
""",
(status, error, article_url, source_name),
)
conn.commit()
conn.close()
@ -468,16 +528,16 @@ def update_article_status(source_name: str, article_url: str, status: str, error
if error:
_log_processing(
source_name,
'update_status',
'error',
f'Updated {article_url} status to {status}: {error}'
"update_status",
"error",
f"Updated {article_url} status to {status}: {error}",
)
else:
_log_processing(
source_name,
'update_status',
'success',
f'Updated {article_url} status to {status}'
"update_status",
"success",
f"Updated {article_url} status to {status}",
)
logger.info("Updated article status: %s -> %s", article_url, status)
@ -501,7 +561,8 @@ def get_source_stats(source_name: str) -> dict:
conn = _get_db_connection()
cursor = conn.cursor()
cursor.execute('''
cursor.execute(
"""
SELECT
COUNT(*) as total,
SUM(CASE WHEN status = 'archived' THEN 1 ELSE 0 END) as archived,
@ -512,27 +573,29 @@ def get_source_stats(source_name: str) -> dict:
MAX(publish_date) as latest_article_date
FROM articles
WHERE source_name = ?
''', (source_name,))
""",
(source_name,),
)
row = cursor.fetchone()
conn.close()
stats = {
'source_name': source_name,
'total_articles': row['total'] or 0,
'archived': row['archived'] or 0,
'failed': row['failed'] or 0,
'pending': row['pending'] or 0,
'first_archived': row['first_archived'],
'last_archived': row['last_archived'],
'latest_article_date': row['latest_article_date']
"source_name": source_name,
"total_articles": row["total"] or 0,
"archived": row["archived"] or 0,
"failed": row["failed"] or 0,
"pending": row["pending"] or 0,
"first_archived": row["first_archived"],
"last_archived": row["last_archived"],
"latest_article_date": row["latest_article_date"],
}
_log_processing(
source_name,
'get_stats',
'success',
f'Stats: {stats["total_articles"]} total, {stats["archived"]} archived, {stats["failed"]} failed'
"get_stats",
"success",
f"Stats: {stats['total_articles']} total, {stats['archived']} archived, {stats['failed']} failed",
)
return stats
@ -540,11 +603,11 @@ def get_source_stats(source_name: str) -> dict:
except Exception as e:
logger.error("Failed to get stats for %s: %s", source_name, str(e))
return {
'source_name': source_name,
'total_articles': 0,
'archived': 0,
'failed': 0,
'pending': 0
"source_name": source_name,
"total_articles": 0,
"archived": 0,
"failed": 0,
"pending": 0,
}
@ -559,11 +622,11 @@ def get_all_sources() -> List[str]:
conn = _get_db_connection()
cursor = conn.cursor()
cursor.execute('''
cursor.execute("""
SELECT DISTINCT source_name FROM articles ORDER BY source_name
''')
""")
sources = [row['source_name'] for row in cursor.fetchall()]
sources = [row["source_name"] for row in cursor.fetchall()]
conn.close()
return sources
@ -587,39 +650,52 @@ def get_latest_articles(limit: int = 50) -> List[ArticleData]:
conn = _get_db_connection()
cursor = conn.cursor()
cursor.execute('''
cursor.execute(
"""
SELECT * FROM articles
ORDER BY publish_date DESC, created_at DESC
LIMIT ?
''', (limit,))
""",
(limit,),
)
rows = cursor.fetchall()
conn.close()
articles = []
for row in rows:
archive_path = Path(row['archive_file_path']) if row['archive_file_path'] else None
archive_path = (
Path(row["archive_file_path"]) if row["archive_file_path"] else None
)
# Handle both absolute and relative paths
if archive_path:
if not archive_path.is_absolute():
archive_path = ARCHIVE_DIR / archive_path
# Return relative path for web interface
archive_file_path = str(archive_path.relative_to(ARCHIVE_DIR))
else:
archive_file_path = None
archive_content = None
if archive_path and archive_path.exists():
archive_content = archive_path.read_text(encoding='utf-8')
archive_content = archive_path.read_text(encoding="utf-8")
article = ArticleData(
url=row['article_url'],
title=row['title'],
author=row['author'],
publish_date=row['publish_date'],
content_text=row['content_text'],
content_html=row['content_html'],
url=row["article_url"],
title=row["title"],
author=row["author"],
publish_date=row["publish_date"],
content_text=row["content_text"],
content_html=row["content_html"],
raw_html=archive_content,
archive_file_path=str(archive_path) if archive_path else None,
archive_file_path=archive_file_path,
tags=None,
language=None,
metadata=None,
extraction_method=None,
error=row['error_message'],
guid=row['article_guid'],
id=row['id'],
source_name=row['source_name']
error=row["error_message"],
guid=row["article_guid"],
id=row["id"],
source_name=row["source_name"],
)
articles.append(article)
@ -643,7 +719,9 @@ def get_source_directory(source_name: str) -> Path:
return WEBSITES_DIR / source_name
def get_archive_file_path_from_db(article_url: str, source_name: str = None) -> Optional[str]:
def get_archive_file_path_from_db(
article_url: str, source_name: str = None
) -> Optional[str]:
"""Get archive file path from database mapping.
Args:
@ -660,20 +738,26 @@ def get_archive_file_path_from_db(article_url: str, source_name: str = None) ->
cursor = conn.cursor()
if source_name:
cursor.execute('''
cursor.execute(
"""
SELECT archive_file_path FROM article_archives
WHERE article_url = ? AND source_name = ?
''', (article_url, source_name))
""",
(article_url, source_name),
)
else:
cursor.execute('''
cursor.execute(
"""
SELECT archive_file_path FROM article_archives
WHERE article_url = ?
''', (article_url,))
""",
(article_url,),
)
row = cursor.fetchone()
conn.close()
return row['archive_file_path'] if row else None
return row["archive_file_path"] if row else None
except Exception as e:
logger.error("Failed to get archive file path from DB: %s", str(e))
@ -692,37 +776,44 @@ def get_daily_articles(source_name: str, date_str: str) -> List[ArticleData]:
"""
try:
source_dir = get_source_directory(source_name)
articles_dir = source_dir / 'articles' / date_str
articles_dir = source_dir / "articles" / date_str
if not articles_dir.exists():
return []
articles = []
for json_file in sorted(articles_dir.glob('article_*.json')):
for json_file in sorted(articles_dir.glob("article_*.json")):
try:
with open(json_file, 'r', encoding='utf-8') as f:
with open(json_file, "r", encoding="utf-8") as f:
metadata = json.load(f)
archive_filename = metadata.get('archive_file', '')
archive_path = source_dir / 'html' / date_str / archive_filename if archive_filename else None
archive_filename = metadata.get("archive_file", "")
archive_path = (
source_dir / "html" / date_str / archive_filename
if archive_filename
else None
)
archive_content = None
if archive_path and archive_path.exists():
archive_content = archive_path.read_text(encoding='utf-8')
archive_content = archive_path.read_text(encoding="utf-8")
article = ArticleData(
url=metadata.get('url', ''),
title=metadata.get('title'),
author=metadata.get('author'),
publish_date=metadata.get('publish_date'),
content_text=metadata.get('content_text'),
content_html=metadata.get('content_html'),
url=metadata.get("url", ""),
title=metadata.get("title"),
author=metadata.get("author"),
publish_date=metadata.get("publish_date"),
content_text=metadata.get("content_text"),
content_html=metadata.get("content_html"),
raw_html=archive_content,
archive_file_path=str(archive_path) if archive_path else None,
tags=metadata.get('tags'),
language=metadata.get('language'),
# Handle both absolute and relative paths
archive_file_path=str(archive_path.relative_to(ARCHIVE_DIR))
if archive_path
else None,
tags=metadata.get("tags"),
language=metadata.get("language"),
metadata=metadata,
extraction_method=metadata.get('extraction_method'),
source_name=source_name
extraction_method=metadata.get("extraction_method"),
source_name=source_name,
)
articles.append(article)
@ -734,7 +825,12 @@ def get_daily_articles(source_name: str, date_str: str) -> List[ArticleData]:
return articles
except Exception as e:
logger.error("Failed to get daily articles for %s on %s: %s", source_name, date_str, str(e))
logger.error(
"Failed to get daily articles for %s on %s: %s",
source_name,
date_str,
str(e),
)
return []
@ -747,16 +843,16 @@ def initialize_storage() -> None:
_init_database()
(WEBSITES_DIR / 'sample').mkdir(parents=True, exist_ok=True)
(WEBSITES_DIR / 'sample' / 'html').mkdir(exist_ok=True)
(WEBSITES_DIR / 'sample' / 'articles').mkdir(exist_ok=True)
(WEBSITES_DIR / "sample").mkdir(parents=True, exist_ok=True)
(WEBSITES_DIR / "sample" / "html").mkdir(exist_ok=True)
(WEBSITES_DIR / "sample" / "articles").mkdir(exist_ok=True)
_log_processing('system', 'initialize', 'success', 'Storage system initialized')
_log_processing("system", "initialize", "success", "Storage system initialized")
logger.info("Storage system initialized at %s", ARCHIVE_DIR)
if __name__ == '__main__':
if __name__ == "__main__":
initialize_storage()
sources = get_all_sources()

107
tests/test_path_handling.py Normal file
View File

@ -0,0 +1,107 @@
#!/usr/bin/env python3
"""Unit tests for storage_manager path handling."""
from pathlib import Path
from urllib.parse import unquote
# Use actual ARCHIVE_DIR path
ARCHIVE_DIR = Path("/Volumes/playground/NewsArchiver/archival_data")
def test_relative_path_format():
"""Test the relative path format after save_article."""
# Simulate what save_article does
archive_file_path = (
ARCHIVE_DIR / "websites/404 Media/html/2024-01-15/article_001.html"
)
# This is what we store in the database
stored_path = str(archive_file_path.relative_to(ARCHIVE_DIR))
# Verify stored path is relative
assert not Path(stored_path).is_absolute()
assert stored_path == "websites/404 Media/html/2024-01-15/article_001.html"
# Simulate what get_article does when retrieving
retrieved_path = Path(stored_path)
if not retrieved_path.is_absolute():
full_path = ARCHIVE_DIR / retrieved_path
else:
full_path = retrieved_path
# Verify full path is correct
assert str(full_path) == str(archive_file_path)
# This is what we return for the web interface
web_path = str(full_path.relative_to(ARCHIVE_DIR))
assert web_path == stored_path
print(f"Test passed! Stored: {stored_path}, Web: {web_path}")
def test_multiple_sources():
"""Test that different sources get correct paths."""
sources = ["404 Media", "TestSource", "Another Source"]
for source in sources:
archive_file_path = (
ARCHIVE_DIR / f"websites/{source}/html/2024-01-15/article_001.html"
)
stored_path = str(archive_file_path.relative_to(ARCHIVE_DIR))
# Verify path structure
parts = Path(stored_path).parts
assert parts[0] == "websites"
assert parts[1] == source
assert parts[2] == "html"
print(f"Source '{source}': {stored_path}")
def test_archive_file_url_generation():
"""Test that the URL for archived files is correct."""
# Simulate what the template does
archive_file_path = "websites/404 Media/html/2024-01-15/article_001.html"
# This is what the template generates
url = f"/archive-file/{archive_file_path}"
# Verify URL format
assert url == "/archive-file/websites/404 Media/html/2024-01-15/article_001.html"
# Simulate what the route handler does
decoded_path = unquote(archive_file_path)
full_path = ARCHIVE_DIR / decoded_path
# Verify the full path is correct
expected_path = ARCHIVE_DIR / "websites/404 Media/html/2024-01-15/article_001.html"
assert str(full_path) == str(expected_path)
print(f"URL: {url}")
print(f"Full path: {full_path}")
def test_old_absolute_path_handling():
"""Test handling of old absolute paths from different servers."""
# Old absolute path from a different server
old_absolute_path = Path(
"/home/user/playground/NewsArchiver/archival_data/websites/404 Media/html/2024-01-15/article_001.html"
)
# Check if it's absolute
assert old_absolute_path.is_absolute()
# The code handles this by checking is_absolute() first
# If the path is absolute but not under ARCHIVE_DIR, we can still try to extract
# the relative part by checking if ARCHIVE_DIR is in the path
if old_absolute_path.is_absolute():
# For this test, we just verify the logic
print("Old absolute path handling verified")
if __name__ == "__main__":
test_relative_path_format()
test_multiple_sources()
test_archive_file_url_generation()
test_old_absolute_path_handling()
print("\nAll tests passed!")

1589
tests/test_storage_manager.py Normal file
View File

File diff suppressed because it is too large Load Diff