Jarian Cottingham
1271f0b21b
- Remove agent/agent.md (dev-time agent context dumps), .DS_Store, committed venv configs (pyvenv.cfg), 0-byte runtime cache - Remove hardcoded /home/userpath from cron_scraper feed lookup - Replace ad-hoc test_implementation.py with pytest tests/test_scraper_cache.py - ruff clean (33 fixes: bare excepts, unused Config, whitespace) - Root pyproject.toml (activates shared Gitea CI), MIT LICENSE, README Tests
62 lines
1.8 KiB
Python
62 lines
1.8 KiB
Python
"""Tests for the scraper article-processing cache."""
|
|
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
|
|
|
from scraper.scraper import ( # noqa: E402
|
|
get_processing_progress,
|
|
load_processed_cache,
|
|
mark_article_processed,
|
|
save_processed_cache,
|
|
)
|
|
|
|
|
|
@pytest.fixture()
|
|
def cache_dir(tmp_path, monkeypatch):
|
|
monkeypatch.chdir(tmp_path)
|
|
(tmp_path / "articles").mkdir()
|
|
return tmp_path
|
|
|
|
|
|
def test_load_missing_cache_returns_empty(cache_dir):
|
|
assert load_processed_cache() == {}
|
|
|
|
|
|
def test_save_and_load_roundtrip(cache_dir):
|
|
save_processed_cache({"a.html": {"status": "completed"}})
|
|
cache = load_processed_cache()
|
|
assert cache["a.html"]["status"] == "completed"
|
|
|
|
|
|
def test_mark_article_processed(cache_dir):
|
|
mark_article_processed("articles/x/article.html")
|
|
cache = load_processed_cache()
|
|
entry = cache["articles/x/article.html"]
|
|
assert entry["status"] == "completed"
|
|
assert entry["embedding_status"] == "pending"
|
|
assert "processed_date" in entry
|
|
assert "last_updated" in entry
|
|
|
|
|
|
def test_processing_progress_empty(cache_dir):
|
|
progress = get_processing_progress()
|
|
assert progress["total_articles"] == 0
|
|
assert progress["completed_articles"] == 0
|
|
assert progress["embedded_articles"] == 0
|
|
assert progress["completion_rate"] == 0
|
|
|
|
|
|
def test_processing_progress_mixed(cache_dir):
|
|
mark_article_processed("a.html")
|
|
mark_article_processed("b.html", status="failed")
|
|
mark_article_processed("c.html", embedding_status="completed")
|
|
progress = get_processing_progress()
|
|
assert progress["total_articles"] == 3
|
|
assert progress["completed_articles"] == 2
|
|
assert progress["embedded_articles"] == 1
|
|
assert progress["completion_rate"] == pytest.approx(2 / 3 * 100)
|