Test content
", - raw_html="Test content
", - extraction_method="singlefile" - ) - - result = save_article(source_name, article_data) - - # Verify article was saved - assert "Saved article" in result - - # Get the article and check that archive_file_path is relative - articles = get_articles_by_source(source_name) - assert len(articles) == 1 - - article = articles[0] - assert article.archive_file_path is not None - archive_path = Path(article.archive_file_path) - - # Path should be relative (not absolute) - assert not archive_path.is_absolute(), f"Expected relative path, got: {article.archive_file_path}" - - # Path should start with 'websites' - assert archive_path.parts[0] == "websites", f"Expected path starting with 'websites', got: {article.archive_file_path}" - - def test_get_article_returns_relative_path(self): - """Test that get_article returns relative paths.""" - source_name = "TestSource2" - article_data = ArticleData( - url="http://example.com/article/2", - title="Test Article 2", - author="Test Author 2", - publish_date="2024-01-16", - content_text="Test content 2", - content_html="Test content 2
", - raw_html="Test content 2
", - extraction_method="singlefile" - ) - - save_article(source_name, article_data) - - # Get articles and check the path - articles = get_articles_by_source(source_name) - assert len(articles) == 1 - - article = articles[0] - archive_file_path = article.archive_file_path - - # Should be a relative path - assert archive_file_path is not None - assert not Path(archive_file_path).is_absolute() - assert archive_file_path.startswith("websites/") - - def test_archive_file_exists(self): - """Test that archived files can be accessed using the relative path.""" - source_name = "TestSource3" - article_data = ArticleData( - url="http://example.com/article/3", - title="Test Article 3", - author="Test Author 3", - publish_date="2024-01-17", - content_text="Test content 3", - content_html="Test content 3
", - raw_html="Test content 3
", - extraction_method="singlefile" - ) - - save_article(source_name, article_data) - - # Get the article - articles = get_articles_by_source(source_name) - article = articles[0] - - # Verify the archive file exists - archive_path = Path(article.archive_file_path) - full_path = ARCHIVE_DIR / archive_path - assert full_path.exists(), f"Archive file should exist at: {full_path}" - - # Verify the content matches - content = full_path.read_text(encoding='utf-8') - assert "Test content 3" in content - - def test_multiple_articles_same_source(self): - """Test saving multiple articles from the same source.""" - source_name = "TestSource4" - - for i in range(3): - article_data = ArticleData( - url=f"http://example.com/article/{i}", - title=f"Test Article {i}", - author="Test Author 4", - publish_date="2024-01-18", - content_text=f"Test content {i}", - content_html=f"Test content {i}
", - raw_html=f"Test content {i}
", - extraction_method="singlefile" - ) - save_article(source_name, article_data) - - # Get all articles - articles = get_articles_by_source(source_name) - assert len(articles) == 3 - - # Check all paths are relative - for article in articles: - assert article.archive_file_path is not None - assert not Path(article.archive_file_path).is_absolute() - - def test_archive_file_path_format(self): - """Test that archive_file_path has the correct format.""" - source_name = "TestSource5" - article_data = ArticleData( - url="http://example.com/article/5", - title="Test Article 5", - author="Test Author 5", - publish_date="2024-01-19", - content_text="Test content 5", - content_html="Test content 5
", - raw_html="Test content 5
", - extraction_method="singlefile" - ) - - save_article(source_name, article_data) - - articles = get_articles_by_source(source_name) - article = articles[0] - - archive_path = Path(article.archive_file_path) - - # Check path structure: websites/source/html/date/article_XXX.html - assert archive_path.parts[0] == "websites" - assert archive_path.parts[1] == source_name - assert archive_path.parts[2] == "html" - assert len(archive_path.parts) >= 4 # At least: websites/source/html/date - - # Check that it's a .html file - assert archive_path.suffix == ".html" - - -if __name__ == "__main__": - pytest.main([__file__, "-v"]) -``` - -```python -#!/usr/bin/env python3 -"""Test runner for NewsArchiver tests.""" - -import subprocess -import sys - -if __name__ == "__main__": - result = subprocess.run( - [sys.executable, "-m", "pytest", "tests", "-v"], - cwd=str(Path(__file__).parent), - capture_output=False - ) - sys.exit(result.returncode) -``` - -Wait, I made an error. Let me fix the test file: - -```python -#!/usr/bin/env python3 -"""Unit tests for storage_manager path handling.""" - -import os -import sqlite3 -import tempfile -import pytest -from pathlib import Path -from unittest.mock import patch, MagicMock - -import sys -from pathlib import Path as PathClass -sys.path.insert(0, str(PathClass(__file__).parent.parent)) - -from storage_manager import ( - save_article, - get_article, - get_articles_by_source, - initialize_storage, - get_all_sources, - ARCHIVE_DIR, - DB_PATH -) -from content_extractor import ArticleData - - -class TestPathHandling: - """Test that paths are stored and retrieved correctly.""" - - def setup_method(self): - """Set up a temporary database for testing.""" - # Use a temporary database - self.temp_db = tempfile.NamedTemporaryFile(delete=False, suffix='.db') - self.temp_db.close() - - # Patch DB_PATH to use temp database - self.patcher = patch.object(storage_manager, 'DB_PATH', Path(self.temp_db.name)) - self.patcher.start() - - # Initialize database - initialize_storage() - - def teardown_method(self): - """Clean up temporary database.""" - self.patcher.stop() - if Path(self.temp_db.name).exists(): - Path(self.temp_db.name).unlink() - - def test_save_article_stores_relative_path(self): - """Test that save_article stores relative paths in the database.""" - source_name = "TestSource" - article_data = ArticleData( - url="http://example.com/article/1", - title="Test Article", - author="Test Author", - publish_date="2024-01-15", - content_text="Test content", - content_html="Test content
", - raw_html="Test content
", - extraction_method="singlefile" - ) - - result = save_article(source_name, article_data) - - # Verify article was saved - assert "Saved article" in result - - # Get the article and check that archive_file_path is relative - articles = get_articles_by_source(source_name) - assert len(articles) == 1 - - article = articles[0] - assert article.archive_file_path is not None - archive_path = Path(article.archive_file_path) - - # Path should be relative (not absolute) - assert not archive_path.is_absolute(), f"Expected relative path, got: {article.archive_file_path}" - - # Path should start with 'websites' - assert archive_path.parts[0] == "websites", f"Expected path starting with 'websites', got: {article.archive_file_path}" - - def test_get_article_returns_relative_path(self): - """Test that get_article returns relative paths.""" - source_name = "TestSource2" - article_data = ArticleData( - url="http://example.com/article/2", - title="Test Article 2", - author="Test Author 2", - publish_date="2024-01-16", - content_text="Test content 2", - content_html="Test content 2
", - raw_html="Test content 2
", - extraction_method="singlefile" - ) - - save_article(source_name, article_data) - - # Get articles and check the path - articles = get_articles_by_source(source_name) - assert len(articles) == 1 - - article = articles[0] - archive_file_path = article.archive_file_path - - # Should be a relative path - assert archive_file_path is not None - assert not Path(archive_file_path).is_absolute() - assert archive_file_path.startswith("websites/") - - def test_archive_file_exists(self): - """Test that archived files can be accessed using the relative path.""" - source_name = "TestSource3" - article_data = ArticleData( - url="http://example.com/article/3", - title="Test Article 3", - author="Test Author 3", - publish_date="2024-01-17", - content_text="Test content 3", - content_html="Test content 3
", - raw_html="Test content 3
", - extraction_method="singlefile" - ) - - save_article(source_name, article_data) - - # Get the article - articles = get_articles_by_source(source_name) - article = articles[0] - - # Verify the archive file exists - archive_path = Path(article.archive_file_path) - full_path = ARCHIVE_DIR / archive_path - assert full_path.exists(), f"Archive file should exist at: {full_path}" - - # Verify the content matches - content = full_path.read_text(encoding='utf-8') - assert "Test content 3" in content - - def test_multiple_articles_same_source(self): - """Test saving multiple articles from the same source.""" - source_name = "TestSource4" - - for i in range(3): - article_data = ArticleData( - url=f"http://example.com/article/{i}", - title=f"Test Article {i}", - author="Test Author 4", - publish_date="2024-01-18", - content_text=f"Test content {i}", - content_html=f"Test content {i}
", - raw_html=f"Test content {i}
", - extraction_method="singlefile" - ) - save_article(source_name, article_data) - - # Get all articles - articles = get_articles_by_source(source_name) - assert len(articles) == 3 - - # Check all paths are relative - for article in articles: - assert article.archive_file_path is not None - assert not Path(article.archive_file_path).is_absolute() - - def test_archive_file_path_format(self): - """Test that archive_file_path has the correct format.""" - source_name = "TestSource5" - article_data = ArticleData( - url="http://example.com/article/5", - title="Test Article 5", - author="Test Author 5", - publish_date="2024-01-19", - content_text="Test content 5", - content_html="Test content 5
", - raw_html="Test content 5
", - extraction_method="singlefile" - ) - - save_article(source_name, article_data) - - articles = get_articles_by_source(source_name) - article = articles[0] - - archive_path = Path(article.archive_file_path) - - # Check path structure: websites/source/html/date/article_XXX.html - assert archive_path.parts[0] == "websites" - assert archive_path.parts[1] == source_name - assert archive_path.parts[2] == "html" - assert len(archive_path.parts) >= 4 # At least: websites/source/html/date - - # Check that it's a .html file - assert archive_path.suffix == ".html" - - -if __name__ == "__main__": - pytest.main([__file__, "-v"]) -``` - -Actually, I realize I need to import the storage_manager module first. Let me create a cleaner test file: - -```python -#!/usr/bin/env python3 -"""Unit tests for storage_manager path handling.""" - -import os import sys import tempfile -import pytest from pathlib import Path -from unittest.mock import patch +from urllib.parse import unquote -# Add parent directory to path -sys.path.insert(0, str(Path(__file__).parent.parent)) - -import storage_manager -from storage_manager import ( - save_article, - get_article, - get_articles_by_source, - initialize_storage, - ARCHIVE_DIR, - DB_PATH -) -from content_extractor import ArticleData - - -class TestPathHandling: - """Test that paths are stored and retrieved correctly.""" - - def setup_method(self): - """Set up a temporary database for testing.""" - # Use a temporary database - self.temp_db = tempfile.NamedTemporaryFile(delete=False, suffix='.db') - self.temp_db.close() - - # Patch DB_PATH to use temp database - self.patcher = patch.object(storage_manager, 'DB_PATH', Path(self.temp_db.name)) - self.patcher.start() - - # Initialize database - initialize_storage() - - def teardown_method(self): - """Clean up temporary database.""" - self.patcher.stop() - if Path(self.temp_db.name).exists(): - Path(self.temp_db.name).unlink() - - def test_save_article_stores_relative_path(self): - """Test that save_article stores relative paths in the database.""" - source_name = "TestSource" - article_data = ArticleData( - url="http://example.com/article/1", - title="Test Article", - author="Test Author", - publish_date="2024-01-15", - content_text="Test content", - content_html="Test content
", - raw_html="Test content
", - extraction_method="singlefile" - ) - - result = save_article(source_name, article_data) - - # Verify article was saved - assert "Saved article" in result - - # Get the article and check that archive_file_path is relative - articles = get_articles_by_source(source_name) - assert len(articles) == 1 - - article = articles[0] - assert article.archive_file_path is not None - archive_path = Path(article.archive_file_path) - - # Path should be relative (not absolute) - assert not archive_path.is_absolute(), f"Expected relative path, got: {article.archive_file_path}" - - # Path should start with 'websites' - assert archive_path.parts[0] == "websites", f"Expected path starting with 'websites', got: {article.archive_file_path}" - - def test_get_article_returns_relative_path(self): - """Test that get_article returns relative paths.""" - source_name = "TestSource2" - article_data = ArticleData( - url="http://example.com/article/2", - title="Test Article 2", - author="Test Author 2", - publish_date="2024-01-16", - content_text="Test content 2", - content_html="Test content 2
", - raw_html="Test content 2
", - extraction_method="singlefile" - ) - - save_article(source_name, article_data) - - # Get articles and check the path - articles = get_articles_by_source(source_name) - assert len(articles) == 1 - - article = articles[0] - archive_file_path = article.archive_file_path - - # Should be a relative path - assert archive_file_path is not None - assert not Path(archive_file_path).is_absolute() - assert archive_file_path.startswith("websites/") - - def test_archive_file_exists(self): - """Test that archived files can be accessed using the relative path.""" - source_name = "TestSource3" - article_data = ArticleData( - url="http://example.com/article/3", - title="Test Article 3", - author="Test Author 3", - publish_date="2024-01-17", - content_text="Test content 3", - content_html="Test content 3
", - raw_html="Test content 3
", - extraction_method="singlefile" - ) - - save_article(source_name, article_data) - - # Get the article - articles = get_articles_by_source(source_name) - article = articles[0] - - # Verify the archive file exists - archive_path = Path(article.archive_file_path) - full_path = ARCHIVE_DIR / archive_path - assert full_path.exists(), f"Archive file should exist at: {full_path}" - - # Verify the content matches - content = full_path.read_text(encoding='utf-8') - assert "Test content 3" in content - - def test_multiple_articles_same_source(self): - """Test saving multiple articles from the same source.""" - source_name = "TestSource4" - - for i in range(3): - article_data = ArticleData( - url=f"http://example.com/article/{i}", - title=f"Test Article {i}", - author="Test Author 4", - publish_date="2024-01-18", - content_text=f"Test content {i}", - content_html=f"Test content {i}
", - raw_html=f"Test content {i}
", - extraction_method="singlefile" - ) - save_article(source_name, article_data) - - # Get all articles - articles = get_articles_by_source(source_name) - assert len(articles) == 3 - - # Check all paths are relative - for article in articles: - assert article.archive_file_path is not None - assert not Path(article.archive_file_path).is_absolute() - - def test_archive_file_path_format(self): - """Test that archive_file_path has the correct format.""" - source_name = "TestSource5" - article_data = ArticleData( - url="http://example.com/article/5", - title="Test Article 5", - author="Test Author 5", - publish_date="2024-01-19", - content_text="Test content 5", - content_html="Test content 5
", - raw_html="Test content 5
", - extraction_method="singlefile" - ) - - save_article(source_name, article_data) - - articles = get_articles_by_source(source_name) - article = articles[0] - - archive_path = Path(article.archive_file_path) - - # Check path structure: websites/source/html/date/article_XXX.html - assert archive_path.parts[0] == "websites" - assert archive_path.parts[1] == source_name - assert archive_path.parts[2] == "html" - assert len(archive_path.parts) >= 4 # At least: websites/source/html/date - - # Check that it's a .html file - assert archive_path.suffix == ".html" - - -if __name__ == "__main__": - pytest.main([__file__, "-v"]) -``` - -Actually, I realize there's an issue - `storage_manager` might not be imported yet when we try to patch it. Let me fix this by importing it first: - -```python -#!/usr/bin/env python3 -"""Unit tests for storage_manager path handling.""" - -import os -import sys -import tempfile -import pytest -from pathlib import Path -from unittest.mock import patch - -# Add parent directory to path -sys.path.insert(0, str(Path(__file__).parent.parent)) - -import storage_manager as storage_manager_module -from storage_manager import ( - save_article, - get_article, - get_articles_by_source, - initialize_storage, - ARCHIVE_DIR, - DB_PATH -) -from content_extractor import ArticleData - - -class TestPathHandling: - """Test that paths are stored and retrieved correctly.""" - - def setup_method(self): - """Set up a temporary database for testing.""" - # Use a temporary database - self.temp_db = tempfile.NamedTemporaryFile(delete=False, suffix='.db') - self.temp_db.close() - - # Patch DB_PATH to use temp database - self.patcher = patch.object(storage_manager_module, 'DB_PATH', Path(self.temp_db.name)) - self.patcher.start() - - # Re-initialize with patched DB_PATH - initialize_storage() - - def teardown_method(self): - """Clean up temporary database.""" - self.patcher.stop() - if Path(self.temp_db.name).exists(): - Path(self.temp_db.name).unlink() - - def test_save_article_stores_relative_path(self): - """Test that save_article stores relative paths in the database.""" - source_name = "TestSource" - article_data = ArticleData( - url="http://example.com/article/1", - title="Test Article", - author="Test Author", - publish_date="2024-01-15", - content_text="Test content", - content_html="Test content
", - raw_html="Test content
", - extraction_method="singlefile" - ) - - result = save_article(source_name, article_data) - - # Verify article was saved - assert "Saved article" in result - - # Get the article and check that archive_file_path is relative - articles = get_articles_by_source(source_name) - assert len(articles) == 1 - - article = articles[0] - assert article.archive_file_path is not None - archive_path = Path(article.archive_file_path) - - # Path should be relative (not absolute) - assert not archive_path.is_absolute(), f"Expected relative path, got: {article.archive_file_path}" - - # Path should start with 'websites' - assert archive_path.parts[0] == "websites", f"Expected path starting with 'websites', got: {article.archive_file_path}" - - def test_get_article_returns_relative_path(self): - """Test that get_article returns relative paths.""" - source_name = "TestSource2" - article_data = ArticleData( - url="http://example.com/article/2", - title="Test Article 2", - author="Test Author 2", - publish_date="2024-01-16", - content_text="Test content 2", - content_html="Test content 2
", - raw_html="Test content 2
", - extraction_method="singlefile" - ) - - save_article(source_name, article_data) - - # Get articles and check the path - articles = get_articles_by_source(source_name) - assert len(articles) == 1 - - article = articles[0] - archive_file_path = article.archive_file_path - - # Should be a relative path - assert archive_file_path is not None - assert not Path(archive_file_path).is_absolute() - assert archive_file_path.startswith("websites/") - - def test_archive_file_exists(self): - """Test that archived files can be accessed using the relative path.""" - source_name = "TestSource3" - article_data = ArticleData( - url="http://example.com/article/3", - title="Test Article 3", - author="Test Author 3", - publish_date="2024-01-17", - content_text="Test content 3", - content_html="Test content 3
", - raw_html="Test content 3
", - extraction_method="singlefile" - ) - - save_article(source_name, article_data) - - # Get the article - articles = get_articles_by_source(source_name) - article = articles[0] - - # Verify the archive file exists - archive_path = Path(article.archive_file_path) - full_path = ARCHIVE_DIR / archive_path - assert full_path.exists(), f"Archive file should exist at: {full_path}" - - # Verify the content matches - content = full_path.read_text(encoding='utf-8') - assert "Test content 3" in content - - def test_multiple_articles_same_source(self): - """Test saving multiple articles from the same source.""" - source_name = "TestSource4" - - for i in range(3): - article_data = ArticleData( - url=f"http://example.com/article/{i}", - title=f"Test Article {i}", - author="Test Author 4", - publish_date="2024-01-18", - content_text=f"Test content {i}", - content_html=f"Test content {i}
", - raw_html=f"Test content {i}
", - extraction_method="singlefile" - ) - save_article(source_name, article_data) - - # Get all articles - articles = get_articles_by_source(source_name) - assert len(articles) == 3 - - # Check all paths are relative - for article in articles: - assert article.archive_file_path is not None - assert not Path(article.archive_file_path).is_absolute() - - def test_archive_file_path_format(self): - """Test that archive_file_path has the correct format.""" - source_name = "TestSource5" - article_data = ArticleData( - url="http://example.com/article/5", - title="Test Article 5", - author="Test Author 5", - publish_date="2024-01-19", - content_text="Test content 5", - content_html="Test content 5
", - raw_html="Test content 5
", - extraction_method="singlefile" - ) - - save_article(source_name, article_data) - - articles = get_articles_by_source(source_name) - article = articles[0] - - archive_path = Path(article.archive_file_path) - - # Check path structure: websites/source/html/date/article_XXX.html - assert archive_path.parts[0] == "websites" - assert archive_path.parts[1] == source_name - assert archive_path.parts[2] == "html" - assert len(archive_path.parts) >= 4 # At least: websites/source/html/date - - # Check that it's a .html file - assert archive_path.suffix == ".html" - - -if __name__ == "__main__": - pytest.main([__file__, "-v"]) -``` - -Actually, I realize I need to handle the case where the module has already been imported. Let me fix this properly: - -```python -#!/usr/bin/env python3 -"""Unit tests for storage_manager path handling.""" - -import os -import sys -import tempfile -import pytest -from pathlib import Path -from unittest.mock import patch - -# Add parent directory to path -sys.path.insert(0, str(Path(__file__).parent.parent)) - -# Import storage_manager first -from storage_manager import ( - save_article, - get_article, - get_articles_by_source, - initialize_storage, - ARCHIVE_DIR, - DB_PATH, - _init_database, - _get_db_connection -) -from content_extractor import ArticleData - - -class TestPathHandling: - """Test that paths are stored and retrieved correctly.""" - - def setup_method(self): - """Set up a temporary database for testing.""" - # Use a temporary database - self.temp_db_path = Path(tempfile.mktemp(suffix='.db')) - - # Temporarily modify DB_PATH - self.original_db_path = DB_PATH - import storage_manager - storage_manager.DB_PATH = self.temp_db_path - - # Initialize database - _init_database() - - def teardown_method(self): - """Clean up temporary database.""" - # Restore original DB_PATH - import storage_manager - storage_manager.DB_PATH = self.original_db_path - - # Clean up temp db - if self.temp_db_path.exists(): - self.temp_db_path.unlink() - - def test_save_article_stores_relative_path(self): - """Test that save_article stores relative paths in the database.""" - source_name = "TestSource" - article_data = ArticleData( - url="http://example.com/article/1", - title="Test Article", - author="Test Author", - publish_date="2024-01-15", - content_text="Test content", - content_html="Test content
", - raw_html="Test content
", - extraction_method="singlefile" - ) - - result = save_article(source_name, article_data) - - # Verify article was saved - assert "Saved article" in result - - # Get the article and check that archive_file_path is relative - articles = get_articles_by_source(source_name) - assert len(articles) == 1 - - article = articles[0] - assert article.archive_file_path is not None - archive_path = Path(article.archive_file_path) - - # Path should be relative (not absolute) - assert not archive_path.is_absolute(), f"Expected relative path, got: {article.archive_file_path}" - - # Path should start with 'websites' - assert archive_path.parts[0] == "websites", f"Expected path starting with 'websites', got: {article.archive_file_path}" - - def test_get_article_returns_relative_path(self): - """Test that get_article returns relative paths.""" - source_name = "TestSource2" - article_data = ArticleData( - url="http://example.com/article/2", - title="Test Article 2", - author="Test Author 2", - publish_date="2024-01-16", - content_text="Test content 2", - content_html="Test content 2
", - raw_html="Test content 2
", - extraction_method="singlefile" - ) - - save_article(source_name, article_data) - - # Get articles and check the path - articles = get_articles_by_source(source_name) - assert len(articles) == 1 - - article = articles[0] - archive_file_path = article.archive_file_path - - # Should be a relative path - assert archive_file_path is not None - assert not Path(archive_file_path).is_absolute() - assert archive_file_path.startswith("websites/") - - def test_archive_file_exists(self): - """Test that archived files can be accessed using the relative path.""" - source_name = "TestSource3" - article_data = ArticleData( - url="http://example.com/article/3", - title="Test Article 3", - author="Test Author 3", - publish_date="2024-01-17", - content_text="Test content 3", - content_html="Test content 3
", - raw_html="Test content 3
", - extraction_method="singlefile" - ) - - save_article(source_name, article_data) - - # Get the article - articles = get_articles_by_source(source_name) - article = articles[0] - - # Verify the archive file exists - archive_path = Path(article.archive_file_path) - full_path = ARCHIVE_DIR / archive_path - assert full_path.exists(), f"Archive file should exist at: {full_path}" - - # Verify the content matches - content = full_path.read_text(encoding='utf-8') - assert "Test content 3" in content - - def test_multiple_articles_same_source(self): - """Test saving multiple articles from the same source.""" - source_name = "TestSource4" - - for i in range(3): - article_data = ArticleData( - url=f"http://example.com/article/{i}", - title=f"Test Article {i}", - author="Test Author 4", - publish_date="2024-01-18", - content_text=f"Test content {i}", - content_html=f"Test content {i}
", - raw_html=f"Test content {i}
", - extraction_method="singlefile" - ) - save_article(source_name, article_data) - - # Get all articles - articles = get_articles_by_source(source_name) - assert len(articles) == 3 - - # Check all paths are relative - for article in articles: - assert article.archive_file_path is not None - assert not Path(article.archive_file_path).is_absolute() - - def test_archive_file_path_format(self): - """Test that archive_file_path has the correct format.""" - source_name = "TestSource5" - article_data = ArticleData( - url="http://example.com/article/5", - title="Test Article 5", - author="Test Author 5", - publish_date="2024-01-19", - content_text="Test content 5", - content_html="Test content 5
", - raw_html="Test content 5
", - extraction_method="singlefile" - ) - - save_article(source_name, article_data) - - articles = get_articles_by_source(source_name) - article = articles[0] - - archive_path = Path(article.archive_file_path) - - # Check path structure: websites/source/html/date/article_XXX.html - assert archive_path.parts[0] == "websites" - assert archive_path.parts[1] == source_name - assert archive_path.parts[2] == "html" - assert len(archive_path.parts) >= 4 # At least: websites/source/html/date - - # Check that it's a .html file - assert archive_path.suffix == ".html" - - -if __name__ == "__main__": - pytest.main([__file__, "-v"]) -``` - -Actually, I realize there's a simpler approach - I'll create a minimal test that just verifies the path handling logic without needing to actually save articles. Let me create a simpler test file: - -```python -#!/usr/bin/env python3 -"""Unit tests for storage_manager path handling.""" - -import sys -from pathlib import Path - -# Add parent directory to path -sys.path.insert(0, str(Path(__file__).parent.parent)) - -import storage_manager - - -def test_relative_path_logic(): - """Test the relative path calculation logic.""" - ARCHIVE_DIR = Path("/Volumes/playground/NewsArchiver/archival_data") - - # Test 1: Relative path - archive_file_path = ARCHIVE_DIR / "websites/404 Media/html/2024-01-15/article_001.html" - relative_path = archive_file_path.relative_to(ARCHIVE_DIR) - assert relative_path == Path("websites/404 Media/html/2024-01-15/article_001.html") - assert not relative_path.is_absolute() - - # Test 2: Absolute path (simulating old entries) - old_archive_path = Path("/home/user/playground/NewsArchiver/archival_data/websites/404 Media/html/2024-01-15/article_001.html") - assert old_archive_path.is_absolute() - - # When we have an old absolute path, we should be able to extract the relative part - # by checking if it starts with ARCHIVE_DIR - if old_archive_path.is_absolute(): - # Try to convert to relative - try: - relative = old_archive_path.relative_to(ARCHIVE_DIR) - print(f"Relative path from old absolute: {relative}") - except ValueError as e: - print(f"Cannot make relative: {e}") - - # Test 3: Path from current ARCHIVE_DIR - current_archive_path = ARCHIVE_DIR / "websites/TestSource/html/2024-01-15/article_002.html" - relative = current_archive_path.relative_to(ARCHIVE_DIR) - assert relative == Path("websites/TestSource/html/2024-01-15/article_002.html") - - print("All path logic tests passed!") - - -if __name__ == "__main__": - test_relative_path_logic() -``` - -Actually, let me just write the proper test file now: - -```python -#!/usr/bin/env python3 -"""Unit tests for storage_manager path handling.""" - -import sys -from pathlib import Path - -# Add parent directory to path -sys.path.insert(0, str(Path(__file__).parent.parent)) - - -def test_path_logic(): - """Test the relative path logic without needing a database.""" - # Simulate ARCHIVE_DIR - ARCHIVE_DIR = Path("/Volumes/playground/NewsArchiver/archival_data") - - # Test 1: Normal relative path calculation - archive_file_path = ARCHIVE_DIR / "websites/404 Media/html/2024-01-15/article_001.html" - relative_path = archive_file_path.relative_to(ARCHIVE_DIR) - assert str(relative_path) == "websites/404 Media/html/2024-01-15/article_001.html" - assert not relative_path.is_absolute() - print(f"Test 1 passed: {relative_path}") - - # Test 2: Old absolute path from different server - old_archive_path = Path("/home/user/playground/NewsArchiver/archival_data/websites/404 Media/html/2024-01-15/article_001.html") - assert old_archive_path.is_absolute() - - # When we have an old absolute path, we need to handle it differently - # The path should start with ARCHIVE_DIR for relative_to to work - try: - relative = old_archive_path.relative_to(ARCHIVE_DIR) - print(f"Test 2 passed: {relative}") - except ValueError as e: - print(f"Test 2 info: Cannot make relative from different base: {e}") - - # Test 3: Path from different ARCHIVE_DIR - different_archive_path = Path("/Volumes/playground/NewsArchiver/archival_data/websites/TestSource/html/2024-01-15/article_002.html") - relative = different_archive_path.relative_to(ARCHIVE_DIR) - assert str(relative) == "websites/TestSource/html/2024-01-15/article_002.html" - print(f"Test 3 passed: {relative}") - - # Test 4: Simulate the full flow - # 1. Save article: store relative path - archive_file = ARCHIVE_DIR / "websites/TestSource/html/2024-01-15/article_003.html" - stored_path = str(archive_file.relative_to(ARCHIVE_DIR)) - assert stored_path == "websites/TestSource/html/2024-01-15/article_003.html" - print(f"Test 4a passed: stored_path = {stored_path}") - - # 2. Retrieve article: convert back to absolute for file access - retrieved_path = Path(stored_path) - if not retrieved_path.is_absolute(): - retrieved_path = ARCHIVE_DIR / retrieved_path - assert str(retrieved_path) == str(archive_file) - print(f"Test 4b passed: retrieved_path = {retrieved_path}") - - # 3. Return relative path for web interface - web_path = retrieved_path.relative_to(ARCHIVE_DIR) - assert str(web_path) == stored_path - print(f"Test 4c passed: web_path = {web_path}") - - print("\nAll path logic tests passed!") - - -if __name__ == "__main__": - test_path_logic() -``` - -This is getting too long. Let me create a proper unit test file that tests the actual code changes: - -```python -#!/usr/bin/env python3 -"""Unit tests for storage_manager path handling.""" - -import sys -from pathlib import Path - -# Add parent directory to path -sys.path.insert(0, str(Path(__file__).parent.parent)) - - -def test_save_article_path_format(): - """Test that save_article stores relative paths.""" - from storage_manager import ARCHIVE_DIR - - # Simulate the path calculation - archive_file_path = ARCHIVE_DIR / "websites/404 Media/html/2024-01-15/article_001.html" - stored_path = str(archive_file_path.relative_to(ARCHIVE_DIR)) - - # Verify the stored path is relative - assert not Path(stored_path).is_absolute() - assert stored_path.startswith("websites/") - print(f"Stored path: {stored_path}") - - # Simulate retrieving and converting back to absolute - retrieved_path = Path(stored_path) - if not retrieved_path.is_absolute(): - full_path = ARCHIVE_DIR / retrieved_path - else: - full_path = retrieved_path - - # Verify the full path is correct - assert str(full_path) == str(archive_file_path) - print(f"Full path: {full_path}") - - # Verify web path is correct - web_path = full_path.relative_to(ARCHIVE_DIR) - assert str(web_path) == stored_path - print(f"Web path: {web_path}") - - print("\ntest_save_article_path_format passed!") - - -def test_old_absolute_path_handling(): - """Test handling of old absolute paths from different servers.""" - ARCHIVE_DIR = Path("/Volumes/playground/NewsArchiver/archival_data") - - # Old absolute path from different server - old_absolute_path = Path("/home/user/playground/NewsArchiver/archival_data/websites/404 Media/html/2024-01-15/article_001.html") - - # Check if it's absolute - assert old_absolute_path.is_absolute() - - # Try to convert to relative - this will fail if paths don't match - try: - relative = old_absolute_path.relative_to(ARCHIVE_DIR) - print(f"Old path converted: {relative}") - except ValueError as e: - print(f"Old path cannot be converted (different base): {e}") - - print("\ntest_old_absolute_path_handling passed!") - - -if __name__ == "__main__": - test_save_article_path_format() - test_old_absolute_path_handling() -``` - -Let me now create the actual test file with the proper content: - -```python -#!/usr/bin/env python3 -"""Unit tests for storage_manager path handling.""" - -import sys -from pathlib import Path - -# Add parent directory to path sys.path.insert(0, str(Path(__file__).parent.parent)) def test_relative_path_format(): - """Test the relative path format after save_article.""" - from storage_manager import ARCHIVE_DIR + """Test relative path format for archive files.""" + archive_dir = Path(tempfile.mkdtemp()) + archive_file_path = archive_dir / "websites/TestSource/html/2024-01-15/article_001.html" + archive_file_path.parent.mkdir(parents=True, exist_ok=True) + archive_file_path.touch() - # Simulate what save_article does - archive_file_path = ARCHIVE_DIR / "websites/404 Media/html/2024-01-15/article_001.html" - - # This is what we store in the database - stored_path = str(archive_file_path.relative_to(ARCHIVE_DIR)) - - # Verify stored path is relative + stored_path = str(archive_file_path.relative_to(archive_dir)) assert not Path(stored_path).is_absolute() - assert stored_path == "websites/404 Media/html/2024-01-15/article_001.html" + assert stored_path == "websites/TestSource/html/2024-01-15/article_001.html" - # Simulate what get_article does when retrieving retrieved_path = Path(stored_path) - if not retrieved_path.is_absolute(): - full_path = ARCHIVE_DIR / retrieved_path - else: - full_path = retrieved_path - - # Verify full path is correct + full_path = archive_dir / retrieved_path if not retrieved_path.is_absolute() else retrieved_path assert str(full_path) == str(archive_file_path) - # This is what we return for the web interface - web_path = str(full_path.relative_to(ARCHIVE_DIR)) + web_path = str(full_path.relative_to(archive_dir)) assert web_path == stored_path + print(f" PASS: stored={stored_path}") - print(f"Test passed! Stored: {stored_path}, Web: {web_path}") + +def test_archive_file_url_generation(): + """Test URL generation for archived files.""" + archive_file_path = "websites/404 Media/html/2024-01-15/article_001.html" + url = f"/archive-file/{archive_file_path}" + assert url == "/archive-file/websites/404 Media/html/2024-01-15/article_001.html" + + decoded_path = unquote(archive_file_path) + archive_dir = Path("/tmp/test_archives") + full_path = archive_dir / decoded_path + assert str(full_path) == str(archive_dir / "websites/404 Media/html/2024-01-15/article_001.html") + print(f" PASS: url={url}") def test_multiple_sources(): - """Test that different sources get correct paths.""" - from storage_manager import ARCHIVE_DIR - - sources = ["404 Media", "TestSource", "Another Source"] - - for source in sources: - archive_file_path = ARCHIVE_DIR / f"websites/{source}/html/2024-01-15/article_001.html" - stored_path = str(archive_file_path.relative_to(ARCHIVE_DIR)) - - # Verify path structure + """Test different sources get correct paths.""" + archive_dir = Path(tempfile.mkdtemp()) + for source in ["404 Media", "TestSource", "Another Source"]: + archive_file_path = archive_dir / f"websites/{source}/html/2024-01-15/article_001.html" + archive_file_path.parent.mkdir(parents=True, exist_ok=True) + stored_path = str(archive_file_path.relative_to(archive_dir)) parts = Path(stored_path).parts assert parts[0] == "websites" assert parts[1] == source assert parts[2] == "html" - - print(f"Source '{source}': {stored_path}") - - -def test_archive_file_url_generation(): - """Test that the URL for archived files is correct.""" - # Simulate what the template does - archive_file_path = "websites/404 Media/html/2024-01-15/article_001.html" - - # This is what the template generates - url = f"/archive-file/{archive_file_path}" - - # Verify URL format - assert url == "/archive-file/websites/404 Media/html/2024-01-15/article_001.html" - - # Simulate what the route handler does - from storage_manager import ARCHIVE_DIR - from urllib.parse import unquote - decoded_path = unquote(archive_file_path) - full_path = ARCHIVE_DIR / decoded_path - - # Verify the full path is correct - assert full_path.exists() or full_path.parent.exists() # File might not exist in test - - print(f"URL: {url}") - print(f"Full path: {full_path}") + print(f" PASS: source='{source}' path={stored_path}") if __name__ == "__main__": test_relative_path_format() - test_multiple_sources() test_archive_file_url_generation() - print("\nAll tests passed!") -``` - -Let me now create the final test file: - -```python -#!/usr/bin/env python3 -"""Unit tests for storage_manager path handling.""" - -import sys -from pathlib import Path - -# Add parent directory to path -sys.path.insert(0, str(Path(__file__).parent.parent)) - - -def test_relative_path_format(): - """Test the relative path format after save_article.""" - from storage_manager import ARCHIVE_DIR - - # Simulate what save_article does - archive_file_path = ARCHIVE_DIR / "websites/404 Media/html/2024-01-15/article_001.html" - - # This is what we store in the database - stored_path = str(archive_file_path.relative_to(ARCHIVE_DIR)) - - # Verify stored path is relative - assert not Path(stored_path).is_absolute() - assert stored_path == "websites/404 Media/html/2024-01-15/article_001.html" - - # Simulate what get_article does when retrieving - retrieved_path = Path(stored_path) - if not retrieved_path.is_absolute(): - full_path = ARCHIVE_DIR / retrieved_path - else: - full_path = retrieved_path - - # Verify full path is correct - assert str(full_path) == str(archive_file_path) - - # This is what we return for the web interface - web_path = str(full_path.relative_to(ARCHIVE_DIR)) - assert web_path == stored_path - - print(f"Test passed! Stored: {stored_path}, Web: {web_path}") - - -def test_multiple_sources(): - """Test that different sources get correct paths.""" - from storage_manager import ARCHIVE_DIR - - sources = ["404 Media", "TestSource", "Another Source"] - - for source in sources: - archive_file_path = ARCHIVE_DIR / f"websites/{source}/html/2024-01-15/article_001.html" - stored_path = str(archive_file_path.relative_to(ARCHIVE_DIR)) - - # Verify path structure - parts = Path(stored_path).parts - assert parts[0] == "websites" - assert parts[1] == source - assert parts[2] == "html" - - print(f"Source '{source}': {stored_path}") - - -def test_archive_file_url_generation(): - """Test that the URL for archived files is correct.""" - # Simulate what the template does - archive_file_path = "websites/404 Media/html/2024-01-15/article_001.html" - - # This is what the template generates - url = f"/archive-file/{archive_file_path}" - - # Verify URL format - assert url == "/archive-file/websites/404 Media/html/2024-01-15/article_001.html" - - # Simulate what the route handler does - from storage_manager import ARCHIVE_DIR - from urllib.parse import unquote - decoded_path = unquote(archive_file_path) - full_path = ARCHIVE_DIR / decoded_path - - # Verify the full path is correct - assert str(full_path) == str(ARCHIVE_DIR / "websites/404 Media/html/2024-01-15/article_001.html") - - print(f"URL: {url}") - print(f"Full path: {full_path}") - - -if __name__ == "__main__": - test_relative_path_format() test_multiple_sources() - test_archive_file_url_generation() - print("\nAll tests passed!") -``` - -```python -#!/usr/bin/env python3 -"""Unit tests for storage_manager path handling.""" - -import sys -from pathlib import Path - -# Add parent directory to path -sys.path.insert(0, str(Path(__file__).parent.parent)) - - -def test_relative_path_format(): - """Test the relative path format after save_article.""" - from storage_manager import ARCHIVE_DIR - - # Simulate what save_article does - archive_file_path = ARCHIVE_DIR / "websites/404 Media/html/2024-01-15/article_001.html" - - # This is what we store in the database - stored_path = str(archive_file_path.relative_to(ARCHIVE_DIR)) - - # Verify stored path is relative - assert not Path(stored_path).is_absolute() - assert stored_path == "websites/404 Media/html/2024-01-15/article_001.html" - - # Simulate what get_article does when retrieving - retrieved_path = Path(stored_path) - if not retrieved_path.is_absolute(): - full_path = ARCHIVE_DIR / retrieved_path - else: - full_path = retrieved_path - - # Verify full path is correct - assert str(full_path) == str(archive_file_path) - - # This is what we return for the web interface - web_path = str(full_path.relative_to(ARCHIVE_DIR)) - assert web_path == stored_path - - print(f"Test passed! Stored: {stored_path}, Web: {web_path}") - - -def test_multiple_sources(): - """Test that different sources get correct paths.""" - from storage_manager import ARCHIVE_DIR - - sources = ["404 Media", "TestSource", "Another Source"] - - for source in sources: - archive_file_path = ARCHIVE_DIR / f"websites/{source}/html/2024-01-15/article_001.html" - stored_path = str(archive_file_path.relative_to(ARCHIVE_DIR)) - - # Verify path structure - parts = Path(stored_path).parts - assert parts[0] == "websites" - assert parts[1] == source - assert parts[2] == "html" - - print(f"Source '{source}': {stored_path}") - - -def test_archive_file_url_generation(): - """Test that the URL for archived files is correct.""" - # Simulate what the template does - archive_file_path = "websites/404 Media/html/2024-01-15/article_001.html" - - # This is what the template generates - url = f"/archive-file/{archive_file_path}" - - # Verify URL format - assert url == "/archive-file/websites/404 Media/html/2024-01-15/article_001.html" - - # Simulate what the route handler does - from storage_manager import ARCHIVE_DIR - from urllib.parse import unquote - decoded_path = unquote(archive_file_path) - full_path = ARCHIVE_DIR / decoded_path - - # Verify the full path is correct - assert str(full_path) == str(ARCHIVE_DIR / "websites/404 Media/html/2024-01-15/article_001.html") - - print(f"URL: {url}") - print(f"Full path: {full_path}") - - -if __name__ == "__main__": - test_relative_path_format() - test_multiple_sources() - test_archive_file_url_generation() - print("\nAll tests passed!") -``` - -Let me create the final file: - -```python -#!/usr/bin/env python3 -"""Unit tests for storage_manager path handling.""" - -import sys -from pathlib import Path - -# Add parent directory to path -sys.path.insert(0, str(Path(__file__).parent.parent)) - - -def test_relative_path_format(): - """Test the relative path format after save_article.""" - from storage_manager import ARCHIVE_DIR - - # Simulate what save_article does - archive_file_path = ARCHIVE_DIR / "websites/404 Media/html/2024-01-15/article_001.html" - - # This is what we store in the database - stored_path = str(archive_file_path.relative_to(ARCHIVE_DIR)) - - # Verify stored path is relative - assert not Path(stored_path).is_absolute() - assert stored_path == "websites/404 Media/html/2024-01-15/article_001.html" - - # Simulate what get_article does when retrieving - retrieved_path = Path(stored_path) - if not retrieved_path.is_absolute(): - full_path = ARCHIVE_DIR / retrieved_path - else: - full_path = retrieved_path - - # Verify full path is correct - assert str(full_path) == str(archive_file_path) - - # This is what we return for the web interface - web_path = str(full_path.relative_to(ARCHIVE_DIR)) - assert web_path == stored_path - - print(f"Test passed! Stored: {stored_path}, Web: {web_path}") - - -def test_multiple_sources(): - """Test that different sources get correct paths.""" - from storage_manager import ARCHIVE_DIR - - sources = ["404 Media", "TestSource", "Another Source"] - - for source in sources: - archive_file_path = ARCHIVE_DIR / f"websites/{source}/html/2024-01-15/article_001.html" - stored_path = str(archive_file_path.relative_to(ARCHIVE_DIR)) - - # Verify path structure - parts = Path(stored_path).parts - assert parts[0] == "websites" - assert parts[1] == source - assert parts[2] == "html" - - print(f"Source '{source}': {stored_path}") - - -def test_archive_file_url_generation(): - """Test that the URL for archived files is correct.""" - # Simulate what the template does - archive_file_path = "websites/404 Media/html/2024-01-15/article_001.html" - - # This is what the template generates - url = f"/archive-file/{archive_file_path}" - - # Verify URL format - assert url == "/archive-file/websites/404 Media/html/2024-01-15/article_001.html" - - # Simulate what the route handler does - from storage_manager import ARCHIVE_DIR - from urllib.parse import unquote - decoded_path = unquote(archive_file_path) - full_path = ARCHIVE_DIR / decoded_path - - # Verify the full path is correct - assert str(full_path) == str(ARCHIVE_DIR / "websites/404 Media/html/2024-01-15/article_001.html") - - print(f"URL: {url}") - print(f"Full path: {full_path}") - - -if __name__ == "__main__": - test_relative_path_format() - test_multiple_sources() - test_archive_file_url_generation() - print("\nAll tests passed!") + print("\nAll storage_manager tests passed!") diff --git a/tests/test_web_interface.py b/tests/test_web_interface.py new file mode 100644 index 0000000..5fd4df7 --- /dev/null +++ b/tests/test_web_interface.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +"""Unit tests for web_interface.py security features.""" + +import subprocess +import sys +import tempfile +from pathlib import Path +from unittest.mock import MagicMock, patch + +sys.path.insert(0, str(Path(__file__).parent.parent)) + + +def test_validate_archive_path_blocks_traversal(): + """Test validate_archive_path function blocks traversal.""" + from web_interface import validate_archive_path, ARCHIVE_DIR + + for bad in ["../../../etc/passwd", "..%2Fetc%2Fpasswd", "../secret.txt"]: + result = validate_archive_path(bad) + assert result is None, f"Should block: {bad}" + print(" PASS: traversal blocked") + + +def test_xss_safe_filter_removed(): + """Test that article.html no longer uses |safe filter.""" + result = subprocess.run( + ["grep", "-rn", "|safe", "templates/article.html"], + capture_output=True, text=True, + ) + assert result.returncode != 0, "article.html should not contain |safe" + print(" PASS: no |safe in article.html") + + +def test_security_headers_present(): + """Test security headers are added to responses.""" + from web_interface import app + client = app.test_client() + resp = client.get("/") + assert resp.headers.get("X-Content-Type-Options") == "nosniff" + assert resp.headers.get("X-Frame-Options") == "DENY" + assert resp.headers.get("X-XSS-Protection") == "1; mode=block" + assert resp.headers.get("Referrer-Policy") == "strict-origin-when-cross-origin" + print(" PASS: security headers present") + + +def test_server_url_not_hardcoded(): + """Test that SERVER_URL comes from env, not hardcoded.""" + result = subprocess.run( + ["grep", "-n", "192\\.168", "web_interface.py"], + capture_output=True, text=True, + ) + assert result.returncode != 0, "No hardcoded internal IPs" + print(" PASS: no hardcoded IPs") + + +def test_no_safe_filter_in_rss(): + """Test RSS/Atom feeds escape content properly.""" + result = subprocess.run( + ["grep", "-rn", "|safe", "templates/rss.xml", "templates/atom.xml"], + capture_output=True, text=True, + ) + assert result.returncode != 0, "No |safe in RSS templates" + print(" PASS: no |safe in RSS/Atom") + + +if __name__ == "__main__": + test_validate_archive_path_blocks_traversal() + test_xss_safe_filter_removed() + test_security_headers_present() + test_server_url_not_hardcoded() + test_no_safe_filter_in_rss() + print("\nAll web interface tests passed!") diff --git a/web_interface.py b/web_interface.py index 35a0672..084d739 100644 --- a/web_interface.py +++ b/web_interface.py @@ -17,7 +17,7 @@ from datetime import datetime, timezone from functools import wraps from pathlib import Path from typing import Optional -from urllib.parse import quote, urlparse +from urllib.parse import quote, unquote, urlparse from flask import ( Flask, @@ -302,7 +302,9 @@ def articles(slug: str): def validate_archive_path(archive_path: str) -> Optional[Path]: - """Validate that an archive path is within ARCHIVE_DIR (prevents path traversal). + """Validate archive path is within ARCHIVE_DIR (prevents path traversal). + + URL-decodes path first to block encoded traversal (..%2F). Args: archive_path: Requested path component @@ -310,8 +312,9 @@ def validate_archive_path(archive_path: str) -> Optional[Path]: Returns: Resolved Path if valid, None if traversal detected """ + decoded = unquote(archive_path) try: - archive_file = (ARCHIVE_DIR / archive_path).resolve() + archive_file = (ARCHIVE_DIR / decoded).resolve() if not str(archive_file).startswith(str(ARCHIVE_DIR)): logger.warning("Path traversal attempt blocked: %s", archive_path) return None @@ -334,10 +337,7 @@ def serve_archive(archive_path): @login_required def serve_archive_file(encoded_path): """Serve archived HTML file from encoded path.""" - import urllib.parse - - archive_path = urllib.parse.unquote(encoded_path) - archive_file = validate_archive_path(archive_path) + archive_file = validate_archive_path(encoded_path) if archive_file and archive_file.exists(): return archive_file.read_text(encoding="utf-8") return "Archive not found", 404