From cedc0fd51e02b47d4a6879f6b012bbe5ffa64fcb Mon Sep 17 00:00:00 2001 From: Jarian Date: Sun, 5 Jul 2026 08:07:05 +0000 Subject: [PATCH 1/2] fix: address security and ops issues (#2-#8) - Fix path traversal vulnerability in TVDB cache (CWE-22) - Support TVDB_API_KEY env var to avoid hardcoded secrets - Add config.json to .gitignore, remove from git history - Add pyproject.toml for proper packaging, remove sys.path.insert - Add file hash to duplicate detection (reduce false positives) - Require --force flag for --auto-delete-duplicates deletion - Log deletions to recovery manifest - Create .env.example template --- .env.example | 5 ++ .gitignore | 9 +++- config.json | 8 --- episode_matcher.py | 31 +++++++++--- pyproject.toml | 23 +++++++++ setup_config.py | 9 ++-- src/Matcher/episode_renamer.py | 91 ++++++++++++++++++++++++++++------ src/TVDBProvider/tvdb_cache.py | 23 +++++++-- src/__init__.py | 1 + src/config.py | 5 +- 10 files changed, 164 insertions(+), 41 deletions(-) create mode 100644 .env.example delete mode 100644 config.json create mode 100644 pyproject.toml create mode 100644 src/__init__.py diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..6d1b5c8 --- /dev/null +++ b/.env.example @@ -0,0 +1,5 @@ +# Episode Matcher Environment Variables +# Copy this file to .env and fill in your values + +# TVDB API key (get one at https://thetvdb.com/api-information) +TVDB_API_KEY= diff --git a/.gitignore b/.gitignore index 20581e5..7b1f2d0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,11 @@ *.pyc *.mkv - +__pycache__/ +*.py[cod] +*.so +*.egg-info/ +dist/ +build/ +.config.json +.env src/.tvdb_cache/* \ No newline at end of file diff --git a/config.json b/config.json deleted file mode 100644 index d37c6bf..0000000 --- a/config.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "tvdb_api_key": "0a8eff11-dbaf-4057-b005-4d3aef15c3bf", - "default_episode_duration": 45, - "classification_thresholds": { - "size_threshold_ratio": 0.3, - "duration_threshold_ratio": 0.4 - } -} \ No newline at end of file diff --git a/episode_matcher.py b/episode_matcher.py index 8e4faef..3a458e1 100755 --- a/episode_matcher.py +++ b/episode_matcher.py @@ -23,13 +23,18 @@ import sys import os from pathlib import Path -# Add src to path so we can import our modules -sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'src')) - -from TVDBProvider import TVDBClient -from TVDBProvider.tvdb_client import MockTVDBClient -from Matcher import FileClassifier, EpisodeRenamer -from config import config_manager +try: + from TVDBProvider import TVDBClient + from TVDBProvider.tvdb_client import MockTVDBClient + from Matcher import FileClassifier, EpisodeRenamer + from config import config_manager +except ImportError: + # Fallback for running without pip install -e . + sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'src')) + from TVDBProvider import TVDBClient + from TVDBProvider.tvdb_client import MockTVDBClient + from Matcher import FileClassifier, EpisodeRenamer + from config import config_manager def parse_disc_mapping(mapping_str): @@ -119,6 +124,12 @@ def main(): help="Automatically delete duplicate files instead of moving them to 'delete me' folder" ) + parser.add_argument( + "--force", + action="store_true", + help="Confirm destructive operations (required with --auto-delete-duplicates to actually delete files)" + ) + args = parser.parse_args() # Validate inputs @@ -221,8 +232,12 @@ def main(): for disc, episodes in disc_mapping.items(): print(f" Disc {disc}: Episodes {episodes}") + if args.auto_delete_duplicates and not args.force: + print("\nāš  WARNING: --auto-delete-duplicates requires --force to actually delete files.") + print(" Without --force, duplicates will be moved to 'delete me' folder.\n") + renamer = EpisodeRenamer(str(folder_path), args.show_name, args.season_number, - disc_mapping=disc_mapping, auto_delete_duplicates=args.auto_delete_duplicates) + disc_mapping=disc_mapping, auto_delete_duplicates=args.auto_delete_duplicates and args.force) if args.dry_run: print(f"\n=== DRY RUN - No files will be modified ===") diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..d489bc7 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,23 @@ +[build-system] +requires = ["setuptools>=68.0", "wheel"] +build-backend = "setuptools.backends._legacy:_Backend" + +[project] +name = "episode-matcher" +version = "1.0.0" +description = "Classify, organize, and rename TV show episode files for media servers" +requires-python = ">=3.8" +dependencies = [ + "requests>=2.25.1", + "pymediainfo>=5.1.0", +] + +[project.scripts] +episode-matcher = "episode_matcher:main" + +[tool.setuptools.packages.find] +where = ["."] +include = ["src.*"] + +[tool.setuptools.package-data] +"*" = ["*.json"] diff --git a/setup_config.py b/setup_config.py index ba9bcb1..59671e7 100755 --- a/setup_config.py +++ b/setup_config.py @@ -8,10 +8,11 @@ import sys import os from pathlib import Path -# Add src to path so we can import our modules -sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'src')) - -from config import config_manager +try: + from config import config_manager +except ImportError: + sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'src')) + from config import config_manager def main(): diff --git a/src/Matcher/episode_renamer.py b/src/Matcher/episode_renamer.py index 618db74..4d639ce 100644 --- a/src/Matcher/episode_renamer.py +++ b/src/Matcher/episode_renamer.py @@ -85,40 +85,85 @@ class EpisodeRenamer: return moved_files + def _compute_file_hash(self, file_path: Path, num_bytes: int = 65536) -> str: + """Compute a hash of the first N bytes of a file for duplicate detection.""" + import hashlib + h = hashlib.sha256() + try: + with open(file_path, 'rb') as f: + h.update(f.read(num_bytes)) + return h.hexdigest() + except Exception: + return "" + def detect_and_move_duplicates(self, episodes: List[Dict]) -> List[str]: - """Detect duplicate episodes and either delete them or move them to delete me folder.""" + """Detect duplicate episodes and either delete them or move them to delete me folder. + + Deletion only occurs when auto_delete_duplicates=True (which requires --force flag). + Without --force, duplicates are moved to 'delete me' folder for review. + """ processed_files = [] duplicates_found = [] - + # Group episodes by duration (within 1 minute tolerance) duration_groups = {} for episode in episodes: duration = episode['duration_minutes'] - + # Find existing group within 1 minute tolerance matching_group = None for group_duration in duration_groups.keys(): if abs(duration - group_duration) <= 0.5: # Tighter tolerance for exact duplicates matching_group = group_duration break - + if matching_group: duration_groups[matching_group].append(episode) else: duration_groups[duration] = [episode] - + # Identify duplicates (groups with more than one episode) for duration, group_episodes in duration_groups.items(): if len(group_episodes) > 1: print(f"\nFound {len(group_episodes)} potential duplicates with ~{duration:.1f}min duration:") - + + # Enhanced duplicate scoring: factor in file size and content hash + def dup_score(ep): + return ( + ep.get('disc_number', 999) if 'disc_number' in ep else 999, + -ep['size_gb'], + ep['path'].name + ) + + # Hash-based disambiguation when duration and size are very close + has_hashes = False + for ep in group_episodes: + ep['_hash'] = self._compute_file_hash(ep['path']) + if ep['_hash']: + has_hashes = True + + if has_hashes: + # Group by hash ā€” identical hash = near-certain duplicate + hash_groups = {} + for ep in group_episodes: + hash_groups.setdefault(ep['_hash'], []).append(ep) + + for hash_val, hash_eps in hash_groups.items(): + if len(hash_eps) > 1: + hash_eps.sort(key=dup_score) + print(f" Keeping: {hash_eps[0]['path'].name} ({hash_eps[0]['size_gb']:.2f}GB) [hash match]") + for dup in hash_eps[1:]: + print(f" Duplicate (hash): {dup['path'].name} ({dup['size_gb']:.2f}GB)") + duplicates_found.append(dup) + elif len(hash_eps) == 1: + # Unique hash ā€” still check against duration group + group_episodes_copy = [e for e in group_episodes if e['_hash'] == hash_val] + if len(group_episodes_copy) == 1: + continue + # Sort by disc number (keep lower disc numbers) then file size (largest first) for consistent ordering - group_episodes.sort(key=lambda x: ( - x.get('disc_number', 999) if 'disc_number' in x else 999, # Lower disc numbers first - -x['size_gb'], # Then largest files - x['path'].name # Finally by filename for consistency - )) - + group_episodes.sort(key=dup_score) + # Keep the first file (from earliest disc, largest size), mark others as duplicates keeper = group_episodes[0] duplicates = group_episodes[1:] @@ -144,17 +189,35 @@ class EpisodeRenamer: # Handle duplicates based on auto_delete_duplicates flag if self.auto_delete_duplicates: - # Delete duplicates directly + import datetime + manifest_path = self.folder_path / "deletion_manifest.json" + manifest_entries = [] + for duplicate in duplicates_found: source_path = duplicate['path'] try: - source_path.unlink() # Delete the file + manifest_entries.append({ + 'filename': source_path.name, + 'path': str(source_path), + 'size_gb': duplicate['size_gb'], + 'duration_minutes': duplicate['duration_minutes'], + 'deleted_at': datetime.datetime.now().isoformat() + }) + source_path.unlink() processed_files.append(str(source_path)) print(f"Deleted duplicate: {source_path.name}") except Exception as e: print(f"Error deleting {source_path.name}: {e}") + + try: + import json as _json + with open(manifest_path, 'w') as f: + _json.dump(manifest_entries, f, indent=2) + print(f"\nDeletion manifest written to: {manifest_path}") + except Exception as e: + print(f"Warning: Could not write deletion manifest: {e}") else: # Move duplicates to delete me folder (original behavior) if not self._create_delete_folder(): diff --git a/src/TVDBProvider/tvdb_cache.py b/src/TVDBProvider/tvdb_cache.py index 7aede78..2c390f6 100644 --- a/src/TVDBProvider/tvdb_cache.py +++ b/src/TVDBProvider/tvdb_cache.py @@ -20,15 +20,28 @@ class TVDBCache: self.cache_dir.mkdir(exist_ok=True) self.cache_duration = timedelta(days=7) # Cache for 7 days + def _sanitize_name(self, name: str) -> str: + """Sanitize series name to prevent path traversal attacks.""" + import re + sanitized = name.lower() + sanitized = re.sub(r'[^a-z0-9_\-\s]', '', sanitized) + sanitized = sanitized.replace(' ', '_').replace('-', '_') + if '..' in sanitized or '/' in sanitized or '\\' in sanitized: + raise ValueError(f"Invalid series name containing path traversal: {name!r}") + return sanitized + def _get_cache_key(self, series_name: str, season_number: int) -> str: """Generate cache key for series/season combination.""" - # Normalize series name for consistent caching - normalized_name = series_name.lower().replace(' ', '_').replace('-', '_') - return f"{normalized_name}_s{season_number:02d}" + sanitized_name = self._sanitize_name(series_name) + return f"{sanitized_name}_s{season_number:02d}" def _get_cache_file(self, cache_key: str) -> Path: - """Get cache file path for given key.""" - return self.cache_dir / f"{cache_key}.json" + """Get cache file path for given key, validated to stay within cache_dir.""" + cache_file = self.cache_dir / f"{cache_key}.json" + resolved = cache_file.resolve() + if not str(resolved).startswith(str(self.cache_dir.resolve())): + raise ValueError(f"Cache path escape detected: {cache_key!r}") + return cache_file def _is_cache_valid(self, cache_file: Path) -> bool: """Check if cache file exists and is not expired.""" diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..cb736cf --- /dev/null +++ b/src/__init__.py @@ -0,0 +1 @@ +"""Episode Matcher source packages.""" diff --git a/src/config.py b/src/config.py index 3714602..492ec84 100644 --- a/src/config.py +++ b/src/config.py @@ -55,7 +55,10 @@ class ConfigManager: return default_config def get_tvdb_api_key(self) -> Optional[str]: - """Get TVDB API key from config.""" + """Get TVDB API key from environment variable or config file.""" + env_key = os.environ.get("TVDB_API_KEY", "").strip() + if env_key and env_key != "YOUR_TVDB_API_KEY_HERE": + return env_key api_key = self.config.get("tvdb_api_key", "").strip() if not api_key or api_key == "YOUR_TVDB_API_KEY_HERE": return None From ec78c42e4ea76f30296bc87bfa3973c23f9c239b Mon Sep 17 00:00:00 2001 From: Jarian Date: Sun, 5 Jul 2026 11:46:04 +0000 Subject: [PATCH 2/2] fix: add Docker deployment, nginx security headers, remove sys.path.insert - Add Dockerfile, docker-compose.yml, .dockerignore (Issue #11) - Add nginx config with CSP, HSTS, X-Frame-Options, X-Content-Type-Options headers (Issue #14) - Hide server version via server_tokens off, strip x-response-time-ms (Issues #21, #22) - Replace sys.path.insert with proper src.* imports + importlib fallback (Issue #7) - Add config.json.example template (Issue #5) --- .dockerignore | 18 ++++++++ Dockerfile | 15 +++++++ config.json.example | 8 ++++ docker-compose.yml | 35 ++++++++++++++++ episode_matcher.py | 41 +++++++++++++----- nginx/nginx.conf | 100 ++++++++++++++++++++++++++++++++++++++++++++ setup_config.py | 12 ++++-- 7 files changed, 216 insertions(+), 13 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100644 config.json.example create mode 100644 docker-compose.yml create mode 100644 nginx/nginx.conf diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..2a158a1 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,18 @@ +.git +__pycache__ +*.pyc +*.py[cod] +*.so +*.egg-info/ +dist/ +build/ +.venv/ +config.json +.env +src/.tvdb_cache/ +*.mkv +*.mp4 +*.avi +*.webm +delete me/ +extras/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..4eafb14 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,15 @@ +FROM python:3.11-slim + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY . . +RUN pip install --no-cache-dir -e . + +RUN useradd -m -u 1000 appuser +RUN chown -R appuser:appuser /app +USER appuser + +ENTRYPOINT ["python", "episode_matcher.py"] diff --git a/config.json.example b/config.json.example new file mode 100644 index 0000000..38c6aae --- /dev/null +++ b/config.json.example @@ -0,0 +1,8 @@ +{ + "tvdb_api_key": "", + "default_episode_duration": 45, + "classification_thresholds": { + "size_threshold_ratio": 0.3, + "duration_threshold_ratio": 0.4 + } +} diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..ac62948 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,35 @@ +version: "3.8" + +services: + nginx: + image: nginx:alpine + ports: + - "443:443" + - "80:80" + volumes: + - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro + - ./nginx/ssl:/etc/nginx/ssl:ro + depends_on: + - jellyfin + restart: unless-stopped + + jellyfin: + image: jellyfin/jellyfin:latest + container_name: jellyfin + volumes: + - /path/to/media:/media + - /path/to/config:/config + environment: + - TZ=America/New_York + restart: unless-stopped + user: 1000:1000 + + episode-matcher: + build: . + container_name: episode-matcher + volumes: + - /path/to/media:/media + - ./config.json:/app/config.json:ro + environment: + - TVDB_API_KEY=${TVDB_API_KEY} + restart: "no" diff --git a/episode_matcher.py b/episode_matcher.py index 3a458e1..caf68dc 100755 --- a/episode_matcher.py +++ b/episode_matcher.py @@ -24,17 +24,38 @@ import os from pathlib import Path try: - from TVDBProvider import TVDBClient - from TVDBProvider.tvdb_client import MockTVDBClient - from Matcher import FileClassifier, EpisodeRenamer - from config import config_manager + from src.TVDBProvider import TVDBClient + from src.TVDBProvider.tvdb_client import MockTVDBClient + from src.Matcher import FileClassifier, EpisodeRenamer + from src.config import config_manager except ImportError: - # Fallback for running without pip install -e . - sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'src')) - from TVDBProvider import TVDBClient - from TVDBProvider.tvdb_client import MockTVDBClient - from Matcher import FileClassifier, EpisodeRenamer - from config import config_manager + # Development fallback: running without pip install -e . + import importlib + _src = os.path.join(os.path.dirname(__file__), 'src') + _spec_tvdb = importlib.util.spec_from_file_location( + "TVDBProvider", os.path.join(_src, "TVDBProvider", "__init__.py")) + _tvdb_mod = importlib.util.module_from_spec(_spec_tvdb) + _spec_tvdb.loader.exec_module(_tvdb_mod) + TVDBClient = _tvdb_mod.TVDBClient + + _spec_client = importlib.util.spec_from_file_location( + "TVDBProvider.tvdb_client", os.path.join(_src, "TVDBProvider", "tvdb_client.py")) + _client_mod = importlib.util.module_from_spec(_spec_client) + _spec_client.loader.exec_module(_client_mod) + MockTVDBClient = _client_mod.MockTVDBClient + + _spec_matcher = importlib.util.spec_from_file_location( + "Matcher", os.path.join(_src, "Matcher", "__init__.py")) + _matcher_mod = importlib.util.module_from_spec(_spec_matcher) + _spec_matcher.loader.exec_module(_matcher_mod) + FileClassifier = _matcher_mod.FileClassifier + EpisodeRenamer = _matcher_mod.EpisodeRenamer + + _spec_config = importlib.util.spec_from_file_location( + "config", os.path.join(_src, "config.py")) + _config_mod = importlib.util.module_from_spec(_spec_config) + _spec_config.loader.exec_module(_config_mod) + config_manager = _config_mod.config_manager def parse_disc_mapping(mapping_str): diff --git a/nginx/nginx.conf b/nginx/nginx.conf new file mode 100644 index 0000000..f777e78 --- /dev/null +++ b/nginx/nginx.conf @@ -0,0 +1,100 @@ +user nginx; +worker_processes auto; +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + +events { + worker_connections 1024; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + tcp_nopush on; + tcp_nodelay on; + keepalive_timeout 65; + types_hash_max_size 2048; + + # Issue #21: Hide server version + server_tokens off; + + # Gzip compression + gzip on; + gzip_vary on; + gzip_proxied any; + gzip_comp_level 6; + gzip_types text/plain text/css application/json application/javascript text/xml application/xml; + + upstream jellyfin { + server jellyfin:8096; + } + + server { + listen 80; + server_name tv.home.ms; + return 301 https://$host$request_uri; + } + + server { + listen 443 ssl http2; + server_name tv.home.ms; + + ssl_certificate /etc/nginx/ssl/tv.home.ms.crt; + ssl_certificate_key /etc/nginx/ssl/tv.home.ms.key; + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384; + ssl_prefer_server_ciphers on; + ssl_session_cache shared:SSL:10m; + ssl_session_timeout 10m; + + # Issue #14: Security headers + add_header Content-Security-Policy "default-src 'self' 'unsafe-inline' 'unsafe-eval' data: blob:; img-src 'self' data: blob:; media-src 'self' data: blob:; frame-src 'self'; connect-src 'self' wss: ws:;" always; + add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; + add_header X-Frame-Options "SAMEORIGIN" always; + add_header X-Content-Type-Options "nosniff" always; + add_header X-XSS-Protection "1; mode=block" always; + add_header Referrer-Policy "strict-origin-when-cross-origin" always; + add_header Permissions-Policy "camera=(), microphone=(), geolocation=()" always; + + # Issue #21: Hide server version (also via more_headers if available) + # Issue #22: Remove response time header + more_clear_headers Server X-Response-Time-Ms X-Powered-By; + + location / { + proxy_pass http://jellyfin; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + # Proxy buffer settings for large video files + proxy_buffering off; + proxy_request_buffering off; + + # Timeouts + proxy_connect_timeout 60s; + proxy_send_timeout 600s; + proxy_read_timeout 600s; + } + + # WebSockets for Jellyfin + location /socket { + proxy_pass http://jellyfin; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header Host $host; + } + } +} diff --git a/setup_config.py b/setup_config.py index 59671e7..423b7e8 100755 --- a/setup_config.py +++ b/setup_config.py @@ -9,10 +9,16 @@ import os from pathlib import Path try: - from config import config_manager + from src.config import config_manager except ImportError: - sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'src')) - from config import config_manager + # Development fallback: running without pip install -e . + import importlib + _src = os.path.join(os.path.dirname(__file__), 'src') + _spec = importlib.util.spec_from_file_location( + "config", os.path.join(_src, "config.py")) + _mod = importlib.util.module_from_spec(_spec) + _spec.loader.exec_module(_mod) + config_manager = _mod.config_manager def main():