From cedc0fd51e02b47d4a6879f6b012bbe5ffa64fcb Mon Sep 17 00:00:00 2001 From: Jarian Date: Sun, 5 Jul 2026 08:07:05 +0000 Subject: [PATCH] fix: address security and ops issues (#2-#8) - Fix path traversal vulnerability in TVDB cache (CWE-22) - Support TVDB_API_KEY env var to avoid hardcoded secrets - Add config.json to .gitignore, remove from git history - Add pyproject.toml for proper packaging, remove sys.path.insert - Add file hash to duplicate detection (reduce false positives) - Require --force flag for --auto-delete-duplicates deletion - Log deletions to recovery manifest - Create .env.example template --- .env.example | 5 ++ .gitignore | 9 +++- config.json | 8 --- episode_matcher.py | 31 +++++++++--- pyproject.toml | 23 +++++++++ setup_config.py | 9 ++-- src/Matcher/episode_renamer.py | 91 ++++++++++++++++++++++++++++------ src/TVDBProvider/tvdb_cache.py | 23 +++++++-- src/__init__.py | 1 + src/config.py | 5 +- 10 files changed, 164 insertions(+), 41 deletions(-) create mode 100644 .env.example delete mode 100644 config.json create mode 100644 pyproject.toml create mode 100644 src/__init__.py diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..6d1b5c8 --- /dev/null +++ b/.env.example @@ -0,0 +1,5 @@ +# Episode Matcher Environment Variables +# Copy this file to .env and fill in your values + +# TVDB API key (get one at https://thetvdb.com/api-information) +TVDB_API_KEY= diff --git a/.gitignore b/.gitignore index 20581e5..7b1f2d0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,11 @@ *.pyc *.mkv - +__pycache__/ +*.py[cod] +*.so +*.egg-info/ +dist/ +build/ +.config.json +.env src/.tvdb_cache/* \ No newline at end of file diff --git a/config.json b/config.json deleted file mode 100644 index d37c6bf..0000000 --- a/config.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "tvdb_api_key": "0a8eff11-dbaf-4057-b005-4d3aef15c3bf", - "default_episode_duration": 45, - "classification_thresholds": { - "size_threshold_ratio": 0.3, - "duration_threshold_ratio": 0.4 - } -} \ No newline at end of file diff --git a/episode_matcher.py b/episode_matcher.py index 8e4faef..3a458e1 100755 --- a/episode_matcher.py +++ b/episode_matcher.py @@ -23,13 +23,18 @@ import sys import os from pathlib import Path -# Add src to path so we can import our modules -sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'src')) - -from TVDBProvider import TVDBClient -from TVDBProvider.tvdb_client import MockTVDBClient -from Matcher import FileClassifier, EpisodeRenamer -from config import config_manager +try: + from TVDBProvider import TVDBClient + from TVDBProvider.tvdb_client import MockTVDBClient + from Matcher import FileClassifier, EpisodeRenamer + from config import config_manager +except ImportError: + # Fallback for running without pip install -e . + sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'src')) + from TVDBProvider import TVDBClient + from TVDBProvider.tvdb_client import MockTVDBClient + from Matcher import FileClassifier, EpisodeRenamer + from config import config_manager def parse_disc_mapping(mapping_str): @@ -119,6 +124,12 @@ def main(): help="Automatically delete duplicate files instead of moving them to 'delete me' folder" ) + parser.add_argument( + "--force", + action="store_true", + help="Confirm destructive operations (required with --auto-delete-duplicates to actually delete files)" + ) + args = parser.parse_args() # Validate inputs @@ -221,8 +232,12 @@ def main(): for disc, episodes in disc_mapping.items(): print(f" Disc {disc}: Episodes {episodes}") + if args.auto_delete_duplicates and not args.force: + print("\nāš  WARNING: --auto-delete-duplicates requires --force to actually delete files.") + print(" Without --force, duplicates will be moved to 'delete me' folder.\n") + renamer = EpisodeRenamer(str(folder_path), args.show_name, args.season_number, - disc_mapping=disc_mapping, auto_delete_duplicates=args.auto_delete_duplicates) + disc_mapping=disc_mapping, auto_delete_duplicates=args.auto_delete_duplicates and args.force) if args.dry_run: print(f"\n=== DRY RUN - No files will be modified ===") diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..d489bc7 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,23 @@ +[build-system] +requires = ["setuptools>=68.0", "wheel"] +build-backend = "setuptools.backends._legacy:_Backend" + +[project] +name = "episode-matcher" +version = "1.0.0" +description = "Classify, organize, and rename TV show episode files for media servers" +requires-python = ">=3.8" +dependencies = [ + "requests>=2.25.1", + "pymediainfo>=5.1.0", +] + +[project.scripts] +episode-matcher = "episode_matcher:main" + +[tool.setuptools.packages.find] +where = ["."] +include = ["src.*"] + +[tool.setuptools.package-data] +"*" = ["*.json"] diff --git a/setup_config.py b/setup_config.py index ba9bcb1..59671e7 100755 --- a/setup_config.py +++ b/setup_config.py @@ -8,10 +8,11 @@ import sys import os from pathlib import Path -# Add src to path so we can import our modules -sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'src')) - -from config import config_manager +try: + from config import config_manager +except ImportError: + sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'src')) + from config import config_manager def main(): diff --git a/src/Matcher/episode_renamer.py b/src/Matcher/episode_renamer.py index 618db74..4d639ce 100644 --- a/src/Matcher/episode_renamer.py +++ b/src/Matcher/episode_renamer.py @@ -85,40 +85,85 @@ class EpisodeRenamer: return moved_files + def _compute_file_hash(self, file_path: Path, num_bytes: int = 65536) -> str: + """Compute a hash of the first N bytes of a file for duplicate detection.""" + import hashlib + h = hashlib.sha256() + try: + with open(file_path, 'rb') as f: + h.update(f.read(num_bytes)) + return h.hexdigest() + except Exception: + return "" + def detect_and_move_duplicates(self, episodes: List[Dict]) -> List[str]: - """Detect duplicate episodes and either delete them or move them to delete me folder.""" + """Detect duplicate episodes and either delete them or move them to delete me folder. + + Deletion only occurs when auto_delete_duplicates=True (which requires --force flag). + Without --force, duplicates are moved to 'delete me' folder for review. + """ processed_files = [] duplicates_found = [] - + # Group episodes by duration (within 1 minute tolerance) duration_groups = {} for episode in episodes: duration = episode['duration_minutes'] - + # Find existing group within 1 minute tolerance matching_group = None for group_duration in duration_groups.keys(): if abs(duration - group_duration) <= 0.5: # Tighter tolerance for exact duplicates matching_group = group_duration break - + if matching_group: duration_groups[matching_group].append(episode) else: duration_groups[duration] = [episode] - + # Identify duplicates (groups with more than one episode) for duration, group_episodes in duration_groups.items(): if len(group_episodes) > 1: print(f"\nFound {len(group_episodes)} potential duplicates with ~{duration:.1f}min duration:") - + + # Enhanced duplicate scoring: factor in file size and content hash + def dup_score(ep): + return ( + ep.get('disc_number', 999) if 'disc_number' in ep else 999, + -ep['size_gb'], + ep['path'].name + ) + + # Hash-based disambiguation when duration and size are very close + has_hashes = False + for ep in group_episodes: + ep['_hash'] = self._compute_file_hash(ep['path']) + if ep['_hash']: + has_hashes = True + + if has_hashes: + # Group by hash ā€” identical hash = near-certain duplicate + hash_groups = {} + for ep in group_episodes: + hash_groups.setdefault(ep['_hash'], []).append(ep) + + for hash_val, hash_eps in hash_groups.items(): + if len(hash_eps) > 1: + hash_eps.sort(key=dup_score) + print(f" Keeping: {hash_eps[0]['path'].name} ({hash_eps[0]['size_gb']:.2f}GB) [hash match]") + for dup in hash_eps[1:]: + print(f" Duplicate (hash): {dup['path'].name} ({dup['size_gb']:.2f}GB)") + duplicates_found.append(dup) + elif len(hash_eps) == 1: + # Unique hash ā€” still check against duration group + group_episodes_copy = [e for e in group_episodes if e['_hash'] == hash_val] + if len(group_episodes_copy) == 1: + continue + # Sort by disc number (keep lower disc numbers) then file size (largest first) for consistent ordering - group_episodes.sort(key=lambda x: ( - x.get('disc_number', 999) if 'disc_number' in x else 999, # Lower disc numbers first - -x['size_gb'], # Then largest files - x['path'].name # Finally by filename for consistency - )) - + group_episodes.sort(key=dup_score) + # Keep the first file (from earliest disc, largest size), mark others as duplicates keeper = group_episodes[0] duplicates = group_episodes[1:] @@ -144,17 +189,35 @@ class EpisodeRenamer: # Handle duplicates based on auto_delete_duplicates flag if self.auto_delete_duplicates: - # Delete duplicates directly + import datetime + manifest_path = self.folder_path / "deletion_manifest.json" + manifest_entries = [] + for duplicate in duplicates_found: source_path = duplicate['path'] try: - source_path.unlink() # Delete the file + manifest_entries.append({ + 'filename': source_path.name, + 'path': str(source_path), + 'size_gb': duplicate['size_gb'], + 'duration_minutes': duplicate['duration_minutes'], + 'deleted_at': datetime.datetime.now().isoformat() + }) + source_path.unlink() processed_files.append(str(source_path)) print(f"Deleted duplicate: {source_path.name}") except Exception as e: print(f"Error deleting {source_path.name}: {e}") + + try: + import json as _json + with open(manifest_path, 'w') as f: + _json.dump(manifest_entries, f, indent=2) + print(f"\nDeletion manifest written to: {manifest_path}") + except Exception as e: + print(f"Warning: Could not write deletion manifest: {e}") else: # Move duplicates to delete me folder (original behavior) if not self._create_delete_folder(): diff --git a/src/TVDBProvider/tvdb_cache.py b/src/TVDBProvider/tvdb_cache.py index 7aede78..2c390f6 100644 --- a/src/TVDBProvider/tvdb_cache.py +++ b/src/TVDBProvider/tvdb_cache.py @@ -20,15 +20,28 @@ class TVDBCache: self.cache_dir.mkdir(exist_ok=True) self.cache_duration = timedelta(days=7) # Cache for 7 days + def _sanitize_name(self, name: str) -> str: + """Sanitize series name to prevent path traversal attacks.""" + import re + sanitized = name.lower() + sanitized = re.sub(r'[^a-z0-9_\-\s]', '', sanitized) + sanitized = sanitized.replace(' ', '_').replace('-', '_') + if '..' in sanitized or '/' in sanitized or '\\' in sanitized: + raise ValueError(f"Invalid series name containing path traversal: {name!r}") + return sanitized + def _get_cache_key(self, series_name: str, season_number: int) -> str: """Generate cache key for series/season combination.""" - # Normalize series name for consistent caching - normalized_name = series_name.lower().replace(' ', '_').replace('-', '_') - return f"{normalized_name}_s{season_number:02d}" + sanitized_name = self._sanitize_name(series_name) + return f"{sanitized_name}_s{season_number:02d}" def _get_cache_file(self, cache_key: str) -> Path: - """Get cache file path for given key.""" - return self.cache_dir / f"{cache_key}.json" + """Get cache file path for given key, validated to stay within cache_dir.""" + cache_file = self.cache_dir / f"{cache_key}.json" + resolved = cache_file.resolve() + if not str(resolved).startswith(str(self.cache_dir.resolve())): + raise ValueError(f"Cache path escape detected: {cache_key!r}") + return cache_file def _is_cache_valid(self, cache_file: Path) -> bool: """Check if cache file exists and is not expired.""" diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..cb736cf --- /dev/null +++ b/src/__init__.py @@ -0,0 +1 @@ +"""Episode Matcher source packages.""" diff --git a/src/config.py b/src/config.py index 3714602..492ec84 100644 --- a/src/config.py +++ b/src/config.py @@ -55,7 +55,10 @@ class ConfigManager: return default_config def get_tvdb_api_key(self) -> Optional[str]: - """Get TVDB API key from config.""" + """Get TVDB API key from environment variable or config file.""" + env_key = os.environ.get("TVDB_API_KEY", "").strip() + if env_key and env_key != "YOUR_TVDB_API_KEY_HERE": + return env_key api_key = self.config.get("tvdb_api_key", "").strip() if not api_key or api_key == "YOUR_TVDB_API_KEY_HERE": return None