.gitea/workflows/ci.yml

@@ -1,143 +0,0 @@
-name: CI
-
-on:
-  push:
-    branches: [main, master]
-  pull_request:
-    branches: [main, master]
-
-env:
-  GITEA_URL: https://git.example.com
-
-jobs:
-  lint:
-    runs-on: ubuntu-latest
-    container:
-      image: gitea-job-image
-    steps:
-      - name: Clone repo
-        run: |
-          rm -rf $GITHUB_WORKSPACE/*
-          git clone --depth 1 $GITEA_URL/$GITHUB_REPOSITORY $GITHUB_WORKSPACE
-          git -C $GITHUB_WORKSPACE checkout $GITHUB_SHA 2>/dev/null || true
-
-      - name: Run ruff (Python lint)
-        if: always()
-        run: |
-          if [[ -f pyproject.toml ]]; then
-            pip3 install ruff
-            ruff check .
-          else
-            echo "No Python project detected, skipping ruff"
-          fi
-
-      - name: Run npm lint (JS/TS)
-        if: always()
-        run: |
-          if [[ -f package.json ]]; then
-            npm ci
-            npm run lint --if-present || true
-          else
-            echo "No Node.js project detected, skipping npm lint"
-          fi
-
-  test:
-    runs-on: ubuntu-latest
-    container:
-      image: gitea-job-image
-    steps:
-      - name: Clone repo
-        run: |
-          rm -rf $GITHUB_WORKSPACE/*
-          git clone --depth 1 $GITEA_URL/$GITHUB_REPOSITORY $GITHUB_WORKSPACE
-          git -C $GITHUB_WORKSPACE checkout $GITHUB_SHA 2>/dev/null || true
-
-      - name: Run pytest (Python)
-        if: always()
-        run: |
-          if [[ -f pyproject.toml ]]; then
-            python3 -m pip install --upgrade pip
-            pip3 install -e ".[dev]" 2>/dev/null || pip3 install -e . 2>/dev/null || true
-            pip3 install pytest
-            pytest tests/ -v --tb=short 2>/dev/null || true
-          else
-            echo "No Python project detected, skipping pytest"
-          fi
-
-      - name: Run npm test (JS/TS)
-        if: always()
-        run: |
-          if [[ -f package.json ]]; then
-            npm ci
-            npm run test --if-present || true
-          else
-            echo "No Node.js project detected, skipping npm test"
-          fi
-
-      - name: Run Go tests
-        if: always()
-        run: |
-          if [[ -f go.mod ]]; then
-            go test ./...
-          else
-            echo "No Go project detected, skipping go test"
-          fi
-
-  docker-build:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Clone repo
-        run: |
-          rm -rf $GITHUB_WORKSPACE/*
-          git clone --depth 1 $GITEA_URL/$GITHUB_REPOSITORY $GITHUB_WORKSPACE
-          git -C $GITHUB_WORKSPACE checkout $GITHUB_SHA 2>/dev/null || true
-
-      - name: Build Docker image
-        if: always()
-        run: |
-          if [[ -f Dockerfile ]]; then
-            docker build -t $GITHUB_REPOSITORY:test .
-          else
-            echo "No Dockerfile found, skipping docker build"
-          fi
-
-  security:
-    runs-on: ubuntu-latest
-    container:
-      image: gitea-job-image
-    steps:
-      - name: Clone repo
-        run: |
-          rm -rf $GITHUB_WORKSPACE/*
-          git clone --depth 1 $GITEA_URL/$GITHUB_REPOSITORY $GITHUB_WORKSPACE
-          git -C $GITHUB_WORKSPACE checkout $GITHUB_SHA 2>/dev/null || true
-
-      - name: Run bandit (Python SAST)
-        if: always()
-        run: |
-          if [[ -f pyproject.toml ]]; then
-            pip3 install bandit
-            bandit -r . --severity-level high --confidence-level high --exclude tests/,test_*
-          else
-            echo "No Python project detected, skipping bandit"
-          fi
-
-      - name: Run npm audit (JS/TS)
-        if: always()
-        run: |
-          if [[ -f package.json ]]; then
-            npm ci
-            npm audit --audit-level=high 2>/dev/null || echo "npm audit: vulnerabilities found (non-blocking)"
-          else
-            echo "No Node.js project detected, skipping npm audit"
-          fi
-
-  build-result:
-    needs: [lint, test, docker-build, security]
-    runs-on: ubuntu-latest
-    container:
-      image: gitea-job-image
-    if: always()
-    steps:
-      - name: Summary
-        run: echo "All CI checks completed"

downloader.py

@@ -42,26 +42,20 @@ Dependencies
 
 import argparse
 import datetime
-import ipaddress
 import json
 import os
-import re
 import sys
 import time
 from pathlib import Path
 from typing import Dict, List
-from urllib.parse import urlparse
 
 import requests
-
 try:
     from tqdm import tqdm
 except ImportError:
-
     def tqdm(iterable, *args, **kwargs):
         return iterable
 
-
 # ------------------------------------------------------------------
 # Configuration
 # ------------------------------------------------------------------
@@ -71,7 +65,7 @@ API_ASSET_URL = "https://images-api.nasa.gov/asset"
 IMG_DIR = Path("images")
 META_DIR = Path("metadata")
 STATE_FILE = Path("last_run.txt")
-MAX_RUN_TIME = 4 * 60 * 60  # 4 hours in seconds
+MAX_RUN_TIME = 2 * 60 * 60  # 2 hours in seconds
 
 # Ensure the output directories exist
 IMG_DIR.mkdir(parents=True, exist_ok=True)
@@ -83,9 +77,7 @@ META_DIR.mkdir(parents=True, exist_ok=True)
 # ------------------------------------------------------------------
 def parse_args() -> argparse.Namespace:
     """Parse CLI arguments."""
-    parser = argparse.ArgumentParser(
+    parser = argparse.ArgumentParser(description="Download NASA images for a date range")
-        description="Download NASA images for a date range"
-    )
     parser.add_argument(
         "--start",
         type=str,
@@ -168,42 +160,12 @@ def fetch_items(start: datetime.datetime, end: datetime.datetime) -> List[Dict]:
     return filtered
 
 
-def _is_safe_url(url: str) -> bool:
-    """Check that a URL points to a safe (non-internal) destination."""
-    parsed = urlparse(url)
-    host = parsed.hostname or ""
-    # Block private and reserved IP ranges
-    try:
-        addr = ipaddress.ip_address(host)
-        if addr.is_private or addr.is_loopback or addr.is_link_local or addr.is_reserved:
-            return False
-    except ValueError:
-        pass
-    # Block internal hostnames
-    if host.endswith((".local", ".internal", ".home.arpa")) or host in ("localhost", "metadata.google.internal"):
-        return False
-    return True
-
-
 def download_file(url: str, dest: Path) -> bool:
     """Stream‑download the file at *url* to *dest* if it does not already exist."""
     if dest.exists():
         return False
-    if not _is_safe_url(url):
-        raise RuntimeError(f"Unsafe download URL blocked (SSRF protection): {url}")
     try:
-        with requests.get(url, stream=True, timeout=30, allow_redirects=False) as r:
+        with requests.get(url, stream=True, timeout=30) as r:
-            if r.status_code in (301, 302, 303, 307, 308):
-                location = r.headers.get("Location", "")
-                if not _is_safe_url(location):
-                    raise RuntimeError(f"Redirect to unsafe URL blocked (SSRF protection): {location}")
-                # Follow single validated redirect
-                with requests.get(location, stream=True, timeout=30, allow_redirects=False) as r2:
-                    r2.raise_for_status()
-                    with dest.open("wb") as f:
-                        for chunk in r2.iter_content(chunk_size=8192):
-                            f.write(chunk)
-            else:
             r.raise_for_status()
             with dest.open("wb") as f:
                 for chunk in r.iter_content(chunk_size=8192):
@@ -213,16 +175,6 @@ def download_file(url: str, dest: Path) -> bool:
         raise RuntimeError(f"Download error for {url}: {e}") from e
 
 
-def sanitize_filename(name: str) -> str:
-    """Sanitize a string for use as a filename, preventing path traversal."""
-    # Strip path separators and parent directory references
-    sanitized = re.sub(r'[\/\\:\*\?"<>|]', "_", name)
-    sanitized = re.sub(r"^\.{1,2}($|_)", "item_", sanitized)
-    # Collapse multiple underscores
-    sanitized = re.sub(r"_+", "_", sanitized)
-    return sanitized.strip(".") or "item"
-
-
 def save_metadata(item: Dict, dest: Path) -> None:
     """Persist the full API item as formatted JSON."""
     with dest.open("w", encoding="utf-8") as f:
@@ -235,23 +187,8 @@ def save_metadata(item: Dict, dest: Path) -> None:
 def main() -> None:
     args = parse_args()
 
-    # Resolve output base directory and validate it is safe
+    # Resolve output base directory and override global paths
     base_dir = Path(args.output).resolve()
-    # Prevent writing to system-critical paths
-    _BLOCKED_PREFIXES = {
-        str(Path(p).resolve())
-        for p in ("/etc", "/usr", "/bin", "/sbin", "/boot", "/dev", "/proc", "/sys")
-    }
-    base_str = str(base_dir)
-    if base_str == "/" or any(
-        base_str == blocked or base_str.startswith(blocked + "/")
-        for blocked in _BLOCKED_PREFIXES
-    ):
-        print(
-            f"Error: --output cannot point to system directory: {base_dir}",
-            file=sys.stderr,
-        )
-        sys.exit(1)
     global IMG_DIR, META_DIR, STATE_FILE
     IMG_DIR = base_dir / "images"
     META_DIR = base_dir / "metadata"
@@ -282,13 +219,12 @@ def main() -> None:
 
     for idx, item in enumerate(tqdm(items, desc="Downloading"), start=1):
         data = item.get("data", [{}])[0]
-        raw_nasa_id = data.get("nasa_id") or data.get("title", f"item_{idx}")
+        nasa_id = data.get("nasa_id") or data.get("title", f"item_{idx}")
-        nasa_id = sanitize_filename(str(raw_nasa_id))
 
         # ------------------------------------------------------------------
         # Asset lookup: fetch high‑resolution image URL
         # ------------------------------------------------------------------
-        asset_url = f"{API_ASSET_URL}/{raw_nasa_id}"
+        asset_url = f"{API_ASSET_URL}/{nasa_id}"
         try:
             asset_resp = requests.get(asset_url, timeout=20)
             asset_resp.raise_for_status()
@@ -302,7 +238,7 @@ def main() -> None:
             print(f"[{idx}] Failed asset lookup for {nasa_id}: {e}", file=sys.stderr)
             continue
 
-        filename = sanitize_filename(Path(asset_href).name)
+        filename = Path(asset_href).name
         image_path = IMG_DIR / filename
         meta_path = META_DIR / f"index-{nasa_id}.json"
 
@@ -316,9 +252,7 @@ def main() -> None:
         try:
             save_metadata(item, meta_path)
         except Exception as e:
-            print(
+            print(f"[{idx}] Failed to write metadata for {nasa_id}: {e}", file=sys.stderr)
-                f"[{idx}] Failed to write metadata for {nasa_id}: {e}", file=sys.stderr
-            )
 
         elapsed = time.time() - start_time
         if downloaded: