Compare commits
No commits in common. "f843270aab61c9acb1ab409608c76cb85c53ae76" and "65623e486fa518a5ca39ebc89928316fe7fedfb6" have entirely different histories.
f843270aab
...
65623e486f
@ -1,143 +0,0 @@
|
||||
name: CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, master]
|
||||
pull_request:
|
||||
branches: [main, master]
|
||||
|
||||
env:
|
||||
GITEA_URL: https://git.example.com
|
||||
|
||||
jobs:
|
||||
lint:
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: gitea-job-image
|
||||
steps:
|
||||
- name: Clone repo
|
||||
run: |
|
||||
rm -rf $GITHUB_WORKSPACE/*
|
||||
git clone --depth 1 $GITEA_URL/$GITHUB_REPOSITORY $GITHUB_WORKSPACE
|
||||
git -C $GITHUB_WORKSPACE checkout $GITHUB_SHA 2>/dev/null || true
|
||||
|
||||
- name: Run ruff (Python lint)
|
||||
if: always()
|
||||
run: |
|
||||
if [[ -f pyproject.toml ]]; then
|
||||
pip3 install ruff
|
||||
ruff check .
|
||||
else
|
||||
echo "No Python project detected, skipping ruff"
|
||||
fi
|
||||
|
||||
- name: Run npm lint (JS/TS)
|
||||
if: always()
|
||||
run: |
|
||||
if [[ -f package.json ]]; then
|
||||
npm ci
|
||||
npm run lint --if-present || true
|
||||
else
|
||||
echo "No Node.js project detected, skipping npm lint"
|
||||
fi
|
||||
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: gitea-job-image
|
||||
steps:
|
||||
- name: Clone repo
|
||||
run: |
|
||||
rm -rf $GITHUB_WORKSPACE/*
|
||||
git clone --depth 1 $GITEA_URL/$GITHUB_REPOSITORY $GITHUB_WORKSPACE
|
||||
git -C $GITHUB_WORKSPACE checkout $GITHUB_SHA 2>/dev/null || true
|
||||
|
||||
- name: Run pytest (Python)
|
||||
if: always()
|
||||
run: |
|
||||
if [[ -f pyproject.toml ]]; then
|
||||
python3 -m pip install --upgrade pip
|
||||
pip3 install -e ".[dev]" 2>/dev/null || pip3 install -e . 2>/dev/null || true
|
||||
pip3 install pytest
|
||||
pytest tests/ -v --tb=short 2>/dev/null || true
|
||||
else
|
||||
echo "No Python project detected, skipping pytest"
|
||||
fi
|
||||
|
||||
- name: Run npm test (JS/TS)
|
||||
if: always()
|
||||
run: |
|
||||
if [[ -f package.json ]]; then
|
||||
npm ci
|
||||
npm run test --if-present || true
|
||||
else
|
||||
echo "No Node.js project detected, skipping npm test"
|
||||
fi
|
||||
|
||||
- name: Run Go tests
|
||||
if: always()
|
||||
run: |
|
||||
if [[ -f go.mod ]]; then
|
||||
go test ./...
|
||||
else
|
||||
echo "No Go project detected, skipping go test"
|
||||
fi
|
||||
|
||||
docker-build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Clone repo
|
||||
run: |
|
||||
rm -rf $GITHUB_WORKSPACE/*
|
||||
git clone --depth 1 $GITEA_URL/$GITHUB_REPOSITORY $GITHUB_WORKSPACE
|
||||
git -C $GITHUB_WORKSPACE checkout $GITHUB_SHA 2>/dev/null || true
|
||||
|
||||
- name: Build Docker image
|
||||
if: always()
|
||||
run: |
|
||||
if [[ -f Dockerfile ]]; then
|
||||
docker build -t $GITHUB_REPOSITORY:test .
|
||||
else
|
||||
echo "No Dockerfile found, skipping docker build"
|
||||
fi
|
||||
|
||||
security:
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: gitea-job-image
|
||||
steps:
|
||||
- name: Clone repo
|
||||
run: |
|
||||
rm -rf $GITHUB_WORKSPACE/*
|
||||
git clone --depth 1 $GITEA_URL/$GITHUB_REPOSITORY $GITHUB_WORKSPACE
|
||||
git -C $GITHUB_WORKSPACE checkout $GITHUB_SHA 2>/dev/null || true
|
||||
|
||||
- name: Run bandit (Python SAST)
|
||||
if: always()
|
||||
run: |
|
||||
if [[ -f pyproject.toml ]]; then
|
||||
pip3 install bandit
|
||||
bandit -r . --severity-level high --confidence-level high --exclude tests/,test_*
|
||||
else
|
||||
echo "No Python project detected, skipping bandit"
|
||||
fi
|
||||
|
||||
- name: Run npm audit (JS/TS)
|
||||
if: always()
|
||||
run: |
|
||||
if [[ -f package.json ]]; then
|
||||
npm ci
|
||||
npm audit --audit-level=high 2>/dev/null || echo "npm audit: vulnerabilities found (non-blocking)"
|
||||
else
|
||||
echo "No Node.js project detected, skipping npm audit"
|
||||
fi
|
||||
|
||||
build-result:
|
||||
needs: [lint, test, docker-build, security]
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: gitea-job-image
|
||||
if: always()
|
||||
steps:
|
||||
- name: Summary
|
||||
run: echo "All CI checks completed"
|
||||
@ -42,26 +42,20 @@ Dependencies
|
||||
|
||||
import argparse
|
||||
import datetime
|
||||
import ipaddress
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Dict, List
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import requests
|
||||
|
||||
try:
|
||||
from tqdm import tqdm
|
||||
except ImportError:
|
||||
|
||||
def tqdm(iterable, *args, **kwargs):
|
||||
return iterable
|
||||
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Configuration
|
||||
# ------------------------------------------------------------------
|
||||
@ -71,7 +65,7 @@ API_ASSET_URL = "https://images-api.nasa.gov/asset"
|
||||
IMG_DIR = Path("images")
|
||||
META_DIR = Path("metadata")
|
||||
STATE_FILE = Path("last_run.txt")
|
||||
MAX_RUN_TIME = 4 * 60 * 60 # 4 hours in seconds
|
||||
MAX_RUN_TIME = 2 * 60 * 60 # 2 hours in seconds
|
||||
|
||||
# Ensure the output directories exist
|
||||
IMG_DIR.mkdir(parents=True, exist_ok=True)
|
||||
@ -83,9 +77,7 @@ META_DIR.mkdir(parents=True, exist_ok=True)
|
||||
# ------------------------------------------------------------------
|
||||
def parse_args() -> argparse.Namespace:
|
||||
"""Parse CLI arguments."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Download NASA images for a date range"
|
||||
)
|
||||
parser = argparse.ArgumentParser(description="Download NASA images for a date range")
|
||||
parser.add_argument(
|
||||
"--start",
|
||||
type=str,
|
||||
@ -168,61 +160,21 @@ def fetch_items(start: datetime.datetime, end: datetime.datetime) -> List[Dict]:
|
||||
return filtered
|
||||
|
||||
|
||||
def _is_safe_url(url: str) -> bool:
|
||||
"""Check that a URL points to a safe (non-internal) destination."""
|
||||
parsed = urlparse(url)
|
||||
host = parsed.hostname or ""
|
||||
# Block private and reserved IP ranges
|
||||
try:
|
||||
addr = ipaddress.ip_address(host)
|
||||
if addr.is_private or addr.is_loopback or addr.is_link_local or addr.is_reserved:
|
||||
return False
|
||||
except ValueError:
|
||||
pass
|
||||
# Block internal hostnames
|
||||
if host.endswith((".local", ".internal", ".home.arpa")) or host in ("localhost", "metadata.google.internal"):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def download_file(url: str, dest: Path) -> bool:
|
||||
"""Stream‑download the file at *url* to *dest* if it does not already exist."""
|
||||
if dest.exists():
|
||||
return False
|
||||
if not _is_safe_url(url):
|
||||
raise RuntimeError(f"Unsafe download URL blocked (SSRF protection): {url}")
|
||||
try:
|
||||
with requests.get(url, stream=True, timeout=30, allow_redirects=False) as r:
|
||||
if r.status_code in (301, 302, 303, 307, 308):
|
||||
location = r.headers.get("Location", "")
|
||||
if not _is_safe_url(location):
|
||||
raise RuntimeError(f"Redirect to unsafe URL blocked (SSRF protection): {location}")
|
||||
# Follow single validated redirect
|
||||
with requests.get(location, stream=True, timeout=30, allow_redirects=False) as r2:
|
||||
r2.raise_for_status()
|
||||
with dest.open("wb") as f:
|
||||
for chunk in r2.iter_content(chunk_size=8192):
|
||||
f.write(chunk)
|
||||
else:
|
||||
r.raise_for_status()
|
||||
with dest.open("wb") as f:
|
||||
for chunk in r.iter_content(chunk_size=8192):
|
||||
f.write(chunk)
|
||||
with requests.get(url, stream=True, timeout=30) as r:
|
||||
r.raise_for_status()
|
||||
with dest.open("wb") as f:
|
||||
for chunk in r.iter_content(chunk_size=8192):
|
||||
f.write(chunk)
|
||||
return True
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"Download error for {url}: {e}") from e
|
||||
|
||||
|
||||
def sanitize_filename(name: str) -> str:
|
||||
"""Sanitize a string for use as a filename, preventing path traversal."""
|
||||
# Strip path separators and parent directory references
|
||||
sanitized = re.sub(r'[\/\\:\*\?"<>|]', "_", name)
|
||||
sanitized = re.sub(r"^\.{1,2}($|_)", "item_", sanitized)
|
||||
# Collapse multiple underscores
|
||||
sanitized = re.sub(r"_+", "_", sanitized)
|
||||
return sanitized.strip(".") or "item"
|
||||
|
||||
|
||||
def save_metadata(item: Dict, dest: Path) -> None:
|
||||
"""Persist the full API item as formatted JSON."""
|
||||
with dest.open("w", encoding="utf-8") as f:
|
||||
@ -235,23 +187,8 @@ def save_metadata(item: Dict, dest: Path) -> None:
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
|
||||
# Resolve output base directory and validate it is safe
|
||||
# Resolve output base directory and override global paths
|
||||
base_dir = Path(args.output).resolve()
|
||||
# Prevent writing to system-critical paths
|
||||
_BLOCKED_PREFIXES = {
|
||||
str(Path(p).resolve())
|
||||
for p in ("/etc", "/usr", "/bin", "/sbin", "/boot", "/dev", "/proc", "/sys")
|
||||
}
|
||||
base_str = str(base_dir)
|
||||
if base_str == "/" or any(
|
||||
base_str == blocked or base_str.startswith(blocked + "/")
|
||||
for blocked in _BLOCKED_PREFIXES
|
||||
):
|
||||
print(
|
||||
f"Error: --output cannot point to system directory: {base_dir}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
sys.exit(1)
|
||||
global IMG_DIR, META_DIR, STATE_FILE
|
||||
IMG_DIR = base_dir / "images"
|
||||
META_DIR = base_dir / "metadata"
|
||||
@ -282,13 +219,12 @@ def main() -> None:
|
||||
|
||||
for idx, item in enumerate(tqdm(items, desc="Downloading"), start=1):
|
||||
data = item.get("data", [{}])[0]
|
||||
raw_nasa_id = data.get("nasa_id") or data.get("title", f"item_{idx}")
|
||||
nasa_id = sanitize_filename(str(raw_nasa_id))
|
||||
nasa_id = data.get("nasa_id") or data.get("title", f"item_{idx}")
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Asset lookup: fetch high‑resolution image URL
|
||||
# ------------------------------------------------------------------
|
||||
asset_url = f"{API_ASSET_URL}/{raw_nasa_id}"
|
||||
asset_url = f"{API_ASSET_URL}/{nasa_id}"
|
||||
try:
|
||||
asset_resp = requests.get(asset_url, timeout=20)
|
||||
asset_resp.raise_for_status()
|
||||
@ -302,7 +238,7 @@ def main() -> None:
|
||||
print(f"[{idx}] Failed asset lookup for {nasa_id}: {e}", file=sys.stderr)
|
||||
continue
|
||||
|
||||
filename = sanitize_filename(Path(asset_href).name)
|
||||
filename = Path(asset_href).name
|
||||
image_path = IMG_DIR / filename
|
||||
meta_path = META_DIR / f"index-{nasa_id}.json"
|
||||
|
||||
@ -316,9 +252,7 @@ def main() -> None:
|
||||
try:
|
||||
save_metadata(item, meta_path)
|
||||
except Exception as e:
|
||||
print(
|
||||
f"[{idx}] Failed to write metadata for {nasa_id}: {e}", file=sys.stderr
|
||||
)
|
||||
print(f"[{idx}] Failed to write metadata for {nasa_id}: {e}", file=sys.stderr)
|
||||
|
||||
elapsed = time.time() - start_time
|
||||
if downloaded:
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user