commit 41d7a6848edcb210f1f3538e903a572be71a314e Author: Jarian Cottingham Date: Wed Sep 10 23:27:33 2025 -0500 Downloader up and working diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..691190f --- /dev/null +++ b/.gitignore @@ -0,0 +1,12 @@ +# Created by venv; see https://docs.python.org/3/library/venv.html + +.ropeproject +.DS_Store +pyvenv.cfg +/bin +/include +/lib + +# Generated folders +/images +/metadata diff --git a/downloader.py b/downloader.py new file mode 100644 index 0000000..250e5fe --- /dev/null +++ b/downloader.py @@ -0,0 +1,255 @@ +#!/usr/bin/env python3 +""" +NASA image downloader. + +The script queries NASA’s public “Image and Video Library” API for +all images between a start and an end date, downloads each image +to the *images* directory and writes a JSON file with the API +metadata next to it. The JSON file is named +`index‑.json`, where `` is the unique ID returned +by the API. + +Now the images are fetched via the asset endpoint so the highest‑resolution +image is downloaded. See the example at the top of this repository for +the asset‑based approach. + +Designed to be run as a nightly daemon or cron job – it keeps a +local record of the last successful run date, so a subsequent +execution will automatically pick up new images and re‑write the +metadata for any images that may have been updated. + +Features +-------- +* idempotent – already‑downloaded images are skipped +* optional time‑budget (max 2 h per run) +* simple state file (`last_run.txt`) stores the end date of the last run +* separate `images/` and `metadata/` directories +* graceful error handling – failures are printed but do not stop the whole run + +Usage +----- +```text +python downloader.py [--start YYYY-MM-DD] [--end YYYY-MM-DD] +``` +If `--start` is omitted, the script reads the date from +`last_run.txt`; if that file does not exist it defaults to three +days ago. `--end` defaults to today. + +Dependencies +------------- +* requests +""" + +import argparse +import datetime +import json +import os +import sys +import time +from pathlib import Path +from typing import Dict, List + +import requests +try: + from tqdm import tqdm +except ImportError: + def tqdm(iterable, *args, **kwargs): + return iterable + +# ------------------------------------------------------------------ +# Configuration +# ------------------------------------------------------------------ +API_SEARCH_URL = "https://images-api.nasa.gov/search" +API_ASSET_URL = "https://images-api.nasa.gov/asset" + +IMG_DIR = Path("images") +META_DIR = Path("metadata") +STATE_FILE = Path("last_run.txt") +MAX_RUN_TIME = 2 * 60 * 60 # 2 hours in seconds + +# Ensure the output directories exist +IMG_DIR.mkdir(exist_ok=True) +META_DIR.mkdir(exist_ok=True) + + +# ------------------------------------------------------------------ +# Helpers +# ------------------------------------------------------------------ +def parse_args() -> argparse.Namespace: + """Parse CLI arguments.""" + parser = argparse.ArgumentParser(description="Download NASA images for a date range") + parser.add_argument( + "--start", + type=str, + help="Start date (YYYY-MM-DD). If omitted, uses date stored in last_run.txt.", + ) + parser.add_argument( + "--end", + type=str, + help="End date (YYYY-MM-DD). Defaults to today.", + ) + return parser.parse_args() + + +def load_state() -> datetime.datetime: + """Return the date stored in STATE_FILE or default to 3 days ago.""" + if STATE_FILE.exists(): + try: + date_str = STATE_FILE.read_text().strip() + return datetime.datetime.strptime(date_str, "%Y-%m-%d") + except Exception: + pass + # default to 3 days ago UTC + return datetime.datetime.utcnow() - datetime.timedelta(days=3) + + +def save_state(end_date: datetime.datetime) -> None: + """Persist the last successful run date.""" + STATE_FILE.write_text(end_date.strftime("%Y-%m-%d")) + + +def fetch_items(start: datetime.datetime, end: datetime.datetime) -> List[Dict]: + """ + Pull all image items from the NASA API in the given date range. + + The API paginates results; we loop until a page returns no items. + Returns the raw ``items`` list from the API. + """ + items: List[Dict] = [] + page = 1 + while True: + params = { + "media_type": "image", + "page": page, + "year_start": start.year, + "year_end": end.year, + } + try: + r = requests.get(API_SEARCH_URL, params=params, timeout=20) + r.raise_for_status() + data = r.json() + except Exception as e: + print(f"API request failed: {e}", file=sys.stderr) + break + batch = data.get("collection", {}).get("items", []) + if not batch or len(batch) == 0: + break + items.extend(batch) + page += 1 + + # Filter by the exact date range + filtered = [] + for itm in items: + d = itm.get("data", [{}])[0] + iso = d.get("date_created") or d.get("date") + if not iso: + continue + try: + # Accept ISO format with or without timezone suffix + dt = datetime.datetime.fromisoformat(iso.rstrip("Z")) + except Exception: + continue + if start <= dt <= end: + filtered.append(itm) + return filtered + + +def download_file(url: str, dest: Path) -> bool: + """Stream‑download the file at *url* to *dest* if it does not already exist.""" + if dest.exists(): + return False + try: + with requests.get(url, stream=True, timeout=30) as r: + r.raise_for_status() + with dest.open("wb") as f: + for chunk in r.iter_content(chunk_size=8192): + f.write(chunk) + return True + except Exception as e: + raise RuntimeError(f"Download error for {url}: {e}") from e + + +def save_metadata(item: Dict, dest: Path) -> None: + """Persist the full API item as formatted JSON.""" + with dest.open("w", encoding="utf-8") as f: + json.dump(item, f, ensure_ascii=False, indent=2) + + +# ------------------------------------------------------------------ +# Main logic +# ------------------------------------------------------------------ +def main() -> None: + args = parse_args() + + start_date = ( + datetime.datetime.strptime(args.start, "%Y-%m-%d") + if args.start + else load_state() + ) + end_date = ( + datetime.datetime.strptime(args.end, "%Y-%m-%d") + if args.end + else datetime.datetime.utcnow() + ) + + if start_date > end_date: + print("Error: start date must be <= end date", file=sys.stderr) + sys.exit(1) + + print(f"Fetching NASA image metadata from {start_date.date()} to {end_date.date()}") + items = fetch_items(start_date, end_date) + print(f"Found {len(items)} items") + + start_time = time.time() + + for idx, item in enumerate(tqdm(items, desc="Downloading"), start=1): + data = item.get("data", [{}])[0] + nasa_id = data.get("nasa_id") or data.get("title", f"item_{idx}") + + # ------------------------------------------------------------------ + # Asset lookup: fetch high‑resolution image URL + # ------------------------------------------------------------------ + asset_url = f"{API_ASSET_URL}/{nasa_id}" + try: + asset_resp = requests.get(asset_url, timeout=20) + asset_resp.raise_for_status() + assets = asset_resp.json() + # Choose the first asset – usually the highest quality + asset_href = assets.get("collection", {}).get("items", [{}])[0].get("href") + if not asset_href: + print(f"[{idx}] No asset link for item {nasa_id}", file=sys.stderr) + continue + except Exception as e: + print(f"[{idx}] Failed asset lookup for {nasa_id}: {e}", file=sys.stderr) + continue + + filename = Path(asset_href).name + image_path = IMG_DIR / filename + meta_path = META_DIR / f"index-{nasa_id}.json" + + try: + downloaded = download_file(asset_href, image_path) + except Exception as e: + print(f"[{idx}] Failed download {asset_href}: {e}", file=sys.stderr) + continue + + # Persist metadata + try: + save_metadata(item, meta_path) + except Exception as e: + print(f"[{idx}] Failed to write metadata for {nasa_id}: {e}", file=sys.stderr) + + elapsed = time.time() - start_time + if downloaded: + time.sleep(3) + if elapsed > MAX_RUN_TIME: + print("Maximum runtime exceeded, stopping early") + break + + # Persist the last successful end date + save_state(end_date) + print("Download complete") + + +if __name__ == "__main__": + main() diff --git a/last_run.txt b/last_run.txt new file mode 100644 index 0000000..4ea7e10 --- /dev/null +++ b/last_run.txt @@ -0,0 +1 @@ +2025-09-11 \ No newline at end of file diff --git a/wrapper.sh b/wrapper.sh new file mode 100644 index 0000000..b0e1cb1 --- /dev/null +++ b/wrapper.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Change to the project root +cd "$(dirname "$0")" + +# Activate virtualenv if you want – optional +# source .venv/bin/activate + +python3 downloader.py