#!/usr/bin/env bash # ------------------------------------------------------------------ # setup_scraper.sh # # Installs a virtual‑env, a tiny wrapper, optional dependencies # and an hourly cron job – *always using the venv’s python/pip*. # ------------------------------------------------------------------ set -euo pipefail # ---------- Configuration ---------- SCRAPER_DIR="/home/user/StockDocs/scraper" VENV_DIR="${SCRAPER_DIR}/venv" SCRIPT_PATH="${SCRAPER_DIR}/cron_scraper.py" WRAPPER_PATH="${SCRAPER_DIR}/run_scraper.sh" LOG_PATH="${SCRAPER_DIR}/cron.log" REQUIREMENTS="${SCRAPER_DIR}/requirements.txt" # Cron line – this will call the wrapper, which in turn calls the # venv’s Python interpreter. CRON_LINE="0 * * * * ${WRAPPER_PATH} >> ${LOG_PATH} 2>&1" # ---------- Helper functions ---------- log() { printf '[setup_scraper] %s\n' "$*"; } error_exit() { printf '[setup_scraper] ERROR: %s\n' "$*" >&2; exit 1; } # ---------- 1️⃣ Create the venv if it’s missing ---------- if [[ ! -d "$VENV_DIR" ]]; then log "Creating virtual‑environment at ${VENV_DIR}" python3 -m venv "$VENV_DIR" || error_exit "Failed to create venv" else log "Virtual‑environment already exists at ${VENV_DIR}" fi # ---------- 2️⃣ Activate the venv *for the rest of this script* ---------- # This changes PATH for the current shell only – it does NOT touch the # system Python. The next line will confirm that we’re really in the venv. # shellcheck source=/dev/null source "${VENV_DIR}/bin/activate" # Quick sanity‑check: make sure we’re using the venv’s python and pip. log "Current python: $(python -c 'import sys;print(sys.executable)')" log "pip version: $(pip --version)" # ---------- 3️⃣ Install dependencies (optional) ---------- if [[ -f "$REQUIREMENTS" ]]; then log "Installing Python packages from ${REQUIREMENTS}" # Use the *venv’s* pip explicitly – this guarantees no system installs. "${VENV_DIR}/bin/pip" install --upgrade pip "${VENV_DIR}/bin/pip" install -r "$REQUIREMENTS" || error_exit "pip install failed" else log "No requirements.txt found – skipping dependency install." fi # ---------- 4️⃣ Create the wrapper script ---------- # The wrapper *does NOT* source the venv any more – it calls the venv’s # Python binary directly. This eliminates the subtle “activate” pitfall. if [[ ! -f "$WRAPPER_PATH" ]]; then log "Creating wrapper script at ${WRAPPER_PATH}" cat > "$WRAPPER_PATH" <<'EOF' #!/usr/bin/env bash # ------------------------------------------------------------------ # run_scraper.sh # # Activates the venv *implicitly* by calling the venv’s Python binary. # ------------------------------------------------------------------ # Absolute path to the venv – change only if you move the venv. VENV_DIR="/home/user/StockDocs/scraper/venv" # Absolute path to the script you want to run. SCRIPT="/home/user/StockDocs/scraper/cron_scraper.py" # Invoke the venv’s Python directly. "${VENV_DIR}/bin/python" "${SCRIPT}" # output is redirected by cron EOF chmod +x "$WRAPPER_PATH" else log "Wrapper script already exists at ${WRAPPER_PATH}" fi # ---------- 5️⃣ Install the cron job ---------- CURRENT_CRON=$(crontab -l 2>/dev/null || true) if echo "$CURRENT_CRON" | grep -Fqx "$CRON_LINE"; then log "Crontab entry already present – nothing to do." else log "Adding new cron entry." # Append the new line and reinstall the crontab. (printf '%s\n' "$CURRENT_CRON" ; printf '%s\n' "$CRON_LINE") | crontab - fi log "✅ Setup complete! ${SCRIPT_PATH} will run every hour via the venv."