92 lines
3.6 KiB
Bash
Executable File
92 lines
3.6 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
# ------------------------------------------------------------------
|
||
# setup_scraper.sh
|
||
#
|
||
# Installs a virtual‑env, a tiny wrapper, optional dependencies
|
||
# and an hourly cron job – *always using the venv’s python/pip*.
|
||
# ------------------------------------------------------------------
|
||
set -euo pipefail
|
||
|
||
# ---------- Configuration ----------
|
||
SCRAPER_DIR="/home/user/StockDocs/scraper"
|
||
VENV_DIR="${SCRAPER_DIR}/venv"
|
||
SCRIPT_PATH="${SCRAPER_DIR}/cron_scraper.py"
|
||
WRAPPER_PATH="${SCRAPER_DIR}/run_scraper.sh"
|
||
LOG_PATH="${SCRAPER_DIR}/cron.log"
|
||
REQUIREMENTS="${SCRAPER_DIR}/requirements.txt"
|
||
|
||
# Cron line – this will call the wrapper, which in turn calls the
|
||
# venv’s Python interpreter.
|
||
CRON_LINE="0 * * * * ${WRAPPER_PATH} >> ${LOG_PATH} 2>&1"
|
||
|
||
# ---------- Helper functions ----------
|
||
log() { printf '[setup_scraper] %s\n' "$*"; }
|
||
error_exit() { printf '[setup_scraper] ERROR: %s\n' "$*" >&2; exit 1; }
|
||
|
||
# ---------- 1️⃣ Create the venv if it’s missing ----------
|
||
if [[ ! -d "$VENV_DIR" ]]; then
|
||
log "Creating virtual‑environment at ${VENV_DIR}"
|
||
python3 -m venv "$VENV_DIR" || error_exit "Failed to create venv"
|
||
else
|
||
log "Virtual‑environment already exists at ${VENV_DIR}"
|
||
fi
|
||
|
||
# ---------- 2️⃣ Activate the venv *for the rest of this script* ----------
|
||
# This changes PATH for the current shell only – it does NOT touch the
|
||
# system Python. The next line will confirm that we’re really in the venv.
|
||
# shellcheck source=/dev/null
|
||
source "${VENV_DIR}/bin/activate"
|
||
|
||
# Quick sanity‑check: make sure we’re using the venv’s python and pip.
|
||
log "Current python: $(python -c 'import sys;print(sys.executable)')"
|
||
log "pip version: $(pip --version)"
|
||
|
||
# ---------- 3️⃣ Install dependencies (optional) ----------
|
||
if [[ -f "$REQUIREMENTS" ]]; then
|
||
log "Installing Python packages from ${REQUIREMENTS}"
|
||
# Use the *venv’s* pip explicitly – this guarantees no system installs.
|
||
"${VENV_DIR}/bin/pip" install --upgrade pip
|
||
"${VENV_DIR}/bin/pip" install -r "$REQUIREMENTS" || error_exit "pip install failed"
|
||
else
|
||
log "No requirements.txt found – skipping dependency install."
|
||
fi
|
||
|
||
# ---------- 4️⃣ Create the wrapper script ----------
|
||
# The wrapper *does NOT* source the venv any more – it calls the venv’s
|
||
# Python binary directly. This eliminates the subtle “activate” pitfall.
|
||
if [[ ! -f "$WRAPPER_PATH" ]]; then
|
||
log "Creating wrapper script at ${WRAPPER_PATH}"
|
||
cat > "$WRAPPER_PATH" <<'EOF'
|
||
#!/usr/bin/env bash
|
||
# ------------------------------------------------------------------
|
||
# run_scraper.sh
|
||
#
|
||
# Activates the venv *implicitly* by calling the venv’s Python binary.
|
||
# ------------------------------------------------------------------
|
||
# Absolute path to the venv – change only if you move the venv.
|
||
VENV_DIR="/home/user/StockDocs/scraper/venv"
|
||
|
||
# Absolute path to the script you want to run.
|
||
SCRIPT="/home/user/StockDocs/scraper/cron_scraper.py"
|
||
|
||
# Invoke the venv’s Python directly.
|
||
"${VENV_DIR}/bin/python" "${SCRIPT}" # output is redirected by cron
|
||
EOF
|
||
chmod +x "$WRAPPER_PATH"
|
||
else
|
||
log "Wrapper script already exists at ${WRAPPER_PATH}"
|
||
fi
|
||
|
||
# ---------- 5️⃣ Install the cron job ----------
|
||
CURRENT_CRON=$(crontab -l 2>/dev/null || true)
|
||
|
||
if echo "$CURRENT_CRON" | grep -Fqx "$CRON_LINE"; then
|
||
log "Crontab entry already present – nothing to do."
|
||
else
|
||
log "Adding new cron entry."
|
||
# Append the new line and reinstall the crontab.
|
||
(printf '%s\n' "$CURRENT_CRON" ; printf '%s\n' "$CRON_LINE") | crontab -
|
||
fi
|
||
|
||
log "✅ Setup complete! ${SCRIPT_PATH} will run every hour via the venv."
|