diff --git a/articleServer/pyvenv.cfg b/articleServer/pyvenv.cfg index 52a9ca7..765fe55 100644 --- a/articleServer/pyvenv.cfg +++ b/articleServer/pyvenv.cfg @@ -1,5 +1,5 @@ -home = /opt/homebrew/opt/python@3.13/bin +home = /usr/bin include-system-site-packages = false -version = 3.13.7 -executable = /opt/homebrew/Cellar/python@3.13/3.13.7/Frameworks/Python.framework/Versions/3.13/bin/python3.13 -command = /opt/homebrew/opt/python@3.13/bin/python3.13 -m venv /Users/user/Projects/StockDocs/articleServer +version = 3.12.3 +executable = /usr/bin/python3.12 +command = /usr/bin/python3 -m venv /home/user/StockDocs/articleServer diff --git a/scraper/.gitignore b/scraper/.gitignore new file mode 100644 index 0000000..029b5b2 --- /dev/null +++ b/scraper/.gitignore @@ -0,0 +1,3 @@ +errors.txt +results.txt +venv diff --git a/scraper/pyvenv.cfg b/scraper/pyvenv.cfg index 50b65da..f163fac 100644 --- a/scraper/pyvenv.cfg +++ b/scraper/pyvenv.cfg @@ -1,5 +1,5 @@ -home = /opt/homebrew/opt/python@3.13/bin +home = /usr/bin include-system-site-packages = false -version = 3.13.7 -executable = /opt/homebrew/Cellar/python@3.13/3.13.7/Frameworks/Python.framework/Versions/3.13/bin/python3.13 -command = /opt/homebrew/opt/python@3.13/bin/python3.13 -m venv /Users/user/Projects/StockDocs/scraper +version = 3.12.3 +executable = /usr/bin/python3.12 +command = /usr/bin/python3 -m venv /home/user/StockDocs/scraper diff --git a/scraper/run_scraper.sh b/scraper/run_scraper.sh new file mode 100755 index 0000000..7871c1a --- /dev/null +++ b/scraper/run_scraper.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# ------------------------------------------------------------------ +# run_scraper.sh +# +# Activates the venv *implicitly* by calling the venv’s Python binary. +# ------------------------------------------------------------------ +# Absolute path to the venv – change only if you move the venv. +VENV_DIR="/home/user/StockDocs/scraper/venv" + +# Absolute path to the script you want to run. +SCRIPT="/home/user/StockDocs/scraper/cron_scraper.py" + +# Invoke the venv’s Python directly. +"${VENV_DIR}/bin/python" "${SCRIPT}" # output is redirected by cron diff --git a/scraper/setup_scraper.sh b/scraper/setup_scraper.sh new file mode 100755 index 0000000..eb847b6 --- /dev/null +++ b/scraper/setup_scraper.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +# ------------------------------------------------------------------ +# setup_scraper.sh +# +# Installs a virtual‑env, a tiny wrapper, optional dependencies +# and an hourly cron job – *always using the venv’s python/pip*. +# ------------------------------------------------------------------ +set -euo pipefail + +# ---------- Configuration ---------- +SCRAPER_DIR="/home/user/StockDocs/scraper" +VENV_DIR="${SCRAPER_DIR}/venv" +SCRIPT_PATH="${SCRAPER_DIR}/cron_scraper.py" +WRAPPER_PATH="${SCRAPER_DIR}/run_scraper.sh" +LOG_PATH="${SCRAPER_DIR}/cron.log" +REQUIREMENTS="${SCRAPER_DIR}/requirements.txt" + +# Cron line – this will call the wrapper, which in turn calls the +# venv’s Python interpreter. +CRON_LINE="0 * * * * ${WRAPPER_PATH} >> ${LOG_PATH} 2>&1" + +# ---------- Helper functions ---------- +log() { printf '[setup_scraper] %s\n' "$*"; } +error_exit() { printf '[setup_scraper] ERROR: %s\n' "$*" >&2; exit 1; } + +# ---------- 1️⃣ Create the venv if it’s missing ---------- +if [[ ! -d "$VENV_DIR" ]]; then + log "Creating virtual‑environment at ${VENV_DIR}" + python3 -m venv "$VENV_DIR" || error_exit "Failed to create venv" +else + log "Virtual‑environment already exists at ${VENV_DIR}" +fi + +# ---------- 2️⃣ Activate the venv *for the rest of this script* ---------- +# This changes PATH for the current shell only – it does NOT touch the +# system Python. The next line will confirm that we’re really in the venv. +# shellcheck source=/dev/null +source "${VENV_DIR}/bin/activate" + +# Quick sanity‑check: make sure we’re using the venv’s python and pip. +log "Current python: $(python -c 'import sys;print(sys.executable)')" +log "pip version: $(pip --version)" + +# ---------- 3️⃣ Install dependencies (optional) ---------- +if [[ -f "$REQUIREMENTS" ]]; then + log "Installing Python packages from ${REQUIREMENTS}" + # Use the *venv’s* pip explicitly – this guarantees no system installs. + "${VENV_DIR}/bin/pip" install --upgrade pip + "${VENV_DIR}/bin/pip" install -r "$REQUIREMENTS" || error_exit "pip install failed" +else + log "No requirements.txt found – skipping dependency install." +fi + +# ---------- 4️⃣ Create the wrapper script ---------- +# The wrapper *does NOT* source the venv any more – it calls the venv’s +# Python binary directly. This eliminates the subtle “activate” pitfall. +if [[ ! -f "$WRAPPER_PATH" ]]; then + log "Creating wrapper script at ${WRAPPER_PATH}" + cat > "$WRAPPER_PATH" <<'EOF' +#!/usr/bin/env bash +# ------------------------------------------------------------------ +# run_scraper.sh +# +# Activates the venv *implicitly* by calling the venv’s Python binary. +# ------------------------------------------------------------------ +# Absolute path to the venv – change only if you move the venv. +VENV_DIR="/home/user/StockDocs/scraper/venv" + +# Absolute path to the script you want to run. +SCRIPT="/home/user/StockDocs/scraper/cron_scraper.py" + +# Invoke the venv’s Python directly. +"${VENV_DIR}/bin/python" "${SCRIPT}" # output is redirected by cron +EOF + chmod +x "$WRAPPER_PATH" +else + log "Wrapper script already exists at ${WRAPPER_PATH}" +fi + +# ---------- 5️⃣ Install the cron job ---------- +CURRENT_CRON=$(crontab -l 2>/dev/null || true) + +if echo "$CURRENT_CRON" | grep -Fqx "$CRON_LINE"; then + log "Crontab entry already present – nothing to do." +else + log "Adding new cron entry." + # Append the new line and reinstall the crontab. + (printf '%s\n' "$CURRENT_CRON" ; printf '%s\n' "$CRON_LINE") | crontab - +fi + +log "✅ Setup complete! ${SCRIPT_PATH} will run every hour via the venv."