Adding a cron job script and a setup script to help with setting up cronjob

articleServer/pyvenv.cfg

@@ -1,5 +1,5 @@
-home = /opt/homebrew/opt/python@3.13/bin
+home = /usr/bin
 include-system-site-packages = false
-version = 3.13.7
+version = 3.12.3
-executable = /opt/homebrew/Cellar/python@3.13/3.13.7/Frameworks/Python.framework/Versions/3.13/bin/python3.13
+executable = /usr/bin/python3.12
-command = /opt/homebrew/opt/python@3.13/bin/python3.13 -m venv /Users/user/Projects/StockDocs/articleServer
+command = /usr/bin/python3 -m venv /home/user/StockDocs/articleServer

scraper/.gitignore

@@ -0,0 +1,3 @@
+errors.txt
+results.txt
+venv

scraper/pyvenv.cfg

@@ -1,5 +1,5 @@
-home = /opt/homebrew/opt/python@3.13/bin
+home = /usr/bin
 include-system-site-packages = false
-version = 3.13.7
+version = 3.12.3
-executable = /opt/homebrew/Cellar/python@3.13/3.13.7/Frameworks/Python.framework/Versions/3.13/bin/python3.13
+executable = /usr/bin/python3.12
-command = /opt/homebrew/opt/python@3.13/bin/python3.13 -m venv /Users/user/Projects/StockDocs/scraper
+command = /usr/bin/python3 -m venv /home/user/StockDocs/scraper

scraper/run_scraper.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+# ------------------------------------------------------------------
+# run_scraper.sh
+#
+# Activates the venv *implicitly* by calling the venv’s Python binary.
+# ------------------------------------------------------------------
+# Absolute path to the venv – change only if you move the venv.
+VENV_DIR="/home/user/StockDocs/scraper/venv"
+
+# Absolute path to the script you want to run.
+SCRIPT="/home/user/StockDocs/scraper/cron_scraper.py"
+
+# Invoke the venv’s Python directly.
+"${VENV_DIR}/bin/python" "${SCRIPT}"  # output is redirected by cron

scraper/setup_scraper.sh

@@ -0,0 +1,91 @@
+#!/usr/bin/env bash
+# ------------------------------------------------------------------
+# setup_scraper.sh
+#
+# Installs a virtual‑env, a tiny wrapper, optional dependencies
+# and an hourly cron job – *always using the venv’s python/pip*.
+# ------------------------------------------------------------------
+set -euo pipefail
+
+# ---------- Configuration ----------
+SCRAPER_DIR="/home/user/StockDocs/scraper"
+VENV_DIR="${SCRAPER_DIR}/venv"
+SCRIPT_PATH="${SCRAPER_DIR}/cron_scraper.py"
+WRAPPER_PATH="${SCRAPER_DIR}/run_scraper.sh"
+LOG_PATH="${SCRAPER_DIR}/cron.log"
+REQUIREMENTS="${SCRAPER_DIR}/requirements.txt"
+
+# Cron line – this will call the wrapper, which in turn calls the
+# venv’s Python interpreter.
+CRON_LINE="0 * * * * ${WRAPPER_PATH} >> ${LOG_PATH} 2>&1"
+
+# ---------- Helper functions ----------
+log() { printf '[setup_scraper] %s\n' "$*"; }
+error_exit() { printf '[setup_scraper] ERROR: %s\n' "$*" >&2; exit 1; }
+
+# ---------- 1️⃣  Create the venv if it’s missing ----------
+if [[ ! -d "$VENV_DIR" ]]; then
+    log "Creating virtual‑environment at ${VENV_DIR}"
+    python3 -m venv "$VENV_DIR" || error_exit "Failed to create venv"
+else
+    log "Virtual‑environment already exists at ${VENV_DIR}"
+fi
+
+# ---------- 2️⃣  Activate the venv *for the rest of this script* ----------
+# This changes PATH for the current shell only – it does NOT touch the
+# system Python.  The next line will confirm that we’re really in the venv.
+# shellcheck source=/dev/null
+source "${VENV_DIR}/bin/activate"
+
+# Quick sanity‑check: make sure we’re using the venv’s python and pip.
+log "Current python: $(python -c 'import sys;print(sys.executable)')"
+log "pip version: $(pip --version)"
+
+# ---------- 3️⃣  Install dependencies (optional) ----------
+if [[ -f "$REQUIREMENTS" ]]; then
+    log "Installing Python packages from ${REQUIREMENTS}"
+    # Use the *venv’s* pip explicitly – this guarantees no system installs.
+    "${VENV_DIR}/bin/pip" install --upgrade pip
+    "${VENV_DIR}/bin/pip" install -r "$REQUIREMENTS" || error_exit "pip install failed"
+else
+    log "No requirements.txt found – skipping dependency install."
+fi
+
+# ---------- 4️⃣  Create the wrapper script ----------
+# The wrapper *does NOT* source the venv any more – it calls the venv’s
+# Python binary directly.  This eliminates the subtle “activate” pitfall.
+if [[ ! -f "$WRAPPER_PATH" ]]; then
+    log "Creating wrapper script at ${WRAPPER_PATH}"
+    cat > "$WRAPPER_PATH" <<'EOF'
+#!/usr/bin/env bash
+# ------------------------------------------------------------------
+# run_scraper.sh
+#
+# Activates the venv *implicitly* by calling the venv’s Python binary.
+# ------------------------------------------------------------------
+# Absolute path to the venv – change only if you move the venv.
+VENV_DIR="/home/user/StockDocs/scraper/venv"
+
+# Absolute path to the script you want to run.
+SCRIPT="/home/user/StockDocs/scraper/cron_scraper.py"
+
+# Invoke the venv’s Python directly.
+"${VENV_DIR}/bin/python" "${SCRIPT}"  # output is redirected by cron
+EOF
+    chmod +x "$WRAPPER_PATH"
+else
+    log "Wrapper script already exists at ${WRAPPER_PATH}"
+fi
+
+# ---------- 5️⃣  Install the cron job ----------
+CURRENT_CRON=$(crontab -l 2>/dev/null || true)
+
+if echo "$CURRENT_CRON" | grep -Fqx "$CRON_LINE"; then
+    log "Crontab entry already present – nothing to do."
+else
+    log "Adding new cron entry."
+    # Append the new line and reinstall the crontab.
+    (printf '%s\n' "$CURRENT_CRON" ; printf '%s\n' "$CRON_LINE") | crontab -
+fi
+
+log "✅  Setup complete!  ${SCRIPT_PATH} will run every hour via the venv."