Jarian Cottingham
e93b5cee3e
- #3: Path traversal fix in /archive and /archive-file routes via resolve() check - #4: SSRF mitigation - env-based SERVER_URL, no hardcoded internal IPs - #5: Stored XSS fix - remove |safe filter from article.html template - #6: Missing import os in scheduler.py (crash on import) - #7: Flask auth (password via NEWSARCHIVER_PASSWORD) + CSRF tokens - #8: Same as #5 (template XSS via |safe) - #9: Motley Fool API key removed - use env var interpolation - #10: Hardcoded paths in setup_cron.sh, stop_services.sh - use BASH_SOURCE - #11: Hardcoded user paths in singlefile_archive.py - use Path.home() - #16: HTTP RSS feeds updated to HTTPS (Barchart, Guardian, BBC, MarketWatch) - #24: SSRF - replace hardcoded 192.168.8.150:5000 with NEWSARCHIVER_SERVER_URL - #25: Command execution details sanitized in error messages - #26: Security headers (X-Content-Type-Options, X-Frame-Options, X-XSS-Protection, Referrer-Policy, CSP) - #27: Auth guard on all routes except RSS/Atom feeds - archive_engine.py: Add missing import os
84 lines
2.6 KiB
Bash
84 lines
2.6 KiB
Bash
#!/bin/bash
|
|
# Setup script for NewsArchiver
|
|
# This script sets up the cron job for automated news archiving
|
|
|
|
set -e
|
|
|
|
SCRIPT_DIR="${NEWSARCHIVER_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)}"
|
|
LOG_FILE="${NEWSARCHIVER_LOG:-/tmp/newsarchiver_cron.log}"
|
|
CRON_JOB="*/30 * * * * /usr/bin/env python3 ${SCRIPT_DIR}/run_archiver.py --interval 30 > ${LOG_FILE} 2>&1"
|
|
|
|
echo "=== NewsArchiver Setup Script ==="
|
|
echo ""
|
|
|
|
# Check Python is available
|
|
if ! command -v python3 &> /dev/null; then
|
|
echo "ERROR: python3 not found"
|
|
exit 1
|
|
fi
|
|
|
|
# Check if running as jarian
|
|
CURRENT_USER=$(whoami)
|
|
if [ "$CURRENT_USER" != "jarian" ]; then
|
|
echo "WARNING: This script is configured for user 'jarian', but you are '$CURRENT_USER'"
|
|
echo "You may need to update the script paths"
|
|
fi
|
|
|
|
# Check if NewsArchiver directory exists
|
|
if [ ! -d "$SCRIPT_DIR" ]; then
|
|
echo "ERROR: NewsArchiver directory not found at $SCRIPT_DIR"
|
|
exit 1
|
|
fi
|
|
|
|
# Check if requirements are installed
|
|
echo "Checking dependencies..."
|
|
cd "$SCRIPT_DIR"
|
|
python3 -c "import flask; import requests; import trafilatura; import feedparser; import apscheduler" 2>/dev/null || {
|
|
echo "Installing dependencies..."
|
|
pip install -r requirements.txt
|
|
}
|
|
|
|
# Check if web server is running
|
|
if ! pgrep -f "run_archiver.py --serve" > /dev/null; then
|
|
echo "Starting web server..."
|
|
nohup python3 "$SCRIPT_DIR/run_archiver.py" --serve --host 0.0.0.0 --port 5000 > /tmp/webserver.log 2>&1 &
|
|
sleep 3
|
|
if curl -s -o /dev/null -w "%{http_code}" http://localhost:5000/ | grep -q "200"; then
|
|
echo "Web server started successfully on port 5000"
|
|
else
|
|
echo "WARNING: Web server may not be responding"
|
|
fi
|
|
else
|
|
echo "Web server is already running"
|
|
fi
|
|
|
|
# Remove old scheduler lock file if exists
|
|
if [ -f "$SCRIPT_DIR/archival_data/.scheduler.lock" ]; then
|
|
rm -f "$SCRIPT_DIR/archival_data/.scheduler.lock"
|
|
echo "Removed stale scheduler lock file"
|
|
fi
|
|
|
|
# Kill any existing scheduler processes
|
|
pkill -f "run_archiver.py --interval" 2>/dev/null || true
|
|
echo "Cleared any existing scheduler processes"
|
|
|
|
# Setup cron job
|
|
echo "Setting up cron job..."
|
|
if crontab -l 2>/dev/null | grep -q "NewsArchiver"; then
|
|
echo "Removing existing NewsArchiver cron job..."
|
|
crontab -l | grep -v "NewsArchiver" | crontab -
|
|
fi
|
|
|
|
echo "$CRON_JOB" | crontab -
|
|
echo "Cron job added successfully"
|
|
|
|
echo ""
|
|
echo "=== Current Cron Jobs ==="
|
|
crontab -l | grep NewsArchiver
|
|
|
|
echo ""
|
|
echo "=== Setup Complete ==="
|
|
echo "- Archiver will run every 30 minutes via cron"
|
|
echo "- Logs written to: $LOG_FILE"
|
|
echo "- Web interface at: http://localhost:5000"
|
|
echo "" |