249 lines
8.6 KiB
YAML
249 lines
8.6 KiB
YAML
name: CI
|
|
|
|
on:
|
|
push:
|
|
branches: [main, master]
|
|
pull_request:
|
|
branches: [main, master]
|
|
|
|
env:
|
|
GITEA_URL: https://git.example.com
|
|
|
|
jobs:
|
|
lint:
|
|
runs-on: ubuntu-latest
|
|
container:
|
|
image: gitea-job-image
|
|
steps:
|
|
- name: Clone repo
|
|
run: |
|
|
rm -rf $GITHUB_WORKSPACE/*
|
|
git clone --depth 1 $GITEA_URL/$GITHUB_REPOSITORY $GITHUB_WORKSPACE
|
|
git -C $GITHUB_WORKSPACE checkout $GITHUB_SHA 2>/dev/null || true
|
|
|
|
- name: Run ruff (Python lint)
|
|
if: always()
|
|
run: |
|
|
if [[ -f pyproject.toml ]]; then
|
|
pip3 install ruff
|
|
ruff check .
|
|
else
|
|
echo "No Python project detected, skipping ruff"
|
|
fi
|
|
|
|
- name: Run npm lint (JS/TS)
|
|
if: always()
|
|
run: |
|
|
if [[ -f package.json ]]; then
|
|
npm ci
|
|
npm run lint --if-present || true
|
|
else
|
|
echo "No Node.js project detected, skipping npm lint"
|
|
fi
|
|
|
|
test:
|
|
runs-on: ubuntu-latest
|
|
container:
|
|
image: gitea-job-image
|
|
steps:
|
|
- name: Clone repo
|
|
run: |
|
|
rm -rf $GITHUB_WORKSPACE/*
|
|
git clone --depth 1 $GITEA_URL/$GITHUB_REPOSITORY $GITHUB_WORKSPACE
|
|
git -C $GITHUB_WORKSPACE checkout $GITHUB_SHA 2>/dev/null || true
|
|
|
|
- name: Run pytest (Python)
|
|
if: always()
|
|
run: |
|
|
if [[ -f pyproject.toml ]]; then
|
|
python3 -m pip install --upgrade pip
|
|
pip3 install -e ".[dev]" 2>/dev/null || pip3 install -e . 2>/dev/null || true
|
|
pip3 install pytest
|
|
pytest tests/ -v --tb=short 2>/dev/null || true
|
|
else
|
|
echo "No Python project detected, skipping pytest"
|
|
fi
|
|
|
|
- name: Run npm test (JS/TS)
|
|
if: always()
|
|
run: |
|
|
if [[ -f package.json ]]; then
|
|
npm ci
|
|
npm run test --if-present || true
|
|
else
|
|
echo "No Node.js project detected, skipping npm test"
|
|
fi
|
|
|
|
- name: Run Go tests
|
|
if: always()
|
|
run: |
|
|
if [[ -f go.mod ]]; then
|
|
go test ./...
|
|
else
|
|
echo "No Go project detected, skipping go test"
|
|
fi
|
|
|
|
e2e:
|
|
runs-on: ubuntu-latest
|
|
needs: [docker-build]
|
|
if: github.event_name == 'pull_request'
|
|
steps:
|
|
- name: Clone repo
|
|
run: |
|
|
rm -rf $GITHUB_WORKSPACE/*
|
|
git clone --depth 1 $GITEA_URL/$GITHUB_REPOSITORY $GITHUB_WORKSPACE
|
|
git -C $GITHUB_WORKSPACE checkout $GITHUB_SHA 2>/dev/null || true
|
|
|
|
- name: Create test network
|
|
run: docker network create newsarchiver-network 2>/dev/null || true
|
|
|
|
- name: Start app container
|
|
run: |
|
|
docker rm -f newsarchiver-e2e 2>/dev/null || true
|
|
docker run -d --name newsarchiver-e2e \
|
|
--network newsarchiver-network \
|
|
-e CI=true \
|
|
-e CI_PORT_OFFSET=1 \
|
|
-e ADMIN_PASSWORD="" \
|
|
-e ARCHIVE_DIR=/data/archives \
|
|
-e DISABLE_RSS_FETCH=1 \
|
|
jarianc/newsarchiverv2:test \
|
|
python run_archiver.py --serve --host 0.0.0.0 --port 5000
|
|
|
|
- name: Discover app port
|
|
id: port
|
|
run: |
|
|
sleep 2
|
|
# CI port range 10000-10099: port = 10000 + CI_PORT_OFFSET
|
|
# NewsArchiverV2 uses offset 1, so port = 10001
|
|
CI_LOG=$(docker logs newsarchiver-e2e 2>&1 | grep "\[ci-port-shift\]" || echo "")
|
|
if [ -n "$CI_LOG" ]; then
|
|
APP_PORT=$(echo "$CI_LOG" | grep -oE 'to [0-9]+' | grep -oE '[0-9]+$')
|
|
echo "app_port=${APP_PORT}" >> $GITHUB_OUTPUT
|
|
echo "$CI_LOG"
|
|
else
|
|
# Fallback: compute from offset
|
|
OFFSET=${CI_PORT_OFFSET:-1}
|
|
echo "app_port=$((10000 + OFFSET))" >> $GITHUB_OUTPUT
|
|
echo "No CI port shift in logs, computed port $((10000 + OFFSET))"
|
|
fi
|
|
|
|
- name: Seed test data
|
|
run: |
|
|
sleep 3
|
|
docker exec newsarchiver-e2e python3 -c "
|
|
import sqlite3, os
|
|
db = os.environ.get('ARCHIVE_DIR', '/app/archival_data') + '/cache.db'
|
|
conn = sqlite3.connect(db)
|
|
c = conn.cursor()
|
|
c.execute('''CREATE TABLE IF NOT EXISTS articles (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT, source_name TEXT NOT NULL,
|
|
article_url TEXT NOT NULL UNIQUE, article_guid TEXT, title TEXT,
|
|
author TEXT, publish_date TEXT, content_text TEXT, content_html TEXT,
|
|
archive_file_path TEXT, metadata_file_path TEXT,
|
|
status TEXT DEFAULT 'pending', error_message TEXT,
|
|
extraction_method TEXT, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP)''')
|
|
c.execute('CREATE INDEX IF NOT EXISTS idx_articles_source ON articles(source_name)')
|
|
c.execute('CREATE INDEX IF NOT EXISTS idx_articles_url ON articles(article_url)')
|
|
c.execute('CREATE INDEX IF NOT EXISTS idx_articles_status ON articles(status)')
|
|
now = '2026-07-07 12:00:00'
|
|
sources = ['Test News', 'Daily Wire', 'Tech Today']
|
|
for i in range(1, 81):
|
|
src = sources[(i-1) % len(sources)]
|
|
c.execute('INSERT OR IGNORE INTO articles (source_name, article_url, title, publish_date, content_text, status, created_at) VALUES (?, ?, ?, ?, ?, ?, ?)',
|
|
(src, f'https://test.com/a/{i}', f'Test Article {i}', f'2026-07-{(i % 28) + 1:02d} 10:00:00', f'Content for article {i}.', 'archived', now))
|
|
conn.commit()
|
|
conn.close()
|
|
print('Seeded 80 test articles across 3 sources')
|
|
"
|
|
|
|
- name: Wait for app
|
|
run: |
|
|
APP_PORT="${{ steps.port.outputs.app_port }}"
|
|
sleep 3
|
|
# Health check from inside container (no host port publish needed)
|
|
for i in $(seq 1 30); do
|
|
docker exec newsarchiver-e2e curl -sf "http://localhost:${APP_PORT}/" && echo "App ready on port ${APP_PORT}" && exit 0
|
|
sleep 2
|
|
done
|
|
echo "App failed to start" && exit 1
|
|
|
|
- name: Run Playwright tests
|
|
run: |
|
|
APP_PORT="${{ steps.port.outputs.app_port }}"
|
|
docker run --rm \
|
|
--network newsarchiver-network \
|
|
-v $GITHUB_WORKSPACE/tests/playwright/tests:/tests/tests \
|
|
-v $GITHUB_WORKSPACE/tests/playwright/playwright.config.ts:/tests/playwright.config.ts \
|
|
-w /tests \
|
|
-e APP_URL=http://newsarchiver-e2e:${APP_PORT} \
|
|
-e PLAYWRIGHT_BROWSERS_PATH=/ms-playwright \
|
|
mcr.microsoft.com/playwright:v1.51.0-jammy \
|
|
sh -c "npm install @playwright/test@1.51.0 && npx playwright@1.51.0 test"
|
|
|
|
- name: Cleanup
|
|
if: always()
|
|
run: |
|
|
docker rm -f newsarchiver-e2e || true
|
|
docker network rm newsarchiver-network 2>/dev/null || true
|
|
|
|
docker-build:
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: Clone repo
|
|
run: |
|
|
rm -rf $GITHUB_WORKSPACE/*
|
|
git clone --depth 1 $GITEA_URL/$GITHUB_REPOSITORY $GITHUB_WORKSPACE
|
|
git -C $GITHUB_WORKSPACE checkout $GITHUB_SHA 2>/dev/null || true
|
|
|
|
- name: Build Docker image
|
|
if: always()
|
|
run: |
|
|
if [[ -f Dockerfile ]]; then
|
|
docker build -t $(echo $GITHUB_REPOSITORY | tr '[:upper:]' '[:lower:]'):test .
|
|
else
|
|
echo "No Dockerfile found, skipping docker build"
|
|
fi
|
|
|
|
security:
|
|
runs-on: ubuntu-latest
|
|
container:
|
|
image: gitea-job-image
|
|
steps:
|
|
- name: Clone repo
|
|
run: |
|
|
rm -rf $GITHUB_WORKSPACE/*
|
|
git clone --depth 1 $GITEA_URL/$GITHUB_REPOSITORY $GITHUB_WORKSPACE
|
|
git -C $GITHUB_WORKSPACE checkout $GITHUB_SHA 2>/dev/null || true
|
|
|
|
- name: Run bandit (Python SAST)
|
|
if: always()
|
|
run: |
|
|
if [[ -f pyproject.toml ]]; then
|
|
pip3 install bandit
|
|
bandit -r . --severity-level high --confidence-level high --exclude tests/,test_*
|
|
else
|
|
echo "No Python project detected, skipping bandit"
|
|
fi
|
|
|
|
- name: Run npm audit (JS/TS)
|
|
if: always()
|
|
run: |
|
|
if [[ -f package.json ]]; then
|
|
npm ci
|
|
npm audit --audit-level=high 2>/dev/null || echo "npm audit: vulnerabilities found (non-blocking)"
|
|
else
|
|
echo "No Node.js project detected, skipping npm audit"
|
|
fi
|
|
|
|
build-result:
|
|
needs: [lint, test, docker-build, security, e2e]
|
|
runs-on: ubuntu-latest
|
|
container:
|
|
image: gitea-job-image
|
|
if: always()
|
|
steps:
|
|
- name: Summary
|
|
run: echo "All CI checks completed"
|