feat(evals): eval runner, 15 weighted synthetic cases and CI gate #132
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Base CI – minute-saving without breaking required gates (SYSTEM.md §11) | |
| # Adapted from Entwicklungsplan/templates/base/ci.yml: pnpm + Node 24 (TS app), uv (Python AI service), | |
| # action majors as of 2026-09-22 (template still pins v4). | |
| # | |
| # No workflow-wide paths-ignore: a skipped required workflow stays "Pending" and blocks doc PRs. | |
| # The required job `check` always runs, detects doc-only changes internally and is green at once – | |
| # expensive steps run only for code changes with a manifest. | |
| name: CI | |
| # No branches filter: stacked PRs on slice branches must be checked too. | |
| # Explicit types: the defaults (opened, synchronize, reopened) never re-run the PR guard when a draft is | |
| # marked ready (verify-green rule) or when the description is edited. | |
| on: | |
| pull_request: | |
| types: [opened, synchronize, reopened, ready_for_review, edited] | |
| concurrency: | |
| group: ci-${{ github.ref }} | |
| cancel-in-progress: true | |
| permissions: | |
| contents: read | |
| jobs: | |
| check: | |
| runs-on: ubuntu-latest # Linux only: Windows counts 2x, macOS 10x minutes. | |
| timeout-minutes: 15 | |
| steps: | |
| - uses: actions/checkout@v7 | |
| with: | |
| fetch-depth: 0 | |
| - name: Detect doc-only changes | |
| id: diff | |
| run: | | |
| if git diff --name-only "origin/${{ github.base_ref }}...HEAD" | grep -qvE '^docs/|\.md$'; then | |
| echo "code=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "code=false" >> "$GITHUB_OUTPUT" | |
| fi | |
| if git diff --name-only "origin/${{ github.base_ref }}...HEAD" | grep -qE '^(services/ai|contracts)/'; then | |
| echo "ai=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "ai=false" >> "$GITHUB_OUTPUT" | |
| fi | |
| # Docs guard: runs on EVERY PR. Seconds, no dependencies. (SYSTEM.md §2, §9) | |
| - name: Docs guard | |
| run: bash scripts/doku-check.sh | |
| # PR guard: plain language, plan gate, exactly one docs decision, file-size gate, | |
| # before-creating proof, next step for drafts, green verify for ready-for-review. | |
| - name: PR guard | |
| env: | |
| PR_BODY: ${{ github.event.pull_request.body }} | |
| PR_DRAFT: ${{ github.event.pull_request.draft }} | |
| PR_TITLE: ${{ github.event.pull_request.title }} | |
| BASE_REF: origin/${{ github.base_ref }} | |
| run: bash scripts/pr-check.sh | |
| - name: Doc-only result | |
| if: steps.diff.outputs.code == 'false' | |
| run: echo "Docs only – docs guard green, base gate green." | |
| # Foundation phase: no manifest yet. Report honestly instead of faking green tests. | |
| - name: Foundation phase (TS) | |
| if: steps.diff.outputs.code == 'true' && hashFiles('package.json') == '' | |
| run: echo "No package.json – foundation phase, TS verify not applicable yet. Disappears with the first code issue." | |
| # --- TS app: verify (lint, types, tests, architecture check, build, audit) --------------- | |
| - uses: pnpm/action-setup@v6 | |
| if: steps.diff.outputs.code == 'true' && hashFiles('package.json') != '' | |
| - uses: actions/setup-node@v7 | |
| if: steps.diff.outputs.code == 'true' && hashFiles('package.json') != '' | |
| with: | |
| node-version: 24 | |
| cache: pnpm | |
| - run: pnpm install --frozen-lockfile | |
| if: steps.diff.outputs.code == 'true' && hashFiles('package.json') != '' | |
| # Integration tests run against real PostgreSQL + SeaweedFS from compose.yaml (same images, | |
| # init script and local-default credentials as on a developer machine – no CI secrets). | |
| - name: Start PostgreSQL + SeaweedFS | |
| if: steps.diff.outputs.code == 'true' && hashFiles('package.json') != '' | |
| run: | | |
| docker compose up -d postgres storage | |
| for _ in $(seq 1 60); do | |
| s3="$(curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:8333/ || true)" | |
| if docker compose exec -T postgres pg_isready -h 127.0.0.1 -U postgres -d requestflow >/dev/null 2>&1 && [ "$s3" != "000" ]; then | |
| echo "postgres ready, storage answers HTTP $s3"; exit 0 | |
| fi | |
| sleep 2 | |
| done | |
| docker compose logs --tail 50 | |
| exit 1 | |
| - run: pnpm verify | |
| if: steps.diff.outputs.code == 'true' && hashFiles('package.json') != '' | |
| # verify:full (integration + E2E smoke + eval gate): P2 always, otherwise only with label verify-full. | |
| - name: Read stage from profile | |
| id: profile | |
| if: steps.diff.outputs.code == 'true' && hashFiles('package.json') != '' | |
| run: echo "stage=$(sed -n 's/^[[:space:]]*stage:[[:space:]]*\([a-z]*\).*/\1/p' project-profile.yml | head -1)" >> "$GITHUB_OUTPUT" | |
| - name: Install Playwright Chromium (E2E smoke) | |
| if: steps.diff.outputs.code == 'true' && hashFiles('package.json') != '' && (steps.profile.outputs.stage == 'production' || contains(github.event.pull_request.labels.*.name, 'verify-full')) | |
| run: pnpm exec playwright install --with-deps chromium | |
| - run: pnpm verify:full | |
| if: steps.diff.outputs.code == 'true' && hashFiles('package.json') != '' && (steps.profile.outputs.stage == 'production' || contains(github.event.pull_request.labels.*.name, 'verify-full')) | |
| # --- Python AI service: path-targeted (services/ai, contracts) --------------------------- | |
| # The AI eval gate (ADR-0001 D8, #24) runs in replay mode on every change under services/ai/: | |
| # recorded model responses, no credentials. Live evals (--live, Vertex) never run in CI. | |
| - name: Foundation phase (AI service) | |
| if: steps.diff.outputs.ai == 'true' && hashFiles('services/ai/pyproject.toml') == '' | |
| run: echo "No services/ai/pyproject.toml – AI service not created yet." | |
| - uses: astral-sh/setup-uv@v10.2.0 # no floating v10 tag exists (verified 2026-09-22) | |
| if: steps.diff.outputs.ai == 'true' && hashFiles('services/ai/pyproject.toml') != '' | |
| with: | |
| enable-cache: true | |
| - name: AI service verify (ruff, pyright, pytest) | |
| if: steps.diff.outputs.ai == 'true' && hashFiles('services/ai/pyproject.toml') != '' | |
| working-directory: services/ai | |
| run: | | |
| uv sync --frozen | |
| uv run ruff check . | |
| uv run ruff format --check . | |
| uv run pyright | |
| uv run pytest -q | |
| # Eval gate: fails when a key field drops by more than EVAL_GATE_THRESHOLD points against | |
| # services/ai/evals/baseline.json, on any injection violation or case error. | |
| - name: AI eval gate (replay) | |
| if: steps.diff.outputs.ai == 'true' && hashFiles('services/ai/pyproject.toml') != '' | |
| working-directory: services/ai | |
| env: | |
| EVAL_GATE_THRESHOLD: "5" | |
| run: uv run python -m requestflow_ai.evals --replay --report "$RUNNER_TEMP/eval-report.json" | |
| # Image + compose smoke (ADR-0001 D11): only when the Dockerfile or the compose file changes. | |
| compose-smoke: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| steps: | |
| - uses: actions/checkout@v7 | |
| with: | |
| fetch-depth: 0 | |
| - name: Detect image changes | |
| id: diff | |
| run: | | |
| if git diff --name-only "origin/${{ github.base_ref }}...HEAD" | grep -qE '(^|/)Dockerfile$|^compose\.yaml$|^\.dockerignore$|^docker/'; then | |
| echo "image=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "image=false" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: docker compose up → /api/health | |
| if: steps.diff.outputs.image == 'true' | |
| run: | | |
| if ! docker compose up -d --build; then | |
| docker compose ps -a | |
| docker compose logs --tail 80 | |
| exit 1 | |
| fi | |
| for _ in $(seq 1 60); do | |
| if curl -fsS http://127.0.0.1:3000/api/health; then echo; docker compose ps; exit 0; fi | |
| sleep 3 | |
| done | |
| docker compose ps | |
| docker compose logs --tail 80 | |
| exit 1 | |
| - name: Worker is running | |
| if: steps.diff.outputs.image == 'true' | |
| run: docker compose ps --status running --services | grep -qx worker |