feat(evals): eval runner, 15 weighted synthetic cases and CI gate

.dependency-cruiser.cjs

@@ -29,7 +29,14 @@ module.exports = {
       severity: "error",
       comment: "Feature modules never open connections or run migrations: createDatabase()/runMigrations() belong to the composition roots (src/app/_server, src/*.ts entrypoints). Types and table definitions (src/db/schema) are fine.",
       from: { path: "^src/features/" },
-      to: { path: "^src/db/(index|client|migrate)\\.ts$", dependencyTypesNot: ["type-only"] },
+      to: { path: "^src/db/(index|client|migrate|job-queue-client)\\.ts$", dependencyTypesNot: ["type-only"] },
+    },
+    {
+      name: "pg-boss-client-only-in-db",
+      severity: "error",
+      comment: "pg-boss opens its own pool: construct it only in src/db (job-queue.ts); features use injected JobSender types.",
+      from: { path: "^src/", pathNot: "^src/db/" },
+      to: { path: "(^|/)node_modules/pg-boss(/|$)", dependencyTypesNot: ["type-only"] },
     },
     {
       name: "not-to-unresolvable",

.env.example

@@ -29,6 +29,53 @@ S3_FORCE_PATH_STYLE=true
 # Host port of the local S3 gateway.
 S3_PORT=8333
 
+# --- Environment -------------------------------------------------------------------------------
+# local | showcase | production. Only `local` accepts the committed local-default auth secret.
+APP_ENV=local
+
+# --- Authentication (Better Auth) ---------------------------------------------------------------
+# Signs sessions and cookies; at least 32 characters. Generate a real one: `openssl rand -base64 32`.
+BETTER_AUTH_SECRET=local-dev-only-secret-change-me-0123456789
+# Public base URL of the web app (cookies, redirects, trusted origin).
+BETTER_AUTH_URL=http://localhost:3000
+# Client IP for the login rate limit. Set to what YOUR reverse proxy writes and list the proxy
+# addresses; without trusted proxies only a single-value header is trusted. If the web container is
+# reachable without a proxy, clients can forge this header – put a proxy in front (see operations.md).
+AUTH_IP_HEADERS=x-forwarded-for
+AUTH_TRUSTED_PROXIES=
+
+# Demo seed only (`pnpm seed:demo`): password of the synthetic demo accounts, local use only.
+SEED_PASSWORD=demo-password-local-only
+
+# --- Upload limits (intake) ---------------------------------------------------------------------
+# Maximum size per file in bytes (default 20 MiB) and files per request (default 10).
+UPLOAD_MAX_FILE_BYTES=20971520
+UPLOAD_MAX_FILES=10
+# Cap of one whole upload request (all files + form overhead); requests without Content-Length are refused.
+UPLOAD_MAX_REQUEST_BYTES=41943040
+
+# --- AI service (services/ai) --------------------------------------------------------------------
+# Called by the worker only. The token must equal the service's AI_SERVICE_TOKEN (at least 24 chars).
+AI_SERVICE_URL=http://127.0.0.1:8000
+AI_SERVICE_TOKEN=local-dev-only-ai-token-0123456789
+# Per-document timeout; a timeout is retried with backoff.
+AI_SERVICE_TIMEOUT_MS=120000
+
+# --- ERP export (ADR-0001 D9) ----------------------------------------------------------------------
+# Base URL of the ERP REST API; the pilot uses the mock inside the web app. The worker exports, the
+# web app serves the mock – both need the same token (at least 24 chars; the local default is
+# refused outside APP_ENV=local).
+# Host value for `pnpm worker` outside Docker; compose sets http://web:3000/api/erp-mock itself – do not
+# copy this line into a .env used by compose.
+ERP_BASE_URL=http://127.0.0.1:3000/api/erp-mock
+ERP_TOKEN=local-dev-only-erp-token-0123456789
+# Per-call timeout (max 20000); a timeout is retried with backoff under the same idempotency key.
+ERP_TIMEOUT_MS=10000
+# The mock route exists only with "true". Fault injection for demos, consumed in order per process:
+# 503 (before storing), lost (stored, answer 503), timeout (hangs) – e.g. ERP_MOCK_FAULTS=503,lost
+ERP_MOCK_ENABLED=true
+ERP_MOCK_FAULTS=
+
 # --- Web ---------------------------------------------------------------------------------------
 # Host port of the web container.
 WEB_PORT=3000

.github/workflows/ci.yml

@@ -101,12 +101,15 @@ jobs:
         id: profile
         if: steps.diff.outputs.code == 'true' && hashFiles('package.json') != ''
         run: echo "stage=$(sed -n 's/^[[:space:]]*stage:[[:space:]]*\([a-z]*\).*/\1/p' project-profile.yml | head -1)" >> "$GITHUB_OUTPUT"
+      - name: Install Playwright Chromium (E2E smoke)
+        if: steps.diff.outputs.code == 'true' && hashFiles('package.json') != '' && (steps.profile.outputs.stage == 'production' || contains(github.event.pull_request.labels.*.name, 'verify-full'))
+        run: pnpm exec playwright install --with-deps chromium
       - run: pnpm verify:full
         if: steps.diff.outputs.code == 'true' && hashFiles('package.json') != '' && (steps.profile.outputs.stage == 'production' || contains(github.event.pull_request.labels.*.name, 'verify-full'))
 
       # --- Python AI service: path-targeted (services/ai, contracts) ---------------------------
-      # The AI eval gate (ADR-0001 D8) joins this block with Epic 2 (#17): it runs on every change
-      # under services/ai/ and needs Vertex credentials via Workload Identity Federation.
+      # The AI eval gate (ADR-0001 D8, #24) runs in replay mode on every change under services/ai/:
+      # recorded model responses, no credentials. Live evals (--live, Vertex) never run in CI.
       - name: Foundation phase (AI service)
         if: steps.diff.outputs.ai == 'true' && hashFiles('services/ai/pyproject.toml') == ''
         run: echo "No services/ai/pyproject.toml – AI service not created yet."
@@ -123,6 +126,14 @@ jobs:
           uv run ruff format --check .
           uv run pyright
           uv run pytest -q
+      # Eval gate: fails when a key field drops by more than EVAL_GATE_THRESHOLD points against
+      # services/ai/evals/baseline.json, on any injection violation or case error.
+      - name: AI eval gate (replay)
+        if: steps.diff.outputs.ai == 'true' && hashFiles('services/ai/pyproject.toml') != ''
+        working-directory: services/ai
+        env:
+          EVAL_GATE_THRESHOLD: "5"
+        run: uv run python -m requestflow_ai.evals --replay --report "$RUNNER_TEMP/eval-report.json"
 
   # Image + compose smoke (ADR-0001 D11): only when the Dockerfile or the compose file changes.
   compose-smoke:

.gitignore

@@ -16,7 +16,7 @@ next-env.d.ts
 # Python
 __pycache__/
 *.py[cod]
-.venv/
+.venv
 .pytest_cache/
 .ruff_cache/
 .mypy_cache/
@@ -38,3 +38,10 @@ Thumbs.db
 
 # Agent worktrees (Claude Code)
 .claude/worktrees/
+
+# Local cloud credentials (never committed)
+.secrets/
+
+# Playwright
+/test-results/
+/playwright-report/

AGENTS.md

@@ -13,7 +13,7 @@ as a real customer engagement for a mid-sized machine-building company. **All da
 - Setup: `pnpm install && uv sync --project services/ai` · Start: `docker compose up` (runs the `setup` deploy step: migrations as `app_owner` + bucket) · Local services only: `docker compose up -d postgres storage`, then `pnpm setup:deploy`
 - `verify:changed` – inner loop: lint (ESLint incl. style rules), typecheck, focused tests of the touched files. `pnpm verify:changed -- <path>` · AI service: `uv run --project services/ai pytest <path>`
 - `verify` – canonical PR proof: lint, types, unit tests **and integration tests against real Postgres + S3 storage** (they prove tenant isolation and exactly-once export on every PR), architecture check (dependency-cruiser), build, `pnpm audit`, plus ruff/pyright/pytest for `services/ai`. `pnpm verify` (needs `docker compose up -d postgres storage`). A PR is not `ready-for-review` while verify fails, cannot run, or the exception is not justified in the PR.
-- `verify:full` – Playwright smoke flow + full AI eval run (until the smoke flow exists it equals `verify`). `pnpm verify:full` – before a release, after risky refactors or with PR label `verify-full`.
+- `verify:full` – `verify` + Playwright smoke flow (upload → worker with an AI stub → review → approve; needs `playwright install chromium` or `PW_CHROMIUM_PATH`); plus the AI eval gate in replay mode (`pnpm evals`). `pnpm verify:full` – before a release, after risky refactors or with PR label `verify-full`.
 - **AI eval gate** (ADR-0001 D8): additionally runs path-targeted in CI on every change under `services/ai/` (prompts, parsing, extraction, model config) – a regression on a key field fails the PR.
 - Test and verify output is trimmed automatically (`scripts/quiet-run.sh` via the hook `filter-test-output.sh`): exit code unchanged, full log path printed; prefix `FLUORY_FULL_OUTPUT=1` once when the cause is unclear.
 - **Docs guard:** `scripts/doku-check.sh` – runs in CI and in `/finish-work`.

CHANGELOG.md

@@ -6,6 +6,42 @@ This file records what changes **in the product** – process and session state
 ## [Unreleased]
 
 ### Added
+- AI eval set and gate: 15 synthetic cases (tables, scans, missing values, prompt injection) measure
+  extraction quality per field; every change to the AI service is checked against a committed
+  baseline and fails when a field gets worse by more than 5 points or an injected value is accepted.
+- More document formats: Outlook `.msg` (with attachments, parsed recursively), Excel `.xlsx` and Word
+  `.docx` are extracted with exact source positions; scanned PDFs can be read with OCR – such values are
+  at most "uncertain". A broken attachment no longer fails the whole request.
+- Extraction schema v2: e-mail, phone and additional requirements as header fields, plus line items
+  (description, quantity, unit, material, dimensions), each with its own status and source quote; German
+  number formats, units and dates are normalised, a bare calendar week ("KW 42") stays at most uncertain.
+- User management for admins (`/users`): see the company's users with role and status, change roles,
+  deactivate (sign-in blocked, sessions ended) and reactivate; invitations and every change are audited.
+  The last active admin of a company cannot be demoted or deactivated; clerks have no access.
+- Guard for tenant isolation: the build fails when a table with company data lacks enforced
+  row-level security or its company policy.
+- Export: approved requests are sent to the ERP (a simulated ERP in the pilot, contract
+  `contracts/erp-export.openapi.yaml`) exactly once – retries after errors or lost answers never create a
+  second record; the request page shows the ERP reference, running retries, and a visible error if the
+  export finally fails (reprocess possible). The ERP mock is off unless `ERP_MOCK_ENABLED=true`.
+- Review (`/requests/:id`): staff see each extracted field with its status (found, uncertain,
+  missing, not verified) beside the source passage, correct values (every correction is kept with
+  who and when), approve the request – which queues it for export – or reject it with a reason.
+- `pnpm verify:full` runs a browser smoke flow (upload → processing → review → correction → approval).
+- Background processing: the worker sends each document to the AI service, stores the extracted
+  fields with their evidence and moves the request to review; failures retry with backoff and end
+  in a visible error with attempts and cause; failed requests can be reprocessed.
+- AI service (`services/ai`): `POST /v1/extract` turns an e-mail or PDF into segments with stable
+  locators and extracts company, contact person and requested delivery date with evidence; a
+  deterministic verifier marks every value whose quote is not in the cited segment as `unverified`.
+- Upload of a quote request (`/requests`): .eml, .msg, .pdf, .xlsx, .docx up to a configured size;
+  originals stored privately, download only for the own company. Request, documents, audit entry and
+  the processing job are created in one step; exact duplicates are flagged and linked.
+- Invite-only login (e-mail + password): admins invite staff into their own company and hand over
+  an invitation link; sign-up without a valid invitation link creates no account. Roles `admin` and `clerk` per company.
+- Tenant isolation: every company-owned table has forced row-level security; data access runs
+  inside `withTenant()`.
+- Login rate limit (stored in the database) and `pnpm seed:demo` with two synthetic companies.
 - Runnable local stack: `docker compose up` starts PostgreSQL 17, SeaweedFS (S3), a one-shot `setup`
   step (migrations + private bucket), the web app and a no-op worker.
 - `GET /api/health` reports database and storage status (200 / 503, no connection details).

compose.yaml

@@ -54,6 +54,16 @@ services:
       S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-local-access-key}
       S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-local-secret-key}
       S3_FORCE_PATH_STYLE: "true"
+      BETTER_AUTH_SECRET: ${BETTER_AUTH_SECRET:-local-dev-only-secret-change-me-0123456789}
+      BETTER_AUTH_URL: ${BETTER_AUTH_URL:-http://localhost:3000}
+      APP_ENV: ${APP_ENV:-local}
+      AI_SERVICE_URL: ${AI_SERVICE_URL:-http://ai:8080}
+      AI_SERVICE_TOKEN: ${AI_SERVICE_TOKEN:-local-dev-only-ai-token-0123456789}
+      # ERP port (ADR-0001 D9): the worker exports to the mock inside `web`; the token is shared.
+      ERP_BASE_URL: ${ERP_BASE_URL:-http://web:3000/api/erp-mock}
+      ERP_TOKEN: ${ERP_TOKEN:-local-dev-only-erp-token-0123456789}
+      ERP_MOCK_ENABLED: ${ERP_MOCK_ENABLED:-true}
+      ERP_MOCK_FAULTS: ${ERP_MOCK_FAULTS:-}
     depends_on:
       postgres:
         condition: service_healthy
@@ -85,6 +95,23 @@ services:
       setup:
         condition: service_completed_successfully
 
+  # AI service (services/ai). Opt-in profile: it refuses to start without Vertex AI credentials
+  # (fail-closed, ADR-0001 D8). Without it, processing jobs retry and end in ERROR with a visible cause.
+  #   docker compose --profile ai up
+  ai:
+    build: ./services/ai
+    profiles: ["ai"]
+    environment:
+      AI_SERVICE_TOKEN: ${AI_SERVICE_TOKEN:-local-dev-only-ai-token-0123456789}
+      VERTEX_PROJECT: ${VERTEX_PROJECT:-}
+      VERTEX_LOCATION: ${VERTEX_LOCATION:-eu}
+      VERTEX_MODEL: ${VERTEX_MODEL:-gemini-3.5-flash}
+      GOOGLE_APPLICATION_CREDENTIALS: /secrets/adc.json
+    volumes:
+      - ${GOOGLE_ADC_FILE:-./.secrets/adc.json}:/secrets/adc.json:ro
+    ports:
+      - "${AI_PORT:-8000}:8080"
+
 volumes:
   pgdata:
   seaweed: