From 850d61fcac0b5f7f249089f0e60bd9a71f2740cd Mon Sep 17 00:00:00 2001 From: tomicodesdev <32748319+tomicodesdev@users.noreply.github.com> Date: Mon, 18 May 2026 02:45:29 +0100 Subject: [PATCH 1/2] Ship Phase 6 historical evals --- .github/workflows/evals.yml | 38 +++ README.md | 3 +- docs/architecture.md | 25 +- docs/operations.md | 19 ++ docs/roadmap.md | 20 +- docs/security.md | 6 + migrations/20260518_000000_eval_history.sql | 70 +++++ procedures/refund-intake.yaml | 3 + scripts/ranse.ts | 64 +++- src/agents/supervisor/email-draft.ts | 4 + src/agents/supervisor/ticket-actions.ts | 6 + src/api/evals.ts | 77 +++++ src/api/routes.ts | 2 + src/evals/anonymize.ts | 127 ++++++++ src/evals/capture.ts | 249 ++++++++++++++++ src/evals/index.ts | 4 + src/evals/replay.ts | 314 ++++++++++++++++++++ src/evals/storage.ts | 263 ++++++++++++++++ src/lib/ids.ts | 3 + src/procedures/runner.ts | 6 + src/types/evals.ts | 111 +++++++ src/ui/api.ts | 18 ++ src/ui/views/EvalsSection.tsx | 125 ++++++++ src/ui/views/Settings.tsx | 3 + tests/evals.test.ts | 227 ++++++++++++++ tests/helpers/workspace-db.ts | 43 +++ 26 files changed, 1820 insertions(+), 10 deletions(-) create mode 100644 .github/workflows/evals.yml create mode 100644 migrations/20260518_000000_eval_history.sql create mode 100644 src/api/evals.ts create mode 100644 src/evals/anonymize.ts create mode 100644 src/evals/capture.ts create mode 100644 src/evals/index.ts create mode 100644 src/evals/replay.ts create mode 100644 src/evals/storage.ts create mode 100644 src/types/evals.ts create mode 100644 src/ui/views/EvalsSection.tsx create mode 100644 tests/evals.test.ts diff --git a/.github/workflows/evals.yml b/.github/workflows/evals.yml new file mode 100644 index 0000000..e27e351 --- /dev/null +++ b/.github/workflows/evals.yml @@ -0,0 +1,38 @@ +name: Evals + +on: + pull_request: + paths: + - "procedures/**" + - "src/agents/specialists/**" + - "src/agents/supervisor/**" + - "src/evals/**" + - "src/llm/**" + - "src/procedures/**" + - "scripts/ranse.ts" + - ".github/workflows/evals.yml" + +jobs: + procedure-evals: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: oven-sh/setup-bun@v2 + - run: bun install --frozen-lockfile + - run: bun run typecheck + - name: Run procedure evals + run: | + shopt -s nullglob + for file in procedures/*.yaml procedures/*.yml procedures/*.json; do + bun scripts/ranse.ts eval "$file" + done + - name: Run hosted historical evals when configured + env: + RANSE_APP_URL: ${{ secrets.RANSE_APP_URL }} + RANSE_COOKIE: ${{ secrets.RANSE_COOKIE }} + run: | + if [ -n "$RANSE_APP_URL" ] && [ -n "$RANSE_COOKIE" ]; then + bun scripts/ranse.ts eval --app-url "$RANSE_APP_URL" --cookie "$RANSE_COOKIE" --ci + else + echo "::notice::Set RANSE_APP_URL and RANSE_COOKIE secrets to gate PRs on hosted historical evals." + fi diff --git a/README.md b/README.md index 4fa7de8..2488db5 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ Ranse turns support email into a real-time, multi-agent support workspace built - **Multi-agent assist** — triage, summarization, knowledge retrieval, reply drafting, escalation, SLA. - **Human approval gates** for every outbound reply, with edit-before-send. - **Multi-provider LLM** — Workers AI out of the box; drop-in Anthropic, OpenAI, Google, Grok, OpenRouter via AI Gateway. +- **Historical evals** — resolved conversations become anonymized replay cases; `ranse eval` catches prompt/procedure regressions before they ship. - **One-click deploy** to your own Cloudflare account — customer-owned from day one. - **Open source** (Apache-2.0). @@ -83,7 +84,7 @@ Then open http://localhost:5173 (or http://localhost:8787 for the Worker directl Ranse is heading from "AI-assisted shared inbox" to a full autonomous customer-service agent — but not as an OSS clone of [Fin](https://fin.ai/) or [Decagon](https://decagon.ai/). The goal is the agent those products *structurally cannot become*: sovereign by construction, per-step model choice, procedures-as-code, MCP-native actions, eval-first against your own ticket history, and a forkable procedure library. -The shape, in short: **retrieval → workspace management → agentic retrieval → autonomous resolution → procedures → MCP actions → evals → procedure library → insights → multi-channel.** Phase 0 (bootstrap, inbound email, supervisor DO, draft + approval), Phase 1 (retrieval foundations), Phase 1.5 (workspace management & tenant isolation), Phase 2 (agentic multi-hop retrieval), Phase 3 (autonomous resolution), Phase 4 (procedures as code), and Phase 5 (MCP-native actions) are shipped. +The shape, in short: **retrieval → workspace management → agentic retrieval → autonomous resolution → procedures → MCP actions → evals → procedure library → insights → multi-channel.** Phase 0 (bootstrap, inbound email, supervisor DO, draft + approval), Phase 1 (retrieval foundations), Phase 1.5 (workspace management & tenant isolation), Phase 2 (agentic multi-hop retrieval), Phase 3 (autonomous resolution), Phase 4 (procedures as code), Phase 5 (MCP-native actions), and Phase 6 (historical evals) are shipped. Full pipeline, principles, and how to contribute to a phase: **[docs/roadmap.md](docs/roadmap.md)**. It's directional, not committed — if you want to work on something further down the list, open a discussion and we'll happily reorder. diff --git a/docs/architecture.md b/docs/architecture.md index 68e149a..31718b0 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -52,7 +52,7 @@ triageAndDraft (runs in DO alarm, async) | System | Purpose | |---|---| | DO SQLite | Workspace state, mailbox counters, BYOK-encrypted secrets | -| D1 | Tickets, messages, audit, approvals, outcomes, feedback, daily rollups, users, sessions, knowledge, LLM config, procedures, MCP registry/tool calls | +| D1 | Tickets, messages, audit, approvals, outcomes, feedback, daily rollups, users, sessions, knowledge, LLM config, procedures, MCP registry/tool calls, eval cases/runs/results | | R2 | Raw MIME, text/html bodies, attachments, exports | | KV | Rate limits, idempotency, lightweight flags | | Vectorize | Per-workspace knowledge chunk embeddings | @@ -101,6 +101,29 @@ ProcedureRunnerAgent MCP server secrets are stored in `UserSecretsStore` under `mcp:`. D1 stores endpoint metadata, discovered tool schemas, annotations, guardrail configuration, and immutable call records. +## Eval flow + +``` +Operator resolves ticket + ├─ setTicketStatus(resolved|closed) + ├─ captureResolvedTicketEvalCase + │ ├─ load ticket + inbound/outbound transcript + │ ├─ anonymize email, phone, and requester-name fields + │ └─ upsert eval_case(source = resolved_ticket) + └─ audit eval.case_captured + +ranse eval / Settings -> Evals + ├─ create eval_run + ├─ for each active eval_case + │ ├─ agenticSearchKnowledge with current retrieval prompts/config + │ ├─ runDraft with current draft prompt/model config + │ ├─ score reply overlap, required terms, confidence signals + │ └─ insert eval_result + └─ mark eval_run passed/failed +``` + +Procedure evals are local and deterministic. `ranse eval ` loads the spec, runs each inline `evals[]` case through `simulateProcedure`, and checks expected status, context paths, and step order before a PR is merged. + ## Scaling model - One `WorkspaceSupervisorAgent` DO per workspace. The email handler pins by `idFromName(workspaceId)` so all events for a workspace funnel through one instance — consistent state, no cross-DO coordination needed. diff --git a/docs/operations.md b/docs/operations.md index 8038be8..a69cc01 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -37,6 +37,25 @@ Operational checks: - Use per-ticket and per-hour limits to protect internal systems from procedure loops. - Inspect `mcp_tool_call` for action history and `audit_event` actions prefixed with `mcp.tool_call_` for timeline-level observability. +## Historical evals + +Resolved tickets are the regression suite. When an operator marks a ticket `resolved` or `closed`, Ranse captures the inbound/outbound transcript into an anonymized `eval_case` if the conversation has both a customer message and a support reply. Operators can also backfill recent cases from **Settings → Evals**. + +CLI workflow: + +```bash +# Run inline evals shipped alongside a procedure spec +bun scripts/ranse.ts eval procedures/refund-intake.yaml + +# Backfill resolved conversations from a deployed workspace +bun scripts/ranse.ts eval capture-resolved --app-url "$RANSE_APP_URL" --cookie "$RANSE_COOKIE" --limit 100 + +# Replay active historical cases through current retrieval + drafting +bun scripts/ranse.ts eval --app-url "$RANSE_APP_URL" --cookie "$RANSE_COOKIE" --threshold 0.35 --ci +``` + +Eval runs write `eval_run` and `eval_result` rows with assertion details. A run fails when any active case regresses. The bundled GitHub Actions workflow always runs procedure evals for relevant PRs; set `RANSE_APP_URL` and `RANSE_COOKIE` repository secrets to make hosted historical replay part of the gate. + ## Escalations The `EscalationAgent` runs on demand. It returns `{ should_escalate, severity, route_to }` and the operator (or an automation rule) picks the handoff target. diff --git a/docs/roadmap.md b/docs/roadmap.md index b85be59..fe48c47 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -49,7 +49,9 @@ Most "AI agent" tools are chat shaped and bolt email on. Real B2B support lives **Phase 5 — MCP-native actions** is shipped. Workspaces can register remote Streamable HTTP MCP servers, discover tools, enforce per-tool guardrails, pause destructive calls for operator approval, and execute `call_action` from procedures with audit-backed tool-call records. -That's now a retrieval-grounded early Fin **Copilot** equivalent with workspace isolation, traceable multi-hop retrieval, a conservative autonomous-send path, a procedure-driven agent loop, and external action execution through the open MCP protocol. Everything below continues the path toward evals, procedure sharing, and insights. +**Phase 6 — Historical evals** is shipped. Resolved tickets are captured as anonymized replay cases, operators can backfill and run evals from Settings, `ranse eval` runs procedure-file and hosted historical suites, and PRs touching prompts/procedures/model logic have an eval workflow. + +That's now a retrieval-grounded early Fin **Copilot** equivalent with workspace isolation, traceable multi-hop retrieval, a conservative autonomous-send path, a procedure-driven agent loop, external action execution through the open MCP protocol, and a regression gate against the workspace's own ticket history. Everything below continues the path toward procedure sharing and insights. ## Phase 1 — Retrieval foundations **Status: shipped.** @@ -117,7 +119,7 @@ Single-pass RAG fails on the hard tickets — the ones where the customer's ques - **Per-hop model routing** (Principle 2): shipped. Planning, judging, rewriting, reranking, and drafting each have independent action keys in workspace model settings. - **`search` as a procedure primitive** (Principle 3): shipped. Phase 4 procedures can call `search(query, scope, max_hops)` with the same traceable retrieval loop. - **Answer Inspection shows the trace**: shipped. Drafts, approval suggestions, and Content Library test searches show hop query → results → judgment → next-query. -- **Eval cases test the loop, not just the final answer** (Principle 5): partially shipped as behavior tests around the loop contract. Historical replay belongs in Phase 6. +- **Eval cases test the loop, not just the final answer** (Principle 5): behavior tests cover the loop contract, and Phase 6 historical replay now runs resolved-ticket cases end-to-end. The `customer_data` search scope still fails closed with an explicit trace; procedures can now retrieve account state by calling registered MCP tools directly through `call_action`. @@ -165,13 +167,17 @@ The `customer_data` search scope still fails closed with an explicit trace; proc - First-party templates are in-repo for Stripe, Shopify, GitHub, Linear, and generic webhook MCP servers so workspaces can register their own protocol-native endpoints without adding connector code to Ranse. ## Phase 6 — Eval against your own ticket history +**Status: shipped.** + *Principle 5* -- Every resolved conversation captured as a replayable eval case (anonymized, configurable PII rules) -- `ranse eval` runs all eval cases against current prompts + procedures, reports regressions -- CI integration: PRs touching `procedures/`, `prompts/`, or model config gate on eval pass -- Historical replay is the primary signal; synthetic-conversation generation is a complement, not a substitute -- Per-procedure eval harness: edge cases, refusal cases, escalation cases all live alongside the procedure file +- Resolved or closed conversations are captured into `eval_case` with transcript, latest customer message, expected reply preview, outcome kinds, and deterministic source fingerprints. +- Capture runs automatically when an operator marks a ticket `resolved` or `closed`, and can be backfilled from **Settings → Evals** or `ranse eval capture-resolved`. +- PII anonymization is applied before persistence, with configurable email, phone, and requester-name redaction rules. +- Hosted eval runs replay active historical cases through current retrieval + draft logic, score overlap and required terms, and store per-case assertions in `eval_result`. +- `ranse eval ` runs inline procedure evals from the YAML/JSON/TS spec. Edge, refusal, escalation, and wait/resume expectations live next to the procedure. +- `.github/workflows/evals.yml` gates PRs touching procedure, prompt/model, supervisor, or eval code on local procedure evals; with `RANSE_APP_URL` and `RANSE_COOKIE` secrets, the same workflow also gates on hosted historical replay. +- Historical replay is the primary signal; synthetic-conversation generation remains a future complement, not a substitute. ## Phase 7 — Procedure library + community *Principle 6* diff --git a/docs/security.md b/docs/security.md index 8f4e7c5..d826dd6 100644 --- a/docs/security.md +++ b/docs/security.md @@ -53,6 +53,12 @@ Default policy: AI may classify, summarize, search, and draft. It **cannot** sen - Guardrails are enforced in the procedure runner before the MCP request leaves Ranse: disabled tools, per-ticket/hour call limits, dollar limits, and allowed customer segments. - Every MCP attempt writes a `mcp_tool_call` row plus an audit event. The request includes a Ranse idempotency key in MCP `_meta` so well-behaved tools can suppress duplicate writes. +## Eval data privacy + +Historical eval cases are stored in the workspace's own D1 database and never leave the customer's Cloudflare account unless the operator exports or queries them. Before persistence, resolved-ticket capture redacts email addresses, phone numbers, and the requester's display name by default. The stored `anonymization_json` records rules and counts only; it does not store the original PII mapping. + +Eval access is restricted to `owner` and `admin` workspace roles because even anonymized conversations can contain commercially sensitive support context. Hosted replay uses the same provider-key and model-routing path as normal drafting, so self-hosters keep the same BYOK and Cloudflare tenant boundaries. + ## Audit trail Every state change writes an `audit_event`: diff --git a/migrations/20260518_000000_eval_history.sql b/migrations/20260518_000000_eval_history.sql new file mode 100644 index 0000000..ce92f11 --- /dev/null +++ b/migrations/20260518_000000_eval_history.sql @@ -0,0 +1,70 @@ +-- Phase 6 historical evals. + +CREATE TABLE IF NOT EXISTS eval_case ( + id TEXT PRIMARY KEY, + workspace_id TEXT NOT NULL, + source TEXT NOT NULL CHECK(source IN ('resolved_ticket','procedure_spec','synthetic','api')), + ticket_id TEXT, + procedure_id TEXT, + procedure_version_id TEXT, + name TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'active' CHECK(status IN ('active','archived')), + input_json TEXT NOT NULL, + expected_json TEXT NOT NULL, + anonymization_json TEXT NOT NULL DEFAULT '{}', + source_fingerprint TEXT NOT NULL, + captured_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + UNIQUE(workspace_id, source, source_fingerprint), + FOREIGN KEY (workspace_id) REFERENCES workspace(id) ON DELETE CASCADE, + FOREIGN KEY (ticket_id) REFERENCES ticket(id) ON DELETE SET NULL, + FOREIGN KEY (procedure_id) REFERENCES procedure(id) ON DELETE SET NULL, + FOREIGN KEY (procedure_version_id) REFERENCES procedure_version(id) ON DELETE SET NULL +); + +CREATE INDEX IF NOT EXISTS idx_eval_case_workspace + ON eval_case(workspace_id, status, captured_at DESC); + +CREATE INDEX IF NOT EXISTS idx_eval_case_ticket + ON eval_case(workspace_id, ticket_id); + +CREATE TABLE IF NOT EXISTS eval_run ( + id TEXT PRIMARY KEY, + workspace_id TEXT NOT NULL, + source TEXT NOT NULL DEFAULT 'api' CHECK(source IN ('api','cli','ci','scheduled')), + status TEXT NOT NULL CHECK(status IN ('running','passed','failed')), + case_count INTEGER NOT NULL DEFAULT 0, + passed_count INTEGER NOT NULL DEFAULT 0, + failed_count INTEGER NOT NULL DEFAULT 0, + regression_count INTEGER NOT NULL DEFAULT 0, + config_json TEXT NOT NULL DEFAULT '{}', + started_at INTEGER NOT NULL, + completed_at INTEGER, + created_at INTEGER NOT NULL, + FOREIGN KEY (workspace_id) REFERENCES workspace(id) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_eval_run_workspace + ON eval_run(workspace_id, created_at DESC); + +CREATE TABLE IF NOT EXISTS eval_result ( + id TEXT PRIMARY KEY, + workspace_id TEXT NOT NULL, + run_id TEXT NOT NULL, + case_id TEXT NOT NULL, + status TEXT NOT NULL CHECK(status IN ('passed','failed','skipped')), + score REAL, + assertions_json TEXT NOT NULL DEFAULT '[]', + actual_json TEXT NOT NULL DEFAULT '{}', + error TEXT, + created_at INTEGER NOT NULL, + FOREIGN KEY (workspace_id) REFERENCES workspace(id) ON DELETE CASCADE, + FOREIGN KEY (run_id) REFERENCES eval_run(id) ON DELETE CASCADE, + FOREIGN KEY (case_id) REFERENCES eval_case(id) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_eval_result_run + ON eval_result(run_id, status); + +CREATE INDEX IF NOT EXISTS idx_eval_result_case + ON eval_result(workspace_id, case_id, created_at DESC); diff --git a/procedures/refund-intake.yaml b/procedures/refund-intake.yaml index 2242436..7f7c07e 100644 --- a/procedures/refund-intake.yaml +++ b/procedures/refund-intake.yaml @@ -22,3 +22,6 @@ evals: subject: Refund request expect: status: completed + steps: + - find_policy + - add_context_note diff --git a/scripts/ranse.ts b/scripts/ranse.ts index 4da11b1..004149f 100644 --- a/scripts/ranse.ts +++ b/scripts/ranse.ts @@ -1,5 +1,6 @@ #!/usr/bin/env bun import { readFile } from 'node:fs/promises'; +import { runProcedureSpecEvals } from '../src/evals/replay'; import { loadProcedureFile } from '../src/procedures/files'; import { simulateProcedure } from '../src/procedures/simulate'; @@ -41,9 +42,53 @@ async function main() { return; } + if (command === 'eval') { + if (args[0] === 'capture-resolved') { + const appUrl = flagValue(args, '--app-url') ?? process.env.RANSE_APP_URL; + const cookie = flagValue(args, '--cookie') ?? process.env.RANSE_COOKIE; + if (!appUrl || !cookie) throw new Error('eval_capture_requires_app_url_and_cookie'); + const res = await fetch(new URL('/api/evals/cases/capture-resolved', appUrl), { + method: 'POST', + headers: { 'content-type': 'application/json', cookie }, + body: JSON.stringify({ limit: numberFlag(args, '--limit') }), + }); + const payload = await jsonOrText(res); + console.log(JSON.stringify(payload, null, 2)); + process.exit(res.ok && (payload as any).ok !== false ? 0 : 1); + } + + const procedureFile = args[0] && !args[0].startsWith('--') ? args[0] : undefined; + if (procedureFile) { + const spec = await loadProcedureFile(procedureFile); + const report = runProcedureSpecEvals(spec); + console.log(JSON.stringify(report, null, 2)); + process.exit(report.status === 'passed' ? 0 : 1); + } + + const appUrl = flagValue(args, '--app-url') ?? process.env.RANSE_APP_URL; + const cookie = flagValue(args, '--cookie') ?? process.env.RANSE_COOKIE; + if (!appUrl || !cookie) throw new Error('eval_requires_procedure_file_or_app_url_and_cookie'); + const res = await fetch(new URL('/api/evals/runs', appUrl), { + method: 'POST', + headers: { 'content-type': 'application/json', cookie }, + body: JSON.stringify({ + limit: numberFlag(args, '--limit'), + threshold: numberFlag(args, '--threshold'), + source: args.includes('--ci') ? 'ci' : 'cli', + }), + }); + const payload = await jsonOrText(res); + console.log(JSON.stringify(payload, null, 2)); + const status = (payload as any)?.run?.status; + process.exit(res.ok && status !== 'failed' ? 0 : 1); + } + console.log(`usage: ranse simulate [--input input.json] - ranse publish --app-url --cookie `); + ranse publish --app-url --cookie + ranse eval + ranse eval --app-url --cookie [--limit n] [--threshold n] [--ci] + ranse eval capture-resolved --app-url --cookie [--limit n]`); } function requiredArg(value: string | undefined, message: string): string { @@ -56,6 +101,23 @@ function flagValue(args_: string[], flag: string): string | undefined { return index >= 0 ? args_[index + 1] : undefined; } +function numberFlag(args_: string[], flag: string): number | undefined { + const value = flagValue(args_, flag); + if (value === undefined) return undefined; + const parsed = Number(value); + if (!Number.isFinite(parsed)) throw new Error(`invalid_number:${flag}`); + return parsed; +} + +async function jsonOrText(res: Response): Promise { + const text = await res.text(); + try { + return JSON.parse(text); + } catch { + return { status: res.status, body: text }; + } +} + main().catch((err) => { console.error(err instanceof Error ? err.message : err); process.exit(1); diff --git a/src/agents/supervisor/email-draft.ts b/src/agents/supervisor/email-draft.ts index a12d575..f2afcd8 100644 --- a/src/agents/supervisor/email-draft.ts +++ b/src/agents/supervisor/email-draft.ts @@ -1,5 +1,6 @@ import type { Env } from '../../env'; import { buildReplyAddress } from '../../email/reply-security'; +import { captureResolvedTicketEvalCase } from '../../evals/capture'; import { createApproval } from '../../lib/approvals'; import { audit } from '../../lib/audit'; import { recordOutcome } from '../../lib/outcomes'; @@ -231,6 +232,9 @@ async function tryAutoSend( citesKnowledgeIds: draft.cites_knowledge_ids, }, }); + await captureResolvedTicketEvalCase(ctx.env, ctx.workspaceId, ticketId).catch((err) => + console.warn('failed to capture autonomous eval case', err), + ); await audit(ctx.env, { workspaceId: ctx.workspaceId, ticketId, diff --git a/src/agents/supervisor/ticket-actions.ts b/src/agents/supervisor/ticket-actions.ts index f07d28b..433b959 100644 --- a/src/agents/supervisor/ticket-actions.ts +++ b/src/agents/supervisor/ticket-actions.ts @@ -1,4 +1,5 @@ import type { Env } from '../../env'; +import { captureResolvedTicketEvalCase } from '../../evals/capture'; import { audit } from '../../lib/audit'; import { ids } from '../../lib/ids'; import { listTicketFeedback, listTicketOutcomes, recordTicketFeedback } from '../../lib/outcomes'; @@ -111,6 +112,11 @@ export async function setTicketStatus( actorId: args.actorUserId, action: `ticket.${args.status}`, }); + if (args.status === 'resolved' || args.status === 'closed') { + await captureResolvedTicketEvalCase(env, workspaceId, args.ticketId, { + actorUserId: args.actorUserId, + }).catch((err) => console.warn('failed to capture resolved ticket eval case', err)); + } await refreshCounts(); } diff --git a/src/api/evals.ts b/src/api/evals.ts new file mode 100644 index 0000000..2309c59 --- /dev/null +++ b/src/api/evals.ts @@ -0,0 +1,77 @@ +import type { Hono } from 'hono'; +import { z } from 'zod'; +import { workspaceConfig } from '../agents/supervisor/settings'; +import { captureResolvedTicketEvalCases } from '../evals/capture'; +import { runEvalSuite } from '../evals/replay'; +import { getEvalRunDetail, listEvalCases, listEvalRuns } from '../evals/storage'; +import { apiError } from '../lib/errors'; +import { type Ctx, OWNER_OR_ADMIN, requireWorkspaceRole } from './context'; + +const anonymizationSchema = z + .object({ + redactEmails: z.boolean().optional(), + redactPhones: z.boolean().optional(), + redactRequesterName: z.boolean().optional(), + }) + .optional(); + +export function registerEvalRoutes(apiApp: Hono) { + apiApp.get('/evals/cases', requireWorkspaceRole(OWNER_OR_ADMIN), async (c) => { + const s = c.get('session'); + const cases = await listEvalCases(c.env, s.workspaceId, { + status: c.req.query('status') === 'archived' ? 'archived' : 'active', + limit: Number(c.req.query('limit') ?? 100), + }); + return c.json({ cases }); + }); + + apiApp.get('/evals/runs', requireWorkspaceRole(OWNER_OR_ADMIN), async (c) => { + const s = c.get('session'); + const runs = await listEvalRuns(c.env, s.workspaceId, Number(c.req.query('limit') ?? 20)); + return c.json({ runs }); + }); + + apiApp.get('/evals/runs/:id', requireWorkspaceRole(OWNER_OR_ADMIN), async (c) => { + const s = c.get('session'); + const detail = await getEvalRunDetail(c.env, s.workspaceId, c.req.param('id')); + if (!detail) return apiError(c, 'not_found', 'That eval run does not exist.'); + return c.json(detail); + }); + + apiApp.post('/evals/cases/capture-resolved', requireWorkspaceRole(OWNER_OR_ADMIN), async (c) => { + const s = c.get('session'); + const body = z + .object({ + limit: z.number().int().min(1).max(200).optional(), + anonymization: anonymizationSchema, + }) + .parse(await c.req.json().catch(() => ({}))); + const result = await captureResolvedTicketEvalCases(c.env, s.workspaceId, { + limit: body.limit, + anonymization: body.anonymization, + actorUserId: s.userId, + }); + return c.json({ ok: result.failed === 0, ...result }); + }); + + apiApp.post('/evals/runs', requireWorkspaceRole(OWNER_OR_ADMIN), async (c) => { + const s = c.get('session'); + const body = z + .object({ + limit: z.number().int().min(1).max(500).optional(), + case_ids: z.array(z.string()).max(500).optional(), + threshold: z.number().min(0.05).max(0.95).optional(), + source: z.enum(['api', 'cli', 'ci', 'scheduled']).optional(), + }) + .parse(await c.req.json().catch(() => ({}))); + const result = await runEvalSuite(c.env, s.workspaceId, { + limit: body.limit, + caseIds: body.case_ids, + threshold: body.threshold, + source: body.source ?? 'api', + workspaceConfig: await workspaceConfig(c.env, s.workspaceId), + }); + const detail = await getEvalRunDetail(c.env, s.workspaceId, result.run.id); + return c.json(detail ?? result); + }); +} diff --git a/src/api/routes.ts b/src/api/routes.ts index b7fc97d..f923881 100644 --- a/src/api/routes.ts +++ b/src/api/routes.ts @@ -1,6 +1,7 @@ import { Hono } from 'hono'; import { registerApprovalRoutes } from './approvals'; import { installApiAuth, type Ctx } from './context'; +import { registerEvalRoutes } from './evals'; import { registerKnowledgeRoutes } from './knowledge'; import { registerMcpRoutes } from './mcp'; import { registerNotificationRoutes } from './notifications'; @@ -19,4 +20,5 @@ registerApprovalRoutes(apiApp); registerKnowledgeRoutes(apiApp); registerProcedureRoutes(apiApp); registerMcpRoutes(apiApp); +registerEvalRoutes(apiApp); registerWorkspaceRoutes(apiApp); diff --git a/src/evals/anonymize.ts b/src/evals/anonymize.ts new file mode 100644 index 0000000..952b5bd --- /dev/null +++ b/src/evals/anonymize.ts @@ -0,0 +1,127 @@ +import type { EvalAnonymizationConfig } from '../types/evals'; + +interface AnonymizerState { + emailMap: Map; + phoneMap: Map; + nameCount: number; +} + +export interface AnonymizationMetadata { + rules: { + redactEmails: boolean; + redactPhones: boolean; + redactRequesterName: boolean; + }; + counts: { + emails: number; + phones: number; + requesterNames: number; + }; +} + +const EMAIL_RE = /\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/gi; +const PHONE_RE = /(? +> & + Pick { + return { + redactEmails: config.redactEmails ?? true, + redactPhones: config.redactPhones ?? true, + redactRequesterName: config.redactRequesterName ?? true, + requesterEmail: config.requesterEmail ?? null, + requesterName: config.requesterName ?? null, + }; +} + +export function anonymizeValue( + value: T, + config: EvalAnonymizationConfig = {}, +): { value: T; metadata: AnonymizationMetadata } { + const normalized = normalizeAnonymizationConfig(config); + const state: AnonymizerState = { + emailMap: new Map(), + phoneMap: new Map(), + nameCount: 0, + }; + const anonymized = anonymizeAny(value, normalized, state) as T; + return { + value: anonymized, + metadata: { + rules: { + redactEmails: normalized.redactEmails, + redactPhones: normalized.redactPhones, + redactRequesterName: normalized.redactRequesterName, + }, + counts: { + emails: state.emailMap.size, + phones: state.phoneMap.size, + requesterNames: state.nameCount, + }, + }, + }; +} + +function anonymizeAny( + value: unknown, + config: ReturnType, + state: AnonymizerState, +): unknown { + if (typeof value === 'string') return anonymizeText(value, config, state); + if (Array.isArray(value)) return value.map((item) => anonymizeAny(item, config, state)); + if (!value || typeof value !== 'object') return value; + const out: Record = {}; + for (const [key, nested] of Object.entries(value)) { + out[key] = anonymizeAny(nested, config, state); + } + return out; +} + +function anonymizeText( + input: string, + config: ReturnType, + state: AnonymizerState, +): string { + let text = input; + if (config.redactRequesterName && config.requesterName?.trim()) { + const name = config.requesterName.trim(); + if (name.length >= 3) { + const re = new RegExp(`\\b${escapeRegExp(name)}\\b`, 'gi'); + text = text.replace(re, () => { + state.nameCount += 1; + return '[customer_name]'; + }); + } + } + if (config.redactEmails) { + text = text.replace(EMAIL_RE, (match) => { + const key = match.toLowerCase(); + let replacement = state.emailMap.get(key); + if (!replacement) { + replacement = `customer_${state.emailMap.size + 1}@example.test`; + state.emailMap.set(key, replacement); + } + return replacement; + }); + } + if (config.redactPhones) { + text = text.replace(PHONE_RE, (match) => { + const digits = match.replace(/\D/g, ''); + if (digits.length < 8) return match; + let replacement = state.phoneMap.get(digits); + if (!replacement) { + replacement = `[phone_${state.phoneMap.size + 1}]`; + state.phoneMap.set(digits, replacement); + } + return replacement; + }); + } + return text; +} + +function escapeRegExp(value: string) { + return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} diff --git a/src/evals/capture.ts b/src/evals/capture.ts new file mode 100644 index 0000000..ed98a49 --- /dev/null +++ b/src/evals/capture.ts @@ -0,0 +1,249 @@ +import type { Env } from '../env'; +import { audit } from '../lib/audit'; +import { sha256Hex } from '../lib/crypto'; +import { getText } from '../lib/storage'; +import type { + EvalAnonymizationConfig, + ResolvedTicketEvalExpected, + ResolvedTicketEvalInput, +} from '../types/evals'; +import { anonymizeValue } from './anonymize'; +import { upsertEvalCase } from './storage'; + +interface TicketRow { + id: string; + subject: string; + status: string; + priority: string; + category: string | null; + requester_email: string; + requester_name: string | null; +} + +interface MessageRow { + id: string; + direction: string; + from_address: string | null; + to_address: string | null; + subject: string | null; + preview: string | null; + body_r2_key: string | null; + sent_at: number; +} + +interface OutcomeRow { + kind: string; +} + +export async function captureResolvedTicketEvalCase( + env: Env, + workspaceId: string, + ticketId: string, + options: { anonymization?: EvalAnonymizationConfig; actorUserId?: string | null } = {}, +): Promise<{ captured: boolean; caseId?: string; reason?: string }> { + const ticket = await env.DB.prepare( + `SELECT id, subject, status, priority, category, requester_email, requester_name + FROM ticket + WHERE id = ? AND workspace_id = ?`, + ) + .bind(ticketId, workspaceId) + .first(); + if (!ticket) return { captured: false, reason: 'ticket_not_found' }; + const resolvedOutcome = await env.DB.prepare( + `SELECT kind FROM ticket_outcome_event + WHERE workspace_id = ? AND ticket_id = ? + AND kind IN ('resolved_autonomously','resolved_via_procedure') + ORDER BY created_at DESC LIMIT 1`, + ) + .bind(workspaceId, ticketId) + .first(); + if (!['resolved', 'closed'].includes(ticket.status) && !resolvedOutcome) { + return { captured: false, reason: 'ticket_not_resolved' }; + } + + const messageRows = await env.DB.prepare( + `SELECT id, direction, from_address, to_address, subject, preview, body_r2_key, sent_at + FROM message_index + WHERE ticket_id = ? AND workspace_id = ? + ORDER BY sent_at ASC`, + ) + .bind(ticketId, workspaceId) + .all(); + const messages = messageRows.results ?? []; + const inbound = [...messages].reverse().find((message) => message.direction === 'inbound'); + const outbound = [...messages].reverse().find((message) => message.direction === 'outbound'); + if (!inbound) return { captured: false, reason: 'no_inbound_message' }; + if (!outbound) return { captured: false, reason: 'no_outbound_message' }; + + const outboundBody = await messageBody(env, outbound); + if (!outboundBody.trim()) return { captured: false, reason: 'empty_expected_reply' }; + + const transcript = await Promise.all( + messages + .filter((message) => ['inbound', 'outbound'].includes(message.direction)) + .map(async (message) => ({ + id: message.id, + direction: message.direction, + from_address: message.from_address, + to_address: message.to_address, + subject: message.subject, + preview: (await messageBody(env, message)).slice(0, 4000), + sent_at: message.sent_at, + })), + ); + const latestCustomerPreview = await messageBody(env, inbound); + const outcomes = await env.DB.prepare( + `SELECT kind FROM ticket_outcome_event + WHERE workspace_id = ? AND ticket_id = ? + ORDER BY created_at ASC`, + ) + .bind(workspaceId, ticketId) + .all(); + + const input: ResolvedTicketEvalInput = { + ticket: { + id: ticket.id, + subject: ticket.subject, + requester_email: ticket.requester_email, + requester_name: ticket.requester_name, + status: ticket.status, + priority: ticket.priority, + category: ticket.category, + }, + transcript, + latest_customer_message: { + subject: inbound.subject ?? ticket.subject, + preview: latestCustomerPreview.slice(0, 4000), + }, + }; + const expected: ResolvedTicketEvalExpected = { + expected_status: ticket.status, + expected_priority: ticket.priority, + expected_category: ticket.category, + expected_reply_preview: outboundBody.slice(0, 4000), + required_terms: extractRequiredTerms(outboundBody), + outcome_kinds: (outcomes.results ?? []).map((row) => row.kind), + }; + const anonymization = { + ...options.anonymization, + requesterEmail: ticket.requester_email, + requesterName: ticket.requester_name, + }; + const anonymizedInput = anonymizeValue(input, anonymization); + const anonymizedExpected = anonymizeValue(expected, anonymization); + const fingerprint = await sha256Hex( + JSON.stringify({ + ticketId, + inboundId: inbound.id, + outboundId: outbound.id, + expected: anonymizedExpected.value.expected_reply_preview, + }), + ); + const evalCase = await upsertEvalCase(env, { + workspaceId, + source: 'resolved_ticket', + ticketId, + name: `Resolved ticket: ${ticket.subject}`.slice(0, 200), + inputJson: JSON.stringify(anonymizedInput.value), + expectedJson: JSON.stringify(anonymizedExpected.value), + anonymizationJson: JSON.stringify({ + input: anonymizedInput.metadata, + expected: anonymizedExpected.metadata, + }), + sourceFingerprint: fingerprint, + }); + await audit(env, { + workspaceId, + ticketId, + actorType: options.actorUserId ? 'user' : 'system', + actorId: options.actorUserId ?? undefined, + action: 'eval.case_captured', + payload: { evalCaseId: evalCase.id, source: 'resolved_ticket' }, + }); + return { captured: true, caseId: evalCase.id }; +} + +export async function captureResolvedTicketEvalCases( + env: Env, + workspaceId: string, + options: { + limit?: number; + anonymization?: EvalAnonymizationConfig; + actorUserId?: string | null; + } = {}, +): Promise<{ captured: number; skipped: number; failed: number; cases: string[] }> { + const limit = Math.min(Math.max(options.limit ?? 50, 1), 200); + const rows = await env.DB.prepare( + `SELECT id FROM ticket + WHERE workspace_id = ? AND status IN ('resolved','closed') + ORDER BY updated_at DESC + LIMIT ?`, + ) + .bind(workspaceId, limit) + .all<{ id: string }>(); + let captured = 0; + let skipped = 0; + let failed = 0; + const cases: string[] = []; + for (const row of rows.results ?? []) { + try { + const result = await captureResolvedTicketEvalCase(env, workspaceId, row.id, options); + if (result.captured && result.caseId) { + captured += 1; + cases.push(result.caseId); + } else { + skipped += 1; + } + } catch (err) { + failed += 1; + console.warn('failed to capture eval case', row.id, err); + } + } + return { captured, skipped, failed, cases }; +} + +async function messageBody(env: Env, message: MessageRow): Promise { + if (message.body_r2_key) { + const body = await getText(env, message.body_r2_key); + if (body) return body; + } + return message.preview ?? ''; +} + +export function extractRequiredTerms(body: string, limit = 8): string[] { + const stop = new Set([ + 'about', + 'after', + 'again', + 'also', + 'because', + 'before', + 'could', + 'customer', + 'hello', + 'please', + 'regards', + 'support', + 'thanks', + 'thank', + 'there', + 'these', + 'those', + 'would', + 'your', + ]); + const words = body + .toLowerCase() + .replace(/https?:\/\/\S+/g, ' ') + .match(/[a-z][a-z0-9_-]{3,}/g); + if (!words) return []; + const counts = new Map(); + for (const word of words) { + if (stop.has(word)) continue; + counts.set(word, (counts.get(word) ?? 0) + 1); + } + return [...counts.entries()] + .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])) + .slice(0, limit) + .map(([word]) => word); +} diff --git a/src/evals/index.ts b/src/evals/index.ts new file mode 100644 index 0000000..732adc6 --- /dev/null +++ b/src/evals/index.ts @@ -0,0 +1,4 @@ +export * from './anonymize'; +export * from './capture'; +export * from './replay'; +export * from './storage'; diff --git a/src/evals/replay.ts b/src/evals/replay.ts new file mode 100644 index 0000000..d4c62c8 --- /dev/null +++ b/src/evals/replay.ts @@ -0,0 +1,314 @@ +import type { Env } from '../env'; +import { runDraft, type DraftResult } from '../agents/specialists/draft'; +import { agenticSearchKnowledge } from '../knowledge'; +import type { AgentConfig } from '../llm/config.types'; +import { simulateProcedure } from '../procedures/simulate'; +import type { + EvalAssertion, + ProcedureEvalReport, + ResolvedTicketEvalExpected, + ResolvedTicketEvalInput, +} from '../types/evals'; +import type { AgenticKnowledgeResult, KnowledgeHit } from '../types/knowledge'; +import type { ProcedureSpec } from '../types/procedure'; +import { completeEvalRun, createEvalRun, insertEvalResult, listEvalCases } from './storage'; + +export interface RunEvalSuiteOptions { + source?: 'api' | 'cli' | 'ci' | 'scheduled'; + limit?: number; + caseIds?: string[]; + threshold?: number; + workspaceConfig?: Partial; + retrievalRunner?: (input: ResolvedTicketEvalInput) => Promise; + draftRunner?: (input: ResolvedTicketEvalInput, knowledge: KnowledgeHit[]) => Promise; +} + +export async function runEvalSuite( + env: Env, + workspaceId: string, + options: RunEvalSuiteOptions = {}, +) { + const threshold = clampThreshold(options.threshold ?? 0.35); + const run = await createEvalRun(env, { + workspaceId, + source: options.source ?? 'api', + config: { threshold, limit: options.limit ?? null, caseIds: options.caseIds ?? null }, + }); + const cases = await listEvalCases(env, workspaceId, { + status: 'active', + limit: options.limit ?? 100, + caseIds: options.caseIds, + }); + let passedCount = 0; + let failedCount = 0; + let regressionCount = 0; + + for (const evalCase of cases) { + try { + if (evalCase.source !== 'resolved_ticket') { + await insertEvalResult(env, { + workspaceId, + runId: run.id, + caseId: evalCase.id, + status: 'skipped', + assertions: [ + { + name: 'source_supported', + passed: false, + message: 'Only resolved_ticket cases run in hosted replay.', + }, + ], + actual: {}, + }); + continue; + } + const input = JSON.parse(evalCase.input_json) as ResolvedTicketEvalInput; + const expected = JSON.parse(evalCase.expected_json) as ResolvedTicketEvalExpected; + const result = await replayResolvedTicketCase(env, workspaceId, input, expected, { + threshold, + workspaceConfig: options.workspaceConfig, + retrievalRunner: options.retrievalRunner, + draftRunner: options.draftRunner, + }); + if (result.status === 'passed') passedCount += 1; + else { + failedCount += 1; + regressionCount += 1; + } + await insertEvalResult(env, { + workspaceId, + runId: run.id, + caseId: evalCase.id, + status: result.status, + score: result.score, + assertions: result.assertions, + actual: result.actual, + error: result.error, + }); + } catch (err) { + failedCount += 1; + regressionCount += 1; + await insertEvalResult(env, { + workspaceId, + runId: run.id, + caseId: evalCase.id, + status: 'failed', + assertions: [{ name: 'replay_completed', passed: false }], + actual: {}, + error: err instanceof Error ? err.message : 'eval_case_failed', + }); + } + } + + const completed = await completeEvalRun(env, workspaceId, run.id, { + caseCount: cases.length, + passedCount, + failedCount, + regressionCount, + }); + return { run: completed }; +} + +export async function replayResolvedTicketCase( + env: Env, + workspaceId: string, + input: ResolvedTicketEvalInput, + expected: ResolvedTicketEvalExpected, + options: { + threshold?: number; + workspaceConfig?: Partial; + retrievalRunner?: (input: ResolvedTicketEvalInput) => Promise; + draftRunner?: ( + input: ResolvedTicketEvalInput, + knowledge: KnowledgeHit[], + ) => Promise; + } = {}, +): Promise<{ + status: 'passed' | 'failed'; + score: number; + assertions: EvalAssertion[]; + actual: Record; + error?: string; +}> { + const threshold = clampThreshold(options.threshold ?? 0.35); + const retrieval = options.retrievalRunner + ? await options.retrievalRunner(input) + : await agenticSearchKnowledge( + env, + workspaceId, + `${input.latest_customer_message.subject}\n${input.latest_customer_message.preview}`, + { limit: 5, maxHops: 3, workspaceConfig: options.workspaceConfig }, + ); + const draft = options.draftRunner + ? await options.draftRunner(input, retrieval.hits) + : await runDraft({ + env, + workspaceId, + ticketId: input.ticket.id, + customerMessage: input.latest_customer_message.preview, + customerName: input.ticket.requester_name ?? undefined, + knowledge: retrieval.hits, + workspaceConfig: options.workspaceConfig, + }); + const score = scoreTextSimilarity(expected.expected_reply_preview, draft.body_markdown); + const required = expected.required_terms.slice(0, 5); + const lowerDraft = draft.body_markdown.toLowerCase(); + const missingTerms = required.filter((term) => !lowerDraft.includes(term.toLowerCase())); + const assertions: EvalAssertion[] = [ + { + name: 'draft_non_empty', + passed: draft.body_markdown.trim().length >= 20, + message: 'Draft should produce a substantive answer.', + }, + { + name: 'semantic_overlap', + passed: score >= threshold, + score, + message: `Expected overlap >= ${threshold}.`, + }, + { + name: 'required_terms', + passed: missingTerms.length <= Math.max(1, Math.floor(required.length / 2)), + message: missingTerms.length ? `Missing terms: ${missingTerms.join(', ')}` : undefined, + }, + { + name: 'confidence_signal', + passed: draft.confidence >= 0.2 || draft.needs_human_review_reasons.length > 0, + score: draft.confidence, + message: 'Low confidence drafts must explicitly ask for review.', + }, + ]; + const passed = assertions.every((assertion) => assertion.passed); + return { + status: passed ? 'passed' : 'failed', + score, + assertions, + actual: { + body_markdown: draft.body_markdown, + subject: draft.subject, + confidence: draft.confidence, + cites_knowledge_ids: draft.cites_knowledge_ids, + retrieval_trace: retrieval.trace, + }, + }; +} + +export function runProcedureSpecEvals(spec: ProcedureSpec): { + status: 'passed' | 'failed'; + case_count: number; + passed_count: number; + failed_count: number; + results: ProcedureEvalReport[]; +} { + const evals = spec.evals ?? []; + const results = evals.map((evalCase) => { + const actual = simulateProcedure(spec, evalCase.input); + const assertions = evaluateProcedureExpectations(actual, evalCase.expect ?? {}); + return { + name: evalCase.name, + status: assertions.every((assertion) => assertion.passed) + ? ('passed' as const) + : ('failed' as const), + assertions, + actual, + }; + }); + const passed = results.filter((result) => result.status === 'passed').length; + const failed = results.length - passed; + return { + status: failed > 0 ? 'failed' : 'passed', + case_count: results.length, + passed_count: passed, + failed_count: failed, + results, + }; +} + +export function evaluateProcedureExpectations( + actual: unknown, + expect: Record, +): EvalAssertion[] { + const assertions: EvalAssertion[] = []; + if ('status' in expect) { + assertions.push({ + name: 'status', + passed: getPath(actual, 'status') === expect.status, + message: `Expected status ${String(expect.status)}, got ${String(getPath(actual, 'status'))}.`, + }); + } + if ('context' in expect && expect.context && typeof expect.context === 'object') { + for (const [path, expectedValue] of Object.entries(expect.context as Record)) { + const actualValue = getPath(actual, `context.${path}`); + assertions.push({ + name: `context.${path}`, + passed: JSON.stringify(actualValue) === JSON.stringify(expectedValue), + message: `Expected ${JSON.stringify(expectedValue)}, got ${JSON.stringify(actualValue)}.`, + }); + } + } + if ('steps' in expect && Array.isArray(expect.steps)) { + const stepIds = + (getPath(actual, 'steps') as Array<{ step_id?: string }> | undefined)?.map( + (step) => step.step_id, + ) ?? []; + assertions.push({ + name: 'steps', + passed: JSON.stringify(stepIds) === JSON.stringify(expect.steps), + message: `Expected steps ${JSON.stringify(expect.steps)}, got ${JSON.stringify(stepIds)}.`, + }); + } + if (assertions.length === 0) { + assertions.push({ + name: 'runs_without_failure', + passed: getPath(actual, 'status') !== 'failed', + }); + } + return assertions; +} + +export function scoreTextSimilarity(expected: string, actual: string): number { + const expectedTokens = tokenSet(expected); + const actualTokens = tokenSet(actual); + if (expectedTokens.size === 0 || actualTokens.size === 0) return 0; + let overlap = 0; + for (const token of expectedTokens) { + if (actualTokens.has(token)) overlap += 1; + } + return Number((overlap / expectedTokens.size).toFixed(4)); +} + +function tokenSet(value: string): Set { + const stop = new Set([ + 'about', + 'after', + 'again', + 'also', + 'because', + 'before', + 'could', + 'hello', + 'please', + 'thank', + 'thanks', + 'there', + 'these', + 'those', + 'would', + 'your', + ]); + return new Set( + (value.toLowerCase().match(/[a-z][a-z0-9_-]{3,}/g) ?? []).filter((token) => !stop.has(token)), + ); +} + +function getPath(value: unknown, path: string): unknown { + return path.split('.').reduce((current, segment) => { + if (!current || typeof current !== 'object') return undefined; + return (current as Record)[segment]; + }, value); +} + +function clampThreshold(value: number) { + if (!Number.isFinite(value)) return 0.35; + return Math.min(Math.max(value, 0.05), 0.95); +} diff --git a/src/evals/storage.ts b/src/evals/storage.ts new file mode 100644 index 0000000..fddec10 --- /dev/null +++ b/src/evals/storage.ts @@ -0,0 +1,263 @@ +import type { Env } from '../env'; +import { ids } from '../lib/ids'; +import type { + EvalCase, + EvalCaseSource, + EvalResult, + EvalResultStatus, + EvalRun, + EvalRunDetail, + EvalRunSource, + EvalRunStatus, +} from '../types/evals'; + +export async function upsertEvalCase( + env: Env, + input: { + workspaceId: string; + source: EvalCaseSource; + ticketId?: string | null; + procedureId?: string | null; + procedureVersionId?: string | null; + name: string; + inputJson: string; + expectedJson: string; + anonymizationJson: string; + sourceFingerprint: string; + }, +): Promise { + const now = Date.now(); + await env.DB.prepare( + `INSERT INTO eval_case ( + id, workspace_id, source, ticket_id, procedure_id, procedure_version_id, + name, status, input_json, expected_json, anonymization_json, source_fingerprint, + captured_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, 'active', ?, ?, ?, ?, ?, ?) + ON CONFLICT(workspace_id, source, source_fingerprint) DO UPDATE SET + ticket_id = excluded.ticket_id, + procedure_id = excluded.procedure_id, + procedure_version_id = excluded.procedure_version_id, + name = excluded.name, + status = 'active', + input_json = excluded.input_json, + expected_json = excluded.expected_json, + anonymization_json = excluded.anonymization_json, + updated_at = excluded.updated_at`, + ) + .bind( + ids.evalCase(), + input.workspaceId, + input.source, + input.ticketId ?? null, + input.procedureId ?? null, + input.procedureVersionId ?? null, + input.name, + input.inputJson, + input.expectedJson, + input.anonymizationJson, + input.sourceFingerprint, + now, + now, + ) + .run(); + const row = await env.DB.prepare( + `SELECT * FROM eval_case + WHERE workspace_id = ? AND source = ? AND source_fingerprint = ?`, + ) + .bind(input.workspaceId, input.source, input.sourceFingerprint) + .first(); + if (!row) throw new Error('eval_case_upsert_failed'); + return row; +} + +export async function listEvalCases( + env: Env, + workspaceId: string, + params: { status?: 'active' | 'archived'; limit?: number; caseIds?: string[] } = {}, +): Promise { + const limit = Math.min(Math.max(params.limit ?? 100, 1), 500); + const bindings: unknown[] = [workspaceId]; + const clauses = ['workspace_id = ?']; + if (params.status) { + clauses.push('status = ?'); + bindings.push(params.status); + } + if (params.caseIds?.length) { + clauses.push(`id IN (${params.caseIds.map(() => '?').join(',')})`); + bindings.push(...params.caseIds); + } + bindings.push(limit); + const rows = await env.DB.prepare( + `SELECT * FROM eval_case + WHERE ${clauses.join(' AND ')} + ORDER BY captured_at DESC + LIMIT ?`, + ) + .bind(...bindings) + .all(); + return rows.results ?? []; +} + +export async function listEvalRuns(env: Env, workspaceId: string, limit = 20): Promise { + const rows = await env.DB.prepare( + `SELECT * FROM eval_run + WHERE workspace_id = ? + ORDER BY created_at DESC + LIMIT ?`, + ) + .bind(workspaceId, Math.min(Math.max(limit, 1), 100)) + .all(); + return rows.results ?? []; +} + +export async function createEvalRun( + env: Env, + input: { + workspaceId: string; + source: EvalRunSource; + config: Record; + }, +): Promise { + const now = Date.now(); + const run: EvalRun = { + id: ids.evalRun(), + workspace_id: input.workspaceId, + source: input.source, + status: 'running', + case_count: 0, + passed_count: 0, + failed_count: 0, + regression_count: 0, + config_json: JSON.stringify(input.config), + started_at: now, + completed_at: null, + created_at: now, + }; + await env.DB.prepare( + `INSERT INTO eval_run ( + id, workspace_id, source, status, case_count, passed_count, failed_count, + regression_count, config_json, started_at, completed_at, created_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + ) + .bind( + run.id, + run.workspace_id, + run.source, + run.status, + run.case_count, + run.passed_count, + run.failed_count, + run.regression_count, + run.config_json, + run.started_at, + run.completed_at, + run.created_at, + ) + .run(); + return run; +} + +export async function insertEvalResult( + env: Env, + input: { + workspaceId: string; + runId: string; + caseId: string; + status: EvalResultStatus; + score?: number | null; + assertions: unknown[]; + actual: unknown; + error?: string | null; + }, +): Promise { + const result: EvalResult = { + id: ids.evalResult(), + workspace_id: input.workspaceId, + run_id: input.runId, + case_id: input.caseId, + status: input.status, + score: input.score ?? null, + assertions_json: JSON.stringify(input.assertions), + actual_json: JSON.stringify(input.actual), + error: input.error ?? null, + created_at: Date.now(), + }; + await env.DB.prepare( + `INSERT INTO eval_result ( + id, workspace_id, run_id, case_id, status, score, assertions_json, + actual_json, error, created_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + ) + .bind( + result.id, + result.workspace_id, + result.run_id, + result.case_id, + result.status, + result.score, + result.assertions_json, + result.actual_json, + result.error, + result.created_at, + ) + .run(); + return result; +} + +export async function completeEvalRun( + env: Env, + workspaceId: string, + runId: string, + counts: { + caseCount: number; + passedCount: number; + failedCount: number; + regressionCount: number; + }, +): Promise { + const status: EvalRunStatus = + counts.failedCount > 0 || counts.regressionCount > 0 ? 'failed' : 'passed'; + await env.DB.prepare( + `UPDATE eval_run + SET status = ?, case_count = ?, passed_count = ?, failed_count = ?, + regression_count = ?, completed_at = ? + WHERE id = ? AND workspace_id = ?`, + ) + .bind( + status, + counts.caseCount, + counts.passedCount, + counts.failedCount, + counts.regressionCount, + Date.now(), + runId, + workspaceId, + ) + .run(); + const row = await env.DB.prepare(`SELECT * FROM eval_run WHERE id = ? AND workspace_id = ?`) + .bind(runId, workspaceId) + .first(); + if (!row) throw new Error('eval_run_not_found'); + return row; +} + +export async function getEvalRunDetail( + env: Env, + workspaceId: string, + runId: string, +): Promise { + const run = await env.DB.prepare(`SELECT * FROM eval_run WHERE id = ? AND workspace_id = ?`) + .bind(runId, workspaceId) + .first(); + if (!run) return null; + const rows = await env.DB.prepare( + `SELECT r.*, c.name AS case_name, c.source AS case_source + FROM eval_result r + JOIN eval_case c ON c.id = r.case_id + WHERE r.run_id = ? AND r.workspace_id = ? + ORDER BY r.created_at ASC`, + ) + .bind(runId, workspaceId) + .all(); + return { run, results: rows.results ?? [] }; +} diff --git a/src/lib/ids.ts b/src/lib/ids.ts index 264375e..61d7ab2 100644 --- a/src/lib/ids.ts +++ b/src/lib/ids.ts @@ -32,4 +32,7 @@ export const ids = { mcpServer: () => id('mcp_srv'), mcpTool: () => id('mcp_tool'), mcpToolCall: () => id('mcp_call'), + evalCase: () => id('eval_case'), + evalRun: () => id('eval_run'), + evalResult: () => id('eval_result'), }; diff --git a/src/procedures/runner.ts b/src/procedures/runner.ts index c56993e..99f1233 100644 --- a/src/procedures/runner.ts +++ b/src/procedures/runner.ts @@ -17,6 +17,7 @@ import type { import type { SendThreadedReply } from '../types/supervisor'; import { makeSendThreadedReply } from '../agents/supervisor/replies'; import { workspaceConfig } from '../agents/supervisor/settings'; +import { captureResolvedTicketEvalCase } from '../evals/capture'; import { getRunBundle, getStepRunByIndex, recordStepRun, updateRun } from './storage'; import { deletePath, @@ -650,6 +651,11 @@ async function setTicketField( action: `procedure.ticket_${field}_set`, payload: { value }, }); + if (field === 'status' && ['resolved', 'closed'].includes(value)) { + await captureResolvedTicketEvalCase(env, workspaceId, ticketId).catch((err) => + console.warn('failed to capture procedure eval case', err), + ); + } } async function escalateTicket( diff --git a/src/types/evals.ts b/src/types/evals.ts new file mode 100644 index 0000000..a8ac5f0 --- /dev/null +++ b/src/types/evals.ts @@ -0,0 +1,111 @@ +export type EvalCaseSource = 'resolved_ticket' | 'procedure_spec' | 'synthetic' | 'api'; +export type EvalCaseStatus = 'active' | 'archived'; +export type EvalRunSource = 'api' | 'cli' | 'ci' | 'scheduled'; +export type EvalRunStatus = 'running' | 'passed' | 'failed'; +export type EvalResultStatus = 'passed' | 'failed' | 'skipped'; + +export interface EvalAnonymizationConfig { + redactEmails?: boolean; + redactPhones?: boolean; + redactRequesterName?: boolean; + requesterEmail?: string | null; + requesterName?: string | null; +} + +export interface EvalCase { + id: string; + workspace_id: string; + source: EvalCaseSource; + ticket_id: string | null; + procedure_id: string | null; + procedure_version_id: string | null; + name: string; + status: EvalCaseStatus; + input_json: string; + expected_json: string; + anonymization_json: string; + source_fingerprint: string; + captured_at: number; + updated_at: number; +} + +export interface EvalRun { + id: string; + workspace_id: string; + source: EvalRunSource; + status: EvalRunStatus; + case_count: number; + passed_count: number; + failed_count: number; + regression_count: number; + config_json: string; + started_at: number; + completed_at: number | null; + created_at: number; +} + +export interface EvalResult { + id: string; + workspace_id: string; + run_id: string; + case_id: string; + status: EvalResultStatus; + score: number | null; + assertions_json: string; + actual_json: string; + error: string | null; + created_at: number; +} + +export interface EvalRunDetail { + run: EvalRun; + results: Array; +} + +export interface ResolvedTicketEvalInput { + ticket: { + id: string; + subject: string; + requester_email: string; + requester_name: string | null; + status: string; + priority: string; + category: string | null; + }; + transcript: Array<{ + id: string; + direction: string; + from_address: string | null; + to_address: string | null; + subject: string | null; + preview: string; + sent_at: number; + }>; + latest_customer_message: { + subject: string; + preview: string; + }; +} + +export interface ResolvedTicketEvalExpected { + expected_status: string; + expected_priority: string; + expected_category: string | null; + expected_reply_preview: string; + required_terms: string[]; + outcome_kinds: string[]; +} + +export interface EvalAssertion { + name: string; + passed: boolean; + message?: string; + score?: number; +} + +export interface ProcedureEvalReport { + name: string; + status: 'passed' | 'failed'; + assertions: EvalAssertion[]; + actual: unknown; +} diff --git a/src/ui/api.ts b/src/ui/api.ts index ce0aef9..2b1bd0a 100644 --- a/src/ui/api.ts +++ b/src/ui/api.ts @@ -12,6 +12,7 @@ import type { } from '../types/procedure'; import type { McpServerListItem, McpTool, McpToolCall, McpToolGuardrail } from '../types/mcp'; import type { AuthMe } from '../types/workspace'; +import type { EvalCase, EvalRun, EvalRunDetail } from '../types/evals'; import { api, uploadFile, uploadKnowledgePdf } from './api-core'; import { workspaceApi } from './api-workspaces'; @@ -24,6 +25,8 @@ export type AnswerInspectionTrace = AgenticRetrievalTrace; export type ProcedureListEntry = ProcedureListItem; export type McpServerEntry = McpServerListItem; export type McpToolEntry = McpTool; +export type EvalCaseEntry = EvalCase; +export type EvalRunEntry = EvalRun; export const API = { setupStatus: () => api<{ completed: boolean }>('/setup/status'), @@ -232,6 +235,21 @@ export const API = { }), listMcpToolCalls: (ticketId: string) => api<{ toolCalls: McpToolCall[] }>(`/api/mcp/tool-calls?ticket_id=${ticketId}`), + listEvalCases: () => api<{ cases: EvalCaseEntry[] }>('/api/evals/cases'), + listEvalRuns: () => api<{ runs: EvalRunEntry[] }>('/api/evals/runs'), + captureResolvedEvalCases: (limit = 50) => + api<{ ok: boolean; captured: number; skipped: number; failed: number; cases: string[] }>( + '/api/evals/cases/capture-resolved', + { + method: 'POST', + body: JSON.stringify({ limit }), + }, + ), + runEvalSuite: (body: { limit?: number; threshold?: number } = {}) => + api('/api/evals/runs', { + method: 'POST', + body: JSON.stringify(body), + }), importResolvedTicketsKnowledge: (limit = 50) => api<{ ok: boolean; imported: number; skipped: number; failed: number }>( '/api/knowledge/import-resolved-tickets', diff --git a/src/ui/views/EvalsSection.tsx b/src/ui/views/EvalsSection.tsx new file mode 100644 index 0000000..4242d6b --- /dev/null +++ b/src/ui/views/EvalsSection.tsx @@ -0,0 +1,125 @@ +import { useEffect, useState } from 'react'; +import { API, type EvalCaseEntry, type EvalRunEntry } from '../api'; + +interface EvalsSectionProps { + onSaved: (message?: string) => void; +} + +export function EvalsSection({ onSaved }: EvalsSectionProps) { + const [cases, setCases] = useState([]); + const [runs, setRuns] = useState([]); + const [limit, setLimit] = useState(50); + const [threshold, setThreshold] = useState(0.35); + const [busy, setBusy] = useState(''); + const [error, setError] = useState(''); + + async function load() { + const [caseRes, runRes] = await Promise.all([API.listEvalCases(), API.listEvalRuns()]); + setCases(caseRes.cases ?? []); + setRuns(runRes.runs ?? []); + } + + useEffect(() => { + load().catch((err) => setError(err.message || 'Failed to load evals')); + }, []); + + async function capture() { + setError(''); + setBusy('capture'); + try { + const result = await API.captureResolvedEvalCases(limit); + await load(); + onSaved(`Captured ${result.captured} eval cases`); + } catch (err: any) { + setError(err.message || 'Capture failed'); + } finally { + setBusy(''); + } + } + + async function run() { + setError(''); + setBusy('run'); + try { + const detail = await API.runEvalSuite({ limit, threshold }); + await load(); + onSaved(`Eval run ${detail.run.status}`); + } catch (err: any) { + setError(err.message || 'Eval run failed'); + } finally { + setBusy(''); + } + } + + return ( + <> +

Evals

+
+
+
+ + setLimit(Number(e.target.value))} + /> +
+
+ + setThreshold(Number(e.target.value))} + /> +
+ + +
+ + {error && ( +
+ {error} +
+ )} + +
+
+
+
Historical cases
+
+ {cases.length} active replay cases from resolved conversations +
+
+ {cases.length} +
+ {runs.slice(0, 5).map((runItem) => ( +
+
+
+ {runItem.status} · {runItem.passed_count}/{runItem.case_count} passed +
+
+ {new Date(runItem.created_at).toLocaleString()} · {runItem.regression_count}{' '} + regressions +
+
+ + {runItem.source} + +
+ ))} + {runs.length === 0 &&
No eval runs yet.
} +
+
+ + ); +} diff --git a/src/ui/views/Settings.tsx b/src/ui/views/Settings.tsx index a59de42..1f15cbb 100644 --- a/src/ui/views/Settings.tsx +++ b/src/ui/views/Settings.tsx @@ -8,6 +8,7 @@ import { WorkspacePlatformSection } from './WorkspacePlatformSection'; import { ModelSettingsSection } from './ModelSettingsSection'; import { McpActionsSection } from './McpActionsSection'; import { ProceduresSection } from './ProceduresSection'; +import { EvalsSection } from './EvalsSection'; const PROVIDERS = ['openai', 'anthropic', 'google-ai-studio', 'grok', 'openrouter']; @@ -246,6 +247,8 @@ export function SettingsView() { + + diff --git a/tests/evals.test.ts b/tests/evals.test.ts new file mode 100644 index 0000000..a115208 --- /dev/null +++ b/tests/evals.test.ts @@ -0,0 +1,227 @@ +import { describe, expect, it, vi } from 'vitest'; +import { apiApp } from '../src/api/routes'; +import { anonymizeValue } from '../src/evals/anonymize'; +import { captureResolvedTicketEvalCase } from '../src/evals/capture'; +import { runEvalSuite, runProcedureSpecEvals } from '../src/evals/replay'; +import { + addMember, + createWorkspaceTestDb, + login, + seedMailbox, + seedUser, + seedWorkspace, +} from './helpers/workspace-db'; + +vi.mock('agents', () => ({ + getAgentByName: () => ({}), + Agent: class {}, + callable: () => () => undefined, +})); + +function seedResolvedConversation(db: ReturnType['db']) { + seedMailbox(db, 'ws_a', 'mb_a', 'support@example.com'); + db.prepare( + `INSERT INTO ticket ( + id, workspace_id, mailbox_id, subject, status, priority, category, last_message_at, + requester_email, requester_name, thread_token, created_at, updated_at + ) VALUES ( + 'tkt_eval', 'ws_a', 'mb_a', 'Refund for order 123', 'resolved', 'normal', 'billing', 20, + 'jane.customer@example.com', 'Jane Customer', 'tok_eval', 1, 20 + )`, + ).run(); + db.prepare( + `INSERT INTO message_index ( + id, ticket_id, workspace_id, direction, from_address, to_address, subject, + preview, sent_at, created_at + ) VALUES (?, 'tkt_eval', 'ws_a', ?, ?, ?, ?, ?, ?, ?)`, + ).run( + 'msg_in', + 'inbound', + 'jane.customer@example.com', + 'support@example.com', + 'Refund for order 123', + 'Hi, I am Jane Customer. Please refund order 123. My phone is +1 415 555 1212.', + 10, + 10, + ); + db.prepare( + `INSERT INTO message_index ( + id, ticket_id, workspace_id, direction, from_address, to_address, subject, + preview, sent_at, created_at + ) VALUES (?, 'tkt_eval', 'ws_a', ?, ?, ?, ?, ?, ?, ?)`, + ).run( + 'msg_out', + 'outbound', + 'support@example.com', + 'jane.customer@example.com', + 'Re: Refund for order 123', + 'Hi Jane Customer, your refund for order 123 has been approved and should arrive within five business days.', + 20, + 20, + ); + db.prepare( + `INSERT INTO ticket_outcome_event ( + id, workspace_id, ticket_id, kind, source, payload_json, created_at + ) VALUES ('out_eval', 'ws_a', 'tkt_eval', 'resolved_autonomously', 'agent', '{}', 20)`, + ).run(); +} + +describe('eval anonymization', () => { + it('redacts stable customer identifiers without storing originals', () => { + const result = anonymizeValue( + { + text: 'Jane Customer wrote from jane.customer@example.com and +1 415 555 1212.', + }, + { + requesterEmail: 'jane.customer@example.com', + requesterName: 'Jane Customer', + }, + ); + + expect(JSON.stringify(result.value)).not.toContain('jane.customer@example.com'); + expect(JSON.stringify(result.value)).not.toContain('Jane Customer'); + expect(JSON.stringify(result.value)).toContain('customer_1@example.test'); + expect(JSON.stringify(result.value)).toContain('[phone_1]'); + expect(JSON.stringify(result.metadata)).not.toContain('jane.customer@example.com'); + }); +}); + +describe('historical eval capture and replay', () => { + it('captures resolved tickets as anonymized replay cases', async () => { + const { db, env } = createWorkspaceTestDb(); + seedWorkspace(db, 'ws_a', 'Alpha'); + seedResolvedConversation(db); + + const result = await captureResolvedTicketEvalCase(env, 'ws_a', 'tkt_eval'); + expect(result.caseId).toBeTruthy(); + const row = db.prepare(`SELECT * FROM eval_case WHERE id = ?`).get(result.caseId!) as any; + + expect(result.captured).toBe(true); + expect(row.source).toBe('resolved_ticket'); + expect(row.input_json).not.toContain('jane.customer@example.com'); + expect(row.input_json).not.toContain('Jane Customer'); + expect(row.input_json).toContain('customer_1@example.test'); + expect(row.expected_json).toContain('refund'); + }); + + it('captures autonomous resolved outcomes even before ticket closure', async () => { + const { db, env } = createWorkspaceTestDb(); + seedWorkspace(db, 'ws_a', 'Alpha'); + seedResolvedConversation(db); + db.prepare(`UPDATE ticket SET status = 'pending' WHERE id = 'tkt_eval'`).run(); + + const result = await captureResolvedTicketEvalCase(env, 'ws_a', 'tkt_eval'); + + expect(result.captured).toBe(true); + }); + + it('runs historical evals and marks regressions', async () => { + const { db, env } = createWorkspaceTestDb(); + seedWorkspace(db, 'ws_a', 'Alpha'); + seedResolvedConversation(db); + await captureResolvedTicketEvalCase(env, 'ws_a', 'tkt_eval'); + + const passing = await runEvalSuite(env, 'ws_a', { + retrievalRunner: async () => ({ + hits: [], + trace: { + plan: { originalQuery: 'refund', scope: 'all', subqueries: ['refund'], maxHops: 1 }, + hops: [], + finalAnswerable: false, + stopReason: 'no_hits', + startedAt: 1, + durationMs: 1, + }, + }), + draftRunner: async () => ({ + subject: 'Re: Refund for order 123', + body_markdown: + 'Your refund for order 123 has been approved and should arrive within five business days.', + tone: 'friendly', + cites_knowledge_ids: [], + confidence: 0.9, + needs_human_review_reasons: [], + }), + }); + const failing = await runEvalSuite(env, 'ws_a', { + retrievalRunner: async () => ({ + hits: [], + trace: { + plan: { originalQuery: 'refund', scope: 'all', subqueries: ['refund'], maxHops: 1 }, + hops: [], + finalAnswerable: false, + stopReason: 'no_hits', + startedAt: 1, + durationMs: 1, + }, + }), + draftRunner: async () => ({ + subject: 'Re: Refund for order 123', + body_markdown: 'We cannot help with this.', + tone: 'friendly', + cites_knowledge_ids: [], + confidence: 0.9, + needs_human_review_reasons: [], + }), + }); + + expect(passing.run.status).toBe('passed'); + expect(failing.run.status).toBe('failed'); + expect(failing.run.regression_count).toBe(1); + }); +}); + +describe('procedure spec evals', () => { + it('checks inline procedure expectations', () => { + const report = runProcedureSpecEvals({ + slug: 'refund-intake', + name: 'Refund intake', + version: '1.0.0', + trigger: { type: 'manual' }, + steps: [ + { id: 'priority', type: 'set_ticket_field', field: 'priority', value: 'high' }, + { id: 'wait', type: 'ask_customer', message: 'Please send the order id.' }, + ], + evals: [ + { + name: 'waits after priority', + input: { ticket: { subject: 'Refund' } }, + expect: { + status: 'waiting', + context: { 'ticket.priority': 'high' }, + steps: ['priority', 'wait'], + }, + }, + ], + }); + + expect(report.status).toBe('passed'); + expect(report.passed_count).toBe(1); + }); +}); + +describe('eval API', () => { + it('lets workspace owners capture resolved conversation evals', async () => { + const { db, env } = createWorkspaceTestDb(); + seedWorkspace(db, 'ws_a', 'Alpha'); + await seedUser(db, 'usr_a', 'owner@example.com'); + addMember(db, 'ws_a', 'usr_a', 'owner'); + seedResolvedConversation(db); + const cookie = await login(env, 'owner@example.com'); + + const res = await apiApp.request( + '/evals/cases/capture-resolved', + { + method: 'POST', + headers: { cookie, 'content-type': 'application/json' }, + body: JSON.stringify({ limit: 5 }), + }, + env, + ); + const body = await res.json(); + + expect(res.status).toBe(200); + expect(body.captured).toBe(1); + expect(db.prepare(`SELECT COUNT(*) AS n FROM eval_case`).get()).toEqual({ n: 1 }); + }); +}); diff --git a/tests/helpers/workspace-db.ts b/tests/helpers/workspace-db.ts index b523b1c..9b0c783 100644 --- a/tests/helpers/workspace-db.ts +++ b/tests/helpers/workspace-db.ts @@ -301,6 +301,49 @@ export function createWorkspaceTestDb() { CREATE UNIQUE INDEX idx_mcp_tool_call_procedure_step ON mcp_tool_call(workspace_id, procedure_run_id, procedure_step_index) WHERE procedure_run_id IS NOT NULL AND procedure_step_index IS NOT NULL; + CREATE TABLE eval_case ( + id TEXT PRIMARY KEY, + workspace_id TEXT NOT NULL, + source TEXT NOT NULL, + ticket_id TEXT, + procedure_id TEXT, + procedure_version_id TEXT, + name TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'active', + input_json TEXT NOT NULL, + expected_json TEXT NOT NULL, + anonymization_json TEXT NOT NULL DEFAULT '{}', + source_fingerprint TEXT NOT NULL, + captured_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + UNIQUE(workspace_id, source, source_fingerprint) + ); + CREATE TABLE eval_run ( + id TEXT PRIMARY KEY, + workspace_id TEXT NOT NULL, + source TEXT NOT NULL DEFAULT 'api', + status TEXT NOT NULL, + case_count INTEGER NOT NULL DEFAULT 0, + passed_count INTEGER NOT NULL DEFAULT 0, + failed_count INTEGER NOT NULL DEFAULT 0, + regression_count INTEGER NOT NULL DEFAULT 0, + config_json TEXT NOT NULL DEFAULT '{}', + started_at INTEGER NOT NULL, + completed_at INTEGER, + created_at INTEGER NOT NULL + ); + CREATE TABLE eval_result ( + id TEXT PRIMARY KEY, + workspace_id TEXT NOT NULL, + run_id TEXT NOT NULL, + case_id TEXT NOT NULL, + status TEXT NOT NULL, + score REAL, + assertions_json TEXT NOT NULL DEFAULT '[]', + actual_json TEXT NOT NULL DEFAULT '{}', + error TEXT, + created_at INTEGER NOT NULL + ); `); const envDb = { From 0375b087c86bf14b65b6be7d2bd3e2a1763cf43c Mon Sep 17 00:00:00 2001 From: tomicodesdev <32748319+tomicodesdev@users.noreply.github.com> Date: Mon, 18 May 2026 02:55:48 +0100 Subject: [PATCH 2/2] Harden Phase 6 eval gates --- docs/architecture.md | 6 ++- docs/operations.md | 10 +++-- docs/roadmap.md | 8 ++-- docs/security.md | 2 +- scripts/ranse.ts | 3 +- src/api/evals.ts | 22 +++++++++- src/evals/anonymize.ts | 28 ++++++++++++ src/evals/capture.ts | 47 ++++++++++++++++++-- src/evals/replay.ts | 82 +++++++++++++++++++++++++++++++---- src/evals/storage.ts | 36 +++++++++++++++ src/ui/api.ts | 28 +++++++++--- src/ui/views/EvalsSection.tsx | 43 +++++++++++++++++- tests/evals.test.ts | 82 ++++++++++++++++++++++++++++++++++- 13 files changed, 366 insertions(+), 31 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 31718b0..3ee61da 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -104,11 +104,12 @@ MCP server secrets are stored in `UserSecretsStore` under `mcp:`. D1 s ## Eval flow ``` -Operator resolves ticket - ├─ setTicketStatus(resolved|closed) +Ticket is resolved + ├─ setTicketStatus(resolved|closed), autonomous outcome, or procedure status step ├─ captureResolvedTicketEvalCase │ ├─ load ticket + inbound/outbound transcript │ ├─ anonymize email, phone, and requester-name fields + │ ├─ reject capture if residual PII remains │ └─ upsert eval_case(source = resolved_ticket) └─ audit eval.case_captured @@ -118,6 +119,7 @@ ranse eval / Settings -> Evals │ ├─ agenticSearchKnowledge with current retrieval prompts/config │ ├─ runDraft with current draft prompt/model config │ ├─ score reply overlap, required terms, confidence signals + │ ├─ compare against latest prior baseline result │ └─ insert eval_result └─ mark eval_run passed/failed ``` diff --git a/docs/operations.md b/docs/operations.md index a69cc01..c05024b 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -39,7 +39,9 @@ Operational checks: ## Historical evals -Resolved tickets are the regression suite. When an operator marks a ticket `resolved` or `closed`, Ranse captures the inbound/outbound transcript into an anonymized `eval_case` if the conversation has both a customer message and a support reply. Operators can also backfill recent cases from **Settings → Evals**. +Resolved tickets are the regression suite. When an operator marks a ticket `resolved` or `closed`, an autonomous reply records a resolved outcome, or a procedure resolves a ticket, Ranse captures the inbound/outbound transcript into an anonymized `eval_case` if the conversation has both a customer message and a support reply. Operators can also backfill recent cases from **Settings → Evals**. + +Capture fails closed if the anonymized payload still contains residual non-placeholder email, phone, or requester-name data. No eval case is written until the redaction rules are safe for that conversation. CLI workflow: @@ -51,10 +53,12 @@ bun scripts/ranse.ts eval procedures/refund-intake.yaml bun scripts/ranse.ts eval capture-resolved --app-url "$RANSE_APP_URL" --cookie "$RANSE_COOKIE" --limit 100 # Replay active historical cases through current retrieval + drafting -bun scripts/ranse.ts eval --app-url "$RANSE_APP_URL" --cookie "$RANSE_COOKIE" --threshold 0.35 --ci +bun scripts/ranse.ts eval --app-url "$RANSE_APP_URL" --cookie "$RANSE_COOKIE" --threshold 0.35 --score-drop 0.15 --ci ``` -Eval runs write `eval_run` and `eval_result` rows with assertion details. A run fails when any active case regresses. The bundled GitHub Actions workflow always runs procedure evals for relevant PRs; set `RANSE_APP_URL` and `RANSE_COOKIE` repository secrets to make hosted historical replay part of the gate. +Eval runs write `eval_run` and `eval_result` rows with assertion details. A run fails when any active case fails or regresses. `regression_count` is reserved for cases with a prior baseline that got worse: previous pass → current fail, or a score drop larger than the configured threshold. Archive noisy cases from **Settings → Evals** instead of deleting them. + +The bundled GitHub Actions workflow always runs procedure evals for relevant PRs; set `RANSE_APP_URL` and `RANSE_COOKIE` repository secrets to make hosted historical replay part of the gate. ## Escalations diff --git a/docs/roadmap.md b/docs/roadmap.md index fe48c47..b2bf1be 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -172,9 +172,11 @@ The `customer_data` search scope still fails closed with an explicit trace; proc *Principle 5* - Resolved or closed conversations are captured into `eval_case` with transcript, latest customer message, expected reply preview, outcome kinds, and deterministic source fingerprints. -- Capture runs automatically when an operator marks a ticket `resolved` or `closed`, and can be backfilled from **Settings → Evals** or `ranse eval capture-resolved`. -- PII anonymization is applied before persistence, with configurable email, phone, and requester-name redaction rules. -- Hosted eval runs replay active historical cases through current retrieval + draft logic, score overlap and required terms, and store per-case assertions in `eval_result`. +- Capture runs automatically when an operator marks a ticket `resolved` or `closed`, when autonomous resolution succeeds, and when a procedure resolves a ticket. Backfill uses both ticket status and resolved outcome events. +- PII anonymization is applied before persistence, with configurable email, phone, and requester-name redaction rules. Residual PII detection fails closed instead of storing a case when redaction leaves sensitive data behind. +- Hosted eval runs replay active historical cases through current retrieval + draft logic, score overlap and required terms, compare against the previous baseline result, and store per-case assertions in `eval_result`. +- Regression gates distinguish first-run failures from true regressions: a previous pass that now fails, or a score drop beyond the configured threshold, increments `regression_count`. +- Operators can archive noisy historical cases from **Settings → Evals** without deleting the audit trail. - `ranse eval ` runs inline procedure evals from the YAML/JSON/TS spec. Edge, refusal, escalation, and wait/resume expectations live next to the procedure. - `.github/workflows/evals.yml` gates PRs touching procedure, prompt/model, supervisor, or eval code on local procedure evals; with `RANSE_APP_URL` and `RANSE_COOKIE` secrets, the same workflow also gates on hosted historical replay. - Historical replay is the primary signal; synthetic-conversation generation remains a future complement, not a substitute. diff --git a/docs/security.md b/docs/security.md index d826dd6..dfe9fdc 100644 --- a/docs/security.md +++ b/docs/security.md @@ -55,7 +55,7 @@ Default policy: AI may classify, summarize, search, and draft. It **cannot** sen ## Eval data privacy -Historical eval cases are stored in the workspace's own D1 database and never leave the customer's Cloudflare account unless the operator exports or queries them. Before persistence, resolved-ticket capture redacts email addresses, phone numbers, and the requester's display name by default. The stored `anonymization_json` records rules and counts only; it does not store the original PII mapping. +Historical eval cases are stored in the workspace's own D1 database and never leave the customer's Cloudflare account unless the operator exports or queries them. Before persistence, resolved-ticket capture redacts email addresses, phone numbers, and the requester's display name by default. Capture then scans the anonymized payload for residual non-placeholder email, phone, or requester-name data and skips the case if any remains. The stored `anonymization_json` records rules and counts only; it does not store the original PII mapping. Eval access is restricted to `owner` and `admin` workspace roles because even anonymized conversations can contain commercially sensitive support context. Hosted replay uses the same provider-key and model-routing path as normal drafting, so self-hosters keep the same BYOK and Cloudflare tenant boundaries. diff --git a/scripts/ranse.ts b/scripts/ranse.ts index 004149f..733f8bd 100644 --- a/scripts/ranse.ts +++ b/scripts/ranse.ts @@ -74,6 +74,7 @@ async function main() { body: JSON.stringify({ limit: numberFlag(args, '--limit'), threshold: numberFlag(args, '--threshold'), + score_drop_threshold: numberFlag(args, '--score-drop'), source: args.includes('--ci') ? 'ci' : 'cli', }), }); @@ -87,7 +88,7 @@ async function main() { ranse simulate [--input input.json] ranse publish --app-url --cookie ranse eval - ranse eval --app-url --cookie [--limit n] [--threshold n] [--ci] + ranse eval --app-url --cookie [--limit n] [--threshold n] [--score-drop n] [--ci] ranse eval capture-resolved --app-url --cookie [--limit n]`); } diff --git a/src/api/evals.ts b/src/api/evals.ts index 2309c59..2dbd6ca 100644 --- a/src/api/evals.ts +++ b/src/api/evals.ts @@ -3,7 +3,12 @@ import { z } from 'zod'; import { workspaceConfig } from '../agents/supervisor/settings'; import { captureResolvedTicketEvalCases } from '../evals/capture'; import { runEvalSuite } from '../evals/replay'; -import { getEvalRunDetail, listEvalCases, listEvalRuns } from '../evals/storage'; +import { + getEvalRunDetail, + listEvalCases, + listEvalRuns, + updateEvalCaseStatus, +} from '../evals/storage'; import { apiError } from '../lib/errors'; import { type Ctx, OWNER_OR_ADMIN, requireWorkspaceRole } from './context'; @@ -31,6 +36,19 @@ export function registerEvalRoutes(apiApp: Hono) { return c.json({ runs }); }); + apiApp.patch('/evals/cases/:id', requireWorkspaceRole(OWNER_OR_ADMIN), async (c) => { + const s = c.get('session'); + const body = z.object({ status: z.enum(['active', 'archived']) }).parse(await c.req.json()); + const evalCase = await updateEvalCaseStatus( + c.env, + s.workspaceId, + c.req.param('id'), + body.status, + ); + if (!evalCase) return apiError(c, 'not_found', 'That eval case does not exist.'); + return c.json({ case: evalCase }); + }); + apiApp.get('/evals/runs/:id', requireWorkspaceRole(OWNER_OR_ADMIN), async (c) => { const s = c.get('session'); const detail = await getEvalRunDetail(c.env, s.workspaceId, c.req.param('id')); @@ -61,6 +79,7 @@ export function registerEvalRoutes(apiApp: Hono) { limit: z.number().int().min(1).max(500).optional(), case_ids: z.array(z.string()).max(500).optional(), threshold: z.number().min(0.05).max(0.95).optional(), + score_drop_threshold: z.number().min(0.01).max(0.75).optional(), source: z.enum(['api', 'cli', 'ci', 'scheduled']).optional(), }) .parse(await c.req.json().catch(() => ({}))); @@ -68,6 +87,7 @@ export function registerEvalRoutes(apiApp: Hono) { limit: body.limit, caseIds: body.case_ids, threshold: body.threshold, + scoreDropThreshold: body.score_drop_threshold, source: body.source ?? 'api', workspaceConfig: await workspaceConfig(c.env, s.workspaceId), }); diff --git a/src/evals/anonymize.ts b/src/evals/anonymize.ts index 952b5bd..284cb0c 100644 --- a/src/evals/anonymize.ts +++ b/src/evals/anonymize.ts @@ -19,6 +19,11 @@ export interface AnonymizationMetadata { }; } +export interface ResidualPiiFinding { + kind: 'email' | 'phone' | 'requester_name'; + value: string; +} + const EMAIL_RE = /\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/gi; const PHONE_RE = /(?( }; } +export function detectResidualPii(value: unknown): ResidualPiiFinding[] { + const findings = new Map(); + for (const text of collectStrings(value)) { + for (const email of text.match(EMAIL_RE) ?? []) { + if (email.toLowerCase().endsWith('@example.test')) continue; + findings.set(`email:${email.toLowerCase()}`, { kind: 'email', value: email }); + } + for (const phone of text.match(PHONE_RE) ?? []) { + const digits = phone.replace(/\D/g, ''); + if (digits.length < 8) continue; + findings.set(`phone:${digits}`, { kind: 'phone', value: phone }); + } + } + return [...findings.values()]; +} + +function collectStrings(value: unknown): string[] { + if (typeof value === 'string') return [value]; + if (Array.isArray(value)) return value.flatMap(collectStrings); + if (!value || typeof value !== 'object') return []; + return Object.values(value).flatMap(collectStrings); +} + function anonymizeAny( value: unknown, config: ReturnType, diff --git a/src/evals/capture.ts b/src/evals/capture.ts index ed98a49..543be36 100644 --- a/src/evals/capture.ts +++ b/src/evals/capture.ts @@ -7,7 +7,7 @@ import type { ResolvedTicketEvalExpected, ResolvedTicketEvalInput, } from '../types/evals'; -import { anonymizeValue } from './anonymize'; +import { anonymizeValue, detectResidualPii, normalizeAnonymizationConfig } from './anonymize'; import { upsertEvalCase } from './storage'; interface TicketRow { @@ -131,6 +131,18 @@ export async function captureResolvedTicketEvalCase( }; const anonymizedInput = anonymizeValue(input, anonymization); const anonymizedExpected = anonymizeValue(expected, anonymization); + const piiCheck = detectAnonymizationLeaks( + { input: anonymizedInput.value, expected: anonymizedExpected.value }, + anonymization, + ); + if (piiCheck.length > 0) { + console.warn('skipping eval capture because anonymization left residual pii', { + workspaceId, + ticketId, + findings: piiCheck.map((finding) => finding.kind), + }); + return { captured: false, reason: 'anonymization_residual_pii' }; + } const fingerprint = await sha256Hex( JSON.stringify({ ticketId, @@ -174,9 +186,16 @@ export async function captureResolvedTicketEvalCases( ): Promise<{ captured: number; skipped: number; failed: number; cases: string[] }> { const limit = Math.min(Math.max(options.limit ?? 50, 1), 200); const rows = await env.DB.prepare( - `SELECT id FROM ticket - WHERE workspace_id = ? AND status IN ('resolved','closed') - ORDER BY updated_at DESC + `SELECT DISTINCT t.id, t.updated_at + FROM ticket t + LEFT JOIN ticket_outcome_event o + ON o.workspace_id = t.workspace_id AND o.ticket_id = t.id + WHERE t.workspace_id = ? + AND ( + t.status IN ('resolved','closed') + OR o.kind IN ('resolved_autonomously','resolved_via_procedure') + ) + ORDER BY t.updated_at DESC LIMIT ?`, ) .bind(workspaceId, limit) @@ -247,3 +266,23 @@ export function extractRequiredTerms(body: string, limit = 8): string[] { .slice(0, limit) .map(([word]) => word); } + +function detectAnonymizationLeaks( + value: unknown, + config: EvalAnonymizationConfig, +): ReturnType { + const normalized = normalizeAnonymizationConfig(config); + const findings = detectResidualPii(value).filter((finding) => { + if (finding.kind === 'email') return normalized.redactEmails; + if (finding.kind === 'phone') return normalized.redactPhones; + return true; + }); + const text = JSON.stringify(value).toLowerCase(); + const requesterName = normalized.requesterName?.trim().toLowerCase(); + if (normalized.redactRequesterName && requesterName && requesterName.length >= 3) { + if (text.includes(requesterName)) { + findings.push({ kind: 'requester_name', value: 'requester_name' }); + } + } + return findings; +} diff --git a/src/evals/replay.ts b/src/evals/replay.ts index d4c62c8..7e411f9 100644 --- a/src/evals/replay.ts +++ b/src/evals/replay.ts @@ -11,13 +11,20 @@ import type { } from '../types/evals'; import type { AgenticKnowledgeResult, KnowledgeHit } from '../types/knowledge'; import type { ProcedureSpec } from '../types/procedure'; -import { completeEvalRun, createEvalRun, insertEvalResult, listEvalCases } from './storage'; +import { + completeEvalRun, + createEvalRun, + getLatestEvalResultForCase, + insertEvalResult, + listEvalCases, +} from './storage'; export interface RunEvalSuiteOptions { source?: 'api' | 'cli' | 'ci' | 'scheduled'; limit?: number; caseIds?: string[]; threshold?: number; + scoreDropThreshold?: number; workspaceConfig?: Partial; retrievalRunner?: (input: ResolvedTicketEvalInput) => Promise; draftRunner?: (input: ResolvedTicketEvalInput, knowledge: KnowledgeHit[]) => Promise; @@ -29,10 +36,17 @@ export async function runEvalSuite( options: RunEvalSuiteOptions = {}, ) { const threshold = clampThreshold(options.threshold ?? 0.35); + const scoreDropThreshold = clampScoreDrop(options.scoreDropThreshold ?? 0.15); const run = await createEvalRun(env, { workspaceId, source: options.source ?? 'api', - config: { threshold, limit: options.limit ?? null, caseIds: options.caseIds ?? null }, + config: { + threshold, + scoreDropThreshold, + limit: options.limit ?? null, + caseIds: options.caseIds ?? null, + workspaceConfig: options.workspaceConfig ?? null, + }, }); const cases = await listEvalCases(env, workspaceId, { status: 'active', @@ -70,24 +84,25 @@ export async function runEvalSuite( retrievalRunner: options.retrievalRunner, draftRunner: options.draftRunner, }); + const previous = await getLatestEvalResultForCase(env, workspaceId, evalCase.id); + const baseline = compareAgainstBaseline(previous, result, scoreDropThreshold); if (result.status === 'passed') passedCount += 1; - else { - failedCount += 1; - regressionCount += 1; - } + else failedCount += 1; + if (baseline.regressed) regressionCount += 1; await insertEvalResult(env, { workspaceId, runId: run.id, caseId: evalCase.id, status: result.status, score: result.score, - assertions: result.assertions, - actual: result.actual, + assertions: [...result.assertions, baseline.assertion], + actual: { ...result.actual, baseline: baseline.actual }, error: result.error, }); } catch (err) { failedCount += 1; - regressionCount += 1; + const previous = await getLatestEvalResultForCase(env, workspaceId, evalCase.id); + if (previous?.status === 'passed') regressionCount += 1; await insertEvalResult(env, { workspaceId, runId: run.id, @@ -312,3 +327,52 @@ function clampThreshold(value: number) { if (!Number.isFinite(value)) return 0.35; return Math.min(Math.max(value, 0.05), 0.95); } + +function clampScoreDrop(value: number) { + if (!Number.isFinite(value)) return 0.15; + return Math.min(Math.max(value, 0.01), 0.75); +} + +function compareAgainstBaseline( + previous: { status: string; score: number | null } | null, + current: { status: 'passed' | 'failed'; score: number }, + scoreDropThreshold: number, +): { + regressed: boolean; + assertion: EvalAssertion; + actual: Record; +} { + if (!previous) { + return { + regressed: false, + assertion: { + name: 'baseline_regression', + passed: true, + message: 'No prior baseline exists for this case.', + }, + actual: { previous_status: null, previous_score: null, score_delta: null }, + }; + } + const scoreDelta = + previous.score === null ? null : Number((current.score - previous.score).toFixed(4)); + const regressed = + (previous.status === 'passed' && current.status === 'failed') || + (scoreDelta !== null && scoreDelta <= -scoreDropThreshold); + return { + regressed, + assertion: { + name: 'baseline_regression', + passed: !regressed, + score: scoreDelta ?? undefined, + message: regressed + ? `Regressed from previous ${previous.status} baseline; score delta ${scoreDelta ?? 'n/a'}.` + : `No regression from previous ${previous.status} baseline.`, + }, + actual: { + previous_status: previous.status, + previous_score: previous.score, + score_delta: scoreDelta, + score_drop_threshold: scoreDropThreshold, + }, + }; +} diff --git a/src/evals/storage.ts b/src/evals/storage.ts index fddec10..93418ba 100644 --- a/src/evals/storage.ts +++ b/src/evals/storage.ts @@ -98,6 +98,23 @@ export async function listEvalCases( return rows.results ?? []; } +export async function updateEvalCaseStatus( + env: Env, + workspaceId: string, + caseId: string, + status: 'active' | 'archived', +): Promise { + await env.DB.prepare( + `UPDATE eval_case SET status = ?, updated_at = ? + WHERE id = ? AND workspace_id = ?`, + ) + .bind(status, Date.now(), caseId, workspaceId) + .run(); + return env.DB.prepare(`SELECT * FROM eval_case WHERE id = ? AND workspace_id = ?`) + .bind(caseId, workspaceId) + .first(); +} + export async function listEvalRuns(env: Env, workspaceId: string, limit = 20): Promise { const rows = await env.DB.prepare( `SELECT * FROM eval_run @@ -204,6 +221,25 @@ export async function insertEvalResult( return result; } +export async function getLatestEvalResultForCase( + env: Env, + workspaceId: string, + caseId: string, +): Promise { + return env.DB.prepare( + `SELECT r.* + FROM eval_result r + JOIN eval_run er ON er.id = r.run_id + WHERE r.workspace_id = ? AND r.case_id = ? + AND r.status IN ('passed','failed') + AND er.status IN ('passed','failed') + ORDER BY r.created_at DESC + LIMIT 1`, + ) + .bind(workspaceId, caseId) + .first(); +} + export async function completeEvalRun( env: Env, workspaceId: string, diff --git a/src/ui/api.ts b/src/ui/api.ts index 2b1bd0a..8d72ded 100644 --- a/src/ui/api.ts +++ b/src/ui/api.ts @@ -200,7 +200,11 @@ export const API = { auth_header_name?: string | null; auth_secret?: string; enabled?: boolean; - }) => api<{ server: McpServerEntry }>('/api/mcp/servers', { method: 'POST', body: JSON.stringify(body) }), + }) => + api<{ server: McpServerEntry }>('/api/mcp/servers', { + method: 'POST', + body: JSON.stringify(body), + }), updateMcpServer: ( id: string, body: { @@ -211,10 +215,17 @@ export const API = { auth_secret?: string; enabled?: boolean; }, - ) => api<{ server: McpServerEntry }>(`/api/mcp/servers/${id}`, { method: 'PATCH', body: JSON.stringify(body) }), - deleteMcpServer: (id: string) => api<{ ok: boolean }>(`/api/mcp/servers/${id}`, { method: 'DELETE' }), + ) => + api<{ server: McpServerEntry }>(`/api/mcp/servers/${id}`, { + method: 'PATCH', + body: JSON.stringify(body), + }), + deleteMcpServer: (id: string) => + api<{ ok: boolean }>(`/api/mcp/servers/${id}`, { method: 'DELETE' }), discoverMcpTools: (serverId: string) => - api<{ ok: boolean; tools: McpToolEntry[] }>(`/api/mcp/servers/${serverId}/discover`, { method: 'POST' }), + api<{ ok: boolean; tools: McpToolEntry[] }>(`/api/mcp/servers/${serverId}/discover`, { + method: 'POST', + }), listMcpTools: (serverId?: string) => api<{ tools: McpToolEntry[] }>(`/api/mcp/tools${serverId ? `?server_id=${serverId}` : ''}`), setMcpGuardrail: ( @@ -237,6 +248,11 @@ export const API = { api<{ toolCalls: McpToolCall[] }>(`/api/mcp/tool-calls?ticket_id=${ticketId}`), listEvalCases: () => api<{ cases: EvalCaseEntry[] }>('/api/evals/cases'), listEvalRuns: () => api<{ runs: EvalRunEntry[] }>('/api/evals/runs'), + updateEvalCase: (id: string, status: 'active' | 'archived') => + api<{ case: EvalCaseEntry }>(`/api/evals/cases/${id}`, { + method: 'PATCH', + body: JSON.stringify({ status }), + }), captureResolvedEvalCases: (limit = 50) => api<{ ok: boolean; captured: number; skipped: number; failed: number; cases: string[] }>( '/api/evals/cases/capture-resolved', @@ -245,7 +261,9 @@ export const API = { body: JSON.stringify({ limit }), }, ), - runEvalSuite: (body: { limit?: number; threshold?: number } = {}) => + runEvalSuite: ( + body: { limit?: number; threshold?: number; score_drop_threshold?: number } = {}, + ) => api('/api/evals/runs', { method: 'POST', body: JSON.stringify(body), diff --git a/src/ui/views/EvalsSection.tsx b/src/ui/views/EvalsSection.tsx index 4242d6b..2abdf27 100644 --- a/src/ui/views/EvalsSection.tsx +++ b/src/ui/views/EvalsSection.tsx @@ -10,6 +10,7 @@ export function EvalsSection({ onSaved }: EvalsSectionProps) { const [runs, setRuns] = useState([]); const [limit, setLimit] = useState(50); const [threshold, setThreshold] = useState(0.35); + const [scoreDropThreshold, setScoreDropThreshold] = useState(0.15); const [busy, setBusy] = useState(''); const [error, setError] = useState(''); @@ -41,7 +42,11 @@ export function EvalsSection({ onSaved }: EvalsSectionProps) { setError(''); setBusy('run'); try { - const detail = await API.runEvalSuite({ limit, threshold }); + const detail = await API.runEvalSuite({ + limit, + threshold, + score_drop_threshold: scoreDropThreshold, + }); await load(); onSaved(`Eval run ${detail.run.status}`); } catch (err: any) { @@ -77,6 +82,17 @@ export function EvalsSection({ onSaved }: EvalsSectionProps) { onChange={(e) => setThreshold(Number(e.target.value))} /> +
+ + setScoreDropThreshold(Number(e.target.value))} + /> +
@@ -101,6 +117,31 @@ export function EvalsSection({ onSaved }: EvalsSectionProps) { {cases.length} + {cases.slice(0, 5).map((evalCase) => ( +
+
+
{evalCase.name}
+
+ {evalCase.source} · {new Date(evalCase.captured_at).toLocaleString()} +
+
+ +
+ ))} {runs.slice(0, 5).map((runItem) => (
diff --git a/tests/evals.test.ts b/tests/evals.test.ts index a115208..60fabe5 100644 --- a/tests/evals.test.ts +++ b/tests/evals.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it, vi } from 'vitest'; import { apiApp } from '../src/api/routes'; -import { anonymizeValue } from '../src/evals/anonymize'; +import { anonymizeValue, detectResidualPii } from '../src/evals/anonymize'; import { captureResolvedTicketEvalCase } from '../src/evals/capture'; import { runEvalSuite, runProcedureSpecEvals } from '../src/evals/replay'; import { @@ -84,6 +84,16 @@ describe('eval anonymization', () => { expect(JSON.stringify(result.value)).toContain('[phone_1]'); expect(JSON.stringify(result.metadata)).not.toContain('jane.customer@example.com'); }); + + it('detects residual PII after anonymization and ignores placeholders', () => { + const findings = detectResidualPii({ + safe: 'customer_1@example.test [phone_1]', + timestamp: 1760000000000, + leaked: 'finance@merchant.example and +1 212 555 0199', + }); + + expect(findings.map((finding) => finding.kind)).toEqual(['email', 'phone']); + }); }); describe('historical eval capture and replay', () => { @@ -169,6 +179,61 @@ describe('historical eval capture and replay', () => { expect(failing.run.status).toBe('failed'); expect(failing.run.regression_count).toBe(1); }); + + it('marks score drops from the previous baseline as regressions', async () => { + const { db, env } = createWorkspaceTestDb(); + seedWorkspace(db, 'ws_a', 'Alpha'); + seedResolvedConversation(db); + await captureResolvedTicketEvalCase(env, 'ws_a', 'tkt_eval'); + const retrievalRunner = async () => ({ + hits: [], + trace: { + plan: { + originalQuery: 'refund', + scope: 'all' as const, + subqueries: ['refund'], + maxHops: 1, + }, + hops: [], + finalAnswerable: false, + stopReason: 'no_hits' as const, + startedAt: 1, + durationMs: 1, + }, + }); + + await runEvalSuite(env, 'ws_a', { + threshold: 0.2, + retrievalRunner, + draftRunner: async () => ({ + subject: 'Re: Refund for order 123', + body_markdown: + 'Hi, your refund for order 123 has been approved and should arrive within five business days.', + tone: 'friendly', + cites_knowledge_ids: [], + confidence: 0.9, + needs_human_review_reasons: [], + }), + }); + const regressed = await runEvalSuite(env, 'ws_a', { + threshold: 0.2, + scoreDropThreshold: 0.15, + retrievalRunner, + draftRunner: async () => ({ + subject: 'Re: Refund for order 123', + body_markdown: 'Your refund has been approved and should arrive.', + tone: 'friendly', + cites_knowledge_ids: [], + confidence: 0.9, + needs_human_review_reasons: [], + }), + }); + + expect(regressed.run.passed_count).toBe(1); + expect(regressed.run.failed_count).toBe(0); + expect(regressed.run.regression_count).toBe(1); + expect(regressed.run.status).toBe('failed'); + }); }); describe('procedure spec evals', () => { @@ -223,5 +288,20 @@ describe('eval API', () => { expect(res.status).toBe(200); expect(body.captured).toBe(1); expect(db.prepare(`SELECT COUNT(*) AS n FROM eval_case`).get()).toEqual({ n: 1 }); + + const archive = await apiApp.request( + `/evals/cases/${body.cases[0]}`, + { + method: 'PATCH', + headers: { cookie, 'content-type': 'application/json' }, + body: JSON.stringify({ status: 'archived' }), + }, + env, + ); + + expect(archive.status).toBe(200); + expect(db.prepare(`SELECT status FROM eval_case WHERE id = ?`).get(body.cases[0])).toEqual({ + status: 'archived', + }); }); });