diff --git a/.github/workflows/evals.yml b/.github/workflows/evals.yml index e27e351..1127b5d 100644 --- a/.github/workflows/evals.yml +++ b/.github/workflows/evals.yml @@ -20,12 +20,16 @@ jobs: - uses: oven-sh/setup-bun@v2 - run: bun install --frozen-lockfile - run: bun run typecheck + - name: Validate procedure library manifest + run: bun scripts/ranse.ts procedure validate-library - name: Run procedure evals run: | shopt -s nullglob for file in procedures/*.yaml procedures/*.yml procedures/*.json; do bun scripts/ranse.ts eval "$file" done + - name: Validate procedure library + run: bunx vitest run tests/procedure-library.test.ts - name: Run hosted historical evals when configured env: RANSE_APP_URL: ${{ secrets.RANSE_APP_URL }} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a8af28c..4733642 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -27,6 +27,13 @@ bun run dev - Keep commits focused. One logical change per PR. - Include a before/after description in the PR body if the change affects UX, APIs, or the setup flow. +## Procedure library contributions + +- Start from `procedure-library/README.md`, `src/procedures/library-data.ts`, and `src/procedures/library-mcp-tools.ts`. +- Every library procedure must include inline `evals`, a generic owner of `ranse-library`, deterministic provenance, and reference MCP tool specs for external system assumptions. Required MCP references must be exercised by `call_action` steps, and write/destructive actions must stay behind approval. +- Run `bun scripts/ranse.ts procedure validate-library`, `bun scripts/ranse.ts procedure add --dir /tmp/ranse-procs --force`, `bun scripts/ranse.ts eval /tmp/ranse-procs/.yaml`, and `bunx vitest run tests/procedure-library.test.ts`. +- Do not bake customer-specific policy text, private route names, or proprietary tool names into shared library procedures. + ## Commit messages Use terse Conventional Commit-style subjects: diff --git a/README.md b/README.md index 2488db5..872b597 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,8 @@ Ranse turns support email into a real-time, multi-agent support workspace built - **Human approval gates** for every outbound reply, with edit-before-send. - **Multi-provider LLM** — Workers AI out of the box; drop-in Anthropic, OpenAI, Google, Grok, OpenRouter via AI Gateway. - **Historical evals** — resolved conversations become anonymized replay cases; `ranse eval` catches prompt/procedure regressions before they ship. +- **Forkable procedure library** — install vetted support workflows with evals, provenance checksums, and MCP reference contracts, then customize them in your repo. +- **Insights loop** — score conversations, surface evidence-backed unresolved intents, draft reviewable KB suggestions with lineage, and detect source-specific drift from successful replies. - **One-click deploy** to your own Cloudflare account — customer-owned from day one. - **Open source** (Apache-2.0). @@ -84,7 +86,7 @@ Then open http://localhost:5173 (or http://localhost:8787 for the Worker directl Ranse is heading from "AI-assisted shared inbox" to a full autonomous customer-service agent — but not as an OSS clone of [Fin](https://fin.ai/) or [Decagon](https://decagon.ai/). The goal is the agent those products *structurally cannot become*: sovereign by construction, per-step model choice, procedures-as-code, MCP-native actions, eval-first against your own ticket history, and a forkable procedure library. -The shape, in short: **retrieval → workspace management → agentic retrieval → autonomous resolution → procedures → MCP actions → evals → procedure library → insights → multi-channel.** Phase 0 (bootstrap, inbound email, supervisor DO, draft + approval), Phase 1 (retrieval foundations), Phase 1.5 (workspace management & tenant isolation), Phase 2 (agentic multi-hop retrieval), Phase 3 (autonomous resolution), Phase 4 (procedures as code), Phase 5 (MCP-native actions), and Phase 6 (historical evals) are shipped. +The shape, in short: **retrieval → workspace management → agentic retrieval → autonomous resolution → procedures → MCP actions → evals → procedure library → insights → multi-channel.** Phase 0 (bootstrap, inbound email, supervisor DO, draft + approval), Phase 1 (retrieval foundations), Phase 1.5 (workspace management & tenant isolation), Phase 2 (agentic multi-hop retrieval), Phase 3 (autonomous resolution), Phase 4 (procedures as code), Phase 5 (MCP-native actions), Phase 6 (historical evals), Phase 7 (procedure library), and Phase 8 (insights & auto-improving KB) are shipped. Full pipeline, principles, and how to contribute to a phase: **[docs/roadmap.md](docs/roadmap.md)**. It's directional, not committed — if you want to work on something further down the list, open a discussion and we'll happily reorder. diff --git a/docs/architecture.md b/docs/architecture.md index 3ee61da..cd23863 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -16,6 +16,7 @@ Function-based, not DOs. They live in `src/agents/specialists/` and return struc - `triage` — category, priority, sentiment, language, spam detection. - `summarize` — thread summary + next-step hint. - `knowledge` — manual/URL/PDF/resolved-ticket ingestion, Workers AI embeddings, Vectorize search, reranking, and keyword fallback. +- `insights` — conversation rubric scoring, aggregate operational metrics, unresolved-intent KB suggestions, and knowledge drift detection. - `draft` — generate a reply with citations; flag review risks. - `escalation` — decide whether to route to a human/team. - `sla` — deterministic, no LLM; computes breach status. @@ -52,7 +53,7 @@ triageAndDraft (runs in DO alarm, async) | System | Purpose | |---|---| | DO SQLite | Workspace state, mailbox counters, BYOK-encrypted secrets | -| D1 | Tickets, messages, audit, approvals, outcomes, feedback, daily rollups, users, sessions, knowledge, LLM config, procedures, MCP registry/tool calls, eval cases/runs/results | +| D1 | Tickets, messages, audit, approvals, outcomes, feedback, daily rollups, users, sessions, knowledge, LLM config, procedures, MCP registry/tool calls, eval cases/runs/results, conversation scores, KB suggestions, drift signals | | R2 | Raw MIME, text/html bodies, attachments, exports | | KV | Rate limits, idempotency, lightweight flags | | Vectorize | Per-workspace knowledge chunk embeddings | @@ -126,6 +127,44 @@ ranse eval / Settings -> Evals Procedure evals are local and deterministic. `ranse eval ` loads the spec, runs each inline `evals[]` case through `simulateProcedure`, and checks expected status, context paths, and step order before a PR is merged. +## Procedure library flow + +``` +Settings -> Procedures + ├─ GET /api/procedures/library + │ └─ compare library MCP contracts against discovered workspace MCP tools + ├─ GET /api/procedures/library/manifest + ├─ POST /api/procedures/library/:slug/install + └─ upsertProcedureVersion(source_kind = seed, source_ref = library:@#sha256:) + +ranse procedure add + ├─ read built-in catalog from src/procedures/library-data.ts + ├─ validate inline evals + ├─ write procedures/.yaml + ├─ write procedures/.mcp.json + └─ write procedures/.provenance.json +``` + +The built-in catalog is code, not database state, so deploys carry the exact procedure specs, evals, and reference MCP contracts reviewed in git. List/detail responses include deterministic SHA-256 provenance, the Ranse procedure schema version, MCP readiness for the selected workspace, and the MCP schema version used for reference ToolAnnotations. Validation requires each required MCP reference to be exercised by a `call_action` step; write and destructive actions cannot opt out of approval. + +## Insights loop + +``` +Weekly cron / manual refresh + ├─ score recent tickets on groundedness, tone, resolution, and customer effort + ├─ aggregate resolution, follow-up, feedback, unresolved-intent, and procedure-latency metrics + ├─ cluster repeated unresolved conversations into confidence-scored KB article suggestions + └─ compare cited KB sources against successful replies for source-specific drift signals + +Insights page + ├─ POST /api/insights/scores/run + ├─ POST /api/insights/kb-suggestions/run + ├─ POST /api/insights/kb-suggestions/:id/accept + └─ POST /api/insights/drift/run +``` + +Suggestions are review records, not automatic content edits. They require repeated unresolved-ticket evidence, store confidence and source-ticket lineage, and accepted suggestions become terminal records linked to the manual knowledge source created through the same ingestion path as the Content Library. + ## Scaling model - One `WorkspaceSupervisorAgent` DO per workspace. The email handler pins by `idFromName(workspaceId)` so all events for a workspace funnel through one instance — consistent state, no cross-DO coordination needed. diff --git a/docs/operations.md b/docs/operations.md index c05024b..76daf75 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -60,6 +60,28 @@ Eval runs write `eval_run` and `eval_result` rows with assertion details. A run The bundled GitHub Actions workflow always runs procedure evals for relevant PRs; set `RANSE_APP_URL` and `RANSE_COOKIE` repository secrets to make hosted historical replay part of the gate. +## Procedure library + +Owners and admins can install vetted procedure templates from **Settings → Procedures**. The catalog shows whether the selected workspace has the required MCP servers and tools discovered before install. Installed library procedures are published as immutable procedure versions with `source_ref = library:@#sha256:`. + +Local fork workflow: + +```bash +bun scripts/ranse.ts procedure list +bun scripts/ranse.ts procedure manifest +bun scripts/ranse.ts procedure validate-library +bun scripts/ranse.ts procedure add shipping-dispute --dir procedures +bun scripts/ranse.ts eval procedures/shipping-dispute.yaml +``` + +The CLI writes the procedure spec, `.mcp.json`, and `.provenance.json`. The provenance file records the library version, source ref, procedure SHA-256 checksum, and standards metadata used when the procedure was forked. Treat the MCP specs as contracts: each required reference is exercised by a `call_action` step, read-only tools may run automatically, and write/destructive tools must remain behind an approval gate unless you deliberately rework the procedure and its evals. + +## Insights + +Owners and admins can open **Insights** to refresh conversation scores, unresolved-intent KB suggestions, and knowledge drift signals. The weekly cron `17 3 * * 1` runs the same maintenance loop automatically inside the worker. + +KB suggestions are generated only from repeated unresolved-ticket clusters and include evidence count, confidence, suggested terms, and source-ticket IDs. Accepted suggestions are idempotently published as manual knowledge sources, linked back through `accepted_source_id`, and then treated as terminal audit records. Dismissed suggestions and resolved drift signals remain in D1 for auditability instead of being deleted. + ## Escalations The `EscalationAgent` runs on demand. It returns `{ should_escalate, severity, route_to }` and the operator (or an automation rule) picks the handoff target. diff --git a/docs/roadmap.md b/docs/roadmap.md index b2bf1be..a34a5d9 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -51,7 +51,11 @@ Most "AI agent" tools are chat shaped and bolt email on. Real B2B support lives **Phase 6 — Historical evals** is shipped. Resolved tickets are captured as anonymized replay cases, operators can backfill and run evals from Settings, `ranse eval` runs procedure-file and hosted historical suites, and PRs touching prompts/procedures/model logic have an eval workflow. -That's now a retrieval-grounded early Fin **Copilot** equivalent with workspace isolation, traceable multi-hop retrieval, a conservative autonomous-send path, a procedure-driven agent loop, external action execution through the open MCP protocol, and a regression gate against the workspace's own ticket history. Everything below continues the path toward procedure sharing and insights. +**Phase 7 — Procedure library** is shipped. Workspaces can install vetted workflows from Settings, fork them locally with `ranse procedure add`, and inspect reference MCP tool contracts plus inline evals before customization. + +**Phase 8 — Insights & auto-improving KB** is shipped. Workspaces get conversation rubric scoring, aggregate insight dashboards, unresolved-intent KB suggestions, accepted-suggestion publishing into the knowledge base, drift signals against successful replies, and weekly scheduled insight maintenance. + +That's now a retrieval-grounded early Fin **Copilot** equivalent with workspace isolation, traceable multi-hop retrieval, a conservative autonomous-send path, a procedure-driven agent loop, external action execution through the open MCP protocol, a regression gate against the workspace's own ticket history, a forkable procedure library, and a sovereign insights loop that turns real support history into reviewed KB improvements. Everything below continues the path toward multi-channel surfaces. ## Phase 1 — Retrieval foundations **Status: shipped.** @@ -182,20 +186,29 @@ The `customer_data` search scope still fails closed with an explicit trace; proc - Historical replay is the primary signal; synthetic-conversation generation remains a future complement, not a substitute. ## Phase 7 — Procedure library + community +**Status: shipped.** + *Principle 6* -- Public repo `getranse/procedures-library` — refund flow, password reset, shipping dispute, subscription cancellation, fraud triage, GDPR data request, etc. -- `ranse procedure add ` clones from library into workspace repo as a starting point -- Each library procedure ships with eval cases and a reference MCP tool spec -- Contribution guidelines for upstreaming generic procedures back from workspaces +- Built-in catalog ships refund intake, password reset, shipping dispute, and GDPR data request workflows. +- Settings exposes the catalog so owners/admins can install procedures directly into the selected workspace, with MCP readiness surfaced before install. +- `ranse procedure list` and `ranse procedure add ` fork procedures into a repo-local `procedures/` directory as YAML or JSON. +- `ranse procedure manifest` exports the full machine-readable catalog for a standalone community mirror. +- Each library procedure ships with inline eval cases, deterministic SHA-256 provenance, and reference MCP tool specs written beside the forked procedure as `.mcp.json` plus `.provenance.json`. +- Library procedures now exercise required MCP contracts through `call_action`; read-only lookups can run automatically, while write/destructive actions pause for operator approval. +- Library validation runs every procedure's inline evals, checksum generation, immutable clone behavior, route permissions, MCP reference matching, and unsafe-action checks in `tests/procedure-library.test.ts`. +- `procedure-library/README.md` and `CONTRIBUTING.md` define the contribution bar for upstreaming generic workflows. A standalone `getranse/procedures-library` repo can now mirror this catalog when community volume warrants it. ## Phase 8 — Insights & auto-improving KB +**Status: shipped.** + *Principle 5 (extends), Principle 1* -- Per-conversation rubric scoring (groundedness, tone, resolution, customer effort) -- Aggregate dashboards: resolution rate, escalation reasons, top unanswered intents, slowest procedures -- **Suggestions agent** clusters unresolved conversations weekly, drafts new KB articles **as PRs to the workspace's content repo** — human review preserved, no surprise edits -- Drift detection: flag KB entries whose answers diverge from recent successful replies +- Per-conversation rubric scoring is stored in D1 for groundedness, tone, resolution, customer effort, and overall quality, with signals preserved as auditable JSON. +- Aggregate dashboards are shipped in the operator console for resolution rate, follow-ups, feedback, low-score conversations, top unresolved intents, escalation reasons, and slowest procedures. +- The suggestions loop clusters repeated unresolved conversations, stores evidence count/confidence/source-ticket lineage, drafts reviewable KB article candidates, and lets an admin accept a suggestion into the workspace knowledge base. Human review is preserved; no content is published silently. +- Drift detection flags cited knowledge sources that no longer cover terms appearing in successful replies tied back to those source chunks. +- Weekly scheduled insight maintenance scores recent conversations, refreshes unresolved-intent suggestions, and detects KB drift inside the customer's Cloudflare account. ## Phase 9 — Multi-channel + voice *Principle 7 — email is the wedge; other channels are derivatives* diff --git a/migrations/20260518_010000_insights.sql b/migrations/20260518_010000_insights.sql new file mode 100644 index 0000000..125cd0f --- /dev/null +++ b/migrations/20260518_010000_insights.sql @@ -0,0 +1,63 @@ +-- Phase 8 insights and auto-improving knowledge base. + +CREATE TABLE IF NOT EXISTS conversation_score ( + id TEXT PRIMARY KEY, + workspace_id TEXT NOT NULL, + ticket_id TEXT NOT NULL, + groundedness_score REAL NOT NULL CHECK(groundedness_score >= 0 AND groundedness_score <= 1), + tone_score REAL NOT NULL CHECK(tone_score >= 0 AND tone_score <= 1), + resolution_score REAL NOT NULL CHECK(resolution_score >= 0 AND resolution_score <= 1), + effort_score REAL NOT NULL CHECK(effort_score >= 0 AND effort_score <= 1), + overall_score REAL NOT NULL CHECK(overall_score >= 0 AND overall_score <= 1), + signals_json TEXT NOT NULL DEFAULT '{}', + scored_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + UNIQUE(workspace_id, ticket_id), + FOREIGN KEY (workspace_id) REFERENCES workspace(id) ON DELETE CASCADE, + FOREIGN KEY (ticket_id) REFERENCES ticket(id) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_conversation_score_workspace + ON conversation_score(workspace_id, overall_score ASC, scored_at DESC); + +CREATE TABLE IF NOT EXISTS kb_suggestion ( + id TEXT PRIMARY KEY, + workspace_id TEXT NOT NULL, + cluster_key TEXT NOT NULL, + title TEXT NOT NULL, + summary TEXT NOT NULL, + body_markdown TEXT NOT NULL, + source_ticket_ids_json TEXT NOT NULL DEFAULT '[]', + suggested_terms_json TEXT NOT NULL DEFAULT '[]', + status TEXT NOT NULL DEFAULT 'open' CHECK(status IN ('open','accepted','dismissed')), + source TEXT NOT NULL DEFAULT 'unresolved_cluster', + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + UNIQUE(workspace_id, cluster_key), + FOREIGN KEY (workspace_id) REFERENCES workspace(id) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_kb_suggestion_workspace + ON kb_suggestion(workspace_id, status, updated_at DESC); + +CREATE TABLE IF NOT EXISTS knowledge_drift_signal ( + id TEXT PRIMARY KEY, + workspace_id TEXT NOT NULL, + source_id TEXT NOT NULL, + signal_hash TEXT NOT NULL, + severity TEXT NOT NULL CHECK(severity IN ('low','medium','high')), + title TEXT NOT NULL, + summary TEXT NOT NULL, + successful_reply_count INTEGER NOT NULL DEFAULT 0, + divergence_terms_json TEXT NOT NULL DEFAULT '[]', + example_ticket_ids_json TEXT NOT NULL DEFAULT '[]', + status TEXT NOT NULL DEFAULT 'open' CHECK(status IN ('open','resolved','dismissed')), + detected_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + UNIQUE(workspace_id, source_id, signal_hash), + FOREIGN KEY (workspace_id) REFERENCES workspace(id) ON DELETE CASCADE, + FOREIGN KEY (source_id) REFERENCES knowledge_source(id) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_knowledge_drift_signal_workspace + ON knowledge_drift_signal(workspace_id, status, severity, detected_at DESC); diff --git a/migrations/20260518_020000_insights_hardening.sql b/migrations/20260518_020000_insights_hardening.sql new file mode 100644 index 0000000..a753974 --- /dev/null +++ b/migrations/20260518_020000_insights_hardening.sql @@ -0,0 +1,10 @@ +-- Phase 8 insights hardening: suggestion confidence and acceptance lineage. + +ALTER TABLE kb_suggestion ADD COLUMN evidence_count INTEGER NOT NULL DEFAULT 0; +ALTER TABLE kb_suggestion ADD COLUMN confidence_score REAL NOT NULL DEFAULT 0; +ALTER TABLE kb_suggestion ADD COLUMN accepted_source_id TEXT; +ALTER TABLE kb_suggestion ADD COLUMN accepted_by_user_id TEXT; +ALTER TABLE kb_suggestion ADD COLUMN accepted_at INTEGER; + +CREATE INDEX IF NOT EXISTS idx_kb_suggestion_acceptance + ON kb_suggestion(workspace_id, status, accepted_at DESC); diff --git a/procedure-library/README.md b/procedure-library/README.md new file mode 100644 index 0000000..aa7b0ef --- /dev/null +++ b/procedure-library/README.md @@ -0,0 +1,39 @@ +# Ranse Procedure Library + +The built-in library is the seed for the future `getranse/procedures-library` community repo. Each entry is forkable, ships with inline evals, and includes reference MCP tool specs for the systems it expects a workspace to expose. Library validation ties those contracts back to real `call_action` steps so templates cannot advertise unused or unsafe external actions. + +## Install locally + +```bash +bun scripts/ranse.ts procedure list +bun scripts/ranse.ts procedure manifest +bun scripts/ranse.ts procedure validate-library +bun scripts/ranse.ts procedure add refund-intake --dir procedures +bun scripts/ranse.ts eval procedures/refund-intake.yaml +``` + +`procedure add` writes three files: + +- `.yaml` or `.json` — the procedure spec to customize, review, and commit. +- `.mcp.json` — reference MCP tool contracts to implement or map to existing servers, including MCP ToolAnnotations. +- `.provenance.json` — immutable library version, source ref, procedure checksum, and standards metadata. + +`procedure manifest` emits the full machine-readable catalog for mirroring into a standalone community repo. `procedure validate-library` reruns schema checks, inline evals, checksum generation, MCP annotation checks, action-reference matching, and approval-safety checks for write/destructive tools. + +## Current entries + +| Slug | Category | MCP references | +|---|---|---| +| `refund-intake` | Billing | Stripe customer lookup and refund creation | +| `password-reset` | Account | Identity lookup and password reset request creation | +| `shipping-dispute` | Shipping | Shopify order search | +| `gdpr-data-request` | Privacy | Privacy request creation | + +## Contribution bar + +- Keep procedures generic enough for another company to fork. +- Include at least one eval case in `evals`. +- Include reference MCP tool specs for every external system assumption, and exercise each required tool from a `call_action` step. +- Include `openWorldHint`, `readOnlyHint`, and destructive/idempotent hints where applicable. +- Default write and destructive actions to approval, and never ask customers for secrets, passwords, or one-time codes. +- Run `bun scripts/ranse.ts procedure validate-library`, `bun scripts/ranse.ts procedure add --dir /tmp/ranse-procs --force`, and `bun run test` before opening a PR. diff --git a/scripts/ranse.ts b/scripts/ranse.ts index 733f8bd..eccf0f5 100644 --- a/scripts/ranse.ts +++ b/scripts/ranse.ts @@ -1,7 +1,15 @@ #!/usr/bin/env bun -import { readFile } from 'node:fs/promises'; +import { mkdir, readFile, stat, writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { stringify as stringifyYaml } from 'yaml'; import { runProcedureSpecEvals } from '../src/evals/replay'; import { loadProcedureFile } from '../src/procedures/files'; +import { + getProcedureLibraryItem, + getProcedureLibraryManifest, + listProcedureLibrary, + validateProcedureLibrary, +} from '../src/procedures/library'; import { simulateProcedure } from '../src/procedures/simulate'; const [, , command, ...args] = process.argv; @@ -84,12 +92,101 @@ async function main() { process.exit(res.ok && status !== 'failed' ? 0 : 1); } + if (command === 'procedure') { + const subcommand = args[0]; + if (subcommand === 'list') { + console.log(JSON.stringify({ procedures: await listProcedureLibrary() }, null, 2)); + return; + } + + if (subcommand === 'manifest') { + console.log(JSON.stringify(await getProcedureLibraryManifest(), null, 2)); + return; + } + + if (subcommand === 'validate-library') { + console.log( + JSON.stringify({ ok: true, procedures: await validateProcedureLibrary() }, null, 2), + ); + return; + } + + if (subcommand === 'add') { + assertKnownFlags(args.slice(2), ['--dir', '--format'], ['--force']); + const slug = requiredArg(args[1], procedureUsage()); + const item = await getProcedureLibraryItem(slug); + if (!item) throw new Error(`procedure_library_item_not_found:${slug}`); + const report = runProcedureSpecEvals(item.spec); + if (report.status !== 'passed') throw new Error(`procedure_library_eval_failed:${slug}`); + + const dir = flagValue(args, '--dir') ?? 'procedures'; + const format = flagValue(args, '--format') ?? 'yaml'; + if (!['yaml', 'json'].includes(format)) throw new Error('unsupported_procedure_format'); + await mkdir(dir, { recursive: true }); + const procedurePath = join(dir, `${item.slug}.${format === 'json' ? 'json' : 'yaml'}`); + const mcpPath = join(dir, `${item.slug}.mcp.json`); + const provenancePath = join(dir, `${item.slug}.provenance.json`); + await writeIfAllowed( + procedurePath, + format === 'json' ? `${JSON.stringify(item.spec, null, 2)}\n` : stringifyYaml(item.spec), + args.includes('--force'), + ); + await writeIfAllowed( + mcpPath, + `${JSON.stringify( + { + protocol: { + name: 'model-context-protocol', + schema_version: item.provenance.standards.mcp_schema, + }, + tools: item.reference_mcp_tools.map((tool) => ({ + server: tool.server, + name: tool.tool, + title: tool.title, + description: tool.description, + inputSchema: tool.input_schema, + annotations: tool.annotations, + })), + }, + null, + 2, + )}\n`, + args.includes('--force'), + ); + await writeIfAllowed( + provenancePath, + `${JSON.stringify(item.provenance, null, 2)}\n`, + args.includes('--force'), + ); + console.log( + JSON.stringify( + { + ok: true, + procedure: procedurePath, + reference_mcp_tools: mcpPath, + provenance: provenancePath, + evals: report, + }, + null, + 2, + ), + ); + return; + } + + throw new Error(procedureUsage()); + } + console.log(`usage: ranse simulate [--input input.json] ranse publish --app-url --cookie ranse eval ranse eval --app-url --cookie [--limit n] [--threshold n] [--score-drop n] [--ci] - ranse eval capture-resolved --app-url --cookie [--limit n]`); + ranse eval capture-resolved --app-url --cookie [--limit n] + ranse procedure list + ranse procedure manifest + ranse procedure validate-library + ranse procedure add [--dir procedures] [--format yaml|json] [--force]`); } function requiredArg(value: string | undefined, message: string): string { @@ -99,7 +196,10 @@ function requiredArg(value: string | undefined, message: string): string { function flagValue(args_: string[], flag: string): string | undefined { const index = args_.indexOf(flag); - return index >= 0 ? args_[index + 1] : undefined; + if (index < 0) return undefined; + const value = args_[index + 1]; + if (!value || value.startsWith('--')) throw new Error(`missing_flag_value:${flag}`); + return value; } function numberFlag(args_: string[], flag: string): number | undefined { @@ -119,6 +219,44 @@ async function jsonOrText(res: Response): Promise { } } +async function writeIfAllowed(path: string, body: string, force: boolean) { + if (!force && (await exists(path))) throw new Error(`file_exists:${path}`); + await writeFile(path, body, 'utf8'); +} + +function assertKnownFlags(args_: string[], flagsWithValues: string[], booleanFlags: string[]) { + for (let index = 0; index < args_.length; index += 1) { + const arg = args_[index]; + if (!arg.startsWith('--')) continue; + if (booleanFlags.includes(arg)) continue; + if (flagsWithValues.includes(arg)) { + const value = args_[index + 1]; + if (!value || value.startsWith('--')) throw new Error(`missing_flag_value:${arg}`); + index += 1; + continue; + } + throw new Error(`unknown_flag:${arg}`); + } +} + +function procedureUsage(): string { + return `usage: + ranse procedure list + ranse procedure manifest + ranse procedure validate-library + ranse procedure add [--dir procedures] [--format yaml|json] [--force]`; +} + +async function exists(path: string): Promise { + try { + await stat(path); + return true; + } catch (err) { + if ((err as NodeJS.ErrnoException).code === 'ENOENT') return false; + throw err; + } +} + main().catch((err) => { console.error(err instanceof Error ? err.message : err); process.exit(1); diff --git a/src/api/insights.ts b/src/api/insights.ts new file mode 100644 index 0000000..023171e --- /dev/null +++ b/src/api/insights.ts @@ -0,0 +1,128 @@ +import type { Hono } from 'hono'; +import { z } from 'zod'; +import { apiError } from '../lib/errors'; +import { + acceptKbSuggestion, + detectKnowledgeDrift, + generateKbSuggestions, + getInsightSummary, + listConversationScores, + listKbSuggestions, + listKnowledgeDriftSignals, + scoreWorkspaceConversations, + updateKbSuggestionStatus, + updateKnowledgeDriftStatus, +} from '../insights'; +import { OWNER_OR_ADMIN, requireWorkspaceRole, type Ctx } from './context'; + +const limitSchema = z.object({ limit: z.number().int().min(1).max(500).optional() }); +const suggestionStatusSchema = z.object({ status: z.enum(['open', 'dismissed']) }); +const driftStatusSchema = z.object({ status: z.enum(['open', 'resolved', 'dismissed']) }); + +export function registerInsightRoutes(apiApp: Hono) { + apiApp.get('/insights/summary', requireWorkspaceRole(OWNER_OR_ADMIN), async (c) => { + const s = c.get('session'); + const days = Math.min(Math.max(Number(c.req.query('days') ?? 30), 1), 365); + return c.json({ summary: await getInsightSummary(c.env, s.workspaceId, days) }); + }); + + apiApp.get('/insights/scores', requireWorkspaceRole(OWNER_OR_ADMIN), async (c) => { + const s = c.get('session'); + const limit = Math.min(Math.max(Number(c.req.query('limit') ?? 50), 1), 200); + return c.json({ scores: await listConversationScores(c.env, s.workspaceId, limit) }); + }); + + apiApp.post('/insights/scores/run', requireWorkspaceRole(OWNER_OR_ADMIN), async (c) => { + const s = c.get('session'); + const body = limitSchema.parse(await c.req.json().catch(() => ({}))); + return c.json(await scoreWorkspaceConversations(c.env, s.workspaceId, body.limit ?? 100)); + }); + + apiApp.get('/insights/kb-suggestions', requireWorkspaceRole(OWNER_OR_ADMIN), async (c) => { + const s = c.get('session'); + const status = c.req.query('status'); + return c.json({ + suggestions: await listKbSuggestions( + c.env, + s.workspaceId, + status === 'open' || status === 'accepted' || status === 'dismissed' ? status : undefined, + ), + }); + }); + + apiApp.post('/insights/kb-suggestions/run', requireWorkspaceRole(OWNER_OR_ADMIN), async (c) => { + const s = c.get('session'); + const body = limitSchema.parse(await c.req.json().catch(() => ({}))); + return c.json(await generateKbSuggestions(c.env, s.workspaceId, body.limit ?? 100)); + }); + + apiApp.patch('/insights/kb-suggestions/:id', requireWorkspaceRole(OWNER_OR_ADMIN), async (c) => { + const s = c.get('session'); + const body = suggestionStatusSchema.parse(await c.req.json()); + try { + const suggestion = await updateKbSuggestionStatus( + c.env, + s.workspaceId, + c.req.param('id'), + body.status, + s.userId, + ); + if (!suggestion) return apiError(c, 'not_found', 'Suggestion not found.'); + return c.json({ suggestion }); + } catch (err) { + if (err instanceof Error && err.message === 'kb_suggestion_accepted') { + return apiError(c, 'conflict', 'Accepted suggestions cannot be changed.', 409); + } + throw err; + } + }); + + apiApp.post( + '/insights/kb-suggestions/:id/accept', + requireWorkspaceRole(OWNER_OR_ADMIN), + async (c) => { + const s = c.get('session'); + try { + const result = await acceptKbSuggestion(c.env, s.workspaceId, c.req.param('id'), s.userId); + if (!result) return apiError(c, 'not_found', 'Suggestion not found.'); + return c.json(result); + } catch (err) { + if (err instanceof Error && err.message === 'kb_suggestion_not_open') { + return apiError(c, 'conflict', 'Only open suggestions can be accepted.', 409); + } + throw err; + } + }, + ); + + apiApp.get('/insights/drift', requireWorkspaceRole(OWNER_OR_ADMIN), async (c) => { + const s = c.get('session'); + const status = c.req.query('status'); + return c.json({ + signals: await listKnowledgeDriftSignals( + c.env, + s.workspaceId, + status === 'open' || status === 'resolved' || status === 'dismissed' ? status : undefined, + ), + }); + }); + + apiApp.post('/insights/drift/run', requireWorkspaceRole(OWNER_OR_ADMIN), async (c) => { + const s = c.get('session'); + return c.json(await detectKnowledgeDrift(c.env, s.workspaceId)); + }); + + apiApp.patch('/insights/drift/:id', requireWorkspaceRole(OWNER_OR_ADMIN), async (c) => { + const s = c.get('session'); + const body = driftStatusSchema.parse(await c.req.json()); + const signal = await updateKnowledgeDriftStatus( + c.env, + s.workspaceId, + c.req.param('id'), + body.status, + s.userId, + ); + if (!signal) return apiError(c, 'not_found', 'Drift signal not found.'); + return c.json({ signal }); + }); +} diff --git a/src/api/procedures.ts b/src/api/procedures.ts index 3e843ac..3aec40e 100644 --- a/src/api/procedures.ts +++ b/src/api/procedures.ts @@ -1,6 +1,13 @@ import type { Hono } from 'hono'; import { z } from 'zod'; import { apiError } from '../lib/errors'; +import { + getProcedureLibraryItem, + getProcedureLibraryManifest, + getProcedureLibraryReadiness, + installProcedureFromLibrary, + listProcedureLibraryWithReadiness, +} from '../procedures/library'; import { resumeProcedureRunner, startProcedureRunner } from '../procedures/orchestration'; import { createProcedureRun, @@ -60,6 +67,56 @@ export function registerProcedureRoutes(apiApp: Hono) { } }); + apiApp.get('/procedures/library', async (c) => { + const s = c.get('session'); + return c.json({ procedures: await listProcedureLibraryWithReadiness(c.env, s.workspaceId) }); + }); + + apiApp.get('/procedures/library/manifest', async (c) => { + return c.json(await getProcedureLibraryManifest()); + }); + + apiApp.get('/procedures/library/:slug', async (c) => { + const item = await getProcedureLibraryItem(c.req.param('slug')); + if (!item) return apiError(c, 'not_found', 'That library procedure does not exist.'); + const s = c.get('session'); + return c.json({ + procedure: { + ...item, + readiness: await getProcedureLibraryReadiness(c.env, s.workspaceId, item.slug), + }, + }); + }); + + apiApp.post( + '/procedures/library/:slug/install', + requireWorkspaceRole(OWNER_OR_ADMIN), + async (c) => { + const s = c.get('session'); + try { + const result = await installProcedureFromLibrary(c.env, { + workspaceId: s.workspaceId, + actorUserId: s.userId, + slug: c.req.param('slug'), + }); + return c.json(result); + } catch (err) { + if (err instanceof Error && err.message === 'procedure_library_item_not_found') { + return apiError(c, 'not_found', 'That library procedure does not exist.'); + } + if (err instanceof Error && err.message === 'procedure_version_conflict') { + return apiError( + c, + 'conflict', + 'A different spec already exists for that procedure version.', + 409, + ); + } + throw err; + } + }, + ); + apiApp.get('/procedures/:id', async (c) => { const s = c.get('session'); const bundle = await getActiveProcedure(c.env, s.workspaceId, c.req.param('id')); diff --git a/src/api/routes.ts b/src/api/routes.ts index f923881..06654ff 100644 --- a/src/api/routes.ts +++ b/src/api/routes.ts @@ -2,6 +2,7 @@ import { Hono } from 'hono'; import { registerApprovalRoutes } from './approvals'; import { installApiAuth, type Ctx } from './context'; import { registerEvalRoutes } from './evals'; +import { registerInsightRoutes } from './insights'; import { registerKnowledgeRoutes } from './knowledge'; import { registerMcpRoutes } from './mcp'; import { registerNotificationRoutes } from './notifications'; @@ -21,4 +22,5 @@ registerKnowledgeRoutes(apiApp); registerProcedureRoutes(apiApp); registerMcpRoutes(apiApp); registerEvalRoutes(apiApp); +registerInsightRoutes(apiApp); registerWorkspaceRoutes(apiApp); diff --git a/src/evals/replay.ts b/src/evals/replay.ts index 7e411f9..30a70ad 100644 --- a/src/evals/replay.ts +++ b/src/evals/replay.ts @@ -272,6 +272,44 @@ export function evaluateProcedureExpectations( message: `Expected steps ${JSON.stringify(expect.steps)}, got ${JSON.stringify(stepIds)}.`, }); } + if ( + 'step_statuses' in expect && + expect.step_statuses && + typeof expect.step_statuses === 'object' + ) { + const steps = + (getPath(actual, 'steps') as Array<{ step_id?: string; status?: string }> | undefined) ?? []; + for (const [stepId, expectedStatus] of Object.entries( + expect.step_statuses as Record, + )) { + const actualStatus = steps.find((step) => step.step_id === stepId)?.status; + assertions.push({ + name: `step_status.${stepId}`, + passed: actualStatus === expectedStatus, + message: `Expected ${String(expectedStatus)}, got ${String(actualStatus)}.`, + }); + } + } + if ('step_inputs' in expect && expect.step_inputs && typeof expect.step_inputs === 'object') { + const steps = + (getPath(actual, 'steps') as Array<{ step_id?: string; input?: unknown }> | undefined) ?? []; + for (const [stepId, expectedPaths] of Object.entries( + expect.step_inputs as Record>, + )) { + const stepInput = steps.find((step) => step.step_id === stepId)?.input; + for (const [path, expectedValue] of Object.entries(expectedPaths)) { + const actualValue = + stepInput && typeof stepInput === 'object' + ? getPath(stepInput as Record, path) + : undefined; + assertions.push({ + name: `step_input.${stepId}.${path}`, + passed: JSON.stringify(actualValue) === JSON.stringify(expectedValue), + message: `Expected ${JSON.stringify(expectedValue)}, got ${JSON.stringify(actualValue)}.`, + }); + } + } + } if (assertions.length === 0) { assertions.push({ name: 'runs_without_failure', diff --git a/src/insights/index.ts b/src/insights/index.ts new file mode 100644 index 0000000..a33c92c --- /dev/null +++ b/src/insights/index.ts @@ -0,0 +1,1157 @@ +import type { Env } from '../env'; +import { audit } from '../lib/audit'; +import { sha256Hex } from '../lib/crypto'; +import { ids } from '../lib/ids'; +import { ingestKnowledgeSource } from '../knowledge'; +import type { + ConversationScore, + InsightSummary, + KbSuggestion, + KbSuggestionStatus, + KnowledgeDriftSignal, + KnowledgeDriftStatus, +} from '../types/insights'; + +const STOP_WORDS = new Set([ + 'about', + 'after', + 'again', + 'also', + 'because', + 'before', + 'could', + 'customer', + 'does', + 'done', + 'for', + 'from', + 'get', + 'have', + 'help', + 'how', + 'into', + 'just', + 'need', + 'order', + 'please', + 'request', + 'send', + 'support', + 'that', + 'their', + 'there', + 'this', + 'ticket', + 'what', + 'when', + 'where', + 'with', + 'your', +]); + +const MIN_SUGGESTION_TICKETS = 2; +const MIN_DRIFT_REPLIES = 2; +const MAX_SOURCE_CHUNKS_FOR_LINEAGE = 50; + +interface TicketRow { + id: string; + workspace_id: string; + subject: string; + status: string; + priority: string; + category: string | null; + requester_email: string; + created_at: number; + updated_at: number; +} + +interface MessageRow { + id: string; + ticket_id: string; + workspace_id: string; + direction: 'inbound' | 'outbound' | 'note'; + preview: string | null; + sent_at: number; + created_at: number; +} + +interface ApprovalRow { + kind: string; + status: string; + proposed_json: string; + risk_reasons_json: string; + created_at: number; +} + +interface OutcomeRow { + kind: string; + confidence_score: number | null; + payload_json: string; + created_at: number; +} + +interface FeedbackRow { + rating: 'positive' | 'negative'; + source: string; + comment: string | null; + created_at: number; +} + +export async function scoreWorkspaceConversations( + env: Env, + workspaceId: string, + limit = 100, +): Promise<{ scored: number; scores: ConversationScore[] }> { + const rows = await env.DB.prepare( + `SELECT id FROM ticket + WHERE workspace_id = ? + ORDER BY updated_at DESC + LIMIT ?`, + ) + .bind(workspaceId, Math.min(Math.max(limit, 1), 500)) + .all<{ id: string }>(); + const scores: ConversationScore[] = []; + for (const row of rows.results ?? []) { + const score = await scoreConversation(env, workspaceId, row.id); + if (score) scores.push(score); + } + return { scored: scores.length, scores }; +} + +export async function scoreConversation( + env: Env, + workspaceId: string, + ticketId: string, +): Promise { + const ticket = await getTicket(env, workspaceId, ticketId); + if (!ticket) return null; + const [messages, approvals, outcomes, feedback] = await Promise.all([ + listMessages(env, workspaceId, ticketId), + listApprovals(env, workspaceId, ticketId), + listOutcomes(env, workspaceId, ticketId), + listFeedback(env, workspaceId, ticketId), + ]); + const inbound = messages.filter((msg) => msg.direction === 'inbound'); + const outbound = messages.filter((msg) => msg.direction === 'outbound'); + const proposed = approvals.map((approval) => safeJson(approval.proposed_json)); + const citedIds = new Set(); + let proposedConfidence = 0; + let groundedTrace = false; + let hasKnowledgeHits = false; + for (const item of proposed) { + for (const cited of asStringArray(item.cites_knowledge_ids)) citedIds.add(cited); + proposedConfidence = Math.max(proposedConfidence, numberOrZero(item.confidence)); + groundedTrace = groundedTrace || hasFinalAnswerableTrace(item.knowledge_trace); + hasKnowledgeHits = + hasKnowledgeHits || (Array.isArray(item.knowledge_hits) && item.knowledge_hits.length > 0); + } + const risks = approvals.flatMap((approval) => + asStringArray(safeJson(approval.risk_reasons_json)), + ); + const outcomeKinds = new Set(outcomes.map((outcome) => outcome.kind)); + const hasPositiveFeedback = feedback.some((item) => item.rating === 'positive'); + const hasNegativeFeedback = feedback.some((item) => item.rating === 'negative'); + const escalated = outcomeKinds.has('escalated'); + const followedUp = outcomeKinds.has('customer_followed_up'); + + const groundedness = scoreGroundedness({ + hasOutbound: outbound.length > 0, + citedCount: citedIds.size, + proposedConfidence, + groundedTrace, + hasKnowledgeHits, + risks, + }); + const tone = scoreTone(outbound.map((msg) => msg.preview ?? '').join('\n')); + const resolution = scoreResolution(ticket.status, { + resolvedByOutcome: + outcomeKinds.has('resolved_autonomously') || outcomeKinds.has('resolved_via_procedure'), + escalated, + followedUp, + hasPositiveFeedback, + hasNegativeFeedback, + }); + const effort = scoreEffort({ + inboundCount: inbound.length, + outboundCount: outbound.length, + escalated, + followedUp, + pendingApprovals: approvals.filter((approval) => approval.status === 'pending').length, + }); + const overall = weightedScore({ groundedness, tone, resolution, effort }); + const now = Date.now(); + const signals = { + inbound_count: inbound.length, + outbound_count: outbound.length, + cited_knowledge_count: citedIds.size, + proposed_confidence: proposedConfidence || null, + risk_reasons: unique(risks), + outcome_kinds: [...outcomeKinds], + feedback: feedback.map((item) => ({ rating: item.rating, source: item.source })), + }; + await env.DB.prepare( + `INSERT INTO conversation_score ( + id, workspace_id, ticket_id, groundedness_score, tone_score, resolution_score, + effort_score, overall_score, signals_json, scored_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(workspace_id, ticket_id) DO UPDATE SET + groundedness_score = excluded.groundedness_score, + tone_score = excluded.tone_score, + resolution_score = excluded.resolution_score, + effort_score = excluded.effort_score, + overall_score = excluded.overall_score, + signals_json = excluded.signals_json, + scored_at = excluded.scored_at, + updated_at = excluded.updated_at`, + ) + .bind( + ids.conversationScore(), + workspaceId, + ticketId, + groundedness, + tone, + resolution, + effort, + overall, + JSON.stringify(signals), + now, + now, + ) + .run(); + return (await getConversationScore(env, workspaceId, ticketId))!; +} + +export async function listConversationScores( + env: Env, + workspaceId: string, + limit = 50, +): Promise { + const rows = await env.DB.prepare( + `SELECT s.*, t.subject, t.status, t.category + FROM conversation_score s + JOIN ticket t ON t.id = s.ticket_id AND t.workspace_id = s.workspace_id + WHERE s.workspace_id = ? + ORDER BY s.overall_score ASC, s.scored_at DESC + LIMIT ?`, + ) + .bind(workspaceId, Math.min(Math.max(limit, 1), 200)) + .all(); + return rows.results ?? []; +} + +export async function getInsightSummary( + env: Env, + workspaceId: string, + days = 30, +): Promise { + const rangeDays = Math.min(Math.max(days, 1), 365); + const since = Date.now() - rangeDays * 24 * 60 * 60 * 1000; + const [ticketRows, outcomes, feedback, scoreRows, unresolved, procedures] = await Promise.all([ + env.DB.prepare(`SELECT status FROM ticket WHERE workspace_id = ? AND created_at >= ?`) + .bind(workspaceId, since) + .all<{ status: string }>(), + env.DB.prepare( + `SELECT kind, payload_json FROM ticket_outcome_event + WHERE workspace_id = ? AND created_at >= ?`, + ) + .bind(workspaceId, since) + .all<{ kind: string; payload_json: string }>(), + env.DB.prepare(`SELECT rating FROM ticket_feedback WHERE workspace_id = ? AND created_at >= ?`) + .bind(workspaceId, since) + .all<{ rating: 'positive' | 'negative' }>(), + env.DB.prepare( + `SELECT s.groundedness_score, s.tone_score, s.resolution_score, s.effort_score, s.overall_score + FROM conversation_score s + JOIN ticket t ON t.id = s.ticket_id AND t.workspace_id = s.workspace_id + WHERE s.workspace_id = ? AND t.created_at >= ?`, + ) + .bind(workspaceId, since) + .all< + Pick< + ConversationScore, + | 'groundedness_score' + | 'tone_score' + | 'resolution_score' + | 'effort_score' + | 'overall_score' + > + >(), + env.DB.prepare( + `SELECT id, subject, category, status FROM ticket + WHERE workspace_id = ? AND status IN ('open','pending') AND updated_at >= ? + ORDER BY updated_at DESC LIMIT 200`, + ) + .bind(workspaceId, since) + .all<{ id: string; subject: string; category: string | null; status: string }>(), + env.DB.prepare( + `SELECT p.id AS procedure_id, p.slug, p.name, r.status, + COALESCE(r.completed_at, r.updated_at) - COALESCE(r.started_at, r.created_at) AS duration_ms + FROM procedure_run r + JOIN "procedure" p ON p.id = r.procedure_id AND p.workspace_id = r.workspace_id + WHERE r.workspace_id = ? AND r.created_at >= ?`, + ) + .bind(workspaceId, since) + .all<{ + procedure_id: string; + slug: string; + name: string; + status: string; + duration_ms: number; + }>(), + ]); + + const tickets = ticketRows.results ?? []; + const outcomeRows = outcomes.results ?? []; + const feedbackRows = feedback.results ?? []; + const resolved = tickets.filter( + (ticket) => ticket.status === 'resolved' || ticket.status === 'closed', + ).length; + return { + range_days: rangeDays, + ticket_count: tickets.length, + resolved_ticket_count: resolved, + resolution_rate: tickets.length ? round4(resolved / tickets.length) : 0, + open_ticket_count: tickets.filter((ticket) => ticket.status === 'open').length, + pending_ticket_count: tickets.filter((ticket) => ticket.status === 'pending').length, + escalated_count: outcomeRows.filter((outcome) => outcome.kind === 'escalated').length, + customer_followed_up_count: outcomeRows.filter( + (outcome) => outcome.kind === 'customer_followed_up', + ).length, + positive_feedback_count: feedbackRows.filter((item) => item.rating === 'positive').length, + negative_feedback_count: feedbackRows.filter((item) => item.rating === 'negative').length, + avg_groundedness_score: averageScore(scoreRows.results ?? [], 'groundedness_score'), + avg_tone_score: averageScore(scoreRows.results ?? [], 'tone_score'), + avg_resolution_score: averageScore(scoreRows.results ?? [], 'resolution_score'), + avg_effort_score: averageScore(scoreRows.results ?? [], 'effort_score'), + avg_overall_score: averageScore(scoreRows.results ?? [], 'overall_score'), + escalation_reasons: topCounts( + outcomeRows + .filter((outcome) => outcome.kind === 'escalated') + .map((outcome) => escalationReason(safeJson(outcome.payload_json))), + 8, + ).map(([reason, count]) => ({ reason, count })), + top_unresolved_intents: topUnresolvedIntents(unresolved.results ?? []), + slowest_procedures: slowestProcedures(procedures.results ?? []), + }; +} + +export async function generateKbSuggestions( + env: Env, + workspaceId: string, + limit = 100, +): Promise<{ generated: number; suggestions: KbSuggestion[] }> { + const tickets = await env.DB.prepare( + `SELECT id, subject, category, status, updated_at + FROM ticket + WHERE workspace_id = ? AND status IN ('open','pending') + ORDER BY updated_at DESC LIMIT ?`, + ) + .bind(workspaceId, Math.min(Math.max(limit, 1), 300)) + .all<{ + id: string; + subject: string; + category: string | null; + status: string; + updated_at: number; + }>(); + const idsByIntent = new Map>(); + for (const ticket of tickets.results ?? []) { + const intent = inferTicketIntent(ticket.category, ticket.subject); + if (!intent) continue; + const rows = idsByIntent.get(intent) ?? []; + rows.push({ id: ticket.id, subject: ticket.subject }); + idsByIntent.set(intent, rows); + } + + const suggestions: KbSuggestion[] = []; + const eligibleClusters = [...idsByIntent.entries()] + .filter(([, rows]) => rows.length >= MIN_SUGGESTION_TICKETS) + .sort((a, b) => b[1].length - a[1].length || a[0].localeCompare(b[0])) + .slice(0, 10); + for (const [intent, rows] of eligibleClusters) { + const clusterKey = `unresolved:${await sha256Hex(intent.toLowerCase())}`; + const sourceTicketIds = rows.slice(0, 20).map((row) => row.id); + const terms = topTerms(rows.map((row) => row.subject).join(' '), 12); + const confidence = suggestionConfidence(rows.length, terms.length); + const title = `Document ${humanizeIntent(intent)}`; + const body = [ + `# ${title}`, + '', + '## Evidence', + `- Unresolved conversations: ${rows.length}`, + `- Suggested terms: ${terms.join(', ') || 'none'}`, + `- Confidence: ${Math.round(confidence * 100)}%`, + '', + '## Customer questions to cover', + ...rows.slice(0, 6).map((row) => `- ${row.subject}`), + '', + '## Draft answer', + 'Add the approved support policy, required customer details, edge cases, and escalation rules here before publishing.', + '', + '## Source tickets', + ...sourceTicketIds.map((ticketId) => `- ${ticketId}`), + ].join('\n'); + await upsertKbSuggestion(env, workspaceId, { + clusterKey, + title, + summary: `${rows.length} unresolved ${humanizeIntent(intent).toLowerCase()} conversation${rows.length === 1 ? '' : 's'} need a reusable answer.`, + body, + sourceTicketIds, + terms, + confidence, + }); + const suggestion = await getKbSuggestionByCluster(env, workspaceId, clusterKey); + if (suggestion?.status === 'open') suggestions.push(suggestion); + } + return { generated: suggestions.length, suggestions }; +} + +export async function listKbSuggestions( + env: Env, + workspaceId: string, + status?: KbSuggestionStatus, +): Promise { + const where = status ? 'WHERE workspace_id = ? AND status = ?' : 'WHERE workspace_id = ?'; + const rows = await env.DB.prepare( + `SELECT * FROM kb_suggestion ${where} ORDER BY updated_at DESC LIMIT 100`, + ) + .bind(...(status ? [workspaceId, status] : [workspaceId])) + .all(); + return rows.results ?? []; +} + +export async function updateKbSuggestionStatus( + env: Env, + workspaceId: string, + suggestionId: string, + status: Exclude, + actorUserId?: string, +): Promise { + const current = await getKbSuggestion(env, workspaceId, suggestionId); + if (!current) return null; + if (current.status === 'accepted') { + throw new Error('kb_suggestion_accepted'); + } + await env.DB.prepare( + `UPDATE kb_suggestion SET status = ?, updated_at = ? WHERE id = ? AND workspace_id = ?`, + ) + .bind(status, Date.now(), suggestionId, workspaceId) + .run(); + const suggestion = await getKbSuggestion(env, workspaceId, suggestionId); + if (suggestion) { + await audit(env, { + workspaceId, + actorType: actorUserId ? 'user' : 'system', + actorId: actorUserId, + action: 'insights.kb_suggestion_status_updated', + payload: { suggestionId, status }, + }); + } + return suggestion; +} + +export async function acceptKbSuggestion( + env: Env, + workspaceId: string, + suggestionId: string, + actorUserId?: string, +): Promise<{ suggestion: KbSuggestion; sourceId: string } | null> { + const suggestion = await getKbSuggestion(env, workspaceId, suggestionId); + if (!suggestion) return null; + if (suggestion.status === 'accepted' && suggestion.accepted_source_id) { + return { suggestion, sourceId: suggestion.accepted_source_id }; + } + if (suggestion.status !== 'open') throw new Error('kb_suggestion_not_open'); + const sourceId = sourceIdForSuggestion(suggestion.id); + const result = await ingestKnowledgeSource(env, workspaceId, { + kind: 'manual', + title: suggestion.title, + body: suggestion.body_markdown, + sourceId, + }); + const now = Date.now(); + await env.DB.prepare( + `UPDATE kb_suggestion + SET status = 'accepted', + accepted_source_id = ?, + accepted_by_user_id = ?, + accepted_at = ?, + updated_at = ? + WHERE id = ? AND workspace_id = ?`, + ) + .bind(result.sourceId, actorUserId ?? null, now, now, suggestionId, workspaceId) + .run(); + const updated = await getKbSuggestion(env, workspaceId, suggestionId); + await audit(env, { + workspaceId, + actorType: actorUserId ? 'user' : 'system', + actorId: actorUserId, + action: 'insights.kb_suggestion_accepted', + payload: { suggestionId, sourceId: result.sourceId }, + }); + return updated ? { suggestion: updated, sourceId: result.sourceId } : null; +} + +export async function detectKnowledgeDrift( + env: Env, + workspaceId: string, +): Promise<{ detected: number; signals: KnowledgeDriftSignal[] }> { + const sources = await env.DB.prepare( + `SELECT s.id, s.title, COALESCE(SUM(c.used_in_answers_count), 0) AS used_count + FROM knowledge_source s + LEFT JOIN knowledge_chunk c ON c.source_id = s.id AND c.workspace_id = s.workspace_id + WHERE s.workspace_id = ? AND s.status = 'ready' + GROUP BY s.id + HAVING COALESCE(SUM(c.used_in_answers_count), 0) > 0 + ORDER BY used_count DESC, s.updated_at DESC LIMIT 50`, + ) + .bind(workspaceId) + .all<{ id: string; title: string; used_count: number }>(); + const signals: KnowledgeDriftSignal[] = []; + for (const source of sources.results ?? []) { + const sourceChunks = await sourceChunksForDrift(env, workspaceId, source.id); + const sourceBody = sourceChunks.map((chunk) => chunk.body).join('\n\n'); + if (!sourceBody.trim()) continue; + const citedTicketIds = await citedTicketIdsForSource( + env, + workspaceId, + sourceChunks.map((chunk) => chunk.id), + ); + const replies = await successfulReplyCorpus(env, workspaceId, citedTicketIds); + if (replies.length < MIN_DRIFT_REPLIES) continue; + const replyTerms = termCounts(replies.map((reply) => reply.preview).join(' ')); + const sourceTerms = new Set(topTerms(sourceBody, 200)); + const divergent = [...replyTerms.entries()] + .filter(([term, count]) => count >= 2 && !sourceTerms.has(term)) + .sort((a, b) => b[1] - a[1]) + .slice(0, 12) + .map(([term]) => term); + if (divergent.length < 3) continue; + const signalHash = await sha256Hex(`${source.id}:${divergent.join('|')}`); + const severity = divergent.length >= 8 ? 'high' : divergent.length >= 5 ? 'medium' : 'low'; + await upsertDriftSignal(env, workspaceId, { + sourceId: source.id, + signalHash, + severity, + title: `${source.title} may be drifting`, + summary: `Recent successful replies mention terms not covered by this source: ${divergent.slice(0, 5).join(', ')}.`, + successfulReplyCount: replies.length, + divergenceTerms: divergent, + exampleTicketIds: unique(replies.map((reply) => reply.ticket_id)).slice(0, 10), + }); + const signal = await getDriftSignalByHash(env, workspaceId, source.id, signalHash); + if (signal?.status === 'open') signals.push(signal); + } + return { detected: signals.length, signals }; +} + +export async function listKnowledgeDriftSignals( + env: Env, + workspaceId: string, + status?: KnowledgeDriftStatus, +): Promise { + const where = status ? 'WHERE workspace_id = ? AND status = ?' : 'WHERE workspace_id = ?'; + const rows = await env.DB.prepare( + `SELECT * FROM knowledge_drift_signal ${where} + ORDER BY CASE severity WHEN 'high' THEN 0 WHEN 'medium' THEN 1 ELSE 2 END, detected_at DESC + LIMIT 100`, + ) + .bind(...(status ? [workspaceId, status] : [workspaceId])) + .all(); + return rows.results ?? []; +} + +export async function updateKnowledgeDriftStatus( + env: Env, + workspaceId: string, + signalId: string, + status: KnowledgeDriftStatus, + actorUserId?: string, +): Promise { + await env.DB.prepare( + `UPDATE knowledge_drift_signal SET status = ?, updated_at = ? WHERE id = ? AND workspace_id = ?`, + ) + .bind(status, Date.now(), signalId, workspaceId) + .run(); + const signal = await getDriftSignal(env, workspaceId, signalId); + if (signal) { + await audit(env, { + workspaceId, + actorType: actorUserId ? 'user' : 'system', + actorId: actorUserId, + action: 'insights.knowledge_drift_status_updated', + payload: { signalId, status }, + }); + } + return signal; +} + +export async function runWorkspaceInsightsMaintenance( + env: Env, + workspaceId: string, +): Promise<{ scored: number; suggestions: number; drift: number }> { + const [scores, suggestions, drift] = await Promise.all([ + scoreWorkspaceConversations(env, workspaceId, 200), + generateKbSuggestions(env, workspaceId, 200), + detectKnowledgeDrift(env, workspaceId), + ]); + return { scored: scores.scored, suggestions: suggestions.generated, drift: drift.detected }; +} + +export async function runAllWorkspaceInsightsMaintenance( + env: Env, +): Promise> { + const rows = await env.DB.prepare( + `SELECT id FROM workspace WHERE archived_at IS NULL AND deleted_at IS NULL ORDER BY created_at ASC`, + ).all<{ id: string }>(); + const results = []; + for (const row of rows.results ?? []) { + const result = await runWorkspaceInsightsMaintenance(env, row.id); + results.push({ workspaceId: row.id, ...result }); + } + return results; +} + +async function getTicket( + env: Env, + workspaceId: string, + ticketId: string, +): Promise { + return env.DB.prepare( + `SELECT id, workspace_id, subject, status, priority, category, requester_email, created_at, updated_at + FROM ticket WHERE workspace_id = ? AND id = ?`, + ) + .bind(workspaceId, ticketId) + .first(); +} + +async function listMessages( + env: Env, + workspaceId: string, + ticketId: string, +): Promise { + const rows = await env.DB.prepare( + `SELECT id, ticket_id, workspace_id, direction, preview, sent_at, created_at + FROM message_index WHERE workspace_id = ? AND ticket_id = ? ORDER BY sent_at ASC`, + ) + .bind(workspaceId, ticketId) + .all(); + return rows.results ?? []; +} + +async function listApprovals( + env: Env, + workspaceId: string, + ticketId: string, +): Promise { + const rows = await env.DB.prepare( + `SELECT kind, status, proposed_json, risk_reasons_json, created_at + FROM approval_request WHERE workspace_id = ? AND ticket_id = ? ORDER BY created_at DESC`, + ) + .bind(workspaceId, ticketId) + .all(); + return rows.results ?? []; +} + +async function listOutcomes( + env: Env, + workspaceId: string, + ticketId: string, +): Promise { + const rows = await env.DB.prepare( + `SELECT kind, confidence_score, payload_json, created_at + FROM ticket_outcome_event WHERE workspace_id = ? AND ticket_id = ? ORDER BY created_at DESC`, + ) + .bind(workspaceId, ticketId) + .all(); + return rows.results ?? []; +} + +async function listFeedback( + env: Env, + workspaceId: string, + ticketId: string, +): Promise { + const rows = await env.DB.prepare( + `SELECT rating, source, comment, created_at + FROM ticket_feedback WHERE workspace_id = ? AND ticket_id = ? ORDER BY created_at DESC`, + ) + .bind(workspaceId, ticketId) + .all(); + return rows.results ?? []; +} + +async function getConversationScore( + env: Env, + workspaceId: string, + ticketId: string, +): Promise { + return env.DB.prepare( + `SELECT s.*, t.subject, t.status, t.category + FROM conversation_score s + JOIN ticket t ON t.id = s.ticket_id AND t.workspace_id = s.workspace_id + WHERE s.workspace_id = ? AND s.ticket_id = ?`, + ) + .bind(workspaceId, ticketId) + .first(); +} + +async function upsertKbSuggestion( + env: Env, + workspaceId: string, + input: { + clusterKey: string; + title: string; + summary: string; + body: string; + sourceTicketIds: string[]; + terms: string[]; + confidence: number; + }, +): Promise { + const now = Date.now(); + await env.DB.prepare( + `INSERT INTO kb_suggestion ( + id, workspace_id, cluster_key, title, summary, body_markdown, + source_ticket_ids_json, suggested_terms_json, evidence_count, confidence_score, + status, source, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'open', 'unresolved_cluster', ?, ?) + ON CONFLICT(workspace_id, cluster_key) DO UPDATE SET + title = CASE WHEN kb_suggestion.status = 'open' THEN excluded.title ELSE kb_suggestion.title END, + summary = CASE WHEN kb_suggestion.status = 'open' THEN excluded.summary ELSE kb_suggestion.summary END, + body_markdown = CASE WHEN kb_suggestion.status = 'open' THEN excluded.body_markdown ELSE kb_suggestion.body_markdown END, + source_ticket_ids_json = CASE WHEN kb_suggestion.status = 'open' THEN excluded.source_ticket_ids_json ELSE kb_suggestion.source_ticket_ids_json END, + suggested_terms_json = CASE WHEN kb_suggestion.status = 'open' THEN excluded.suggested_terms_json ELSE kb_suggestion.suggested_terms_json END, + evidence_count = CASE WHEN kb_suggestion.status = 'open' THEN excluded.evidence_count ELSE kb_suggestion.evidence_count END, + confidence_score = CASE WHEN kb_suggestion.status = 'open' THEN excluded.confidence_score ELSE kb_suggestion.confidence_score END, + updated_at = CASE WHEN kb_suggestion.status = 'open' THEN excluded.updated_at ELSE kb_suggestion.updated_at END`, + ) + .bind( + ids.kbSuggestion(), + workspaceId, + input.clusterKey, + input.title, + input.summary, + input.body, + JSON.stringify(input.sourceTicketIds), + JSON.stringify(input.terms), + input.sourceTicketIds.length, + input.confidence, + now, + now, + ) + .run(); +} + +async function getKbSuggestionByCluster( + env: Env, + workspaceId: string, + clusterKey: string, +): Promise { + return env.DB.prepare(`SELECT * FROM kb_suggestion WHERE workspace_id = ? AND cluster_key = ?`) + .bind(workspaceId, clusterKey) + .first(); +} + +async function getKbSuggestion( + env: Env, + workspaceId: string, + suggestionId: string, +): Promise { + return env.DB.prepare(`SELECT * FROM kb_suggestion WHERE workspace_id = ? AND id = ?`) + .bind(workspaceId, suggestionId) + .first(); +} + +async function successfulReplyCorpus( + env: Env, + workspaceId: string, + ticketIds?: string[], +): Promise> { + if (ticketIds && ticketIds.length === 0) return []; + const ticketFilter = ticketIds?.length + ? `AND t.id IN (${ticketIds.map(() => '?').join(',')})` + : ''; + const rows = await env.DB.prepare( + `SELECT DISTINCT t.id AS ticket_id, m.preview + FROM ticket t + JOIN message_index m ON m.ticket_id = t.id AND m.workspace_id = t.workspace_id + LEFT JOIN ticket_feedback f ON f.ticket_id = t.id AND f.workspace_id = t.workspace_id + LEFT JOIN ticket_outcome_event o ON o.ticket_id = t.id AND o.workspace_id = t.workspace_id + WHERE t.workspace_id = ? + AND m.direction = 'outbound' + AND m.preview IS NOT NULL + ${ticketFilter} + AND ( + t.status IN ('resolved','closed') + OR f.rating = 'positive' + OR o.kind IN ('resolved_autonomously','resolved_via_procedure') + ) + ORDER BY m.sent_at DESC LIMIT 100`, + ) + .bind(workspaceId, ...(ticketIds ?? [])) + .all<{ ticket_id: string; preview: string }>(); + return rows.results ?? []; +} + +async function sourceChunksForDrift( + env: Env, + workspaceId: string, + sourceId: string, +): Promise> { + const rows = await env.DB.prepare( + `SELECT id, body FROM knowledge_chunk WHERE workspace_id = ? AND source_id = ? ORDER BY ordinal ASC`, + ) + .bind(workspaceId, sourceId) + .all<{ id: string; body: string }>(); + return rows.results ?? []; +} + +async function citedTicketIdsForSource( + env: Env, + workspaceId: string, + chunkIds: string[], +): Promise { + const lineageIds = chunkIds.slice(0, MAX_SOURCE_CHUNKS_FOR_LINEAGE); + if (lineageIds.length === 0) return []; + const conditions = lineageIds.map(() => `proposed_json LIKE ? ESCAPE '\\'`).join(' OR '); + const rows = await env.DB.prepare( + `SELECT DISTINCT ticket_id + FROM approval_request + WHERE workspace_id = ? + AND (${conditions}) + ORDER BY created_at DESC + LIMIT 100`, + ) + .bind(workspaceId, ...lineageIds.map((id) => `%${escapeLike(JSON.stringify(id))}%`)) + .all<{ ticket_id: string }>(); + return (rows.results ?? []).map((row) => row.ticket_id); +} + +async function upsertDriftSignal( + env: Env, + workspaceId: string, + input: { + sourceId: string; + signalHash: string; + severity: 'low' | 'medium' | 'high'; + title: string; + summary: string; + successfulReplyCount: number; + divergenceTerms: string[]; + exampleTicketIds: string[]; + }, +): Promise { + const now = Date.now(); + await env.DB.prepare( + `INSERT INTO knowledge_drift_signal ( + id, workspace_id, source_id, signal_hash, severity, title, summary, + successful_reply_count, divergence_terms_json, example_ticket_ids_json, + status, detected_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'open', ?, ?) + ON CONFLICT(workspace_id, source_id, signal_hash) DO UPDATE SET + severity = excluded.severity, + title = excluded.title, + summary = excluded.summary, + successful_reply_count = excluded.successful_reply_count, + divergence_terms_json = excluded.divergence_terms_json, + example_ticket_ids_json = excluded.example_ticket_ids_json, + updated_at = CASE WHEN knowledge_drift_signal.status = 'open' THEN excluded.updated_at ELSE knowledge_drift_signal.updated_at END`, + ) + .bind( + ids.knowledgeDriftSignal(), + workspaceId, + input.sourceId, + input.signalHash, + input.severity, + input.title, + input.summary, + input.successfulReplyCount, + JSON.stringify(input.divergenceTerms), + JSON.stringify(input.exampleTicketIds), + now, + now, + ) + .run(); +} + +async function getDriftSignalByHash( + env: Env, + workspaceId: string, + sourceId: string, + signalHash: string, +): Promise { + return env.DB.prepare( + `SELECT * FROM knowledge_drift_signal WHERE workspace_id = ? AND source_id = ? AND signal_hash = ?`, + ) + .bind(workspaceId, sourceId, signalHash) + .first(); +} + +async function getDriftSignal( + env: Env, + workspaceId: string, + signalId: string, +): Promise { + return env.DB.prepare(`SELECT * FROM knowledge_drift_signal WHERE workspace_id = ? AND id = ?`) + .bind(workspaceId, signalId) + .first(); +} + +function scoreGroundedness(input: { + hasOutbound: boolean; + citedCount: number; + proposedConfidence: number; + groundedTrace: boolean; + hasKnowledgeHits: boolean; + risks: string[]; +}): number { + if (!input.hasOutbound) return 0.2; + let score = input.citedCount > 0 ? 0.78 : input.hasKnowledgeHits ? 0.62 : 0.52; + if (input.groundedTrace) score += 0.08; + if (input.proposedConfidence > 0) score += Math.min(0.1, input.proposedConfidence * 0.1); + if (input.risks.some((risk) => /insufficient|uncited|weak_retrieval|stale/i.test(risk))) { + score -= 0.25; + } + return clamp01(score); +} + +function scoreTone(text: string): number { + if (!text.trim()) return 0.5; + const lower = text.toLowerCase(); + let score = 0.82; + if (/\b(thank|thanks|please|happy to help|i can help)\b/.test(lower)) score += 0.08; + if (/\b(stupid|obvious|not our problem|as stated|you failed|you must)\b/.test(lower)) + score -= 0.28; + if (/[A-Z]{12,}/.test(text)) score -= 0.12; + if (lower.length < 40) score -= 0.08; + return clamp01(score); +} + +function scoreResolution( + status: string, + signals: { + resolvedByOutcome: boolean; + escalated: boolean; + followedUp: boolean; + hasPositiveFeedback: boolean; + hasNegativeFeedback: boolean; + }, +): number { + let score = + status === 'resolved' || status === 'closed' ? 0.82 : status === 'pending' ? 0.5 : 0.28; + if (signals.resolvedByOutcome) score += 0.1; + if (signals.escalated) score -= 0.15; + if (signals.followedUp) score -= 0.22; + if (signals.hasPositiveFeedback) score += 0.1; + if (signals.hasNegativeFeedback) score -= 0.25; + return clamp01(score); +} + +function scoreEffort(input: { + inboundCount: number; + outboundCount: number; + escalated: boolean; + followedUp: boolean; + pendingApprovals: number; +}): number { + let score = 0.94; + score -= Math.max(0, input.inboundCount - 1) * 0.08; + score -= Math.max(0, input.outboundCount - 2) * 0.05; + if (input.escalated) score -= 0.12; + if (input.followedUp) score -= 0.18; + if (input.pendingApprovals > 0) score -= 0.08; + return clamp01(score); +} + +function weightedScore(scores: { + groundedness: number; + tone: number; + resolution: number; + effort: number; +}): number { + return round4( + scores.groundedness * 0.3 + scores.tone * 0.2 + scores.resolution * 0.35 + scores.effort * 0.15, + ); +} + +function averageScore>(rows: T[], key: keyof T): number | null { + if (rows.length === 0) return null; + return round4(rows.reduce((sum, row) => sum + row[key], 0) / rows.length); +} + +function topUnresolvedIntents( + tickets: Array<{ id: string; subject: string; category: string | null }>, +): InsightSummary['top_unresolved_intents'] { + const groups = new Map(); + for (const ticket of tickets) { + const intent = inferTicketIntent(ticket.category, ticket.subject); + const current = groups.get(intent) ?? { count: 0, example: ticket.id }; + current.count += 1; + groups.set(intent, current); + } + return [...groups.entries()] + .sort((a, b) => b[1].count - a[1].count) + .slice(0, 8) + .map(([intent, value]) => ({ + intent, + count: value.count, + example_ticket_id: value.example, + })); +} + +function slowestProcedures( + rows: Array<{ + procedure_id: string; + slug: string; + name: string; + status: string; + duration_ms: number; + }>, +): InsightSummary['slowest_procedures'] { + const groups = new Map< + string, + { + procedure_id: string; + slug: string; + name: string; + durations: number[]; + waiting: number; + failed: number; + } + >(); + for (const row of rows) { + const current = groups.get(row.procedure_id) ?? { + procedure_id: row.procedure_id, + slug: row.slug, + name: row.name, + durations: [], + waiting: 0, + failed: 0, + }; + current.durations.push(Math.max(0, row.duration_ms ?? 0)); + if (row.status === 'waiting') current.waiting += 1; + if (row.status === 'failed') current.failed += 1; + groups.set(row.procedure_id, current); + } + return [...groups.values()] + .map((group) => ({ + procedure_id: group.procedure_id, + slug: group.slug, + name: group.name, + run_count: group.durations.length, + avg_duration_ms: Math.round( + group.durations.reduce((sum, value) => sum + value, 0) / group.durations.length, + ), + waiting_count: group.waiting, + failed_count: group.failed, + })) + .sort((a, b) => b.avg_duration_ms - a.avg_duration_ms) + .slice(0, 8); +} + +function escalationReason(payload: Record): string { + const reason = String( + payload.reason ?? payload.routeTo ?? payload.route_to ?? payload.severity ?? '', + ).trim(); + return reason.slice(0, 120) || 'Escalated'; +} + +function inferTicketIntent(category: string | null, subject: string): string { + const subjectTerms = topTerms(subject, 2); + const normalizedCategory = category?.trim().toLowerCase(); + if (normalizedCategory && subjectTerms.length > 0) { + return `${normalizedCategory} ${subjectTerms.join(' ')}`; + } + return subjectTerms.length ? subjectTerms.join(' ') : normalizedCategory || 'uncategorized'; +} + +function humanizeIntent(intent: string): string { + return ( + intent + .split(/[\s_-]+/) + .filter(Boolean) + .map((word) => word.charAt(0).toUpperCase() + word.slice(1)) + .join(' ') || 'Support Topic' + ); +} + +function topTerms(text: string, limit: number): string[] { + return [...termCounts(text).entries()] + .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])) + .slice(0, limit) + .map(([term]) => term); +} + +function termCounts(text: string): Map { + const counts = new Map(); + for (const term of text.toLowerCase().match(/[a-z0-9][a-z0-9'-]{2,}/g) ?? []) { + const normalized = term.replace(/^['-]+|['-]+$/g, ''); + if (normalized.length < 3 || STOP_WORDS.has(normalized)) continue; + counts.set(normalized, (counts.get(normalized) ?? 0) + 1); + } + return counts; +} + +function topCounts(values: string[], limit: number): Array<[string, number]> { + const counts = new Map(); + for (const value of values) counts.set(value, (counts.get(value) ?? 0) + 1); + return [...counts.entries()] + .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])) + .slice(0, limit); +} + +function suggestionConfidence(ticketCount: number, termCount: number): number { + return clamp01(Math.min(0.95, 0.48 + ticketCount * 0.1 + Math.min(termCount, 8) * 0.025)); +} + +function sourceIdForSuggestion(suggestionId: string): string { + const suffix = suggestionId.startsWith('kb_sug_') + ? suggestionId.slice('kb_sug_'.length) + : suggestionId; + return `ksrc_sug_${suffix}`; +} + +function escapeLike(value: string): string { + return value.replace(/[\\%_]/g, (match) => `\\${match}`); +} + +function safeJson(value: string): Record { + try { + const parsed = JSON.parse(value); + return parsed && typeof parsed === 'object' ? parsed : {}; + } catch { + return {}; + } +} + +function asStringArray(value: unknown): string[] { + return Array.isArray(value) + ? value.filter((item): item is string => typeof item === 'string') + : []; +} + +function numberOrZero(value: unknown): number { + return typeof value === 'number' && Number.isFinite(value) ? value : 0; +} + +function hasFinalAnswerableTrace(value: unknown): boolean { + return ( + typeof value === 'object' && + value !== null && + 'finalAnswerable' in value && + value.finalAnswerable === true + ); +} + +function unique(values: T[]): T[] { + return [...new Set(values)]; +} + +function clamp01(value: number): number { + return round4(Math.min(1, Math.max(0, value))); +} + +function round4(value: number): number { + return Number(value.toFixed(4)); +} diff --git a/src/jobs/scheduled.ts b/src/jobs/scheduled.ts index 2f26aef..4d573d3 100644 --- a/src/jobs/scheduled.ts +++ b/src/jobs/scheduled.ts @@ -1,8 +1,13 @@ import type { ExecutionContext, ScheduledController } from '@cloudflare/workers-types'; import type { Env } from '../env'; +import { runAllWorkspaceInsightsMaintenance } from '../insights'; import { runSLASweep } from './sla-sweep'; -export async function handleScheduled(controller: ScheduledController, env: Env, ctx: ExecutionContext): Promise { +export async function handleScheduled( + controller: ScheduledController, + env: Env, + ctx: ExecutionContext, +): Promise { switch (controller.cron) { case '*/5 * * * *': ctx.waitUntil( @@ -11,6 +16,13 @@ export async function handleScheduled(controller: ScheduledController, env: Env, .catch((e) => console.error('sla-sweep failed', e)), ); break; + case '17 3 * * 1': + ctx.waitUntil( + runAllWorkspaceInsightsMaintenance(env) + .then((r) => console.log('insights-maintenance', r)) + .catch((e) => console.error('insights-maintenance failed', e)), + ); + break; default: console.warn('unhandled cron', controller.cron); } diff --git a/src/lib/ids.ts b/src/lib/ids.ts index 61d7ab2..b9e676f 100644 --- a/src/lib/ids.ts +++ b/src/lib/ids.ts @@ -35,4 +35,7 @@ export const ids = { evalCase: () => id('eval_case'), evalRun: () => id('eval_run'), evalResult: () => id('eval_result'), + conversationScore: () => id('score'), + kbSuggestion: () => id('kb_sug'), + knowledgeDriftSignal: () => id('drift'), }; diff --git a/src/procedures/library-data.ts b/src/procedures/library-data.ts new file mode 100644 index 0000000..b1fb62a --- /dev/null +++ b/src/procedures/library-data.ts @@ -0,0 +1,380 @@ +import type { + ProcedureLibraryItem, + ProcedureLibraryMcpToolSpec, + ProcedureSpec, +} from '../types/procedure'; +import { identityTools, privacyTools, shopifyTools, stripeRefundTools } from './library-mcp-tools'; +import { normalizeProcedureSpec } from './schema'; + +export type ProcedureLibrarySeedItem = Omit; + +const refundIntake = normalizeProcedureSpec({ + slug: 'refund-intake', + name: 'Refund intake', + version: '1.0.0', + description: + 'Collect refund context, inspect policy evidence, lookup the customer, and gate approved refunds.', + owner: 'ranse-library', + trigger: { type: 'manual' }, + steps: [ + { + id: 'find_policy', + type: 'search', + query: 'refund policy for {{ ticket.subject }}', + scope: 'knowledge', + max_hops: 2, + save_as: 'policy', + }, + { + id: 'lookup_customer', + type: 'call_action', + tool: 'stripe.customers.search', + args: { email: '{{ customer.email }}' }, + requires_approval: false, + save_as: 'stripe_customer', + }, + { + id: 'refund_gate', + type: 'if', + condition: { var: 'refund.approved', equals: true }, + // biome-ignore lint/suspicious/noThenProperty: Procedure specs intentionally use if/then/else terminology. + then: [ + { + id: 'create_refund', + type: 'call_action', + tool: 'stripe.refunds.create', + args: { + charge_id: '{{ refund.charge_id }}', + amount_cents: '{{ refund.amount_cents }}', + }, + requires_approval: true, + save_as: 'refund_result', + }, + ], + else: [ + { + id: 'add_context_note', + type: 'add_note', + body: 'Refund intake started. Top policy hit: {{ policy.hits.0.title }}', + }, + ], + }, + ], + evals: [ + { + name: 'basic_refund_ticket', + input: { + ticket: { subject: 'Refund request' }, + customer: { email: 'customer@example.com' }, + refund: { approved: false }, + }, + expect: { + status: 'completed', + steps: ['find_policy', 'lookup_customer', 'refund_gate', 'add_context_note'], + step_statuses: { lookup_customer: 'completed' }, + }, + }, + { + name: 'approved_refund_waits_for_operator', + input: { + ticket: { subject: 'Refund request' }, + customer: { email: 'customer@example.com' }, + refund: { approved: true, charge_id: 'ch_123', amount_cents: 2500 }, + }, + expect: { + status: 'waiting', + steps: ['find_policy', 'lookup_customer', 'refund_gate', 'create_refund'], + step_statuses: { create_refund: 'waiting' }, + step_inputs: { + create_refund: { 'args.charge_id': 'ch_123', 'args.amount_cents': 2500 }, + }, + }, + }, + ], +}); + +const passwordReset = normalizeProcedureSpec({ + slug: 'password-reset', + name: 'Password reset', + version: '1.0.0', + description: 'Verifies account-recovery policy and collects the information needed for a reset.', + owner: 'ranse-library', + trigger: { type: 'intent', intent: 'password_reset' }, + steps: [ + { + id: 'find_policy', + type: 'search', + query: 'password reset account recovery verification policy', + scope: 'knowledge', + max_hops: 2, + save_as: 'policy', + }, + { + id: 'has_identifier', + type: 'if', + condition: { var: 'customer.identifier', exists: true }, + // biome-ignore lint/suspicious/noThenProperty: Procedure specs intentionally use if/then/else terminology. + then: [ + { + id: 'lookup_identity', + type: 'call_action', + tool: 'identity.users.lookup', + args: { identifier: '{{ customer.identifier }}' }, + requires_approval: false, + save_as: 'identity_lookup', + }, + { + id: 'has_user_id', + type: 'if', + condition: { var: 'customer.user_id', exists: true }, + // biome-ignore lint/suspicious/noThenProperty: Procedure specs intentionally use if/then/else terminology. + then: [ + { + id: 'create_reset_request', + type: 'call_action', + tool: 'identity.password_resets.create', + args: { user_id: '{{ customer.user_id }}' }, + requires_approval: true, + save_as: 'password_reset_request', + }, + ], + else: [ + { + id: 'ask_user_id', + type: 'ask_customer', + subject: 'Re: {{ ticket.subject }}', + message: + 'I found the account context. Please confirm the account email or username again, and do not include passwords or one-time codes.', + }, + ], + }, + ], + else: [ + { + id: 'ask_identifier', + type: 'ask_customer', + subject: 'Re: {{ ticket.subject }}', + message: + 'I can help with that. Please send the account email or username, and do not include your password or one-time codes.', + }, + ], + }, + ], + evals: [ + { + name: 'waits_for_identifier', + input: { ticket: { subject: 'I cannot log in' } }, + expect: { status: 'waiting', steps: ['find_policy', 'has_identifier', 'ask_identifier'] }, + }, + { + name: 'reset_request_waits_for_operator', + input: { + ticket: { subject: 'I cannot log in' }, + customer: { identifier: 'customer@example.com', user_id: 'user_123' }, + }, + expect: { + status: 'waiting', + steps: [ + 'find_policy', + 'has_identifier', + 'lookup_identity', + 'has_user_id', + 'create_reset_request', + ], + step_statuses: { lookup_identity: 'completed', create_reset_request: 'waiting' }, + step_inputs: { + create_reset_request: { 'args.user_id': 'user_123' }, + }, + }, + }, + ], +}); + +const shippingDispute = normalizeProcedureSpec({ + slug: 'shipping-dispute', + name: 'Shipping dispute', + version: '1.0.0', + description: 'Triage delayed, missing, or damaged shipments and collect order context.', + owner: 'ranse-library', + trigger: { type: 'intent', intent: 'shipping_dispute' }, + steps: [ + { + id: 'find_shipping_policy', + type: 'search', + query: 'shipping delay missing damaged order policy {{ ticket.subject }}', + scope: 'knowledge', + max_hops: 3, + save_as: 'shipping_policy', + }, + { id: 'set_category', type: 'set_ticket_field', field: 'category', value: 'shipping' }, + { + id: 'has_order_query', + type: 'if', + condition: { var: 'order.query', exists: true }, + // biome-ignore lint/suspicious/noThenProperty: Procedure specs intentionally use if/then/else terminology. + then: [ + { + id: 'search_order', + type: 'call_action', + tool: 'shopify.orders.search', + args: { query: '{{ order.query }}' }, + requires_approval: false, + save_as: 'order_matches', + }, + { + id: 'add_order_note', + type: 'add_note', + body: 'Shipping dispute prepared for {{ order.query }}.', + }, + ], + else: [ + { + id: 'ask_order', + type: 'ask_customer', + subject: 'Re: {{ ticket.subject }}', + message: + 'Please send your order number and confirm whether the shipment is delayed, missing, or arrived damaged.', + }, + ], + }, + ], + evals: [ + { + name: 'collects_order_context', + input: { ticket: { subject: 'Package never arrived' } }, + expect: { + status: 'waiting', + context: { 'ticket.category': 'shipping' }, + steps: ['find_shipping_policy', 'set_category', 'has_order_query', 'ask_order'], + }, + }, + { + name: 'looks_up_known_order', + input: { + ticket: { subject: 'Package never arrived' }, + order: { query: '#1001' }, + }, + expect: { + status: 'completed', + context: { 'ticket.category': 'shipping' }, + steps: [ + 'find_shipping_policy', + 'set_category', + 'has_order_query', + 'search_order', + 'add_order_note', + ], + step_statuses: { search_order: 'completed' }, + step_inputs: { + search_order: { 'args.query': '#1001' }, + }, + }, + }, + ], +}); + +const gdprRequest = normalizeProcedureSpec({ + slug: 'gdpr-data-request', + name: 'GDPR data request', + version: '1.0.0', + description: 'Escalates privacy data access or deletion requests with the required urgency.', + owner: 'ranse-library', + trigger: { type: 'intent', intent: 'privacy_request' }, + steps: [ + { id: 'set_priority', type: 'set_ticket_field', field: 'priority', value: 'high' }, + { id: 'set_category', type: 'set_ticket_field', field: 'category', value: 'privacy' }, + { + id: 'escalate_privacy', + type: 'escalate_to', + route_to: 'privacy', + severity: 'high', + reason: 'Potential data access/deletion request requires privacy-owner review.', + }, + { + id: 'create_privacy_request', + type: 'call_action', + tool: 'privacy.requests.create', + args: { ticket_id: '{{ ticket_id }}' }, + requires_approval: true, + save_as: 'privacy_request', + }, + ], + evals: [ + { + name: 'escalates_privacy_request', + input: { ticket_id: 'tkt_privacy', ticket: { subject: 'Delete my account data' } }, + expect: { + status: 'waiting', + context: { 'ticket.priority': 'high', 'ticket.category': 'privacy' }, + steps: ['set_priority', 'set_category', 'escalate_privacy', 'create_privacy_request'], + step_statuses: { create_privacy_request: 'waiting' }, + step_inputs: { + create_privacy_request: { 'args.ticket_id': 'tkt_privacy' }, + }, + }, + }, + ], +}); + +export const PROCEDURE_LIBRARY: ProcedureLibrarySeedItem[] = [ + entry( + refundIntake, + 'billing', + 'Collect refund context with policy evidence.', + 'medium', + ['refund', 'billing', 'policy'], + ['stripe'], + stripeRefundTools, + ), + entry( + passwordReset, + 'account', + 'Collect safe account-recovery context without requesting secrets.', + 'medium', + ['login', 'identity', 'security'], + ['identity'], + identityTools, + ), + entry( + shippingDispute, + 'shipping', + 'Prepare delayed, missing, or damaged shipment tickets.', + 'low', + ['shipping', 'orders', 'returns'], + ['shopify'], + shopifyTools, + ), + entry( + gdprRequest, + 'privacy', + 'Escalate privacy data requests to the right owner.', + 'high', + ['privacy', 'gdpr', 'escalation'], + ['privacy'], + privacyTools, + ), +]; + +function entry( + spec: ProcedureSpec, + category: ProcedureLibraryItem['category'], + summary: string, + riskLevel: ProcedureLibraryItem['risk_level'], + tags: string[], + requiredMcpServers: string[], + referenceMcpTools: ProcedureLibraryMcpToolSpec[], +): ProcedureLibrarySeedItem { + return { + slug: spec.slug, + name: spec.name, + summary, + category, + tags, + risk_level: riskLevel, + required_mcp_servers: requiredMcpServers, + eval_count: spec.evals?.length ?? 0, + version: spec.version, + spec, + reference_mcp_tools: referenceMcpTools, + }; +} diff --git a/src/procedures/library-mcp-tools.ts b/src/procedures/library-mcp-tools.ts new file mode 100644 index 0000000..bd1cc2a --- /dev/null +++ b/src/procedures/library-mcp-tools.ts @@ -0,0 +1,113 @@ +import type { ProcedureLibraryMcpToolSpec } from '../types/procedure'; + +export const stripeRefundTools: ProcedureLibraryMcpToolSpec[] = [ + { + server: 'stripe', + tool: 'customers.search', + title: 'Search Stripe customers', + description: + 'Find a customer record by email or customer id before reviewing refund eligibility.', + input_schema: { + type: 'object', + properties: { email: { type: 'string' } }, + required: ['email'], + }, + annotations: { + readOnlyHint: true, + destructiveHint: false, + openWorldHint: true, + }, + }, + { + server: 'stripe', + tool: 'refunds.create', + title: 'Create Stripe refund', + description: 'Create a refund after policy and operator approval gates pass.', + input_schema: { + type: 'object', + properties: { charge_id: { type: 'string' }, amount_cents: { type: 'integer' } }, + required: ['charge_id', 'amount_cents'], + }, + annotations: { + readOnlyHint: false, + destructiveHint: true, + idempotentHint: true, + openWorldHint: true, + }, + }, +]; + +export const identityTools: ProcedureLibraryMcpToolSpec[] = [ + { + server: 'identity', + tool: 'users.lookup', + title: 'Lookup identity user', + description: 'Lookup the account record after the customer provides an email or username.', + input_schema: { + type: 'object', + properties: { identifier: { type: 'string' } }, + required: ['identifier'], + }, + annotations: { + readOnlyHint: true, + destructiveHint: false, + openWorldHint: false, + }, + }, + { + server: 'identity', + tool: 'password_resets.create', + title: 'Create password-reset ticket', + description: 'Create an internal password-reset request after verification gates pass.', + input_schema: { + type: 'object', + properties: { user_id: { type: 'string' } }, + required: ['user_id'], + }, + annotations: { + readOnlyHint: false, + destructiveHint: false, + idempotentHint: true, + openWorldHint: false, + }, + }, +]; + +export const shopifyTools: ProcedureLibraryMcpToolSpec[] = [ + { + server: 'shopify', + tool: 'orders.search', + title: 'Search Shopify orders', + description: 'Find candidate orders by email, order number, or tracking number.', + input_schema: { + type: 'object', + properties: { query: { type: 'string' } }, + required: ['query'], + }, + annotations: { + readOnlyHint: true, + destructiveHint: false, + openWorldHint: true, + }, + }, +]; + +export const privacyTools: ProcedureLibraryMcpToolSpec[] = [ + { + server: 'privacy', + tool: 'requests.create', + title: 'Create privacy request', + description: 'Open a tracked data-access or deletion request in the privacy system.', + input_schema: { + type: 'object', + properties: { ticket_id: { type: 'string' } }, + required: ['ticket_id'], + }, + annotations: { + readOnlyHint: false, + destructiveHint: false, + idempotentHint: true, + openWorldHint: false, + }, + }, +]; diff --git a/src/procedures/library.ts b/src/procedures/library.ts new file mode 100644 index 0000000..705deb9 --- /dev/null +++ b/src/procedures/library.ts @@ -0,0 +1,267 @@ +import type { Env } from '../env'; +import { sha256Hex } from '../lib/crypto'; +import type { + ProcedureLibraryEntry, + ProcedureLibraryItem, + ProcedureLibraryMcpToolSpec, + ProcedureLibraryManifest, + ProcedureLibraryProvenance, + ProcedureLibraryReadiness, + ProcedureLibraryReadinessTool, + ProcedureLibraryStandards, + ProcedureStep, +} from '../types/procedure'; +import { runProcedureSpecEvals } from '../evals/replay'; +import { listMcpServers, listMcpTools, normalizeMcpServerName } from '../mcp/storage'; +import { PROCEDURE_LIBRARY } from './library-data'; +import { normalizeProcedureSpec, stableStringify } from './schema'; +import { upsertProcedureVersion } from './storage'; + +export const PROCEDURE_LIBRARY_VERSION = '2026-05-18'; +export const PROCEDURE_LIBRARY_STANDARDS: ProcedureLibraryStandards = { + procedure_schema: 'ranse.procedure.v1', + mcp_schema: '2025-11-25', +}; + +export async function listProcedureLibrary(): Promise { + return Promise.all( + PROCEDURE_LIBRARY.map(async (item) => { + const entry = await hydrateLibraryItem(item); + const { spec: _spec, reference_mcp_tools: _tools, ...summary } = entry; + return summary; + }), + ); +} + +export async function listProcedureLibraryWithReadiness( + env: Env, + workspaceId: string, +): Promise { + const [servers, tools] = await Promise.all([ + listMcpServers(env, workspaceId), + listMcpTools(env, workspaceId), + ]); + return Promise.all( + PROCEDURE_LIBRARY.map(async (item) => { + const entry = await hydrateLibraryItem(item); + const { spec: _spec, reference_mcp_tools: _tools, ...summary } = entry; + return { ...summary, readiness: readinessFromInventory(entry, servers, tools) }; + }), + ); +} + +export async function getProcedureLibraryItem(slug: string): Promise { + const item = PROCEDURE_LIBRARY.find((entry) => entry.slug === slug); + return item ? hydrateLibraryItem(item) : null; +} + +export async function getProcedureLibraryManifest(): Promise { + return { + manifest_version: PROCEDURE_LIBRARY_VERSION, + standards: { ...PROCEDURE_LIBRARY_STANDARDS }, + procedures: await Promise.all(PROCEDURE_LIBRARY.map(hydrateLibraryItem)), + }; +} + +export async function getProcedureLibraryReadiness( + env: Env, + workspaceId: string, + slug: string, +): Promise { + const item = await getProcedureLibraryItem(slug); + return item ? assessProcedureLibraryReadiness(env, workspaceId, item) : null; +} + +export async function assessProcedureLibraryReadiness( + env: Env, + workspaceId: string, + item: ProcedureLibraryItem, +): Promise { + const [servers, tools] = await Promise.all([ + listMcpServers(env, workspaceId), + listMcpTools(env, workspaceId), + ]); + return readinessFromInventory(item, servers, tools); +} + +function readinessFromInventory( + item: ProcedureLibraryItem, + servers: Awaited>, + tools: Awaited>, +): ProcedureLibraryReadiness { + const serversByName = new Map(servers.map((server) => [server.name, server])); + const toolsByServer = new Map>(); + for (const tool of tools) { + const names = toolsByServer.get(tool.server_id) ?? new Set(); + names.add(tool.name); + toolsByServer.set(tool.server_id, names); + } + + const readinessTools = item.reference_mcp_tools.map((reference) => { + const serverName = normalizeMcpServerName(reference.server); + const server = serversByName.get(serverName); + const hasTool = server ? toolsByServer.get(server.id)?.has(reference.tool) === true : false; + const status: ProcedureLibraryReadinessTool['status'] = !server + ? 'missing_server' + : server.enabled !== 1 + ? 'server_disabled' + : hasTool + ? 'ready' + : 'missing_tool'; + return { + server: serverName, + tool: reference.tool, + usage: reference.usage ?? 'required', + status, + destructive: reference.annotations?.destructiveHint === true, + read_only: reference.annotations?.readOnlyHint === true, + }; + }); + const required = readinessTools.filter((tool) => tool.usage !== 'optional'); + const readyRequired = required.filter((tool) => tool.status === 'ready'); + return { + status: readyRequired.length === required.length ? 'ready' : 'needs_setup', + ready_tool_count: readyRequired.length, + required_tool_count: required.length, + tools: readinessTools, + }; +} + +export async function validateProcedureLibrary(): Promise> { + const seen = new Set(); + const results: Array<{ slug: string; ok: true }> = []; + for (const item of PROCEDURE_LIBRARY) { + if (seen.has(item.slug)) throw new Error(`procedure_library_duplicate_slug:${item.slug}`); + seen.add(item.slug); + if (!item.spec.evals?.length) throw new Error(`procedure_library_missing_evals:${item.slug}`); + if (!item.reference_mcp_tools.length) { + throw new Error(`procedure_library_missing_mcp_tools:${item.slug}`); + } + for (const tool of item.reference_mcp_tools) { + if (!tool.server || !tool.tool || !tool.input_schema) { + throw new Error(`procedure_library_invalid_mcp_tool:${item.slug}`); + } + if (tool.annotations?.openWorldHint === undefined) { + throw new Error(`procedure_library_missing_open_world_hint:${item.slug}:${tool.tool}`); + } + } + normalizeProcedureSpec(item.spec); + validateLibraryActionContracts(item); + const report = runProcedureSpecEvals(item.spec); + if (report.status !== 'passed') throw new Error(`procedure_library_eval_failed:${item.slug}`); + const hydrated = await hydrateLibraryItem(item); + if (hydrated.provenance.spec_checksum.length !== 64) { + throw new Error(`procedure_library_invalid_checksum:${item.slug}`); + } + results.push({ slug: item.slug, ok: true as const }); + } + return results; +} + +export async function installProcedureFromLibrary( + env: Env, + input: { workspaceId: string; actorUserId: string; slug: string }, +) { + const item = await getProcedureLibraryItem(input.slug); + if (!item) throw new Error('procedure_library_item_not_found'); + return upsertProcedureVersion(env, { + workspaceId: input.workspaceId, + actorUserId: input.actorUserId, + spec: item.spec, + sourceKind: 'seed', + sourceRef: item.provenance.source_ref, + }); +} + +async function hydrateLibraryItem( + item: Omit, +): Promise { + const clone = cloneJson(item); + return { + ...clone, + tags: [...clone.tags], + required_mcp_servers: [...clone.required_mcp_servers], + spec: cloneJson(clone.spec), + reference_mcp_tools: cloneJson(clone.reference_mcp_tools), + provenance: await procedureLibraryProvenance(clone), + }; +} + +async function procedureLibraryProvenance( + item: Omit, +): Promise { + const checksum = await sha256Hex(stableStringify(item.spec)); + return { + source: 'ranse-library', + source_ref: `library:${item.slug}@${item.version}#sha256:${checksum}`, + library_version: PROCEDURE_LIBRARY_VERSION, + spec_checksum_algorithm: 'sha256', + spec_checksum: checksum, + standards: { ...PROCEDURE_LIBRARY_STANDARDS }, + }; +} + +function cloneJson(value: T): T { + return JSON.parse(JSON.stringify(value)) as T; +} + +function validateLibraryActionContracts(item: Omit): void { + const actions = collectCallActionSteps(item.spec.steps); + const actionRefs = new Set(actions.map((step) => step.tool)); + const referenceByQualifiedName = new Map( + item.reference_mcp_tools.map((tool) => [qualifiedMcpToolName(tool), tool]), + ); + const requiredServers = new Set(item.required_mcp_servers.map(normalizeMcpServerName)); + const referencedServers = new Set( + item.reference_mcp_tools.map((tool) => normalizeMcpServerName(tool.server)), + ); + + for (const server of referencedServers) { + if (!requiredServers.has(server)) { + throw new Error(`procedure_library_missing_required_server:${item.slug}:${server}`); + } + } + + for (const step of actions) { + const reference = referenceByQualifiedName.get(step.tool); + if (!reference) { + throw new Error(`procedure_library_missing_mcp_reference:${item.slug}:${step.tool}`); + } + if (reference.annotations?.destructiveHint === true && step.requires_approval === false) { + throw new Error( + `procedure_library_destructive_action_without_approval:${item.slug}:${step.id}`, + ); + } + if (reference.annotations?.readOnlyHint !== true && step.requires_approval === false) { + throw new Error(`procedure_library_write_action_without_approval:${item.slug}:${step.id}`); + } + } + + for (const reference of item.reference_mcp_tools) { + if ((reference.usage ?? 'required') === 'optional') continue; + const qualifiedName = qualifiedMcpToolName(reference); + if (!actionRefs.has(qualifiedName)) { + throw new Error(`procedure_library_unused_mcp_reference:${item.slug}:${qualifiedName}`); + } + } +} + +function collectCallActionSteps( + steps: ProcedureStep[], +): Array> { + const actions: Array> = []; + for (const step of steps) { + if (step.type === 'call_action') actions.push(step); + if (step.type === 'if') + actions.push( + ...collectCallActionSteps(step.then), + ...collectCallActionSteps(step.else ?? []), + ); + if (step.type === 'loop') actions.push(...collectCallActionSteps(step.steps)); + } + return actions; +} + +function qualifiedMcpToolName(tool: ProcedureLibraryMcpToolSpec): string { + return `${normalizeMcpServerName(tool.server)}.${tool.tool}`; +} diff --git a/src/procedures/template.ts b/src/procedures/template.ts index ab554d0..8b12b0e 100644 --- a/src/procedures/template.ts +++ b/src/procedures/template.ts @@ -1,6 +1,7 @@ import type { ProcedureCondition } from '../types/procedure'; const TEMPLATE_EXPR = /\{\{\s*([a-zA-Z0-9_.:-]+)\s*\}\}/g; +const TEMPLATE_VALUE_EXPR = /^\{\{\s*([a-zA-Z0-9_.:-]+)\s*\}\}$/; export function getPath(context: Record, path: string): unknown { const parts = path.split('.').filter(Boolean); @@ -51,7 +52,14 @@ export function renderTemplate(template: string, context: Record(value: T, context: Record): T { - if (typeof value === 'string') return renderTemplate(value, context) as T; + if (typeof value === 'string') { + const wholeValue = value.match(TEMPLATE_VALUE_EXPR); + if (wholeValue) { + const resolved = getPath(context, wholeValue[1]); + return (resolved === null || resolved === undefined ? '' : resolved) as T; + } + return renderTemplate(value, context) as T; + } if (Array.isArray(value)) return value.map((item) => renderValue(item, context)) as T; if (value && typeof value === 'object') { return Object.fromEntries( diff --git a/src/types/insights.ts b/src/types/insights.ts new file mode 100644 index 0000000..12b9689 --- /dev/null +++ b/src/types/insights.ts @@ -0,0 +1,85 @@ +export type KbSuggestionStatus = 'open' | 'accepted' | 'dismissed'; +export type KnowledgeDriftStatus = 'open' | 'resolved' | 'dismissed'; +export type KnowledgeDriftSeverity = 'low' | 'medium' | 'high'; + +export interface ConversationScore { + id: string; + workspace_id: string; + ticket_id: string; + groundedness_score: number; + tone_score: number; + resolution_score: number; + effort_score: number; + overall_score: number; + signals_json: string; + scored_at: number; + updated_at: number; + subject?: string; + status?: string; + category?: string | null; +} + +export interface InsightSummary { + range_days: number; + ticket_count: number; + resolved_ticket_count: number; + resolution_rate: number; + open_ticket_count: number; + pending_ticket_count: number; + escalated_count: number; + customer_followed_up_count: number; + positive_feedback_count: number; + negative_feedback_count: number; + avg_groundedness_score: number | null; + avg_tone_score: number | null; + avg_resolution_score: number | null; + avg_effort_score: number | null; + avg_overall_score: number | null; + escalation_reasons: Array<{ reason: string; count: number }>; + top_unresolved_intents: Array<{ intent: string; count: number; example_ticket_id: string }>; + slowest_procedures: Array<{ + procedure_id: string; + slug: string; + name: string; + run_count: number; + avg_duration_ms: number; + waiting_count: number; + failed_count: number; + }>; +} + +export interface KbSuggestion { + id: string; + workspace_id: string; + cluster_key: string; + title: string; + summary: string; + body_markdown: string; + source_ticket_ids_json: string; + suggested_terms_json: string; + evidence_count: number; + confidence_score: number; + status: KbSuggestionStatus; + source: string; + accepted_source_id: string | null; + accepted_by_user_id: string | null; + accepted_at: number | null; + created_at: number; + updated_at: number; +} + +export interface KnowledgeDriftSignal { + id: string; + workspace_id: string; + source_id: string; + signal_hash: string; + severity: KnowledgeDriftSeverity; + title: string; + summary: string; + successful_reply_count: number; + divergence_terms_json: string; + example_ticket_ids_json: string; + status: KnowledgeDriftStatus; + detected_at: number; + updated_at: number; +} diff --git a/src/types/procedure.ts b/src/types/procedure.ts index 68b8bde..5e1a6c5 100644 --- a/src/types/procedure.ts +++ b/src/types/procedure.ts @@ -2,6 +2,7 @@ import type { KnowledgeSearchScope } from './knowledge'; export type ProcedureTriggerType = 'manual' | 'ticket_created' | 'intent'; export type ProcedureSourceKind = 'api' | 'git' | 'seed'; +export type ProcedureLibraryCategory = 'billing' | 'account' | 'shipping' | 'privacy' | 'triage'; export type ProcedureRunStatus = | 'queued' | 'running' @@ -10,7 +11,11 @@ export type ProcedureRunStatus = | 'failed' | 'cancelled'; export type ProcedureStepRunStatus = 'running' | 'waiting' | 'completed' | 'failed' | 'skipped'; -export type ProcedureEventType = 'customer_reply' | 'approval_decided' | 'manual_resume' | 'timeout'; +export type ProcedureEventType = + | 'customer_reply' + | 'approval_decided' + | 'manual_resume' + | 'timeout'; export interface ProcedureTrigger { type: ProcedureTriggerType; @@ -90,6 +95,76 @@ export interface ProcedureSpec { evals?: Array<{ name: string; input: Record; expect?: Record }>; } +export interface ProcedureLibraryMcpToolSpec { + server: string; + tool: string; + title: string; + description: string; + usage?: 'required' | 'optional'; + input_schema: Record; + annotations?: { + readOnlyHint?: boolean; + destructiveHint?: boolean; + idempotentHint?: boolean; + openWorldHint?: boolean; + }; +} + +export interface ProcedureLibraryStandards { + procedure_schema: 'ranse.procedure.v1'; + mcp_schema: '2025-11-25'; +} + +export interface ProcedureLibraryProvenance { + source: 'ranse-library'; + source_ref: string; + library_version: string; + spec_checksum_algorithm: 'sha256'; + spec_checksum: string; + standards: ProcedureLibraryStandards; +} + +export interface ProcedureLibraryEntry { + slug: string; + name: string; + summary: string; + category: ProcedureLibraryCategory; + tags: string[]; + risk_level: 'low' | 'medium' | 'high'; + required_mcp_servers: string[]; + eval_count: number; + version: string; + provenance: ProcedureLibraryProvenance; + readiness?: ProcedureLibraryReadiness; +} + +export interface ProcedureLibraryItem extends ProcedureLibraryEntry { + spec: ProcedureSpec; + reference_mcp_tools: ProcedureLibraryMcpToolSpec[]; +} + +export interface ProcedureLibraryManifest { + manifest_version: string; + standards: ProcedureLibraryStandards; + procedures: ProcedureLibraryItem[]; +} + +export interface ProcedureLibraryReadinessTool { + server: string; + tool: string; + usage: 'required' | 'optional'; + status: 'ready' | 'missing_server' | 'server_disabled' | 'missing_tool'; + destructive: boolean; + read_only: boolean; +} + +export interface ProcedureLibraryReadiness { + status: 'ready' | 'needs_setup'; + ready_tool_count: number; + required_tool_count: number; + tools: ProcedureLibraryReadinessTool[]; +} + export interface ProcedureListItem { id: string; slug: string; diff --git a/src/ui/api.ts b/src/ui/api.ts index 8d72ded..d318913 100644 --- a/src/ui/api.ts +++ b/src/ui/api.ts @@ -5,6 +5,8 @@ import type { KnowledgeSourceListItem, } from '../types/knowledge'; import type { + ProcedureLibraryItem, + ProcedureLibraryEntry, ProcedureListItem, ProcedureRun, ProcedureRunDetail, @@ -13,6 +15,14 @@ import type { import type { McpServerListItem, McpTool, McpToolCall, McpToolGuardrail } from '../types/mcp'; import type { AuthMe } from '../types/workspace'; import type { EvalCase, EvalRun, EvalRunDetail } from '../types/evals'; +import type { + ConversationScore, + InsightSummary, + KbSuggestion, + KbSuggestionStatus, + KnowledgeDriftSignal, + KnowledgeDriftStatus, +} from '../types/insights'; import { api, uploadFile, uploadKnowledgePdf } from './api-core'; import { workspaceApi } from './api-workspaces'; @@ -23,10 +33,16 @@ export type KnowledgeSearchHit = KnowledgeHit; export type AnswerInspectionHit = KnowledgeInspectionHit; export type AnswerInspectionTrace = AgenticRetrievalTrace; export type ProcedureListEntry = ProcedureListItem; +export type ProcedureLibraryListEntry = ProcedureLibraryEntry; +export type ProcedureLibraryDetail = ProcedureLibraryItem; export type McpServerEntry = McpServerListItem; export type McpToolEntry = McpTool; export type EvalCaseEntry = EvalCase; export type EvalRunEntry = EvalRun; +export type ConversationScoreEntry = ConversationScore; +export type InsightSummaryEntry = InsightSummary; +export type KbSuggestionEntry = KbSuggestion; +export type KnowledgeDriftSignalEntry = KnowledgeDriftSignal; export const API = { setupStatus: () => api<{ completed: boolean }>('/setup/status'), @@ -154,6 +170,17 @@ export const API = { trace?: AnswerInspectionTrace; }>('/api/knowledge/search', { method: 'POST', body: JSON.stringify({ query, limit }) }), listProcedures: () => api<{ procedures: ProcedureListEntry[] }>('/api/procedures'), + listProcedureLibrary: () => + api<{ procedures: ProcedureLibraryListEntry[] }>('/api/procedures/library'), + procedureLibraryItem: (slug: string) => + api<{ procedure: ProcedureLibraryDetail }>(`/api/procedures/library/${slug}`), + installProcedureLibraryItem: (slug: string) => + api<{ procedure: ProcedureListItem; version: unknown; created: boolean }>( + `/api/procedures/library/${slug}/install`, + { + method: 'POST', + }, + ), procedure: (id: string) => api<{ procedure: ProcedureListItem; @@ -268,6 +295,46 @@ export const API = { method: 'POST', body: JSON.stringify(body), }), + insightSummary: (days = 30) => + api<{ summary: InsightSummaryEntry }>(`/api/insights/summary?days=${days}`), + listConversationScores: (limit = 50) => + api<{ scores: ConversationScoreEntry[] }>(`/api/insights/scores?limit=${limit}`), + runConversationScoring: (limit = 100) => + api<{ scored: number; scores: ConversationScoreEntry[] }>('/api/insights/scores/run', { + method: 'POST', + body: JSON.stringify({ limit }), + }), + listKbSuggestions: (status: KbSuggestionStatus = 'open') => + api<{ suggestions: KbSuggestionEntry[] }>(`/api/insights/kb-suggestions?status=${status}`), + generateKbSuggestions: (limit = 100) => + api<{ generated: number; suggestions: KbSuggestionEntry[] }>( + '/api/insights/kb-suggestions/run', + { + method: 'POST', + body: JSON.stringify({ limit }), + }, + ), + updateKbSuggestion: (id: string, status: Exclude) => + api<{ suggestion: KbSuggestionEntry }>(`/api/insights/kb-suggestions/${id}`, { + method: 'PATCH', + body: JSON.stringify({ status }), + }), + acceptKbSuggestion: (id: string) => + api<{ suggestion: KbSuggestionEntry; sourceId: string }>( + `/api/insights/kb-suggestions/${id}/accept`, + { method: 'POST' }, + ), + listKnowledgeDrift: (status: KnowledgeDriftStatus = 'open') => + api<{ signals: KnowledgeDriftSignalEntry[] }>(`/api/insights/drift?status=${status}`), + runKnowledgeDrift: () => + api<{ detected: number; signals: KnowledgeDriftSignalEntry[] }>('/api/insights/drift/run', { + method: 'POST', + }), + updateKnowledgeDrift: (id: string, status: KnowledgeDriftStatus) => + api<{ signal: KnowledgeDriftSignalEntry }>(`/api/insights/drift/${id}`, { + method: 'PATCH', + body: JSON.stringify({ status }), + }), importResolvedTicketsKnowledge: (limit = 50) => api<{ ok: boolean; imported: number; skipped: number; failed: number }>( '/api/knowledge/import-resolved-tickets', diff --git a/src/ui/app.tsx b/src/ui/app.tsx index 0fd138e..04e02ed 100644 --- a/src/ui/app.tsx +++ b/src/ui/app.tsx @@ -5,17 +5,24 @@ import { LoginView } from './views/Login'; import { InboxView } from './views/Inbox'; import { TicketView } from './views/Ticket'; import { SettingsView } from './views/Settings'; +import { InsightsView } from './views/Insights'; import { InviteAcceptView } from './views/InviteAccept'; import { WorkspaceGate } from './views/WorkspaceGate'; import { WorkspaceSwitcher } from './WorkspaceSwitcher'; import type { AuthMe } from '../types/workspace'; -type Route = { name: 'inbox' } | { name: 'ticket'; id: string } | { name: 'settings' } | { name: 'invite'; token: string }; +type Route = + | { name: 'inbox' } + | { name: 'ticket'; id: string } + | { name: 'insights' } + | { name: 'settings' } + | { name: 'invite'; token: string }; function parseRoute(): Route { const path = window.location.pathname; if (path.startsWith('/invite/')) return { name: 'invite', token: path.slice('/invite/'.length) }; if (path.startsWith('/t/')) return { name: 'ticket', id: path.slice(3) }; + if (path === '/insights') return { name: 'insights' }; if (path === '/settings') return { name: 'settings' }; return { name: 'inbox' }; } @@ -40,7 +47,9 @@ export function App() { setStage('app'); } - useEffect(() => { loadSession().catch(() => setStage('login')); }, []); + useEffect(() => { + loadSession().catch(() => setStage('login')); + }, []); useEffect(() => { const onPop = () => setRoute(parseRoute()); @@ -53,30 +62,83 @@ export function App() { setRoute(parseRoute()); } - if (stage === 'loading') return
Loading…
; + if (stage === 'loading') + return ( +
+
Loading…
+
+ ); if (stage === 'setup') return window.location.assign('/')} />; if (stage === 'login') { - return window.location.assign(route.name === 'invite' ? `/invite/${route.token}` : '/')} />; + return ( + + window.location.assign(route.name === 'invite' ? `/invite/${route.token}` : '/') + } + /> + ); } - if (route.name === 'invite') return window.location.assign('/')} />; - if (me && !me.currentWorkspaceId) return loadSession()} />; + if (route.name === 'invite') + return window.location.assign('/')} />; + if (me && !me.currentWorkspaceId) + return loadSession()} />; return (
diff --git a/src/ui/styles/components.css b/src/ui/styles/components.css index 1fa08a9..f1b06f0 100644 --- a/src/ui/styles/components.css +++ b/src/ui/styles/components.css @@ -214,8 +214,32 @@ background: rgba(255, 255, 255, 0.02); } +.insight-grid { + display: grid; + grid-template-columns: repeat(4, minmax(0, 1fr)); + gap: 12px; + margin-bottom: 16px; +} +.metric-card { + display: grid; + gap: 4px; + font-size: 24px; + font-weight: 650; +} +.insight-layout { + display: grid; + grid-template-columns: minmax(0, 1fr) minmax(0, 1fr); + gap: 16px; + align-items: start; + margin-bottom: 16px; +} + @media (max-width: 900px) { .ticket-detail { grid-template-columns: 1fr; } + .insight-grid, + .insight-layout { + grid-template-columns: 1fr; + } } diff --git a/src/ui/views/Insights.tsx b/src/ui/views/Insights.tsx new file mode 100644 index 0000000..8145bba --- /dev/null +++ b/src/ui/views/Insights.tsx @@ -0,0 +1,267 @@ +import { useEffect, useState } from 'react'; +import { + API, + type ConversationScoreEntry, + type InsightSummaryEntry, + type KbSuggestionEntry, + type KnowledgeDriftSignalEntry, +} from '../api'; + +export function InsightsView() { + const [summary, setSummary] = useState(null); + const [scores, setScores] = useState([]); + const [suggestions, setSuggestions] = useState([]); + const [drift, setDrift] = useState([]); + const [busy, setBusy] = useState(''); + const [error, setError] = useState(''); + + async function load() { + const [summaryRes, scoreRes, suggestionRes, driftRes] = await Promise.all([ + API.insightSummary(30), + API.listConversationScores(10), + API.listKbSuggestions('open'), + API.listKnowledgeDrift('open'), + ]); + setSummary(summaryRes.summary); + setScores(scoreRes.scores ?? []); + setSuggestions(suggestionRes.suggestions ?? []); + setDrift(driftRes.signals ?? []); + } + + useEffect(() => { + load().catch((err) => setError(err.message || 'Failed to load insights')); + }, []); + + async function runAll() { + setError(''); + setBusy('run'); + try { + await Promise.all([ + API.runConversationScoring(200), + API.generateKbSuggestions(200), + API.runKnowledgeDrift(), + ]); + await load(); + } catch (err) { + setError(errorMessage(err, 'Insight refresh failed')); + } finally { + setBusy(''); + } + } + + return ( + <> +
+

Insights

+ +
+ + {error &&
{error}
} + +
+ + + + +
+ +
+
+

Rubric

+
+ {scores.map((item) => ( +
+
+
{item.subject ?? item.ticket_id}
+
+ {item.status} · {item.category ?? 'uncategorized'} · grounded{' '} + {score(item.groundedness_score)} · effort {score(item.effort_score)} +
+
+ = 0.75 ? 'resolved' : ''}`}> + {score(item.overall_score)} + +
+ ))} + {scores.length === 0 &&
No scorecards yet.
} +
+
+ +
+

Unanswered intents

+
+ {(summary?.top_unresolved_intents ?? []).map((item) => ( +
+
+
{item.intent}
+
+ Example {item.example_ticket_id} +
+
+ {item.count} +
+ ))} + {(summary?.top_unresolved_intents ?? []).length === 0 && ( +
No unresolved clusters.
+ )} +
+
+
+ +
+
+

KB suggestions

+
+ {suggestions.map((item) => ( +
+
+
{item.title}
+
+ {item.summary} ·{' '} + {item.evidence_count || jsonArray(item.source_ticket_ids_json).length} tickets ·{' '} + {score(item.confidence_score)} confidence +
+
+
+ + +
+
+ ))} + {suggestions.length === 0 &&
No suggestions open.
} +
+
+ +
+

Drift

+
+ {drift.map((item) => ( +
+
+
{item.title}
+
+ {item.summary} +
+
+ +
+ ))} + {drift.length === 0 &&
No open drift signals.
} +
+
+
+ +
+

Procedure latency

+
+ {(summary?.slowest_procedures ?? []).map((item) => ( +
+
+
{item.name}
+
+ {item.slug} · {item.run_count} runs · {item.waiting_count} waiting ·{' '} + {item.failed_count} failed +
+
+ {duration(item.avg_duration_ms)} +
+ ))} + {(summary?.slowest_procedures ?? []).length === 0 && ( +
No procedure runs in range.
+ )} +
+
+ + ); +} + +function Metric({ label, value }: { label: string; value: string }) { + return ( +
+
+ {label} +
+
{value}
+
+ ); +} + +function score(value?: number | null): string { + return value === null || value === undefined ? '-' : Math.round(value * 100).toString(); +} + +function percent(value?: number | null): string { + return value === null || value === undefined ? '0%' : `${Math.round(value * 100)}%`; +} + +function duration(ms: number): string { + if (ms < 60_000) return `${Math.max(1, Math.round(ms / 1000))}s`; + if (ms < 60 * 60_000) return `${Math.round(ms / 60_000)}m`; + return `${Math.round(ms / 3_600_000)}h`; +} + +function errorMessage(err: unknown, fallback: string): string { + return err instanceof Error && err.message ? err.message : fallback; +} + +function jsonArray(value: string): unknown[] { + try { + const parsed = JSON.parse(value); + return Array.isArray(parsed) ? parsed : []; + } catch { + return []; + } +} diff --git a/src/ui/views/ProceduresSection.tsx b/src/ui/views/ProceduresSection.tsx index 1a33025..8a0c877 100644 --- a/src/ui/views/ProceduresSection.tsx +++ b/src/ui/views/ProceduresSection.tsx @@ -1,5 +1,5 @@ import { useEffect, useState } from 'react'; -import { API, type ProcedureListEntry } from '../api'; +import { API, type ProcedureLibraryListEntry, type ProcedureListEntry } from '../api'; interface ProceduresSectionProps { onSaved: (message?: string) => void; @@ -32,12 +32,18 @@ const DEFAULT_SPEC = `{ export function ProceduresSection({ onSaved }: ProceduresSectionProps) { const [procedures, setProcedures] = useState([]); + const [library, setLibrary] = useState([]); const [draft, setDraft] = useState(DEFAULT_SPEC); const [error, setError] = useState(''); + const [busy, setBusy] = useState(''); async function load() { - const res = await API.listProcedures(); - setProcedures(res.procedures ?? []); + const [procedureRes, libraryRes] = await Promise.all([ + API.listProcedures(), + API.listProcedureLibrary(), + ]); + setProcedures(procedureRes.procedures ?? []); + setLibrary(libraryRes.procedures ?? []); } useEffect(() => { @@ -48,6 +54,72 @@ export function ProceduresSection({ onSaved }: ProceduresSectionProps) { <>

Procedures

+
+ {library.map((item) => ( +
+
+
{item.name}
+
+ {item.category} · {item.risk_level} risk · v{item.version} · {item.eval_count}{' '} + evals · {item.provenance.spec_checksum.slice(0, 12)} + {item.required_mcp_servers.length > 0 + ? ` · MCP: ${item.required_mcp_servers.join(', ')}` + : ''} +
+
+ {item.summary} +
+ {item.readiness && ( +
+ {readinessLabel(item)} +
+ )} +
+
+ + +
+
+ ))} +
+
{procedures.map((procedure) => (
@@ -91,3 +163,15 @@ export function ProceduresSection({ onSaved }: ProceduresSectionProps) { ); } + +function readinessLabel(item: ProcedureLibraryListEntry): string { + if (!item.readiness) return ''; + if (item.readiness.status === 'ready') { + return `MCP ready: ${item.readiness.ready_tool_count}/${item.readiness.required_tool_count} required tools`; + } + const missing = item.readiness.tools + .filter((tool) => tool.usage !== 'optional' && tool.status !== 'ready') + .slice(0, 3) + .map((tool) => `${tool.server}.${tool.tool}`); + return `MCP setup needed: ${missing.join(', ')}`; +} diff --git a/tests/helpers/workspace-db.ts b/tests/helpers/workspace-db.ts index 9b0c783..71d9ae9 100644 --- a/tests/helpers/workspace-db.ts +++ b/tests/helpers/workspace-db.ts @@ -157,7 +157,41 @@ export function createWorkspaceTestDb() { workspace_id TEXT NOT NULL, kind TEXT NOT NULL, title TEXT NOT NULL, + url TEXT, + r2_key TEXT, + ticket_id TEXT, + message_id TEXT, + content_hash TEXT, status TEXT NOT NULL, + chunk_count INTEGER NOT NULL DEFAULT 0, + last_crawled_at INTEGER, + last_indexed_at INTEGER, + error TEXT, + created_at INTEGER NOT NULL DEFAULT 1, + updated_at INTEGER NOT NULL + ); + CREATE TABLE knowledge_chunk ( + id TEXT PRIMARY KEY, + workspace_id TEXT NOT NULL, + source_id TEXT NOT NULL, + ordinal INTEGER NOT NULL, + title TEXT NOT NULL, + body TEXT NOT NULL, + snippet TEXT NOT NULL, + url TEXT, + vector_id TEXT NOT NULL, + content_hash TEXT NOT NULL, + used_in_answers_count INTEGER NOT NULL DEFAULT 0, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL + ); + CREATE TABLE knowledge_doc ( + id TEXT PRIMARY KEY, + workspace_id TEXT NOT NULL, + title TEXT NOT NULL, + body TEXT NOT NULL, + url TEXT, + created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL ); CREATE TABLE notification_channel ( @@ -344,6 +378,56 @@ export function createWorkspaceTestDb() { error TEXT, created_at INTEGER NOT NULL ); + CREATE TABLE conversation_score ( + id TEXT PRIMARY KEY, + workspace_id TEXT NOT NULL, + ticket_id TEXT NOT NULL, + groundedness_score REAL NOT NULL, + tone_score REAL NOT NULL, + resolution_score REAL NOT NULL, + effort_score REAL NOT NULL, + overall_score REAL NOT NULL, + signals_json TEXT NOT NULL DEFAULT '{}', + scored_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + UNIQUE(workspace_id, ticket_id) + ); + CREATE TABLE kb_suggestion ( + id TEXT PRIMARY KEY, + workspace_id TEXT NOT NULL, + cluster_key TEXT NOT NULL, + title TEXT NOT NULL, + summary TEXT NOT NULL, + body_markdown TEXT NOT NULL, + source_ticket_ids_json TEXT NOT NULL DEFAULT '[]', + suggested_terms_json TEXT NOT NULL DEFAULT '[]', + evidence_count INTEGER NOT NULL DEFAULT 0, + confidence_score REAL NOT NULL DEFAULT 0, + status TEXT NOT NULL DEFAULT 'open', + source TEXT NOT NULL DEFAULT 'unresolved_cluster', + accepted_source_id TEXT, + accepted_by_user_id TEXT, + accepted_at INTEGER, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + UNIQUE(workspace_id, cluster_key) + ); + CREATE TABLE knowledge_drift_signal ( + id TEXT PRIMARY KEY, + workspace_id TEXT NOT NULL, + source_id TEXT NOT NULL, + signal_hash TEXT NOT NULL, + severity TEXT NOT NULL, + title TEXT NOT NULL, + summary TEXT NOT NULL, + successful_reply_count INTEGER NOT NULL DEFAULT 0, + divergence_terms_json TEXT NOT NULL DEFAULT '[]', + example_ticket_ids_json TEXT NOT NULL DEFAULT '[]', + status TEXT NOT NULL DEFAULT 'open', + detected_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + UNIQUE(workspace_id, source_id, signal_hash) + ); `); const envDb = { diff --git a/tests/insights.test.ts b/tests/insights.test.ts new file mode 100644 index 0000000..cf16bce --- /dev/null +++ b/tests/insights.test.ts @@ -0,0 +1,264 @@ +import { describe, expect, it, vi } from 'vitest'; +import { apiApp } from '../src/api/routes'; +import { + acceptKbSuggestion, + detectKnowledgeDrift, + generateKbSuggestions, + getInsightSummary, + scoreConversation, + updateKbSuggestionStatus, +} from '../src/insights'; +import { + addMember, + createWorkspaceTestDb, + login, + seedMailbox, + seedUser, + seedWorkspace, +} from './helpers/workspace-db'; + +vi.mock('agents', () => ({ + getAgentByName: () => ({}), + Agent: class {}, + callable: () => () => undefined, +})); + +function seedTicket( + db: ReturnType['db'], + patch: Partial<{ + id: string; + status: string; + subject: string; + category: string | null; + now: number; + }> = {}, +) { + const now = patch.now ?? Date.now(); + db.prepare( + `INSERT INTO ticket ( + id, workspace_id, mailbox_id, subject, status, priority, category, last_message_at, + requester_email, thread_token, created_at, updated_at + ) VALUES (?, 'ws_a', 'mb_a', ?, ?, 'normal', ?, ?, 'customer@example.com', ?, ?, ?)`, + ).run( + patch.id ?? 'tkt_1', + patch.subject ?? 'Refund request', + patch.status ?? 'open', + patch.category ?? null, + now, + `tok_${patch.id ?? 'tkt_1'}`, + now, + now, + ); +} + +function seedMessage( + db: ReturnType['db'], + input: { + id: string; + ticketId: string; + direction: 'inbound' | 'outbound'; + preview: string; + now?: number; + }, +) { + const now = input.now ?? Date.now(); + db.prepare( + `INSERT INTO message_index ( + id, ticket_id, workspace_id, direction, preview, sent_at, created_at + ) VALUES (?, ?, 'ws_a', ?, ?, ?, ?)`, + ).run(input.id, input.ticketId, input.direction, input.preview, now, now); +} + +describe('insights', () => { + it('scores conversations and aggregates workspace insight metrics', async () => { + const { db, env } = createWorkspaceTestDb(); + seedWorkspace(db, 'ws_a', 'Alpha'); + seedMailbox(db, 'ws_a', 'mb_a', 'support@example.com'); + seedTicket(db, { id: 'tkt_score', status: 'resolved', category: 'billing' }); + seedMessage(db, { + id: 'msg_in', + ticketId: 'tkt_score', + direction: 'inbound', + preview: 'Can I get a refund?', + }); + seedMessage(db, { + id: 'msg_out', + ticketId: 'tkt_score', + direction: 'outbound', + preview: 'Thanks for reaching out. I can help with that refund.', + }); + db.prepare( + `INSERT INTO approval_request ( + id, workspace_id, ticket_id, kind, status, proposed_json, risk_reasons_json, created_at + ) VALUES ('apr_1', 'ws_a', 'tkt_score', 'draft_reply', 'approved', ?, '[]', ?)`, + ).run( + JSON.stringify({ + confidence: 0.96, + cites_knowledge_ids: ['kchk_refund'], + knowledge_hits: [{ id: 'kchk_refund' }], + knowledge_trace: { finalAnswerable: true }, + }), + Date.now(), + ); + db.prepare( + `INSERT INTO ticket_outcome_event ( + id, workspace_id, ticket_id, kind, source, payload_json, created_at + ) VALUES ('out_1', 'ws_a', 'tkt_score', 'resolved_autonomously', 'agent', '{}', ?)`, + ).run(Date.now()); + db.prepare( + `INSERT INTO ticket_feedback ( + id, workspace_id, ticket_id, rating, source, created_at + ) VALUES ('fb_1', 'ws_a', 'tkt_score', 'positive', 'customer', ?)`, + ).run(Date.now()); + + const score = await scoreConversation(env, 'ws_a', 'tkt_score'); + const summary = await getInsightSummary(env, 'ws_a', 30); + + expect(score?.overall_score).toBeGreaterThan(0.8); + expect(score?.groundedness_score).toBeGreaterThan(0.85); + expect(summary.resolution_rate).toBe(1); + expect(summary.avg_overall_score).toBeGreaterThan(0.8); + }); + + it('generates reviewable KB suggestions and accepts them into knowledge', async () => { + const { db, env } = createWorkspaceTestDb(); + seedWorkspace(db, 'ws_a', 'Alpha'); + seedMailbox(db, 'ws_a', 'mb_a', 'support@example.com'); + seedTicket(db, { + id: 'tkt_unanswered_1', + status: 'open', + subject: 'Subscription invoice credit question', + category: 'billing', + }); + seedTicket(db, { + id: 'tkt_unanswered_2', + status: 'pending', + subject: 'Need invoice credit for subscription', + category: 'billing', + }); + + const generated = await generateKbSuggestions(env, 'ws_a'); + const accepted = await acceptKbSuggestion(env, 'ws_a', generated.suggestions[0].id, 'usr_a'); + const acceptedAgain = await acceptKbSuggestion( + env, + 'ws_a', + generated.suggestions[0].id, + 'usr_a', + ); + + expect(generated.generated).toBe(1); + expect(generated.suggestions[0].evidence_count).toBe(2); + expect(generated.suggestions[0].confidence_score).toBeGreaterThan(0.7); + expect(generated.suggestions[0].source_ticket_ids_json).toContain('tkt_unanswered_1'); + expect(accepted?.sourceId).toMatch(/^ksrc_/); + expect(acceptedAgain?.sourceId).toBe(accepted?.sourceId); + expect(db.prepare(`SELECT status FROM kb_suggestion`).get()).toEqual({ status: 'accepted' }); + expect(db.prepare(`SELECT accepted_source_id FROM kb_suggestion`).get()).toEqual({ + accepted_source_id: accepted?.sourceId, + }); + expect(db.prepare(`SELECT COUNT(*) AS n FROM knowledge_source`).get()).toEqual({ n: 1 }); + }); + + it('does not generate KB suggestions from thin single-ticket evidence', async () => { + const { db, env } = createWorkspaceTestDb(); + seedWorkspace(db, 'ws_a', 'Alpha'); + seedMailbox(db, 'ws_a', 'mb_a', 'support@example.com'); + seedTicket(db, { + id: 'tkt_one_off', + status: 'open', + subject: 'One off custom invoice memo', + category: 'billing', + }); + + const generated = await generateKbSuggestions(env, 'ws_a'); + + expect(generated.generated).toBe(0); + expect(db.prepare(`SELECT COUNT(*) AS n FROM kb_suggestion`).get()).toEqual({ n: 0 }); + }); + + it('keeps accepted KB suggestions terminal for status updates', async () => { + const { db, env } = createWorkspaceTestDb(); + seedWorkspace(db, 'ws_a', 'Alpha'); + seedMailbox(db, 'ws_a', 'mb_a', 'support@example.com'); + seedTicket(db, { + id: 'tkt_terminal_1', + status: 'open', + subject: 'Need invoice credit for subscription', + category: 'billing', + }); + seedTicket(db, { + id: 'tkt_terminal_2', + status: 'open', + subject: 'Subscription invoice credit request', + category: 'billing', + }); + const generated = await generateKbSuggestions(env, 'ws_a'); + await acceptKbSuggestion(env, 'ws_a', generated.suggestions[0].id, 'usr_a'); + + await expect( + updateKbSuggestionStatus(env, 'ws_a', generated.suggestions[0].id, 'dismissed', 'usr_a'), + ).rejects.toThrow('kb_suggestion_accepted'); + }); + + it('detects knowledge drift from successful replies', async () => { + const { db, env } = createWorkspaceTestDb(); + seedWorkspace(db, 'ws_a', 'Alpha'); + seedMailbox(db, 'ws_a', 'mb_a', 'support@example.com'); + db.prepare( + `INSERT INTO knowledge_source (id, workspace_id, kind, title, status, chunk_count, created_at, updated_at) + VALUES ('ksrc_policy', 'ws_a', 'manual', 'Refund policy', 'ready', 1, 1, 1)`, + ).run(); + db.prepare( + `INSERT INTO knowledge_chunk ( + id, workspace_id, source_id, ordinal, title, body, snippet, vector_id, content_hash, + used_in_answers_count, created_at, updated_at + ) VALUES ( + 'kchk_policy', 'ws_a', 'ksrc_policy', 0, 'Refund policy', + 'Refund policy covers returned items and order cancellation.', + 'Refund policy covers returned items.', 'vec_policy', 'hash_policy', 2, 1, 1 + )`, + ).run(); + for (const id of ['tkt_drift_1', 'tkt_drift_2']) { + seedTicket(db, { id, status: 'resolved', subject: 'Billing help' }); + seedMessage(db, { + id: `msg_${id}`, + ticketId: id, + direction: 'outbound', + preview: 'We applied a subscription invoice credit and adjusted the renewal invoice.', + }); + db.prepare( + `INSERT INTO approval_request ( + id, workspace_id, ticket_id, kind, status, proposed_json, risk_reasons_json, created_at + ) VALUES (?, 'ws_a', ?, 'draft_reply', 'approved', ?, '[]', ?)`, + ).run(`apr_${id}`, id, JSON.stringify({ cites_knowledge_ids: ['kchk_policy'] }), Date.now()); + } + for (const id of ['tkt_unrelated_1', 'tkt_unrelated_2']) { + seedTicket(db, { id, status: 'resolved', subject: 'Shipping help' }); + seedMessage(db, { + id: `msg_${id}`, + ticketId: id, + direction: 'outbound', + preview: 'Warehouse dispatch tracking labels carrier pickup manifest.', + }); + } + + const result = await detectKnowledgeDrift(env, 'ws_a'); + + expect(result.detected).toBe(1); + expect(result.signals[0].severity).toBe('medium'); + expect(result.signals[0].divergence_terms_json).toContain('subscription'); + expect(result.signals[0].divergence_terms_json).not.toContain('warehouse'); + }); + + it('protects insight APIs behind workspace admin roles', async () => { + const { db, env } = createWorkspaceTestDb(); + seedWorkspace(db, 'ws_a', 'Alpha'); + await seedUser(db, 'usr_viewer', 'viewer@example.com'); + addMember(db, 'ws_a', 'usr_viewer', 'viewer'); + const cookie = await login(env, 'viewer@example.com'); + + const res = await apiApp.request('/insights/summary', { headers: { cookie } }, env); + + expect(res.status).toBe(403); + }); +}); diff --git a/tests/procedure-library.test.ts b/tests/procedure-library.test.ts new file mode 100644 index 0000000..0f749a3 --- /dev/null +++ b/tests/procedure-library.test.ts @@ -0,0 +1,196 @@ +import { describe, expect, it, vi } from 'vitest'; +import { apiApp } from '../src/api/routes'; +import { createMcpServer, upsertDiscoveredMcpTools } from '../src/mcp/storage'; +import { + getProcedureLibraryItem, + getProcedureLibraryManifest, + getProcedureLibraryReadiness, + listProcedureLibrary, + validateProcedureLibrary, +} from '../src/procedures/library'; +import { runProcedureSpecEvals } from '../src/evals/replay'; +import { + addMember, + createWorkspaceTestDb, + login, + seedUser, + seedWorkspace, +} from './helpers/workspace-db'; + +vi.mock('agents', () => ({ + getAgentByName: () => ({}), + Agent: class {}, + callable: () => () => undefined, +})); + +describe('procedure library', () => { + it('ships validated procedures with evals, checksums, and MCP tool specs', async () => { + const entries = await listProcedureLibrary(); + + expect(entries.map((entry) => entry.slug)).toEqual([ + 'refund-intake', + 'password-reset', + 'shipping-dispute', + 'gdpr-data-request', + ]); + expect(await validateProcedureLibrary()).toHaveLength(entries.length); + for (const entry of entries) { + expect((entry as any).spec).toBeUndefined(); + expect(entry.provenance.spec_checksum).toMatch(/^[a-f0-9]{64}$/); + expect(entry.provenance.source_ref).toContain(`#sha256:${entry.provenance.spec_checksum}`); + const item = await getProcedureLibraryItem(entry.slug); + expect(item?.reference_mcp_tools.length).toBeGreaterThan(0); + expect(item?.spec.evals?.length).toBeGreaterThan(0); + expect(runProcedureSpecEvals(item!.spec).status).toBe('passed'); + const actionTools = collectActionTools(item!.spec.steps); + for (const tool of item!.reference_mcp_tools) { + expect(tool.annotations?.openWorldHint).not.toBeUndefined(); + expect(actionTools).toContain(`${tool.server}.${tool.tool}`); + const matchingActions = collectActions(item!.spec.steps).filter( + (step) => step.tool === `${tool.server}.${tool.tool}`, + ); + if (tool.annotations?.readOnlyHint !== true) { + expect(matchingActions.every((step) => step.requires_approval !== false)).toBe(true); + } + } + } + }); + + it('returns immutable library clones and a full manifest', async () => { + const entries = await listProcedureLibrary(); + entries[0].tags.push('mutated'); + expect((await listProcedureLibrary())[0].tags).not.toContain('mutated'); + + const detail = await getProcedureLibraryItem('refund-intake'); + detail!.spec.name = 'Mutated'; + expect((await getProcedureLibraryItem('refund-intake'))?.spec.name).toBe('Refund intake'); + + const manifest = await getProcedureLibraryManifest(); + expect(manifest.manifest_version).toBe('2026-05-18'); + expect(manifest.standards.mcp_schema).toBe('2025-11-25'); + expect(manifest.procedures).toHaveLength(entries.length); + }); + + it('lets workspace owners install a library procedure through the API', async () => { + const { db, env } = createWorkspaceTestDb(); + seedWorkspace(db, 'ws_a', 'Alpha'); + await seedUser(db, 'usr_a', 'owner@example.com'); + addMember(db, 'ws_a', 'usr_a', 'owner'); + const cookie = await login(env, 'owner@example.com'); + + const listRes = await apiApp.request('/procedures/library', { headers: { cookie } }, env); + const listBody = await listRes.json(); + const manifestRes = await apiApp.request( + '/procedures/library/manifest', + { headers: { cookie } }, + env, + ); + const detailRes = await apiApp.request( + '/procedures/library/password-reset', + { headers: { cookie } }, + env, + ); + const detailBody = await detailRes.json(); + const installRes = await apiApp.request( + '/procedures/library/password-reset/install', + { method: 'POST', headers: { cookie } }, + env, + ); + const installBody = await installRes.json(); + + expect(listRes.status).toBe(200); + expect(listBody.procedures.some((entry: any) => entry.slug === 'password-reset')).toBe(true); + expect(listBody.procedures[0].spec).toBeUndefined(); + expect(listBody.procedures[0].readiness.status).toBe('needs_setup'); + expect(manifestRes.status).toBe(200); + expect(detailRes.status).toBe(200); + expect(detailBody.procedure.provenance.spec_checksum).toMatch(/^[a-f0-9]{64}$/); + expect(installRes.status).toBe(200); + expect(installBody.procedure.slug).toBe('password-reset'); + const stored = db.prepare(`SELECT source_kind, source_ref FROM procedure_version`).get() as { + source_kind: string; + source_ref: string; + }; + expect(stored).toEqual({ + source_kind: 'seed', + source_ref: detailBody.procedure.provenance.source_ref, + }); + expect(stored.source_ref).toContain('#sha256:'); + }); + + it('reports MCP readiness for library installs', async () => { + const { db, env } = createWorkspaceTestDb(); + seedWorkspace(db, 'ws_a', 'Alpha'); + const missing = await getProcedureLibraryReadiness(env, 'ws_a', 'refund-intake'); + expect(missing?.status).toBe('needs_setup'); + expect(missing?.tools.map((tool) => tool.status)).toEqual(['missing_server', 'missing_server']); + + const server = await createMcpServer(env, { + workspaceId: 'ws_a', + name: 'stripe', + endpointUrl: 'https://mcp.example.com/stripe', + }); + await upsertDiscoveredMcpTools(env, 'ws_a', server.id, [ + { + name: 'customers.search', + inputSchema: {}, + annotations: { readOnlyHint: true, destructiveHint: false, openWorldHint: true }, + }, + { + name: 'refunds.create', + inputSchema: {}, + annotations: { + readOnlyHint: false, + destructiveHint: true, + idempotentHint: true, + openWorldHint: true, + }, + }, + ]); + + const ready = await getProcedureLibraryReadiness(env, 'ws_a', 'refund-intake'); + expect(ready).toMatchObject({ + status: 'ready', + ready_tool_count: 2, + required_tool_count: 2, + }); + }); + + it('fails closed on unknown procedures and non-admin installs', async () => { + const { db, env } = createWorkspaceTestDb(); + seedWorkspace(db, 'ws_a', 'Alpha'); + await seedUser(db, 'usr_viewer', 'viewer@example.com'); + addMember(db, 'ws_a', 'usr_viewer', 'viewer'); + const cookie = await login(env, 'viewer@example.com'); + + const missingRes = await apiApp.request( + '/procedures/library/not-real', + { headers: { cookie } }, + env, + ); + const installRes = await apiApp.request( + '/procedures/library/refund-intake/install', + { method: 'POST', headers: { cookie } }, + env, + ); + + expect(missingRes.status).toBe(404); + expect(installRes.status).toBe(403); + }); +}); + +function collectActionTools(steps: any[]): string[] { + return collectActions(steps).map((step) => step.tool); +} + +function collectActions(steps: any[]): any[] { + const actions: any[] = []; + for (const step of steps) { + if (step.type === 'call_action') actions.push(step); + if (step.type === 'if') { + actions.push(...collectActions(step.then ?? []), ...collectActions(step.else ?? [])); + } + if (step.type === 'loop') actions.push(...collectActions(step.steps ?? [])); + } + return actions; +} diff --git a/wrangler.jsonc b/wrangler.jsonc index 1618ff5..3c46ca6 100644 --- a/wrangler.jsonc +++ b/wrangler.jsonc @@ -86,7 +86,7 @@ "send_email": [{ "name": "EMAIL" }], "triggers": { - "crons": ["*/5 * * * *"] + "crons": ["*/5 * * * *", "17 3 * * 1"] }, "unsafe": {