From eb03a18428a5e2afbf964ff590833e346660cce4 Mon Sep 17 00:00:00 2001 From: Rahul Tomar Date: Fri, 6 Nov 2026 15:36:38 +0100 Subject: [PATCH] What a proposal does: behaviour before and after Agent proposals now open on a behaviour tab instead of the code diff. Compare behaviour runs every example on the base commit and with the proposal's files, in the browser, and compares every output cycle by cycle (the extra cycles after each example included). Each difference is put in words (lib/compare.ts): a shift of the same behaviour reads "led now changes 1 cycle earlier", anything else "busy at cycle 40 was 1, now 0 (different in 3 cycles)"; with no difference it says the change only touches cases the examples don't try. For editors it also compares how well each version is tested: examples passing, coverage and each state machine's moves added or removed, from POST /agent/{id}/compare (design::quality runs the gate on both trees). Checked in the browser with two test proposals added to the local database (no model call): counting to N-2 was described cycle by cycle with the first example going from passes to fails; starting the counter at 1 was described as the LED changing 1 cycle earlier. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 4 + apps/web/src/api.ts | 9 ++ apps/web/src/components/Agent.tsx | 168 +++++++++++++++++++++++++++++- apps/web/src/lib/compare.ts | 99 ++++++++++++++++++ apps/web/src/styles.css | 13 +++ crates/wafer-api/src/agent.rs | 15 +++ crates/wafer-api/src/design.rs | 28 +++++ crates/wafer-api/src/lib.rs | 1 + crates/wafer-api/tests/agent.rs | 16 +++ docs/api.md | 1 + docs/user-guide/agent.md | 12 +++ 11 files changed, 362 insertions(+), 4 deletions(-) create mode 100644 apps/web/src/lib/compare.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index a360c9d..bf3c7a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ All notable changes to Wafersmith are recorded here. The format follows ### Added +- **What a proposal does**: agent proposals open on a behaviour tab. **Compare behaviour** runs + every example before and after the change (in the browser) and describes each output that + behaves differently ("`led` now changes 1 cycle earlier"), and compares examples passing, + coverage and state-machine moves (`POST /agent/{id}/compare`). - **Review report**: one printable page per project (Health → Review report): what the design is, its examples as plain sentences with their results, coverage and planted mistakes, its state machines, the latest build and what still needs an expert, and where it came from (the AI's diff --git a/apps/web/src/api.ts b/apps/web/src/api.ts index f24faa1..06a95ae 100644 --- a/apps/web/src/api.ts +++ b/apps/web/src/api.ts @@ -276,6 +276,14 @@ export interface MistakesReport { elapsed_ms: number; } +/** How well a version of the design is tested (`POST /agent/{id}/compare`). */ +export interface Quality { + examples: number; + passing: number; + coverage: Coverage | null; + machines: Machine[]; +} + export interface SpecRun { commit: string; examples: ExampleResult[]; @@ -656,6 +664,7 @@ export const api = { mergeProposal: (pid: string) => request("POST", `/agent/${enc(pid)}/merge`), discardProposal: (pid: string) => request("POST", `/agent/${enc(pid)}/discard`), regate: (pid: string) => request<{ report: GateReport; reproduced: boolean }>("POST", `/agent/${enc(pid)}/regate`), + compareProposal: (pid: string) => request<{ before: Quality; after: Quality }>("POST", `/agent/${enc(pid)}/compare`), createJob: (id: string, body: { commit?: string; config?: string } = {}) => request("POST", `/projects/${enc(id)}/jobs`, body), listJobs: (id: string) => request("GET", `/projects/${enc(id)}/jobs`), diff --git a/apps/web/src/components/Agent.tsx b/apps/web/src/components/Agent.tsx index 7e4dc20..59ddce3 100644 --- a/apps/web/src/components/Agent.tsx +++ b/apps/web/src/components/Agent.tsx @@ -1,6 +1,19 @@ import { useEffect, useState, type FormEvent } from "react"; -import { api, ApiError, type EvidenceStep, type GateReport, type Proposal, type ProposalListItem, type ProposalStatus } from "../api"; +import { + api, + ApiError, + type Blocks, + type EvidenceStep, + type GateReport, + type Proposal, + type ProposalListItem, + type ProposalStatus, + type Quality, + type Spec, +} from "../api"; +import { compareExamples, type ExampleChange } from "../lib/compare"; import { hunks, lineDiff } from "../lib/diff"; +import { SimClient } from "../sim/client"; const POLL_MS = 1500; @@ -112,7 +125,7 @@ export function AgentList({ ); } -type Tab = "changes" | "gate" | "evidence"; +type Tab = "behaviour" | "changes" | "gate" | "evidence"; /** Review a proposal: diff, gate, evidence; merge when the gate passed. */ export function AgentView({ @@ -149,7 +162,8 @@ export function AgentView({ if (stopped) return; setP(next); if (next.status === "running") timer = window.setTimeout(poll, POLL_MS); - else setTab((t) => (t === "evidence" && Object.keys(next.files).length ? "changes" : t)); + // A finished proposal with changes opens on what it does, not on the code. + else setTab((t) => (t === "evidence" && Object.keys(next.files).length ? "behaviour" : t)); } catch (e) { if (!stopped) setError((e as Error).message); } @@ -190,7 +204,7 @@ export function AgentView({ )}
- {(["changes", "gate", "evidence"] as Tab[]).map((t) => ( + {(["behaviour", "changes", "gate", "evidence"] as Tab[]).map((t) => ( @@ -248,6 +262,7 @@ export function AgentView({ ))}
)} + {tab === "behaviour" && } {tab === "gate" && setRegate(await api.regate(p.id))} />} {tab === "evidence" && } @@ -393,3 +408,148 @@ function Evidence({ p }: { p: Proposal }) { ); } + +/** What the proposal does: each example before and after it, and how well the design is tested. */ +function Behaviour({ p, canRun }: { p: Proposal; canRun: boolean }) { + const [examples, setExamples] = useState(null); + const [quality, setQuality] = useState<{ before: Quality; after: Quality } | null>(null); + const [busy, setBusy] = useState(false); + const [error, setError] = useState(null); + + async function compare() { + setBusy(true); + setError(null); + const client = new SimClient(); + try { + const snap = await api.snapshot(p.project_id, p.base_commit); + const before = snap.files; + const after = { ...before, ...p.files }; + const specText = before["spec.json"]; + if (!specText) throw new Error("the project has no spec, so there are no examples to compare"); + const spec = JSON.parse(specText) as Spec; + const chip = before["blocks.json"] ? (JSON.parse(before["blocks.json"]) as Blocks) : null; + const [ex, q] = await Promise.all([ + compareExamples(client, spec, chip, before, after), + canRun ? api.compareProposal(p.id) : Promise.resolve(null), + ]); + setExamples(ex); + setQuality(q); + } catch (e) { + setError((e as Error).message); + } finally { + client.dispose(); + setBusy(false); + } + } + + if (p.status === "running") return

The agent is still working.

; + const changed = examples?.filter((e) => e.differences.length || e.before !== e.after) ?? []; + return ( +
+

+ Runs every example on the project as it was ({p.base_commit.slice(0, 8)}) and with this change, and compares + every output cycle by cycle, including the cycles after each example ends. +

+ + {error &&

{error}

} + {examples && ( + <> +

The examples

+ {changed.length === 0 ? ( +

+ Every example behaves exactly as before. The change only affects cases the examples don't try, or none at all; the + coverage below says whether that leaves something untested. +

+ ) : ( +

+ {changed.length} of {examples.length} example{examples.length === 1 ? "" : "s"} behave differently: +

+ )} +
    + {examples + .filter((e) => e.differences.length || e.before !== e.after || e.problem) + .map((e) => ( +
  • + {e.title}{" "} + + {verdict(e.before)} → {verdict(e.after)} + + {e.problem &&

    {e.problem}

    } +
      + {e.differences.map((d) => ( +
    • {withCode(d)}
      • + ))} +
    +
    • + ))} +
+ + )} + {quality && ( + <> +

How well it is tested

+ + + + + + + + + + + {moves(quality).map(([label, text]) => ( + + + + + ))} + +
Examples passing + {quality.before.passing} of {quality.before.examples} → {quality.after.passing} of {quality.after.examples} +
Cases the examples try + {cov(quality.before)} → {cov(quality.after)} +
{label}{text}
+ {(quality.after.coverage?.gaps.length ?? 0) > (quality.before.coverage?.gaps.length ?? 0) && ( +

+ With this change the examples leave more cases untried; Health lists them once it is merged. +

+ )} + + )} + {examples && !canRun &&

The coverage comparison needs an editor of the project.

} +
+ ); +} + +const verdict = (v: boolean | null) => (v === null ? "couldn't run" : v ? "passes" : "fails"); +const cov = (q: Quality) => (q.coverage ? `${q.coverage.covered} of ${q.coverage.total}` : "none"); + +/** State machines and their moves: what the change adds or removes. */ +function moves(q: { before: Quality; after: Quality }): [string, string][] { + const key = (m: Quality["machines"][number]) => `${m.module}.${m.state}`; + const list = (m: Quality["machines"][number]) => m.transitions.map((t) => `${t.from} → ${t.to} when ${t.when}`); + const out: [string, string][] = []; + const names = new Set([...q.before.machines.map(key), ...q.after.machines.map(key)]); + for (const name of names) { + const b = q.before.machines.find((m) => key(m) === name); + const a = q.after.machines.find((m) => key(m) === name); + const bl = b ? list(b) : []; + const al = a ? list(a) : []; + const added = al.filter((x) => !bl.includes(x)); + const removed = bl.filter((x) => !al.includes(x)); + const label = `State machine ${name}`; + if (!b) out.push([label, `new, with ${al.length} moves`]); + else if (!a) out.push([label, "removed"]); + else if (added.length || removed.length) + out.push([label, [...added.map((x) => `+ ${x.replace(/`/g, "")}`), ...removed.map((x) => `− ${x.replace(/`/g, "")}`)].join("; ")]); + else out.push([label, `unchanged (${al.length} moves)`]); + } + return out; +} + +function withCode(text: string) { + return text.split("`").map((part, i) => (i % 2 ? {part} : {part})); +} diff --git a/apps/web/src/lib/compare.ts b/apps/web/src/lib/compare.ts new file mode 100644 index 0000000..d13cc8e --- /dev/null +++ b/apps/web/src/lib/compare.ts @@ -0,0 +1,99 @@ +// What a change does to the design's behaviour: every example simulated before and after it, and +// each output compared cycle by cycle (including the cycles after the example ends), described in +// plain words ("busy now changes 1 cycle earlier"). + +import type { Blocks, Spec } from "../api"; +import type { SimClient } from "../sim/client"; +import { extendTestbench, fmt, replay, targetSpec } from "./replay"; + +export interface ExampleChange { + id: string; + title: string; + /** Whether the example passed before and after; null if it couldn't run. */ + before: boolean | null; + after: boolean | null; + /** One line per output that behaves differently. */ + differences: string[]; + problem: string | null; +} + +/** How far a shifted copy of the same behaviour may be (cycles). */ +const MAX_SHIFT = 5; + +export async function compareExamples( + client: SimClient, + spec: Spec, + chip: Blocks | null, + before: Record, + after: Record, +): Promise { + const out: ExampleChange[] = []; + for (const ex of spec.examples) { + const tbPath = `tb/examples/tb_ex_${ex.id}.v`; + const tb = before[tbPath]; + const tspec = targetSpec(spec, ex, chip); + const change: ExampleChange = { id: ex.id, title: ex.title, before: null, after: null, differences: [], problem: null }; + if (!tb) { + change.problem = "its testbench isn't in the project (save the spec)"; + out.push(change); + continue; + } + try { + const run = async (files: Record) => { + const list = Object.entries(files) + .filter(([p]) => /\.s?v$/.test(p) && !p.startsWith("tb/")) + .map(([name, source]) => ({ name, source })); + list.push({ name: tbPath, source: extendTestbench(tb, tspec) }); + return replay(tspec, ex, await client.run(list, `tb_ex_${ex.id}`)); + }; + const b = await run(before); + const a = await run(after); + change.before = !b.firstFail && !b.problem; + change.after = !a.firstFail && !a.problem; + for (const rb of b.rows.filter((r) => r.dir === "out")) { + const ra = a.rows.find((r) => r.name === rb.name); + if (!ra) continue; + const d = describe(rb.name, rb.width, rb.values, ra.values); + if (d) change.differences.push(d); + } + } catch (e) { + change.problem = (e as Error).message; + } + out.push(change); + } + return out; +} + +/** How `after` differs from `before` for one output, or null if it doesn't. */ +export function describe(name: string, width: number, before: (bigint | null)[], after: (bigint | null)[]): string | null { + const n = Math.min(before.length, after.length); + const differ: number[] = []; + for (let c = 0; c < n; c++) if (before[c] !== null && after[c] !== null && before[c] !== after[c]) differ.push(c); + if (!differ.length) return null; + // The same behaviour moved in time: after[c] == before[c - k] wherever both are known. + for (let k = 1; k <= MAX_SHIFT; k++) { + for (const dir of [1, -1]) { + const s = k * dir; + let ok = true; + let compared = 0; + for (let c = Math.max(0, s); c < n && c - s < n; c++) { + const x = after[c]; + const y = before[c - s]; + if (x === null || y === null) continue; + compared++; + if (x !== y) { + ok = false; + break; + } + } + if (ok && compared > 0) { + return `\`${name}\` now changes ${k} cycle${k === 1 ? "" : "s"} ${dir > 0 ? "later" : "earlier"} (first difference at cycle ${differ[0]})`; + } + } + } + const c = differ[0]; + return ( + `\`${name}\` at cycle ${c} was ${fmt(before[c], width)}, now ${fmt(after[c], width)}` + + (differ.length > 1 ? ` (different in ${differ.length} cycles)` : "") + ); +} diff --git a/apps/web/src/styles.css b/apps/web/src/styles.css index ccc04b4..ddea342 100644 --- a/apps/web/src/styles.css +++ b/apps/web/src/styles.css @@ -2632,3 +2632,16 @@ a.button { overflow: visible; } } + +/* The behaviour comparison of an agent proposal. */ +.behaviour h4 { + margin: 14px 0 4px; +} + +.behaviour-list { + padding-left: 18px; +} + +.behaviour-list > li { + margin-bottom: 8px; +} diff --git a/crates/wafer-api/src/agent.rs b/crates/wafer-api/src/agent.rs index fc9100d..91e8db2 100644 --- a/crates/wafer-api/src/agent.rs +++ b/crates/wafer-api/src/agent.rs @@ -333,6 +333,21 @@ async fn run_gate(tree: BTreeMap) -> ApiResult { Ok(tokio::task::spawn_blocking(move || gate::run(&tree)).await?) } +/// What the proposal does to how well the design is tested: examples passing, coverage and state +/// machines, at its base commit and with its changes (editors: it runs the gate twice). +pub async fn compare(State(state): State, user: AuthUser, Path(id): Path) -> ApiResult> { + let p = load(&state, &user, id, Role::Editor).await?; + let before: BTreeMap = + crate::workspace::text_files(state.repos.clone(), p.project_id, p.base_commit.clone(), MAX_FILES_TEXT) + .await? + .into_iter() + .collect(); + let after = proposed_tree(&state, &p).await?; + let (b, a) = + tokio::task::spawn_blocking(move || (crate::design::quality(&before), crate::design::quality(&after))).await?; + Ok(Json(json!({ "before": b, "after": a }))) +} + /// Re-runs the gate on the proposal's exact inputs and says whether it reproduces. pub async fn regate(State(state): State, user: AuthUser, Path(id): Path) -> ApiResult> { let p = load(&state, &user, id, Role::Editor).await?; diff --git a/crates/wafer-api/src/design.rs b/crates/wafer-api/src/design.rs index 8e91775..454b20b 100644 --- a/crates/wafer-api/src/design.rs +++ b/crates/wafer-api/src/design.rs @@ -223,6 +223,34 @@ pub async fn run(State(state): State, user: AuthUser, Path(id): Path, + pub machines: Vec, +} + +/// [`Quality`] of `files` (spec, design and example testbenches). Blocks; runs the gate. +pub(crate) fn quality(files: &BTreeMap) -> Quality { + let tops: Vec = files + .get(SPEC_FILE) + .and_then(|t| serde_json::from_str::(t).ok()) + .map(|s| s.examples.iter().map(|e| spec::tb_name(&e.id)).collect()) + .unwrap_or_default(); + let mut machines = machines_in(files); + let report = gate::run(files); + let runs: Vec<&TestbenchRun> = report.runs.iter().filter(|r| tops.contains(&r.top)).collect(); + crate::fsm::mark(&mut machines, &runs); + Quality { + examples: tops.len(), + passing: runs.iter().filter(|r| r.verdict == "pass").count(), + coverage: crate::coverage::of(&runs), + machines, + } +} + /// The design's state machines (`crate::fsm`), or none if the design doesn't parse. fn machines_in(files: &BTreeMap) -> Vec { let hdl: Vec<(&str, &str)> = files diff --git a/crates/wafer-api/src/lib.rs b/crates/wafer-api/src/lib.rs index 426d845..592704a 100644 --- a/crates/wafer-api/src/lib.rs +++ b/crates/wafer-api/src/lib.rs @@ -85,6 +85,7 @@ pub fn router(state: AppState) -> Router { .route("/agent/{id}/merge", post(agent::merge)) .route("/agent/{id}/discard", post(agent::discard)) .route("/agent/{id}/regate", post(agent::regate)) + .route("/agent/{id}/compare", post(agent::compare)) .route("/jobs/{id}", get(jobs::get)) .route("/jobs/{id}/log", get(jobs::log)) .route("/jobs/{id}/cancel", post(jobs::cancel)) diff --git a/crates/wafer-api/tests/agent.rs b/crates/wafer-api/tests/agent.rs index f1fe06a..295e29b 100644 --- a/crates/wafer-api/tests/agent.rs +++ b/crates/wafer-api/tests/agent.rs @@ -210,6 +210,22 @@ async fn proposal_gate_evidence_and_merge(db: PgPool) { call(&e.addr, &ada, reqwest::Method::POST, &format!("/agent/{}/regate", a["id"].as_str().unwrap()), None).await; assert_eq!(rg["reproduced"], true, "{rg}"); + // Before and after the proposal: this project has no spec, so no examples or coverage, and + // neither version has a state machine. + let (s, cmp) = + call(&e.addr, &ada, reqwest::Method::POST, &format!("/agent/{}/compare", a["id"].as_str().unwrap()), None) + .await; + assert_eq!(s, 200, "{cmp}"); + for side in ["before", "after"] { + assert_eq!(cmp[side]["examples"], 0, "{cmp}"); + assert!(cmp[side]["coverage"].is_null() && cmp[side]["machines"] == json!([]), "{cmp}"); + } + let bob = token("bob", "bob@example.org"); + let (s, _) = + call(&e.addr, &bob, reqwest::Method::POST, &format!("/agent/{}/compare", a["id"].as_str().unwrap()), None) + .await; + assert_eq!(s, 404, "only members can compare"); + // A second proposal on the same base, to be made stale by merging the first. let b = propose(&e, &ada, &project, "Add a parity generator (again)").await; assert_eq!(b["status"], "ready"); diff --git a/docs/api.md b/docs/api.md index b7a7db3..f32b555 100644 --- a/docs/api.md +++ b/docs/api.md @@ -92,6 +92,7 @@ The spec format and the example step semantics are in [user-guide/design-view.md | GET | `/projects/{id}/agent` | viewer | Recent proposals. | | GET | `/agent/{id}` | viewer | Proposal: changed files, their base content, gate report, evidence log. | | POST | `/agent/{id}/merge` | editor | Merge; the gate runs again and must pass, and the base must still be `main`. | +| POST | `/agent/{id}/compare` | editor | How well the design is tested at the proposal's base commit and with its changes: `{before, after}`, each `{examples, passing, coverage, machines}`. | | POST | `/agent/{id}/regate` | editor | Re-run the gate on the recorded inputs and say whether it reproduces. | | POST | `/agent/{id}/discard` | editor | Discard a finished proposal. | diff --git a/docs/user-guide/agent.md b/docs/user-guide/agent.md index d3924a5..0b4c097 100644 --- a/docs/user-guide/agent.md +++ b/docs/user-guide/agent.md @@ -16,6 +16,18 @@ through five tools: - `run_simulation`, with the same simulator as the browser; - `submit_proposal`, to finish with a commit title and a summary for the reviewer. +## What the change does + +A proposal opens on **behaviour**: **Compare behaviour** runs every example on the project as it +was and with the change, and compares every output cycle by cycle, including the cycles after +each example ends. It says which examples behave differently and how, in words: "`led` now changes +1 cycle earlier", or "`busy` at cycle 40 was 1, now 0 (different in 3 cycles)". When nothing +differs, the change only affects cases the examples don't try (or nothing at all). + +Underneath, for editors, it compares how well each version is tested: examples passing, the cases +the examples try (coverage), and each state machine's moves added or removed +(`POST /agent/{id}/compare`). **Changes** still shows the code diff. + ## The simulation gate When the agent finishes, the server runs every testbench on the files the proposal would produce