What a proposal does: behaviour before and after

CHANGELOG.md

@@ -8,6 +8,10 @@ All notable changes to Wafersmith are recorded here. The format follows
 
 ### Added
 
+- **What a proposal does**: agent proposals open on a behaviour tab. **Compare behaviour** runs
+  every example before and after the change (in the browser) and describes each output that
+  behaves differently ("`led` now changes 1 cycle earlier"), and compares examples passing,
+  coverage and state-machine moves (`POST /agent/{id}/compare`).
 - **Review report**: one printable page per project (Health → Review report): what the design is,
   its examples as plain sentences with their results, coverage and planted mistakes, its state
   machines, the latest build and what still needs an expert, and where it came from (the AI's

apps/web/src/api.ts

@@ -276,6 +276,14 @@ export interface MistakesReport {
   elapsed_ms: number;
 }
 
+/** How well a version of the design is tested (`POST /agent/{id}/compare`). */
+export interface Quality {
+  examples: number;
+  passing: number;
+  coverage: Coverage | null;
+  machines: Machine[];
+}
+
 export interface SpecRun {
   commit: string;
   examples: ExampleResult[];
@@ -656,6 +664,7 @@ export const api = {
   mergeProposal: (pid: string) => request<Proposal>("POST", `/agent/${enc(pid)}/merge`),
   discardProposal: (pid: string) => request<Proposal>("POST", `/agent/${enc(pid)}/discard`),
   regate: (pid: string) => request<{ report: GateReport; reproduced: boolean }>("POST", `/agent/${enc(pid)}/regate`),
+  compareProposal: (pid: string) => request<{ before: Quality; after: Quality }>("POST", `/agent/${enc(pid)}/compare`),
   createJob: (id: string, body: { commit?: string; config?: string } = {}) =>
     request<Job>("POST", `/projects/${enc(id)}/jobs`, body),
   listJobs: (id: string) => request<Job[]>("GET", `/projects/${enc(id)}/jobs`),

apps/web/src/components/Agent.tsx

@@ -1,6 +1,19 @@
 import { useEffect, useState, type FormEvent } from "react";
-import { api, ApiError, type EvidenceStep, type GateReport, type Proposal, type ProposalListItem, type ProposalStatus } from "../api";
+import {
+  api,
+  ApiError,
+  type Blocks,
+  type EvidenceStep,
+  type GateReport,
+  type Proposal,
+  type ProposalListItem,
+  type ProposalStatus,
+  type Quality,
+  type Spec,
+} from "../api";
+import { compareExamples, type ExampleChange } from "../lib/compare";
 import { hunks, lineDiff } from "../lib/diff";
+import { SimClient } from "../sim/client";
 
 const POLL_MS = 1500;
 
@@ -112,7 +125,7 @@ export function AgentList({
   );
 }
 
-type Tab = "changes" | "gate" | "evidence";
+type Tab = "behaviour" | "changes" | "gate" | "evidence";
 
 /** Review a proposal: diff, gate, evidence; merge when the gate passed. */
 export function AgentView({
@@ -149,7 +162,8 @@ export function AgentView({
         if (stopped) return;
         setP(next);
         if (next.status === "running") timer = window.setTimeout(poll, POLL_MS);
-        else setTab((t) => (t === "evidence" && Object.keys(next.files).length ? "changes" : t));
+        // A finished proposal with changes opens on what it does, not on the code.
+        else setTab((t) => (t === "evidence" && Object.keys(next.files).length ? "behaviour" : t));
       } catch (e) {
         if (!stopped) setError((e as Error).message);
       }
@@ -190,7 +204,7 @@ export function AgentView({
         )}
         <span className="spacer" />
         <div className="run-tabs" role="tablist">
-          {(["changes", "gate", "evidence"] as Tab[]).map((t) => (
+          {(["behaviour", "changes", "gate", "evidence"] as Tab[]).map((t) => (
             <button key={t} role="tab" aria-selected={tab === t} className={tab === t ? "active" : ""} onClick={() => setTab(t)}>
               {t}
             </button>
@@ -248,6 +262,7 @@ export function AgentView({
               ))}
             </div>
           )}
+          {tab === "behaviour" && <Behaviour p={p} canRun={canMerge} />}
           {tab === "gate" && <Gate report={p.gate} regate={regate} onRegate={async () => setRegate(await api.regate(p.id))} />}
           {tab === "evidence" && <Evidence p={p} />}
         </div>
@@ -393,3 +408,148 @@ function Evidence({ p }: { p: Proposal }) {
     </div>
   );
 }
+
+/** What the proposal does: each example before and after it, and how well the design is tested. */
+function Behaviour({ p, canRun }: { p: Proposal; canRun: boolean }) {
+  const [examples, setExamples] = useState<ExampleChange[] | null>(null);
+  const [quality, setQuality] = useState<{ before: Quality; after: Quality } | null>(null);
+  const [busy, setBusy] = useState(false);
+  const [error, setError] = useState<string | null>(null);
+
+  async function compare() {
+    setBusy(true);
+    setError(null);
+    const client = new SimClient();
+    try {
+      const snap = await api.snapshot(p.project_id, p.base_commit);
+      const before = snap.files;
+      const after = { ...before, ...p.files };
+      const specText = before["spec.json"];
+      if (!specText) throw new Error("the project has no spec, so there are no examples to compare");
+      const spec = JSON.parse(specText) as Spec;
+      const chip = before["blocks.json"] ? (JSON.parse(before["blocks.json"]) as Blocks) : null;
+      const [ex, q] = await Promise.all([
+        compareExamples(client, spec, chip, before, after),
+        canRun ? api.compareProposal(p.id) : Promise.resolve(null),
+      ]);
+      setExamples(ex);
+      setQuality(q);
+    } catch (e) {
+      setError((e as Error).message);
+    } finally {
+      client.dispose();
+      setBusy(false);
+    }
+  }
+
+  if (p.status === "running") return <p className="muted pad">The agent is still working.</p>;
+  const changed = examples?.filter((e) => e.differences.length || e.before !== e.after) ?? [];
+  return (
+    <div className="pad behaviour">
+      <p className="small">
+        Runs every example on the project as it was (<code>{p.base_commit.slice(0, 8)}</code>) and with this change, and compares
+        every output cycle by cycle, including the cycles after each example ends.
+      </p>
+      <button className="primary small" onClick={compare} disabled={busy}>
+        {busy ? "Comparing…" : examples ? "Compare again" : "Compare behaviour"}
+      </button>
+      {error && <p className="error small">{error}</p>}
+      {examples && (
+        <>
+          <h4>The examples</h4>
+          {changed.length === 0 ? (
+            <p className="ok small">
+              Every example behaves exactly as before. The change only affects cases the examples don't try, or none at all; the
+              coverage below says whether that leaves something untested.
+            </p>
+          ) : (
+            <p className="small">
+              {changed.length} of {examples.length} example{examples.length === 1 ? "" : "s"} behave differently:
+            </p>
+          )}
+          <ul className="behaviour-list">
+            {examples
+              .filter((e) => e.differences.length || e.before !== e.after || e.problem)
+              .map((e) => (
+                <li key={e.id}>
+                  <strong>{e.title}</strong>{" "}
+                  <span className="muted small">
+                    {verdict(e.before)} → {verdict(e.after)}
+                  </span>
+                  {e.problem && <p className="error small">{e.problem}</p>}
+                  <ul className="small">
+                    {e.differences.map((d) => (
+                      <li key={d}>{withCode(d)}</li>
+                    ))}
+                  </ul>
+                </li>
+              ))}
+          </ul>
+        </>
+      )}
+      {quality && (
+        <>
+          <h4>How well it is tested</h4>
+          <table className="report-table small">
+            <tbody>
+              <tr>
+                <th>Examples passing</th>
+                <td>
+                  {quality.before.passing} of {quality.before.examples} → {quality.after.passing} of {quality.after.examples}
+                </td>
+              </tr>
+              <tr>
+                <th>Cases the examples try</th>
+                <td>
+                  {cov(quality.before)} → {cov(quality.after)}
+                </td>
+              </tr>
+              {moves(quality).map(([label, text]) => (
+                <tr key={label}>
+                  <th>{label}</th>
+                  <td>{text}</td>
+                </tr>
+              ))}
+            </tbody>
+          </table>
+          {(quality.after.coverage?.gaps.length ?? 0) > (quality.before.coverage?.gaps.length ?? 0) && (
+            <p className="warn small">
+              With this change the examples leave more cases untried; Health lists them once it is merged.
+            </p>
+          )}
+        </>
+      )}
+      {examples && !canRun && <p className="muted small">The coverage comparison needs an editor of the project.</p>}
+    </div>
+  );
+}
+
+const verdict = (v: boolean | null) => (v === null ? "couldn't run" : v ? "passes" : "fails");
+const cov = (q: Quality) => (q.coverage ? `${q.coverage.covered} of ${q.coverage.total}` : "none");
+
+/** State machines and their moves: what the change adds or removes. */
+function moves(q: { before: Quality; after: Quality }): [string, string][] {
+  const key = (m: Quality["machines"][number]) => `${m.module}.${m.state}`;
+  const list = (m: Quality["machines"][number]) => m.transitions.map((t) => `${t.from} → ${t.to} when ${t.when}`);
+  const out: [string, string][] = [];
+  const names = new Set([...q.before.machines.map(key), ...q.after.machines.map(key)]);
+  for (const name of names) {
+    const b = q.before.machines.find((m) => key(m) === name);
+    const a = q.after.machines.find((m) => key(m) === name);
+    const bl = b ? list(b) : [];
+    const al = a ? list(a) : [];
+    const added = al.filter((x) => !bl.includes(x));
+    const removed = bl.filter((x) => !al.includes(x));
+    const label = `State machine ${name}`;
+    if (!b) out.push([label, `new, with ${al.length} moves`]);
+    else if (!a) out.push([label, "removed"]);
+    else if (added.length || removed.length)
+      out.push([label, [...added.map((x) => `+ ${x.replace(/`/g, "")}`), ...removed.map((x) => `− ${x.replace(/`/g, "")}`)].join("; ")]);
+    else out.push([label, `unchanged (${al.length} moves)`]);
+  }
+  return out;
+}
+
+function withCode(text: string) {
+  return text.split("`").map((part, i) => (i % 2 ? <code key={i}>{part}</code> : <span key={i}>{part}</span>));
+}

apps/web/src/lib/compare.ts

@@ -0,0 +1,99 @@
+// What a change does to the design's behaviour: every example simulated before and after it, and
+// each output compared cycle by cycle (including the cycles after the example ends), described in
+// plain words ("busy now changes 1 cycle earlier").
+
+import type { Blocks, Spec } from "../api";
+import type { SimClient } from "../sim/client";
+import { extendTestbench, fmt, replay, targetSpec } from "./replay";
+
+export interface ExampleChange {
+  id: string;
+  title: string;
+  /** Whether the example passed before and after; null if it couldn't run. */
+  before: boolean | null;
+  after: boolean | null;
+  /** One line per output that behaves differently. */
+  differences: string[];
+  problem: string | null;
+}
+
+/** How far a shifted copy of the same behaviour may be (cycles). */
+const MAX_SHIFT = 5;
+
+export async function compareExamples(
+  client: SimClient,
+  spec: Spec,
+  chip: Blocks | null,
+  before: Record<string, string>,
+  after: Record<string, string>,
+): Promise<ExampleChange[]> {
+  const out: ExampleChange[] = [];
+  for (const ex of spec.examples) {
+    const tbPath = `tb/examples/tb_ex_${ex.id}.v`;
+    const tb = before[tbPath];
+    const tspec = targetSpec(spec, ex, chip);
+    const change: ExampleChange = { id: ex.id, title: ex.title, before: null, after: null, differences: [], problem: null };
+    if (!tb) {
+      change.problem = "its testbench isn't in the project (save the spec)";
+      out.push(change);
+      continue;
+    }
+    try {
+      const run = async (files: Record<string, string>) => {
+        const list = Object.entries(files)
+          .filter(([p]) => /\.s?v$/.test(p) && !p.startsWith("tb/"))
+          .map(([name, source]) => ({ name, source }));
+        list.push({ name: tbPath, source: extendTestbench(tb, tspec) });
+        return replay(tspec, ex, await client.run(list, `tb_ex_${ex.id}`));
+      };
+      const b = await run(before);
+      const a = await run(after);
+      change.before = !b.firstFail && !b.problem;
+      change.after = !a.firstFail && !a.problem;
+      for (const rb of b.rows.filter((r) => r.dir === "out")) {
+        const ra = a.rows.find((r) => r.name === rb.name);
+        if (!ra) continue;
+        const d = describe(rb.name, rb.width, rb.values, ra.values);
+        if (d) change.differences.push(d);
+      }
+    } catch (e) {
+      change.problem = (e as Error).message;
+    }
+    out.push(change);
+  }
+  return out;
+}
+
+/** How `after` differs from `before` for one output, or null if it doesn't. */
+export function describe(name: string, width: number, before: (bigint | null)[], after: (bigint | null)[]): string | null {
+  const n = Math.min(before.length, after.length);
+  const differ: number[] = [];
+  for (let c = 0; c < n; c++) if (before[c] !== null && after[c] !== null && before[c] !== after[c]) differ.push(c);
+  if (!differ.length) return null;
+  // The same behaviour moved in time: after[c] == before[c - k] wherever both are known.
+  for (let k = 1; k <= MAX_SHIFT; k++) {
+    for (const dir of [1, -1]) {
+      const s = k * dir;
+      let ok = true;
+      let compared = 0;
+      for (let c = Math.max(0, s); c < n && c - s < n; c++) {
+        const x = after[c];
+        const y = before[c - s];
+        if (x === null || y === null) continue;
+        compared++;
+        if (x !== y) {
+          ok = false;
+          break;
+        }
+      }
+      if (ok && compared > 0) {
+        return `\`${name}\` now changes ${k} cycle${k === 1 ? "" : "s"} ${dir > 0 ? "later" : "earlier"} (first difference at cycle ${differ[0]})`;
+      }
+    }
+  }
+  const c = differ[0];
+  return (
+    `\`${name}\` at cycle ${c} was ${fmt(before[c], width)}, now ${fmt(after[c], width)}` +
+    (differ.length > 1 ? ` (different in ${differ.length} cycles)` : "")
+  );
+}

apps/web/src/styles.css

@@ -2632,3 +2632,16 @@ a.button {
     overflow: visible;
   }
 }
+
+/* The behaviour comparison of an agent proposal. */
+.behaviour h4 {
+  margin: 14px 0 4px;
+}
+
+.behaviour-list {
+  padding-left: 18px;
+}
+
+.behaviour-list > li {
+  margin-bottom: 8px;
+}

crates/wafer-api/src/agent.rs

@@ -333,6 +333,21 @@ async fn run_gate(tree: BTreeMap<String, String>) -> ApiResult<GateReport> {
     Ok(tokio::task::spawn_blocking(move || gate::run(&tree)).await?)
 }
 
+/// What the proposal does to how well the design is tested: examples passing, coverage and state
+/// machines, at its base commit and with its changes (editors: it runs the gate twice).
+pub async fn compare(State(state): State<AppState>, user: AuthUser, Path(id): Path<Uuid>) -> ApiResult<Json<Value>> {
+    let p = load(&state, &user, id, Role::Editor).await?;
+    let before: BTreeMap<String, String> =
+        crate::workspace::text_files(state.repos.clone(), p.project_id, p.base_commit.clone(), MAX_FILES_TEXT)
+            .await?
+            .into_iter()
+            .collect();
+    let after = proposed_tree(&state, &p).await?;
+    let (b, a) =
+        tokio::task::spawn_blocking(move || (crate::design::quality(&before), crate::design::quality(&after))).await?;
+    Ok(Json(json!({ "before": b, "after": a })))
+}
+
 /// Re-runs the gate on the proposal's exact inputs and says whether it reproduces.
 pub async fn regate(State(state): State<AppState>, user: AuthUser, Path(id): Path<Uuid>) -> ApiResult<Json<Value>> {
     let p = load(&state, &user, id, Role::Editor).await?;