Find an example: search the design's inputs for an untried case or a slipped mistake

CHANGELOG.md

@@ -8,6 +8,12 @@ All notable changes to Wafersmith are recorded here. The format follows
 
 ### Added
 
+- **Find an example**: for a case no example tries, or a planted mistake that slipped through,
+  Health can have the simulator search the design's inputs for an example that reaches it. The
+  example found is cut to the shortest and simplest, ends with checks of what the design did, and
+  opens in Examples unsaved, for a person to check. No AI and no cost; repeatable.
+  `POST /projects/{id}/spec/find-example`. On the starter designs with only their first example,
+  it found every untried case and 33 of 34 slipped mistakes (the last changes nothing visible).
 - **The example: use what the design did**: when an example fails and the design is right, one
   click rewrites the failing check from the simulation (moved a few cycles, a longer wait-until, a
   shorter hold, or the value the design gave), re-simulates until it passes, and lists the changes

apps/web/src/api.ts

@@ -215,7 +215,7 @@ export interface ExampleResult {
 export interface CoverageGap {
   file: string;
   line: number;
-  kind: "then" | "else" | "arm" | "default";
+  kind: "then" | "else" | "arm" | "default" | "part" | "side";
   what: string;
   labels: string | null;
   /** In words, with the condition in backticks: "`full` is never true in the examples". */
@@ -276,6 +276,19 @@ export interface MistakesReport {
   elapsed_ms: number;
 }
 
+/** What "Find an example" searches for: a coverage gap or a planted mistake that slipped through. */
+export type FindTarget =
+  | { for: "gap"; file: string; line: number; text: string }
+  | { for: "mistake"; file: string; line: number; kind: string; original: string; mistake: string };
+
+/** A draft example the search found (not added to the spec), or why there is none. */
+export interface FoundExample {
+  example: SpecExample | null;
+  tried: number;
+  elapsed_ms: number;
+  message: string | null;
+}
+
 /** How well a version of the design is tested (`POST /agent/{id}/compare`). */
 export interface Quality {
   examples: number;
@@ -631,6 +644,7 @@ export const api = {
     request<{ commit: string; changed: boolean }>("PUT", `/projects/${enc(id)}/spec`, body),
   runSpec: (id: string) => request<SpecRun>("POST", `/projects/${enc(id)}/spec/run`),
   plantMistakes: (id: string) => request<MistakesReport>("POST", `/projects/${enc(id)}/spec/mistakes`),
+  findExample: (id: string, target: FindTarget) => request<FoundExample>("POST", `/projects/${enc(id)}/spec/find-example`, target),
   exampleTestbench: (id: string, body: { spec: Spec; example: SpecExample }) =>
     request<{ path: string; top: string; source: string }>("POST", `/projects/${enc(id)}/spec/testbench`, body),
   draftSpec: (id: string, body: { instruction: string; spec?: Spec | null }) =>

apps/web/src/components/Design.tsx

@@ -4,6 +4,7 @@ import {
   isFinished,
   type Blocks,
   type ExampleResult,
+  type FindTarget,
   type Job,
   type MistakesReport,
   type Spec,
@@ -85,8 +86,8 @@ export function DesignFlow({ projectId, startIntent, onWide, head, canEdit, onOp
   const [busy, setBusy] = useState<string | null>(null);
   const [note, setNote] = useState<string | null>(null);
   const [run, setRun] = useState<SpecRun | null>(null);
-  // Text for "Add an example in words", from a coverage gap in Health.
-  const [exampleSeed, setExampleSeed] = useState<{ text: string } | null>(null);
+  // From Health: text for "Add an example in words", or a found example to select.
+  const [exampleSeed, setExampleSeed] = useState<{ text?: string; select?: string } | null>(null);
   const [budget, setBudget] = useState<{ today_usd: number; limit_usd: number | null } | null>(null);
   const refreshBudget = useCallback(() => {
     api.aiUsage().then(setBudget, () => setBudget(null));
@@ -292,6 +293,16 @@ export function DesignFlow({ projectId, startIntent, onWide, head, canEdit, onOp
               setExampleSeed({ text });
               setStep("examples");
             }}
+            onFoundExample={(ex) => {
+              if (!draft) return;
+              // Unsaved found examples may already use the id the server picked.
+              const ids = new Set(draft.examples.map((e) => e.id));
+              let id = ex.id;
+              for (let n = 1; ids.has(id); n++) id = `found_${n}`;
+              update({ examples: [...draft.examples, { ...ex, id }] });
+              setExampleSeed({ select: id });
+              setStep("examples");
+            }}
           />
         )}
       </div>
@@ -621,8 +632,8 @@ function Examples({
   fixExample,
   seed,
 }: {
-  /** Text to start "Add an example in words" with (a new object each time). */
-  seed: { text: string } | null;
+  /** Text to start "Add an example in words" with, or an example to select (a new object each time). */
+  seed: { text?: string; select?: string } | null;
   projectId: string;
   head: string;
   /** The chip from the Blocks view, which examples can target. */
@@ -647,7 +658,11 @@ function Examples({
   const [tried, setTried] = useState<string | null>(null);
   useEffect(() => {
     if (!seed) return;
-    setWords(seed.text);
+    if (seed.select) {
+      setSelId(seed.select);
+      return;
+    }
+    setWords(seed.text ?? "");
     window.setTimeout(() => {
       const box = document.getElementById("words") as HTMLTextAreaElement | null;
       box?.scrollIntoView({ block: "center" });
@@ -1250,10 +1265,13 @@ function Health({
   onShowRun,
   onOpenFile,
   onAddExample,
+  onFoundExample,
 }: {
   onOpenFile: (path: string) => void;
   /** Opens Examples with "Add an example in words" filled in. */
   onAddExample: (text: string) => void;
+  /** Adds an example "Find an example" found to the spec (unsaved) and opens it. */
+  onFoundExample: (ex: SpecExample) => void;
   projectId: string;
   head: string | null;
   /** From the spec, else the flow config's CLOCK_PERIOD. */
@@ -1377,9 +1395,12 @@ function Health({
               <span className="small">{withCode(g.text)}</span>
               <span className="coverage-actions">
                 {canEdit && g.kind !== "default" && (
-                  <button className="small" onClick={() => onAddExample(g.suggestion)}>
-                    Add an example
-                  </button>
+                  <>
+                    <FindExample projectId={projectId} target={{ for: "gap", file: g.file, line: g.line, text: g.text }} onFound={onFoundExample} />
+                    <button className="small" onClick={() => onAddExample(g.suggestion)} title="Describe an example in words for the assistant to write">
+                      Add in words
+                    </button>
+                  </>
                 )}
                 <button className="link small" onClick={() => onOpenFile(g.file)} title={`${g.file}, line ${g.line}`}>
                   {g.file.split("/").pop()}:{g.line}
@@ -1423,9 +1444,16 @@ function Health({
                   <span className="small">{withCode(g.text)}</span>
                   <span className="coverage-actions">
                     {canEdit && (
-                      <button className="small" onClick={() => onAddExample(g.suggestion)}>
-                        Add an example
-                      </button>
+                      <>
+                        <FindExample
+                          projectId={projectId}
+                          target={{ for: "mistake", file: g.file, line: g.line, kind: g.kind, original: g.original, mistake: g.mistake }}
+                          onFound={onFoundExample}
+                        />
+                        <button className="small" onClick={() => onAddExample(g.suggestion)} title="Describe an example in words for the assistant to write">
+                          Add in words
+                        </button>
+                      </>
                     )}
                     <button className="link small" onClick={() => onOpenFile(g.file)} title={`${g.file}, line ${g.line}`}>
                       {g.file.split("/").pop()}:{g.line}
@@ -1574,3 +1602,36 @@ const fmt = (v: unknown) => (typeof v === "number" ? Number(v.toPrecision(4)).to
 function withCode(text: string) {
   return text.split("`").map((part, i) => (i % 2 ? <code key={i}>{part}</code> : <span key={i}>{part}</span>));
 }
+
+/** "Find an example": the simulator searches the design's inputs for an example that reaches the
+ * target, which then opens in Examples as an unsaved draft. */
+function FindExample({ projectId, target, onFound }: { projectId: string; target: FindTarget; onFound: (ex: SpecExample) => void }) {
+  const [state, setState] = useState<"idle" | "searching" | string>("idle");
+  async function search() {
+    setState("searching");
+    try {
+      const r = await api.findExample(projectId, target);
+      if (r.example) {
+        setState("idle");
+        onFound(r.example);
+      } else {
+        setState(r.message ?? "Nothing found.");
+      }
+    } catch (e) {
+      setState((e as Error).message);
+    }
+  }
+  return (
+    <>
+      <button
+        className="small primary"
+        onClick={search}
+        disabled={state === "searching"}
+        title="The simulator tries the design's inputs until it finds an example for this, which you then check"
+      >
+        {state === "searching" ? "Searching…" : "Find an example"}
+      </button>
+      {state !== "idle" && state !== "searching" && <span className="muted small find-note">{state}</span>}
+    </>
+  );
+}

apps/web/src/styles.css

@@ -2304,10 +2304,18 @@ select.small {
 
 .coverage-actions {
   display: flex;
+  flex-wrap: wrap;
+  justify-content: flex-end;
   align-items: center;
   gap: 8px;
 }
 
+/* Why "Find an example" found nothing, under its buttons. */
+.coverage-actions .find-note {
+  flex-basis: 100%;
+  text-align: right;
+}
+
 /* Try it: the design as a small board. */
 .tryit {
   display: flex;

crates/wafer-api/src/coverage.rs

@@ -58,6 +58,11 @@ pub fn of(runs: &[&TestbenchRun]) -> Option<Coverage> {
     Some(Coverage { covered: hits.values().filter(|n| **n > 0).count(), total: hits.len(), gaps })
 }
 
+/// The gap `p` would be, in words (the `text` of [`Gap`]).
+pub(crate) fn text_of(p: &CoverPoint) -> String {
+    gap(&p.file, p.line, &p.kind, &p.what).text
+}
+
 fn counts(p: &CoverPoint) -> bool {
     !p.file.starts_with("tb/") && !matches!(p.kind, CoverKind::Default { written: false })
 }

crates/wafer-api/src/design.rs

@@ -289,6 +289,32 @@ pub async fn mistakes(
     report.map(Json).map_err(ApiError::Conflict)
 }
 
+/// How long a search for an example may take (shrinking the one found may take half again).
+const FIND_BUDGET: std::time::Duration = std::time::Duration::from_secs(20);
+
+/// "Find an example" (`crate::search`) for a coverage gap or a planted mistake that slipped
+/// through, at the branch tip. Returns a draft example; nothing is saved. Editors: it costs server
+/// time.
+pub async fn find_example(
+    State(state): State<AppState>,
+    user: AuthUser,
+    Path(id): Path<Uuid>,
+    Json(target): Json<crate::search::Target>,
+) -> ApiResult<Json<crate::search::Found>> {
+    authorize(&state, &user, id, Role::Editor).await?;
+    let commit = head(&state, id).await?;
+    let files = files_at(&state, id, &commit).await?;
+    let spec = match files.get(SPEC_FILE) {
+        Some(t) => parse_spec(t)?,
+        None => return Err(ApiError::NotFound("this project has no spec yet".into())),
+    };
+    let found = tokio::task::spawn_blocking(move || {
+        gate::with_slot(|| wafer_sim::on_big_stack(|| crate::search::find(&spec, &files, &target, FIND_BUDGET)))
+    })
+    .await?;
+    found.map(Json).map_err(ApiError::Conflict)
+}
+
 #[derive(Debug, Deserialize)]
 pub struct TestbenchReq {
     /// The spec being edited (unsaved edits included).

crates/wafer-api/src/lib.rs

@@ -18,6 +18,7 @@ pub mod licence;
 pub mod lsp;
 pub mod mistakes;
 pub mod projects;
+pub mod search;
 pub mod spec;
 pub mod spend;
 pub mod starters;
@@ -73,6 +74,7 @@ pub fn router(state: AppState) -> Router {
         .route("/projects/{id}/spec/run", post(design::run))
         .route("/projects/{id}/spec/testbench", post(design::testbench))
         .route("/projects/{id}/spec/mistakes", post(design::mistakes))
+        .route("/projects/{id}/spec/find-example", post(design::find_example))
         .route("/projects/{id}/spec/draft", post(design::draft))
         .route("/starters", get(starters::list))
         .route("/projects/{id}/starters/{starter}", post(starters::add))

crates/wafer-api/src/mistakes.rs

@@ -104,6 +104,35 @@ pub fn check(
     Ok(Report { caught, tried, sites: sites.len(), slipped, elapsed_ms: started.elapsed().as_millis() as u64 })
 }
 
+/// The design with one planted mistake in it: the one at `file`/`line` that turns `original` into
+/// `mistake`, as [`check`] reported it. None if the design no longer has it.
+pub(crate) fn planted(
+    modules: &[Module],
+    files: &BTreeMap<String, String>,
+    file: &str,
+    line: u32,
+    original: &str,
+    mistake: &str,
+) -> Option<Vec<Module>> {
+    let mine =
+        |m: &Module| !m.file.starts_with("tb/") && files.get(&m.file).is_some_and(|c| !crate::blocks::is_generated(c));
+    let mut probe = modules.to_vec();
+    let mut w = Walker::new(None);
+    for m in probe.iter_mut().filter(|m| mine(m)) {
+        w.module(m);
+    }
+    let i = w
+        .sites
+        .iter()
+        .position(|s| s.file == file && s.line == line && s.original == original && s.mistake == mistake)?;
+    let mut ms = modules.to_vec();
+    let mut w = Walker::new(Some(i));
+    for m in ms.iter_mut().filter(|m| mine(m)) {
+        w.module(m);
+    }
+    Some(ms)
+}
+
 /// Walks the design in a fixed order, numbering every place a mistake could go. Without a target
 /// it lists them; with one it plants that mistake and leaves the rest as they are.
 struct Walker {