Find examples for everything: one search for every gap and slipped mistake

CHANGELOG.md

@@ -8,6 +8,12 @@ All notable changes to Wafersmith are recorded here. The format follows
 
 ### Added
 
+- **Find examples for everything**: Health searches for examples for every untried case and every
+  planted mistake that slipped through at once. Each example found is checked against the targets
+  still open, so one example covers several (on the up/down counter with one example, 2 examples
+  covered all 6; in the test, 3 covered 12). The examples found are listed as sentences with what
+  each covers, to untick or add together; those not found say why. No AI and no cost.
+  `POST /api/projects/{id}/spec/find-examples`.
 - **Why? through memories**: in "Why is it this value?", a value read from a memory (`dout =
   mem[rptr]`) now names the word it was (`mem[0] = 17`), and that word can be clicked like any
   value, back to the line that stored it (`mem[wptr] <= din`) and why it ran. The simulator records

apps/web/src/api.ts

@@ -289,6 +289,16 @@ export interface FoundExample {
   message: string | null;
 }
 
+/** "Find examples for everything" (`POST .../spec/find-examples`): draft examples that between
+ * them cover the targets asked for, each with the targets it covers (indexes into the request),
+ * and why the rest weren't found. */
+export interface FoundExamples {
+  drafts: { example: SpecExample; covers: number[] }[];
+  left: { target: number; message: string }[];
+  tried: number;
+  elapsed_ms: number;
+}
+
 /** "Try a small fix": small repairs after which every example passes (`POST .../spec/small-fix`). */
 export interface SmallFixes {
   fixes: SmallFix[];
@@ -671,6 +681,8 @@ export const api = {
   smallFix: (id: string, focus: { file: string; line: number }[]) =>
     request<SmallFixes>("POST", `/projects/${enc(id)}/spec/small-fix`, { focus }),
   findExample: (id: string, target: FindTarget) => request<FoundExample>("POST", `/projects/${enc(id)}/spec/find-example`, target),
+  findExamples: (id: string, targets: FindTarget[]) =>
+    request<FoundExamples>("POST", `/projects/${enc(id)}/spec/find-examples`, { targets }),
   exampleTestbench: (id: string, body: { spec: Spec; example: SpecExample }) =>
     request<{ path: string; top: string; source: string }>("POST", `/projects/${enc(id)}/spec/testbench`, body),
   draftSpec: (id: string, body: { instruction: string; spec?: Spec | null }) =>

apps/web/src/components/Design.tsx

@@ -5,6 +5,7 @@ import {
   type Blocks,
   type ExampleResult,
   type FindTarget,
+  type FoundExamples,
   type Job,
   type MistakesReport,
   type SmallFix,
@@ -24,6 +25,7 @@ import { TryIt } from "./TryIt";
 import { SimClient } from "../sim/client";
 import { compareExamples } from "../lib/compare";
 import { BlocksStep } from "./Blocks";
+import { narrate } from "../lib/narrate";
 
 type Step = "describe" | "examples" | "blocks" | "health";
 
@@ -297,14 +299,19 @@ export function DesignFlow({ projectId, startIntent, onWide, head, canEdit, onOp
               setExampleSeed({ text });
               setStep("examples");
             }}
-            onFoundExample={(ex) => {
-              if (!draft) return;
-              // Unsaved found examples may already use the id the server picked.
+            spec={draft}
+            onFoundExamples={(found) => {
+              if (!draft || !found.length) return;
+              // Unsaved found examples may already use the ids the server picked.
               const ids = new Set(draft.examples.map((e) => e.id));
-              let id = ex.id;
-              for (let n = 1; ids.has(id); n++) id = `found_${n}`;
-              update({ examples: [...draft.examples, { ...ex, id }] });
-              setExampleSeed({ select: id });
+              const added = found.map((ex) => {
+                let id = ex.id;
+                for (let n = 1; ids.has(id); n++) id = `found_${n}`;
+                ids.add(id);
+                return { ...ex, id };
+              });
+              update({ examples: [...draft.examples, ...added] });
+              setExampleSeed({ select: added[0].id });
               setStep("examples");
             }}
           />
@@ -1573,13 +1580,16 @@ function Health({
   onShowRun,
   onOpenFile,
   onAddExample,
-  onFoundExample,
+  onFoundExamples,
+  spec,
 }: {
   onOpenFile: (path: string) => void;
   /** Opens Examples with "Add an example in words" filled in. */
   onAddExample: (text: string) => void;
-  /** Adds an example "Find an example" found to the spec (unsaved) and opens it. */
-  onFoundExample: (ex: SpecExample) => void;
+  /** Adds examples "Find an example" found to the spec (unsaved) and opens the first. */
+  onFoundExamples: (exs: SpecExample[]) => void;
+  /** The spec as being edited, to describe found examples in words. */
+  spec: Spec | null;
   projectId: string;
   head: string | null;
   /** From the spec, else the flow config's CLOCK_PERIOD. */
@@ -1609,6 +1619,7 @@ function Health({
   }
   const [error, setError] = useState<string | null>(null);
   const [tick, setTick] = useState(0);
+  const onFoundExample = (ex: SpecExample) => onFoundExamples([ex]);
 
   useEffect(() => {
     api.listJobs(projectId).then(setJobs, (e: Error) => setError(e.message));
@@ -1784,6 +1795,31 @@ function Health({
       ),
     });
   }
+  // Every case no example tries and every planted mistake that slipped through, for one search.
+  const report = mistakes && mistakes.head === head ? mistakes.report : null;
+  const findable: { target: FindTarget; text: string }[] = [
+    ...(cov?.gaps ?? []).filter((g) => g.kind !== "default").map((g) => ({ target: { for: "gap" as const, file: g.file, line: g.line, text: g.text }, text: g.text })),
+    ...(report?.slipped ?? []).map((g) => ({
+      target: { for: "mistake" as const, file: g.file, line: g.line, kind: g.kind, original: g.original, mistake: g.mistake },
+      text: g.text,
+    })),
+  ].slice(0, 60);
+  if (canEdit && spec && findable.length > 1) {
+    const gapsN = findable.filter((f) => f.target.for === "gap").length;
+    const slippedN = findable.length - gapsN;
+    cards.push({
+      id: "find-all",
+      ok: null,
+      title: "Find examples for everything",
+      plain:
+        `The simulator can search for examples for ${[gapsN && `the ${gapsN} untried ${gapsN === 1 ? "case" : "cases"}`, slippedN && `the ${slippedN} slipped ${slippedN === 1 ? "mistake" : "mistakes"}`].filter(Boolean).join(" and ")} above at once, using one example for several where it can. You check each one before it is added.` +
+        (!report && exTotal > 0 ? " Plant mistakes first to include those too." : ""),
+      explain:
+        "It searches for one target at a time, starting from your examples. Each example it finds is run against the targets still open, and every one it also covers is ticked off, so the set stays small. The checks at the end of each example are what the design did: if one looks wrong, you have found a bug.",
+      expert: `search per target ≤ 10 s, all ≤ 90 s; inputs from the examples, changed and fresh; cut to the shortest; covers = takes the branch / kills the mutant`,
+      extra: <FindAll key={`${head}:${findable.length}`} projectId={projectId} spec={spec} items={findable} onAdd={onFoundExamples} />,
+    });
+  }
   for (const [i, m] of (run?.machines ?? []).entries()) {
     const untested = m.transitions.filter((t) => t.covered === false).length;
     cards.push({
@@ -1911,6 +1947,124 @@ function withCode(text: string) {
   return text.split("`").map((part, i) => (i % 2 ? <code key={i}>{part}</code> : <span key={i}>{part}</span>));
 }
 
+/** What an example covers, in a few words: "tries `up` = 0 while …", "catches `count + 1` as
+ * `count - 1` (line 14)". */
+function covers(item: { target: FindTarget; text: string }): string {
+  const t = item.target;
+  if (t.for === "gap") return item.text.replace(/^Never tried: /, "tries ");
+  if (t.kind === "forget") return `catches \`${t.original}\` left out (line ${t.line})`;
+  if (t.kind === "reverse") return `catches \`${t.original}\` the other way round (line ${t.line})`;
+  return `catches \`${t.original}\` as \`${t.mistake}\` (line ${t.line})`;
+}
+
+/** "Find examples for everything": one search for every untried case and slipped mistake, then a
+ * list of the examples found to check and add together. */
+function FindAll({
+  projectId,
+  spec,
+  items,
+  onAdd,
+}: {
+  projectId: string;
+  spec: Spec;
+  items: { target: FindTarget; text: string }[];
+  onAdd: (exs: SpecExample[]) => void;
+}) {
+  const [busy, setBusy] = useState(false);
+  const [error, setError] = useState<string | null>(null);
+  const [found, setFound] = useState<FoundExamples | null>(null);
+  const [skip, setSkip] = useState<Set<number>>(new Set());
+  async function search() {
+    setBusy(true);
+    setError(null);
+    try {
+      setFound(await api.findExamples(projectId, items.map((i) => i.target)));
+      setSkip(new Set());
+    } catch (e) {
+      setError((e as Error).message);
+    } finally {
+      setBusy(false);
+    }
+  }
+  const covered = found ? found.drafts.reduce((n, d) => n + d.covers.length, 0) : 0;
+  const chosen = found ? found.drafts.filter((_, i) => !skip.has(i)) : [];
+  return (
+    <div className="design-col find-all" style={{ gap: 6 }}>
+      <div className="design-row">
+        <button className="small primary" onClick={search} disabled={busy}>
+          {busy ? "Searching… (up to 90 s)" : found ? "Search again" : `Find examples for all ${items.length}`}
+        </button>
+        {error && <span className="error small">{error}</span>}
+      </div>
+      {found && (
+        <>
+          <p className="small" style={{ margin: 0 }}>
+            {found.drafts.length
+              ? `${found.drafts.length} ${found.drafts.length === 1 ? "example covers" : "examples cover"} ${covered} of ${items.length}. Read each one: the checks at the end are what the design did, so make sure that is what it should do.`
+              : `No example found for any of the ${items.length}.`}{" "}
+            <span className="muted">
+              ({found.tried} input sequences tried in {found.elapsed_ms < 100 ? "under 0.1" : (found.elapsed_ms / 1000).toFixed(1)} s)
+            </span>
+          </p>
+          {found.drafts.length > 0 && (
+            <ul className="find-all-list">
+              {found.drafts.map((d, i) => (
+                <li key={d.example.id}>
+                  <label className="find-all-pick">
+                    <input
+                      type="checkbox"
+                      checked={!skip.has(i)}
+                      onChange={() => {
+                        const next = new Set(skip);
+                        if (next.has(i)) next.delete(i);
+                        else next.add(i);
+                        setSkip(next);
+                      }}
+                    />
+                    <strong>{d.example.title}</strong>
+                  </label>
+                  <span className="small">{narrate(spec, d.example)}</span>
+                  <span className="muted small">
+                    Covers: {d.covers.map((c, k) => (
+                      <span key={c}>
+                        {k > 0 && "; "}
+                        {withCode(covers(items[c]))}
+                      </span>
+                    ))}
+                  </span>
+                </li>
+              ))}
+            </ul>
+          )}
+          {found.left.length > 0 && (
+            <details className="small">
+              <summary>
+                {found.left.length} not found
+              </summary>
+              <ul className="coverage-gaps">
+                {found.left.map((u) => (
+                  <li key={u.target}>
+                    <span>{withCode(items[u.target].text)}</span>
+                    <span className="muted">{u.message}</span>
+                  </li>
+                ))}
+              </ul>
+            </details>
+          )}
+          {found.drafts.length > 0 && (
+            <div className="design-row">
+              <button className="small primary" disabled={!chosen.length} onClick={() => onAdd(chosen.map((d) => d.example))}>
+                Add {chosen.length} {chosen.length === 1 ? "example" : "examples"}
+              </button>
+              <span className="muted small">They open in Examples, unsaved, to save with the rest.</span>
+            </div>
+          )}
+        </>
+      )}
+    </div>
+  );
+}
+
 /** "Find an example": the simulator searches the design's inputs for an example that reaches the
  * target, which then opens in Examples as an unsaved draft. */
 function FindExample({ projectId, target, onFound }: { projectId: string; target: FindTarget; onFound: (ex: SpecExample) => void }) {

apps/web/src/styles.css

@@ -2316,6 +2316,30 @@ select.small {
   text-align: right;
 }
 
+/* "Find examples for everything": the examples found, to check and add. */
+.find-all-list {
+  list-style: none;
+  margin: 0;
+  padding: 0;
+  display: flex;
+  flex-direction: column;
+  gap: 8px;
+}
+
+.find-all-list li {
+  display: flex;
+  flex-direction: column;
+  gap: 2px;
+  padding: 6px 0;
+  border-top: 1px solid var(--border);
+}
+
+.find-all-pick {
+  display: flex;
+  align-items: center;
+  gap: 6px;
+}
+
 /* Try it: the design as a small board. */
 .tryit {
   display: flex;

crates/wafer-api/src/design.rs

@@ -315,6 +315,47 @@ pub async fn find_example(
     found.map(Json).map_err(ApiError::Conflict)
 }
 
+/// "Find examples for everything": the most targets one request may ask for, how long the search
+/// for each may take, and how long all of them may.
+const FIND_ALL_MAX: usize = 60;
+const FIND_ALL_EACH: std::time::Duration = std::time::Duration::from_secs(10);
+const FIND_ALL_BUDGET: std::time::Duration = std::time::Duration::from_secs(90);
+
+#[derive(Debug, Deserialize)]
+pub struct FindAllReq {
+    pub targets: Vec<crate::search::Target>,
+}
+
+/// "Find examples for everything" (`crate::search::find_all`) at the branch tip: draft examples
+/// that between them cover the coverage gaps and slipped mistakes asked for, and why the rest
+/// weren't found. Nothing is saved. Editors: it costs server time.
+pub async fn find_examples(
+    State(state): State<AppState>,
+    user: AuthUser,
+    Path(id): Path<Uuid>,
+    Json(req): Json<FindAllReq>,
+) -> ApiResult<Json<crate::search::FoundAll>> {
+    authorize(&state, &user, id, Role::Editor).await?;
+    if req.targets.is_empty() || req.targets.len() > FIND_ALL_MAX {
+        return Err(ApiError::BadRequest(format!("ask for 1 to {FIND_ALL_MAX} targets")));
+    }
+    let commit = head(&state, id).await?;
+    let files = files_at(&state, id, &commit).await?;
+    let spec = match files.get(SPEC_FILE) {
+        Some(t) => parse_spec(t)?,
+        None => return Err(ApiError::NotFound("this project has no spec yet".into())),
+    };
+    let found = tokio::task::spawn_blocking(move || {
+        gate::with_slot(|| {
+            wafer_sim::on_big_stack(|| {
+                crate::search::find_all(&spec, &files, &req.targets, FIND_ALL_EACH, FIND_ALL_BUDGET)
+            })
+        })
+    })
+    .await?;
+    found.map(Json).map_err(ApiError::Conflict)
+}
+
 /// How long "Try a small fix" may search.
 const FIX_BUDGET: std::time::Duration = std::time::Duration::from_secs(20);
 

crates/wafer-api/src/lib.rs

@@ -76,6 +76,7 @@ pub fn router(state: AppState) -> Router {
         .route("/projects/{id}/spec/testbench", post(design::testbench))
         .route("/projects/{id}/spec/mistakes", post(design::mistakes))
         .route("/projects/{id}/spec/find-example", post(design::find_example))
+        .route("/projects/{id}/spec/find-examples", post(design::find_examples))
         .route("/projects/{id}/spec/small-fix", post(design::small_fix))
         .route("/projects/{id}/spec/draft", post(design::draft))
         .route("/starters", get(starters::list))