From c632ad33e90c85efa45822b6a274e968d93b3bb0 Mon Sep 17 00:00:00 2001 From: Rahul Tomar Date: Fri, 6 Nov 2026 16:05:08 +0100 Subject: [PATCH] Find an example: search the design's inputs for an untried case or a slipped mistake Health's coverage gaps and slipped planted mistakes get a "Find an example" button. The simulator tries the examples' inputs (as they are and changed) and fresh sequences of values the design uses, keeps the first that reaches the case (or makes the mistaken copy differ at an output), cuts it to the shortest, undoes input changes it doesn't need, and ends with checks of what the design did. The draft opens in Examples, unsaved, for a person to check. Deterministic; no AI, no cost. POST /projects/{id}/spec/find-example. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 6 + apps/web/src/api.ts | 16 +- apps/web/src/components/Design.tsx | 83 +++- apps/web/src/styles.css | 8 + crates/wafer-api/src/coverage.rs | 5 + crates/wafer-api/src/design.rs | 26 ++ crates/wafer-api/src/lib.rs | 2 + crates/wafer-api/src/mistakes.rs | 29 ++ crates/wafer-api/src/search.rs | 637 +++++++++++++++++++++++++++++ crates/wafer-api/src/spec.rs | 2 +- docs/api.md | 1 + docs/user-guide/design-view.md | 25 +- 12 files changed, 821 insertions(+), 19 deletions(-) create mode 100644 crates/wafer-api/src/search.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index fa81fdb..5f496b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,12 @@ All notable changes to Wafersmith are recorded here. The format follows ### Added +- **Find an example**: for a case no example tries, or a planted mistake that slipped through, + Health can have the simulator search the design's inputs for an example that reaches it. The + example found is cut to the shortest and simplest, ends with checks of what the design did, and + opens in Examples unsaved, for a person to check. No AI and no cost; repeatable. + `POST /projects/{id}/spec/find-example`. On the starter designs with only their first example, + it found every untried case and 33 of 34 slipped mistakes (the last changes nothing visible). - **The example: use what the design did**: when an example fails and the design is right, one click rewrites the failing check from the simulation (moved a few cycles, a longer wait-until, a shorter hold, or the value the design gave), re-simulates until it passes, and lists the changes diff --git a/apps/web/src/api.ts b/apps/web/src/api.ts index 06a95ae..e47d5eb 100644 --- a/apps/web/src/api.ts +++ b/apps/web/src/api.ts @@ -215,7 +215,7 @@ export interface ExampleResult { export interface CoverageGap { file: string; line: number; - kind: "then" | "else" | "arm" | "default"; + kind: "then" | "else" | "arm" | "default" | "part" | "side"; what: string; labels: string | null; /** In words, with the condition in backticks: "`full` is never true in the examples". */ @@ -276,6 +276,19 @@ export interface MistakesReport { elapsed_ms: number; } +/** What "Find an example" searches for: a coverage gap or a planted mistake that slipped through. */ +export type FindTarget = + | { for: "gap"; file: string; line: number; text: string } + | { for: "mistake"; file: string; line: number; kind: string; original: string; mistake: string }; + +/** A draft example the search found (not added to the spec), or why there is none. */ +export interface FoundExample { + example: SpecExample | null; + tried: number; + elapsed_ms: number; + message: string | null; +} + /** How well a version of the design is tested (`POST /agent/{id}/compare`). */ export interface Quality { examples: number; @@ -631,6 +644,7 @@ export const api = { request<{ commit: string; changed: boolean }>("PUT", `/projects/${enc(id)}/spec`, body), runSpec: (id: string) => request("POST", `/projects/${enc(id)}/spec/run`), plantMistakes: (id: string) => request("POST", `/projects/${enc(id)}/spec/mistakes`), + findExample: (id: string, target: FindTarget) => request("POST", `/projects/${enc(id)}/spec/find-example`, target), exampleTestbench: (id: string, body: { spec: Spec; example: SpecExample }) => request<{ path: string; top: string; source: string }>("POST", `/projects/${enc(id)}/spec/testbench`, body), draftSpec: (id: string, body: { instruction: string; spec?: Spec | null }) => diff --git a/apps/web/src/components/Design.tsx b/apps/web/src/components/Design.tsx index 6238198..fe606b4 100644 --- a/apps/web/src/components/Design.tsx +++ b/apps/web/src/components/Design.tsx @@ -4,6 +4,7 @@ import { isFinished, type Blocks, type ExampleResult, + type FindTarget, type Job, type MistakesReport, type Spec, @@ -85,8 +86,8 @@ export function DesignFlow({ projectId, startIntent, onWide, head, canEdit, onOp const [busy, setBusy] = useState(null); const [note, setNote] = useState(null); const [run, setRun] = useState(null); - // Text for "Add an example in words", from a coverage gap in Health. - const [exampleSeed, setExampleSeed] = useState<{ text: string } | null>(null); + // From Health: text for "Add an example in words", or a found example to select. + const [exampleSeed, setExampleSeed] = useState<{ text?: string; select?: string } | null>(null); const [budget, setBudget] = useState<{ today_usd: number; limit_usd: number | null } | null>(null); const refreshBudget = useCallback(() => { api.aiUsage().then(setBudget, () => setBudget(null)); @@ -292,6 +293,16 @@ export function DesignFlow({ projectId, startIntent, onWide, head, canEdit, onOp setExampleSeed({ text }); setStep("examples"); }} + onFoundExample={(ex) => { + if (!draft) return; + // Unsaved found examples may already use the id the server picked. + const ids = new Set(draft.examples.map((e) => e.id)); + let id = ex.id; + for (let n = 1; ids.has(id); n++) id = `found_${n}`; + update({ examples: [...draft.examples, { ...ex, id }] }); + setExampleSeed({ select: id }); + setStep("examples"); + }} /> )} @@ -621,8 +632,8 @@ function Examples({ fixExample, seed, }: { - /** Text to start "Add an example in words" with (a new object each time). */ - seed: { text: string } | null; + /** Text to start "Add an example in words" with, or an example to select (a new object each time). */ + seed: { text?: string; select?: string } | null; projectId: string; head: string; /** The chip from the Blocks view, which examples can target. */ @@ -647,7 +658,11 @@ function Examples({ const [tried, setTried] = useState(null); useEffect(() => { if (!seed) return; - setWords(seed.text); + if (seed.select) { + setSelId(seed.select); + return; + } + setWords(seed.text ?? ""); window.setTimeout(() => { const box = document.getElementById("words") as HTMLTextAreaElement | null; box?.scrollIntoView({ block: "center" }); @@ -1250,10 +1265,13 @@ function Health({ onShowRun, onOpenFile, onAddExample, + onFoundExample, }: { onOpenFile: (path: string) => void; /** Opens Examples with "Add an example in words" filled in. */ onAddExample: (text: string) => void; + /** Adds an example "Find an example" found to the spec (unsaved) and opens it. */ + onFoundExample: (ex: SpecExample) => void; projectId: string; head: string | null; /** From the spec, else the flow config's CLOCK_PERIOD. */ @@ -1377,9 +1395,12 @@ function Health({ {withCode(g.text)} {canEdit && g.kind !== "default" && ( - + <> + + + )} + <> + + + )} + {state !== "idle" && state !== "searching" && {state}} + + ); +} diff --git a/apps/web/src/styles.css b/apps/web/src/styles.css index a85c004..b7a8bc4 100644 --- a/apps/web/src/styles.css +++ b/apps/web/src/styles.css @@ -2304,10 +2304,18 @@ select.small { .coverage-actions { display: flex; + flex-wrap: wrap; + justify-content: flex-end; align-items: center; gap: 8px; } +/* Why "Find an example" found nothing, under its buttons. */ +.coverage-actions .find-note { + flex-basis: 100%; + text-align: right; +} + /* Try it: the design as a small board. */ .tryit { display: flex; diff --git a/crates/wafer-api/src/coverage.rs b/crates/wafer-api/src/coverage.rs index 33ad4e7..e135758 100644 --- a/crates/wafer-api/src/coverage.rs +++ b/crates/wafer-api/src/coverage.rs @@ -58,6 +58,11 @@ pub fn of(runs: &[&TestbenchRun]) -> Option { Some(Coverage { covered: hits.values().filter(|n| **n > 0).count(), total: hits.len(), gaps }) } +/// The gap `p` would be, in words (the `text` of [`Gap`]). +pub(crate) fn text_of(p: &CoverPoint) -> String { + gap(&p.file, p.line, &p.kind, &p.what).text +} + fn counts(p: &CoverPoint) -> bool { !p.file.starts_with("tb/") && !matches!(p.kind, CoverKind::Default { written: false }) } diff --git a/crates/wafer-api/src/design.rs b/crates/wafer-api/src/design.rs index 454b20b..5bf9a3b 100644 --- a/crates/wafer-api/src/design.rs +++ b/crates/wafer-api/src/design.rs @@ -289,6 +289,32 @@ pub async fn mistakes( report.map(Json).map_err(ApiError::Conflict) } +/// How long a search for an example may take (shrinking the one found may take half again). +const FIND_BUDGET: std::time::Duration = std::time::Duration::from_secs(20); + +/// "Find an example" (`crate::search`) for a coverage gap or a planted mistake that slipped +/// through, at the branch tip. Returns a draft example; nothing is saved. Editors: it costs server +/// time. +pub async fn find_example( + State(state): State, + user: AuthUser, + Path(id): Path, + Json(target): Json, +) -> ApiResult> { + authorize(&state, &user, id, Role::Editor).await?; + let commit = head(&state, id).await?; + let files = files_at(&state, id, &commit).await?; + let spec = match files.get(SPEC_FILE) { + Some(t) => parse_spec(t)?, + None => return Err(ApiError::NotFound("this project has no spec yet".into())), + }; + let found = tokio::task::spawn_blocking(move || { + gate::with_slot(|| wafer_sim::on_big_stack(|| crate::search::find(&spec, &files, &target, FIND_BUDGET))) + }) + .await?; + found.map(Json).map_err(ApiError::Conflict) +} + #[derive(Debug, Deserialize)] pub struct TestbenchReq { /// The spec being edited (unsaved edits included). diff --git a/crates/wafer-api/src/lib.rs b/crates/wafer-api/src/lib.rs index 592704a..d7ac88e 100644 --- a/crates/wafer-api/src/lib.rs +++ b/crates/wafer-api/src/lib.rs @@ -18,6 +18,7 @@ pub mod licence; pub mod lsp; pub mod mistakes; pub mod projects; +pub mod search; pub mod spec; pub mod spend; pub mod starters; @@ -73,6 +74,7 @@ pub fn router(state: AppState) -> Router { .route("/projects/{id}/spec/run", post(design::run)) .route("/projects/{id}/spec/testbench", post(design::testbench)) .route("/projects/{id}/spec/mistakes", post(design::mistakes)) + .route("/projects/{id}/spec/find-example", post(design::find_example)) .route("/projects/{id}/spec/draft", post(design::draft)) .route("/starters", get(starters::list)) .route("/projects/{id}/starters/{starter}", post(starters::add)) diff --git a/crates/wafer-api/src/mistakes.rs b/crates/wafer-api/src/mistakes.rs index 5524494..9fb9f9e 100644 --- a/crates/wafer-api/src/mistakes.rs +++ b/crates/wafer-api/src/mistakes.rs @@ -104,6 +104,35 @@ pub fn check( Ok(Report { caught, tried, sites: sites.len(), slipped, elapsed_ms: started.elapsed().as_millis() as u64 }) } +/// The design with one planted mistake in it: the one at `file`/`line` that turns `original` into +/// `mistake`, as [`check`] reported it. None if the design no longer has it. +pub(crate) fn planted( + modules: &[Module], + files: &BTreeMap, + file: &str, + line: u32, + original: &str, + mistake: &str, +) -> Option> { + let mine = + |m: &Module| !m.file.starts_with("tb/") && files.get(&m.file).is_some_and(|c| !crate::blocks::is_generated(c)); + let mut probe = modules.to_vec(); + let mut w = Walker::new(None); + for m in probe.iter_mut().filter(|m| mine(m)) { + w.module(m); + } + let i = w + .sites + .iter() + .position(|s| s.file == file && s.line == line && s.original == original && s.mistake == mistake)?; + let mut ms = modules.to_vec(); + let mut w = Walker::new(Some(i)); + for m in ms.iter_mut().filter(|m| mine(m)) { + w.module(m); + } + Some(ms) +} + /// Walks the design in a fixed order, numbering every place a mistake could go. Without a target /// it lists them; with one it plants that mistake and leaves the rest as they are. struct Walker { diff --git a/crates/wafer-api/src/search.rs b/crates/wafer-api/src/search.rs new file mode 100644 index 0000000..c841750 --- /dev/null +++ b/crates/wafer-api/src/search.rs @@ -0,0 +1,637 @@ +//! "Find an example": searches for inputs that make the design do something no example pins down +//! yet, and writes them as a draft example for a person to check. Two targets: a coverage gap (a +//! branch no example takes) and a planted mistake that slipped through (inputs under which the +//! mistaken design behaves differently from the real one). +//! +//! The search simulates input sequences: the existing examples' inputs, as they are and with +//! changes, and fresh ones made of values the design and its examples use (0, 1, all ones, the +//! design's constants) held for a cycle or for many. It keeps the first sequence that reaches the +//! target, cuts it to the shortest that still does, undoes every input change it doesn't need, and +//! ends with checks of what the design did. The same design and spec always give the same example. + +use std::collections::{BTreeMap, BTreeSet}; +use std::time::{Duration, Instant}; + +use serde::{Deserialize, Serialize}; +use wafer_sim::SimResult; +use wafer_sim::ast::Module; + +use crate::spec::{Example, Spec, Step}; + +/// What to find an example for. +#[derive(Debug, Clone, Deserialize)] +#[serde(tag = "for", rename_all = "snake_case")] +pub enum Target { + /// A coverage gap, as `crate::coverage` reported it. + Gap { file: String, line: u32, text: String }, + /// A planted mistake that slipped through, as `crate::mistakes` reported it. + Mistake { file: String, line: u32, kind: String, original: String, mistake: String }, +} + +#[derive(Debug, Clone, Serialize)] +pub struct Found { + /// The draft example (not added to the spec); None if the search found nothing. + pub example: Option, + /// Input sequences simulated. + pub tried: usize, + pub elapsed_ms: u64, + /// Why nothing was found, in words. + pub message: Option, +} + +/// A stretch of an example: every input's value from its start, held for `hold` cycles. +#[derive(Debug, Clone, PartialEq)] +struct Seg { + set: Vec, + hold: u32, +} + +/// The longest input sequence tried, in cycles. +const MAX_CYCLES: u32 = 2000; +/// How long a stretch of inputs is held: mostly short, sometimes long enough for a counter. +const HOLDS: [u32; 12] = [1, 1, 1, 1, 2, 2, 3, 5, 8, 20, 60, 250]; +const NOTE: &str = "what the design did when found: check it is right"; + +/// Searches for an example that reaches `target`, within `budget`. Fails (in words) if the design +/// doesn't parse, has no inputs to set, or no longer has the target. Call on a big stack. +pub fn find(spec: &Spec, files: &BTreeMap, target: &Target, budget: Duration) -> Result { + let started = Instant::now(); + let hdl: Vec<(&str, &str)> = files + .iter() + .filter(|(p, _)| crate::gate::is_hdl(p) && !p.starts_with("tb/")) + .map(|(p, c)| (p.as_str(), c.as_str())) + .collect(); + let (design, _) = wafer_sim::parse(&hdl).map_err(|e| format!("the design doesn't parse: {e}"))?; + let mutant = match target { + Target::Gap { .. } => None, + Target::Mistake { file, line, original, mistake, .. } => Some( + crate::mistakes::planted(&design, files, file, *line, original, mistake) + .ok_or("that mistake can't be planted any more: the design has changed since it was checked")?, + ), + }; + let clock = spec.clock.as_ref().map(|c| c.port.as_str()); + let reset = spec.reset.as_ref().map(|r| r.port.as_str()); + let inputs: Vec<(String, u32)> = spec + .ports + .iter() + .filter(|p| p.dir == "in" && Some(p.name.as_str()) != clock && Some(p.name.as_str()) != reset) + .map(|p| (p.name.clone(), p.width.min(64))) + .collect(); + if inputs.is_empty() { + return Err("the design has no inputs to set, so there is nothing to search".into()); + } + let outputs = spec.ports.iter().filter(|p| p.dir == "out").map(|p| (p.name.clone(), p.width.min(64))).collect(); + let ids: BTreeSet<&str> = spec.examples.iter().map(|e| e.id.as_str()).collect(); + let id = (1..).map(|n| format!("found_{n}")).find(|i| !ids.contains(i.as_str())).expect("a free id"); + let mut s = Search { + spec: Spec { examples: Vec::new(), ..spec.clone() }, + design: &design, + mutant, + target, + pools: pools(spec, &inputs, &hdl), + inputs, + outputs, + id, + tried: 0, + exists: false, + rng: Rng(0x9E37_79B9_7F4A_7C15), + }; + + let mut found = None; + let starts: Vec> = spec.examples.iter().filter_map(|e| s.segments(e)).collect(); + // The examples' inputs as they are first (a mistake may already show at an output no example + // checks), then changed, then fresh. + let mut queue: Vec> = starts.clone(); + let mut round = 0; + while started.elapsed() < budget { + let cand = match queue.pop() { + Some(c) => c, + None => { + round += 1; + s.candidate(&starts, round) + } + }; + if let Some(k) = s.reaches(&cand) { + found = Some(s.cut(&cand, k)); + break; + } + if round == 1 && !s.exists && matches!(target, Target::Gap { .. }) { + return Err("that case isn't in the design any more: run the examples again".into()); + } + } + let example = match found { + Some(segs) => { + let segs = shrink(&mut s, segs, started, budget); + s.finish(segs) + } + None => None, + }; + let message = example.is_none().then(|| match target { + Target::Gap { .. } => format!( + "No input sequence the search tried ({} in {} s) reached this case. It may be impossible by design, or need \ + inputs in an order the search didn't try: describe the example in words instead.", + s.tried, + budget.as_secs() + ), + Target::Mistake { .. } => format!( + "No input sequence the search tried ({} in {} s) made the mistaken design behave differently at its \ + outputs. The mistake may change nothing that can be seen, so no example can catch it.", + s.tried, + budget.as_secs() + ), + }); + Ok(Found { example, tried: s.tried, elapsed_ms: started.elapsed().as_millis() as u64, message }) +} + +struct Search<'a> { + /// The spec without its examples: the search's examples are run one at a time. + spec: Spec, + design: &'a [Module], + /// The design with the mistake planted, when the target is a mistake. + mutant: Option>, + target: &'a Target, + inputs: Vec<(String, u32)>, + outputs: Vec<(String, u32)>, + /// Values worth trying, per input. + pools: Vec>, + id: String, + tried: usize, + /// Whether the gap's branch was seen in a run at all (it is, unless the design changed). + exists: bool, + rng: Rng, +} + +impl Search<'_> { + /// Runs `ex` on `modules`; None if it doesn't build or finish. + fn run(&mut self, modules: &[Module], ex: &Example) -> Option { + self.tried += 1; + let one = Spec { examples: vec![ex.clone()], ..self.spec.clone() }; + let tb = one.testbench(ex); + let (tbm, ts) = wafer_sim::parse(&[("tb/examples/search.v", tb.as_str())]).ok()?; + let mut all = modules.to_vec(); + all.extend(tbm); + let r = wafer_sim::run_parsed(&all, ts, &crate::spec::tb_name(&ex.id), crate::gate::limits()).ok()?; + r.finished.then_some(r) + } + + /// The example made of `segs`, then `tail`. + fn example(&self, segs: &[Seg], tail: Vec) -> Example { + let mut steps: Vec = Vec::new(); + let mut now = vec![0u64; self.inputs.len()]; + for s in segs { + for (i, (name, w)) in self.inputs.iter().enumerate() { + if s.set[i] != now[i] { + steps.push(step("set", name, &write(s.set[i], *w), 0, "")); + now[i] = s.set[i]; + } + } + match steps.last_mut() { + Some(l) if l.op == "wait" => l.cycles += s.hold, + _ => steps.push(step("wait", "", "", s.hold, "")), + } + } + steps.extend(tail); + Example { + id: self.id.clone(), + title: String::new(), + given: String::new(), + expect: String::new(), + params: Vec::new(), + steps, + target: None, + } + } + + /// An existing example's inputs as stretches (its checks dropped); None for an example with + /// settings of its own or for the chip, which run a different design. + fn segments(&self, e: &Example) -> Option> { + if !e.params.is_empty() || e.target.as_deref().is_some_and(|t| t != self.spec.top) { + return None; + } + let mut now = vec![0u64; self.inputs.len()]; + let mut segs = Vec::new(); + let mut pending = false; + for st in &e.steps { + match st.op.as_str() { + "set" => { + if let Some(i) = self.inputs.iter().position(|(n, _)| *n == st.signal) { + now[i] = crate::spec::parse_value(&st.value).unwrap_or(0) & mask(self.inputs[i].1); + pending = true; + } + } + "wait" | "wait_until" | "expect_for" => { + segs.push(Seg { set: now.clone(), hold: st.cycles.max(1) }); + pending = false; + } + _ => {} + } + } + if pending || segs.is_empty() { + segs.push(Seg { set: now, hold: 1 }); + } + Some(segs) + } + + /// A new sequence to try: an example's with its end changed, or a fresh one. + fn candidate(&mut self, starts: &[Vec], round: usize) -> Vec { + let mut segs = if !starts.is_empty() && !round.is_multiple_of(3) { + let base = &starts[self.rng.below(starts.len())]; + let keep = self.rng.below(base.len() + 1); + let mut segs = base[..keep].to_vec(); + if self.rng.below(2) == 0 && keep > 0 { + // Change one stretch in the middle and keep the rest. + let at = self.rng.below(keep); + segs[at] = self.seg(&segs[at].set.clone()); + segs.extend_from_slice(&base[keep..]); + } + segs + } else { + Vec::new() + }; + for _ in 0..1 + self.rng.below(8) { + let prev = segs.last().map_or_else(|| vec![0; self.inputs.len()], |s: &Seg| s.set.clone()); + segs.push(self.seg(&prev)); + } + truncate(&segs, MAX_CYCLES) + } + + /// A stretch after `prev`: some inputs changed (a flip, a value worth trying or any value). + fn seg(&mut self, prev: &[u64]) -> Seg { + let p = [15, 40, 80][self.rng.below(3)]; + let mut set = prev.to_vec(); + for (i, (_, w)) in self.inputs.iter().enumerate() { + if self.rng.below(100) >= p { + continue; + } + set[i] = if *w == 1 { + prev[i] ^ 1 + } else if self.rng.below(5) < 3 { + self.pools[i][self.rng.below(self.pools[i].len())] + } else { + self.rng.next() & mask(*w) + }; + } + Seg { set, hold: HOLDS[self.rng.below(HOLDS.len())] } + } + + /// The cycle by which `segs` reach the target, if they do: for a mistake the first cycle at + /// which an output differs; for a gap the end (cut later). + fn reaches(&mut self, segs: &[Seg]) -> Option { + let segs = one_more(segs)?; + let ex = self.example(&segs, Vec::new()); + let total = cycles(&segs); + match self.target { + Target::Gap { file, line, text } => { + let r = self.run(self.design, &ex)?; + let mut hit = false; + for (p, n) in &r.coverage { + if p.file == *file && p.line == *line && crate::coverage::text_of(p) == *text { + self.exists = true; + hit |= *n > 0; + } + } + hit.then_some(total) + } + Target::Mistake { .. } => { + let a = self.run(self.design, &ex)?; + let mutant = self.mutant.clone().expect("a mistake has a mutant"); + let b = match self.run(&mutant, &ex) { + Some(b) => b, + // The mistake hangs or breaks the design: noticeable from the first cycle. + None => return Some(1), + }; + (1..=total).find(|&k| { + let t = self.sample(k); + let top = crate::spec::tb_name(&ex.id); + self.outputs.iter().any(|(o, _)| at(&a, &top, o, t) != at(&b, &top, o, t)) + }) + } + } + } + + /// The time just before the example's `k`th cycle ends: what a check after `k` cycles sees. + fn sample(&self, k: u32) -> u64 { + let period = 2 * self.spec.half_ps(); + let base = if self.spec.reset.is_some() { 2 } else { 1 }; + (base + k as u64) * period - 1 + } + + /// The shortest start of `segs` that reaches the target, given it does by cycle `k`. + fn cut(&mut self, segs: &[Seg], k: u32) -> Vec { + match self.target { + Target::Mistake { .. } => truncate(&one_more(segs).unwrap_or_default(), k), + Target::Gap { .. } => { + // Reaching a branch only gets more likely with more cycles: search the cut point. + let (mut lo, mut hi) = (1, k.max(1)); + while lo < hi { + let mid = (lo + hi) / 2; + if self.reaches(&truncate(segs, mid)).is_some() { + hi = mid; + } else { + lo = mid + 1; + } + } + truncate(segs, hi) + } + } + } + + /// The draft example for `segs` (which reach the target), with checks of what the design did, + /// if it passes on the design (and, for a mistake, fails with it planted). + fn finish(&mut self, segs: Vec) -> Option { + // A gap's sequence is checked one cycle on, as `reaches` ran it; a mistake's was cut at the + // cycle the difference shows. + let segs = if self.mutant.is_none() { one_more(&segs)? } else { segs }; + let k = cycles(&segs); + let probe = self.example(&segs, Vec::new()); + let top = crate::spec::tb_name(&probe.id); + let t = self.sample(k); + let a = self.run(self.design, &probe)?; + let b = self.mutant.clone().map(|m| self.run(&m, &probe)); + let mut tail = Vec::new(); + for (o, w) in &self.outputs { + let v = at(&a, &top, o, t)?; + // For a mistake, only the outputs that tell the two apart. + if let Some(Some(b)) = &b + && at(b, &top, o, t) == Some(v) + { + continue; + } + tail.push(step("expect", o, &write(v, *w), 0, NOTE)); + } + if tail.is_empty() && b.is_none() { + return None; + } + let mut ex = self.example(&segs, tail); + let passes = |r: &Option| r.as_ref().is_some_and(|r| crate::gate::verdict(&r.log).0 == "pass"); + if !passes(&self.run(self.design, &ex)) { + return None; + } + if let Some(m) = self.mutant.clone() + && passes(&self.run(&m, &ex)) + { + return None; + } + let plain = |s: &str| s.replace('`', ""); + let (title, given) = match self.target { + Target::Gap { text, .. } => ( + format!("Tries {}", plain(text.trim_start_matches("Never tried: "))), + "Found by searching the design's inputs for a case no example tried.".to_string(), + ), + Target::Mistake { kind, original, mistake, line, .. } => ( + match kind.as_str() { + "reverse" => format!("Notices if {} were the other way round", plain(original)), + "forget" => format!("Notices if {} were left out", plain(original)), + _ => format!("Notices if {} were {}", plain(original), plain(mistake)), + }, + format!( + "Found by searching the design's inputs for a difference this mistake (line {line}) would make." + ), + ), + }; + ex.title = title; + ex.given = given; + ex.expect = "The checks at the end are what the design did: make sure that is what it should do.".into(); + Some(ex) + } +} + +/// Undoes every input change and shortens every hold that the target doesn't need, while time is +/// left. +fn shrink(s: &mut Search, mut segs: Vec, started: Instant, budget: Duration) -> Vec { + // Shrinking may take half the budget again. + let time = |_: &Search| started.elapsed() < budget + budget / 2; + for i in (0..segs.len()).rev() { + for j in 0..s.inputs.len() { + let prev = if i == 0 { 0 } else { segs[i - 1].set[j] }; + if segs[i].set[j] == prev || !time(s) { + continue; + } + let mut c = segs.clone(); + c[i].set[j] = prev; + if let Some(k) = s.reaches(&c) { + segs = s.cut(&c, k); + if i >= segs.len() { + break; + } + } + } + while i < segs.len() && segs[i].hold > 1 && time(s) { + let mut c = segs.clone(); + c[i].hold /= 2; + match s.reaches(&c) { + Some(k) => segs = s.cut(&c, k), + None => break, + } + } + } + segs +} + +/// `segs` and one more cycle with the same inputs, so the last stretch's inputs take effect. +fn one_more(segs: &[Seg]) -> Option> { + let mut out = segs.to_vec(); + out.push(Seg { set: segs.last()?.set.clone(), hold: 1 }); + Some(out) +} + +/// The first `n` cycles of `segs`. +fn truncate(segs: &[Seg], n: u32) -> Vec { + let mut out = Vec::new(); + let mut left = n; + for s in segs { + if left == 0 { + break; + } + out.push(Seg { set: s.set.clone(), hold: s.hold.min(left) }); + left -= s.hold.min(left); + } + out +} + +fn cycles(segs: &[Seg]) -> u32 { + segs.iter().map(|s| s.hold).sum() +} + +/// Values worth trying for each input: 0, 1, all ones, half, the numbers written in the design +/// (and one either side), and the values the examples set. +fn pools(spec: &Spec, inputs: &[(String, u32)], hdl: &[(&str, &str)]) -> Vec> { + let mut consts = BTreeSet::new(); + for (_, src) in hdl { + for tok in src.split(|c: char| !(c.is_ascii_alphanumeric() || c == '\'' || c == '_')) { + if tok.starts_with(|c: char| c.is_ascii_digit() || c == '\'') + && let Some(v) = crate::spec::parse_value(tok) + { + consts.extend([v, v.wrapping_sub(1), v.wrapping_add(1)]); + } + if consts.len() > 200 { + break; + } + } + } + for p in &spec.params { + if let Some(v) = crate::spec::parse_value(&p.value) { + consts.extend([v, v.wrapping_sub(1), v.wrapping_add(1)]); + } + } + inputs + .iter() + .map(|(name, w)| { + let m = mask(*w); + let mut pool: BTreeSet = [0, 1, m, m >> 1, (m >> 1) + 1].into_iter().map(|v| v & m).collect(); + pool.extend(consts.iter().map(|v| v & m)); + for e in &spec.examples { + for st in e.steps.iter().filter(|s| s.op == "set" && s.signal == *name) { + if let Some(v) = crate::spec::parse_value(&st.value) { + pool.insert(v & m); + } + } + } + pool.into_iter().collect() + }) + .collect() +} + +/// `port`'s value in the testbench `top` at time `t`. +fn at(r: &SimResult, top: &str, port: &str, t: u64) -> Option { + let name = format!("{top}.{port}"); + let s = r.signals.iter().find(|s| s.name == name)?; + let i = s.times.partition_point(|&x| x <= t); + (i > 0).then(|| s.values[i - 1]) +} + +fn mask(w: u32) -> u64 { + if w >= 64 { u64::MAX } else { (1u64 << w) - 1 } +} + +/// A value as a person would write it: decimal up to 16 bits, hexadecimal beyond. +fn write(v: u64, w: u32) -> String { + if w <= 16 { v.to_string() } else { format!("0x{v:X}") } +} + +fn step(op: &str, signal: &str, value: &str, cycles: u32, note: &str) -> Step { + Step { op: op.into(), signal: signal.into(), value: value.into(), cycles, note: note.into() } +} + +/// xorshift64*: small, fast and the same everywhere, so a search is repeatable. +struct Rng(u64); + +impl Rng { + fn next(&mut self) -> u64 { + self.0 ^= self.0 >> 12; + self.0 ^= self.0 << 25; + self.0 ^= self.0 >> 27; + self.0.wrapping_mul(0x2545_F491_4F6C_DD1D) + } + + fn below(&mut self, n: usize) -> usize { + (self.next() % n.max(1) as u64) as usize + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The up/down counter's design with only some of its examples, as a project would have it. + fn project(keep: &[usize]) -> (Spec, BTreeMap) { + let st = crate::starters::get("updown").expect("starter"); + let mut spec = st.spec.clone(); + spec.examples = keep.iter().map(|&i| spec.examples[i].clone()).collect(); + (spec.clone(), files(&spec)) + } + + fn files(spec: &Spec) -> BTreeMap { + let st = crate::starters::get("updown").expect("starter"); + let mut files: BTreeMap = st + .project_files(&st.title) + .into_iter() + .filter(|(p, _)| !p.starts_with("tb/")) + .filter_map(|(p, c)| String::from_utf8(c).ok().map(|c| (p, c))) + .collect(); + files.extend(spec.testbenches()); + files + } + + fn gaps(spec: &Spec) -> Vec { + let report = crate::gate::run(&files(spec)); + assert!(report.passed, "{}", report.summary); + let runs: Vec<_> = report.runs.iter().collect(); + crate::coverage::of(&runs).expect("coverage").gaps + } + + fn search(spec: &Spec, target: &Target) -> Found { + let f = files(spec); + wafer_sim::on_big_stack(|| find(spec, &f, target, Duration::from_secs(20))).expect("searches") + } + + #[test] + fn finds_examples_that_try_untried_cases() { + let (mut spec, _) = project(&[1]); + let before = gaps(&spec); + let targets: Vec = before + .iter() + .filter(|g| g.kind != "default") + .take(3) + .map(|g| Target::Gap { file: g.file.clone(), line: g.line, text: g.text.clone() }) + .collect(); + assert!(!targets.is_empty(), "the weak examples leave gaps"); + for t in &targets { + let found = search(&spec, t); + let ex = found.example.clone().unwrap_or_else(|| panic!("{t:?}: {:?}", found.message)); + assert!(ex.title.starts_with("Tries "), "{}", ex.title); + assert!(ex.steps.iter().any(|s| s.op == "expect" && s.note == NOTE), "{:?}", ex.steps); + // The same design and spec give the same example. + assert_eq!(search(&spec, t).example, Some(ex.clone())); + spec.examples.push(ex); + } + let after = gaps(&spec); + for t in &targets { + let Target::Gap { text, line, .. } = t else { unreachable!() }; + assert!(!after.iter().any(|g| g.text == *text && g.line == *line), "{text} is still a gap"); + } + } + + #[test] + fn finds_examples_that_catch_slipped_mistakes() { + let (mut spec, f) = project(&[1]); + let tops: Vec = spec.examples.iter().map(|e| crate::spec::tb_name(&e.id)).collect(); + let check = |f: &BTreeMap, tops: &[String]| { + wafer_sim::on_big_stack(|| crate::mistakes::check(f, tops, 40, Duration::from_secs(60))).expect("checks") + }; + let weak = check(&f, &tops); + let s = weak.slipped.iter().find(|s| s.original == "count + 1").expect("count + 1 slips through"); + let target = Target::Mistake { + file: s.file.clone(), + line: s.line, + kind: s.kind.clone(), + original: s.original.clone(), + mistake: s.mistake.clone(), + }; + let found = search(&spec, &target); + let ex = found.example.unwrap_or_else(|| panic!("{:?}", found.message)); + assert_eq!(ex.title, "Notices if count + 1 were count - 1"); + spec.examples.push(ex); + let tops: Vec = spec.examples.iter().map(|e| crate::spec::tb_name(&e.id)).collect(); + let strong = check(&files(&spec), &tops); + assert!(!strong.slipped.iter().any(|x| x.original == "count + 1"), "{strong:#?}"); + assert!(strong.slipped.len() < weak.slipped.len()); + } + + #[test] + fn unreachable_targets_say_why() { + let (spec, f) = project(&[1]); + let t = Target::Gap { file: "src/updown.v".into(), line: 1, text: "Never tried: nothing".into() }; + let err = wafer_sim::on_big_stack(|| find(&spec, &f, &t, Duration::from_secs(2))).unwrap_err(); + assert!(err.contains("isn't in the design"), "{err}"); + let t = Target::Mistake { + file: "src/updown.v".into(), + line: 1, + kind: "compare".into(), + original: "a".into(), + mistake: "b".into(), + }; + let err = wafer_sim::on_big_stack(|| find(&spec, &f, &t, Duration::from_secs(2))).unwrap_err(); + assert!(err.contains("can't be planted"), "{err}"); + } +} diff --git a/crates/wafer-api/src/spec.rs b/crates/wafer-api/src/spec.rs index 6a75c6f..eb8846e 100644 --- a/crates/wafer-api/src/spec.rs +++ b/crates/wafer-api/src/spec.rs @@ -381,7 +381,7 @@ impl Spec { } /// Half a clock period in whole picoseconds (5 ns without a clock), so any frequency works. - fn half_ps(&self) -> u64 { + pub(crate) fn half_ps(&self) -> u64 { self.clock.as_ref().map(|c| (500_000.0 / c.mhz).round().max(1.0) as u64).unwrap_or(5_000) } diff --git a/docs/api.md b/docs/api.md index f32b555..2c0e459 100644 --- a/docs/api.md +++ b/docs/api.md @@ -49,6 +49,7 @@ shows how to get one locally). `GET /healthz` (outside `/api`) needs no token. | PUT | `/projects/{id}/spec` | editor | `{spec, parent, message?}`: commits the spec and the regenerated `tb/examples/` testbenches. `400` lists the problems. | | POST | `/projects/{id}/spec/run` | editor | Runs every testbench at `main`; results per example (and the other testbenches), and `coverage`: `{covered, total, gaps: [{file, line, kind, what, labels, text, suggestion}]}`, the design's branches the examples take (`null` without examples or branches), and `machines`: the design's state machines `{module, file, state, states, values, transitions: [{from, to, when, line, covered}], reset}`. | | POST | `/projects/{id}/spec/mistakes` | editor | Plants up to 40 small mistakes in the design, one at a time, and runs the examples against each (at most 30 s): `{caught, tried, sites, slipped: [{file, line, kind, original, mistake, text, suggestion}], elapsed_ms}`. `409` if an example already fails. | +| POST | `/projects/{id}/spec/find-example` | editor | Searches the design's inputs for an example that reaches a target: `{"for": "gap", file, line, text}` (a coverage gap) or `{"for": "mistake", file, line, kind, original, mistake}` (a planted mistake that slipped through), as `spec/run` and `spec/mistakes` report them. Returns `{example, tried, elapsed_ms, message}`: a draft example (not saved) whose final checks hold what the design did, or `null` and why. At most about 30 s. `409` if the design doesn't parse, has no inputs, or no longer has the target. | | POST | `/projects/{id}/spec/testbench` | viewer | `{spec, example}`: the testbench the example compiles to (`{path, top, source}`), without saving; `400` lists the problems. Used by the Try it playground. | | POST | `/projects/{id}/spec/draft` | editor | `{instruction, spec?}`: the assistant writes or revises the spec; nothing is saved. Returns `{spec, note, model, usage, usd, elapsed_ms}`; `usd` is priced for the model that served it and counts against the AI budget. | diff --git a/docs/user-guide/design-view.md b/docs/user-guide/design-view.md index 5a2156f..5a19953 100644 --- a/docs/user-guide/design-view.md +++ b/docs/user-guide/design-view.md @@ -111,17 +111,30 @@ mean?** and the expert numbers underneath: (in `wr && !full`, `full` both 0 and 1 when it is decided), and both sides of each `?:` choice. Each one they never try is shown in plain terms ("Never tried: `en` = 0", "Never tried: `state` = 2", "Never tried: `full` ≠ 0 while deciding `wr && !full`"). Passing examples only vouch for the cases they - try. **Add an example** opens Examples with the missing case written out as a new example's - description; the file and line link opens the code. A `default` choice that no example reaches + try. **Find an example** searches for one (see below); **Add in words** opens Examples with the + missing case written out as a new example's description; the file and line link opens the code. A `default` choice that no example reaches is shown but doesn't hold the project back, since it is often impossible by design. - **Would your examples catch a mistake?**: **Plant mistakes and check** makes small, typical mistakes in a copy of the design, one at a time (a comparison turned round, a number one off, a condition reversed, a `!` left out, `+` for `-`, an assignment left out), and runs every example against each. It reports how many some example caught, and each one that slipped through ("If - `cnt == N - 1` were `cnt == N` (one off, line 15), every example would still pass"), with **Add - an example** and a link to the line. Your files aren't changed. A few planted mistakes change - nothing at all and can't be caught. Up to 40 mistakes, spread over the design, in at most 30 - seconds. + `cnt == N - 1` were `cnt == N` (one off, line 15), every example would still pass"), with **Find + an example**, **Add in words** and a link to the line. Your files aren't changed. A few planted + mistakes change nothing at all and can't be caught. Up to 40 mistakes, spread over the design, + in at most 30 seconds. + + **Find an example**, for a case no example tries or a mistake that slipped through, has the + simulator search the design's inputs (no AI, nothing to pay). It tries your examples' inputs, + as they are and with changes, and fresh sequences of values the design uses (0, 1, all ones, + the numbers in its code), held for a cycle or many. The first sequence that reaches the case, + or makes the mistaken copy behave differently at an output, is cut to the shortest that still + does and stripped of every input change it doesn't need. It ends with checks of the outputs + (for a mistake, the ones that tell the two apart), holding what the design did. The example + opens in Examples, unsaved, titled "Tries …" or "Notices if …": **check that what it expects + is what the design should do**. If it isn't, you have found a bug. Save the spec to keep it. + The search takes at most about 30 seconds, and the same design and spec always give the same + example. If nothing is found, it says why: the case may be impossible by design, or the mistake + may change nothing that can be seen. - **How it works**: each state machine found in the Verilog, drawn as states (boxes, with the one after reset marked) and numbered moves between them, listed with when they happen ("state 0 → state 1 when `rx` = 0"). Moves an example takes are solid; dashed red ones no example takes yet;