diff --git a/CHANGELOG.md b/CHANGELOG.md index 8825515..7785e8a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,15 @@ All notable changes to Wafersmith are recorded here. The format follows ### Added +- **Try a small fix**: a failing example offers "The design: try a small fix". Without AI, the + server tries the small repairs that undo the most common mistakes (a comparison turned around, a + limit one off in either direction, a condition reversed, a missing or extra `!`/`~`, `&&` for + `||`, `-` for `+`, an assigned value one off), the lines "Why not?" points at first, and keeps + those after which every example passes. Each is shown as a one-line diff that keeps the line as + written (sized numbers too), with what it changes in the examples' outputs, and **Use this + change** commits it. Every mistake "Would your examples catch a mistake?" plants in the starters + and an example notices is repaired back to the exact original line (tested). + `POST /api/projects/{id}/spec/small-fix`. - **Why wasn't it the expected value?**: a failing example's account ends with a link that lists every line of the design that could have written the value the example expected, and for each the decision that kept it from running ("`cnt == N - 1` was false (cnt = 2)"), or that it ran and diff --git a/apps/web/src/api.ts b/apps/web/src/api.ts index e47d5eb..edee20f 100644 --- a/apps/web/src/api.ts +++ b/apps/web/src/api.ts @@ -289,6 +289,30 @@ export interface FoundExample { message: string | null; } +/** "Try a small fix": small repairs after which every example passes (`POST .../spec/small-fix`). */ +export interface SmallFixes { + fixes: SmallFix[]; + tried: number; + sites: number; + /** The examples failing before any repair. */ + failing: string[]; + elapsed_ms: number; + message: string; +} + +export interface SmallFix { + file: string; + line: number; + kind: string; + original: string; + replacement: string; + /** "Line 18 of blinker.v: `cnt == N - 1` → `cnt == N`; all 3 examples pass." */ + text: string; + /** The line as it is and as it would be, and the whole file changed; null if the change can't + * be placed in the text. */ + edit: { before: string; after: string; content: string } | null; +} + /** How well a version of the design is tested (`POST /agent/{id}/compare`). */ export interface Quality { examples: number; @@ -644,6 +668,8 @@ export const api = { request<{ commit: string; changed: boolean }>("PUT", `/projects/${enc(id)}/spec`, body), runSpec: (id: string) => request("POST", `/projects/${enc(id)}/spec/run`), plantMistakes: (id: string) => request("POST", `/projects/${enc(id)}/spec/mistakes`), + smallFix: (id: string, focus: { file: string; line: number }[]) => + request("POST", `/projects/${enc(id)}/spec/small-fix`, { focus }), findExample: (id: string, target: FindTarget) => request("POST", `/projects/${enc(id)}/spec/find-example`, target), exampleTestbench: (id: string, body: { spec: Spec; example: SpecExample }) => request<{ path: string; top: string; source: string }>("POST", `/projects/${enc(id)}/spec/testbench`, body), diff --git a/apps/web/src/components/Design.tsx b/apps/web/src/components/Design.tsx index fee30a8..cb89252 100644 --- a/apps/web/src/components/Design.tsx +++ b/apps/web/src/components/Design.tsx @@ -7,6 +7,8 @@ import { type FindTarget, type Job, type MistakesReport, + type SmallFix, + type SmallFixes, type Spec, type SpecExample, type SpecRun, @@ -20,6 +22,7 @@ import { decided, missed, why, whyNot, whyNotText, whyText, type Because, type R import { StateMachine } from "./StateMachine"; import { TryIt } from "./TryIt"; import { SimClient } from "../sim/client"; +import { compareExamples } from "../lib/compare"; import { BlocksStep } from "./Blocks"; type Step = "describe" | "examples" | "blocks" | "health"; @@ -982,6 +985,57 @@ function Happened({ }; // "Why not?": why the first failing check didn't see the value the example expected. const [notWhy, setNotWhy] = useState(null); + // "Try a small fix": repairs the server found, what each does to the outputs, and the commit + // they apply to. + const [small, setSmall] = useState<{ result: SmallFixes; effects: string[][]; parent: string | null } | "working" | null>(null); + const [smallError, setSmallError] = useState(null); + async function trySmallFix() { + if (!rep) return; + setSmall("working"); + setSmallError(null); + try { + // The lines "Why not?" and "Why?" point at are tried first. + const f = rep.firstFail; + const w = notWhy ?? (f && f.failCycle !== undefined ? whyNot(rep, `${rep.top}.${f.signal}`, f.failCycle, f.expected) : null); + const focus = [ + ...(w?.candidates ?? []).flatMap((c) => [c.line, ...c.passed.map((p) => p.line), ...(c.blocked ? [c.blocked.line] : [])].map((line) => ({ file: c.file, line }))), + ...trail.flatMap((b) => (b.file && b.line ? [b.line, ...b.conds.map((c) => c.line)].map((line) => ({ file: b.file!, line })) : [])), + ]; + const result = await api.smallFix(projectId, focus); + const snap = await api.snapshot(projectId); + const client = new SimClient(); + const effects: string[][] = []; + try { + for (const fx of result.fixes) { + if (!fx.edit) { + effects.push([]); + continue; + } + const changes = await compareExamples(client, baseSpec, null, snap.files, { ...snap.files, [fx.file]: fx.edit.content }); + effects.push(changes.flatMap((c) => c.differences.map((d) => `${c.title}: ${d}`))); + } + } finally { + client.dispose(); + } + setSmall({ result, effects, parent: snap.commit }); + } catch (e) { + setSmallError((e as Error).message); + setSmall(null); + } + } + async function applySmallFix(fx: SmallFix, parent: string | null) { + if (!fx.edit) return; + try { + await api.commit(projectId, { + message: `Small fix: line ${fx.line} of ${fx.file.split("/").pop()}, ${fx.original} → ${fx.replacement}`, + parent, + changes: [{ op: "write", path: fx.file, content: fx.edit.content }], + }); + setSmall(null); + } catch (e) { + setSmallError((e as Error).message); + } + } // "The design is right": a change to the example that makes it pass, worked out and re-checked. const [fix, setFix] = useState(null); async function acceptDesign() { @@ -1048,7 +1102,11 @@ function Happened({ const every = rep.cycles > 200 ? 50 : rep.cycles > 80 ? 20 : rep.cycles > 30 ? 10 : 5; const failing = rep.firstFail; // The AI gets the explanation too, when someone followed one. - const traced = [notWhy && whyNotText(notWhy), ...trail.map(whyText)].filter(Boolean); + const traced = [ + notWhy && whyNotText(notWhy), + ...trail.map(whyText), + small && typeof small === "object" && !small.result.fixes.length && `Small changes tried without the AI: ${small.result.message}`, + ].filter(Boolean); const story = traced.length ? `${rep.story} Why, traced in the simulation: ${traced.join(" ")}` : rep.story; const picked = trail[0]; @@ -1141,6 +1199,57 @@ function Happened({ + + + )} + {small === "working" &&

Trying small changes to the design, running every example after each…

} + {smallError &&

Couldn't try small fixes: {smallError}

} + {small && typeof small === "object" && ( +
+
+ {small.result.fixes.length ? "Small changes to the design that make every example pass" : "No small fix"} + +
+

+ {small.result.message} + {!small.result.fixes.length && " Ask the AI to fix the design; it is told what was tried."} +

+ {small.result.fixes.map((fx, i) => ( +
+
{withCode(fx.text)}
+ {fx.edit && ( + 
+                  - {fx.edit.before.trim()}
+                  {"\n"}
+                  + {fx.edit.after.trim()}
+
+ )} + {small.effects[i]?.length > 0 && What it changes in the examples:} + {small.effects[i]?.length > 0 && ( +
    + {small.effects[i].map((d) => ( +
  • {withCode(d)}
    • + ))} +
+ )} + +
+ ))}
)} {fix === "working" &&

Working out the change and checking it…

} diff --git a/apps/web/src/styles.css b/apps/web/src/styles.css index 8728cdd..b81558b 100644 --- a/apps/web/src/styles.css +++ b/apps/web/src/styles.css @@ -2682,3 +2682,27 @@ a.button { .why-chain button.link { padding: 0; } + +/* "Try a small fix": each repair found, as a one-line diff. */ +.small-fix { + display: flex; + flex-direction: column; + align-items: flex-start; + gap: 4px; + padding-top: 8px; + border-top: 1px solid var(--border); +} + +.small-fix-diff { + margin: 0; + white-space: pre-wrap; + font-size: 12px; +} + +.small-fix-diff .del { + color: var(--danger); +} + +.small-fix-diff .add { + color: var(--ok); +} diff --git a/crates/wafer-api/src/design.rs b/crates/wafer-api/src/design.rs index 5bf9a3b..c228047 100644 --- a/crates/wafer-api/src/design.rs +++ b/crates/wafer-api/src/design.rs @@ -315,6 +315,40 @@ pub async fn find_example( found.map(Json).map_err(ApiError::Conflict) } +/// How long "Try a small fix" may search. +const FIX_BUDGET: std::time::Duration = std::time::Duration::from_secs(20); + +#[derive(Debug, Deserialize, Default)] +pub struct SmallFixReq { + /// Lines to try first (those "Why not?" pointed at). + #[serde(default)] + pub focus: Vec, +} + +/// "Try a small fix" (`crate::fixes`) at the branch tip: small repairs after which every example +/// passes, each with the edit to make. Nothing is saved. Editors: it costs server time. +pub async fn small_fix( + State(state): State, + user: AuthUser, + Path(id): Path, + Json(req): Json, +) -> ApiResult> { + authorize(&state, &user, id, Role::Editor).await?; + let commit = head(&state, id).await?; + let files = files_at(&state, id, &commit).await?; + let spec = match files.get(SPEC_FILE) { + Some(t) => parse_spec(t)?, + None => return Err(ApiError::NotFound("this project has no spec yet".into())), + }; + let tops: Vec = spec.examples.iter().map(|e| spec::tb_name(&e.id)).collect(); + let focus: Vec = req.focus.into_iter().take(64).collect(); + let found = tokio::task::spawn_blocking(move || { + gate::with_slot(|| wafer_sim::on_big_stack(|| crate::fixes::find(&files, &tops, &focus, FIX_BUDGET))) + }) + .await?; + found.map(Json).map_err(ApiError::Conflict) +} + #[derive(Debug, Deserialize)] pub struct TestbenchReq { /// The spec being edited (unsaved edits included). diff --git a/crates/wafer-api/src/fixes.rs b/crates/wafer-api/src/fixes.rs new file mode 100644 index 0000000..27b9738 --- /dev/null +++ b/crates/wafer-api/src/fixes.rs @@ -0,0 +1,458 @@ +//! "Try a small fix": when an example fails, tries the small repairs that undo the most common +//! mistakes (a comparison turned around, a constant one off, a condition reversed, a `!` too many, +//! `&&` for `||`, `-` for `+`, an assigned value one off), one at a time, and keeps those after +//! which every example passes. No AI: each repair is checked by simulating all the examples. +//! +//! Repairs are made in the parsed design, like planted mistakes (`crate::mistakes`), lines the +//! person points at (from "Why not?") first. A repair found is turned back into an edit of one +//! line of the file, keeping the rest of the line as written. + +use std::collections::BTreeMap; +use std::time::{Duration, Instant}; + +use serde::{Deserialize, Serialize}; +use wafer_sim::ast::Module; + +use crate::mistakes::Walker; + +#[derive(Debug, Clone, Deserialize, Default)] +pub struct Focus { + pub file: String, + pub line: u32, +} + +#[derive(Debug, Clone, Serialize, PartialEq)] +pub struct Fixes { + pub fixes: Vec, + /// Repairs tried, of `sites` possible. + pub tried: usize, + pub sites: usize, + /// The examples failing before any repair. + pub failing: Vec, + pub elapsed_ms: u64, + /// In words: what was found, or why nothing was. + pub message: String, +} + +#[derive(Debug, Clone, Serialize, PartialEq)] +pub struct Fix { + pub file: String, + pub line: u32, + /// As `crate::mistakes::Slipped::kind`, plus `value` (an assigned constant). + pub kind: String, + pub original: String, + pub replacement: String, + /// "Line 18 of blinker.v: `cnt == N - 1` → `cnt == N`; every example passes." + pub text: String, + /// The line as it is and as it would be; None when the change can't be placed in the text + /// (it spans lines, or is written very differently from how it prints). + pub edit: Option, +} + +#[derive(Debug, Clone, Serialize, PartialEq)] +pub struct Edit { + pub before: String, + pub after: String, + /// The whole file with the change made. + pub content: String, +} + +/// Fixes kept, at most. +const MAX_FIXES: usize = 3; + +/// Repairs that make every example in `examples` (testbench tops) pass, within `budget`. Fails (in +/// words) if the design doesn't parse or every example already passes. Call on a big stack. +pub fn find( + files: &BTreeMap, + examples: &[String], + focus: &[Focus], + budget: Duration, +) -> Result { + let started = Instant::now(); + let hdl: Vec<(&str, &str)> = + files.iter().filter(|(p, _)| crate::gate::is_hdl(p)).map(|(p, c)| (p.as_str(), c.as_str())).collect(); + let (modules, timescale) = wafer_sim::parse(&hdl).map_err(|e| format!("the design doesn't parse: {e}"))?; + let limits = crate::gate::limits(); + let passes = |ms: &[Module], top: &str| match wafer_sim::run_parsed(ms, timescale, top, limits) { + Ok(r) => r.finished && crate::gate::verdict(&r.log).0 == "pass", + Err(_) => false, + }; + let failing: Vec = examples.iter().filter(|t| !passes(&modules, t)).cloned().collect(); + if failing.is_empty() { + return Err("every example passes already".into()); + } + // The others, after the failing ones: a repair has to fix those first. + let order: Vec<&String> = failing.iter().chain(examples.iter().filter(|t| !failing.contains(t))).collect(); + + let mine = + |m: &Module| !m.file.starts_with("tb/") && files.get(&m.file).is_some_and(|c| !crate::blocks::is_generated(c)); + let mut probe = modules.clone(); + let mut w = Walker::repairs(None); + for m in probe.iter_mut().filter(|m| mine(m)) { + w.module(m); + } + let sites = w.sites; + // The lines pointed at first, then the rest in the order of the design. + let near = |i: &usize| { + let s = &sites[*i]; + !focus.iter().any(|f| f.file == s.file && f.line == s.line) + }; + let mut chosen: Vec = (0..sites.len()).collect(); + chosen.sort_by_key(|i| near(i)); + + let (mut fixes, mut tried) = (Vec::new(), 0); + for i in chosen { + if started.elapsed() > budget || fixes.len() >= MAX_FIXES { + break; + } + let mut ms = modules.clone(); + let mut w = Walker::repairs(Some(i)); + for m in ms.iter_mut().filter(|m| mine(m)) { + w.module(m); + } + tried += 1; + if order.iter().all(|t| passes(&ms, t)) { + let s = &sites[i]; + let name = s.file.rsplit('/').next().unwrap_or(&s.file); + fixes.push(Fix { + file: s.file.clone(), + line: s.line, + kind: s.kind.clone(), + text: format!( + "Line {} of {name}: `{}` → `{}`; {} pass{}.", + s.line, + s.original, + s.mistake, + if examples.len() == 1 { + "the example".to_string() + } else { + format!("all {} examples", examples.len()) + }, + if examples.len() == 1 { "es" } else { "" } + ), + edit: files.get(&s.file).and_then(|src| edit(src, s.line, &s.original, &s.mistake)), + original: s.original.clone(), + replacement: s.mistake.clone(), + }); + } + } + let message = if !fixes.is_empty() { + format!( + "Found {} small change{} after which every example passes ({tried} tried).", + fixes.len(), + if fixes.len() == 1 { "" } else { "s" } + ) + } else if tried < sites.len() { + format!( + "No small change tried makes every example pass ({tried} of {} tried before the time ran out).", + sites.len() + ) + } else { + format!("None of the {tried} small changes there are makes every example pass.") + }; + Ok(Fixes { + fixes, + tried, + sites: sites.len(), + failing: failing.iter().map(|t| t.trim_start_matches("tb_ex_").to_string()).collect(), + elapsed_ms: started.elapsed().as_millis() as u64, + message, + }) +} + +/// A token of a line of Verilog, with where it is in the line. +#[derive(Debug, Clone, PartialEq)] +struct Tok { + text: String, + /// For a number: its value, so `4'd7` matches `7`. + value: Option, + start: usize, + end: usize, +} + +fn tokens(s: &str) -> Vec { + const OPS: [&str; 22] = [ + "<<<", ">>>", "===", "!==", "<=", ">=", "==", "!=", "&&", "||", "<<", ">>", "~^", "^~", "~&", "~|", "+:", "-:", + "**", "->", "::", "'{", + ]; + let b = s.as_bytes(); + let mut out = Vec::new(); + let mut i = 0; + while i < b.len() { + let c = b[i] as char; + if c.is_whitespace() { + i += 1; + continue; + } + let start = i; + if c.is_ascii_alphabetic() || c == '_' || c == '$' || c == '`' { + i += 1; + while i < b.len() && ((b[i] as char).is_ascii_alphanumeric() || b[i] == b'_' || b[i] == b'$') { + i += 1; + } + out.push(Tok { text: s[start..i].into(), value: None, start, end: i }); + continue; + } + if c.is_ascii_digit() || (c == '\'' && i + 1 < b.len() && b"sSbBoOdDhH".contains(&b[i + 1])) { + while i < b.len() && ((b[i] as char).is_ascii_digit() || b[i] == b'_') { + i += 1; + } + let mut j = i; + while j < b.len() && b[j] == b' ' { + j += 1; + } + if j < b.len() && b[j] == b'\'' { + i = j + 1; + if i < b.len() && (b[i] == b's' || b[i] == b'S') { + i += 1; + } + i += 1; // the base + while i < b.len() && b[i] == b' ' { + i += 1; + } + while i < b.len() && ((b[i] as char).is_ascii_hexdigit() || b"_xXzZ?".contains(&b[i])) { + i += 1; + } + } + let text = s[start..i].to_string(); + let value = number(&text); + out.push(Tok { text, value, start, end: i }); + continue; + } + let op = OPS.iter().find(|o| s[i..].starts_with(*o)).map_or(1, |o| o.len()); + i += op; + out.push(Tok { text: s[start..i].into(), value: None, start, end: i }); + } + out +} + +/// The value of a Verilog number (`12`, `4'd7`, `8'hff`, `1'b1`); None with x/z bits. +fn number(t: &str) -> Option { + let t: String = t.chars().filter(|c| *c != '_' && *c != ' ').collect(); + match t.split_once('\'') { + None => t.parse().ok(), + Some((_, rest)) => { + let rest = rest.trim_start_matches(['s', 'S']); + let (base, digits) = rest.split_at(1); + let radix = match base.to_ascii_lowercase().as_str() { + "b" => 2, + "o" => 8, + "d" => 10, + "h" => 16, + _ => return None, + }; + u64::from_str_radix(digits, radix).ok() + } + } +} + +fn same(a: &Tok, b: &Tok) -> bool { + match (a.value, b.value) { + (Some(x), Some(y)) => x == y, + _ => a.text == b.text, + } +} + +/// `src` with `original` (as the syntax tree prints it) on line `line` replaced by `replacement`: +/// numbers as written in the line where they are unchanged, and in its style (`4'd8`) where the +/// change is to a number. +fn edit(src: &str, line: u32, original: &str, replacement: &str) -> Option { + let lines: Vec<&str> = src.split_inclusive('\n').collect(); + let raw = *lines.get(line.checked_sub(1)? as usize)?; + let text = raw.trim_end_matches(['\n', '\r']); + let have = tokens(text); + let want = tokens(original); + if want.is_empty() { + return None; + } + let at = (0..have.len()).find_map(|k| matched(&have, &want, k))?; + let span: Vec<&Tok> = at.iter().map(|&i| &have[i]).collect(); + let written: Vec<&str> = span.iter().map(|t| &text[t.start..t.end]).collect(); + + // The replacement, its numbers written the way the line writes them. + let new = tokens(replacement); + let aligned = new.len() == want.len(); + let mut out = String::new(); + let mut last = 0; + for (k, t) in new.iter().enumerate() { + out.push_str(&replacement[last..t.start]); + last = t.end; + let Some(v) = t.value else { + out.push_str(&t.text); + continue; + }; + let spelled = if aligned && span[k].value.is_some() { + // The number at the same place: as written, or changed in its style. + if span[k].value == Some(v) { Some(written[k].to_string()) } else { restyle(written[k], v) } + } else { + // A number of the original, as written. + span.iter().zip(&written).find(|(o, _)| o.value == Some(v)).map(|(_, w)| w.to_string()) + }; + out.push_str(&spelled.unwrap_or_else(|| t.text.clone())); + } + out.push_str(&replacement[last..]); + + let after = format!("{}{}{}", &text[..span[0].start], out, &text[span[span.len() - 1].end..]); + let mut content = String::with_capacity(src.len() + 8); + for (i, l) in lines.iter().enumerate() { + if i + 1 == line as usize { + content.push_str(&after); + content.push_str(&l[text.len()..]); + } else { + content.push_str(l); + } + } + Some(Edit { before: text.to_string(), after, content }) +} + +/// Where `want` appears in `have` from token `from` on: the token matching each, allowing +/// parentheses the printed form leaves out (`!(en)` for `!en`) as long as they pair up. +fn matched(have: &[Tok], want: &[Tok], from: usize) -> Option> { + let mut at = Vec::with_capacity(want.len()); + let (mut i, mut depth) = (from, 0i32); + for w in want { + loop { + let h = have.get(i)?; + if same(w, h) { + break; + } + // Only inside the match, and only parentheses. + match h.text.as_str() { + "(" if !at.is_empty() => depth += 1, + ")" if !at.is_empty() && depth > 0 => depth -= 1, + _ => return None, + } + i += 1; + } + at.push(i); + i += 1; + } + // Close what was skipped open. + while depth > 0 { + if have.get(i)?.text != ")" { + return None; + } + depth -= 1; + at.push(i); + i += 1; + } + Some(at) +} + +/// `v` written like `like` (`4'd7` → `4'd8`, `1'b0` → `1'b1`); None if `like` isn't a sized number. +fn restyle(like: &str, v: u64) -> Option { + let (w, rest) = like.split_once('\'')?; + let signed = rest.starts_with(['s', 'S']); + let base = rest.trim_start_matches(['s', 'S']).chars().next()?; + let digits = match base.to_ascii_lowercase() { + 'b' => format!("{v:b}"), + 'o' => format!("{v:o}"), + 'h' => format!("{v:x}"), + _ => v.to_string(), + }; + Some(format!("{}'{}{base}{digits}", w.trim(), if signed { "s" } else { "" })) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn project(id: &str) -> (BTreeMap, Vec) { + let st = crate::starters::get(id).expect("starter"); + let files = st + .project_files(&st.title) + .into_iter() + .filter_map(|(p, c)| String::from_utf8(c).ok().map(|c| (p, c))) + .collect(); + (files, st.spec.examples.iter().map(|e| crate::spec::tb_name(&e.id)).collect()) + } + + fn run(files: &BTreeMap, examples: &[String], focus: &[Focus]) -> Result { + wafer_sim::on_big_stack(|| find(files, examples, focus, Duration::from_secs(60))) + } + + /// Breaks a starter in one place and expects the repair (and the line as it was) back. + fn repairs(id: &str, file: &str, good: &str, bad: &str) -> Fixes { + let (mut files, examples) = project(id); + let src = files[file].clone(); + assert_eq!(src.matches(good).count(), 1, "{good}"); + let line = src[..src.find(good).unwrap()].matches('\n').count() as u32 + 1; + files.insert(file.into(), src.replacen(good, bad, 1)); + let r = run(&files, &examples, &[Focus { file: file.into(), line }]).expect("searches"); + let back = r.fixes.iter().find(|f| f.edit.as_ref().is_some_and(|e| e.content == src)); + assert!(back.is_some(), "{id}: {bad} → {good}: {r:#?}"); + r + } + + #[test] + fn typical_mistakes_are_repaired_back_to_the_line_as_written() { + // Off by one: the edit keeps the line, only the condition changes. + let r = repairs("blinker", "src/blinker.v", "cnt == N - 1", "cnt == N"); + let f = &r.fixes[0]; + assert_eq!((f.original.as_str(), f.replacement.as_str()), ("cnt == N", "cnt == N - 1")); + assert_eq!(f.text, format!("Line {} of blinker.v: `cnt == N` → `cnt == N - 1`; all 2 examples pass.", f.line)); + let e = f.edit.as_ref().unwrap(); + assert!(e.before.contains("cnt == N)") && e.after.contains("cnt == N - 1)"), "{e:?}"); + assert_eq!(r.failing.len(), 1); + + // A comparison turned around, `-` for `+`, and a sized constant one off (written as before). + repairs("updown", "src/updown.v", "up && !down) count <= count + 4'd1", "up && !down) count <= count - 4'd1"); + repairs("watchdog", "src/watchdog.v", "bark <= 1'b1", "bark <= 1'b0"); + } + + /// Every mistake "Would your examples catch a mistake?" plants in a starter (a spread of them; + /// leaving lines out aside), written into the file, is repaired back to the line as it was. + #[test] + fn planted_mistakes_in_the_starters_are_repaired_exactly() { + let mut repaired = 0; + for st in crate::starters::all() { + let (files, examples) = project(&st.id); + let hdl: Vec<(&str, &str)> = + files.iter().filter(|(p, _)| p.ends_with(".v")).map(|(p, c)| (p.as_str(), c.as_str())).collect(); + let (mut ms, _) = wafer_sim::parse(&hdl).unwrap(); + let mut w = Walker::new(None); + ms.iter_mut().filter(|m| !m.file.starts_with("tb/")).for_each(|m| w.module(m)); + let every = (w.sites.len() / 6).max(1); + for s in w.sites.iter().step_by(every).filter(|s| s.kind != "forget") { + let planted = edit(&files[&s.file], s.line, &s.original, &s.mistake).expect("placed in the text"); + let mut broken = files.clone(); + broken.insert(s.file.clone(), planted.content); + let Ok(r) = run(&broken, &examples, &[Focus { file: s.file.clone(), line: s.line }]) else { + continue; // no example notices it + }; + assert!( + r.fixes.iter().any(|f| f.edit.as_ref().is_some_and(|e| e.content == files[&s.file])), + "{}: {} → {}: {r:#?}", + st.id, + s.original, + s.mistake + ); + repaired += 1; + } + } + assert!(repaired > 60, "{repaired}"); + } + + #[test] + fn passing_designs_are_refused() { + let (files, examples) = project("blinker"); + assert_eq!(run(&files, &examples, &[]).unwrap_err(), "every example passes already"); + } + + #[test] + fn edits_keep_how_the_line_is_written() { + let src = "a\n else if (cnt == 16'd7) begin // seven\nb\n"; + let e = edit(src, 2, "cnt == 7", "cnt == 8").unwrap(); + assert_eq!(e.after, " else if (cnt == 16'd8) begin // seven"); + assert_eq!(e.content, "a\n else if (cnt == 16'd8) begin // seven\nb\n"); + let e = edit(" if (!rst_n) x <= 1'b0;\n", 1, "!rst_n", "rst_n").unwrap(); + assert_eq!(e.after, " if (rst_n) x <= 1'b0;"); + let e = edit(" led <= 1'b0;\n", 1, "led <= 0", "led <= 1").unwrap(); + assert_eq!(e.after, " led <= 1'b1;"); + let e = edit(" if (a&&b) y = 0;\n", 1, "a && b", "a || b").unwrap(); + assert_eq!(e.after, " if (a || b) y = 0;"); + let e = edit(" if (!(en)) y = 0;\n", 1, "!en", "en").unwrap(); + assert_eq!(e.after, " if (en) y = 0;", "parentheses the printed form leaves out"); + assert!(edit(" x = 1;\n", 1, "y == 2", "y == 3").is_none(), "not on that line"); + } +} diff --git a/crates/wafer-api/src/lib.rs b/crates/wafer-api/src/lib.rs index d7ac88e..40834ef 100644 --- a/crates/wafer-api/src/lib.rs +++ b/crates/wafer-api/src/lib.rs @@ -9,6 +9,7 @@ pub mod collab; pub mod coverage; pub mod design; pub mod error; +pub mod fixes; pub mod fsm; pub mod gate; pub mod gls; @@ -75,6 +76,7 @@ pub fn router(state: AppState) -> Router { .route("/projects/{id}/spec/testbench", post(design::testbench)) .route("/projects/{id}/spec/mistakes", post(design::mistakes)) .route("/projects/{id}/spec/find-example", post(design::find_example)) + .route("/projects/{id}/spec/small-fix", post(design::small_fix)) .route("/projects/{id}/spec/draft", post(design::draft)) .route("/starters", get(starters::list)) .route("/projects/{id}/starters/{starter}", post(starters::add)) diff --git a/crates/wafer-api/src/mistakes.rs b/crates/wafer-api/src/mistakes.rs index 9fb9f9e..6d4c55a 100644 --- a/crates/wafer-api/src/mistakes.rs +++ b/crates/wafer-api/src/mistakes.rs @@ -135,16 +135,23 @@ pub(crate) fn planted( /// Walks the design in a fixed order, numbering every place a mistake could go. Without a target /// it lists them; with one it plants that mistake and leaves the rest as they are. -struct Walker { +pub(crate) struct Walker { target: Option, next: usize, file: String, - sites: Vec, + pub(crate) sites: Vec, + /// Looking for repairs rather than mistakes (`crate::fixes`): constants are tried one off in + /// both directions, assigned constants changed too, and nothing is left out. + fix: bool, } impl Walker { - fn new(target: Option) -> Self { - Walker { target, next: 0, file: String::new(), sites: Vec::new() } + pub(crate) fn new(target: Option) -> Self { + Walker { target, next: 0, file: String::new(), sites: Vec::new(), fix: false } + } + + pub(crate) fn repairs(target: Option) -> Self { + Walker { fix: true, ..Walker::new(target) } } /// Numbers one possible mistake; true if it is the one to plant. @@ -175,7 +182,7 @@ impl Walker { self.target == Some(i) } - fn module(&mut self, m: &mut Module) { + pub(crate) fn module(&mut self, m: &mut Module) { self.file = m.file.clone(); for item in &mut m.items { match item { @@ -193,8 +200,13 @@ impl Walker { Stmt::Block(v) => v.iter_mut().for_each(|x| self.stmt(x)), Stmt::If(c, t, e) => { let line = c.line().unwrap_or(0); - if self.site(line, "reverse", c.to_string(), format!("!({c})")) { - *c = Expr::Unary("!", Box::new(c.clone())); + let flipped = Expr::Unary("!", Box::new(c.clone())); + // A repair writes it as one would (`!rst_n`) and leaves a `!` already there to the + // repair that takes it out. + let mistake = if self.fix { flipped.to_string() } else { format!("!({c})") }; + if !(self.fix && matches!(c, Expr::Unary("!", _))) && self.site(line, "reverse", c.to_string(), mistake) + { + *c = flipped; } self.expr(c, line); self.stmt(t); @@ -214,7 +226,37 @@ impl Walker { // after reset); leaving it out would mostly report mistakes that change nothing. let zero = matches!(rhs, Expr::Num(n) if n.value == 0); let text = format!("{lhs} {arrow} {rhs}"); - if !zero && self.site(line, "forget", text, String::new()) { + if self.fix { + // A signal assigned as it is that should have been inverted (`led <= ~led`). + if let Expr::Ident(..) = rhs { + let inv = Expr::Unary("~", Box::new(rhs.clone())); + if self.site(line, "reverse_value", text.clone(), format!("{lhs} {arrow} {inv}")) { + *rhs = inv; + return; + } + } + // The value assigned, one off (for one bit: the other value). + if let Expr::Num(n) = rhs { + let n = *n; + let wide = n.width.is_none_or(|w| w > 1); + let mut tries = vec![n.value.wrapping_add(1)]; + if n.value > 0 { + tries.insert(0, n.value - 1); + } + for v in tries { + if !wide && v > 1 { + continue; + } + let mut m = n; + m.value = v; + let to = format!("{lhs} {arrow} {}", Expr::Num(m)); + if self.site(line, "value", text.clone(), to) { + *rhs = Expr::Num(m); + return; + } + } + } + } else if !zero && self.site(line, "forget", text, String::new()) { *s = Stmt::Null; return; } @@ -252,22 +294,68 @@ impl Walker { return; } } + // A repair: a `!` missing from one side of `&&` / `||` (`up && down` → `up && !down`). + if self.fix && matches!(*op, "&&" | "||") { + for right in [false, true] { + let side: Box = if right { b.clone() } else { a.clone() }; + if !matches!(side.as_ref(), Expr::Ident(..)) { + continue; + } + let not = Box::new(Expr::Unary("!", side)); + let (na, nb) = if right { (a.clone(), not.clone()) } else { (not.clone(), b.clone()) }; + if self.site(line, "reverse_value", original.clone(), Expr::Binary(op, na, nb).to_string()) { + if right { + *b = not; + } else { + *a = not; + } + return; + } + } + } if matches!(*op, "==" | "!=" | "<" | "<=" | ">" | ">=") { // The constant compared with, one off: `cnt == 7` → `cnt == 8`, `cnt == N - 1` → `cnt == N`. - let one_off = match b.as_ref() { + let one = |v: u64| { + Box::new(Expr::Num(wafer_sim::lexer::Num { + value: v, + xz: 0, + width: None, + signed: false, + fill: false, + real: false, + })) + }; + let mut one_off = Vec::new(); + match b.as_ref() { Expr::Num(n) => { let mut m = *n; m.value = m.value.wrapping_add(1); - Some(Expr::Num(m)) + one_off.push(Expr::Num(m)); + if self.fix && n.value > 0 { + m.value = n.value - 1; + one_off.push(Expr::Num(m)); + } } - Expr::Binary("-", x, one) if matches!(one.as_ref(), Expr::Num(n) if n.value == 1) => { - Some(x.as_ref().clone()) + Expr::Binary(pm @ ("-" | "+"), x, k) if matches!(k.as_ref(), Expr::Num(n) if n.value == 1) => { + if *pm == "-" || self.fix { + one_off.push(x.as_ref().clone()); + } } - _ => None, - }; - if let Some(nb) = one_off { + Expr::Ident(..) if self.fix => { + one_off.push(Expr::Binary("-", b.clone(), one(1))); + one_off.push(Expr::Binary("+", b.clone(), one(1))); + } + _ => {} + } + if self.fix + && let Expr::Binary("-", x, k) = b.as_ref() + && matches!(k.as_ref(), Expr::Num(n) if n.value == 1) + { + one_off.push(Expr::Binary("-", x.clone(), one(2))); + } + for nb in one_off { let mistake = Expr::Binary(op, a.clone(), Box::new(nb.clone())).to_string(); - if self.site(line, "off_by_one", original, mistake) { + if self.site(line, "off_by_one", original.clone(), mistake) { **b = nb; return; } diff --git a/docs/api.md b/docs/api.md index 2c0e459..49f14ab 100644 --- a/docs/api.md +++ b/docs/api.md @@ -50,6 +50,7 @@ shows how to get one locally). `GET /healthz` (outside `/api`) needs no token. | POST | `/projects/{id}/spec/run` | editor | Runs every testbench at `main`; results per example (and the other testbenches), and `coverage`: `{covered, total, gaps: [{file, line, kind, what, labels, text, suggestion}]}`, the design's branches the examples take (`null` without examples or branches), and `machines`: the design's state machines `{module, file, state, states, values, transitions: [{from, to, when, line, covered}], reset}`. | | POST | `/projects/{id}/spec/mistakes` | editor | Plants up to 40 small mistakes in the design, one at a time, and runs the examples against each (at most 30 s): `{caught, tried, sites, slipped: [{file, line, kind, original, mistake, text, suggestion}], elapsed_ms}`. `409` if an example already fails. | | POST | `/projects/{id}/spec/find-example` | editor | Searches the design's inputs for an example that reaches a target: `{"for": "gap", file, line, text}` (a coverage gap) or `{"for": "mistake", file, line, kind, original, mistake}` (a planted mistake that slipped through), as `spec/run` and `spec/mistakes` report them. Returns `{example, tried, elapsed_ms, message}`: a draft example (not saved) whose final checks hold what the design did, or `null` and why. At most about 30 s. `409` if the design doesn't parse, has no inputs, or no longer has the target. | +| POST | `/projects/{id}/spec/small-fix` | editor | "Try a small fix": with `{focus: [{file, line}]}` (lines to try first, optional), tries small repairs of the design at the branch tip one at a time and keeps those after which every example passes. Returns `{fixes: [{file, line, kind, original, replacement, text, edit: {before, after, content} \| null}], tried, sites, failing, elapsed_ms, message}`; `edit.content` is the whole file changed, for a commit. Nothing is saved. At most about 20 s. `409` if the design doesn't parse or every example passes already. | | POST | `/projects/{id}/spec/testbench` | viewer | `{spec, example}`: the testbench the example compiles to (`{path, top, source}`), without saving; `400` lists the problems. Used by the Try it playground. | | POST | `/projects/{id}/spec/draft` | editor | `{instruction, spec?}`: the assistant writes or revises the spec; nothing is saved. Returns `{spec, note, model, usage, usd, elapsed_ms}`; `usd` is priced for the model that served it and counts against the AI budget. | diff --git a/docs/user-guide/design-view.md b/docs/user-guide/design-view.md index 7f93425..fe11525 100644 --- a/docs/user-guide/design-view.md +++ b/docs/user-guide/design-view.md @@ -111,7 +111,7 @@ Lines that only write other constants (`valid <= 0` when 1 was expected) are lef If no line can write the expected value, it says so: the design needs one. This answer, too, goes with a request to the AI. -Three buttons answer the one question to decide, **Which is wrong?**: +Four buttons answer the one question to decide, **Which is wrong?**: - **The design: ask the AI to fix it**, with this explanation; - **The example: use what the design did**: rewrites the failing check from the simulation, @@ -124,7 +124,16 @@ Three buttons answer the one question to decide, **Which is wrong?**: and saving the spec keeps them. Use it when the design is right and the example's timing or value was a guess; - **The example: ask the assistant to correct it**: re-checks the example against the spec in - words and corrects it. + words and corrects it; +- **The design: try a small fix**: tries, without AI, the small changes that undo the most common + mistakes, one at a time: a comparison the other way round (`<` for `<=`, `==` for `!=`), a + constant or limit one off in either direction (`cnt == N` → `cnt == N - 1`), a condition + reversed, a `!` or `~` too many or missing, `&&` for `||`, `-` for `+`, and an assigned value one + off (`1'b0` for `1'b1`). The lines "Why not?" and "Why?" pointed at are tried first. After each + change every example is simulated, and only changes after which all of them pass are offered + (up to three), each as a one-line diff that keeps the rest of the line as written, with what it + changes in the examples' outputs ("`led` at cycle 4 was 0, now 1"). **Use this change** commits + it. If none works, it says how many it tried, and asking the AI then tells it what was tried. ## Health