Why wasn't it the expected value? Explain a failing check

CHANGELOG.md

@@ -8,6 +8,13 @@ All notable changes to Wafersmith are recorded here. The format follows
 
 ### Added
 
+- **Why wasn't it the expected value?**: a failing example's account ends with a link that lists
+  every line of the design that could have written the value the example expected, and for each
+  the decision that kept it from running ("`cnt == N - 1` was false (cnt = 2)"), or that it ran and
+  computed something else. Its values can be clicked to follow the cause back, and the answer goes
+  with a request to the AI. The simulator now records every `if`/`case` decision with the branch
+  taken (`SimResult::decisions`) and which signal each assignment writes (`Origin::writes`). Runs
+  in the browser; no AI and no cost.
 - **Why is it this value?**: click any value in an example's "What actually happened" timeline to
   see the line of the design that wrote it, why that line ran (each `if`/`case` around it and
   which way it went, with the values decided on) and the values it used; each of those can be

apps/web/src/components/Design.tsx

@@ -16,7 +16,7 @@ import {
 } from "../api";
 import { extendTestbench, fmt as fmtValue, replay, targetSpec, type Replay } from "../lib/replay";
 import { findFix, type Fix } from "../lib/fixexample";
-import { decided, why, whyText, type Because, type Read } from "../lib/why";
+import { decided, missed, why, whyNot, whyNotText, whyText, type Because, type Read, type WhyNot } from "../lib/why";
 import { StateMachine } from "./StateMachine";
 import { TryIt } from "./TryIt";
 import { SimClient } from "../sim/client";
@@ -980,6 +980,8 @@ function Happened({
     const b = rep && why(rep, full, t);
     if (b) setTrail((tr) => [...tr.slice(0, after), b]);
   };
+  // "Why not?": why the first failing check didn't see the value the example expected.
+  const [notWhy, setNotWhy] = useState<WhyNot | null>(null);
   // "The design is right": a change to the example that makes it pass, worked out and re-checked.
   const [fix, setFix] = useState<Fix | "working" | "none" | null>(null);
   async function acceptDesign() {
@@ -1030,6 +1032,7 @@ function Happened({
       if (!cancelled) {
         setRep(replay(spec, ex, sim));
         setTrail([]);
+        setNotWhy(null);
       }
     })().catch((e: Error) => !cancelled && setError(e.message));
     return () => {
@@ -1045,7 +1048,8 @@ function Happened({
   const every = rep.cycles > 200 ? 50 : rep.cycles > 80 ? 20 : rep.cycles > 30 ? 10 : 5;
   const failing = rep.firstFail;
   // The AI gets the explanation too, when someone followed one.
-  const story = trail.length ? `${rep.story} Why, traced in the simulation: ${trail.map(whyText).join(" ")}` : rep.story;
+  const traced = [notWhy && whyNotText(notWhy), ...trail.map(whyText)].filter(Boolean);
+  const story = traced.length ? `${rep.story} Why, traced in the simulation: ${traced.join(" ")}` : rep.story;
   const picked = trail[0];
 
   return (
@@ -1055,8 +1059,27 @@ function Happened({
         <span className={`status ${failing ? "status-failed" : "status-succeeded"}`}>{failing ? "differs from the example" : "matches the example"}</span>
       </div>
       <p className={`small ${failing ? "" : "muted"}`} style={{ margin: 0 }}>
-        {rep.story}
+        {rep.story}{" "}
+        {failing && failing.failCycle !== undefined && !notWhy && (
+          <button
+            className="link small"
+            onClick={() => setNotWhy(whyNot(rep, `${rep.top}.${failing.signal}`, failing.failCycle!, failing.expected))}
+          >
+            Why wasn't it {failing.expected.toString()}?
+          </button>
+        )}
       </p>
+      {notWhy && (
+        <WhyNotPanel
+          w={notWhy}
+          onAsk={(r) => {
+            if (r.t === null) return;
+            setPickedCycle(null);
+            ask(r.full, r.t, 0);
+          }}
+          onClose={() => setNotWhy(null)}
+        />
+      )}
       {rep.problem && <p className="warn small">Note: {rep.problem}.</p>}
       <div className="design-scroll" ref={box}>
         <div className="hp-grid" style={{ gridTemplateColumns: `72px ${width}px` }}>
@@ -1244,6 +1267,76 @@ function HpRow({
   );
 }
 
+/** "Why not?": the lines that could have written the value the example expected, and what kept
+ * each from doing so; its values can be asked about like any other. */
+function WhyNotPanel({ w, onAsk, onClose }: { w: WhyNot; onAsk: (r: Read) => void; onClose: () => void }) {
+  const chip = (r: Read) => (
+    <button key={r.full} className="link small mono" disabled={r.t === null} onClick={() => onAsk(r)} title={`Why is ${r.name} ${r.value}?`}>
+      {r.name} = {r.value}
+    </button>
+  );
+  const chips = (rs: Read[]) => rs.map((r, j) => [j > 0 && ", ", chip(r)]);
+  return (
+    <div className="design-card why">
+      <div className="design-row" style={{ justifyContent: "space-between" }}>
+        <strong className="small">
+          Why wasn't <span className="mono">{w.name}</span> {w.expected} at cycle {w.cycle}?
+        </strong>
+        <button className="link small" onClick={onClose}>
+          Close
+        </button>
+      </div>
+      {w.unrecorded && <p className="warn small">The run is too long to have recorded every decision up to this cycle.</p>}
+      {w.candidates.length === 0 ? (
+        <p className="small" style={{ margin: 0 }}>
+          No line of the design can write {w.expected} to <span className="mono">{w.name}</span>
+          {w.others > 0 && <> (the {w.others === 1 ? "line that writes it only writes" : `${w.others} lines that write it only write`} other values)</>}. The design
+          needs a line that does.
+        </p>
+      ) : (
+        <ol className="why-chain">
+          {w.candidates.map((c, k) => (
+            <li key={k} className="small">
+              Line {c.line} of {c.file.split("/").pop()}: <code>{c.code}</code>
+              {c.verdict === "never" && <div>It hadn't run yet: nothing had reached its decisions by cycle {w.cycle}.</div>}
+              {c.verdict === "blocked" && c.blocked && (
+                <div>
+                  Didn't run for cycle {c.cycle}
+                  {c.passed.length > 0 && (
+                    <>
+                      {" "}
+                      (
+                      {c.passed.map((p, j) => (
+                        <span key={j}>
+                          {j > 0 && ", "}
+                          {withCode(decided(p))}
+                        </span>
+                      ))}
+                      )
+                    </>
+                  )}{" "}
+                  because <strong>{withCode(missed(c.blocked))}</strong>
+                  {c.blocked.reads.length > 0 && <> ({chips(c.blocked.reads)})</>}.
+                </div>
+              )}
+              {c.verdict === "ran" && (
+                <div>
+                  It ran{c.cycle !== undefined && <> for cycle {c.cycle}</>}
+                  {c.reads.length > 0 && <>, with {chips(c.reads)}</>}, and gave {w.name} another value (or a later line changed it).
+                </div>
+              )}
+            </li>
+          ))}
+        </ol>
+      )}
+      <p className="muted small" style={{ margin: 0 }}>
+        {w.candidates.length > 0 && w.others > 0 && <>Not listed: {w.others} line{w.others === 1 ? "" : "s"} writing other constants. </>}
+        Click a value to see why it was that. When you ask the AI to fix something, this goes with it.
+      </p>
+    </div>
+  );
+}
+
 /** "Why is it this value?": each value asked about, explained; the values an explanation mentions
  * can be asked about in turn, building a chain back to the cause. */
 function WhyPanel({ trail, onAsk, onClose }: { trail: Because[]; onAsk: (r: Read, level: number) => void; onClose: () => void }) {

apps/web/src/lib/why.ts

@@ -2,7 +2,7 @@
 // that led to it and the values it read, from the simulator's record of what wrote each change.
 // Port connections are followed through; the example's own inputs end the chain.
 
-import { NO_ORIGIN, type SimSignal } from "../sim/protocol";
+import { NO_ORIGIN, type SimOrigin, type SimSignal } from "../sim/protocol";
 import type { Replay } from "./replay";
 
 /** A value an explanation mentions, which can be explained in turn. */
@@ -127,3 +127,170 @@ export function whyText(b: Because): string {
     "."
   );
 }
+
+/** A line of the design that could have given a signal the value an example expected, and what
+ * happened to it the last time it could have run. */
+export interface Candidate {
+  file: string;
+  line: number;
+  code: string;
+  /** `blocked`: a decision went another way; `ran`: it ran (and computed something else, or a
+   * later line changed the value again); `never`: nothing had decided it yet. */
+  verdict: "blocked" | "ran" | "never";
+  /** The cycle that last decision is seen at. */
+  cycle?: number;
+  /** The decisions that went its way, outermost first, then (for `blocked`) the one that didn't. */
+  passed: Because["conds"];
+  blocked?: Because["conds"][number];
+  /** For `ran`: the values it computed from. */
+  reads: Read[];
+}
+
+export interface WhyNot {
+  name: string;
+  full: string;
+  expected: string;
+  actual: string;
+  cycle: number;
+  /** The lines that write the signal and could have written the expected value. */
+  candidates: Candidate[];
+  /** Lines that write the signal but only another constant (`valid <= 0` when 1 was expected). */
+  others: number;
+  /** The run made too many decisions to record them all, and this moment is past the record. */
+  unrecorded: boolean;
+}
+
+/** A Verilog number (`1`, `4'hf`, `8'd12`), or null for anything else. */
+function constant(s: string): bigint | null {
+  const v = s.trim().replace(/_/g, "").toLowerCase();
+  try {
+    const m = v.match(/^\d*'s?([hbod])([0-9a-f]+)$/);
+    if (m) return BigInt({ h: "0x", b: "0b", o: "0o", d: "" }[m[1] as "h"] + m[2]);
+    return /^\d+$/.test(v) ? BigInt(v) : null;
+  } catch {
+    return null;
+  }
+}
+
+/** Why `full` wasn't `expected` at a cycle's check point: each line that could have written that
+ * value, and the decision that kept it from running (or that it did run). */
+export function whyNot(rep: Replay, full: string, cycle: number, expected: bigint): WhyNot | null {
+  const t = rep.at(cycle);
+  if (t === undefined) return null;
+  const signals = new Map(rep.sim.signals.map((s) => [s.name, s]));
+  const name = (f: string) => {
+    const rest = f.startsWith(`${rep.top}.`) ? f.slice(rep.top.length + 1) : f;
+    return rest.startsWith("dut.") ? rest.slice(4) : rest;
+  };
+  const read = (f: string, at: number): Read => {
+    const s = signals.get(f);
+    const i = s ? changeAt(s, at) : -1;
+    return { name: name(f), full: f, value: s && i >= 0 ? s.values[i].toString() : "?", t: s && i >= 0 ? at : null };
+  };
+  // Through port connections to the signal the design writes.
+  let cur = full;
+  for (let hop = 0; hop < 32; hop++) {
+    const s = signals.get(cur);
+    const i = s ? changeAt(s, t) : -1;
+    if (!s || i < 0 || s.by[i] === NO_ORIGIN) break;
+    const o = rep.sim.origins[s.by[i]];
+    if (!/^(input|output) \S+ of /.test(o.text) || o.reads.length !== 1) break;
+    cur = o.reads[0];
+  }
+  const shown = signals.get(full);
+  const at = shown ? changeAt(shown, t) : -1;
+  const out: WhyNot = {
+    name: name(full),
+    full,
+    expected: expected.toString(),
+    actual: shown && at >= 0 ? shown.values[at].toString() : "?",
+    cycle,
+    candidates: [],
+    others: 0,
+    unrecorded: false,
+  };
+
+  const { times, branches, decisionOf, cut } = rep.sim.decisions;
+  out.unrecorded = cut && (times.length === 0 || times[times.length - 1] < t);
+  // The last decision `d` made at or before `t` (or, from `from` on, at time `at` exactly).
+  const lastBefore = (d: number) => {
+    let lo = 0;
+    let hi = times.length - 1;
+    let end = -1;
+    while (lo <= hi) {
+      const mid = (lo + hi) >> 1;
+      if (times[mid] <= t) {
+        end = mid;
+        lo = mid + 1;
+      } else hi = mid - 1;
+    }
+    for (let j = end; j >= 0; j--) if (decisionOf[branches[j]] === d) return j;
+    return -1;
+  };
+  const nextAt = (d: number, from: number) => {
+    for (let j = from + 1; j < times.length && times[j] === times[from]; j++) if (decisionOf[branches[j]] === d) return j;
+    return -1;
+  };
+
+  const seen = new Set<string>();
+  for (const o of rep.sim.origins) {
+    if (o.writes !== cur || /^(input|output) \S+ of /.test(o.text) || o.text.endsWith("(its starting value)")) continue;
+    const key = `${o.file}:${o.line}:${o.text}`;
+    if (seen.has(key)) continue;
+    seen.add(key);
+    // A line that only ever writes another constant can't be the one.
+    const m = o.text.match(/<?=/);
+    const k = m?.index === undefined ? null : constant(o.text.slice(m.index + m[0].length));
+    if (k !== null && k !== expected) {
+      out.others++;
+      continue;
+    }
+    const nb = nonblocking(o.text);
+    const valuesAt = (tm: number) => (nb ? Math.max(0, tm - 1) : tm);
+    const cond = (c: SimOrigin["conds"][number], tm: number) => ({ line: c.line, code: c.text, taken: c.taken, reads: c.reads.map((r) => read(r, valuesAt(tm))) });
+    const cand: Candidate = { file: o.file, line: o.line, code: o.text, verdict: "ran", passed: [], reads: [] };
+    if (o.conds.length) {
+      let j = lastBefore(decisionOf[o.conds[0].point]);
+      if (j < 0) cand.verdict = "never";
+      else {
+        cand.cycle = rep.cycleAt(times[j]);
+        for (let n = 0; n < o.conds.length; n++) {
+          const c = o.conds[n];
+          if (n > 0) j = nextAt(decisionOf[c.point], j);
+          if (j < 0) break;
+          if (branches[j] !== c.point) {
+            cand.verdict = "blocked";
+            cand.blocked = cond(c, times[j]);
+            break;
+          }
+          cand.passed.push(cond(c, times[j]));
+        }
+        if (cand.verdict === "ran") cand.reads = o.reads.map((r) => read(r, valuesAt(times[j])));
+      }
+    } else cand.reads = o.reads.map((r) => read(r, valuesAt(t)));
+    out.candidates.push(cand);
+  }
+  return out;
+}
+
+/** How a decision went the other way: "`state == STOP` was false", "`state` wasn't DONE". */
+export function missed(c: Because["conds"][number]): string {
+  if (c.taken === "true") return `\`${c.code}\` was false`;
+  if (c.taken === "false") return `\`${c.code}\` was true`;
+  if (c.taken === "default") return `\`${c.code}\` matched one of the listed values`;
+  return `\`${c.code}\` wasn't ${c.taken.replace(/, /g, " or ")}`;
+}
+
+/** The answer as one paragraph, for the AI. */
+export function whyNotText(w: WhyNot): string {
+  const vals = (rs: Read[]) => rs.map((r) => `${r.name} = ${r.value}`).join(", ");
+  const parts = w.candidates.map((c) => {
+    const head = `line ${c.line} of ${c.file}, \`${c.code}\``;
+    if (c.verdict === "never") return `${head}, had never run by then`;
+    if (c.verdict === "blocked" && c.blocked)
+      return `${head}, didn't run for cycle ${c.cycle} because ${missed(c.blocked)}${c.blocked.reads.length ? ` (${vals(c.blocked.reads)})` : ""}`;
+    return `${head}, ran${c.cycle !== undefined ? ` for cycle ${c.cycle}` : ""}${c.reads.length ? ` with ${vals(c.reads)}` : ""}`;
+  });
+  const head = `Why ${w.name} wasn't ${w.expected} at cycle ${w.cycle} (it was ${w.actual}):`;
+  return parts.length ? `${head} ${parts.join("; ")}.` : `${head} no line of the design writes ${w.expected} to it.`;
+}

apps/web/src/sim/protocol.ts

@@ -22,8 +22,22 @@ export interface SimOrigin {
   text: string;
   /** Full names of the signals its value is computed from. */
   reads: string[];
-  /** The `if`/`case` decisions it is inside, outermost first, and which way each went. */
-  conds: { line: number; text: string; reads: string[]; taken: string }[];
+  /** Full name of the signal it writes; empty when it writes several. */
+  writes: string;
+  /** The `if`/`case` decisions it is inside, outermost first, and which way each went for it to
+   * run; `point` is that branch, as in `SimDecisions`. */
+  conds: { line: number; text: string; reads: string[]; taken: string; point: number }[];
+}
+
+/** Every `if`/`case` decision the run made, in order. */
+export interface SimDecisions {
+  times: Float64Array;
+  /** The branch taken (a coverage point number). */
+  branches: Uint32Array;
+  /** For each coverage point: the decision it is a branch of, or NO_ORIGIN for one that isn't. */
+  decisionOf: Uint32Array;
+  /** The run made more decisions than were recorded (only the first are). */
+  cut: boolean;
 }
 
 export interface SimResult {
@@ -37,6 +51,7 @@ export interface SimResult {
   ms: number;
   signals: SimSignal[];
   origins: SimOrigin[];
+  decisions: SimDecisions;
 }
 
 export type SimRequest =

apps/web/src/sim/worker.ts

@@ -22,6 +22,12 @@ self.onmessage = async (ev: MessageEvent<SimRequest>) => {
       const ms = performance.now() - t0;
       const names = last.names;
       const widths = last.widths;
+      const decisions = {
+        times: last.decisionTimes(),
+        branches: last.decisionBranches(),
+        decisionOf: last.branchDecision(),
+        cut: last.decisionsCut,
+      };
       const signals = names.map((name, i) => ({ name, width: widths[i], times: last!.times(i), values: last!.values(i), by: last!.by(i) }));
       reply(
         {
@@ -38,9 +44,10 @@ self.onmessage = async (ev: MessageEvent<SimRequest>) => {
             ms,
             signals,
             origins: JSON.parse(last.origins()),
+            decisions,
           },
         },
-        signals.flatMap((s) => [s.times.buffer, s.values.buffer, s.by.buffer]),
+        [...signals.flatMap((s) => [s.times.buffer, s.values.buffer, s.by.buffer]), decisions.times.buffer, decisions.branches.buffer],
       );
     }
   } catch (e) {