fix(calibration): run the loop, give it an entry point, and stop the guard reading prose as a call

hooks-core/calibration.mjs

@@ -29,7 +29,20 @@
  * that cannot support one.
  */
 
-import { record, readMetrics } from './metrics.mjs';
+// readBalance, not readMetrics. Both readers used the windowed firehose, which returns at most
+// 2,000,000 bytes and the last 5,000 lines -- and forecast events were not among the protected
+// kinds, so they lived in a log dominated by per-tool-call records (metrics.mjs's own comment
+// cites 4,735 capture events inside one window).
+//
+// Two failures followed. In observeOutcome, on a busy project the forecast had scrolled past the
+// tail by the time compaction fired, so the outcome was discarded with no record, no error and no
+// counter. In reliability, outcomes vanished faster than they accumulated, so rows.length could
+// never reach MIN_SCORED and calibrate returned 'not yet calibrated (n/8)' permanently -- leaving
+// a user unable to distinguish "never had data" from "the data is being thrown away".
+//
+// 'forecast' and 'forecast-outcome' are now balance kinds, so they are written to balance.jsonl
+// as well and read back unwindowed.
+import { record, readBalance } from './metrics.mjs';
 
 /** Horizon buckets, in turns. Error behaves differently across these ranges. */
 const HORIZONS = [
@@ -51,8 +64,15 @@ function horizonOf(turns) {
   return HORIZONS.find((h) => turns <= h.max).name;
 }
 
-/** Records a prediction so it can be scored when the outcome arrives. */
-export function logForecast(dir, { sessionId, predictedTurns, used, capacity }) {
+/**
+ * Records a prediction so it can be scored when the outcome arrives.
+ *
+ * `turns` is the session's turn count AT PREDICTION TIME, and it is what makes the outcome
+ * computable at all: predictedTurns means "turns from here until compaction", so the ground truth
+ * is the turns that ELAPSED, not the turn number compaction happened on. Without it the caller at
+ * compaction has nothing to subtract from and would have to score an interval against an absolute.
+ */
+export function logForecast(dir, { sessionId, predictedTurns, used, capacity, turns }) {
   if (!Number.isFinite(predictedTurns)) return;
   record(dir, {
     kind: 'forecast',
@@ -61,6 +81,7 @@ export function logForecast(dir, { sessionId, predictedTurns, used, capacity })
     horizon: horizonOf(predictedTurns),
     used,
     capacity,
+    turns: Number.isFinite(turns) ? turns : null,
   });
 }
 
@@ -71,21 +92,59 @@ export function logForecast(dir, { sessionId, predictedTurns, used, capacity })
  * Called when compaction fires: the number of turns that elapsed since the
  * prediction is the ground truth it was predicting.
  */
-export function observeOutcome(dir, { sessionId, actualTurns }) {
-  if (!Number.isFinite(actualTurns)) return;
-  const open = readMetrics(dir)
-    .filter((e) => e.kind === 'forecast' && e.sessionId === sessionId && !e.scored)
+export function observeOutcome(dir, { sessionId, actualTurns, atTurn }) {
+  // EITHER THE ELAPSED COUNT OR THE TURN IT HAPPENED ON.
+  //
+  // The only real caller -- the PreCompact hook -- knows what turn compaction fired on, not how
+  // many turns have passed since a prediction it did not make. Deriving the interval here keeps
+  // that subtraction next to the record that defines it, rather than making every caller
+  // re-discover which forecast is open in order to compute its own ground truth.
+  const elapsed = Number.isFinite(actualTurns)
+    ? actualTurns
+    : null;
+  if (elapsed === null && !Number.isFinite(atTurn)) return;
+
+  // CLOSURE IS DERIVED FROM THE OUTCOME ROWS, not from a flag on the forecast.
+  //
+  // The old filter was `!e.scored`, but nothing ever writes `scored`: logForecast records
+  // kind/sessionId/predictedTurns/horizon/used/capacity, and record() is an append-only JSONL
+  // writer with no update path, so a forecast can never gain the field. The predicate was always
+  // true, and the last forecast for a session was therefore re-closable without limit.
+  //
+  // A session where compaction fires twice with no intervening logForecast popped the SAME
+  // forecast twice, producing two outcome rows for one prediction. reliability counts ROWS, not
+  // predictions, so the hit rate and the bias mean both inflate and MIN_SCORED can be satisfied
+  // by a single prediction re-closed eight times -- precisely the "threshold from a single
+  // sample" this module exists to prevent. The existing test used a fresh sessionId per
+  // iteration, so it never exposed it.
+  const events = readBalance(dir);
+  const closed = new Set(
+    events.filter((e) => e.kind === 'forecast-outcome' && e.forecastId).map((e) => e.forecastId),
+  );
+  const open = events
+    .filter((e) => e.kind === 'forecast' && e.sessionId === sessionId && !closed.has(e.id))
     .pop();
   if (!open) return;
 
+  // Derived only once the open forecast is known, since the interval is measured from ITS turn.
+  // A forecast recorded before `turns` existed has none, and there is nothing to subtract from --
+  // scoring it against an absolute turn number would manufacture an error rather than measure one,
+  // so it is left open instead.
+  const actual = elapsed !== null ? elapsed
+    : (Number.isFinite(open.turns) ? atTurn - open.turns : null);
+  if (actual === null || !Number.isFinite(actual)) return;
+
   record(dir, {
     kind: 'forecast-outcome',
     sessionId,
+    // Stamped so the next call can see this forecast is spoken for. Without it the set above is
+    // empty and the same defect returns.
+    forecastId: open.id,
     horizon: open.horizon,
     predictedTurns: open.predictedTurns,
-    actualTurns,
-    error: actualTurns - open.predictedTurns,
-    hit: Math.abs(actualTurns - open.predictedTurns) <= TOLERANCE,
+    actualTurns: actual,
+    error: actual - open.predictedTurns,
+    hit: Math.abs(actual - open.predictedTurns) <= TOLERANCE,
   });
 }
 
@@ -96,7 +155,7 @@ export function observeOutcome(dir, { sessionId, actualTurns }) {
  * actionable, where "62% accurate" is not.
  */
 export function reliability(dir) {
-  const outcomes = readMetrics(dir).filter((e) => e.kind === 'forecast-outcome');
+  const outcomes = readBalance(dir).filter((e) => e.kind === 'forecast-outcome');
   const buckets = {};
 
   for (const horizon of HORIZONS) {
@@ -162,7 +221,28 @@ export function calibrate(dir, predictedTurns) {
 
   // Refit: shift by the observed bias for this horizon. A bucket that runs long
   // gets pulled in, and vice versa.
-  const adjusted = Math.max(0, Math.round(predictedTurns + bucket.bias));
+  //
+  // NOT CLAMPED WITH Math.max, which converted nonsense into a plausible-looking emergency. bias
+  // is a plain arithmetic mean of signed error, and a mean is not robust to the shape the
+  // 50%-hit-rate floor admits: a near bucket with errors [0,0,0,0,-30,-30,-30,-30] has four hits,
+  // so it clears the floor and is called calibrated, and its bias is -15. A raw forecast of 3
+  // turns then became Math.max(0, Math.round(3 - 15)) = 0 and was published as fact, with the
+  // note "within 3 on 50% of 8 past forecasts" -- a zero-turn deadline the forecaster never
+  // produced, from a bucket that is wrong half the time.
+  //
+  // A correction at least as large as the thing it corrects is not a correction.
+  const adjusted = Math.round(predictedTurns + bucket.bias);
+  if (Math.abs(bucket.bias) >= predictedTurns || adjusted < 1) {
+    return {
+      publishable: false,
+      predictedTurns,
+      horizon,
+      bias: bucket.bias,
+      reason: `the observed bias at this horizon (${Math.round(bucket.bias)} turns) is as large as ` +
+        `the forecast itself (${predictedTurns}), so the correction would replace it rather than ` +
+        'adjust it',
+    };
+  }
 
   return {
     publishable: true,

hooks-core/forecast.mjs

@@ -32,6 +32,7 @@
 import { readMetrics, readBalance, isFixtureAnchor, BALANCE_KINDS } from './metrics.mjs';
 import { aggregateConsolidation } from './consolidate.mjs';
 import { previewQuality } from './expand.mjs';
+import { logForecast, calibrate } from './calibration.mjs';
 
 /** Turns of headroom below which the runway is worth interrupting for. */
 export const ACTIONABLE_RUNWAY = 8;
@@ -310,7 +311,33 @@ export function forecastPanel(dir, session = {}, findings = []) {
 
   if (air) {
     parts.runway = air;
-    lines.push(runwayLine(air));
+
+    // THE FORECAST KEEPS ITS OWN SCORE -- from here, because this is the only place a forecast is
+    // actually made. calibration.mjs was written for exactly this and had zero shipped importers:
+    // no forecast was ever logged, no outcome ever observed, reliability always saw an empty set,
+    // and calibrate was unreachable. So the shipped panel printed precisely the uncalibrated
+    // number that module's docstring calls "a vibe with a typeface".
+    //
+    // Logged BEFORE it is corrected. The raw prediction is what gets scored against the outcome;
+    // scoring the corrected one would fold the correction back into the next correction and the
+    // bias would chase its own tail.
+    logForecast(dir, {
+      sessionId: session.sessionId,
+      predictedTurns: air.withGraph,
+      used: session.used,
+      capacity: session.capacity,
+      // The turn this was predicted FROM. predictedTurns is an interval, so without its origin
+      // the outcome at compaction has nothing to subtract from.
+      turns: session.turns,
+    });
+
+    const score = calibrate(dir, air.withGraph);
+    parts.calibration = score;
+    // Published only when the horizon has earned it. An unpublishable score does not replace the
+    // number -- it is simply absent, exactly as the uncalibrated case reads today.
+    lines.push(score.publishable
+      ? `${runwayLine({ ...air, withGraph: score.predictedTurns })}\n  (corrected: ${score.note})`
+      : runwayLine(air));
   }
 
   if (rate) {

hooks-core/metrics.mjs

@@ -66,7 +66,17 @@ const balancePath = (dir) => join(dir, 'balance.jsonl');
  * starve the measurement. Exported because any reader that needs the unwindowed truth needs to
  * know which kinds it applies to, and a second copy of this set elsewhere would drift.
  */
-export const BALANCE_KINDS = new Set(['inject', 'harvest', 'substitute']);
+export const BALANCE_KINDS = new Set([
+  'inject',
+  'harvest',
+  'substitute',
+  // The forecast's own score. Both kinds are rare and accumulate slowly -- MIN_SCORED is 8 --
+  // while the firehose they shared is dominated by per-tool-call records, so they were evicted
+  // faster than they could accumulate: calibrate() returned 'not yet calibrated (n/8)' forever
+  // and an outcome whose forecast had scrolled away was dropped with no record at all.
+  'forecast',
+  'forecast-outcome',
+]);
 
 /**
  * Is this touch in the holdout arm?

hooks-core/policy.mjs

@@ -270,7 +270,7 @@ function statePath(sessionId, agent) {
 
 /** A usable state object, whatever was on disk. */
 function emptyState() {
-  return { seen: {}, denied: {}, injected: [], actCounts: {} };
+  return { seen: {}, denied: {}, injected: [], actCounts: {}, forecast: null };
 }
 
 /**
@@ -310,6 +310,16 @@ export function loadState(sessionId, agent) {
         parsed.actCounts && typeof parsed.actCounts === 'object' && !Array.isArray(parsed.actCounts)
           ? parsed.actCounts
           : {},
+      // THE SAME REASON A THIRD TIME. The forecast throttle records when the panel was last
+      // computed and what runway it last SHOWED, and both are meaningless within a single hook
+      // process -- every tool call is a new one. Dropped here, `checkedAt` would reset on every
+      // call and the panel would rebuild itself, transcript parse and all, on each tool use; and
+      // `shown` would reset, so worthSurfacing would compare against nothing and re-interrupt with
+      // the same runway forever.
+      forecast:
+        parsed.forecast && typeof parsed.forecast === 'object' && !Array.isArray(parsed.forecast)
+          ? parsed.forecast
+          : null,
     };
   } catch {
     return emptyState();
@@ -381,6 +391,18 @@ export function saveState(sessionId, state, agent) {
         }
         return out;
       })(),
+      // LATEST CHECK WINS, which is the opposite direction from the fields above and correct for
+      // this one. `seen`, `denied`, `injected` and `actCounts` are all append-only, so a union or
+      // a max is the safe merge. The forecast throttle is a POINT IN TIME: taking the older of two
+      // concurrent checks would re-open the window and let the panel be rebuilt immediately, which
+      // is the cost the throttle exists to bound.
+      forecast: (() => {
+        const mine = state.forecast || null;
+        const theirs = current.forecast || null;
+        if (!mine) return theirs;
+        if (!theirs) return mine;
+        return (mine.checkedAt || 0) >= (theirs.checkedAt || 0) ? mine : theirs;
+      })(),
     };
 
     // Write-then-rename so a reader never observes a half-written file.