fix(forecast): let the panel publish a result that is bad for the product

hooks-core/calibration.mjs

@@ -29,7 +29,20 @@
  * that cannot support one.
  */
 
-import { record, readMetrics } from './metrics.mjs';
+// readBalance, not readMetrics. Both readers used the windowed firehose, which returns at most
+// 2,000,000 bytes and the last 5,000 lines -- and forecast events were not among the protected
+// kinds, so they lived in a log dominated by per-tool-call records (metrics.mjs's own comment
+// cites 4,735 capture events inside one window).
+//
+// Two failures followed. In observeOutcome, on a busy project the forecast had scrolled past the
+// tail by the time compaction fired, so the outcome was discarded with no record, no error and no
+// counter. In reliability, outcomes vanished faster than they accumulated, so rows.length could
+// never reach MIN_SCORED and calibrate returned 'not yet calibrated (n/8)' permanently -- leaving
+// a user unable to distinguish "never had data" from "the data is being thrown away".
+//
+// 'forecast' and 'forecast-outcome' are now balance kinds, so they are written to balance.jsonl
+// as well and read back unwindowed.
+import { record, readBalance } from './metrics.mjs';
 
 /** Horizon buckets, in turns. Error behaves differently across these ranges. */
 const HORIZONS = [
@@ -51,8 +64,15 @@ function horizonOf(turns) {
   return HORIZONS.find((h) => turns <= h.max).name;
 }
 
-/** Records a prediction so it can be scored when the outcome arrives. */
-export function logForecast(dir, { sessionId, predictedTurns, used, capacity }) {
+/**
+ * Records a prediction so it can be scored when the outcome arrives.
+ *
+ * `turns` is the session's turn count AT PREDICTION TIME, and it is what makes the outcome
+ * computable at all: predictedTurns means "turns from here until compaction", so the ground truth
+ * is the turns that ELAPSED, not the turn number compaction happened on. Without it the caller at
+ * compaction has nothing to subtract from and would have to score an interval against an absolute.
+ */
+export function logForecast(dir, { sessionId, predictedTurns, used, capacity, turns }) {
   if (!Number.isFinite(predictedTurns)) return;
   record(dir, {
     kind: 'forecast',
@@ -61,6 +81,7 @@ export function logForecast(dir, { sessionId, predictedTurns, used, capacity })
     horizon: horizonOf(predictedTurns),
     used,
     capacity,
+    turns: Number.isFinite(turns) ? turns : null,
   });
 }
 
@@ -71,21 +92,74 @@ export function logForecast(dir, { sessionId, predictedTurns, used, capacity })
  * Called when compaction fires: the number of turns that elapsed since the
  * prediction is the ground truth it was predicting.
  */
-export function observeOutcome(dir, { sessionId, actualTurns }) {
-  if (!Number.isFinite(actualTurns)) return;
-  const open = readMetrics(dir)
-    .filter((e) => e.kind === 'forecast' && e.sessionId === sessionId && !e.scored)
+export function observeOutcome(dir, { sessionId, actualTurns, atTurn }) {
+  // EITHER THE ELAPSED COUNT OR THE TURN IT HAPPENED ON.
+  //
+  // The only real caller -- the PreCompact hook -- knows what turn compaction fired on, not how
+  // many turns have passed since a prediction it did not make. Deriving the interval here keeps
+  // that subtraction next to the record that defines it, rather than making every caller
+  // re-discover which forecast is open in order to compute its own ground truth.
+  const elapsed = Number.isFinite(actualTurns)
+    ? actualTurns
+    : null;
+  if (elapsed === null && !Number.isFinite(atTurn)) return;
+
+  // CLOSURE IS DERIVED FROM THE OUTCOME ROWS, not from a flag on the forecast.
+  //
+  // The old filter was `!e.scored`, but nothing ever writes `scored`: logForecast records
+  // kind/sessionId/predictedTurns/horizon/used/capacity, and record() is an append-only JSONL
+  // writer with no update path, so a forecast can never gain the field. The predicate was always
+  // true, and the last forecast for a session was therefore re-closable without limit.
+  //
+  // A session where compaction fires twice with no intervening logForecast popped the SAME
+  // forecast twice, producing two outcome rows for one prediction. reliability counts ROWS, not
+  // predictions, so the hit rate and the bias mean both inflate and MIN_SCORED can be satisfied
+  // by a single prediction re-closed eight times -- precisely the "threshold from a single
+  // sample" this module exists to prevent. The existing test used a fresh sessionId per
+  // iteration, so it never exposed it.
+  const events = readBalance(dir);
+  const closed = new Set(
+    events.filter((e) => e.kind === 'forecast-outcome' && e.forecastId).map((e) => e.forecastId),
+  );
+  // ID-BEARING ROWS ONLY, AND NEWEST BY TIME RATHER THAN BY POSITION.
+  //
+  // Two separate hazards, both from readBalance's shape. It returns balance-log rows followed by
+  // the legacy firehose rows it migrates, unsorted -- so `.pop()` returns the last row in the
+  // array, which is not the newest forecast. And legacy rows predate `id`, so `closed.has(e.id)`
+  // is `closed.has(undefined)`, always false: such a forecast can be closed again and again, and
+  // each outcome stores `forecastId: undefined`, which JSON.stringify drops entirely. That is the
+  // repeat-scoring defect this function was written to fix, reappearing through the back door.
+  const open = events
+    .filter((e) => e.kind === 'forecast' && e.sessionId === sessionId && e.id && !closed.has(e.id))
+    .sort((a, b) => (a.at ?? 0) - (b.at ?? 0))
     .pop();
   if (!open) return;
 
+  // Derived only once the open forecast is known, since the interval is measured from ITS turn.
+  // A forecast recorded before `turns` existed has none, and there is nothing to subtract from --
+  // scoring it against an absolute turn number would manufacture an error rather than measure one,
+  // so it is left open instead.
+  const actual = elapsed !== null ? elapsed
+    : (Number.isFinite(open.turns) ? atTurn - open.turns : null);
+  // AT LEAST ONE TURN MUST HAVE PASSED. `atTurn` is the transcript's current turn count and
+  // `open.turns` the count at prediction time, so a truncated, rotated or replaced transcript
+  // makes the difference zero or negative. Recording that as ground truth feeds a large negative
+  // `error` into `reliability`, which shifts `bucket.bias`, which is what `calibrate` publishes a
+  // corrected forecast from -- so one bad transcript would skew every later prediction. Leaving
+  // the forecast open loses nothing: the next compaction closes it against a sane count.
+  if (actual === null || !Number.isFinite(actual) || actual < 1) return;
+
   record(dir, {
     kind: 'forecast-outcome',
     sessionId,
+    // Stamped so the next call can see this forecast is spoken for. Without it the set above is
+    // empty and the same defect returns.
+    forecastId: open.id,
     horizon: open.horizon,
     predictedTurns: open.predictedTurns,
-    actualTurns,
-    error: actualTurns - open.predictedTurns,
-    hit: Math.abs(actualTurns - open.predictedTurns) <= TOLERANCE,
+    actualTurns: actual,
+    error: actual - open.predictedTurns,
+    hit: Math.abs(actual - open.predictedTurns) <= TOLERANCE,
   });
 }
 
@@ -96,7 +170,7 @@ export function observeOutcome(dir, { sessionId, actualTurns }) {
  * actionable, where "62% accurate" is not.
  */
 export function reliability(dir) {
-  const outcomes = readMetrics(dir).filter((e) => e.kind === 'forecast-outcome');
+  const outcomes = readBalance(dir).filter((e) => e.kind === 'forecast-outcome');
   const buckets = {};
 
   for (const horizon of HORIZONS) {
@@ -162,7 +236,28 @@ export function calibrate(dir, predictedTurns) {
 
   // Refit: shift by the observed bias for this horizon. A bucket that runs long
   // gets pulled in, and vice versa.
-  const adjusted = Math.max(0, Math.round(predictedTurns + bucket.bias));
+  //
+  // NOT CLAMPED WITH Math.max, which converted nonsense into a plausible-looking emergency. bias
+  // is a plain arithmetic mean of signed error, and a mean is not robust to the shape the
+  // 50%-hit-rate floor admits: a near bucket with errors [0,0,0,0,-30,-30,-30,-30] has four hits,
+  // so it clears the floor and is called calibrated, and its bias is -15. A raw forecast of 3
+  // turns then became Math.max(0, Math.round(3 - 15)) = 0 and was published as fact, with the
+  // note "within 3 on 50% of 8 past forecasts" -- a zero-turn deadline the forecaster never
+  // produced, from a bucket that is wrong half the time.
+  //
+  // A correction at least as large as the thing it corrects is not a correction.
+  const adjusted = Math.round(predictedTurns + bucket.bias);
+  if (Math.abs(bucket.bias) >= predictedTurns || adjusted < 1) {
+    return {
+      publishable: false,
+      predictedTurns,
+      horizon,
+      bias: bucket.bias,
+      reason: `the observed bias at this horizon (${Math.round(bucket.bias)} turns) is as large as ` +
+        `the forecast itself (${predictedTurns}), so the correction would replace it rather than ` +
+        'adjust it',
+    };
+  }
 
   return {
     publishable: true,