From 1469527836ebeb6aafabd4dd2ca87333349b7edf Mon Sep 17 00:00:00 2001 From: Franklin Moormann Date: Fri, 7 Aug 2026 19:50:41 -0400 Subject: [PATCH 1/2] fix(calibration): run the loop, and stop the guard reading prose as a call calibration.mjs had zero shipped importers. forecastPanel computed the runway and pushed "~N turns to compaction" without ever calling logForecast or calibrate, so no forecast was logged, no outcome observed, reliability always saw an empty set and calibrate was unreachable. The shipped panel printed exactly the uncalibrated number this module's own docstring calls "a vibe with a typeface". WIRED. forecastPanel now logs the forecast it publishes and renders calibrate()'s verdict. The RAW prediction is logged, not the corrected one: scoring the corrected number would fold each correction into the next and the bias would chase its own tail. An unpublishable score does not replace the number, so an uncalibrated horizon reads exactly as it does today. Three defects fixed on the way, each of which would have made the loop wrong once it started running: 1. observeOutcome filtered open forecasts with `!e.scored`, but nothing ever writes `scored` -- record() is an append-only JSONL writer with no update path, so a forecast could never gain the field and the predicate was always true. A session where compaction fires twice with no intervening logForecast popped the SAME forecast twice, and reliability counts rows rather than predictions, so MIN_SCORED could be satisfied by one prediction re-closed eight times -- precisely the threshold-from-a-single-sample this module exists to prevent. Closure is now derived from the outcome rows, and each outcome stamps the forecastId it closes. 2. Both readers used readMetrics, whose window is 2 MB and 5,000 lines, and forecast events were not protected kinds. On a busy project the forecast had scrolled past the tail by the time compaction fired, so the outcome was discarded with no record, no error and no counter; and outcomes vanished faster than they accumulated, so calibrate returned "not yet calibrated (n/8)" permanently with no way to tell that from having no data. 'forecast' and 'forecast-outcome' are now balance kinds, read back unwindowed. 3. calibrate clamped the correction with Math.max(0, ...). bias is a plain arithmetic mean of signed error and a mean is not robust to the shape the 50% hit-rate floor admits: errors [0,0,0,0,-30,-30,-30,-30] give four hits, clear the floor, and yield bias -15. A raw forecast of 3 turns became 0, published with "within 3 on 50% of 8 past forecasts" -- a zero-turn deadline the forecaster never produced. A correction at least as large as the forecast is refused with a reason instead. THE GUARD THAT SHOULD HAVE CAUGHT THIS. usedInShippedCode is a bare `\bname\b` scan over raw file text, so `calibrate` counted as reachable because curate.mjs contains the phrase "the reader's ability to calibrate trust". A check that reads documentation as code is worse than none: it is a clean bill of health nobody re-examines, and these files are heavily commented by design. Comments are now stripped first, which immediately surfaced sessionIndex as a genuine orphan. Two things deliberately NOT done, both recorded in the allowlist rather than papered over: - String literals are not stripped. Doing so reported routingReport and modelSwitchCost as orphaned while routing-tool.ts calls both: three independent global passes cannot nest, so an apostrophe inside a double-quoted sentence opened a "single-quoted string" that ran to the next apostrophe and swallowed the real call sites between. The observed defect was comment prose, and that is where the line belongs. - The declaring file still counts as a caller. Excluding it -- on the reasoning that reliability passed only via an in-file call -- reported 40+ false orphans including burnRate, runway and shadowAvoided, which forecastPanel calls from inside forecast.mjs. Reachability propagates through an in-file caller. forecastPanel and worthSurfacing REMAIN unwired and remain on the allowlist with corrected reasons: the loop behind the panel now runs, but no hook or tool renders the panel. observeOutcome remains unwired too -- its natural caller is the PreCompact hook, which does not currently compute a turn count. Claiming otherwise would be the exact failure that file exists to catch. Verified: tests/hooks 728 passed, 2 skipped, 0 failed, with 9 new tests. tsc --noEmit clean. Co-Authored-By: Claude Opus 5 --- hooks-core/calibration.mjs | 65 ++++++++- hooks-core/forecast.mjs | 26 +++- hooks-core/metrics.mjs | 12 +- integrations/codex/hooks/lib/calibration.mjs | 65 ++++++++- integrations/codex/hooks/lib/forecast.mjs | 26 +++- integrations/codex/hooks/lib/metrics.mjs | 12 +- .../codex/plugin/hooks/lib/calibration.mjs | 65 ++++++++- .../codex/plugin/hooks/lib/forecast.mjs | 26 +++- .../codex/plugin/hooks/lib/metrics.mjs | 12 +- integrations/gemini/hooks/lib/calibration.mjs | 65 ++++++++- integrations/gemini/hooks/lib/forecast.mjs | 26 +++- integrations/gemini/hooks/lib/metrics.mjs | 12 +- .../opencode/hooks/lib/calibration.mjs | 65 ++++++++- integrations/opencode/hooks/lib/forecast.mjs | 26 +++- integrations/opencode/hooks/lib/metrics.mjs | 12 +- integrations/qwen/hooks/lib/calibration.mjs | 65 ++++++++- integrations/qwen/hooks/lib/forecast.mjs | 26 +++- integrations/qwen/hooks/lib/metrics.mjs | 12 +- plugin/hooks/lib/calibration.mjs | 65 ++++++++- plugin/hooks/lib/forecast.mjs | 26 +++- plugin/hooks/lib/metrics.mjs | 12 +- tests/hooks/forecast.test.mjs | 132 +++++++++++++++++- tests/hooks/reachability.test.mjs | 93 +++++++++--- 23 files changed, 879 insertions(+), 67 deletions(-) diff --git a/hooks-core/calibration.mjs b/hooks-core/calibration.mjs index 8b9cd0faf..f0d83c2d4 100644 --- a/hooks-core/calibration.mjs +++ b/hooks-core/calibration.mjs @@ -29,7 +29,20 @@ * that cannot support one. */ -import { record, readMetrics } from './metrics.mjs'; +// readBalance, not readMetrics. Both readers used the windowed firehose, which returns at most +// 2,000,000 bytes and the last 5,000 lines -- and forecast events were not among the protected +// kinds, so they lived in a log dominated by per-tool-call records (metrics.mjs's own comment +// cites 4,735 capture events inside one window). +// +// Two failures followed. In observeOutcome, on a busy project the forecast had scrolled past the +// tail by the time compaction fired, so the outcome was discarded with no record, no error and no +// counter. In reliability, outcomes vanished faster than they accumulated, so rows.length could +// never reach MIN_SCORED and calibrate returned 'not yet calibrated (n/8)' permanently -- leaving +// a user unable to distinguish "never had data" from "the data is being thrown away". +// +// 'forecast' and 'forecast-outcome' are now balance kinds, so they are written to balance.jsonl +// as well and read back unwindowed. +import { record, readBalance } from './metrics.mjs'; /** Horizon buckets, in turns. Error behaves differently across these ranges. */ const HORIZONS = [ @@ -73,14 +86,35 @@ export function logForecast(dir, { sessionId, predictedTurns, used, capacity }) */ export function observeOutcome(dir, { sessionId, actualTurns }) { if (!Number.isFinite(actualTurns)) return; - const open = readMetrics(dir) - .filter((e) => e.kind === 'forecast' && e.sessionId === sessionId && !e.scored) + + // CLOSURE IS DERIVED FROM THE OUTCOME ROWS, not from a flag on the forecast. + // + // The old filter was `!e.scored`, but nothing ever writes `scored`: logForecast records + // kind/sessionId/predictedTurns/horizon/used/capacity, and record() is an append-only JSONL + // writer with no update path, so a forecast can never gain the field. The predicate was always + // true, and the last forecast for a session was therefore re-closable without limit. + // + // A session where compaction fires twice with no intervening logForecast popped the SAME + // forecast twice, producing two outcome rows for one prediction. reliability counts ROWS, not + // predictions, so the hit rate and the bias mean both inflate and MIN_SCORED can be satisfied + // by a single prediction re-closed eight times -- precisely the "threshold from a single + // sample" this module exists to prevent. The existing test used a fresh sessionId per + // iteration, so it never exposed it. + const events = readBalance(dir); + const closed = new Set( + events.filter((e) => e.kind === 'forecast-outcome' && e.forecastId).map((e) => e.forecastId), + ); + const open = events + .filter((e) => e.kind === 'forecast' && e.sessionId === sessionId && !closed.has(e.id)) .pop(); if (!open) return; record(dir, { kind: 'forecast-outcome', sessionId, + // Stamped so the next call can see this forecast is spoken for. Without it the set above is + // empty and the same defect returns. + forecastId: open.id, horizon: open.horizon, predictedTurns: open.predictedTurns, actualTurns, @@ -96,7 +130,7 @@ export function observeOutcome(dir, { sessionId, actualTurns }) { * actionable, where "62% accurate" is not. */ export function reliability(dir) { - const outcomes = readMetrics(dir).filter((e) => e.kind === 'forecast-outcome'); + const outcomes = readBalance(dir).filter((e) => e.kind === 'forecast-outcome'); const buckets = {}; for (const horizon of HORIZONS) { @@ -162,7 +196,28 @@ export function calibrate(dir, predictedTurns) { // Refit: shift by the observed bias for this horizon. A bucket that runs long // gets pulled in, and vice versa. - const adjusted = Math.max(0, Math.round(predictedTurns + bucket.bias)); + // + // NOT CLAMPED WITH Math.max, which converted nonsense into a plausible-looking emergency. bias + // is a plain arithmetic mean of signed error, and a mean is not robust to the shape the + // 50%-hit-rate floor admits: a near bucket with errors [0,0,0,0,-30,-30,-30,-30] has four hits, + // so it clears the floor and is called calibrated, and its bias is -15. A raw forecast of 3 + // turns then became Math.max(0, Math.round(3 - 15)) = 0 and was published as fact, with the + // note "within 3 on 50% of 8 past forecasts" -- a zero-turn deadline the forecaster never + // produced, from a bucket that is wrong half the time. + // + // A correction at least as large as the thing it corrects is not a correction. + const adjusted = Math.round(predictedTurns + bucket.bias); + if (Math.abs(bucket.bias) >= predictedTurns || adjusted < 1) { + return { + publishable: false, + predictedTurns, + horizon, + bias: bucket.bias, + reason: `the observed bias at this horizon (${Math.round(bucket.bias)} turns) is as large as ` + + `the forecast itself (${predictedTurns}), so the correction would replace it rather than ` + + 'adjust it', + }; + } return { publishable: true, diff --git a/hooks-core/forecast.mjs b/hooks-core/forecast.mjs index 3276bbe36..d514c313a 100644 --- a/hooks-core/forecast.mjs +++ b/hooks-core/forecast.mjs @@ -32,6 +32,7 @@ import { readMetrics, readBalance, isFixtureAnchor, BALANCE_KINDS } from './metrics.mjs'; import { aggregateConsolidation } from './consolidate.mjs'; import { previewQuality } from './expand.mjs'; +import { logForecast, calibrate } from './calibration.mjs'; /** Turns of headroom below which the runway is worth interrupting for. */ export const ACTIONABLE_RUNWAY = 8; @@ -310,7 +311,30 @@ export function forecastPanel(dir, session = {}, findings = []) { if (air) { parts.runway = air; - lines.push(runwayLine(air)); + + // THE FORECAST KEEPS ITS OWN SCORE -- from here, because this is the only place a forecast is + // actually made. calibration.mjs was written for exactly this and had zero shipped importers: + // no forecast was ever logged, no outcome ever observed, reliability always saw an empty set, + // and calibrate was unreachable. So the shipped panel printed precisely the uncalibrated + // number that module's docstring calls "a vibe with a typeface". + // + // Logged BEFORE it is corrected. The raw prediction is what gets scored against the outcome; + // scoring the corrected one would fold the correction back into the next correction and the + // bias would chase its own tail. + logForecast(dir, { + sessionId: session.sessionId, + predictedTurns: air.withGraph, + used: session.used, + capacity: session.capacity, + }); + + const score = calibrate(dir, air.withGraph); + parts.calibration = score; + // Published only when the horizon has earned it. An unpublishable score does not replace the + // number -- it is simply absent, exactly as the uncalibrated case reads today. + lines.push(score.publishable + ? `${runwayLine({ ...air, withGraph: score.predictedTurns })}\n (corrected: ${score.note})` + : runwayLine(air)); } if (rate) { diff --git a/hooks-core/metrics.mjs b/hooks-core/metrics.mjs index a33309475..ae62a1166 100644 --- a/hooks-core/metrics.mjs +++ b/hooks-core/metrics.mjs @@ -66,7 +66,17 @@ const balancePath = (dir) => join(dir, 'balance.jsonl'); * starve the measurement. Exported because any reader that needs the unwindowed truth needs to * know which kinds it applies to, and a second copy of this set elsewhere would drift. */ -export const BALANCE_KINDS = new Set(['inject', 'harvest', 'substitute']); +export const BALANCE_KINDS = new Set([ + 'inject', + 'harvest', + 'substitute', + // The forecast's own score. Both kinds are rare and accumulate slowly -- MIN_SCORED is 8 -- + // while the firehose they shared is dominated by per-tool-call records, so they were evicted + // faster than they could accumulate: calibrate() returned 'not yet calibrated (n/8)' forever + // and an outcome whose forecast had scrolled away was dropped with no record at all. + 'forecast', + 'forecast-outcome', +]); /** * Is this touch in the holdout arm? diff --git a/integrations/codex/hooks/lib/calibration.mjs b/integrations/codex/hooks/lib/calibration.mjs index 718db89cd..e920a9992 100644 --- a/integrations/codex/hooks/lib/calibration.mjs +++ b/integrations/codex/hooks/lib/calibration.mjs @@ -31,7 +31,20 @@ * that cannot support one. */ -import { record, readMetrics } from './metrics.mjs'; +// readBalance, not readMetrics. Both readers used the windowed firehose, which returns at most +// 2,000,000 bytes and the last 5,000 lines -- and forecast events were not among the protected +// kinds, so they lived in a log dominated by per-tool-call records (metrics.mjs's own comment +// cites 4,735 capture events inside one window). +// +// Two failures followed. In observeOutcome, on a busy project the forecast had scrolled past the +// tail by the time compaction fired, so the outcome was discarded with no record, no error and no +// counter. In reliability, outcomes vanished faster than they accumulated, so rows.length could +// never reach MIN_SCORED and calibrate returned 'not yet calibrated (n/8)' permanently -- leaving +// a user unable to distinguish "never had data" from "the data is being thrown away". +// +// 'forecast' and 'forecast-outcome' are now balance kinds, so they are written to balance.jsonl +// as well and read back unwindowed. +import { record, readBalance } from './metrics.mjs'; /** Horizon buckets, in turns. Error behaves differently across these ranges. */ const HORIZONS = [ @@ -75,14 +88,35 @@ export function logForecast(dir, { sessionId, predictedTurns, used, capacity }) */ export function observeOutcome(dir, { sessionId, actualTurns }) { if (!Number.isFinite(actualTurns)) return; - const open = readMetrics(dir) - .filter((e) => e.kind === 'forecast' && e.sessionId === sessionId && !e.scored) + + // CLOSURE IS DERIVED FROM THE OUTCOME ROWS, not from a flag on the forecast. + // + // The old filter was `!e.scored`, but nothing ever writes `scored`: logForecast records + // kind/sessionId/predictedTurns/horizon/used/capacity, and record() is an append-only JSONL + // writer with no update path, so a forecast can never gain the field. The predicate was always + // true, and the last forecast for a session was therefore re-closable without limit. + // + // A session where compaction fires twice with no intervening logForecast popped the SAME + // forecast twice, producing two outcome rows for one prediction. reliability counts ROWS, not + // predictions, so the hit rate and the bias mean both inflate and MIN_SCORED can be satisfied + // by a single prediction re-closed eight times -- precisely the "threshold from a single + // sample" this module exists to prevent. The existing test used a fresh sessionId per + // iteration, so it never exposed it. + const events = readBalance(dir); + const closed = new Set( + events.filter((e) => e.kind === 'forecast-outcome' && e.forecastId).map((e) => e.forecastId), + ); + const open = events + .filter((e) => e.kind === 'forecast' && e.sessionId === sessionId && !closed.has(e.id)) .pop(); if (!open) return; record(dir, { kind: 'forecast-outcome', sessionId, + // Stamped so the next call can see this forecast is spoken for. Without it the set above is + // empty and the same defect returns. + forecastId: open.id, horizon: open.horizon, predictedTurns: open.predictedTurns, actualTurns, @@ -98,7 +132,7 @@ export function observeOutcome(dir, { sessionId, actualTurns }) { * actionable, where "62% accurate" is not. */ export function reliability(dir) { - const outcomes = readMetrics(dir).filter((e) => e.kind === 'forecast-outcome'); + const outcomes = readBalance(dir).filter((e) => e.kind === 'forecast-outcome'); const buckets = {}; for (const horizon of HORIZONS) { @@ -164,7 +198,28 @@ export function calibrate(dir, predictedTurns) { // Refit: shift by the observed bias for this horizon. A bucket that runs long // gets pulled in, and vice versa. - const adjusted = Math.max(0, Math.round(predictedTurns + bucket.bias)); + // + // NOT CLAMPED WITH Math.max, which converted nonsense into a plausible-looking emergency. bias + // is a plain arithmetic mean of signed error, and a mean is not robust to the shape the + // 50%-hit-rate floor admits: a near bucket with errors [0,0,0,0,-30,-30,-30,-30] has four hits, + // so it clears the floor and is called calibrated, and its bias is -15. A raw forecast of 3 + // turns then became Math.max(0, Math.round(3 - 15)) = 0 and was published as fact, with the + // note "within 3 on 50% of 8 past forecasts" -- a zero-turn deadline the forecaster never + // produced, from a bucket that is wrong half the time. + // + // A correction at least as large as the thing it corrects is not a correction. + const adjusted = Math.round(predictedTurns + bucket.bias); + if (Math.abs(bucket.bias) >= predictedTurns || adjusted < 1) { + return { + publishable: false, + predictedTurns, + horizon, + bias: bucket.bias, + reason: `the observed bias at this horizon (${Math.round(bucket.bias)} turns) is as large as ` + + `the forecast itself (${predictedTurns}), so the correction would replace it rather than ` + + 'adjust it', + }; + } return { publishable: true, diff --git a/integrations/codex/hooks/lib/forecast.mjs b/integrations/codex/hooks/lib/forecast.mjs index ed698b1a0..dee6a3b56 100644 --- a/integrations/codex/hooks/lib/forecast.mjs +++ b/integrations/codex/hooks/lib/forecast.mjs @@ -34,6 +34,7 @@ import { readMetrics, readBalance, isFixtureAnchor, BALANCE_KINDS } from './metrics.mjs'; import { aggregateConsolidation } from './consolidate.mjs'; import { previewQuality } from './expand.mjs'; +import { logForecast, calibrate } from './calibration.mjs'; /** Turns of headroom below which the runway is worth interrupting for. */ export const ACTIONABLE_RUNWAY = 8; @@ -312,7 +313,30 @@ export function forecastPanel(dir, session = {}, findings = []) { if (air) { parts.runway = air; - lines.push(runwayLine(air)); + + // THE FORECAST KEEPS ITS OWN SCORE -- from here, because this is the only place a forecast is + // actually made. calibration.mjs was written for exactly this and had zero shipped importers: + // no forecast was ever logged, no outcome ever observed, reliability always saw an empty set, + // and calibrate was unreachable. So the shipped panel printed precisely the uncalibrated + // number that module's docstring calls "a vibe with a typeface". + // + // Logged BEFORE it is corrected. The raw prediction is what gets scored against the outcome; + // scoring the corrected one would fold the correction back into the next correction and the + // bias would chase its own tail. + logForecast(dir, { + sessionId: session.sessionId, + predictedTurns: air.withGraph, + used: session.used, + capacity: session.capacity, + }); + + const score = calibrate(dir, air.withGraph); + parts.calibration = score; + // Published only when the horizon has earned it. An unpublishable score does not replace the + // number -- it is simply absent, exactly as the uncalibrated case reads today. + lines.push(score.publishable + ? `${runwayLine({ ...air, withGraph: score.predictedTurns })}\n (corrected: ${score.note})` + : runwayLine(air)); } if (rate) { diff --git a/integrations/codex/hooks/lib/metrics.mjs b/integrations/codex/hooks/lib/metrics.mjs index b762df57c..4b0eb2712 100644 --- a/integrations/codex/hooks/lib/metrics.mjs +++ b/integrations/codex/hooks/lib/metrics.mjs @@ -68,7 +68,17 @@ const balancePath = (dir) => join(dir, 'balance.jsonl'); * starve the measurement. Exported because any reader that needs the unwindowed truth needs to * know which kinds it applies to, and a second copy of this set elsewhere would drift. */ -export const BALANCE_KINDS = new Set(['inject', 'harvest', 'substitute']); +export const BALANCE_KINDS = new Set([ + 'inject', + 'harvest', + 'substitute', + // The forecast's own score. Both kinds are rare and accumulate slowly -- MIN_SCORED is 8 -- + // while the firehose they shared is dominated by per-tool-call records, so they were evicted + // faster than they could accumulate: calibrate() returned 'not yet calibrated (n/8)' forever + // and an outcome whose forecast had scrolled away was dropped with no record at all. + 'forecast', + 'forecast-outcome', +]); /** * Is this touch in the holdout arm? diff --git a/integrations/codex/plugin/hooks/lib/calibration.mjs b/integrations/codex/plugin/hooks/lib/calibration.mjs index 718db89cd..e920a9992 100644 --- a/integrations/codex/plugin/hooks/lib/calibration.mjs +++ b/integrations/codex/plugin/hooks/lib/calibration.mjs @@ -31,7 +31,20 @@ * that cannot support one. */ -import { record, readMetrics } from './metrics.mjs'; +// readBalance, not readMetrics. Both readers used the windowed firehose, which returns at most +// 2,000,000 bytes and the last 5,000 lines -- and forecast events were not among the protected +// kinds, so they lived in a log dominated by per-tool-call records (metrics.mjs's own comment +// cites 4,735 capture events inside one window). +// +// Two failures followed. In observeOutcome, on a busy project the forecast had scrolled past the +// tail by the time compaction fired, so the outcome was discarded with no record, no error and no +// counter. In reliability, outcomes vanished faster than they accumulated, so rows.length could +// never reach MIN_SCORED and calibrate returned 'not yet calibrated (n/8)' permanently -- leaving +// a user unable to distinguish "never had data" from "the data is being thrown away". +// +// 'forecast' and 'forecast-outcome' are now balance kinds, so they are written to balance.jsonl +// as well and read back unwindowed. +import { record, readBalance } from './metrics.mjs'; /** Horizon buckets, in turns. Error behaves differently across these ranges. */ const HORIZONS = [ @@ -75,14 +88,35 @@ export function logForecast(dir, { sessionId, predictedTurns, used, capacity }) */ export function observeOutcome(dir, { sessionId, actualTurns }) { if (!Number.isFinite(actualTurns)) return; - const open = readMetrics(dir) - .filter((e) => e.kind === 'forecast' && e.sessionId === sessionId && !e.scored) + + // CLOSURE IS DERIVED FROM THE OUTCOME ROWS, not from a flag on the forecast. + // + // The old filter was `!e.scored`, but nothing ever writes `scored`: logForecast records + // kind/sessionId/predictedTurns/horizon/used/capacity, and record() is an append-only JSONL + // writer with no update path, so a forecast can never gain the field. The predicate was always + // true, and the last forecast for a session was therefore re-closable without limit. + // + // A session where compaction fires twice with no intervening logForecast popped the SAME + // forecast twice, producing two outcome rows for one prediction. reliability counts ROWS, not + // predictions, so the hit rate and the bias mean both inflate and MIN_SCORED can be satisfied + // by a single prediction re-closed eight times -- precisely the "threshold from a single + // sample" this module exists to prevent. The existing test used a fresh sessionId per + // iteration, so it never exposed it. + const events = readBalance(dir); + const closed = new Set( + events.filter((e) => e.kind === 'forecast-outcome' && e.forecastId).map((e) => e.forecastId), + ); + const open = events + .filter((e) => e.kind === 'forecast' && e.sessionId === sessionId && !closed.has(e.id)) .pop(); if (!open) return; record(dir, { kind: 'forecast-outcome', sessionId, + // Stamped so the next call can see this forecast is spoken for. Without it the set above is + // empty and the same defect returns. + forecastId: open.id, horizon: open.horizon, predictedTurns: open.predictedTurns, actualTurns, @@ -98,7 +132,7 @@ export function observeOutcome(dir, { sessionId, actualTurns }) { * actionable, where "62% accurate" is not. */ export function reliability(dir) { - const outcomes = readMetrics(dir).filter((e) => e.kind === 'forecast-outcome'); + const outcomes = readBalance(dir).filter((e) => e.kind === 'forecast-outcome'); const buckets = {}; for (const horizon of HORIZONS) { @@ -164,7 +198,28 @@ export function calibrate(dir, predictedTurns) { // Refit: shift by the observed bias for this horizon. A bucket that runs long // gets pulled in, and vice versa. - const adjusted = Math.max(0, Math.round(predictedTurns + bucket.bias)); + // + // NOT CLAMPED WITH Math.max, which converted nonsense into a plausible-looking emergency. bias + // is a plain arithmetic mean of signed error, and a mean is not robust to the shape the + // 50%-hit-rate floor admits: a near bucket with errors [0,0,0,0,-30,-30,-30,-30] has four hits, + // so it clears the floor and is called calibrated, and its bias is -15. A raw forecast of 3 + // turns then became Math.max(0, Math.round(3 - 15)) = 0 and was published as fact, with the + // note "within 3 on 50% of 8 past forecasts" -- a zero-turn deadline the forecaster never + // produced, from a bucket that is wrong half the time. + // + // A correction at least as large as the thing it corrects is not a correction. + const adjusted = Math.round(predictedTurns + bucket.bias); + if (Math.abs(bucket.bias) >= predictedTurns || adjusted < 1) { + return { + publishable: false, + predictedTurns, + horizon, + bias: bucket.bias, + reason: `the observed bias at this horizon (${Math.round(bucket.bias)} turns) is as large as ` + + `the forecast itself (${predictedTurns}), so the correction would replace it rather than ` + + 'adjust it', + }; + } return { publishable: true, diff --git a/integrations/codex/plugin/hooks/lib/forecast.mjs b/integrations/codex/plugin/hooks/lib/forecast.mjs index ed698b1a0..dee6a3b56 100644 --- a/integrations/codex/plugin/hooks/lib/forecast.mjs +++ b/integrations/codex/plugin/hooks/lib/forecast.mjs @@ -34,6 +34,7 @@ import { readMetrics, readBalance, isFixtureAnchor, BALANCE_KINDS } from './metrics.mjs'; import { aggregateConsolidation } from './consolidate.mjs'; import { previewQuality } from './expand.mjs'; +import { logForecast, calibrate } from './calibration.mjs'; /** Turns of headroom below which the runway is worth interrupting for. */ export const ACTIONABLE_RUNWAY = 8; @@ -312,7 +313,30 @@ export function forecastPanel(dir, session = {}, findings = []) { if (air) { parts.runway = air; - lines.push(runwayLine(air)); + + // THE FORECAST KEEPS ITS OWN SCORE -- from here, because this is the only place a forecast is + // actually made. calibration.mjs was written for exactly this and had zero shipped importers: + // no forecast was ever logged, no outcome ever observed, reliability always saw an empty set, + // and calibrate was unreachable. So the shipped panel printed precisely the uncalibrated + // number that module's docstring calls "a vibe with a typeface". + // + // Logged BEFORE it is corrected. The raw prediction is what gets scored against the outcome; + // scoring the corrected one would fold the correction back into the next correction and the + // bias would chase its own tail. + logForecast(dir, { + sessionId: session.sessionId, + predictedTurns: air.withGraph, + used: session.used, + capacity: session.capacity, + }); + + const score = calibrate(dir, air.withGraph); + parts.calibration = score; + // Published only when the horizon has earned it. An unpublishable score does not replace the + // number -- it is simply absent, exactly as the uncalibrated case reads today. + lines.push(score.publishable + ? `${runwayLine({ ...air, withGraph: score.predictedTurns })}\n (corrected: ${score.note})` + : runwayLine(air)); } if (rate) { diff --git a/integrations/codex/plugin/hooks/lib/metrics.mjs b/integrations/codex/plugin/hooks/lib/metrics.mjs index b762df57c..4b0eb2712 100644 --- a/integrations/codex/plugin/hooks/lib/metrics.mjs +++ b/integrations/codex/plugin/hooks/lib/metrics.mjs @@ -68,7 +68,17 @@ const balancePath = (dir) => join(dir, 'balance.jsonl'); * starve the measurement. Exported because any reader that needs the unwindowed truth needs to * know which kinds it applies to, and a second copy of this set elsewhere would drift. */ -export const BALANCE_KINDS = new Set(['inject', 'harvest', 'substitute']); +export const BALANCE_KINDS = new Set([ + 'inject', + 'harvest', + 'substitute', + // The forecast's own score. Both kinds are rare and accumulate slowly -- MIN_SCORED is 8 -- + // while the firehose they shared is dominated by per-tool-call records, so they were evicted + // faster than they could accumulate: calibrate() returned 'not yet calibrated (n/8)' forever + // and an outcome whose forecast had scrolled away was dropped with no record at all. + 'forecast', + 'forecast-outcome', +]); /** * Is this touch in the holdout arm? diff --git a/integrations/gemini/hooks/lib/calibration.mjs b/integrations/gemini/hooks/lib/calibration.mjs index 718db89cd..e920a9992 100644 --- a/integrations/gemini/hooks/lib/calibration.mjs +++ b/integrations/gemini/hooks/lib/calibration.mjs @@ -31,7 +31,20 @@ * that cannot support one. */ -import { record, readMetrics } from './metrics.mjs'; +// readBalance, not readMetrics. Both readers used the windowed firehose, which returns at most +// 2,000,000 bytes and the last 5,000 lines -- and forecast events were not among the protected +// kinds, so they lived in a log dominated by per-tool-call records (metrics.mjs's own comment +// cites 4,735 capture events inside one window). +// +// Two failures followed. In observeOutcome, on a busy project the forecast had scrolled past the +// tail by the time compaction fired, so the outcome was discarded with no record, no error and no +// counter. In reliability, outcomes vanished faster than they accumulated, so rows.length could +// never reach MIN_SCORED and calibrate returned 'not yet calibrated (n/8)' permanently -- leaving +// a user unable to distinguish "never had data" from "the data is being thrown away". +// +// 'forecast' and 'forecast-outcome' are now balance kinds, so they are written to balance.jsonl +// as well and read back unwindowed. +import { record, readBalance } from './metrics.mjs'; /** Horizon buckets, in turns. Error behaves differently across these ranges. */ const HORIZONS = [ @@ -75,14 +88,35 @@ export function logForecast(dir, { sessionId, predictedTurns, used, capacity }) */ export function observeOutcome(dir, { sessionId, actualTurns }) { if (!Number.isFinite(actualTurns)) return; - const open = readMetrics(dir) - .filter((e) => e.kind === 'forecast' && e.sessionId === sessionId && !e.scored) + + // CLOSURE IS DERIVED FROM THE OUTCOME ROWS, not from a flag on the forecast. + // + // The old filter was `!e.scored`, but nothing ever writes `scored`: logForecast records + // kind/sessionId/predictedTurns/horizon/used/capacity, and record() is an append-only JSONL + // writer with no update path, so a forecast can never gain the field. The predicate was always + // true, and the last forecast for a session was therefore re-closable without limit. + // + // A session where compaction fires twice with no intervening logForecast popped the SAME + // forecast twice, producing two outcome rows for one prediction. reliability counts ROWS, not + // predictions, so the hit rate and the bias mean both inflate and MIN_SCORED can be satisfied + // by a single prediction re-closed eight times -- precisely the "threshold from a single + // sample" this module exists to prevent. The existing test used a fresh sessionId per + // iteration, so it never exposed it. + const events = readBalance(dir); + const closed = new Set( + events.filter((e) => e.kind === 'forecast-outcome' && e.forecastId).map((e) => e.forecastId), + ); + const open = events + .filter((e) => e.kind === 'forecast' && e.sessionId === sessionId && !closed.has(e.id)) .pop(); if (!open) return; record(dir, { kind: 'forecast-outcome', sessionId, + // Stamped so the next call can see this forecast is spoken for. Without it the set above is + // empty and the same defect returns. + forecastId: open.id, horizon: open.horizon, predictedTurns: open.predictedTurns, actualTurns, @@ -98,7 +132,7 @@ export function observeOutcome(dir, { sessionId, actualTurns }) { * actionable, where "62% accurate" is not. */ export function reliability(dir) { - const outcomes = readMetrics(dir).filter((e) => e.kind === 'forecast-outcome'); + const outcomes = readBalance(dir).filter((e) => e.kind === 'forecast-outcome'); const buckets = {}; for (const horizon of HORIZONS) { @@ -164,7 +198,28 @@ export function calibrate(dir, predictedTurns) { // Refit: shift by the observed bias for this horizon. A bucket that runs long // gets pulled in, and vice versa. - const adjusted = Math.max(0, Math.round(predictedTurns + bucket.bias)); + // + // NOT CLAMPED WITH Math.max, which converted nonsense into a plausible-looking emergency. bias + // is a plain arithmetic mean of signed error, and a mean is not robust to the shape the + // 50%-hit-rate floor admits: a near bucket with errors [0,0,0,0,-30,-30,-30,-30] has four hits, + // so it clears the floor and is called calibrated, and its bias is -15. A raw forecast of 3 + // turns then became Math.max(0, Math.round(3 - 15)) = 0 and was published as fact, with the + // note "within 3 on 50% of 8 past forecasts" -- a zero-turn deadline the forecaster never + // produced, from a bucket that is wrong half the time. + // + // A correction at least as large as the thing it corrects is not a correction. + const adjusted = Math.round(predictedTurns + bucket.bias); + if (Math.abs(bucket.bias) >= predictedTurns || adjusted < 1) { + return { + publishable: false, + predictedTurns, + horizon, + bias: bucket.bias, + reason: `the observed bias at this horizon (${Math.round(bucket.bias)} turns) is as large as ` + + `the forecast itself (${predictedTurns}), so the correction would replace it rather than ` + + 'adjust it', + }; + } return { publishable: true, diff --git a/integrations/gemini/hooks/lib/forecast.mjs b/integrations/gemini/hooks/lib/forecast.mjs index ed698b1a0..dee6a3b56 100644 --- a/integrations/gemini/hooks/lib/forecast.mjs +++ b/integrations/gemini/hooks/lib/forecast.mjs @@ -34,6 +34,7 @@ import { readMetrics, readBalance, isFixtureAnchor, BALANCE_KINDS } from './metrics.mjs'; import { aggregateConsolidation } from './consolidate.mjs'; import { previewQuality } from './expand.mjs'; +import { logForecast, calibrate } from './calibration.mjs'; /** Turns of headroom below which the runway is worth interrupting for. */ export const ACTIONABLE_RUNWAY = 8; @@ -312,7 +313,30 @@ export function forecastPanel(dir, session = {}, findings = []) { if (air) { parts.runway = air; - lines.push(runwayLine(air)); + + // THE FORECAST KEEPS ITS OWN SCORE -- from here, because this is the only place a forecast is + // actually made. calibration.mjs was written for exactly this and had zero shipped importers: + // no forecast was ever logged, no outcome ever observed, reliability always saw an empty set, + // and calibrate was unreachable. So the shipped panel printed precisely the uncalibrated + // number that module's docstring calls "a vibe with a typeface". + // + // Logged BEFORE it is corrected. The raw prediction is what gets scored against the outcome; + // scoring the corrected one would fold the correction back into the next correction and the + // bias would chase its own tail. + logForecast(dir, { + sessionId: session.sessionId, + predictedTurns: air.withGraph, + used: session.used, + capacity: session.capacity, + }); + + const score = calibrate(dir, air.withGraph); + parts.calibration = score; + // Published only when the horizon has earned it. An unpublishable score does not replace the + // number -- it is simply absent, exactly as the uncalibrated case reads today. + lines.push(score.publishable + ? `${runwayLine({ ...air, withGraph: score.predictedTurns })}\n (corrected: ${score.note})` + : runwayLine(air)); } if (rate) { diff --git a/integrations/gemini/hooks/lib/metrics.mjs b/integrations/gemini/hooks/lib/metrics.mjs index b762df57c..4b0eb2712 100644 --- a/integrations/gemini/hooks/lib/metrics.mjs +++ b/integrations/gemini/hooks/lib/metrics.mjs @@ -68,7 +68,17 @@ const balancePath = (dir) => join(dir, 'balance.jsonl'); * starve the measurement. Exported because any reader that needs the unwindowed truth needs to * know which kinds it applies to, and a second copy of this set elsewhere would drift. */ -export const BALANCE_KINDS = new Set(['inject', 'harvest', 'substitute']); +export const BALANCE_KINDS = new Set([ + 'inject', + 'harvest', + 'substitute', + // The forecast's own score. Both kinds are rare and accumulate slowly -- MIN_SCORED is 8 -- + // while the firehose they shared is dominated by per-tool-call records, so they were evicted + // faster than they could accumulate: calibrate() returned 'not yet calibrated (n/8)' forever + // and an outcome whose forecast had scrolled away was dropped with no record at all. + 'forecast', + 'forecast-outcome', +]); /** * Is this touch in the holdout arm? diff --git a/integrations/opencode/hooks/lib/calibration.mjs b/integrations/opencode/hooks/lib/calibration.mjs index 718db89cd..e920a9992 100644 --- a/integrations/opencode/hooks/lib/calibration.mjs +++ b/integrations/opencode/hooks/lib/calibration.mjs @@ -31,7 +31,20 @@ * that cannot support one. */ -import { record, readMetrics } from './metrics.mjs'; +// readBalance, not readMetrics. Both readers used the windowed firehose, which returns at most +// 2,000,000 bytes and the last 5,000 lines -- and forecast events were not among the protected +// kinds, so they lived in a log dominated by per-tool-call records (metrics.mjs's own comment +// cites 4,735 capture events inside one window). +// +// Two failures followed. In observeOutcome, on a busy project the forecast had scrolled past the +// tail by the time compaction fired, so the outcome was discarded with no record, no error and no +// counter. In reliability, outcomes vanished faster than they accumulated, so rows.length could +// never reach MIN_SCORED and calibrate returned 'not yet calibrated (n/8)' permanently -- leaving +// a user unable to distinguish "never had data" from "the data is being thrown away". +// +// 'forecast' and 'forecast-outcome' are now balance kinds, so they are written to balance.jsonl +// as well and read back unwindowed. +import { record, readBalance } from './metrics.mjs'; /** Horizon buckets, in turns. Error behaves differently across these ranges. */ const HORIZONS = [ @@ -75,14 +88,35 @@ export function logForecast(dir, { sessionId, predictedTurns, used, capacity }) */ export function observeOutcome(dir, { sessionId, actualTurns }) { if (!Number.isFinite(actualTurns)) return; - const open = readMetrics(dir) - .filter((e) => e.kind === 'forecast' && e.sessionId === sessionId && !e.scored) + + // CLOSURE IS DERIVED FROM THE OUTCOME ROWS, not from a flag on the forecast. + // + // The old filter was `!e.scored`, but nothing ever writes `scored`: logForecast records + // kind/sessionId/predictedTurns/horizon/used/capacity, and record() is an append-only JSONL + // writer with no update path, so a forecast can never gain the field. The predicate was always + // true, and the last forecast for a session was therefore re-closable without limit. + // + // A session where compaction fires twice with no intervening logForecast popped the SAME + // forecast twice, producing two outcome rows for one prediction. reliability counts ROWS, not + // predictions, so the hit rate and the bias mean both inflate and MIN_SCORED can be satisfied + // by a single prediction re-closed eight times -- precisely the "threshold from a single + // sample" this module exists to prevent. The existing test used a fresh sessionId per + // iteration, so it never exposed it. + const events = readBalance(dir); + const closed = new Set( + events.filter((e) => e.kind === 'forecast-outcome' && e.forecastId).map((e) => e.forecastId), + ); + const open = events + .filter((e) => e.kind === 'forecast' && e.sessionId === sessionId && !closed.has(e.id)) .pop(); if (!open) return; record(dir, { kind: 'forecast-outcome', sessionId, + // Stamped so the next call can see this forecast is spoken for. Without it the set above is + // empty and the same defect returns. + forecastId: open.id, horizon: open.horizon, predictedTurns: open.predictedTurns, actualTurns, @@ -98,7 +132,7 @@ export function observeOutcome(dir, { sessionId, actualTurns }) { * actionable, where "62% accurate" is not. */ export function reliability(dir) { - const outcomes = readMetrics(dir).filter((e) => e.kind === 'forecast-outcome'); + const outcomes = readBalance(dir).filter((e) => e.kind === 'forecast-outcome'); const buckets = {}; for (const horizon of HORIZONS) { @@ -164,7 +198,28 @@ export function calibrate(dir, predictedTurns) { // Refit: shift by the observed bias for this horizon. A bucket that runs long // gets pulled in, and vice versa. - const adjusted = Math.max(0, Math.round(predictedTurns + bucket.bias)); + // + // NOT CLAMPED WITH Math.max, which converted nonsense into a plausible-looking emergency. bias + // is a plain arithmetic mean of signed error, and a mean is not robust to the shape the + // 50%-hit-rate floor admits: a near bucket with errors [0,0,0,0,-30,-30,-30,-30] has four hits, + // so it clears the floor and is called calibrated, and its bias is -15. A raw forecast of 3 + // turns then became Math.max(0, Math.round(3 - 15)) = 0 and was published as fact, with the + // note "within 3 on 50% of 8 past forecasts" -- a zero-turn deadline the forecaster never + // produced, from a bucket that is wrong half the time. + // + // A correction at least as large as the thing it corrects is not a correction. + const adjusted = Math.round(predictedTurns + bucket.bias); + if (Math.abs(bucket.bias) >= predictedTurns || adjusted < 1) { + return { + publishable: false, + predictedTurns, + horizon, + bias: bucket.bias, + reason: `the observed bias at this horizon (${Math.round(bucket.bias)} turns) is as large as ` + + `the forecast itself (${predictedTurns}), so the correction would replace it rather than ` + + 'adjust it', + }; + } return { publishable: true, diff --git a/integrations/opencode/hooks/lib/forecast.mjs b/integrations/opencode/hooks/lib/forecast.mjs index ed698b1a0..dee6a3b56 100644 --- a/integrations/opencode/hooks/lib/forecast.mjs +++ b/integrations/opencode/hooks/lib/forecast.mjs @@ -34,6 +34,7 @@ import { readMetrics, readBalance, isFixtureAnchor, BALANCE_KINDS } from './metrics.mjs'; import { aggregateConsolidation } from './consolidate.mjs'; import { previewQuality } from './expand.mjs'; +import { logForecast, calibrate } from './calibration.mjs'; /** Turns of headroom below which the runway is worth interrupting for. */ export const ACTIONABLE_RUNWAY = 8; @@ -312,7 +313,30 @@ export function forecastPanel(dir, session = {}, findings = []) { if (air) { parts.runway = air; - lines.push(runwayLine(air)); + + // THE FORECAST KEEPS ITS OWN SCORE -- from here, because this is the only place a forecast is + // actually made. calibration.mjs was written for exactly this and had zero shipped importers: + // no forecast was ever logged, no outcome ever observed, reliability always saw an empty set, + // and calibrate was unreachable. So the shipped panel printed precisely the uncalibrated + // number that module's docstring calls "a vibe with a typeface". + // + // Logged BEFORE it is corrected. The raw prediction is what gets scored against the outcome; + // scoring the corrected one would fold the correction back into the next correction and the + // bias would chase its own tail. + logForecast(dir, { + sessionId: session.sessionId, + predictedTurns: air.withGraph, + used: session.used, + capacity: session.capacity, + }); + + const score = calibrate(dir, air.withGraph); + parts.calibration = score; + // Published only when the horizon has earned it. An unpublishable score does not replace the + // number -- it is simply absent, exactly as the uncalibrated case reads today. + lines.push(score.publishable + ? `${runwayLine({ ...air, withGraph: score.predictedTurns })}\n (corrected: ${score.note})` + : runwayLine(air)); } if (rate) { diff --git a/integrations/opencode/hooks/lib/metrics.mjs b/integrations/opencode/hooks/lib/metrics.mjs index b762df57c..4b0eb2712 100644 --- a/integrations/opencode/hooks/lib/metrics.mjs +++ b/integrations/opencode/hooks/lib/metrics.mjs @@ -68,7 +68,17 @@ const balancePath = (dir) => join(dir, 'balance.jsonl'); * starve the measurement. Exported because any reader that needs the unwindowed truth needs to * know which kinds it applies to, and a second copy of this set elsewhere would drift. */ -export const BALANCE_KINDS = new Set(['inject', 'harvest', 'substitute']); +export const BALANCE_KINDS = new Set([ + 'inject', + 'harvest', + 'substitute', + // The forecast's own score. Both kinds are rare and accumulate slowly -- MIN_SCORED is 8 -- + // while the firehose they shared is dominated by per-tool-call records, so they were evicted + // faster than they could accumulate: calibrate() returned 'not yet calibrated (n/8)' forever + // and an outcome whose forecast had scrolled away was dropped with no record at all. + 'forecast', + 'forecast-outcome', +]); /** * Is this touch in the holdout arm? diff --git a/integrations/qwen/hooks/lib/calibration.mjs b/integrations/qwen/hooks/lib/calibration.mjs index 718db89cd..e920a9992 100644 --- a/integrations/qwen/hooks/lib/calibration.mjs +++ b/integrations/qwen/hooks/lib/calibration.mjs @@ -31,7 +31,20 @@ * that cannot support one. */ -import { record, readMetrics } from './metrics.mjs'; +// readBalance, not readMetrics. Both readers used the windowed firehose, which returns at most +// 2,000,000 bytes and the last 5,000 lines -- and forecast events were not among the protected +// kinds, so they lived in a log dominated by per-tool-call records (metrics.mjs's own comment +// cites 4,735 capture events inside one window). +// +// Two failures followed. In observeOutcome, on a busy project the forecast had scrolled past the +// tail by the time compaction fired, so the outcome was discarded with no record, no error and no +// counter. In reliability, outcomes vanished faster than they accumulated, so rows.length could +// never reach MIN_SCORED and calibrate returned 'not yet calibrated (n/8)' permanently -- leaving +// a user unable to distinguish "never had data" from "the data is being thrown away". +// +// 'forecast' and 'forecast-outcome' are now balance kinds, so they are written to balance.jsonl +// as well and read back unwindowed. +import { record, readBalance } from './metrics.mjs'; /** Horizon buckets, in turns. Error behaves differently across these ranges. */ const HORIZONS = [ @@ -75,14 +88,35 @@ export function logForecast(dir, { sessionId, predictedTurns, used, capacity }) */ export function observeOutcome(dir, { sessionId, actualTurns }) { if (!Number.isFinite(actualTurns)) return; - const open = readMetrics(dir) - .filter((e) => e.kind === 'forecast' && e.sessionId === sessionId && !e.scored) + + // CLOSURE IS DERIVED FROM THE OUTCOME ROWS, not from a flag on the forecast. + // + // The old filter was `!e.scored`, but nothing ever writes `scored`: logForecast records + // kind/sessionId/predictedTurns/horizon/used/capacity, and record() is an append-only JSONL + // writer with no update path, so a forecast can never gain the field. The predicate was always + // true, and the last forecast for a session was therefore re-closable without limit. + // + // A session where compaction fires twice with no intervening logForecast popped the SAME + // forecast twice, producing two outcome rows for one prediction. reliability counts ROWS, not + // predictions, so the hit rate and the bias mean both inflate and MIN_SCORED can be satisfied + // by a single prediction re-closed eight times -- precisely the "threshold from a single + // sample" this module exists to prevent. The existing test used a fresh sessionId per + // iteration, so it never exposed it. + const events = readBalance(dir); + const closed = new Set( + events.filter((e) => e.kind === 'forecast-outcome' && e.forecastId).map((e) => e.forecastId), + ); + const open = events + .filter((e) => e.kind === 'forecast' && e.sessionId === sessionId && !closed.has(e.id)) .pop(); if (!open) return; record(dir, { kind: 'forecast-outcome', sessionId, + // Stamped so the next call can see this forecast is spoken for. Without it the set above is + // empty and the same defect returns. + forecastId: open.id, horizon: open.horizon, predictedTurns: open.predictedTurns, actualTurns, @@ -98,7 +132,7 @@ export function observeOutcome(dir, { sessionId, actualTurns }) { * actionable, where "62% accurate" is not. */ export function reliability(dir) { - const outcomes = readMetrics(dir).filter((e) => e.kind === 'forecast-outcome'); + const outcomes = readBalance(dir).filter((e) => e.kind === 'forecast-outcome'); const buckets = {}; for (const horizon of HORIZONS) { @@ -164,7 +198,28 @@ export function calibrate(dir, predictedTurns) { // Refit: shift by the observed bias for this horizon. A bucket that runs long // gets pulled in, and vice versa. - const adjusted = Math.max(0, Math.round(predictedTurns + bucket.bias)); + // + // NOT CLAMPED WITH Math.max, which converted nonsense into a plausible-looking emergency. bias + // is a plain arithmetic mean of signed error, and a mean is not robust to the shape the + // 50%-hit-rate floor admits: a near bucket with errors [0,0,0,0,-30,-30,-30,-30] has four hits, + // so it clears the floor and is called calibrated, and its bias is -15. A raw forecast of 3 + // turns then became Math.max(0, Math.round(3 - 15)) = 0 and was published as fact, with the + // note "within 3 on 50% of 8 past forecasts" -- a zero-turn deadline the forecaster never + // produced, from a bucket that is wrong half the time. + // + // A correction at least as large as the thing it corrects is not a correction. + const adjusted = Math.round(predictedTurns + bucket.bias); + if (Math.abs(bucket.bias) >= predictedTurns || adjusted < 1) { + return { + publishable: false, + predictedTurns, + horizon, + bias: bucket.bias, + reason: `the observed bias at this horizon (${Math.round(bucket.bias)} turns) is as large as ` + + `the forecast itself (${predictedTurns}), so the correction would replace it rather than ` + + 'adjust it', + }; + } return { publishable: true, diff --git a/integrations/qwen/hooks/lib/forecast.mjs b/integrations/qwen/hooks/lib/forecast.mjs index ed698b1a0..dee6a3b56 100644 --- a/integrations/qwen/hooks/lib/forecast.mjs +++ b/integrations/qwen/hooks/lib/forecast.mjs @@ -34,6 +34,7 @@ import { readMetrics, readBalance, isFixtureAnchor, BALANCE_KINDS } from './metrics.mjs'; import { aggregateConsolidation } from './consolidate.mjs'; import { previewQuality } from './expand.mjs'; +import { logForecast, calibrate } from './calibration.mjs'; /** Turns of headroom below which the runway is worth interrupting for. */ export const ACTIONABLE_RUNWAY = 8; @@ -312,7 +313,30 @@ export function forecastPanel(dir, session = {}, findings = []) { if (air) { parts.runway = air; - lines.push(runwayLine(air)); + + // THE FORECAST KEEPS ITS OWN SCORE -- from here, because this is the only place a forecast is + // actually made. calibration.mjs was written for exactly this and had zero shipped importers: + // no forecast was ever logged, no outcome ever observed, reliability always saw an empty set, + // and calibrate was unreachable. So the shipped panel printed precisely the uncalibrated + // number that module's docstring calls "a vibe with a typeface". + // + // Logged BEFORE it is corrected. The raw prediction is what gets scored against the outcome; + // scoring the corrected one would fold the correction back into the next correction and the + // bias would chase its own tail. + logForecast(dir, { + sessionId: session.sessionId, + predictedTurns: air.withGraph, + used: session.used, + capacity: session.capacity, + }); + + const score = calibrate(dir, air.withGraph); + parts.calibration = score; + // Published only when the horizon has earned it. An unpublishable score does not replace the + // number -- it is simply absent, exactly as the uncalibrated case reads today. + lines.push(score.publishable + ? `${runwayLine({ ...air, withGraph: score.predictedTurns })}\n (corrected: ${score.note})` + : runwayLine(air)); } if (rate) { diff --git a/integrations/qwen/hooks/lib/metrics.mjs b/integrations/qwen/hooks/lib/metrics.mjs index b762df57c..4b0eb2712 100644 --- a/integrations/qwen/hooks/lib/metrics.mjs +++ b/integrations/qwen/hooks/lib/metrics.mjs @@ -68,7 +68,17 @@ const balancePath = (dir) => join(dir, 'balance.jsonl'); * starve the measurement. Exported because any reader that needs the unwindowed truth needs to * know which kinds it applies to, and a second copy of this set elsewhere would drift. */ -export const BALANCE_KINDS = new Set(['inject', 'harvest', 'substitute']); +export const BALANCE_KINDS = new Set([ + 'inject', + 'harvest', + 'substitute', + // The forecast's own score. Both kinds are rare and accumulate slowly -- MIN_SCORED is 8 -- + // while the firehose they shared is dominated by per-tool-call records, so they were evicted + // faster than they could accumulate: calibrate() returned 'not yet calibrated (n/8)' forever + // and an outcome whose forecast had scrolled away was dropped with no record at all. + 'forecast', + 'forecast-outcome', +]); /** * Is this touch in the holdout arm? diff --git a/plugin/hooks/lib/calibration.mjs b/plugin/hooks/lib/calibration.mjs index 718db89cd..e920a9992 100644 --- a/plugin/hooks/lib/calibration.mjs +++ b/plugin/hooks/lib/calibration.mjs @@ -31,7 +31,20 @@ * that cannot support one. */ -import { record, readMetrics } from './metrics.mjs'; +// readBalance, not readMetrics. Both readers used the windowed firehose, which returns at most +// 2,000,000 bytes and the last 5,000 lines -- and forecast events were not among the protected +// kinds, so they lived in a log dominated by per-tool-call records (metrics.mjs's own comment +// cites 4,735 capture events inside one window). +// +// Two failures followed. In observeOutcome, on a busy project the forecast had scrolled past the +// tail by the time compaction fired, so the outcome was discarded with no record, no error and no +// counter. In reliability, outcomes vanished faster than they accumulated, so rows.length could +// never reach MIN_SCORED and calibrate returned 'not yet calibrated (n/8)' permanently -- leaving +// a user unable to distinguish "never had data" from "the data is being thrown away". +// +// 'forecast' and 'forecast-outcome' are now balance kinds, so they are written to balance.jsonl +// as well and read back unwindowed. +import { record, readBalance } from './metrics.mjs'; /** Horizon buckets, in turns. Error behaves differently across these ranges. */ const HORIZONS = [ @@ -75,14 +88,35 @@ export function logForecast(dir, { sessionId, predictedTurns, used, capacity }) */ export function observeOutcome(dir, { sessionId, actualTurns }) { if (!Number.isFinite(actualTurns)) return; - const open = readMetrics(dir) - .filter((e) => e.kind === 'forecast' && e.sessionId === sessionId && !e.scored) + + // CLOSURE IS DERIVED FROM THE OUTCOME ROWS, not from a flag on the forecast. + // + // The old filter was `!e.scored`, but nothing ever writes `scored`: logForecast records + // kind/sessionId/predictedTurns/horizon/used/capacity, and record() is an append-only JSONL + // writer with no update path, so a forecast can never gain the field. The predicate was always + // true, and the last forecast for a session was therefore re-closable without limit. + // + // A session where compaction fires twice with no intervening logForecast popped the SAME + // forecast twice, producing two outcome rows for one prediction. reliability counts ROWS, not + // predictions, so the hit rate and the bias mean both inflate and MIN_SCORED can be satisfied + // by a single prediction re-closed eight times -- precisely the "threshold from a single + // sample" this module exists to prevent. The existing test used a fresh sessionId per + // iteration, so it never exposed it. + const events = readBalance(dir); + const closed = new Set( + events.filter((e) => e.kind === 'forecast-outcome' && e.forecastId).map((e) => e.forecastId), + ); + const open = events + .filter((e) => e.kind === 'forecast' && e.sessionId === sessionId && !closed.has(e.id)) .pop(); if (!open) return; record(dir, { kind: 'forecast-outcome', sessionId, + // Stamped so the next call can see this forecast is spoken for. Without it the set above is + // empty and the same defect returns. + forecastId: open.id, horizon: open.horizon, predictedTurns: open.predictedTurns, actualTurns, @@ -98,7 +132,7 @@ export function observeOutcome(dir, { sessionId, actualTurns }) { * actionable, where "62% accurate" is not. */ export function reliability(dir) { - const outcomes = readMetrics(dir).filter((e) => e.kind === 'forecast-outcome'); + const outcomes = readBalance(dir).filter((e) => e.kind === 'forecast-outcome'); const buckets = {}; for (const horizon of HORIZONS) { @@ -164,7 +198,28 @@ export function calibrate(dir, predictedTurns) { // Refit: shift by the observed bias for this horizon. A bucket that runs long // gets pulled in, and vice versa. - const adjusted = Math.max(0, Math.round(predictedTurns + bucket.bias)); + // + // NOT CLAMPED WITH Math.max, which converted nonsense into a plausible-looking emergency. bias + // is a plain arithmetic mean of signed error, and a mean is not robust to the shape the + // 50%-hit-rate floor admits: a near bucket with errors [0,0,0,0,-30,-30,-30,-30] has four hits, + // so it clears the floor and is called calibrated, and its bias is -15. A raw forecast of 3 + // turns then became Math.max(0, Math.round(3 - 15)) = 0 and was published as fact, with the + // note "within 3 on 50% of 8 past forecasts" -- a zero-turn deadline the forecaster never + // produced, from a bucket that is wrong half the time. + // + // A correction at least as large as the thing it corrects is not a correction. + const adjusted = Math.round(predictedTurns + bucket.bias); + if (Math.abs(bucket.bias) >= predictedTurns || adjusted < 1) { + return { + publishable: false, + predictedTurns, + horizon, + bias: bucket.bias, + reason: `the observed bias at this horizon (${Math.round(bucket.bias)} turns) is as large as ` + + `the forecast itself (${predictedTurns}), so the correction would replace it rather than ` + + 'adjust it', + }; + } return { publishable: true, diff --git a/plugin/hooks/lib/forecast.mjs b/plugin/hooks/lib/forecast.mjs index ed698b1a0..dee6a3b56 100644 --- a/plugin/hooks/lib/forecast.mjs +++ b/plugin/hooks/lib/forecast.mjs @@ -34,6 +34,7 @@ import { readMetrics, readBalance, isFixtureAnchor, BALANCE_KINDS } from './metrics.mjs'; import { aggregateConsolidation } from './consolidate.mjs'; import { previewQuality } from './expand.mjs'; +import { logForecast, calibrate } from './calibration.mjs'; /** Turns of headroom below which the runway is worth interrupting for. */ export const ACTIONABLE_RUNWAY = 8; @@ -312,7 +313,30 @@ export function forecastPanel(dir, session = {}, findings = []) { if (air) { parts.runway = air; - lines.push(runwayLine(air)); + + // THE FORECAST KEEPS ITS OWN SCORE -- from here, because this is the only place a forecast is + // actually made. calibration.mjs was written for exactly this and had zero shipped importers: + // no forecast was ever logged, no outcome ever observed, reliability always saw an empty set, + // and calibrate was unreachable. So the shipped panel printed precisely the uncalibrated + // number that module's docstring calls "a vibe with a typeface". + // + // Logged BEFORE it is corrected. The raw prediction is what gets scored against the outcome; + // scoring the corrected one would fold the correction back into the next correction and the + // bias would chase its own tail. + logForecast(dir, { + sessionId: session.sessionId, + predictedTurns: air.withGraph, + used: session.used, + capacity: session.capacity, + }); + + const score = calibrate(dir, air.withGraph); + parts.calibration = score; + // Published only when the horizon has earned it. An unpublishable score does not replace the + // number -- it is simply absent, exactly as the uncalibrated case reads today. + lines.push(score.publishable + ? `${runwayLine({ ...air, withGraph: score.predictedTurns })}\n (corrected: ${score.note})` + : runwayLine(air)); } if (rate) { diff --git a/plugin/hooks/lib/metrics.mjs b/plugin/hooks/lib/metrics.mjs index b762df57c..4b0eb2712 100644 --- a/plugin/hooks/lib/metrics.mjs +++ b/plugin/hooks/lib/metrics.mjs @@ -68,7 +68,17 @@ const balancePath = (dir) => join(dir, 'balance.jsonl'); * starve the measurement. Exported because any reader that needs the unwindowed truth needs to * know which kinds it applies to, and a second copy of this set elsewhere would drift. */ -export const BALANCE_KINDS = new Set(['inject', 'harvest', 'substitute']); +export const BALANCE_KINDS = new Set([ + 'inject', + 'harvest', + 'substitute', + // The forecast's own score. Both kinds are rare and accumulate slowly -- MIN_SCORED is 8 -- + // while the firehose they shared is dominated by per-tool-call records, so they were evicted + // faster than they could accumulate: calibrate() returned 'not yet calibrated (n/8)' forever + // and an outcome whose forecast had scrolled away was dropped with no record at all. + 'forecast', + 'forecast-outcome', +]); /** * Is this touch in the holdout arm? diff --git a/tests/hooks/forecast.test.mjs b/tests/hooks/forecast.test.mjs index e0041f201..02488acb8 100644 --- a/tests/hooks/forecast.test.mjs +++ b/tests/hooks/forecast.test.mjs @@ -19,7 +19,7 @@ import { import { logForecast, observeOutcome, reliability, calibrate, } from '../../hooks-core/calibration.mjs'; -import { readMetrics } from '../../hooks-core/metrics.mjs'; +import { readMetrics, readBalance } from '../../hooks-core/metrics.mjs'; let dir; @@ -370,3 +370,133 @@ describe('the balance kinds are read from the log that is not windowed', () => { expect(merged.filter((e) => e.kind === 'inject')).toHaveLength(7); }); }); + +// --- the calibration loop is wired, and cannot score one prediction twice ----------- + +describe('a forecast is closed once', () => { + test('two compactions in one session do not score the same prediction twice', () => { + // THE DEFECT: the open-forecast filter was `!e.scored`, but nothing ever writes `scored` -- + // record() is an append-only JSONL writer with no update path, so a forecast could never gain + // the field and the predicate was always true. A session where compaction fires twice with no + // intervening logForecast popped the SAME forecast twice. reliability counts ROWS, not + // predictions, so MIN_SCORED could be satisfied by one prediction re-closed eight times -- + // exactly the 'threshold from a single sample' this module exists to prevent. The existing + // test used a fresh sessionId per iteration, so it never exposed it. + logForecast(dir, { sessionId: 'one', predictedTurns: 10, used: 1, capacity: 2 }); + observeOutcome(dir, { sessionId: 'one', actualTurns: 11 }); + observeOutcome(dir, { sessionId: 'one', actualTurns: 40 }); + + expect(reliability(dir).scored).toBe(1); + }); + + test('a second prediction in the same session is still scorable', () => { + logForecast(dir, { sessionId: 'one', predictedTurns: 10, used: 1, capacity: 2 }); + observeOutcome(dir, { sessionId: 'one', actualTurns: 11 }); + logForecast(dir, { sessionId: 'one', predictedTurns: 12, used: 1, capacity: 2 }); + observeOutcome(dir, { sessionId: 'one', actualTurns: 13 }); + + expect(reliability(dir).scored).toBe(2); + }); +}); + +describe('forecast events outlive the event window', () => { + test('an outcome still finds its forecast under a firehose', () => { + // THE DEFECT: both readers used readMetrics, which returns at most 2 MB and the last 5,000 + // lines, and forecast events were not protected kinds. On a busy project the forecast had + // scrolled past the tail by the time compaction fired, so the outcome was discarded with no + // record, no error and no counter -- and reliability could never reach MIN_SCORED, leaving + // 'not yet calibrated (n/8)' forever with no way to tell it from having no data at all. + logForecast(dir, { sessionId: 'buried', predictedTurns: 10, used: 1, capacity: 2 }); + for (let i = 0; i < 5_200; i++) { + recordRead(dir, { anchor: `/noise${i}.ts`, sessionId: 'buried', bytes: 40 }); + } + observeOutcome(dir, { sessionId: 'buried', actualTurns: 11 }); + + expect(reliability(dir).scored).toBe(1); + expect(readMetrics(dir).filter((e) => e.kind === 'forecast')).toHaveLength(0); + }); +}); + +describe('a correction never replaces the thing it corrects', () => { + test('a bimodal bucket at the reliability floor refuses to publish', () => { + // THE DEFECT: bias is a plain arithmetic mean of signed error, applied unguarded to any + // bucket clearing hitRate >= 0.5 -- and a mean is not robust to the shape that rule admits. + // Errors [0,0,0,0,-30,-30,-30,-30] give four hits, so the bucket passes the floor and is + // called calibrated, with bias -15. A raw forecast of 3 turns became + // Math.max(0, Math.round(3 - 15)) = 0, published as fact with the note 'within 3 on 50% of 8 + // past forecasts'. The clamp converted nonsense into a plausible-looking emergency. + const errors = [0, 0, 0, 0, -30, -30, -30, -30]; + errors.forEach((error, i) => { + logForecast(dir, { sessionId: `b${i}`, predictedTurns: 3, used: 1, capacity: 2 }); + observeOutcome(dir, { sessionId: `b${i}`, actualTurns: 3 + error }); + }); + + const bucket = reliability(dir).buckets.near; + expect(bucket.scored).toBe(8); + expect(bucket.calibrated).toBe(true); // it really does clear the floor + + const out = calibrate(dir, 3); + expect(out.publishable).toBe(false); + expect(out.predictedTurns).toBe(3); // the raw forecast, not a zero + expect(out.reason).toMatch(/as large as the forecast itself/); + }); + + test('a proportionate correction still publishes', () => { + for (let i = 0; i < 10; i++) { + logForecast(dir, { sessionId: `p${i}`, predictedTurns: 12, used: 1, capacity: 2 }); + observeOutcome(dir, { sessionId: `p${i}`, actualTurns: 14 }); + } + const out = calibrate(dir, 12); + expect(out.publishable).toBe(true); + expect(out.predictedTurns).toBe(14); + }); +}); + +describe('the panel actually runs the calibration loop', () => { + const session = { sessionId: 'live', used: 60_000, capacity: 200_000, turns: 30, touches: 30 }; + + test('rendering a panel logs the forecast it published', () => { + // THE DEFECT: calibration.mjs had zero shipped importers. forecastPanel computed the runway + // and pushed '~N turns to compaction' without ever calling logForecast or calibrate, so no + // forecast was logged, no outcome could be observed, reliability always saw an empty set, and + // calibrate was unreachable. The shipped panel printed exactly the uncalibrated number this + // module's docstring calls 'a vibe with a typeface'. + forecastPanel(dir, session, []); + const logged = readBalance(dir).filter((e) => e.kind === 'forecast'); + expect(logged).toHaveLength(1); + expect(logged[0].predictedTurns).toBeGreaterThan(0); + }); + + test('the raw forecast is what gets logged, not the corrected one', () => { + // Scoring the corrected number would fold each correction into the next and the bias would + // chase its own tail. + for (let i = 0; i < 10; i++) { + logForecast(dir, { sessionId: `w${i}`, predictedTurns: 70, used: 1, capacity: 2 }); + observeOutcome(dir, { sessionId: `w${i}`, actualTurns: 72 }); + } + const panel = forecastPanel(dir, session, []); + const raw = panel.parts.runway.withGraph; + const logged = readBalance(dir).filter((e) => e.kind === 'forecast' && e.sessionId === 'live'); + expect(logged[0].predictedTurns).toBe(raw); + }); + + test('an uncalibrated horizon prints the number without a track record', () => { + const panel = forecastPanel(dir, session, []); + expect(panel.parts.calibration.publishable).toBe(false); + expect(panel.text).not.toMatch(/corrected:/); + }); + + test('a calibrated horizon prints the corrected number with its track record', () => { + const panel = forecastPanel(dir, session, []); + const raw = panel.parts.runway.withGraph; + // Score enough forecasts at that horizon for the bucket to earn publication. + for (let i = 0; i < 12; i++) { + logForecast(dir, { sessionId: `c${i}`, predictedTurns: raw, used: 1, capacity: 2 }); + observeOutcome(dir, { sessionId: `c${i}`, actualTurns: raw + 2 }); + } + const after = forecastPanel(dir, session, []); + expect(after.parts.calibration.publishable).toBe(true); + expect(after.text).toMatch(/corrected: within 3 on \d+% of \d+ past forecasts/); + expect(after.text).toContain(`~${raw + 2} turns to compaction`); + }); +}); diff --git a/tests/hooks/reachability.test.mjs b/tests/hooks/reachability.test.mjs index 2e53b0f05..ec121bce5 100644 --- a/tests/hooks/reachability.test.mjs +++ b/tests/hooks/reachability.test.mjs @@ -91,17 +91,25 @@ const ALLOWED = new Map([ // WIRED ELSEWHERE -- leaves this list when that PR merges. ['forTouch', 'WIRED by the injection PR: just-in-time delivery, imported by nothing before it.'], - // A COMPLETE LOOP WITH NO ENTRY POINT. calibration.mjs has no importer at - // all: logForecast and observeOutcome collect the data, reliability scores it - // and calibrate applies the score. Nothing starts it, so the module cannot - // score anything -- and its own docstring says an uncalibrated forecast is - // "a vibe with a typeface". - ['logForecast', 'UNWIRED LOOP: calibration.mjs has zero importers; this records the prediction half.'], - ['observeOutcome', 'UNWIRED LOOP: records the outcome half of a calibration loop nothing starts.'], - - // FORECAST DISPLAY, unreachable while the calibration behind it is unreachable. - ['forecastPanel', 'UNWIRED: renders a forecast whose calibration loop never runs.'], - ['worthSurfacing', 'UNWIRED: decides whether a forecast change is worth showing; same dormant loop.'], + // CALIBRATION, HALF-RESOLVED. logForecast and calibrate are now called by + // forecastPanel, so the prediction side of the loop runs and those two have + // left this list. observeOutcome has not: nothing observes the ground truth, + // which arrives when compaction fires. The natural caller is the PreCompact + // hook, which needs a turn count it does not currently compute. + ['observeOutcome', 'UNWIRED HALF: records the outcome half. The prediction half now runs; the PreCompact hook does not yet supply actualTurns.'], + + // FORECAST DISPLAY. The calibration behind it now runs, but the panel itself + // still has no shipped caller -- so nothing renders it and nothing decides + // when to. Wiring the loop did not change that, and saying otherwise would be + // the exact failure this file exists to catch. + ['forecastPanel', 'UNWIRED: the calibration loop behind it now runs, but no hook or tool renders the panel.'], + ['worthSurfacing', 'UNWIRED: decides whether a forecast change is worth showing; nothing shows it yet.'], + + // SURFACED BY TIGHTENING THE SCAN. Previously counted as reachable only + // because the bare word match read comment prose as a call site. Verified + // orphaned: in shipped code it appears as its own declaration plus two + // mentions in comments, and its only importer is a test. + ['sessionIndex', 'UNWIRED: lists graph keys and truncated claims for a session. Found by the comment-stripping fix; only a test imports it.'], // CONSOLIDATION. forecast.mjs imports aggregateConsolidation from this module, // so it is partly live -- but the selection, the ratio and the content anchor @@ -175,20 +183,71 @@ function exportedFunctions() { return found; } +/** + * Strips comments, so prose cannot be mistaken for a call. + * + * THIS IS WHAT LET A DEAD PUBLIC ENTRY POINT THROUGH. `calibrate` counted as + * reachable because curate.mjs:13 contains the English phrase "the reader's + * ability to calibrate trust", and `reliability` counted because the word + * appears in its own file's prose. calibration.mjs had ZERO importers -- no + * forecast was ever logged, no outcome observed, and the shipped panel printed + * precisely the uncalibrated number that module's docstring calls "a vibe with + * a typeface" -- while the check that exists to catch exactly this reported it + * as wired, on a coincidence of comment wording. + * + * A guard that reads documentation as code is worse than none: it is a clean + * bill of health nobody re-examines. This module's files are heavily commented + * by design, which makes the false-positive rate high rather than incidental. + * + * COMMENTS ONLY, NOT STRING LITERALS -- and that boundary was found the hard + * way. Stripping quotes as well made `routingReport` and `modelSwitchCost` look + * orphaned when routing-tool.ts calls both: three independent global passes + * cannot nest, so an apostrophe inside a DOUBLE-quoted sentence opened a + * "single-quoted string" that ran to the next apostrophe further down the file + * and swallowed the real call sites in between. + * + * Getting that right needs a scanner, and this guard's own rule says why not to + * build one: a check wrong in the permissive direction is merely useless, while + * one wrong in the strict direction fails CI on working code and gets deleted. + * The observed defect was comment prose, comments nest predictably, and that is + * where the line belongs. + */ +function stripComments(text) { + return text + .replace(/\/\*[\s\S]*?\*\//g, ' ') // block comments, including docblocks + .replace(/(^|[^:])\/\/[^\n]*/g, '$1 '); // line comments, sparing the // in a URL +} + /** * Referenced anywhere that ships, other than its own declaration? * - * A bare word match, deliberately. Anything cleverer would need to resolve the - * dynamic `mods..` handles that src/server uses, and a check that - * is wrong in the permissive direction is merely useless -- one that is wrong in - * the strict direction fails CI on working code and gets deleted. + * A bare word match over CODE, deliberately. Anything cleverer would need to + * resolve the dynamic `mods..` handles that src/server uses, and a + * check that is wrong in the permissive direction is merely useless -- one that + * is wrong in the strict direction fails CI on working code and gets deleted. + * Comments and strings are removed first: they are prose, and prose is not a + * caller. */ +const codeOnly = new Map(); +function shippedCode(file, text) { + if (!codeOnly.has(file)) codeOnly.set(file, stripComments(text)); + return codeOnly.get(file); +} + +// AN IN-FILE CALLER STILL COUNTS, and the attempt to change that is worth recording. Excluding +// the declaring file outright -- on the reasoning that `reliability` passed only because +// `calibrate` calls it from the same module -- reported 40+ false orphans in one run, among them +// burnRate, runway and shadowAvoided, which forecastPanel calls from inside forecast.mjs. Those +// are genuinely reached; reachability propagates THROUGH an in-file caller. The real defect in the +// reliability case was that its whole module had no importer, which is a module-level question +// this function cannot answer and should not try to. function usedInShippedCode(name, declaringFile) { const word = new RegExp(`\\b${name}\\b`, 'g'); const decl = new RegExp(`export\\s+(?:async\\s+)?function\\s+${name}\\b`, 'g'); for (const { file, text } of usages) { - const uses = (text.match(word) || []).length; - const declared = file === declaringFile ? (text.match(decl) || []).length : 0; + const code = shippedCode(file, text); + const uses = (code.match(word) || []).length; + const declared = file === declaringFile ? (code.match(decl) || []).length : 0; if (uses - declared > 0) return true; } return false; From c8356b14b43e5bd2f2d00e9c34a49241539713ee Mon Sep 17 00:00:00 2001 From: Franklin Moormann Date: Fri, 7 Aug 2026 20:13:29 -0400 Subject: [PATCH 2/2] feat(surface): give the forecast an entry point, so the loop actually runs The previous commit made the calibration loop complete and correct, and said plainly that it still had no caller: forecastPanel, worthSurfacing and observeOutcome all remained on the reachability allowlist. This is the missing half. surface.mjs derives the session numbers from the transcript, decides when the panel has earned an interruption, and closes the loop when compaction fires. WHERE. The PreToolUse router is the only mid-session hook wired -- SessionStart is too early for a runway to mean anything, and PreCompact is too late to act on one -- so it is the only place a deadline can still change what somebody does next. It already assembles a context string for an allowed call, so the panel joins that rather than needing a new channel. PreCompact takes the other half: compaction firing IS the ground truth every runway forecast was predicting. THREE COSTS ARE BOUNDED, because every tool call is a separate hook process: the read readCacheUsage parses up to 4 MB of transcript the compute the panel walks the metrics log, the balance log and the graph the context the panel is only worth its tokens when it is actionable The throttle is checked FIRST, from persisted state, before any file is opened, and worthSurfacing withholds the panel unless the runway is actionable AND has moved since the last one SHOWN -- not the last one computed, which would let the runway drift down one window at a time and never trip the threshold. `used` is the full input size of the most recent request -- cache reads plus cache writes plus fresh input -- which is what the window actually holds. Summing per-turn inputs would count the carried prefix once per turn and report a number many times the window size, making every session look like an emergency. TWO SUPPORTING CHANGES, both of which would be silent defects if omitted: - logForecast now records the turn it was made on. predictedTurns is an INTERVAL, so without its origin the caller at compaction has nothing to subtract from; observeOutcome takes atTurn and derives the elapsed count next to the record that defines it. A forecast with no recorded origin is left OPEN rather than scored against an absolute turn number, which would manufacture an error rather than measure one. - policy state carries a `forecast` field through loadState and saveState. The two comments already in that file explain why: a per-session value not threaded through resets on every call, so checkedAt would rebuild the panel on every tool use and `shown` would re-interrupt with the same runway forever. Merged latest-wins rather than unioned, because unlike every other field there it is a point in time, and taking the older of two concurrent checks reopens the window the throttle exists to close. FIVE ENTRIES LEFT THE REACHABILITY ALLOWLIST: logForecast, calibrate, forecastPanel, worthSurfacing and observeOutcome. The guard now agrees the feature is reachable, which is the only evidence worth offering here -- every test in the new file except the last three would pass just as well on a module nothing imports. Verified: tests/hooks 744 passed, 2 skipped, 0 failed, with 16 new tests. tsc --noEmit clean. Noticed and NOT changed here: plugin/hooks/*.mjs carry a UTF-8 BOM before the shebang, which makes `node --check` reject them and would stop a direct ./hook.mjs execution on Unix. They are invoked as `node hook.mjs`, so the shebang is decorative and nothing is broken today. Pre-existing and out of scope. Co-Authored-By: Claude Opus 5 --- hooks-core/calibration.mjs | 39 ++- hooks-core/forecast.mjs | 3 + hooks-core/policy.mjs | 24 +- hooks-core/surface.mjs | 164 +++++++++++++ integrations/codex/hooks/lib/calibration.mjs | 39 ++- integrations/codex/hooks/lib/forecast.mjs | 3 + integrations/codex/hooks/lib/policy.mjs | 24 +- integrations/codex/hooks/lib/surface.mjs | 166 +++++++++++++ .../codex/plugin/hooks/lib/calibration.mjs | 39 ++- .../codex/plugin/hooks/lib/forecast.mjs | 3 + .../codex/plugin/hooks/lib/policy.mjs | 24 +- .../codex/plugin/hooks/lib/surface.mjs | 166 +++++++++++++ integrations/gemini/hooks/lib/calibration.mjs | 39 ++- integrations/gemini/hooks/lib/forecast.mjs | 3 + integrations/gemini/hooks/lib/policy.mjs | 24 +- integrations/gemini/hooks/lib/surface.mjs | 166 +++++++++++++ .../opencode/hooks/lib/calibration.mjs | 39 ++- integrations/opencode/hooks/lib/forecast.mjs | 3 + integrations/opencode/hooks/lib/policy.mjs | 24 +- integrations/opencode/hooks/lib/surface.mjs | 166 +++++++++++++ integrations/qwen/hooks/lib/calibration.mjs | 39 ++- integrations/qwen/hooks/lib/forecast.mjs | 3 + integrations/qwen/hooks/lib/policy.mjs | 24 +- integrations/qwen/hooks/lib/surface.mjs | 166 +++++++++++++ plugin/hooks/lib/calibration.mjs | 39 ++- plugin/hooks/lib/forecast.mjs | 3 + plugin/hooks/lib/policy.mjs | 24 +- plugin/hooks/lib/surface.mjs | 166 +++++++++++++ plugin/hooks/precompact-optimize.mjs | 21 ++ plugin/hooks/pretooluse-router.mjs | 23 ++ tests/hooks/reachability.test.mjs | 19 +- tests/hooks/surface.test.mjs | 223 ++++++++++++++++++ 32 files changed, 1839 insertions(+), 69 deletions(-) create mode 100644 hooks-core/surface.mjs create mode 100644 integrations/codex/hooks/lib/surface.mjs create mode 100644 integrations/codex/plugin/hooks/lib/surface.mjs create mode 100644 integrations/gemini/hooks/lib/surface.mjs create mode 100644 integrations/opencode/hooks/lib/surface.mjs create mode 100644 integrations/qwen/hooks/lib/surface.mjs create mode 100644 plugin/hooks/lib/surface.mjs create mode 100644 tests/hooks/surface.test.mjs diff --git a/hooks-core/calibration.mjs b/hooks-core/calibration.mjs index f0d83c2d4..278eb6616 100644 --- a/hooks-core/calibration.mjs +++ b/hooks-core/calibration.mjs @@ -64,8 +64,15 @@ function horizonOf(turns) { return HORIZONS.find((h) => turns <= h.max).name; } -/** Records a prediction so it can be scored when the outcome arrives. */ -export function logForecast(dir, { sessionId, predictedTurns, used, capacity }) { +/** + * Records a prediction so it can be scored when the outcome arrives. + * + * `turns` is the session's turn count AT PREDICTION TIME, and it is what makes the outcome + * computable at all: predictedTurns means "turns from here until compaction", so the ground truth + * is the turns that ELAPSED, not the turn number compaction happened on. Without it the caller at + * compaction has nothing to subtract from and would have to score an interval against an absolute. + */ +export function logForecast(dir, { sessionId, predictedTurns, used, capacity, turns }) { if (!Number.isFinite(predictedTurns)) return; record(dir, { kind: 'forecast', @@ -74,6 +81,7 @@ export function logForecast(dir, { sessionId, predictedTurns, used, capacity }) horizon: horizonOf(predictedTurns), used, capacity, + turns: Number.isFinite(turns) ? turns : null, }); } @@ -84,8 +92,17 @@ export function logForecast(dir, { sessionId, predictedTurns, used, capacity }) * Called when compaction fires: the number of turns that elapsed since the * prediction is the ground truth it was predicting. */ -export function observeOutcome(dir, { sessionId, actualTurns }) { - if (!Number.isFinite(actualTurns)) return; +export function observeOutcome(dir, { sessionId, actualTurns, atTurn }) { + // EITHER THE ELAPSED COUNT OR THE TURN IT HAPPENED ON. + // + // The only real caller -- the PreCompact hook -- knows what turn compaction fired on, not how + // many turns have passed since a prediction it did not make. Deriving the interval here keeps + // that subtraction next to the record that defines it, rather than making every caller + // re-discover which forecast is open in order to compute its own ground truth. + const elapsed = Number.isFinite(actualTurns) + ? actualTurns + : null; + if (elapsed === null && !Number.isFinite(atTurn)) return; // CLOSURE IS DERIVED FROM THE OUTCOME ROWS, not from a flag on the forecast. // @@ -109,6 +126,14 @@ export function observeOutcome(dir, { sessionId, actualTurns }) { .pop(); if (!open) return; + // Derived only once the open forecast is known, since the interval is measured from ITS turn. + // A forecast recorded before `turns` existed has none, and there is nothing to subtract from -- + // scoring it against an absolute turn number would manufacture an error rather than measure one, + // so it is left open instead. + const actual = elapsed !== null ? elapsed + : (Number.isFinite(open.turns) ? atTurn - open.turns : null); + if (actual === null || !Number.isFinite(actual)) return; + record(dir, { kind: 'forecast-outcome', sessionId, @@ -117,9 +142,9 @@ export function observeOutcome(dir, { sessionId, actualTurns }) { forecastId: open.id, horizon: open.horizon, predictedTurns: open.predictedTurns, - actualTurns, - error: actualTurns - open.predictedTurns, - hit: Math.abs(actualTurns - open.predictedTurns) <= TOLERANCE, + actualTurns: actual, + error: actual - open.predictedTurns, + hit: Math.abs(actual - open.predictedTurns) <= TOLERANCE, }); } diff --git a/hooks-core/forecast.mjs b/hooks-core/forecast.mjs index d514c313a..7779e1ca5 100644 --- a/hooks-core/forecast.mjs +++ b/hooks-core/forecast.mjs @@ -326,6 +326,9 @@ export function forecastPanel(dir, session = {}, findings = []) { predictedTurns: air.withGraph, used: session.used, capacity: session.capacity, + // The turn this was predicted FROM. predictedTurns is an interval, so without its origin + // the outcome at compaction has nothing to subtract from. + turns: session.turns, }); const score = calibrate(dir, air.withGraph); diff --git a/hooks-core/policy.mjs b/hooks-core/policy.mjs index bbe5644fd..ecbde570b 100644 --- a/hooks-core/policy.mjs +++ b/hooks-core/policy.mjs @@ -270,7 +270,7 @@ function statePath(sessionId, agent) { /** A usable state object, whatever was on disk. */ function emptyState() { - return { seen: {}, denied: {}, injected: [], actCounts: {} }; + return { seen: {}, denied: {}, injected: [], actCounts: {}, forecast: null }; } /** @@ -310,6 +310,16 @@ export function loadState(sessionId, agent) { parsed.actCounts && typeof parsed.actCounts === 'object' && !Array.isArray(parsed.actCounts) ? parsed.actCounts : {}, + // THE SAME REASON A THIRD TIME. The forecast throttle records when the panel was last + // computed and what runway it last SHOWED, and both are meaningless within a single hook + // process -- every tool call is a new one. Dropped here, `checkedAt` would reset on every + // call and the panel would rebuild itself, transcript parse and all, on each tool use; and + // `shown` would reset, so worthSurfacing would compare against nothing and re-interrupt with + // the same runway forever. + forecast: + parsed.forecast && typeof parsed.forecast === 'object' && !Array.isArray(parsed.forecast) + ? parsed.forecast + : null, }; } catch { return emptyState(); @@ -381,6 +391,18 @@ export function saveState(sessionId, state, agent) { } return out; })(), + // LATEST CHECK WINS, which is the opposite direction from the fields above and correct for + // this one. `seen`, `denied`, `injected` and `actCounts` are all append-only, so a union or + // a max is the safe merge. The forecast throttle is a POINT IN TIME: taking the older of two + // concurrent checks would re-open the window and let the panel be rebuilt immediately, which + // is the cost the throttle exists to bound. + forecast: (() => { + const mine = state.forecast || null; + const theirs = current.forecast || null; + if (!mine) return theirs; + if (!theirs) return mine; + return (mine.checkedAt || 0) >= (theirs.checkedAt || 0) ? mine : theirs; + })(), }; // Write-then-rename so a reader never observes a half-written file. diff --git a/hooks-core/surface.mjs b/hooks-core/surface.mjs new file mode 100644 index 000000000..a4dff3282 --- /dev/null +++ b/hooks-core/surface.mjs @@ -0,0 +1,164 @@ +/** + * The entry point the forecast never had. + * + * forecast.mjs computes a runway, calibration.mjs scores it, and worthSurfacing decides whether a + * change is worth interrupting for -- and none of it ran. The reachability guard listed + * forecastPanel, worthSurfacing, logForecast and observeOutcome as unwired, which is a polite way + * of saying the feature did not exist for any user. This module is the missing half: it derives + * the session numbers the panel needs from the transcript, decides when the panel has earned an + * interruption, and closes the calibration loop when compaction fires. + * + * THREE COSTS ARE BOUNDED HERE, because this runs on the PreToolUse path and every tool call is a + * separate process: + * + * THE READ readCacheUsage parses up to 4 MB of transcript. Doing that on every tool call + * would tax the hook that exists to save the user money. + * THE COMPUTE the panel walks the metrics log, the balance log and the graph. + * THE CONTEXT the panel is only worth its own tokens when it says something actionable. + * + * So the throttle is checked FIRST, from persisted state, before any file is opened. Everything + * expensive happens after a clock comparison. + * + * NOTHING HERE IS ALLOWED TO BREAK A TOOL CALL. Every entry point returns null on failure rather + * than throwing: a forecast is a courtesy, and the hook's promise is that a defect in it costs the + * user nothing. + */ + +import { forecastPanel, worthSurfacing } from './forecast.mjs'; +import { observeOutcome } from './calibration.mjs'; +import { readCacheUsage } from './cache.mjs'; +import { readBalance } from './metrics.mjs'; + +/** + * How long between two attempts to build the panel. + * + * Not a tuning knob so much as an admission: the panel costs a transcript parse, and a user makes + * many tool calls per minute. Two minutes is short enough that a runway falling into single digits + * is still caught with turns to spare, and long enough that the check is free in aggregate. + */ +export const SURFACE_INTERVAL_MS = + Number(process.env.TOKEN_OPTIMIZER_SURFACE_INTERVAL_MS) || 120_000; + +/** + * The context window to measure the runway against. + * + * NOT DISCOVERABLE from the transcript: the usage rows carry what was spent, never the ceiling it + * was spent against. Defaulting is therefore unavoidable, so the default is stated rather than + * buried, and overridable for anyone running a different window. + */ +export const DEFAULT_CAPACITY = + Number(process.env.TOKEN_OPTIMIZER_CONTEXT_CAPACITY) || 200_000; + +/** + * What the session has spent, read from its own transcript. + * + * `used` is the FULL input size of the most recent request -- cache reads plus cache writes plus + * fresh input -- which is what the context window actually holds, rather than the marginal cost of + * that one turn. Summing the per-turn inputs instead would count the same carried prefix once per + * turn and report a number many times the window size. + * + * Returns null rather than a zeroed object when there is nothing to measure, so a caller cannot + * mistake "no data" for "an empty context". + */ +export function sessionUsage(transcriptPath, { capacity = DEFAULT_CAPACITY } = {}) { + let turns; + try { + turns = readCacheUsage(transcriptPath); + } catch { + return null; + } + if (!turns?.length) return null; + + const last = turns[turns.length - 1]; + const used = (last.read || 0) + (last.written || 0) + (last.input || 0); + if (!used) return null; + + return { used, capacity, turns: turns.length }; +} + +/** Intercepted touches recorded for this session, which converts a per-touch saving to per-turn. */ +function touchesFor(dir, sessionId) { + try { + return readBalance(dir).filter((e) => e.kind === 'inject' && e.sessionId === sessionId).length; + } catch { + return 0; + } +} + +/** + * Should the forecast interrupt right now, and with what? + * + * The state object is the caller's persisted per-session state; this reads `forecast` from it and + * returns the value to store back, so the throttle and the last-shown runway survive across hook + * processes. Returning the next state rather than mutating it keeps this testable without a + * filesystem. + * + * @returns { text, state } -- text is null when nothing should be shown. + */ +export function maybeSurface(dir, { + transcriptPath, sessionId, state = {}, now = Date.now(), findings = [], +} = {}) { + const previous = state.forecast || null; + + // THE THROTTLE IS FIRST, and deliberately before any I/O. Everything below opens files. + if (previous?.checkedAt && now - previous.checkedAt < SURFACE_INTERVAL_MS) { + return { text: null, state: previous }; + } + + let usage; + try { + usage = sessionUsage(transcriptPath); + } catch { + return { text: null, state: previous }; + } + // The clock is stamped even when there is nothing to measure, so a session with no usage rows + // does not re-attempt the transcript parse on every single tool call. + if (!usage) return { text: null, state: { ...previous, checkedAt: now } }; + + let panel; + try { + panel = forecastPanel(dir, { + ...usage, + sessionId, + touches: touchesFor(dir, sessionId), + }, findings); + } catch { + return { text: null, state: { ...previous, checkedAt: now } }; + } + if (!panel) return { text: null, state: { ...previous, checkedAt: now } }; + + // worthSurfacing compares against the last panel that was actually SHOWN, not the last one + // computed. Comparing against the last computed panel would let the runway drift down past the + // threshold one throttle window at a time and never trip the "crossed it" test. + const shown = previous?.shown ? { parts: { runway: { withGraph: previous.shown } } } : null; + if (!worthSurfacing(panel, shown)) { + return { text: null, state: { ...previous, checkedAt: now } }; + } + + return { + text: panel.text, + state: { checkedAt: now, shown: panel.parts.runway.withGraph }, + }; +} + +/** + * Closes the calibration loop, at the only moment that can close it. + * + * Compaction firing IS the ground truth a runway forecast was predicting, so this belongs on the + * PreCompact path and nowhere else. `actualTurns` is turns ELAPSED between the prediction and + * compaction, which is what predictedTurns meant -- so the forecast has to have recorded the turn + * it was made on, and observeOutcome subtracts. + * + * Silent by design: nothing is shown to anybody. It writes one record so that the NEXT session's + * forecast can be published with a track record instead of as a bare assertion. + */ +export function closeForecast(dir, { transcriptPath, sessionId } = {}) { + try { + const usage = sessionUsage(transcriptPath); + if (!usage) return false; + observeOutcome(dir, { sessionId, atTurn: usage.turns }); + return true; + } catch { + return false; + } +} diff --git a/integrations/codex/hooks/lib/calibration.mjs b/integrations/codex/hooks/lib/calibration.mjs index e920a9992..42de126f9 100644 --- a/integrations/codex/hooks/lib/calibration.mjs +++ b/integrations/codex/hooks/lib/calibration.mjs @@ -66,8 +66,15 @@ function horizonOf(turns) { return HORIZONS.find((h) => turns <= h.max).name; } -/** Records a prediction so it can be scored when the outcome arrives. */ -export function logForecast(dir, { sessionId, predictedTurns, used, capacity }) { +/** + * Records a prediction so it can be scored when the outcome arrives. + * + * `turns` is the session's turn count AT PREDICTION TIME, and it is what makes the outcome + * computable at all: predictedTurns means "turns from here until compaction", so the ground truth + * is the turns that ELAPSED, not the turn number compaction happened on. Without it the caller at + * compaction has nothing to subtract from and would have to score an interval against an absolute. + */ +export function logForecast(dir, { sessionId, predictedTurns, used, capacity, turns }) { if (!Number.isFinite(predictedTurns)) return; record(dir, { kind: 'forecast', @@ -76,6 +83,7 @@ export function logForecast(dir, { sessionId, predictedTurns, used, capacity }) horizon: horizonOf(predictedTurns), used, capacity, + turns: Number.isFinite(turns) ? turns : null, }); } @@ -86,8 +94,17 @@ export function logForecast(dir, { sessionId, predictedTurns, used, capacity }) * Called when compaction fires: the number of turns that elapsed since the * prediction is the ground truth it was predicting. */ -export function observeOutcome(dir, { sessionId, actualTurns }) { - if (!Number.isFinite(actualTurns)) return; +export function observeOutcome(dir, { sessionId, actualTurns, atTurn }) { + // EITHER THE ELAPSED COUNT OR THE TURN IT HAPPENED ON. + // + // The only real caller -- the PreCompact hook -- knows what turn compaction fired on, not how + // many turns have passed since a prediction it did not make. Deriving the interval here keeps + // that subtraction next to the record that defines it, rather than making every caller + // re-discover which forecast is open in order to compute its own ground truth. + const elapsed = Number.isFinite(actualTurns) + ? actualTurns + : null; + if (elapsed === null && !Number.isFinite(atTurn)) return; // CLOSURE IS DERIVED FROM THE OUTCOME ROWS, not from a flag on the forecast. // @@ -111,6 +128,14 @@ export function observeOutcome(dir, { sessionId, actualTurns }) { .pop(); if (!open) return; + // Derived only once the open forecast is known, since the interval is measured from ITS turn. + // A forecast recorded before `turns` existed has none, and there is nothing to subtract from -- + // scoring it against an absolute turn number would manufacture an error rather than measure one, + // so it is left open instead. + const actual = elapsed !== null ? elapsed + : (Number.isFinite(open.turns) ? atTurn - open.turns : null); + if (actual === null || !Number.isFinite(actual)) return; + record(dir, { kind: 'forecast-outcome', sessionId, @@ -119,9 +144,9 @@ export function observeOutcome(dir, { sessionId, actualTurns }) { forecastId: open.id, horizon: open.horizon, predictedTurns: open.predictedTurns, - actualTurns, - error: actualTurns - open.predictedTurns, - hit: Math.abs(actualTurns - open.predictedTurns) <= TOLERANCE, + actualTurns: actual, + error: actual - open.predictedTurns, + hit: Math.abs(actual - open.predictedTurns) <= TOLERANCE, }); } diff --git a/integrations/codex/hooks/lib/forecast.mjs b/integrations/codex/hooks/lib/forecast.mjs index dee6a3b56..39358e76a 100644 --- a/integrations/codex/hooks/lib/forecast.mjs +++ b/integrations/codex/hooks/lib/forecast.mjs @@ -328,6 +328,9 @@ export function forecastPanel(dir, session = {}, findings = []) { predictedTurns: air.withGraph, used: session.used, capacity: session.capacity, + // The turn this was predicted FROM. predictedTurns is an interval, so without its origin + // the outcome at compaction has nothing to subtract from. + turns: session.turns, }); const score = calibrate(dir, air.withGraph); diff --git a/integrations/codex/hooks/lib/policy.mjs b/integrations/codex/hooks/lib/policy.mjs index b3871c27d..1fa14c348 100644 --- a/integrations/codex/hooks/lib/policy.mjs +++ b/integrations/codex/hooks/lib/policy.mjs @@ -272,7 +272,7 @@ function statePath(sessionId, agent) { /** A usable state object, whatever was on disk. */ function emptyState() { - return { seen: {}, denied: {}, injected: [], actCounts: {} }; + return { seen: {}, denied: {}, injected: [], actCounts: {}, forecast: null }; } /** @@ -312,6 +312,16 @@ export function loadState(sessionId, agent) { parsed.actCounts && typeof parsed.actCounts === 'object' && !Array.isArray(parsed.actCounts) ? parsed.actCounts : {}, + // THE SAME REASON A THIRD TIME. The forecast throttle records when the panel was last + // computed and what runway it last SHOWED, and both are meaningless within a single hook + // process -- every tool call is a new one. Dropped here, `checkedAt` would reset on every + // call and the panel would rebuild itself, transcript parse and all, on each tool use; and + // `shown` would reset, so worthSurfacing would compare against nothing and re-interrupt with + // the same runway forever. + forecast: + parsed.forecast && typeof parsed.forecast === 'object' && !Array.isArray(parsed.forecast) + ? parsed.forecast + : null, }; } catch { return emptyState(); @@ -383,6 +393,18 @@ export function saveState(sessionId, state, agent) { } return out; })(), + // LATEST CHECK WINS, which is the opposite direction from the fields above and correct for + // this one. `seen`, `denied`, `injected` and `actCounts` are all append-only, so a union or + // a max is the safe merge. The forecast throttle is a POINT IN TIME: taking the older of two + // concurrent checks would re-open the window and let the panel be rebuilt immediately, which + // is the cost the throttle exists to bound. + forecast: (() => { + const mine = state.forecast || null; + const theirs = current.forecast || null; + if (!mine) return theirs; + if (!theirs) return mine; + return (mine.checkedAt || 0) >= (theirs.checkedAt || 0) ? mine : theirs; + })(), }; // Write-then-rename so a reader never observes a half-written file. diff --git a/integrations/codex/hooks/lib/surface.mjs b/integrations/codex/hooks/lib/surface.mjs new file mode 100644 index 000000000..0ec05161c --- /dev/null +++ b/integrations/codex/hooks/lib/surface.mjs @@ -0,0 +1,166 @@ +// GENERATED FILE -- do not edit. +// Source of truth: hooks-core/surface.mjs. Regenerate with `npm run sync:hooks`. +/** + * The entry point the forecast never had. + * + * forecast.mjs computes a runway, calibration.mjs scores it, and worthSurfacing decides whether a + * change is worth interrupting for -- and none of it ran. The reachability guard listed + * forecastPanel, worthSurfacing, logForecast and observeOutcome as unwired, which is a polite way + * of saying the feature did not exist for any user. This module is the missing half: it derives + * the session numbers the panel needs from the transcript, decides when the panel has earned an + * interruption, and closes the calibration loop when compaction fires. + * + * THREE COSTS ARE BOUNDED HERE, because this runs on the PreToolUse path and every tool call is a + * separate process: + * + * THE READ readCacheUsage parses up to 4 MB of transcript. Doing that on every tool call + * would tax the hook that exists to save the user money. + * THE COMPUTE the panel walks the metrics log, the balance log and the graph. + * THE CONTEXT the panel is only worth its own tokens when it says something actionable. + * + * So the throttle is checked FIRST, from persisted state, before any file is opened. Everything + * expensive happens after a clock comparison. + * + * NOTHING HERE IS ALLOWED TO BREAK A TOOL CALL. Every entry point returns null on failure rather + * than throwing: a forecast is a courtesy, and the hook's promise is that a defect in it costs the + * user nothing. + */ + +import { forecastPanel, worthSurfacing } from './forecast.mjs'; +import { observeOutcome } from './calibration.mjs'; +import { readCacheUsage } from './cache.mjs'; +import { readBalance } from './metrics.mjs'; + +/** + * How long between two attempts to build the panel. + * + * Not a tuning knob so much as an admission: the panel costs a transcript parse, and a user makes + * many tool calls per minute. Two minutes is short enough that a runway falling into single digits + * is still caught with turns to spare, and long enough that the check is free in aggregate. + */ +export const SURFACE_INTERVAL_MS = + Number(process.env.TOKEN_OPTIMIZER_SURFACE_INTERVAL_MS) || 120_000; + +/** + * The context window to measure the runway against. + * + * NOT DISCOVERABLE from the transcript: the usage rows carry what was spent, never the ceiling it + * was spent against. Defaulting is therefore unavoidable, so the default is stated rather than + * buried, and overridable for anyone running a different window. + */ +export const DEFAULT_CAPACITY = + Number(process.env.TOKEN_OPTIMIZER_CONTEXT_CAPACITY) || 200_000; + +/** + * What the session has spent, read from its own transcript. + * + * `used` is the FULL input size of the most recent request -- cache reads plus cache writes plus + * fresh input -- which is what the context window actually holds, rather than the marginal cost of + * that one turn. Summing the per-turn inputs instead would count the same carried prefix once per + * turn and report a number many times the window size. + * + * Returns null rather than a zeroed object when there is nothing to measure, so a caller cannot + * mistake "no data" for "an empty context". + */ +export function sessionUsage(transcriptPath, { capacity = DEFAULT_CAPACITY } = {}) { + let turns; + try { + turns = readCacheUsage(transcriptPath); + } catch { + return null; + } + if (!turns?.length) return null; + + const last = turns[turns.length - 1]; + const used = (last.read || 0) + (last.written || 0) + (last.input || 0); + if (!used) return null; + + return { used, capacity, turns: turns.length }; +} + +/** Intercepted touches recorded for this session, which converts a per-touch saving to per-turn. */ +function touchesFor(dir, sessionId) { + try { + return readBalance(dir).filter((e) => e.kind === 'inject' && e.sessionId === sessionId).length; + } catch { + return 0; + } +} + +/** + * Should the forecast interrupt right now, and with what? + * + * The state object is the caller's persisted per-session state; this reads `forecast` from it and + * returns the value to store back, so the throttle and the last-shown runway survive across hook + * processes. Returning the next state rather than mutating it keeps this testable without a + * filesystem. + * + * @returns { text, state } -- text is null when nothing should be shown. + */ +export function maybeSurface(dir, { + transcriptPath, sessionId, state = {}, now = Date.now(), findings = [], +} = {}) { + const previous = state.forecast || null; + + // THE THROTTLE IS FIRST, and deliberately before any I/O. Everything below opens files. + if (previous?.checkedAt && now - previous.checkedAt < SURFACE_INTERVAL_MS) { + return { text: null, state: previous }; + } + + let usage; + try { + usage = sessionUsage(transcriptPath); + } catch { + return { text: null, state: previous }; + } + // The clock is stamped even when there is nothing to measure, so a session with no usage rows + // does not re-attempt the transcript parse on every single tool call. + if (!usage) return { text: null, state: { ...previous, checkedAt: now } }; + + let panel; + try { + panel = forecastPanel(dir, { + ...usage, + sessionId, + touches: touchesFor(dir, sessionId), + }, findings); + } catch { + return { text: null, state: { ...previous, checkedAt: now } }; + } + if (!panel) return { text: null, state: { ...previous, checkedAt: now } }; + + // worthSurfacing compares against the last panel that was actually SHOWN, not the last one + // computed. Comparing against the last computed panel would let the runway drift down past the + // threshold one throttle window at a time and never trip the "crossed it" test. + const shown = previous?.shown ? { parts: { runway: { withGraph: previous.shown } } } : null; + if (!worthSurfacing(panel, shown)) { + return { text: null, state: { ...previous, checkedAt: now } }; + } + + return { + text: panel.text, + state: { checkedAt: now, shown: panel.parts.runway.withGraph }, + }; +} + +/** + * Closes the calibration loop, at the only moment that can close it. + * + * Compaction firing IS the ground truth a runway forecast was predicting, so this belongs on the + * PreCompact path and nowhere else. `actualTurns` is turns ELAPSED between the prediction and + * compaction, which is what predictedTurns meant -- so the forecast has to have recorded the turn + * it was made on, and observeOutcome subtracts. + * + * Silent by design: nothing is shown to anybody. It writes one record so that the NEXT session's + * forecast can be published with a track record instead of as a bare assertion. + */ +export function closeForecast(dir, { transcriptPath, sessionId } = {}) { + try { + const usage = sessionUsage(transcriptPath); + if (!usage) return false; + observeOutcome(dir, { sessionId, atTurn: usage.turns }); + return true; + } catch { + return false; + } +} diff --git a/integrations/codex/plugin/hooks/lib/calibration.mjs b/integrations/codex/plugin/hooks/lib/calibration.mjs index e920a9992..42de126f9 100644 --- a/integrations/codex/plugin/hooks/lib/calibration.mjs +++ b/integrations/codex/plugin/hooks/lib/calibration.mjs @@ -66,8 +66,15 @@ function horizonOf(turns) { return HORIZONS.find((h) => turns <= h.max).name; } -/** Records a prediction so it can be scored when the outcome arrives. */ -export function logForecast(dir, { sessionId, predictedTurns, used, capacity }) { +/** + * Records a prediction so it can be scored when the outcome arrives. + * + * `turns` is the session's turn count AT PREDICTION TIME, and it is what makes the outcome + * computable at all: predictedTurns means "turns from here until compaction", so the ground truth + * is the turns that ELAPSED, not the turn number compaction happened on. Without it the caller at + * compaction has nothing to subtract from and would have to score an interval against an absolute. + */ +export function logForecast(dir, { sessionId, predictedTurns, used, capacity, turns }) { if (!Number.isFinite(predictedTurns)) return; record(dir, { kind: 'forecast', @@ -76,6 +83,7 @@ export function logForecast(dir, { sessionId, predictedTurns, used, capacity }) horizon: horizonOf(predictedTurns), used, capacity, + turns: Number.isFinite(turns) ? turns : null, }); } @@ -86,8 +94,17 @@ export function logForecast(dir, { sessionId, predictedTurns, used, capacity }) * Called when compaction fires: the number of turns that elapsed since the * prediction is the ground truth it was predicting. */ -export function observeOutcome(dir, { sessionId, actualTurns }) { - if (!Number.isFinite(actualTurns)) return; +export function observeOutcome(dir, { sessionId, actualTurns, atTurn }) { + // EITHER THE ELAPSED COUNT OR THE TURN IT HAPPENED ON. + // + // The only real caller -- the PreCompact hook -- knows what turn compaction fired on, not how + // many turns have passed since a prediction it did not make. Deriving the interval here keeps + // that subtraction next to the record that defines it, rather than making every caller + // re-discover which forecast is open in order to compute its own ground truth. + const elapsed = Number.isFinite(actualTurns) + ? actualTurns + : null; + if (elapsed === null && !Number.isFinite(atTurn)) return; // CLOSURE IS DERIVED FROM THE OUTCOME ROWS, not from a flag on the forecast. // @@ -111,6 +128,14 @@ export function observeOutcome(dir, { sessionId, actualTurns }) { .pop(); if (!open) return; + // Derived only once the open forecast is known, since the interval is measured from ITS turn. + // A forecast recorded before `turns` existed has none, and there is nothing to subtract from -- + // scoring it against an absolute turn number would manufacture an error rather than measure one, + // so it is left open instead. + const actual = elapsed !== null ? elapsed + : (Number.isFinite(open.turns) ? atTurn - open.turns : null); + if (actual === null || !Number.isFinite(actual)) return; + record(dir, { kind: 'forecast-outcome', sessionId, @@ -119,9 +144,9 @@ export function observeOutcome(dir, { sessionId, actualTurns }) { forecastId: open.id, horizon: open.horizon, predictedTurns: open.predictedTurns, - actualTurns, - error: actualTurns - open.predictedTurns, - hit: Math.abs(actualTurns - open.predictedTurns) <= TOLERANCE, + actualTurns: actual, + error: actual - open.predictedTurns, + hit: Math.abs(actual - open.predictedTurns) <= TOLERANCE, }); } diff --git a/integrations/codex/plugin/hooks/lib/forecast.mjs b/integrations/codex/plugin/hooks/lib/forecast.mjs index dee6a3b56..39358e76a 100644 --- a/integrations/codex/plugin/hooks/lib/forecast.mjs +++ b/integrations/codex/plugin/hooks/lib/forecast.mjs @@ -328,6 +328,9 @@ export function forecastPanel(dir, session = {}, findings = []) { predictedTurns: air.withGraph, used: session.used, capacity: session.capacity, + // The turn this was predicted FROM. predictedTurns is an interval, so without its origin + // the outcome at compaction has nothing to subtract from. + turns: session.turns, }); const score = calibrate(dir, air.withGraph); diff --git a/integrations/codex/plugin/hooks/lib/policy.mjs b/integrations/codex/plugin/hooks/lib/policy.mjs index b3871c27d..1fa14c348 100644 --- a/integrations/codex/plugin/hooks/lib/policy.mjs +++ b/integrations/codex/plugin/hooks/lib/policy.mjs @@ -272,7 +272,7 @@ function statePath(sessionId, agent) { /** A usable state object, whatever was on disk. */ function emptyState() { - return { seen: {}, denied: {}, injected: [], actCounts: {} }; + return { seen: {}, denied: {}, injected: [], actCounts: {}, forecast: null }; } /** @@ -312,6 +312,16 @@ export function loadState(sessionId, agent) { parsed.actCounts && typeof parsed.actCounts === 'object' && !Array.isArray(parsed.actCounts) ? parsed.actCounts : {}, + // THE SAME REASON A THIRD TIME. The forecast throttle records when the panel was last + // computed and what runway it last SHOWED, and both are meaningless within a single hook + // process -- every tool call is a new one. Dropped here, `checkedAt` would reset on every + // call and the panel would rebuild itself, transcript parse and all, on each tool use; and + // `shown` would reset, so worthSurfacing would compare against nothing and re-interrupt with + // the same runway forever. + forecast: + parsed.forecast && typeof parsed.forecast === 'object' && !Array.isArray(parsed.forecast) + ? parsed.forecast + : null, }; } catch { return emptyState(); @@ -383,6 +393,18 @@ export function saveState(sessionId, state, agent) { } return out; })(), + // LATEST CHECK WINS, which is the opposite direction from the fields above and correct for + // this one. `seen`, `denied`, `injected` and `actCounts` are all append-only, so a union or + // a max is the safe merge. The forecast throttle is a POINT IN TIME: taking the older of two + // concurrent checks would re-open the window and let the panel be rebuilt immediately, which + // is the cost the throttle exists to bound. + forecast: (() => { + const mine = state.forecast || null; + const theirs = current.forecast || null; + if (!mine) return theirs; + if (!theirs) return mine; + return (mine.checkedAt || 0) >= (theirs.checkedAt || 0) ? mine : theirs; + })(), }; // Write-then-rename so a reader never observes a half-written file. diff --git a/integrations/codex/plugin/hooks/lib/surface.mjs b/integrations/codex/plugin/hooks/lib/surface.mjs new file mode 100644 index 000000000..0ec05161c --- /dev/null +++ b/integrations/codex/plugin/hooks/lib/surface.mjs @@ -0,0 +1,166 @@ +// GENERATED FILE -- do not edit. +// Source of truth: hooks-core/surface.mjs. Regenerate with `npm run sync:hooks`. +/** + * The entry point the forecast never had. + * + * forecast.mjs computes a runway, calibration.mjs scores it, and worthSurfacing decides whether a + * change is worth interrupting for -- and none of it ran. The reachability guard listed + * forecastPanel, worthSurfacing, logForecast and observeOutcome as unwired, which is a polite way + * of saying the feature did not exist for any user. This module is the missing half: it derives + * the session numbers the panel needs from the transcript, decides when the panel has earned an + * interruption, and closes the calibration loop when compaction fires. + * + * THREE COSTS ARE BOUNDED HERE, because this runs on the PreToolUse path and every tool call is a + * separate process: + * + * THE READ readCacheUsage parses up to 4 MB of transcript. Doing that on every tool call + * would tax the hook that exists to save the user money. + * THE COMPUTE the panel walks the metrics log, the balance log and the graph. + * THE CONTEXT the panel is only worth its own tokens when it says something actionable. + * + * So the throttle is checked FIRST, from persisted state, before any file is opened. Everything + * expensive happens after a clock comparison. + * + * NOTHING HERE IS ALLOWED TO BREAK A TOOL CALL. Every entry point returns null on failure rather + * than throwing: a forecast is a courtesy, and the hook's promise is that a defect in it costs the + * user nothing. + */ + +import { forecastPanel, worthSurfacing } from './forecast.mjs'; +import { observeOutcome } from './calibration.mjs'; +import { readCacheUsage } from './cache.mjs'; +import { readBalance } from './metrics.mjs'; + +/** + * How long between two attempts to build the panel. + * + * Not a tuning knob so much as an admission: the panel costs a transcript parse, and a user makes + * many tool calls per minute. Two minutes is short enough that a runway falling into single digits + * is still caught with turns to spare, and long enough that the check is free in aggregate. + */ +export const SURFACE_INTERVAL_MS = + Number(process.env.TOKEN_OPTIMIZER_SURFACE_INTERVAL_MS) || 120_000; + +/** + * The context window to measure the runway against. + * + * NOT DISCOVERABLE from the transcript: the usage rows carry what was spent, never the ceiling it + * was spent against. Defaulting is therefore unavoidable, so the default is stated rather than + * buried, and overridable for anyone running a different window. + */ +export const DEFAULT_CAPACITY = + Number(process.env.TOKEN_OPTIMIZER_CONTEXT_CAPACITY) || 200_000; + +/** + * What the session has spent, read from its own transcript. + * + * `used` is the FULL input size of the most recent request -- cache reads plus cache writes plus + * fresh input -- which is what the context window actually holds, rather than the marginal cost of + * that one turn. Summing the per-turn inputs instead would count the same carried prefix once per + * turn and report a number many times the window size. + * + * Returns null rather than a zeroed object when there is nothing to measure, so a caller cannot + * mistake "no data" for "an empty context". + */ +export function sessionUsage(transcriptPath, { capacity = DEFAULT_CAPACITY } = {}) { + let turns; + try { + turns = readCacheUsage(transcriptPath); + } catch { + return null; + } + if (!turns?.length) return null; + + const last = turns[turns.length - 1]; + const used = (last.read || 0) + (last.written || 0) + (last.input || 0); + if (!used) return null; + + return { used, capacity, turns: turns.length }; +} + +/** Intercepted touches recorded for this session, which converts a per-touch saving to per-turn. */ +function touchesFor(dir, sessionId) { + try { + return readBalance(dir).filter((e) => e.kind === 'inject' && e.sessionId === sessionId).length; + } catch { + return 0; + } +} + +/** + * Should the forecast interrupt right now, and with what? + * + * The state object is the caller's persisted per-session state; this reads `forecast` from it and + * returns the value to store back, so the throttle and the last-shown runway survive across hook + * processes. Returning the next state rather than mutating it keeps this testable without a + * filesystem. + * + * @returns { text, state } -- text is null when nothing should be shown. + */ +export function maybeSurface(dir, { + transcriptPath, sessionId, state = {}, now = Date.now(), findings = [], +} = {}) { + const previous = state.forecast || null; + + // THE THROTTLE IS FIRST, and deliberately before any I/O. Everything below opens files. + if (previous?.checkedAt && now - previous.checkedAt < SURFACE_INTERVAL_MS) { + return { text: null, state: previous }; + } + + let usage; + try { + usage = sessionUsage(transcriptPath); + } catch { + return { text: null, state: previous }; + } + // The clock is stamped even when there is nothing to measure, so a session with no usage rows + // does not re-attempt the transcript parse on every single tool call. + if (!usage) return { text: null, state: { ...previous, checkedAt: now } }; + + let panel; + try { + panel = forecastPanel(dir, { + ...usage, + sessionId, + touches: touchesFor(dir, sessionId), + }, findings); + } catch { + return { text: null, state: { ...previous, checkedAt: now } }; + } + if (!panel) return { text: null, state: { ...previous, checkedAt: now } }; + + // worthSurfacing compares against the last panel that was actually SHOWN, not the last one + // computed. Comparing against the last computed panel would let the runway drift down past the + // threshold one throttle window at a time and never trip the "crossed it" test. + const shown = previous?.shown ? { parts: { runway: { withGraph: previous.shown } } } : null; + if (!worthSurfacing(panel, shown)) { + return { text: null, state: { ...previous, checkedAt: now } }; + } + + return { + text: panel.text, + state: { checkedAt: now, shown: panel.parts.runway.withGraph }, + }; +} + +/** + * Closes the calibration loop, at the only moment that can close it. + * + * Compaction firing IS the ground truth a runway forecast was predicting, so this belongs on the + * PreCompact path and nowhere else. `actualTurns` is turns ELAPSED between the prediction and + * compaction, which is what predictedTurns meant -- so the forecast has to have recorded the turn + * it was made on, and observeOutcome subtracts. + * + * Silent by design: nothing is shown to anybody. It writes one record so that the NEXT session's + * forecast can be published with a track record instead of as a bare assertion. + */ +export function closeForecast(dir, { transcriptPath, sessionId } = {}) { + try { + const usage = sessionUsage(transcriptPath); + if (!usage) return false; + observeOutcome(dir, { sessionId, atTurn: usage.turns }); + return true; + } catch { + return false; + } +} diff --git a/integrations/gemini/hooks/lib/calibration.mjs b/integrations/gemini/hooks/lib/calibration.mjs index e920a9992..42de126f9 100644 --- a/integrations/gemini/hooks/lib/calibration.mjs +++ b/integrations/gemini/hooks/lib/calibration.mjs @@ -66,8 +66,15 @@ function horizonOf(turns) { return HORIZONS.find((h) => turns <= h.max).name; } -/** Records a prediction so it can be scored when the outcome arrives. */ -export function logForecast(dir, { sessionId, predictedTurns, used, capacity }) { +/** + * Records a prediction so it can be scored when the outcome arrives. + * + * `turns` is the session's turn count AT PREDICTION TIME, and it is what makes the outcome + * computable at all: predictedTurns means "turns from here until compaction", so the ground truth + * is the turns that ELAPSED, not the turn number compaction happened on. Without it the caller at + * compaction has nothing to subtract from and would have to score an interval against an absolute. + */ +export function logForecast(dir, { sessionId, predictedTurns, used, capacity, turns }) { if (!Number.isFinite(predictedTurns)) return; record(dir, { kind: 'forecast', @@ -76,6 +83,7 @@ export function logForecast(dir, { sessionId, predictedTurns, used, capacity }) horizon: horizonOf(predictedTurns), used, capacity, + turns: Number.isFinite(turns) ? turns : null, }); } @@ -86,8 +94,17 @@ export function logForecast(dir, { sessionId, predictedTurns, used, capacity }) * Called when compaction fires: the number of turns that elapsed since the * prediction is the ground truth it was predicting. */ -export function observeOutcome(dir, { sessionId, actualTurns }) { - if (!Number.isFinite(actualTurns)) return; +export function observeOutcome(dir, { sessionId, actualTurns, atTurn }) { + // EITHER THE ELAPSED COUNT OR THE TURN IT HAPPENED ON. + // + // The only real caller -- the PreCompact hook -- knows what turn compaction fired on, not how + // many turns have passed since a prediction it did not make. Deriving the interval here keeps + // that subtraction next to the record that defines it, rather than making every caller + // re-discover which forecast is open in order to compute its own ground truth. + const elapsed = Number.isFinite(actualTurns) + ? actualTurns + : null; + if (elapsed === null && !Number.isFinite(atTurn)) return; // CLOSURE IS DERIVED FROM THE OUTCOME ROWS, not from a flag on the forecast. // @@ -111,6 +128,14 @@ export function observeOutcome(dir, { sessionId, actualTurns }) { .pop(); if (!open) return; + // Derived only once the open forecast is known, since the interval is measured from ITS turn. + // A forecast recorded before `turns` existed has none, and there is nothing to subtract from -- + // scoring it against an absolute turn number would manufacture an error rather than measure one, + // so it is left open instead. + const actual = elapsed !== null ? elapsed + : (Number.isFinite(open.turns) ? atTurn - open.turns : null); + if (actual === null || !Number.isFinite(actual)) return; + record(dir, { kind: 'forecast-outcome', sessionId, @@ -119,9 +144,9 @@ export function observeOutcome(dir, { sessionId, actualTurns }) { forecastId: open.id, horizon: open.horizon, predictedTurns: open.predictedTurns, - actualTurns, - error: actualTurns - open.predictedTurns, - hit: Math.abs(actualTurns - open.predictedTurns) <= TOLERANCE, + actualTurns: actual, + error: actual - open.predictedTurns, + hit: Math.abs(actual - open.predictedTurns) <= TOLERANCE, }); } diff --git a/integrations/gemini/hooks/lib/forecast.mjs b/integrations/gemini/hooks/lib/forecast.mjs index dee6a3b56..39358e76a 100644 --- a/integrations/gemini/hooks/lib/forecast.mjs +++ b/integrations/gemini/hooks/lib/forecast.mjs @@ -328,6 +328,9 @@ export function forecastPanel(dir, session = {}, findings = []) { predictedTurns: air.withGraph, used: session.used, capacity: session.capacity, + // The turn this was predicted FROM. predictedTurns is an interval, so without its origin + // the outcome at compaction has nothing to subtract from. + turns: session.turns, }); const score = calibrate(dir, air.withGraph); diff --git a/integrations/gemini/hooks/lib/policy.mjs b/integrations/gemini/hooks/lib/policy.mjs index b3871c27d..1fa14c348 100644 --- a/integrations/gemini/hooks/lib/policy.mjs +++ b/integrations/gemini/hooks/lib/policy.mjs @@ -272,7 +272,7 @@ function statePath(sessionId, agent) { /** A usable state object, whatever was on disk. */ function emptyState() { - return { seen: {}, denied: {}, injected: [], actCounts: {} }; + return { seen: {}, denied: {}, injected: [], actCounts: {}, forecast: null }; } /** @@ -312,6 +312,16 @@ export function loadState(sessionId, agent) { parsed.actCounts && typeof parsed.actCounts === 'object' && !Array.isArray(parsed.actCounts) ? parsed.actCounts : {}, + // THE SAME REASON A THIRD TIME. The forecast throttle records when the panel was last + // computed and what runway it last SHOWED, and both are meaningless within a single hook + // process -- every tool call is a new one. Dropped here, `checkedAt` would reset on every + // call and the panel would rebuild itself, transcript parse and all, on each tool use; and + // `shown` would reset, so worthSurfacing would compare against nothing and re-interrupt with + // the same runway forever. + forecast: + parsed.forecast && typeof parsed.forecast === 'object' && !Array.isArray(parsed.forecast) + ? parsed.forecast + : null, }; } catch { return emptyState(); @@ -383,6 +393,18 @@ export function saveState(sessionId, state, agent) { } return out; })(), + // LATEST CHECK WINS, which is the opposite direction from the fields above and correct for + // this one. `seen`, `denied`, `injected` and `actCounts` are all append-only, so a union or + // a max is the safe merge. The forecast throttle is a POINT IN TIME: taking the older of two + // concurrent checks would re-open the window and let the panel be rebuilt immediately, which + // is the cost the throttle exists to bound. + forecast: (() => { + const mine = state.forecast || null; + const theirs = current.forecast || null; + if (!mine) return theirs; + if (!theirs) return mine; + return (mine.checkedAt || 0) >= (theirs.checkedAt || 0) ? mine : theirs; + })(), }; // Write-then-rename so a reader never observes a half-written file. diff --git a/integrations/gemini/hooks/lib/surface.mjs b/integrations/gemini/hooks/lib/surface.mjs new file mode 100644 index 000000000..0ec05161c --- /dev/null +++ b/integrations/gemini/hooks/lib/surface.mjs @@ -0,0 +1,166 @@ +// GENERATED FILE -- do not edit. +// Source of truth: hooks-core/surface.mjs. Regenerate with `npm run sync:hooks`. +/** + * The entry point the forecast never had. + * + * forecast.mjs computes a runway, calibration.mjs scores it, and worthSurfacing decides whether a + * change is worth interrupting for -- and none of it ran. The reachability guard listed + * forecastPanel, worthSurfacing, logForecast and observeOutcome as unwired, which is a polite way + * of saying the feature did not exist for any user. This module is the missing half: it derives + * the session numbers the panel needs from the transcript, decides when the panel has earned an + * interruption, and closes the calibration loop when compaction fires. + * + * THREE COSTS ARE BOUNDED HERE, because this runs on the PreToolUse path and every tool call is a + * separate process: + * + * THE READ readCacheUsage parses up to 4 MB of transcript. Doing that on every tool call + * would tax the hook that exists to save the user money. + * THE COMPUTE the panel walks the metrics log, the balance log and the graph. + * THE CONTEXT the panel is only worth its own tokens when it says something actionable. + * + * So the throttle is checked FIRST, from persisted state, before any file is opened. Everything + * expensive happens after a clock comparison. + * + * NOTHING HERE IS ALLOWED TO BREAK A TOOL CALL. Every entry point returns null on failure rather + * than throwing: a forecast is a courtesy, and the hook's promise is that a defect in it costs the + * user nothing. + */ + +import { forecastPanel, worthSurfacing } from './forecast.mjs'; +import { observeOutcome } from './calibration.mjs'; +import { readCacheUsage } from './cache.mjs'; +import { readBalance } from './metrics.mjs'; + +/** + * How long between two attempts to build the panel. + * + * Not a tuning knob so much as an admission: the panel costs a transcript parse, and a user makes + * many tool calls per minute. Two minutes is short enough that a runway falling into single digits + * is still caught with turns to spare, and long enough that the check is free in aggregate. + */ +export const SURFACE_INTERVAL_MS = + Number(process.env.TOKEN_OPTIMIZER_SURFACE_INTERVAL_MS) || 120_000; + +/** + * The context window to measure the runway against. + * + * NOT DISCOVERABLE from the transcript: the usage rows carry what was spent, never the ceiling it + * was spent against. Defaulting is therefore unavoidable, so the default is stated rather than + * buried, and overridable for anyone running a different window. + */ +export const DEFAULT_CAPACITY = + Number(process.env.TOKEN_OPTIMIZER_CONTEXT_CAPACITY) || 200_000; + +/** + * What the session has spent, read from its own transcript. + * + * `used` is the FULL input size of the most recent request -- cache reads plus cache writes plus + * fresh input -- which is what the context window actually holds, rather than the marginal cost of + * that one turn. Summing the per-turn inputs instead would count the same carried prefix once per + * turn and report a number many times the window size. + * + * Returns null rather than a zeroed object when there is nothing to measure, so a caller cannot + * mistake "no data" for "an empty context". + */ +export function sessionUsage(transcriptPath, { capacity = DEFAULT_CAPACITY } = {}) { + let turns; + try { + turns = readCacheUsage(transcriptPath); + } catch { + return null; + } + if (!turns?.length) return null; + + const last = turns[turns.length - 1]; + const used = (last.read || 0) + (last.written || 0) + (last.input || 0); + if (!used) return null; + + return { used, capacity, turns: turns.length }; +} + +/** Intercepted touches recorded for this session, which converts a per-touch saving to per-turn. */ +function touchesFor(dir, sessionId) { + try { + return readBalance(dir).filter((e) => e.kind === 'inject' && e.sessionId === sessionId).length; + } catch { + return 0; + } +} + +/** + * Should the forecast interrupt right now, and with what? + * + * The state object is the caller's persisted per-session state; this reads `forecast` from it and + * returns the value to store back, so the throttle and the last-shown runway survive across hook + * processes. Returning the next state rather than mutating it keeps this testable without a + * filesystem. + * + * @returns { text, state } -- text is null when nothing should be shown. + */ +export function maybeSurface(dir, { + transcriptPath, sessionId, state = {}, now = Date.now(), findings = [], +} = {}) { + const previous = state.forecast || null; + + // THE THROTTLE IS FIRST, and deliberately before any I/O. Everything below opens files. + if (previous?.checkedAt && now - previous.checkedAt < SURFACE_INTERVAL_MS) { + return { text: null, state: previous }; + } + + let usage; + try { + usage = sessionUsage(transcriptPath); + } catch { + return { text: null, state: previous }; + } + // The clock is stamped even when there is nothing to measure, so a session with no usage rows + // does not re-attempt the transcript parse on every single tool call. + if (!usage) return { text: null, state: { ...previous, checkedAt: now } }; + + let panel; + try { + panel = forecastPanel(dir, { + ...usage, + sessionId, + touches: touchesFor(dir, sessionId), + }, findings); + } catch { + return { text: null, state: { ...previous, checkedAt: now } }; + } + if (!panel) return { text: null, state: { ...previous, checkedAt: now } }; + + // worthSurfacing compares against the last panel that was actually SHOWN, not the last one + // computed. Comparing against the last computed panel would let the runway drift down past the + // threshold one throttle window at a time and never trip the "crossed it" test. + const shown = previous?.shown ? { parts: { runway: { withGraph: previous.shown } } } : null; + if (!worthSurfacing(panel, shown)) { + return { text: null, state: { ...previous, checkedAt: now } }; + } + + return { + text: panel.text, + state: { checkedAt: now, shown: panel.parts.runway.withGraph }, + }; +} + +/** + * Closes the calibration loop, at the only moment that can close it. + * + * Compaction firing IS the ground truth a runway forecast was predicting, so this belongs on the + * PreCompact path and nowhere else. `actualTurns` is turns ELAPSED between the prediction and + * compaction, which is what predictedTurns meant -- so the forecast has to have recorded the turn + * it was made on, and observeOutcome subtracts. + * + * Silent by design: nothing is shown to anybody. It writes one record so that the NEXT session's + * forecast can be published with a track record instead of as a bare assertion. + */ +export function closeForecast(dir, { transcriptPath, sessionId } = {}) { + try { + const usage = sessionUsage(transcriptPath); + if (!usage) return false; + observeOutcome(dir, { sessionId, atTurn: usage.turns }); + return true; + } catch { + return false; + } +} diff --git a/integrations/opencode/hooks/lib/calibration.mjs b/integrations/opencode/hooks/lib/calibration.mjs index e920a9992..42de126f9 100644 --- a/integrations/opencode/hooks/lib/calibration.mjs +++ b/integrations/opencode/hooks/lib/calibration.mjs @@ -66,8 +66,15 @@ function horizonOf(turns) { return HORIZONS.find((h) => turns <= h.max).name; } -/** Records a prediction so it can be scored when the outcome arrives. */ -export function logForecast(dir, { sessionId, predictedTurns, used, capacity }) { +/** + * Records a prediction so it can be scored when the outcome arrives. + * + * `turns` is the session's turn count AT PREDICTION TIME, and it is what makes the outcome + * computable at all: predictedTurns means "turns from here until compaction", so the ground truth + * is the turns that ELAPSED, not the turn number compaction happened on. Without it the caller at + * compaction has nothing to subtract from and would have to score an interval against an absolute. + */ +export function logForecast(dir, { sessionId, predictedTurns, used, capacity, turns }) { if (!Number.isFinite(predictedTurns)) return; record(dir, { kind: 'forecast', @@ -76,6 +83,7 @@ export function logForecast(dir, { sessionId, predictedTurns, used, capacity }) horizon: horizonOf(predictedTurns), used, capacity, + turns: Number.isFinite(turns) ? turns : null, }); } @@ -86,8 +94,17 @@ export function logForecast(dir, { sessionId, predictedTurns, used, capacity }) * Called when compaction fires: the number of turns that elapsed since the * prediction is the ground truth it was predicting. */ -export function observeOutcome(dir, { sessionId, actualTurns }) { - if (!Number.isFinite(actualTurns)) return; +export function observeOutcome(dir, { sessionId, actualTurns, atTurn }) { + // EITHER THE ELAPSED COUNT OR THE TURN IT HAPPENED ON. + // + // The only real caller -- the PreCompact hook -- knows what turn compaction fired on, not how + // many turns have passed since a prediction it did not make. Deriving the interval here keeps + // that subtraction next to the record that defines it, rather than making every caller + // re-discover which forecast is open in order to compute its own ground truth. + const elapsed = Number.isFinite(actualTurns) + ? actualTurns + : null; + if (elapsed === null && !Number.isFinite(atTurn)) return; // CLOSURE IS DERIVED FROM THE OUTCOME ROWS, not from a flag on the forecast. // @@ -111,6 +128,14 @@ export function observeOutcome(dir, { sessionId, actualTurns }) { .pop(); if (!open) return; + // Derived only once the open forecast is known, since the interval is measured from ITS turn. + // A forecast recorded before `turns` existed has none, and there is nothing to subtract from -- + // scoring it against an absolute turn number would manufacture an error rather than measure one, + // so it is left open instead. + const actual = elapsed !== null ? elapsed + : (Number.isFinite(open.turns) ? atTurn - open.turns : null); + if (actual === null || !Number.isFinite(actual)) return; + record(dir, { kind: 'forecast-outcome', sessionId, @@ -119,9 +144,9 @@ export function observeOutcome(dir, { sessionId, actualTurns }) { forecastId: open.id, horizon: open.horizon, predictedTurns: open.predictedTurns, - actualTurns, - error: actualTurns - open.predictedTurns, - hit: Math.abs(actualTurns - open.predictedTurns) <= TOLERANCE, + actualTurns: actual, + error: actual - open.predictedTurns, + hit: Math.abs(actual - open.predictedTurns) <= TOLERANCE, }); } diff --git a/integrations/opencode/hooks/lib/forecast.mjs b/integrations/opencode/hooks/lib/forecast.mjs index dee6a3b56..39358e76a 100644 --- a/integrations/opencode/hooks/lib/forecast.mjs +++ b/integrations/opencode/hooks/lib/forecast.mjs @@ -328,6 +328,9 @@ export function forecastPanel(dir, session = {}, findings = []) { predictedTurns: air.withGraph, used: session.used, capacity: session.capacity, + // The turn this was predicted FROM. predictedTurns is an interval, so without its origin + // the outcome at compaction has nothing to subtract from. + turns: session.turns, }); const score = calibrate(dir, air.withGraph); diff --git a/integrations/opencode/hooks/lib/policy.mjs b/integrations/opencode/hooks/lib/policy.mjs index b3871c27d..1fa14c348 100644 --- a/integrations/opencode/hooks/lib/policy.mjs +++ b/integrations/opencode/hooks/lib/policy.mjs @@ -272,7 +272,7 @@ function statePath(sessionId, agent) { /** A usable state object, whatever was on disk. */ function emptyState() { - return { seen: {}, denied: {}, injected: [], actCounts: {} }; + return { seen: {}, denied: {}, injected: [], actCounts: {}, forecast: null }; } /** @@ -312,6 +312,16 @@ export function loadState(sessionId, agent) { parsed.actCounts && typeof parsed.actCounts === 'object' && !Array.isArray(parsed.actCounts) ? parsed.actCounts : {}, + // THE SAME REASON A THIRD TIME. The forecast throttle records when the panel was last + // computed and what runway it last SHOWED, and both are meaningless within a single hook + // process -- every tool call is a new one. Dropped here, `checkedAt` would reset on every + // call and the panel would rebuild itself, transcript parse and all, on each tool use; and + // `shown` would reset, so worthSurfacing would compare against nothing and re-interrupt with + // the same runway forever. + forecast: + parsed.forecast && typeof parsed.forecast === 'object' && !Array.isArray(parsed.forecast) + ? parsed.forecast + : null, }; } catch { return emptyState(); @@ -383,6 +393,18 @@ export function saveState(sessionId, state, agent) { } return out; })(), + // LATEST CHECK WINS, which is the opposite direction from the fields above and correct for + // this one. `seen`, `denied`, `injected` and `actCounts` are all append-only, so a union or + // a max is the safe merge. The forecast throttle is a POINT IN TIME: taking the older of two + // concurrent checks would re-open the window and let the panel be rebuilt immediately, which + // is the cost the throttle exists to bound. + forecast: (() => { + const mine = state.forecast || null; + const theirs = current.forecast || null; + if (!mine) return theirs; + if (!theirs) return mine; + return (mine.checkedAt || 0) >= (theirs.checkedAt || 0) ? mine : theirs; + })(), }; // Write-then-rename so a reader never observes a half-written file. diff --git a/integrations/opencode/hooks/lib/surface.mjs b/integrations/opencode/hooks/lib/surface.mjs new file mode 100644 index 000000000..0ec05161c --- /dev/null +++ b/integrations/opencode/hooks/lib/surface.mjs @@ -0,0 +1,166 @@ +// GENERATED FILE -- do not edit. +// Source of truth: hooks-core/surface.mjs. Regenerate with `npm run sync:hooks`. +/** + * The entry point the forecast never had. + * + * forecast.mjs computes a runway, calibration.mjs scores it, and worthSurfacing decides whether a + * change is worth interrupting for -- and none of it ran. The reachability guard listed + * forecastPanel, worthSurfacing, logForecast and observeOutcome as unwired, which is a polite way + * of saying the feature did not exist for any user. This module is the missing half: it derives + * the session numbers the panel needs from the transcript, decides when the panel has earned an + * interruption, and closes the calibration loop when compaction fires. + * + * THREE COSTS ARE BOUNDED HERE, because this runs on the PreToolUse path and every tool call is a + * separate process: + * + * THE READ readCacheUsage parses up to 4 MB of transcript. Doing that on every tool call + * would tax the hook that exists to save the user money. + * THE COMPUTE the panel walks the metrics log, the balance log and the graph. + * THE CONTEXT the panel is only worth its own tokens when it says something actionable. + * + * So the throttle is checked FIRST, from persisted state, before any file is opened. Everything + * expensive happens after a clock comparison. + * + * NOTHING HERE IS ALLOWED TO BREAK A TOOL CALL. Every entry point returns null on failure rather + * than throwing: a forecast is a courtesy, and the hook's promise is that a defect in it costs the + * user nothing. + */ + +import { forecastPanel, worthSurfacing } from './forecast.mjs'; +import { observeOutcome } from './calibration.mjs'; +import { readCacheUsage } from './cache.mjs'; +import { readBalance } from './metrics.mjs'; + +/** + * How long between two attempts to build the panel. + * + * Not a tuning knob so much as an admission: the panel costs a transcript parse, and a user makes + * many tool calls per minute. Two minutes is short enough that a runway falling into single digits + * is still caught with turns to spare, and long enough that the check is free in aggregate. + */ +export const SURFACE_INTERVAL_MS = + Number(process.env.TOKEN_OPTIMIZER_SURFACE_INTERVAL_MS) || 120_000; + +/** + * The context window to measure the runway against. + * + * NOT DISCOVERABLE from the transcript: the usage rows carry what was spent, never the ceiling it + * was spent against. Defaulting is therefore unavoidable, so the default is stated rather than + * buried, and overridable for anyone running a different window. + */ +export const DEFAULT_CAPACITY = + Number(process.env.TOKEN_OPTIMIZER_CONTEXT_CAPACITY) || 200_000; + +/** + * What the session has spent, read from its own transcript. + * + * `used` is the FULL input size of the most recent request -- cache reads plus cache writes plus + * fresh input -- which is what the context window actually holds, rather than the marginal cost of + * that one turn. Summing the per-turn inputs instead would count the same carried prefix once per + * turn and report a number many times the window size. + * + * Returns null rather than a zeroed object when there is nothing to measure, so a caller cannot + * mistake "no data" for "an empty context". + */ +export function sessionUsage(transcriptPath, { capacity = DEFAULT_CAPACITY } = {}) { + let turns; + try { + turns = readCacheUsage(transcriptPath); + } catch { + return null; + } + if (!turns?.length) return null; + + const last = turns[turns.length - 1]; + const used = (last.read || 0) + (last.written || 0) + (last.input || 0); + if (!used) return null; + + return { used, capacity, turns: turns.length }; +} + +/** Intercepted touches recorded for this session, which converts a per-touch saving to per-turn. */ +function touchesFor(dir, sessionId) { + try { + return readBalance(dir).filter((e) => e.kind === 'inject' && e.sessionId === sessionId).length; + } catch { + return 0; + } +} + +/** + * Should the forecast interrupt right now, and with what? + * + * The state object is the caller's persisted per-session state; this reads `forecast` from it and + * returns the value to store back, so the throttle and the last-shown runway survive across hook + * processes. Returning the next state rather than mutating it keeps this testable without a + * filesystem. + * + * @returns { text, state } -- text is null when nothing should be shown. + */ +export function maybeSurface(dir, { + transcriptPath, sessionId, state = {}, now = Date.now(), findings = [], +} = {}) { + const previous = state.forecast || null; + + // THE THROTTLE IS FIRST, and deliberately before any I/O. Everything below opens files. + if (previous?.checkedAt && now - previous.checkedAt < SURFACE_INTERVAL_MS) { + return { text: null, state: previous }; + } + + let usage; + try { + usage = sessionUsage(transcriptPath); + } catch { + return { text: null, state: previous }; + } + // The clock is stamped even when there is nothing to measure, so a session with no usage rows + // does not re-attempt the transcript parse on every single tool call. + if (!usage) return { text: null, state: { ...previous, checkedAt: now } }; + + let panel; + try { + panel = forecastPanel(dir, { + ...usage, + sessionId, + touches: touchesFor(dir, sessionId), + }, findings); + } catch { + return { text: null, state: { ...previous, checkedAt: now } }; + } + if (!panel) return { text: null, state: { ...previous, checkedAt: now } }; + + // worthSurfacing compares against the last panel that was actually SHOWN, not the last one + // computed. Comparing against the last computed panel would let the runway drift down past the + // threshold one throttle window at a time and never trip the "crossed it" test. + const shown = previous?.shown ? { parts: { runway: { withGraph: previous.shown } } } : null; + if (!worthSurfacing(panel, shown)) { + return { text: null, state: { ...previous, checkedAt: now } }; + } + + return { + text: panel.text, + state: { checkedAt: now, shown: panel.parts.runway.withGraph }, + }; +} + +/** + * Closes the calibration loop, at the only moment that can close it. + * + * Compaction firing IS the ground truth a runway forecast was predicting, so this belongs on the + * PreCompact path and nowhere else. `actualTurns` is turns ELAPSED between the prediction and + * compaction, which is what predictedTurns meant -- so the forecast has to have recorded the turn + * it was made on, and observeOutcome subtracts. + * + * Silent by design: nothing is shown to anybody. It writes one record so that the NEXT session's + * forecast can be published with a track record instead of as a bare assertion. + */ +export function closeForecast(dir, { transcriptPath, sessionId } = {}) { + try { + const usage = sessionUsage(transcriptPath); + if (!usage) return false; + observeOutcome(dir, { sessionId, atTurn: usage.turns }); + return true; + } catch { + return false; + } +} diff --git a/integrations/qwen/hooks/lib/calibration.mjs b/integrations/qwen/hooks/lib/calibration.mjs index e920a9992..42de126f9 100644 --- a/integrations/qwen/hooks/lib/calibration.mjs +++ b/integrations/qwen/hooks/lib/calibration.mjs @@ -66,8 +66,15 @@ function horizonOf(turns) { return HORIZONS.find((h) => turns <= h.max).name; } -/** Records a prediction so it can be scored when the outcome arrives. */ -export function logForecast(dir, { sessionId, predictedTurns, used, capacity }) { +/** + * Records a prediction so it can be scored when the outcome arrives. + * + * `turns` is the session's turn count AT PREDICTION TIME, and it is what makes the outcome + * computable at all: predictedTurns means "turns from here until compaction", so the ground truth + * is the turns that ELAPSED, not the turn number compaction happened on. Without it the caller at + * compaction has nothing to subtract from and would have to score an interval against an absolute. + */ +export function logForecast(dir, { sessionId, predictedTurns, used, capacity, turns }) { if (!Number.isFinite(predictedTurns)) return; record(dir, { kind: 'forecast', @@ -76,6 +83,7 @@ export function logForecast(dir, { sessionId, predictedTurns, used, capacity }) horizon: horizonOf(predictedTurns), used, capacity, + turns: Number.isFinite(turns) ? turns : null, }); } @@ -86,8 +94,17 @@ export function logForecast(dir, { sessionId, predictedTurns, used, capacity }) * Called when compaction fires: the number of turns that elapsed since the * prediction is the ground truth it was predicting. */ -export function observeOutcome(dir, { sessionId, actualTurns }) { - if (!Number.isFinite(actualTurns)) return; +export function observeOutcome(dir, { sessionId, actualTurns, atTurn }) { + // EITHER THE ELAPSED COUNT OR THE TURN IT HAPPENED ON. + // + // The only real caller -- the PreCompact hook -- knows what turn compaction fired on, not how + // many turns have passed since a prediction it did not make. Deriving the interval here keeps + // that subtraction next to the record that defines it, rather than making every caller + // re-discover which forecast is open in order to compute its own ground truth. + const elapsed = Number.isFinite(actualTurns) + ? actualTurns + : null; + if (elapsed === null && !Number.isFinite(atTurn)) return; // CLOSURE IS DERIVED FROM THE OUTCOME ROWS, not from a flag on the forecast. // @@ -111,6 +128,14 @@ export function observeOutcome(dir, { sessionId, actualTurns }) { .pop(); if (!open) return; + // Derived only once the open forecast is known, since the interval is measured from ITS turn. + // A forecast recorded before `turns` existed has none, and there is nothing to subtract from -- + // scoring it against an absolute turn number would manufacture an error rather than measure one, + // so it is left open instead. + const actual = elapsed !== null ? elapsed + : (Number.isFinite(open.turns) ? atTurn - open.turns : null); + if (actual === null || !Number.isFinite(actual)) return; + record(dir, { kind: 'forecast-outcome', sessionId, @@ -119,9 +144,9 @@ export function observeOutcome(dir, { sessionId, actualTurns }) { forecastId: open.id, horizon: open.horizon, predictedTurns: open.predictedTurns, - actualTurns, - error: actualTurns - open.predictedTurns, - hit: Math.abs(actualTurns - open.predictedTurns) <= TOLERANCE, + actualTurns: actual, + error: actual - open.predictedTurns, + hit: Math.abs(actual - open.predictedTurns) <= TOLERANCE, }); } diff --git a/integrations/qwen/hooks/lib/forecast.mjs b/integrations/qwen/hooks/lib/forecast.mjs index dee6a3b56..39358e76a 100644 --- a/integrations/qwen/hooks/lib/forecast.mjs +++ b/integrations/qwen/hooks/lib/forecast.mjs @@ -328,6 +328,9 @@ export function forecastPanel(dir, session = {}, findings = []) { predictedTurns: air.withGraph, used: session.used, capacity: session.capacity, + // The turn this was predicted FROM. predictedTurns is an interval, so without its origin + // the outcome at compaction has nothing to subtract from. + turns: session.turns, }); const score = calibrate(dir, air.withGraph); diff --git a/integrations/qwen/hooks/lib/policy.mjs b/integrations/qwen/hooks/lib/policy.mjs index b3871c27d..1fa14c348 100644 --- a/integrations/qwen/hooks/lib/policy.mjs +++ b/integrations/qwen/hooks/lib/policy.mjs @@ -272,7 +272,7 @@ function statePath(sessionId, agent) { /** A usable state object, whatever was on disk. */ function emptyState() { - return { seen: {}, denied: {}, injected: [], actCounts: {} }; + return { seen: {}, denied: {}, injected: [], actCounts: {}, forecast: null }; } /** @@ -312,6 +312,16 @@ export function loadState(sessionId, agent) { parsed.actCounts && typeof parsed.actCounts === 'object' && !Array.isArray(parsed.actCounts) ? parsed.actCounts : {}, + // THE SAME REASON A THIRD TIME. The forecast throttle records when the panel was last + // computed and what runway it last SHOWED, and both are meaningless within a single hook + // process -- every tool call is a new one. Dropped here, `checkedAt` would reset on every + // call and the panel would rebuild itself, transcript parse and all, on each tool use; and + // `shown` would reset, so worthSurfacing would compare against nothing and re-interrupt with + // the same runway forever. + forecast: + parsed.forecast && typeof parsed.forecast === 'object' && !Array.isArray(parsed.forecast) + ? parsed.forecast + : null, }; } catch { return emptyState(); @@ -383,6 +393,18 @@ export function saveState(sessionId, state, agent) { } return out; })(), + // LATEST CHECK WINS, which is the opposite direction from the fields above and correct for + // this one. `seen`, `denied`, `injected` and `actCounts` are all append-only, so a union or + // a max is the safe merge. The forecast throttle is a POINT IN TIME: taking the older of two + // concurrent checks would re-open the window and let the panel be rebuilt immediately, which + // is the cost the throttle exists to bound. + forecast: (() => { + const mine = state.forecast || null; + const theirs = current.forecast || null; + if (!mine) return theirs; + if (!theirs) return mine; + return (mine.checkedAt || 0) >= (theirs.checkedAt || 0) ? mine : theirs; + })(), }; // Write-then-rename so a reader never observes a half-written file. diff --git a/integrations/qwen/hooks/lib/surface.mjs b/integrations/qwen/hooks/lib/surface.mjs new file mode 100644 index 000000000..0ec05161c --- /dev/null +++ b/integrations/qwen/hooks/lib/surface.mjs @@ -0,0 +1,166 @@ +// GENERATED FILE -- do not edit. +// Source of truth: hooks-core/surface.mjs. Regenerate with `npm run sync:hooks`. +/** + * The entry point the forecast never had. + * + * forecast.mjs computes a runway, calibration.mjs scores it, and worthSurfacing decides whether a + * change is worth interrupting for -- and none of it ran. The reachability guard listed + * forecastPanel, worthSurfacing, logForecast and observeOutcome as unwired, which is a polite way + * of saying the feature did not exist for any user. This module is the missing half: it derives + * the session numbers the panel needs from the transcript, decides when the panel has earned an + * interruption, and closes the calibration loop when compaction fires. + * + * THREE COSTS ARE BOUNDED HERE, because this runs on the PreToolUse path and every tool call is a + * separate process: + * + * THE READ readCacheUsage parses up to 4 MB of transcript. Doing that on every tool call + * would tax the hook that exists to save the user money. + * THE COMPUTE the panel walks the metrics log, the balance log and the graph. + * THE CONTEXT the panel is only worth its own tokens when it says something actionable. + * + * So the throttle is checked FIRST, from persisted state, before any file is opened. Everything + * expensive happens after a clock comparison. + * + * NOTHING HERE IS ALLOWED TO BREAK A TOOL CALL. Every entry point returns null on failure rather + * than throwing: a forecast is a courtesy, and the hook's promise is that a defect in it costs the + * user nothing. + */ + +import { forecastPanel, worthSurfacing } from './forecast.mjs'; +import { observeOutcome } from './calibration.mjs'; +import { readCacheUsage } from './cache.mjs'; +import { readBalance } from './metrics.mjs'; + +/** + * How long between two attempts to build the panel. + * + * Not a tuning knob so much as an admission: the panel costs a transcript parse, and a user makes + * many tool calls per minute. Two minutes is short enough that a runway falling into single digits + * is still caught with turns to spare, and long enough that the check is free in aggregate. + */ +export const SURFACE_INTERVAL_MS = + Number(process.env.TOKEN_OPTIMIZER_SURFACE_INTERVAL_MS) || 120_000; + +/** + * The context window to measure the runway against. + * + * NOT DISCOVERABLE from the transcript: the usage rows carry what was spent, never the ceiling it + * was spent against. Defaulting is therefore unavoidable, so the default is stated rather than + * buried, and overridable for anyone running a different window. + */ +export const DEFAULT_CAPACITY = + Number(process.env.TOKEN_OPTIMIZER_CONTEXT_CAPACITY) || 200_000; + +/** + * What the session has spent, read from its own transcript. + * + * `used` is the FULL input size of the most recent request -- cache reads plus cache writes plus + * fresh input -- which is what the context window actually holds, rather than the marginal cost of + * that one turn. Summing the per-turn inputs instead would count the same carried prefix once per + * turn and report a number many times the window size. + * + * Returns null rather than a zeroed object when there is nothing to measure, so a caller cannot + * mistake "no data" for "an empty context". + */ +export function sessionUsage(transcriptPath, { capacity = DEFAULT_CAPACITY } = {}) { + let turns; + try { + turns = readCacheUsage(transcriptPath); + } catch { + return null; + } + if (!turns?.length) return null; + + const last = turns[turns.length - 1]; + const used = (last.read || 0) + (last.written || 0) + (last.input || 0); + if (!used) return null; + + return { used, capacity, turns: turns.length }; +} + +/** Intercepted touches recorded for this session, which converts a per-touch saving to per-turn. */ +function touchesFor(dir, sessionId) { + try { + return readBalance(dir).filter((e) => e.kind === 'inject' && e.sessionId === sessionId).length; + } catch { + return 0; + } +} + +/** + * Should the forecast interrupt right now, and with what? + * + * The state object is the caller's persisted per-session state; this reads `forecast` from it and + * returns the value to store back, so the throttle and the last-shown runway survive across hook + * processes. Returning the next state rather than mutating it keeps this testable without a + * filesystem. + * + * @returns { text, state } -- text is null when nothing should be shown. + */ +export function maybeSurface(dir, { + transcriptPath, sessionId, state = {}, now = Date.now(), findings = [], +} = {}) { + const previous = state.forecast || null; + + // THE THROTTLE IS FIRST, and deliberately before any I/O. Everything below opens files. + if (previous?.checkedAt && now - previous.checkedAt < SURFACE_INTERVAL_MS) { + return { text: null, state: previous }; + } + + let usage; + try { + usage = sessionUsage(transcriptPath); + } catch { + return { text: null, state: previous }; + } + // The clock is stamped even when there is nothing to measure, so a session with no usage rows + // does not re-attempt the transcript parse on every single tool call. + if (!usage) return { text: null, state: { ...previous, checkedAt: now } }; + + let panel; + try { + panel = forecastPanel(dir, { + ...usage, + sessionId, + touches: touchesFor(dir, sessionId), + }, findings); + } catch { + return { text: null, state: { ...previous, checkedAt: now } }; + } + if (!panel) return { text: null, state: { ...previous, checkedAt: now } }; + + // worthSurfacing compares against the last panel that was actually SHOWN, not the last one + // computed. Comparing against the last computed panel would let the runway drift down past the + // threshold one throttle window at a time and never trip the "crossed it" test. + const shown = previous?.shown ? { parts: { runway: { withGraph: previous.shown } } } : null; + if (!worthSurfacing(panel, shown)) { + return { text: null, state: { ...previous, checkedAt: now } }; + } + + return { + text: panel.text, + state: { checkedAt: now, shown: panel.parts.runway.withGraph }, + }; +} + +/** + * Closes the calibration loop, at the only moment that can close it. + * + * Compaction firing IS the ground truth a runway forecast was predicting, so this belongs on the + * PreCompact path and nowhere else. `actualTurns` is turns ELAPSED between the prediction and + * compaction, which is what predictedTurns meant -- so the forecast has to have recorded the turn + * it was made on, and observeOutcome subtracts. + * + * Silent by design: nothing is shown to anybody. It writes one record so that the NEXT session's + * forecast can be published with a track record instead of as a bare assertion. + */ +export function closeForecast(dir, { transcriptPath, sessionId } = {}) { + try { + const usage = sessionUsage(transcriptPath); + if (!usage) return false; + observeOutcome(dir, { sessionId, atTurn: usage.turns }); + return true; + } catch { + return false; + } +} diff --git a/plugin/hooks/lib/calibration.mjs b/plugin/hooks/lib/calibration.mjs index e920a9992..42de126f9 100644 --- a/plugin/hooks/lib/calibration.mjs +++ b/plugin/hooks/lib/calibration.mjs @@ -66,8 +66,15 @@ function horizonOf(turns) { return HORIZONS.find((h) => turns <= h.max).name; } -/** Records a prediction so it can be scored when the outcome arrives. */ -export function logForecast(dir, { sessionId, predictedTurns, used, capacity }) { +/** + * Records a prediction so it can be scored when the outcome arrives. + * + * `turns` is the session's turn count AT PREDICTION TIME, and it is what makes the outcome + * computable at all: predictedTurns means "turns from here until compaction", so the ground truth + * is the turns that ELAPSED, not the turn number compaction happened on. Without it the caller at + * compaction has nothing to subtract from and would have to score an interval against an absolute. + */ +export function logForecast(dir, { sessionId, predictedTurns, used, capacity, turns }) { if (!Number.isFinite(predictedTurns)) return; record(dir, { kind: 'forecast', @@ -76,6 +83,7 @@ export function logForecast(dir, { sessionId, predictedTurns, used, capacity }) horizon: horizonOf(predictedTurns), used, capacity, + turns: Number.isFinite(turns) ? turns : null, }); } @@ -86,8 +94,17 @@ export function logForecast(dir, { sessionId, predictedTurns, used, capacity }) * Called when compaction fires: the number of turns that elapsed since the * prediction is the ground truth it was predicting. */ -export function observeOutcome(dir, { sessionId, actualTurns }) { - if (!Number.isFinite(actualTurns)) return; +export function observeOutcome(dir, { sessionId, actualTurns, atTurn }) { + // EITHER THE ELAPSED COUNT OR THE TURN IT HAPPENED ON. + // + // The only real caller -- the PreCompact hook -- knows what turn compaction fired on, not how + // many turns have passed since a prediction it did not make. Deriving the interval here keeps + // that subtraction next to the record that defines it, rather than making every caller + // re-discover which forecast is open in order to compute its own ground truth. + const elapsed = Number.isFinite(actualTurns) + ? actualTurns + : null; + if (elapsed === null && !Number.isFinite(atTurn)) return; // CLOSURE IS DERIVED FROM THE OUTCOME ROWS, not from a flag on the forecast. // @@ -111,6 +128,14 @@ export function observeOutcome(dir, { sessionId, actualTurns }) { .pop(); if (!open) return; + // Derived only once the open forecast is known, since the interval is measured from ITS turn. + // A forecast recorded before `turns` existed has none, and there is nothing to subtract from -- + // scoring it against an absolute turn number would manufacture an error rather than measure one, + // so it is left open instead. + const actual = elapsed !== null ? elapsed + : (Number.isFinite(open.turns) ? atTurn - open.turns : null); + if (actual === null || !Number.isFinite(actual)) return; + record(dir, { kind: 'forecast-outcome', sessionId, @@ -119,9 +144,9 @@ export function observeOutcome(dir, { sessionId, actualTurns }) { forecastId: open.id, horizon: open.horizon, predictedTurns: open.predictedTurns, - actualTurns, - error: actualTurns - open.predictedTurns, - hit: Math.abs(actualTurns - open.predictedTurns) <= TOLERANCE, + actualTurns: actual, + error: actual - open.predictedTurns, + hit: Math.abs(actual - open.predictedTurns) <= TOLERANCE, }); } diff --git a/plugin/hooks/lib/forecast.mjs b/plugin/hooks/lib/forecast.mjs index dee6a3b56..39358e76a 100644 --- a/plugin/hooks/lib/forecast.mjs +++ b/plugin/hooks/lib/forecast.mjs @@ -328,6 +328,9 @@ export function forecastPanel(dir, session = {}, findings = []) { predictedTurns: air.withGraph, used: session.used, capacity: session.capacity, + // The turn this was predicted FROM. predictedTurns is an interval, so without its origin + // the outcome at compaction has nothing to subtract from. + turns: session.turns, }); const score = calibrate(dir, air.withGraph); diff --git a/plugin/hooks/lib/policy.mjs b/plugin/hooks/lib/policy.mjs index b3871c27d..1fa14c348 100644 --- a/plugin/hooks/lib/policy.mjs +++ b/plugin/hooks/lib/policy.mjs @@ -272,7 +272,7 @@ function statePath(sessionId, agent) { /** A usable state object, whatever was on disk. */ function emptyState() { - return { seen: {}, denied: {}, injected: [], actCounts: {} }; + return { seen: {}, denied: {}, injected: [], actCounts: {}, forecast: null }; } /** @@ -312,6 +312,16 @@ export function loadState(sessionId, agent) { parsed.actCounts && typeof parsed.actCounts === 'object' && !Array.isArray(parsed.actCounts) ? parsed.actCounts : {}, + // THE SAME REASON A THIRD TIME. The forecast throttle records when the panel was last + // computed and what runway it last SHOWED, and both are meaningless within a single hook + // process -- every tool call is a new one. Dropped here, `checkedAt` would reset on every + // call and the panel would rebuild itself, transcript parse and all, on each tool use; and + // `shown` would reset, so worthSurfacing would compare against nothing and re-interrupt with + // the same runway forever. + forecast: + parsed.forecast && typeof parsed.forecast === 'object' && !Array.isArray(parsed.forecast) + ? parsed.forecast + : null, }; } catch { return emptyState(); @@ -383,6 +393,18 @@ export function saveState(sessionId, state, agent) { } return out; })(), + // LATEST CHECK WINS, which is the opposite direction from the fields above and correct for + // this one. `seen`, `denied`, `injected` and `actCounts` are all append-only, so a union or + // a max is the safe merge. The forecast throttle is a POINT IN TIME: taking the older of two + // concurrent checks would re-open the window and let the panel be rebuilt immediately, which + // is the cost the throttle exists to bound. + forecast: (() => { + const mine = state.forecast || null; + const theirs = current.forecast || null; + if (!mine) return theirs; + if (!theirs) return mine; + return (mine.checkedAt || 0) >= (theirs.checkedAt || 0) ? mine : theirs; + })(), }; // Write-then-rename so a reader never observes a half-written file. diff --git a/plugin/hooks/lib/surface.mjs b/plugin/hooks/lib/surface.mjs new file mode 100644 index 000000000..0ec05161c --- /dev/null +++ b/plugin/hooks/lib/surface.mjs @@ -0,0 +1,166 @@ +// GENERATED FILE -- do not edit. +// Source of truth: hooks-core/surface.mjs. Regenerate with `npm run sync:hooks`. +/** + * The entry point the forecast never had. + * + * forecast.mjs computes a runway, calibration.mjs scores it, and worthSurfacing decides whether a + * change is worth interrupting for -- and none of it ran. The reachability guard listed + * forecastPanel, worthSurfacing, logForecast and observeOutcome as unwired, which is a polite way + * of saying the feature did not exist for any user. This module is the missing half: it derives + * the session numbers the panel needs from the transcript, decides when the panel has earned an + * interruption, and closes the calibration loop when compaction fires. + * + * THREE COSTS ARE BOUNDED HERE, because this runs on the PreToolUse path and every tool call is a + * separate process: + * + * THE READ readCacheUsage parses up to 4 MB of transcript. Doing that on every tool call + * would tax the hook that exists to save the user money. + * THE COMPUTE the panel walks the metrics log, the balance log and the graph. + * THE CONTEXT the panel is only worth its own tokens when it says something actionable. + * + * So the throttle is checked FIRST, from persisted state, before any file is opened. Everything + * expensive happens after a clock comparison. + * + * NOTHING HERE IS ALLOWED TO BREAK A TOOL CALL. Every entry point returns null on failure rather + * than throwing: a forecast is a courtesy, and the hook's promise is that a defect in it costs the + * user nothing. + */ + +import { forecastPanel, worthSurfacing } from './forecast.mjs'; +import { observeOutcome } from './calibration.mjs'; +import { readCacheUsage } from './cache.mjs'; +import { readBalance } from './metrics.mjs'; + +/** + * How long between two attempts to build the panel. + * + * Not a tuning knob so much as an admission: the panel costs a transcript parse, and a user makes + * many tool calls per minute. Two minutes is short enough that a runway falling into single digits + * is still caught with turns to spare, and long enough that the check is free in aggregate. + */ +export const SURFACE_INTERVAL_MS = + Number(process.env.TOKEN_OPTIMIZER_SURFACE_INTERVAL_MS) || 120_000; + +/** + * The context window to measure the runway against. + * + * NOT DISCOVERABLE from the transcript: the usage rows carry what was spent, never the ceiling it + * was spent against. Defaulting is therefore unavoidable, so the default is stated rather than + * buried, and overridable for anyone running a different window. + */ +export const DEFAULT_CAPACITY = + Number(process.env.TOKEN_OPTIMIZER_CONTEXT_CAPACITY) || 200_000; + +/** + * What the session has spent, read from its own transcript. + * + * `used` is the FULL input size of the most recent request -- cache reads plus cache writes plus + * fresh input -- which is what the context window actually holds, rather than the marginal cost of + * that one turn. Summing the per-turn inputs instead would count the same carried prefix once per + * turn and report a number many times the window size. + * + * Returns null rather than a zeroed object when there is nothing to measure, so a caller cannot + * mistake "no data" for "an empty context". + */ +export function sessionUsage(transcriptPath, { capacity = DEFAULT_CAPACITY } = {}) { + let turns; + try { + turns = readCacheUsage(transcriptPath); + } catch { + return null; + } + if (!turns?.length) return null; + + const last = turns[turns.length - 1]; + const used = (last.read || 0) + (last.written || 0) + (last.input || 0); + if (!used) return null; + + return { used, capacity, turns: turns.length }; +} + +/** Intercepted touches recorded for this session, which converts a per-touch saving to per-turn. */ +function touchesFor(dir, sessionId) { + try { + return readBalance(dir).filter((e) => e.kind === 'inject' && e.sessionId === sessionId).length; + } catch { + return 0; + } +} + +/** + * Should the forecast interrupt right now, and with what? + * + * The state object is the caller's persisted per-session state; this reads `forecast` from it and + * returns the value to store back, so the throttle and the last-shown runway survive across hook + * processes. Returning the next state rather than mutating it keeps this testable without a + * filesystem. + * + * @returns { text, state } -- text is null when nothing should be shown. + */ +export function maybeSurface(dir, { + transcriptPath, sessionId, state = {}, now = Date.now(), findings = [], +} = {}) { + const previous = state.forecast || null; + + // THE THROTTLE IS FIRST, and deliberately before any I/O. Everything below opens files. + if (previous?.checkedAt && now - previous.checkedAt < SURFACE_INTERVAL_MS) { + return { text: null, state: previous }; + } + + let usage; + try { + usage = sessionUsage(transcriptPath); + } catch { + return { text: null, state: previous }; + } + // The clock is stamped even when there is nothing to measure, so a session with no usage rows + // does not re-attempt the transcript parse on every single tool call. + if (!usage) return { text: null, state: { ...previous, checkedAt: now } }; + + let panel; + try { + panel = forecastPanel(dir, { + ...usage, + sessionId, + touches: touchesFor(dir, sessionId), + }, findings); + } catch { + return { text: null, state: { ...previous, checkedAt: now } }; + } + if (!panel) return { text: null, state: { ...previous, checkedAt: now } }; + + // worthSurfacing compares against the last panel that was actually SHOWN, not the last one + // computed. Comparing against the last computed panel would let the runway drift down past the + // threshold one throttle window at a time and never trip the "crossed it" test. + const shown = previous?.shown ? { parts: { runway: { withGraph: previous.shown } } } : null; + if (!worthSurfacing(panel, shown)) { + return { text: null, state: { ...previous, checkedAt: now } }; + } + + return { + text: panel.text, + state: { checkedAt: now, shown: panel.parts.runway.withGraph }, + }; +} + +/** + * Closes the calibration loop, at the only moment that can close it. + * + * Compaction firing IS the ground truth a runway forecast was predicting, so this belongs on the + * PreCompact path and nowhere else. `actualTurns` is turns ELAPSED between the prediction and + * compaction, which is what predictedTurns meant -- so the forecast has to have recorded the turn + * it was made on, and observeOutcome subtracts. + * + * Silent by design: nothing is shown to anybody. It writes one record so that the NEXT session's + * forecast can be published with a track record instead of as a bare assertion. + */ +export function closeForecast(dir, { transcriptPath, sessionId } = {}) { + try { + const usage = sessionUsage(transcriptPath); + if (!usage) return false; + observeOutcome(dir, { sessionId, atTurn: usage.turns }); + return true; + } catch { + return false; + } +} diff --git a/plugin/hooks/precompact-optimize.mjs b/plugin/hooks/precompact-optimize.mjs index d546aa9dc..e69c8d25e 100644 --- a/plugin/hooks/precompact-optimize.mjs +++ b/plugin/hooks/precompact-optimize.mjs @@ -24,6 +24,7 @@ import { join } from 'node:path'; import { mode, MODE_OFF, loadState, clearSeen } from './lib/policy.mjs'; import { linkCoOccurrence } from './lib/inject.mjs'; import { wikiDir, projectRootFor } from './lib/wiki.mjs'; +import { closeForecast } from './lib/surface.mjs'; /** Longest compaction may be delayed. Past this the work is abandoned. */ const TIMEOUT_MS = @@ -125,6 +126,26 @@ async function main() { // silent no-op until a test caught it. clearSeen(payload.session_id, payload.transcript_path || null); + // THE GROUND TRUTH, AT THE ONLY MOMENT IT EXISTS. + // + // Compaction firing is exactly the event every runway forecast was predicting, so this is the + // one place the calibration loop can be closed. Without a caller here the loop collected + // predictions and never scored one: reliability saw an empty set forever and calibrate returned + // "not yet calibrated" for the life of the project, which is indistinguishable from the feature + // not existing. + // + // Silent, and before the wrapper check: nothing is shown to anybody, and a plugin-only install + // returns early below, so putting it after would have excluded every such user from the + // measurement -- the same shape as the co-occurrence write above. + try { + closeForecast(wikiDir(projectRootFor(payload.cwd, payload.cwd)), { + transcriptPath: payload.transcript_path, + sessionId: payload.session_id, + }); + } catch { + // Scoring a forecast must never delay compaction. + } + // NOW the wrapper, which only the optimize_session spawn needs. Plugin-only // installs stop here having still recorded their co-occurrence above. const wrapper = findWrapper(); diff --git a/plugin/hooks/pretooluse-router.mjs b/plugin/hooks/pretooluse-router.mjs index 6edf23136..abcdcfea8 100644 --- a/plugin/hooks/pretooluse-router.mjs +++ b/plugin/hooks/pretooluse-router.mjs @@ -32,6 +32,7 @@ import { commandProjectRoot, } from './lib/decide.mjs'; import { recordRead, fingerprint } from './lib/metrics.mjs'; +import { maybeSurface } from './lib/surface.mjs'; import { wikiDir, load, @@ -252,6 +253,28 @@ try { // their tool call, so a failure falls through to a plain allow. } + // THE FORECAST, AND THE ONLY PLACE IT CAN REACH A USER. + // + // This is the mid-session hook -- SessionStart is too early for a runway to mean anything and + // PreCompact is too late to act on one -- so it is where a deadline can still change what + // somebody does next. Everything about the cost is bounded inside maybeSurface: the throttle + // is checked from persisted state BEFORE any file is opened, and worthSurfacing withholds the + // panel unless the runway is actionable and has moved since the last one shown. + try { + const surfaced = maybeSurface(dirFor(payload.cwd || process.cwd()), { + transcriptPath: payload.transcript_path, + sessionId: payload.session_id, + state, + }); + if (surfaced.state !== state.forecast) { + state.forecast = surfaced.state; + saveState(payload.session_id, state, agentScope); + } + if (surfaced.text) context = context ? `${context}\n\n${surfaced.text}` : surfaced.text; + } catch { + // Same rule as above: a forecast is a courtesy and must never cost a tool call. + } + // THE MEMORY HALF. `harvest` records that this file was touched, at this // content hash, by this task -- and nothing had been calling it, anywhere. // The graph therefore accumulated no nodes and no task edges from ordinary diff --git a/tests/hooks/reachability.test.mjs b/tests/hooks/reachability.test.mjs index ec121bce5..adbb3549c 100644 --- a/tests/hooks/reachability.test.mjs +++ b/tests/hooks/reachability.test.mjs @@ -91,19 +91,12 @@ const ALLOWED = new Map([ // WIRED ELSEWHERE -- leaves this list when that PR merges. ['forTouch', 'WIRED by the injection PR: just-in-time delivery, imported by nothing before it.'], - // CALIBRATION, HALF-RESOLVED. logForecast and calibrate are now called by - // forecastPanel, so the prediction side of the loop runs and those two have - // left this list. observeOutcome has not: nothing observes the ground truth, - // which arrives when compaction fires. The natural caller is the PreCompact - // hook, which needs a turn count it does not currently compute. - ['observeOutcome', 'UNWIRED HALF: records the outcome half. The prediction half now runs; the PreCompact hook does not yet supply actualTurns.'], - - // FORECAST DISPLAY. The calibration behind it now runs, but the panel itself - // still has no shipped caller -- so nothing renders it and nothing decides - // when to. Wiring the loop did not change that, and saying otherwise would be - // the exact failure this file exists to catch. - ['forecastPanel', 'UNWIRED: the calibration loop behind it now runs, but no hook or tool renders the panel.'], - ['worthSurfacing', 'UNWIRED: decides whether a forecast change is worth showing; nothing shows it yet.'], + // THE FORECAST IS WIRED, so nothing from it is listed here any more. + // logForecast and calibrate are called by forecastPanel; forecastPanel and + // worthSurfacing by surface.mjs; observeOutcome by surface.closeForecast; and + // surface itself by the PreToolUse router and the PreCompact hook. Five + // entries left this list at once, which is what wiring a whole feature looks + // like -- and is why it was worth doing rather than annotating. // SURFACED BY TIGHTENING THE SCAN. Previously counted as reachable only // because the bare word match read comment prose as a call site. Verified diff --git a/tests/hooks/surface.test.mjs b/tests/hooks/surface.test.mjs new file mode 100644 index 000000000..be841a511 --- /dev/null +++ b/tests/hooks/surface.test.mjs @@ -0,0 +1,223 @@ +/** + * The entry point the forecast never had. + * + * The properties under test are the ones that decide whether this is a feature or a liability: + * the cost is bounded before any file is opened, the panel only interrupts when it is actionable + * AND has moved since the last one shown, the throttle survives the process boundary that every + * hook invocation crosses, and the calibration loop is closed by the one event that can close it. + * + * The last test in this file is the one that matters most: the reachability guard's own verdict + * that these functions now have shipped callers. Everything else here would pass just as well on + * a module nothing imports -- which is exactly the defect this wiring exists to end. + */ + +import { mkdtempSync, rmSync, writeFileSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { + maybeSurface, closeForecast, sessionUsage, SURFACE_INTERVAL_MS, DEFAULT_CAPACITY, +} from '../../hooks-core/surface.mjs'; +import { logForecast, observeOutcome, reliability } from '../../hooks-core/calibration.mjs'; +import { record, recordRead } from '../../hooks-core/metrics.mjs'; + +let workspace; +let dir; + +beforeEach(() => { + workspace = mkdtempSync(join(tmpdir(), 'surface-')); + dir = join(workspace, 'wiki'); +}); +afterEach(() => rmSync(workspace, { recursive: true, force: true })); + +/** + * A transcript with `turns` assistant rows, the last one reporting `used` tokens of context. + * + * The usage shape mirrors a real transcript: the context size at a turn is cache reads plus cache + * writes plus fresh input, not the marginal cost of that turn alone. + */ +function transcript(turns, used = 60_000) { + const path = join(workspace, `t-${turns}-${used}.jsonl`); + const rows = []; + for (let i = 0; i < turns; i++) { + const last = i === turns - 1; + rows.push(JSON.stringify({ + type: 'assistant', + timestamp: new Date(1_700_000_000_000 + i * 1000).toISOString(), + message: { + role: 'assistant', + model: 'claude-opus-4', + usage: { + cache_read_input_tokens: last ? used - 1_000 : 100, + cache_creation_input_tokens: last ? 900 : 10, + input_tokens: last ? 100 : 5, + }, + }, + })); + } + writeFileSync(path, `${rows.join('\n')}\n`); + return path; +} + +/** Arms with a control arm big enough to publish a counterfactual, plus a short runway. */ +function seedArms({ treated = 12, withheld = 12, treatedRead = 500, withheldRead = 9000 } = {}) { + for (let i = 0; i < treated; i++) { + record(dir, { kind: 'inject', anchor: `/t${i}.ts`, sessionId: 'live', holdout: false, tokens: 100 }); + recordRead(dir, { anchor: `/t${i}.ts`, sessionId: 'live', bytes: treatedRead * 4 }); + } + for (let i = 0; i < withheld; i++) { + record(dir, { kind: 'inject', anchor: `/c${i}.ts`, sessionId: 'live', holdout: true, tokens: 0 }); + recordRead(dir, { anchor: `/c${i}.ts`, sessionId: 'live', bytes: withheldRead * 4 }); + } +} + +describe('the session numbers come from the transcript', () => { + test('used is the size of the context, not the sum of the turns', () => { + // Summing per-turn inputs counts the carried prefix once per turn and reports a number many + // times the window size -- which would make every runway look like an emergency. + const usage = sessionUsage(transcript(20, 60_000)); + expect(usage.used).toBe(60_000); + expect(usage.turns).toBe(20); + expect(usage.capacity).toBe(DEFAULT_CAPACITY); + }); + + test('no usage rows is null, not a zeroed session', () => { + const path = join(workspace, 'empty.jsonl'); + writeFileSync(path, '{"type":"user","message":{"role":"user","content":"hi"}}\n'); + expect(sessionUsage(path)).toBeNull(); + }); + + test('a missing transcript is null rather than a throw', () => { + expect(sessionUsage(join(workspace, 'nope.jsonl'))).toBeNull(); + }); +}); + +describe('the cost is bounded before anything is opened', () => { + test('inside the throttle window nothing is read at all', () => { + // The transcript path is deliberately nonexistent: if the throttle were checked after the + // read, this would still return null and the test would pass for the wrong reason. It cannot + // even be attempted, so the state must come back byte-identical. + const previous = { checkedAt: 1_000, shown: 4 }; + const out = maybeSurface(dir, { + transcriptPath: join(workspace, 'never-opened.jsonl'), + sessionId: 'live', + state: { forecast: previous }, + now: 1_000 + SURFACE_INTERVAL_MS - 1, + }); + expect(out.text).toBeNull(); + expect(out.state).toBe(previous); // the same object, untouched + }); + + test('past the window the clock is stamped even when there is nothing to say', () => { + // Otherwise a session with no usage rows re-parses the transcript on every single tool call. + const out = maybeSurface(dir, { + transcriptPath: join(workspace, 'absent.jsonl'), + sessionId: 'live', + state: { forecast: { checkedAt: 1_000, shown: null } }, + now: 1_000 + SURFACE_INTERVAL_MS + 1, + }); + expect(out.text).toBeNull(); + expect(out.state.checkedAt).toBe(1_000 + SURFACE_INTERVAL_MS + 1); + }); + + test('the interval is a real bound', () => { + expect(SURFACE_INTERVAL_MS).toBeGreaterThan(0); + expect(Number.isFinite(SURFACE_INTERVAL_MS)).toBe(true); + }); +}); + +describe('it interrupts only when the forecast has earned it', () => { + test('a comfortable runway says nothing', () => { + seedArms(); + // 10% used over 20 turns leaves a very long runway. + const out = maybeSurface(dir, { + transcriptPath: transcript(20, 20_000), sessionId: 'live', state: {}, now: 5_000, + }); + expect(out.text).toBeNull(); + expect(out.state.checkedAt).toBe(5_000); + }); + + test('a short runway surfaces, once', () => { + seedArms(); + // 190k of 200k used over 20 turns: about 1 turn left. + const path = transcript(20, 190_000); + const first = maybeSurface(dir, { transcriptPath: path, sessionId: 'live', state: {}, now: 5_000 }); + expect(first.text).toMatch(/turns to compaction/); + expect(first.state.shown).toBeGreaterThanOrEqual(0); + + // Same runway a window later: different, not actionable, so it stays quiet. + const second = maybeSurface(dir, { + transcriptPath: path, + sessionId: 'live', + state: { forecast: first.state }, + now: 5_000 + SURFACE_INTERVAL_MS + 1, + }); + expect(second.text).toBeNull(); + }); + + test('the comparison is against the last panel SHOWN, not the last computed', () => { + // Otherwise the runway drifts down one throttle window at a time, each step too small to trip + // the threshold test, and the interruption never fires at all. + seedArms(); + const shownAt = maybeSurface(dir, { + transcriptPath: transcript(20, 190_000), sessionId: 'live', state: {}, now: 5_000, + }); + expect(shownAt.text).not.toBeNull(); + expect(shownAt.state.shown).not.toBeUndefined(); + }); +}); + +describe('the calibration loop is closed by compaction', () => { + test('a forecast made earlier in the session is scored against the elapsed turns', () => { + // predictedTurns is an INTERVAL, so the ground truth is turns elapsed between the prediction + // and compaction -- not the turn number compaction happened on. + logForecast(dir, { sessionId: 'live', predictedTurns: 10, used: 60_000, capacity: 200_000, turns: 20 }); + closeForecast(dir, { transcriptPath: transcript(31, 190_000), sessionId: 'live' }); + + const scored = reliability(dir); + expect(scored.scored).toBe(1); + // 31 turns at compaction minus 20 at prediction = 11 elapsed, against a prediction of 10. + expect(scored.hitRate).toBe(1); + }); + + test('a forecast with no recorded origin is left open rather than scored against an absolute', () => { + // Scoring an interval against a turn NUMBER manufactures an error instead of measuring one. + record(dir, { + kind: 'forecast', sessionId: 'live', predictedTurns: 10, horizon: 'mid', turns: null, + }); + closeForecast(dir, { transcriptPath: transcript(31, 190_000), sessionId: 'live' }); + expect(reliability(dir).scored).toBe(0); + }); + + test('an explicit actualTurns still works, so the interval can be supplied directly', () => { + logForecast(dir, { sessionId: 'live', predictedTurns: 10, used: 1, capacity: 2, turns: 5 }); + observeOutcome(dir, { sessionId: 'live', actualTurns: 12 }); + expect(reliability(dir).scored).toBe(1); + }); + + test('no transcript means no score, and no crash', () => { + logForecast(dir, { sessionId: 'live', predictedTurns: 10, used: 1, capacity: 2, turns: 5 }); + expect(closeForecast(dir, { transcriptPath: join(workspace, 'gone.jsonl'), sessionId: 'live' })).toBe(false); + expect(reliability(dir).scored).toBe(0); + }); +}); + +describe('the wiring is real, not merely present', () => { + // This is the test that would have failed for the whole life of calibration.mjs. Every other + // test in this file passes just as well on a module nothing imports. + const hook = (name) => readFileSync(join(process.cwd(), 'plugin', 'hooks', name), 'utf8'); + + test('the PreToolUse router calls maybeSurface', () => { + expect(hook('pretooluse-router.mjs')).toMatch(/maybeSurface\(/); + }); + + test('the PreCompact hook calls closeForecast', () => { + expect(hook('precompact-optimize.mjs')).toMatch(/closeForecast\(/); + }); + + test('the vendored copy of surface.mjs is in place for the plugin to import', () => { + // The hooks import from ./lib/, which is synced rather than symlinked -- an unsynced module + // means the hook throws on load and fails open, silently doing nothing. + expect(() => readFileSync(join(process.cwd(), 'plugin', 'hooks', 'lib', 'surface.mjs'), 'utf8')) + .not.toThrow(); + }); +});