diff --git a/hooks-core/calibration.mjs b/hooks-core/calibration.mjs index 8b9cd0faf..278eb6616 100644 --- a/hooks-core/calibration.mjs +++ b/hooks-core/calibration.mjs @@ -29,7 +29,20 @@ * that cannot support one. */ -import { record, readMetrics } from './metrics.mjs'; +// readBalance, not readMetrics. Both readers used the windowed firehose, which returns at most +// 2,000,000 bytes and the last 5,000 lines -- and forecast events were not among the protected +// kinds, so they lived in a log dominated by per-tool-call records (metrics.mjs's own comment +// cites 4,735 capture events inside one window). +// +// Two failures followed. In observeOutcome, on a busy project the forecast had scrolled past the +// tail by the time compaction fired, so the outcome was discarded with no record, no error and no +// counter. In reliability, outcomes vanished faster than they accumulated, so rows.length could +// never reach MIN_SCORED and calibrate returned 'not yet calibrated (n/8)' permanently -- leaving +// a user unable to distinguish "never had data" from "the data is being thrown away". +// +// 'forecast' and 'forecast-outcome' are now balance kinds, so they are written to balance.jsonl +// as well and read back unwindowed. +import { record, readBalance } from './metrics.mjs'; /** Horizon buckets, in turns. Error behaves differently across these ranges. */ const HORIZONS = [ @@ -51,8 +64,15 @@ function horizonOf(turns) { return HORIZONS.find((h) => turns <= h.max).name; } -/** Records a prediction so it can be scored when the outcome arrives. */ -export function logForecast(dir, { sessionId, predictedTurns, used, capacity }) { +/** + * Records a prediction so it can be scored when the outcome arrives. + * + * `turns` is the session's turn count AT PREDICTION TIME, and it is what makes the outcome + * computable at all: predictedTurns means "turns from here until compaction", so the ground truth + * is the turns that ELAPSED, not the turn number compaction happened on. Without it the caller at + * compaction has nothing to subtract from and would have to score an interval against an absolute. + */ +export function logForecast(dir, { sessionId, predictedTurns, used, capacity, turns }) { if (!Number.isFinite(predictedTurns)) return; record(dir, { kind: 'forecast', @@ -61,6 +81,7 @@ export function logForecast(dir, { sessionId, predictedTurns, used, capacity }) horizon: horizonOf(predictedTurns), used, capacity, + turns: Number.isFinite(turns) ? turns : null, }); } @@ -71,21 +92,59 @@ export function logForecast(dir, { sessionId, predictedTurns, used, capacity }) * Called when compaction fires: the number of turns that elapsed since the * prediction is the ground truth it was predicting. */ -export function observeOutcome(dir, { sessionId, actualTurns }) { - if (!Number.isFinite(actualTurns)) return; - const open = readMetrics(dir) - .filter((e) => e.kind === 'forecast' && e.sessionId === sessionId && !e.scored) +export function observeOutcome(dir, { sessionId, actualTurns, atTurn }) { + // EITHER THE ELAPSED COUNT OR THE TURN IT HAPPENED ON. + // + // The only real caller -- the PreCompact hook -- knows what turn compaction fired on, not how + // many turns have passed since a prediction it did not make. Deriving the interval here keeps + // that subtraction next to the record that defines it, rather than making every caller + // re-discover which forecast is open in order to compute its own ground truth. + const elapsed = Number.isFinite(actualTurns) + ? actualTurns + : null; + if (elapsed === null && !Number.isFinite(atTurn)) return; + + // CLOSURE IS DERIVED FROM THE OUTCOME ROWS, not from a flag on the forecast. + // + // The old filter was `!e.scored`, but nothing ever writes `scored`: logForecast records + // kind/sessionId/predictedTurns/horizon/used/capacity, and record() is an append-only JSONL + // writer with no update path, so a forecast can never gain the field. The predicate was always + // true, and the last forecast for a session was therefore re-closable without limit. + // + // A session where compaction fires twice with no intervening logForecast popped the SAME + // forecast twice, producing two outcome rows for one prediction. reliability counts ROWS, not + // predictions, so the hit rate and the bias mean both inflate and MIN_SCORED can be satisfied + // by a single prediction re-closed eight times -- precisely the "threshold from a single + // sample" this module exists to prevent. The existing test used a fresh sessionId per + // iteration, so it never exposed it. + const events = readBalance(dir); + const closed = new Set( + events.filter((e) => e.kind === 'forecast-outcome' && e.forecastId).map((e) => e.forecastId), + ); + const open = events + .filter((e) => e.kind === 'forecast' && e.sessionId === sessionId && !closed.has(e.id)) .pop(); if (!open) return; + // Derived only once the open forecast is known, since the interval is measured from ITS turn. + // A forecast recorded before `turns` existed has none, and there is nothing to subtract from -- + // scoring it against an absolute turn number would manufacture an error rather than measure one, + // so it is left open instead. + const actual = elapsed !== null ? elapsed + : (Number.isFinite(open.turns) ? atTurn - open.turns : null); + if (actual === null || !Number.isFinite(actual)) return; + record(dir, { kind: 'forecast-outcome', sessionId, + // Stamped so the next call can see this forecast is spoken for. Without it the set above is + // empty and the same defect returns. + forecastId: open.id, horizon: open.horizon, predictedTurns: open.predictedTurns, - actualTurns, - error: actualTurns - open.predictedTurns, - hit: Math.abs(actualTurns - open.predictedTurns) <= TOLERANCE, + actualTurns: actual, + error: actual - open.predictedTurns, + hit: Math.abs(actual - open.predictedTurns) <= TOLERANCE, }); } @@ -96,7 +155,7 @@ export function observeOutcome(dir, { sessionId, actualTurns }) { * actionable, where "62% accurate" is not. */ export function reliability(dir) { - const outcomes = readMetrics(dir).filter((e) => e.kind === 'forecast-outcome'); + const outcomes = readBalance(dir).filter((e) => e.kind === 'forecast-outcome'); const buckets = {}; for (const horizon of HORIZONS) { @@ -162,7 +221,28 @@ export function calibrate(dir, predictedTurns) { // Refit: shift by the observed bias for this horizon. A bucket that runs long // gets pulled in, and vice versa. - const adjusted = Math.max(0, Math.round(predictedTurns + bucket.bias)); + // + // NOT CLAMPED WITH Math.max, which converted nonsense into a plausible-looking emergency. bias + // is a plain arithmetic mean of signed error, and a mean is not robust to the shape the + // 50%-hit-rate floor admits: a near bucket with errors [0,0,0,0,-30,-30,-30,-30] has four hits, + // so it clears the floor and is called calibrated, and its bias is -15. A raw forecast of 3 + // turns then became Math.max(0, Math.round(3 - 15)) = 0 and was published as fact, with the + // note "within 3 on 50% of 8 past forecasts" -- a zero-turn deadline the forecaster never + // produced, from a bucket that is wrong half the time. + // + // A correction at least as large as the thing it corrects is not a correction. + const adjusted = Math.round(predictedTurns + bucket.bias); + if (Math.abs(bucket.bias) >= predictedTurns || adjusted < 1) { + return { + publishable: false, + predictedTurns, + horizon, + bias: bucket.bias, + reason: `the observed bias at this horizon (${Math.round(bucket.bias)} turns) is as large as ` + + `the forecast itself (${predictedTurns}), so the correction would replace it rather than ` + + 'adjust it', + }; + } return { publishable: true, diff --git a/hooks-core/forecast.mjs b/hooks-core/forecast.mjs index 3276bbe36..7779e1ca5 100644 --- a/hooks-core/forecast.mjs +++ b/hooks-core/forecast.mjs @@ -32,6 +32,7 @@ import { readMetrics, readBalance, isFixtureAnchor, BALANCE_KINDS } from './metrics.mjs'; import { aggregateConsolidation } from './consolidate.mjs'; import { previewQuality } from './expand.mjs'; +import { logForecast, calibrate } from './calibration.mjs'; /** Turns of headroom below which the runway is worth interrupting for. */ export const ACTIONABLE_RUNWAY = 8; @@ -310,7 +311,33 @@ export function forecastPanel(dir, session = {}, findings = []) { if (air) { parts.runway = air; - lines.push(runwayLine(air)); + + // THE FORECAST KEEPS ITS OWN SCORE -- from here, because this is the only place a forecast is + // actually made. calibration.mjs was written for exactly this and had zero shipped importers: + // no forecast was ever logged, no outcome ever observed, reliability always saw an empty set, + // and calibrate was unreachable. So the shipped panel printed precisely the uncalibrated + // number that module's docstring calls "a vibe with a typeface". + // + // Logged BEFORE it is corrected. The raw prediction is what gets scored against the outcome; + // scoring the corrected one would fold the correction back into the next correction and the + // bias would chase its own tail. + logForecast(dir, { + sessionId: session.sessionId, + predictedTurns: air.withGraph, + used: session.used, + capacity: session.capacity, + // The turn this was predicted FROM. predictedTurns is an interval, so without its origin + // the outcome at compaction has nothing to subtract from. + turns: session.turns, + }); + + const score = calibrate(dir, air.withGraph); + parts.calibration = score; + // Published only when the horizon has earned it. An unpublishable score does not replace the + // number -- it is simply absent, exactly as the uncalibrated case reads today. + lines.push(score.publishable + ? `${runwayLine({ ...air, withGraph: score.predictedTurns })}\n (corrected: ${score.note})` + : runwayLine(air)); } if (rate) { diff --git a/hooks-core/metrics.mjs b/hooks-core/metrics.mjs index a33309475..ae62a1166 100644 --- a/hooks-core/metrics.mjs +++ b/hooks-core/metrics.mjs @@ -66,7 +66,17 @@ const balancePath = (dir) => join(dir, 'balance.jsonl'); * starve the measurement. Exported because any reader that needs the unwindowed truth needs to * know which kinds it applies to, and a second copy of this set elsewhere would drift. */ -export const BALANCE_KINDS = new Set(['inject', 'harvest', 'substitute']); +export const BALANCE_KINDS = new Set([ + 'inject', + 'harvest', + 'substitute', + // The forecast's own score. Both kinds are rare and accumulate slowly -- MIN_SCORED is 8 -- + // while the firehose they shared is dominated by per-tool-call records, so they were evicted + // faster than they could accumulate: calibrate() returned 'not yet calibrated (n/8)' forever + // and an outcome whose forecast had scrolled away was dropped with no record at all. + 'forecast', + 'forecast-outcome', +]); /** * Is this touch in the holdout arm? diff --git a/hooks-core/policy.mjs b/hooks-core/policy.mjs index bbe5644fd..ecbde570b 100644 --- a/hooks-core/policy.mjs +++ b/hooks-core/policy.mjs @@ -270,7 +270,7 @@ function statePath(sessionId, agent) { /** A usable state object, whatever was on disk. */ function emptyState() { - return { seen: {}, denied: {}, injected: [], actCounts: {} }; + return { seen: {}, denied: {}, injected: [], actCounts: {}, forecast: null }; } /** @@ -310,6 +310,16 @@ export function loadState(sessionId, agent) { parsed.actCounts && typeof parsed.actCounts === 'object' && !Array.isArray(parsed.actCounts) ? parsed.actCounts : {}, + // THE SAME REASON A THIRD TIME. The forecast throttle records when the panel was last + // computed and what runway it last SHOWED, and both are meaningless within a single hook + // process -- every tool call is a new one. Dropped here, `checkedAt` would reset on every + // call and the panel would rebuild itself, transcript parse and all, on each tool use; and + // `shown` would reset, so worthSurfacing would compare against nothing and re-interrupt with + // the same runway forever. + forecast: + parsed.forecast && typeof parsed.forecast === 'object' && !Array.isArray(parsed.forecast) + ? parsed.forecast + : null, }; } catch { return emptyState(); @@ -381,6 +391,18 @@ export function saveState(sessionId, state, agent) { } return out; })(), + // LATEST CHECK WINS, which is the opposite direction from the fields above and correct for + // this one. `seen`, `denied`, `injected` and `actCounts` are all append-only, so a union or + // a max is the safe merge. The forecast throttle is a POINT IN TIME: taking the older of two + // concurrent checks would re-open the window and let the panel be rebuilt immediately, which + // is the cost the throttle exists to bound. + forecast: (() => { + const mine = state.forecast || null; + const theirs = current.forecast || null; + if (!mine) return theirs; + if (!theirs) return mine; + return (mine.checkedAt || 0) >= (theirs.checkedAt || 0) ? mine : theirs; + })(), }; // Write-then-rename so a reader never observes a half-written file. diff --git a/hooks-core/surface.mjs b/hooks-core/surface.mjs new file mode 100644 index 000000000..a4dff3282 --- /dev/null +++ b/hooks-core/surface.mjs @@ -0,0 +1,164 @@ +/** + * The entry point the forecast never had. + * + * forecast.mjs computes a runway, calibration.mjs scores it, and worthSurfacing decides whether a + * change is worth interrupting for -- and none of it ran. The reachability guard listed + * forecastPanel, worthSurfacing, logForecast and observeOutcome as unwired, which is a polite way + * of saying the feature did not exist for any user. This module is the missing half: it derives + * the session numbers the panel needs from the transcript, decides when the panel has earned an + * interruption, and closes the calibration loop when compaction fires. + * + * THREE COSTS ARE BOUNDED HERE, because this runs on the PreToolUse path and every tool call is a + * separate process: + * + * THE READ readCacheUsage parses up to 4 MB of transcript. Doing that on every tool call + * would tax the hook that exists to save the user money. + * THE COMPUTE the panel walks the metrics log, the balance log and the graph. + * THE CONTEXT the panel is only worth its own tokens when it says something actionable. + * + * So the throttle is checked FIRST, from persisted state, before any file is opened. Everything + * expensive happens after a clock comparison. + * + * NOTHING HERE IS ALLOWED TO BREAK A TOOL CALL. Every entry point returns null on failure rather + * than throwing: a forecast is a courtesy, and the hook's promise is that a defect in it costs the + * user nothing. + */ + +import { forecastPanel, worthSurfacing } from './forecast.mjs'; +import { observeOutcome } from './calibration.mjs'; +import { readCacheUsage } from './cache.mjs'; +import { readBalance } from './metrics.mjs'; + +/** + * How long between two attempts to build the panel. + * + * Not a tuning knob so much as an admission: the panel costs a transcript parse, and a user makes + * many tool calls per minute. Two minutes is short enough that a runway falling into single digits + * is still caught with turns to spare, and long enough that the check is free in aggregate. + */ +export const SURFACE_INTERVAL_MS = + Number(process.env.TOKEN_OPTIMIZER_SURFACE_INTERVAL_MS) || 120_000; + +/** + * The context window to measure the runway against. + * + * NOT DISCOVERABLE from the transcript: the usage rows carry what was spent, never the ceiling it + * was spent against. Defaulting is therefore unavoidable, so the default is stated rather than + * buried, and overridable for anyone running a different window. + */ +export const DEFAULT_CAPACITY = + Number(process.env.TOKEN_OPTIMIZER_CONTEXT_CAPACITY) || 200_000; + +/** + * What the session has spent, read from its own transcript. + * + * `used` is the FULL input size of the most recent request -- cache reads plus cache writes plus + * fresh input -- which is what the context window actually holds, rather than the marginal cost of + * that one turn. Summing the per-turn inputs instead would count the same carried prefix once per + * turn and report a number many times the window size. + * + * Returns null rather than a zeroed object when there is nothing to measure, so a caller cannot + * mistake "no data" for "an empty context". + */ +export function sessionUsage(transcriptPath, { capacity = DEFAULT_CAPACITY } = {}) { + let turns; + try { + turns = readCacheUsage(transcriptPath); + } catch { + return null; + } + if (!turns?.length) return null; + + const last = turns[turns.length - 1]; + const used = (last.read || 0) + (last.written || 0) + (last.input || 0); + if (!used) return null; + + return { used, capacity, turns: turns.length }; +} + +/** Intercepted touches recorded for this session, which converts a per-touch saving to per-turn. */ +function touchesFor(dir, sessionId) { + try { + return readBalance(dir).filter((e) => e.kind === 'inject' && e.sessionId === sessionId).length; + } catch { + return 0; + } +} + +/** + * Should the forecast interrupt right now, and with what? + * + * The state object is the caller's persisted per-session state; this reads `forecast` from it and + * returns the value to store back, so the throttle and the last-shown runway survive across hook + * processes. Returning the next state rather than mutating it keeps this testable without a + * filesystem. + * + * @returns { text, state } -- text is null when nothing should be shown. + */ +export function maybeSurface(dir, { + transcriptPath, sessionId, state = {}, now = Date.now(), findings = [], +} = {}) { + const previous = state.forecast || null; + + // THE THROTTLE IS FIRST, and deliberately before any I/O. Everything below opens files. + if (previous?.checkedAt && now - previous.checkedAt < SURFACE_INTERVAL_MS) { + return { text: null, state: previous }; + } + + let usage; + try { + usage = sessionUsage(transcriptPath); + } catch { + return { text: null, state: previous }; + } + // The clock is stamped even when there is nothing to measure, so a session with no usage rows + // does not re-attempt the transcript parse on every single tool call. + if (!usage) return { text: null, state: { ...previous, checkedAt: now } }; + + let panel; + try { + panel = forecastPanel(dir, { + ...usage, + sessionId, + touches: touchesFor(dir, sessionId), + }, findings); + } catch { + return { text: null, state: { ...previous, checkedAt: now } }; + } + if (!panel) return { text: null, state: { ...previous, checkedAt: now } }; + + // worthSurfacing compares against the last panel that was actually SHOWN, not the last one + // computed. Comparing against the last computed panel would let the runway drift down past the + // threshold one throttle window at a time and never trip the "crossed it" test. + const shown = previous?.shown ? { parts: { runway: { withGraph: previous.shown } } } : null; + if (!worthSurfacing(panel, shown)) { + return { text: null, state: { ...previous, checkedAt: now } }; + } + + return { + text: panel.text, + state: { checkedAt: now, shown: panel.parts.runway.withGraph }, + }; +} + +/** + * Closes the calibration loop, at the only moment that can close it. + * + * Compaction firing IS the ground truth a runway forecast was predicting, so this belongs on the + * PreCompact path and nowhere else. `actualTurns` is turns ELAPSED between the prediction and + * compaction, which is what predictedTurns meant -- so the forecast has to have recorded the turn + * it was made on, and observeOutcome subtracts. + * + * Silent by design: nothing is shown to anybody. It writes one record so that the NEXT session's + * forecast can be published with a track record instead of as a bare assertion. + */ +export function closeForecast(dir, { transcriptPath, sessionId } = {}) { + try { + const usage = sessionUsage(transcriptPath); + if (!usage) return false; + observeOutcome(dir, { sessionId, atTurn: usage.turns }); + return true; + } catch { + return false; + } +} diff --git a/integrations/codex/hooks/lib/calibration.mjs b/integrations/codex/hooks/lib/calibration.mjs index 718db89cd..42de126f9 100644 --- a/integrations/codex/hooks/lib/calibration.mjs +++ b/integrations/codex/hooks/lib/calibration.mjs @@ -31,7 +31,20 @@ * that cannot support one. */ -import { record, readMetrics } from './metrics.mjs'; +// readBalance, not readMetrics. Both readers used the windowed firehose, which returns at most +// 2,000,000 bytes and the last 5,000 lines -- and forecast events were not among the protected +// kinds, so they lived in a log dominated by per-tool-call records (metrics.mjs's own comment +// cites 4,735 capture events inside one window). +// +// Two failures followed. In observeOutcome, on a busy project the forecast had scrolled past the +// tail by the time compaction fired, so the outcome was discarded with no record, no error and no +// counter. In reliability, outcomes vanished faster than they accumulated, so rows.length could +// never reach MIN_SCORED and calibrate returned 'not yet calibrated (n/8)' permanently -- leaving +// a user unable to distinguish "never had data" from "the data is being thrown away". +// +// 'forecast' and 'forecast-outcome' are now balance kinds, so they are written to balance.jsonl +// as well and read back unwindowed. +import { record, readBalance } from './metrics.mjs'; /** Horizon buckets, in turns. Error behaves differently across these ranges. */ const HORIZONS = [ @@ -53,8 +66,15 @@ function horizonOf(turns) { return HORIZONS.find((h) => turns <= h.max).name; } -/** Records a prediction so it can be scored when the outcome arrives. */ -export function logForecast(dir, { sessionId, predictedTurns, used, capacity }) { +/** + * Records a prediction so it can be scored when the outcome arrives. + * + * `turns` is the session's turn count AT PREDICTION TIME, and it is what makes the outcome + * computable at all: predictedTurns means "turns from here until compaction", so the ground truth + * is the turns that ELAPSED, not the turn number compaction happened on. Without it the caller at + * compaction has nothing to subtract from and would have to score an interval against an absolute. + */ +export function logForecast(dir, { sessionId, predictedTurns, used, capacity, turns }) { if (!Number.isFinite(predictedTurns)) return; record(dir, { kind: 'forecast', @@ -63,6 +83,7 @@ export function logForecast(dir, { sessionId, predictedTurns, used, capacity }) horizon: horizonOf(predictedTurns), used, capacity, + turns: Number.isFinite(turns) ? turns : null, }); } @@ -73,21 +94,59 @@ export function logForecast(dir, { sessionId, predictedTurns, used, capacity }) * Called when compaction fires: the number of turns that elapsed since the * prediction is the ground truth it was predicting. */ -export function observeOutcome(dir, { sessionId, actualTurns }) { - if (!Number.isFinite(actualTurns)) return; - const open = readMetrics(dir) - .filter((e) => e.kind === 'forecast' && e.sessionId === sessionId && !e.scored) +export function observeOutcome(dir, { sessionId, actualTurns, atTurn }) { + // EITHER THE ELAPSED COUNT OR THE TURN IT HAPPENED ON. + // + // The only real caller -- the PreCompact hook -- knows what turn compaction fired on, not how + // many turns have passed since a prediction it did not make. Deriving the interval here keeps + // that subtraction next to the record that defines it, rather than making every caller + // re-discover which forecast is open in order to compute its own ground truth. + const elapsed = Number.isFinite(actualTurns) + ? actualTurns + : null; + if (elapsed === null && !Number.isFinite(atTurn)) return; + + // CLOSURE IS DERIVED FROM THE OUTCOME ROWS, not from a flag on the forecast. + // + // The old filter was `!e.scored`, but nothing ever writes `scored`: logForecast records + // kind/sessionId/predictedTurns/horizon/used/capacity, and record() is an append-only JSONL + // writer with no update path, so a forecast can never gain the field. The predicate was always + // true, and the last forecast for a session was therefore re-closable without limit. + // + // A session where compaction fires twice with no intervening logForecast popped the SAME + // forecast twice, producing two outcome rows for one prediction. reliability counts ROWS, not + // predictions, so the hit rate and the bias mean both inflate and MIN_SCORED can be satisfied + // by a single prediction re-closed eight times -- precisely the "threshold from a single + // sample" this module exists to prevent. The existing test used a fresh sessionId per + // iteration, so it never exposed it. + const events = readBalance(dir); + const closed = new Set( + events.filter((e) => e.kind === 'forecast-outcome' && e.forecastId).map((e) => e.forecastId), + ); + const open = events + .filter((e) => e.kind === 'forecast' && e.sessionId === sessionId && !closed.has(e.id)) .pop(); if (!open) return; + // Derived only once the open forecast is known, since the interval is measured from ITS turn. + // A forecast recorded before `turns` existed has none, and there is nothing to subtract from -- + // scoring it against an absolute turn number would manufacture an error rather than measure one, + // so it is left open instead. + const actual = elapsed !== null ? elapsed + : (Number.isFinite(open.turns) ? atTurn - open.turns : null); + if (actual === null || !Number.isFinite(actual)) return; + record(dir, { kind: 'forecast-outcome', sessionId, + // Stamped so the next call can see this forecast is spoken for. Without it the set above is + // empty and the same defect returns. + forecastId: open.id, horizon: open.horizon, predictedTurns: open.predictedTurns, - actualTurns, - error: actualTurns - open.predictedTurns, - hit: Math.abs(actualTurns - open.predictedTurns) <= TOLERANCE, + actualTurns: actual, + error: actual - open.predictedTurns, + hit: Math.abs(actual - open.predictedTurns) <= TOLERANCE, }); } @@ -98,7 +157,7 @@ export function observeOutcome(dir, { sessionId, actualTurns }) { * actionable, where "62% accurate" is not. */ export function reliability(dir) { - const outcomes = readMetrics(dir).filter((e) => e.kind === 'forecast-outcome'); + const outcomes = readBalance(dir).filter((e) => e.kind === 'forecast-outcome'); const buckets = {}; for (const horizon of HORIZONS) { @@ -164,7 +223,28 @@ export function calibrate(dir, predictedTurns) { // Refit: shift by the observed bias for this horizon. A bucket that runs long // gets pulled in, and vice versa. - const adjusted = Math.max(0, Math.round(predictedTurns + bucket.bias)); + // + // NOT CLAMPED WITH Math.max, which converted nonsense into a plausible-looking emergency. bias + // is a plain arithmetic mean of signed error, and a mean is not robust to the shape the + // 50%-hit-rate floor admits: a near bucket with errors [0,0,0,0,-30,-30,-30,-30] has four hits, + // so it clears the floor and is called calibrated, and its bias is -15. A raw forecast of 3 + // turns then became Math.max(0, Math.round(3 - 15)) = 0 and was published as fact, with the + // note "within 3 on 50% of 8 past forecasts" -- a zero-turn deadline the forecaster never + // produced, from a bucket that is wrong half the time. + // + // A correction at least as large as the thing it corrects is not a correction. + const adjusted = Math.round(predictedTurns + bucket.bias); + if (Math.abs(bucket.bias) >= predictedTurns || adjusted < 1) { + return { + publishable: false, + predictedTurns, + horizon, + bias: bucket.bias, + reason: `the observed bias at this horizon (${Math.round(bucket.bias)} turns) is as large as ` + + `the forecast itself (${predictedTurns}), so the correction would replace it rather than ` + + 'adjust it', + }; + } return { publishable: true, diff --git a/integrations/codex/hooks/lib/forecast.mjs b/integrations/codex/hooks/lib/forecast.mjs index ed698b1a0..39358e76a 100644 --- a/integrations/codex/hooks/lib/forecast.mjs +++ b/integrations/codex/hooks/lib/forecast.mjs @@ -34,6 +34,7 @@ import { readMetrics, readBalance, isFixtureAnchor, BALANCE_KINDS } from './metrics.mjs'; import { aggregateConsolidation } from './consolidate.mjs'; import { previewQuality } from './expand.mjs'; +import { logForecast, calibrate } from './calibration.mjs'; /** Turns of headroom below which the runway is worth interrupting for. */ export const ACTIONABLE_RUNWAY = 8; @@ -312,7 +313,33 @@ export function forecastPanel(dir, session = {}, findings = []) { if (air) { parts.runway = air; - lines.push(runwayLine(air)); + + // THE FORECAST KEEPS ITS OWN SCORE -- from here, because this is the only place a forecast is + // actually made. calibration.mjs was written for exactly this and had zero shipped importers: + // no forecast was ever logged, no outcome ever observed, reliability always saw an empty set, + // and calibrate was unreachable. So the shipped panel printed precisely the uncalibrated + // number that module's docstring calls "a vibe with a typeface". + // + // Logged BEFORE it is corrected. The raw prediction is what gets scored against the outcome; + // scoring the corrected one would fold the correction back into the next correction and the + // bias would chase its own tail. + logForecast(dir, { + sessionId: session.sessionId, + predictedTurns: air.withGraph, + used: session.used, + capacity: session.capacity, + // The turn this was predicted FROM. predictedTurns is an interval, so without its origin + // the outcome at compaction has nothing to subtract from. + turns: session.turns, + }); + + const score = calibrate(dir, air.withGraph); + parts.calibration = score; + // Published only when the horizon has earned it. An unpublishable score does not replace the + // number -- it is simply absent, exactly as the uncalibrated case reads today. + lines.push(score.publishable + ? `${runwayLine({ ...air, withGraph: score.predictedTurns })}\n (corrected: ${score.note})` + : runwayLine(air)); } if (rate) { diff --git a/integrations/codex/hooks/lib/metrics.mjs b/integrations/codex/hooks/lib/metrics.mjs index b762df57c..4b0eb2712 100644 --- a/integrations/codex/hooks/lib/metrics.mjs +++ b/integrations/codex/hooks/lib/metrics.mjs @@ -68,7 +68,17 @@ const balancePath = (dir) => join(dir, 'balance.jsonl'); * starve the measurement. Exported because any reader that needs the unwindowed truth needs to * know which kinds it applies to, and a second copy of this set elsewhere would drift. */ -export const BALANCE_KINDS = new Set(['inject', 'harvest', 'substitute']); +export const BALANCE_KINDS = new Set([ + 'inject', + 'harvest', + 'substitute', + // The forecast's own score. Both kinds are rare and accumulate slowly -- MIN_SCORED is 8 -- + // while the firehose they shared is dominated by per-tool-call records, so they were evicted + // faster than they could accumulate: calibrate() returned 'not yet calibrated (n/8)' forever + // and an outcome whose forecast had scrolled away was dropped with no record at all. + 'forecast', + 'forecast-outcome', +]); /** * Is this touch in the holdout arm? diff --git a/integrations/codex/hooks/lib/policy.mjs b/integrations/codex/hooks/lib/policy.mjs index b3871c27d..1fa14c348 100644 --- a/integrations/codex/hooks/lib/policy.mjs +++ b/integrations/codex/hooks/lib/policy.mjs @@ -272,7 +272,7 @@ function statePath(sessionId, agent) { /** A usable state object, whatever was on disk. */ function emptyState() { - return { seen: {}, denied: {}, injected: [], actCounts: {} }; + return { seen: {}, denied: {}, injected: [], actCounts: {}, forecast: null }; } /** @@ -312,6 +312,16 @@ export function loadState(sessionId, agent) { parsed.actCounts && typeof parsed.actCounts === 'object' && !Array.isArray(parsed.actCounts) ? parsed.actCounts : {}, + // THE SAME REASON A THIRD TIME. The forecast throttle records when the panel was last + // computed and what runway it last SHOWED, and both are meaningless within a single hook + // process -- every tool call is a new one. Dropped here, `checkedAt` would reset on every + // call and the panel would rebuild itself, transcript parse and all, on each tool use; and + // `shown` would reset, so worthSurfacing would compare against nothing and re-interrupt with + // the same runway forever. + forecast: + parsed.forecast && typeof parsed.forecast === 'object' && !Array.isArray(parsed.forecast) + ? parsed.forecast + : null, }; } catch { return emptyState(); @@ -383,6 +393,18 @@ export function saveState(sessionId, state, agent) { } return out; })(), + // LATEST CHECK WINS, which is the opposite direction from the fields above and correct for + // this one. `seen`, `denied`, `injected` and `actCounts` are all append-only, so a union or + // a max is the safe merge. The forecast throttle is a POINT IN TIME: taking the older of two + // concurrent checks would re-open the window and let the panel be rebuilt immediately, which + // is the cost the throttle exists to bound. + forecast: (() => { + const mine = state.forecast || null; + const theirs = current.forecast || null; + if (!mine) return theirs; + if (!theirs) return mine; + return (mine.checkedAt || 0) >= (theirs.checkedAt || 0) ? mine : theirs; + })(), }; // Write-then-rename so a reader never observes a half-written file. diff --git a/integrations/codex/hooks/lib/surface.mjs b/integrations/codex/hooks/lib/surface.mjs new file mode 100644 index 000000000..0ec05161c --- /dev/null +++ b/integrations/codex/hooks/lib/surface.mjs @@ -0,0 +1,166 @@ +// GENERATED FILE -- do not edit. +// Source of truth: hooks-core/surface.mjs. Regenerate with `npm run sync:hooks`. +/** + * The entry point the forecast never had. + * + * forecast.mjs computes a runway, calibration.mjs scores it, and worthSurfacing decides whether a + * change is worth interrupting for -- and none of it ran. The reachability guard listed + * forecastPanel, worthSurfacing, logForecast and observeOutcome as unwired, which is a polite way + * of saying the feature did not exist for any user. This module is the missing half: it derives + * the session numbers the panel needs from the transcript, decides when the panel has earned an + * interruption, and closes the calibration loop when compaction fires. + * + * THREE COSTS ARE BOUNDED HERE, because this runs on the PreToolUse path and every tool call is a + * separate process: + * + * THE READ readCacheUsage parses up to 4 MB of transcript. Doing that on every tool call + * would tax the hook that exists to save the user money. + * THE COMPUTE the panel walks the metrics log, the balance log and the graph. + * THE CONTEXT the panel is only worth its own tokens when it says something actionable. + * + * So the throttle is checked FIRST, from persisted state, before any file is opened. Everything + * expensive happens after a clock comparison. + * + * NOTHING HERE IS ALLOWED TO BREAK A TOOL CALL. Every entry point returns null on failure rather + * than throwing: a forecast is a courtesy, and the hook's promise is that a defect in it costs the + * user nothing. + */ + +import { forecastPanel, worthSurfacing } from './forecast.mjs'; +import { observeOutcome } from './calibration.mjs'; +import { readCacheUsage } from './cache.mjs'; +import { readBalance } from './metrics.mjs'; + +/** + * How long between two attempts to build the panel. + * + * Not a tuning knob so much as an admission: the panel costs a transcript parse, and a user makes + * many tool calls per minute. Two minutes is short enough that a runway falling into single digits + * is still caught with turns to spare, and long enough that the check is free in aggregate. + */ +export const SURFACE_INTERVAL_MS = + Number(process.env.TOKEN_OPTIMIZER_SURFACE_INTERVAL_MS) || 120_000; + +/** + * The context window to measure the runway against. + * + * NOT DISCOVERABLE from the transcript: the usage rows carry what was spent, never the ceiling it + * was spent against. Defaulting is therefore unavoidable, so the default is stated rather than + * buried, and overridable for anyone running a different window. + */ +export const DEFAULT_CAPACITY = + Number(process.env.TOKEN_OPTIMIZER_CONTEXT_CAPACITY) || 200_000; + +/** + * What the session has spent, read from its own transcript. + * + * `used` is the FULL input size of the most recent request -- cache reads plus cache writes plus + * fresh input -- which is what the context window actually holds, rather than the marginal cost of + * that one turn. Summing the per-turn inputs instead would count the same carried prefix once per + * turn and report a number many times the window size. + * + * Returns null rather than a zeroed object when there is nothing to measure, so a caller cannot + * mistake "no data" for "an empty context". + */ +export function sessionUsage(transcriptPath, { capacity = DEFAULT_CAPACITY } = {}) { + let turns; + try { + turns = readCacheUsage(transcriptPath); + } catch { + return null; + } + if (!turns?.length) return null; + + const last = turns[turns.length - 1]; + const used = (last.read || 0) + (last.written || 0) + (last.input || 0); + if (!used) return null; + + return { used, capacity, turns: turns.length }; +} + +/** Intercepted touches recorded for this session, which converts a per-touch saving to per-turn. */ +function touchesFor(dir, sessionId) { + try { + return readBalance(dir).filter((e) => e.kind === 'inject' && e.sessionId === sessionId).length; + } catch { + return 0; + } +} + +/** + * Should the forecast interrupt right now, and with what? + * + * The state object is the caller's persisted per-session state; this reads `forecast` from it and + * returns the value to store back, so the throttle and the last-shown runway survive across hook + * processes. Returning the next state rather than mutating it keeps this testable without a + * filesystem. + * + * @returns { text, state } -- text is null when nothing should be shown. + */ +export function maybeSurface(dir, { + transcriptPath, sessionId, state = {}, now = Date.now(), findings = [], +} = {}) { + const previous = state.forecast || null; + + // THE THROTTLE IS FIRST, and deliberately before any I/O. Everything below opens files. + if (previous?.checkedAt && now - previous.checkedAt < SURFACE_INTERVAL_MS) { + return { text: null, state: previous }; + } + + let usage; + try { + usage = sessionUsage(transcriptPath); + } catch { + return { text: null, state: previous }; + } + // The clock is stamped even when there is nothing to measure, so a session with no usage rows + // does not re-attempt the transcript parse on every single tool call. + if (!usage) return { text: null, state: { ...previous, checkedAt: now } }; + + let panel; + try { + panel = forecastPanel(dir, { + ...usage, + sessionId, + touches: touchesFor(dir, sessionId), + }, findings); + } catch { + return { text: null, state: { ...previous, checkedAt: now } }; + } + if (!panel) return { text: null, state: { ...previous, checkedAt: now } }; + + // worthSurfacing compares against the last panel that was actually SHOWN, not the last one + // computed. Comparing against the last computed panel would let the runway drift down past the + // threshold one throttle window at a time and never trip the "crossed it" test. + const shown = previous?.shown ? { parts: { runway: { withGraph: previous.shown } } } : null; + if (!worthSurfacing(panel, shown)) { + return { text: null, state: { ...previous, checkedAt: now } }; + } + + return { + text: panel.text, + state: { checkedAt: now, shown: panel.parts.runway.withGraph }, + }; +} + +/** + * Closes the calibration loop, at the only moment that can close it. + * + * Compaction firing IS the ground truth a runway forecast was predicting, so this belongs on the + * PreCompact path and nowhere else. `actualTurns` is turns ELAPSED between the prediction and + * compaction, which is what predictedTurns meant -- so the forecast has to have recorded the turn + * it was made on, and observeOutcome subtracts. + * + * Silent by design: nothing is shown to anybody. It writes one record so that the NEXT session's + * forecast can be published with a track record instead of as a bare assertion. + */ +export function closeForecast(dir, { transcriptPath, sessionId } = {}) { + try { + const usage = sessionUsage(transcriptPath); + if (!usage) return false; + observeOutcome(dir, { sessionId, atTurn: usage.turns }); + return true; + } catch { + return false; + } +} diff --git a/integrations/codex/plugin/hooks/lib/calibration.mjs b/integrations/codex/plugin/hooks/lib/calibration.mjs index 718db89cd..42de126f9 100644 --- a/integrations/codex/plugin/hooks/lib/calibration.mjs +++ b/integrations/codex/plugin/hooks/lib/calibration.mjs @@ -31,7 +31,20 @@ * that cannot support one. */ -import { record, readMetrics } from './metrics.mjs'; +// readBalance, not readMetrics. Both readers used the windowed firehose, which returns at most +// 2,000,000 bytes and the last 5,000 lines -- and forecast events were not among the protected +// kinds, so they lived in a log dominated by per-tool-call records (metrics.mjs's own comment +// cites 4,735 capture events inside one window). +// +// Two failures followed. In observeOutcome, on a busy project the forecast had scrolled past the +// tail by the time compaction fired, so the outcome was discarded with no record, no error and no +// counter. In reliability, outcomes vanished faster than they accumulated, so rows.length could +// never reach MIN_SCORED and calibrate returned 'not yet calibrated (n/8)' permanently -- leaving +// a user unable to distinguish "never had data" from "the data is being thrown away". +// +// 'forecast' and 'forecast-outcome' are now balance kinds, so they are written to balance.jsonl +// as well and read back unwindowed. +import { record, readBalance } from './metrics.mjs'; /** Horizon buckets, in turns. Error behaves differently across these ranges. */ const HORIZONS = [ @@ -53,8 +66,15 @@ function horizonOf(turns) { return HORIZONS.find((h) => turns <= h.max).name; } -/** Records a prediction so it can be scored when the outcome arrives. */ -export function logForecast(dir, { sessionId, predictedTurns, used, capacity }) { +/** + * Records a prediction so it can be scored when the outcome arrives. + * + * `turns` is the session's turn count AT PREDICTION TIME, and it is what makes the outcome + * computable at all: predictedTurns means "turns from here until compaction", so the ground truth + * is the turns that ELAPSED, not the turn number compaction happened on. Without it the caller at + * compaction has nothing to subtract from and would have to score an interval against an absolute. + */ +export function logForecast(dir, { sessionId, predictedTurns, used, capacity, turns }) { if (!Number.isFinite(predictedTurns)) return; record(dir, { kind: 'forecast', @@ -63,6 +83,7 @@ export function logForecast(dir, { sessionId, predictedTurns, used, capacity }) horizon: horizonOf(predictedTurns), used, capacity, + turns: Number.isFinite(turns) ? turns : null, }); } @@ -73,21 +94,59 @@ export function logForecast(dir, { sessionId, predictedTurns, used, capacity }) * Called when compaction fires: the number of turns that elapsed since the * prediction is the ground truth it was predicting. */ -export function observeOutcome(dir, { sessionId, actualTurns }) { - if (!Number.isFinite(actualTurns)) return; - const open = readMetrics(dir) - .filter((e) => e.kind === 'forecast' && e.sessionId === sessionId && !e.scored) +export function observeOutcome(dir, { sessionId, actualTurns, atTurn }) { + // EITHER THE ELAPSED COUNT OR THE TURN IT HAPPENED ON. + // + // The only real caller -- the PreCompact hook -- knows what turn compaction fired on, not how + // many turns have passed since a prediction it did not make. Deriving the interval here keeps + // that subtraction next to the record that defines it, rather than making every caller + // re-discover which forecast is open in order to compute its own ground truth. + const elapsed = Number.isFinite(actualTurns) + ? actualTurns + : null; + if (elapsed === null && !Number.isFinite(atTurn)) return; + + // CLOSURE IS DERIVED FROM THE OUTCOME ROWS, not from a flag on the forecast. + // + // The old filter was `!e.scored`, but nothing ever writes `scored`: logForecast records + // kind/sessionId/predictedTurns/horizon/used/capacity, and record() is an append-only JSONL + // writer with no update path, so a forecast can never gain the field. The predicate was always + // true, and the last forecast for a session was therefore re-closable without limit. + // + // A session where compaction fires twice with no intervening logForecast popped the SAME + // forecast twice, producing two outcome rows for one prediction. reliability counts ROWS, not + // predictions, so the hit rate and the bias mean both inflate and MIN_SCORED can be satisfied + // by a single prediction re-closed eight times -- precisely the "threshold from a single + // sample" this module exists to prevent. The existing test used a fresh sessionId per + // iteration, so it never exposed it. + const events = readBalance(dir); + const closed = new Set( + events.filter((e) => e.kind === 'forecast-outcome' && e.forecastId).map((e) => e.forecastId), + ); + const open = events + .filter((e) => e.kind === 'forecast' && e.sessionId === sessionId && !closed.has(e.id)) .pop(); if (!open) return; + // Derived only once the open forecast is known, since the interval is measured from ITS turn. + // A forecast recorded before `turns` existed has none, and there is nothing to subtract from -- + // scoring it against an absolute turn number would manufacture an error rather than measure one, + // so it is left open instead. + const actual = elapsed !== null ? elapsed + : (Number.isFinite(open.turns) ? atTurn - open.turns : null); + if (actual === null || !Number.isFinite(actual)) return; + record(dir, { kind: 'forecast-outcome', sessionId, + // Stamped so the next call can see this forecast is spoken for. Without it the set above is + // empty and the same defect returns. + forecastId: open.id, horizon: open.horizon, predictedTurns: open.predictedTurns, - actualTurns, - error: actualTurns - open.predictedTurns, - hit: Math.abs(actualTurns - open.predictedTurns) <= TOLERANCE, + actualTurns: actual, + error: actual - open.predictedTurns, + hit: Math.abs(actual - open.predictedTurns) <= TOLERANCE, }); } @@ -98,7 +157,7 @@ export function observeOutcome(dir, { sessionId, actualTurns }) { * actionable, where "62% accurate" is not. */ export function reliability(dir) { - const outcomes = readMetrics(dir).filter((e) => e.kind === 'forecast-outcome'); + const outcomes = readBalance(dir).filter((e) => e.kind === 'forecast-outcome'); const buckets = {}; for (const horizon of HORIZONS) { @@ -164,7 +223,28 @@ export function calibrate(dir, predictedTurns) { // Refit: shift by the observed bias for this horizon. A bucket that runs long // gets pulled in, and vice versa. - const adjusted = Math.max(0, Math.round(predictedTurns + bucket.bias)); + // + // NOT CLAMPED WITH Math.max, which converted nonsense into a plausible-looking emergency. bias + // is a plain arithmetic mean of signed error, and a mean is not robust to the shape the + // 50%-hit-rate floor admits: a near bucket with errors [0,0,0,0,-30,-30,-30,-30] has four hits, + // so it clears the floor and is called calibrated, and its bias is -15. A raw forecast of 3 + // turns then became Math.max(0, Math.round(3 - 15)) = 0 and was published as fact, with the + // note "within 3 on 50% of 8 past forecasts" -- a zero-turn deadline the forecaster never + // produced, from a bucket that is wrong half the time. + // + // A correction at least as large as the thing it corrects is not a correction. + const adjusted = Math.round(predictedTurns + bucket.bias); + if (Math.abs(bucket.bias) >= predictedTurns || adjusted < 1) { + return { + publishable: false, + predictedTurns, + horizon, + bias: bucket.bias, + reason: `the observed bias at this horizon (${Math.round(bucket.bias)} turns) is as large as ` + + `the forecast itself (${predictedTurns}), so the correction would replace it rather than ` + + 'adjust it', + }; + } return { publishable: true, diff --git a/integrations/codex/plugin/hooks/lib/forecast.mjs b/integrations/codex/plugin/hooks/lib/forecast.mjs index ed698b1a0..39358e76a 100644 --- a/integrations/codex/plugin/hooks/lib/forecast.mjs +++ b/integrations/codex/plugin/hooks/lib/forecast.mjs @@ -34,6 +34,7 @@ import { readMetrics, readBalance, isFixtureAnchor, BALANCE_KINDS } from './metrics.mjs'; import { aggregateConsolidation } from './consolidate.mjs'; import { previewQuality } from './expand.mjs'; +import { logForecast, calibrate } from './calibration.mjs'; /** Turns of headroom below which the runway is worth interrupting for. */ export const ACTIONABLE_RUNWAY = 8; @@ -312,7 +313,33 @@ export function forecastPanel(dir, session = {}, findings = []) { if (air) { parts.runway = air; - lines.push(runwayLine(air)); + + // THE FORECAST KEEPS ITS OWN SCORE -- from here, because this is the only place a forecast is + // actually made. calibration.mjs was written for exactly this and had zero shipped importers: + // no forecast was ever logged, no outcome ever observed, reliability always saw an empty set, + // and calibrate was unreachable. So the shipped panel printed precisely the uncalibrated + // number that module's docstring calls "a vibe with a typeface". + // + // Logged BEFORE it is corrected. The raw prediction is what gets scored against the outcome; + // scoring the corrected one would fold the correction back into the next correction and the + // bias would chase its own tail. + logForecast(dir, { + sessionId: session.sessionId, + predictedTurns: air.withGraph, + used: session.used, + capacity: session.capacity, + // The turn this was predicted FROM. predictedTurns is an interval, so without its origin + // the outcome at compaction has nothing to subtract from. + turns: session.turns, + }); + + const score = calibrate(dir, air.withGraph); + parts.calibration = score; + // Published only when the horizon has earned it. An unpublishable score does not replace the + // number -- it is simply absent, exactly as the uncalibrated case reads today. + lines.push(score.publishable + ? `${runwayLine({ ...air, withGraph: score.predictedTurns })}\n (corrected: ${score.note})` + : runwayLine(air)); } if (rate) { diff --git a/integrations/codex/plugin/hooks/lib/metrics.mjs b/integrations/codex/plugin/hooks/lib/metrics.mjs index b762df57c..4b0eb2712 100644 --- a/integrations/codex/plugin/hooks/lib/metrics.mjs +++ b/integrations/codex/plugin/hooks/lib/metrics.mjs @@ -68,7 +68,17 @@ const balancePath = (dir) => join(dir, 'balance.jsonl'); * starve the measurement. Exported because any reader that needs the unwindowed truth needs to * know which kinds it applies to, and a second copy of this set elsewhere would drift. */ -export const BALANCE_KINDS = new Set(['inject', 'harvest', 'substitute']); +export const BALANCE_KINDS = new Set([ + 'inject', + 'harvest', + 'substitute', + // The forecast's own score. Both kinds are rare and accumulate slowly -- MIN_SCORED is 8 -- + // while the firehose they shared is dominated by per-tool-call records, so they were evicted + // faster than they could accumulate: calibrate() returned 'not yet calibrated (n/8)' forever + // and an outcome whose forecast had scrolled away was dropped with no record at all. + 'forecast', + 'forecast-outcome', +]); /** * Is this touch in the holdout arm? diff --git a/integrations/codex/plugin/hooks/lib/policy.mjs b/integrations/codex/plugin/hooks/lib/policy.mjs index b3871c27d..1fa14c348 100644 --- a/integrations/codex/plugin/hooks/lib/policy.mjs +++ b/integrations/codex/plugin/hooks/lib/policy.mjs @@ -272,7 +272,7 @@ function statePath(sessionId, agent) { /** A usable state object, whatever was on disk. */ function emptyState() { - return { seen: {}, denied: {}, injected: [], actCounts: {} }; + return { seen: {}, denied: {}, injected: [], actCounts: {}, forecast: null }; } /** @@ -312,6 +312,16 @@ export function loadState(sessionId, agent) { parsed.actCounts && typeof parsed.actCounts === 'object' && !Array.isArray(parsed.actCounts) ? parsed.actCounts : {}, + // THE SAME REASON A THIRD TIME. The forecast throttle records when the panel was last + // computed and what runway it last SHOWED, and both are meaningless within a single hook + // process -- every tool call is a new one. Dropped here, `checkedAt` would reset on every + // call and the panel would rebuild itself, transcript parse and all, on each tool use; and + // `shown` would reset, so worthSurfacing would compare against nothing and re-interrupt with + // the same runway forever. + forecast: + parsed.forecast && typeof parsed.forecast === 'object' && !Array.isArray(parsed.forecast) + ? parsed.forecast + : null, }; } catch { return emptyState(); @@ -383,6 +393,18 @@ export function saveState(sessionId, state, agent) { } return out; })(), + // LATEST CHECK WINS, which is the opposite direction from the fields above and correct for + // this one. `seen`, `denied`, `injected` and `actCounts` are all append-only, so a union or + // a max is the safe merge. The forecast throttle is a POINT IN TIME: taking the older of two + // concurrent checks would re-open the window and let the panel be rebuilt immediately, which + // is the cost the throttle exists to bound. + forecast: (() => { + const mine = state.forecast || null; + const theirs = current.forecast || null; + if (!mine) return theirs; + if (!theirs) return mine; + return (mine.checkedAt || 0) >= (theirs.checkedAt || 0) ? mine : theirs; + })(), }; // Write-then-rename so a reader never observes a half-written file. diff --git a/integrations/codex/plugin/hooks/lib/surface.mjs b/integrations/codex/plugin/hooks/lib/surface.mjs new file mode 100644 index 000000000..0ec05161c --- /dev/null +++ b/integrations/codex/plugin/hooks/lib/surface.mjs @@ -0,0 +1,166 @@ +// GENERATED FILE -- do not edit. +// Source of truth: hooks-core/surface.mjs. Regenerate with `npm run sync:hooks`. +/** + * The entry point the forecast never had. + * + * forecast.mjs computes a runway, calibration.mjs scores it, and worthSurfacing decides whether a + * change is worth interrupting for -- and none of it ran. The reachability guard listed + * forecastPanel, worthSurfacing, logForecast and observeOutcome as unwired, which is a polite way + * of saying the feature did not exist for any user. This module is the missing half: it derives + * the session numbers the panel needs from the transcript, decides when the panel has earned an + * interruption, and closes the calibration loop when compaction fires. + * + * THREE COSTS ARE BOUNDED HERE, because this runs on the PreToolUse path and every tool call is a + * separate process: + * + * THE READ readCacheUsage parses up to 4 MB of transcript. Doing that on every tool call + * would tax the hook that exists to save the user money. + * THE COMPUTE the panel walks the metrics log, the balance log and the graph. + * THE CONTEXT the panel is only worth its own tokens when it says something actionable. + * + * So the throttle is checked FIRST, from persisted state, before any file is opened. Everything + * expensive happens after a clock comparison. + * + * NOTHING HERE IS ALLOWED TO BREAK A TOOL CALL. Every entry point returns null on failure rather + * than throwing: a forecast is a courtesy, and the hook's promise is that a defect in it costs the + * user nothing. + */ + +import { forecastPanel, worthSurfacing } from './forecast.mjs'; +import { observeOutcome } from './calibration.mjs'; +import { readCacheUsage } from './cache.mjs'; +import { readBalance } from './metrics.mjs'; + +/** + * How long between two attempts to build the panel. + * + * Not a tuning knob so much as an admission: the panel costs a transcript parse, and a user makes + * many tool calls per minute. Two minutes is short enough that a runway falling into single digits + * is still caught with turns to spare, and long enough that the check is free in aggregate. + */ +export const SURFACE_INTERVAL_MS = + Number(process.env.TOKEN_OPTIMIZER_SURFACE_INTERVAL_MS) || 120_000; + +/** + * The context window to measure the runway against. + * + * NOT DISCOVERABLE from the transcript: the usage rows carry what was spent, never the ceiling it + * was spent against. Defaulting is therefore unavoidable, so the default is stated rather than + * buried, and overridable for anyone running a different window. + */ +export const DEFAULT_CAPACITY = + Number(process.env.TOKEN_OPTIMIZER_CONTEXT_CAPACITY) || 200_000; + +/** + * What the session has spent, read from its own transcript. + * + * `used` is the FULL input size of the most recent request -- cache reads plus cache writes plus + * fresh input -- which is what the context window actually holds, rather than the marginal cost of + * that one turn. Summing the per-turn inputs instead would count the same carried prefix once per + * turn and report a number many times the window size. + * + * Returns null rather than a zeroed object when there is nothing to measure, so a caller cannot + * mistake "no data" for "an empty context". + */ +export function sessionUsage(transcriptPath, { capacity = DEFAULT_CAPACITY } = {}) { + let turns; + try { + turns = readCacheUsage(transcriptPath); + } catch { + return null; + } + if (!turns?.length) return null; + + const last = turns[turns.length - 1]; + const used = (last.read || 0) + (last.written || 0) + (last.input || 0); + if (!used) return null; + + return { used, capacity, turns: turns.length }; +} + +/** Intercepted touches recorded for this session, which converts a per-touch saving to per-turn. */ +function touchesFor(dir, sessionId) { + try { + return readBalance(dir).filter((e) => e.kind === 'inject' && e.sessionId === sessionId).length; + } catch { + return 0; + } +} + +/** + * Should the forecast interrupt right now, and with what? + * + * The state object is the caller's persisted per-session state; this reads `forecast` from it and + * returns the value to store back, so the throttle and the last-shown runway survive across hook + * processes. Returning the next state rather than mutating it keeps this testable without a + * filesystem. + * + * @returns { text, state } -- text is null when nothing should be shown. + */ +export function maybeSurface(dir, { + transcriptPath, sessionId, state = {}, now = Date.now(), findings = [], +} = {}) { + const previous = state.forecast || null; + + // THE THROTTLE IS FIRST, and deliberately before any I/O. Everything below opens files. + if (previous?.checkedAt && now - previous.checkedAt < SURFACE_INTERVAL_MS) { + return { text: null, state: previous }; + } + + let usage; + try { + usage = sessionUsage(transcriptPath); + } catch { + return { text: null, state: previous }; + } + // The clock is stamped even when there is nothing to measure, so a session with no usage rows + // does not re-attempt the transcript parse on every single tool call. + if (!usage) return { text: null, state: { ...previous, checkedAt: now } }; + + let panel; + try { + panel = forecastPanel(dir, { + ...usage, + sessionId, + touches: touchesFor(dir, sessionId), + }, findings); + } catch { + return { text: null, state: { ...previous, checkedAt: now } }; + } + if (!panel) return { text: null, state: { ...previous, checkedAt: now } }; + + // worthSurfacing compares against the last panel that was actually SHOWN, not the last one + // computed. Comparing against the last computed panel would let the runway drift down past the + // threshold one throttle window at a time and never trip the "crossed it" test. + const shown = previous?.shown ? { parts: { runway: { withGraph: previous.shown } } } : null; + if (!worthSurfacing(panel, shown)) { + return { text: null, state: { ...previous, checkedAt: now } }; + } + + return { + text: panel.text, + state: { checkedAt: now, shown: panel.parts.runway.withGraph }, + }; +} + +/** + * Closes the calibration loop, at the only moment that can close it. + * + * Compaction firing IS the ground truth a runway forecast was predicting, so this belongs on the + * PreCompact path and nowhere else. `actualTurns` is turns ELAPSED between the prediction and + * compaction, which is what predictedTurns meant -- so the forecast has to have recorded the turn + * it was made on, and observeOutcome subtracts. + * + * Silent by design: nothing is shown to anybody. It writes one record so that the NEXT session's + * forecast can be published with a track record instead of as a bare assertion. + */ +export function closeForecast(dir, { transcriptPath, sessionId } = {}) { + try { + const usage = sessionUsage(transcriptPath); + if (!usage) return false; + observeOutcome(dir, { sessionId, atTurn: usage.turns }); + return true; + } catch { + return false; + } +} diff --git a/integrations/gemini/hooks/lib/calibration.mjs b/integrations/gemini/hooks/lib/calibration.mjs index 718db89cd..42de126f9 100644 --- a/integrations/gemini/hooks/lib/calibration.mjs +++ b/integrations/gemini/hooks/lib/calibration.mjs @@ -31,7 +31,20 @@ * that cannot support one. */ -import { record, readMetrics } from './metrics.mjs'; +// readBalance, not readMetrics. Both readers used the windowed firehose, which returns at most +// 2,000,000 bytes and the last 5,000 lines -- and forecast events were not among the protected +// kinds, so they lived in a log dominated by per-tool-call records (metrics.mjs's own comment +// cites 4,735 capture events inside one window). +// +// Two failures followed. In observeOutcome, on a busy project the forecast had scrolled past the +// tail by the time compaction fired, so the outcome was discarded with no record, no error and no +// counter. In reliability, outcomes vanished faster than they accumulated, so rows.length could +// never reach MIN_SCORED and calibrate returned 'not yet calibrated (n/8)' permanently -- leaving +// a user unable to distinguish "never had data" from "the data is being thrown away". +// +// 'forecast' and 'forecast-outcome' are now balance kinds, so they are written to balance.jsonl +// as well and read back unwindowed. +import { record, readBalance } from './metrics.mjs'; /** Horizon buckets, in turns. Error behaves differently across these ranges. */ const HORIZONS = [ @@ -53,8 +66,15 @@ function horizonOf(turns) { return HORIZONS.find((h) => turns <= h.max).name; } -/** Records a prediction so it can be scored when the outcome arrives. */ -export function logForecast(dir, { sessionId, predictedTurns, used, capacity }) { +/** + * Records a prediction so it can be scored when the outcome arrives. + * + * `turns` is the session's turn count AT PREDICTION TIME, and it is what makes the outcome + * computable at all: predictedTurns means "turns from here until compaction", so the ground truth + * is the turns that ELAPSED, not the turn number compaction happened on. Without it the caller at + * compaction has nothing to subtract from and would have to score an interval against an absolute. + */ +export function logForecast(dir, { sessionId, predictedTurns, used, capacity, turns }) { if (!Number.isFinite(predictedTurns)) return; record(dir, { kind: 'forecast', @@ -63,6 +83,7 @@ export function logForecast(dir, { sessionId, predictedTurns, used, capacity }) horizon: horizonOf(predictedTurns), used, capacity, + turns: Number.isFinite(turns) ? turns : null, }); } @@ -73,21 +94,59 @@ export function logForecast(dir, { sessionId, predictedTurns, used, capacity }) * Called when compaction fires: the number of turns that elapsed since the * prediction is the ground truth it was predicting. */ -export function observeOutcome(dir, { sessionId, actualTurns }) { - if (!Number.isFinite(actualTurns)) return; - const open = readMetrics(dir) - .filter((e) => e.kind === 'forecast' && e.sessionId === sessionId && !e.scored) +export function observeOutcome(dir, { sessionId, actualTurns, atTurn }) { + // EITHER THE ELAPSED COUNT OR THE TURN IT HAPPENED ON. + // + // The only real caller -- the PreCompact hook -- knows what turn compaction fired on, not how + // many turns have passed since a prediction it did not make. Deriving the interval here keeps + // that subtraction next to the record that defines it, rather than making every caller + // re-discover which forecast is open in order to compute its own ground truth. + const elapsed = Number.isFinite(actualTurns) + ? actualTurns + : null; + if (elapsed === null && !Number.isFinite(atTurn)) return; + + // CLOSURE IS DERIVED FROM THE OUTCOME ROWS, not from a flag on the forecast. + // + // The old filter was `!e.scored`, but nothing ever writes `scored`: logForecast records + // kind/sessionId/predictedTurns/horizon/used/capacity, and record() is an append-only JSONL + // writer with no update path, so a forecast can never gain the field. The predicate was always + // true, and the last forecast for a session was therefore re-closable without limit. + // + // A session where compaction fires twice with no intervening logForecast popped the SAME + // forecast twice, producing two outcome rows for one prediction. reliability counts ROWS, not + // predictions, so the hit rate and the bias mean both inflate and MIN_SCORED can be satisfied + // by a single prediction re-closed eight times -- precisely the "threshold from a single + // sample" this module exists to prevent. The existing test used a fresh sessionId per + // iteration, so it never exposed it. + const events = readBalance(dir); + const closed = new Set( + events.filter((e) => e.kind === 'forecast-outcome' && e.forecastId).map((e) => e.forecastId), + ); + const open = events + .filter((e) => e.kind === 'forecast' && e.sessionId === sessionId && !closed.has(e.id)) .pop(); if (!open) return; + // Derived only once the open forecast is known, since the interval is measured from ITS turn. + // A forecast recorded before `turns` existed has none, and there is nothing to subtract from -- + // scoring it against an absolute turn number would manufacture an error rather than measure one, + // so it is left open instead. + const actual = elapsed !== null ? elapsed + : (Number.isFinite(open.turns) ? atTurn - open.turns : null); + if (actual === null || !Number.isFinite(actual)) return; + record(dir, { kind: 'forecast-outcome', sessionId, + // Stamped so the next call can see this forecast is spoken for. Without it the set above is + // empty and the same defect returns. + forecastId: open.id, horizon: open.horizon, predictedTurns: open.predictedTurns, - actualTurns, - error: actualTurns - open.predictedTurns, - hit: Math.abs(actualTurns - open.predictedTurns) <= TOLERANCE, + actualTurns: actual, + error: actual - open.predictedTurns, + hit: Math.abs(actual - open.predictedTurns) <= TOLERANCE, }); } @@ -98,7 +157,7 @@ export function observeOutcome(dir, { sessionId, actualTurns }) { * actionable, where "62% accurate" is not. */ export function reliability(dir) { - const outcomes = readMetrics(dir).filter((e) => e.kind === 'forecast-outcome'); + const outcomes = readBalance(dir).filter((e) => e.kind === 'forecast-outcome'); const buckets = {}; for (const horizon of HORIZONS) { @@ -164,7 +223,28 @@ export function calibrate(dir, predictedTurns) { // Refit: shift by the observed bias for this horizon. A bucket that runs long // gets pulled in, and vice versa. - const adjusted = Math.max(0, Math.round(predictedTurns + bucket.bias)); + // + // NOT CLAMPED WITH Math.max, which converted nonsense into a plausible-looking emergency. bias + // is a plain arithmetic mean of signed error, and a mean is not robust to the shape the + // 50%-hit-rate floor admits: a near bucket with errors [0,0,0,0,-30,-30,-30,-30] has four hits, + // so it clears the floor and is called calibrated, and its bias is -15. A raw forecast of 3 + // turns then became Math.max(0, Math.round(3 - 15)) = 0 and was published as fact, with the + // note "within 3 on 50% of 8 past forecasts" -- a zero-turn deadline the forecaster never + // produced, from a bucket that is wrong half the time. + // + // A correction at least as large as the thing it corrects is not a correction. + const adjusted = Math.round(predictedTurns + bucket.bias); + if (Math.abs(bucket.bias) >= predictedTurns || adjusted < 1) { + return { + publishable: false, + predictedTurns, + horizon, + bias: bucket.bias, + reason: `the observed bias at this horizon (${Math.round(bucket.bias)} turns) is as large as ` + + `the forecast itself (${predictedTurns}), so the correction would replace it rather than ` + + 'adjust it', + }; + } return { publishable: true, diff --git a/integrations/gemini/hooks/lib/forecast.mjs b/integrations/gemini/hooks/lib/forecast.mjs index ed698b1a0..39358e76a 100644 --- a/integrations/gemini/hooks/lib/forecast.mjs +++ b/integrations/gemini/hooks/lib/forecast.mjs @@ -34,6 +34,7 @@ import { readMetrics, readBalance, isFixtureAnchor, BALANCE_KINDS } from './metrics.mjs'; import { aggregateConsolidation } from './consolidate.mjs'; import { previewQuality } from './expand.mjs'; +import { logForecast, calibrate } from './calibration.mjs'; /** Turns of headroom below which the runway is worth interrupting for. */ export const ACTIONABLE_RUNWAY = 8; @@ -312,7 +313,33 @@ export function forecastPanel(dir, session = {}, findings = []) { if (air) { parts.runway = air; - lines.push(runwayLine(air)); + + // THE FORECAST KEEPS ITS OWN SCORE -- from here, because this is the only place a forecast is + // actually made. calibration.mjs was written for exactly this and had zero shipped importers: + // no forecast was ever logged, no outcome ever observed, reliability always saw an empty set, + // and calibrate was unreachable. So the shipped panel printed precisely the uncalibrated + // number that module's docstring calls "a vibe with a typeface". + // + // Logged BEFORE it is corrected. The raw prediction is what gets scored against the outcome; + // scoring the corrected one would fold the correction back into the next correction and the + // bias would chase its own tail. + logForecast(dir, { + sessionId: session.sessionId, + predictedTurns: air.withGraph, + used: session.used, + capacity: session.capacity, + // The turn this was predicted FROM. predictedTurns is an interval, so without its origin + // the outcome at compaction has nothing to subtract from. + turns: session.turns, + }); + + const score = calibrate(dir, air.withGraph); + parts.calibration = score; + // Published only when the horizon has earned it. An unpublishable score does not replace the + // number -- it is simply absent, exactly as the uncalibrated case reads today. + lines.push(score.publishable + ? `${runwayLine({ ...air, withGraph: score.predictedTurns })}\n (corrected: ${score.note})` + : runwayLine(air)); } if (rate) { diff --git a/integrations/gemini/hooks/lib/metrics.mjs b/integrations/gemini/hooks/lib/metrics.mjs index b762df57c..4b0eb2712 100644 --- a/integrations/gemini/hooks/lib/metrics.mjs +++ b/integrations/gemini/hooks/lib/metrics.mjs @@ -68,7 +68,17 @@ const balancePath = (dir) => join(dir, 'balance.jsonl'); * starve the measurement. Exported because any reader that needs the unwindowed truth needs to * know which kinds it applies to, and a second copy of this set elsewhere would drift. */ -export const BALANCE_KINDS = new Set(['inject', 'harvest', 'substitute']); +export const BALANCE_KINDS = new Set([ + 'inject', + 'harvest', + 'substitute', + // The forecast's own score. Both kinds are rare and accumulate slowly -- MIN_SCORED is 8 -- + // while the firehose they shared is dominated by per-tool-call records, so they were evicted + // faster than they could accumulate: calibrate() returned 'not yet calibrated (n/8)' forever + // and an outcome whose forecast had scrolled away was dropped with no record at all. + 'forecast', + 'forecast-outcome', +]); /** * Is this touch in the holdout arm? diff --git a/integrations/gemini/hooks/lib/policy.mjs b/integrations/gemini/hooks/lib/policy.mjs index b3871c27d..1fa14c348 100644 --- a/integrations/gemini/hooks/lib/policy.mjs +++ b/integrations/gemini/hooks/lib/policy.mjs @@ -272,7 +272,7 @@ function statePath(sessionId, agent) { /** A usable state object, whatever was on disk. */ function emptyState() { - return { seen: {}, denied: {}, injected: [], actCounts: {} }; + return { seen: {}, denied: {}, injected: [], actCounts: {}, forecast: null }; } /** @@ -312,6 +312,16 @@ export function loadState(sessionId, agent) { parsed.actCounts && typeof parsed.actCounts === 'object' && !Array.isArray(parsed.actCounts) ? parsed.actCounts : {}, + // THE SAME REASON A THIRD TIME. The forecast throttle records when the panel was last + // computed and what runway it last SHOWED, and both are meaningless within a single hook + // process -- every tool call is a new one. Dropped here, `checkedAt` would reset on every + // call and the panel would rebuild itself, transcript parse and all, on each tool use; and + // `shown` would reset, so worthSurfacing would compare against nothing and re-interrupt with + // the same runway forever. + forecast: + parsed.forecast && typeof parsed.forecast === 'object' && !Array.isArray(parsed.forecast) + ? parsed.forecast + : null, }; } catch { return emptyState(); @@ -383,6 +393,18 @@ export function saveState(sessionId, state, agent) { } return out; })(), + // LATEST CHECK WINS, which is the opposite direction from the fields above and correct for + // this one. `seen`, `denied`, `injected` and `actCounts` are all append-only, so a union or + // a max is the safe merge. The forecast throttle is a POINT IN TIME: taking the older of two + // concurrent checks would re-open the window and let the panel be rebuilt immediately, which + // is the cost the throttle exists to bound. + forecast: (() => { + const mine = state.forecast || null; + const theirs = current.forecast || null; + if (!mine) return theirs; + if (!theirs) return mine; + return (mine.checkedAt || 0) >= (theirs.checkedAt || 0) ? mine : theirs; + })(), }; // Write-then-rename so a reader never observes a half-written file. diff --git a/integrations/gemini/hooks/lib/surface.mjs b/integrations/gemini/hooks/lib/surface.mjs new file mode 100644 index 000000000..0ec05161c --- /dev/null +++ b/integrations/gemini/hooks/lib/surface.mjs @@ -0,0 +1,166 @@ +// GENERATED FILE -- do not edit. +// Source of truth: hooks-core/surface.mjs. Regenerate with `npm run sync:hooks`. +/** + * The entry point the forecast never had. + * + * forecast.mjs computes a runway, calibration.mjs scores it, and worthSurfacing decides whether a + * change is worth interrupting for -- and none of it ran. The reachability guard listed + * forecastPanel, worthSurfacing, logForecast and observeOutcome as unwired, which is a polite way + * of saying the feature did not exist for any user. This module is the missing half: it derives + * the session numbers the panel needs from the transcript, decides when the panel has earned an + * interruption, and closes the calibration loop when compaction fires. + * + * THREE COSTS ARE BOUNDED HERE, because this runs on the PreToolUse path and every tool call is a + * separate process: + * + * THE READ readCacheUsage parses up to 4 MB of transcript. Doing that on every tool call + * would tax the hook that exists to save the user money. + * THE COMPUTE the panel walks the metrics log, the balance log and the graph. + * THE CONTEXT the panel is only worth its own tokens when it says something actionable. + * + * So the throttle is checked FIRST, from persisted state, before any file is opened. Everything + * expensive happens after a clock comparison. + * + * NOTHING HERE IS ALLOWED TO BREAK A TOOL CALL. Every entry point returns null on failure rather + * than throwing: a forecast is a courtesy, and the hook's promise is that a defect in it costs the + * user nothing. + */ + +import { forecastPanel, worthSurfacing } from './forecast.mjs'; +import { observeOutcome } from './calibration.mjs'; +import { readCacheUsage } from './cache.mjs'; +import { readBalance } from './metrics.mjs'; + +/** + * How long between two attempts to build the panel. + * + * Not a tuning knob so much as an admission: the panel costs a transcript parse, and a user makes + * many tool calls per minute. Two minutes is short enough that a runway falling into single digits + * is still caught with turns to spare, and long enough that the check is free in aggregate. + */ +export const SURFACE_INTERVAL_MS = + Number(process.env.TOKEN_OPTIMIZER_SURFACE_INTERVAL_MS) || 120_000; + +/** + * The context window to measure the runway against. + * + * NOT DISCOVERABLE from the transcript: the usage rows carry what was spent, never the ceiling it + * was spent against. Defaulting is therefore unavoidable, so the default is stated rather than + * buried, and overridable for anyone running a different window. + */ +export const DEFAULT_CAPACITY = + Number(process.env.TOKEN_OPTIMIZER_CONTEXT_CAPACITY) || 200_000; + +/** + * What the session has spent, read from its own transcript. + * + * `used` is the FULL input size of the most recent request -- cache reads plus cache writes plus + * fresh input -- which is what the context window actually holds, rather than the marginal cost of + * that one turn. Summing the per-turn inputs instead would count the same carried prefix once per + * turn and report a number many times the window size. + * + * Returns null rather than a zeroed object when there is nothing to measure, so a caller cannot + * mistake "no data" for "an empty context". + */ +export function sessionUsage(transcriptPath, { capacity = DEFAULT_CAPACITY } = {}) { + let turns; + try { + turns = readCacheUsage(transcriptPath); + } catch { + return null; + } + if (!turns?.length) return null; + + const last = turns[turns.length - 1]; + const used = (last.read || 0) + (last.written || 0) + (last.input || 0); + if (!used) return null; + + return { used, capacity, turns: turns.length }; +} + +/** Intercepted touches recorded for this session, which converts a per-touch saving to per-turn. */ +function touchesFor(dir, sessionId) { + try { + return readBalance(dir).filter((e) => e.kind === 'inject' && e.sessionId === sessionId).length; + } catch { + return 0; + } +} + +/** + * Should the forecast interrupt right now, and with what? + * + * The state object is the caller's persisted per-session state; this reads `forecast` from it and + * returns the value to store back, so the throttle and the last-shown runway survive across hook + * processes. Returning the next state rather than mutating it keeps this testable without a + * filesystem. + * + * @returns { text, state } -- text is null when nothing should be shown. + */ +export function maybeSurface(dir, { + transcriptPath, sessionId, state = {}, now = Date.now(), findings = [], +} = {}) { + const previous = state.forecast || null; + + // THE THROTTLE IS FIRST, and deliberately before any I/O. Everything below opens files. + if (previous?.checkedAt && now - previous.checkedAt < SURFACE_INTERVAL_MS) { + return { text: null, state: previous }; + } + + let usage; + try { + usage = sessionUsage(transcriptPath); + } catch { + return { text: null, state: previous }; + } + // The clock is stamped even when there is nothing to measure, so a session with no usage rows + // does not re-attempt the transcript parse on every single tool call. + if (!usage) return { text: null, state: { ...previous, checkedAt: now } }; + + let panel; + try { + panel = forecastPanel(dir, { + ...usage, + sessionId, + touches: touchesFor(dir, sessionId), + }, findings); + } catch { + return { text: null, state: { ...previous, checkedAt: now } }; + } + if (!panel) return { text: null, state: { ...previous, checkedAt: now } }; + + // worthSurfacing compares against the last panel that was actually SHOWN, not the last one + // computed. Comparing against the last computed panel would let the runway drift down past the + // threshold one throttle window at a time and never trip the "crossed it" test. + const shown = previous?.shown ? { parts: { runway: { withGraph: previous.shown } } } : null; + if (!worthSurfacing(panel, shown)) { + return { text: null, state: { ...previous, checkedAt: now } }; + } + + return { + text: panel.text, + state: { checkedAt: now, shown: panel.parts.runway.withGraph }, + }; +} + +/** + * Closes the calibration loop, at the only moment that can close it. + * + * Compaction firing IS the ground truth a runway forecast was predicting, so this belongs on the + * PreCompact path and nowhere else. `actualTurns` is turns ELAPSED between the prediction and + * compaction, which is what predictedTurns meant -- so the forecast has to have recorded the turn + * it was made on, and observeOutcome subtracts. + * + * Silent by design: nothing is shown to anybody. It writes one record so that the NEXT session's + * forecast can be published with a track record instead of as a bare assertion. + */ +export function closeForecast(dir, { transcriptPath, sessionId } = {}) { + try { + const usage = sessionUsage(transcriptPath); + if (!usage) return false; + observeOutcome(dir, { sessionId, atTurn: usage.turns }); + return true; + } catch { + return false; + } +} diff --git a/integrations/opencode/hooks/lib/calibration.mjs b/integrations/opencode/hooks/lib/calibration.mjs index 718db89cd..42de126f9 100644 --- a/integrations/opencode/hooks/lib/calibration.mjs +++ b/integrations/opencode/hooks/lib/calibration.mjs @@ -31,7 +31,20 @@ * that cannot support one. */ -import { record, readMetrics } from './metrics.mjs'; +// readBalance, not readMetrics. Both readers used the windowed firehose, which returns at most +// 2,000,000 bytes and the last 5,000 lines -- and forecast events were not among the protected +// kinds, so they lived in a log dominated by per-tool-call records (metrics.mjs's own comment +// cites 4,735 capture events inside one window). +// +// Two failures followed. In observeOutcome, on a busy project the forecast had scrolled past the +// tail by the time compaction fired, so the outcome was discarded with no record, no error and no +// counter. In reliability, outcomes vanished faster than they accumulated, so rows.length could +// never reach MIN_SCORED and calibrate returned 'not yet calibrated (n/8)' permanently -- leaving +// a user unable to distinguish "never had data" from "the data is being thrown away". +// +// 'forecast' and 'forecast-outcome' are now balance kinds, so they are written to balance.jsonl +// as well and read back unwindowed. +import { record, readBalance } from './metrics.mjs'; /** Horizon buckets, in turns. Error behaves differently across these ranges. */ const HORIZONS = [ @@ -53,8 +66,15 @@ function horizonOf(turns) { return HORIZONS.find((h) => turns <= h.max).name; } -/** Records a prediction so it can be scored when the outcome arrives. */ -export function logForecast(dir, { sessionId, predictedTurns, used, capacity }) { +/** + * Records a prediction so it can be scored when the outcome arrives. + * + * `turns` is the session's turn count AT PREDICTION TIME, and it is what makes the outcome + * computable at all: predictedTurns means "turns from here until compaction", so the ground truth + * is the turns that ELAPSED, not the turn number compaction happened on. Without it the caller at + * compaction has nothing to subtract from and would have to score an interval against an absolute. + */ +export function logForecast(dir, { sessionId, predictedTurns, used, capacity, turns }) { if (!Number.isFinite(predictedTurns)) return; record(dir, { kind: 'forecast', @@ -63,6 +83,7 @@ export function logForecast(dir, { sessionId, predictedTurns, used, capacity }) horizon: horizonOf(predictedTurns), used, capacity, + turns: Number.isFinite(turns) ? turns : null, }); } @@ -73,21 +94,59 @@ export function logForecast(dir, { sessionId, predictedTurns, used, capacity }) * Called when compaction fires: the number of turns that elapsed since the * prediction is the ground truth it was predicting. */ -export function observeOutcome(dir, { sessionId, actualTurns }) { - if (!Number.isFinite(actualTurns)) return; - const open = readMetrics(dir) - .filter((e) => e.kind === 'forecast' && e.sessionId === sessionId && !e.scored) +export function observeOutcome(dir, { sessionId, actualTurns, atTurn }) { + // EITHER THE ELAPSED COUNT OR THE TURN IT HAPPENED ON. + // + // The only real caller -- the PreCompact hook -- knows what turn compaction fired on, not how + // many turns have passed since a prediction it did not make. Deriving the interval here keeps + // that subtraction next to the record that defines it, rather than making every caller + // re-discover which forecast is open in order to compute its own ground truth. + const elapsed = Number.isFinite(actualTurns) + ? actualTurns + : null; + if (elapsed === null && !Number.isFinite(atTurn)) return; + + // CLOSURE IS DERIVED FROM THE OUTCOME ROWS, not from a flag on the forecast. + // + // The old filter was `!e.scored`, but nothing ever writes `scored`: logForecast records + // kind/sessionId/predictedTurns/horizon/used/capacity, and record() is an append-only JSONL + // writer with no update path, so a forecast can never gain the field. The predicate was always + // true, and the last forecast for a session was therefore re-closable without limit. + // + // A session where compaction fires twice with no intervening logForecast popped the SAME + // forecast twice, producing two outcome rows for one prediction. reliability counts ROWS, not + // predictions, so the hit rate and the bias mean both inflate and MIN_SCORED can be satisfied + // by a single prediction re-closed eight times -- precisely the "threshold from a single + // sample" this module exists to prevent. The existing test used a fresh sessionId per + // iteration, so it never exposed it. + const events = readBalance(dir); + const closed = new Set( + events.filter((e) => e.kind === 'forecast-outcome' && e.forecastId).map((e) => e.forecastId), + ); + const open = events + .filter((e) => e.kind === 'forecast' && e.sessionId === sessionId && !closed.has(e.id)) .pop(); if (!open) return; + // Derived only once the open forecast is known, since the interval is measured from ITS turn. + // A forecast recorded before `turns` existed has none, and there is nothing to subtract from -- + // scoring it against an absolute turn number would manufacture an error rather than measure one, + // so it is left open instead. + const actual = elapsed !== null ? elapsed + : (Number.isFinite(open.turns) ? atTurn - open.turns : null); + if (actual === null || !Number.isFinite(actual)) return; + record(dir, { kind: 'forecast-outcome', sessionId, + // Stamped so the next call can see this forecast is spoken for. Without it the set above is + // empty and the same defect returns. + forecastId: open.id, horizon: open.horizon, predictedTurns: open.predictedTurns, - actualTurns, - error: actualTurns - open.predictedTurns, - hit: Math.abs(actualTurns - open.predictedTurns) <= TOLERANCE, + actualTurns: actual, + error: actual - open.predictedTurns, + hit: Math.abs(actual - open.predictedTurns) <= TOLERANCE, }); } @@ -98,7 +157,7 @@ export function observeOutcome(dir, { sessionId, actualTurns }) { * actionable, where "62% accurate" is not. */ export function reliability(dir) { - const outcomes = readMetrics(dir).filter((e) => e.kind === 'forecast-outcome'); + const outcomes = readBalance(dir).filter((e) => e.kind === 'forecast-outcome'); const buckets = {}; for (const horizon of HORIZONS) { @@ -164,7 +223,28 @@ export function calibrate(dir, predictedTurns) { // Refit: shift by the observed bias for this horizon. A bucket that runs long // gets pulled in, and vice versa. - const adjusted = Math.max(0, Math.round(predictedTurns + bucket.bias)); + // + // NOT CLAMPED WITH Math.max, which converted nonsense into a plausible-looking emergency. bias + // is a plain arithmetic mean of signed error, and a mean is not robust to the shape the + // 50%-hit-rate floor admits: a near bucket with errors [0,0,0,0,-30,-30,-30,-30] has four hits, + // so it clears the floor and is called calibrated, and its bias is -15. A raw forecast of 3 + // turns then became Math.max(0, Math.round(3 - 15)) = 0 and was published as fact, with the + // note "within 3 on 50% of 8 past forecasts" -- a zero-turn deadline the forecaster never + // produced, from a bucket that is wrong half the time. + // + // A correction at least as large as the thing it corrects is not a correction. + const adjusted = Math.round(predictedTurns + bucket.bias); + if (Math.abs(bucket.bias) >= predictedTurns || adjusted < 1) { + return { + publishable: false, + predictedTurns, + horizon, + bias: bucket.bias, + reason: `the observed bias at this horizon (${Math.round(bucket.bias)} turns) is as large as ` + + `the forecast itself (${predictedTurns}), so the correction would replace it rather than ` + + 'adjust it', + }; + } return { publishable: true, diff --git a/integrations/opencode/hooks/lib/forecast.mjs b/integrations/opencode/hooks/lib/forecast.mjs index ed698b1a0..39358e76a 100644 --- a/integrations/opencode/hooks/lib/forecast.mjs +++ b/integrations/opencode/hooks/lib/forecast.mjs @@ -34,6 +34,7 @@ import { readMetrics, readBalance, isFixtureAnchor, BALANCE_KINDS } from './metrics.mjs'; import { aggregateConsolidation } from './consolidate.mjs'; import { previewQuality } from './expand.mjs'; +import { logForecast, calibrate } from './calibration.mjs'; /** Turns of headroom below which the runway is worth interrupting for. */ export const ACTIONABLE_RUNWAY = 8; @@ -312,7 +313,33 @@ export function forecastPanel(dir, session = {}, findings = []) { if (air) { parts.runway = air; - lines.push(runwayLine(air)); + + // THE FORECAST KEEPS ITS OWN SCORE -- from here, because this is the only place a forecast is + // actually made. calibration.mjs was written for exactly this and had zero shipped importers: + // no forecast was ever logged, no outcome ever observed, reliability always saw an empty set, + // and calibrate was unreachable. So the shipped panel printed precisely the uncalibrated + // number that module's docstring calls "a vibe with a typeface". + // + // Logged BEFORE it is corrected. The raw prediction is what gets scored against the outcome; + // scoring the corrected one would fold the correction back into the next correction and the + // bias would chase its own tail. + logForecast(dir, { + sessionId: session.sessionId, + predictedTurns: air.withGraph, + used: session.used, + capacity: session.capacity, + // The turn this was predicted FROM. predictedTurns is an interval, so without its origin + // the outcome at compaction has nothing to subtract from. + turns: session.turns, + }); + + const score = calibrate(dir, air.withGraph); + parts.calibration = score; + // Published only when the horizon has earned it. An unpublishable score does not replace the + // number -- it is simply absent, exactly as the uncalibrated case reads today. + lines.push(score.publishable + ? `${runwayLine({ ...air, withGraph: score.predictedTurns })}\n (corrected: ${score.note})` + : runwayLine(air)); } if (rate) { diff --git a/integrations/opencode/hooks/lib/metrics.mjs b/integrations/opencode/hooks/lib/metrics.mjs index b762df57c..4b0eb2712 100644 --- a/integrations/opencode/hooks/lib/metrics.mjs +++ b/integrations/opencode/hooks/lib/metrics.mjs @@ -68,7 +68,17 @@ const balancePath = (dir) => join(dir, 'balance.jsonl'); * starve the measurement. Exported because any reader that needs the unwindowed truth needs to * know which kinds it applies to, and a second copy of this set elsewhere would drift. */ -export const BALANCE_KINDS = new Set(['inject', 'harvest', 'substitute']); +export const BALANCE_KINDS = new Set([ + 'inject', + 'harvest', + 'substitute', + // The forecast's own score. Both kinds are rare and accumulate slowly -- MIN_SCORED is 8 -- + // while the firehose they shared is dominated by per-tool-call records, so they were evicted + // faster than they could accumulate: calibrate() returned 'not yet calibrated (n/8)' forever + // and an outcome whose forecast had scrolled away was dropped with no record at all. + 'forecast', + 'forecast-outcome', +]); /** * Is this touch in the holdout arm? diff --git a/integrations/opencode/hooks/lib/policy.mjs b/integrations/opencode/hooks/lib/policy.mjs index b3871c27d..1fa14c348 100644 --- a/integrations/opencode/hooks/lib/policy.mjs +++ b/integrations/opencode/hooks/lib/policy.mjs @@ -272,7 +272,7 @@ function statePath(sessionId, agent) { /** A usable state object, whatever was on disk. */ function emptyState() { - return { seen: {}, denied: {}, injected: [], actCounts: {} }; + return { seen: {}, denied: {}, injected: [], actCounts: {}, forecast: null }; } /** @@ -312,6 +312,16 @@ export function loadState(sessionId, agent) { parsed.actCounts && typeof parsed.actCounts === 'object' && !Array.isArray(parsed.actCounts) ? parsed.actCounts : {}, + // THE SAME REASON A THIRD TIME. The forecast throttle records when the panel was last + // computed and what runway it last SHOWED, and both are meaningless within a single hook + // process -- every tool call is a new one. Dropped here, `checkedAt` would reset on every + // call and the panel would rebuild itself, transcript parse and all, on each tool use; and + // `shown` would reset, so worthSurfacing would compare against nothing and re-interrupt with + // the same runway forever. + forecast: + parsed.forecast && typeof parsed.forecast === 'object' && !Array.isArray(parsed.forecast) + ? parsed.forecast + : null, }; } catch { return emptyState(); @@ -383,6 +393,18 @@ export function saveState(sessionId, state, agent) { } return out; })(), + // LATEST CHECK WINS, which is the opposite direction from the fields above and correct for + // this one. `seen`, `denied`, `injected` and `actCounts` are all append-only, so a union or + // a max is the safe merge. The forecast throttle is a POINT IN TIME: taking the older of two + // concurrent checks would re-open the window and let the panel be rebuilt immediately, which + // is the cost the throttle exists to bound. + forecast: (() => { + const mine = state.forecast || null; + const theirs = current.forecast || null; + if (!mine) return theirs; + if (!theirs) return mine; + return (mine.checkedAt || 0) >= (theirs.checkedAt || 0) ? mine : theirs; + })(), }; // Write-then-rename so a reader never observes a half-written file. diff --git a/integrations/opencode/hooks/lib/surface.mjs b/integrations/opencode/hooks/lib/surface.mjs new file mode 100644 index 000000000..0ec05161c --- /dev/null +++ b/integrations/opencode/hooks/lib/surface.mjs @@ -0,0 +1,166 @@ +// GENERATED FILE -- do not edit. +// Source of truth: hooks-core/surface.mjs. Regenerate with `npm run sync:hooks`. +/** + * The entry point the forecast never had. + * + * forecast.mjs computes a runway, calibration.mjs scores it, and worthSurfacing decides whether a + * change is worth interrupting for -- and none of it ran. The reachability guard listed + * forecastPanel, worthSurfacing, logForecast and observeOutcome as unwired, which is a polite way + * of saying the feature did not exist for any user. This module is the missing half: it derives + * the session numbers the panel needs from the transcript, decides when the panel has earned an + * interruption, and closes the calibration loop when compaction fires. + * + * THREE COSTS ARE BOUNDED HERE, because this runs on the PreToolUse path and every tool call is a + * separate process: + * + * THE READ readCacheUsage parses up to 4 MB of transcript. Doing that on every tool call + * would tax the hook that exists to save the user money. + * THE COMPUTE the panel walks the metrics log, the balance log and the graph. + * THE CONTEXT the panel is only worth its own tokens when it says something actionable. + * + * So the throttle is checked FIRST, from persisted state, before any file is opened. Everything + * expensive happens after a clock comparison. + * + * NOTHING HERE IS ALLOWED TO BREAK A TOOL CALL. Every entry point returns null on failure rather + * than throwing: a forecast is a courtesy, and the hook's promise is that a defect in it costs the + * user nothing. + */ + +import { forecastPanel, worthSurfacing } from './forecast.mjs'; +import { observeOutcome } from './calibration.mjs'; +import { readCacheUsage } from './cache.mjs'; +import { readBalance } from './metrics.mjs'; + +/** + * How long between two attempts to build the panel. + * + * Not a tuning knob so much as an admission: the panel costs a transcript parse, and a user makes + * many tool calls per minute. Two minutes is short enough that a runway falling into single digits + * is still caught with turns to spare, and long enough that the check is free in aggregate. + */ +export const SURFACE_INTERVAL_MS = + Number(process.env.TOKEN_OPTIMIZER_SURFACE_INTERVAL_MS) || 120_000; + +/** + * The context window to measure the runway against. + * + * NOT DISCOVERABLE from the transcript: the usage rows carry what was spent, never the ceiling it + * was spent against. Defaulting is therefore unavoidable, so the default is stated rather than + * buried, and overridable for anyone running a different window. + */ +export const DEFAULT_CAPACITY = + Number(process.env.TOKEN_OPTIMIZER_CONTEXT_CAPACITY) || 200_000; + +/** + * What the session has spent, read from its own transcript. + * + * `used` is the FULL input size of the most recent request -- cache reads plus cache writes plus + * fresh input -- which is what the context window actually holds, rather than the marginal cost of + * that one turn. Summing the per-turn inputs instead would count the same carried prefix once per + * turn and report a number many times the window size. + * + * Returns null rather than a zeroed object when there is nothing to measure, so a caller cannot + * mistake "no data" for "an empty context". + */ +export function sessionUsage(transcriptPath, { capacity = DEFAULT_CAPACITY } = {}) { + let turns; + try { + turns = readCacheUsage(transcriptPath); + } catch { + return null; + } + if (!turns?.length) return null; + + const last = turns[turns.length - 1]; + const used = (last.read || 0) + (last.written || 0) + (last.input || 0); + if (!used) return null; + + return { used, capacity, turns: turns.length }; +} + +/** Intercepted touches recorded for this session, which converts a per-touch saving to per-turn. */ +function touchesFor(dir, sessionId) { + try { + return readBalance(dir).filter((e) => e.kind === 'inject' && e.sessionId === sessionId).length; + } catch { + return 0; + } +} + +/** + * Should the forecast interrupt right now, and with what? + * + * The state object is the caller's persisted per-session state; this reads `forecast` from it and + * returns the value to store back, so the throttle and the last-shown runway survive across hook + * processes. Returning the next state rather than mutating it keeps this testable without a + * filesystem. + * + * @returns { text, state } -- text is null when nothing should be shown. + */ +export function maybeSurface(dir, { + transcriptPath, sessionId, state = {}, now = Date.now(), findings = [], +} = {}) { + const previous = state.forecast || null; + + // THE THROTTLE IS FIRST, and deliberately before any I/O. Everything below opens files. + if (previous?.checkedAt && now - previous.checkedAt < SURFACE_INTERVAL_MS) { + return { text: null, state: previous }; + } + + let usage; + try { + usage = sessionUsage(transcriptPath); + } catch { + return { text: null, state: previous }; + } + // The clock is stamped even when there is nothing to measure, so a session with no usage rows + // does not re-attempt the transcript parse on every single tool call. + if (!usage) return { text: null, state: { ...previous, checkedAt: now } }; + + let panel; + try { + panel = forecastPanel(dir, { + ...usage, + sessionId, + touches: touchesFor(dir, sessionId), + }, findings); + } catch { + return { text: null, state: { ...previous, checkedAt: now } }; + } + if (!panel) return { text: null, state: { ...previous, checkedAt: now } }; + + // worthSurfacing compares against the last panel that was actually SHOWN, not the last one + // computed. Comparing against the last computed panel would let the runway drift down past the + // threshold one throttle window at a time and never trip the "crossed it" test. + const shown = previous?.shown ? { parts: { runway: { withGraph: previous.shown } } } : null; + if (!worthSurfacing(panel, shown)) { + return { text: null, state: { ...previous, checkedAt: now } }; + } + + return { + text: panel.text, + state: { checkedAt: now, shown: panel.parts.runway.withGraph }, + }; +} + +/** + * Closes the calibration loop, at the only moment that can close it. + * + * Compaction firing IS the ground truth a runway forecast was predicting, so this belongs on the + * PreCompact path and nowhere else. `actualTurns` is turns ELAPSED between the prediction and + * compaction, which is what predictedTurns meant -- so the forecast has to have recorded the turn + * it was made on, and observeOutcome subtracts. + * + * Silent by design: nothing is shown to anybody. It writes one record so that the NEXT session's + * forecast can be published with a track record instead of as a bare assertion. + */ +export function closeForecast(dir, { transcriptPath, sessionId } = {}) { + try { + const usage = sessionUsage(transcriptPath); + if (!usage) return false; + observeOutcome(dir, { sessionId, atTurn: usage.turns }); + return true; + } catch { + return false; + } +} diff --git a/integrations/qwen/hooks/lib/calibration.mjs b/integrations/qwen/hooks/lib/calibration.mjs index 718db89cd..42de126f9 100644 --- a/integrations/qwen/hooks/lib/calibration.mjs +++ b/integrations/qwen/hooks/lib/calibration.mjs @@ -31,7 +31,20 @@ * that cannot support one. */ -import { record, readMetrics } from './metrics.mjs'; +// readBalance, not readMetrics. Both readers used the windowed firehose, which returns at most +// 2,000,000 bytes and the last 5,000 lines -- and forecast events were not among the protected +// kinds, so they lived in a log dominated by per-tool-call records (metrics.mjs's own comment +// cites 4,735 capture events inside one window). +// +// Two failures followed. In observeOutcome, on a busy project the forecast had scrolled past the +// tail by the time compaction fired, so the outcome was discarded with no record, no error and no +// counter. In reliability, outcomes vanished faster than they accumulated, so rows.length could +// never reach MIN_SCORED and calibrate returned 'not yet calibrated (n/8)' permanently -- leaving +// a user unable to distinguish "never had data" from "the data is being thrown away". +// +// 'forecast' and 'forecast-outcome' are now balance kinds, so they are written to balance.jsonl +// as well and read back unwindowed. +import { record, readBalance } from './metrics.mjs'; /** Horizon buckets, in turns. Error behaves differently across these ranges. */ const HORIZONS = [ @@ -53,8 +66,15 @@ function horizonOf(turns) { return HORIZONS.find((h) => turns <= h.max).name; } -/** Records a prediction so it can be scored when the outcome arrives. */ -export function logForecast(dir, { sessionId, predictedTurns, used, capacity }) { +/** + * Records a prediction so it can be scored when the outcome arrives. + * + * `turns` is the session's turn count AT PREDICTION TIME, and it is what makes the outcome + * computable at all: predictedTurns means "turns from here until compaction", so the ground truth + * is the turns that ELAPSED, not the turn number compaction happened on. Without it the caller at + * compaction has nothing to subtract from and would have to score an interval against an absolute. + */ +export function logForecast(dir, { sessionId, predictedTurns, used, capacity, turns }) { if (!Number.isFinite(predictedTurns)) return; record(dir, { kind: 'forecast', @@ -63,6 +83,7 @@ export function logForecast(dir, { sessionId, predictedTurns, used, capacity }) horizon: horizonOf(predictedTurns), used, capacity, + turns: Number.isFinite(turns) ? turns : null, }); } @@ -73,21 +94,59 @@ export function logForecast(dir, { sessionId, predictedTurns, used, capacity }) * Called when compaction fires: the number of turns that elapsed since the * prediction is the ground truth it was predicting. */ -export function observeOutcome(dir, { sessionId, actualTurns }) { - if (!Number.isFinite(actualTurns)) return; - const open = readMetrics(dir) - .filter((e) => e.kind === 'forecast' && e.sessionId === sessionId && !e.scored) +export function observeOutcome(dir, { sessionId, actualTurns, atTurn }) { + // EITHER THE ELAPSED COUNT OR THE TURN IT HAPPENED ON. + // + // The only real caller -- the PreCompact hook -- knows what turn compaction fired on, not how + // many turns have passed since a prediction it did not make. Deriving the interval here keeps + // that subtraction next to the record that defines it, rather than making every caller + // re-discover which forecast is open in order to compute its own ground truth. + const elapsed = Number.isFinite(actualTurns) + ? actualTurns + : null; + if (elapsed === null && !Number.isFinite(atTurn)) return; + + // CLOSURE IS DERIVED FROM THE OUTCOME ROWS, not from a flag on the forecast. + // + // The old filter was `!e.scored`, but nothing ever writes `scored`: logForecast records + // kind/sessionId/predictedTurns/horizon/used/capacity, and record() is an append-only JSONL + // writer with no update path, so a forecast can never gain the field. The predicate was always + // true, and the last forecast for a session was therefore re-closable without limit. + // + // A session where compaction fires twice with no intervening logForecast popped the SAME + // forecast twice, producing two outcome rows for one prediction. reliability counts ROWS, not + // predictions, so the hit rate and the bias mean both inflate and MIN_SCORED can be satisfied + // by a single prediction re-closed eight times -- precisely the "threshold from a single + // sample" this module exists to prevent. The existing test used a fresh sessionId per + // iteration, so it never exposed it. + const events = readBalance(dir); + const closed = new Set( + events.filter((e) => e.kind === 'forecast-outcome' && e.forecastId).map((e) => e.forecastId), + ); + const open = events + .filter((e) => e.kind === 'forecast' && e.sessionId === sessionId && !closed.has(e.id)) .pop(); if (!open) return; + // Derived only once the open forecast is known, since the interval is measured from ITS turn. + // A forecast recorded before `turns` existed has none, and there is nothing to subtract from -- + // scoring it against an absolute turn number would manufacture an error rather than measure one, + // so it is left open instead. + const actual = elapsed !== null ? elapsed + : (Number.isFinite(open.turns) ? atTurn - open.turns : null); + if (actual === null || !Number.isFinite(actual)) return; + record(dir, { kind: 'forecast-outcome', sessionId, + // Stamped so the next call can see this forecast is spoken for. Without it the set above is + // empty and the same defect returns. + forecastId: open.id, horizon: open.horizon, predictedTurns: open.predictedTurns, - actualTurns, - error: actualTurns - open.predictedTurns, - hit: Math.abs(actualTurns - open.predictedTurns) <= TOLERANCE, + actualTurns: actual, + error: actual - open.predictedTurns, + hit: Math.abs(actual - open.predictedTurns) <= TOLERANCE, }); } @@ -98,7 +157,7 @@ export function observeOutcome(dir, { sessionId, actualTurns }) { * actionable, where "62% accurate" is not. */ export function reliability(dir) { - const outcomes = readMetrics(dir).filter((e) => e.kind === 'forecast-outcome'); + const outcomes = readBalance(dir).filter((e) => e.kind === 'forecast-outcome'); const buckets = {}; for (const horizon of HORIZONS) { @@ -164,7 +223,28 @@ export function calibrate(dir, predictedTurns) { // Refit: shift by the observed bias for this horizon. A bucket that runs long // gets pulled in, and vice versa. - const adjusted = Math.max(0, Math.round(predictedTurns + bucket.bias)); + // + // NOT CLAMPED WITH Math.max, which converted nonsense into a plausible-looking emergency. bias + // is a plain arithmetic mean of signed error, and a mean is not robust to the shape the + // 50%-hit-rate floor admits: a near bucket with errors [0,0,0,0,-30,-30,-30,-30] has four hits, + // so it clears the floor and is called calibrated, and its bias is -15. A raw forecast of 3 + // turns then became Math.max(0, Math.round(3 - 15)) = 0 and was published as fact, with the + // note "within 3 on 50% of 8 past forecasts" -- a zero-turn deadline the forecaster never + // produced, from a bucket that is wrong half the time. + // + // A correction at least as large as the thing it corrects is not a correction. + const adjusted = Math.round(predictedTurns + bucket.bias); + if (Math.abs(bucket.bias) >= predictedTurns || adjusted < 1) { + return { + publishable: false, + predictedTurns, + horizon, + bias: bucket.bias, + reason: `the observed bias at this horizon (${Math.round(bucket.bias)} turns) is as large as ` + + `the forecast itself (${predictedTurns}), so the correction would replace it rather than ` + + 'adjust it', + }; + } return { publishable: true, diff --git a/integrations/qwen/hooks/lib/forecast.mjs b/integrations/qwen/hooks/lib/forecast.mjs index ed698b1a0..39358e76a 100644 --- a/integrations/qwen/hooks/lib/forecast.mjs +++ b/integrations/qwen/hooks/lib/forecast.mjs @@ -34,6 +34,7 @@ import { readMetrics, readBalance, isFixtureAnchor, BALANCE_KINDS } from './metrics.mjs'; import { aggregateConsolidation } from './consolidate.mjs'; import { previewQuality } from './expand.mjs'; +import { logForecast, calibrate } from './calibration.mjs'; /** Turns of headroom below which the runway is worth interrupting for. */ export const ACTIONABLE_RUNWAY = 8; @@ -312,7 +313,33 @@ export function forecastPanel(dir, session = {}, findings = []) { if (air) { parts.runway = air; - lines.push(runwayLine(air)); + + // THE FORECAST KEEPS ITS OWN SCORE -- from here, because this is the only place a forecast is + // actually made. calibration.mjs was written for exactly this and had zero shipped importers: + // no forecast was ever logged, no outcome ever observed, reliability always saw an empty set, + // and calibrate was unreachable. So the shipped panel printed precisely the uncalibrated + // number that module's docstring calls "a vibe with a typeface". + // + // Logged BEFORE it is corrected. The raw prediction is what gets scored against the outcome; + // scoring the corrected one would fold the correction back into the next correction and the + // bias would chase its own tail. + logForecast(dir, { + sessionId: session.sessionId, + predictedTurns: air.withGraph, + used: session.used, + capacity: session.capacity, + // The turn this was predicted FROM. predictedTurns is an interval, so without its origin + // the outcome at compaction has nothing to subtract from. + turns: session.turns, + }); + + const score = calibrate(dir, air.withGraph); + parts.calibration = score; + // Published only when the horizon has earned it. An unpublishable score does not replace the + // number -- it is simply absent, exactly as the uncalibrated case reads today. + lines.push(score.publishable + ? `${runwayLine({ ...air, withGraph: score.predictedTurns })}\n (corrected: ${score.note})` + : runwayLine(air)); } if (rate) { diff --git a/integrations/qwen/hooks/lib/metrics.mjs b/integrations/qwen/hooks/lib/metrics.mjs index b762df57c..4b0eb2712 100644 --- a/integrations/qwen/hooks/lib/metrics.mjs +++ b/integrations/qwen/hooks/lib/metrics.mjs @@ -68,7 +68,17 @@ const balancePath = (dir) => join(dir, 'balance.jsonl'); * starve the measurement. Exported because any reader that needs the unwindowed truth needs to * know which kinds it applies to, and a second copy of this set elsewhere would drift. */ -export const BALANCE_KINDS = new Set(['inject', 'harvest', 'substitute']); +export const BALANCE_KINDS = new Set([ + 'inject', + 'harvest', + 'substitute', + // The forecast's own score. Both kinds are rare and accumulate slowly -- MIN_SCORED is 8 -- + // while the firehose they shared is dominated by per-tool-call records, so they were evicted + // faster than they could accumulate: calibrate() returned 'not yet calibrated (n/8)' forever + // and an outcome whose forecast had scrolled away was dropped with no record at all. + 'forecast', + 'forecast-outcome', +]); /** * Is this touch in the holdout arm? diff --git a/integrations/qwen/hooks/lib/policy.mjs b/integrations/qwen/hooks/lib/policy.mjs index b3871c27d..1fa14c348 100644 --- a/integrations/qwen/hooks/lib/policy.mjs +++ b/integrations/qwen/hooks/lib/policy.mjs @@ -272,7 +272,7 @@ function statePath(sessionId, agent) { /** A usable state object, whatever was on disk. */ function emptyState() { - return { seen: {}, denied: {}, injected: [], actCounts: {} }; + return { seen: {}, denied: {}, injected: [], actCounts: {}, forecast: null }; } /** @@ -312,6 +312,16 @@ export function loadState(sessionId, agent) { parsed.actCounts && typeof parsed.actCounts === 'object' && !Array.isArray(parsed.actCounts) ? parsed.actCounts : {}, + // THE SAME REASON A THIRD TIME. The forecast throttle records when the panel was last + // computed and what runway it last SHOWED, and both are meaningless within a single hook + // process -- every tool call is a new one. Dropped here, `checkedAt` would reset on every + // call and the panel would rebuild itself, transcript parse and all, on each tool use; and + // `shown` would reset, so worthSurfacing would compare against nothing and re-interrupt with + // the same runway forever. + forecast: + parsed.forecast && typeof parsed.forecast === 'object' && !Array.isArray(parsed.forecast) + ? parsed.forecast + : null, }; } catch { return emptyState(); @@ -383,6 +393,18 @@ export function saveState(sessionId, state, agent) { } return out; })(), + // LATEST CHECK WINS, which is the opposite direction from the fields above and correct for + // this one. `seen`, `denied`, `injected` and `actCounts` are all append-only, so a union or + // a max is the safe merge. The forecast throttle is a POINT IN TIME: taking the older of two + // concurrent checks would re-open the window and let the panel be rebuilt immediately, which + // is the cost the throttle exists to bound. + forecast: (() => { + const mine = state.forecast || null; + const theirs = current.forecast || null; + if (!mine) return theirs; + if (!theirs) return mine; + return (mine.checkedAt || 0) >= (theirs.checkedAt || 0) ? mine : theirs; + })(), }; // Write-then-rename so a reader never observes a half-written file. diff --git a/integrations/qwen/hooks/lib/surface.mjs b/integrations/qwen/hooks/lib/surface.mjs new file mode 100644 index 000000000..0ec05161c --- /dev/null +++ b/integrations/qwen/hooks/lib/surface.mjs @@ -0,0 +1,166 @@ +// GENERATED FILE -- do not edit. +// Source of truth: hooks-core/surface.mjs. Regenerate with `npm run sync:hooks`. +/** + * The entry point the forecast never had. + * + * forecast.mjs computes a runway, calibration.mjs scores it, and worthSurfacing decides whether a + * change is worth interrupting for -- and none of it ran. The reachability guard listed + * forecastPanel, worthSurfacing, logForecast and observeOutcome as unwired, which is a polite way + * of saying the feature did not exist for any user. This module is the missing half: it derives + * the session numbers the panel needs from the transcript, decides when the panel has earned an + * interruption, and closes the calibration loop when compaction fires. + * + * THREE COSTS ARE BOUNDED HERE, because this runs on the PreToolUse path and every tool call is a + * separate process: + * + * THE READ readCacheUsage parses up to 4 MB of transcript. Doing that on every tool call + * would tax the hook that exists to save the user money. + * THE COMPUTE the panel walks the metrics log, the balance log and the graph. + * THE CONTEXT the panel is only worth its own tokens when it says something actionable. + * + * So the throttle is checked FIRST, from persisted state, before any file is opened. Everything + * expensive happens after a clock comparison. + * + * NOTHING HERE IS ALLOWED TO BREAK A TOOL CALL. Every entry point returns null on failure rather + * than throwing: a forecast is a courtesy, and the hook's promise is that a defect in it costs the + * user nothing. + */ + +import { forecastPanel, worthSurfacing } from './forecast.mjs'; +import { observeOutcome } from './calibration.mjs'; +import { readCacheUsage } from './cache.mjs'; +import { readBalance } from './metrics.mjs'; + +/** + * How long between two attempts to build the panel. + * + * Not a tuning knob so much as an admission: the panel costs a transcript parse, and a user makes + * many tool calls per minute. Two minutes is short enough that a runway falling into single digits + * is still caught with turns to spare, and long enough that the check is free in aggregate. + */ +export const SURFACE_INTERVAL_MS = + Number(process.env.TOKEN_OPTIMIZER_SURFACE_INTERVAL_MS) || 120_000; + +/** + * The context window to measure the runway against. + * + * NOT DISCOVERABLE from the transcript: the usage rows carry what was spent, never the ceiling it + * was spent against. Defaulting is therefore unavoidable, so the default is stated rather than + * buried, and overridable for anyone running a different window. + */ +export const DEFAULT_CAPACITY = + Number(process.env.TOKEN_OPTIMIZER_CONTEXT_CAPACITY) || 200_000; + +/** + * What the session has spent, read from its own transcript. + * + * `used` is the FULL input size of the most recent request -- cache reads plus cache writes plus + * fresh input -- which is what the context window actually holds, rather than the marginal cost of + * that one turn. Summing the per-turn inputs instead would count the same carried prefix once per + * turn and report a number many times the window size. + * + * Returns null rather than a zeroed object when there is nothing to measure, so a caller cannot + * mistake "no data" for "an empty context". + */ +export function sessionUsage(transcriptPath, { capacity = DEFAULT_CAPACITY } = {}) { + let turns; + try { + turns = readCacheUsage(transcriptPath); + } catch { + return null; + } + if (!turns?.length) return null; + + const last = turns[turns.length - 1]; + const used = (last.read || 0) + (last.written || 0) + (last.input || 0); + if (!used) return null; + + return { used, capacity, turns: turns.length }; +} + +/** Intercepted touches recorded for this session, which converts a per-touch saving to per-turn. */ +function touchesFor(dir, sessionId) { + try { + return readBalance(dir).filter((e) => e.kind === 'inject' && e.sessionId === sessionId).length; + } catch { + return 0; + } +} + +/** + * Should the forecast interrupt right now, and with what? + * + * The state object is the caller's persisted per-session state; this reads `forecast` from it and + * returns the value to store back, so the throttle and the last-shown runway survive across hook + * processes. Returning the next state rather than mutating it keeps this testable without a + * filesystem. + * + * @returns { text, state } -- text is null when nothing should be shown. + */ +export function maybeSurface(dir, { + transcriptPath, sessionId, state = {}, now = Date.now(), findings = [], +} = {}) { + const previous = state.forecast || null; + + // THE THROTTLE IS FIRST, and deliberately before any I/O. Everything below opens files. + if (previous?.checkedAt && now - previous.checkedAt < SURFACE_INTERVAL_MS) { + return { text: null, state: previous }; + } + + let usage; + try { + usage = sessionUsage(transcriptPath); + } catch { + return { text: null, state: previous }; + } + // The clock is stamped even when there is nothing to measure, so a session with no usage rows + // does not re-attempt the transcript parse on every single tool call. + if (!usage) return { text: null, state: { ...previous, checkedAt: now } }; + + let panel; + try { + panel = forecastPanel(dir, { + ...usage, + sessionId, + touches: touchesFor(dir, sessionId), + }, findings); + } catch { + return { text: null, state: { ...previous, checkedAt: now } }; + } + if (!panel) return { text: null, state: { ...previous, checkedAt: now } }; + + // worthSurfacing compares against the last panel that was actually SHOWN, not the last one + // computed. Comparing against the last computed panel would let the runway drift down past the + // threshold one throttle window at a time and never trip the "crossed it" test. + const shown = previous?.shown ? { parts: { runway: { withGraph: previous.shown } } } : null; + if (!worthSurfacing(panel, shown)) { + return { text: null, state: { ...previous, checkedAt: now } }; + } + + return { + text: panel.text, + state: { checkedAt: now, shown: panel.parts.runway.withGraph }, + }; +} + +/** + * Closes the calibration loop, at the only moment that can close it. + * + * Compaction firing IS the ground truth a runway forecast was predicting, so this belongs on the + * PreCompact path and nowhere else. `actualTurns` is turns ELAPSED between the prediction and + * compaction, which is what predictedTurns meant -- so the forecast has to have recorded the turn + * it was made on, and observeOutcome subtracts. + * + * Silent by design: nothing is shown to anybody. It writes one record so that the NEXT session's + * forecast can be published with a track record instead of as a bare assertion. + */ +export function closeForecast(dir, { transcriptPath, sessionId } = {}) { + try { + const usage = sessionUsage(transcriptPath); + if (!usage) return false; + observeOutcome(dir, { sessionId, atTurn: usage.turns }); + return true; + } catch { + return false; + } +} diff --git a/plugin/hooks/lib/calibration.mjs b/plugin/hooks/lib/calibration.mjs index 718db89cd..42de126f9 100644 --- a/plugin/hooks/lib/calibration.mjs +++ b/plugin/hooks/lib/calibration.mjs @@ -31,7 +31,20 @@ * that cannot support one. */ -import { record, readMetrics } from './metrics.mjs'; +// readBalance, not readMetrics. Both readers used the windowed firehose, which returns at most +// 2,000,000 bytes and the last 5,000 lines -- and forecast events were not among the protected +// kinds, so they lived in a log dominated by per-tool-call records (metrics.mjs's own comment +// cites 4,735 capture events inside one window). +// +// Two failures followed. In observeOutcome, on a busy project the forecast had scrolled past the +// tail by the time compaction fired, so the outcome was discarded with no record, no error and no +// counter. In reliability, outcomes vanished faster than they accumulated, so rows.length could +// never reach MIN_SCORED and calibrate returned 'not yet calibrated (n/8)' permanently -- leaving +// a user unable to distinguish "never had data" from "the data is being thrown away". +// +// 'forecast' and 'forecast-outcome' are now balance kinds, so they are written to balance.jsonl +// as well and read back unwindowed. +import { record, readBalance } from './metrics.mjs'; /** Horizon buckets, in turns. Error behaves differently across these ranges. */ const HORIZONS = [ @@ -53,8 +66,15 @@ function horizonOf(turns) { return HORIZONS.find((h) => turns <= h.max).name; } -/** Records a prediction so it can be scored when the outcome arrives. */ -export function logForecast(dir, { sessionId, predictedTurns, used, capacity }) { +/** + * Records a prediction so it can be scored when the outcome arrives. + * + * `turns` is the session's turn count AT PREDICTION TIME, and it is what makes the outcome + * computable at all: predictedTurns means "turns from here until compaction", so the ground truth + * is the turns that ELAPSED, not the turn number compaction happened on. Without it the caller at + * compaction has nothing to subtract from and would have to score an interval against an absolute. + */ +export function logForecast(dir, { sessionId, predictedTurns, used, capacity, turns }) { if (!Number.isFinite(predictedTurns)) return; record(dir, { kind: 'forecast', @@ -63,6 +83,7 @@ export function logForecast(dir, { sessionId, predictedTurns, used, capacity }) horizon: horizonOf(predictedTurns), used, capacity, + turns: Number.isFinite(turns) ? turns : null, }); } @@ -73,21 +94,59 @@ export function logForecast(dir, { sessionId, predictedTurns, used, capacity }) * Called when compaction fires: the number of turns that elapsed since the * prediction is the ground truth it was predicting. */ -export function observeOutcome(dir, { sessionId, actualTurns }) { - if (!Number.isFinite(actualTurns)) return; - const open = readMetrics(dir) - .filter((e) => e.kind === 'forecast' && e.sessionId === sessionId && !e.scored) +export function observeOutcome(dir, { sessionId, actualTurns, atTurn }) { + // EITHER THE ELAPSED COUNT OR THE TURN IT HAPPENED ON. + // + // The only real caller -- the PreCompact hook -- knows what turn compaction fired on, not how + // many turns have passed since a prediction it did not make. Deriving the interval here keeps + // that subtraction next to the record that defines it, rather than making every caller + // re-discover which forecast is open in order to compute its own ground truth. + const elapsed = Number.isFinite(actualTurns) + ? actualTurns + : null; + if (elapsed === null && !Number.isFinite(atTurn)) return; + + // CLOSURE IS DERIVED FROM THE OUTCOME ROWS, not from a flag on the forecast. + // + // The old filter was `!e.scored`, but nothing ever writes `scored`: logForecast records + // kind/sessionId/predictedTurns/horizon/used/capacity, and record() is an append-only JSONL + // writer with no update path, so a forecast can never gain the field. The predicate was always + // true, and the last forecast for a session was therefore re-closable without limit. + // + // A session where compaction fires twice with no intervening logForecast popped the SAME + // forecast twice, producing two outcome rows for one prediction. reliability counts ROWS, not + // predictions, so the hit rate and the bias mean both inflate and MIN_SCORED can be satisfied + // by a single prediction re-closed eight times -- precisely the "threshold from a single + // sample" this module exists to prevent. The existing test used a fresh sessionId per + // iteration, so it never exposed it. + const events = readBalance(dir); + const closed = new Set( + events.filter((e) => e.kind === 'forecast-outcome' && e.forecastId).map((e) => e.forecastId), + ); + const open = events + .filter((e) => e.kind === 'forecast' && e.sessionId === sessionId && !closed.has(e.id)) .pop(); if (!open) return; + // Derived only once the open forecast is known, since the interval is measured from ITS turn. + // A forecast recorded before `turns` existed has none, and there is nothing to subtract from -- + // scoring it against an absolute turn number would manufacture an error rather than measure one, + // so it is left open instead. + const actual = elapsed !== null ? elapsed + : (Number.isFinite(open.turns) ? atTurn - open.turns : null); + if (actual === null || !Number.isFinite(actual)) return; + record(dir, { kind: 'forecast-outcome', sessionId, + // Stamped so the next call can see this forecast is spoken for. Without it the set above is + // empty and the same defect returns. + forecastId: open.id, horizon: open.horizon, predictedTurns: open.predictedTurns, - actualTurns, - error: actualTurns - open.predictedTurns, - hit: Math.abs(actualTurns - open.predictedTurns) <= TOLERANCE, + actualTurns: actual, + error: actual - open.predictedTurns, + hit: Math.abs(actual - open.predictedTurns) <= TOLERANCE, }); } @@ -98,7 +157,7 @@ export function observeOutcome(dir, { sessionId, actualTurns }) { * actionable, where "62% accurate" is not. */ export function reliability(dir) { - const outcomes = readMetrics(dir).filter((e) => e.kind === 'forecast-outcome'); + const outcomes = readBalance(dir).filter((e) => e.kind === 'forecast-outcome'); const buckets = {}; for (const horizon of HORIZONS) { @@ -164,7 +223,28 @@ export function calibrate(dir, predictedTurns) { // Refit: shift by the observed bias for this horizon. A bucket that runs long // gets pulled in, and vice versa. - const adjusted = Math.max(0, Math.round(predictedTurns + bucket.bias)); + // + // NOT CLAMPED WITH Math.max, which converted nonsense into a plausible-looking emergency. bias + // is a plain arithmetic mean of signed error, and a mean is not robust to the shape the + // 50%-hit-rate floor admits: a near bucket with errors [0,0,0,0,-30,-30,-30,-30] has four hits, + // so it clears the floor and is called calibrated, and its bias is -15. A raw forecast of 3 + // turns then became Math.max(0, Math.round(3 - 15)) = 0 and was published as fact, with the + // note "within 3 on 50% of 8 past forecasts" -- a zero-turn deadline the forecaster never + // produced, from a bucket that is wrong half the time. + // + // A correction at least as large as the thing it corrects is not a correction. + const adjusted = Math.round(predictedTurns + bucket.bias); + if (Math.abs(bucket.bias) >= predictedTurns || adjusted < 1) { + return { + publishable: false, + predictedTurns, + horizon, + bias: bucket.bias, + reason: `the observed bias at this horizon (${Math.round(bucket.bias)} turns) is as large as ` + + `the forecast itself (${predictedTurns}), so the correction would replace it rather than ` + + 'adjust it', + }; + } return { publishable: true, diff --git a/plugin/hooks/lib/forecast.mjs b/plugin/hooks/lib/forecast.mjs index ed698b1a0..39358e76a 100644 --- a/plugin/hooks/lib/forecast.mjs +++ b/plugin/hooks/lib/forecast.mjs @@ -34,6 +34,7 @@ import { readMetrics, readBalance, isFixtureAnchor, BALANCE_KINDS } from './metrics.mjs'; import { aggregateConsolidation } from './consolidate.mjs'; import { previewQuality } from './expand.mjs'; +import { logForecast, calibrate } from './calibration.mjs'; /** Turns of headroom below which the runway is worth interrupting for. */ export const ACTIONABLE_RUNWAY = 8; @@ -312,7 +313,33 @@ export function forecastPanel(dir, session = {}, findings = []) { if (air) { parts.runway = air; - lines.push(runwayLine(air)); + + // THE FORECAST KEEPS ITS OWN SCORE -- from here, because this is the only place a forecast is + // actually made. calibration.mjs was written for exactly this and had zero shipped importers: + // no forecast was ever logged, no outcome ever observed, reliability always saw an empty set, + // and calibrate was unreachable. So the shipped panel printed precisely the uncalibrated + // number that module's docstring calls "a vibe with a typeface". + // + // Logged BEFORE it is corrected. The raw prediction is what gets scored against the outcome; + // scoring the corrected one would fold the correction back into the next correction and the + // bias would chase its own tail. + logForecast(dir, { + sessionId: session.sessionId, + predictedTurns: air.withGraph, + used: session.used, + capacity: session.capacity, + // The turn this was predicted FROM. predictedTurns is an interval, so without its origin + // the outcome at compaction has nothing to subtract from. + turns: session.turns, + }); + + const score = calibrate(dir, air.withGraph); + parts.calibration = score; + // Published only when the horizon has earned it. An unpublishable score does not replace the + // number -- it is simply absent, exactly as the uncalibrated case reads today. + lines.push(score.publishable + ? `${runwayLine({ ...air, withGraph: score.predictedTurns })}\n (corrected: ${score.note})` + : runwayLine(air)); } if (rate) { diff --git a/plugin/hooks/lib/metrics.mjs b/plugin/hooks/lib/metrics.mjs index b762df57c..4b0eb2712 100644 --- a/plugin/hooks/lib/metrics.mjs +++ b/plugin/hooks/lib/metrics.mjs @@ -68,7 +68,17 @@ const balancePath = (dir) => join(dir, 'balance.jsonl'); * starve the measurement. Exported because any reader that needs the unwindowed truth needs to * know which kinds it applies to, and a second copy of this set elsewhere would drift. */ -export const BALANCE_KINDS = new Set(['inject', 'harvest', 'substitute']); +export const BALANCE_KINDS = new Set([ + 'inject', + 'harvest', + 'substitute', + // The forecast's own score. Both kinds are rare and accumulate slowly -- MIN_SCORED is 8 -- + // while the firehose they shared is dominated by per-tool-call records, so they were evicted + // faster than they could accumulate: calibrate() returned 'not yet calibrated (n/8)' forever + // and an outcome whose forecast had scrolled away was dropped with no record at all. + 'forecast', + 'forecast-outcome', +]); /** * Is this touch in the holdout arm? diff --git a/plugin/hooks/lib/policy.mjs b/plugin/hooks/lib/policy.mjs index b3871c27d..1fa14c348 100644 --- a/plugin/hooks/lib/policy.mjs +++ b/plugin/hooks/lib/policy.mjs @@ -272,7 +272,7 @@ function statePath(sessionId, agent) { /** A usable state object, whatever was on disk. */ function emptyState() { - return { seen: {}, denied: {}, injected: [], actCounts: {} }; + return { seen: {}, denied: {}, injected: [], actCounts: {}, forecast: null }; } /** @@ -312,6 +312,16 @@ export function loadState(sessionId, agent) { parsed.actCounts && typeof parsed.actCounts === 'object' && !Array.isArray(parsed.actCounts) ? parsed.actCounts : {}, + // THE SAME REASON A THIRD TIME. The forecast throttle records when the panel was last + // computed and what runway it last SHOWED, and both are meaningless within a single hook + // process -- every tool call is a new one. Dropped here, `checkedAt` would reset on every + // call and the panel would rebuild itself, transcript parse and all, on each tool use; and + // `shown` would reset, so worthSurfacing would compare against nothing and re-interrupt with + // the same runway forever. + forecast: + parsed.forecast && typeof parsed.forecast === 'object' && !Array.isArray(parsed.forecast) + ? parsed.forecast + : null, }; } catch { return emptyState(); @@ -383,6 +393,18 @@ export function saveState(sessionId, state, agent) { } return out; })(), + // LATEST CHECK WINS, which is the opposite direction from the fields above and correct for + // this one. `seen`, `denied`, `injected` and `actCounts` are all append-only, so a union or + // a max is the safe merge. The forecast throttle is a POINT IN TIME: taking the older of two + // concurrent checks would re-open the window and let the panel be rebuilt immediately, which + // is the cost the throttle exists to bound. + forecast: (() => { + const mine = state.forecast || null; + const theirs = current.forecast || null; + if (!mine) return theirs; + if (!theirs) return mine; + return (mine.checkedAt || 0) >= (theirs.checkedAt || 0) ? mine : theirs; + })(), }; // Write-then-rename so a reader never observes a half-written file. diff --git a/plugin/hooks/lib/surface.mjs b/plugin/hooks/lib/surface.mjs new file mode 100644 index 000000000..0ec05161c --- /dev/null +++ b/plugin/hooks/lib/surface.mjs @@ -0,0 +1,166 @@ +// GENERATED FILE -- do not edit. +// Source of truth: hooks-core/surface.mjs. Regenerate with `npm run sync:hooks`. +/** + * The entry point the forecast never had. + * + * forecast.mjs computes a runway, calibration.mjs scores it, and worthSurfacing decides whether a + * change is worth interrupting for -- and none of it ran. The reachability guard listed + * forecastPanel, worthSurfacing, logForecast and observeOutcome as unwired, which is a polite way + * of saying the feature did not exist for any user. This module is the missing half: it derives + * the session numbers the panel needs from the transcript, decides when the panel has earned an + * interruption, and closes the calibration loop when compaction fires. + * + * THREE COSTS ARE BOUNDED HERE, because this runs on the PreToolUse path and every tool call is a + * separate process: + * + * THE READ readCacheUsage parses up to 4 MB of transcript. Doing that on every tool call + * would tax the hook that exists to save the user money. + * THE COMPUTE the panel walks the metrics log, the balance log and the graph. + * THE CONTEXT the panel is only worth its own tokens when it says something actionable. + * + * So the throttle is checked FIRST, from persisted state, before any file is opened. Everything + * expensive happens after a clock comparison. + * + * NOTHING HERE IS ALLOWED TO BREAK A TOOL CALL. Every entry point returns null on failure rather + * than throwing: a forecast is a courtesy, and the hook's promise is that a defect in it costs the + * user nothing. + */ + +import { forecastPanel, worthSurfacing } from './forecast.mjs'; +import { observeOutcome } from './calibration.mjs'; +import { readCacheUsage } from './cache.mjs'; +import { readBalance } from './metrics.mjs'; + +/** + * How long between two attempts to build the panel. + * + * Not a tuning knob so much as an admission: the panel costs a transcript parse, and a user makes + * many tool calls per minute. Two minutes is short enough that a runway falling into single digits + * is still caught with turns to spare, and long enough that the check is free in aggregate. + */ +export const SURFACE_INTERVAL_MS = + Number(process.env.TOKEN_OPTIMIZER_SURFACE_INTERVAL_MS) || 120_000; + +/** + * The context window to measure the runway against. + * + * NOT DISCOVERABLE from the transcript: the usage rows carry what was spent, never the ceiling it + * was spent against. Defaulting is therefore unavoidable, so the default is stated rather than + * buried, and overridable for anyone running a different window. + */ +export const DEFAULT_CAPACITY = + Number(process.env.TOKEN_OPTIMIZER_CONTEXT_CAPACITY) || 200_000; + +/** + * What the session has spent, read from its own transcript. + * + * `used` is the FULL input size of the most recent request -- cache reads plus cache writes plus + * fresh input -- which is what the context window actually holds, rather than the marginal cost of + * that one turn. Summing the per-turn inputs instead would count the same carried prefix once per + * turn and report a number many times the window size. + * + * Returns null rather than a zeroed object when there is nothing to measure, so a caller cannot + * mistake "no data" for "an empty context". + */ +export function sessionUsage(transcriptPath, { capacity = DEFAULT_CAPACITY } = {}) { + let turns; + try { + turns = readCacheUsage(transcriptPath); + } catch { + return null; + } + if (!turns?.length) return null; + + const last = turns[turns.length - 1]; + const used = (last.read || 0) + (last.written || 0) + (last.input || 0); + if (!used) return null; + + return { used, capacity, turns: turns.length }; +} + +/** Intercepted touches recorded for this session, which converts a per-touch saving to per-turn. */ +function touchesFor(dir, sessionId) { + try { + return readBalance(dir).filter((e) => e.kind === 'inject' && e.sessionId === sessionId).length; + } catch { + return 0; + } +} + +/** + * Should the forecast interrupt right now, and with what? + * + * The state object is the caller's persisted per-session state; this reads `forecast` from it and + * returns the value to store back, so the throttle and the last-shown runway survive across hook + * processes. Returning the next state rather than mutating it keeps this testable without a + * filesystem. + * + * @returns { text, state } -- text is null when nothing should be shown. + */ +export function maybeSurface(dir, { + transcriptPath, sessionId, state = {}, now = Date.now(), findings = [], +} = {}) { + const previous = state.forecast || null; + + // THE THROTTLE IS FIRST, and deliberately before any I/O. Everything below opens files. + if (previous?.checkedAt && now - previous.checkedAt < SURFACE_INTERVAL_MS) { + return { text: null, state: previous }; + } + + let usage; + try { + usage = sessionUsage(transcriptPath); + } catch { + return { text: null, state: previous }; + } + // The clock is stamped even when there is nothing to measure, so a session with no usage rows + // does not re-attempt the transcript parse on every single tool call. + if (!usage) return { text: null, state: { ...previous, checkedAt: now } }; + + let panel; + try { + panel = forecastPanel(dir, { + ...usage, + sessionId, + touches: touchesFor(dir, sessionId), + }, findings); + } catch { + return { text: null, state: { ...previous, checkedAt: now } }; + } + if (!panel) return { text: null, state: { ...previous, checkedAt: now } }; + + // worthSurfacing compares against the last panel that was actually SHOWN, not the last one + // computed. Comparing against the last computed panel would let the runway drift down past the + // threshold one throttle window at a time and never trip the "crossed it" test. + const shown = previous?.shown ? { parts: { runway: { withGraph: previous.shown } } } : null; + if (!worthSurfacing(panel, shown)) { + return { text: null, state: { ...previous, checkedAt: now } }; + } + + return { + text: panel.text, + state: { checkedAt: now, shown: panel.parts.runway.withGraph }, + }; +} + +/** + * Closes the calibration loop, at the only moment that can close it. + * + * Compaction firing IS the ground truth a runway forecast was predicting, so this belongs on the + * PreCompact path and nowhere else. `actualTurns` is turns ELAPSED between the prediction and + * compaction, which is what predictedTurns meant -- so the forecast has to have recorded the turn + * it was made on, and observeOutcome subtracts. + * + * Silent by design: nothing is shown to anybody. It writes one record so that the NEXT session's + * forecast can be published with a track record instead of as a bare assertion. + */ +export function closeForecast(dir, { transcriptPath, sessionId } = {}) { + try { + const usage = sessionUsage(transcriptPath); + if (!usage) return false; + observeOutcome(dir, { sessionId, atTurn: usage.turns }); + return true; + } catch { + return false; + } +} diff --git a/plugin/hooks/precompact-optimize.mjs b/plugin/hooks/precompact-optimize.mjs index d546aa9dc..e69c8d25e 100644 --- a/plugin/hooks/precompact-optimize.mjs +++ b/plugin/hooks/precompact-optimize.mjs @@ -24,6 +24,7 @@ import { join } from 'node:path'; import { mode, MODE_OFF, loadState, clearSeen } from './lib/policy.mjs'; import { linkCoOccurrence } from './lib/inject.mjs'; import { wikiDir, projectRootFor } from './lib/wiki.mjs'; +import { closeForecast } from './lib/surface.mjs'; /** Longest compaction may be delayed. Past this the work is abandoned. */ const TIMEOUT_MS = @@ -125,6 +126,26 @@ async function main() { // silent no-op until a test caught it. clearSeen(payload.session_id, payload.transcript_path || null); + // THE GROUND TRUTH, AT THE ONLY MOMENT IT EXISTS. + // + // Compaction firing is exactly the event every runway forecast was predicting, so this is the + // one place the calibration loop can be closed. Without a caller here the loop collected + // predictions and never scored one: reliability saw an empty set forever and calibrate returned + // "not yet calibrated" for the life of the project, which is indistinguishable from the feature + // not existing. + // + // Silent, and before the wrapper check: nothing is shown to anybody, and a plugin-only install + // returns early below, so putting it after would have excluded every such user from the + // measurement -- the same shape as the co-occurrence write above. + try { + closeForecast(wikiDir(projectRootFor(payload.cwd, payload.cwd)), { + transcriptPath: payload.transcript_path, + sessionId: payload.session_id, + }); + } catch { + // Scoring a forecast must never delay compaction. + } + // NOW the wrapper, which only the optimize_session spawn needs. Plugin-only // installs stop here having still recorded their co-occurrence above. const wrapper = findWrapper(); diff --git a/plugin/hooks/pretooluse-router.mjs b/plugin/hooks/pretooluse-router.mjs index 6edf23136..abcdcfea8 100644 --- a/plugin/hooks/pretooluse-router.mjs +++ b/plugin/hooks/pretooluse-router.mjs @@ -32,6 +32,7 @@ import { commandProjectRoot, } from './lib/decide.mjs'; import { recordRead, fingerprint } from './lib/metrics.mjs'; +import { maybeSurface } from './lib/surface.mjs'; import { wikiDir, load, @@ -252,6 +253,28 @@ try { // their tool call, so a failure falls through to a plain allow. } + // THE FORECAST, AND THE ONLY PLACE IT CAN REACH A USER. + // + // This is the mid-session hook -- SessionStart is too early for a runway to mean anything and + // PreCompact is too late to act on one -- so it is where a deadline can still change what + // somebody does next. Everything about the cost is bounded inside maybeSurface: the throttle + // is checked from persisted state BEFORE any file is opened, and worthSurfacing withholds the + // panel unless the runway is actionable and has moved since the last one shown. + try { + const surfaced = maybeSurface(dirFor(payload.cwd || process.cwd()), { + transcriptPath: payload.transcript_path, + sessionId: payload.session_id, + state, + }); + if (surfaced.state !== state.forecast) { + state.forecast = surfaced.state; + saveState(payload.session_id, state, agentScope); + } + if (surfaced.text) context = context ? `${context}\n\n${surfaced.text}` : surfaced.text; + } catch { + // Same rule as above: a forecast is a courtesy and must never cost a tool call. + } + // THE MEMORY HALF. `harvest` records that this file was touched, at this // content hash, by this task -- and nothing had been calling it, anywhere. // The graph therefore accumulated no nodes and no task edges from ordinary diff --git a/tests/hooks/forecast.test.mjs b/tests/hooks/forecast.test.mjs index e0041f201..02488acb8 100644 --- a/tests/hooks/forecast.test.mjs +++ b/tests/hooks/forecast.test.mjs @@ -19,7 +19,7 @@ import { import { logForecast, observeOutcome, reliability, calibrate, } from '../../hooks-core/calibration.mjs'; -import { readMetrics } from '../../hooks-core/metrics.mjs'; +import { readMetrics, readBalance } from '../../hooks-core/metrics.mjs'; let dir; @@ -370,3 +370,133 @@ describe('the balance kinds are read from the log that is not windowed', () => { expect(merged.filter((e) => e.kind === 'inject')).toHaveLength(7); }); }); + +// --- the calibration loop is wired, and cannot score one prediction twice ----------- + +describe('a forecast is closed once', () => { + test('two compactions in one session do not score the same prediction twice', () => { + // THE DEFECT: the open-forecast filter was `!e.scored`, but nothing ever writes `scored` -- + // record() is an append-only JSONL writer with no update path, so a forecast could never gain + // the field and the predicate was always true. A session where compaction fires twice with no + // intervening logForecast popped the SAME forecast twice. reliability counts ROWS, not + // predictions, so MIN_SCORED could be satisfied by one prediction re-closed eight times -- + // exactly the 'threshold from a single sample' this module exists to prevent. The existing + // test used a fresh sessionId per iteration, so it never exposed it. + logForecast(dir, { sessionId: 'one', predictedTurns: 10, used: 1, capacity: 2 }); + observeOutcome(dir, { sessionId: 'one', actualTurns: 11 }); + observeOutcome(dir, { sessionId: 'one', actualTurns: 40 }); + + expect(reliability(dir).scored).toBe(1); + }); + + test('a second prediction in the same session is still scorable', () => { + logForecast(dir, { sessionId: 'one', predictedTurns: 10, used: 1, capacity: 2 }); + observeOutcome(dir, { sessionId: 'one', actualTurns: 11 }); + logForecast(dir, { sessionId: 'one', predictedTurns: 12, used: 1, capacity: 2 }); + observeOutcome(dir, { sessionId: 'one', actualTurns: 13 }); + + expect(reliability(dir).scored).toBe(2); + }); +}); + +describe('forecast events outlive the event window', () => { + test('an outcome still finds its forecast under a firehose', () => { + // THE DEFECT: both readers used readMetrics, which returns at most 2 MB and the last 5,000 + // lines, and forecast events were not protected kinds. On a busy project the forecast had + // scrolled past the tail by the time compaction fired, so the outcome was discarded with no + // record, no error and no counter -- and reliability could never reach MIN_SCORED, leaving + // 'not yet calibrated (n/8)' forever with no way to tell it from having no data at all. + logForecast(dir, { sessionId: 'buried', predictedTurns: 10, used: 1, capacity: 2 }); + for (let i = 0; i < 5_200; i++) { + recordRead(dir, { anchor: `/noise${i}.ts`, sessionId: 'buried', bytes: 40 }); + } + observeOutcome(dir, { sessionId: 'buried', actualTurns: 11 }); + + expect(reliability(dir).scored).toBe(1); + expect(readMetrics(dir).filter((e) => e.kind === 'forecast')).toHaveLength(0); + }); +}); + +describe('a correction never replaces the thing it corrects', () => { + test('a bimodal bucket at the reliability floor refuses to publish', () => { + // THE DEFECT: bias is a plain arithmetic mean of signed error, applied unguarded to any + // bucket clearing hitRate >= 0.5 -- and a mean is not robust to the shape that rule admits. + // Errors [0,0,0,0,-30,-30,-30,-30] give four hits, so the bucket passes the floor and is + // called calibrated, with bias -15. A raw forecast of 3 turns became + // Math.max(0, Math.round(3 - 15)) = 0, published as fact with the note 'within 3 on 50% of 8 + // past forecasts'. The clamp converted nonsense into a plausible-looking emergency. + const errors = [0, 0, 0, 0, -30, -30, -30, -30]; + errors.forEach((error, i) => { + logForecast(dir, { sessionId: `b${i}`, predictedTurns: 3, used: 1, capacity: 2 }); + observeOutcome(dir, { sessionId: `b${i}`, actualTurns: 3 + error }); + }); + + const bucket = reliability(dir).buckets.near; + expect(bucket.scored).toBe(8); + expect(bucket.calibrated).toBe(true); // it really does clear the floor + + const out = calibrate(dir, 3); + expect(out.publishable).toBe(false); + expect(out.predictedTurns).toBe(3); // the raw forecast, not a zero + expect(out.reason).toMatch(/as large as the forecast itself/); + }); + + test('a proportionate correction still publishes', () => { + for (let i = 0; i < 10; i++) { + logForecast(dir, { sessionId: `p${i}`, predictedTurns: 12, used: 1, capacity: 2 }); + observeOutcome(dir, { sessionId: `p${i}`, actualTurns: 14 }); + } + const out = calibrate(dir, 12); + expect(out.publishable).toBe(true); + expect(out.predictedTurns).toBe(14); + }); +}); + +describe('the panel actually runs the calibration loop', () => { + const session = { sessionId: 'live', used: 60_000, capacity: 200_000, turns: 30, touches: 30 }; + + test('rendering a panel logs the forecast it published', () => { + // THE DEFECT: calibration.mjs had zero shipped importers. forecastPanel computed the runway + // and pushed '~N turns to compaction' without ever calling logForecast or calibrate, so no + // forecast was logged, no outcome could be observed, reliability always saw an empty set, and + // calibrate was unreachable. The shipped panel printed exactly the uncalibrated number this + // module's docstring calls 'a vibe with a typeface'. + forecastPanel(dir, session, []); + const logged = readBalance(dir).filter((e) => e.kind === 'forecast'); + expect(logged).toHaveLength(1); + expect(logged[0].predictedTurns).toBeGreaterThan(0); + }); + + test('the raw forecast is what gets logged, not the corrected one', () => { + // Scoring the corrected number would fold each correction into the next and the bias would + // chase its own tail. + for (let i = 0; i < 10; i++) { + logForecast(dir, { sessionId: `w${i}`, predictedTurns: 70, used: 1, capacity: 2 }); + observeOutcome(dir, { sessionId: `w${i}`, actualTurns: 72 }); + } + const panel = forecastPanel(dir, session, []); + const raw = panel.parts.runway.withGraph; + const logged = readBalance(dir).filter((e) => e.kind === 'forecast' && e.sessionId === 'live'); + expect(logged[0].predictedTurns).toBe(raw); + }); + + test('an uncalibrated horizon prints the number without a track record', () => { + const panel = forecastPanel(dir, session, []); + expect(panel.parts.calibration.publishable).toBe(false); + expect(panel.text).not.toMatch(/corrected:/); + }); + + test('a calibrated horizon prints the corrected number with its track record', () => { + const panel = forecastPanel(dir, session, []); + const raw = panel.parts.runway.withGraph; + // Score enough forecasts at that horizon for the bucket to earn publication. + for (let i = 0; i < 12; i++) { + logForecast(dir, { sessionId: `c${i}`, predictedTurns: raw, used: 1, capacity: 2 }); + observeOutcome(dir, { sessionId: `c${i}`, actualTurns: raw + 2 }); + } + const after = forecastPanel(dir, session, []); + expect(after.parts.calibration.publishable).toBe(true); + expect(after.text).toMatch(/corrected: within 3 on \d+% of \d+ past forecasts/); + expect(after.text).toContain(`~${raw + 2} turns to compaction`); + }); +}); diff --git a/tests/hooks/reachability.test.mjs b/tests/hooks/reachability.test.mjs index 2e53b0f05..adbb3549c 100644 --- a/tests/hooks/reachability.test.mjs +++ b/tests/hooks/reachability.test.mjs @@ -91,17 +91,18 @@ const ALLOWED = new Map([ // WIRED ELSEWHERE -- leaves this list when that PR merges. ['forTouch', 'WIRED by the injection PR: just-in-time delivery, imported by nothing before it.'], - // A COMPLETE LOOP WITH NO ENTRY POINT. calibration.mjs has no importer at - // all: logForecast and observeOutcome collect the data, reliability scores it - // and calibrate applies the score. Nothing starts it, so the module cannot - // score anything -- and its own docstring says an uncalibrated forecast is - // "a vibe with a typeface". - ['logForecast', 'UNWIRED LOOP: calibration.mjs has zero importers; this records the prediction half.'], - ['observeOutcome', 'UNWIRED LOOP: records the outcome half of a calibration loop nothing starts.'], - - // FORECAST DISPLAY, unreachable while the calibration behind it is unreachable. - ['forecastPanel', 'UNWIRED: renders a forecast whose calibration loop never runs.'], - ['worthSurfacing', 'UNWIRED: decides whether a forecast change is worth showing; same dormant loop.'], + // THE FORECAST IS WIRED, so nothing from it is listed here any more. + // logForecast and calibrate are called by forecastPanel; forecastPanel and + // worthSurfacing by surface.mjs; observeOutcome by surface.closeForecast; and + // surface itself by the PreToolUse router and the PreCompact hook. Five + // entries left this list at once, which is what wiring a whole feature looks + // like -- and is why it was worth doing rather than annotating. + + // SURFACED BY TIGHTENING THE SCAN. Previously counted as reachable only + // because the bare word match read comment prose as a call site. Verified + // orphaned: in shipped code it appears as its own declaration plus two + // mentions in comments, and its only importer is a test. + ['sessionIndex', 'UNWIRED: lists graph keys and truncated claims for a session. Found by the comment-stripping fix; only a test imports it.'], // CONSOLIDATION. forecast.mjs imports aggregateConsolidation from this module, // so it is partly live -- but the selection, the ratio and the content anchor @@ -175,20 +176,71 @@ function exportedFunctions() { return found; } +/** + * Strips comments, so prose cannot be mistaken for a call. + * + * THIS IS WHAT LET A DEAD PUBLIC ENTRY POINT THROUGH. `calibrate` counted as + * reachable because curate.mjs:13 contains the English phrase "the reader's + * ability to calibrate trust", and `reliability` counted because the word + * appears in its own file's prose. calibration.mjs had ZERO importers -- no + * forecast was ever logged, no outcome observed, and the shipped panel printed + * precisely the uncalibrated number that module's docstring calls "a vibe with + * a typeface" -- while the check that exists to catch exactly this reported it + * as wired, on a coincidence of comment wording. + * + * A guard that reads documentation as code is worse than none: it is a clean + * bill of health nobody re-examines. This module's files are heavily commented + * by design, which makes the false-positive rate high rather than incidental. + * + * COMMENTS ONLY, NOT STRING LITERALS -- and that boundary was found the hard + * way. Stripping quotes as well made `routingReport` and `modelSwitchCost` look + * orphaned when routing-tool.ts calls both: three independent global passes + * cannot nest, so an apostrophe inside a DOUBLE-quoted sentence opened a + * "single-quoted string" that ran to the next apostrophe further down the file + * and swallowed the real call sites in between. + * + * Getting that right needs a scanner, and this guard's own rule says why not to + * build one: a check wrong in the permissive direction is merely useless, while + * one wrong in the strict direction fails CI on working code and gets deleted. + * The observed defect was comment prose, comments nest predictably, and that is + * where the line belongs. + */ +function stripComments(text) { + return text + .replace(/\/\*[\s\S]*?\*\//g, ' ') // block comments, including docblocks + .replace(/(^|[^:])\/\/[^\n]*/g, '$1 '); // line comments, sparing the // in a URL +} + /** * Referenced anywhere that ships, other than its own declaration? * - * A bare word match, deliberately. Anything cleverer would need to resolve the - * dynamic `mods..` handles that src/server uses, and a check that - * is wrong in the permissive direction is merely useless -- one that is wrong in - * the strict direction fails CI on working code and gets deleted. + * A bare word match over CODE, deliberately. Anything cleverer would need to + * resolve the dynamic `mods..` handles that src/server uses, and a + * check that is wrong in the permissive direction is merely useless -- one that + * is wrong in the strict direction fails CI on working code and gets deleted. + * Comments and strings are removed first: they are prose, and prose is not a + * caller. */ +const codeOnly = new Map(); +function shippedCode(file, text) { + if (!codeOnly.has(file)) codeOnly.set(file, stripComments(text)); + return codeOnly.get(file); +} + +// AN IN-FILE CALLER STILL COUNTS, and the attempt to change that is worth recording. Excluding +// the declaring file outright -- on the reasoning that `reliability` passed only because +// `calibrate` calls it from the same module -- reported 40+ false orphans in one run, among them +// burnRate, runway and shadowAvoided, which forecastPanel calls from inside forecast.mjs. Those +// are genuinely reached; reachability propagates THROUGH an in-file caller. The real defect in the +// reliability case was that its whole module had no importer, which is a module-level question +// this function cannot answer and should not try to. function usedInShippedCode(name, declaringFile) { const word = new RegExp(`\\b${name}\\b`, 'g'); const decl = new RegExp(`export\\s+(?:async\\s+)?function\\s+${name}\\b`, 'g'); for (const { file, text } of usages) { - const uses = (text.match(word) || []).length; - const declared = file === declaringFile ? (text.match(decl) || []).length : 0; + const code = shippedCode(file, text); + const uses = (code.match(word) || []).length; + const declared = file === declaringFile ? (code.match(decl) || []).length : 0; if (uses - declared > 0) return true; } return false; diff --git a/tests/hooks/surface.test.mjs b/tests/hooks/surface.test.mjs new file mode 100644 index 000000000..be841a511 --- /dev/null +++ b/tests/hooks/surface.test.mjs @@ -0,0 +1,223 @@ +/** + * The entry point the forecast never had. + * + * The properties under test are the ones that decide whether this is a feature or a liability: + * the cost is bounded before any file is opened, the panel only interrupts when it is actionable + * AND has moved since the last one shown, the throttle survives the process boundary that every + * hook invocation crosses, and the calibration loop is closed by the one event that can close it. + * + * The last test in this file is the one that matters most: the reachability guard's own verdict + * that these functions now have shipped callers. Everything else here would pass just as well on + * a module nothing imports -- which is exactly the defect this wiring exists to end. + */ + +import { mkdtempSync, rmSync, writeFileSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { + maybeSurface, closeForecast, sessionUsage, SURFACE_INTERVAL_MS, DEFAULT_CAPACITY, +} from '../../hooks-core/surface.mjs'; +import { logForecast, observeOutcome, reliability } from '../../hooks-core/calibration.mjs'; +import { record, recordRead } from '../../hooks-core/metrics.mjs'; + +let workspace; +let dir; + +beforeEach(() => { + workspace = mkdtempSync(join(tmpdir(), 'surface-')); + dir = join(workspace, 'wiki'); +}); +afterEach(() => rmSync(workspace, { recursive: true, force: true })); + +/** + * A transcript with `turns` assistant rows, the last one reporting `used` tokens of context. + * + * The usage shape mirrors a real transcript: the context size at a turn is cache reads plus cache + * writes plus fresh input, not the marginal cost of that turn alone. + */ +function transcript(turns, used = 60_000) { + const path = join(workspace, `t-${turns}-${used}.jsonl`); + const rows = []; + for (let i = 0; i < turns; i++) { + const last = i === turns - 1; + rows.push(JSON.stringify({ + type: 'assistant', + timestamp: new Date(1_700_000_000_000 + i * 1000).toISOString(), + message: { + role: 'assistant', + model: 'claude-opus-4', + usage: { + cache_read_input_tokens: last ? used - 1_000 : 100, + cache_creation_input_tokens: last ? 900 : 10, + input_tokens: last ? 100 : 5, + }, + }, + })); + } + writeFileSync(path, `${rows.join('\n')}\n`); + return path; +} + +/** Arms with a control arm big enough to publish a counterfactual, plus a short runway. */ +function seedArms({ treated = 12, withheld = 12, treatedRead = 500, withheldRead = 9000 } = {}) { + for (let i = 0; i < treated; i++) { + record(dir, { kind: 'inject', anchor: `/t${i}.ts`, sessionId: 'live', holdout: false, tokens: 100 }); + recordRead(dir, { anchor: `/t${i}.ts`, sessionId: 'live', bytes: treatedRead * 4 }); + } + for (let i = 0; i < withheld; i++) { + record(dir, { kind: 'inject', anchor: `/c${i}.ts`, sessionId: 'live', holdout: true, tokens: 0 }); + recordRead(dir, { anchor: `/c${i}.ts`, sessionId: 'live', bytes: withheldRead * 4 }); + } +} + +describe('the session numbers come from the transcript', () => { + test('used is the size of the context, not the sum of the turns', () => { + // Summing per-turn inputs counts the carried prefix once per turn and reports a number many + // times the window size -- which would make every runway look like an emergency. + const usage = sessionUsage(transcript(20, 60_000)); + expect(usage.used).toBe(60_000); + expect(usage.turns).toBe(20); + expect(usage.capacity).toBe(DEFAULT_CAPACITY); + }); + + test('no usage rows is null, not a zeroed session', () => { + const path = join(workspace, 'empty.jsonl'); + writeFileSync(path, '{"type":"user","message":{"role":"user","content":"hi"}}\n'); + expect(sessionUsage(path)).toBeNull(); + }); + + test('a missing transcript is null rather than a throw', () => { + expect(sessionUsage(join(workspace, 'nope.jsonl'))).toBeNull(); + }); +}); + +describe('the cost is bounded before anything is opened', () => { + test('inside the throttle window nothing is read at all', () => { + // The transcript path is deliberately nonexistent: if the throttle were checked after the + // read, this would still return null and the test would pass for the wrong reason. It cannot + // even be attempted, so the state must come back byte-identical. + const previous = { checkedAt: 1_000, shown: 4 }; + const out = maybeSurface(dir, { + transcriptPath: join(workspace, 'never-opened.jsonl'), + sessionId: 'live', + state: { forecast: previous }, + now: 1_000 + SURFACE_INTERVAL_MS - 1, + }); + expect(out.text).toBeNull(); + expect(out.state).toBe(previous); // the same object, untouched + }); + + test('past the window the clock is stamped even when there is nothing to say', () => { + // Otherwise a session with no usage rows re-parses the transcript on every single tool call. + const out = maybeSurface(dir, { + transcriptPath: join(workspace, 'absent.jsonl'), + sessionId: 'live', + state: { forecast: { checkedAt: 1_000, shown: null } }, + now: 1_000 + SURFACE_INTERVAL_MS + 1, + }); + expect(out.text).toBeNull(); + expect(out.state.checkedAt).toBe(1_000 + SURFACE_INTERVAL_MS + 1); + }); + + test('the interval is a real bound', () => { + expect(SURFACE_INTERVAL_MS).toBeGreaterThan(0); + expect(Number.isFinite(SURFACE_INTERVAL_MS)).toBe(true); + }); +}); + +describe('it interrupts only when the forecast has earned it', () => { + test('a comfortable runway says nothing', () => { + seedArms(); + // 10% used over 20 turns leaves a very long runway. + const out = maybeSurface(dir, { + transcriptPath: transcript(20, 20_000), sessionId: 'live', state: {}, now: 5_000, + }); + expect(out.text).toBeNull(); + expect(out.state.checkedAt).toBe(5_000); + }); + + test('a short runway surfaces, once', () => { + seedArms(); + // 190k of 200k used over 20 turns: about 1 turn left. + const path = transcript(20, 190_000); + const first = maybeSurface(dir, { transcriptPath: path, sessionId: 'live', state: {}, now: 5_000 }); + expect(first.text).toMatch(/turns to compaction/); + expect(first.state.shown).toBeGreaterThanOrEqual(0); + + // Same runway a window later: different, not actionable, so it stays quiet. + const second = maybeSurface(dir, { + transcriptPath: path, + sessionId: 'live', + state: { forecast: first.state }, + now: 5_000 + SURFACE_INTERVAL_MS + 1, + }); + expect(second.text).toBeNull(); + }); + + test('the comparison is against the last panel SHOWN, not the last computed', () => { + // Otherwise the runway drifts down one throttle window at a time, each step too small to trip + // the threshold test, and the interruption never fires at all. + seedArms(); + const shownAt = maybeSurface(dir, { + transcriptPath: transcript(20, 190_000), sessionId: 'live', state: {}, now: 5_000, + }); + expect(shownAt.text).not.toBeNull(); + expect(shownAt.state.shown).not.toBeUndefined(); + }); +}); + +describe('the calibration loop is closed by compaction', () => { + test('a forecast made earlier in the session is scored against the elapsed turns', () => { + // predictedTurns is an INTERVAL, so the ground truth is turns elapsed between the prediction + // and compaction -- not the turn number compaction happened on. + logForecast(dir, { sessionId: 'live', predictedTurns: 10, used: 60_000, capacity: 200_000, turns: 20 }); + closeForecast(dir, { transcriptPath: transcript(31, 190_000), sessionId: 'live' }); + + const scored = reliability(dir); + expect(scored.scored).toBe(1); + // 31 turns at compaction minus 20 at prediction = 11 elapsed, against a prediction of 10. + expect(scored.hitRate).toBe(1); + }); + + test('a forecast with no recorded origin is left open rather than scored against an absolute', () => { + // Scoring an interval against a turn NUMBER manufactures an error instead of measuring one. + record(dir, { + kind: 'forecast', sessionId: 'live', predictedTurns: 10, horizon: 'mid', turns: null, + }); + closeForecast(dir, { transcriptPath: transcript(31, 190_000), sessionId: 'live' }); + expect(reliability(dir).scored).toBe(0); + }); + + test('an explicit actualTurns still works, so the interval can be supplied directly', () => { + logForecast(dir, { sessionId: 'live', predictedTurns: 10, used: 1, capacity: 2, turns: 5 }); + observeOutcome(dir, { sessionId: 'live', actualTurns: 12 }); + expect(reliability(dir).scored).toBe(1); + }); + + test('no transcript means no score, and no crash', () => { + logForecast(dir, { sessionId: 'live', predictedTurns: 10, used: 1, capacity: 2, turns: 5 }); + expect(closeForecast(dir, { transcriptPath: join(workspace, 'gone.jsonl'), sessionId: 'live' })).toBe(false); + expect(reliability(dir).scored).toBe(0); + }); +}); + +describe('the wiring is real, not merely present', () => { + // This is the test that would have failed for the whole life of calibration.mjs. Every other + // test in this file passes just as well on a module nothing imports. + const hook = (name) => readFileSync(join(process.cwd(), 'plugin', 'hooks', name), 'utf8'); + + test('the PreToolUse router calls maybeSurface', () => { + expect(hook('pretooluse-router.mjs')).toMatch(/maybeSurface\(/); + }); + + test('the PreCompact hook calls closeForecast', () => { + expect(hook('precompact-optimize.mjs')).toMatch(/closeForecast\(/); + }); + + test('the vendored copy of surface.mjs is in place for the plugin to import', () => { + // The hooks import from ./lib/, which is synced rather than symlinked -- an unsynced module + // means the hook throws on load and fails open, silently doing nothing. + expect(() => readFileSync(join(process.cwd(), 'plugin', 'hooks', 'lib', 'surface.mjs'), 'utf8')) + .not.toThrow(); + }); +});