fix(keepwarm,lessons): make the tripwire reachable, the gaps per-session, the anchors real

hooks-core/keepwarm.mjs

@@ -24,7 +24,7 @@
  * whole thing stops and says so, which is the same rule the forecast follows.
  */
 
-import { record, readMetrics } from './metrics.mjs';
+import { record, readMetrics, readBalance } from './metrics.mjs';
 import { WRITE_MULTIPLIER, READ_MULTIPLIER } from './cache.mjs';
 
 /**
@@ -54,14 +54,37 @@ export const TRIPWIRE_MIN = 10;
  * was called.
  */
 export function gapDistribution(dir, { events = readMetrics(dir) } = {}) {
-  const stamps = events.map((e) => e.at).filter(Number.isFinite).sort((a, b) => a - b);
+  // PARTITIONED BY SESSION, because the interval between the last event of one session and the
+  // first of the next is not a gap between turns at all. The log spans days, so an overnight
+  // sixteen-hour boundary was counted as one -- dominating p90 and p99 and diluting
+  // probabilityWithin in the conservative direction, so ttlTier answered "neither tier pays" on
+  // projects where it would have paid. Silent, because that bias only ever declines to act.
+  //
+  // A long gap WITHIN a session is kept. It is real evidence that caching does not pay there, and
+  // dropping it would bias the answer the other way -- making keep-warm look better than it is,
+  // which is the direction this project cares about most.
+  //
+  // Events carrying no sessionId are pooled into one group rather than discarded: most kinds do
+  // carry one, and discarding the rest would throw away whole projects' history for a technicality.
+  const bySession = new Map();
+  for (const event of events) {
+    if (!Number.isFinite(event?.at)) continue;
+    const key = event.sessionId || '';
+    if (!bySession.has(key)) bySession.set(key, []);
+    bySession.get(key).push(event.at);
+  }
+
+  const stamps = [...bySession.values()].flat();
   if (stamps.length < 8) return null;
 
   const gaps = [];
-  for (let i = 1; i < stamps.length; i++) {
-    const gap = stamps[i] - stamps[i - 1];
-    // A burst of events inside one turn is not a gap between turns.
-    if (gap > 250) gaps.push(gap);
+  for (const session of bySession.values()) {
+    session.sort((a, b) => a - b);
+    for (let i = 1; i < session.length; i++) {
+      const gap = session[i] - session[i - 1];
+      // A burst of events inside one turn is not a gap between turns.
+      if (gap > 250) gaps.push(gap);
+    }
   }
   if (gaps.length < 6) return null;
 
@@ -207,7 +230,16 @@ export function recordRefreshOutcome(dir, { tier, prefixTokens, hit }) {
  * refreshes, keep-warm stops -- and says which tier and by how much, so the
  * stop is a finding rather than a silent behaviour change.
  */
-export function tripwire(dir, { events = readMetrics(dir) } = {}) {
+/**
+ * The backstop, read from the log that is NOT windowed.
+ *
+ * readBalance rather than readMetrics, and that distinction is the difference between a backstop
+ * that works and one that cannot. There is one outcome per refresh, in a log dominated by reads
+ * and captures, so through the 5000-event window the ten TRIPWIRE_MIN demands aged out before the
+ * tenth was written -- and this returned "only N/10 refreshes observed" for the life of the
+ * project. A guard that can never reach its own threshold is not a guard.
+ */
+export function tripwire(dir, { events = readBalance(dir) } = {}) {
   const outcomes = events.filter((e) => e.kind === 'keepwarm' && e.action === 'outcome');
   if (outcomes.length < TRIPWIRE_MIN) {
     return { tripped: false, observed: outcomes.length, reason: `only ${outcomes.length}/${TRIPWIRE_MIN} refreshes observed` };
@@ -232,8 +264,17 @@ export function tripwire(dir, { events = readMetrics(dir) } = {}) {
  * The decision as it should actually be taken: expected value, with the
  * tripwire able to veto it.
  */
-export function shouldKeepWarm(dir, { prefixTokens, events = readMetrics(dir) } = {}) {
-  const trip = tripwire(dir, { events });
+export function shouldKeepWarm(dir, {
+  prefixTokens,
+  events = readMetrics(dir),
+  // TWO SOURCES, because the two consumers want different things. gapDistribution wants the
+  // firehose -- every event is a timestamp and the recent ones describe the current rhythm. The
+  // tripwire wants the unwindowed balance log, because its outcomes are rare and are exactly what
+  // the window evicts. Passing one shared array to both, as this used to, meant whichever reader
+  // was chosen was wrong for one of them.
+  outcomes = readBalance(dir),
+} = {}) {
+  const trip = tripwire(dir, { events: outcomes });
   if (trip.tripped) return { action: 'skip', reason: trip.reason, trippedWire: true };
 
   const gaps = gapDistribution(dir, { events });

hooks-core/lessons.mjs

@@ -29,6 +29,7 @@
 
 import { ORIGIN_HARVESTED, ORIGIN_HUMAN } from './curate.mjs';
 import { safeTrigger } from './inject.mjs';
+import { isFsSafePath } from './paths.mjs';
 
 /** Lessons longer than this are prose, not instructions. */
 const MAX_CLAIM = 400;
@@ -170,7 +171,7 @@ export function isImperative(claim) {
  * Returns { lessons, rejected } so a caller can report what was dropped rather
  * than silently storing less than it was given.
  */
-export function validateLessons(raw, turns) {
+export function validateLessons(raw, turns, { knownFiles = null } = {}) {
   let parsed = raw;
   if (typeof raw === 'string') {
     try {
@@ -236,8 +237,19 @@ export function validateLessons(raw, turns) {
       // A verified human correction is as close to ground truth as this system
       // gets. An unverified paraphrase is a model's reading of one.
       confidence: verified ? 0.95 : 0.6,
+      // HELD TO THE FILES THE SESSION ACTUALLY TOUCHED, exactly as the finding path already is.
+      // harvest-worker calls validate() with `knownFiles: filesIn(digest)` and says why in its own
+      // comment -- "a model that invents a plausible path cannot anchor a finding to it" -- while
+      // this path accepted any non-empty string. The identical hallucination therefore walked
+      // straight through, and a lesson anchored to a file nobody opened is then injected on every
+      // future touch of it: a permanent instruction attached to code it was never about.
+      //
+      // isFsSafePath as well, because these strings reach the filesystem later through the graph.
+      // A path holding U+10FFFF ABORTS the process inside libuv rather than throwing, so no
+      // try/catch downstream can contain it.
       anchors: Array.isArray(item?.anchors)
-        ? item.anchors.filter((a) => typeof a === 'string' && a.trim())
+        ? item.anchors.filter((a) => typeof a === 'string' && a.trim() && isFsSafePath(a)
+            && (!knownFiles || knownFiles.includes(a)))
         : [],
     });
   }

hooks-core/metrics.mjs

@@ -76,6 +76,11 @@ export const BALANCE_KINDS = new Set([
   // and an outcome whose forecast had scrolled away was dropped with no record at all.
   'forecast',
   'forecast-outcome',
+  // The keep-warm tripwire's evidence. It needs TRIPWIRE_MIN outcomes before it may have an
+  // opinion, and there is one outcome per refresh -- so through the windowed reader the tenth
+  // aged out before it was written and the backstop was structurally unable to fire, reporting
+  // "only N/10 refreshes observed" for the life of the project.
+  'keepwarm',
 ]);
 
 /**

integrations/codex/hooks/lib/keepwarm.mjs

@@ -26,7 +26,7 @@
  * whole thing stops and says so, which is the same rule the forecast follows.
  */
 
-import { record, readMetrics } from './metrics.mjs';
+import { record, readMetrics, readBalance } from './metrics.mjs';
 import { WRITE_MULTIPLIER, READ_MULTIPLIER } from './cache.mjs';
 
 /**
@@ -56,14 +56,37 @@ export const TRIPWIRE_MIN = 10;
  * was called.
  */
 export function gapDistribution(dir, { events = readMetrics(dir) } = {}) {
-  const stamps = events.map((e) => e.at).filter(Number.isFinite).sort((a, b) => a - b);
+  // PARTITIONED BY SESSION, because the interval between the last event of one session and the
+  // first of the next is not a gap between turns at all. The log spans days, so an overnight
+  // sixteen-hour boundary was counted as one -- dominating p90 and p99 and diluting
+  // probabilityWithin in the conservative direction, so ttlTier answered "neither tier pays" on
+  // projects where it would have paid. Silent, because that bias only ever declines to act.
+  //
+  // A long gap WITHIN a session is kept. It is real evidence that caching does not pay there, and
+  // dropping it would bias the answer the other way -- making keep-warm look better than it is,
+  // which is the direction this project cares about most.
+  //
+  // Events carrying no sessionId are pooled into one group rather than discarded: most kinds do
+  // carry one, and discarding the rest would throw away whole projects' history for a technicality.
+  const bySession = new Map();
+  for (const event of events) {
+    if (!Number.isFinite(event?.at)) continue;
+    const key = event.sessionId || '';
+    if (!bySession.has(key)) bySession.set(key, []);
+    bySession.get(key).push(event.at);
+  }
+
+  const stamps = [...bySession.values()].flat();
   if (stamps.length < 8) return null;
 
   const gaps = [];
-  for (let i = 1; i < stamps.length; i++) {
-    const gap = stamps[i] - stamps[i - 1];
-    // A burst of events inside one turn is not a gap between turns.
-    if (gap > 250) gaps.push(gap);
+  for (const session of bySession.values()) {
+    session.sort((a, b) => a - b);
+    for (let i = 1; i < session.length; i++) {
+      const gap = session[i] - session[i - 1];
+      // A burst of events inside one turn is not a gap between turns.
+      if (gap > 250) gaps.push(gap);
+    }
   }
   if (gaps.length < 6) return null;
 
@@ -209,7 +232,16 @@ export function recordRefreshOutcome(dir, { tier, prefixTokens, hit }) {
  * refreshes, keep-warm stops -- and says which tier and by how much, so the
  * stop is a finding rather than a silent behaviour change.
  */
-export function tripwire(dir, { events = readMetrics(dir) } = {}) {
+/**
+ * The backstop, read from the log that is NOT windowed.
+ *
+ * readBalance rather than readMetrics, and that distinction is the difference between a backstop
+ * that works and one that cannot. There is one outcome per refresh, in a log dominated by reads
+ * and captures, so through the 5000-event window the ten TRIPWIRE_MIN demands aged out before the
+ * tenth was written -- and this returned "only N/10 refreshes observed" for the life of the
+ * project. A guard that can never reach its own threshold is not a guard.
+ */
+export function tripwire(dir, { events = readBalance(dir) } = {}) {
   const outcomes = events.filter((e) => e.kind === 'keepwarm' && e.action === 'outcome');
   if (outcomes.length < TRIPWIRE_MIN) {
     return { tripped: false, observed: outcomes.length, reason: `only ${outcomes.length}/${TRIPWIRE_MIN} refreshes observed` };
@@ -234,8 +266,17 @@ export function tripwire(dir, { events = readMetrics(dir) } = {}) {
  * The decision as it should actually be taken: expected value, with the
  * tripwire able to veto it.
  */
-export function shouldKeepWarm(dir, { prefixTokens, events = readMetrics(dir) } = {}) {
-  const trip = tripwire(dir, { events });
+export function shouldKeepWarm(dir, {
+  prefixTokens,
+  events = readMetrics(dir),
+  // TWO SOURCES, because the two consumers want different things. gapDistribution wants the
+  // firehose -- every event is a timestamp and the recent ones describe the current rhythm. The
+  // tripwire wants the unwindowed balance log, because its outcomes are rare and are exactly what
+  // the window evicts. Passing one shared array to both, as this used to, meant whichever reader
+  // was chosen was wrong for one of them.
+  outcomes = readBalance(dir),
+} = {}) {
+  const trip = tripwire(dir, { events: outcomes });
   if (trip.tripped) return { action: 'skip', reason: trip.reason, trippedWire: true };
 
   const gaps = gapDistribution(dir, { events });

integrations/codex/hooks/lib/lessons.mjs

@@ -31,6 +31,7 @@
 
 import { ORIGIN_HARVESTED, ORIGIN_HUMAN } from './curate.mjs';
 import { safeTrigger } from './inject.mjs';
+import { isFsSafePath } from './paths.mjs';
 
 /** Lessons longer than this are prose, not instructions. */
 const MAX_CLAIM = 400;
@@ -172,7 +173,7 @@ export function isImperative(claim) {
  * Returns { lessons, rejected } so a caller can report what was dropped rather
  * than silently storing less than it was given.
  */
-export function validateLessons(raw, turns) {
+export function validateLessons(raw, turns, { knownFiles = null } = {}) {
   let parsed = raw;
   if (typeof raw === 'string') {
     try {
@@ -238,8 +239,19 @@ export function validateLessons(raw, turns) {
       // A verified human correction is as close to ground truth as this system
       // gets. An unverified paraphrase is a model's reading of one.
       confidence: verified ? 0.95 : 0.6,
+      // HELD TO THE FILES THE SESSION ACTUALLY TOUCHED, exactly as the finding path already is.
+      // harvest-worker calls validate() with `knownFiles: filesIn(digest)` and says why in its own
+      // comment -- "a model that invents a plausible path cannot anchor a finding to it" -- while
+      // this path accepted any non-empty string. The identical hallucination therefore walked
+      // straight through, and a lesson anchored to a file nobody opened is then injected on every
+      // future touch of it: a permanent instruction attached to code it was never about.
+      //
+      // isFsSafePath as well, because these strings reach the filesystem later through the graph.
+      // A path holding U+10FFFF ABORTS the process inside libuv rather than throwing, so no
+      // try/catch downstream can contain it.
       anchors: Array.isArray(item?.anchors)
-        ? item.anchors.filter((a) => typeof a === 'string' && a.trim())
+        ? item.anchors.filter((a) => typeof a === 'string' && a.trim() && isFsSafePath(a)
+            && (!knownFiles || knownFiles.includes(a)))
         : [],
     });
   }

integrations/codex/hooks/lib/metrics.mjs

@@ -78,6 +78,11 @@ export const BALANCE_KINDS = new Set([
   // and an outcome whose forecast had scrolled away was dropped with no record at all.
   'forecast',
   'forecast-outcome',
+  // The keep-warm tripwire's evidence. It needs TRIPWIRE_MIN outcomes before it may have an
+  // opinion, and there is one outcome per refresh -- so through the windowed reader the tenth
+  // aged out before it was written and the backstop was structurally unable to fire, reporting
+  // "only N/10 refreshes observed" for the life of the project.
+  'keepwarm',
 ]);
 
 /**

integrations/codex/plugin/hooks/lib/keepwarm.mjs

@@ -26,7 +26,7 @@
  * whole thing stops and says so, which is the same rule the forecast follows.
  */
 
-import { record, readMetrics } from './metrics.mjs';
+import { record, readMetrics, readBalance } from './metrics.mjs';
 import { WRITE_MULTIPLIER, READ_MULTIPLIER } from './cache.mjs';
 
 /**
@@ -56,14 +56,37 @@ export const TRIPWIRE_MIN = 10;
  * was called.
  */
 export function gapDistribution(dir, { events = readMetrics(dir) } = {}) {
-  const stamps = events.map((e) => e.at).filter(Number.isFinite).sort((a, b) => a - b);
+  // PARTITIONED BY SESSION, because the interval between the last event of one session and the
+  // first of the next is not a gap between turns at all. The log spans days, so an overnight
+  // sixteen-hour boundary was counted as one -- dominating p90 and p99 and diluting
+  // probabilityWithin in the conservative direction, so ttlTier answered "neither tier pays" on
+  // projects where it would have paid. Silent, because that bias only ever declines to act.
+  //
+  // A long gap WITHIN a session is kept. It is real evidence that caching does not pay there, and
+  // dropping it would bias the answer the other way -- making keep-warm look better than it is,
+  // which is the direction this project cares about most.
+  //
+  // Events carrying no sessionId are pooled into one group rather than discarded: most kinds do
+  // carry one, and discarding the rest would throw away whole projects' history for a technicality.
+  const bySession = new Map();
+  for (const event of events) {
+    if (!Number.isFinite(event?.at)) continue;
+    const key = event.sessionId || '';
+    if (!bySession.has(key)) bySession.set(key, []);
+    bySession.get(key).push(event.at);
+  }
+
+  const stamps = [...bySession.values()].flat();
   if (stamps.length < 8) return null;
 
   const gaps = [];
-  for (let i = 1; i < stamps.length; i++) {
-    const gap = stamps[i] - stamps[i - 1];
-    // A burst of events inside one turn is not a gap between turns.
-    if (gap > 250) gaps.push(gap);
+  for (const session of bySession.values()) {
+    session.sort((a, b) => a - b);
+    for (let i = 1; i < session.length; i++) {
+      const gap = session[i] - session[i - 1];
+      // A burst of events inside one turn is not a gap between turns.
+      if (gap > 250) gaps.push(gap);
+    }
   }
   if (gaps.length < 6) return null;
 
@@ -209,7 +232,16 @@ export function recordRefreshOutcome(dir, { tier, prefixTokens, hit }) {
  * refreshes, keep-warm stops -- and says which tier and by how much, so the
  * stop is a finding rather than a silent behaviour change.
  */
-export function tripwire(dir, { events = readMetrics(dir) } = {}) {
+/**
+ * The backstop, read from the log that is NOT windowed.
+ *
+ * readBalance rather than readMetrics, and that distinction is the difference between a backstop
+ * that works and one that cannot. There is one outcome per refresh, in a log dominated by reads
+ * and captures, so through the 5000-event window the ten TRIPWIRE_MIN demands aged out before the
+ * tenth was written -- and this returned "only N/10 refreshes observed" for the life of the
+ * project. A guard that can never reach its own threshold is not a guard.
+ */
+export function tripwire(dir, { events = readBalance(dir) } = {}) {
   const outcomes = events.filter((e) => e.kind === 'keepwarm' && e.action === 'outcome');
   if (outcomes.length < TRIPWIRE_MIN) {
     return { tripped: false, observed: outcomes.length, reason: `only ${outcomes.length}/${TRIPWIRE_MIN} refreshes observed` };
@@ -234,8 +266,17 @@ export function tripwire(dir, { events = readMetrics(dir) } = {}) {
  * The decision as it should actually be taken: expected value, with the
  * tripwire able to veto it.
  */
-export function shouldKeepWarm(dir, { prefixTokens, events = readMetrics(dir) } = {}) {
-  const trip = tripwire(dir, { events });
+export function shouldKeepWarm(dir, {
+  prefixTokens,
+  events = readMetrics(dir),
+  // TWO SOURCES, because the two consumers want different things. gapDistribution wants the
+  // firehose -- every event is a timestamp and the recent ones describe the current rhythm. The
+  // tripwire wants the unwindowed balance log, because its outcomes are rare and are exactly what
+  // the window evicts. Passing one shared array to both, as this used to, meant whichever reader
+  // was chosen was wrong for one of them.
+  outcomes = readBalance(dir),
+} = {}) {
+  const trip = tripwire(dir, { events: outcomes });
   if (trip.tripped) return { action: 'skip', reason: trip.reason, trippedWire: true };
 
   const gaps = gapDistribution(dir, { events });