From 66defa2bb446ad139cef656915ea19583565becd Mon Sep 17 00:00:00 2001 From: Franklin Moormann Date: Sat, 17 Oct 2026 13:59:14 -0400 Subject: [PATCH 1/5] feat(bench): bench/recall, calls avoided across a derive-then-reuse pair of sessions M8 of #357, built and self-tested; nothing has been run against a model. Three pairs of THOL tasks, each a real defect in a small standard-library Python project: day buckets that use the machine's local time (tzbook), a window documented in seconds and read in milliseconds (quotasvc), and migrations sorted as strings (migrun). `derive` asks for the root cause; `reuse` starts a fresh session in a fresh workspace and asks a question whose answer depends on it. - verifiers score from ground truth in fixtures/truth, which the agent never sees, and award nothing for an empty workspace, a changed fixture file, an empty reply, or a reuse answer hedged with a wrong one. - recall.py: `plan` writes the docker commands for the bench/thol rig, one empty graph per arm, pair and repetition, mounted for that pair's two sessions only; `run` refuses without --approve-spend; `report` counts calls avoided only for repetitions both arms answered correctly. - test_recall.py (16 tests, standard library only): every verifier against answers whose score is known first, no workspace file or prompt stating a conclusion, each fixture carrying its defect, the runner's isolation and its refusal. Runs in a new CI job, Recall Benchmark Self-Test. - fixtures are pinned to LF, because the verifiers hash them. Co-Authored-By: Claude Opus 5.5 --- .gitattributes | 4 + .github/workflows/quality-gates.yml | 21 ++ bench/recall/README.md | 52 ++++ bench/recall/fixtures/migrun/README.md | 5 + .../migrun/migrations/10_backfill_email.sql | 1 + .../migrun/migrations/1_create_users.sql | 1 + .../migrun/migrations/2_add_email.sql | 1 + .../migrun/migrations/3_create_orders.sql | 1 + .../migrun/migrations/4_index_orders_user.sql | 1 + .../migrun/migrations/5_create_products.sql | 1 + .../migrun/migrations/6_add_product_price.sql | 1 + .../migrations/7_create_order_lines.sql | 1 + .../migrun/migrations/8_seed_admin.sql | 1 + .../migrun/migrations/9_index_lines_order.sql | 1 + .../recall/fixtures/migrun/migrun/__init__.py | 1 + bench/recall/fixtures/migrun/migrun/runner.py | 20 ++ .../recall/fixtures/migrun/tests/__init__.py | 0 .../fixtures/migrun/tests/test_runner.py | 18 ++ bench/recall/fixtures/quotasvc/README.md | 12 + .../fixtures/quotasvc/quotasvc/__init__.py | 1 + .../fixtures/quotasvc/quotasvc/config.py | 14 + .../fixtures/quotasvc/quotasvc/limiter.py | 24 ++ .../fixtures/quotasvc/tests/__init__.py | 0 .../fixtures/quotasvc/tests/test_limiter.py | 21 ++ .../recall/fixtures/truth/recall-migrate.json | 63 ++++ bench/recall/fixtures/truth/recall-quota.json | 50 ++++ bench/recall/fixtures/truth/recall-tz.json | 49 ++++ bench/recall/fixtures/tzbook/README.md | 5 + .../recall/fixtures/tzbook/tests/__init__.py | 0 .../fixtures/tzbook/tests/test_buckets.py | 34 +++ .../recall/fixtures/tzbook/tzbook/__init__.py | 1 + .../recall/fixtures/tzbook/tzbook/buckets.py | 22 ++ bench/recall/hash_fixtures.py | 49 ++++ bench/recall/recall.py | 160 +++++++++++ bench/recall/tasks/_recall.py | 53 ++++ bench/recall/tasks/_vlib.py | 56 ++++ .../tasks/recall-migrate-derive/prompt.md | 6 + .../tasks/recall-migrate-derive/task.json | 1 + .../tasks/recall-migrate-derive/verify.py | 7 + .../tasks/recall-migrate-reuse/prompt.md | 5 + .../tasks/recall-migrate-reuse/task.json | 1 + .../tasks/recall-migrate-reuse/verify.py | 7 + .../tasks/recall-quota-derive/prompt.md | 7 + .../tasks/recall-quota-derive/task.json | 1 + .../tasks/recall-quota-derive/verify.py | 7 + .../recall/tasks/recall-quota-reuse/prompt.md | 4 + .../recall/tasks/recall-quota-reuse/task.json | 1 + .../recall/tasks/recall-quota-reuse/verify.py | 7 + bench/recall/tasks/recall-tz-derive/prompt.md | 7 + bench/recall/tasks/recall-tz-derive/task.json | 1 + bench/recall/tasks/recall-tz-derive/verify.py | 7 + bench/recall/tasks/recall-tz-reuse/prompt.md | 6 + bench/recall/tasks/recall-tz-reuse/task.json | 1 + bench/recall/tasks/recall-tz-reuse/verify.py | 7 + bench/recall/test_recall.py | 271 ++++++++++++++++++ 55 files changed, 1099 insertions(+) create mode 100644 bench/recall/README.md create mode 100644 bench/recall/fixtures/migrun/README.md create mode 100644 bench/recall/fixtures/migrun/migrations/10_backfill_email.sql create mode 100644 bench/recall/fixtures/migrun/migrations/1_create_users.sql create mode 100644 bench/recall/fixtures/migrun/migrations/2_add_email.sql create mode 100644 bench/recall/fixtures/migrun/migrations/3_create_orders.sql create mode 100644 bench/recall/fixtures/migrun/migrations/4_index_orders_user.sql create mode 100644 bench/recall/fixtures/migrun/migrations/5_create_products.sql create mode 100644 bench/recall/fixtures/migrun/migrations/6_add_product_price.sql create mode 100644 bench/recall/fixtures/migrun/migrations/7_create_order_lines.sql create mode 100644 bench/recall/fixtures/migrun/migrations/8_seed_admin.sql create mode 100644 bench/recall/fixtures/migrun/migrations/9_index_lines_order.sql create mode 100644 bench/recall/fixtures/migrun/migrun/__init__.py create mode 100644 bench/recall/fixtures/migrun/migrun/runner.py create mode 100644 bench/recall/fixtures/migrun/tests/__init__.py create mode 100644 bench/recall/fixtures/migrun/tests/test_runner.py create mode 100644 bench/recall/fixtures/quotasvc/README.md create mode 100644 bench/recall/fixtures/quotasvc/quotasvc/__init__.py create mode 100644 bench/recall/fixtures/quotasvc/quotasvc/config.py create mode 100644 bench/recall/fixtures/quotasvc/quotasvc/limiter.py create mode 100644 bench/recall/fixtures/quotasvc/tests/__init__.py create mode 100644 bench/recall/fixtures/quotasvc/tests/test_limiter.py create mode 100644 bench/recall/fixtures/truth/recall-migrate.json create mode 100644 bench/recall/fixtures/truth/recall-quota.json create mode 100644 bench/recall/fixtures/truth/recall-tz.json create mode 100644 bench/recall/fixtures/tzbook/README.md create mode 100644 bench/recall/fixtures/tzbook/tests/__init__.py create mode 100644 bench/recall/fixtures/tzbook/tests/test_buckets.py create mode 100644 bench/recall/fixtures/tzbook/tzbook/__init__.py create mode 100644 bench/recall/fixtures/tzbook/tzbook/buckets.py create mode 100644 bench/recall/hash_fixtures.py create mode 100644 bench/recall/recall.py create mode 100644 bench/recall/tasks/_recall.py create mode 100644 bench/recall/tasks/_vlib.py create mode 100644 bench/recall/tasks/recall-migrate-derive/prompt.md create mode 100644 bench/recall/tasks/recall-migrate-derive/task.json create mode 100644 bench/recall/tasks/recall-migrate-derive/verify.py create mode 100644 bench/recall/tasks/recall-migrate-reuse/prompt.md create mode 100644 bench/recall/tasks/recall-migrate-reuse/task.json create mode 100644 bench/recall/tasks/recall-migrate-reuse/verify.py create mode 100644 bench/recall/tasks/recall-quota-derive/prompt.md create mode 100644 bench/recall/tasks/recall-quota-derive/task.json create mode 100644 bench/recall/tasks/recall-quota-derive/verify.py create mode 100644 bench/recall/tasks/recall-quota-reuse/prompt.md create mode 100644 bench/recall/tasks/recall-quota-reuse/task.json create mode 100644 bench/recall/tasks/recall-quota-reuse/verify.py create mode 100644 bench/recall/tasks/recall-tz-derive/prompt.md create mode 100644 bench/recall/tasks/recall-tz-derive/task.json create mode 100644 bench/recall/tasks/recall-tz-derive/verify.py create mode 100644 bench/recall/tasks/recall-tz-reuse/prompt.md create mode 100644 bench/recall/tasks/recall-tz-reuse/task.json create mode 100644 bench/recall/tasks/recall-tz-reuse/verify.py create mode 100644 bench/recall/test_recall.py diff --git a/.gitattributes b/.gitattributes index ba4815bee..c9bdb0ffd 100644 --- a/.gitattributes +++ b/.gitattributes @@ -15,3 +15,7 @@ bench/tools/fixtures/** text eol=lf # compared against, exact bytes. bench/compression/corpus.snapshot.json text eol=lf bench/compression/token-counts.json text eol=lf + +# The recall verifiers hash the fixture files to prove a session changed nothing, +# so a file's bytes must be the same on every checkout. +bench/recall/fixtures/** text eol=lf diff --git a/.github/workflows/quality-gates.yml b/.github/workflows/quality-gates.yml index e2241547f..007031f9a 100644 --- a/.github/workflows/quality-gates.yml +++ b/.github/workflows/quality-gates.yml @@ -707,6 +707,27 @@ jobs: # `tiktoken` and `dist/`; `bench/tools/reduction` needs `tiktoken`. They are # instrument checks and not README checks, so they are run as their own step -- # a failure names which half of the job went red without reading the log. + # THE RECALL BENCHMARK'S VERIFIERS, EXERCISED WITHOUT RUNNING A SESSION. A + # verifier that is wrong is wrong silently -- it scores real, billed sessions + # and nobody re-reads the replies -- so every one is run here against answers + # whose score is known first. Standard library only, so no install. + recall-selftest: + name: Recall Benchmark Self-Test + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.13' + + - name: Run the recall verifiers against known answers + run: python3 -m unittest discover -s bench/recall -p "test_*.py" -v + bench-proof: name: Proof Harness, Instruments That Need a Build, README Figures runs-on: ubuntu-latest diff --git a/bench/recall/README.md b/bench/recall/README.md new file mode 100644 index 000000000..377893799 --- /dev/null +++ b/bench/recall/README.md @@ -0,0 +1,52 @@ +# bench/recall: calls avoided across sessions + +Does what one session works out save the next session tool calls? Every +single-session benchmark, THOL included, is blind to this by construction: +each run starts from nothing. This one runs sessions in pairs. + +| half | asks | +| -------- | --------------------------------------------------------------- | +| `derive` | work something out that the workspace does not state | +| `reuse` | a fresh session, fresh workspace: a question that depends on it | + +Three pairs, each a real defect in a small standard-library Python project: + +| pair | fixture | derived | reused | +| ---------------- | ---------- | ---------------------------------------- | ----------------------------------------- | +| `recall-tz` | `tzbook` | day buckets use the machine's local time | the variable CI sets to make it pass | +| `recall-quota` | `quotasvc` | a window documented in s is read in ms | the value that gives a 90-second window | +| `recall-migrate` | `migrun` | migrations sort as strings | which migration a fresh database runs first | + +The tasks are THOL tasks (`prompt.md`, `task.json`, `verify.py`) and run on the +rig in `bench/thol`. Each arm, pair and repetition gets its own empty graph +directory, mounted for that pair's two sessions only. **Calls avoided** is the +control arm's reuse tool calls minus the arm's, counted only for repetitions in +which both answered correctly: a cheap wrong answer is not a saving. + +## Verifiers + +Scored from ground truth in `fixtures/truth/`, which the agent never sees. A +verifier awards nothing for an empty workspace, a changed fixture file, an empty +reply, or (on `reuse`) the right answer hedged with a wrong one. + + python3 -m unittest discover -s bench/recall -p "test_*.py" + +runs every verifier against answers whose score is known in advance, checks that +no workspace file or prompt states a pair's conclusion, and checks that each +fixture really carries its defect. It runs no session and spends nothing; CI +runs it on every pull request. + +## Running it + + python3 bench/recall/recall.py plan out/recall --reps 3 + python3 bench/recall/recall.py run out/recall --approve-spend + python3 bench/recall/recall.py report out/recall/results/results.sqlite + +`plan` prints every docker command and spends nothing. `run` refuses without +`--approve-spend`, because every step is a real, billed Claude Code session. +The default arms are `control` and `token-optimizer-proxy-knowledge`, whose +manifest points the graph at `/proxy-graph`. + +Changing a fixture? Re-hash it, or every verifier will score a real session 0: + + python3 bench/recall/hash_fixtures.py \ No newline at end of file diff --git a/bench/recall/fixtures/migrun/README.md b/bench/recall/fixtures/migrun/README.md new file mode 100644 index 000000000..790ed1174 --- /dev/null +++ b/bench/recall/fixtures/migrun/README.md @@ -0,0 +1,5 @@ +# migrun + +Applies the SQL files in `migrations/` to a SQLite database, in order. + + python3 -m unittest discover -s tests -t . \ No newline at end of file diff --git a/bench/recall/fixtures/migrun/migrations/10_backfill_email.sql b/bench/recall/fixtures/migrun/migrations/10_backfill_email.sql new file mode 100644 index 000000000..3ad186673 --- /dev/null +++ b/bench/recall/fixtures/migrun/migrations/10_backfill_email.sql @@ -0,0 +1 @@ +UPDATE users SET email = name || '@example.invalid' WHERE email IS NULL; diff --git a/bench/recall/fixtures/migrun/migrations/1_create_users.sql b/bench/recall/fixtures/migrun/migrations/1_create_users.sql new file mode 100644 index 000000000..15400af86 --- /dev/null +++ b/bench/recall/fixtures/migrun/migrations/1_create_users.sql @@ -0,0 +1 @@ +CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT NOT NULL); diff --git a/bench/recall/fixtures/migrun/migrations/2_add_email.sql b/bench/recall/fixtures/migrun/migrations/2_add_email.sql new file mode 100644 index 000000000..79fc1d861 --- /dev/null +++ b/bench/recall/fixtures/migrun/migrations/2_add_email.sql @@ -0,0 +1 @@ +ALTER TABLE users ADD COLUMN email TEXT; diff --git a/bench/recall/fixtures/migrun/migrations/3_create_orders.sql b/bench/recall/fixtures/migrun/migrations/3_create_orders.sql new file mode 100644 index 000000000..478369124 --- /dev/null +++ b/bench/recall/fixtures/migrun/migrations/3_create_orders.sql @@ -0,0 +1 @@ +CREATE TABLE orders (id INTEGER PRIMARY KEY, user_id INTEGER NOT NULL, total INTEGER NOT NULL); diff --git a/bench/recall/fixtures/migrun/migrations/4_index_orders_user.sql b/bench/recall/fixtures/migrun/migrations/4_index_orders_user.sql new file mode 100644 index 000000000..f62240a19 --- /dev/null +++ b/bench/recall/fixtures/migrun/migrations/4_index_orders_user.sql @@ -0,0 +1 @@ +CREATE INDEX orders_user ON orders (user_id); diff --git a/bench/recall/fixtures/migrun/migrations/5_create_products.sql b/bench/recall/fixtures/migrun/migrations/5_create_products.sql new file mode 100644 index 000000000..f19f3c42f --- /dev/null +++ b/bench/recall/fixtures/migrun/migrations/5_create_products.sql @@ -0,0 +1 @@ +CREATE TABLE products (id INTEGER PRIMARY KEY, title TEXT NOT NULL); diff --git a/bench/recall/fixtures/migrun/migrations/6_add_product_price.sql b/bench/recall/fixtures/migrun/migrations/6_add_product_price.sql new file mode 100644 index 000000000..de2369b51 --- /dev/null +++ b/bench/recall/fixtures/migrun/migrations/6_add_product_price.sql @@ -0,0 +1 @@ +ALTER TABLE products ADD COLUMN price INTEGER NOT NULL DEFAULT 0; diff --git a/bench/recall/fixtures/migrun/migrations/7_create_order_lines.sql b/bench/recall/fixtures/migrun/migrations/7_create_order_lines.sql new file mode 100644 index 000000000..69e17460d --- /dev/null +++ b/bench/recall/fixtures/migrun/migrations/7_create_order_lines.sql @@ -0,0 +1 @@ +CREATE TABLE order_lines (order_id INTEGER NOT NULL, product_id INTEGER NOT NULL, quantity INTEGER NOT NULL); diff --git a/bench/recall/fixtures/migrun/migrations/8_seed_admin.sql b/bench/recall/fixtures/migrun/migrations/8_seed_admin.sql new file mode 100644 index 000000000..89c90055e --- /dev/null +++ b/bench/recall/fixtures/migrun/migrations/8_seed_admin.sql @@ -0,0 +1 @@ +INSERT INTO users (id, name) VALUES (1, 'admin'); diff --git a/bench/recall/fixtures/migrun/migrations/9_index_lines_order.sql b/bench/recall/fixtures/migrun/migrations/9_index_lines_order.sql new file mode 100644 index 000000000..726f68ce0 --- /dev/null +++ b/bench/recall/fixtures/migrun/migrations/9_index_lines_order.sql @@ -0,0 +1 @@ +CREATE INDEX lines_order ON order_lines (order_id); diff --git a/bench/recall/fixtures/migrun/migrun/__init__.py b/bench/recall/fixtures/migrun/migrun/__init__.py new file mode 100644 index 000000000..5a5d070ed --- /dev/null +++ b/bench/recall/fixtures/migrun/migrun/__init__.py @@ -0,0 +1 @@ +"""A small SQL migration runner.""" \ No newline at end of file diff --git a/bench/recall/fixtures/migrun/migrun/runner.py b/bench/recall/fixtures/migrun/migrun/runner.py new file mode 100644 index 000000000..b87baf169 --- /dev/null +++ b/bench/recall/fixtures/migrun/migrun/runner.py @@ -0,0 +1,20 @@ +"""Apply every migration in a directory to a database, in order.""" + +import os +import sqlite3 + + +def pending(directory): + """The migration files to apply, in the order they are applied.""" + return sorted(name for name in os.listdir(directory) if name.endswith(".sql")) + + +def migrate(database, directory): + """Apply each pending migration; return the names applied.""" + applied = [] + with sqlite3.connect(database) as connection: + for name in pending(directory): + with open(os.path.join(directory, name), encoding="utf-8") as handle: + connection.executescript(handle.read()) + applied.append(name) + return applied \ No newline at end of file diff --git a/bench/recall/fixtures/migrun/tests/__init__.py b/bench/recall/fixtures/migrun/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/bench/recall/fixtures/migrun/tests/test_runner.py b/bench/recall/fixtures/migrun/tests/test_runner.py new file mode 100644 index 000000000..b45b57b0f --- /dev/null +++ b/bench/recall/fixtures/migrun/tests/test_runner.py @@ -0,0 +1,18 @@ +import os +import tempfile +import unittest + +from migrun.runner import migrate + +MIGRATIONS = os.path.join(os.path.dirname(__file__), "..", "migrations") + + +class FreshDatabaseTest(unittest.TestCase): + def test_every_migration_applies_to_a_fresh_database(self): + with tempfile.TemporaryDirectory() as directory: + applied = migrate(os.path.join(directory, "app.db"), MIGRATIONS) + self.assertEqual(len(applied), 10) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/bench/recall/fixtures/quotasvc/README.md b/bench/recall/fixtures/quotasvc/README.md new file mode 100644 index 000000000..6013d3ec3 --- /dev/null +++ b/bench/recall/fixtures/quotasvc/README.md @@ -0,0 +1,12 @@ +# quotasvc + +A fixed-window request limiter. + +## Configuration + +| variable | meaning | default | +| -------------- | ----------------------------------------- | ------- | +| `QUOTA_LIMIT` | requests allowed per window | `100` | +| `QUOTA_WINDOW` | length of the window, in seconds | `60` | + + python3 -m unittest discover -s tests -t . \ No newline at end of file diff --git a/bench/recall/fixtures/quotasvc/quotasvc/__init__.py b/bench/recall/fixtures/quotasvc/quotasvc/__init__.py new file mode 100644 index 000000000..5c9886a77 --- /dev/null +++ b/bench/recall/fixtures/quotasvc/quotasvc/__init__.py @@ -0,0 +1 @@ +"""A fixed-window request limiter.""" \ No newline at end of file diff --git a/bench/recall/fixtures/quotasvc/quotasvc/config.py b/bench/recall/fixtures/quotasvc/quotasvc/config.py new file mode 100644 index 000000000..06a83730b --- /dev/null +++ b/bench/recall/fixtures/quotasvc/quotasvc/config.py @@ -0,0 +1,14 @@ +"""Read the limiter's settings from the environment.""" + +DEFAULT_LIMIT = 100 +DEFAULT_WINDOW = 60_000 + + +def limit(env): + raw = env.get("QUOTA_LIMIT") + return int(raw) if raw else DEFAULT_LIMIT + + +def window(env): + raw = env.get("QUOTA_WINDOW") + return int(raw) if raw else DEFAULT_WINDOW \ No newline at end of file diff --git a/bench/recall/fixtures/quotasvc/quotasvc/limiter.py b/bench/recall/fixtures/quotasvc/quotasvc/limiter.py new file mode 100644 index 000000000..56ebe40e2 --- /dev/null +++ b/bench/recall/fixtures/quotasvc/quotasvc/limiter.py @@ -0,0 +1,24 @@ +"""Allow at most `limit` requests in each fixed window of `window_ms`.""" + + +class Limiter: + def __init__(self, limit, window_ms): + self.limit = limit + self.window_ms = window_ms + self._window_start = None + self._used = 0 + + def allow(self, now_ms): + if self._window_start is None or now_ms - self._window_start >= self.window_ms: + self._window_start = now_ms + self._used = 0 + if self._used < self.limit: + self._used += 1 + return True + return False + + +def from_env(env): + from . import config + + return Limiter(config.limit(env), config.window(env)) \ No newline at end of file diff --git a/bench/recall/fixtures/quotasvc/tests/__init__.py b/bench/recall/fixtures/quotasvc/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/bench/recall/fixtures/quotasvc/tests/test_limiter.py b/bench/recall/fixtures/quotasvc/tests/test_limiter.py new file mode 100644 index 000000000..31b70425d --- /dev/null +++ b/bench/recall/fixtures/quotasvc/tests/test_limiter.py @@ -0,0 +1,21 @@ +import unittest + +from quotasvc.limiter import Limiter, from_env + + +class LimiterTest(unittest.TestCase): + def test_limit_holds_inside_a_window(self): + limiter = Limiter(limit=3, window_ms=1000) + self.assertEqual([limiter.allow(t) for t in (0, 10, 20, 30)], [True, True, True, False]) + + def test_window_resets(self): + limiter = Limiter(limit=1, window_ms=1000) + self.assertEqual([limiter.allow(t) for t in (0, 500, 1000)], [True, False, True]) + + def test_defaults(self): + limiter = from_env({}) + self.assertEqual((limiter.limit, limiter.window_ms), (100, 60_000)) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/bench/recall/fixtures/truth/recall-migrate.json b/bench/recall/fixtures/truth/recall-migrate.json new file mode 100644 index 000000000..f206c3f04 --- /dev/null +++ b/bench/recall/fixtures/truth/recall-migrate.json @@ -0,0 +1,63 @@ +{ + "pair": "recall-migrate", + "fixture": "migrun", + "marker": "migrun/runner.py", + "derive": { + "groups": [ + [ + "lexicographic", + "alphabetical", + "as strings", + "as text", + "string order", + "string sort", + "lexical", + "not numerically" + ], + [ + "10_" + ] + ] + }, + "reuse": { + "accept": [ + "\\b10_backfill_email(?:\\.sql)?\\b" + ], + "decoys": [ + "\\b1_create_users(?:\\.sql)?\\b" + ] + }, + "reference": { + "derive": "pending() sorts the file names as strings, so 10_backfill_email.sql sorts before 1_create_users.sql and runs first, before the users table exists.", + "reuse": "10_backfill_email.sql" + }, + "decoyAnswers": [ + "1_create_users.sql", + "2_add_email.sql", + "1_create_users.sql or 10_backfill_email.sql" + ], + "absent": [ + "lexicographic", + "alphabetical", + "sorts before", + "as strings", + "10_backfill_email.sql is" + ], + "fixture_hashes": { + "migrations/10_backfill_email.sql": "067c4b59f8c0ac9cc0ff87912dd5957b477a72841b8851a628954a9329f05997", + "migrations/1_create_users.sql": "f202577af96fe5ff413f6456e176f80eb994ac7d349d89f8707fa32f79cef931", + "migrations/2_add_email.sql": "7a3a4c70d5af51f931ef9c9e1b12d7ae59d117a77416e39b47abf3041544bf39", + "migrations/3_create_orders.sql": "e39eca2d89e2212e9b6fca8d55344c2a3ed8394229c6763dc042f121b3acb927", + "migrations/4_index_orders_user.sql": "01b2714a415725a3e92af7cb65e0e79068ba1f6b9559854d53b8855043705ecc", + "migrations/5_create_products.sql": "762ecf5aff555e446eed504c7c3878f0e5f2084b79fa3cbacdf00932ba956c03", + "migrations/6_add_product_price.sql": "c5561bbeb757ed258a9c284ecf14a2085f9db2cf3a61232834fa993074bab1c9", + "migrations/7_create_order_lines.sql": "4703c49d3ccc9be96d01ca728491b106e3aacd1408c26e91dffe301590cb8f8c", + "migrations/8_seed_admin.sql": "06a434cdae0e1d462a2adec402c3ac4833608be8dc103df6f867d10115e721e5", + "migrations/9_index_lines_order.sql": "77f1869d9b43704092799765f8cc5ae08dbd1ca9ba27289dc78b901bf3fd51aa", + "migrun/__init__.py": "59560137168ce2e1b73443a6887a9383f0c55bf02b6159cf85323df44df8bf88", + "migrun/runner.py": "b10b746c5760acc3afe4ea0803785ea2868455b9cfc48af3f190358e5cf4c9c6", + "README.md": "31662aa5abf7529d393810a38366f542ff624b09c876003d89f40d1596914776", + "tests/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "tests/test_runner.py": "1dbc46f80a4bbb687c9ab6bfaea041d275cf29bb5aca5fe9d010e8d1c82589fd" + } +} diff --git a/bench/recall/fixtures/truth/recall-quota.json b/bench/recall/fixtures/truth/recall-quota.json new file mode 100644 index 000000000..2a938da18 --- /dev/null +++ b/bench/recall/fixtures/truth/recall-quota.json @@ -0,0 +1,50 @@ +{ + "pair": "recall-quota", + "fixture": "quotasvc", + "marker": "quotasvc/config.py", + "derive": { + "groups": [ + [ + "millisecond", + "window_ms" + ], + [ + "second" + ] + ] + }, + "reuse": { + "accept": [ + "(? dict: + root = FIXTURES / fixture + return { + path.relative_to(root).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest() + for path in sorted(root.rglob("*")) + if path.is_file() and "__pycache__" not in path.parts + } + + +def main(check: bool) -> int: + stale = [] + for file in sorted(TRUTH.glob("*.json")): + record = json.loads(file.read_text()) + want = hashes(record["fixture"]) + if record.get("fixture_hashes") != want: + stale.append(file.name) + if not check: + record["fixture_hashes"] = want + file.write_text(json.dumps(record, indent=2) + "\n", newline="\n") + if check and stale: + print("stale fixture hashes: " + ", ".join(stale)) + return 1 + print(("checked " if check else "wrote ") + f"{len(list(TRUTH.glob('*.json')))} truth file(s)") + return 0 + + +if __name__ == "__main__": + sys.exit(main("--check" in sys.argv[1:])) \ No newline at end of file diff --git a/bench/recall/recall.py b/bench/recall/recall.py new file mode 100644 index 000000000..3f62cdab3 --- /dev/null +++ b/bench/recall/recall.py @@ -0,0 +1,160 @@ +"""bench/recall: does what one session works out save the next one calls? + + python3 bench/recall/recall.py plan [--arms A,B] [--reps N] + python3 bench/recall/recall.py run --approve-spend + python3 bench/recall/recall.py report + +Each pair is two THOL tasks run back to back on the rig (bench/thol): `derive` +works something out that the workspace does not state, then `reuse` starts a +FRESH session in a fresh copy of the same workspace and asks a question whose +answer depends on it. The knowledge graph is the one thing that may survive +from one to the other: every (arm, pair, repetition) gets its own empty graph +directory, mounted at /proxy-graph for both sessions and for nothing else, so +nothing can leak between pairs or between arms. + +THE MEASUREMENT IS CALLS AVOIDED, COUNTED ONLY WHERE BOTH ARMS ARE RIGHT. The +control arm has no graph, so its reuse session must do the work again; an arm +that carries the conclusion should answer with fewer tool calls. A wrong reuse +answer is not a cheap one, so a pair counts toward calls avoided only for the +repetitions in which both arms scored 1. + +`plan` writes the exact docker commands and the per-session arm-order files +and spends nothing. `run` executes them and refuses without --approve-spend, +because every session is a real Claude Code run. +""" + +import argparse +import json +from contextlib import closing +import shlex +import sqlite3 +import subprocess +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent +RIG = HERE.parent / "thol" +THOL_HOME = "/home/bench/thol" +DEFAULT_ARMS = "control,token-optimizer-proxy-knowledge" + + +def pairs(): + records = ( + json.loads(p.read_text(encoding="utf-8")) + for p in (HERE / "fixtures" / "truth").glob("*.json") + ) + return sorted(records, key=lambda t: t["pair"]) + + +def session(pair, role, arm, rep, out, image, graph): + """One docker invocation of the rig's campaign mode for one session.""" + task = f"{pair['pair']}-{role}" + order = out / "orders" / f"{arm}.json" + label = f"recall {pair['pair']} {arm} rep{rep} {role}" + mounts = [ + (RIG / "auth", "/auth", "ro"), + (out / "results", "/results", "rw"), + (graph, "/proxy-graph", "rw"), + (order, "/arm-order.json", "ro"), + (HERE / "tasks" / "_recall.py", f"{THOL_HOME}/tasks/_recall.py", "ro"), + (HERE / "tasks" / task, f"{THOL_HOME}/tasks/{task}", "ro"), + (HERE / "fixtures" / pair["fixture"], f"{THOL_HOME}/fixtures/out/{pair['fixture']}", "ro"), + ( + HERE / "fixtures" / "truth" / f"{pair['pair']}.json", + f"{THOL_HOME}/fixtures/truth/{pair['pair']}.json", + "ro", + ), + ] + command = ["docker", "run", "--rm"] + for host, inside, mode in mounts: + command += ["-v", f"{host}:{inside}:{mode}"] + command += [ + "-e", "THOL_ARM_ORDER_FILE=/arm-order.json", + "-e", f"THOL_CAMPAIGN={label}", + image, "campaign", "-c", arm, "-t", task, "--reps", "1", + ] + return command + + +def plan(args): + out = Path(args.out).resolve() + arms = args.arms.split(",") + if len(set(arms)) != len(arms) or "control" not in arms: + sys.exit("recall: arms must be distinct and include control, which calls avoided is measured against") + (out / "orders").mkdir(parents=True, exist_ok=True) + (out / "results").mkdir(exist_ok=True) + for arm in arms: + (out / "orders" / f"{arm}.json").write_text( + json.dumps({"arms": [arm], "orders": [[arm]]}) + "\n", encoding="utf-8" + ) + steps = [] + for rep in range(1, args.reps + 1): + for pair in pairs(): + for arm in arms: + graph = out / "graphs" / f"{pair['pair']}-{arm}-rep{rep}" + graph.mkdir(parents=True, exist_ok=True) + for role in ("derive", "reuse"): + steps.append(session(pair, role, arm, rep, out, args.image, graph)) + (out / "plan.json").write_text(json.dumps(steps, indent=1) + "\n", encoding="utf-8") + for step in steps: + print(shlex.join(str(part) for part in step)) + print(f"\n{len(steps)} session(s) planned in {out}; nothing has run and nothing was spent.") + + +def run(args): + if not args.approve_spend: + sys.exit("recall: refusing to run without --approve-spend; every step is a real, billed Claude Code session") + out = Path(args.out).resolve() + steps = json.loads((out / "plan.json").read_text(encoding="utf-8")) + for n, step in enumerate(steps, 1): + print(f"[{n}/{len(steps)}] {step[step.index('-t') + 1]} as {step[step.index('-c') + 1]}", flush=True) + if subprocess.run(step).returncode != 0: + sys.exit(f"recall: step {n} failed; the steps before it are recorded and the plan can be re-run from it") + + +def report(args): + with closing(sqlite3.connect(args.results)) as db: + rows = db.execute( + "SELECT competitor, task, rep, score, tool_calls, num_turns, total_cost_usd FROM runs " + "WHERE task LIKE 'recall-%-reuse' AND status = 'ok'" + ).fetchall() + by = {} + for arm, task, rep, score, calls, turns, cost in rows: + by.setdefault(task, {}).setdefault(arm, {})[rep] = (score, calls, turns, cost) + print("pair arm right calls turns cost avoided") + for task in sorted(by): + control = by[task].get("control", {}) + for arm in sorted(by[task]): + runs = by[task][arm] + both = [r for r in runs if r in control and runs[r][0] == 1 and control[r][0] == 1] + avoided = ( + sum(control[r][1] - runs[r][1] for r in both) / len(both) + if both and arm != "control" else None + ) + right = sum(1 for r in runs.values() if r[0] == 1) + mean = lambda i: sum(r[i] or 0 for r in runs.values()) / len(runs) + print( + f"{task[:-6]:17} {arm:33} {right}/{len(runs):<4} {mean(1):6.1f} {mean(2):6.1f} " + f"{mean(3):6.3f} {'-' if avoided is None else f'{avoided:.1f} over {len(both)}'}" + ) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + sub = parser.add_subparsers(dest="command", required=True) + p = sub.add_parser("plan") + p.add_argument("out") + p.add_argument("--arms", default=DEFAULT_ARMS) + p.add_argument("--reps", type=int, default=3) + p.add_argument("--image", default="thol-rig:local") + r = sub.add_parser("run") + r.add_argument("out") + r.add_argument("--approve-spend", action="store_true") + q = sub.add_parser("report") + q.add_argument("results") + args = parser.parse_args() + {"plan": plan, "run": run, "report": report}[args.command](args) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/bench/recall/tasks/_recall.py b/bench/recall/tasks/_recall.py new file mode 100644 index 000000000..8ca9bfefa --- /dev/null +++ b/bench/recall/tasks/_recall.py @@ -0,0 +1,53 @@ +"""What every recall verifier does, so the six of them cannot disagree. + +A recall task is a PAIR. Session 1 (`derive`) is asked to work something out +that the workspace does not state. Session 2 (`reuse`) starts cold, in a fresh +copy of the same workspace, and is asked a question whose answer depends on +that conclusion. The pair measures whether what session 1 established reaches +session 2 -- the calls it avoids are the measurement, and a correct answer is +the precondition for counting them. + +THE ANSWER IS SCORED FROM GROUND TRUTH THE AGENT NEVER SEES (TRUTH_DIR), and a +verifier awards nothing unless three things hold first: + + - the workspace is the fixture: an empty or missing workspace scores 0, so a + session that never ran cannot pass on a lucky reply; + - no fixture file was changed: both prompts say not to, and a session that + "answers" by rewriting the code has answered a different question; + - there is a reply at all. + +A `reuse` answer that names the right value AND a decoy is a hedge, and scores +0: a list of candidates is not a conclusion. +""" + +import re + +from _vlib import WS, answer, emit, hashes_intact, truth + + +def verify(pair: str, role: str) -> None: + t = truth(f"{pair}.json") + if not (WS / t["marker"]).is_file(): + emit(0.0, f"the workspace is not the {t['fixture']} fixture (no {t['marker']})") + if not hashes_intact(WS, t["fixture_hashes"]): + emit(0.0, "a fixture file was changed or deleted; the prompt says to change none") + text = answer().strip() + if not text: + emit(0.0, "no final reply") + if role == "derive": + missing = [ + group for group in t["derive"]["groups"] + if not any(term.lower() in text.lower() for term in group) + ] + if missing: + emit(0.0, "the explanation names none of: " + "; ".join(" | ".join(g) for g in missing)) + emit(1.0, "the root cause is named") + if role == "reuse": + right = any(re.search(p, text, re.IGNORECASE) for p in t["reuse"]["accept"]) + hedge = [p for p in t["reuse"]["decoys"] if re.search(p, text, re.IGNORECASE)] + if right and not hedge: + emit(1.0, "the answer depends on the derived conclusion and is correct") + if right: + emit(0.0, "the right answer was given alongside a wrong one: " + ", ".join(hedge)) + emit(0.0, "not the answer the derived conclusion implies") + emit(0.0, f"unknown role {role!r}") \ No newline at end of file diff --git a/bench/recall/tasks/_vlib.py b/bench/recall/tasks/_vlib.py new file mode 100644 index 000000000..acfe8239c --- /dev/null +++ b/bench/recall/tasks/_vlib.py @@ -0,0 +1,56 @@ +"""The THOL verifier protocol, for running these verifiers outside the rig. + +THOL (github.com/pi-infected/token-harness-optimizer-leaderboard, MIT) runs a +task's verifier as `python3 verify.py` with three environment variables: + + WORKSPACE - the run workspace, after the session + ANSWER_FILE - a file holding the agent's final reply + TRUTH_DIR - fixtures/truth, which the agent never sees + +and reads `score=<0..1>` from its output. Inside the rig the verifiers import +THOL's own `_vlib.py`, which sits beside every task there; this file is never +mounted into the rig. It exists so `test_recall.py` can run every verifier +offline against the same contract, and it implements only the part of that +contract these verifiers use. +""" + +import hashlib +import json +import os +import sys +from pathlib import Path + +WS = Path(os.environ["WORKSPACE"]) +TRUTH_DIR = Path(os.environ["TRUTH_DIR"]) +ANSWER_FILE = os.environ.get("ANSWER_FILE", "") + + +def answer() -> str: + try: + return Path(ANSWER_FILE).read_text() + except OSError: + return "" + + +def truth(name: str): + return json.loads((TRUTH_DIR / name).read_text()) + + +def emit(score: float, detail: str = ""): + print(f"score={max(0.0, min(1.0, score)):.4f}") + if detail: + for line in detail.splitlines(): + print(f"detail={line}") + sys.exit(0) + + +def sha(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def hashes_intact(root: Path, expected: dict) -> bool: + for rel, h in expected.items(): + p = root / rel + if not p.exists() or sha(p) != h: + return False + return True \ No newline at end of file diff --git a/bench/recall/tasks/recall-migrate-derive/prompt.md b/bench/recall/tasks/recall-migrate-derive/prompt.md new file mode 100644 index 000000000..50dd43e29 --- /dev/null +++ b/bench/recall/tasks/recall-migrate-derive/prompt.md @@ -0,0 +1,6 @@ +Applying the migrations in the project in the current directory to a fresh +database fails (`python3 -m unittest discover -s tests -t .` shows it). + +Find the root cause. Do not change any file. + +In your final reply, state the root cause in one or two sentences. \ No newline at end of file diff --git a/bench/recall/tasks/recall-migrate-derive/task.json b/bench/recall/tasks/recall-migrate-derive/task.json new file mode 100644 index 000000000..d5646da42 --- /dev/null +++ b/bench/recall/tasks/recall-migrate-derive/task.json @@ -0,0 +1 @@ +{"id":"recall-migrate-derive","title":"recall: migration order (derive)","kind":"code","workspace":{"fixture":"migrun"},"max_turns":30,"timeout_s":900,"success_threshold":1.0} diff --git a/bench/recall/tasks/recall-migrate-derive/verify.py b/bench/recall/tasks/recall-migrate-derive/verify.py new file mode 100644 index 000000000..d5ee4a808 --- /dev/null +++ b/bench/recall/tasks/recall-migrate-derive/verify.py @@ -0,0 +1,7 @@ +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from _recall import verify # noqa: E402 + +verify("recall-migrate", "derive") \ No newline at end of file diff --git a/bench/recall/tasks/recall-migrate-reuse/prompt.md b/bench/recall/tasks/recall-migrate-reuse/prompt.md new file mode 100644 index 000000000..4c2ffb820 --- /dev/null +++ b/bench/recall/tasks/recall-migrate-reuse/prompt.md @@ -0,0 +1,5 @@ +With the project in the current directory exactly as it is now, and with no +file changed, a fresh database is migrated. + +Reply with the file name of the migration that is applied first, and nothing +else. \ No newline at end of file diff --git a/bench/recall/tasks/recall-migrate-reuse/task.json b/bench/recall/tasks/recall-migrate-reuse/task.json new file mode 100644 index 000000000..ba21661f8 --- /dev/null +++ b/bench/recall/tasks/recall-migrate-reuse/task.json @@ -0,0 +1 @@ +{"id":"recall-migrate-reuse","title":"recall: migration order (reuse)","kind":"code","workspace":{"fixture":"migrun"},"max_turns":30,"timeout_s":900,"success_threshold":1.0} diff --git a/bench/recall/tasks/recall-migrate-reuse/verify.py b/bench/recall/tasks/recall-migrate-reuse/verify.py new file mode 100644 index 000000000..6dc68a0f0 --- /dev/null +++ b/bench/recall/tasks/recall-migrate-reuse/verify.py @@ -0,0 +1,7 @@ +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from _recall import verify # noqa: E402 + +verify("recall-migrate", "reuse") \ No newline at end of file diff --git a/bench/recall/tasks/recall-quota-derive/prompt.md b/bench/recall/tasks/recall-quota-derive/prompt.md new file mode 100644 index 000000000..2abacd859 --- /dev/null +++ b/bench/recall/tasks/recall-quota-derive/prompt.md @@ -0,0 +1,7 @@ +Operators of the service in the current directory set `QUOTA_WINDOW=60`, +expecting a one-minute window, and clients get roughly a thousand times more +requests through than `QUOTA_LIMIT` allows. + +Find the root cause. Do not change any file. + +In your final reply, state the root cause in one or two sentences. \ No newline at end of file diff --git a/bench/recall/tasks/recall-quota-derive/task.json b/bench/recall/tasks/recall-quota-derive/task.json new file mode 100644 index 000000000..f3e172d23 --- /dev/null +++ b/bench/recall/tasks/recall-quota-derive/task.json @@ -0,0 +1 @@ +{"id":"recall-quota-derive","title":"recall: request window units (derive)","kind":"code","workspace":{"fixture":"quotasvc"},"max_turns":30,"timeout_s":900,"success_threshold":1.0} diff --git a/bench/recall/tasks/recall-quota-derive/verify.py b/bench/recall/tasks/recall-quota-derive/verify.py new file mode 100644 index 000000000..10f269c57 --- /dev/null +++ b/bench/recall/tasks/recall-quota-derive/verify.py @@ -0,0 +1,7 @@ +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from _recall import verify # noqa: E402 + +verify("recall-quota", "derive") \ No newline at end of file diff --git a/bench/recall/tasks/recall-quota-reuse/prompt.md b/bench/recall/tasks/recall-quota-reuse/prompt.md new file mode 100644 index 000000000..e876defbf --- /dev/null +++ b/bench/recall/tasks/recall-quota-reuse/prompt.md @@ -0,0 +1,4 @@ +On the service in the current directory, exactly as it is now and with no file +changed, an operator wants each request window to last 90 seconds. + +Reply with the exact value they must give `QUOTA_WINDOW`, and nothing else. \ No newline at end of file diff --git a/bench/recall/tasks/recall-quota-reuse/task.json b/bench/recall/tasks/recall-quota-reuse/task.json new file mode 100644 index 000000000..122b1ddda --- /dev/null +++ b/bench/recall/tasks/recall-quota-reuse/task.json @@ -0,0 +1 @@ +{"id":"recall-quota-reuse","title":"recall: request window units (reuse)","kind":"code","workspace":{"fixture":"quotasvc"},"max_turns":30,"timeout_s":900,"success_threshold":1.0} diff --git a/bench/recall/tasks/recall-quota-reuse/verify.py b/bench/recall/tasks/recall-quota-reuse/verify.py new file mode 100644 index 000000000..244d0be2b --- /dev/null +++ b/bench/recall/tasks/recall-quota-reuse/verify.py @@ -0,0 +1,7 @@ +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from _recall import verify # noqa: E402 + +verify("recall-quota", "reuse") \ No newline at end of file diff --git a/bench/recall/tasks/recall-tz-derive/prompt.md b/bench/recall/tasks/recall-tz-derive/prompt.md new file mode 100644 index 000000000..3dee7b15a --- /dev/null +++ b/bench/recall/tasks/recall-tz-derive/prompt.md @@ -0,0 +1,7 @@ +The test suite of the Python project in the current directory +(`python3 -m unittest discover -s tests -t .`) passes on some machines and +fails on others, with exactly the same code and the same Python version. + +Find the root cause. Do not change any file. + +In your final reply, state the root cause in one or two sentences. \ No newline at end of file diff --git a/bench/recall/tasks/recall-tz-derive/task.json b/bench/recall/tasks/recall-tz-derive/task.json new file mode 100644 index 000000000..703af9d1a --- /dev/null +++ b/bench/recall/tasks/recall-tz-derive/task.json @@ -0,0 +1 @@ +{"id":"recall-tz-derive","title":"recall: timezone-dependent day buckets (derive)","kind":"code","workspace":{"fixture":"tzbook"},"max_turns":30,"timeout_s":900,"success_threshold":1.0} diff --git a/bench/recall/tasks/recall-tz-derive/verify.py b/bench/recall/tasks/recall-tz-derive/verify.py new file mode 100644 index 000000000..39671738c --- /dev/null +++ b/bench/recall/tasks/recall-tz-derive/verify.py @@ -0,0 +1,7 @@ +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from _recall import verify # noqa: E402 + +verify("recall-tz", "derive") \ No newline at end of file diff --git a/bench/recall/tasks/recall-tz-reuse/prompt.md b/bench/recall/tasks/recall-tz-reuse/prompt.md new file mode 100644 index 000000000..a98da62b6 --- /dev/null +++ b/bench/recall/tasks/recall-tz-reuse/prompt.md @@ -0,0 +1,6 @@ +The CI machines for the Python project in the current directory must run its +test suite (`python3 -m unittest discover -s tests -t .`) so that it passes on +every machine, without any file being changed. + +Reply with the single environment variable assignment CI should set, in the +form NAME=value, and nothing else. \ No newline at end of file diff --git a/bench/recall/tasks/recall-tz-reuse/task.json b/bench/recall/tasks/recall-tz-reuse/task.json new file mode 100644 index 000000000..bc36d0d11 --- /dev/null +++ b/bench/recall/tasks/recall-tz-reuse/task.json @@ -0,0 +1 @@ +{"id":"recall-tz-reuse","title":"recall: timezone-dependent day buckets (reuse)","kind":"code","workspace":{"fixture":"tzbook"},"max_turns":30,"timeout_s":900,"success_threshold":1.0} diff --git a/bench/recall/tasks/recall-tz-reuse/verify.py b/bench/recall/tasks/recall-tz-reuse/verify.py new file mode 100644 index 000000000..c3eb9dc23 --- /dev/null +++ b/bench/recall/tasks/recall-tz-reuse/verify.py @@ -0,0 +1,7 @@ +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from _recall import verify # noqa: E402 + +verify("recall-tz", "reuse") \ No newline at end of file diff --git a/bench/recall/test_recall.py b/bench/recall/test_recall.py new file mode 100644 index 000000000..5e51630c1 --- /dev/null +++ b/bench/recall/test_recall.py @@ -0,0 +1,271 @@ +"""Offline self-test of bench/recall: no session is run and nothing is spent. + + python3 -m unittest discover -s bench/recall -p "test_*.py" + +A recall measurement is only as good as its verifiers, and a verifier that is +wrong is wrong silently: it scores real sessions, nobody re-reads the replies, +and a benchmark that passes everything or nothing looks exactly like one that +works. So every verifier is exercised here against answers whose score is +known before it runs, and every fixture is checked to carry the defect its +pair is about. +""" + +import importlib.util +import json +import os +import shutil +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +HERE = Path(__file__).resolve().parent +TASKS = HERE / "tasks" +FIXTURES = HERE / "fixtures" +TRUTH = FIXTURES / "truth" +PAIRS = sorted(json.loads(p.read_text())["pair"] for p in TRUTH.glob("*.json")) + + +def record(pair): + return json.loads((TRUTH / f"{pair}.json").read_text()) + + +def score(pair, role, reply, workspace): + with tempfile.TemporaryDirectory() as scratch: + answer = Path(scratch) / "answer.txt" + answer.write_text(reply) + run = subprocess.run( + [sys.executable, str(TASKS / f"{pair}-{role}" / "verify.py")], + env={**os.environ, "WORKSPACE": str(workspace), "ANSWER_FILE": str(answer), "TRUTH_DIR": str(TRUTH)}, + capture_output=True, + text=True, + timeout=60, + ) + if run.returncode != 0: + raise AssertionError(f"verifier broke: {run.stderr}") + line = next(l for l in run.stdout.splitlines() if l.startswith("score=")) + return float(line.split("=", 1)[1]) + + +class Workspace: + """A fresh copy of a pair's fixture, as THOL hands one to each session.""" + + def __init__(self, pair): + self.fixture = record(pair)["fixture"] + + def __enter__(self): + self.tmp = tempfile.TemporaryDirectory() + self.path = Path(self.tmp.name) / "ws" + shutil.copytree(FIXTURES / self.fixture, self.path, ignore=shutil.ignore_patterns("__pycache__")) + return self.path + + def __exit__(self, *exc): + self.tmp.cleanup() + + +class TheTasksAreWellFormed(unittest.TestCase): + def test_every_pair_has_both_halves(self): + self.assertEqual(len(PAIRS), 3) + for pair in PAIRS: + for role in ("derive", "reuse"): + task = json.loads((TASKS / f"{pair}-{role}" / "task.json").read_text()) + self.assertEqual(task["id"], f"{pair}-{role}") + self.assertEqual(task["workspace"]["fixture"], record(pair)["fixture"]) + self.assertTrue((FIXTURES / task["workspace"]["fixture"]).is_dir()) + + def test_the_fixture_hashes_are_current(self): + spec = importlib.util.spec_from_file_location("hash_fixtures", HERE / "hash_fixtures.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + self.assertEqual(module.main(check=True), 0) + + +class TheConclusionIsNotInTheWorkspace(unittest.TestCase): + """The reuse answer must come from session 1 or from doing the work again.""" + + def test_no_workspace_file_or_prompt_states_it(self): + for pair in PAIRS: + t = record(pair) + texts = { + str(p.relative_to(FIXTURES)): p.read_text(encoding="utf-8").lower() + for p in (FIXTURES / t["fixture"]).rglob("*") + if p.is_file() and "__pycache__" not in p.parts + } + for role in ("derive", "reuse"): + texts[f"{pair}-{role}/prompt.md"] = (TASKS / f"{pair}-{role}" / "prompt.md").read_text(encoding="utf-8").lower() + for phrase in t["absent"]: + for name, text in texts.items(): + self.assertNotIn(phrase.lower(), text, f"{pair}: {name} states {phrase!r}") + + +class TheVerifiersScoreWhatTheySay(unittest.TestCase): + def test_the_reference_answers_score_full_credit(self): + for pair in PAIRS: + t = record(pair) + with Workspace(pair) as ws: + self.assertEqual(score(pair, "derive", t["reference"]["derive"], ws), 1.0, pair) + self.assertEqual(score(pair, "reuse", t["reference"]["reuse"], ws), 1.0, pair) + + def test_an_empty_reply_scores_nothing(self): + for pair in PAIRS: + with Workspace(pair) as ws: + for role in ("derive", "reuse"): + self.assertEqual(score(pair, role, " \n", ws), 0.0, f"{pair}-{role}") + + def test_wrong_and_hedged_answers_score_nothing(self): + for pair in PAIRS: + with Workspace(pair) as ws: + for wrong in record(pair)["decoyAnswers"]: + self.assertEqual(score(pair, "reuse", wrong, ws), 0.0, f"{pair}: {wrong!r}") + + def test_an_empty_workspace_scores_nothing(self): + for pair in PAIRS: + t = record(pair) + with tempfile.TemporaryDirectory() as empty: + for role in ("derive", "reuse"): + self.assertEqual(score(pair, role, t["reference"][role], empty), 0.0, f"{pair}-{role}") + + def test_a_changed_fixture_scores_nothing(self): + for pair in PAIRS: + t = record(pair) + with Workspace(pair) as ws: + marker = ws / t["marker"] + marker.write_text(marker.read_text() + "\n# edited\n") + for role in ("derive", "reuse"): + self.assertEqual(score(pair, role, t["reference"][role], ws), 0.0, f"{pair}-{role}") + + def test_a_derive_reply_that_misses_the_cause_scores_nothing(self): + vague = "The tests fail because of a bug in the code; it needs fixing." + for pair in PAIRS: + with Workspace(pair) as ws: + self.assertEqual(score(pair, "derive", vague, ws), 0.0, pair) + + +class TheFixturesCarryTheirDefect(unittest.TestCase): + def test_migrations_sort_as_strings(self): + sys.path.insert(0, str(FIXTURES / "migrun")) + try: + from migrun.runner import pending + + self.assertEqual(pending(FIXTURES / "migrun" / "migrations")[0], "10_backfill_email.sql") + finally: + sys.path.remove(str(FIXTURES / "migrun")) + with Workspace("recall-migrate") as ws: + suite = subprocess.run( + [sys.executable, "-m", "unittest", "discover", "-s", "tests", "-t", "."], + cwd=ws, capture_output=True, text=True, timeout=120, + ) + self.assertNotEqual(suite.returncode, 0, "a fresh database must fail to migrate") + + def test_the_window_is_read_in_milliseconds(self): + sys.path.insert(0, str(FIXTURES / "quotasvc")) + try: + from quotasvc.limiter import from_env + + self.assertEqual(from_env({"QUOTA_WINDOW": "60"}).window_ms, 60) + finally: + sys.path.remove(str(FIXTURES / "quotasvc")) + self.assertIn("in seconds", (FIXTURES / "quotasvc" / "README.md").read_text()) + + @unittest.skipIf(not hasattr(__import__("time"), "tzset"), "time.tzset is POSIX-only") + def test_the_days_depend_on_the_local_time_zone(self): + def suite(zone): + with Workspace("recall-tz") as ws: + return subprocess.run( + [sys.executable, "-m", "unittest", "discover", "-s", "tests", "-t", "."], + cwd=ws, env={**os.environ, "TZ": zone}, capture_output=True, text=True, timeout=120, + ).returncode + + self.assertEqual(suite("UTC"), 0) + self.assertNotEqual(suite("Pacific/Honolulu"), 0) + + +def runner(): + spec = importlib.util.spec_from_file_location("recall", HERE / "recall.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class TheRunnerKeepsThePairsApart(unittest.TestCase): + """Nothing may reach a reuse session except what its own derive session left.""" + + def plan(self, out, reps=2): + import argparse + import contextlib + import io + + with contextlib.redirect_stdout(io.StringIO()): + runner().plan(argparse.Namespace(out=str(out), arms="control,token-optimizer-proxy-knowledge", reps=reps, image="img")) + return json.loads((Path(out) / "plan.json").read_text(encoding="utf-8")) + + def test_each_pair_arm_and_repetition_has_its_own_graph_shared_by_its_two_sessions(self): + with tempfile.TemporaryDirectory() as out: + steps = self.plan(out) + self.assertEqual(len(steps), 3 * 2 * 2 * 2) + graphs = {} + for step in steps: + graph = next(v.split(":/proxy-graph:")[0] for v in step if ":/proxy-graph:" in v) + task = step[step.index("-t") + 1] + graphs.setdefault(graph, []).append(task) + self.assertEqual(len(graphs), 3 * 2 * 2) + for tasks in graphs.values(): + self.assertEqual(len(tasks), 2) + self.assertTrue(tasks[0].endswith("-derive") and tasks[1].endswith("-reuse")) + self.assertEqual(tasks[0][: -len("-derive")], tasks[1][: -len("-reuse")]) + + def test_a_session_sees_only_its_own_task_fixture_and_truth(self): + with tempfile.TemporaryDirectory() as out: + steps = self.plan(out, reps=1) + for step in steps: + task = step[step.index("-t") + 1] + pair = task.rsplit("-", 1)[0] + mounts = [v for i, v in enumerate(step) if i and step[i - 1] == "-v"] + tasks = [m for m in mounts if "/tasks/recall-" in m] + truths = [m for m in mounts if "/fixtures/truth/" in m] + self.assertEqual(len(tasks), 1, task) + self.assertIn(f"/tasks/{task}:", tasks[0]) + self.assertEqual(len(truths), 1, task) + self.assertIn(f"/fixtures/truth/{pair}.json:", truths[0]) + + def test_running_without_approval_is_refused(self): + import argparse + + with tempfile.TemporaryDirectory() as out: + self.plan(out, reps=1) + with self.assertRaises(SystemExit) as refused: + runner().run(argparse.Namespace(out=out, approve_spend=False)) + self.assertIn("approve-spend", str(refused.exception)) + + def test_calls_avoided_counts_only_repetitions_both_arms_got_right(self): + import argparse + import contextlib + import io + import sqlite3 + + with tempfile.TemporaryDirectory() as out: + db = Path(out) / "results.sqlite" + with contextlib.closing(sqlite3.connect(db)) as c, c: + c.execute( + "CREATE TABLE runs (competitor TEXT, task TEXT, rep INTEGER, score REAL, " + "tool_calls INTEGER, num_turns INTEGER, total_cost_usd REAL, status TEXT)" + ) + rows = [ + ("control", "recall-tz-reuse", 1, 1.0, 12, 8, 0.2, "ok"), + ("control", "recall-tz-reuse", 2, 1.0, 10, 7, 0.2, "ok"), + ("knowledge", "recall-tz-reuse", 1, 1.0, 2, 2, 0.05, "ok"), + # Cheap and wrong: must not count as calls avoided. + ("knowledge", "recall-tz-reuse", 2, 0.0, 1, 1, 0.01, "ok"), + ] + c.executemany("INSERT INTO runs VALUES (?,?,?,?,?,?,?,?)", rows) + buffer = io.StringIO() + with contextlib.redirect_stdout(buffer): + runner().report(argparse.Namespace(results=str(db))) + line = next(l for l in buffer.getvalue().splitlines() if "knowledge" in l) + self.assertIn("10.0 over 1", line) + self.assertIn("1/2", line) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file From e7161d1bed77055bc443f0cc7a84a503b63e32dd Mon Sep 17 00:00:00 2001 From: Franklin Moormann Date: Sat, 17 Oct 2026 20:58:06 -0400 Subject: [PATCH 2/5] perf(hooks): deliver the wiki_write guidance after real work, not at session start M5 of #357: fixed per-session context dominates a small session's cost. The "Record what you work out" guidance was the largest part of the SessionStart policy block -- 1,579 characters, 341 tokens (cl100k) of a ~511-token block -- and every session paid for it, including the ones that never do anything worth recording. It is now delivered once, as post-tool context, on the first edit or after GUIDANCE_AFTER_CALLS (10) tool calls, whichever comes first, and only where wiki_write is available. The SessionStart block is 170 tokens. - the wording is byte-identical (moved into recordingGuidance()), because it was tuned by measurement (0.3 vs 0.9 on the last-message rule). - `recordingGuided` is a persisted state field, in emptyState, loadState and saveState's merge, set-once like recordingNudged. - tests: the guidance-content assertions read recordingGuidance(); assist mode checks it is absent at start, delivered on the first edit, not repeated, and delivered after ten calls with no edit; the codex lifecycle test now expects it on the first edit. assist-mode.test.mjs gets its own state directory -- it shared the machine's, so one run inherited another's "already said" flags. - 22 vendored hook copies resynced (npm run sync:hooks). Not measured end to end: a THOL campaign is the measurement and needs spend approval. Hooks suite: 125 suites, 2393 passed. Co-Authored-By: Claude Opus 5.5 --- hooks-core/adapter.mjs | 48 ++++++++++--- hooks-core/policy.mjs | 8 +++ .../hooks/token-optimizer/lib/adapter.mjs | 48 ++++++++++--- .../hooks/token-optimizer/lib/policy.mjs | 8 +++ integrations/codex/hooks/lib/adapter.mjs | 48 ++++++++++--- integrations/codex/hooks/lib/policy.mjs | 8 +++ .../codex/plugin/hooks/lib/adapter.mjs | 48 ++++++++++--- .../codex/plugin/hooks/lib/policy.mjs | 8 +++ .../copilot/.github/hooks/lib/adapter.mjs | 48 ++++++++++--- .../copilot/.github/hooks/lib/policy.mjs | 8 +++ integrations/cursor/hooks/lib/adapter.mjs | 48 ++++++++++--- integrations/cursor/hooks/lib/policy.mjs | 8 +++ integrations/gemini/hooks/lib/adapter.mjs | 48 ++++++++++--- integrations/gemini/hooks/lib/policy.mjs | 8 +++ integrations/kilo/hooks/lib/adapter.mjs | 48 ++++++++++--- integrations/kilo/hooks/lib/policy.mjs | 8 +++ integrations/opencode/hooks/lib/adapter.mjs | 48 ++++++++++--- integrations/opencode/hooks/lib/policy.mjs | 8 +++ integrations/qwen/hooks/lib/adapter.mjs | 48 ++++++++++--- integrations/qwen/hooks/lib/policy.mjs | 8 +++ integrations/windsurf/hooks/lib/adapter.mjs | 48 ++++++++++--- integrations/windsurf/hooks/lib/policy.mjs | 8 +++ plugin/hooks/lib/adapter.mjs | 48 ++++++++++--- plugin/hooks/lib/policy.mjs | 8 +++ tests/hooks/assist-mode.test.mjs | 71 +++++++++++++++++-- tests/hooks/codex-semantic-harvest.test.mjs | 9 ++- .../hooks/mcp-capability-negotiation.test.mjs | 12 +++- tests/hooks/policy-asks-for-findings.test.mjs | 34 +++++---- 28 files changed, 645 insertions(+), 153 deletions(-) diff --git a/hooks-core/adapter.mjs b/hooks-core/adapter.mjs index ebcbf278e..ca459ba77 100644 --- a/hooks-core/adapter.mjs +++ b/hooks-core/adapter.mjs @@ -890,6 +890,21 @@ function gradeObservedTools(state, event, payload, raw) { } /** The session-start notice, shared verbatim so no client drifts its own copy. */ +/** Tool calls after which a session that has edited nothing still gets the recording guidance. */ +export const GUIDANCE_AFTER_CALLS = 10; + +/** + * The wiki_write guidance. It was part of the SessionStart block until #357 + * (M5): it is the largest part of that block, and it only matters once there is + * something to record, so it is now delivered once, on the first post-tool + * event after the session shows real work, and a session that never gets that + * far never pays for it. The wording is unchanged, because it was measured -- + * see the last paragraph. + */ +export function recordingGuidance() { + return `## Record what you work out\n\nCall wiki_write when you establish something durable about this project, while\nyou still hold the context. Every claim needs at least one anchor -- a real file\npath, or path#symbol -- because an unanchored claim can never be checked against\nthe code again and would be served as current forever; unanchored writes are\nrefused.\n\nWorth recording: a decision and why the alternative was rejected, a failure and\nwhat actually caused it, a command that turned out to be the one that works.\nNot worth recording: what the code plainly says. Every wiki_write must include\nthe concrete evidence, when it applies, a calibrated confidence label, its\nscope, and what would invalidate it. Prefer the thing someone had to work out,\nbecause that exists nowhere in the source tree.\n\nRECORD BEFORE YOU ANSWER, NEVER AFTER. Your final message must be the answer to\nthe task -- never a confirmation that you saved a note. Measured: a run whose\nlast message was "Recorded. The finding captured: ..." scored 0.3 where the\nidentical task answered directly scored 0.9. The work had been done correctly\nand the artifact was written; only the last message was wrong, and the last\nmessage is what a verifier, a reviewer and the person who asked all read.${extractionNotice()}`; +} + export function policyText( canDeny, availableTools = undefined, @@ -970,16 +985,10 @@ export function policyText( ] .filter(Boolean) .join(' '); - const recording = tools.has('wiki_write') - ? `\n\n## Record what you work out\n\nCall wiki_write when you establish something durable about this project, while\nyou still hold the context. Every claim needs at least one anchor -- a real file\npath, or path#symbol -- because an unanchored claim can never be checked against\nthe code again and would be served as current forever; unanchored writes are\nrefused.\n\nWorth recording: a decision and why the alternative was rejected, a failure and\nwhat actually caused it, a command that turned out to be the one that works.\nNot worth recording: what the code plainly says. Every wiki_write must include\nthe concrete evidence, when it applies, a calibrated confidence label, its\nscope, and what would invalidate it. Prefer the thing someone had to work out,\nbecause that exists nowhere in the source tree.\n\nRECORD BEFORE YOU ANSWER, NEVER AFTER. Your final message must be the answer to\nthe task -- never a confirmation that you saved a note. Measured: a run whose\nlast message was "Recorded. The finding captured: ..." scored 0.3 where the\nidentical task answered directly scored 0.9. The work had been done correctly\nand the artifact was written; only the last message was wrong, and the last\nmessage is what a verifier, a reviewer and the person who asked all read.${extractionNotice()}` - // WAS 158 CHARACTERS OF SELF-DESCRIPTION. "Structural graph capture remains - // active through lifecycle hooks. Durable semantic MCP writes are not - // requested because the writer schema is not proven available." -- true, - // and it changed no decision the model ever made with it. Fixed context is - // charged on every session; it has to earn its place by altering behaviour. - : ''; - - return `# Token optimization is active\n\n${outputDiscipline()}\n\n${connection}${routing}${enforcement}${utilities ? `\n\n${utilities}` : ''}${recording}${projectBriefing()}`; + // THE RECORDING GUIDANCE IS DELIVERED LATER, NOT HERE (#357, M5): see + // recordingGuidance. A session that never does real work never pays for it. + + return `# Token optimization is active\n\n${outputDiscipline()}\n\n${connection}${routing}${enforcement}${utilities ? `\n\n${utilities}` : ''}${projectBriefing()}`; } /** @@ -1868,9 +1877,26 @@ async function runHook(clientName, event, invocation) { // has already run) either way. Say it once per session per kind -- see // adviseOnce -- rather than on every matching call. const advisory = verdict && adviseOnce(state, verdict) ? verdict.reason : null; + // THE RECORDING GUIDANCE, ONCE, AFTER REAL WORK (#357, M5). It used to ride in + // every SessionStart, where it is the largest fixed cost a small session pays + // and buys nothing until there is something to record. The first edit is the + // clearest sign there will be; GUIDANCE_AFTER_CALLS tool calls is the fallback + // for an investigation that edits nothing but still works things out. + const guided = + event === 'post-tool' && + !state.recordingGuided && + toolEvidence.names.has('wiki_write') && + ((Number(state.edits) || 0) >= 1 || + (Number(state.toolCalls) || 0) >= GUIDANCE_AFTER_CALLS); + if (guided) state.recordingGuided = true; saveState(payload.session_id, state, agentScope); - const context = [advisory, graphContext, recordingContext] + const context = [ + advisory, + graphContext, + guided ? recordingGuidance() : null, + recordingContext, + ] .filter(Boolean) .join('\n\n'); if (context) { diff --git a/hooks-core/policy.mjs b/hooks-core/policy.mjs index 0c72a950d..64014cb5a 100644 --- a/hooks-core/policy.mjs +++ b/hooks-core/policy.mjs @@ -392,6 +392,9 @@ function emptyState() { editedFiles: [], harvestedEdits: 0, recordingNudged: false, + // The recording guidance has been delivered (#357, M5); see + // recordingGuidance in adapter.mjs. + recordingGuided: false, // Facts the search advisory has already delivered this session. Without // this in the default, loadState returned an object with no `advised`, the // router rebuilt an empty Set on every hook process, and the same advisory @@ -485,6 +488,7 @@ export function loadState(sessionId, agent) { ? parsed.harvestedEdits : 0, recordingNudged: parsed.recordingNudged === true, + recordingGuided: parsed.recordingGuided === true, // Every hook call is a new process, so a field left out of this list is // forgotten between calls: that is how "said once" kept being said on // every search, and how a repeated WebFetch was never recognised @@ -649,6 +653,10 @@ export function saveState(sessionId, state, agent) { recordingNudged: Boolean( current.recordingNudged || state.recordingNudged ), + // Same rule: guidance said once stays said. + recordingGuided: Boolean( + current.recordingGuided || state.recordingGuided + ), // UNION, AND CAPPED. Two hook processes can each deliver an advisory // before either writes, so last-writer-wins would forget one and repeat // it. A fact once told stays told -- that is the whole point of the set diff --git a/integrations/cline/hooks/token-optimizer/lib/adapter.mjs b/integrations/cline/hooks/token-optimizer/lib/adapter.mjs index 89eea2645..a75f15023 100644 --- a/integrations/cline/hooks/token-optimizer/lib/adapter.mjs +++ b/integrations/cline/hooks/token-optimizer/lib/adapter.mjs @@ -892,6 +892,21 @@ function gradeObservedTools(state, event, payload, raw) { } /** The session-start notice, shared verbatim so no client drifts its own copy. */ +/** Tool calls after which a session that has edited nothing still gets the recording guidance. */ +export const GUIDANCE_AFTER_CALLS = 10; + +/** + * The wiki_write guidance. It was part of the SessionStart block until #357 + * (M5): it is the largest part of that block, and it only matters once there is + * something to record, so it is now delivered once, on the first post-tool + * event after the session shows real work, and a session that never gets that + * far never pays for it. The wording is unchanged, because it was measured -- + * see the last paragraph. + */ +export function recordingGuidance() { + return `## Record what you work out\n\nCall wiki_write when you establish something durable about this project, while\nyou still hold the context. Every claim needs at least one anchor -- a real file\npath, or path#symbol -- because an unanchored claim can never be checked against\nthe code again and would be served as current forever; unanchored writes are\nrefused.\n\nWorth recording: a decision and why the alternative was rejected, a failure and\nwhat actually caused it, a command that turned out to be the one that works.\nNot worth recording: what the code plainly says. Every wiki_write must include\nthe concrete evidence, when it applies, a calibrated confidence label, its\nscope, and what would invalidate it. Prefer the thing someone had to work out,\nbecause that exists nowhere in the source tree.\n\nRECORD BEFORE YOU ANSWER, NEVER AFTER. Your final message must be the answer to\nthe task -- never a confirmation that you saved a note. Measured: a run whose\nlast message was "Recorded. The finding captured: ..." scored 0.3 where the\nidentical task answered directly scored 0.9. The work had been done correctly\nand the artifact was written; only the last message was wrong, and the last\nmessage is what a verifier, a reviewer and the person who asked all read.${extractionNotice()}`; +} + export function policyText( canDeny, availableTools = undefined, @@ -972,16 +987,10 @@ export function policyText( ] .filter(Boolean) .join(' '); - const recording = tools.has('wiki_write') - ? `\n\n## Record what you work out\n\nCall wiki_write when you establish something durable about this project, while\nyou still hold the context. Every claim needs at least one anchor -- a real file\npath, or path#symbol -- because an unanchored claim can never be checked against\nthe code again and would be served as current forever; unanchored writes are\nrefused.\n\nWorth recording: a decision and why the alternative was rejected, a failure and\nwhat actually caused it, a command that turned out to be the one that works.\nNot worth recording: what the code plainly says. Every wiki_write must include\nthe concrete evidence, when it applies, a calibrated confidence label, its\nscope, and what would invalidate it. Prefer the thing someone had to work out,\nbecause that exists nowhere in the source tree.\n\nRECORD BEFORE YOU ANSWER, NEVER AFTER. Your final message must be the answer to\nthe task -- never a confirmation that you saved a note. Measured: a run whose\nlast message was "Recorded. The finding captured: ..." scored 0.3 where the\nidentical task answered directly scored 0.9. The work had been done correctly\nand the artifact was written; only the last message was wrong, and the last\nmessage is what a verifier, a reviewer and the person who asked all read.${extractionNotice()}` - // WAS 158 CHARACTERS OF SELF-DESCRIPTION. "Structural graph capture remains - // active through lifecycle hooks. Durable semantic MCP writes are not - // requested because the writer schema is not proven available." -- true, - // and it changed no decision the model ever made with it. Fixed context is - // charged on every session; it has to earn its place by altering behaviour. - : ''; - - return `# Token optimization is active\n\n${outputDiscipline()}\n\n${connection}${routing}${enforcement}${utilities ? `\n\n${utilities}` : ''}${recording}${projectBriefing()}`; + // THE RECORDING GUIDANCE IS DELIVERED LATER, NOT HERE (#357, M5): see + // recordingGuidance. A session that never does real work never pays for it. + + return `# Token optimization is active\n\n${outputDiscipline()}\n\n${connection}${routing}${enforcement}${utilities ? `\n\n${utilities}` : ''}${projectBriefing()}`; } /** @@ -1870,9 +1879,26 @@ async function runHook(clientName, event, invocation) { // has already run) either way. Say it once per session per kind -- see // adviseOnce -- rather than on every matching call. const advisory = verdict && adviseOnce(state, verdict) ? verdict.reason : null; + // THE RECORDING GUIDANCE, ONCE, AFTER REAL WORK (#357, M5). It used to ride in + // every SessionStart, where it is the largest fixed cost a small session pays + // and buys nothing until there is something to record. The first edit is the + // clearest sign there will be; GUIDANCE_AFTER_CALLS tool calls is the fallback + // for an investigation that edits nothing but still works things out. + const guided = + event === 'post-tool' && + !state.recordingGuided && + toolEvidence.names.has('wiki_write') && + ((Number(state.edits) || 0) >= 1 || + (Number(state.toolCalls) || 0) >= GUIDANCE_AFTER_CALLS); + if (guided) state.recordingGuided = true; saveState(payload.session_id, state, agentScope); - const context = [advisory, graphContext, recordingContext] + const context = [ + advisory, + graphContext, + guided ? recordingGuidance() : null, + recordingContext, + ] .filter(Boolean) .join('\n\n'); if (context) { diff --git a/integrations/cline/hooks/token-optimizer/lib/policy.mjs b/integrations/cline/hooks/token-optimizer/lib/policy.mjs index c9eb1f896..a161899d3 100644 --- a/integrations/cline/hooks/token-optimizer/lib/policy.mjs +++ b/integrations/cline/hooks/token-optimizer/lib/policy.mjs @@ -394,6 +394,9 @@ function emptyState() { editedFiles: [], harvestedEdits: 0, recordingNudged: false, + // The recording guidance has been delivered (#357, M5); see + // recordingGuidance in adapter.mjs. + recordingGuided: false, // Facts the search advisory has already delivered this session. Without // this in the default, loadState returned an object with no `advised`, the // router rebuilt an empty Set on every hook process, and the same advisory @@ -487,6 +490,7 @@ export function loadState(sessionId, agent) { ? parsed.harvestedEdits : 0, recordingNudged: parsed.recordingNudged === true, + recordingGuided: parsed.recordingGuided === true, // Every hook call is a new process, so a field left out of this list is // forgotten between calls: that is how "said once" kept being said on // every search, and how a repeated WebFetch was never recognised @@ -651,6 +655,10 @@ export function saveState(sessionId, state, agent) { recordingNudged: Boolean( current.recordingNudged || state.recordingNudged ), + // Same rule: guidance said once stays said. + recordingGuided: Boolean( + current.recordingGuided || state.recordingGuided + ), // UNION, AND CAPPED. Two hook processes can each deliver an advisory // before either writes, so last-writer-wins would forget one and repeat // it. A fact once told stays told -- that is the whole point of the set diff --git a/integrations/codex/hooks/lib/adapter.mjs b/integrations/codex/hooks/lib/adapter.mjs index 89eea2645..a75f15023 100644 --- a/integrations/codex/hooks/lib/adapter.mjs +++ b/integrations/codex/hooks/lib/adapter.mjs @@ -892,6 +892,21 @@ function gradeObservedTools(state, event, payload, raw) { } /** The session-start notice, shared verbatim so no client drifts its own copy. */ +/** Tool calls after which a session that has edited nothing still gets the recording guidance. */ +export const GUIDANCE_AFTER_CALLS = 10; + +/** + * The wiki_write guidance. It was part of the SessionStart block until #357 + * (M5): it is the largest part of that block, and it only matters once there is + * something to record, so it is now delivered once, on the first post-tool + * event after the session shows real work, and a session that never gets that + * far never pays for it. The wording is unchanged, because it was measured -- + * see the last paragraph. + */ +export function recordingGuidance() { + return `## Record what you work out\n\nCall wiki_write when you establish something durable about this project, while\nyou still hold the context. Every claim needs at least one anchor -- a real file\npath, or path#symbol -- because an unanchored claim can never be checked against\nthe code again and would be served as current forever; unanchored writes are\nrefused.\n\nWorth recording: a decision and why the alternative was rejected, a failure and\nwhat actually caused it, a command that turned out to be the one that works.\nNot worth recording: what the code plainly says. Every wiki_write must include\nthe concrete evidence, when it applies, a calibrated confidence label, its\nscope, and what would invalidate it. Prefer the thing someone had to work out,\nbecause that exists nowhere in the source tree.\n\nRECORD BEFORE YOU ANSWER, NEVER AFTER. Your final message must be the answer to\nthe task -- never a confirmation that you saved a note. Measured: a run whose\nlast message was "Recorded. The finding captured: ..." scored 0.3 where the\nidentical task answered directly scored 0.9. The work had been done correctly\nand the artifact was written; only the last message was wrong, and the last\nmessage is what a verifier, a reviewer and the person who asked all read.${extractionNotice()}`; +} + export function policyText( canDeny, availableTools = undefined, @@ -972,16 +987,10 @@ export function policyText( ] .filter(Boolean) .join(' '); - const recording = tools.has('wiki_write') - ? `\n\n## Record what you work out\n\nCall wiki_write when you establish something durable about this project, while\nyou still hold the context. Every claim needs at least one anchor -- a real file\npath, or path#symbol -- because an unanchored claim can never be checked against\nthe code again and would be served as current forever; unanchored writes are\nrefused.\n\nWorth recording: a decision and why the alternative was rejected, a failure and\nwhat actually caused it, a command that turned out to be the one that works.\nNot worth recording: what the code plainly says. Every wiki_write must include\nthe concrete evidence, when it applies, a calibrated confidence label, its\nscope, and what would invalidate it. Prefer the thing someone had to work out,\nbecause that exists nowhere in the source tree.\n\nRECORD BEFORE YOU ANSWER, NEVER AFTER. Your final message must be the answer to\nthe task -- never a confirmation that you saved a note. Measured: a run whose\nlast message was "Recorded. The finding captured: ..." scored 0.3 where the\nidentical task answered directly scored 0.9. The work had been done correctly\nand the artifact was written; only the last message was wrong, and the last\nmessage is what a verifier, a reviewer and the person who asked all read.${extractionNotice()}` - // WAS 158 CHARACTERS OF SELF-DESCRIPTION. "Structural graph capture remains - // active through lifecycle hooks. Durable semantic MCP writes are not - // requested because the writer schema is not proven available." -- true, - // and it changed no decision the model ever made with it. Fixed context is - // charged on every session; it has to earn its place by altering behaviour. - : ''; - - return `# Token optimization is active\n\n${outputDiscipline()}\n\n${connection}${routing}${enforcement}${utilities ? `\n\n${utilities}` : ''}${recording}${projectBriefing()}`; + // THE RECORDING GUIDANCE IS DELIVERED LATER, NOT HERE (#357, M5): see + // recordingGuidance. A session that never does real work never pays for it. + + return `# Token optimization is active\n\n${outputDiscipline()}\n\n${connection}${routing}${enforcement}${utilities ? `\n\n${utilities}` : ''}${projectBriefing()}`; } /** @@ -1870,9 +1879,26 @@ async function runHook(clientName, event, invocation) { // has already run) either way. Say it once per session per kind -- see // adviseOnce -- rather than on every matching call. const advisory = verdict && adviseOnce(state, verdict) ? verdict.reason : null; + // THE RECORDING GUIDANCE, ONCE, AFTER REAL WORK (#357, M5). It used to ride in + // every SessionStart, where it is the largest fixed cost a small session pays + // and buys nothing until there is something to record. The first edit is the + // clearest sign there will be; GUIDANCE_AFTER_CALLS tool calls is the fallback + // for an investigation that edits nothing but still works things out. + const guided = + event === 'post-tool' && + !state.recordingGuided && + toolEvidence.names.has('wiki_write') && + ((Number(state.edits) || 0) >= 1 || + (Number(state.toolCalls) || 0) >= GUIDANCE_AFTER_CALLS); + if (guided) state.recordingGuided = true; saveState(payload.session_id, state, agentScope); - const context = [advisory, graphContext, recordingContext] + const context = [ + advisory, + graphContext, + guided ? recordingGuidance() : null, + recordingContext, + ] .filter(Boolean) .join('\n\n'); if (context) { diff --git a/integrations/codex/hooks/lib/policy.mjs b/integrations/codex/hooks/lib/policy.mjs index c9eb1f896..a161899d3 100644 --- a/integrations/codex/hooks/lib/policy.mjs +++ b/integrations/codex/hooks/lib/policy.mjs @@ -394,6 +394,9 @@ function emptyState() { editedFiles: [], harvestedEdits: 0, recordingNudged: false, + // The recording guidance has been delivered (#357, M5); see + // recordingGuidance in adapter.mjs. + recordingGuided: false, // Facts the search advisory has already delivered this session. Without // this in the default, loadState returned an object with no `advised`, the // router rebuilt an empty Set on every hook process, and the same advisory @@ -487,6 +490,7 @@ export function loadState(sessionId, agent) { ? parsed.harvestedEdits : 0, recordingNudged: parsed.recordingNudged === true, + recordingGuided: parsed.recordingGuided === true, // Every hook call is a new process, so a field left out of this list is // forgotten between calls: that is how "said once" kept being said on // every search, and how a repeated WebFetch was never recognised @@ -651,6 +655,10 @@ export function saveState(sessionId, state, agent) { recordingNudged: Boolean( current.recordingNudged || state.recordingNudged ), + // Same rule: guidance said once stays said. + recordingGuided: Boolean( + current.recordingGuided || state.recordingGuided + ), // UNION, AND CAPPED. Two hook processes can each deliver an advisory // before either writes, so last-writer-wins would forget one and repeat // it. A fact once told stays told -- that is the whole point of the set diff --git a/integrations/codex/plugin/hooks/lib/adapter.mjs b/integrations/codex/plugin/hooks/lib/adapter.mjs index 89eea2645..a75f15023 100644 --- a/integrations/codex/plugin/hooks/lib/adapter.mjs +++ b/integrations/codex/plugin/hooks/lib/adapter.mjs @@ -892,6 +892,21 @@ function gradeObservedTools(state, event, payload, raw) { } /** The session-start notice, shared verbatim so no client drifts its own copy. */ +/** Tool calls after which a session that has edited nothing still gets the recording guidance. */ +export const GUIDANCE_AFTER_CALLS = 10; + +/** + * The wiki_write guidance. It was part of the SessionStart block until #357 + * (M5): it is the largest part of that block, and it only matters once there is + * something to record, so it is now delivered once, on the first post-tool + * event after the session shows real work, and a session that never gets that + * far never pays for it. The wording is unchanged, because it was measured -- + * see the last paragraph. + */ +export function recordingGuidance() { + return `## Record what you work out\n\nCall wiki_write when you establish something durable about this project, while\nyou still hold the context. Every claim needs at least one anchor -- a real file\npath, or path#symbol -- because an unanchored claim can never be checked against\nthe code again and would be served as current forever; unanchored writes are\nrefused.\n\nWorth recording: a decision and why the alternative was rejected, a failure and\nwhat actually caused it, a command that turned out to be the one that works.\nNot worth recording: what the code plainly says. Every wiki_write must include\nthe concrete evidence, when it applies, a calibrated confidence label, its\nscope, and what would invalidate it. Prefer the thing someone had to work out,\nbecause that exists nowhere in the source tree.\n\nRECORD BEFORE YOU ANSWER, NEVER AFTER. Your final message must be the answer to\nthe task -- never a confirmation that you saved a note. Measured: a run whose\nlast message was "Recorded. The finding captured: ..." scored 0.3 where the\nidentical task answered directly scored 0.9. The work had been done correctly\nand the artifact was written; only the last message was wrong, and the last\nmessage is what a verifier, a reviewer and the person who asked all read.${extractionNotice()}`; +} + export function policyText( canDeny, availableTools = undefined, @@ -972,16 +987,10 @@ export function policyText( ] .filter(Boolean) .join(' '); - const recording = tools.has('wiki_write') - ? `\n\n## Record what you work out\n\nCall wiki_write when you establish something durable about this project, while\nyou still hold the context. Every claim needs at least one anchor -- a real file\npath, or path#symbol -- because an unanchored claim can never be checked against\nthe code again and would be served as current forever; unanchored writes are\nrefused.\n\nWorth recording: a decision and why the alternative was rejected, a failure and\nwhat actually caused it, a command that turned out to be the one that works.\nNot worth recording: what the code plainly says. Every wiki_write must include\nthe concrete evidence, when it applies, a calibrated confidence label, its\nscope, and what would invalidate it. Prefer the thing someone had to work out,\nbecause that exists nowhere in the source tree.\n\nRECORD BEFORE YOU ANSWER, NEVER AFTER. Your final message must be the answer to\nthe task -- never a confirmation that you saved a note. Measured: a run whose\nlast message was "Recorded. The finding captured: ..." scored 0.3 where the\nidentical task answered directly scored 0.9. The work had been done correctly\nand the artifact was written; only the last message was wrong, and the last\nmessage is what a verifier, a reviewer and the person who asked all read.${extractionNotice()}` - // WAS 158 CHARACTERS OF SELF-DESCRIPTION. "Structural graph capture remains - // active through lifecycle hooks. Durable semantic MCP writes are not - // requested because the writer schema is not proven available." -- true, - // and it changed no decision the model ever made with it. Fixed context is - // charged on every session; it has to earn its place by altering behaviour. - : ''; - - return `# Token optimization is active\n\n${outputDiscipline()}\n\n${connection}${routing}${enforcement}${utilities ? `\n\n${utilities}` : ''}${recording}${projectBriefing()}`; + // THE RECORDING GUIDANCE IS DELIVERED LATER, NOT HERE (#357, M5): see + // recordingGuidance. A session that never does real work never pays for it. + + return `# Token optimization is active\n\n${outputDiscipline()}\n\n${connection}${routing}${enforcement}${utilities ? `\n\n${utilities}` : ''}${projectBriefing()}`; } /** @@ -1870,9 +1879,26 @@ async function runHook(clientName, event, invocation) { // has already run) either way. Say it once per session per kind -- see // adviseOnce -- rather than on every matching call. const advisory = verdict && adviseOnce(state, verdict) ? verdict.reason : null; + // THE RECORDING GUIDANCE, ONCE, AFTER REAL WORK (#357, M5). It used to ride in + // every SessionStart, where it is the largest fixed cost a small session pays + // and buys nothing until there is something to record. The first edit is the + // clearest sign there will be; GUIDANCE_AFTER_CALLS tool calls is the fallback + // for an investigation that edits nothing but still works things out. + const guided = + event === 'post-tool' && + !state.recordingGuided && + toolEvidence.names.has('wiki_write') && + ((Number(state.edits) || 0) >= 1 || + (Number(state.toolCalls) || 0) >= GUIDANCE_AFTER_CALLS); + if (guided) state.recordingGuided = true; saveState(payload.session_id, state, agentScope); - const context = [advisory, graphContext, recordingContext] + const context = [ + advisory, + graphContext, + guided ? recordingGuidance() : null, + recordingContext, + ] .filter(Boolean) .join('\n\n'); if (context) { diff --git a/integrations/codex/plugin/hooks/lib/policy.mjs b/integrations/codex/plugin/hooks/lib/policy.mjs index c9eb1f896..a161899d3 100644 --- a/integrations/codex/plugin/hooks/lib/policy.mjs +++ b/integrations/codex/plugin/hooks/lib/policy.mjs @@ -394,6 +394,9 @@ function emptyState() { editedFiles: [], harvestedEdits: 0, recordingNudged: false, + // The recording guidance has been delivered (#357, M5); see + // recordingGuidance in adapter.mjs. + recordingGuided: false, // Facts the search advisory has already delivered this session. Without // this in the default, loadState returned an object with no `advised`, the // router rebuilt an empty Set on every hook process, and the same advisory @@ -487,6 +490,7 @@ export function loadState(sessionId, agent) { ? parsed.harvestedEdits : 0, recordingNudged: parsed.recordingNudged === true, + recordingGuided: parsed.recordingGuided === true, // Every hook call is a new process, so a field left out of this list is // forgotten between calls: that is how "said once" kept being said on // every search, and how a repeated WebFetch was never recognised @@ -651,6 +655,10 @@ export function saveState(sessionId, state, agent) { recordingNudged: Boolean( current.recordingNudged || state.recordingNudged ), + // Same rule: guidance said once stays said. + recordingGuided: Boolean( + current.recordingGuided || state.recordingGuided + ), // UNION, AND CAPPED. Two hook processes can each deliver an advisory // before either writes, so last-writer-wins would forget one and repeat // it. A fact once told stays told -- that is the whole point of the set diff --git a/integrations/copilot/.github/hooks/lib/adapter.mjs b/integrations/copilot/.github/hooks/lib/adapter.mjs index 89eea2645..a75f15023 100644 --- a/integrations/copilot/.github/hooks/lib/adapter.mjs +++ b/integrations/copilot/.github/hooks/lib/adapter.mjs @@ -892,6 +892,21 @@ function gradeObservedTools(state, event, payload, raw) { } /** The session-start notice, shared verbatim so no client drifts its own copy. */ +/** Tool calls after which a session that has edited nothing still gets the recording guidance. */ +export const GUIDANCE_AFTER_CALLS = 10; + +/** + * The wiki_write guidance. It was part of the SessionStart block until #357 + * (M5): it is the largest part of that block, and it only matters once there is + * something to record, so it is now delivered once, on the first post-tool + * event after the session shows real work, and a session that never gets that + * far never pays for it. The wording is unchanged, because it was measured -- + * see the last paragraph. + */ +export function recordingGuidance() { + return `## Record what you work out\n\nCall wiki_write when you establish something durable about this project, while\nyou still hold the context. Every claim needs at least one anchor -- a real file\npath, or path#symbol -- because an unanchored claim can never be checked against\nthe code again and would be served as current forever; unanchored writes are\nrefused.\n\nWorth recording: a decision and why the alternative was rejected, a failure and\nwhat actually caused it, a command that turned out to be the one that works.\nNot worth recording: what the code plainly says. Every wiki_write must include\nthe concrete evidence, when it applies, a calibrated confidence label, its\nscope, and what would invalidate it. Prefer the thing someone had to work out,\nbecause that exists nowhere in the source tree.\n\nRECORD BEFORE YOU ANSWER, NEVER AFTER. Your final message must be the answer to\nthe task -- never a confirmation that you saved a note. Measured: a run whose\nlast message was "Recorded. The finding captured: ..." scored 0.3 where the\nidentical task answered directly scored 0.9. The work had been done correctly\nand the artifact was written; only the last message was wrong, and the last\nmessage is what a verifier, a reviewer and the person who asked all read.${extractionNotice()}`; +} + export function policyText( canDeny, availableTools = undefined, @@ -972,16 +987,10 @@ export function policyText( ] .filter(Boolean) .join(' '); - const recording = tools.has('wiki_write') - ? `\n\n## Record what you work out\n\nCall wiki_write when you establish something durable about this project, while\nyou still hold the context. Every claim needs at least one anchor -- a real file\npath, or path#symbol -- because an unanchored claim can never be checked against\nthe code again and would be served as current forever; unanchored writes are\nrefused.\n\nWorth recording: a decision and why the alternative was rejected, a failure and\nwhat actually caused it, a command that turned out to be the one that works.\nNot worth recording: what the code plainly says. Every wiki_write must include\nthe concrete evidence, when it applies, a calibrated confidence label, its\nscope, and what would invalidate it. Prefer the thing someone had to work out,\nbecause that exists nowhere in the source tree.\n\nRECORD BEFORE YOU ANSWER, NEVER AFTER. Your final message must be the answer to\nthe task -- never a confirmation that you saved a note. Measured: a run whose\nlast message was "Recorded. The finding captured: ..." scored 0.3 where the\nidentical task answered directly scored 0.9. The work had been done correctly\nand the artifact was written; only the last message was wrong, and the last\nmessage is what a verifier, a reviewer and the person who asked all read.${extractionNotice()}` - // WAS 158 CHARACTERS OF SELF-DESCRIPTION. "Structural graph capture remains - // active through lifecycle hooks. Durable semantic MCP writes are not - // requested because the writer schema is not proven available." -- true, - // and it changed no decision the model ever made with it. Fixed context is - // charged on every session; it has to earn its place by altering behaviour. - : ''; - - return `# Token optimization is active\n\n${outputDiscipline()}\n\n${connection}${routing}${enforcement}${utilities ? `\n\n${utilities}` : ''}${recording}${projectBriefing()}`; + // THE RECORDING GUIDANCE IS DELIVERED LATER, NOT HERE (#357, M5): see + // recordingGuidance. A session that never does real work never pays for it. + + return `# Token optimization is active\n\n${outputDiscipline()}\n\n${connection}${routing}${enforcement}${utilities ? `\n\n${utilities}` : ''}${projectBriefing()}`; } /** @@ -1870,9 +1879,26 @@ async function runHook(clientName, event, invocation) { // has already run) either way. Say it once per session per kind -- see // adviseOnce -- rather than on every matching call. const advisory = verdict && adviseOnce(state, verdict) ? verdict.reason : null; + // THE RECORDING GUIDANCE, ONCE, AFTER REAL WORK (#357, M5). It used to ride in + // every SessionStart, where it is the largest fixed cost a small session pays + // and buys nothing until there is something to record. The first edit is the + // clearest sign there will be; GUIDANCE_AFTER_CALLS tool calls is the fallback + // for an investigation that edits nothing but still works things out. + const guided = + event === 'post-tool' && + !state.recordingGuided && + toolEvidence.names.has('wiki_write') && + ((Number(state.edits) || 0) >= 1 || + (Number(state.toolCalls) || 0) >= GUIDANCE_AFTER_CALLS); + if (guided) state.recordingGuided = true; saveState(payload.session_id, state, agentScope); - const context = [advisory, graphContext, recordingContext] + const context = [ + advisory, + graphContext, + guided ? recordingGuidance() : null, + recordingContext, + ] .filter(Boolean) .join('\n\n'); if (context) { diff --git a/integrations/copilot/.github/hooks/lib/policy.mjs b/integrations/copilot/.github/hooks/lib/policy.mjs index c9eb1f896..a161899d3 100644 --- a/integrations/copilot/.github/hooks/lib/policy.mjs +++ b/integrations/copilot/.github/hooks/lib/policy.mjs @@ -394,6 +394,9 @@ function emptyState() { editedFiles: [], harvestedEdits: 0, recordingNudged: false, + // The recording guidance has been delivered (#357, M5); see + // recordingGuidance in adapter.mjs. + recordingGuided: false, // Facts the search advisory has already delivered this session. Without // this in the default, loadState returned an object with no `advised`, the // router rebuilt an empty Set on every hook process, and the same advisory @@ -487,6 +490,7 @@ export function loadState(sessionId, agent) { ? parsed.harvestedEdits : 0, recordingNudged: parsed.recordingNudged === true, + recordingGuided: parsed.recordingGuided === true, // Every hook call is a new process, so a field left out of this list is // forgotten between calls: that is how "said once" kept being said on // every search, and how a repeated WebFetch was never recognised @@ -651,6 +655,10 @@ export function saveState(sessionId, state, agent) { recordingNudged: Boolean( current.recordingNudged || state.recordingNudged ), + // Same rule: guidance said once stays said. + recordingGuided: Boolean( + current.recordingGuided || state.recordingGuided + ), // UNION, AND CAPPED. Two hook processes can each deliver an advisory // before either writes, so last-writer-wins would forget one and repeat // it. A fact once told stays told -- that is the whole point of the set diff --git a/integrations/cursor/hooks/lib/adapter.mjs b/integrations/cursor/hooks/lib/adapter.mjs index 89eea2645..a75f15023 100644 --- a/integrations/cursor/hooks/lib/adapter.mjs +++ b/integrations/cursor/hooks/lib/adapter.mjs @@ -892,6 +892,21 @@ function gradeObservedTools(state, event, payload, raw) { } /** The session-start notice, shared verbatim so no client drifts its own copy. */ +/** Tool calls after which a session that has edited nothing still gets the recording guidance. */ +export const GUIDANCE_AFTER_CALLS = 10; + +/** + * The wiki_write guidance. It was part of the SessionStart block until #357 + * (M5): it is the largest part of that block, and it only matters once there is + * something to record, so it is now delivered once, on the first post-tool + * event after the session shows real work, and a session that never gets that + * far never pays for it. The wording is unchanged, because it was measured -- + * see the last paragraph. + */ +export function recordingGuidance() { + return `## Record what you work out\n\nCall wiki_write when you establish something durable about this project, while\nyou still hold the context. Every claim needs at least one anchor -- a real file\npath, or path#symbol -- because an unanchored claim can never be checked against\nthe code again and would be served as current forever; unanchored writes are\nrefused.\n\nWorth recording: a decision and why the alternative was rejected, a failure and\nwhat actually caused it, a command that turned out to be the one that works.\nNot worth recording: what the code plainly says. Every wiki_write must include\nthe concrete evidence, when it applies, a calibrated confidence label, its\nscope, and what would invalidate it. Prefer the thing someone had to work out,\nbecause that exists nowhere in the source tree.\n\nRECORD BEFORE YOU ANSWER, NEVER AFTER. Your final message must be the answer to\nthe task -- never a confirmation that you saved a note. Measured: a run whose\nlast message was "Recorded. The finding captured: ..." scored 0.3 where the\nidentical task answered directly scored 0.9. The work had been done correctly\nand the artifact was written; only the last message was wrong, and the last\nmessage is what a verifier, a reviewer and the person who asked all read.${extractionNotice()}`; +} + export function policyText( canDeny, availableTools = undefined, @@ -972,16 +987,10 @@ export function policyText( ] .filter(Boolean) .join(' '); - const recording = tools.has('wiki_write') - ? `\n\n## Record what you work out\n\nCall wiki_write when you establish something durable about this project, while\nyou still hold the context. Every claim needs at least one anchor -- a real file\npath, or path#symbol -- because an unanchored claim can never be checked against\nthe code again and would be served as current forever; unanchored writes are\nrefused.\n\nWorth recording: a decision and why the alternative was rejected, a failure and\nwhat actually caused it, a command that turned out to be the one that works.\nNot worth recording: what the code plainly says. Every wiki_write must include\nthe concrete evidence, when it applies, a calibrated confidence label, its\nscope, and what would invalidate it. Prefer the thing someone had to work out,\nbecause that exists nowhere in the source tree.\n\nRECORD BEFORE YOU ANSWER, NEVER AFTER. Your final message must be the answer to\nthe task -- never a confirmation that you saved a note. Measured: a run whose\nlast message was "Recorded. The finding captured: ..." scored 0.3 where the\nidentical task answered directly scored 0.9. The work had been done correctly\nand the artifact was written; only the last message was wrong, and the last\nmessage is what a verifier, a reviewer and the person who asked all read.${extractionNotice()}` - // WAS 158 CHARACTERS OF SELF-DESCRIPTION. "Structural graph capture remains - // active through lifecycle hooks. Durable semantic MCP writes are not - // requested because the writer schema is not proven available." -- true, - // and it changed no decision the model ever made with it. Fixed context is - // charged on every session; it has to earn its place by altering behaviour. - : ''; - - return `# Token optimization is active\n\n${outputDiscipline()}\n\n${connection}${routing}${enforcement}${utilities ? `\n\n${utilities}` : ''}${recording}${projectBriefing()}`; + // THE RECORDING GUIDANCE IS DELIVERED LATER, NOT HERE (#357, M5): see + // recordingGuidance. A session that never does real work never pays for it. + + return `# Token optimization is active\n\n${outputDiscipline()}\n\n${connection}${routing}${enforcement}${utilities ? `\n\n${utilities}` : ''}${projectBriefing()}`; } /** @@ -1870,9 +1879,26 @@ async function runHook(clientName, event, invocation) { // has already run) either way. Say it once per session per kind -- see // adviseOnce -- rather than on every matching call. const advisory = verdict && adviseOnce(state, verdict) ? verdict.reason : null; + // THE RECORDING GUIDANCE, ONCE, AFTER REAL WORK (#357, M5). It used to ride in + // every SessionStart, where it is the largest fixed cost a small session pays + // and buys nothing until there is something to record. The first edit is the + // clearest sign there will be; GUIDANCE_AFTER_CALLS tool calls is the fallback + // for an investigation that edits nothing but still works things out. + const guided = + event === 'post-tool' && + !state.recordingGuided && + toolEvidence.names.has('wiki_write') && + ((Number(state.edits) || 0) >= 1 || + (Number(state.toolCalls) || 0) >= GUIDANCE_AFTER_CALLS); + if (guided) state.recordingGuided = true; saveState(payload.session_id, state, agentScope); - const context = [advisory, graphContext, recordingContext] + const context = [ + advisory, + graphContext, + guided ? recordingGuidance() : null, + recordingContext, + ] .filter(Boolean) .join('\n\n'); if (context) { diff --git a/integrations/cursor/hooks/lib/policy.mjs b/integrations/cursor/hooks/lib/policy.mjs index c9eb1f896..a161899d3 100644 --- a/integrations/cursor/hooks/lib/policy.mjs +++ b/integrations/cursor/hooks/lib/policy.mjs @@ -394,6 +394,9 @@ function emptyState() { editedFiles: [], harvestedEdits: 0, recordingNudged: false, + // The recording guidance has been delivered (#357, M5); see + // recordingGuidance in adapter.mjs. + recordingGuided: false, // Facts the search advisory has already delivered this session. Without // this in the default, loadState returned an object with no `advised`, the // router rebuilt an empty Set on every hook process, and the same advisory @@ -487,6 +490,7 @@ export function loadState(sessionId, agent) { ? parsed.harvestedEdits : 0, recordingNudged: parsed.recordingNudged === true, + recordingGuided: parsed.recordingGuided === true, // Every hook call is a new process, so a field left out of this list is // forgotten between calls: that is how "said once" kept being said on // every search, and how a repeated WebFetch was never recognised @@ -651,6 +655,10 @@ export function saveState(sessionId, state, agent) { recordingNudged: Boolean( current.recordingNudged || state.recordingNudged ), + // Same rule: guidance said once stays said. + recordingGuided: Boolean( + current.recordingGuided || state.recordingGuided + ), // UNION, AND CAPPED. Two hook processes can each deliver an advisory // before either writes, so last-writer-wins would forget one and repeat // it. A fact once told stays told -- that is the whole point of the set diff --git a/integrations/gemini/hooks/lib/adapter.mjs b/integrations/gemini/hooks/lib/adapter.mjs index 89eea2645..a75f15023 100644 --- a/integrations/gemini/hooks/lib/adapter.mjs +++ b/integrations/gemini/hooks/lib/adapter.mjs @@ -892,6 +892,21 @@ function gradeObservedTools(state, event, payload, raw) { } /** The session-start notice, shared verbatim so no client drifts its own copy. */ +/** Tool calls after which a session that has edited nothing still gets the recording guidance. */ +export const GUIDANCE_AFTER_CALLS = 10; + +/** + * The wiki_write guidance. It was part of the SessionStart block until #357 + * (M5): it is the largest part of that block, and it only matters once there is + * something to record, so it is now delivered once, on the first post-tool + * event after the session shows real work, and a session that never gets that + * far never pays for it. The wording is unchanged, because it was measured -- + * see the last paragraph. + */ +export function recordingGuidance() { + return `## Record what you work out\n\nCall wiki_write when you establish something durable about this project, while\nyou still hold the context. Every claim needs at least one anchor -- a real file\npath, or path#symbol -- because an unanchored claim can never be checked against\nthe code again and would be served as current forever; unanchored writes are\nrefused.\n\nWorth recording: a decision and why the alternative was rejected, a failure and\nwhat actually caused it, a command that turned out to be the one that works.\nNot worth recording: what the code plainly says. Every wiki_write must include\nthe concrete evidence, when it applies, a calibrated confidence label, its\nscope, and what would invalidate it. Prefer the thing someone had to work out,\nbecause that exists nowhere in the source tree.\n\nRECORD BEFORE YOU ANSWER, NEVER AFTER. Your final message must be the answer to\nthe task -- never a confirmation that you saved a note. Measured: a run whose\nlast message was "Recorded. The finding captured: ..." scored 0.3 where the\nidentical task answered directly scored 0.9. The work had been done correctly\nand the artifact was written; only the last message was wrong, and the last\nmessage is what a verifier, a reviewer and the person who asked all read.${extractionNotice()}`; +} + export function policyText( canDeny, availableTools = undefined, @@ -972,16 +987,10 @@ export function policyText( ] .filter(Boolean) .join(' '); - const recording = tools.has('wiki_write') - ? `\n\n## Record what you work out\n\nCall wiki_write when you establish something durable about this project, while\nyou still hold the context. Every claim needs at least one anchor -- a real file\npath, or path#symbol -- because an unanchored claim can never be checked against\nthe code again and would be served as current forever; unanchored writes are\nrefused.\n\nWorth recording: a decision and why the alternative was rejected, a failure and\nwhat actually caused it, a command that turned out to be the one that works.\nNot worth recording: what the code plainly says. Every wiki_write must include\nthe concrete evidence, when it applies, a calibrated confidence label, its\nscope, and what would invalidate it. Prefer the thing someone had to work out,\nbecause that exists nowhere in the source tree.\n\nRECORD BEFORE YOU ANSWER, NEVER AFTER. Your final message must be the answer to\nthe task -- never a confirmation that you saved a note. Measured: a run whose\nlast message was "Recorded. The finding captured: ..." scored 0.3 where the\nidentical task answered directly scored 0.9. The work had been done correctly\nand the artifact was written; only the last message was wrong, and the last\nmessage is what a verifier, a reviewer and the person who asked all read.${extractionNotice()}` - // WAS 158 CHARACTERS OF SELF-DESCRIPTION. "Structural graph capture remains - // active through lifecycle hooks. Durable semantic MCP writes are not - // requested because the writer schema is not proven available." -- true, - // and it changed no decision the model ever made with it. Fixed context is - // charged on every session; it has to earn its place by altering behaviour. - : ''; - - return `# Token optimization is active\n\n${outputDiscipline()}\n\n${connection}${routing}${enforcement}${utilities ? `\n\n${utilities}` : ''}${recording}${projectBriefing()}`; + // THE RECORDING GUIDANCE IS DELIVERED LATER, NOT HERE (#357, M5): see + // recordingGuidance. A session that never does real work never pays for it. + + return `# Token optimization is active\n\n${outputDiscipline()}\n\n${connection}${routing}${enforcement}${utilities ? `\n\n${utilities}` : ''}${projectBriefing()}`; } /** @@ -1870,9 +1879,26 @@ async function runHook(clientName, event, invocation) { // has already run) either way. Say it once per session per kind -- see // adviseOnce -- rather than on every matching call. const advisory = verdict && adviseOnce(state, verdict) ? verdict.reason : null; + // THE RECORDING GUIDANCE, ONCE, AFTER REAL WORK (#357, M5). It used to ride in + // every SessionStart, where it is the largest fixed cost a small session pays + // and buys nothing until there is something to record. The first edit is the + // clearest sign there will be; GUIDANCE_AFTER_CALLS tool calls is the fallback + // for an investigation that edits nothing but still works things out. + const guided = + event === 'post-tool' && + !state.recordingGuided && + toolEvidence.names.has('wiki_write') && + ((Number(state.edits) || 0) >= 1 || + (Number(state.toolCalls) || 0) >= GUIDANCE_AFTER_CALLS); + if (guided) state.recordingGuided = true; saveState(payload.session_id, state, agentScope); - const context = [advisory, graphContext, recordingContext] + const context = [ + advisory, + graphContext, + guided ? recordingGuidance() : null, + recordingContext, + ] .filter(Boolean) .join('\n\n'); if (context) { diff --git a/integrations/gemini/hooks/lib/policy.mjs b/integrations/gemini/hooks/lib/policy.mjs index c9eb1f896..a161899d3 100644 --- a/integrations/gemini/hooks/lib/policy.mjs +++ b/integrations/gemini/hooks/lib/policy.mjs @@ -394,6 +394,9 @@ function emptyState() { editedFiles: [], harvestedEdits: 0, recordingNudged: false, + // The recording guidance has been delivered (#357, M5); see + // recordingGuidance in adapter.mjs. + recordingGuided: false, // Facts the search advisory has already delivered this session. Without // this in the default, loadState returned an object with no `advised`, the // router rebuilt an empty Set on every hook process, and the same advisory @@ -487,6 +490,7 @@ export function loadState(sessionId, agent) { ? parsed.harvestedEdits : 0, recordingNudged: parsed.recordingNudged === true, + recordingGuided: parsed.recordingGuided === true, // Every hook call is a new process, so a field left out of this list is // forgotten between calls: that is how "said once" kept being said on // every search, and how a repeated WebFetch was never recognised @@ -651,6 +655,10 @@ export function saveState(sessionId, state, agent) { recordingNudged: Boolean( current.recordingNudged || state.recordingNudged ), + // Same rule: guidance said once stays said. + recordingGuided: Boolean( + current.recordingGuided || state.recordingGuided + ), // UNION, AND CAPPED. Two hook processes can each deliver an advisory // before either writes, so last-writer-wins would forget one and repeat // it. A fact once told stays told -- that is the whole point of the set diff --git a/integrations/kilo/hooks/lib/adapter.mjs b/integrations/kilo/hooks/lib/adapter.mjs index 89eea2645..a75f15023 100644 --- a/integrations/kilo/hooks/lib/adapter.mjs +++ b/integrations/kilo/hooks/lib/adapter.mjs @@ -892,6 +892,21 @@ function gradeObservedTools(state, event, payload, raw) { } /** The session-start notice, shared verbatim so no client drifts its own copy. */ +/** Tool calls after which a session that has edited nothing still gets the recording guidance. */ +export const GUIDANCE_AFTER_CALLS = 10; + +/** + * The wiki_write guidance. It was part of the SessionStart block until #357 + * (M5): it is the largest part of that block, and it only matters once there is + * something to record, so it is now delivered once, on the first post-tool + * event after the session shows real work, and a session that never gets that + * far never pays for it. The wording is unchanged, because it was measured -- + * see the last paragraph. + */ +export function recordingGuidance() { + return `## Record what you work out\n\nCall wiki_write when you establish something durable about this project, while\nyou still hold the context. Every claim needs at least one anchor -- a real file\npath, or path#symbol -- because an unanchored claim can never be checked against\nthe code again and would be served as current forever; unanchored writes are\nrefused.\n\nWorth recording: a decision and why the alternative was rejected, a failure and\nwhat actually caused it, a command that turned out to be the one that works.\nNot worth recording: what the code plainly says. Every wiki_write must include\nthe concrete evidence, when it applies, a calibrated confidence label, its\nscope, and what would invalidate it. Prefer the thing someone had to work out,\nbecause that exists nowhere in the source tree.\n\nRECORD BEFORE YOU ANSWER, NEVER AFTER. Your final message must be the answer to\nthe task -- never a confirmation that you saved a note. Measured: a run whose\nlast message was "Recorded. The finding captured: ..." scored 0.3 where the\nidentical task answered directly scored 0.9. The work had been done correctly\nand the artifact was written; only the last message was wrong, and the last\nmessage is what a verifier, a reviewer and the person who asked all read.${extractionNotice()}`; +} + export function policyText( canDeny, availableTools = undefined, @@ -972,16 +987,10 @@ export function policyText( ] .filter(Boolean) .join(' '); - const recording = tools.has('wiki_write') - ? `\n\n## Record what you work out\n\nCall wiki_write when you establish something durable about this project, while\nyou still hold the context. Every claim needs at least one anchor -- a real file\npath, or path#symbol -- because an unanchored claim can never be checked against\nthe code again and would be served as current forever; unanchored writes are\nrefused.\n\nWorth recording: a decision and why the alternative was rejected, a failure and\nwhat actually caused it, a command that turned out to be the one that works.\nNot worth recording: what the code plainly says. Every wiki_write must include\nthe concrete evidence, when it applies, a calibrated confidence label, its\nscope, and what would invalidate it. Prefer the thing someone had to work out,\nbecause that exists nowhere in the source tree.\n\nRECORD BEFORE YOU ANSWER, NEVER AFTER. Your final message must be the answer to\nthe task -- never a confirmation that you saved a note. Measured: a run whose\nlast message was "Recorded. The finding captured: ..." scored 0.3 where the\nidentical task answered directly scored 0.9. The work had been done correctly\nand the artifact was written; only the last message was wrong, and the last\nmessage is what a verifier, a reviewer and the person who asked all read.${extractionNotice()}` - // WAS 158 CHARACTERS OF SELF-DESCRIPTION. "Structural graph capture remains - // active through lifecycle hooks. Durable semantic MCP writes are not - // requested because the writer schema is not proven available." -- true, - // and it changed no decision the model ever made with it. Fixed context is - // charged on every session; it has to earn its place by altering behaviour. - : ''; - - return `# Token optimization is active\n\n${outputDiscipline()}\n\n${connection}${routing}${enforcement}${utilities ? `\n\n${utilities}` : ''}${recording}${projectBriefing()}`; + // THE RECORDING GUIDANCE IS DELIVERED LATER, NOT HERE (#357, M5): see + // recordingGuidance. A session that never does real work never pays for it. + + return `# Token optimization is active\n\n${outputDiscipline()}\n\n${connection}${routing}${enforcement}${utilities ? `\n\n${utilities}` : ''}${projectBriefing()}`; } /** @@ -1870,9 +1879,26 @@ async function runHook(clientName, event, invocation) { // has already run) either way. Say it once per session per kind -- see // adviseOnce -- rather than on every matching call. const advisory = verdict && adviseOnce(state, verdict) ? verdict.reason : null; + // THE RECORDING GUIDANCE, ONCE, AFTER REAL WORK (#357, M5). It used to ride in + // every SessionStart, where it is the largest fixed cost a small session pays + // and buys nothing until there is something to record. The first edit is the + // clearest sign there will be; GUIDANCE_AFTER_CALLS tool calls is the fallback + // for an investigation that edits nothing but still works things out. + const guided = + event === 'post-tool' && + !state.recordingGuided && + toolEvidence.names.has('wiki_write') && + ((Number(state.edits) || 0) >= 1 || + (Number(state.toolCalls) || 0) >= GUIDANCE_AFTER_CALLS); + if (guided) state.recordingGuided = true; saveState(payload.session_id, state, agentScope); - const context = [advisory, graphContext, recordingContext] + const context = [ + advisory, + graphContext, + guided ? recordingGuidance() : null, + recordingContext, + ] .filter(Boolean) .join('\n\n'); if (context) { diff --git a/integrations/kilo/hooks/lib/policy.mjs b/integrations/kilo/hooks/lib/policy.mjs index c9eb1f896..a161899d3 100644 --- a/integrations/kilo/hooks/lib/policy.mjs +++ b/integrations/kilo/hooks/lib/policy.mjs @@ -394,6 +394,9 @@ function emptyState() { editedFiles: [], harvestedEdits: 0, recordingNudged: false, + // The recording guidance has been delivered (#357, M5); see + // recordingGuidance in adapter.mjs. + recordingGuided: false, // Facts the search advisory has already delivered this session. Without // this in the default, loadState returned an object with no `advised`, the // router rebuilt an empty Set on every hook process, and the same advisory @@ -487,6 +490,7 @@ export function loadState(sessionId, agent) { ? parsed.harvestedEdits : 0, recordingNudged: parsed.recordingNudged === true, + recordingGuided: parsed.recordingGuided === true, // Every hook call is a new process, so a field left out of this list is // forgotten between calls: that is how "said once" kept being said on // every search, and how a repeated WebFetch was never recognised @@ -651,6 +655,10 @@ export function saveState(sessionId, state, agent) { recordingNudged: Boolean( current.recordingNudged || state.recordingNudged ), + // Same rule: guidance said once stays said. + recordingGuided: Boolean( + current.recordingGuided || state.recordingGuided + ), // UNION, AND CAPPED. Two hook processes can each deliver an advisory // before either writes, so last-writer-wins would forget one and repeat // it. A fact once told stays told -- that is the whole point of the set diff --git a/integrations/opencode/hooks/lib/adapter.mjs b/integrations/opencode/hooks/lib/adapter.mjs index 89eea2645..a75f15023 100644 --- a/integrations/opencode/hooks/lib/adapter.mjs +++ b/integrations/opencode/hooks/lib/adapter.mjs @@ -892,6 +892,21 @@ function gradeObservedTools(state, event, payload, raw) { } /** The session-start notice, shared verbatim so no client drifts its own copy. */ +/** Tool calls after which a session that has edited nothing still gets the recording guidance. */ +export const GUIDANCE_AFTER_CALLS = 10; + +/** + * The wiki_write guidance. It was part of the SessionStart block until #357 + * (M5): it is the largest part of that block, and it only matters once there is + * something to record, so it is now delivered once, on the first post-tool + * event after the session shows real work, and a session that never gets that + * far never pays for it. The wording is unchanged, because it was measured -- + * see the last paragraph. + */ +export function recordingGuidance() { + return `## Record what you work out\n\nCall wiki_write when you establish something durable about this project, while\nyou still hold the context. Every claim needs at least one anchor -- a real file\npath, or path#symbol -- because an unanchored claim can never be checked against\nthe code again and would be served as current forever; unanchored writes are\nrefused.\n\nWorth recording: a decision and why the alternative was rejected, a failure and\nwhat actually caused it, a command that turned out to be the one that works.\nNot worth recording: what the code plainly says. Every wiki_write must include\nthe concrete evidence, when it applies, a calibrated confidence label, its\nscope, and what would invalidate it. Prefer the thing someone had to work out,\nbecause that exists nowhere in the source tree.\n\nRECORD BEFORE YOU ANSWER, NEVER AFTER. Your final message must be the answer to\nthe task -- never a confirmation that you saved a note. Measured: a run whose\nlast message was "Recorded. The finding captured: ..." scored 0.3 where the\nidentical task answered directly scored 0.9. The work had been done correctly\nand the artifact was written; only the last message was wrong, and the last\nmessage is what a verifier, a reviewer and the person who asked all read.${extractionNotice()}`; +} + export function policyText( canDeny, availableTools = undefined, @@ -972,16 +987,10 @@ export function policyText( ] .filter(Boolean) .join(' '); - const recording = tools.has('wiki_write') - ? `\n\n## Record what you work out\n\nCall wiki_write when you establish something durable about this project, while\nyou still hold the context. Every claim needs at least one anchor -- a real file\npath, or path#symbol -- because an unanchored claim can never be checked against\nthe code again and would be served as current forever; unanchored writes are\nrefused.\n\nWorth recording: a decision and why the alternative was rejected, a failure and\nwhat actually caused it, a command that turned out to be the one that works.\nNot worth recording: what the code plainly says. Every wiki_write must include\nthe concrete evidence, when it applies, a calibrated confidence label, its\nscope, and what would invalidate it. Prefer the thing someone had to work out,\nbecause that exists nowhere in the source tree.\n\nRECORD BEFORE YOU ANSWER, NEVER AFTER. Your final message must be the answer to\nthe task -- never a confirmation that you saved a note. Measured: a run whose\nlast message was "Recorded. The finding captured: ..." scored 0.3 where the\nidentical task answered directly scored 0.9. The work had been done correctly\nand the artifact was written; only the last message was wrong, and the last\nmessage is what a verifier, a reviewer and the person who asked all read.${extractionNotice()}` - // WAS 158 CHARACTERS OF SELF-DESCRIPTION. "Structural graph capture remains - // active through lifecycle hooks. Durable semantic MCP writes are not - // requested because the writer schema is not proven available." -- true, - // and it changed no decision the model ever made with it. Fixed context is - // charged on every session; it has to earn its place by altering behaviour. - : ''; - - return `# Token optimization is active\n\n${outputDiscipline()}\n\n${connection}${routing}${enforcement}${utilities ? `\n\n${utilities}` : ''}${recording}${projectBriefing()}`; + // THE RECORDING GUIDANCE IS DELIVERED LATER, NOT HERE (#357, M5): see + // recordingGuidance. A session that never does real work never pays for it. + + return `# Token optimization is active\n\n${outputDiscipline()}\n\n${connection}${routing}${enforcement}${utilities ? `\n\n${utilities}` : ''}${projectBriefing()}`; } /** @@ -1870,9 +1879,26 @@ async function runHook(clientName, event, invocation) { // has already run) either way. Say it once per session per kind -- see // adviseOnce -- rather than on every matching call. const advisory = verdict && adviseOnce(state, verdict) ? verdict.reason : null; + // THE RECORDING GUIDANCE, ONCE, AFTER REAL WORK (#357, M5). It used to ride in + // every SessionStart, where it is the largest fixed cost a small session pays + // and buys nothing until there is something to record. The first edit is the + // clearest sign there will be; GUIDANCE_AFTER_CALLS tool calls is the fallback + // for an investigation that edits nothing but still works things out. + const guided = + event === 'post-tool' && + !state.recordingGuided && + toolEvidence.names.has('wiki_write') && + ((Number(state.edits) || 0) >= 1 || + (Number(state.toolCalls) || 0) >= GUIDANCE_AFTER_CALLS); + if (guided) state.recordingGuided = true; saveState(payload.session_id, state, agentScope); - const context = [advisory, graphContext, recordingContext] + const context = [ + advisory, + graphContext, + guided ? recordingGuidance() : null, + recordingContext, + ] .filter(Boolean) .join('\n\n'); if (context) { diff --git a/integrations/opencode/hooks/lib/policy.mjs b/integrations/opencode/hooks/lib/policy.mjs index c9eb1f896..a161899d3 100644 --- a/integrations/opencode/hooks/lib/policy.mjs +++ b/integrations/opencode/hooks/lib/policy.mjs @@ -394,6 +394,9 @@ function emptyState() { editedFiles: [], harvestedEdits: 0, recordingNudged: false, + // The recording guidance has been delivered (#357, M5); see + // recordingGuidance in adapter.mjs. + recordingGuided: false, // Facts the search advisory has already delivered this session. Without // this in the default, loadState returned an object with no `advised`, the // router rebuilt an empty Set on every hook process, and the same advisory @@ -487,6 +490,7 @@ export function loadState(sessionId, agent) { ? parsed.harvestedEdits : 0, recordingNudged: parsed.recordingNudged === true, + recordingGuided: parsed.recordingGuided === true, // Every hook call is a new process, so a field left out of this list is // forgotten between calls: that is how "said once" kept being said on // every search, and how a repeated WebFetch was never recognised @@ -651,6 +655,10 @@ export function saveState(sessionId, state, agent) { recordingNudged: Boolean( current.recordingNudged || state.recordingNudged ), + // Same rule: guidance said once stays said. + recordingGuided: Boolean( + current.recordingGuided || state.recordingGuided + ), // UNION, AND CAPPED. Two hook processes can each deliver an advisory // before either writes, so last-writer-wins would forget one and repeat // it. A fact once told stays told -- that is the whole point of the set diff --git a/integrations/qwen/hooks/lib/adapter.mjs b/integrations/qwen/hooks/lib/adapter.mjs index 89eea2645..a75f15023 100644 --- a/integrations/qwen/hooks/lib/adapter.mjs +++ b/integrations/qwen/hooks/lib/adapter.mjs @@ -892,6 +892,21 @@ function gradeObservedTools(state, event, payload, raw) { } /** The session-start notice, shared verbatim so no client drifts its own copy. */ +/** Tool calls after which a session that has edited nothing still gets the recording guidance. */ +export const GUIDANCE_AFTER_CALLS = 10; + +/** + * The wiki_write guidance. It was part of the SessionStart block until #357 + * (M5): it is the largest part of that block, and it only matters once there is + * something to record, so it is now delivered once, on the first post-tool + * event after the session shows real work, and a session that never gets that + * far never pays for it. The wording is unchanged, because it was measured -- + * see the last paragraph. + */ +export function recordingGuidance() { + return `## Record what you work out\n\nCall wiki_write when you establish something durable about this project, while\nyou still hold the context. Every claim needs at least one anchor -- a real file\npath, or path#symbol -- because an unanchored claim can never be checked against\nthe code again and would be served as current forever; unanchored writes are\nrefused.\n\nWorth recording: a decision and why the alternative was rejected, a failure and\nwhat actually caused it, a command that turned out to be the one that works.\nNot worth recording: what the code plainly says. Every wiki_write must include\nthe concrete evidence, when it applies, a calibrated confidence label, its\nscope, and what would invalidate it. Prefer the thing someone had to work out,\nbecause that exists nowhere in the source tree.\n\nRECORD BEFORE YOU ANSWER, NEVER AFTER. Your final message must be the answer to\nthe task -- never a confirmation that you saved a note. Measured: a run whose\nlast message was "Recorded. The finding captured: ..." scored 0.3 where the\nidentical task answered directly scored 0.9. The work had been done correctly\nand the artifact was written; only the last message was wrong, and the last\nmessage is what a verifier, a reviewer and the person who asked all read.${extractionNotice()}`; +} + export function policyText( canDeny, availableTools = undefined, @@ -972,16 +987,10 @@ export function policyText( ] .filter(Boolean) .join(' '); - const recording = tools.has('wiki_write') - ? `\n\n## Record what you work out\n\nCall wiki_write when you establish something durable about this project, while\nyou still hold the context. Every claim needs at least one anchor -- a real file\npath, or path#symbol -- because an unanchored claim can never be checked against\nthe code again and would be served as current forever; unanchored writes are\nrefused.\n\nWorth recording: a decision and why the alternative was rejected, a failure and\nwhat actually caused it, a command that turned out to be the one that works.\nNot worth recording: what the code plainly says. Every wiki_write must include\nthe concrete evidence, when it applies, a calibrated confidence label, its\nscope, and what would invalidate it. Prefer the thing someone had to work out,\nbecause that exists nowhere in the source tree.\n\nRECORD BEFORE YOU ANSWER, NEVER AFTER. Your final message must be the answer to\nthe task -- never a confirmation that you saved a note. Measured: a run whose\nlast message was "Recorded. The finding captured: ..." scored 0.3 where the\nidentical task answered directly scored 0.9. The work had been done correctly\nand the artifact was written; only the last message was wrong, and the last\nmessage is what a verifier, a reviewer and the person who asked all read.${extractionNotice()}` - // WAS 158 CHARACTERS OF SELF-DESCRIPTION. "Structural graph capture remains - // active through lifecycle hooks. Durable semantic MCP writes are not - // requested because the writer schema is not proven available." -- true, - // and it changed no decision the model ever made with it. Fixed context is - // charged on every session; it has to earn its place by altering behaviour. - : ''; - - return `# Token optimization is active\n\n${outputDiscipline()}\n\n${connection}${routing}${enforcement}${utilities ? `\n\n${utilities}` : ''}${recording}${projectBriefing()}`; + // THE RECORDING GUIDANCE IS DELIVERED LATER, NOT HERE (#357, M5): see + // recordingGuidance. A session that never does real work never pays for it. + + return `# Token optimization is active\n\n${outputDiscipline()}\n\n${connection}${routing}${enforcement}${utilities ? `\n\n${utilities}` : ''}${projectBriefing()}`; } /** @@ -1870,9 +1879,26 @@ async function runHook(clientName, event, invocation) { // has already run) either way. Say it once per session per kind -- see // adviseOnce -- rather than on every matching call. const advisory = verdict && adviseOnce(state, verdict) ? verdict.reason : null; + // THE RECORDING GUIDANCE, ONCE, AFTER REAL WORK (#357, M5). It used to ride in + // every SessionStart, where it is the largest fixed cost a small session pays + // and buys nothing until there is something to record. The first edit is the + // clearest sign there will be; GUIDANCE_AFTER_CALLS tool calls is the fallback + // for an investigation that edits nothing but still works things out. + const guided = + event === 'post-tool' && + !state.recordingGuided && + toolEvidence.names.has('wiki_write') && + ((Number(state.edits) || 0) >= 1 || + (Number(state.toolCalls) || 0) >= GUIDANCE_AFTER_CALLS); + if (guided) state.recordingGuided = true; saveState(payload.session_id, state, agentScope); - const context = [advisory, graphContext, recordingContext] + const context = [ + advisory, + graphContext, + guided ? recordingGuidance() : null, + recordingContext, + ] .filter(Boolean) .join('\n\n'); if (context) { diff --git a/integrations/qwen/hooks/lib/policy.mjs b/integrations/qwen/hooks/lib/policy.mjs index c9eb1f896..a161899d3 100644 --- a/integrations/qwen/hooks/lib/policy.mjs +++ b/integrations/qwen/hooks/lib/policy.mjs @@ -394,6 +394,9 @@ function emptyState() { editedFiles: [], harvestedEdits: 0, recordingNudged: false, + // The recording guidance has been delivered (#357, M5); see + // recordingGuidance in adapter.mjs. + recordingGuided: false, // Facts the search advisory has already delivered this session. Without // this in the default, loadState returned an object with no `advised`, the // router rebuilt an empty Set on every hook process, and the same advisory @@ -487,6 +490,7 @@ export function loadState(sessionId, agent) { ? parsed.harvestedEdits : 0, recordingNudged: parsed.recordingNudged === true, + recordingGuided: parsed.recordingGuided === true, // Every hook call is a new process, so a field left out of this list is // forgotten between calls: that is how "said once" kept being said on // every search, and how a repeated WebFetch was never recognised @@ -651,6 +655,10 @@ export function saveState(sessionId, state, agent) { recordingNudged: Boolean( current.recordingNudged || state.recordingNudged ), + // Same rule: guidance said once stays said. + recordingGuided: Boolean( + current.recordingGuided || state.recordingGuided + ), // UNION, AND CAPPED. Two hook processes can each deliver an advisory // before either writes, so last-writer-wins would forget one and repeat // it. A fact once told stays told -- that is the whole point of the set diff --git a/integrations/windsurf/hooks/lib/adapter.mjs b/integrations/windsurf/hooks/lib/adapter.mjs index 89eea2645..a75f15023 100644 --- a/integrations/windsurf/hooks/lib/adapter.mjs +++ b/integrations/windsurf/hooks/lib/adapter.mjs @@ -892,6 +892,21 @@ function gradeObservedTools(state, event, payload, raw) { } /** The session-start notice, shared verbatim so no client drifts its own copy. */ +/** Tool calls after which a session that has edited nothing still gets the recording guidance. */ +export const GUIDANCE_AFTER_CALLS = 10; + +/** + * The wiki_write guidance. It was part of the SessionStart block until #357 + * (M5): it is the largest part of that block, and it only matters once there is + * something to record, so it is now delivered once, on the first post-tool + * event after the session shows real work, and a session that never gets that + * far never pays for it. The wording is unchanged, because it was measured -- + * see the last paragraph. + */ +export function recordingGuidance() { + return `## Record what you work out\n\nCall wiki_write when you establish something durable about this project, while\nyou still hold the context. Every claim needs at least one anchor -- a real file\npath, or path#symbol -- because an unanchored claim can never be checked against\nthe code again and would be served as current forever; unanchored writes are\nrefused.\n\nWorth recording: a decision and why the alternative was rejected, a failure and\nwhat actually caused it, a command that turned out to be the one that works.\nNot worth recording: what the code plainly says. Every wiki_write must include\nthe concrete evidence, when it applies, a calibrated confidence label, its\nscope, and what would invalidate it. Prefer the thing someone had to work out,\nbecause that exists nowhere in the source tree.\n\nRECORD BEFORE YOU ANSWER, NEVER AFTER. Your final message must be the answer to\nthe task -- never a confirmation that you saved a note. Measured: a run whose\nlast message was "Recorded. The finding captured: ..." scored 0.3 where the\nidentical task answered directly scored 0.9. The work had been done correctly\nand the artifact was written; only the last message was wrong, and the last\nmessage is what a verifier, a reviewer and the person who asked all read.${extractionNotice()}`; +} + export function policyText( canDeny, availableTools = undefined, @@ -972,16 +987,10 @@ export function policyText( ] .filter(Boolean) .join(' '); - const recording = tools.has('wiki_write') - ? `\n\n## Record what you work out\n\nCall wiki_write when you establish something durable about this project, while\nyou still hold the context. Every claim needs at least one anchor -- a real file\npath, or path#symbol -- because an unanchored claim can never be checked against\nthe code again and would be served as current forever; unanchored writes are\nrefused.\n\nWorth recording: a decision and why the alternative was rejected, a failure and\nwhat actually caused it, a command that turned out to be the one that works.\nNot worth recording: what the code plainly says. Every wiki_write must include\nthe concrete evidence, when it applies, a calibrated confidence label, its\nscope, and what would invalidate it. Prefer the thing someone had to work out,\nbecause that exists nowhere in the source tree.\n\nRECORD BEFORE YOU ANSWER, NEVER AFTER. Your final message must be the answer to\nthe task -- never a confirmation that you saved a note. Measured: a run whose\nlast message was "Recorded. The finding captured: ..." scored 0.3 where the\nidentical task answered directly scored 0.9. The work had been done correctly\nand the artifact was written; only the last message was wrong, and the last\nmessage is what a verifier, a reviewer and the person who asked all read.${extractionNotice()}` - // WAS 158 CHARACTERS OF SELF-DESCRIPTION. "Structural graph capture remains - // active through lifecycle hooks. Durable semantic MCP writes are not - // requested because the writer schema is not proven available." -- true, - // and it changed no decision the model ever made with it. Fixed context is - // charged on every session; it has to earn its place by altering behaviour. - : ''; - - return `# Token optimization is active\n\n${outputDiscipline()}\n\n${connection}${routing}${enforcement}${utilities ? `\n\n${utilities}` : ''}${recording}${projectBriefing()}`; + // THE RECORDING GUIDANCE IS DELIVERED LATER, NOT HERE (#357, M5): see + // recordingGuidance. A session that never does real work never pays for it. + + return `# Token optimization is active\n\n${outputDiscipline()}\n\n${connection}${routing}${enforcement}${utilities ? `\n\n${utilities}` : ''}${projectBriefing()}`; } /** @@ -1870,9 +1879,26 @@ async function runHook(clientName, event, invocation) { // has already run) either way. Say it once per session per kind -- see // adviseOnce -- rather than on every matching call. const advisory = verdict && adviseOnce(state, verdict) ? verdict.reason : null; + // THE RECORDING GUIDANCE, ONCE, AFTER REAL WORK (#357, M5). It used to ride in + // every SessionStart, where it is the largest fixed cost a small session pays + // and buys nothing until there is something to record. The first edit is the + // clearest sign there will be; GUIDANCE_AFTER_CALLS tool calls is the fallback + // for an investigation that edits nothing but still works things out. + const guided = + event === 'post-tool' && + !state.recordingGuided && + toolEvidence.names.has('wiki_write') && + ((Number(state.edits) || 0) >= 1 || + (Number(state.toolCalls) || 0) >= GUIDANCE_AFTER_CALLS); + if (guided) state.recordingGuided = true; saveState(payload.session_id, state, agentScope); - const context = [advisory, graphContext, recordingContext] + const context = [ + advisory, + graphContext, + guided ? recordingGuidance() : null, + recordingContext, + ] .filter(Boolean) .join('\n\n'); if (context) { diff --git a/integrations/windsurf/hooks/lib/policy.mjs b/integrations/windsurf/hooks/lib/policy.mjs index c9eb1f896..a161899d3 100644 --- a/integrations/windsurf/hooks/lib/policy.mjs +++ b/integrations/windsurf/hooks/lib/policy.mjs @@ -394,6 +394,9 @@ function emptyState() { editedFiles: [], harvestedEdits: 0, recordingNudged: false, + // The recording guidance has been delivered (#357, M5); see + // recordingGuidance in adapter.mjs. + recordingGuided: false, // Facts the search advisory has already delivered this session. Without // this in the default, loadState returned an object with no `advised`, the // router rebuilt an empty Set on every hook process, and the same advisory @@ -487,6 +490,7 @@ export function loadState(sessionId, agent) { ? parsed.harvestedEdits : 0, recordingNudged: parsed.recordingNudged === true, + recordingGuided: parsed.recordingGuided === true, // Every hook call is a new process, so a field left out of this list is // forgotten between calls: that is how "said once" kept being said on // every search, and how a repeated WebFetch was never recognised @@ -651,6 +655,10 @@ export function saveState(sessionId, state, agent) { recordingNudged: Boolean( current.recordingNudged || state.recordingNudged ), + // Same rule: guidance said once stays said. + recordingGuided: Boolean( + current.recordingGuided || state.recordingGuided + ), // UNION, AND CAPPED. Two hook processes can each deliver an advisory // before either writes, so last-writer-wins would forget one and repeat // it. A fact once told stays told -- that is the whole point of the set diff --git a/plugin/hooks/lib/adapter.mjs b/plugin/hooks/lib/adapter.mjs index 89eea2645..a75f15023 100644 --- a/plugin/hooks/lib/adapter.mjs +++ b/plugin/hooks/lib/adapter.mjs @@ -892,6 +892,21 @@ function gradeObservedTools(state, event, payload, raw) { } /** The session-start notice, shared verbatim so no client drifts its own copy. */ +/** Tool calls after which a session that has edited nothing still gets the recording guidance. */ +export const GUIDANCE_AFTER_CALLS = 10; + +/** + * The wiki_write guidance. It was part of the SessionStart block until #357 + * (M5): it is the largest part of that block, and it only matters once there is + * something to record, so it is now delivered once, on the first post-tool + * event after the session shows real work, and a session that never gets that + * far never pays for it. The wording is unchanged, because it was measured -- + * see the last paragraph. + */ +export function recordingGuidance() { + return `## Record what you work out\n\nCall wiki_write when you establish something durable about this project, while\nyou still hold the context. Every claim needs at least one anchor -- a real file\npath, or path#symbol -- because an unanchored claim can never be checked against\nthe code again and would be served as current forever; unanchored writes are\nrefused.\n\nWorth recording: a decision and why the alternative was rejected, a failure and\nwhat actually caused it, a command that turned out to be the one that works.\nNot worth recording: what the code plainly says. Every wiki_write must include\nthe concrete evidence, when it applies, a calibrated confidence label, its\nscope, and what would invalidate it. Prefer the thing someone had to work out,\nbecause that exists nowhere in the source tree.\n\nRECORD BEFORE YOU ANSWER, NEVER AFTER. Your final message must be the answer to\nthe task -- never a confirmation that you saved a note. Measured: a run whose\nlast message was "Recorded. The finding captured: ..." scored 0.3 where the\nidentical task answered directly scored 0.9. The work had been done correctly\nand the artifact was written; only the last message was wrong, and the last\nmessage is what a verifier, a reviewer and the person who asked all read.${extractionNotice()}`; +} + export function policyText( canDeny, availableTools = undefined, @@ -972,16 +987,10 @@ export function policyText( ] .filter(Boolean) .join(' '); - const recording = tools.has('wiki_write') - ? `\n\n## Record what you work out\n\nCall wiki_write when you establish something durable about this project, while\nyou still hold the context. Every claim needs at least one anchor -- a real file\npath, or path#symbol -- because an unanchored claim can never be checked against\nthe code again and would be served as current forever; unanchored writes are\nrefused.\n\nWorth recording: a decision and why the alternative was rejected, a failure and\nwhat actually caused it, a command that turned out to be the one that works.\nNot worth recording: what the code plainly says. Every wiki_write must include\nthe concrete evidence, when it applies, a calibrated confidence label, its\nscope, and what would invalidate it. Prefer the thing someone had to work out,\nbecause that exists nowhere in the source tree.\n\nRECORD BEFORE YOU ANSWER, NEVER AFTER. Your final message must be the answer to\nthe task -- never a confirmation that you saved a note. Measured: a run whose\nlast message was "Recorded. The finding captured: ..." scored 0.3 where the\nidentical task answered directly scored 0.9. The work had been done correctly\nand the artifact was written; only the last message was wrong, and the last\nmessage is what a verifier, a reviewer and the person who asked all read.${extractionNotice()}` - // WAS 158 CHARACTERS OF SELF-DESCRIPTION. "Structural graph capture remains - // active through lifecycle hooks. Durable semantic MCP writes are not - // requested because the writer schema is not proven available." -- true, - // and it changed no decision the model ever made with it. Fixed context is - // charged on every session; it has to earn its place by altering behaviour. - : ''; - - return `# Token optimization is active\n\n${outputDiscipline()}\n\n${connection}${routing}${enforcement}${utilities ? `\n\n${utilities}` : ''}${recording}${projectBriefing()}`; + // THE RECORDING GUIDANCE IS DELIVERED LATER, NOT HERE (#357, M5): see + // recordingGuidance. A session that never does real work never pays for it. + + return `# Token optimization is active\n\n${outputDiscipline()}\n\n${connection}${routing}${enforcement}${utilities ? `\n\n${utilities}` : ''}${projectBriefing()}`; } /** @@ -1870,9 +1879,26 @@ async function runHook(clientName, event, invocation) { // has already run) either way. Say it once per session per kind -- see // adviseOnce -- rather than on every matching call. const advisory = verdict && adviseOnce(state, verdict) ? verdict.reason : null; + // THE RECORDING GUIDANCE, ONCE, AFTER REAL WORK (#357, M5). It used to ride in + // every SessionStart, where it is the largest fixed cost a small session pays + // and buys nothing until there is something to record. The first edit is the + // clearest sign there will be; GUIDANCE_AFTER_CALLS tool calls is the fallback + // for an investigation that edits nothing but still works things out. + const guided = + event === 'post-tool' && + !state.recordingGuided && + toolEvidence.names.has('wiki_write') && + ((Number(state.edits) || 0) >= 1 || + (Number(state.toolCalls) || 0) >= GUIDANCE_AFTER_CALLS); + if (guided) state.recordingGuided = true; saveState(payload.session_id, state, agentScope); - const context = [advisory, graphContext, recordingContext] + const context = [ + advisory, + graphContext, + guided ? recordingGuidance() : null, + recordingContext, + ] .filter(Boolean) .join('\n\n'); if (context) { diff --git a/plugin/hooks/lib/policy.mjs b/plugin/hooks/lib/policy.mjs index c9eb1f896..a161899d3 100644 --- a/plugin/hooks/lib/policy.mjs +++ b/plugin/hooks/lib/policy.mjs @@ -394,6 +394,9 @@ function emptyState() { editedFiles: [], harvestedEdits: 0, recordingNudged: false, + // The recording guidance has been delivered (#357, M5); see + // recordingGuidance in adapter.mjs. + recordingGuided: false, // Facts the search advisory has already delivered this session. Without // this in the default, loadState returned an object with no `advised`, the // router rebuilt an empty Set on every hook process, and the same advisory @@ -487,6 +490,7 @@ export function loadState(sessionId, agent) { ? parsed.harvestedEdits : 0, recordingNudged: parsed.recordingNudged === true, + recordingGuided: parsed.recordingGuided === true, // Every hook call is a new process, so a field left out of this list is // forgotten between calls: that is how "said once" kept being said on // every search, and how a repeated WebFetch was never recognised @@ -651,6 +655,10 @@ export function saveState(sessionId, state, agent) { recordingNudged: Boolean( current.recordingNudged || state.recordingNudged ), + // Same rule: guidance said once stays said. + recordingGuided: Boolean( + current.recordingGuided || state.recordingGuided + ), // UNION, AND CAPPED. Two hook processes can each deliver an advisory // before either writes, so last-writer-wins would forget one and repeat // it. A fact once told stays told -- that is the whole point of the set diff --git a/tests/hooks/assist-mode.test.mjs b/tests/hooks/assist-mode.test.mjs index 5a166bdc7..675d334fd 100644 --- a/tests/hooks/assist-mode.test.mjs +++ b/tests/hooks/assist-mode.test.mjs @@ -30,8 +30,10 @@ import { mode, MODE_ASSIST, MODE_ENFORCE, MODE_OFF } from '../../hooks-core/poli const HERE = dirname(fileURLToPath(import.meta.url)); const ROUTER = join(HERE, '..', '..', 'plugin', 'hooks', 'pretooluse-router.mjs'); const SESSION_START = join(HERE, '..', '..', 'plugin', 'hooks', 'session-start.mjs'); +const POST_TOOL = join(HERE, '..', '..', 'plugin', 'hooks', 'post-tool.mjs'); const ISOLATED_GRAPH = mkdtempSync(join(tmpdir(), 'assist-graph-')); +const ISOLATED_STATE = mkdtempSync(join(tmpdir(), 'assist-state-')); let workspace; let big; @@ -62,6 +64,10 @@ function runHook(script, payload, modeValue) { 'smart_read,smart_write,smart_edit,smart_glob,smart_grep,wiki_write', TOKEN_OPTIMIZER_WIKI_DIR: ISOLATED_GRAPH, TOKEN_OPTIMIZER_SHARED_DIR: ISOLATED_GRAPH, + // Session state is per run, not per machine: the session ids restart at + // 1 every run, so a shared state root handed one run the "already said" + // flags an earlier run had saved. + TOKEN_OPTIMIZER_STATE_DIR: ISOLATED_STATE, }, }); } @@ -146,18 +152,73 @@ describe('assist does not advertise routing it will not perform', () => { // The hook actually ran and emitted its payload. Three absences alone are // also satisfied by a crashed hook returning '' -- which would suppress the // routing advisory for entirely the wrong reason, and look identical here. - expect(out).toContain('Record what you work out'); + expect(out).toContain('Token optimization is active'); expect(out).not.toContain('Prefer only the registered'); expect(out).not.toContain('These are recommendations'); expect(out).not.toContain('denied only when'); }); - test('but the graph guidance survives, which is the point of assist', () => { - // Guards against over-correction: suppressing the whole payload would make - // assist indistinguishable from off. - expect(startUnder('assist')).toContain('Record what you work out'); + test('the graph guidance survives, deferred to the first sign of real work (#357, M5)', () => { + // Guards against over-correction: suppressing the guidance would make + // assist indistinguishable from off. It is no longer in SessionStart, + // where every session paid for it, but the first post-tool event after an + // edit delivers it -- once. + const session = `assist-deferred-${++sessionSeq}`; + const start = runHook( + SESSION_START, + { hook_event_name: 'SessionStart', cwd: workspace, session_id: session }, + 'assist' + ).stdout; + expect(start).toContain('Token optimization is active'); + expect(start).not.toContain('Record what you work out'); + const edit = { + hook_event_name: 'PostToolUse', + cwd: workspace, + session_id: session, + tool_name: 'Edit', + tool_input: { file_path: big, old_string: 'x', new_string: 'y' }, + tool_response: { success: true }, + }; + expect(runHook(POST_TOOL, edit, 'assist').stdout).toContain( + 'Record what you work out' + ); + expect(runHook(POST_TOOL, edit, 'assist').stdout).not.toContain( + 'Record what you work out' + ); }); + test('an investigation that edits nothing gets it after GUIDANCE_AFTER_CALLS tool calls', () => { + const session = `assist-calls-${++sessionSeq}`; + const bash = { + hook_event_name: 'PostToolUse', + cwd: workspace, + session_id: session, + tool_name: 'Bash', + tool_input: { command: 'git status' }, + tool_response: { stdout: 'clean', exitCode: 0 }, + }; + const read = (n) => + runHook( + ROUTER, + { + hook_event_name: 'PreToolUse', + cwd: workspace, + session_id: session, + tool_name: 'Read', + tool_input: { file_path: join(workspace, `f${n}.ts`) }, + }, + 'assist' + ); + for (let n = 0; n < 9; n++) read(n); + expect(runHook(POST_TOOL, bash, 'assist').stdout).not.toContain( + 'Record what you work out' + ); + read(9); + expect(runHook(POST_TOOL, bash, 'assist').stdout).toContain( + 'Record what you work out' + ); + }, 60_000); + test('enforce still advertises routing, so this is scoped to assist', () => { const out = startUnder('enforce'); expect(out).toContain('Prefer only the registered'); diff --git a/tests/hooks/codex-semantic-harvest.test.mjs b/tests/hooks/codex-semantic-harvest.test.mjs index 0f61fc3e2..d7fc7fa61 100644 --- a/tests/hooks/codex-semantic-harvest.test.mjs +++ b/tests/hooks/codex-semantic-harvest.test.mjs @@ -68,10 +68,17 @@ describe('Codex semantic harvest lifecycle', () => { writeFileSync(first, 'export const first = 1;\n'); writeFileSync(second, 'export const second = 2;\n'); - expect(run(POST, editPayload('semantic-1', first))).toBeNull(); + // The first edit is the first sign of real work, so it carries the deferred + // recording guidance (#357, M5) -- once. The nudge naming the files waits + // for the second edit, as before. + const opening = run(POST, editPayload('semantic-1', first)); + expect(opening.hookSpecificOutput.additionalContext).toContain( + 'Record what you work out' + ); const output = run(POST, editPayload('semantic-1', second)); const context = output.hookSpecificOutput.additionalContext; + expect(context).not.toContain('Record what you work out'); expect(context).toContain('wiki_write'); expect(context).toContain('first.ts'); expect(context).toContain('second.ts'); diff --git a/tests/hooks/mcp-capability-negotiation.test.mjs b/tests/hooks/mcp-capability-negotiation.test.mjs index 3361d1e9e..3fe4227b2 100644 --- a/tests/hooks/mcp-capability-negotiation.test.mjs +++ b/tests/hooks/mcp-capability-negotiation.test.mjs @@ -260,9 +260,17 @@ describe('session policy honesty', () => { }); test('mentions only schemas proven present', () => { - const text = policyText(true, new Set(['smart_grep', 'wiki_write']), true); + // optimize_session, not wiki_write: since #357 (M5) the wiki_write guidance + // is delivered after real work rather than at SessionStart, so this block + // no longer names it either way. + const text = policyText( + true, + new Set(['smart_grep', 'optimize_session']), + true + ); expect(text).toContain('smart_grep'); - expect(text).toContain('wiki_write'); + expect(text).toContain('optimize_session'); expect(text).not.toContain('smart_read'); + expect(text).not.toContain('get_optimization_report'); }); }); diff --git a/tests/hooks/policy-asks-for-findings.test.mjs b/tests/hooks/policy-asks-for-findings.test.mjs index be886035e..f71fa6672 100644 --- a/tests/hooks/policy-asks-for-findings.test.mjs +++ b/tests/hooks/policy-asks-for-findings.test.mjs @@ -29,10 +29,15 @@ const ADAPTER = pathToFileURL( ).href; let policyText; +let recordingGuidance; let volatileLines; +// THE GUIDANCE MOVED OUT OF policyText (#357, M5): SessionStart no longer +// carries it, and the first post-tool event after real work delivers +// recordingGuidance() once instead. What it must say is unchanged, so these +// assertions read it from where it now lives. beforeAll(async () => { - ({ policyText } = await import(ADAPTER)); + ({ policyText, recordingGuidance } = await import(ADAPTER)); ({ volatileLines } = await import( pathToFileURL(join(process.cwd(), 'hooks-core', 'cache.mjs')).href )); @@ -40,12 +45,12 @@ beforeAll(async () => { describe('the standing policy asks for findings', () => { it('names the tool that records them', () => { - expect(policyText(true)).toMatch(/wiki_write/); + expect(recordingGuidance()).toMatch(/wiki_write/); }); it('states the anchor requirement, which the tool enforces', () => { // Without this the agent learns the rule by having writes refused. - expect(policyText(true)).toMatch(/anchor/i); + expect(recordingGuidance()).toMatch(/anchor/i); }); it('says what is worth recording, not merely that recording exists', () => { @@ -53,14 +58,16 @@ describe('the standing policy asks for findings', () => { // specific -- what was TRIED AND REJECTED, and why, because that exists // nowhere in the source tree -- and the briefing has to carry that same // specificity or it produces restatements of what the code already says. - expect(policyText(true)).toMatch(/rejected|failure|decision/i); + expect(recordingGuidance()).toMatch(/rejected|failure|decision/i); }); - it('keeps the briefing in both enforcement modes', () => { - // policyText(false) is the advise-only wording used by clients that cannot - // deny a tool call. Recording findings is not an enforcement feature, so it - // must not disappear along with the denial language. - expect(policyText(false)).toMatch(/wiki_write/); + it('is the same briefing in both enforcement modes, and in neither SessionStart', () => { + // Recording findings is not an enforcement feature, so it must not depend on + // the denial wording either mode uses: it is one text, delivered after the + // session has done real work, and neither policy block repeats it. + expect(recordingGuidance()).toMatch(/wiki_write/); + expect(policyText(true)).not.toMatch(/Record what you work out/); + expect(policyText(false)).not.toMatch(/Record what you work out/); }); }); @@ -108,7 +115,7 @@ describe('the standing policy states what will extract findings after the sessio // that was buried deepest: no credential, no billing, no digest sent. process.env.TOKEN_OPTIMIZER_HARVEST_ENDPOINT = 'http://127.0.0.1:11434/v1/chat/completions'; - const text = policyText(true); + const text = recordingGuidance(); expect(text).toMatch(/local model endpoint is configured/i); expect(text).toMatch(/free and private/i); @@ -117,7 +124,7 @@ describe('the standing policy states what will extract findings after the sessio it('names what a credentialed harvest sends, and what it never sends', () => { process.env.TOKEN_OPTIMIZER_API_KEY = 'sk-test'; - const text = policyText(true); + const text = recordingGuidance(); expect(text).toMatch(/bounded digest/i); expect(text).toMatch(/never file contents/i); @@ -126,7 +133,7 @@ describe('the standing policy states what will extract findings after the sessio it('on a machine with no credential, states the consequence and names the free option', () => { // THE DEFAULT STATE, and the whole reason this line exists. `off:no-key` is // CI, corporate laptops and every subscription-only login. - const text = policyText(true); + const text = recordingGuidance(); expect(text).toMatch(/no separate extractor will run/i); expect(text).toMatch(/derived locally/i); @@ -138,7 +145,7 @@ describe('the standing policy states what will extract findings after the sessio // and this one is charged to every session's prefix. process.env.TOKEN_OPTIMIZER_HARVEST = '0'; process.env.TOKEN_OPTIMIZER_API_KEY = 'sk-test'; - const text = policyText(true); + const text = recordingGuidance(); expect(text).toMatch(/off by configuration/i); expect(text).toMatch(/derived locally/i); @@ -163,6 +170,7 @@ describe('the standing policy states what will extract findings after the sessio for (const key of KEYS) delete process.env[key]; Object.assign(process.env, env); expect(volatileLines(policyText(true))).toHaveLength(0); + expect(volatileLines(recordingGuidance())).toHaveLength(0); } }); }); From fecfa618750735530f7201b6efa9a3232f5f8e41 Mon Sep 17 00:00:00 2001 From: Franklin Moormann Date: Sat, 17 Oct 2026 21:14:33 -0400 Subject: [PATCH 3/5] fix(bench): keep recall truth out of the session container; resumable units Review fixes on #507, and two defects found while making `run` resumable: - truth is no longer mounted into the session's container: THOL runs agent and verifier as one user on one filesystem, so the agent could read it. Inside the rig the verifier records a deferred 0; workspaces are kept and the new `score` command runs every verifier on the host; `report` uses those. - one unit = one pair, arm and repetition, with a new directory, empty graph and its own THOL results volume per attempt. `run` records finished units and skips them, and a failed unit restarts from an empty graph. - found while doing that: THOL_CAMPAIGN was a free-text label, but THOL treats it as the Claude Code version pin and exits on a mismatch, so no run could have started; and with every session at --reps 1 in one shared volume, THOL's own resume would have skipped repetitions 2 and 3 as done. It is now the pin (--campaign, default as run-campaign.sh), and volumes are per unit. - `plan` refuses a non-empty output directory, so no earlier graph survives. - derive is scored on regex conclusions with contradiction patterns: the right words in a sentence that denies the cause score 0. The quota pair's "second" term matched inside "millisecond"; it is now a word match. - the fixture check compares the whole inventory, so an added file (e.g. migrations/0_x.sql) scores 0; bytecode and dot-directories are ignored. - TZ=UTC+10 and other offsets no longer match the UTC answer. test_recall.py: 23 tests (1 skipped on Windows). Each of the three verifier fixes was mutated out and its test failed. Co-Authored-By: Claude Opus 5.5 --- bench/recall/README.md | 27 +- .../recall/fixtures/truth/recall-migrate.json | 28 +- bench/recall/fixtures/truth/recall-quota.json | 22 +- bench/recall/fixtures/truth/recall-tz.json | 30 ++- bench/recall/recall.py | 234 +++++++++++------ bench/recall/tasks/_recall.py | 62 ++++- bench/recall/tasks/_vlib.py | 12 +- bench/recall/test_recall.py | 239 ++++++++++++++---- 8 files changed, 466 insertions(+), 188 deletions(-) diff --git a/bench/recall/README.md b/bench/recall/README.md index 377893799..df5526ff8 100644 --- a/bench/recall/README.md +++ b/bench/recall/README.md @@ -18,16 +18,23 @@ Three pairs, each a real defect in a small standard-library Python project: | `recall-migrate` | `migrun` | migrations sort as strings | which migration a fresh database runs first | The tasks are THOL tasks (`prompt.md`, `task.json`, `verify.py`) and run on the -rig in `bench/thol`. Each arm, pair and repetition gets its own empty graph -directory, mounted for that pair's two sessions only. **Calls avoided** is the +rig in `bench/thol`. Each arm, pair and repetition is one unit: every attempt +at it gets a new directory with an empty graph and its own THOL results +volume, shared by that unit's two sessions only. **Calls avoided** is the control arm's reuse tool calls minus the arm's, counted only for repetitions in which both answered correctly: a cheap wrong answer is not a saving. +**The truth never enters a session's container.** THOL runs the agent and its +verifier as one user on one filesystem, so a mounted truth file is one the +agent could read. None is mounted: in the container the verifier records a +deferred 0, workspaces are kept, and `score` re-runs every verifier on the host. + ## Verifiers Scored from ground truth in `fixtures/truth/`, which the agent never sees. A -verifier awards nothing for an empty workspace, a changed fixture file, an empty -reply, or (on `reuse`) the right answer hedged with a wrong one. +verifier awards nothing for an empty workspace, a fixture file changed, removed +or added, an empty reply, a `derive` explanation that states the cause in the +right words but denies it, or a `reuse` answer hedged with a wrong one. python3 -m unittest discover -s bench/recall -p "test_*.py" @@ -40,10 +47,14 @@ runs it on every pull request. python3 bench/recall/recall.py plan out/recall --reps 3 python3 bench/recall/recall.py run out/recall --approve-spend - python3 bench/recall/recall.py report out/recall/results/results.sqlite - -`plan` prints every docker command and spends nothing. `run` refuses without -`--approve-spend`, because every step is a real, billed Claude Code session. + python3 bench/recall/recall.py score out/recall + python3 bench/recall/recall.py report out/recall + +`plan` needs a new directory and prints every docker command; it spends +nothing. `run` refuses without `--approve-spend`, because every session is a +real, billed Claude Code session. Re-running `run` after a failure skips every +finished unit and restarts the failed one from an empty graph. `--campaign` is +THOL's Claude Code version pin, not a label. The default arms are `control` and `token-optimizer-proxy-knowledge`, whose manifest points the graph at `/proxy-graph`. diff --git a/bench/recall/fixtures/truth/recall-migrate.json b/bench/recall/fixtures/truth/recall-migrate.json index f206c3f04..58b097b50 100644 --- a/bench/recall/fixtures/truth/recall-migrate.json +++ b/bench/recall/fixtures/truth/recall-migrate.json @@ -3,20 +3,13 @@ "fixture": "migrun", "marker": "migrun/runner.py", "derive": { - "groups": [ - [ - "lexicographic", - "alphabetical", - "as strings", - "as text", - "string order", - "string sort", - "lexical", - "not numerically" - ], - [ - "10_" - ] + "require": [ + "lexicograph|alphabetic|as strings|as text|string (?:order|sort)|lexical|not numerically|character by character", + "\\b10_" + ], + "contradict": [ + "\\b1_create_users(?:\\.sql)?\\s+(?:is\\s+first|(?:is\\s+|will\\s+be\\s+)?(?:run|runs|applied|applies|comes|sorts|goes)\\s+first)", + "(? [--arms A,B] [--reps N] + python3 bench/recall/recall.py plan [--arms A,B] [--reps N] python3 bench/recall/recall.py run --approve-spend - python3 bench/recall/recall.py report + python3 bench/recall/recall.py score + python3 bench/recall/recall.py report Each pair is two THOL tasks run back to back on the rig (bench/thol): `derive` works something out that the workspace does not state, then `reuse` starts a FRESH session in a fresh copy of the same workspace and asks a question whose answer depends on it. The knowledge graph is the one thing that may survive -from one to the other: every (arm, pair, repetition) gets its own empty graph -directory, mounted at /proxy-graph for both sessions and for nothing else, so -nothing can leak between pairs or between arms. +from one to the other. + +ONE UNIT = ONE PAIR, ONE ARM, ONE REPETITION, AND IT IS ATOMIC. Every attempt +at a unit gets a new directory holding a new, empty graph and its own THOL +results volume. Its own volume because THOL resumes by skipping any +(competitor, task, rep) already recorded at the same Claude Code version, and +every session here is `--reps 1`: in a shared volume the second repetition of +a task would be skipped as done. A unit is recorded as done only when both of +its sessions finished; `run` skips done units, so re-running after a failure +repeats no completed (billed) unit, and the failed one starts again from an +empty graph rather than inheriting what half of it wrote. + +THE TRUTH NEVER ENTERS A SESSION'S CONTAINER. THOL runs the agent and its +verifier as one user on one filesystem, so a mounted truth file is one the +agent can read. No truth is mounted; the in-container verifier records a +deferred 0, workspaces are kept (`--keep-workspaces`), and `score` runs each +verifier on the host against the kept workspace and answer. `report` uses +those scores only. THE MEASUREMENT IS CALLS AVOIDED, COUNTED ONLY WHERE BOTH ARMS ARE RIGHT. The -control arm has no graph, so its reuse session must do the work again; an arm -that carries the conclusion should answer with fewer tool calls. A wrong reuse -answer is not a cheap one, so a pair counts toward calls avoided only for the -repetitions in which both arms scored 1. - -`plan` writes the exact docker commands and the per-session arm-order files -and spends nothing. `run` executes them and refuses without --approve-spend, -because every session is a real Claude Code run. +control arm has no graph, so its reuse session must do the work again. A wrong +reuse answer is not a cheap one, so a repetition counts toward calls avoided +only when both arms' reuse answers scored 1. + +`plan` spends nothing. `run` refuses without --approve-spend, because every +session is a real, billed Claude Code session. """ import argparse import json -from contextlib import closing +import os import shlex import sqlite3 import subprocess import sys +from contextlib import closing from pathlib import Path HERE = Path(__file__).resolve().parent RIG = HERE.parent / "thol" THOL_HOME = "/home/bench/thol" DEFAULT_ARMS = "control,token-optimizer-proxy-knowledge" +# THOL_CAMPAIGN IS A VERSION PIN, NOT A LABEL: runner.py exits unless the +# installed Claude Code's `--version` starts with it. Same default as +# bench/thol/run-campaign.sh. +DEFAULT_CAMPAIGN = "2.1.251 (Claude Code)" +ROLES = ("derive", "reuse") def pairs(): @@ -46,96 +66,167 @@ def pairs(): return sorted(records, key=lambda t: t["pair"]) -def session(pair, role, arm, rep, out, image, graph): +def session(unit, role, attempt_dir, out, plan): """One docker invocation of the rig's campaign mode for one session.""" - task = f"{pair['pair']}-{role}" - order = out / "orders" / f"{arm}.json" - label = f"recall {pair['pair']} {arm} rep{rep} {role}" + task = f"{unit['pair']}-{role}" mounts = [ (RIG / "auth", "/auth", "ro"), - (out / "results", "/results", "rw"), - (graph, "/proxy-graph", "rw"), - (order, "/arm-order.json", "ro"), + (attempt_dir / "results", "/results", "rw"), + (attempt_dir / "graph", "/proxy-graph", "rw"), + (out / "orders" / f"{unit['arm']}.json", "/arm-order.json", "ro"), (HERE / "tasks" / "_recall.py", f"{THOL_HOME}/tasks/_recall.py", "ro"), (HERE / "tasks" / task, f"{THOL_HOME}/tasks/{task}", "ro"), - (HERE / "fixtures" / pair["fixture"], f"{THOL_HOME}/fixtures/out/{pair['fixture']}", "ro"), - ( - HERE / "fixtures" / "truth" / f"{pair['pair']}.json", - f"{THOL_HOME}/fixtures/truth/{pair['pair']}.json", - "ro", - ), + (HERE / "fixtures" / unit["fixture"], f"{THOL_HOME}/fixtures/out/{unit['fixture']}", "ro"), ] command = ["docker", "run", "--rm"] for host, inside, mode in mounts: command += ["-v", f"{host}:{inside}:{mode}"] command += [ "-e", "THOL_ARM_ORDER_FILE=/arm-order.json", - "-e", f"THOL_CAMPAIGN={label}", - image, "campaign", "-c", arm, "-t", task, "--reps", "1", + "-e", f"THOL_CAMPAIGN={plan['campaign']}", + plan["image"], "campaign", + "-c", unit["arm"], "-t", task, "--reps", "1", "--keep-workspaces", ] return command +def load(out): + return json.loads((out / "plan.json").read_text(encoding="utf-8")) + + +def progress(out): + path = out / "progress.json" + if path.exists(): + return json.loads(path.read_text(encoding="utf-8")) + return {"done": {}, "attempts": {}} + + +def save_progress(out, state): + tmp = out / "progress.json.tmp" + tmp.write_text(json.dumps(state, indent=1) + "\n", encoding="utf-8") + os.replace(tmp, out / "progress.json") + + def plan(args): out = Path(args.out).resolve() + # A NEW DIRECTORY, ALWAYS. Re-planning into an old one would hand the next + # derive session whatever graph the last plan's sessions wrote. + if out.exists() and any(out.iterdir()): + sys.exit(f"recall: {out} is not empty; plan into a new directory") arms = args.arms.split(",") if len(set(arms)) != len(arms) or "control" not in arms: sys.exit("recall: arms must be distinct and include control, which calls avoided is measured against") - (out / "orders").mkdir(parents=True, exist_ok=True) - (out / "results").mkdir(exist_ok=True) + (out / "orders").mkdir(parents=True) for arm in arms: (out / "orders" / f"{arm}.json").write_text( json.dumps({"arms": [arm], "orders": [[arm]]}) + "\n", encoding="utf-8" ) - steps = [] - for rep in range(1, args.reps + 1): - for pair in pairs(): - for arm in arms: - graph = out / "graphs" / f"{pair['pair']}-{arm}-rep{rep}" - graph.mkdir(parents=True, exist_ok=True) - for role in ("derive", "reuse"): - steps.append(session(pair, role, arm, rep, out, args.image, graph)) - (out / "plan.json").write_text(json.dumps(steps, indent=1) + "\n", encoding="utf-8") - for step in steps: - print(shlex.join(str(part) for part in step)) - print(f"\n{len(steps)} session(s) planned in {out}; nothing has run and nothing was spent.") - - -def run(args): + units = [ + {"id": f"{p['pair']}-{arm}-rep{rep}", "pair": p["pair"], "fixture": p["fixture"], "arm": arm, "rep": rep} + for rep in range(1, args.reps + 1) + for p in pairs() + for arm in arms + ] + record = {"campaign": args.campaign, "image": args.image, "units": units} + (out / "plan.json").write_text(json.dumps(record, indent=1) + "\n", encoding="utf-8") + for unit in units: + for role in ROLES: + print(shlex.join(str(part) for part in session(unit, role, out / "units" / unit["id"] / "try1", out, record))) + print(f"\n{len(units)} unit(s), {2 * len(units)} session(s) planned in {out}; nothing has run and nothing was spent.") + + +def run(args, execute=subprocess.run): if not args.approve_spend: sys.exit("recall: refusing to run without --approve-spend; every step is a real, billed Claude Code session") out = Path(args.out).resolve() - steps = json.loads((out / "plan.json").read_text(encoding="utf-8")) - for n, step in enumerate(steps, 1): - print(f"[{n}/{len(steps)}] {step[step.index('-t') + 1]} as {step[step.index('-c') + 1]}", flush=True) - if subprocess.run(step).returncode != 0: - sys.exit(f"recall: step {n} failed; the steps before it are recorded and the plan can be re-run from it") + record = load(out) + state = progress(out) + for n, unit in enumerate(record["units"], 1): + if unit["id"] in state["done"]: + continue + attempt = state["attempts"].get(unit["id"], 0) + 1 + state["attempts"][unit["id"]] = attempt + save_progress(out, state) + attempt_dir = out / "units" / unit["id"] / f"try{attempt}" + (attempt_dir / "graph").mkdir(parents=True) + (attempt_dir / "results").mkdir() + for role in ROLES: + print(f"[{n}/{len(record['units'])}] {unit['id']} {role} (attempt {attempt})", flush=True) + if execute(session(unit, role, attempt_dir, out, record)).returncode != 0: + sys.exit( + f"recall: {unit['id']} {role} failed. Completed units are recorded; run again " + "to resume, and this unit restarts from an empty graph." + ) + state["done"][unit["id"]] = attempt_dir.relative_to(out).as_posix() + save_progress(out, state) + + +def sessions_of(out, unit_dir): + """THOL's rows for one attempt, with where each run's workspace and answer are.""" + results = out / unit_dir / "results" + with closing(sqlite3.connect(results / "results.sqlite")) as db: + rows = db.execute( + "SELECT run_id, task, tool_calls, num_turns, total_cost_usd FROM runs WHERE status = 'ok'" + ).fetchall() + for run_id, task, calls, turns, cost in rows: + yield { + "run_id": run_id, "task": task, "calls": calls, "turns": turns, "cost": cost, + "workspace": results / "runs" / run_id / "workspace", + "answer": results / "artifacts" / run_id / "answer.txt", + } + + +def verify_on_host(task, workspace, answer): + """Run a task's verifier against the real truth, outside any session container.""" + env = {**os.environ, "WORKSPACE": str(workspace), "ANSWER_FILE": str(answer), + "TRUTH_DIR": str(HERE / "fixtures" / "truth")} + result = subprocess.run([sys.executable, str(HERE / "tasks" / task / "verify.py")], + env=env, capture_output=True, text=True, timeout=600) + line = next((l for l in result.stdout.splitlines() if l.startswith("score=")), None) + return None if line is None else float(line.split("=", 1)[1]) + + +def score(args): + out = Path(args.out).resolve() + record = load(out) + state = progress(out) + scores = [] + for unit in record["units"]: + unit_dir = state["done"].get(unit["id"]) + if unit_dir is None: + continue + for s in sessions_of(out, unit_dir): + role = s["task"].rsplit("-", 1)[1] + scores.append({ + "unit": unit["id"], "pair": unit["pair"], "arm": unit["arm"], "rep": unit["rep"], + "role": role, "calls": s["calls"], "turns": s["turns"], "cost": s["cost"], + "score": verify_on_host(s["task"], s["workspace"], s["answer"]), + }) + (out / "scores.json").write_text(json.dumps(scores, indent=1) + "\n", encoding="utf-8") + print(f"scored {len(scores)} session(s) from {len(state['done'])} completed unit(s)") def report(args): - with closing(sqlite3.connect(args.results)) as db: - rows = db.execute( - "SELECT competitor, task, rep, score, tool_calls, num_turns, total_cost_usd FROM runs " - "WHERE task LIKE 'recall-%-reuse' AND status = 'ok'" - ).fetchall() + out = Path(args.out).resolve() + rows = [r for r in json.loads((out / "scores.json").read_text(encoding="utf-8")) if r["role"] == "reuse"] by = {} - for arm, task, rep, score, calls, turns, cost in rows: - by.setdefault(task, {}).setdefault(arm, {})[rep] = (score, calls, turns, cost) + for r in rows: + by.setdefault(r["pair"], {}).setdefault(r["arm"], {})[r["rep"]] = r print("pair arm right calls turns cost avoided") - for task in sorted(by): - control = by[task].get("control", {}) - for arm in sorted(by[task]): - runs = by[task][arm] - both = [r for r in runs if r in control and runs[r][0] == 1 and control[r][0] == 1] + for pair in sorted(by): + control = by[pair].get("control", {}) + for arm in sorted(by[pair]): + runs = by[pair][arm] + both = [k for k in runs if k in control and runs[k]["score"] == 1 and control[k]["score"] == 1] avoided = ( - sum(control[r][1] - runs[r][1] for r in both) / len(both) + sum(control[k]["calls"] - runs[k]["calls"] for k in both) / len(both) if both and arm != "control" else None ) - right = sum(1 for r in runs.values() if r[0] == 1) - mean = lambda i: sum(r[i] or 0 for r in runs.values()) / len(runs) + right = sum(1 for r in runs.values() if r["score"] == 1) + mean = lambda key: sum(r[key] or 0 for r in runs.values()) / len(runs) print( - f"{task[:-6]:17} {arm:33} {right}/{len(runs):<4} {mean(1):6.1f} {mean(2):6.1f} " - f"{mean(3):6.3f} {'-' if avoided is None else f'{avoided:.1f} over {len(both)}'}" + f"{pair:17} {arm:33} {right}/{len(runs):<4} {mean('calls'):6.1f} {mean('turns'):6.1f} " + f"{mean('cost'):6.3f} {'-' if avoided is None else f'{avoided:.1f} over {len(both)}'}" ) @@ -147,13 +238,14 @@ def main(): p.add_argument("--arms", default=DEFAULT_ARMS) p.add_argument("--reps", type=int, default=3) p.add_argument("--image", default="thol-rig:local") + p.add_argument("--campaign", default=DEFAULT_CAMPAIGN) r = sub.add_parser("run") r.add_argument("out") r.add_argument("--approve-spend", action="store_true") - q = sub.add_parser("report") - q.add_argument("results") + for name in ("score", "report"): + sub.add_parser(name).add_argument("out") args = parser.parse_args() - {"plan": plan, "run": run, "report": report}[args.command](args) + {"plan": plan, "run": run, "score": score, "report": report}[args.command](args) if __name__ == "__main__": diff --git a/bench/recall/tasks/_recall.py b/bench/recall/tasks/_recall.py index 8ca9bfefa..536db01ba 100644 --- a/bench/recall/tasks/_recall.py +++ b/bench/recall/tasks/_recall.py @@ -12,36 +12,74 @@ - the workspace is the fixture: an empty or missing workspace scores 0, so a session that never ran cannot pass on a lucky reply; - - no fixture file was changed: both prompts say not to, and a session that - "answers" by rewriting the code has answered a different question; + - no fixture file was changed, removed or ADDED: both prompts say not to, + and a session that "answers" by rewriting the code has answered a + different question. An added file counts, because one can change what the + code does without touching a listed file -- `migrations/0_x.sql` runs + before every other migration. Bytecode caches and dot-directories (a + client's own state, such as `.token-optimizer/`) are not the fixture; - there is a reply at all. A `reuse` answer that names the right value AND a decoy is a hedge, and scores -0: a list of candidates is not a conclusion. +0: a list of candidates is not a conclusion. A `derive` answer must state the +pair's conclusion (every `require` pattern) and must not deny it (no +`contradict` pattern): the right words in a sentence that says the opposite +are not the right answer. + +THE TRUTH NEVER ENTERS THE SESSION'S CONTAINER. THOL runs the agent and the +verifier as one user on one filesystem, so a truth file mounted for the +verifier is a file the agent can `cat`. recall.py mounts none; inside the rig +this verifier finds no truth and records a deferred 0, and `recall.py score` +runs it again on the host against the kept workspace and answer, which is the +score the report uses. """ import re -from _vlib import WS, answer, emit, hashes_intact, truth +from _vlib import TRUTH_DIR, WS, answer, emit, sha, truth + +IGNORED_DIRS = {"__pycache__"} + + +def inventory(root): + """Every fixture-relevant file under root, as posix relative paths.""" + files = set() + for path in root.rglob("*"): + rel = path.relative_to(root) + if not path.is_file() or any(p in IGNORED_DIRS or p.startswith(".") for p in rel.parts[:-1]): + continue + if path.suffix == ".pyc": + continue + files.add(rel.as_posix()) + return files + + +def fixture_intact(root, expected): + """True when the workspace holds exactly the fixture's files, unchanged.""" + if inventory(root) != set(expected): + return False + return all(sha(root / rel) == digest for rel, digest in expected.items()) def verify(pair: str, role: str) -> None: + if not (TRUTH_DIR / f"{pair}.json").is_file(): + emit(0.0, "deferred: scored outside the session container by bench/recall/recall.py score") t = truth(f"{pair}.json") if not (WS / t["marker"]).is_file(): emit(0.0, f"the workspace is not the {t['fixture']} fixture (no {t['marker']})") - if not hashes_intact(WS, t["fixture_hashes"]): - emit(0.0, "a fixture file was changed or deleted; the prompt says to change none") + if not fixture_intact(WS, t["fixture_hashes"]): + emit(0.0, "a fixture file was changed, removed or added; the prompt says to change none") text = answer().strip() if not text: emit(0.0, "no final reply") if role == "derive": - missing = [ - group for group in t["derive"]["groups"] - if not any(term.lower() in text.lower() for term in group) - ] + missing = [p for p in t["derive"]["require"] if not re.search(p, text, re.IGNORECASE)] if missing: - emit(0.0, "the explanation names none of: " + "; ".join(" | ".join(g) for g in missing)) - emit(1.0, "the root cause is named") + emit(0.0, "the explanation does not state: " + "; ".join(missing)) + denied = [p for p in t["derive"]["contradict"] if re.search(p, text, re.IGNORECASE)] + if denied: + emit(0.0, "the explanation contradicts the conclusion: " + "; ".join(denied)) + emit(1.0, "the root cause is stated") if role == "reuse": right = any(re.search(p, text, re.IGNORECASE) for p in t["reuse"]["accept"]) hedge = [p for p in t["reuse"]["decoys"] if re.search(p, text, re.IGNORECASE)] diff --git a/bench/recall/tasks/_vlib.py b/bench/recall/tasks/_vlib.py index acfe8239c..9f3238102 100644 --- a/bench/recall/tasks/_vlib.py +++ b/bench/recall/tasks/_vlib.py @@ -11,7 +11,9 @@ THOL's own `_vlib.py`, which sits beside every task there; this file is never mounted into the rig. It exists so `test_recall.py` can run every verifier offline against the same contract, and it implements only the part of that -contract these verifiers use. +contract these verifiers use. THOL's own hashes_intact is deliberately not +mirrored: it checks only the files it is given, so a file ADDED to the +workspace passes it; _recall.py checks the whole inventory instead. """ import hashlib @@ -46,11 +48,3 @@ def emit(score: float, detail: str = ""): def sha(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() - - -def hashes_intact(root: Path, expected: dict) -> bool: - for rel, h in expected.items(): - p = root / rel - if not p.exists() or sha(p) != h: - return False - return True \ No newline at end of file diff --git a/bench/recall/test_recall.py b/bench/recall/test_recall.py index 5e51630c1..8909d9add 100644 --- a/bench/recall/test_recall.py +++ b/bench/recall/test_recall.py @@ -135,6 +135,35 @@ def test_a_changed_fixture_scores_nothing(self): for role in ("derive", "reuse"): self.assertEqual(score(pair, role, t["reference"][role], ws), 0.0, f"{pair}-{role}") + def test_a_derive_reply_that_denies_the_cause_in_the_right_words_scores_nothing(self): + # "Sorting is alphabetical, so 1_create_users.sql runs first" names both + # the sort and 10_ and says the opposite of what the fixture does. + for pair in PAIRS: + with Workspace(pair) as ws: + for wrong in record(pair)["deriveDecoys"]: + self.assertEqual(score(pair, "derive", wrong, ws), 0.0, f"{pair}: {wrong!r}") + + def test_an_added_fixture_file_scores_nothing(self): + # migrations/0_x.sql changes which migration runs first without + # touching any listed file, so the listed hashes alone cannot see it. + for pair in PAIRS: + t = record(pair) + with Workspace(pair) as ws: + extra = ws / Path(t["marker"]).parent / "0_added.sql" + extra.write_text("SELECT 1;\n") + for role in ("derive", "reuse"): + self.assertEqual(score(pair, role, t["reference"][role], ws), 0.0, f"{pair}-{role}") + + def test_bytecode_and_a_clients_own_state_are_not_the_fixture(self): + for pair in PAIRS: + t = record(pair) + with Workspace(pair) as ws: + (ws / "__pycache__").mkdir() + (ws / "__pycache__" / "x.cpython-313.pyc").write_bytes(b"\0") + (ws / ".token-optimizer" / "wiki").mkdir(parents=True) + (ws / ".token-optimizer" / "wiki" / "nodes.jsonl").write_text("{}\n") + self.assertEqual(score(pair, "reuse", t["reference"]["reuse"], ws), 1.0, pair) + def test_a_derive_reply_that_misses_the_cause_scores_nothing(self): vague = "The tests fail because of a bug in the code; it needs fixing." for pair in PAIRS: @@ -188,80 +217,184 @@ def runner(): return module +def quiet(fn, *args, **kwargs): + import contextlib + import io + + with contextlib.redirect_stdout(io.StringIO()): + return fn(*args, **kwargs) + + +def make_plan(out, reps=2): + import argparse + + quiet(runner().plan, argparse.Namespace( + out=str(out), arms="control,knowledge", reps=reps, image="img", campaign="2.1.251 (Claude Code)")) + return json.loads((Path(out) / "plan.json").read_text(encoding="utf-8")) + + +def mounts_of(step): + return [v for i, v in enumerate(step) if i and step[i - 1] == "-v"] + + +class Ok: + returncode = 0 + + +class Fail: + returncode = 1 + + class TheRunnerKeepsThePairsApart(unittest.TestCase): """Nothing may reach a reuse session except what its own derive session left.""" - def plan(self, out, reps=2): - import argparse - import contextlib - import io + def test_a_plan_is_one_unit_per_pair_arm_and_repetition(self): + with tempfile.TemporaryDirectory() as out: + plan = make_plan(Path(out) / "p") + self.assertEqual(len(plan["units"]), 3 * 2 * 2) + self.assertEqual(len({u["id"] for u in plan["units"]}), len(plan["units"])) - with contextlib.redirect_stdout(io.StringIO()): - runner().plan(argparse.Namespace(out=str(out), arms="control,token-optimizer-proxy-knowledge", reps=reps, image="img")) - return json.loads((Path(out) / "plan.json").read_text(encoding="utf-8")) + def test_planning_into_a_used_directory_is_refused(self): + with tempfile.TemporaryDirectory() as out: + make_plan(Path(out) / "p") + with self.assertRaises(SystemExit) as refused: + make_plan(Path(out) / "p") + self.assertIn("not empty", str(refused.exception)) - def test_each_pair_arm_and_repetition_has_its_own_graph_shared_by_its_two_sessions(self): + def test_a_session_sees_its_own_task_and_fixture_and_never_the_truth(self): + r = runner() with tempfile.TemporaryDirectory() as out: - steps = self.plan(out) - self.assertEqual(len(steps), 3 * 2 * 2 * 2) - graphs = {} - for step in steps: - graph = next(v.split(":/proxy-graph:")[0] for v in step if ":/proxy-graph:" in v) - task = step[step.index("-t") + 1] - graphs.setdefault(graph, []).append(task) - self.assertEqual(len(graphs), 3 * 2 * 2) - for tasks in graphs.values(): - self.assertEqual(len(tasks), 2) - self.assertTrue(tasks[0].endswith("-derive") and tasks[1].endswith("-reuse")) - self.assertEqual(tasks[0][: -len("-derive")], tasks[1][: -len("-reuse")]) - - def test_a_session_sees_only_its_own_task_fixture_and_truth(self): + out = Path(out) / "p" + plan = make_plan(out, reps=1) + for unit in plan["units"]: + for role in r.ROLES: + step = r.session(unit, role, out / "units" / unit["id"] / "try1", out, plan) + mounts = mounts_of(step) + self.assertEqual([m for m in mounts if "truth" in m], [], unit["id"]) + tasks = [m for m in mounts if "/tasks/recall-" in m] + self.assertEqual(len(tasks), 1) + self.assertIn(f"/tasks/{unit['pair']}-{role}:", tasks[0]) + self.assertIn("--keep-workspaces", step) + self.assertIn("THOL_CAMPAIGN=2.1.251 (Claude Code)", step) + + def test_the_two_sessions_of_an_attempt_share_one_graph_and_results_volume(self): + r = runner() with tempfile.TemporaryDirectory() as out: - steps = self.plan(out, reps=1) - for step in steps: - task = step[step.index("-t") + 1] - pair = task.rsplit("-", 1)[0] - mounts = [v for i, v in enumerate(step) if i and step[i - 1] == "-v"] - tasks = [m for m in mounts if "/tasks/recall-" in m] - truths = [m for m in mounts if "/fixtures/truth/" in m] - self.assertEqual(len(tasks), 1, task) - self.assertIn(f"/tasks/{task}:", tasks[0]) - self.assertEqual(len(truths), 1, task) - self.assertIn(f"/fixtures/truth/{pair}.json:", truths[0]) + out = Path(out) / "p" + plan = make_plan(out, reps=1) + unit = plan["units"][0] + attempt = out / "units" / unit["id"] / "try1" + steps = [r.session(unit, role, attempt, out, plan) for role in r.ROLES] + for wanted in (":/proxy-graph:", ":/results:"): + found = {next(m for m in mounts_of(s) if wanted in m) for s in steps} + self.assertEqual(len(found), 1, wanted) def test_running_without_approval_is_refused(self): import argparse with tempfile.TemporaryDirectory() as out: - self.plan(out, reps=1) + make_plan(Path(out) / "p", reps=1) with self.assertRaises(SystemExit) as refused: - runner().run(argparse.Namespace(out=out, approve_spend=False)) + runner().run(argparse.Namespace(out=str(Path(out) / "p"), approve_spend=False)) self.assertIn("approve-spend", str(refused.exception)) - def test_calls_avoided_counts_only_repetitions_both_arms_got_right(self): + def test_a_rerun_repeats_no_finished_unit_and_restarts_a_failed_one_clean(self): import argparse - import contextlib - import io - import sqlite3 + r = runner() with tempfile.TemporaryDirectory() as out: - db = Path(out) / "results.sqlite" - with contextlib.closing(sqlite3.connect(db)) as c, c: - c.execute( - "CREATE TABLE runs (competitor TEXT, task TEXT, rep INTEGER, score REAL, " - "tool_calls INTEGER, num_turns INTEGER, total_cost_usd REAL, status TEXT)" + out = Path(out) / "p" + plan = make_plan(out, reps=1) + calls = [] + + def flaky(step): + calls.append(step) + graph = next(m for m in mounts_of(step) if ":/proxy-graph:" in m).split(":/proxy-graph:")[0] + # Each session leaves something in the graph, as a real one would. + (Path(graph) / f"left-{len(calls)}").write_text("x") + return Fail() if len(calls) == 4 else Ok() + + args = argparse.Namespace(out=str(out), approve_spend=True) + with self.assertRaises(SystemExit): + quiet(r.run, args, execute=flaky) + self.assertEqual(len(calls), 4) + first = plan["units"][0]["id"] + failed = plan["units"][1]["id"] + self.assertIn(first, r.progress(out)["done"]) + self.assertNotIn(failed, r.progress(out)["done"]) + + calls.clear() + quiet(r.run, args, execute=lambda step: (calls.append(step), Ok())[1]) + touched = {m for s in calls for m in mounts_of(s) if ":/results:" in m} + self.assertFalse(any(f"units{os.sep}{first}{os.sep}" in m or f"units/{first}/" in m for m in touched)) + self.assertEqual(len(calls), 2 * (len(plan["units"]) - 1)) + retry = out / "units" / failed / "try2" / "graph" + self.assertEqual(r.progress(out)["done"][failed], f"units/{failed}/try2") + self.assertTrue(retry.is_dir()) + self.assertFalse(any(p.name.startswith("left-") and int(p.name[5:]) <= 4 for p in retry.iterdir())) + + +class ScoringHappensOnTheHost(unittest.TestCase): + def test_inside_the_container_the_verifier_defers(self): + with Workspace("recall-tz") as ws, tempfile.TemporaryDirectory() as empty_truth: + with tempfile.TemporaryDirectory() as scratch: + answer = Path(scratch) / "answer.txt" + answer.write_text(record("recall-tz")["reference"]["reuse"]) + run = subprocess.run( + [sys.executable, str(TASKS / "recall-tz-reuse" / "verify.py")], + env={**os.environ, "WORKSPACE": str(ws), "ANSWER_FILE": str(answer), "TRUTH_DIR": empty_truth}, + capture_output=True, text=True, timeout=60, ) - rows = [ - ("control", "recall-tz-reuse", 1, 1.0, 12, 8, 0.2, "ok"), - ("control", "recall-tz-reuse", 2, 1.0, 10, 7, 0.2, "ok"), - ("knowledge", "recall-tz-reuse", 1, 1.0, 2, 2, 0.05, "ok"), - # Cheap and wrong: must not count as calls avoided. - ("knowledge", "recall-tz-reuse", 2, 0.0, 1, 1, 0.01, "ok"), - ] - c.executemany("INSERT INTO runs VALUES (?,?,?,?,?,?,?,?)", rows) + self.assertIn("score=0.0000", run.stdout) + self.assertIn("deferred", run.stdout) + + def test_score_reads_the_kept_workspace_and_answer_and_report_counts_both_right(self): + import argparse + import sqlite3 + from contextlib import closing + + r = runner() + with tempfile.TemporaryDirectory() as out: + out = Path(out) / "p" + plan = make_plan(out, reps=2) + replies = { # (arm, rep) -> (reuse reply, reuse tool calls) + ("control", 1): ("TZ=UTC", 12), + ("control", 2): ("TZ=UTC", 10), + ("knowledge", 1): ("TZ=UTC", 2), + ("knowledge", 2): ("LANG=C", 1), # cheap and wrong: not a saving + } + state = {"done": {}, "attempts": {}} + for unit in plan["units"]: + if unit["pair"] != "recall-tz": + continue + unit_dir = out / "units" / unit["id"] / "try1" + results = unit_dir / "results" + results.mkdir(parents=True) + with closing(sqlite3.connect(results / "results.sqlite")) as db, db: + db.execute("CREATE TABLE runs (run_id TEXT, task TEXT, tool_calls INTEGER, " + "num_turns INTEGER, total_cost_usd REAL, status TEXT)") + for role in ("derive", "reuse"): + run_id = f"{unit['id']}-{role}" + reply, calls = (record("recall-tz")["reference"]["derive"], 5) if role == "derive" \ + else replies[(unit["arm"], unit["rep"])] + shutil.copytree(FIXTURES / "tzbook", results / "runs" / run_id / "workspace", + ignore=shutil.ignore_patterns("__pycache__")) + (results / "artifacts" / run_id).mkdir(parents=True) + (results / "artifacts" / run_id / "answer.txt").write_text(reply) + db.execute("INSERT INTO runs VALUES (?,?,?,?,?,?)", + (run_id, f"recall-tz-{role}", calls, calls, 0.1, "ok")) + state["done"][unit["id"]] = unit_dir.relative_to(out).as_posix() + r.save_progress(out, state) + quiet(r.score, argparse.Namespace(out=str(out))) + scores = json.loads((out / "scores.json").read_text(encoding="utf-8")) + self.assertEqual(len(scores), 8) + self.assertTrue(all(s["score"] == 1.0 for s in scores if s["role"] == "derive")) + import contextlib + import io + buffer = io.StringIO() with contextlib.redirect_stdout(buffer): - runner().report(argparse.Namespace(results=str(db))) + r.report(argparse.Namespace(out=str(out))) line = next(l for l in buffer.getvalue().splitlines() if "knowledge" in l) self.assertIn("10.0 over 1", line) self.assertIn("1/2", line) From c28bbe06c67ad81e0af3e1c3a6ad391fdb3fa25f Mon Sep 17 00:00:00 2001 From: Franklin Moormann Date: Sat, 17 Oct 2026 21:52:01 -0400 Subject: [PATCH 4/5] fix(bench)+fix(hooks): loud recall scoring, completion needs ok rows; gate deferred guidance Second review round on #507: - run: a unit is done only when THOL recorded an `ok` row for both roles. It exits 0 after a timeout too, and score reads only ok rows, so a unit could be marked done with a session missing and never retried. - score: a missing workspace, answer or truth file, a verifier that crashed or printed no score, and a score that deferred on the host all stop `score` with the reason and write nothing, instead of entering scores.json as 0. - derive: a causal phrase that blames something other than the pair's cause ("uses time.localtime, but the failures come from float rounding") is a contradiction, for all three pairs, each with a mixed decoy answer. - hooks: the deferred recording guidance needs the harvest feature, as the nudge does, so experiment arms with harvest off get no recording instructions; and a client whose context style is silent (Windsurf) no longer spends the once-only flag on a message it cannot deliver. Every fix was mutated out and its test failed. recall: 26 tests (1 skipped on Windows); hooks: 126 suites, 2396 passed. Co-Authored-By: Claude Opus 5.5 --- .../recall/fixtures/truth/recall-migrate.json | 6 +- bench/recall/fixtures/truth/recall-quota.json | 6 +- bench/recall/fixtures/truth/recall-tz.json | 6 +- bench/recall/recall.py | 58 +++++++++++-- bench/recall/test_recall.py | 73 +++++++++++++++- hooks-core/adapter.mjs | 7 ++ .../hooks/token-optimizer/lib/adapter.mjs | 7 ++ integrations/codex/hooks/lib/adapter.mjs | 7 ++ .../codex/plugin/hooks/lib/adapter.mjs | 7 ++ .../copilot/.github/hooks/lib/adapter.mjs | 7 ++ integrations/cursor/hooks/lib/adapter.mjs | 7 ++ integrations/gemini/hooks/lib/adapter.mjs | 7 ++ integrations/kilo/hooks/lib/adapter.mjs | 7 ++ integrations/opencode/hooks/lib/adapter.mjs | 7 ++ integrations/qwen/hooks/lib/adapter.mjs | 7 ++ integrations/windsurf/hooks/lib/adapter.mjs | 7 ++ plugin/hooks/lib/adapter.mjs | 7 ++ .../recording-guidance-delivery.test.mjs | 85 +++++++++++++++++++ 18 files changed, 305 insertions(+), 13 deletions(-) create mode 100644 tests/hooks/recording-guidance-delivery.test.mjs diff --git a/bench/recall/fixtures/truth/recall-migrate.json b/bench/recall/fixtures/truth/recall-migrate.json index 58b097b50..09870a808 100644 --- a/bench/recall/fixtures/truth/recall-migrate.json +++ b/bench/recall/fixtures/truth/recall-migrate.json @@ -9,7 +9,8 @@ ], "contradict": [ "\\b1_create_users(?:\\.sql)?\\s+(?:is\\s+first|(?:is\\s+|will\\s+be\\s+)?(?:run|runs|applied|applies|comes|sorts|goes)\\s+first)", - "(?= 1 || (Number(state.toolCalls) || 0) >= GUIDANCE_AFTER_CALLS); diff --git a/integrations/cline/hooks/token-optimizer/lib/adapter.mjs b/integrations/cline/hooks/token-optimizer/lib/adapter.mjs index a75f15023..e972bb51d 100644 --- a/integrations/cline/hooks/token-optimizer/lib/adapter.mjs +++ b/integrations/cline/hooks/token-optimizer/lib/adapter.mjs @@ -1884,9 +1884,16 @@ async function runHook(clientName, event, invocation) { // and buys nothing until there is something to record. The first edit is the // clearest sign there will be; GUIDANCE_AFTER_CALLS tool calls is the fallback // for an investigation that edits nothing but still works things out. + // + // Only where it can arrive and is wanted: an experiment arm with harvest off + // gets no recording instructions (the nudge above has the same gate), and a + // client whose context style is silent (Windsurf) cannot deliver any context, + // so it must not spend the once-only flag on a message nobody receives. const guided = event === 'post-tool' && !state.recordingGuided && + features.harvest && + contextOutput(client, eventName, '') !== null && toolEvidence.names.has('wiki_write') && ((Number(state.edits) || 0) >= 1 || (Number(state.toolCalls) || 0) >= GUIDANCE_AFTER_CALLS); diff --git a/integrations/codex/hooks/lib/adapter.mjs b/integrations/codex/hooks/lib/adapter.mjs index a75f15023..e972bb51d 100644 --- a/integrations/codex/hooks/lib/adapter.mjs +++ b/integrations/codex/hooks/lib/adapter.mjs @@ -1884,9 +1884,16 @@ async function runHook(clientName, event, invocation) { // and buys nothing until there is something to record. The first edit is the // clearest sign there will be; GUIDANCE_AFTER_CALLS tool calls is the fallback // for an investigation that edits nothing but still works things out. + // + // Only where it can arrive and is wanted: an experiment arm with harvest off + // gets no recording instructions (the nudge above has the same gate), and a + // client whose context style is silent (Windsurf) cannot deliver any context, + // so it must not spend the once-only flag on a message nobody receives. const guided = event === 'post-tool' && !state.recordingGuided && + features.harvest && + contextOutput(client, eventName, '') !== null && toolEvidence.names.has('wiki_write') && ((Number(state.edits) || 0) >= 1 || (Number(state.toolCalls) || 0) >= GUIDANCE_AFTER_CALLS); diff --git a/integrations/codex/plugin/hooks/lib/adapter.mjs b/integrations/codex/plugin/hooks/lib/adapter.mjs index a75f15023..e972bb51d 100644 --- a/integrations/codex/plugin/hooks/lib/adapter.mjs +++ b/integrations/codex/plugin/hooks/lib/adapter.mjs @@ -1884,9 +1884,16 @@ async function runHook(clientName, event, invocation) { // and buys nothing until there is something to record. The first edit is the // clearest sign there will be; GUIDANCE_AFTER_CALLS tool calls is the fallback // for an investigation that edits nothing but still works things out. + // + // Only where it can arrive and is wanted: an experiment arm with harvest off + // gets no recording instructions (the nudge above has the same gate), and a + // client whose context style is silent (Windsurf) cannot deliver any context, + // so it must not spend the once-only flag on a message nobody receives. const guided = event === 'post-tool' && !state.recordingGuided && + features.harvest && + contextOutput(client, eventName, '') !== null && toolEvidence.names.has('wiki_write') && ((Number(state.edits) || 0) >= 1 || (Number(state.toolCalls) || 0) >= GUIDANCE_AFTER_CALLS); diff --git a/integrations/copilot/.github/hooks/lib/adapter.mjs b/integrations/copilot/.github/hooks/lib/adapter.mjs index a75f15023..e972bb51d 100644 --- a/integrations/copilot/.github/hooks/lib/adapter.mjs +++ b/integrations/copilot/.github/hooks/lib/adapter.mjs @@ -1884,9 +1884,16 @@ async function runHook(clientName, event, invocation) { // and buys nothing until there is something to record. The first edit is the // clearest sign there will be; GUIDANCE_AFTER_CALLS tool calls is the fallback // for an investigation that edits nothing but still works things out. + // + // Only where it can arrive and is wanted: an experiment arm with harvest off + // gets no recording instructions (the nudge above has the same gate), and a + // client whose context style is silent (Windsurf) cannot deliver any context, + // so it must not spend the once-only flag on a message nobody receives. const guided = event === 'post-tool' && !state.recordingGuided && + features.harvest && + contextOutput(client, eventName, '') !== null && toolEvidence.names.has('wiki_write') && ((Number(state.edits) || 0) >= 1 || (Number(state.toolCalls) || 0) >= GUIDANCE_AFTER_CALLS); diff --git a/integrations/cursor/hooks/lib/adapter.mjs b/integrations/cursor/hooks/lib/adapter.mjs index a75f15023..e972bb51d 100644 --- a/integrations/cursor/hooks/lib/adapter.mjs +++ b/integrations/cursor/hooks/lib/adapter.mjs @@ -1884,9 +1884,16 @@ async function runHook(clientName, event, invocation) { // and buys nothing until there is something to record. The first edit is the // clearest sign there will be; GUIDANCE_AFTER_CALLS tool calls is the fallback // for an investigation that edits nothing but still works things out. + // + // Only where it can arrive and is wanted: an experiment arm with harvest off + // gets no recording instructions (the nudge above has the same gate), and a + // client whose context style is silent (Windsurf) cannot deliver any context, + // so it must not spend the once-only flag on a message nobody receives. const guided = event === 'post-tool' && !state.recordingGuided && + features.harvest && + contextOutput(client, eventName, '') !== null && toolEvidence.names.has('wiki_write') && ((Number(state.edits) || 0) >= 1 || (Number(state.toolCalls) || 0) >= GUIDANCE_AFTER_CALLS); diff --git a/integrations/gemini/hooks/lib/adapter.mjs b/integrations/gemini/hooks/lib/adapter.mjs index a75f15023..e972bb51d 100644 --- a/integrations/gemini/hooks/lib/adapter.mjs +++ b/integrations/gemini/hooks/lib/adapter.mjs @@ -1884,9 +1884,16 @@ async function runHook(clientName, event, invocation) { // and buys nothing until there is something to record. The first edit is the // clearest sign there will be; GUIDANCE_AFTER_CALLS tool calls is the fallback // for an investigation that edits nothing but still works things out. + // + // Only where it can arrive and is wanted: an experiment arm with harvest off + // gets no recording instructions (the nudge above has the same gate), and a + // client whose context style is silent (Windsurf) cannot deliver any context, + // so it must not spend the once-only flag on a message nobody receives. const guided = event === 'post-tool' && !state.recordingGuided && + features.harvest && + contextOutput(client, eventName, '') !== null && toolEvidence.names.has('wiki_write') && ((Number(state.edits) || 0) >= 1 || (Number(state.toolCalls) || 0) >= GUIDANCE_AFTER_CALLS); diff --git a/integrations/kilo/hooks/lib/adapter.mjs b/integrations/kilo/hooks/lib/adapter.mjs index a75f15023..e972bb51d 100644 --- a/integrations/kilo/hooks/lib/adapter.mjs +++ b/integrations/kilo/hooks/lib/adapter.mjs @@ -1884,9 +1884,16 @@ async function runHook(clientName, event, invocation) { // and buys nothing until there is something to record. The first edit is the // clearest sign there will be; GUIDANCE_AFTER_CALLS tool calls is the fallback // for an investigation that edits nothing but still works things out. + // + // Only where it can arrive and is wanted: an experiment arm with harvest off + // gets no recording instructions (the nudge above has the same gate), and a + // client whose context style is silent (Windsurf) cannot deliver any context, + // so it must not spend the once-only flag on a message nobody receives. const guided = event === 'post-tool' && !state.recordingGuided && + features.harvest && + contextOutput(client, eventName, '') !== null && toolEvidence.names.has('wiki_write') && ((Number(state.edits) || 0) >= 1 || (Number(state.toolCalls) || 0) >= GUIDANCE_AFTER_CALLS); diff --git a/integrations/opencode/hooks/lib/adapter.mjs b/integrations/opencode/hooks/lib/adapter.mjs index a75f15023..e972bb51d 100644 --- a/integrations/opencode/hooks/lib/adapter.mjs +++ b/integrations/opencode/hooks/lib/adapter.mjs @@ -1884,9 +1884,16 @@ async function runHook(clientName, event, invocation) { // and buys nothing until there is something to record. The first edit is the // clearest sign there will be; GUIDANCE_AFTER_CALLS tool calls is the fallback // for an investigation that edits nothing but still works things out. + // + // Only where it can arrive and is wanted: an experiment arm with harvest off + // gets no recording instructions (the nudge above has the same gate), and a + // client whose context style is silent (Windsurf) cannot deliver any context, + // so it must not spend the once-only flag on a message nobody receives. const guided = event === 'post-tool' && !state.recordingGuided && + features.harvest && + contextOutput(client, eventName, '') !== null && toolEvidence.names.has('wiki_write') && ((Number(state.edits) || 0) >= 1 || (Number(state.toolCalls) || 0) >= GUIDANCE_AFTER_CALLS); diff --git a/integrations/qwen/hooks/lib/adapter.mjs b/integrations/qwen/hooks/lib/adapter.mjs index a75f15023..e972bb51d 100644 --- a/integrations/qwen/hooks/lib/adapter.mjs +++ b/integrations/qwen/hooks/lib/adapter.mjs @@ -1884,9 +1884,16 @@ async function runHook(clientName, event, invocation) { // and buys nothing until there is something to record. The first edit is the // clearest sign there will be; GUIDANCE_AFTER_CALLS tool calls is the fallback // for an investigation that edits nothing but still works things out. + // + // Only where it can arrive and is wanted: an experiment arm with harvest off + // gets no recording instructions (the nudge above has the same gate), and a + // client whose context style is silent (Windsurf) cannot deliver any context, + // so it must not spend the once-only flag on a message nobody receives. const guided = event === 'post-tool' && !state.recordingGuided && + features.harvest && + contextOutput(client, eventName, '') !== null && toolEvidence.names.has('wiki_write') && ((Number(state.edits) || 0) >= 1 || (Number(state.toolCalls) || 0) >= GUIDANCE_AFTER_CALLS); diff --git a/integrations/windsurf/hooks/lib/adapter.mjs b/integrations/windsurf/hooks/lib/adapter.mjs index a75f15023..e972bb51d 100644 --- a/integrations/windsurf/hooks/lib/adapter.mjs +++ b/integrations/windsurf/hooks/lib/adapter.mjs @@ -1884,9 +1884,16 @@ async function runHook(clientName, event, invocation) { // and buys nothing until there is something to record. The first edit is the // clearest sign there will be; GUIDANCE_AFTER_CALLS tool calls is the fallback // for an investigation that edits nothing but still works things out. + // + // Only where it can arrive and is wanted: an experiment arm with harvest off + // gets no recording instructions (the nudge above has the same gate), and a + // client whose context style is silent (Windsurf) cannot deliver any context, + // so it must not spend the once-only flag on a message nobody receives. const guided = event === 'post-tool' && !state.recordingGuided && + features.harvest && + contextOutput(client, eventName, '') !== null && toolEvidence.names.has('wiki_write') && ((Number(state.edits) || 0) >= 1 || (Number(state.toolCalls) || 0) >= GUIDANCE_AFTER_CALLS); diff --git a/plugin/hooks/lib/adapter.mjs b/plugin/hooks/lib/adapter.mjs index a75f15023..e972bb51d 100644 --- a/plugin/hooks/lib/adapter.mjs +++ b/plugin/hooks/lib/adapter.mjs @@ -1884,9 +1884,16 @@ async function runHook(clientName, event, invocation) { // and buys nothing until there is something to record. The first edit is the // clearest sign there will be; GUIDANCE_AFTER_CALLS tool calls is the fallback // for an investigation that edits nothing but still works things out. + // + // Only where it can arrive and is wanted: an experiment arm with harvest off + // gets no recording instructions (the nudge above has the same gate), and a + // client whose context style is silent (Windsurf) cannot deliver any context, + // so it must not spend the once-only flag on a message nobody receives. const guided = event === 'post-tool' && !state.recordingGuided && + features.harvest && + contextOutput(client, eventName, '') !== null && toolEvidence.names.has('wiki_write') && ((Number(state.edits) || 0) >= 1 || (Number(state.toolCalls) || 0) >= GUIDANCE_AFTER_CALLS); diff --git a/tests/hooks/recording-guidance-delivery.test.mjs b/tests/hooks/recording-guidance-delivery.test.mjs new file mode 100644 index 000000000..1e4451ffc --- /dev/null +++ b/tests/hooks/recording-guidance-delivery.test.mjs @@ -0,0 +1,85 @@ +/** + * The deferred recording guidance (#357, M5) reaches only the sessions that can + * receive it and are meant to. + * + * Delivery is once per session, so the flag that says "delivered" must not be + * spent where nothing is delivered: an experiment arm with harvest off gets no + * recording instructions at all, and a client that cannot carry context + * (Windsurf's silent style) must leave the flag unset. + */ +import { afterAll, describe, expect, test } from '@jest/globals'; +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { join, dirname } from 'node:path'; +import { tmpdir } from 'node:os'; +import { fileURLToPath } from 'node:url'; + +const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..', '..'); +const CLAUDE_POST = join(ROOT, 'plugin', 'hooks', 'post-tool.mjs'); +const WINDSURF_POST = join(ROOT, 'integrations', 'windsurf', 'hooks', 'post-tool.mjs'); + +const scratch = mkdtempSync(join(tmpdir(), 'recording-delivery-')); +afterAll(() => rmSync(scratch, { recursive: true, force: true })); + +function hook(script, payload, env = {}) { + const state = mkdtempSync(join(scratch, 'state-')); + const result = spawnSync(process.execPath, [script], { + input: JSON.stringify(payload), + encoding: 'utf8', + timeout: 30_000, + env: { + ...process.env, + TOKEN_OPTIMIZER_MODE: 'assist', + TOKEN_OPTIMIZER_MCP_CAPABILITIES: 'smart_edit,wiki_write', + TOKEN_OPTIMIZER_WIKI_DIR: join(scratch, 'graph'), + TOKEN_OPTIMIZER_SHARED_DIR: join(scratch, 'graph'), + TOKEN_OPTIMIZER_STATE_DIR: state, + ...env, + }, + }); + const saved = readdirSync(state) + .filter((name) => name.endsWith('.json')) + .map((name) => JSON.parse(readFileSync(join(state, name), 'utf8'))); + return { stdout: result.stdout, saved }; +} + +const file = join(scratch, 'edited.ts'); +writeFileSync(file, 'export const a = 1;\n'); + +const claudeEdit = (session) => ({ + hook_event_name: 'PostToolUse', + session_id: session, + cwd: scratch, + tool_name: 'Edit', + tool_input: { file_path: file, old_string: '1', new_string: '2' }, + tool_response: { success: true }, +}); + +describe('the deferred recording guidance', () => { + test('arrives on the first edit in the full arm (the control for the tests below)', () => { + const { stdout, saved } = hook(CLAUDE_POST, claudeEdit('full-arm')); + expect(stdout).toContain('Record what you work out'); + expect(saved.some((s) => s.recordingGuided === true)).toBe(true); + }); + + test('never arrives in an arm with harvest off, and the flag stays unspent', () => { + for (const arm of ['optimizer', 'retrieval', 'baseline']) { + const { stdout, saved } = hook(CLAUDE_POST, claudeEdit(`arm-${arm}`), { + TOKEN_OPTIMIZER_EXPERIMENT_ARM: arm, + }); + expect(stdout).not.toContain('Record what you work out'); + expect(saved.some((s) => s.recordingGuided === true)).toBe(false); + } + }); + + test('is not marked delivered for a client that cannot carry context', () => { + const { stdout, saved } = hook(WINDSURF_POST, { + agent_action_name: 'post_write_code', + trajectory_id: 'windsurf-silent', + tool_info: { file_path: file, working_directory: scratch, edits: [] }, + }); + expect(stdout.trim()).toBe(''); + expect(saved.length).toBeGreaterThan(0); + expect(saved.some((s) => s.recordingGuided === true)).toBe(false); + }); +}); \ No newline at end of file From 7c404a3f24b2ecafd2236cf937dc49210a9bf2db Mon Sep 17 00:00:00 2001 From: Franklin Moormann Date: Sat, 17 Oct 2026 22:52:04 -0400 Subject: [PATCH 5/5] fix(bench): name wrong causes, and drop stale scores on a rescore The derive contradiction check rejected any causal clause that did not mention the right cause within 60 characters. That rejected a correct two-sentence answer and accepted "stems from QUOTA_LIMIT being measured in seconds" because "seconds" appeared later. Each pair now lists its known wrong causes, and a clause contradicts only when it names one before the right cause. deriveAccepts holds correct answers that must score 1. score now deletes scores.json before rescoring, so a failed rescore cannot leave report reading an earlier, smaller run. Co-Authored-By: Claude Opus 5.5 --- .../recall/fixtures/truth/recall-migrate.json | 5 ++++- bench/recall/fixtures/truth/recall-quota.json | 8 ++++++-- bench/recall/fixtures/truth/recall-tz.json | 5 ++++- bench/recall/recall.py | 7 ++++++- bench/recall/test_recall.py | 19 +++++++++++++++++++ 5 files changed, 39 insertions(+), 5 deletions(-) diff --git a/bench/recall/fixtures/truth/recall-migrate.json b/bench/recall/fixtures/truth/recall-migrate.json index 09870a808..3eccd6cd6 100644 --- a/bench/recall/fixtures/truth/recall-migrate.json +++ b/bench/recall/fixtures/truth/recall-migrate.json @@ -10,7 +10,7 @@ "contradict": [ "\\b1_create_users(?:\\.sql)?\\s+(?:is\\s+first|(?:is\\s+|will\\s+be\\s+)?(?:run|runs|applied|applies|comes|sorts|goes)\\s+first)", "(?