feat(bench)+perf(hooks): bench/recall, and defer the wiki guidance past session start

.gitattributes

@@ -15,3 +15,7 @@ bench/tools/fixtures/** text eol=lf
 # compared against, exact bytes.
 bench/compression/corpus.snapshot.json text eol=lf
 bench/compression/token-counts.json text eol=lf
+
+# The recall verifiers hash the fixture files to prove a session changed nothing,
+# so a file's bytes must be the same on every checkout.
+bench/recall/fixtures/** text eol=lf

.github/workflows/quality-gates.yml

@@ -711,6 +711,27 @@ jobs:
   # `tiktoken` and `dist/`; `bench/tools/reduction` needs `tiktoken`. They are
   # instrument checks and not README checks, so they are run as their own step --
   # a failure names which half of the job went red without reading the log.
+  # THE RECALL BENCHMARK'S VERIFIERS, EXERCISED WITHOUT RUNNING A SESSION. A
+  # verifier that is wrong is wrong silently -- it scores real, billed sessions
+  # and nobody re-reads the replies -- so every one is run here against answers
+  # whose score is known first. Standard library only, so no install.
+  recall-selftest:
+    name: Recall Benchmark Self-Test
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.13'
+
+      - name: Run the recall verifiers against known answers
+        run: python3 -m unittest discover -s bench/recall -p "test_*.py" -v
+
   bench-proof:
     name: Proof Harness, Instruments That Need a Build, README Figures
     runs-on: ubuntu-latest

bench/recall/README.md

@@ -0,0 +1,63 @@
+# bench/recall: calls avoided across sessions
+
+Does what one session works out save the next session tool calls? Every
+single-session benchmark, THOL included, is blind to this by construction:
+each run starts from nothing. This one runs sessions in pairs.
+
+| half     | asks                                                            |
+| -------- | --------------------------------------------------------------- |
+| `derive` | work something out that the workspace does not state            |
+| `reuse`  | a fresh session, fresh workspace: a question that depends on it |
+
+Three pairs, each a real defect in a small standard-library Python project:
+
+| pair             | fixture    | derived                                  | reused                                    |
+| ---------------- | ---------- | ---------------------------------------- | ----------------------------------------- |
+| `recall-tz`      | `tzbook`   | day buckets use the machine's local time | the variable CI sets to make it pass      |
+| `recall-quota`   | `quotasvc` | a window documented in s is read in ms   | the value that gives a 90-second window   |
+| `recall-migrate` | `migrun`   | migrations sort as strings               | which migration a fresh database runs first |
+
+The tasks are THOL tasks (`prompt.md`, `task.json`, `verify.py`) and run on the
+rig in `bench/thol`. Each arm, pair and repetition is one unit: every attempt
+at it gets a new directory with an empty graph and its own THOL results
+volume, shared by that unit's two sessions only. **Calls avoided** is the
+control arm's reuse tool calls minus the arm's, counted only for repetitions in
+which both answered correctly: a cheap wrong answer is not a saving.
+
+**The truth never enters a session's container.** THOL runs the agent and its
+verifier as one user on one filesystem, so a mounted truth file is one the
+agent could read. None is mounted: in the container the verifier records a
+deferred 0, workspaces are kept, and `score` re-runs every verifier on the host.
+
+## Verifiers
+
+Scored from ground truth in `fixtures/truth/`, which the agent never sees. A
+verifier awards nothing for an empty workspace, a fixture file changed, removed
+or added, an empty reply, a `derive` explanation that states the cause in the
+right words but denies it, or a `reuse` answer hedged with a wrong one.
+
+    python3 -m unittest discover -s bench/recall -p "test_*.py"
+
+runs every verifier against answers whose score is known in advance, checks that
+no workspace file or prompt states a pair's conclusion, and checks that each
+fixture really carries its defect. It runs no session and spends nothing; CI
+runs it on every pull request.
+
+## Running it
+
+    python3 bench/recall/recall.py plan out/recall --reps 3
+    python3 bench/recall/recall.py run out/recall --approve-spend
+    python3 bench/recall/recall.py score out/recall
+    python3 bench/recall/recall.py report out/recall
+
+`plan` needs a new directory and prints every docker command; it spends
+nothing. `run` refuses without `--approve-spend`, because every session is a
+real, billed Claude Code session. Re-running `run` after a failure skips every
+finished unit and restarts the failed one from an empty graph. `--campaign` is
+THOL's Claude Code version pin, not a label.
+The default arms are `control` and `token-optimizer-proxy-knowledge`, whose
+manifest points the graph at `/proxy-graph`.
+
+Changing a fixture? Re-hash it, or every verifier will score a real session 0:
+
+    python3 bench/recall/hash_fixtures.py

bench/recall/fixtures/migrun/README.md

@@ -0,0 +1,5 @@
+# migrun
+
+Applies the SQL files in `migrations/` to a SQLite database, in order.
+
+    python3 -m unittest discover -s tests -t .

bench/recall/fixtures/migrun/migrations/10_backfill_email.sql

@@ -0,0 +1 @@
+UPDATE users SET email = name || '@example.invalid' WHERE email IS NULL;

bench/recall/fixtures/migrun/migrations/1_create_users.sql

@@ -0,0 +1 @@
+CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT NOT NULL);

bench/recall/fixtures/migrun/migrations/2_add_email.sql

@@ -0,0 +1 @@
+ALTER TABLE users ADD COLUMN email TEXT;

bench/recall/fixtures/migrun/migrations/3_create_orders.sql

@@ -0,0 +1 @@
+CREATE TABLE orders (id INTEGER PRIMARY KEY, user_id INTEGER NOT NULL, total INTEGER NOT NULL);

bench/recall/fixtures/migrun/migrations/4_index_orders_user.sql

@@ -0,0 +1 @@
+CREATE INDEX orders_user ON orders (user_id);

bench/recall/fixtures/migrun/migrations/5_create_products.sql

@@ -0,0 +1 @@
+CREATE TABLE products (id INTEGER PRIMARY KEY, title TEXT NOT NULL);

bench/recall/fixtures/migrun/migrations/6_add_product_price.sql

@@ -0,0 +1 @@
+ALTER TABLE products ADD COLUMN price INTEGER NOT NULL DEFAULT 0;

bench/recall/fixtures/migrun/migrations/7_create_order_lines.sql

@@ -0,0 +1 @@
+CREATE TABLE order_lines (order_id INTEGER NOT NULL, product_id INTEGER NOT NULL, quantity INTEGER NOT NULL);

bench/recall/fixtures/migrun/migrations/8_seed_admin.sql

@@ -0,0 +1 @@
+INSERT INTO users (id, name) VALUES (1, 'admin');

bench/recall/fixtures/migrun/migrations/9_index_lines_order.sql

@@ -0,0 +1 @@
+CREATE INDEX lines_order ON order_lines (order_id);

bench/recall/fixtures/migrun/migrun/__init__.py

@@ -0,0 +1 @@
+"""A small SQL migration runner."""

bench/recall/fixtures/migrun/migrun/runner.py

@@ -0,0 +1,20 @@
+"""Apply every migration in a directory to a database, in order."""
+
+import os
+import sqlite3
+
+
+def pending(directory):
+    """The migration files to apply, in the order they are applied."""
+    return sorted(name for name in os.listdir(directory) if name.endswith(".sql"))
+
+
+def migrate(database, directory):
+    """Apply each pending migration; return the names applied."""
+    applied = []
+    with sqlite3.connect(database) as connection:
+        for name in pending(directory):
+            with open(os.path.join(directory, name), encoding="utf-8") as handle:
+                connection.executescript(handle.read())
+            applied.append(name)
+    return applied

bench/recall/fixtures/migrun/tests/test_runner.py

@@ -0,0 +1,18 @@
+import os
+import tempfile
+import unittest
+
+from migrun.runner import migrate
+
+MIGRATIONS = os.path.join(os.path.dirname(__file__), "..", "migrations")
+
+
+class FreshDatabaseTest(unittest.TestCase):
+    def test_every_migration_applies_to_a_fresh_database(self):
+        with tempfile.TemporaryDirectory() as directory:
+            applied = migrate(os.path.join(directory, "app.db"), MIGRATIONS)
+        self.assertEqual(len(applied), 10)
+
+
+if __name__ == "__main__":
+    unittest.main()

bench/recall/fixtures/quotasvc/README.md

@@ -0,0 +1,12 @@
+# quotasvc
+
+A fixed-window request limiter.
+
+## Configuration
+
+| variable       | meaning                                   | default |
+| -------------- | ----------------------------------------- | ------- |
+| `QUOTA_LIMIT`  | requests allowed per window               | `100`   |
+| `QUOTA_WINDOW` | length of the window, in seconds          | `60`    |
+
+    python3 -m unittest discover -s tests -t .

bench/recall/fixtures/quotasvc/quotasvc/__init__.py

@@ -0,0 +1 @@
+"""A fixed-window request limiter."""

bench/recall/fixtures/quotasvc/quotasvc/config.py

@@ -0,0 +1,14 @@
+"""Read the limiter's settings from the environment."""
+
+DEFAULT_LIMIT = 100
+DEFAULT_WINDOW = 60_000
+
+
+def limit(env):
+    raw = env.get("QUOTA_LIMIT")
+    return int(raw) if raw else DEFAULT_LIMIT
+
+
+def window(env):
+    raw = env.get("QUOTA_WINDOW")
+    return int(raw) if raw else DEFAULT_WINDOW

bench/recall/fixtures/quotasvc/quotasvc/limiter.py

@@ -0,0 +1,24 @@
+"""Allow at most `limit` requests in each fixed window of `window_ms`."""
+
+
+class Limiter:
+    def __init__(self, limit, window_ms):
+        self.limit = limit
+        self.window_ms = window_ms
+        self._window_start = None
+        self._used = 0
+
+    def allow(self, now_ms):
+        if self._window_start is None or now_ms - self._window_start >= self.window_ms:
+            self._window_start = now_ms
+            self._used = 0
+        if self._used < self.limit:
+            self._used += 1
+            return True
+        return False
+
+
+def from_env(env):
+    from . import config
+
+    return Limiter(config.limit(env), config.window(env))

bench/recall/fixtures/quotasvc/tests/test_limiter.py

@@ -0,0 +1,21 @@
+import unittest
+
+from quotasvc.limiter import Limiter, from_env
+
+
+class LimiterTest(unittest.TestCase):
+    def test_limit_holds_inside_a_window(self):
+        limiter = Limiter(limit=3, window_ms=1000)
+        self.assertEqual([limiter.allow(t) for t in (0, 10, 20, 30)], [True, True, True, False])
+
+    def test_window_resets(self):
+        limiter = Limiter(limit=1, window_ms=1000)
+        self.assertEqual([limiter.allow(t) for t in (0, 500, 1000)], [True, False, True])
+
+    def test_defaults(self):
+        limiter = from_env({})
+        self.assertEqual((limiter.limit, limiter.window_ms), (100, 60_000))
+
+
+if __name__ == "__main__":
+    unittest.main()

bench/recall/fixtures/truth/recall-migrate.json

@@ -0,0 +1,66 @@
+{
+  "pair": "recall-migrate",
+  "fixture": "migrun",
+  "marker": "migrun/runner.py",
+  "derive": {
+    "require": [
+      "lexicograph|alphabetic|as strings|as text|string (?:order|sort)|lexical|not numerically|character by character",
+      "\\b10_"
+    ],
+    "contradict": [
+      "\\b1_create_users(?:\\.sql)?\\s+(?:is\\s+first|(?:is\\s+|will\\s+be\\s+)?(?:run|runs|applied|applies|comes|sorts|goes)\\s+first)",
+      "(?<!not )(?<!n't )(?<!never )\\b(?:sorted|sorts|ordered|orders)\\s+numerically",
+      "\\b(?:failures?|failing|fails|the (?:bug|problem|issue)|(?:root )?cause)\\b[^.;]{0,30}?\\b(?:comes?|stems?|results?|arises?|is caused|are caused|is due|are due|lies|is)\\s+(?:from|by|to|in)\\s+(?:(?!sort|order|lexicograph|alphabetic|\\b10_|pending\\()(?:[^.;]|\\.(?!\\s|$))){0,60}?(?:sqlite|ALTER TABLE|syntax|foreign key|transaction|\\block|permission|encoding|race\\b|concurren)"
+    ]
+  },
+  "reuse": {
+    "accept": [
+      "\\b10_backfill_email(?:\\.sql)?\\b"
+    ],
+    "decoys": [
+      "\\b1_create_users(?:\\.sql)?\\b"
+    ]
+  },
+  "reference": {
+    "derive": "pending() sorts the file names as strings, so 10_backfill_email.sql sorts before 1_create_users.sql and runs first, before the users table exists.",
+    "reuse": "10_backfill_email.sql"
+  },
+  "decoyAnswers": [
+    "1_create_users.sql",
+    "2_add_email.sql",
+    "1_create_users.sql or 10_backfill_email.sql"
+  ],
+  "absent": [
+    "lexicographic",
+    "alphabetical",
+    "sorts before",
+    "as strings",
+    "10_backfill_email.sql is"
+  ],
+  "fixture_hashes": {
+    "migrations/10_backfill_email.sql": "067c4b59f8c0ac9cc0ff87912dd5957b477a72841b8851a628954a9329f05997",
+    "migrations/1_create_users.sql": "f202577af96fe5ff413f6456e176f80eb994ac7d349d89f8707fa32f79cef931",
+    "migrations/2_add_email.sql": "7a3a4c70d5af51f931ef9c9e1b12d7ae59d117a77416e39b47abf3041544bf39",
+    "migrations/3_create_orders.sql": "e39eca2d89e2212e9b6fca8d55344c2a3ed8394229c6763dc042f121b3acb927",
+    "migrations/4_index_orders_user.sql": "01b2714a415725a3e92af7cb65e0e79068ba1f6b9559854d53b8855043705ecc",
+    "migrations/5_create_products.sql": "762ecf5aff555e446eed504c7c3878f0e5f2084b79fa3cbacdf00932ba956c03",
+    "migrations/6_add_product_price.sql": "c5561bbeb757ed258a9c284ecf14a2085f9db2cf3a61232834fa993074bab1c9",
+    "migrations/7_create_order_lines.sql": "4703c49d3ccc9be96d01ca728491b106e3aacd1408c26e91dffe301590cb8f8c",
+    "migrations/8_seed_admin.sql": "06a434cdae0e1d462a2adec402c3ac4833608be8dc103df6f867d10115e721e5",
+    "migrations/9_index_lines_order.sql": "77f1869d9b43704092799765f8cc5ae08dbd1ca9ba27289dc78b901bf3fd51aa",
+    "migrun/__init__.py": "59560137168ce2e1b73443a6887a9383f0c55bf02b6159cf85323df44df8bf88",
+    "migrun/runner.py": "b10b746c5760acc3afe4ea0803785ea2868455b9cfc48af3f190358e5cf4c9c6",
+    "README.md": "31662aa5abf7529d393810a38366f542ff624b09c876003d89f40d1596914776",
+    "tests/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+    "tests/test_runner.py": "1dbc46f80a4bbb687c9ab6bfaea041d275cf29bb5aca5fe9d010e8d1c82589fd"
+  },
+  "deriveDecoys": [
+    "Sorting is alphabetical, so 1_create_users.sql runs first, not 10_backfill_email.sql; the bug is elsewhere.",
+    "The migrations are sorted numerically, so 10_backfill_email.sql is simply missing a table.",
+    "sqlite3 does not support ALTER TABLE ADD COLUMN.",
+    "The files sort alphabetically and 10_backfill_email.sql is there, but the failure comes from sqlite3 rejecting ALTER TABLE."
+  ],
+  "deriveAccepts": [
+    "The failure comes from migration 10 referencing a missing table. The files sort alphabetically, so 10_backfill_email.sql runs first."
+  ]
+}

bench/recall/fixtures/truth/recall-quota.json

@@ -0,0 +1,60 @@
+{
+  "pair": "recall-quota",
+  "fixture": "quotasvc",
+  "marker": "quotasvc/config.py",
+  "derive": {
+    "require": [
+      "milli\\s*-?seconds?|\\bms\\b|window_ms",
+      "\\bseconds?\\b"
+    ],
+    "contradict": [
+      "(?<!not )(?<!n't )(?<!never )\\b(?:read|interpreted|treated|used|parsed|handled)\\s+(?:correctly\\s+)?(?:in|as)\\s+seconds\\b",
+      "\\b(?:isn't|is not|never|not)\\s+(?:read|interpreted|treated|used)\\s+(?:in|as)\\s+milli\\s*-?seconds",
+      "\\b(?:failures?|failing|fails|the (?:bug|problem|issue)|(?:root )?cause)\\b[^.;]{0,30}?\\b(?:comes?|stems?|results?|arises?|is caused|are caused|is due|are due|lies|is)\\s+(?:from|by|to|in)\\s+(?:(?!QUOTA_WINDOW|window_ms|config\\.window|\\bwindow\\b|milli\\s*-?second|\\bms\\b)(?:[^.;]|\\.(?!\\s|$))){0,60}?(?:QUOTA_LIMIT|counter|never reset|not reset|race\\b|concurren|clock|cache|off[- ]by[- ]one|rounding|time\\s*zone)"
+    ]
+  },
+  "reuse": {
+    "accept": [
+      "(?<![\\d.,_])90[,_]?000(?![\\d.,_])"
+    ],
+    "decoys": [
+      "(?<![\\d.,_])90(?![\\d.,_])",
+      "(?<![\\d.,_])1\\.5(?![\\d])",
+      "(?<![\\d.,_])90s\\b"
+    ]
+  },
+  "reference": {
+    "derive": "config.window() hands the QUOTA_WINDOW value to the limiter unconverted, and the limiter reads it as window_ms, so a value the README documents in seconds is used as milliseconds: 60 becomes a 60 ms window.",
+    "reuse": "90000"
+  },
+  "decoyAnswers": [
+    "90",
+    "1.5",
+    "90s",
+    "90000 or 90"
+  ],
+  "absent": [
+    "90000",
+    "90_000",
+    "90,000",
+    "millisecond"
+  ],
+  "fixture_hashes": {
+    "quotasvc/__init__.py": "3c98cb40b0993425157593dd9d5f7450ec79bc97c719c2ddbf14b71a4289b12a",
+    "quotasvc/config.py": "61208092ddd6766cf7be63ba3ca25ff49aa33282fe72b2fe1c46da9839b90b09",
+    "quotasvc/limiter.py": "da1c754a02ea04e0695b55a7b179e85ac8797c638c529ce7ecfb4bf101b2b3fc",
+    "README.md": "6803a5c9e41c6ab2ecd0165161747bd7610d3c6a447b2656ac1726f753a681d5",
+    "tests/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+    "tests/test_limiter.py": "a581cb0066f3c1ce7fab365010d974f682ffa2aebd7bf945701002ff8f00b9ba"
+  },
+  "deriveDecoys": [
+    "QUOTA_WINDOW is read correctly in seconds, not milliseconds; the limit counter is never reset.",
+    "The window is not interpreted as milliseconds; the problem is that QUOTA_LIMIT is ignored.",
+    "A race between concurrent requests lets extra requests through.",
+    "QUOTA_WINDOW is documented in seconds and window_ms is milliseconds, but the problem is in the counter, which never resets.",
+    "The bug stems from QUOTA_LIMIT being measured in seconds; QUOTA_WINDOW is in seconds and window_ms is in milliseconds."
+  ],
+  "deriveAccepts": [
+    "The bug comes from QUOTA_WINDOW being given in seconds while window_ms expects milliseconds."
+  ]
+}