Expand clean screening with executable contract fixtures

bench/README.md

@@ -44,6 +44,32 @@ This screen runs every fixture in local diff-file mode. It does no forge I/O. Th
 
 Inspect the fixture IDs and source before a live run in [`fixtures/cases.ts`](fixtures/cases.ts). A selected-case screen requires `--case <fixture-id>` together with `--screen-profile <path>`, which supplies the exact provider and price contract. A live screen is development evidence, not a hosted admission or a comparison with another model.
 
+## Expanded clean bank
+
+The 25-case clean bank combines the 13 admission clean cases with 12 supplemental cases in [`fixtures/clean-screen.ts`](fixtures/clean-screen.ts). The [clean-screen entrypoint](src/clean-screen.ts) passes the exported `cleanScreenCases` to the existing `runLive` API. Default screens and admission retain the 70-case corpus and its attested evaluator inputs. The supplemental cases cover authorization, expiry, asynchronous ordering, cancellation, retry limits, pagination, integer precision, defaults, lock release, SQL parameters, array ownership, and partial updates. Behavioral tests exercise both versions of each executable module, including rejection paths and boundary values.
+
+Live diff-file screening sends the diff. Supplemental source comments therefore contain the argument and behavior contracts; the `Object.hasOwn` case also supplies package metadata declaring Node.js 22 or later. Metadata outside the diff is not evidence available to this screen.
+
+After the build and dependency setup above, select a checked-in profile:
+
+| `REVIEW_MODEL` | `SCREEN_PROFILE` |
+| --- | --- |
+| `openai/gpt-5.6-luna` | [clean-screen-luna.json](clean-screen-luna.json) |
+| `z-ai/glm-5.2` | [clean-screen-glm.json](clean-screen-glm.json) |
+
+Each profile contains one generator, an empty `scorerChain`, the canonical `providerGenerationModels` identity, and explicit provider route and price bounds. The screen disables the scorer when the `scorerModel` option is absent. For Luna, set `export REVIEW_MODEL=openai/gpt-5.6-luna SCREEN_PROFILE=clean-screen-luna.json`; use the GLM row for its run. With the model credential in the environment and GNU `timeout` installed, run from `bench/`:
+
+```sh
+POSTIL_LLM_REQUEST_TIMEOUT_SECS=30 POSTIL_LLM_TOTAL_TIMEOUT_SECS=60 \
+timeout 780s bun run src/clean-screen.ts
+```
+
+Each invocation retains a separate report and per-case evidence. Its `supplementalScreen` field records a separate framed SHA-256 digest of the supplemental fixture module and entrypoint; `summary.evaluatorSha256` identifies the default evaluator. The process fails if every case is unavailable; partial reports retain unavailable cases for inspection. The CLI safety cap is 180 seconds per case. Report final review silence, final findings, suppressed findings with reasons, and unavailable cases separately, with the 13 legacy and 12 supplemental cases identified. A silent final review can contain suppressed model findings.
+
+The clean-bank-v2 evidence identifies the [measured fixture and evaluator source](https://github.com/postil-dev/postil-cli/tree/a7e7c67235519fff79c6e82c44550ac29255dcdc) and its [exact invocation](https://github.com/postil-dev/postil-cli/blob/a7e7c67235519fff79c6e82c44550ac29255dcdc/bench/README.md#expanded-clean-bank). The command above uses the same 25 fixture payloads with the default evaluator source digest; its reports are separate evidence.
+
+Compare models only when the selected cases, fixture hash, evaluator hash, binary hash, retry settings, and concurrency match. Provider routes remain explicit. Evidence identifies the fixture/evaluator source by immutable commit and the measured executable by SHA-256; use `POSTIL_BIN` to select that executable. A different build produces separate evidence. This authored bank is not held-out validation, and one observation per fixture does not establish a stable false-positive rate.
+
 ## Managed qualification
 
 Managed qualification exercises an ordered generator and scorer pair through the mock forge and a real provider. It requires an exact pair, provider identity and route, three complete repeats, and a release build whose embedded profile matches the worktree. Run the manual [managed admission workflow](../.github/workflows/bench-live.yml) for the attested hosted path. `bun run verify-admission` validates checked-in admission evidence.

bench/clean-screen-glm.json

@@ -0,0 +1,22 @@
+{
+  "benchmarkProviderIdentity": "openrouter:managed-routing",
+  "upstreamProviderIdentity": "Z.AI",
+  "upstreamProviderRoute": "z-ai/fp8",
+  "apiBase": "https://openrouter.ai:443/api/v1",
+  "apiFormat": "openai-compatible",
+  "generatorChain": [
+    "z-ai/glm-5.2"
+  ],
+  "consensus": 1,
+  "scorerChain": [],
+  "providerGenerationModels": {
+    "z-ai/glm-5.2": "z-ai/glm-5.2-20260616"
+  },
+  "modelPriceBounds": [
+    {
+      "model": "z-ai/glm-5.2",
+      "inputMicrosPerMillionTokens": 1400000,
+      "outputMicrosPerMillionTokens": 4400000
+    }
+  ]
+}

bench/clean-screen-luna.json

@@ -0,0 +1,22 @@
+{
+  "benchmarkProviderIdentity": "openrouter:managed-routing",
+  "upstreamProviderIdentity": "Azure",
+  "upstreamProviderRoute": "azure/eu",
+  "apiBase": "https://openrouter.ai:443/api/v1",
+  "apiFormat": "openai-compatible",
+  "generatorChain": [
+    "openai/gpt-5.6-luna"
+  ],
+  "consensus": 1,
+  "scorerChain": [],
+  "providerGenerationModels": {
+    "openai/gpt-5.6-luna": "openai/gpt-5.6-luna-20260709"
+  },
+  "modelPriceBounds": [
+    {
+      "model": "openai/gpt-5.6-luna",
+      "inputMicrosPerMillionTokens": 220000,
+      "outputMicrosPerMillionTokens": 1320000
+    }
+  ]
+}

bench/fixtures/cases.test.ts

@@ -1,5 +1,137 @@
 import { describe, expect, test } from "bun:test";
-import { makeDiff } from "./cases";
+import { cases, makeDiff } from "./cases";
+import { supplementalCleanCases, supplementalCleanSpecs } from "./clean-screen";
+import { benchmarkCase, parseUnifiedDiffFiles } from "../src/harness";
+
+test("supplemental modules supply complete evidence and unique clean targets", () => {
+  const ids = [...cases, ...supplementalCleanCases].map((input) => input.id);
+  expect(new Set(ids).size).toBe(ids.length);
+  expect(supplementalCleanCases).toHaveLength(12);
+  for (const input of supplementalCleanCases) {
+    const parsed = benchmarkCase.parse(input);
+    expect(parsed.groundTruth.findings).toEqual([]);
+    expect(parsed.modelOutput.findings).toEqual([]);
+    const file = parseUnifiedDiffFiles(parsed.diff).find((file) => file.path === parsed.allowedContext.files[0].path)!;
+    expect(file.after.replace(/^ /gm, "").trim()).toBe(parsed.allowedContext.files[0].content.trim());
+    expect(parsed.diff).toContain(parsed.allowedContext.docs[0].content);
+  }
+});
+
+for (const version of ["before", "after"] as const) {
+  const program = (id: string): any => {
+    const spec = supplementalCleanSpecs.find((entry) => entry.id === `clean-${id}`)!;
+    return new Function(`${spec[version].replace("export ", "")}; return run;`)();
+  };
+  describe(`supplemental clean behavior: ${version}`, () => {
+    test("tenant checks allow the owner and reject a different tenant", () => {
+      const run = program("tenant-guard-extraction");
+      expect(run({ tenantId: "a" }, { tenantId: "a", name: "report" })).toBe("report");
+      expect(() => run({ tenantId: "b" }, { tenantId: "a", name: "report" })).toThrow("Forbidden");
+    });
+    test("expiry includes the boundary and preserves falsy cached values", () => {
+      const run = program("cache-expiry-boundary");
+      expect(run(null, 10)).toBeNull();
+      expect(run({ expiresAt: 10, value: false }, 9)).toBe(false);
+      expect(run({ expiresAt: 10, value: "stale" }, 10)).toBeNull();
+      expect(run({ expiresAt: 10, value: "stale" }, 11)).toBeNull();
+    });
+    test("concurrent reads preserve order and propagate rejection", async () => {
+      const run = program("concurrent-result-order");
+      const pending = new Map<number, (value: string) => void>();
+      const result = run([1, 2], (id: number) => new Promise<string>((resolve) => pending.set(id, resolve)));
+      expect([...pending.keys()]).toEqual([1, 2]);
+      pending.get(2)!("second"); pending.get(1)!("first");
+      expect(await result).toEqual(["first", "second"]);
+      expect(await run([], () => { throw new Error("unexpected read"); })).toEqual([]);
+      await expect(run([1], async () => { throw new Error("read failed"); })).rejects.toThrow("read failed");
+    });
+    test("cancellation reaches the request with the original signal", async () => {
+      const run = program("abort-signal-forwarding");
+      const controller = new AbortController();
+      const result = run((_url: string, options: RequestInit) => {
+        expect(options.method).toBe("GET");
+        expect(options.signal).toBe(controller.signal);
+        return new Promise((_resolve, reject) => options.signal!.addEventListener("abort", () => reject(new Error("aborted")), { once: true }));
+      }, "https://example.test/items", controller.signal);
+      controller.abort();
+      await expect(result).rejects.toThrow("aborted");
+    });
+    test("retry budget stops at the boundary and propagates failures", async () => {
+      const run = program("retry-limit-extraction");
+      let calls = 0;
+      const retry = async () => { calls++; return "retried"; };
+      expect(await run(2, retry, 3)).toBe("retried");
+      expect(await run(3, retry, 3)).toBe("exhausted");
+      expect(await run(4, retry, 3)).toBe("exhausted");
+      expect(calls).toBe(1);
+      await expect(run(0, async () => { throw new Error("retry failed"); }, 3)).rejects.toThrow("retry failed");
+    });
+    test("pagination excludes the cursor without gaps or repeats", () => {
+      const run = program("pagination-exclusive-cursor");
+      const rows = [1, 3, 5, 8].map((id) => ({ id }));
+      expect(run(rows, 1, 2)).toEqual([{ id: 3 }, { id: 5 }]);
+      expect(run(rows, 5, 2)).toEqual([{ id: 8 }]);
+      expect(run(rows, 8, 2)).toEqual([]);
+      expect(run([], 0, 2)).toEqual([]);
+    });
+    test("serialization retains exact integer digits", () => {
+      const run = program("bigint-json-serialization");
+      for (const value of [0n, 9007199254740993n, -9007199254740993n]) {
+        expect(JSON.parse(run(value))).toEqual({ totalMicros: String(value) });
+      }
+    });
+    test("configuration distinguishes zero from missing values", () => {
+      const run = program("config-preserves-zero");
+      expect(run({})).toBe(3); expect(run({ retries: null })).toBe(3);
+      expect(run({ retries: 0 })).toBe(0); expect(run({ retries: 5 })).toBe(5);
+    });
+    test("lock stays held until the write settles on success and failure", async () => {
+      const run = program("lock-finally-release");
+      for (const fails of [false, true]) {
+        const events: string[] = [];
+        let finish!: () => void;
+        const result = run(async () => { events.push("lock"); return () => events.push("release"); }, async () => {
+          events.push("write");
+          await new Promise<void>((resolve) => { finish = resolve; });
+          events.push("settled");
+          if (fails) throw new Error("write failed");
+          return 42;
+        });
+        await Promise.resolve();
+        expect(events).toEqual(["lock", "write"]);
+        finish();
+        if (fails) await expect(result).rejects.toThrow("write failed");
+        else expect(await result).toBe(42);
+        expect(events).toEqual(["lock", "write", "settled", "release"]);
+      }
+    });
+    test("untrusted query input remains a separate bound parameter", async () => {
+      const run = program("parameterized-query-extraction");
+      const input = "' OR 1=1 --";
+      const result = await run({ query: async (sql: string, params: unknown[]) => {
+        expect(sql).toBe("SELECT id FROM people WHERE name = $1");
+        expect(params).toEqual([input]);
+        return [{ id: 7 }];
+      } }, input);
+      expect(result).toEqual([{ id: 7 }]);
+    });
+    test("sorting preserves caller ownership and descending numeric order", () => {
+      const run = program("input-array-copy-sort");
+      const input = Object.freeze([2, -1, 10, 2]);
+      expect(run(input)).toEqual([10, 2, 2, -1]);
+      expect(input).toEqual([2, -1, 10, 2]);
+      expect(run([])).toEqual([]);
+    });
+    test("partial updates distinguish absence, null, and inherited properties", () => {
+      const run = program("optional-field-presence");
+      const current = Object.freeze({ displayName: "Name", id: 1 });
+      expect(run(current, {})).toEqual(current);
+      expect(run(current, { displayName: null })).toEqual({ displayName: null, id: 1 });
+      expect(run(current, { displayName: "" })).toEqual({ displayName: "", id: 1 });
+      expect(run(current, Object.create({ displayName: "inherited" }))).toEqual(current);
+    });
+  });
+}
 
 describe("makeDiff", () => {
   test("uses one-line hunk counts for one-line fixtures", () => {