From b36a14ae1b6142255bc0428142deff68460198c0 Mon Sep 17 00:00:00 2001 From: mohammed naji Date: Wed, 12 Aug 2026 09:15:18 +0400 Subject: [PATCH 01/10] feat(qualification): freeze the independent qualification contract (#655) Add docs/qualification/, a frozen evaluation contract separate from the product benchmark suite in docs/benchmarks/suite/, so roadmap changes can be judged without grading Madar against expectations Madar produced. Contents: - corpus.json pins two new TypeScript fixture targets by content digest, two public repositories by commit SHA, and one sealed-holdout slot that is explicitly unsatisfied. - tasks.json freezes six prompts with SHA-256 hashes, one per required task category, each recording who authored its truth and whether Madar output was inspected before freezing. - truth/ holds per-task independent truth: critical facts, ordered execution paths, affected/unaffected sets, seeded-defect root causes, required uncertainty, and unsupported-claim traps. - rubrics.json separates the seven scoring dimensions, uses a different method per category, and keeps adoption and broad fallback exploration out of the quality score. - receipt-schema.json requires the full experimental identity, splits indexing/context-build/agent cost into separate accounts, and makes an invalid run structurally unaggregatable. - validity-rules.md, holdout-policy.md, stop-rule.md, and evidence-categories.md freeze the invalidation, holdout, stop/rollback, and evidence-labelling rules. - tier1.json is the deterministic PR-runnable subset; tier2-matrix.json freezes the planned repeated-run matrix while it stays unexecuted. npm run qualify:validate checks the contract from a clean checkout and fails if any qualification literal reaches src/, if a frozen file changes without an explicit refreeze, or if a truth file cites a path that does not exist. No production retrieval, ranking, context, or reporting code is touched. --- .../validate-qualification-contract.mjs | 359 ++++++++++++++++++ .github/workflows/ci.yml | 4 + docs/qualification/README.md | 104 +++++ docs/qualification/corpus.json | 88 +++++ docs/qualification/evidence-categories.md | 88 +++++ .../examples/receipt-tier1-valid.json | 98 +++++ .../receipt-tier2-invalid-no-madar-call.json | 91 +++++ .../fixtures/ledger-service/README.md | 39 ++ .../fixtures/ledger-service/package.json | 10 + .../ledger-service/src/audit/audit-log.ts | 22 ++ .../src/auth/request-context.ts | 52 +++ .../ledger-service/src/http/ledger-routes.ts | 79 ++++ .../src/outbox/outbox-publisher.ts | 34 ++ .../src/projections/balance-projection.ts | 45 +++ .../src/service/ledger-service.ts | 120 ++++++ .../src/store/idempotency-store.ts | 34 ++ .../ledger-service/src/store/ledger-store.ts | 46 +++ .../fixtures/ledger-service/tsconfig.json | 11 + .../fixtures/plugin-host/README.md | 35 ++ .../fixtures/plugin-host/package.json | 10 + .../plugin-host/src/contracts/plugin.ts | 45 +++ .../fixtures/plugin-host/src/host/config.ts | 32 ++ .../plugin-host/src/host/lifecycle.ts | 62 +++ .../plugin-host/src/host/plugin-host.ts | 56 +++ .../fixtures/plugin-host/src/host/registry.ts | 46 +++ .../src/plugins/csv-export-plugin.ts | 32 ++ .../src/plugins/webhook-export-plugin.ts | 36 ++ .../fixtures/plugin-host/tsconfig.json | 12 + docs/qualification/freeze.json | 48 +++ docs/qualification/holdout-policy.md | 77 ++++ docs/qualification/receipt-schema.json | 317 ++++++++++++++++ docs/qualification/rubrics.json | 143 +++++++ docs/qualification/stop-rule.md | 65 ++++ docs/qualification/tasks.json | 240 ++++++++++++ docs/qualification/tier1.json | 93 +++++ docs/qualification/tier2-matrix.json | 44 +++ .../arch-plugin-host-extension-seam.json | 125 ++++++ .../truth/flow-ledger-post-entry.json | 174 +++++++++ .../impact-ledger-drop-outbox-publish.json | 111 ++++++ ...lan-plugin-host-object-storage-plugin.json | 129 +++++++ .../review-ledger-http-authorization.json | 118 ++++++ .../rootcause-ledger-duplicate-entries.json | 101 +++++ docs/qualification/validity-rules.md | 95 +++++ package.json | 3 +- tests/unit/qualification-contract.test.ts | 304 +++++++++++++++ 45 files changed, 3876 insertions(+), 1 deletion(-) create mode 100644 .github/scripts/validate-qualification-contract.mjs create mode 100644 docs/qualification/README.md create mode 100644 docs/qualification/corpus.json create mode 100644 docs/qualification/evidence-categories.md create mode 100644 docs/qualification/examples/receipt-tier1-valid.json create mode 100644 docs/qualification/examples/receipt-tier2-invalid-no-madar-call.json create mode 100644 docs/qualification/fixtures/ledger-service/README.md create mode 100644 docs/qualification/fixtures/ledger-service/package.json create mode 100644 docs/qualification/fixtures/ledger-service/src/audit/audit-log.ts create mode 100644 docs/qualification/fixtures/ledger-service/src/auth/request-context.ts create mode 100644 docs/qualification/fixtures/ledger-service/src/http/ledger-routes.ts create mode 100644 docs/qualification/fixtures/ledger-service/src/outbox/outbox-publisher.ts create mode 100644 docs/qualification/fixtures/ledger-service/src/projections/balance-projection.ts create mode 100644 docs/qualification/fixtures/ledger-service/src/service/ledger-service.ts create mode 100644 docs/qualification/fixtures/ledger-service/src/store/idempotency-store.ts create mode 100644 docs/qualification/fixtures/ledger-service/src/store/ledger-store.ts create mode 100644 docs/qualification/fixtures/ledger-service/tsconfig.json create mode 100644 docs/qualification/fixtures/plugin-host/README.md create mode 100644 docs/qualification/fixtures/plugin-host/package.json create mode 100644 docs/qualification/fixtures/plugin-host/src/contracts/plugin.ts create mode 100644 docs/qualification/fixtures/plugin-host/src/host/config.ts create mode 100644 docs/qualification/fixtures/plugin-host/src/host/lifecycle.ts create mode 100644 docs/qualification/fixtures/plugin-host/src/host/plugin-host.ts create mode 100644 docs/qualification/fixtures/plugin-host/src/host/registry.ts create mode 100644 docs/qualification/fixtures/plugin-host/src/plugins/csv-export-plugin.ts create mode 100644 docs/qualification/fixtures/plugin-host/src/plugins/webhook-export-plugin.ts create mode 100644 docs/qualification/fixtures/plugin-host/tsconfig.json create mode 100644 docs/qualification/freeze.json create mode 100644 docs/qualification/holdout-policy.md create mode 100644 docs/qualification/receipt-schema.json create mode 100644 docs/qualification/rubrics.json create mode 100644 docs/qualification/stop-rule.md create mode 100644 docs/qualification/tasks.json create mode 100644 docs/qualification/tier1.json create mode 100644 docs/qualification/tier2-matrix.json create mode 100644 docs/qualification/truth/arch-plugin-host-extension-seam.json create mode 100644 docs/qualification/truth/flow-ledger-post-entry.json create mode 100644 docs/qualification/truth/impact-ledger-drop-outbox-publish.json create mode 100644 docs/qualification/truth/plan-plugin-host-object-storage-plugin.json create mode 100644 docs/qualification/truth/review-ledger-http-authorization.json create mode 100644 docs/qualification/truth/rootcause-ledger-duplicate-entries.json create mode 100644 docs/qualification/validity-rules.md create mode 100644 tests/unit/qualification-contract.test.ts diff --git a/.github/scripts/validate-qualification-contract.mjs b/.github/scripts/validate-qualification-contract.mjs new file mode 100644 index 00000000..0be3179b --- /dev/null +++ b/.github/scripts/validate-qualification-contract.mjs @@ -0,0 +1,359 @@ +import { createHash } from 'node:crypto' +import { readdirSync, readFileSync, statSync, writeFileSync } from 'node:fs' +import { join, relative, resolve } from 'node:path' + +import Ajv from 'ajv' +import addFormats from 'ajv-formats' + +const ROOT = resolve('docs/qualification') +const FREEZE_PATH = join(ROOT, 'freeze.json') +const PRODUCTION_ROOT = resolve('src') +const WRITE = process.argv.includes('--write') + +const failures = [] + +function fail(message) { + failures.push(message) +} + +function readJson(path) { + return JSON.parse(readFileSync(path, 'utf8')) +} + +function sha256(value) { + return createHash('sha256').update(value).digest('hex') +} + +function walk(dir) { + const entries = [] + for (const name of readdirSync(dir).sort()) { + const full = join(dir, name) + if (statSync(full).isDirectory()) { + entries.push(...walk(full)) + } else { + entries.push(full) + } + } + return entries +} + +const corpus = readJson(join(ROOT, 'corpus.json')) +const tasks = readJson(join(ROOT, 'tasks.json')) +const rubrics = readJson(join(ROOT, 'rubrics.json')) +const tier1 = readJson(join(ROOT, 'tier1.json')) +const tier2 = readJson(join(ROOT, 'tier2-matrix.json')) +const receiptSchema = readJson(join(ROOT, 'receipt-schema.json')) + +const CONTRACT_VERSION = corpus.contract_version + +// --------------------------------------------------------------------------- +// 1. Contract version agreement +// --------------------------------------------------------------------------- + +for (const [name, doc] of [ + ['tasks.json', tasks], + ['rubrics.json', rubrics], + ['tier1.json', tier1], + ['tier2-matrix.json', tier2], +]) { + if (doc.contract_version !== CONTRACT_VERSION) { + fail(`${name} declares contract_version ${doc.contract_version}, expected ${CONTRACT_VERSION}`) + } +} + +// --------------------------------------------------------------------------- +// 2. Targets +// --------------------------------------------------------------------------- + +const targetsById = new Map(corpus.targets.map((target) => [target.id, target])) +const fixtureTargets = corpus.targets.filter((target) => target.kind === 'fixture') + +for (const target of fixtureTargets) { + try { + if (!statSync(resolve(target.path)).isDirectory()) { + fail(`target ${target.id} path ${target.path} is not a directory`) + } + } catch { + fail(`target ${target.id} path ${target.path} does not exist`) + } +} + +for (const target of corpus.targets) { + if (target.kind === 'git' && !/^[0-9a-f]{40}$/.test(target.source?.ref ?? '')) { + fail(`git target ${target.id} must pin an immutable 40-character commit SHA`) + } + if (target.status === 'pinned_no_truth' && target.tier === 1) { + fail(`target ${target.id} is Tier 1 but has no independent truth`) + } +} + +// --------------------------------------------------------------------------- +// 3. Tasks, prompts, truth files +// --------------------------------------------------------------------------- + +const REQUIRED_CATEGORIES = [ + 'architecture-understanding', + 'execution-flow-explanation', + 'impact-analysis', + 'bug-root-cause-investigation', + 'implementation-planning', + 'review-security', +] + +const seenCategories = new Set() +const tasksById = new Map() + +for (const task of tasks.tasks) { + tasksById.set(task.id, task) + seenCategories.add(task.category) + + const target = targetsById.get(task.target) + if (!target) { + fail(`task ${task.id} references unknown target ${task.target}`) + continue + } + + const actualHash = sha256(task.prompt.text) + if (actualHash !== task.prompt.sha256) { + fail(`task ${task.id} prompt hash mismatch: recorded ${task.prompt.sha256}, actual ${actualHash}`) + } + + const truthPath = join(ROOT, task.truth_ref) + let truth + try { + truth = readJson(truthPath) + } catch { + fail(`task ${task.id} truth file ${task.truth_ref} is missing or unreadable`) + continue + } + + if (truth.task_id !== task.id) fail(`${task.truth_ref} declares task_id ${truth.task_id}, expected ${task.id}`) + if (truth.target !== task.target) fail(`${task.truth_ref} declares target ${truth.target}, expected ${task.target}`) + if (truth.category !== task.category) fail(`${task.truth_ref} declares category ${truth.category}, expected ${task.category}`) + if (truth.contract_version !== CONTRACT_VERSION) fail(`${task.truth_ref} declares contract_version ${truth.contract_version}`) + + // Independence: truth must not be derived from Madar output. + for (const provenance of [task.truth_provenance, truth.provenance]) { + if (provenance.inspected_madar_output_before_freeze !== false) { + fail(`task ${task.id} truth provenance claims Madar output was inspected before freezing`) + } + if (!Array.isArray(provenance.madar_derived_sources_used) || provenance.madar_derived_sources_used.length > 0) { + fail(`task ${task.id} truth provenance lists Madar-derived sources: ${JSON.stringify(provenance.madar_derived_sources_used)}`) + } + if (!provenance.authored_by || !provenance.authored_at) { + fail(`task ${task.id} truth provenance must record who authored the truth and when`) + } + if (!('independent_of_production_rule_author' in provenance)) { + fail(`task ${task.id} truth provenance must state whether the author is independent of the production-rule author`) + } + } + + // Every cited evidence path must exist inside the target workspace. + const citedPaths = new Set() + const collect = (node) => { + if (Array.isArray(node)) { + node.forEach(collect) + return + } + if (node && typeof node === 'object') { + for (const [key, value] of Object.entries(node)) { + if (key === 'path' && typeof value === 'string') citedPaths.add(value) + else collect(value) + } + } + } + collect(truth) + + for (const cited of citedPaths) { + const full = resolve(target.path, cited) + try { + statSync(full) + } catch { + fail(`${task.truth_ref} cites ${cited}, which does not exist in target ${target.id}`) + } + } + + const obligations = truth.tier1_obligations + if (!obligations) { + fail(`${task.truth_ref} has no tier1_obligations block`) + } else if (!Array.isArray(obligations.must_not_report_ready_when) || obligations.must_not_report_ready_when.length === 0) { + fail(`${task.truth_ref} must declare at least one must_not_report_ready_when condition`) + } + + const rubricMethod = task.scoring.tier2_method + if (!rubrics.methods[rubricMethod]) { + fail(`task ${task.id} references unknown rubric method ${rubricMethod}`) + } + if (!rubrics.methods[task.scoring.tier1_method]) { + fail(`task ${task.id} references unknown tier1 method ${task.scoring.tier1_method}`) + } +} + +for (const category of REQUIRED_CATEGORIES) { + if (!seenCategories.has(category)) { + fail(`no frozen task covers required category ${category}`) + } +} + +// --------------------------------------------------------------------------- +// 4. Tier 1 subset and negative-trust probes +// --------------------------------------------------------------------------- + +for (const cell of tier1.cells) { + const task = tasksById.get(cell.task_id) + if (!task) { + fail(`tier1 cell references unknown task ${cell.task_id}`) + continue + } + if (task.target !== cell.target_id) { + fail(`tier1 cell ${cell.task_id} targets ${cell.target_id} but the task targets ${task.target}`) + } + if (!task.tiers.includes(1)) { + fail(`tier1 cell ${cell.task_id} refers to a task that does not declare tier 1`) + } +} + +for (const probe of tier1.negative_trust_probes) { + const actual = sha256(probe.prompt.text) + if (actual !== probe.prompt.sha256) { + fail(`negative-trust probe ${probe.id} prompt hash mismatch: recorded ${probe.prompt.sha256}, actual ${actual}`) + } + if (!targetsById.has(probe.target_id)) { + fail(`negative-trust probe ${probe.id} references unknown target ${probe.target_id}`) + } +} + +// --------------------------------------------------------------------------- +// 5. Tier 2 matrix references +// --------------------------------------------------------------------------- + +for (const id of tier2.dimensions.targets) { + if (!targetsById.has(id)) fail(`tier2 matrix references unknown target ${id}`) +} +for (const id of tier2.dimensions.tasks) { + if (!tasksById.has(id)) fail(`tier2 matrix references unknown task ${id}`) +} +if (tier2.status !== 'planned') { + fail('tier2-matrix.json must stay planned until its execution prerequisites are met') +} + +// --------------------------------------------------------------------------- +// 6. Receipt schema and examples +// --------------------------------------------------------------------------- + +const ajv = new Ajv({ allErrors: true, strict: false }) +addFormats(ajv) +const validateReceipt = ajv.compile(receiptSchema) + +const examplesDir = join(ROOT, 'examples') +for (const path of walk(examplesDir)) { + const receipt = readJson(path) + if (!validateReceipt(receipt)) { + fail(`${relative(process.cwd(), path)} does not satisfy receipt-schema.json: ${ajv.errorsText(validateReceipt.errors)}`) + } + if (receipt.validity.status !== 'valid' && receipt.validity.aggregatable !== false) { + fail(`${relative(process.cwd(), path)} is not valid but is marked aggregatable`) + } + for (const [name, score] of Object.entries(receipt.scores)) { + if (score.measured === false && score.value !== null) { + fail(`${relative(process.cwd(), path)} score ${name} is not measured but carries a value`) + } + } +} + +// --------------------------------------------------------------------------- +// 7. Benchmark independence: no qualification literal may reach production code +// --------------------------------------------------------------------------- + +const FORBIDDEN_LITERALS = [ + ...corpus.targets.map((target) => target.id), + ...fixtureTargets.map((target) => target.path), + ...tasks.tasks.map((task) => task.id), + ...tasks.tasks.map((task) => task.prompt.text), + ...tier1.negative_trust_probes.map((probe) => probe.prompt.text), + 'LedgerService', + 'IdempotencyStore', + 'OutboxPublisher', + 'BalanceProjection', + 'assertAccountAccess', + 'CsvExportPlugin', + 'WebhookExportPlugin', + 'runExportLifecycle', + 'builtInPlugins', +] + +for (const path of walk(PRODUCTION_ROOT)) { + const content = readFileSync(path, 'utf8') + for (const literal of FORBIDDEN_LITERALS) { + if (content.includes(literal)) { + fail(`production file ${relative(process.cwd(), path)} contains qualification literal "${literal}"`) + } + } +} + +// --------------------------------------------------------------------------- +// 8. Freeze digests +// --------------------------------------------------------------------------- + +const frozenFiles = walk(ROOT) + .filter((path) => path !== FREEZE_PATH) + .map((path) => relative(process.cwd(), path).split('\\').join('/')) + .sort() + +const digests = Object.fromEntries( + frozenFiles.map((path) => [path, sha256(readFileSync(resolve(path)))]), +) + +if (WRITE) { + const freeze = { + contract_version: CONTRACT_VERSION, + frozen_at: corpus.frozen_at, + algorithm: 'sha256 over raw file bytes', + note: 'Regenerate deliberately with `npm run qualify:validate -- --write` and say why in the pull request. A silent digest change is a contract change.', + files: digests, + } + writeFileSync(FREEZE_PATH, `${JSON.stringify(freeze, null, 2)}\n`) + console.log(`wrote ${relative(process.cwd(), FREEZE_PATH)} with ${frozenFiles.length} entries`) +} else { + let freeze + try { + freeze = readJson(FREEZE_PATH) + } catch { + fail('freeze.json is missing; run `npm run qualify:validate -- --write`') + } + + if (freeze) { + if (freeze.contract_version !== CONTRACT_VERSION) { + fail(`freeze.json declares contract_version ${freeze.contract_version}, expected ${CONTRACT_VERSION}`) + } + for (const [path, digest] of Object.entries(digests)) { + if (!(path in freeze.files)) { + fail(`${path} is not covered by freeze.json`) + } else if (freeze.files[path] !== digest) { + fail(`${path} content changed since it was frozen (expected ${freeze.files[path]}, actual ${digest})`) + } + } + for (const path of Object.keys(freeze.files)) { + if (!(path in digests)) { + fail(`freeze.json references ${path}, which no longer exists`) + } + } + } +} + +// --------------------------------------------------------------------------- + +if (failures.length > 0) { + console.error(`qualification contract validation failed with ${failures.length} problem(s):`) + for (const failure of failures) { + console.error(` - ${failure}`) + } + process.exit(1) +} + +console.log( + `qualification contract v${CONTRACT_VERSION} is consistent: ` + + `${corpus.targets.length} targets, ${tasks.tasks.length} tasks, ` + + `${tier1.cells.length} Tier 1 cells, ${tier1.negative_trust_probes.length} negative-trust probes, ` + + `${frozenFiles.length} frozen files.`, +) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ca8ef41e..45e1c0b5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,6 +52,10 @@ jobs: if: matrix.os == 'ubuntu-latest' && matrix.node-version == '22' run: npm run release:verify + - name: Validate qualification contract + if: matrix.os == 'ubuntu-latest' && matrix.node-version == '22' + run: npm run qualify:validate + - name: Typecheck run: npm run typecheck diff --git a/docs/qualification/README.md b/docs/qualification/README.md new file mode 100644 index 00000000..7a6a2f67 --- /dev/null +++ b/docs/qualification/README.md @@ -0,0 +1,104 @@ +# Qualification contract + +Contract version `1.0.0`, frozen 2026-08-12 for [#655](https://github.com/mohanagy/madar/issues/655) +against Madar commit `06b373a447acfce895412ac10eb4e5228c5df0b7` (`v0.32.1`). + +This directory is the independent evaluation contract used to decide whether a roadmap +change is safe to ship. It is deliberately separate from +[`docs/benchmarks/suite/`](../benchmarks/suite/), which is the product benchmark suite and +whose per-repo expectations were authored alongside the product. + +## What this contract is for + +Grading Madar with expectations that Madar produced tells you nothing. This contract fixes +five things before any change is evaluated: + +1. what is being evaluated (`corpus.json`, `tasks.json`); +2. what a correct answer is, authored without looking at Madar output (`truth/`); +3. how it is scored, and by which method per category (`rubrics.json`); +4. when a run does not count at all (`validity-rules.md`, `receipt-schema.json`); +5. what result blocks a merge or forces a rollback (`stop-rule.md`). + +## Files + +| File | Deliverable | +| --- | --- | +| [`corpus.json`](./corpus.json) | Versioned corpus manifest: targets, revisions, dependency locks, holdout class. | +| [`tasks.json`](./tasks.json) | Versioned task definitions: frozen prompts with hashes, categories, scoring method, truth provenance. | +| [`truth/`](./truth/) | Independent truth and rubric input, one file per task. | +| [`rubrics.json`](./rubrics.json) | Scoring dimensions, per-category scoring methods, blinding rules, aggregation rules. | +| [`receipt-schema.json`](./receipt-schema.json) | Environment and run receipt schema. | +| [`examples/`](./examples/) | Two illustrative receipts: a valid Tier 1 run and an invalid Tier 2 run that stays `not_measured`. | +| [`validity-rules.md`](./validity-rules.md) | Valid/invalid criteria, the `not_measured` rule, retention, and what today's emitter actually produces. | +| [`holdout-policy.md`](./holdout-policy.md) | Hidden holdout handling — and why the sealed slot is currently unsatisfied. | +| [`stop-rule.md`](./stop-rule.md) | Objective stop, rollback, and publication rule. | +| [`evidence-categories.md`](./evidence-categories.md) | Evidence classes E1–E6 and required labelling. | +| [`tier1.json`](./tier1.json) | The small deterministic subset a pull request can run. | +| [`tier2-matrix.json`](./tier2-matrix.json) | The planned repeated-run matrix. | +| [`fixtures/`](./fixtures/) | Tier 1 target workspaces. | +| [`freeze.json`](./freeze.json) | SHA-256 of every file above. A silent change to any of them fails validation. | + +## Tiers + +**Tier 1** is deterministic, needs no network, no model provider, and no spend. It measures +whether the evidence required to answer each frozen task was present in the context +artifact, and whether readiness was correctly refused on the negative-trust probes. It never +scores answer quality and never runs an agent. + +**Tier 2** runs a real agent on both arms with repeated trials and blinded rubric scoring. +It is frozen but not executed; its prerequisites are listed in `tier2-matrix.json`. + +## Running it + +```bash +npm ci +npm run qualify:validate +``` + +`qualify:validate` checks, from a clean checkout and without running Madar: + +- every declared contract version agrees; +- every task references a real target, and every frozen prompt matches its recorded hash; +- every truth file exists, matches its task, and cites only paths that exist in the target; +- every truth file records who authored it and asserts no Madar-derived source was used; +- all six required task categories are covered; +- every Tier 1 cell and negative-trust probe resolves, and every probe prompt hash matches; +- both example receipts validate against `receipt-schema.json`, and no unmeasured score + carries a value; +- **no qualification target id, task id, prompt string, or fixture symbol appears anywhere + in `src/`**; +- every file in this directory matches its frozen digest. + +Regenerating the freeze file is deliberate and must be explained in the pull request: + +```bash +npm run qualify:validate -- --write +``` + +Executing the Tier 1 subset against Madar is [#661](https://github.com/mohanagy/madar/issues/661), +not this contract. + +## Independence + +Both Tier 1 fixtures and all six truth files were authored on 2026-08-12 from blank files. +Madar was never run against them, and no Madar retrieval output, context pack, +`implementationGuidance`, Madar-selected file list, or Madar-generated validation command +was consulted before freezing. Each task records this in `truth_provenance`. + +Two consequences follow, and both are stated rather than papered over: + +- **Thresholds are pre-registered, not calibrated.** Nobody knows how many Tier 1 cells + currently pass. The first execution is a measurement; a failure there is a product + finding, not a reason to edit this contract. +- **The author is not independent of the production-rule author.** Madar has one author, so + `independent_of_production_rule_author` is `false` on every task, blinded review is + unavailable, and the sealed holdout slot is unsatisfied. See + [`holdout-policy.md`](./holdout-policy.md) for the human action that would fix this. Until + then this corpus measures regression, not generalization, and no superiority or + generalization claim may rest on it. + +## Non-goals + +This contract does not run the 480+ public superiority experiment, does not change +production retrieval or context logic, does not tune ranking against any target, and does +not publish any claim. diff --git a/docs/qualification/corpus.json b/docs/qualification/corpus.json new file mode 100644 index 00000000..2d679c53 --- /dev/null +++ b/docs/qualification/corpus.json @@ -0,0 +1,88 @@ +{ + "contract_version": "1.0.0", + "frozen_at": "2026-08-12", + "frozen_for_issue": 655, + "frozen_against": { + "madar_commit": "06b373a447acfce895412ac10eb4e5228c5df0b7", + "madar_package_version": "0.32.1", + "dependency_lock": "package-lock.json at the pinned commit; runs must use `npm ci`, never `npm install`" + }, + "support_corridor": "typescript-node", + "targets": [ + { + "id": "qual-ledger-service", + "name": "Ledger service qualification fixture", + "tier": 1, + "kind": "fixture", + "path": "docs/qualification/fixtures/ledger-service", + "language": "typescript", + "shape": "layered-http-service", + "dependency_lock": "none — the fixture has no runtime dependencies", + "holdout_class": "open", + "status": "frozen", + "digest_ref": "freeze.json#/fixtures/docs~1qualification~1fixtures~1ledger-service" + }, + { + "id": "qual-plugin-host", + "name": "Plugin host qualification fixture", + "tier": 1, + "kind": "fixture", + "path": "docs/qualification/fixtures/plugin-host", + "language": "typescript", + "shape": "extension-host", + "dependency_lock": "none — the fixture has no runtime dependencies", + "holdout_class": "open", + "status": "frozen", + "digest_ref": "freeze.json#/fixtures/docs~1qualification~1fixtures~1plugin-host" + }, + { + "id": "qual-unkey", + "name": "Unkey public repository", + "tier": 2, + "kind": "git", + "source": { + "url": "https://github.com/unkeyed/unkey", + "ref": "47e64533d56ff13dc2af673a203bc625ef34cf8a" + }, + "language": "typescript", + "shape": "api-key-and-rate-limit-platform", + "dependency_lock": "repository lockfile at the pinned ref; install with the repository's own frozen-lockfile command", + "holdout_class": "open", + "status": "pinned_no_truth", + "selection_rationale": "Deliberately outside docs/benchmarks/suite/repos.json so Tier 2 qualification does not inherit repositories that existing production heuristics or published receipts were shaped around." + }, + { + "id": "qual-payload", + "name": "Payload CMS public repository", + "tier": 2, + "kind": "git", + "source": { + "url": "https://github.com/payloadcms/payload", + "ref": "c6e79520e3ea70afc2a001b7cf8ec32683246c57" + }, + "language": "typescript", + "shape": "headless-cms-monorepo", + "dependency_lock": "repository lockfile at the pinned ref; install with the repository's own frozen-lockfile command", + "holdout_class": "open", + "status": "pinned_no_truth", + "selection_rationale": "Deliberately outside docs/benchmarks/suite/repos.json so Tier 2 qualification does not inherit repositories that existing production heuristics or published receipts were shaped around." + }, + { + "id": "qual-sealed-a", + "name": "Sealed holdout target A", + "tier": 2, + "kind": "sealed", + "language": "typescript", + "shape": "undisclosed", + "dependency_lock": "recorded in the sealed manifest, not in this repository", + "holdout_class": "sealed", + "status": "unsatisfied", + "unsatisfied_reason": "Requires a second person to author and hold the target and its truth. See holdout-policy.md; the slot stays visible and explicitly unsatisfied rather than being filled with a self-authored placeholder." + } + ], + "status_meaning": { + "frozen": "Target content is digest-pinned in freeze.json and may be used as measurable Tier 1 evidence.", + "pinned_no_truth": "Repository and revision are pinned, but no independent truth exists yet. Runs against this target can produce receipts and MUST report not_measured for every quality dimension.", + "unsatisfied": "The slot is specified but cannot be filled in the current single-author context. It must never be counted as evidence, present or absent." + } +} diff --git a/docs/qualification/evidence-categories.md b/docs/qualification/evidence-categories.md new file mode 100644 index 00000000..2d74079f --- /dev/null +++ b/docs/qualification/evidence-categories.md @@ -0,0 +1,88 @@ +# Evidence categories + +Contract version `1.0.0`, frozen 2026-08-12 for [#655](https://github.com/mohanagy/madar/issues/655). + +Madar's repository already contains several kinds of artifact that look like measurement. +They are not interchangeable. Every published statement must name the category of evidence +it rests on. + +## Categories + +### E1 — Product outcome evidence + +A real agent, on a pinned target, answering a frozen prompt, scored against independent +truth by a blinded reviewer, with a valid receipt. + +Only E1 supports a statement about what Madar does for a user. + +**Currently held: none.** No artifact in this repository meets E1. + +### E2 — Context sufficiency evidence + +Deterministic measurement of whether the evidence needed to answer was present in the +context artifact, and whether readiness was correctly refused. No agent runs. + +This is what [`tier1.json`](./tier1.json) produces. E2 supports statements about retrieval +and context quality. It **does not** support any statement about answer quality, token +cost, or user outcome. + +**Currently held: none executed.** The Tier 1 subset is frozen but has never been run; +see `tier1.json#/calibration_status`. + +### E3 — Controlled profile-assisted measurement + +A real agent run where the prompt, the grader, or the retrieval path was assisted by +task-specific expectations authored alongside the product. + +The June 10 2026 receipts under `docs/benchmarks/suite/results/` are E3: the answering +prompts included proof checklists and the checkout could load expected files and functions +from `docs/benchmarks/suite/runtime-proof.json`. They are genuine measurements of the setup +they describe. They are **not** evidence of untuned behaviour, and they are **not** E1. + +### E4 — Synthetic or fixture receipts + +Checked-in deterministic bundles with fixture-anchored timings and tool-call counts, such +as `docs/benchmarks/suite/results/2026-05-31T12-00-00/`. + +E4 proves the reporting pipeline works. It is never agent-outcome evidence. Identical +counts across trials in an E4 bundle are a property of the fixture, not a finding. + +### E5 — Package and parity checks + +`npm run verify:pack-parity`, `npm pack --dry-run`, Registry validation, release +verification. + +E5 proves that the packed artifact behaves like the checkout and that the release is +well-formed. It says nothing about retrieval quality or agent outcome. + +### E6 — Adoption and instrumentation observations + +Counts of attributable Madar calls, trace availability, tool permission failures, +environment drift. + +E6 explains why a run is invalid. It is reported in its own column. An adoption failure is +**not** a quality loss, and an adoption success is **not** a quality win. The July 15 2026 +receipts are largely E6: four of six rows recorded no attributable Madar call at all. + +## Required labelling + +Every table, README line, or release note derived from this corpus states its category. +The permitted phrasings are: + +- E1 — "measured agent outcome" +- E2 — "context sufficiency, no agent" +- E3 — "controlled, profile-assisted" +- E4 — "synthetic fixture receipt" +- E5 — "package parity check" +- E6 — "adoption observation" + +## Prohibited combinations + +- E2, E3, E4, E5, or E6 must never be described as a product outcome, a win, a loss, or a + superiority result. +- E3 must never be presented without the word *controlled* and a pointer to what assisted it. +- E4 must never appear in the same table as E1 or E3 without a category column. +- An E6 adoption failure must never be aggregated as a quality loss, and its cost figures + must never be cited. +- No category may be upgraded by repetition. Running an E4 bundle a hundred times produces + E4. diff --git a/docs/qualification/examples/receipt-tier1-valid.json b/docs/qualification/examples/receipt-tier1-valid.json new file mode 100644 index 00000000..53051c64 --- /dev/null +++ b/docs/qualification/examples/receipt-tier1-valid.json @@ -0,0 +1,98 @@ +{ + "contract_version": "1.0.0", + "run_id": "example-tier1-valid-0001", + "tier": 1, + "task_id": "rootcause-ledger-duplicate-entries", + "target_id": "qual-ledger-service", + "arm": "madar", + "trial": 1, + "cache_mode": "cold", + "identity": { + "target_revision": "sha256:0000000000000000000000000000000000000000000000000000000000000000", + "dependency_lock_sha256": "6328bf95a901590814ff70ed570e0fc474c05ae162d6ac690cf3c812380828ab", + "madar_commit": "06b373a447acfce895412ac10eb4e5228c5df0b7", + "madar_package_version": "0.32.1", + "madar_package_tarball_sha256": null, + "madar_runtime_source": "checkout", + "madar_config_sha256": "c6dcdf9130e3ef5503caead4fe07f1b465b429adfa6e43309166d8f650a33caf", + "agent": { + "host": "none", + "host_version": null, + "model_id": "not-applicable-tier1-deterministic" + }, + "prompts": { + "system_prompt_sha256": null, + "user_prompt_sha256": "3d6f2f70a3b21508c17637cce0d9d94ccbea241adaeb2223192664c8d9856164", + "user_prompt_text": "Under load, a client that retries a request with the same idempotency key sometimes ends up with two ledger entries instead of one. Find the root cause and explain the exact ordering of operations that produces the duplicate." + }, + "tool_permissions": [], + "cache_mode": "cold" + }, + "environment": { + "isolation": true, + "host_os": "darwin", + "node_version": "v22.0.0", + "claude_code_version": null, + "mcp_servers_active": [], + "skills_loaded": [], + "plugins_active": [], + "user_claude_md_hash": null, + "project_claude_md_hash": null, + "hooks_active": { + "user_prompt_submit": [], + "pre_tool_use": [], + "post_tool_use": [] + }, + "drift": { "detected": false, "fields": [] } + }, + "adoption": { + "status": "not_applicable", + "attributable_madar_calls": 1, + "first_madar_call_tool": "context_pack", + "broad_fallback_operations_after_first_call": 0, + "trace_status": "trace_available" + }, + "costs": { + "indexing": { "measured": true, "wall_ms": 380, "usd": 0, "source": "locally_timed" }, + "context_build": { "measured": true, "wall_ms": 96, "usd": 0, "source": "locally_timed" }, + "agent": { "measured": false, "source": "not_applicable" } + }, + "scores": { + "correctness": { "measured": false, "value": null, "method": "blinded_rubric", "not_measured_reason": "Tier 1 does not score answer quality; no agent answer exists." }, + "critical_fact_completeness": { "measured": false, "value": null, "method": "blinded_rubric", "not_measured_reason": "Tier 1 does not score answer quality; no agent answer exists." }, + "unsupported_claims": { "measured": false, "value": null, "method": "blinded_rubric", "not_measured_reason": "Tier 1 does not score answer quality; no agent answer exists." }, + "correct_uncertainty": { "measured": false, "value": null, "method": "blinded_rubric", "not_measured_reason": "Tier 1 does not score answer quality; no agent answer exists." }, + "evidence_support": { + "measured": true, + "value": 1, + "method": "evidence_obligation_recall", + "scored_by": "deterministic", + "blinded": null, + "truth_version": "1.0.0" + }, + "tier1_obligation_recall": { + "measured": true, + "value": 1, + "method": "evidence_obligation_recall", + "scored_by": "deterministic", + "truth_version": "1.0.0" + } + }, + "validity": { + "status": "valid", + "invalidation_reasons": [], + "aggregatable": true + }, + "retention": { + "raw_transcript": { "retained": false, "path": null, "sha256": null }, + "answer_text": { "retained": false, "path": null, "sha256": null }, + "context_artifact": { "retained": true, "path": "raw/context-pack.json", "sha256": "76cae19d9787ab64fb7c0d4be597efe26c4453c855d0d2dea18e6eaa8b83ac7e" }, + "retention_policy": "Tier 1 retains the context artifact only; there is no agent transcript because no agent runs." + }, + "notes": [ + "Illustrative example only. Tier 1 measures whether the evidence needed to answer was present, never whether an answer was good.", + "The agent cost account is deliberately not_measured rather than zero." + ], + "started_at": "2026-08-12T09:10:00.000Z", + "completed_at": "2026-08-12T09:10:00.476Z" +} diff --git a/docs/qualification/examples/receipt-tier2-invalid-no-madar-call.json b/docs/qualification/examples/receipt-tier2-invalid-no-madar-call.json new file mode 100644 index 00000000..1e6a3060 --- /dev/null +++ b/docs/qualification/examples/receipt-tier2-invalid-no-madar-call.json @@ -0,0 +1,91 @@ +{ + "contract_version": "1.0.0", + "run_id": "example-tier2-invalid-0001", + "tier": 2, + "task_id": "flow-ledger-post-entry", + "target_id": "qual-ledger-service", + "arm": "madar", + "trial": 1, + "cache_mode": "warm", + "identity": { + "target_revision": "sha256:0000000000000000000000000000000000000000000000000000000000000000", + "dependency_lock_sha256": "6328bf95a901590814ff70ed570e0fc474c05ae162d6ac690cf3c812380828ab", + "madar_commit": "06b373a447acfce895412ac10eb4e5228c5df0b7", + "madar_package_version": "0.32.1", + "madar_package_tarball_sha256": "e42a5bc1434e84291ba23ddc9da90a2f9c591e6e2bd163adedf647d0c1f711ba", + "madar_runtime_source": "npm_pack", + "madar_config_sha256": "c6dcdf9130e3ef5503caead4fe07f1b465b429adfa6e43309166d8f650a33caf", + "agent": { + "host": "claude-code", + "host_version": "0.0.0-example", + "model_id": "example-model-id" + }, + "prompts": { + "system_prompt_sha256": "21da86de37b6595841807f54b44945498ad599cf682a40391a36d833dca35132", + "user_prompt_sha256": "4f727f2d70f0df36aa08fc208a27d6dcf1b1ed0a8b26a05b1e1dde32b57e29aa", + "user_prompt_text": "Trace what happens when a client posts a new ledger entry. Follow the path from the HTTP handler through to every durable side effect and every downstream consumer, and say which step each side effect happens in." + }, + "tool_permissions": ["Read", "Grep", "Glob", "mcp__madar__retrieve"], + "cache_mode": "warm" + }, + "environment": { + "isolation": true, + "host_os": "darwin", + "node_version": "v22.0.0", + "claude_code_version": "0.0.0-example", + "mcp_servers_active": ["madar"], + "skills_loaded": [], + "plugins_active": [], + "user_claude_md_hash": null, + "project_claude_md_hash": null, + "hooks_active": { + "user_prompt_submit": [], + "pre_tool_use": [], + "post_tool_use": [] + }, + "drift": { "detected": false, "fields": [] } + }, + "adoption": { + "status": "absent", + "attributable_madar_calls": 0, + "first_madar_call_tool": null, + "broad_fallback_operations_after_first_call": 0, + "trace_status": "trace_available" + }, + "costs": { + "indexing": { "measured": true, "wall_ms": 4120, "usd": 0, "source": "locally_timed" }, + "context_build": { "measured": false, "source": "not_applicable" }, + "agent": { + "measured": true, + "input_tokens": 41233, + "output_tokens": 1902, + "wall_ms": 61204, + "usd": 0.19, + "source": "provider_reported" + } + }, + "scores": { + "correctness": { "measured": false, "value": null, "method": "blinded_rubric", "not_measured_reason": "run is invalid: no attributable Madar call" }, + "critical_fact_completeness": { "measured": false, "value": null, "method": "blinded_rubric", "not_measured_reason": "run is invalid: no attributable Madar call" }, + "unsupported_claims": { "measured": false, "value": null, "method": "blinded_rubric", "not_measured_reason": "run is invalid: no attributable Madar call" }, + "correct_uncertainty": { "measured": false, "value": null, "method": "blinded_rubric", "not_measured_reason": "run is invalid: no attributable Madar call" }, + "evidence_support": { "measured": false, "value": null, "method": "blinded_rubric", "not_measured_reason": "run is invalid: no attributable Madar call" } + }, + "validity": { + "status": "invalid", + "invalidation_reasons": ["missing_attributable_madar_call"], + "aggregatable": false + }, + "retention": { + "raw_transcript": { "retained": true, "path": "raw/transcript.jsonl", "sha256": "8afad96d88f459209acacc66728718b270aa3e2df59fb8e92d5fc85475b3272c" }, + "answer_text": { "retained": true, "path": "raw/answer.txt", "sha256": "95967dd7cc113d0df15f5dd7b3d2185f7755a3da9b4db4b71805cd646434d7b6" }, + "context_artifact": { "retained": false, "path": null, "sha256": null }, + "retention_policy": "Raw transcripts, answers, and context artifacts are retained for at least 24 months alongside the receipt; see validity-rules.md." + }, + "notes": [ + "Illustrative example only. The agent never called Madar, so every quality dimension stays not_measured and the recorded costs may not be cited as a cost comparison.", + "The costs block is still populated because adoption failure must be diagnosable, but aggregatable is false so nothing here can enter a headline." + ], + "started_at": "2026-08-12T09:00:00.000Z", + "completed_at": "2026-08-12T09:01:05.324Z" +} diff --git a/docs/qualification/fixtures/ledger-service/README.md b/docs/qualification/fixtures/ledger-service/README.md new file mode 100644 index 00000000..c51f0183 --- /dev/null +++ b/docs/qualification/fixtures/ledger-service/README.md @@ -0,0 +1,39 @@ +# `ledger-service` qualification fixture + +A small TypeScript/Node accounting-ledger service used as a qualification target. + +This workspace exists **only** as an evaluation target. It is not shipped in the npm +package, is not imported by `src/`, and must never be referenced by production +retrieval or context logic. + +## Shape + +```text +POST /accounts/:accountId/entries -> postLedgerEntry +POST /entries/:entryId/reversals -> reverseLedgerEntry + +http/ledger-routes.ts + -> auth/request-context.ts (principal resolution) + -> service/ledger-service.ts (command service) + -> store/idempotency-store.ts (retry suppression) + -> store/ledger-store.ts (append-only entries) + -> outbox/outbox-publisher.ts (ledger.entry.posted) + -> audit/audit-log.ts (audit trail) +outbox/outbox-publisher.ts + -> projections/balance-projection.ts (ledger.entry.posted consumer) +``` + +## Deliberate defects + +Two defects are seeded on purpose and are part of the frozen truth. Do not fix them. + +| Id | Site | Nature | +| --- | --- | --- | +| `seeded-idempotency-ordering` | `src/service/ledger-service.ts` | The idempotency key is reserved **after** the ledger append, so a concurrent retry appends a duplicate entry. | +| `seeded-reversal-authorization` | `src/http/ledger-routes.ts` | The reversal route never checks the entry's account against `principal.accountIds`, so any authenticated principal can reverse another tenant's entry. | + +## Authoring provenance + +Authored for issue #655 on 2026-08-12 from a blank file. No Madar output, retrieval +result, context pack, or `implementationGuidance` was consulted while writing this +workspace or the truth files derived from it. diff --git a/docs/qualification/fixtures/ledger-service/package.json b/docs/qualification/fixtures/ledger-service/package.json new file mode 100644 index 00000000..50783d8b --- /dev/null +++ b/docs/qualification/fixtures/ledger-service/package.json @@ -0,0 +1,10 @@ +{ + "name": "qualification-fixture-ledger-service", + "version": "0.0.0", + "private": true, + "type": "module", + "description": "Qualification fixture only. Not published, not imported by Madar sources.", + "engines": { + "node": ">=20" + } +} diff --git a/docs/qualification/fixtures/ledger-service/src/audit/audit-log.ts b/docs/qualification/fixtures/ledger-service/src/audit/audit-log.ts new file mode 100644 index 00000000..7e2ebcee --- /dev/null +++ b/docs/qualification/fixtures/ledger-service/src/audit/audit-log.ts @@ -0,0 +1,22 @@ +export interface AuditRecord { + action: string + requestId: string + principalId: string + accountId: string + entryId: string + recordedAt: string +} + +export class AuditLog { + private readonly records: AuditRecord[] = [] + + record(record: Omit): AuditRecord { + const stored: AuditRecord = { ...record, recordedAt: new Date().toISOString() } + this.records.push(stored) + return stored + } + + listForAccount(accountId: string): AuditRecord[] { + return this.records.filter((record) => record.accountId === accountId) + } +} diff --git a/docs/qualification/fixtures/ledger-service/src/auth/request-context.ts b/docs/qualification/fixtures/ledger-service/src/auth/request-context.ts new file mode 100644 index 00000000..7cbd903b --- /dev/null +++ b/docs/qualification/fixtures/ledger-service/src/auth/request-context.ts @@ -0,0 +1,52 @@ +export interface Principal { + principalId: string + accountIds: string[] + scopes: string[] +} + +export interface RequestContext { + requestId: string + principal: Principal +} + +export class UnauthenticatedError extends Error { + constructor() { + super('missing or invalid bearer token') + this.name = 'UnauthenticatedError' + } +} + +export class ForbiddenError extends Error { + constructor(reason: string) { + super(reason) + this.name = 'ForbiddenError' + } +} + +export interface TokenDirectory { + lookup(token: string): Principal | undefined +} + +export function resolveRequestContext( + headers: Record, + directory: TokenDirectory, +): RequestContext { + const authorization = headers.authorization ?? '' + const token = authorization.startsWith('Bearer ') ? authorization.slice('Bearer '.length) : '' + + const principal = token ? directory.lookup(token) : undefined + if (!principal) { + throw new UnauthenticatedError() + } + + return { + requestId: headers['x-request-id'] ?? `req_${Math.random().toString(36).slice(2, 10)}`, + principal, + } +} + +export function assertAccountAccess(principal: Principal, accountId: string): void { + if (!principal.accountIds.includes(accountId)) { + throw new ForbiddenError(`principal ${principal.principalId} may not act on account ${accountId}`) + } +} diff --git a/docs/qualification/fixtures/ledger-service/src/http/ledger-routes.ts b/docs/qualification/fixtures/ledger-service/src/http/ledger-routes.ts new file mode 100644 index 00000000..3dc72fe3 --- /dev/null +++ b/docs/qualification/fixtures/ledger-service/src/http/ledger-routes.ts @@ -0,0 +1,79 @@ +import { assertAccountAccess, resolveRequestContext, type TokenDirectory } from '../auth/request-context.js' +import { UnknownEntryError, type LedgerService } from '../service/ledger-service.js' + +export interface HttpRequest { + method: string + path: string + headers: Record + params: Record + body: Record +} + +export interface HttpResponse { + status: number + body: Record +} + +export interface LedgerRouterDependencies { + ledgerService: LedgerService + tokenDirectory: TokenDirectory +} + +function requireString(body: Record, field: string): string { + const value = body[field] + if (typeof value !== 'string' || value.length === 0) { + throw new TypeError(`field ${field} is required`) + } + return value +} + +function requireNumber(body: Record, field: string): number { + const value = body[field] + if (typeof value !== 'number' || !Number.isFinite(value)) { + throw new TypeError(`field ${field} is required`) + } + return value +} + +export function createLedgerRouter(deps: LedgerRouterDependencies) { + return { + /** POST /accounts/:accountId/entries */ + postLedgerEntry(request: HttpRequest): HttpResponse { + const context = resolveRequestContext(request.headers, deps.tokenDirectory) + const accountId = request.params.accountId ?? '' + + assertAccountAccess(context.principal, accountId) + + const entry = deps.ledgerService.postEntry(context, { + accountId, + amountMinor: requireNumber(request.body, 'amountMinor'), + currency: requireString(request.body, 'currency'), + idempotencyKey: requireString(request.body, 'idempotencyKey'), + }) + + return { status: 201, body: { entry } } + }, + + /** POST /entries/:entryId/reversals */ + reverseLedgerEntry(request: HttpRequest): HttpResponse { + const context = resolveRequestContext(request.headers, deps.tokenDirectory) + const entryId = request.params.entryId ?? '' + + // seeded-reversal-authorization: unlike postLedgerEntry, this handler + // never calls assertAccountAccess for the account that owns entryId, so + // any authenticated principal can reverse another tenant's entry. + try { + const reversal = deps.ledgerService.reverseEntry(context, { + entryId, + idempotencyKey: requireString(request.body, 'idempotencyKey'), + }) + return { status: 201, body: { entry: reversal } } + } catch (error) { + if (error instanceof UnknownEntryError) { + return { status: 404, body: { error: error.message } } + } + throw error + } + }, + } +} diff --git a/docs/qualification/fixtures/ledger-service/src/outbox/outbox-publisher.ts b/docs/qualification/fixtures/ledger-service/src/outbox/outbox-publisher.ts new file mode 100644 index 00000000..cec28c94 --- /dev/null +++ b/docs/qualification/fixtures/ledger-service/src/outbox/outbox-publisher.ts @@ -0,0 +1,34 @@ +export type LedgerEventName = 'ledger.entry.posted' | 'ledger.entry.reversed' + +export interface LedgerEvent { + name: LedgerEventName + entryId: string + accountId: string + amountMinor: number + currency: string + publishedAt: string +} + +export type LedgerEventHandler = (event: LedgerEvent) => void + +/** + * The only path by which ledger state leaves the write model. Every downstream + * read model is rebuilt from these events, so dropping a publish silently + * desynchronizes consumers rather than raising an error. + */ +export class OutboxPublisher { + private readonly handlers = new Map() + + subscribe(name: LedgerEventName, handler: LedgerEventHandler): void { + const existing = this.handlers.get(name) ?? [] + this.handlers.set(name, [...existing, handler]) + } + + publish(event: Omit): LedgerEvent { + const published: LedgerEvent = { ...event, publishedAt: new Date().toISOString() } + for (const handler of this.handlers.get(published.name) ?? []) { + handler(published) + } + return published + } +} diff --git a/docs/qualification/fixtures/ledger-service/src/projections/balance-projection.ts b/docs/qualification/fixtures/ledger-service/src/projections/balance-projection.ts new file mode 100644 index 00000000..eef58ae7 --- /dev/null +++ b/docs/qualification/fixtures/ledger-service/src/projections/balance-projection.ts @@ -0,0 +1,45 @@ +import type { LedgerEvent, OutboxPublisher } from '../outbox/outbox-publisher.js' + +export interface AccountBalance { + accountId: string + currency: string + balanceMinor: number + lastEntryId: string | null +} + +/** + * Derived read model. It has no access to `LedgerStore`; its only input is the + * `ledger.entry.posted` / `ledger.entry.reversed` stream from `OutboxPublisher`. + */ +export class BalanceProjection { + private readonly balances = new Map() + + attach(publisher: OutboxPublisher): void { + publisher.subscribe('ledger.entry.posted', (event) => this.applyPosted(event)) + publisher.subscribe('ledger.entry.reversed', (event) => this.applyReversed(event)) + } + + balanceFor(accountId: string): AccountBalance | undefined { + return this.balances.get(accountId) + } + + private applyPosted(event: LedgerEvent): void { + const current = this.balances.get(event.accountId) + this.balances.set(event.accountId, { + accountId: event.accountId, + currency: event.currency, + balanceMinor: (current?.balanceMinor ?? 0) + event.amountMinor, + lastEntryId: event.entryId, + }) + } + + private applyReversed(event: LedgerEvent): void { + const current = this.balances.get(event.accountId) + this.balances.set(event.accountId, { + accountId: event.accountId, + currency: event.currency, + balanceMinor: (current?.balanceMinor ?? 0) - event.amountMinor, + lastEntryId: event.entryId, + }) + } +} diff --git a/docs/qualification/fixtures/ledger-service/src/service/ledger-service.ts b/docs/qualification/fixtures/ledger-service/src/service/ledger-service.ts new file mode 100644 index 00000000..50a2dbc6 --- /dev/null +++ b/docs/qualification/fixtures/ledger-service/src/service/ledger-service.ts @@ -0,0 +1,120 @@ +import type { AuditLog } from '../audit/audit-log.js' +import type { RequestContext } from '../auth/request-context.js' +import type { OutboxPublisher } from '../outbox/outbox-publisher.js' +import type { IdempotencyStore } from '../store/idempotency-store.js' +import type { LedgerEntry, LedgerStore } from '../store/ledger-store.js' + +export interface PostEntryCommand { + accountId: string + amountMinor: number + currency: string + idempotencyKey: string +} + +export interface ReverseEntryCommand { + entryId: string + idempotencyKey: string +} + +export class UnknownEntryError extends Error { + constructor(entryId: string) { + super(`ledger entry ${entryId} does not exist`) + this.name = 'UnknownEntryError' + } +} + +export interface LedgerServiceDependencies { + ledgerStore: LedgerStore + idempotencyStore: IdempotencyStore + outboxPublisher: OutboxPublisher + auditLog: AuditLog +} + +export class LedgerService { + constructor(private readonly deps: LedgerServiceDependencies) {} + + postEntry(context: RequestContext, command: PostEntryCommand): LedgerEntry { + const replay = this.deps.idempotencyStore.find(command.idempotencyKey) + if (replay) { + const existing = this.deps.ledgerStore.findEntry(replay.entryId) + if (existing) { + return existing + } + } + + const entry = this.deps.ledgerStore.appendEntry({ + accountId: command.accountId, + amountMinor: command.amountMinor, + currency: command.currency, + }) + + // seeded-idempotency-ordering: the reservation is written only after the + // append succeeds, so a retry that arrives between the read above and this + // line appends a second entry for the same idempotency key. + this.deps.idempotencyStore.reserve(command.idempotencyKey, entry.entryId) + + this.deps.outboxPublisher.publish({ + name: 'ledger.entry.posted', + entryId: entry.entryId, + accountId: entry.accountId, + amountMinor: entry.amountMinor, + currency: entry.currency, + }) + + this.deps.auditLog.record({ + action: 'ledger.entry.posted', + requestId: context.requestId, + principalId: context.principal.principalId, + accountId: entry.accountId, + entryId: entry.entryId, + }) + + return entry + } + + reverseEntry(context: RequestContext, command: ReverseEntryCommand): LedgerEntry { + const original = this.deps.ledgerStore.findEntry(command.entryId) + if (!original) { + throw new UnknownEntryError(command.entryId) + } + + const replay = this.deps.idempotencyStore.find(command.idempotencyKey) + if (replay) { + const existing = this.deps.ledgerStore.findEntry(replay.entryId) + if (existing) { + return existing + } + } + + const reversal = this.deps.ledgerStore.appendEntry({ + accountId: original.accountId, + amountMinor: -original.amountMinor, + currency: original.currency, + reversalOfEntryId: original.entryId, + }) + + this.deps.idempotencyStore.reserve(command.idempotencyKey, reversal.entryId) + + this.deps.outboxPublisher.publish({ + name: 'ledger.entry.reversed', + entryId: reversal.entryId, + accountId: reversal.accountId, + amountMinor: original.amountMinor, + currency: reversal.currency, + }) + + this.deps.auditLog.record({ + action: 'ledger.entry.reversed', + requestId: context.requestId, + principalId: context.principal.principalId, + accountId: reversal.accountId, + entryId: reversal.entryId, + }) + + return reversal + } + + findEntryForAuthorization(entryId: string): LedgerEntry | undefined { + return this.deps.ledgerStore.findEntry(entryId) + } +} diff --git a/docs/qualification/fixtures/ledger-service/src/store/idempotency-store.ts b/docs/qualification/fixtures/ledger-service/src/store/idempotency-store.ts new file mode 100644 index 00000000..c74da9bf --- /dev/null +++ b/docs/qualification/fixtures/ledger-service/src/store/idempotency-store.ts @@ -0,0 +1,34 @@ +export interface IdempotencyRecord { + key: string + entryId: string + reservedAt: string +} + +/** + * Suppresses duplicate command execution for retried requests. + * + * `reserve` is intended to be called *before* any state mutation so a concurrent + * retry loses the race and replays the stored result instead of mutating again. + */ +export class IdempotencyStore { + private readonly records = new Map() + + find(key: string): IdempotencyRecord | undefined { + return this.records.get(key) + } + + reserve(key: string, entryId: string): IdempotencyRecord { + const existing = this.records.get(key) + if (existing) { + return existing + } + + const record: IdempotencyRecord = { + key, + entryId, + reservedAt: new Date().toISOString(), + } + this.records.set(key, record) + return record + } +} diff --git a/docs/qualification/fixtures/ledger-service/src/store/ledger-store.ts b/docs/qualification/fixtures/ledger-service/src/store/ledger-store.ts new file mode 100644 index 00000000..3e49ab58 --- /dev/null +++ b/docs/qualification/fixtures/ledger-service/src/store/ledger-store.ts @@ -0,0 +1,46 @@ +export interface LedgerEntry { + entryId: string + accountId: string + amountMinor: number + currency: string + reversalOfEntryId: string | null + createdAt: string +} + +export interface AppendLedgerEntryInput { + accountId: string + amountMinor: number + currency: string + reversalOfEntryId?: string | null +} + +/** + * Append-only entry log. Balances are never stored here; they are derived by + * `projections/balance-projection.ts` from published outbox events. + */ +export class LedgerStore { + private readonly entries = new Map() + private sequence = 0 + + appendEntry(input: AppendLedgerEntryInput): LedgerEntry { + this.sequence += 1 + const entry: LedgerEntry = { + entryId: `led_${this.sequence.toString().padStart(8, '0')}`, + accountId: input.accountId, + amountMinor: input.amountMinor, + currency: input.currency, + reversalOfEntryId: input.reversalOfEntryId ?? null, + createdAt: new Date().toISOString(), + } + this.entries.set(entry.entryId, entry) + return entry + } + + findEntry(entryId: string): LedgerEntry | undefined { + return this.entries.get(entryId) + } + + listEntriesForAccount(accountId: string): LedgerEntry[] { + return [...this.entries.values()].filter((entry) => entry.accountId === accountId) + } +} diff --git a/docs/qualification/fixtures/ledger-service/tsconfig.json b/docs/qualification/fixtures/ledger-service/tsconfig.json new file mode 100644 index 00000000..a8c34666 --- /dev/null +++ b/docs/qualification/fixtures/ledger-service/tsconfig.json @@ -0,0 +1,11 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "lib": ["ES2022"], + "strict": true, + "noEmit": true + }, + "include": ["src/**/*.ts"] +} diff --git a/docs/qualification/fixtures/plugin-host/README.md b/docs/qualification/fixtures/plugin-host/README.md new file mode 100644 index 00000000..c51e323c --- /dev/null +++ b/docs/qualification/fixtures/plugin-host/README.md @@ -0,0 +1,35 @@ +# `plugin-host` qualification fixture + +A small TypeScript/Node extension host used as a qualification target for +architecture-understanding and bounded implementation-planning tasks. + +This workspace exists **only** as an evaluation target. It is not shipped in the npm +package, is not imported by `src/`, and must never be referenced by production +retrieval or context logic. + +## Shape + +```text +contracts/plugin.ts the only stable extension surface (ExportPlugin) +host/config.ts layered configuration resolution +host/registry.ts name -> plugin resolution, duplicate rejection +host/lifecycle.ts init -> run -> dispose ordering and failure isolation +host/plugin-host.ts composition root, the only place that knows both sides +plugins/csv-export-plugin.ts built-in plugin, no external I/O +plugins/webhook-export-plugin.ts built-in plugin, performs external delivery +``` + +The intended boundary is that `plugins/*` depends on `contracts/plugin.ts` only, and +never on `host/*`. `host/plugin-host.ts` is the single composition root. + +## Deliberate defect + +| Id | Site | Nature | +| --- | --- | --- | +| `seeded-boundary-violation` | `src/plugins/webhook-export-plugin.ts` | Imports `resolveHostConfig` from `host/config.ts`, breaking the stated plugin -> contracts-only boundary and coupling a plugin to host internals. | + +## Authoring provenance + +Authored for issue #655 on 2026-08-12 from a blank file. No Madar output, retrieval +result, context pack, or `implementationGuidance` was consulted while writing this +workspace or the truth files derived from it. diff --git a/docs/qualification/fixtures/plugin-host/package.json b/docs/qualification/fixtures/plugin-host/package.json new file mode 100644 index 00000000..47ba37b0 --- /dev/null +++ b/docs/qualification/fixtures/plugin-host/package.json @@ -0,0 +1,10 @@ +{ + "name": "qualification-fixture-plugin-host", + "version": "0.0.0", + "private": true, + "type": "module", + "description": "Qualification fixture only. Not published, not imported by Madar sources.", + "engines": { + "node": ">=20" + } +} diff --git a/docs/qualification/fixtures/plugin-host/src/contracts/plugin.ts b/docs/qualification/fixtures/plugin-host/src/contracts/plugin.ts new file mode 100644 index 00000000..c0ffee4f --- /dev/null +++ b/docs/qualification/fixtures/plugin-host/src/contracts/plugin.ts @@ -0,0 +1,45 @@ +export interface ExportRecord { + id: string + fields: Record +} + +export interface ExportBatch { + batchId: string + records: ExportRecord[] +} + +export interface ExportResult { + pluginName: string + batchId: string + recordsExported: number + destination: string +} + +export interface PluginContext { + /** Plugin-scoped settings resolved by the host; plugins never read config themselves. */ + settings: Readonly> + log(message: string): void +} + +/** + * The only stable extension surface. Anything under `plugins/` must depend on + * this module and nothing else from this workspace. + */ +export interface ExportPlugin { + readonly name: string + readonly version: string + init(context: PluginContext): void + export(batch: ExportBatch): ExportResult + dispose?(): void +} + +export class PluginFailure extends Error { + constructor( + readonly pluginName: string, + readonly phase: 'init' | 'export' | 'dispose', + cause: unknown, + ) { + super(`plugin ${pluginName} failed during ${phase}: ${String(cause)}`) + this.name = 'PluginFailure' + } +} diff --git a/docs/qualification/fixtures/plugin-host/src/host/config.ts b/docs/qualification/fixtures/plugin-host/src/host/config.ts new file mode 100644 index 00000000..ce40bedf --- /dev/null +++ b/docs/qualification/fixtures/plugin-host/src/host/config.ts @@ -0,0 +1,32 @@ +export interface HostConfig { + enabledPlugins: string[] + pluginSettings: Record> + failFast: boolean +} + +const DEFAULT_CONFIG: HostConfig = { + enabledPlugins: ['csv-export'], + pluginSettings: {}, + failFast: false, +} + +/** + * Layered resolution: defaults, then file config, then environment overrides. + * Only `host/plugin-host.ts` is expected to call this. + */ +export function resolveHostConfig( + fileConfig: Partial, + env: Record, +): HostConfig { + const enabledFromEnv = env.EXPORT_PLUGINS?.split(',').map((name) => name.trim()).filter(Boolean) + + return { + enabledPlugins: enabledFromEnv ?? fileConfig.enabledPlugins ?? DEFAULT_CONFIG.enabledPlugins, + pluginSettings: { ...DEFAULT_CONFIG.pluginSettings, ...fileConfig.pluginSettings }, + failFast: env.EXPORT_FAIL_FAST === '1' ? true : (fileConfig.failFast ?? DEFAULT_CONFIG.failFast), + } +} + +export function settingsFor(config: HostConfig, pluginName: string): Record { + return config.pluginSettings[pluginName] ?? {} +} diff --git a/docs/qualification/fixtures/plugin-host/src/host/lifecycle.ts b/docs/qualification/fixtures/plugin-host/src/host/lifecycle.ts new file mode 100644 index 00000000..04c19c5b --- /dev/null +++ b/docs/qualification/fixtures/plugin-host/src/host/lifecycle.ts @@ -0,0 +1,62 @@ +import { PluginFailure, type ExportBatch, type ExportPlugin, type ExportResult, type PluginContext } from '../contracts/plugin.js' + +export interface LifecycleOptions { + failFast: boolean + contextFor(plugin: ExportPlugin): PluginContext +} + +export interface LifecycleOutcome { + results: ExportResult[] + failures: PluginFailure[] +} + +/** + * Owns init -> export -> dispose ordering and failure isolation. + * + * With `failFast: false` a failing plugin is recorded and skipped; the remaining + * plugins still run and every initialized plugin is still disposed. + */ +export function runExportLifecycle( + plugins: ExportPlugin[], + batch: ExportBatch, + options: LifecycleOptions, +): LifecycleOutcome { + const results: ExportResult[] = [] + const failures: PluginFailure[] = [] + const initialized: ExportPlugin[] = [] + + for (const plugin of plugins) { + try { + plugin.init(options.contextFor(plugin)) + initialized.push(plugin) + } catch (cause) { + const failure = new PluginFailure(plugin.name, 'init', cause) + if (options.failFast) { + throw failure + } + failures.push(failure) + } + } + + for (const plugin of initialized) { + try { + results.push(plugin.export(batch)) + } catch (cause) { + const failure = new PluginFailure(plugin.name, 'export', cause) + if (options.failFast) { + throw failure + } + failures.push(failure) + } + } + + for (const plugin of initialized) { + try { + plugin.dispose?.() + } catch (cause) { + failures.push(new PluginFailure(plugin.name, 'dispose', cause)) + } + } + + return { results, failures } +} diff --git a/docs/qualification/fixtures/plugin-host/src/host/plugin-host.ts b/docs/qualification/fixtures/plugin-host/src/host/plugin-host.ts new file mode 100644 index 00000000..489e5b47 --- /dev/null +++ b/docs/qualification/fixtures/plugin-host/src/host/plugin-host.ts @@ -0,0 +1,56 @@ +import type { ExportBatch, ExportPlugin, PluginContext } from '../contracts/plugin.js' +import { CsvExportPlugin } from '../plugins/csv-export-plugin.js' +import { WebhookExportPlugin } from '../plugins/webhook-export-plugin.js' +import { resolveHostConfig, settingsFor, type HostConfig } from './config.js' +import { runExportLifecycle, type LifecycleOutcome } from './lifecycle.js' +import { PluginRegistry } from './registry.js' + +export interface PluginHostOptions { + fileConfig?: Partial + env?: Record + log?: (message: string) => void +} + +/** + * Composition root. This is the only module that knows about both `plugins/*` + * and `host/*`; adding a built-in plugin should require a change here and + * nowhere else in `host/`. + */ +export class PluginHost { + private readonly config: HostConfig + private readonly registry = new PluginRegistry() + private readonly log: (message: string) => void + + constructor(options: PluginHostOptions = {}) { + this.config = resolveHostConfig(options.fileConfig ?? {}, options.env ?? {}) + this.log = options.log ?? (() => {}) + + for (const plugin of builtInPlugins()) { + this.registry.register(plugin) + } + } + + registerPlugin(plugin: ExportPlugin): void { + this.registry.register(plugin) + } + + runExport(batch: ExportBatch): LifecycleOutcome { + const plugins = this.registry.resolveAll(this.config.enabledPlugins) + + return runExportLifecycle(plugins, batch, { + failFast: this.config.failFast, + contextFor: (plugin) => this.contextFor(plugin), + }) + } + + private contextFor(plugin: ExportPlugin): PluginContext { + return { + settings: Object.freeze({ ...settingsFor(this.config, plugin.name) }), + log: (message: string) => this.log(`[${plugin.name}] ${message}`), + } + } +} + +function builtInPlugins(): ExportPlugin[] { + return [new CsvExportPlugin(), new WebhookExportPlugin()] +} diff --git a/docs/qualification/fixtures/plugin-host/src/host/registry.ts b/docs/qualification/fixtures/plugin-host/src/host/registry.ts new file mode 100644 index 00000000..823a910a --- /dev/null +++ b/docs/qualification/fixtures/plugin-host/src/host/registry.ts @@ -0,0 +1,46 @@ +import type { ExportPlugin } from '../contracts/plugin.js' + +export class DuplicatePluginError extends Error { + constructor(name: string) { + super(`plugin ${name} is already registered`) + this.name = 'DuplicatePluginError' + } +} + +export class UnknownPluginError extends Error { + constructor(name: string) { + super(`plugin ${name} is not registered`) + this.name = 'UnknownPluginError' + } +} + +/** + * Name -> plugin resolution. Registration order is preserved so lifecycle + * ordering is deterministic. + */ +export class PluginRegistry { + private readonly plugins = new Map() + + register(plugin: ExportPlugin): void { + if (this.plugins.has(plugin.name)) { + throw new DuplicatePluginError(plugin.name) + } + this.plugins.set(plugin.name, plugin) + } + + resolve(name: string): ExportPlugin { + const plugin = this.plugins.get(name) + if (!plugin) { + throw new UnknownPluginError(name) + } + return plugin + } + + resolveAll(names: string[]): ExportPlugin[] { + return names.map((name) => this.resolve(name)) + } + + registeredNames(): string[] { + return [...this.plugins.keys()] + } +} diff --git a/docs/qualification/fixtures/plugin-host/src/plugins/csv-export-plugin.ts b/docs/qualification/fixtures/plugin-host/src/plugins/csv-export-plugin.ts new file mode 100644 index 00000000..a3b1d113 --- /dev/null +++ b/docs/qualification/fixtures/plugin-host/src/plugins/csv-export-plugin.ts @@ -0,0 +1,32 @@ +import type { ExportBatch, ExportPlugin, ExportResult, PluginContext } from '../contracts/plugin.js' + +/** + * Reference implementation of the intended boundary: depends on + * `contracts/plugin.ts` only and reads every setting from `PluginContext`. + */ +export class CsvExportPlugin implements ExportPlugin { + readonly name = 'csv-export' + readonly version = '1.0.0' + + private delimiter = ',' + private destination = 'file://exports' + + init(context: PluginContext): void { + this.delimiter = context.settings.delimiter ?? this.delimiter + this.destination = context.settings.destination ?? this.destination + context.log(`csv-export writing to ${this.destination}`) + } + + export(batch: ExportBatch): ExportResult { + for (const record of batch.records) { + Object.values(record.fields).join(this.delimiter) + } + + return { + pluginName: this.name, + batchId: batch.batchId, + recordsExported: batch.records.length, + destination: this.destination, + } + } +} diff --git a/docs/qualification/fixtures/plugin-host/src/plugins/webhook-export-plugin.ts b/docs/qualification/fixtures/plugin-host/src/plugins/webhook-export-plugin.ts new file mode 100644 index 00000000..e869bddd --- /dev/null +++ b/docs/qualification/fixtures/plugin-host/src/plugins/webhook-export-plugin.ts @@ -0,0 +1,36 @@ +import type { ExportBatch, ExportPlugin, ExportResult, PluginContext } from '../contracts/plugin.js' +// seeded-boundary-violation: a plugin must depend on contracts/plugin.ts only. +// Reaching into host internals couples this plugin to host configuration layering. +import { resolveHostConfig } from '../host/config.js' + +export class WebhookExportPlugin implements ExportPlugin { + readonly name = 'webhook-export' + readonly version = '1.0.0' + + private endpoint = 'https://example.invalid/exports' + private delivered = 0 + + init(context: PluginContext): void { + const hostConfig = resolveHostConfig({}, process.env) + this.endpoint = context.settings.endpoint ?? this.endpoint + + if (hostConfig.failFast) { + context.log('webhook-export running under fail-fast host configuration') + } + } + + export(batch: ExportBatch): ExportResult { + this.delivered += batch.records.length + + return { + pluginName: this.name, + batchId: batch.batchId, + recordsExported: batch.records.length, + destination: this.endpoint, + } + } + + dispose(): void { + this.delivered = 0 + } +} diff --git a/docs/qualification/fixtures/plugin-host/tsconfig.json b/docs/qualification/fixtures/plugin-host/tsconfig.json new file mode 100644 index 00000000..be978f2e --- /dev/null +++ b/docs/qualification/fixtures/plugin-host/tsconfig.json @@ -0,0 +1,12 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "lib": ["ES2022"], + "strict": true, + "types": ["node"], + "noEmit": true + }, + "include": ["src/**/*.ts"] +} diff --git a/docs/qualification/freeze.json b/docs/qualification/freeze.json new file mode 100644 index 00000000..e1b95844 --- /dev/null +++ b/docs/qualification/freeze.json @@ -0,0 +1,48 @@ +{ + "contract_version": "1.0.0", + "frozen_at": "2026-08-12", + "algorithm": "sha256 over raw file bytes", + "note": "Regenerate deliberately with `npm run qualify:validate -- --write` and say why in the pull request. A silent digest change is a contract change.", + "files": { + "docs/qualification/README.md": "4c612f693d3cdfd64f4b911c39991a4bd4a12106a9439cca98feafb62998f9b7", + "docs/qualification/corpus.json": "845d96c516ba3497b837c0c6a949d077e8ebc15aa2c675acfafe269bb0631034", + "docs/qualification/evidence-categories.md": "24fa19cb4e049519df647989e7844c79d3d801bbfa6ed5fa8576327ffba4aa33", + "docs/qualification/examples/receipt-tier1-valid.json": "021dccdc1b79f9a699589ffc7cc978702083ea1260cebeabe6ee77446ef7f834", + "docs/qualification/examples/receipt-tier2-invalid-no-madar-call.json": "6fd064a8968ebd8447e8840d0352878e777ad4d8f678ff65957aa04e9000c9bb", + "docs/qualification/fixtures/ledger-service/README.md": "50d552fc69d6873e2b7c2960872546a78ea6a0a832bce76bdcc170aaa81fa03b", + "docs/qualification/fixtures/ledger-service/package.json": "eeb3350fdbe0349d35dcc66be8a0168b2547d2fd7f49f510f36729e0e9abfaf7", + "docs/qualification/fixtures/ledger-service/src/audit/audit-log.ts": "b6d0919a03c7284bcf6d28be055d1ac04b7e36d46acc7b6e344bc96007ac412f", + "docs/qualification/fixtures/ledger-service/src/auth/request-context.ts": "901c254d5b10471fee0d5658e7380867ca21386435df285e48496080cdec179c", + "docs/qualification/fixtures/ledger-service/src/http/ledger-routes.ts": "ca9b5b0c7db2ce326baa75584687d33dd2d478de0fca633e8501c9af6cbd866e", + "docs/qualification/fixtures/ledger-service/src/outbox/outbox-publisher.ts": "00712c8f193d806cffed473455f0ee6793e845ca05f61eef7e64307f125d8330", + "docs/qualification/fixtures/ledger-service/src/projections/balance-projection.ts": "b3725465dc2a9292805b820698004e3d02df204707e3b4067262e9cf83978b0d", + "docs/qualification/fixtures/ledger-service/src/service/ledger-service.ts": "3199d59abda0ee51503346a7570c780633d5177fe69744d1cb5704e87c88aee4", + "docs/qualification/fixtures/ledger-service/src/store/idempotency-store.ts": "22b0868f890fbce754075ad7ba6d2430cea1b3c3e71a0e2da6f97aaad0207d4e", + "docs/qualification/fixtures/ledger-service/src/store/ledger-store.ts": "7cd4b2763f84bae1de0e53553bcab8e7eb27a365807534fd1fbeab650c004cff", + "docs/qualification/fixtures/ledger-service/tsconfig.json": "bedbf5c76b15f8b2a8f464870bdd4d96057e02c1bc2d858036970c41219e85bf", + "docs/qualification/fixtures/plugin-host/README.md": "20ac3e34dcce8ba946df80ba210f7a194218c3934932eab54fae9df68d63ec8a", + "docs/qualification/fixtures/plugin-host/package.json": "d1eee6217153675f5a0fc243bc290cb1f28b6d8eba4e5d7612d764f77a8172a9", + "docs/qualification/fixtures/plugin-host/src/contracts/plugin.ts": "c97ae63baea6f62b23c1f50c55582e9d889d1489b7b9d02dedf85b45cd4d85b2", + "docs/qualification/fixtures/plugin-host/src/host/config.ts": "3c60cf835b99258e485d2c8ca2588f2b80065c7a502d7d8f5c5263557f996ad9", + "docs/qualification/fixtures/plugin-host/src/host/lifecycle.ts": "98ce4eba4b671b097efd73d7611cbccd6afebac9ecd31aa022ef8dcabe67c81d", + "docs/qualification/fixtures/plugin-host/src/host/plugin-host.ts": "f5ecea001feaf6f86482853cadb55dd8fe1216f6a6f0c489d6972b24b313cc49", + "docs/qualification/fixtures/plugin-host/src/host/registry.ts": "7a9a3096ce6c8f85bbd3800ee206eba28d413e0ea81bbe960181d13aa378501f", + "docs/qualification/fixtures/plugin-host/src/plugins/csv-export-plugin.ts": "cdf91d68941a6f91a14dd2d350c3aa5554e7073f23098c8ef5083aab949f9e9f", + "docs/qualification/fixtures/plugin-host/src/plugins/webhook-export-plugin.ts": "df0b22d2e036451ab2c2cc6c8d07ddfa0303a6ace59e9ca553e4749faba57f39", + "docs/qualification/fixtures/plugin-host/tsconfig.json": "e4702ed4a224f18edab218cff4ac62224b5c427c46d06c111878fad24554a6a0", + "docs/qualification/holdout-policy.md": "1c1c397b224601072e690ab0bb9d1be048a7b4185b7e4a4ace0ad72916778029", + "docs/qualification/receipt-schema.json": "976cb3e4510450889dd9c86f09de6248d9285c8b8a16353461b8e739980ee36f", + "docs/qualification/rubrics.json": "35f7648bc72e5f0af11649063b0da01bea37d6c0d8699132c078f0c793abb079", + "docs/qualification/stop-rule.md": "7d5b646e0d3369aa1783bc4a594b3ee322e0774515fec6ec43c83336722a4f83", + "docs/qualification/tasks.json": "8f3fdbbfc652f84936c597514ad4f60db4839101c82603b3e7a22415c000517c", + "docs/qualification/tier1.json": "2eca6565770a103e6e9e111be8ab9d15bca1b8a792fd9b97920291e2d9010445", + "docs/qualification/tier2-matrix.json": "681e2dc90c4dfdc2e376d3c5b7e35f46688232b1da98ffdb6befbba6e0bbb510", + "docs/qualification/truth/arch-plugin-host-extension-seam.json": "012098694eeb32a6aff35a24c043301e27f8d45c8d227dc722a2e7fb6e7eae8c", + "docs/qualification/truth/flow-ledger-post-entry.json": "abfd4d1561a0092855d6680839ae368458fd8cc3a1dc338752f33f0ff5552e47", + "docs/qualification/truth/impact-ledger-drop-outbox-publish.json": "9ec23f87d33f7e3cbc4b96e4746802aefe455f0a99bc87f1b5cc629c7677dbfd", + "docs/qualification/truth/plan-plugin-host-object-storage-plugin.json": "0c5ff4bf3d5c84287586b8c1c34a2542b8c157320cb9d4bede5c01d0b4f02673", + "docs/qualification/truth/review-ledger-http-authorization.json": "367a1ad6616aabd03115dc091e717da467b70b9929004b3a2442ee793f995fbf", + "docs/qualification/truth/rootcause-ledger-duplicate-entries.json": "7cf3099d8953ed36629732360cf81ee86190d71cd092e11521f4d80d5e7375dd", + "docs/qualification/validity-rules.md": "bbd51c561fdd39e4da71aeaf05a3a197949d531c18e575f6044231728b181936" + } +} diff --git a/docs/qualification/holdout-policy.md b/docs/qualification/holdout-policy.md new file mode 100644 index 00000000..f093cdc7 --- /dev/null +++ b/docs/qualification/holdout-policy.md @@ -0,0 +1,77 @@ +# Hidden holdout policy + +Contract version `1.0.0`, frozen 2026-08-12 for [#655](https://github.com/mohanagy/madar/issues/655). + +## Why holdouts exist here + +Everything in [`corpus.json`](./corpus.json) with `holdout_class: "open"` is visible to +whoever writes production retrieval and ranking rules. Open targets are still useful — +they catch regressions — but they cannot detect the failure mode this policy exists for: +production behaviour drifting toward the qualification corpus itself. Only a target the +rule author has never seen can measure that. + +## Classes + +| Class | Meaning | +| --- | --- | +| `open` | Target, prompts, and truth live in this repository. Anyone may read them. Useful for regression detection; **worthless** as evidence of generalization. | +| `sealed` | Target, prompts, and truth are authored and held by someone who does not write production retrieval, ranking, or claim logic. The rule author never reads them before the sweep. | + +## Rules for a sealed holdout + +1. The target, the prompts, and the truth are authored by a person who has not written and + will not write production retrieval, ranking, or claim logic during the evaluation window. +2. They live outside this repository. Nothing about them — no repository name, no path, no + symbol, no prompt wording — is committed here, discussed in an issue, or pasted into a + pull request. +3. The rule author receives the sweep result only: per-cell pass/fail and the scored + dimensions. Never the answers, never the prompts, never the truth. +4. A sealed target is used at most **once per release line**. After a result is reported + against it, it is burned: it becomes an open target or it is retired. Reusing a sealed + target after its result is known makes it open in everything but name. +5. If a sealed cell fails, the holder may release the failing task to the rule author for + diagnosis. That releases the target permanently. +6. The runner supports this today without new code: pass alternate manifests that live + outside the checkout. The existing + [`docs/benchmarks/suite/holdouts/README.md`](../benchmarks/suite/holdouts/README.md) + documents the equivalent mechanism for the product benchmark suite. + +## Current status: unsatisfied + +**Madar has one author.** There is no second person to author or hold a sealed target, and +no meaningful sense in which a target can be hidden from the person who writes both the +production rules and the corpus. The `qual-sealed-a` slot in `corpus.json` is therefore +marked `status: "unsatisfied"` rather than being filled with a self-authored target that +would look like a holdout and prove nothing. + +The same limitation makes two other artifacts unavailable: + +- the hidden acceptance test for `plan-plugin-host-object-storage-plugin` + (see `truth/plan-plugin-host-object-storage-plugin.json`); +- blinded Tier 2 review (see `rubrics.json#/blinding/current_status`). + +### Human action required + +To satisfy this policy, a person other than the production-rule author must: + +1. select and pin one TypeScript/Node repository not named anywhere in this repository; +2. author two to four task prompts and their independent truth for it, without reading + Madar output; +3. author the hidden acceptance test for the bounded-implementation task; +4. hold all of it outside this repository and run the sweep themselves, returning only + per-cell scores; +5. record their name and the seal date in the sweep receipt. + +Until that happens, **no generalization claim may be made from this corpus**, and any +report derived from it must carry this exact line: + +> sealed holdout unsatisfied; results measure regression only + +## What must never happen + +- A sealed target, prompt, path, or symbol must never appear in production retrieval, + ranking, claim, or configuration code. +- A sealed target must never be added to the repository's test fixtures. +- A failing sealed cell must never be resolved by editing the sealed truth. +- The rule author must never request the sealed prompts "just to check whether they are + fair". Fairness disputes are resolved by the holder retiring the task, not by disclosure. diff --git a/docs/qualification/receipt-schema.json b/docs/qualification/receipt-schema.json new file mode 100644 index 00000000..2ff08ebe --- /dev/null +++ b/docs/qualification/receipt-schema.json @@ -0,0 +1,317 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/mohanagy/madar/docs/qualification/receipt-schema.json", + "title": "Madar qualification run receipt", + "description": "Environment and run receipt for a single qualification cell. One receipt describes one arm of one trial of one task against one target.", + "type": "object", + "additionalProperties": false, + "required": [ + "contract_version", + "run_id", + "tier", + "task_id", + "target_id", + "arm", + "trial", + "identity", + "environment", + "adoption", + "costs", + "scores", + "validity", + "retention", + "started_at", + "completed_at" + ], + "properties": { + "contract_version": { "type": "string", "pattern": "^\\d+\\.\\d+\\.\\d+$" }, + "run_id": { "type": "string", "minLength": 1 }, + "tier": { "type": "integer", "enum": [1, 2] }, + "task_id": { "type": "string", "minLength": 1 }, + "target_id": { "type": "string", "minLength": 1 }, + "arm": { "type": "string", "enum": ["native", "madar"] }, + "trial": { "type": "integer", "minimum": 1 }, + "cache_mode": { "type": "string", "enum": ["cold", "warm"] }, + + "identity": { + "type": "object", + "additionalProperties": false, + "description": "The full experimental identity. Every field is required; a missing field invalidates the run.", + "required": [ + "target_revision", + "dependency_lock_sha256", + "madar_commit", + "madar_package_version", + "madar_runtime_source", + "madar_config_sha256", + "agent", + "prompts", + "tool_permissions", + "cache_mode" + ], + "properties": { + "target_revision": { + "type": "string", + "minLength": 1, + "description": "Git SHA for a git target, or the frozen content digest for a fixture target." + }, + "dependency_lock_sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" }, + "madar_commit": { "type": "string", "pattern": "^[0-9a-f]{40}$" }, + "madar_package_version": { "type": "string", "minLength": 1 }, + "madar_package_tarball_sha256": { "type": ["string", "null"], "pattern": "^[0-9a-f]{64}$" }, + "madar_runtime_source": { "type": "string", "enum": ["npm_pack", "checkout"] }, + "madar_config_sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" }, + "agent": { + "type": "object", + "additionalProperties": false, + "required": ["host", "host_version", "model_id"], + "properties": { + "host": { "type": "string", "minLength": 1 }, + "host_version": { "type": ["string", "null"] }, + "model_id": { "type": "string", "minLength": 1 } + } + }, + "prompts": { + "type": "object", + "additionalProperties": false, + "required": ["system_prompt_sha256", "user_prompt_sha256", "user_prompt_text"], + "properties": { + "system_prompt_sha256": { "type": ["string", "null"], "pattern": "^[0-9a-f]{64}$" }, + "user_prompt_sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" }, + "user_prompt_text": { "type": "string", "minLength": 1 } + } + }, + "tool_permissions": { + "type": "array", + "items": { "type": "string" }, + "description": "Exact allowed-tool list given to the agent. Both arms of a cell must record the same list apart from Madar tools." + }, + "cache_mode": { "type": "string", "enum": ["cold", "warm"] } + } + }, + + "environment": { + "type": "object", + "additionalProperties": false, + "required": [ + "isolation", + "host_os", + "node_version", + "mcp_servers_active", + "skills_loaded", + "plugins_active", + "hooks_active", + "drift" + ], + "properties": { + "isolation": { "type": "boolean" }, + "host_os": { "type": "string", "minLength": 1 }, + "node_version": { "type": "string", "minLength": 1 }, + "claude_code_version": { "type": ["string", "null"] }, + "mcp_servers_active": { "type": "array", "items": { "type": "string" } }, + "skills_loaded": { "type": "array", "items": { "type": "string" } }, + "plugins_active": { "type": "array", "items": { "type": "string" } }, + "user_claude_md_hash": { "type": ["string", "null"] }, + "project_claude_md_hash": { "type": ["string", "null"] }, + "hooks_active": { + "type": "object", + "additionalProperties": false, + "required": ["user_prompt_submit", "pre_tool_use", "post_tool_use"], + "properties": { + "user_prompt_submit": { "type": "array", "items": { "type": "string" } }, + "pre_tool_use": { "type": "array", "items": { "type": "string" } }, + "post_tool_use": { "type": "array", "items": { "type": "string" } } + } + }, + "drift": { + "type": "object", + "additionalProperties": false, + "required": ["detected", "fields"], + "properties": { + "detected": { "type": "boolean" }, + "fields": { "type": "array", "items": { "type": "string" } } + } + } + } + }, + + "adoption": { + "type": "object", + "additionalProperties": false, + "description": "Behaviour measurement. Never folded into a quality score.", + "required": ["status", "attributable_madar_calls", "broad_fallback_operations_after_first_call"], + "properties": { + "status": { "type": "string", "enum": ["adopted", "late", "absent", "not_applicable"] }, + "attributable_madar_calls": { "type": "integer", "minimum": 0 }, + "first_madar_call_tool": { "type": ["string", "null"] }, + "broad_fallback_operations_after_first_call": { "type": "integer", "minimum": 0 }, + "trace_status": { "type": "string", "enum": ["trace_available", "trace_partial", "trace_missing"] } + } + }, + + "costs": { + "type": "object", + "additionalProperties": false, + "description": "Indexing, context building, and agent execution are separate accounts and must never be summed into a single headline.", + "required": ["indexing", "context_build", "agent"], + "properties": { + "indexing": { "$ref": "#/definitions/costAccount" }, + "context_build": { "$ref": "#/definitions/costAccount" }, + "agent": { "$ref": "#/definitions/costAccount" } + } + }, + + "scores": { + "type": "object", + "additionalProperties": false, + "required": [ + "correctness", + "critical_fact_completeness", + "unsupported_claims", + "correct_uncertainty", + "evidence_support" + ], + "properties": { + "correctness": { "$ref": "#/definitions/score" }, + "critical_fact_completeness": { "$ref": "#/definitions/score" }, + "unsupported_claims": { "$ref": "#/definitions/score" }, + "correct_uncertainty": { "$ref": "#/definitions/score" }, + "evidence_support": { "$ref": "#/definitions/score" }, + "tier1_obligation_recall": { "$ref": "#/definitions/score" } + } + }, + + "validity": { + "type": "object", + "additionalProperties": false, + "required": ["status", "invalidation_reasons", "aggregatable"], + "properties": { + "status": { "type": "string", "enum": ["valid", "degraded", "invalid"] }, + "invalidation_reasons": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "missing_attributable_madar_call", + "prompt_contract_failure", + "answer_contract_failure", + "target_revision_mismatch", + "package_revision_mismatch", + "dependency_lock_mismatch", + "isolation_failure", + "incomplete_transcript", + "incomplete_receipt", + "judge_failure", + "environment_mismatch", + "quality_gate_failure", + "truth_unavailable", + "blinding_unavailable" + ] + } + }, + "aggregatable": { + "type": "boolean", + "description": "MUST be false whenever status is not valid. A false value forbids this receipt from entering any cost, latency, or quality aggregate." + } + } + }, + + "retention": { + "type": "object", + "additionalProperties": false, + "required": ["raw_transcript", "answer_text", "context_artifact", "retention_policy"], + "properties": { + "raw_transcript": { "$ref": "#/definitions/artifactRef" }, + "answer_text": { "$ref": "#/definitions/artifactRef" }, + "context_artifact": { "$ref": "#/definitions/artifactRef" }, + "retention_policy": { "type": "string", "minLength": 1 } + } + }, + + "notes": { "type": "array", "items": { "type": "string" } }, + "started_at": { "type": "string", "format": "date-time" }, + "completed_at": { "type": "string", "format": "date-time" } + }, + + "allOf": [ + { + "description": "A run that is not valid can never be aggregatable.", + "if": { + "properties": { "validity": { "properties": { "status": { "enum": ["degraded", "invalid"] } }, "required": ["status"] } }, + "required": ["validity"] + }, + "then": { + "properties": { "validity": { "properties": { "aggregatable": { "const": false } } } } + } + }, + { + "description": "An invalid run must give a reason.", + "if": { + "properties": { "validity": { "properties": { "status": { "const": "invalid" } }, "required": ["status"] } }, + "required": ["validity"] + }, + "then": { + "properties": { "validity": { "properties": { "invalidation_reasons": { "minItems": 1 } } } } + } + } + ], + + "definitions": { + "costAccount": { + "type": "object", + "additionalProperties": false, + "required": ["measured"], + "properties": { + "measured": { "type": "boolean" }, + "input_tokens": { "type": ["integer", "null"], "minimum": 0 }, + "output_tokens": { "type": ["integer", "null"], "minimum": 0 }, + "cache_creation_input_tokens": { "type": ["integer", "null"], "minimum": 0 }, + "wall_ms": { "type": ["integer", "null"], "minimum": 0 }, + "usd": { "type": ["number", "null"], "minimum": 0 }, + "source": { + "type": "string", + "enum": ["provider_reported", "locally_timed", "unknown", "not_applicable"] + } + } + }, + "score": { + "type": "object", + "additionalProperties": false, + "required": ["measured", "value", "method"], + "properties": { + "measured": { "type": "boolean" }, + "value": { + "type": ["number", "string", "null"], + "description": "MUST be null whenever measured is false. A not_measured dimension never carries a number." + }, + "method": { "type": "string", "minLength": 1 }, + "scored_by": { "type": ["string", "null"] }, + "blinded": { "type": ["boolean", "null"] }, + "truth_version": { "type": ["string", "null"] }, + "not_measured_reason": { "type": ["string", "null"] } + }, + "allOf": [ + { + "if": { "properties": { "measured": { "const": false } }, "required": ["measured"] }, + "then": { + "properties": { + "value": { "type": "null" }, + "not_measured_reason": { "type": "string", "minLength": 1 } + }, + "required": ["not_measured_reason"] + } + } + ] + }, + "artifactRef": { + "type": "object", + "additionalProperties": false, + "required": ["retained", "path", "sha256"], + "properties": { + "retained": { "type": "boolean" }, + "path": { "type": ["string", "null"] }, + "sha256": { "type": ["string", "null"], "pattern": "^[0-9a-f]{64}$" } + } + } + } +} diff --git a/docs/qualification/rubrics.json b/docs/qualification/rubrics.json new file mode 100644 index 00000000..c0fee2fb --- /dev/null +++ b/docs/qualification/rubrics.json @@ -0,0 +1,143 @@ +{ + "contract_version": "1.0.0", + "frozen_at": "2026-08-12", + "dimensions": { + "correctness": { + "definition": "Every assertion the answer makes about the target is true of the pinned source.", + "scale": { "0": "contains a false assertion about the target", "1": "no false assertions, but material gaps", "2": "no false assertions" }, + "scored_by": "blinded_human", + "tiers": [2], + "gating": true + }, + "critical_fact_completeness": { + "definition": "Fraction of the truth file's critical facts that the answer states. Facts marked supporting do not count in the denominator.", + "scale": "ratio in [0,1] with the per-task critical_facts_required_for_pass set treated as mandatory", + "scored_by": "blinded_human", + "tiers": [2], + "gating": true + }, + "unsupported_claims": { + "definition": "Count of assertions matching the task's unsupported_claim_traps, plus any other assertion a blinded reviewer cannot trace to the pinned source.", + "scale": "non-negative integer; lower is better", + "scored_by": "blinded_human", + "tiers": [2], + "gating": true + }, + "correct_uncertainty": { + "definition": "Fraction of the task's correct_uncertainty requirements the answer honours. Both directions are penalised: asserting something the source cannot support, and hedging something the source states plainly.", + "scale": "ratio in [0,1]", + "scored_by": "blinded_human", + "tiers": [2], + "gating": true + }, + "evidence_support": { + "definition": "Fraction of critical assertions accompanied by a citation to a real path and symbol in the pinned target that actually contains the cited content.", + "scale": "ratio in [0,1]", + "scored_by": "both", + "tiers": [1, 2], + "gating": true, + "note": "The path/symbol existence half is deterministic. Whether the cited code supports the assertion is blinded-human only." + }, + "intended_tool_adoption": { + "definition": "Whether the run made an attributable Madar call before broad repository exploration, when the task contract requires one.", + "scale": { "adopted": "attributable Madar call precedes broad exploration", "late": "attributable call occurred after broad exploration", "absent": "no attributable Madar call" }, + "scored_by": "deterministic", + "tiers": [2], + "gating": false, + "note": "Adoption is a behaviour measurement, NOT a context-quality measurement. It is reported in its own column and must never be folded into a quality score. absent makes the run invalid for quality comparison; it is not a quality loss." + }, + "broad_fallback_exploration": { + "definition": "Count of broad repository operations (directory-wide search, unscoped glob, full-file reads outside the evidence set) performed after the first Madar call.", + "scale": "non-negative integer; lower is better", + "scored_by": "deterministic", + "tiers": [2], + "gating": false, + "note": "Reported separately from quality and from cost. A high value with a passing quality score means the context pack was insufficient, not that the answer was bad." + } + }, + "methods": { + "evidence_obligation_recall": { + "tier": 1, + "deterministic": true, + "inputs": ["the context artifact produced for the frozen prompt", "the task truth file's tier1_obligations"], + "procedure": [ + "Collect the evidence set (paths and symbols) the artifact presents as supporting material.", + "required_evidence_paths recall must be >= min_critical_fact_recall.", + "required_evidence_symbols recall must be >= min_critical_fact_recall.", + "Every path cited by the artifact must exist in the pinned target.", + "If any must_not_report_ready_when condition holds, the artifact must not report a ready state." + ], + "outcome": ["pass", "fail", "not_measured"], + "explicitly_not_measured": "Answer quality. Tier 1 measures whether the evidence needed to answer was present, not whether an agent answered well." + }, + "blinded_rubric": { + "tier": 2, + "deterministic": false, + "procedure": [ + "The reviewer receives the answer text, the task prompt, and the truth file.", + "The reviewer does NOT receive the arm label, the token counts, the latency, or the transcript.", + "The reviewer scores correctness, critical_fact_completeness, unsupported_claims, correct_uncertainty, and evidence_support.", + "Arm labels are revealed only after every answer in the cell is scored." + ], + "pass_condition": "correctness == 2 AND every id in critical_facts_required_for_pass is present AND unsupported_claims == 0" + }, + "ordered_path_rubric": { + "tier": 2, + "deterministic": false, + "extends": "blinded_rubric", + "additional_procedure": [ + "For each pair in order_sensitive_pairs, the answer must place the first element before the second.", + "An answer that names every step but inverts an order_sensitive_pair fails correctness." + ] + }, + "affected_set_precision_recall": { + "tier": 2, + "deterministic": false, + "extends": "blinded_rubric", + "additional_procedure": [ + "recall = |named ∩ recall_denominator| / |recall_denominator|", + "Each member of precision_penalty_set named as affected counts as one unsupported claim.", + "The loud-vs-silent judgement is scored under correctness, not under recall." + ] + }, + "single_root_cause_adjudication": { + "tier": 2, + "deterministic": false, + "extends": "blinded_rubric", + "additional_procedure": [ + "The answer passes only if it names a cause in accepted_root_cause_ids as THE cause.", + "Listing the correct cause among several candidate causes without committing scores as a partial: critical_fact_completeness credit, correctness 1, no pass.", + "Every id in uncertainty_required_for_pass must be honoured." + ] + }, + "seeded_defect_detection": { + "tier": 2, + "deterministic": false, + "extends": "blinded_rubric", + "additional_procedure": [ + "Every id in required_detections must be reported with a citation that proves it.", + "Each reported finding matching false_positive_set counts as one unsupported claim.", + "Findings in acceptable_additional_findings are neither required nor penalised." + ] + } + }, + "blinding": { + "required_for_tier": 2, + "rules": [ + "Answers are stripped of arm labels, tool traces, and cost data before review.", + "Answers from both arms of a cell are shuffled and reviewed in one pass.", + "The reviewer must not be the person who authored the change under evaluation.", + "Reviewer identity, review date, and the truth file version are recorded on every score." + ], + "current_status": "unsatisfied", + "current_status_reason": "Single-author repository. See holdout-policy.md; Tier 2 scores produced without an independent reviewer must be labelled self_reviewed and are not publishable evidence." + }, + "aggregation": { + "rules": [ + "Quality dimensions are aggregated per task and per repo. There is no blended cross-task headline.", + "Runs whose validity is not valid contribute to no aggregate other than the invalid-run count.", + "Adoption and broad_fallback_exploration are reported as their own columns and never merged into a quality score.", + "Cost and latency are reported only for cells whose gating quality dimensions passed." + ] + } +} diff --git a/docs/qualification/stop-rule.md b/docs/qualification/stop-rule.md new file mode 100644 index 00000000..082b5698 --- /dev/null +++ b/docs/qualification/stop-rule.md @@ -0,0 +1,65 @@ +# Stop, rollback, and publication rule + +Contract version `1.0.0`, frozen 2026-08-12 for [#655](https://github.com/mohanagy/madar/issues/655). + +This rule exists to be objective enough to block a pull request or a release without a +judgement call. Each condition is written so that a reviewer can answer yes or no from the +receipts alone. + +## S1 — Stop conditions (a change must not ship) + +A roadmap change **must not merge**, and must be rolled back or disabled if already +merged, when any of the following holds against the frozen corpus. + +| Id | Condition | Objective test | +| --- | --- | --- | +| S1.1 | Critical-fact completeness regresses beyond the pre-registered margin | For any task, the post-change critical-fact completeness is lower than the pre-change value by more than the non-inferiority margin **0.05**, on `n_valid >= 5` paired trials in the same cell. | +| S1.2 | Unsupported claims increase materially | For any task, the post-change mean unsupported-claim count exceeds the pre-change mean by **more than 0.5 claims per answer**, or any single answer introduces an unsupported claim listed in that task's `unsupported_claim_traps` that the pre-change arm did not make. | +| S1.3 | False-ready behaviour appears | Any negative-trust probe in [`tier1.json`](./tier1.json) reports a ready state, or any evidence set contains a path or symbol that does not exist in the pinned target. This is a **single-occurrence** trip: one instance blocks. | +| S1.4 | Host adoption falls below the phase target | Across the Tier 2 sweep, fewer than the phase target of Madar-arm runs have `adoption.status` of `adopted`. Phase 0 target: **not set** — adoption is measured and reported, and a *decrease* of more than 10 percentage points against the previous recorded sweep blocks. | +| S1.5 | Graph or artifact integrity fails | Any graph-integrity invariant from #656–#659 fails, or an artifact fails its round-trip or old-reader-rejection check. Single-occurrence trip. | +| S1.6 | Results depend on qualification-repository literals | Any qualification fixture path, symbol, prompt string, repository id, or a near-equivalent special case appears in production retrieval, ranking, context, or claim logic. Single-occurrence trip; checked deterministically by `npm run qualify:validate`. | +| S1.7 | Output differences remain unexplained | A retrieval, pack, graph, or artifact output differs from the pre-change baseline and the pull request does not explain the difference. Updating a snapshot is not an explanation. | +| S1.8 | Cost improves only by reducing outcome quality | A token, latency, or cost improvement is reported for a cell whose correctness or critical-fact completeness is not non-inferior under S1.1. | + +## S2 — Rollback + +When a stop condition is discovered after merge: + +1. Disable the change at the narrowest available seam — feature flag, default flip, or + revert of the specific commit — the same day it is confirmed. +2. Do not fix forward on the protected branch while a stop condition is tripped. +3. File the finding as a linked issue with the receipt paths that prove it. +4. Re-run the affected Tier 1 subset after the rollback and attach the receipt showing the + condition cleared. +5. If the change already shipped to npm, the release notes are amended and the affected + claim is withdrawn before anything else ships. + +## S3 — Publication + +A claim derived from this corpus may be published only when **all** of the following hold. +Any single failure means the claim is not published in any weakened form either. + +1. Every cell backing the claim has `validity.status: "valid"` and `aggregatable: true`. +2. `n_invalid` is published beside `n_valid` for every row. +3. Correctness and critical-fact completeness passed **before** any cost or latency figure + is shown. +4. Tier 2 scores were produced by a blinded reviewer who did not author the change. +5. The sealed holdout is satisfied, or the report makes no generalization claim and carries + the line `sealed holdout unsatisfied; results measure regression only`. +6. The claim is narrower than or equal to the evidence: per-target and per-task, never a + blended headline. +7. The evidence class is labelled per [`evidence-categories.md`](./evidence-categories.md). + +## S4 — What may never be used to clear a stop condition + +- Editing a truth file, a rubric threshold, or a prompt after seeing a result. +- Adding a qualification path, symbol, prompt, or repository name to production logic. +- Re-running a failing cell until it passes and reporting the passing run. +- Marking a measured failure as `not_measured`. +- Substituting a different task, target, or prompt for the one that failed. +- Narrowing the corpus so the failing cell is no longer in it. + +Changing the frozen contract is possible, but only by bumping `contract_version`, stating +what changed and why in the pull request, and re-baselining every affected cell. A contract +change never retroactively clears a recorded stop condition. diff --git a/docs/qualification/tasks.json b/docs/qualification/tasks.json new file mode 100644 index 00000000..ba8109eb --- /dev/null +++ b/docs/qualification/tasks.json @@ -0,0 +1,240 @@ +{ + "contract_version": "1.0.0", + "frozen_at": "2026-08-12", + "prompt_hash_algorithm": "sha256 over the exact UTF-8 prompt text, no trailing newline", + "tasks": [ + { + "id": "arch-plugin-host-extension-seam", + "name": "Explain the extension architecture and its dependency direction", + "category": "architecture-understanding", + "target": "qual-plugin-host", + "tiers": [ + 1, + 2 + ], + "status": "frozen", + "prompt": { + "text": "Describe the extension architecture of this workspace. What is the stable extension surface, what are the module layers and the allowed dependency direction between them, and does any module currently depend in the wrong direction?", + "sha256": "3f4837829b97f6bdd273a9f751ee31141b928e7d77c5bfe850ed1fb2ff05bfd8" + }, + "truth_ref": "truth/arch-plugin-host-extension-seam.json", + "scoring": { + "tier1_method": "evidence_obligation_recall", + "tier2_method": "blinded_rubric", + "rubric_ref": "rubrics.json#/methods/blinded_rubric" + }, + "validity_requirements": { + "requires_attributable_madar_call": true, + "requires_answer_within_prompt_contract": true, + "requires_complete_transcript": true + }, + "truth_provenance": { + "authored_by": "madar-655-qualification-agent", + "author_role": "benchmark author", + "authored_at": "2026-08-12", + "derived_from": [ + "fixture source authored in the same change, read directly" + ], + "madar_derived_sources_used": [], + "inspected_madar_output_before_freeze": false, + "independent_of_production_rule_author": false, + "review_status": "unreviewed" + } + }, + { + "id": "flow-ledger-post-entry", + "name": "Trace the post-entry execution flow to every durable side effect", + "category": "execution-flow-explanation", + "target": "qual-ledger-service", + "tiers": [ + 1, + 2 + ], + "status": "frozen", + "prompt": { + "text": "Trace what happens when a client posts a new ledger entry. Follow the path from the HTTP handler through to every durable side effect and every downstream consumer, and say which step each side effect happens in.", + "sha256": "4f727f2d70f0df36aa08fc208a27d6dcf1b1ed0a8b26a05b1e1dde32b57e29aa" + }, + "truth_ref": "truth/flow-ledger-post-entry.json", + "scoring": { + "tier1_method": "evidence_obligation_recall", + "tier2_method": "ordered_path_rubric", + "rubric_ref": "rubrics.json#/methods/ordered_path_rubric" + }, + "validity_requirements": { + "requires_attributable_madar_call": true, + "requires_answer_within_prompt_contract": true, + "requires_complete_transcript": true + }, + "truth_provenance": { + "authored_by": "madar-655-qualification-agent", + "author_role": "benchmark author", + "authored_at": "2026-08-12", + "derived_from": [ + "fixture source authored in the same change, read directly" + ], + "madar_derived_sources_used": [], + "inspected_madar_output_before_freeze": false, + "independent_of_production_rule_author": false, + "review_status": "unreviewed" + } + }, + { + "id": "impact-ledger-drop-outbox-publish", + "name": "Impact of dropping outbox publication on the write path", + "category": "impact-analysis", + "target": "qual-ledger-service", + "tiers": [ + 1, + 2 + ], + "status": "frozen", + "prompt": { + "text": "What breaks if the write path stops emitting outbox events after a ledger entry is appended? List every module whose behaviour changes and say whether the failure is loud or silent.", + "sha256": "9ccbea6fc04c64b868676c858d2f34c0e808bca307e3481ccda62e3c9713cb6c" + }, + "truth_ref": "truth/impact-ledger-drop-outbox-publish.json", + "scoring": { + "tier1_method": "evidence_obligation_recall", + "tier2_method": "affected_set_precision_recall", + "rubric_ref": "rubrics.json#/methods/affected_set_precision_recall" + }, + "validity_requirements": { + "requires_attributable_madar_call": true, + "requires_answer_within_prompt_contract": true, + "requires_complete_transcript": true + }, + "truth_provenance": { + "authored_by": "madar-655-qualification-agent", + "author_role": "benchmark author", + "authored_at": "2026-08-12", + "derived_from": [ + "fixture source authored in the same change, read directly" + ], + "madar_derived_sources_used": [], + "inspected_madar_output_before_freeze": false, + "independent_of_production_rule_author": false, + "review_status": "unreviewed" + } + }, + { + "id": "rootcause-ledger-duplicate-entries", + "name": "Root-cause a duplicate entry under retried requests", + "category": "bug-root-cause-investigation", + "target": "qual-ledger-service", + "tiers": [ + 1, + 2 + ], + "status": "frozen", + "prompt": { + "text": "Under load, a client that retries a request with the same idempotency key sometimes ends up with two ledger entries instead of one. Find the root cause and explain the exact ordering of operations that produces the duplicate.", + "sha256": "3d6f2f70a3b21508c17637cce0d9d94ccbea241adaeb2223192664c8d9856164" + }, + "truth_ref": "truth/rootcause-ledger-duplicate-entries.json", + "scoring": { + "tier1_method": "evidence_obligation_recall", + "tier2_method": "single_root_cause_adjudication", + "rubric_ref": "rubrics.json#/methods/single_root_cause_adjudication" + }, + "validity_requirements": { + "requires_attributable_madar_call": true, + "requires_answer_within_prompt_contract": true, + "requires_complete_transcript": true + }, + "truth_provenance": { + "authored_by": "madar-655-qualification-agent", + "author_role": "benchmark author", + "authored_at": "2026-08-12", + "derived_from": [ + "seeded defect introduced deliberately while authoring the fixture" + ], + "madar_derived_sources_used": [], + "inspected_madar_output_before_freeze": false, + "independent_of_production_rule_author": false, + "review_status": "unreviewed" + } + }, + { + "id": "plan-plugin-host-object-storage-plugin", + "name": "Plan a bounded new built-in plugin", + "category": "implementation-planning", + "target": "qual-plugin-host", + "tiers": [ + 1, + 2 + ], + "status": "frozen", + "prompt": { + "text": "Plan the change needed to add a new built-in export plugin that writes batches to object storage. Do not widen the extension surface and do not change host modules that are unrelated to registering a plugin. List the files you would add or change and why each one is required.", + "sha256": "b031c780c0810bf79463045b1b7494ec431860173514c87a2572918a7603674f" + }, + "truth_ref": "truth/plan-plugin-host-object-storage-plugin.json", + "scoring": { + "tier1_method": "evidence_obligation_recall", + "tier2_method": "blinded_rubric", + "rubric_ref": "rubrics.json#/methods/blinded_rubric", + "hidden_acceptance_test": { + "required": true, + "status": "unavailable", + "reason": "A hidden acceptance test must be authored and held by someone other than the production-rule author. See holdout-policy.md. Until it exists, this task's Tier 2 implementation score is not_measured and only the plan rubric applies." + } + }, + "validity_requirements": { + "requires_attributable_madar_call": true, + "requires_answer_within_prompt_contract": true, + "requires_complete_transcript": true + }, + "truth_provenance": { + "authored_by": "madar-655-qualification-agent", + "author_role": "benchmark author", + "authored_at": "2026-08-12", + "derived_from": [ + "fixture source authored in the same change, read directly" + ], + "madar_derived_sources_used": [], + "inspected_madar_output_before_freeze": false, + "independent_of_production_rule_author": false, + "review_status": "unreviewed" + } + }, + { + "id": "review-ledger-http-authorization", + "name": "Review the ledger HTTP surface for authorization defects", + "category": "review-security", + "target": "qual-ledger-service", + "tiers": [ + 1, + 2 + ], + "status": "frozen", + "prompt": { + "text": "Review the ledger HTTP surface for authorization defects before merge. For each defect, name the handler, say what an attacker can do, and cite the code that proves it.", + "sha256": "d9a634a7bc95d610223b7d0d3c827b514d14f7871e1b248cc46285821f37ea73" + }, + "truth_ref": "truth/review-ledger-http-authorization.json", + "scoring": { + "tier1_method": "evidence_obligation_recall", + "tier2_method": "seeded_defect_detection", + "rubric_ref": "rubrics.json#/methods/seeded_defect_detection" + }, + "validity_requirements": { + "requires_attributable_madar_call": true, + "requires_answer_within_prompt_contract": true, + "requires_complete_transcript": true + }, + "truth_provenance": { + "authored_by": "madar-655-qualification-agent", + "author_role": "benchmark author", + "authored_at": "2026-08-12", + "derived_from": [ + "seeded defect introduced deliberately while authoring the fixture" + ], + "madar_derived_sources_used": [], + "inspected_madar_output_before_freeze": false, + "independent_of_production_rule_author": false, + "review_status": "unreviewed" + } + } + ] +} diff --git a/docs/qualification/tier1.json b/docs/qualification/tier1.json new file mode 100644 index 00000000..7d264b54 --- /dev/null +++ b/docs/qualification/tier1.json @@ -0,0 +1,93 @@ +{ + "contract_version": "1.0.0", + "frozen_at": "2026-08-12", + "purpose": "The deterministic subset that a pull request can run. It measures whether the evidence needed to answer each frozen task was present in the context artifact, and whether readiness was refused when it should have been. It never scores answer quality and never runs an agent.", + "properties": { + "deterministic": true, + "requires_network": false, + "requires_model_provider": false, + "requires_api_spend": false, + "trials_per_cell": 1, + "trials_rationale": "One trial is sufficient because no sampling occurs. If a Tier 1 cell is ever observed to be non-deterministic, that is itself a defect and the cell must be reported as invalid, not retried until it passes." + }, + "cells": [ + { + "task_id": "arch-plugin-host-extension-seam", + "target_id": "qual-plugin-host", + "method": "evidence_obligation_recall" + }, + { + "task_id": "flow-ledger-post-entry", + "target_id": "qual-ledger-service", + "method": "evidence_obligation_recall" + }, + { + "task_id": "impact-ledger-drop-outbox-publish", + "target_id": "qual-ledger-service", + "method": "evidence_obligation_recall" + }, + { + "task_id": "rootcause-ledger-duplicate-entries", + "target_id": "qual-ledger-service", + "method": "evidence_obligation_recall" + }, + { + "task_id": "plan-plugin-host-object-storage-plugin", + "target_id": "qual-plugin-host", + "method": "evidence_obligation_recall" + }, + { + "task_id": "review-ledger-http-authorization", + "target_id": "qual-ledger-service", + "method": "evidence_obligation_recall" + } + ], + "negative_trust_probes": [ + { + "id": "neg-ledger-absent-feature", + "target_id": "qual-ledger-service", + "prompt": { + "text": "How does this service enforce per-currency spending limits before an entry is accepted?", + "sha256": "306a894807d16ea4167cf85826dad35504cf9594a93b1ef286b6682dc278efe2" + }, + "ground_truth": "No spending-limit logic exists anywhere in the workspace. The only validation on the write path is bearer-token authentication, account membership on the post route, and type checks on amountMinor, currency, and idempotencyKey.", + "required_behaviour": [ + "The artifact must not report a ready state.", + "The artifact must declare that the requested behaviour was not found rather than presenting the nearest validation code as if it implemented limits.", + "No fabricated path or symbol may appear in the evidence set." + ], + "failure_is": "false_ready" + }, + { + "id": "neg-plugin-host-absent-loader", + "target_id": "qual-plugin-host", + "prompt": { + "text": "Where does the host load third-party plugins from disk at runtime, and how are they sandboxed?", + "sha256": "bd88479cb5d0b2d190d681cfe5f31123cbe4c72d09c064c153aee1e70c802209" + }, + "ground_truth": "There is no dynamic loading and no sandboxing. builtInPlugins() returns a hardcoded array, and registerPlugin accepts an already-constructed object from the caller.", + "required_behaviour": [ + "The artifact must not report a ready state.", + "The artifact must state that no dynamic loading or sandboxing mechanism exists.", + "Presenting resolveHostConfig or PluginRegistry as a loader counts as a false-ready failure." + ], + "failure_is": "false_ready" + } + ], + "gate": { + "applies_to": "every pull request that touches retrieval, context building, graph construction, or ranking", + "pass_condition": "every cell passes its evidence_obligation_recall threshold AND every negative_trust_probe satisfies its required_behaviour", + "on_failure": "the pull request is blocked; see stop-rule.md", + "forbidden_remedies": [ + "Adding a qualification path, symbol, prompt, or repository name to production retrieval, ranking, or claim logic.", + "Relaxing a truth file to match observed output.", + "Lowering min_critical_fact_recall to make a cell pass.", + "Marking a failing cell not_measured. not_measured is for runs that could not be measured, never for runs that were measured and failed." + ] + }, + "calibration_status": { + "state": "pre_registered_uncalibrated", + "explanation": "The thresholds in the truth files were written from the fixture source before Madar was ever run against these fixtures, and the author did not inspect Madar output before freezing. No cell in this subset has a recorded pass or fail yet, so it is unknown how many currently pass.", + "consequence": "The first execution of this subset (issue #661) is a measurement, not a regression check. A failing cell on first execution is a product finding to be filed as a linked issue, not a reason to edit this contract." + } +} diff --git a/docs/qualification/tier2-matrix.json b/docs/qualification/tier2-matrix.json new file mode 100644 index 00000000..b06c75a8 --- /dev/null +++ b/docs/qualification/tier2-matrix.json @@ -0,0 +1,44 @@ +{ + "contract_version": "1.0.0", + "frozen_at": "2026-08-12", + "status": "planned", + "status_meaning": "The matrix shape, arms, repeat count, and reporting rules are frozen now so they cannot be chosen after seeing results. No Tier 2 cell has been executed under this contract.", + "blocked_by": [ + "Independent truth for the Tier 2 git targets does not exist (corpus.json marks both as pinned_no_truth).", + "Blinded review is unavailable in a single-author context (rubrics.json#/blinding/current_status).", + "The sealed holdout slot is unsatisfied (holdout-policy.md)." + ], + "dimensions": { + "targets": ["qual-ledger-service", "qual-plugin-host", "qual-unkey", "qual-payload", "qual-sealed-a"], + "tasks": [ + "arch-plugin-host-extension-seam", + "flow-ledger-post-entry", + "impact-ledger-drop-outbox-publish", + "rootcause-ledger-duplicate-entries", + "plan-plugin-host-object-storage-plugin", + "review-ledger-http-authorization" + ], + "arms": ["native", "madar"], + "cache_modes": ["cold", "warm"], + "trials_per_cell": 5 + }, + "trial_rationale": "Five trials per cell is the smallest count that lets a per-cell median be reported with a visible min/max spread while keeping a full sweep affordable. It is not powered for a small effect size. Any claim that depends on a difference smaller than the observed per-cell spread must be reported as not established, whatever the medians say.", + "pairing": "Both arms of a cell run against the same target revision, the same prompt text, the same tool permission list apart from Madar tools, the same cache mode, and the same trial index. A cell where the two arms differ in any identity field is invalid, not a result.", + "reporting": { + "unit": "one row per target per task per cache mode", + "statistics": ["median", "min", "max", "n_valid", "n_invalid"], + "forbidden": [ + "A blended cross-target or cross-task headline number.", + "Reporting cost or latency for a cell whose gating quality dimensions did not pass.", + "Dropping invalid runs silently; n_invalid is published next to n_valid on every row.", + "Reporting a win for a cell where the Madar arm had adoption status absent." + ] + }, + "ordering_rule": "Quality gates are evaluated first. Cost, token, and latency columns are populated only for cells that already passed correctness and critical-fact completeness. This ordering is part of the contract, not a presentation choice.", + "execution_prerequisites": [ + "Every task in the sweep has a truth file with review_status other than unreviewed.", + "A reviewer who did not author the change under evaluation is available for blinded scoring.", + "Isolation mode is active and the environment receipt matches the pinned contract.", + "The sealed holdout slot is either filled or the sweep is published with the holdout column explicitly marked unsatisfied." + ] +} diff --git a/docs/qualification/truth/arch-plugin-host-extension-seam.json b/docs/qualification/truth/arch-plugin-host-extension-seam.json new file mode 100644 index 00000000..d3654820 --- /dev/null +++ b/docs/qualification/truth/arch-plugin-host-extension-seam.json @@ -0,0 +1,125 @@ +{ + "contract_version": "1.0.0", + "task_id": "arch-plugin-host-extension-seam", + "target": "qual-plugin-host", + "category": "architecture-understanding", + "provenance": { + "authored_by": "madar-655-qualification-agent", + "authored_at": "2026-08-12", + "derived_from": ["fixture source authored in the same change, read directly"], + "madar_derived_sources_used": [], + "inspected_madar_output_before_freeze": false, + "independent_of_production_rule_author": false, + "review_status": "unreviewed" + }, + "critical_facts": [ + { + "id": "extension-surface", + "statement": "The stable extension surface is the ExportPlugin interface in src/contracts/plugin.ts; every plugin implements it.", + "criticality": "critical", + "evidence": [{ "path": "src/contracts/plugin.ts", "symbol": "ExportPlugin" }] + }, + { + "id": "layer-direction", + "statement": "The intended dependency direction is contracts <- plugins and contracts <- host; plugins must not depend on host.", + "criticality": "critical", + "evidence": [ + { "path": "src/contracts/plugin.ts", "symbol": "ExportPlugin" }, + { "path": "src/plugins/csv-export-plugin.ts", "symbol": "CsvExportPlugin" } + ] + }, + { + "id": "composition-root", + "statement": "src/host/plugin-host.ts is the composition root and the only module that imports both host internals and concrete plugins.", + "criticality": "critical", + "evidence": [ + { "path": "src/host/plugin-host.ts", "symbol": "PluginHost" }, + { "path": "src/host/plugin-host.ts", "symbol": "builtInPlugins" } + ] + }, + { + "id": "registry-resolution", + "statement": "PluginRegistry maps names to plugins, preserves registration order, and rejects duplicate names with DuplicatePluginError.", + "criticality": "critical", + "evidence": [{ "path": "src/host/registry.ts", "symbol": "PluginRegistry" }] + }, + { + "id": "lifecycle-ownership", + "statement": "runExportLifecycle owns init -> export -> dispose ordering and, when failFast is false, isolates a failing plugin while still running and disposing the rest.", + "criticality": "critical", + "evidence": [{ "path": "src/host/lifecycle.ts", "symbol": "runExportLifecycle" }] + }, + { + "id": "config-layering", + "statement": "resolveHostConfig layers defaults, then file config, then environment overrides; plugins never read configuration themselves and receive only scoped settings through PluginContext.", + "criticality": "supporting", + "evidence": [ + { "path": "src/host/config.ts", "symbol": "resolveHostConfig" }, + { "path": "src/host/plugin-host.ts", "symbol": "PluginHost.contextFor" } + ] + }, + { + "id": "boundary-violation", + "statement": "src/plugins/webhook-export-plugin.ts imports resolveHostConfig from host/config.ts. This is the only wrong-direction dependency in the workspace.", + "criticality": "critical", + "seeded_defect_id": "seeded-boundary-violation", + "evidence": [{ "path": "src/plugins/webhook-export-plugin.ts", "symbol": "WebhookExportPlugin.init" }] + } + ], + "correct_uncertainty": [ + { + "id": "no-enforcement", + "requirement": "The dependency direction is a convention. Nothing in the workspace enforces it — there is no lint rule, no build boundary, and no runtime check. An answer that presents the direction as enforced is wrong." + }, + { + "id": "static-only", + "requirement": "All of this is read from static imports. No runtime trace exists, so claims about what actually loads at runtime are hypotheses." + } + ], + "unsupported_claim_traps": [ + { + "id": "dynamic-discovery", + "claim": "Plugins are discovered dynamically at runtime from a plugins directory or from configuration.", + "why_false": "builtInPlugins() in src/host/plugin-host.ts returns a hardcoded array. Configuration only selects which already-registered names are enabled." + }, + { + "id": "registry-enforces-boundary", + "claim": "PluginRegistry or PluginHost enforces that plugins only depend on contracts.", + "why_false": "Neither module inspects plugin imports. The boundary is documentation only, which is exactly why the webhook plugin can violate it." + }, + { + "id": "csv-also-violates", + "claim": "CsvExportPlugin also reaches into host modules.", + "why_false": "src/plugins/csv-export-plugin.ts imports from ../contracts/plugin.js only." + }, + { + "id": "failfast-still-disposes", + "claim": "With failFast enabled, already-initialized plugins are still disposed after a failure.", + "why_false": "runExportLifecycle throws the PluginFailure immediately when failFast is true, so the dispose loop is never reached." + } + ], + "tier1_obligations": { + "required_evidence_paths": [ + "src/contracts/plugin.ts", + "src/host/plugin-host.ts", + "src/host/registry.ts", + "src/host/lifecycle.ts", + "src/plugins/webhook-export-plugin.ts" + ], + "required_evidence_symbols": ["ExportPlugin", "PluginHost", "PluginRegistry", "runExportLifecycle"], + "min_critical_fact_recall": 1.0, + "must_not_report_ready_when": [ + "any required_evidence_path is absent from the evidence set", + "the wrong-direction import in src/plugins/webhook-export-plugin.ts is not represented in the graph" + ] + }, + "tier2_scoring": { + "method": "blinded_rubric", + "critical_facts_required_for_pass": [ + "extension-surface", + "layer-direction", + "composition-root", + "boundary-violation" + ] + } +} diff --git a/docs/qualification/truth/flow-ledger-post-entry.json b/docs/qualification/truth/flow-ledger-post-entry.json new file mode 100644 index 00000000..03b08ea5 --- /dev/null +++ b/docs/qualification/truth/flow-ledger-post-entry.json @@ -0,0 +1,174 @@ +{ + "contract_version": "1.0.0", + "task_id": "flow-ledger-post-entry", + "target": "qual-ledger-service", + "category": "execution-flow-explanation", + "provenance": { + "authored_by": "madar-655-qualification-agent", + "authored_at": "2026-08-12", + "derived_from": ["fixture source authored in the same change, read directly"], + "madar_derived_sources_used": [], + "inspected_madar_output_before_freeze": false, + "independent_of_production_rule_author": false, + "review_status": "unreviewed" + }, + "ordered_path": [ + { + "step": 1, + "id": "http-entrypoint", + "statement": "POST /accounts/:accountId/entries is handled by postLedgerEntry from createLedgerRouter.", + "criticality": "critical", + "evidence": [{ "path": "src/http/ledger-routes.ts", "symbol": "createLedgerRouter.postLedgerEntry" }] + }, + { + "step": 2, + "id": "authentication", + "statement": "resolveRequestContext parses the bearer token and throws UnauthenticatedError when the token is missing or unknown.", + "criticality": "critical", + "evidence": [{ "path": "src/auth/request-context.ts", "symbol": "resolveRequestContext" }] + }, + { + "step": 3, + "id": "authorization", + "statement": "assertAccountAccess rejects the request with ForbiddenError when the path accountId is not in principal.accountIds.", + "criticality": "critical", + "evidence": [{ "path": "src/auth/request-context.ts", "symbol": "assertAccountAccess" }] + }, + { + "step": 4, + "id": "body-validation", + "statement": "requireNumber and requireString validate amountMinor, currency, and idempotencyKey before the service is called.", + "criticality": "supporting", + "evidence": [{ "path": "src/http/ledger-routes.ts", "symbol": "requireString" }] + }, + { + "step": 5, + "id": "replay-check", + "statement": "LedgerService.postEntry first asks IdempotencyStore.find for a prior reservation and returns the stored entry when one exists.", + "criticality": "critical", + "evidence": [ + { "path": "src/service/ledger-service.ts", "symbol": "LedgerService.postEntry" }, + { "path": "src/store/idempotency-store.ts", "symbol": "IdempotencyStore.find" } + ] + }, + { + "step": 6, + "id": "ledger-append", + "statement": "LedgerStore.appendEntry appends a new immutable entry and allocates the entryId. Durable side effect.", + "criticality": "critical", + "side_effect": "durable", + "evidence": [{ "path": "src/store/ledger-store.ts", "symbol": "LedgerStore.appendEntry" }] + }, + { + "step": 7, + "id": "idempotency-reserve", + "statement": "IdempotencyStore.reserve records the key -> entryId mapping. This happens AFTER the append. Durable side effect.", + "criticality": "critical", + "side_effect": "durable", + "evidence": [{ "path": "src/store/idempotency-store.ts", "symbol": "IdempotencyStore.reserve" }] + }, + { + "step": 8, + "id": "outbox-publish", + "statement": "OutboxPublisher.publish emits ledger.entry.posted synchronously to every subscribed handler. Durable side effect.", + "criticality": "critical", + "side_effect": "durable", + "evidence": [{ "path": "src/outbox/outbox-publisher.ts", "symbol": "OutboxPublisher.publish" }] + }, + { + "step": 9, + "id": "balance-projection", + "statement": "BalanceProjection.applyPosted is the downstream consumer of ledger.entry.posted and updates the derived account balance.", + "criticality": "critical", + "evidence": [{ "path": "src/projections/balance-projection.ts", "symbol": "BalanceProjection.attach" }] + }, + { + "step": 10, + "id": "audit-record", + "statement": "AuditLog.record writes the ledger.entry.posted audit row AFTER the outbox publish. Durable side effect.", + "criticality": "critical", + "side_effect": "durable", + "evidence": [{ "path": "src/audit/audit-log.ts", "symbol": "AuditLog.record" }] + }, + { + "step": 11, + "id": "response", + "statement": "The handler returns 201 with the created entry.", + "criticality": "supporting", + "evidence": [{ "path": "src/http/ledger-routes.ts", "symbol": "createLedgerRouter.postLedgerEntry" }] + } + ], + "durable_side_effects": ["ledger-append", "idempotency-reserve", "outbox-publish", "audit-record"], + "correct_uncertainty": [ + { + "id": "projection-not-wired", + "requirement": "No module in this workspace calls BalanceProjection.attach. The projection consumes ledger.entry.posted only if a composition root subscribes it, and no such composition root exists here. An answer that states the projection is definitely wired is over-claiming; an answer that omits the consumer entirely is incomplete." + }, + { + "id": "static-hypothesis", + "requirement": "The path is reconstructed from static call sites, not from an observed runtime trace." + } + ], + "unsupported_claim_traps": [ + { + "id": "async-outbox", + "claim": "Outbox events are persisted to a table or queue and delivered asynchronously by a worker.", + "why_false": "OutboxPublisher.publish invokes subscribed handlers inline; there is no store, no queue, and no worker." + }, + { + "id": "audit-before-outbox", + "claim": "The audit record is written before the outbox event is published.", + "why_false": "In LedgerService.postEntry the publish call precedes the auditLog.record call." + }, + { + "id": "reserve-before-append", + "claim": "The idempotency key is reserved before the ledger entry is appended.", + "why_false": "reserve is called after appendEntry. This ordering is the seeded defect exercised by rootcause-ledger-duplicate-entries." + }, + { + "id": "store-computes-balance", + "claim": "LedgerStore maintains or returns account balances.", + "why_false": "LedgerStore only appends and reads entries; balances exist only in BalanceProjection." + } + ], + "tier1_obligations": { + "required_evidence_paths": [ + "src/http/ledger-routes.ts", + "src/auth/request-context.ts", + "src/service/ledger-service.ts", + "src/store/ledger-store.ts", + "src/store/idempotency-store.ts", + "src/outbox/outbox-publisher.ts", + "src/audit/audit-log.ts", + "src/projections/balance-projection.ts" + ], + "required_evidence_symbols": [ + "postLedgerEntry", + "LedgerService", + "LedgerStore", + "OutboxPublisher", + "AuditLog", + "BalanceProjection" + ], + "min_critical_fact_recall": 1.0, + "must_not_report_ready_when": [ + "any required_evidence_path is absent from the evidence set", + "the event edge from OutboxPublisher.publish to BalanceProjection is neither present nor declared as unresolved" + ] + }, + "tier2_scoring": { + "method": "ordered_path_rubric", + "order_sensitive_pairs": [ + ["ledger-append", "idempotency-reserve"], + ["outbox-publish", "audit-record"] + ], + "critical_facts_required_for_pass": [ + "http-entrypoint", + "replay-check", + "ledger-append", + "outbox-publish", + "audit-record", + "balance-projection" + ] + } +} diff --git a/docs/qualification/truth/impact-ledger-drop-outbox-publish.json b/docs/qualification/truth/impact-ledger-drop-outbox-publish.json new file mode 100644 index 00000000..919abb4f --- /dev/null +++ b/docs/qualification/truth/impact-ledger-drop-outbox-publish.json @@ -0,0 +1,111 @@ +{ + "contract_version": "1.0.0", + "task_id": "impact-ledger-drop-outbox-publish", + "target": "qual-ledger-service", + "category": "impact-analysis", + "provenance": { + "authored_by": "madar-655-qualification-agent", + "authored_at": "2026-08-12", + "derived_from": ["fixture source authored in the same change, read directly"], + "madar_derived_sources_used": [], + "inspected_madar_output_before_freeze": false, + "independent_of_production_rule_author": false, + "review_status": "unreviewed" + }, + "change_under_analysis": "Remove the OutboxPublisher.publish calls from LedgerService.postEntry and LedgerService.reverseEntry.", + "affected_set": [ + { + "id": "service-call-sites", + "path": "src/service/ledger-service.ts", + "symbols": ["LedgerService.postEntry", "LedgerService.reverseEntry"], + "effect": "Both call sites disappear; the service no longer emits anything on the write path.", + "failure_mode": "silent", + "criticality": "critical" + }, + { + "id": "publisher-dead", + "path": "src/outbox/outbox-publisher.ts", + "symbols": ["OutboxPublisher.publish"], + "effect": "publish has no remaining callers; subscriptions registered through subscribe never fire.", + "failure_mode": "silent", + "criticality": "critical" + }, + { + "id": "projection-stale", + "path": "src/projections/balance-projection.ts", + "symbols": ["BalanceProjection.applyPosted", "BalanceProjection.applyReversed", "BalanceProjection.balanceFor"], + "effect": "The derived read model stops advancing. balanceFor keeps returning the last value before the change, or undefined, while the ledger keeps growing.", + "failure_mode": "silent", + "criticality": "critical" + } + ], + "unaffected_set": [ + { + "path": "src/store/ledger-store.ts", + "reason": "The append-only entry log is the write model and is unchanged; entries are still created and readable." + }, + { + "path": "src/audit/audit-log.ts", + "reason": "AuditLog.record is called directly by LedgerService and does not depend on the outbox." + }, + { + "path": "src/store/idempotency-store.ts", + "reason": "Retry suppression is independent of event publication." + }, + { + "path": "src/auth/request-context.ts", + "reason": "Authentication and authorization run before the service and are unaffected." + }, + { + "path": "src/http/ledger-routes.ts", + "reason": "Both handlers still return 201 with the created entry; the HTTP contract does not change." + } + ], + "correct_uncertainty": [ + { + "id": "no-wiring-site", + "requirement": "No module in this workspace calls BalanceProjection.attach, so the projection's subscription is inferred from the event names it subscribes to rather than observed at a wiring site. The impact on the projection must be stated as conditional on that wiring." + } + ], + "unsupported_claim_traps": [ + { + "id": "loud-failure", + "claim": "Removing the publish calls causes an error, a failed request, or a visible test failure.", + "why_false": "Nothing checks that an event was published. Every failure mode here is silent divergence between the write model and the read model." + }, + { + "id": "ledger-store-affected", + "claim": "LedgerStore stops recording entries or loses data.", + "why_false": "appendEntry is called before publish and is untouched." + }, + { + "id": "audit-affected", + "claim": "The audit trail is lost.", + "why_false": "AuditLog.record is a direct call from LedgerService, not an outbox subscriber." + } + ], + "tier1_obligations": { + "required_evidence_paths": [ + "src/service/ledger-service.ts", + "src/outbox/outbox-publisher.ts", + "src/projections/balance-projection.ts" + ], + "required_evidence_symbols": ["OutboxPublisher", "BalanceProjection", "LedgerService"], + "min_critical_fact_recall": 1.0, + "must_not_report_ready_when": [ + "the consumer side of ledger.entry.posted is missing from the evidence set and is not declared as unresolved" + ] + }, + "tier2_scoring": { + "method": "affected_set_precision_recall", + "recall_denominator": ["service-call-sites", "publisher-dead", "projection-stale"], + "precision_penalty_set": [ + "src/store/ledger-store.ts", + "src/audit/audit-log.ts", + "src/store/idempotency-store.ts", + "src/auth/request-context.ts", + "src/http/ledger-routes.ts" + ], + "critical_facts_required_for_pass": ["projection-stale"] + } +} diff --git a/docs/qualification/truth/plan-plugin-host-object-storage-plugin.json b/docs/qualification/truth/plan-plugin-host-object-storage-plugin.json new file mode 100644 index 00000000..c7ef0088 --- /dev/null +++ b/docs/qualification/truth/plan-plugin-host-object-storage-plugin.json @@ -0,0 +1,129 @@ +{ + "contract_version": "1.0.0", + "task_id": "plan-plugin-host-object-storage-plugin", + "target": "qual-plugin-host", + "category": "implementation-planning", + "provenance": { + "authored_by": "madar-655-qualification-agent", + "authored_at": "2026-08-12", + "derived_from": ["fixture source authored in the same change, read directly"], + "madar_derived_sources_used": [], + "inspected_madar_output_before_freeze": false, + "independent_of_production_rule_author": false, + "review_status": "unreviewed" + }, + "reference_plan": { + "files_to_add": [ + { + "new_path": "src/plugins/object-storage-export-plugin.ts", + "why": "New ExportPlugin implementation. Must import from ../contracts/plugin.js only and read bucket/prefix/endpoint from PluginContext.settings." + } + ], + "files_to_change": [ + { + "path": "src/host/plugin-host.ts", + "why": "builtInPlugins() is the single registration point for built-in plugins; the new plugin is constructed and returned there." + } + ], + "files_that_must_not_change": [ + { + "path": "src/contracts/plugin.ts", + "why": "Adding an object-storage-specific method or field widens the extension surface, which the prompt forbids." + }, + { + "path": "src/host/registry.ts", + "why": "Registration is name-based and already generic; a new plugin needs no registry change." + }, + { + "path": "src/host/lifecycle.ts", + "why": "init/export/dispose ordering is already generic." + }, + { + "path": "src/host/config.ts", + "why": "Enablement flows through HostConfig.enabledPlugins and the EXPORT_PLUGINS environment override, both of which are already name-driven." + } + ], + "enablement": "The plugin runs only when its name appears in HostConfig.enabledPlugins or in the EXPORT_PLUGINS environment override. No code change is required to make that possible.", + "optional": [ + "Implement dispose() to flush or close the storage client, since runExportLifecycle calls dispose on every initialized plugin." + ] + }, + "critical_facts": [ + { + "id": "single-registration-point", + "statement": "builtInPlugins() in src/host/plugin-host.ts is the only place a built-in plugin is registered.", + "criticality": "critical", + "evidence": [{ "path": "src/host/plugin-host.ts", "symbol": "builtInPlugins" }] + }, + { + "id": "contract-only-dependency", + "statement": "The new plugin must depend on src/contracts/plugin.ts only, following CsvExportPlugin rather than WebhookExportPlugin.", + "criticality": "critical", + "evidence": [ + { "path": "src/plugins/csv-export-plugin.ts", "symbol": "CsvExportPlugin" }, + { "path": "src/contracts/plugin.ts", "symbol": "ExportPlugin" } + ] + }, + { + "id": "settings-through-context", + "statement": "Plugin configuration arrives through PluginContext.settings, which PluginHost.contextFor scopes per plugin name.", + "criticality": "critical", + "evidence": [{ "path": "src/host/plugin-host.ts", "symbol": "PluginHost.contextFor" }] + }, + { + "id": "no-surface-widening", + "statement": "src/contracts/plugin.ts must not change.", + "criticality": "critical", + "evidence": [{ "path": "src/contracts/plugin.ts", "symbol": "ExportPlugin" }] + } + ], + "correct_uncertainty": [ + { + "id": "no-object-storage-dependency", + "requirement": "The workspace has no object-storage client and no runtime dependencies at all. A plan must either introduce one explicitly as a new dependency or state that the transport is left abstract; silently assuming an SDK is already available is wrong." + }, + { + "id": "no-tests-present", + "requirement": "The workspace contains no test files, so a plan that claims existing tests will cover the change is unsupported." + } + ], + "unsupported_claim_traps": [ + { + "id": "extend-contract", + "claim": "Add an ObjectStorage-specific method, option, or field to the ExportPlugin interface.", + "why_false": "That widens the extension surface, which the prompt forbids, and would force every existing plugin to change." + }, + { + "id": "registry-change", + "claim": "PluginRegistry must be modified to know about the new plugin.", + "why_false": "register() and resolve() are name-based and already generic." + }, + { + "id": "copy-webhook", + "claim": "Follow WebhookExportPlugin and read host configuration directly with resolveHostConfig.", + "why_false": "That is the seeded boundary violation; copying it reintroduces a wrong-direction dependency." + } + ], + "tier1_obligations": { + "required_evidence_paths": [ + "src/contracts/plugin.ts", + "src/host/plugin-host.ts", + "src/plugins/csv-export-plugin.ts" + ], + "required_evidence_symbols": ["ExportPlugin", "builtInPlugins", "PluginContext"], + "min_critical_fact_recall": 1.0, + "must_not_report_ready_when": [ + "src/host/plugin-host.ts is absent from the evidence set" + ] + }, + "tier2_scoring": { + "method": "blinded_rubric", + "critical_facts_required_for_pass": ["single-registration-point", "contract-only-dependency", "no-surface-widening"], + "hidden_acceptance_test": { + "required": true, + "status": "unavailable", + "blocking_reason": "An executable hidden acceptance test must be authored and held by a person other than the production-rule author. In the current single-author context it cannot be produced credibly. Until it exists the bounded-implementation score for this task is not_measured and only the plan rubric is scored.", + "human_action_required": "A second maintainer authors an acceptance test asserting that a new plugin can be added by touching exactly src/plugins/.ts and builtInPlugins(), and stores it outside this repository per holdout-policy.md." + } + } +} diff --git a/docs/qualification/truth/review-ledger-http-authorization.json b/docs/qualification/truth/review-ledger-http-authorization.json new file mode 100644 index 00000000..f393d8be --- /dev/null +++ b/docs/qualification/truth/review-ledger-http-authorization.json @@ -0,0 +1,118 @@ +{ + "contract_version": "1.0.0", + "task_id": "review-ledger-http-authorization", + "target": "qual-ledger-service", + "category": "review-security", + "provenance": { + "authored_by": "madar-655-qualification-agent", + "authored_at": "2026-08-12", + "derived_from": ["seeded defect introduced deliberately while authoring the fixture"], + "madar_derived_sources_used": [], + "inspected_madar_output_before_freeze": false, + "independent_of_production_rule_author": false, + "review_status": "unreviewed" + }, + "seeded_defect": { + "id": "seeded-reversal-authorization", + "path": "src/http/ledger-routes.ts", + "symbol": "createLedgerRouter.reverseLedgerEntry", + "description": "reverseLedgerEntry authenticates the caller but never authorizes them against the account that owns entryId. postLedgerEntry calls assertAccountAccess; the reversal handler does not.", + "attacker_capability": "Any principal holding a valid token can reverse a ledger entry belonging to an account they do not own, which writes a negative entry, emits ledger.entry.reversed, and corrupts the other tenant's derived balance.", + "proving_evidence": [ + { "path": "src/http/ledger-routes.ts", "symbol": "createLedgerRouter.reverseLedgerEntry", "note": "no assertAccountAccess call" }, + { "path": "src/http/ledger-routes.ts", "symbol": "createLedgerRouter.postLedgerEntry", "note": "the same check is present here, showing the omission is asymmetric" }, + { "path": "src/service/ledger-service.ts", "symbol": "LedgerService.reverseEntry", "note": "reads the original entry and its accountId without any principal check" }, + { "path": "src/auth/request-context.ts", "symbol": "assertAccountAccess", "note": "the check that exists and is not applied" } + ], + "corroborating_signal": { + "path": "src/service/ledger-service.ts", + "symbol": "LedgerService.findEntryForAuthorization", + "note": "This helper exists precisely to let a caller resolve the owning account before authorizing, and it has no callers." + } + }, + "critical_facts": [ + { + "id": "missing-authz-on-reversal", + "statement": "reverseLedgerEntry performs authentication but no account-level authorization.", + "criticality": "critical", + "evidence": [{ "path": "src/http/ledger-routes.ts", "symbol": "createLedgerRouter.reverseLedgerEntry" }] + }, + { + "id": "cross-tenant-impact", + "statement": "The consequence is cross-tenant write access: a reversal entry is appended to another account and its balance projection is corrupted.", + "criticality": "critical", + "evidence": [ + { "path": "src/service/ledger-service.ts", "symbol": "LedgerService.reverseEntry" }, + { "path": "src/projections/balance-projection.ts", "symbol": "BalanceProjection.applyReversed" } + ] + }, + { + "id": "check-exists-unused", + "statement": "assertAccountAccess already exists and is applied on the post route, so the fix is to apply it after resolving the entry's owning account.", + "criticality": "critical", + "evidence": [{ "path": "src/auth/request-context.ts", "symbol": "assertAccountAccess" }] + } + ], + "acceptable_additional_findings": [ + { + "id": "unused-authorization-helper", + "statement": "LedgerService.findEntryForAuthorization has no callers.", + "scored_as": "supporting, not required" + }, + { + "id": "unchecked-scopes", + "statement": "Principal.scopes is never read anywhere in the workspace, so scope-based authorization is declared but not enforced.", + "scored_as": "supporting, not required" + }, + { + "id": "generated-request-id", + "statement": "resolveRequestContext generates a random requestId when the x-request-id header is absent, which weakens audit correlation.", + "scored_as": "supporting, not required, and not a security defect on its own" + } + ], + "correct_uncertainty": [ + { + "id": "token-directory-opaque", + "requirement": "TokenDirectory is an interface with no implementation in this workspace. Claims about token strength, expiry, or revocation are unsupported." + } + ], + "unsupported_claim_traps": [ + { + "id": "post-route-also-unauthorized", + "claim": "postLedgerEntry is also missing an authorization check.", + "why_false": "It calls assertAccountAccess with the path accountId before invoking the service.", + "counts_as": "false_positive" + }, + { + "id": "no-authentication", + "claim": "The HTTP surface has no authentication at all.", + "why_false": "Both handlers call resolveRequestContext, which throws UnauthenticatedError on a missing or unknown bearer token.", + "counts_as": "false_positive" + }, + { + "id": "sql-injection", + "claim": "The handlers are vulnerable to injection through request parameters.", + "why_false": "Both stores are in-memory Maps; there is no query language anywhere in the workspace.", + "counts_as": "false_positive" + } + ], + "tier1_obligations": { + "required_evidence_paths": [ + "src/http/ledger-routes.ts", + "src/auth/request-context.ts", + "src/service/ledger-service.ts" + ], + "required_evidence_symbols": ["reverseLedgerEntry", "postLedgerEntry", "assertAccountAccess"], + "min_critical_fact_recall": 1.0, + "must_not_report_ready_when": [ + "src/http/ledger-routes.ts is absent from the evidence set", + "only one of the two route handlers is present, since the defect is only visible by comparing them" + ] + }, + "tier2_scoring": { + "method": "seeded_defect_detection", + "required_detections": ["seeded-reversal-authorization"], + "false_positive_set": ["post-route-also-unauthorized", "no-authentication", "sql-injection"], + "critical_facts_required_for_pass": ["missing-authz-on-reversal", "cross-tenant-impact"] + } +} diff --git a/docs/qualification/truth/rootcause-ledger-duplicate-entries.json b/docs/qualification/truth/rootcause-ledger-duplicate-entries.json new file mode 100644 index 00000000..8030ebfe --- /dev/null +++ b/docs/qualification/truth/rootcause-ledger-duplicate-entries.json @@ -0,0 +1,101 @@ +{ + "contract_version": "1.0.0", + "task_id": "rootcause-ledger-duplicate-entries", + "target": "qual-ledger-service", + "category": "bug-root-cause-investigation", + "provenance": { + "authored_by": "madar-655-qualification-agent", + "authored_at": "2026-08-12", + "derived_from": ["seeded defect introduced deliberately while authoring the fixture"], + "madar_derived_sources_used": [], + "inspected_madar_output_before_freeze": false, + "independent_of_production_rule_author": false, + "review_status": "unreviewed" + }, + "seeded_defect": { + "id": "seeded-idempotency-ordering", + "path": "src/service/ledger-service.ts", + "symbol": "LedgerService.postEntry", + "root_cause": "The idempotency key is reserved after the ledger append instead of before it. IdempotencyStore.find and IdempotencyStore.reserve form a check-then-act pair with a state mutation in between, so two concurrent requests carrying the same key can both observe no reservation and both call LedgerStore.appendEntry.", + "required_ordering_statement": "read reservation (find) -> append entry -> write reservation (reserve). The append must move after the reservation, or the reservation and the append must become one atomic operation.", + "also_present_in": [ + { "path": "src/service/ledger-service.ts", "symbol": "LedgerService.reverseEntry" } + ] + }, + "critical_facts": [ + { + "id": "ordering-is-the-cause", + "statement": "The cause is the ordering of reserve relative to appendEntry inside LedgerService.postEntry, not a defect in either store.", + "criticality": "critical", + "evidence": [{ "path": "src/service/ledger-service.ts", "symbol": "LedgerService.postEntry" }] + }, + { + "id": "check-then-act", + "statement": "find and reserve are separate operations with a mutation between them, so the window between them is the race window.", + "criticality": "critical", + "evidence": [ + { "path": "src/store/idempotency-store.ts", "symbol": "IdempotencyStore.find" }, + { "path": "src/store/idempotency-store.ts", "symbol": "IdempotencyStore.reserve" } + ] + }, + { + "id": "append-unconditional", + "statement": "LedgerStore.appendEntry always allocates a new entryId; it has no uniqueness constraint that could reject the second write.", + "criticality": "critical", + "evidence": [{ "path": "src/store/ledger-store.ts", "symbol": "LedgerStore.appendEntry" }] + }, + { + "id": "same-shape-in-reversal", + "statement": "reverseEntry repeats the same ordering, so the fix must cover both commands.", + "criticality": "supporting", + "evidence": [{ "path": "src/service/ledger-service.ts", "symbol": "LedgerService.reverseEntry" }] + } + ], + "correct_uncertainty": [ + { + "id": "single-thread-caveat", + "requirement": "The stores are synchronous in-memory Maps, so within one Node thread the find/append/reserve sequence cannot actually be interleaved. The ordering is still wrong, and it becomes observable as soon as either store performs real I/O or the process is replicated. A correct answer names the ordering defect AND states that the duplicate is not reproducible against the current in-memory implementation without introducing an await or a second process." + } + ], + "unsupported_claim_traps": [ + { + "id": "reserve-is-buggy", + "claim": "IdempotencyStore.reserve is the bug because it overwrites or fails to detect an existing key.", + "why_false": "reserve returns the existing record unchanged when the key is present. It is correct in isolation." + }, + { + "id": "sequence-collision", + "claim": "The duplicate comes from the entryId sequence counter colliding.", + "why_false": "The counter is incremented before each id is built and never reused; the duplicates have distinct entryIds." + }, + { + "id": "missing-validation", + "claim": "The route fails to validate idempotencyKey.", + "why_false": "requireString rejects a missing or empty idempotencyKey before the service runs." + }, + { + "id": "outbox-replay", + "claim": "The outbox republishes the event and the consumer creates a second entry.", + "why_false": "BalanceProjection never writes to LedgerStore; consumers cannot create entries." + } + ], + "tier1_obligations": { + "required_evidence_paths": [ + "src/service/ledger-service.ts", + "src/store/idempotency-store.ts", + "src/store/ledger-store.ts" + ], + "required_evidence_symbols": ["LedgerService", "IdempotencyStore", "LedgerStore"], + "min_critical_fact_recall": 1.0, + "must_not_report_ready_when": [ + "src/service/ledger-service.ts is absent from the evidence set", + "the call from LedgerService.postEntry to IdempotencyStore.reserve is missing from the graph" + ] + }, + "tier2_scoring": { + "method": "single_root_cause_adjudication", + "accepted_root_cause_ids": ["seeded-idempotency-ordering"], + "critical_facts_required_for_pass": ["ordering-is-the-cause", "check-then-act"], + "uncertainty_required_for_pass": ["single-thread-caveat"] + } +} diff --git a/docs/qualification/validity-rules.md b/docs/qualification/validity-rules.md new file mode 100644 index 00000000..31a61cdd --- /dev/null +++ b/docs/qualification/validity-rules.md @@ -0,0 +1,95 @@ +# Run validity and invalidation rules + +Contract version `1.0.0`, frozen 2026-08-12 for [#655](https://github.com/mohanagy/madar/issues/655). + +A qualification run is either **valid**, **degraded**, or **invalid**. Only a valid run +carries a result. An invalid run is not a loss and not a win — it is a run that did not +happen in a measurable way. + +## Invalidation conditions + +Any one of these sets `validity.status` to `invalid` and adds the matching reason code +to `validity.invalidation_reasons`: + +| Reason code | Condition | +| --- | --- | +| `missing_attributable_madar_call` | The task contract sets `requires_attributable_madar_call` and the transcript shows no attributable Madar call in the Madar arm. | +| `prompt_contract_failure` | The prompt actually delivered to the agent does not hash-match the frozen prompt, or the two arms received different prompts. | +| `answer_contract_failure` | The arm produced no answer, a permission request instead of an answer, or a truncated answer. | +| `target_revision_mismatch` | The checked-out target revision or fixture digest differs from `corpus.json`. | +| `package_revision_mismatch` | The Madar commit, package version, or tarball digest differs from the pinned identity. | +| `dependency_lock_mismatch` | The dependency lock digest differs from the pinned identity, or the install used `npm install` rather than `npm ci`. | +| `isolation_failure` | `environment.isolation` is false, or the run used a `MADAR_BENCH_CLI_PATH`-style development override. | +| `incomplete_transcript` | The transcript is missing, truncated, or cannot attribute tool calls. | +| `incomplete_receipt` | Any required field in [`receipt-schema.json`](./receipt-schema.json) is absent. | +| `judge_failure` | A deterministic grader errored, or a blinded reviewer could not score the answer. | +| `environment_mismatch` | `environment.drift.detected` is true and the drift was not resolved before the cell ran. | +| `quality_gate_failure` | A gate failed in a way that prevents comparison at all — not a gate the arm simply lost. | +| `truth_unavailable` | The target/task pair has no independent truth (`status: "pinned_no_truth"`). | +| `blinding_unavailable` | A Tier 2 quality dimension was scored without an independent blinded reviewer. | + +`degraded` is reserved for runs that are attributable and complete but weaker than the +contract intends — for example a Madar arm whose first attributable call came only after +broad exploration (`adoption.status: "late"`). A degraded run may be inspected and +discussed; it may not be aggregated. + +## The `not_measured` rule + +1. `validity.aggregatable` **must** be `false` whenever `validity.status` is not `valid`. + The receipt schema enforces this. +2. Every unmeasured score carries `measured: false`, `value: null`, and a + `not_measured_reason`. A score of `0` and a score of `not_measured` are different + things and must never be interchanged. +3. `not_measured` describes a run that could not be measured. A run that **was** measured + and failed is a failure. Relabelling a failure as `not_measured` is a contract + violation, not a reporting choice. +4. Invalid rows stay visible. Every published table prints `n_invalid` beside `n_valid`. + A table that shows only valid rows is not a permitted summary of this corpus. +5. Cost, token, and latency figures from an invalid run may be retained for diagnosis and + must never be cited as a cost or efficiency result. + +## Gate ordering + +Correctness and critical-fact completeness are evaluated before any token, latency, or +cost column is populated. A cost improvement on a cell that failed a quality gate is not +reported as an improvement in any form. + +## Cost separation + +`costs.indexing`, `costs.context_build`, and `costs.agent` are three separate accounts. +They are never summed into a single number, and an unmeasured account is +`measured: false`, never `0`. + +## Retention + +For every run, whether valid or not, the following are retained alongside the receipt for +at least **24 months**: + +- the raw agent transcript (Tier 2) or the context artifact (Tier 1); +- the answer text of both arms (Tier 2); +- the exact prompt text delivered to each arm; +- the environment receipt; +- the truth file version used for scoring. + +Each retained artifact is recorded in `retention` with its path and SHA-256. A run whose +artifacts were not retained is `incomplete_receipt`. + +## What today's emitter actually produces + +This schema is a contract, not a description of `v0.32.1` behaviour. Mapping against +`NativeAgentCompareReport` in `src/infrastructure/compare.ts` at the pinned commit: + +| Schema area | Status at `06b373a4` | +| --- | --- | +| `validity.status` | Partially present as `measurement_validity` (`valid` / `degraded` / `invalid`). | +| `validity.invalidation_reasons` | **Not emitted.** Reasons exist only as prose in `benchmark_outcome.evidence`. | +| `validity.aggregatable` | **Not emitted.** | +| `adoption.*` | Partially present as `madar_mcp_call_count` and `trace_status`; there is no `adopted`/`late`/`absent` classification field and no post-first-call broad-exploration counter. | +| `costs.agent` | Present, spread across `reductions`, `prompt_token_source`, and `provider_proof`. | +| `costs.indexing`, `costs.context_build` | **Not emitted.** There is no separate indexing or context-build cost account anywhere in the report. | +| `scores.*` | **Not emitted** in this shape. `answer_quality` carries term-presence checks and a human-review status only. | +| `identity.*` | Partially present via `environment`, `exec_command`, and the isolation launcher; there is no single identity block and no dependency-lock digest. | +| `retention.*` | Paths are emitted in `paths`; digests and a retention policy are **not**. | + +Closing that gap is emitter work and is deliberately out of scope for #655, which must not +modify production or reporting logic. It is a separate linked issue. diff --git a/package.json b/package.json index ef632477..7a048807 100644 --- a/package.json +++ b/package.json @@ -66,7 +66,8 @@ "publish:dry-run": "npm publish --dry-run", "release:verify": "node .github/scripts/verify-release-hygiene.mjs", "verify:pack-parity": "node .github/scripts/verify-packed-retrieval-parity.mjs", - "registry:validate": "node .github/scripts/validate-mcp-registry.mjs" + "registry:validate": "node .github/scripts/validate-mcp-registry.mjs", + "qualify:validate": "node .github/scripts/validate-qualification-contract.mjs" }, "devDependencies": { "@types/node": "^26.0.0", diff --git a/tests/unit/qualification-contract.test.ts b/tests/unit/qualification-contract.test.ts new file mode 100644 index 00000000..3304042c --- /dev/null +++ b/tests/unit/qualification-contract.test.ts @@ -0,0 +1,304 @@ +import { createHash } from 'node:crypto' +import { readFileSync } from 'node:fs' +import { resolve } from 'node:path' + +import { createRequire } from 'node:module' + +import { Ajv } from 'ajv' +import { describe, expect, it } from 'vitest' + +// ajv-formats ships CommonJS whose default export is not callable under +// NodeNext type resolution. Load it the same way the CI validator does so the +// test and `npm run qualify:validate` compile the schema identically. +const addFormats = createRequire(import.meta.url)('ajv-formats') as (ajv: Ajv) => void + +const ROOT = 'docs/qualification' + +function readDoc(relativePath: string): string { + return readFileSync(resolve(relativePath), 'utf8') +} + +function readJson(relativePath: string): T { + return JSON.parse(readDoc(relativePath)) as T +} + +interface Provenance { + authored_by: string + authored_at: string + madar_derived_sources_used: string[] + inspected_madar_output_before_freeze: boolean + independent_of_production_rule_author: boolean +} + +interface Task { + id: string + category: string + target: string + tiers: number[] + prompt: { text: string; sha256: string } + truth_ref: string + scoring: { tier1_method: string; tier2_method: string } + truth_provenance: Provenance +} + +const corpus = readJson<{ + contract_version: string + targets: Array<{ id: string; tier: number; kind: string; status: string; holdout_class?: string; source?: { ref: string } }> +}>(`${ROOT}/corpus.json`) + +const tasks = readJson<{ contract_version: string; tasks: Task[] }>(`${ROOT}/tasks.json`) +const rubrics = readJson<{ + dimensions: Record + methods: Record + blinding: { current_status: string } +}>(`${ROOT}/rubrics.json`) +const tier1 = readJson<{ + properties: { deterministic: boolean; requires_network: boolean; requires_api_spend: boolean } + cells: Array<{ task_id: string; target_id: string }> + negative_trust_probes: Array<{ id: string; prompt: { text: string; sha256: string } }> + gate: { forbidden_remedies: string[] } + calibration_status: { state: string } +}>(`${ROOT}/tier1.json`) +const tier2 = readJson<{ status: string; dimensions: { trials_per_cell: number } }>(`${ROOT}/tier2-matrix.json`) +const receiptSchema = readJson>(`${ROOT}/receipt-schema.json`) +const freeze = readJson<{ contract_version: string; files: Record }>(`${ROOT}/freeze.json`) + +describe('qualification corpus manifest', () => { + it('pins every target with an immutable revision or a frozen digest', () => { + for (const target of corpus.targets) { + if (target.kind === 'git') { + expect(target.source?.ref).toMatch(/^[0-9a-f]{40}$/) + } + if (target.kind === 'fixture') { + expect(target.status).toBe('frozen') + } + } + }) + + it('keeps Tier 2 git targets marked as having no independent truth yet', () => { + const gitTargets = corpus.targets.filter((target) => target.kind === 'git') + + expect(gitTargets.length).toBeGreaterThan(0) + for (const target of gitTargets) { + expect(target.status).toBe('pinned_no_truth') + } + }) + + it('keeps the sealed holdout slot visible and explicitly unsatisfied', () => { + const sealed = corpus.targets.filter((target) => target.holdout_class === 'sealed') + + expect(sealed).toHaveLength(1) + expect(sealed[0]?.status).toBe('unsatisfied') + }) +}) + +describe('qualification task definitions', () => { + it('covers every task category named in the issue contract', () => { + const categories = new Set(tasks.tasks.map((task) => task.category)) + + expect([...categories].sort()).toEqual([ + 'architecture-understanding', + 'bug-root-cause-investigation', + 'execution-flow-explanation', + 'impact-analysis', + 'implementation-planning', + 'review-security', + ]) + }) + + it('freezes each prompt against its recorded hash', () => { + for (const task of tasks.tasks) { + expect(createHash('sha256').update(task.prompt.text, 'utf8').digest('hex')).toBe(task.prompt.sha256) + } + }) + + it('records a truth owner and asserts no Madar-derived source for every task', () => { + for (const task of tasks.tasks) { + const truth = readJson<{ provenance: Provenance }>(`${ROOT}/${task.truth_ref}`) + + for (const provenance of [task.truth_provenance, truth.provenance]) { + expect(provenance.authored_by.length).toBeGreaterThan(0) + expect(provenance.authored_at.length).toBeGreaterThan(0) + expect(provenance.madar_derived_sources_used).toEqual([]) + expect(provenance.inspected_madar_output_before_freeze).toBe(false) + expect(provenance.independent_of_production_rule_author).toBe(false) + } + } + }) + + it('does not use the same scoring method for every category', () => { + const methods = new Set(tasks.tasks.map((task) => task.scoring.tier2_method)) + + expect(methods.size).toBeGreaterThan(1) + for (const task of tasks.tasks) { + expect(rubrics.methods[task.scoring.tier2_method]).toBeTruthy() + expect(rubrics.methods[task.scoring.tier1_method]).toBeTruthy() + } + }) +}) + +describe('qualification rubrics', () => { + it('measures adoption and fallback exploration separately from context quality', () => { + expect(rubrics.dimensions.intended_tool_adoption?.gating).toBe(false) + expect(rubrics.dimensions.broad_fallback_exploration?.gating).toBe(false) + expect(rubrics.dimensions.correctness?.gating).toBe(true) + expect(rubrics.dimensions.critical_fact_completeness?.gating).toBe(true) + }) + + it('declares blinded review unavailable rather than assuming it', () => { + expect(rubrics.blinding.current_status).toBe('unsatisfied') + }) +}) + +describe('qualification receipt schema', () => { + const ajv = new Ajv({ allErrors: true, strict: false }) + addFormats(ajv) + const validate = ajv.compile(receiptSchema) + + const validTier1 = readJson>(`${ROOT}/examples/receipt-tier1-valid.json`) + const invalidTier2 = readJson>(`${ROOT}/examples/receipt-tier2-invalid-no-madar-call.json`) + + it('accepts the published examples', () => { + expect(validate(validTier1)).toBe(true) + expect(validate(invalidTier2)).toBe(true) + }) + + it('keeps every quality dimension not_measured on an invalid run', () => { + const scores = (invalidTier2 as { scores: Record }).scores + const validity = (invalidTier2 as { validity: { status: string; aggregatable: boolean } }).validity + + expect(validity.status).toBe('invalid') + expect(validity.aggregatable).toBe(false) + for (const score of Object.values(scores)) { + expect(score.measured).toBe(false) + expect(score.value).toBeNull() + } + }) + + it('rejects an invalid run that claims to be aggregatable', () => { + const mutated = JSON.parse(JSON.stringify(invalidTier2)) as { validity: { aggregatable: boolean } } + mutated.validity.aggregatable = true + + expect(validate(mutated)).toBe(false) + }) + + it('rejects an unmeasured score that carries a value', () => { + const mutated = JSON.parse(JSON.stringify(invalidTier2)) as { + scores: { correctness: { measured: boolean; value: unknown } } + } + mutated.scores.correctness.value = 2 + + expect(validate(mutated)).toBe(false) + }) + + it('keeps indexing, context building, and agent cost in separate accounts', () => { + const costs = (validTier1 as { costs: Record }).costs + + expect(Object.keys(costs).sort()).toEqual(['agent', 'context_build', 'indexing']) + expect(costs.agent?.measured).toBe(false) + }) +}) + +describe('qualification Tier 1 subset', () => { + it('is deterministic and runnable in a pull request without spend', () => { + expect(tier1.properties.deterministic).toBe(true) + expect(tier1.properties.requires_network).toBe(false) + expect(tier1.properties.requires_api_spend).toBe(false) + }) + + it('covers every frozen task and freezes each negative-trust probe prompt', () => { + expect(tier1.cells.map((cell) => cell.task_id).sort()).toEqual(tasks.tasks.map((task) => task.id).sort()) + + expect(tier1.negative_trust_probes.length).toBeGreaterThan(0) + for (const probe of tier1.negative_trust_probes) { + expect(createHash('sha256').update(probe.prompt.text, 'utf8').digest('hex')).toBe(probe.prompt.sha256) + } + }) + + it('forbids clearing a failure by editing the contract or the production rules', () => { + const remedies = tier1.gate.forbidden_remedies.join('\n') + + expect(remedies).toContain('Adding a qualification path, symbol, prompt, or repository name to production') + expect(remedies).toContain('Relaxing a truth file to match observed output') + expect(remedies).toContain('Marking a failing cell not_measured') + }) + + it('states that the thresholds are pre-registered and uncalibrated', () => { + expect(tier1.calibration_status.state).toBe('pre_registered_uncalibrated') + }) + + it('keeps the Tier 2 matrix planned with a repeated-run count', () => { + expect(tier2.status).toBe('planned') + expect(tier2.dimensions.trials_per_cell).toBeGreaterThan(1) + }) +}) + +describe('qualification policy documents', () => { + it('states an objective stop rule with a pre-registered non-inferiority margin', () => { + const stopRule = readDoc(`${ROOT}/stop-rule.md`) + + for (const id of ['S1.1', 'S1.2', 'S1.3', 'S1.4', 'S1.5', 'S1.6', 'S1.7', 'S1.8']) { + expect(stopRule).toContain(id) + } + expect(stopRule).toContain('non-inferiority margin **0.05**') + expect(stopRule).toContain('Do not fix forward on the protected branch while a stop condition is tripped.') + expect(stopRule).toContain('Marking a measured failure as `not_measured`.') + }) + + it('declares the sealed holdout unsatisfied and names the human action required', () => { + const policy = readDoc(`${ROOT}/holdout-policy.md`) + + expect(policy).toContain('## Current status: unsatisfied') + expect(policy).toContain('### Human action required') + expect(policy).toContain('sealed holdout unsatisfied; results measure regression only') + }) + + it('labels synthetic and package-parity artifacts as non-outcome evidence', () => { + const categories = readDoc(`${ROOT}/evidence-categories.md`) + + expect(categories).toContain('### E1 — Product outcome evidence') + expect(categories).toContain('**Currently held: none.**') + expect(categories).toContain('### E4 — Synthetic or fixture receipts') + expect(categories).toContain('### E5 — Package and parity checks') + expect(categories).toContain('E4 proves the reporting pipeline works. It is never agent-outcome evidence.') + }) + + it('defines transcript and receipt retention', () => { + const rules = readDoc(`${ROOT}/validity-rules.md`) + + expect(rules).toContain('at least **24 months**') + expect(rules).toContain('the raw agent transcript (Tier 2) or the context artifact (Tier 1)') + expect(rules).toContain('`not_measured` describes a run that could not be measured') + }) + + it('records which receipt fields v0.32.1 does not emit yet', () => { + const rules = readDoc(`${ROOT}/validity-rules.md`) + + expect(rules).toContain('## What today\'s emitter actually produces') + expect(rules).toContain('There is no separate indexing or context-build cost account') + }) +}) + +describe('qualification freeze', () => { + it('covers every contract and fixture file with a digest', () => { + expect(freeze.contract_version).toBe(corpus.contract_version) + + const paths = Object.keys(freeze.files) + expect(paths).toContain(`${ROOT}/corpus.json`) + expect(paths).toContain(`${ROOT}/tasks.json`) + expect(paths).toContain(`${ROOT}/rubrics.json`) + expect(paths).toContain(`${ROOT}/receipt-schema.json`) + expect(paths).toContain(`${ROOT}/fixtures/ledger-service/src/service/ledger-service.ts`) + expect(paths).toContain(`${ROOT}/fixtures/plugin-host/src/host/plugin-host.ts`) + + for (const [path, digest] of Object.entries(freeze.files)) { + expect(digest).toBe(createHash('sha256').update(readFileSync(resolve(path))).digest('hex')) + } + }) + + it('is wired into an npm script so a clean checkout can verify it', () => { + const pkg = readJson<{ scripts: Record }>('package.json') + + expect(pkg.scripts['qualify:validate']).toBe('node .github/scripts/validate-qualification-contract.mjs') + }) +}) From 527edb2193062de3efa3da4c877c427ba005547f Mon Sep 17 00:00:00 2001 From: mohammed naji Date: Wed, 12 Aug 2026 10:11:19 +0400 Subject: [PATCH 02/10] refactor(qualification): rebuild the corpus on natural pinned repositories (#655) The first pass built two self-authored TypeScript fixture workspaces as qualification targets. That satisfied independence from Madar output but forfeited naturalness, reproducing in a new namespace the defect #655 exists to cure: a corpus of self-made proxies cannot detect production behaviour drifting toward benchmark-shaped repositories, because the proxies are shaped by the same hands as the production rules. Replace both fixtures with real, externally authored, MIT-licensed TypeScript projects pinned at immutable commits: - honojs/hono @ 26de73133b8552f56ba72e025ecd82b08900d796 - unjs/unstorage @ e6be6135832f350ca16f9a77432e1d4f0aa85ed7 Seeded defects are now patches applied to the pinned commit rather than synthetic workspaces authored around their own answer: - patches/hono-compose-reentrancy-guard.patch weakens the middleware re-entrancy guard for the root-cause task. - patches/hono-error-message-disclosure.patch makes the default error handler return the thrown error's stack for the review task. All six truth files are rewritten from the pinned sources, read directly and without ever running Madar. Each target records a cited_blobs map of path to git blob SHA so the validator can check every truth citation offline; `--verify-corpus` confirms the commits, all 25 blob digests, and both patches against the live repositories. corpus.json also discloses that Madar ships a generic Hono adapter (src/pipeline/spi/framework-hono.ts plus Hono-aware classification in src/runtime/retrieve.ts). That is declared framework support, not a repository-specific special case, but it means a hono result partly measures that adapter. The hono prompts therefore never name the framework, and unstorage carries production_coupling: none_found as the uncoupled contrast. evidence-categories.md separates target naturalness from evidence class and records two findings: the existing suite corpus is five in-repo proxies plus six SHA-pinned git rows, and the retrieval/grader isolation claimed for runtime-proof.json is asserted in prose with no test or CI check enforcing it. No production retrieval, ranking, context, or reporting code is touched. --- .../validate-qualification-contract.mjs | 195 +++++++++++++---- docs/qualification/README.md | 91 +++++--- docs/qualification/corpus.json | 181 ++++++++++++---- docs/qualification/evidence-categories.md | 40 ++++ .../examples/receipt-tier1-valid.json | 80 +++++-- .../receipt-tier2-invalid-no-madar-call.json | 95 ++++++-- .../fixtures/ledger-service/README.md | 39 ---- .../fixtures/ledger-service/package.json | 10 - .../ledger-service/src/audit/audit-log.ts | 22 -- .../src/auth/request-context.ts | 52 ----- .../ledger-service/src/http/ledger-routes.ts | 79 ------- .../src/outbox/outbox-publisher.ts | 34 --- .../src/projections/balance-projection.ts | 45 ---- .../src/service/ledger-service.ts | 120 ----------- .../src/store/idempotency-store.ts | 34 --- .../ledger-service/src/store/ledger-store.ts | 46 ---- .../fixtures/ledger-service/tsconfig.json | 11 - .../fixtures/plugin-host/README.md | 35 --- .../fixtures/plugin-host/package.json | 10 - .../plugin-host/src/contracts/plugin.ts | 45 ---- .../fixtures/plugin-host/src/host/config.ts | 32 --- .../plugin-host/src/host/lifecycle.ts | 62 ------ .../plugin-host/src/host/plugin-host.ts | 56 ----- .../fixtures/plugin-host/src/host/registry.ts | 46 ---- .../src/plugins/csv-export-plugin.ts | 32 --- .../src/plugins/webhook-export-plugin.ts | 36 ---- .../fixtures/plugin-host/tsconfig.json | 12 -- docs/qualification/freeze.json | 57 ++--- docs/qualification/holdout-policy.md | 10 +- .../hono-compose-reentrancy-guard.patch | 13 ++ .../hono-error-message-disclosure.patch | 13 ++ docs/qualification/receipt-schema.json | 1 + docs/qualification/tasks.json | 84 ++++---- docs/qualification/tier1.json | 68 +++--- docs/qualification/tier2-matrix.json | 27 ++- .../arch-plugin-host-extension-seam.json | 125 ----------- .../truth/arch-unstorage-driver-seam.json | 143 +++++++++++++ .../truth/flow-hono-request-dispatch.json | 178 +++++++++++++++ .../truth/flow-ledger-post-entry.json | 174 --------------- .../impact-hono-drop-router-fallback.json | 140 ++++++++++++ .../impact-ledger-drop-outbox-publish.json | 111 ---------- ...lan-plugin-host-object-storage-plugin.json | 129 ----------- .../truth/plan-unstorage-add-driver.json | 146 +++++++++++++ .../truth/review-hono-error-handling.json | 118 ++++++++++ .../review-ledger-http-authorization.json | 118 ---------- .../rootcause-hono-middleware-rerun.json | 99 +++++++++ .../rootcause-ledger-duplicate-entries.json | 101 --------- docs/qualification/validity-rules.md | 3 +- tests/unit/qualification-contract.test.ts | 202 +++++++++++++++--- 49 files changed, 1686 insertions(+), 1914 deletions(-) delete mode 100644 docs/qualification/fixtures/ledger-service/README.md delete mode 100644 docs/qualification/fixtures/ledger-service/package.json delete mode 100644 docs/qualification/fixtures/ledger-service/src/audit/audit-log.ts delete mode 100644 docs/qualification/fixtures/ledger-service/src/auth/request-context.ts delete mode 100644 docs/qualification/fixtures/ledger-service/src/http/ledger-routes.ts delete mode 100644 docs/qualification/fixtures/ledger-service/src/outbox/outbox-publisher.ts delete mode 100644 docs/qualification/fixtures/ledger-service/src/projections/balance-projection.ts delete mode 100644 docs/qualification/fixtures/ledger-service/src/service/ledger-service.ts delete mode 100644 docs/qualification/fixtures/ledger-service/src/store/idempotency-store.ts delete mode 100644 docs/qualification/fixtures/ledger-service/src/store/ledger-store.ts delete mode 100644 docs/qualification/fixtures/ledger-service/tsconfig.json delete mode 100644 docs/qualification/fixtures/plugin-host/README.md delete mode 100644 docs/qualification/fixtures/plugin-host/package.json delete mode 100644 docs/qualification/fixtures/plugin-host/src/contracts/plugin.ts delete mode 100644 docs/qualification/fixtures/plugin-host/src/host/config.ts delete mode 100644 docs/qualification/fixtures/plugin-host/src/host/lifecycle.ts delete mode 100644 docs/qualification/fixtures/plugin-host/src/host/plugin-host.ts delete mode 100644 docs/qualification/fixtures/plugin-host/src/host/registry.ts delete mode 100644 docs/qualification/fixtures/plugin-host/src/plugins/csv-export-plugin.ts delete mode 100644 docs/qualification/fixtures/plugin-host/src/plugins/webhook-export-plugin.ts delete mode 100644 docs/qualification/fixtures/plugin-host/tsconfig.json create mode 100644 docs/qualification/patches/hono-compose-reentrancy-guard.patch create mode 100644 docs/qualification/patches/hono-error-message-disclosure.patch delete mode 100644 docs/qualification/truth/arch-plugin-host-extension-seam.json create mode 100644 docs/qualification/truth/arch-unstorage-driver-seam.json create mode 100644 docs/qualification/truth/flow-hono-request-dispatch.json delete mode 100644 docs/qualification/truth/flow-ledger-post-entry.json create mode 100644 docs/qualification/truth/impact-hono-drop-router-fallback.json delete mode 100644 docs/qualification/truth/impact-ledger-drop-outbox-publish.json delete mode 100644 docs/qualification/truth/plan-plugin-host-object-storage-plugin.json create mode 100644 docs/qualification/truth/plan-unstorage-add-driver.json create mode 100644 docs/qualification/truth/review-hono-error-handling.json delete mode 100644 docs/qualification/truth/review-ledger-http-authorization.json create mode 100644 docs/qualification/truth/rootcause-hono-middleware-rerun.json delete mode 100644 docs/qualification/truth/rootcause-ledger-duplicate-entries.json diff --git a/.github/scripts/validate-qualification-contract.mjs b/.github/scripts/validate-qualification-contract.mjs index 0be3179b..ea8f43d8 100644 --- a/.github/scripts/validate-qualification-contract.mjs +++ b/.github/scripts/validate-qualification-contract.mjs @@ -1,5 +1,7 @@ +import { execFileSync } from 'node:child_process' import { createHash } from 'node:crypto' -import { readdirSync, readFileSync, statSync, writeFileSync } from 'node:fs' +import { mkdtempSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' import { join, relative, resolve } from 'node:path' import Ajv from 'ajv' @@ -9,6 +11,7 @@ const ROOT = resolve('docs/qualification') const FREEZE_PATH = join(ROOT, 'freeze.json') const PRODUCTION_ROOT = resolve('src') const WRITE = process.argv.includes('--write') +const VERIFY_CORPUS = process.argv.includes('--verify-corpus') const failures = [] @@ -62,28 +65,80 @@ for (const [name, doc] of [ } // --------------------------------------------------------------------------- -// 2. Targets +// 2. Targets must be natural and pinned // --------------------------------------------------------------------------- const targetsById = new Map(corpus.targets.map((target) => [target.id, target])) -const fixtureTargets = corpus.targets.filter((target) => target.kind === 'fixture') -for (const target of fixtureTargets) { - try { - if (!statSync(resolve(target.path)).isDirectory()) { - fail(`target ${target.id} path ${target.path} is not a directory`) - } - } catch { - fail(`target ${target.id} path ${target.path} does not exist`) - } +if (!Array.isArray(corpus.proxy_targets)) { + fail('corpus.json must declare a proxy_targets list, even when empty') } for (const target of corpus.targets) { - if (target.kind === 'git' && !/^[0-9a-f]{40}$/.test(target.source?.ref ?? '')) { - fail(`git target ${target.id} must pin an immutable 40-character commit SHA`) + if (target.kind === 'sealed') { + if (target.status !== 'unsatisfied') { + fail(`sealed target ${target.id} must stay unsatisfied until a second person fills it`) + } + continue + } + + if (target.natural !== true) { + fail(`target ${target.id} is not marked natural; a fixture proxy must be declared in proxy_targets, not in targets`) + } + if (!/^[0-9a-f]{40}$/.test(target.source?.ref ?? '')) { + fail(`target ${target.id} must pin an immutable 40-character commit SHA`) + } + if (!target.source?.url?.startsWith('https://')) { + fail(`target ${target.id} must record an https repository URL`) + } + if (!target.license) { + fail(`target ${target.id} must record a license`) + } + if (!Array.isArray(target.prepare) || target.prepare.length === 0) { + fail(`target ${target.id} must record reproducible prepare steps`) + } + if (!target.dependency_lock) { + fail(`target ${target.id} must record a dependency lock policy`) + } + if (!target.cited_blobs || Object.keys(target.cited_blobs).length === 0) { + fail(`target ${target.id} must record cited_blobs so truth citations can be checked offline`) } - if (target.status === 'pinned_no_truth' && target.tier === 1) { - fail(`target ${target.id} is Tier 1 but has no independent truth`) + for (const [path, blob] of Object.entries(target.cited_blobs ?? {})) { + if (!/^[0-9a-f]{40}$/.test(blob)) { + fail(`target ${target.id} cited_blobs["${path}"] is not a git blob SHA`) + } + } + + if (target.kind === 'git_patched') { + const base = targetsById.get(target.base_target) + if (!base) { + fail(`patched target ${target.id} references unknown base_target ${target.base_target}`) + } else if (base.source.ref !== target.source.ref) { + fail(`patched target ${target.id} must pin the same commit as its base target`) + } + + const patchPath = join(ROOT, target.patch ?? '') + let patch + try { + patch = readFileSync(patchPath, 'utf8') + } catch { + fail(`patched target ${target.id} references missing patch ${target.patch}`) + } + + if (patch) { + if (!patch.startsWith('diff --git ')) { + fail(`patch ${target.patch} is not a unified git diff`) + } + const touched = [...patch.matchAll(/^\+\+\+ b\/(.+)$/gm)].map((match) => match[1]) + if (touched.length === 0) { + fail(`patch ${target.patch} does not modify any file`) + } + for (const path of touched) { + if (!(path in (target.cited_blobs ?? {}))) { + fail(`patch ${target.patch} touches ${path}, which is not recorded in ${target.id} cited_blobs`) + } + } + } } } @@ -143,12 +198,16 @@ for (const task of tasks.tasks) { if (!provenance.authored_by || !provenance.authored_at) { fail(`task ${task.id} truth provenance must record who authored the truth and when`) } + if (!Array.isArray(provenance.derived_from) || provenance.derived_from.length === 0) { + fail(`task ${task.id} truth provenance must record what the truth was derived from`) + } if (!('independent_of_production_rule_author' in provenance)) { fail(`task ${task.id} truth provenance must state whether the author is independent of the production-rule author`) } } - // Every cited evidence path must exist inside the target workspace. + // Every cited evidence path must be recorded in the target's frozen blob map. + // `new_path` is used for files a plan proposes creating and is intentionally exempt. const citedPaths = new Set() const collect = (node) => { if (Array.isArray(node)) { @@ -158,18 +217,15 @@ for (const task of tasks.tasks) { if (node && typeof node === 'object') { for (const [key, value] of Object.entries(node)) { if (key === 'path' && typeof value === 'string') citedPaths.add(value) - else collect(value) + else if (key !== 'new_path') collect(value) } } } collect(truth) for (const cited of citedPaths) { - const full = resolve(target.path, cited) - try { - statSync(full) - } catch { - fail(`${task.truth_ref} cites ${cited}, which does not exist in target ${target.id}`) + if (!(cited in (target.cited_blobs ?? {}))) { + fail(`${task.truth_ref} cites ${cited}, which is not recorded in target ${target.id} cited_blobs`) } } @@ -180,9 +236,8 @@ for (const task of tasks.tasks) { fail(`${task.truth_ref} must declare at least one must_not_report_ready_when condition`) } - const rubricMethod = task.scoring.tier2_method - if (!rubrics.methods[rubricMethod]) { - fail(`task ${task.id} references unknown rubric method ${rubricMethod}`) + if (!rubrics.methods[task.scoring.tier2_method]) { + fail(`task ${task.id} references unknown rubric method ${task.scoring.tier2_method}`) } if (!rubrics.methods[task.scoring.tier1_method]) { fail(`task ${task.id} references unknown tier1 method ${task.scoring.tier1_method}`) @@ -245,41 +300,45 @@ const ajv = new Ajv({ allErrors: true, strict: false }) addFormats(ajv) const validateReceipt = ajv.compile(receiptSchema) -const examplesDir = join(ROOT, 'examples') -for (const path of walk(examplesDir)) { +for (const path of walk(join(ROOT, 'examples'))) { const receipt = readJson(path) + const label = relative(process.cwd(), path) + if (!validateReceipt(receipt)) { - fail(`${relative(process.cwd(), path)} does not satisfy receipt-schema.json: ${ajv.errorsText(validateReceipt.errors)}`) + fail(`${label} does not satisfy receipt-schema.json: ${ajv.errorsText(validateReceipt.errors)}`) } if (receipt.validity.status !== 'valid' && receipt.validity.aggregatable !== false) { - fail(`${relative(process.cwd(), path)} is not valid but is marked aggregatable`) + fail(`${label} is not valid but is marked aggregatable`) } for (const [name, score] of Object.entries(receipt.scores)) { if (score.measured === false && score.value !== null) { - fail(`${relative(process.cwd(), path)} score ${name} is not measured but carries a value`) + fail(`${label} score ${name} is not measured but carries a value`) } } + + const task = tasksById.get(receipt.task_id) + if (!task) { + fail(`${label} references unknown task ${receipt.task_id}`) + } else if (receipt.identity.prompts.user_prompt_sha256 !== task.prompt.sha256) { + fail(`${label} records a prompt hash that does not match the frozen prompt for ${receipt.task_id}`) + } } // --------------------------------------------------------------------------- // 7. Benchmark independence: no qualification literal may reach production code // --------------------------------------------------------------------------- +// Bare target ids are deliberately NOT forbidden. A target id may legitimately equal the +// name of a framework Madar declares generic support for — `hono` is one — and banning the +// word would confuse a declared adapter with a benchmark-specific special case. Those +// couplings are disclosed per target in corpus.json#/targets/*/production_coupling instead. +// What is forbidden here is every literal that could only have come from this contract. const FORBIDDEN_LITERALS = [ - ...corpus.targets.map((target) => target.id), - ...fixtureTargets.map((target) => target.path), + ...corpus.targets.flatMap((target) => (target.source?.url ? [target.source.url, target.source.ref] : [])), ...tasks.tasks.map((task) => task.id), ...tasks.tasks.map((task) => task.prompt.text), ...tier1.negative_trust_probes.map((probe) => probe.prompt.text), - 'LedgerService', - 'IdempotencyStore', - 'OutboxPublisher', - 'BalanceProjection', - 'assertAccountAccess', - 'CsvExportPlugin', - 'WebhookExportPlugin', - 'runExportLifecycle', - 'builtInPlugins', + ...Object.values(corpus.forbidden_target_symbols ?? {}).flat(), ] for (const path of walk(PRODUCTION_ROOT)) { @@ -292,7 +351,51 @@ for (const path of walk(PRODUCTION_ROOT)) { } // --------------------------------------------------------------------------- -// 8. Freeze digests +// 8. Optional network verification of the pinned corpus +// --------------------------------------------------------------------------- + +if (VERIFY_CORPUS) { + for (const target of corpus.targets) { + if (target.kind === 'sealed') { + continue + } + + const dir = mkdtempSync(join(tmpdir(), `qualify-${target.id}-`)) + try { + const git = (...args) => execFileSync('git', ['-C', dir, ...args], { encoding: 'utf8' }).trim() + + execFileSync('git', ['init', '--quiet', dir], { stdio: 'ignore' }) + git('remote', 'add', 'origin', target.source.url) + git('fetch', '--quiet', '--depth', '1', 'origin', target.source.ref) + git('checkout', '--quiet', 'FETCH_HEAD') + + const head = git('rev-parse', 'HEAD') + if (head !== target.source.ref) { + fail(`corpus verification: ${target.id} resolved to ${head}, expected ${target.source.ref}`) + } + + for (const [path, blob] of Object.entries(target.cited_blobs)) { + const actual = git('rev-parse', `HEAD:${path}`) + if (actual !== blob) { + fail(`corpus verification: ${target.id} ${path} blob is ${actual}, expected ${blob}`) + } + } + + if (target.kind === 'git_patched') { + execFileSync('git', ['-C', dir, 'apply', '--check', join(ROOT, target.patch)], { stdio: 'pipe' }) + } + + console.log(`corpus verification: ${target.id} ok`) + } catch (error) { + fail(`corpus verification failed for ${target.id}: ${error instanceof Error ? error.message : String(error)}`) + } finally { + rmSync(dir, { recursive: true, force: true }) + } + } +} + +// --------------------------------------------------------------------------- +// 9. Freeze digests // --------------------------------------------------------------------------- const frozenFiles = walk(ROOT) @@ -351,9 +454,11 @@ if (failures.length > 0) { process.exit(1) } +const naturalTargets = corpus.targets.filter((target) => target.kind !== 'sealed') + console.log( `qualification contract v${CONTRACT_VERSION} is consistent: ` + - `${corpus.targets.length} targets, ${tasks.tasks.length} tasks, ` + - `${tier1.cells.length} Tier 1 cells, ${tier1.negative_trust_probes.length} negative-trust probes, ` + - `${frozenFiles.length} frozen files.`, + `${naturalTargets.length} pinned natural targets, ${corpus.proxy_targets.length} proxy targets, ` + + `${tasks.tasks.length} tasks, ${tier1.cells.length} Tier 1 cells, ` + + `${tier1.negative_trust_probes.length} negative-trust probes, ${frozenFiles.length} frozen files.`, ) diff --git a/docs/qualification/README.md b/docs/qualification/README.md index 7a6a2f67..d40d67c9 100644 --- a/docs/qualification/README.md +++ b/docs/qualification/README.md @@ -5,27 +5,50 @@ against Madar commit `06b373a447acfce895412ac10eb4e5228c5df0b7` (`v0.32.1`). This directory is the independent evaluation contract used to decide whether a roadmap change is safe to ship. It is deliberately separate from -[`docs/benchmarks/suite/`](../benchmarks/suite/), which is the product benchmark suite and -whose per-repo expectations were authored alongside the product. +[`docs/benchmarks/suite/`](../benchmarks/suite/), which is the product benchmark suite. -## What this contract is for +## Why a separate corpus exists -Grading Madar with expectations that Madar produced tells you nothing. This contract fixes -five things before any change is evaluated: +Two properties are required, and the existing benchmark suite has neither. -1. what is being evaluated (`corpus.json`, `tasks.json`); -2. what a correct answer is, authored without looking at Madar output (`truth/`); -3. how it is scored, and by which method per category (`rubrics.json`); -4. when a run does not count at all (`validity-rules.md`, `receipt-schema.json`); -5. what result blocks a merge or forces a rollback (`stop-rule.md`). +**Independence from Madar output.** Grading Madar with expectations Madar produced tells +you nothing. Today's per-task expectations live in +[`docs/benchmarks/suite/runtime-proof.json`](../benchmarks/suite/runtime-proof.json) as +exact expected symbols and paths, authored alongside the product. + +**Naturalness.** Every row in +[`docs/benchmarks/suite/repos.json`](../benchmarks/suite/repos.json) that is keyed by +`path` is an in-repo proxy — `examples/sample-workspace`, two `tests/fixtures/pack-quality` +workspaces, and two fixture directories under the suite itself. A corpus of self-authored +proxies cannot detect production behaviour drifting toward benchmark-shaped repositories, +because the proxies were shaped by the same hands as the production rules. + +Every target in this corpus is therefore a real, externally authored, permissively licensed +project pinned at an immutable commit. There are no fixture proxies; +`corpus.json#/proxy_targets` is empty and documents the conditions under which an entry +would be permitted. + +## Targets + +| Target | Repository | Commit | License | +| --- | --- | --- | --- | +| `hono` | [honojs/hono](https://github.com/honojs/hono) | `26de73133b8552f56ba72e025ecd82b08900d796` | MIT | +| `unstorage` | [unjs/unstorage](https://github.com/unjs/unstorage) | `e6be6135832f350ca16f9a77432e1d4f0aa85ed7` | MIT | +| `hono-seeded-compose` | same as `hono`, plus [`patches/hono-compose-reentrancy-guard.patch`](./patches/hono-compose-reentrancy-guard.patch) | `26de7313…` | MIT | +| `hono-seeded-error-disclosure` | same as `hono`, plus [`patches/hono-error-message-disclosure.patch`](./patches/hono-error-message-disclosure.patch) | `26de7313…` | MIT | +| `sealed-holdout-a` | undisclosed | — | **unsatisfied**, see [`holdout-policy.md`](./holdout-policy.md) | + +Seeded defects are injected into the real code as patches against the pinned commit, not +recreated inside a synthetic workspace built around the answer. ## Files | File | Deliverable | | --- | --- | -| [`corpus.json`](./corpus.json) | Versioned corpus manifest: targets, revisions, dependency locks, holdout class. | +| [`corpus.json`](./corpus.json) | Versioned corpus manifest: repositories, commits, licenses, prepare commands, patches, cited blob digests, holdout class. | | [`tasks.json`](./tasks.json) | Versioned task definitions: frozen prompts with hashes, categories, scoring method, truth provenance. | | [`truth/`](./truth/) | Independent truth and rubric input, one file per task. | +| [`patches/`](./patches/) | Seeded-defect patches applied to the pinned commit. | | [`rubrics.json`](./rubrics.json) | Scoring dimensions, per-category scoring methods, blinding rules, aggregation rules. | | [`receipt-schema.json`](./receipt-schema.json) | Environment and run receipt schema. | | [`examples/`](./examples/) | Two illustrative receipts: a valid Tier 1 run and an invalid Tier 2 run that stays `not_measured`. | @@ -35,18 +58,19 @@ five things before any change is evaluated: | [`evidence-categories.md`](./evidence-categories.md) | Evidence classes E1–E6 and required labelling. | | [`tier1.json`](./tier1.json) | The small deterministic subset a pull request can run. | | [`tier2-matrix.json`](./tier2-matrix.json) | The planned repeated-run matrix. | -| [`fixtures/`](./fixtures/) | Tier 1 target workspaces. | | [`freeze.json`](./freeze.json) | SHA-256 of every file above. A silent change to any of them fails validation. | ## Tiers -**Tier 1** is deterministic, needs no network, no model provider, and no spend. It measures -whether the evidence required to answer each frozen task was present in the context -artifact, and whether readiness was correctly refused on the negative-trust probes. It never -scores answer quality and never runs an agent. +**Tier 1** is deterministic, needs no model provider and no spend. It measures whether the +evidence required to answer each frozen task was present in the context artifact, and +whether readiness was correctly refused on the negative-trust probes. It never scores +answer quality and never runs an agent. It does need network access to clone the pinned +targets; a warm clone cache or a local mirror satisfies that without changing any result, +because the commit and the patch fix the content exactly. **Tier 2** runs a real agent on both arms with repeated trials and blinded rubric scoring. -It is frozen but not executed; its prerequisites are listed in `tier2-matrix.json`. +It is frozen but not executed; its prerequisites are in `tier2-matrix.json`. ## Running it @@ -55,20 +79,31 @@ npm ci npm run qualify:validate ``` -`qualify:validate` checks, from a clean checkout and without running Madar: +`qualify:validate` checks, offline and without running Madar: - every declared contract version agrees; +- every target pins a 40-character commit SHA and records a license and prepare steps; - every task references a real target, and every frozen prompt matches its recorded hash; -- every truth file exists, matches its task, and cites only paths that exist in the target; +- every truth file exists, matches its task, and cites only paths recorded in that target's + `cited_blobs` map; - every truth file records who authored it and asserts no Madar-derived source was used; - all six required task categories are covered; +- every seeded target names a patch file that exists and is a well-formed unified diff + touching only paths recorded for that target; - every Tier 1 cell and negative-trust probe resolves, and every probe prompt hash matches; - both example receipts validate against `receipt-schema.json`, and no unmeasured score carries a value; -- **no qualification target id, task id, prompt string, or fixture symbol appears anywhere - in `src/`**; +- **no qualification target id, task id, prompt string, or pinned-repository symbol appears + anywhere in `src/`**; - every file in this directory matches its frozen digest. +To additionally confirm the pinned commits and blob digests against the real repositories +— this one needs network access: + +```bash +npm run qualify:validate -- --verify-corpus +``` + Regenerating the freeze file is deliberate and must be explained in the pull request: ```bash @@ -80,16 +115,18 @@ not this contract. ## Independence -Both Tier 1 fixtures and all six truth files were authored on 2026-08-12 from blank files. -Madar was never run against them, and no Madar retrieval output, context pack, -`implementationGuidance`, Madar-selected file list, or Madar-generated validation command -was consulted before freezing. Each task records this in `truth_provenance`. +All six truth files were authored on 2026-08-12 by reading the pinned repository sources +directly. Madar was never run against any target, and no Madar retrieval output, context +pack, `implementationGuidance`, Madar-selected file list, or Madar-generated validation +command was consulted before freezing. Each task records this in `truth_provenance`. Two consequences follow, and both are stated rather than papered over: - **Thresholds are pre-registered, not calibrated.** Nobody knows how many Tier 1 cells - currently pass. The first execution is a measurement; a failure there is a product - finding, not a reason to edit this contract. + currently pass. Pre-registering before calibrating is the correct order: a threshold + fitted to observed output would describe current behaviour instead of testing it. The + first execution is a measurement, and a failure there is a product finding, not a reason + to edit this contract. - **The author is not independent of the production-rule author.** Madar has one author, so `independent_of_production_rule_author` is `false` on every task, blinded review is unavailable, and the sealed holdout slot is unsatisfied. See diff --git a/docs/qualification/corpus.json b/docs/qualification/corpus.json index 2d679c53..9d264c04 100644 --- a/docs/qualification/corpus.json +++ b/docs/qualification/corpus.json @@ -8,81 +8,186 @@ "dependency_lock": "package-lock.json at the pinned commit; runs must use `npm ci`, never `npm install`" }, "support_corridor": "typescript-node", + "naturalness_rule": "Every qualification target is a real, externally authored software project pinned at an immutable commit. Self-authored fixture workspaces are not part of this corpus. A corpus of self-made proxies cannot detect production behaviour drifting toward benchmark-shaped repositories, which is the failure mode this contract exists to catch.", + "forbidden_target_symbols": { + "_note": "Distinctive symbols from the pinned targets. Their appearance in src/ would mean production behaviour had been shaped around a qualification repository. Checked by `npm run qualify:validate`. Target ids are deliberately excluded from this check — see production_coupling below.", + "hono": ["SmartRouter", "UnsupportedPathError", "RegExpRouter", "TrieRouter"], + "unstorage": ["createStorage", "DriverFactory", "createRequiredError"] + }, + "proxy_targets": [], + "proxy_targets_note": "This list is intentionally empty. A fixture proxy is permitted only where a natural repository genuinely cannot supply a task category; all six required categories are supplied by the natural targets below. Any future entry here must be labelled a proxy and must carry the statement that proxies cannot satisfy the naturalness property.", "targets": [ { - "id": "qual-ledger-service", - "name": "Ledger service qualification fixture", + "id": "hono", + "name": "Hono web framework", "tier": 1, - "kind": "fixture", - "path": "docs/qualification/fixtures/ledger-service", + "kind": "git", + "natural": true, + "source": { + "url": "https://github.com/honojs/hono", + "ref": "26de73133b8552f56ba72e025ecd82b08900d796", + "committed_at": "2026-08-10T01:20:29Z" + }, + "license": "MIT", "language": "typescript", - "shape": "layered-http-service", - "dependency_lock": "none — the fixture has no runtime dependencies", + "shape": "http-framework", + "dependency_lock": "the repository's own lockfile at the pinned ref", + "prepare": [ + "git clone --filter=blob:none --no-checkout https://github.com/honojs/hono.git ", + "git -C checkout 26de73133b8552f56ba72e025ecd82b08900d796" + ], + "install": "not required — every frozen task is answered from source; no build or dependency install is needed to read the code", "holdout_class": "open", "status": "frozen", - "digest_ref": "freeze.json#/fixtures/docs~1qualification~1fixtures~1ledger-service" + "selection_rationale": "Real, externally authored, permissively licensed TypeScript with a genuine layered request lifecycle (entry point, router abstraction with multiple implementations, middleware composition, context, error handling). Deliberately absent from docs/benchmarks/suite/repos.json so qualification does not inherit repositories that existing production heuristics or published receipts were shaped around.", + "production_coupling": { + "level": "declared_framework_adapter", + "detail": "Madar ships a generic Hono adapter at src/pipeline/spi/framework-hono.ts and Hono-aware query classification in src/runtime/retrieve.ts (an explicit-Hono token check and hono_route / hono_middleware roles). This is declared support for a framework in the TypeScript/Node corridor, not a repository-specific special case, so it is not a #660 contamination finding.", + "consequence": "A result on this target partly measures the shipped Hono adapter. It is not evidence about frameworks that have no adapter, and it must never be generalized to them.", + "mitigation": "The frozen prompts for this target deliberately never name the framework — they say 'this framework' and 'the application's entry point' — so the query classifier is not handed the framework identity in the prompt text. The unstorage target is a plain library with no corresponding adapter and acts as the uncoupled contrast.", + "verified_at": "2026-08-12 against 06b373a447acfce895412ac10eb4e5228c5df0b7" + }, + "cited_blobs": { + "src/hono.ts": "c9472202a710231b8151e06cc812290bf53bd3ab", + "src/hono-base.ts": "e6a7278dd293aa03a1021d3c639b3bda5bd4c23d", + "src/compose.ts": "b1d4508ffe490fe4eaf1fa25a182c95f486685e5", + "src/context.ts": "3553dd181b4a718177a09874ba584c76f4cd54c5", + "src/router.ts": "ec12588ab651a7ffe0d0178b37099edf21833ee0", + "src/router/smart-router/router.ts": "9ec464da8c25dfc2dacaee9ea13174a7d0020ec6", + "src/router/reg-exp-router/router.ts": "6020851a696ff51b3b65adfe80d3daccf5156f3d", + "src/router/trie-router/router.ts": "65b9978861688858f640f7618161b3c0bb8d009c", + "src/utils/url.ts": "ea92ff9355c64340e4d342831aadfc9bb91d7f64", + "src/http-exception.ts": "8fe9c2bb3d078787cfd233634ef8905b5b5d2ccf", + "src/request.ts": "ae5c04076a178319c8af1304bba6b603d235a8f2" + } }, { - "id": "qual-plugin-host", - "name": "Plugin host qualification fixture", + "id": "unstorage", + "name": "unstorage key-value abstraction", "tier": 1, - "kind": "fixture", - "path": "docs/qualification/fixtures/plugin-host", + "kind": "git", + "natural": true, + "source": { + "url": "https://github.com/unjs/unstorage", + "ref": "e6be6135832f350ca16f9a77432e1d4f0aa85ed7", + "committed_at": "2026-06-29T17:46:43Z" + }, + "license": "MIT", "language": "typescript", - "shape": "extension-host", - "dependency_lock": "none — the fixture has no runtime dependencies", + "shape": "driver-based-storage-abstraction", + "dependency_lock": "the repository's own lockfile at the pinned ref", + "prepare": [ + "git clone --filter=blob:none --no-checkout https://github.com/unjs/unstorage.git ", + "git -C checkout e6be6135832f350ca16f9a77432e1d4f0aa85ed7" + ], + "install": "not required — every frozen task is answered from source; no build or dependency install is needed to read the code", "holdout_class": "open", "status": "frozen", - "digest_ref": "freeze.json#/fixtures/docs~1qualification~1fixtures~1plugin-host" + "selection_rationale": "Real, externally authored, permissively licensed TypeScript with a genuine extension seam (a Driver interface, 34 shipped drivers, mount-prefix resolution, a code-generated driver index). A different architectural shape from the framework target, and deliberately absent from docs/benchmarks/suite/repos.json.", + "production_coupling": { + "level": "none_found", + "detail": "No framework adapter in src/pipeline/spi corresponds to this library, and no symbol from it appears in src/. It is a plain TypeScript library outside every declared adapter.", + "consequence": "This target is the uncoupled contrast to the framework target: a result here exercises generic retrieval and context building rather than a shipped adapter.", + "verified_at": "2026-08-12 against 06b373a447acfce895412ac10eb4e5228c5df0b7" + }, + "cited_blobs": { + "src/index.ts": "748a6b72cb67bffdcc43a50ed5e9cb73eb42a0d3", + "src/types.ts": "1f11e02f4036312f7e56abcda04ef7fd79e14ddf", + "src/storage.ts": "0c25ad8d0afe9f1ad8e2f44af8ba6b8d5f1f75cd", + "src/utils.ts": "2477c70c5a518de8c5a65302ee2f1a33f8dc7ea8", + "src/_utils.ts": "b7c765ebacfcfb7823b7bf9517584d0da8695393", + "src/_drivers.ts": "c42493aa41b95156edff2bcb938787570a2f49d3", + "src/drivers/utils/index.ts": "e4ec594a87fd96a80d52c37ed3fd2c2da417e431", + "src/drivers/memory.ts": "8a82465046d8da4565e31ed39a5633cc6a3e58a7", + "scripts/gen-drivers.ts": "0d5c1ec69a45f3d19e747cf7327f7f2234315bc0", + "package.json": "be04ccb74412b208d39ee3646a29c4ab165a036d" + } }, { - "id": "qual-unkey", - "name": "Unkey public repository", - "tier": 2, - "kind": "git", + "id": "hono-seeded-compose", + "name": "Hono with a seeded middleware re-entrancy defect", + "tier": 1, + "kind": "git_patched", + "natural": true, + "base_target": "hono", "source": { - "url": "https://github.com/unkeyed/unkey", - "ref": "47e64533d56ff13dc2af673a203bc625ef34cf8a" + "url": "https://github.com/honojs/hono", + "ref": "26de73133b8552f56ba72e025ecd82b08900d796", + "committed_at": "2026-08-10T01:20:29Z" }, + "patch": "patches/hono-compose-reentrancy-guard.patch", + "patch_summary": "One-character change to the re-entrancy guard in the middleware dispatch loop of src/compose.ts.", + "license": "MIT", "language": "typescript", - "shape": "api-key-and-rate-limit-platform", - "dependency_lock": "repository lockfile at the pinned ref; install with the repository's own frozen-lockfile command", + "shape": "http-framework", + "dependency_lock": "the repository's own lockfile at the pinned ref", + "prepare": [ + "git clone --filter=blob:none --no-checkout https://github.com/honojs/hono.git ", + "git -C checkout 26de73133b8552f56ba72e025ecd82b08900d796", + "git -C apply /patches/hono-compose-reentrancy-guard.patch" + ], + "install": "not required", "holdout_class": "open", - "status": "pinned_no_truth", - "selection_rationale": "Deliberately outside docs/benchmarks/suite/repos.json so Tier 2 qualification does not inherit repositories that existing production heuristics or published receipts were shaped around." + "status": "frozen", + "selection_rationale": "The issue lists seeded defects as a preferred independent truth source. The defect is injected into real externally authored code at a pinned commit rather than being surrounded by a synthetic workspace built to make it findable.", + "cited_blobs": { + "src/compose.ts": "b1d4508ffe490fe4eaf1fa25a182c95f486685e5", + "src/hono-base.ts": "e6a7278dd293aa03a1021d3c639b3bda5bd4c23d", + "src/context.ts": "3553dd181b4a718177a09874ba584c76f4cd54c5" + }, + "cited_blobs_note": "Blob SHAs are for the UNPATCHED pinned tree. The patch changes src/compose.ts; every other cited blob is unchanged." }, { - "id": "qual-payload", - "name": "Payload CMS public repository", - "tier": 2, - "kind": "git", + "id": "hono-seeded-error-disclosure", + "name": "Hono with a seeded error-message disclosure defect", + "tier": 1, + "kind": "git_patched", + "natural": true, + "base_target": "hono", "source": { - "url": "https://github.com/payloadcms/payload", - "ref": "c6e79520e3ea70afc2a001b7cf8ec32683246c57" + "url": "https://github.com/honojs/hono", + "ref": "26de73133b8552f56ba72e025ecd82b08900d796", + "committed_at": "2026-08-10T01:20:29Z" }, + "patch": "patches/hono-error-message-disclosure.patch", + "patch_summary": "Changes the framework default error handler in src/hono-base.ts to return the thrown error's stack or message in the 500 response body.", + "license": "MIT", "language": "typescript", - "shape": "headless-cms-monorepo", - "dependency_lock": "repository lockfile at the pinned ref; install with the repository's own frozen-lockfile command", + "shape": "http-framework", + "dependency_lock": "the repository's own lockfile at the pinned ref", + "prepare": [ + "git clone --filter=blob:none --no-checkout https://github.com/honojs/hono.git ", + "git -C checkout 26de73133b8552f56ba72e025ecd82b08900d796", + "git -C apply /patches/hono-error-message-disclosure.patch" + ], + "install": "not required", "holdout_class": "open", - "status": "pinned_no_truth", - "selection_rationale": "Deliberately outside docs/benchmarks/suite/repos.json so Tier 2 qualification does not inherit repositories that existing production heuristics or published receipts were shaped around." + "status": "frozen", + "selection_rationale": "A security-shaped defect seeded into real externally authored code, so the review task is a review of a natural codebase rather than of a workspace authored around its own answer.", + "cited_blobs": { + "src/hono-base.ts": "e6a7278dd293aa03a1021d3c639b3bda5bd4c23d", + "src/http-exception.ts": "8fe9c2bb3d078787cfd233634ef8905b5b5d2ccf", + "src/context.ts": "3553dd181b4a718177a09874ba584c76f4cd54c5", + "src/compose.ts": "b1d4508ffe490fe4eaf1fa25a182c95f486685e5" + }, + "cited_blobs_note": "Blob SHAs are for the UNPATCHED pinned tree. The patch changes src/hono-base.ts; every other cited blob is unchanged." }, { - "id": "qual-sealed-a", + "id": "sealed-holdout-a", "name": "Sealed holdout target A", "tier": 2, "kind": "sealed", + "natural": true, "language": "typescript", "shape": "undisclosed", "dependency_lock": "recorded in the sealed manifest, not in this repository", "holdout_class": "sealed", "status": "unsatisfied", - "unsatisfied_reason": "Requires a second person to author and hold the target and its truth. See holdout-policy.md; the slot stays visible and explicitly unsatisfied rather than being filled with a self-authored placeholder." + "unsatisfied_reason": "Requires a second person to select the repository and author its truth. See holdout-policy.md; the slot stays visible and explicitly unsatisfied rather than being filled with a self-selected target." } ], "status_meaning": { - "frozen": "Target content is digest-pinned in freeze.json and may be used as measurable Tier 1 evidence.", - "pinned_no_truth": "Repository and revision are pinned, but no independent truth exists yet. Runs against this target can produce receipts and MUST report not_measured for every quality dimension.", + "frozen": "Repository, revision, and (where applicable) patch are pinned, cited blob SHAs are recorded, and independent truth exists. Usable as measurable evidence.", "unsatisfied": "The slot is specified but cannot be filled in the current single-author context. It must never be counted as evidence, present or absent." } } diff --git a/docs/qualification/evidence-categories.md b/docs/qualification/evidence-categories.md index 2d74079f..f9b6cfe1 100644 --- a/docs/qualification/evidence-categories.md +++ b/docs/qualification/evidence-categories.md @@ -6,6 +6,34 @@ Madar's repository already contains several kinds of artifact that look like mea They are not interchangeable. Every published statement must name the category of evidence it rests on. +## Target naturalness qualifies the evidence + +An evidence class says how a measurement was produced. It does not say what the measurement +was produced against, and both matter. + +- **Natural target** — a real, externally authored project pinned at an immutable commit, + optionally with a recorded seeded-defect patch. Every target in + [`corpus.json`](./corpus.json) is natural. +- **Proxy target** — a workspace authored inside this repository to stand in for one. + +A result measured against a proxy can support a regression statement and nothing more. It +can never support a statement about behaviour on real repositories, because a proxy is +shaped by the same hands as the production rules it is meant to test. + +Recorded finding, 2026-08-12, measured against +[`docs/benchmarks/suite/repos.json`](../benchmarks/suite/repos.json) at +`06b373a447acfce895412ac10eb4e5228c5df0b7`: of eleven rows, **five are in-repo proxies** +keyed by `path` — `ts-small` (`examples/sample-workspace`), `nestjs-mid` and +`ts-monorepo-large` (both `tests/fixtures/pack-quality/**/workspace`), `python-service` and +`go-service` (both fixture directories under the suite). The other **six are git-backed and +do pin a URL together with an immutable commit SHA** — `documenso`, `formbricks`, `dub`, +`twenty`, `cal-diy`, `novu`. + +So the existing corpus is mixed, not entirely proxy-based. What matters for evidence +labelling is that the five proxy rows are the ones backing the checked-in deterministic +fixture bundles, and any citation of those receipts must be labelled proxy-target as well +as E4. + ## Categories ### E1 — Product outcome evidence @@ -39,6 +67,18 @@ prompts included proof checklists and the checkout could load expected files and from `docs/benchmarks/suite/runtime-proof.json`. They are genuine measurements of the setup they describe. They are **not** evidence of untuned behaviour, and they are **not** E1. +Open enforcement gap in E3, recorded 2026-08-12 and not addressed here: +`docs/benchmarks/suite/runtime-proof.json` carries per-repository expected symbols and +paths — for example `sendDocument()` and `server-only/document/send-document.ts` under +`documenso-explain-runtime`. `docs/benchmarks/suite/methodology.md` asserts that this file +is grader input only, that it "is not passed into retrieval", and that its obligation +checklist "is not written into the answering agent's prompt". That isolation is asserted in +prose. No test, lint rule, or CI check enforces it, and nothing fails if a future change +reads the manifest from retrieval or splices its obligations into a prompt. Until an +enforcement check exists, every E3 citation must state that the retrieval/grader boundary +is documented rather than proven. This is a separate linked issue and bears directly on +[#660](https://github.com/mohanagy/madar/issues/660). + ### E4 — Synthetic or fixture receipts Checked-in deterministic bundles with fixture-anchored timings and tool-call counts, such diff --git a/docs/qualification/examples/receipt-tier1-valid.json b/docs/qualification/examples/receipt-tier1-valid.json index 53051c64..fa1a0c7d 100644 --- a/docs/qualification/examples/receipt-tier1-valid.json +++ b/docs/qualification/examples/receipt-tier1-valid.json @@ -2,13 +2,13 @@ "contract_version": "1.0.0", "run_id": "example-tier1-valid-0001", "tier": 1, - "task_id": "rootcause-ledger-duplicate-entries", - "target_id": "qual-ledger-service", + "task_id": "rootcause-hono-middleware-rerun", + "target_id": "hono-seeded-compose", "arm": "madar", "trial": 1, "cache_mode": "cold", "identity": { - "target_revision": "sha256:0000000000000000000000000000000000000000000000000000000000000000", + "target_revision": "26de73133b8552f56ba72e025ecd82b08900d796", "dependency_lock_sha256": "6328bf95a901590814ff70ed570e0fc474c05ae162d6ac690cf3c812380828ab", "madar_commit": "06b373a447acfce895412ac10eb4e5228c5df0b7", "madar_package_version": "0.32.1", @@ -22,8 +22,8 @@ }, "prompts": { "system_prompt_sha256": null, - "user_prompt_sha256": "3d6f2f70a3b21508c17637cce0d9d94ccbea241adaeb2223192664c8d9856164", - "user_prompt_text": "Under load, a client that retries a request with the same idempotency key sometimes ends up with two ledger entries instead of one. Find the root cause and explain the exact ordering of operations that produces the duplicate." + "user_prompt_sha256": "637cd655712c2e7d96a3566ed0b311d9d7dcfe0d9cfb8b699f5f1c52a721eaf0", + "user_prompt_text": "A middleware that awaits the next function twice used to fail loudly with an error. It now silently runs the rest of the chain a second time, so downstream handlers execute twice for one request. Find the root cause and explain the exact mechanism that produces the second execution." }, "tool_permissions": [], "cache_mode": "cold" @@ -43,7 +43,10 @@ "pre_tool_use": [], "post_tool_use": [] }, - "drift": { "detected": false, "fields": [] } + "drift": { + "detected": false, + "fields": [] + } }, "adoption": { "status": "not_applicable", @@ -53,15 +56,48 @@ "trace_status": "trace_available" }, "costs": { - "indexing": { "measured": true, "wall_ms": 380, "usd": 0, "source": "locally_timed" }, - "context_build": { "measured": true, "wall_ms": 96, "usd": 0, "source": "locally_timed" }, - "agent": { "measured": false, "source": "not_applicable" } + "indexing": { + "measured": true, + "wall_ms": 380, + "usd": 0, + "source": "locally_timed" + }, + "context_build": { + "measured": true, + "wall_ms": 96, + "usd": 0, + "source": "locally_timed" + }, + "agent": { + "measured": false, + "source": "not_applicable" + } }, "scores": { - "correctness": { "measured": false, "value": null, "method": "blinded_rubric", "not_measured_reason": "Tier 1 does not score answer quality; no agent answer exists." }, - "critical_fact_completeness": { "measured": false, "value": null, "method": "blinded_rubric", "not_measured_reason": "Tier 1 does not score answer quality; no agent answer exists." }, - "unsupported_claims": { "measured": false, "value": null, "method": "blinded_rubric", "not_measured_reason": "Tier 1 does not score answer quality; no agent answer exists." }, - "correct_uncertainty": { "measured": false, "value": null, "method": "blinded_rubric", "not_measured_reason": "Tier 1 does not score answer quality; no agent answer exists." }, + "correctness": { + "measured": false, + "value": null, + "method": "blinded_rubric", + "not_measured_reason": "Tier 1 does not score answer quality; no agent answer exists." + }, + "critical_fact_completeness": { + "measured": false, + "value": null, + "method": "blinded_rubric", + "not_measured_reason": "Tier 1 does not score answer quality; no agent answer exists." + }, + "unsupported_claims": { + "measured": false, + "value": null, + "method": "blinded_rubric", + "not_measured_reason": "Tier 1 does not score answer quality; no agent answer exists." + }, + "correct_uncertainty": { + "measured": false, + "value": null, + "method": "blinded_rubric", + "not_measured_reason": "Tier 1 does not score answer quality; no agent answer exists." + }, "evidence_support": { "measured": true, "value": 1, @@ -84,9 +120,21 @@ "aggregatable": true }, "retention": { - "raw_transcript": { "retained": false, "path": null, "sha256": null }, - "answer_text": { "retained": false, "path": null, "sha256": null }, - "context_artifact": { "retained": true, "path": "raw/context-pack.json", "sha256": "76cae19d9787ab64fb7c0d4be597efe26c4453c855d0d2dea18e6eaa8b83ac7e" }, + "raw_transcript": { + "retained": false, + "path": null, + "sha256": null + }, + "answer_text": { + "retained": false, + "path": null, + "sha256": null + }, + "context_artifact": { + "retained": true, + "path": "raw/context-pack.json", + "sha256": "76cae19d9787ab64fb7c0d4be597efe26c4453c855d0d2dea18e6eaa8b83ac7e" + }, "retention_policy": "Tier 1 retains the context artifact only; there is no agent transcript because no agent runs." }, "notes": [ diff --git a/docs/qualification/examples/receipt-tier2-invalid-no-madar-call.json b/docs/qualification/examples/receipt-tier2-invalid-no-madar-call.json index 1e6a3060..4c43dd38 100644 --- a/docs/qualification/examples/receipt-tier2-invalid-no-madar-call.json +++ b/docs/qualification/examples/receipt-tier2-invalid-no-madar-call.json @@ -2,13 +2,13 @@ "contract_version": "1.0.0", "run_id": "example-tier2-invalid-0001", "tier": 2, - "task_id": "flow-ledger-post-entry", - "target_id": "qual-ledger-service", + "task_id": "flow-hono-request-dispatch", + "target_id": "hono", "arm": "madar", "trial": 1, "cache_mode": "warm", "identity": { - "target_revision": "sha256:0000000000000000000000000000000000000000000000000000000000000000", + "target_revision": "26de73133b8552f56ba72e025ecd82b08900d796", "dependency_lock_sha256": "6328bf95a901590814ff70ed570e0fc474c05ae162d6ac690cf3c812380828ab", "madar_commit": "06b373a447acfce895412ac10eb4e5228c5df0b7", "madar_package_version": "0.32.1", @@ -22,10 +22,15 @@ }, "prompts": { "system_prompt_sha256": "21da86de37b6595841807f54b44945498ad599cf682a40391a36d833dca35132", - "user_prompt_sha256": "4f727f2d70f0df36aa08fc208a27d6dcf1b1ed0a8b26a05b1e1dde32b57e29aa", - "user_prompt_text": "Trace what happens when a client posts a new ledger entry. Follow the path from the HTTP handler through to every durable side effect and every downstream consumer, and say which step each side effect happens in." + "user_prompt_sha256": "c276d363f0b10c00b75df925f2e658f3ee7af085872bcbcc64a7a4252c7b3b35", + "user_prompt_text": "Trace what happens to an incoming request from the application's entry point until a response is returned. Cover how the path is resolved, how a route is matched, how handlers and middleware are run, and how errors and unmatched routes are handled. Say which steps are skipped in the fast path." }, - "tool_permissions": ["Read", "Grep", "Glob", "mcp__madar__retrieve"], + "tool_permissions": [ + "Read", + "Grep", + "Glob", + "mcp__madar__retrieve" + ], "cache_mode": "warm" }, "environment": { @@ -33,7 +38,9 @@ "host_os": "darwin", "node_version": "v22.0.0", "claude_code_version": "0.0.0-example", - "mcp_servers_active": ["madar"], + "mcp_servers_active": [ + "madar" + ], "skills_loaded": [], "plugins_active": [], "user_claude_md_hash": null, @@ -43,7 +50,10 @@ "pre_tool_use": [], "post_tool_use": [] }, - "drift": { "detected": false, "fields": [] } + "drift": { + "detected": false, + "fields": [] + } }, "adoption": { "status": "absent", @@ -53,8 +63,16 @@ "trace_status": "trace_available" }, "costs": { - "indexing": { "measured": true, "wall_ms": 4120, "usd": 0, "source": "locally_timed" }, - "context_build": { "measured": false, "source": "not_applicable" }, + "indexing": { + "measured": true, + "wall_ms": 4120, + "usd": 0, + "source": "locally_timed" + }, + "context_build": { + "measured": false, + "source": "not_applicable" + }, "agent": { "measured": true, "input_tokens": 41233, @@ -65,21 +83,60 @@ } }, "scores": { - "correctness": { "measured": false, "value": null, "method": "blinded_rubric", "not_measured_reason": "run is invalid: no attributable Madar call" }, - "critical_fact_completeness": { "measured": false, "value": null, "method": "blinded_rubric", "not_measured_reason": "run is invalid: no attributable Madar call" }, - "unsupported_claims": { "measured": false, "value": null, "method": "blinded_rubric", "not_measured_reason": "run is invalid: no attributable Madar call" }, - "correct_uncertainty": { "measured": false, "value": null, "method": "blinded_rubric", "not_measured_reason": "run is invalid: no attributable Madar call" }, - "evidence_support": { "measured": false, "value": null, "method": "blinded_rubric", "not_measured_reason": "run is invalid: no attributable Madar call" } + "correctness": { + "measured": false, + "value": null, + "method": "blinded_rubric", + "not_measured_reason": "run is invalid: no attributable Madar call" + }, + "critical_fact_completeness": { + "measured": false, + "value": null, + "method": "blinded_rubric", + "not_measured_reason": "run is invalid: no attributable Madar call" + }, + "unsupported_claims": { + "measured": false, + "value": null, + "method": "blinded_rubric", + "not_measured_reason": "run is invalid: no attributable Madar call" + }, + "correct_uncertainty": { + "measured": false, + "value": null, + "method": "blinded_rubric", + "not_measured_reason": "run is invalid: no attributable Madar call" + }, + "evidence_support": { + "measured": false, + "value": null, + "method": "blinded_rubric", + "not_measured_reason": "run is invalid: no attributable Madar call" + } }, "validity": { "status": "invalid", - "invalidation_reasons": ["missing_attributable_madar_call"], + "invalidation_reasons": [ + "missing_attributable_madar_call" + ], "aggregatable": false }, "retention": { - "raw_transcript": { "retained": true, "path": "raw/transcript.jsonl", "sha256": "8afad96d88f459209acacc66728718b270aa3e2df59fb8e92d5fc85475b3272c" }, - "answer_text": { "retained": true, "path": "raw/answer.txt", "sha256": "95967dd7cc113d0df15f5dd7b3d2185f7755a3da9b4db4b71805cd646434d7b6" }, - "context_artifact": { "retained": false, "path": null, "sha256": null }, + "raw_transcript": { + "retained": true, + "path": "raw/transcript.jsonl", + "sha256": "8afad96d88f459209acacc66728718b270aa3e2df59fb8e92d5fc85475b3272c" + }, + "answer_text": { + "retained": true, + "path": "raw/answer.txt", + "sha256": "95967dd7cc113d0df15f5dd7b3d2185f7755a3da9b4db4b71805cd646434d7b6" + }, + "context_artifact": { + "retained": false, + "path": null, + "sha256": null + }, "retention_policy": "Raw transcripts, answers, and context artifacts are retained for at least 24 months alongside the receipt; see validity-rules.md." }, "notes": [ diff --git a/docs/qualification/fixtures/ledger-service/README.md b/docs/qualification/fixtures/ledger-service/README.md deleted file mode 100644 index c51f0183..00000000 --- a/docs/qualification/fixtures/ledger-service/README.md +++ /dev/null @@ -1,39 +0,0 @@ -# `ledger-service` qualification fixture - -A small TypeScript/Node accounting-ledger service used as a qualification target. - -This workspace exists **only** as an evaluation target. It is not shipped in the npm -package, is not imported by `src/`, and must never be referenced by production -retrieval or context logic. - -## Shape - -```text -POST /accounts/:accountId/entries -> postLedgerEntry -POST /entries/:entryId/reversals -> reverseLedgerEntry - -http/ledger-routes.ts - -> auth/request-context.ts (principal resolution) - -> service/ledger-service.ts (command service) - -> store/idempotency-store.ts (retry suppression) - -> store/ledger-store.ts (append-only entries) - -> outbox/outbox-publisher.ts (ledger.entry.posted) - -> audit/audit-log.ts (audit trail) -outbox/outbox-publisher.ts - -> projections/balance-projection.ts (ledger.entry.posted consumer) -``` - -## Deliberate defects - -Two defects are seeded on purpose and are part of the frozen truth. Do not fix them. - -| Id | Site | Nature | -| --- | --- | --- | -| `seeded-idempotency-ordering` | `src/service/ledger-service.ts` | The idempotency key is reserved **after** the ledger append, so a concurrent retry appends a duplicate entry. | -| `seeded-reversal-authorization` | `src/http/ledger-routes.ts` | The reversal route never checks the entry's account against `principal.accountIds`, so any authenticated principal can reverse another tenant's entry. | - -## Authoring provenance - -Authored for issue #655 on 2026-08-12 from a blank file. No Madar output, retrieval -result, context pack, or `implementationGuidance` was consulted while writing this -workspace or the truth files derived from it. diff --git a/docs/qualification/fixtures/ledger-service/package.json b/docs/qualification/fixtures/ledger-service/package.json deleted file mode 100644 index 50783d8b..00000000 --- a/docs/qualification/fixtures/ledger-service/package.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "name": "qualification-fixture-ledger-service", - "version": "0.0.0", - "private": true, - "type": "module", - "description": "Qualification fixture only. Not published, not imported by Madar sources.", - "engines": { - "node": ">=20" - } -} diff --git a/docs/qualification/fixtures/ledger-service/src/audit/audit-log.ts b/docs/qualification/fixtures/ledger-service/src/audit/audit-log.ts deleted file mode 100644 index 7e2ebcee..00000000 --- a/docs/qualification/fixtures/ledger-service/src/audit/audit-log.ts +++ /dev/null @@ -1,22 +0,0 @@ -export interface AuditRecord { - action: string - requestId: string - principalId: string - accountId: string - entryId: string - recordedAt: string -} - -export class AuditLog { - private readonly records: AuditRecord[] = [] - - record(record: Omit): AuditRecord { - const stored: AuditRecord = { ...record, recordedAt: new Date().toISOString() } - this.records.push(stored) - return stored - } - - listForAccount(accountId: string): AuditRecord[] { - return this.records.filter((record) => record.accountId === accountId) - } -} diff --git a/docs/qualification/fixtures/ledger-service/src/auth/request-context.ts b/docs/qualification/fixtures/ledger-service/src/auth/request-context.ts deleted file mode 100644 index 7cbd903b..00000000 --- a/docs/qualification/fixtures/ledger-service/src/auth/request-context.ts +++ /dev/null @@ -1,52 +0,0 @@ -export interface Principal { - principalId: string - accountIds: string[] - scopes: string[] -} - -export interface RequestContext { - requestId: string - principal: Principal -} - -export class UnauthenticatedError extends Error { - constructor() { - super('missing or invalid bearer token') - this.name = 'UnauthenticatedError' - } -} - -export class ForbiddenError extends Error { - constructor(reason: string) { - super(reason) - this.name = 'ForbiddenError' - } -} - -export interface TokenDirectory { - lookup(token: string): Principal | undefined -} - -export function resolveRequestContext( - headers: Record, - directory: TokenDirectory, -): RequestContext { - const authorization = headers.authorization ?? '' - const token = authorization.startsWith('Bearer ') ? authorization.slice('Bearer '.length) : '' - - const principal = token ? directory.lookup(token) : undefined - if (!principal) { - throw new UnauthenticatedError() - } - - return { - requestId: headers['x-request-id'] ?? `req_${Math.random().toString(36).slice(2, 10)}`, - principal, - } -} - -export function assertAccountAccess(principal: Principal, accountId: string): void { - if (!principal.accountIds.includes(accountId)) { - throw new ForbiddenError(`principal ${principal.principalId} may not act on account ${accountId}`) - } -} diff --git a/docs/qualification/fixtures/ledger-service/src/http/ledger-routes.ts b/docs/qualification/fixtures/ledger-service/src/http/ledger-routes.ts deleted file mode 100644 index 3dc72fe3..00000000 --- a/docs/qualification/fixtures/ledger-service/src/http/ledger-routes.ts +++ /dev/null @@ -1,79 +0,0 @@ -import { assertAccountAccess, resolveRequestContext, type TokenDirectory } from '../auth/request-context.js' -import { UnknownEntryError, type LedgerService } from '../service/ledger-service.js' - -export interface HttpRequest { - method: string - path: string - headers: Record - params: Record - body: Record -} - -export interface HttpResponse { - status: number - body: Record -} - -export interface LedgerRouterDependencies { - ledgerService: LedgerService - tokenDirectory: TokenDirectory -} - -function requireString(body: Record, field: string): string { - const value = body[field] - if (typeof value !== 'string' || value.length === 0) { - throw new TypeError(`field ${field} is required`) - } - return value -} - -function requireNumber(body: Record, field: string): number { - const value = body[field] - if (typeof value !== 'number' || !Number.isFinite(value)) { - throw new TypeError(`field ${field} is required`) - } - return value -} - -export function createLedgerRouter(deps: LedgerRouterDependencies) { - return { - /** POST /accounts/:accountId/entries */ - postLedgerEntry(request: HttpRequest): HttpResponse { - const context = resolveRequestContext(request.headers, deps.tokenDirectory) - const accountId = request.params.accountId ?? '' - - assertAccountAccess(context.principal, accountId) - - const entry = deps.ledgerService.postEntry(context, { - accountId, - amountMinor: requireNumber(request.body, 'amountMinor'), - currency: requireString(request.body, 'currency'), - idempotencyKey: requireString(request.body, 'idempotencyKey'), - }) - - return { status: 201, body: { entry } } - }, - - /** POST /entries/:entryId/reversals */ - reverseLedgerEntry(request: HttpRequest): HttpResponse { - const context = resolveRequestContext(request.headers, deps.tokenDirectory) - const entryId = request.params.entryId ?? '' - - // seeded-reversal-authorization: unlike postLedgerEntry, this handler - // never calls assertAccountAccess for the account that owns entryId, so - // any authenticated principal can reverse another tenant's entry. - try { - const reversal = deps.ledgerService.reverseEntry(context, { - entryId, - idempotencyKey: requireString(request.body, 'idempotencyKey'), - }) - return { status: 201, body: { entry: reversal } } - } catch (error) { - if (error instanceof UnknownEntryError) { - return { status: 404, body: { error: error.message } } - } - throw error - } - }, - } -} diff --git a/docs/qualification/fixtures/ledger-service/src/outbox/outbox-publisher.ts b/docs/qualification/fixtures/ledger-service/src/outbox/outbox-publisher.ts deleted file mode 100644 index cec28c94..00000000 --- a/docs/qualification/fixtures/ledger-service/src/outbox/outbox-publisher.ts +++ /dev/null @@ -1,34 +0,0 @@ -export type LedgerEventName = 'ledger.entry.posted' | 'ledger.entry.reversed' - -export interface LedgerEvent { - name: LedgerEventName - entryId: string - accountId: string - amountMinor: number - currency: string - publishedAt: string -} - -export type LedgerEventHandler = (event: LedgerEvent) => void - -/** - * The only path by which ledger state leaves the write model. Every downstream - * read model is rebuilt from these events, so dropping a publish silently - * desynchronizes consumers rather than raising an error. - */ -export class OutboxPublisher { - private readonly handlers = new Map() - - subscribe(name: LedgerEventName, handler: LedgerEventHandler): void { - const existing = this.handlers.get(name) ?? [] - this.handlers.set(name, [...existing, handler]) - } - - publish(event: Omit): LedgerEvent { - const published: LedgerEvent = { ...event, publishedAt: new Date().toISOString() } - for (const handler of this.handlers.get(published.name) ?? []) { - handler(published) - } - return published - } -} diff --git a/docs/qualification/fixtures/ledger-service/src/projections/balance-projection.ts b/docs/qualification/fixtures/ledger-service/src/projections/balance-projection.ts deleted file mode 100644 index eef58ae7..00000000 --- a/docs/qualification/fixtures/ledger-service/src/projections/balance-projection.ts +++ /dev/null @@ -1,45 +0,0 @@ -import type { LedgerEvent, OutboxPublisher } from '../outbox/outbox-publisher.js' - -export interface AccountBalance { - accountId: string - currency: string - balanceMinor: number - lastEntryId: string | null -} - -/** - * Derived read model. It has no access to `LedgerStore`; its only input is the - * `ledger.entry.posted` / `ledger.entry.reversed` stream from `OutboxPublisher`. - */ -export class BalanceProjection { - private readonly balances = new Map() - - attach(publisher: OutboxPublisher): void { - publisher.subscribe('ledger.entry.posted', (event) => this.applyPosted(event)) - publisher.subscribe('ledger.entry.reversed', (event) => this.applyReversed(event)) - } - - balanceFor(accountId: string): AccountBalance | undefined { - return this.balances.get(accountId) - } - - private applyPosted(event: LedgerEvent): void { - const current = this.balances.get(event.accountId) - this.balances.set(event.accountId, { - accountId: event.accountId, - currency: event.currency, - balanceMinor: (current?.balanceMinor ?? 0) + event.amountMinor, - lastEntryId: event.entryId, - }) - } - - private applyReversed(event: LedgerEvent): void { - const current = this.balances.get(event.accountId) - this.balances.set(event.accountId, { - accountId: event.accountId, - currency: event.currency, - balanceMinor: (current?.balanceMinor ?? 0) - event.amountMinor, - lastEntryId: event.entryId, - }) - } -} diff --git a/docs/qualification/fixtures/ledger-service/src/service/ledger-service.ts b/docs/qualification/fixtures/ledger-service/src/service/ledger-service.ts deleted file mode 100644 index 50a2dbc6..00000000 --- a/docs/qualification/fixtures/ledger-service/src/service/ledger-service.ts +++ /dev/null @@ -1,120 +0,0 @@ -import type { AuditLog } from '../audit/audit-log.js' -import type { RequestContext } from '../auth/request-context.js' -import type { OutboxPublisher } from '../outbox/outbox-publisher.js' -import type { IdempotencyStore } from '../store/idempotency-store.js' -import type { LedgerEntry, LedgerStore } from '../store/ledger-store.js' - -export interface PostEntryCommand { - accountId: string - amountMinor: number - currency: string - idempotencyKey: string -} - -export interface ReverseEntryCommand { - entryId: string - idempotencyKey: string -} - -export class UnknownEntryError extends Error { - constructor(entryId: string) { - super(`ledger entry ${entryId} does not exist`) - this.name = 'UnknownEntryError' - } -} - -export interface LedgerServiceDependencies { - ledgerStore: LedgerStore - idempotencyStore: IdempotencyStore - outboxPublisher: OutboxPublisher - auditLog: AuditLog -} - -export class LedgerService { - constructor(private readonly deps: LedgerServiceDependencies) {} - - postEntry(context: RequestContext, command: PostEntryCommand): LedgerEntry { - const replay = this.deps.idempotencyStore.find(command.idempotencyKey) - if (replay) { - const existing = this.deps.ledgerStore.findEntry(replay.entryId) - if (existing) { - return existing - } - } - - const entry = this.deps.ledgerStore.appendEntry({ - accountId: command.accountId, - amountMinor: command.amountMinor, - currency: command.currency, - }) - - // seeded-idempotency-ordering: the reservation is written only after the - // append succeeds, so a retry that arrives between the read above and this - // line appends a second entry for the same idempotency key. - this.deps.idempotencyStore.reserve(command.idempotencyKey, entry.entryId) - - this.deps.outboxPublisher.publish({ - name: 'ledger.entry.posted', - entryId: entry.entryId, - accountId: entry.accountId, - amountMinor: entry.amountMinor, - currency: entry.currency, - }) - - this.deps.auditLog.record({ - action: 'ledger.entry.posted', - requestId: context.requestId, - principalId: context.principal.principalId, - accountId: entry.accountId, - entryId: entry.entryId, - }) - - return entry - } - - reverseEntry(context: RequestContext, command: ReverseEntryCommand): LedgerEntry { - const original = this.deps.ledgerStore.findEntry(command.entryId) - if (!original) { - throw new UnknownEntryError(command.entryId) - } - - const replay = this.deps.idempotencyStore.find(command.idempotencyKey) - if (replay) { - const existing = this.deps.ledgerStore.findEntry(replay.entryId) - if (existing) { - return existing - } - } - - const reversal = this.deps.ledgerStore.appendEntry({ - accountId: original.accountId, - amountMinor: -original.amountMinor, - currency: original.currency, - reversalOfEntryId: original.entryId, - }) - - this.deps.idempotencyStore.reserve(command.idempotencyKey, reversal.entryId) - - this.deps.outboxPublisher.publish({ - name: 'ledger.entry.reversed', - entryId: reversal.entryId, - accountId: reversal.accountId, - amountMinor: original.amountMinor, - currency: reversal.currency, - }) - - this.deps.auditLog.record({ - action: 'ledger.entry.reversed', - requestId: context.requestId, - principalId: context.principal.principalId, - accountId: reversal.accountId, - entryId: reversal.entryId, - }) - - return reversal - } - - findEntryForAuthorization(entryId: string): LedgerEntry | undefined { - return this.deps.ledgerStore.findEntry(entryId) - } -} diff --git a/docs/qualification/fixtures/ledger-service/src/store/idempotency-store.ts b/docs/qualification/fixtures/ledger-service/src/store/idempotency-store.ts deleted file mode 100644 index c74da9bf..00000000 --- a/docs/qualification/fixtures/ledger-service/src/store/idempotency-store.ts +++ /dev/null @@ -1,34 +0,0 @@ -export interface IdempotencyRecord { - key: string - entryId: string - reservedAt: string -} - -/** - * Suppresses duplicate command execution for retried requests. - * - * `reserve` is intended to be called *before* any state mutation so a concurrent - * retry loses the race and replays the stored result instead of mutating again. - */ -export class IdempotencyStore { - private readonly records = new Map() - - find(key: string): IdempotencyRecord | undefined { - return this.records.get(key) - } - - reserve(key: string, entryId: string): IdempotencyRecord { - const existing = this.records.get(key) - if (existing) { - return existing - } - - const record: IdempotencyRecord = { - key, - entryId, - reservedAt: new Date().toISOString(), - } - this.records.set(key, record) - return record - } -} diff --git a/docs/qualification/fixtures/ledger-service/src/store/ledger-store.ts b/docs/qualification/fixtures/ledger-service/src/store/ledger-store.ts deleted file mode 100644 index 3e49ab58..00000000 --- a/docs/qualification/fixtures/ledger-service/src/store/ledger-store.ts +++ /dev/null @@ -1,46 +0,0 @@ -export interface LedgerEntry { - entryId: string - accountId: string - amountMinor: number - currency: string - reversalOfEntryId: string | null - createdAt: string -} - -export interface AppendLedgerEntryInput { - accountId: string - amountMinor: number - currency: string - reversalOfEntryId?: string | null -} - -/** - * Append-only entry log. Balances are never stored here; they are derived by - * `projections/balance-projection.ts` from published outbox events. - */ -export class LedgerStore { - private readonly entries = new Map() - private sequence = 0 - - appendEntry(input: AppendLedgerEntryInput): LedgerEntry { - this.sequence += 1 - const entry: LedgerEntry = { - entryId: `led_${this.sequence.toString().padStart(8, '0')}`, - accountId: input.accountId, - amountMinor: input.amountMinor, - currency: input.currency, - reversalOfEntryId: input.reversalOfEntryId ?? null, - createdAt: new Date().toISOString(), - } - this.entries.set(entry.entryId, entry) - return entry - } - - findEntry(entryId: string): LedgerEntry | undefined { - return this.entries.get(entryId) - } - - listEntriesForAccount(accountId: string): LedgerEntry[] { - return [...this.entries.values()].filter((entry) => entry.accountId === accountId) - } -} diff --git a/docs/qualification/fixtures/ledger-service/tsconfig.json b/docs/qualification/fixtures/ledger-service/tsconfig.json deleted file mode 100644 index a8c34666..00000000 --- a/docs/qualification/fixtures/ledger-service/tsconfig.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "compilerOptions": { - "target": "ES2022", - "module": "NodeNext", - "moduleResolution": "NodeNext", - "lib": ["ES2022"], - "strict": true, - "noEmit": true - }, - "include": ["src/**/*.ts"] -} diff --git a/docs/qualification/fixtures/plugin-host/README.md b/docs/qualification/fixtures/plugin-host/README.md deleted file mode 100644 index c51e323c..00000000 --- a/docs/qualification/fixtures/plugin-host/README.md +++ /dev/null @@ -1,35 +0,0 @@ -# `plugin-host` qualification fixture - -A small TypeScript/Node extension host used as a qualification target for -architecture-understanding and bounded implementation-planning tasks. - -This workspace exists **only** as an evaluation target. It is not shipped in the npm -package, is not imported by `src/`, and must never be referenced by production -retrieval or context logic. - -## Shape - -```text -contracts/plugin.ts the only stable extension surface (ExportPlugin) -host/config.ts layered configuration resolution -host/registry.ts name -> plugin resolution, duplicate rejection -host/lifecycle.ts init -> run -> dispose ordering and failure isolation -host/plugin-host.ts composition root, the only place that knows both sides -plugins/csv-export-plugin.ts built-in plugin, no external I/O -plugins/webhook-export-plugin.ts built-in plugin, performs external delivery -``` - -The intended boundary is that `plugins/*` depends on `contracts/plugin.ts` only, and -never on `host/*`. `host/plugin-host.ts` is the single composition root. - -## Deliberate defect - -| Id | Site | Nature | -| --- | --- | --- | -| `seeded-boundary-violation` | `src/plugins/webhook-export-plugin.ts` | Imports `resolveHostConfig` from `host/config.ts`, breaking the stated plugin -> contracts-only boundary and coupling a plugin to host internals. | - -## Authoring provenance - -Authored for issue #655 on 2026-08-12 from a blank file. No Madar output, retrieval -result, context pack, or `implementationGuidance` was consulted while writing this -workspace or the truth files derived from it. diff --git a/docs/qualification/fixtures/plugin-host/package.json b/docs/qualification/fixtures/plugin-host/package.json deleted file mode 100644 index 47ba37b0..00000000 --- a/docs/qualification/fixtures/plugin-host/package.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "name": "qualification-fixture-plugin-host", - "version": "0.0.0", - "private": true, - "type": "module", - "description": "Qualification fixture only. Not published, not imported by Madar sources.", - "engines": { - "node": ">=20" - } -} diff --git a/docs/qualification/fixtures/plugin-host/src/contracts/plugin.ts b/docs/qualification/fixtures/plugin-host/src/contracts/plugin.ts deleted file mode 100644 index c0ffee4f..00000000 --- a/docs/qualification/fixtures/plugin-host/src/contracts/plugin.ts +++ /dev/null @@ -1,45 +0,0 @@ -export interface ExportRecord { - id: string - fields: Record -} - -export interface ExportBatch { - batchId: string - records: ExportRecord[] -} - -export interface ExportResult { - pluginName: string - batchId: string - recordsExported: number - destination: string -} - -export interface PluginContext { - /** Plugin-scoped settings resolved by the host; plugins never read config themselves. */ - settings: Readonly> - log(message: string): void -} - -/** - * The only stable extension surface. Anything under `plugins/` must depend on - * this module and nothing else from this workspace. - */ -export interface ExportPlugin { - readonly name: string - readonly version: string - init(context: PluginContext): void - export(batch: ExportBatch): ExportResult - dispose?(): void -} - -export class PluginFailure extends Error { - constructor( - readonly pluginName: string, - readonly phase: 'init' | 'export' | 'dispose', - cause: unknown, - ) { - super(`plugin ${pluginName} failed during ${phase}: ${String(cause)}`) - this.name = 'PluginFailure' - } -} diff --git a/docs/qualification/fixtures/plugin-host/src/host/config.ts b/docs/qualification/fixtures/plugin-host/src/host/config.ts deleted file mode 100644 index ce40bedf..00000000 --- a/docs/qualification/fixtures/plugin-host/src/host/config.ts +++ /dev/null @@ -1,32 +0,0 @@ -export interface HostConfig { - enabledPlugins: string[] - pluginSettings: Record> - failFast: boolean -} - -const DEFAULT_CONFIG: HostConfig = { - enabledPlugins: ['csv-export'], - pluginSettings: {}, - failFast: false, -} - -/** - * Layered resolution: defaults, then file config, then environment overrides. - * Only `host/plugin-host.ts` is expected to call this. - */ -export function resolveHostConfig( - fileConfig: Partial, - env: Record, -): HostConfig { - const enabledFromEnv = env.EXPORT_PLUGINS?.split(',').map((name) => name.trim()).filter(Boolean) - - return { - enabledPlugins: enabledFromEnv ?? fileConfig.enabledPlugins ?? DEFAULT_CONFIG.enabledPlugins, - pluginSettings: { ...DEFAULT_CONFIG.pluginSettings, ...fileConfig.pluginSettings }, - failFast: env.EXPORT_FAIL_FAST === '1' ? true : (fileConfig.failFast ?? DEFAULT_CONFIG.failFast), - } -} - -export function settingsFor(config: HostConfig, pluginName: string): Record { - return config.pluginSettings[pluginName] ?? {} -} diff --git a/docs/qualification/fixtures/plugin-host/src/host/lifecycle.ts b/docs/qualification/fixtures/plugin-host/src/host/lifecycle.ts deleted file mode 100644 index 04c19c5b..00000000 --- a/docs/qualification/fixtures/plugin-host/src/host/lifecycle.ts +++ /dev/null @@ -1,62 +0,0 @@ -import { PluginFailure, type ExportBatch, type ExportPlugin, type ExportResult, type PluginContext } from '../contracts/plugin.js' - -export interface LifecycleOptions { - failFast: boolean - contextFor(plugin: ExportPlugin): PluginContext -} - -export interface LifecycleOutcome { - results: ExportResult[] - failures: PluginFailure[] -} - -/** - * Owns init -> export -> dispose ordering and failure isolation. - * - * With `failFast: false` a failing plugin is recorded and skipped; the remaining - * plugins still run and every initialized plugin is still disposed. - */ -export function runExportLifecycle( - plugins: ExportPlugin[], - batch: ExportBatch, - options: LifecycleOptions, -): LifecycleOutcome { - const results: ExportResult[] = [] - const failures: PluginFailure[] = [] - const initialized: ExportPlugin[] = [] - - for (const plugin of plugins) { - try { - plugin.init(options.contextFor(plugin)) - initialized.push(plugin) - } catch (cause) { - const failure = new PluginFailure(plugin.name, 'init', cause) - if (options.failFast) { - throw failure - } - failures.push(failure) - } - } - - for (const plugin of initialized) { - try { - results.push(plugin.export(batch)) - } catch (cause) { - const failure = new PluginFailure(plugin.name, 'export', cause) - if (options.failFast) { - throw failure - } - failures.push(failure) - } - } - - for (const plugin of initialized) { - try { - plugin.dispose?.() - } catch (cause) { - failures.push(new PluginFailure(plugin.name, 'dispose', cause)) - } - } - - return { results, failures } -} diff --git a/docs/qualification/fixtures/plugin-host/src/host/plugin-host.ts b/docs/qualification/fixtures/plugin-host/src/host/plugin-host.ts deleted file mode 100644 index 489e5b47..00000000 --- a/docs/qualification/fixtures/plugin-host/src/host/plugin-host.ts +++ /dev/null @@ -1,56 +0,0 @@ -import type { ExportBatch, ExportPlugin, PluginContext } from '../contracts/plugin.js' -import { CsvExportPlugin } from '../plugins/csv-export-plugin.js' -import { WebhookExportPlugin } from '../plugins/webhook-export-plugin.js' -import { resolveHostConfig, settingsFor, type HostConfig } from './config.js' -import { runExportLifecycle, type LifecycleOutcome } from './lifecycle.js' -import { PluginRegistry } from './registry.js' - -export interface PluginHostOptions { - fileConfig?: Partial - env?: Record - log?: (message: string) => void -} - -/** - * Composition root. This is the only module that knows about both `plugins/*` - * and `host/*`; adding a built-in plugin should require a change here and - * nowhere else in `host/`. - */ -export class PluginHost { - private readonly config: HostConfig - private readonly registry = new PluginRegistry() - private readonly log: (message: string) => void - - constructor(options: PluginHostOptions = {}) { - this.config = resolveHostConfig(options.fileConfig ?? {}, options.env ?? {}) - this.log = options.log ?? (() => {}) - - for (const plugin of builtInPlugins()) { - this.registry.register(plugin) - } - } - - registerPlugin(plugin: ExportPlugin): void { - this.registry.register(plugin) - } - - runExport(batch: ExportBatch): LifecycleOutcome { - const plugins = this.registry.resolveAll(this.config.enabledPlugins) - - return runExportLifecycle(plugins, batch, { - failFast: this.config.failFast, - contextFor: (plugin) => this.contextFor(plugin), - }) - } - - private contextFor(plugin: ExportPlugin): PluginContext { - return { - settings: Object.freeze({ ...settingsFor(this.config, plugin.name) }), - log: (message: string) => this.log(`[${plugin.name}] ${message}`), - } - } -} - -function builtInPlugins(): ExportPlugin[] { - return [new CsvExportPlugin(), new WebhookExportPlugin()] -} diff --git a/docs/qualification/fixtures/plugin-host/src/host/registry.ts b/docs/qualification/fixtures/plugin-host/src/host/registry.ts deleted file mode 100644 index 823a910a..00000000 --- a/docs/qualification/fixtures/plugin-host/src/host/registry.ts +++ /dev/null @@ -1,46 +0,0 @@ -import type { ExportPlugin } from '../contracts/plugin.js' - -export class DuplicatePluginError extends Error { - constructor(name: string) { - super(`plugin ${name} is already registered`) - this.name = 'DuplicatePluginError' - } -} - -export class UnknownPluginError extends Error { - constructor(name: string) { - super(`plugin ${name} is not registered`) - this.name = 'UnknownPluginError' - } -} - -/** - * Name -> plugin resolution. Registration order is preserved so lifecycle - * ordering is deterministic. - */ -export class PluginRegistry { - private readonly plugins = new Map() - - register(plugin: ExportPlugin): void { - if (this.plugins.has(plugin.name)) { - throw new DuplicatePluginError(plugin.name) - } - this.plugins.set(plugin.name, plugin) - } - - resolve(name: string): ExportPlugin { - const plugin = this.plugins.get(name) - if (!plugin) { - throw new UnknownPluginError(name) - } - return plugin - } - - resolveAll(names: string[]): ExportPlugin[] { - return names.map((name) => this.resolve(name)) - } - - registeredNames(): string[] { - return [...this.plugins.keys()] - } -} diff --git a/docs/qualification/fixtures/plugin-host/src/plugins/csv-export-plugin.ts b/docs/qualification/fixtures/plugin-host/src/plugins/csv-export-plugin.ts deleted file mode 100644 index a3b1d113..00000000 --- a/docs/qualification/fixtures/plugin-host/src/plugins/csv-export-plugin.ts +++ /dev/null @@ -1,32 +0,0 @@ -import type { ExportBatch, ExportPlugin, ExportResult, PluginContext } from '../contracts/plugin.js' - -/** - * Reference implementation of the intended boundary: depends on - * `contracts/plugin.ts` only and reads every setting from `PluginContext`. - */ -export class CsvExportPlugin implements ExportPlugin { - readonly name = 'csv-export' - readonly version = '1.0.0' - - private delimiter = ',' - private destination = 'file://exports' - - init(context: PluginContext): void { - this.delimiter = context.settings.delimiter ?? this.delimiter - this.destination = context.settings.destination ?? this.destination - context.log(`csv-export writing to ${this.destination}`) - } - - export(batch: ExportBatch): ExportResult { - for (const record of batch.records) { - Object.values(record.fields).join(this.delimiter) - } - - return { - pluginName: this.name, - batchId: batch.batchId, - recordsExported: batch.records.length, - destination: this.destination, - } - } -} diff --git a/docs/qualification/fixtures/plugin-host/src/plugins/webhook-export-plugin.ts b/docs/qualification/fixtures/plugin-host/src/plugins/webhook-export-plugin.ts deleted file mode 100644 index e869bddd..00000000 --- a/docs/qualification/fixtures/plugin-host/src/plugins/webhook-export-plugin.ts +++ /dev/null @@ -1,36 +0,0 @@ -import type { ExportBatch, ExportPlugin, ExportResult, PluginContext } from '../contracts/plugin.js' -// seeded-boundary-violation: a plugin must depend on contracts/plugin.ts only. -// Reaching into host internals couples this plugin to host configuration layering. -import { resolveHostConfig } from '../host/config.js' - -export class WebhookExportPlugin implements ExportPlugin { - readonly name = 'webhook-export' - readonly version = '1.0.0' - - private endpoint = 'https://example.invalid/exports' - private delivered = 0 - - init(context: PluginContext): void { - const hostConfig = resolveHostConfig({}, process.env) - this.endpoint = context.settings.endpoint ?? this.endpoint - - if (hostConfig.failFast) { - context.log('webhook-export running under fail-fast host configuration') - } - } - - export(batch: ExportBatch): ExportResult { - this.delivered += batch.records.length - - return { - pluginName: this.name, - batchId: batch.batchId, - recordsExported: batch.records.length, - destination: this.endpoint, - } - } - - dispose(): void { - this.delivered = 0 - } -} diff --git a/docs/qualification/fixtures/plugin-host/tsconfig.json b/docs/qualification/fixtures/plugin-host/tsconfig.json deleted file mode 100644 index be978f2e..00000000 --- a/docs/qualification/fixtures/plugin-host/tsconfig.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "compilerOptions": { - "target": "ES2022", - "module": "NodeNext", - "moduleResolution": "NodeNext", - "lib": ["ES2022"], - "strict": true, - "types": ["node"], - "noEmit": true - }, - "include": ["src/**/*.ts"] -} diff --git a/docs/qualification/freeze.json b/docs/qualification/freeze.json index e1b95844..f7581ed4 100644 --- a/docs/qualification/freeze.json +++ b/docs/qualification/freeze.json @@ -4,45 +4,26 @@ "algorithm": "sha256 over raw file bytes", "note": "Regenerate deliberately with `npm run qualify:validate -- --write` and say why in the pull request. A silent digest change is a contract change.", "files": { - "docs/qualification/README.md": "4c612f693d3cdfd64f4b911c39991a4bd4a12106a9439cca98feafb62998f9b7", - "docs/qualification/corpus.json": "845d96c516ba3497b837c0c6a949d077e8ebc15aa2c675acfafe269bb0631034", - "docs/qualification/evidence-categories.md": "24fa19cb4e049519df647989e7844c79d3d801bbfa6ed5fa8576327ffba4aa33", - "docs/qualification/examples/receipt-tier1-valid.json": "021dccdc1b79f9a699589ffc7cc978702083ea1260cebeabe6ee77446ef7f834", - "docs/qualification/examples/receipt-tier2-invalid-no-madar-call.json": "6fd064a8968ebd8447e8840d0352878e777ad4d8f678ff65957aa04e9000c9bb", - "docs/qualification/fixtures/ledger-service/README.md": "50d552fc69d6873e2b7c2960872546a78ea6a0a832bce76bdcc170aaa81fa03b", - "docs/qualification/fixtures/ledger-service/package.json": "eeb3350fdbe0349d35dcc66be8a0168b2547d2fd7f49f510f36729e0e9abfaf7", - "docs/qualification/fixtures/ledger-service/src/audit/audit-log.ts": "b6d0919a03c7284bcf6d28be055d1ac04b7e36d46acc7b6e344bc96007ac412f", - "docs/qualification/fixtures/ledger-service/src/auth/request-context.ts": "901c254d5b10471fee0d5658e7380867ca21386435df285e48496080cdec179c", - "docs/qualification/fixtures/ledger-service/src/http/ledger-routes.ts": "ca9b5b0c7db2ce326baa75584687d33dd2d478de0fca633e8501c9af6cbd866e", - "docs/qualification/fixtures/ledger-service/src/outbox/outbox-publisher.ts": "00712c8f193d806cffed473455f0ee6793e845ca05f61eef7e64307f125d8330", - "docs/qualification/fixtures/ledger-service/src/projections/balance-projection.ts": "b3725465dc2a9292805b820698004e3d02df204707e3b4067262e9cf83978b0d", - "docs/qualification/fixtures/ledger-service/src/service/ledger-service.ts": "3199d59abda0ee51503346a7570c780633d5177fe69744d1cb5704e87c88aee4", - "docs/qualification/fixtures/ledger-service/src/store/idempotency-store.ts": "22b0868f890fbce754075ad7ba6d2430cea1b3c3e71a0e2da6f97aaad0207d4e", - "docs/qualification/fixtures/ledger-service/src/store/ledger-store.ts": "7cd4b2763f84bae1de0e53553bcab8e7eb27a365807534fd1fbeab650c004cff", - "docs/qualification/fixtures/ledger-service/tsconfig.json": "bedbf5c76b15f8b2a8f464870bdd4d96057e02c1bc2d858036970c41219e85bf", - "docs/qualification/fixtures/plugin-host/README.md": "20ac3e34dcce8ba946df80ba210f7a194218c3934932eab54fae9df68d63ec8a", - "docs/qualification/fixtures/plugin-host/package.json": "d1eee6217153675f5a0fc243bc290cb1f28b6d8eba4e5d7612d764f77a8172a9", - "docs/qualification/fixtures/plugin-host/src/contracts/plugin.ts": "c97ae63baea6f62b23c1f50c55582e9d889d1489b7b9d02dedf85b45cd4d85b2", - "docs/qualification/fixtures/plugin-host/src/host/config.ts": "3c60cf835b99258e485d2c8ca2588f2b80065c7a502d7d8f5c5263557f996ad9", - "docs/qualification/fixtures/plugin-host/src/host/lifecycle.ts": "98ce4eba4b671b097efd73d7611cbccd6afebac9ecd31aa022ef8dcabe67c81d", - "docs/qualification/fixtures/plugin-host/src/host/plugin-host.ts": "f5ecea001feaf6f86482853cadb55dd8fe1216f6a6f0c489d6972b24b313cc49", - "docs/qualification/fixtures/plugin-host/src/host/registry.ts": "7a9a3096ce6c8f85bbd3800ee206eba28d413e0ea81bbe960181d13aa378501f", - "docs/qualification/fixtures/plugin-host/src/plugins/csv-export-plugin.ts": "cdf91d68941a6f91a14dd2d350c3aa5554e7073f23098c8ef5083aab949f9e9f", - "docs/qualification/fixtures/plugin-host/src/plugins/webhook-export-plugin.ts": "df0b22d2e036451ab2c2cc6c8d07ddfa0303a6ace59e9ca553e4749faba57f39", - "docs/qualification/fixtures/plugin-host/tsconfig.json": "e4702ed4a224f18edab218cff4ac62224b5c427c46d06c111878fad24554a6a0", - "docs/qualification/holdout-policy.md": "1c1c397b224601072e690ab0bb9d1be048a7b4185b7e4a4ace0ad72916778029", - "docs/qualification/receipt-schema.json": "976cb3e4510450889dd9c86f09de6248d9285c8b8a16353461b8e739980ee36f", + "docs/qualification/README.md": "4261326a246885f0fe50fbbd382d845af6b71b63d472fde5b1bd2e0320307020", + "docs/qualification/corpus.json": "faa4511d5d255357fdc6c0b5b28b7f139a84871456b914c8425cb1cf1688ee03", + "docs/qualification/evidence-categories.md": "1f75b04229b805050584eadda201bc67906224d34f562afa2098558296a952cd", + "docs/qualification/examples/receipt-tier1-valid.json": "6bb66edfaeaba4ccb97af33b0d9d5fcada2a5c5a686ebd919e46049722f825dc", + "docs/qualification/examples/receipt-tier2-invalid-no-madar-call.json": "d20124547ba9839c8254e150d3aa06c8961e423e593b5893cbab36e77e829ad8", + "docs/qualification/holdout-policy.md": "92dee1bc216d69f6a651750cf2c95f7f7736df9b1a9c29cc566d8ffeae1894d0", + "docs/qualification/patches/hono-compose-reentrancy-guard.patch": "9355c5bbb05cd5ae4d998ace18d6381f0cba4fd080203d4d02579da3dcf6dea4", + "docs/qualification/patches/hono-error-message-disclosure.patch": "edb79059b72b4f27f5dc8341ba2d9a3617901c402da9ef1f9daf5503e6528d6f", + "docs/qualification/receipt-schema.json": "2266deb3afee2c2bc352dc7ee584b37472f2c240d2e5a233b44fc946ab645994", "docs/qualification/rubrics.json": "35f7648bc72e5f0af11649063b0da01bea37d6c0d8699132c078f0c793abb079", "docs/qualification/stop-rule.md": "7d5b646e0d3369aa1783bc4a594b3ee322e0774515fec6ec43c83336722a4f83", - "docs/qualification/tasks.json": "8f3fdbbfc652f84936c597514ad4f60db4839101c82603b3e7a22415c000517c", - "docs/qualification/tier1.json": "2eca6565770a103e6e9e111be8ab9d15bca1b8a792fd9b97920291e2d9010445", - "docs/qualification/tier2-matrix.json": "681e2dc90c4dfdc2e376d3c5b7e35f46688232b1da98ffdb6befbba6e0bbb510", - "docs/qualification/truth/arch-plugin-host-extension-seam.json": "012098694eeb32a6aff35a24c043301e27f8d45c8d227dc722a2e7fb6e7eae8c", - "docs/qualification/truth/flow-ledger-post-entry.json": "abfd4d1561a0092855d6680839ae368458fd8cc3a1dc338752f33f0ff5552e47", - "docs/qualification/truth/impact-ledger-drop-outbox-publish.json": "9ec23f87d33f7e3cbc4b96e4746802aefe455f0a99bc87f1b5cc629c7677dbfd", - "docs/qualification/truth/plan-plugin-host-object-storage-plugin.json": "0c5ff4bf3d5c84287586b8c1c34a2542b8c157320cb9d4bede5c01d0b4f02673", - "docs/qualification/truth/review-ledger-http-authorization.json": "367a1ad6616aabd03115dc091e717da467b70b9929004b3a2442ee793f995fbf", - "docs/qualification/truth/rootcause-ledger-duplicate-entries.json": "7cf3099d8953ed36629732360cf81ee86190d71cd092e11521f4d80d5e7375dd", - "docs/qualification/validity-rules.md": "bbd51c561fdd39e4da71aeaf05a3a197949d531c18e575f6044231728b181936" + "docs/qualification/tasks.json": "61f7e5185cd4586fc529709d911a724ba748e37b3e5718ed505e429458015117", + "docs/qualification/tier1.json": "5faa20899960c5245030b48c1e652ab0ea0ebaf69c8fdaca44fa37f184aa6292", + "docs/qualification/tier2-matrix.json": "d4a0a071219041f4766382c63ad651b11a64de6b80f5ab7ae4b8435eda985008", + "docs/qualification/truth/arch-unstorage-driver-seam.json": "82e3a9d52c8a19716b30eceb87ca2a2eb9e2107dd3d07498145c0aae70586b89", + "docs/qualification/truth/flow-hono-request-dispatch.json": "eb39c4e14a1932c5440fcdac2c6ba20928fba89120879f2667c8bfafabb612fb", + "docs/qualification/truth/impact-hono-drop-router-fallback.json": "b500e8d16d32069ed4c6ab8014199e01cd0f0ae4d785ee96eed212b3eaa98f2d", + "docs/qualification/truth/plan-unstorage-add-driver.json": "e6332e047475f88ff41358b17211a22b3096be992a3f523673362163ea9f37f2", + "docs/qualification/truth/review-hono-error-handling.json": "5e0653e0fa613cc7a4ee72aad1175d7a95ccce603aaf68afe5cc59b2e15854dd", + "docs/qualification/truth/rootcause-hono-middleware-rerun.json": "6e2005ced5ba7ff768d1b9b3135eb909e98d6c1ead04712bc1b0185c480e429d", + "docs/qualification/validity-rules.md": "a8dba75f71462fdf6bf9cda41fe5ded2b1541cb2eb6e8897d4f318a05d38133c" } } diff --git a/docs/qualification/holdout-policy.md b/docs/qualification/holdout-policy.md index f093cdc7..80e540b8 100644 --- a/docs/qualification/holdout-policy.md +++ b/docs/qualification/holdout-policy.md @@ -10,6 +10,11 @@ they catch regressions — but they cannot detect the failure mode this policy e production behaviour drifting toward the qualification corpus itself. Only a target the rule author has never seen can measure that. +Naturalness and hiddenness are separate properties and neither substitutes for the other. +Every open target in this corpus is a real externally authored repository, which removes +the risk that the target was shaped around its own answer. It does not remove the risk that +production rules are shaped around the target once it is known. + ## Classes | Class | Meaning | @@ -54,7 +59,8 @@ The same limitation makes two other artifacts unavailable: To satisfy this policy, a person other than the production-rule author must: -1. select and pin one TypeScript/Node repository not named anywhere in this repository; +1. select and pin one real, permissively licensed TypeScript/Node repository not named + anywhere in this repository; 2. author two to four task prompts and their independent truth for it, without reading Madar output; 3. author the hidden acceptance test for the bounded-implementation task; @@ -72,6 +78,8 @@ report derived from it must carry this exact line: - A sealed target, prompt, path, or symbol must never appear in production retrieval, ranking, claim, or configuration code. - A sealed target must never be added to the repository's test fixtures. +- A sealed slot must never be filled with a self-authored fixture workspace. That would + satisfy neither naturalness nor hiddenness while appearing to satisfy both. - A failing sealed cell must never be resolved by editing the sealed truth. - The rule author must never request the sealed prompts "just to check whether they are fair". Fairness disputes are resolved by the holder retiring the task, not by disclosure. diff --git a/docs/qualification/patches/hono-compose-reentrancy-guard.patch b/docs/qualification/patches/hono-compose-reentrancy-guard.patch new file mode 100644 index 00000000..7f5c6335 --- /dev/null +++ b/docs/qualification/patches/hono-compose-reentrancy-guard.patch @@ -0,0 +1,13 @@ +diff --git a/src/compose.ts b/src/compose.ts +index b1d4508..76f84c3 100644 +--- a/src/compose.ts ++++ b/src/compose.ts +@@ -30,7 +30,7 @@ export const compose = ( + * @returns {Promise} - A promise that resolves to the context. + */ + async function dispatch(i: number): Promise { +- if (i <= index) { ++ if (i < index) { + throw new Error('next() called multiple times') + } + index = i diff --git a/docs/qualification/patches/hono-error-message-disclosure.patch b/docs/qualification/patches/hono-error-message-disclosure.patch new file mode 100644 index 00000000..8459b37d --- /dev/null +++ b/docs/qualification/patches/hono-error-message-disclosure.patch @@ -0,0 +1,13 @@ +diff --git a/src/hono-base.ts b/src/hono-base.ts +index e6a7278..6af89b0 100644 +--- a/src/hono-base.ts ++++ b/src/hono-base.ts +@@ -38,7 +38,7 @@ const errorHandler: ErrorHandler = (err, c) => { + return c.newResponse(res.body, res) + } + console.error(err) +- return c.text('Internal Server Error', 500) ++ return c.text(`Internal Server Error: ${err.stack ?? err.message}`, 500) + } + + type GetPath = (request: Request, options?: { env?: E['Bindings'] }) => string diff --git a/docs/qualification/receipt-schema.json b/docs/qualification/receipt-schema.json index 2ff08ebe..5fe01af8 100644 --- a/docs/qualification/receipt-schema.json +++ b/docs/qualification/receipt-schema.json @@ -196,6 +196,7 @@ "prompt_contract_failure", "answer_contract_failure", "target_revision_mismatch", + "patch_application_failure", "package_revision_mismatch", "dependency_lock_mismatch", "isolation_failure", diff --git a/docs/qualification/tasks.json b/docs/qualification/tasks.json index ba8109eb..e06bdad6 100644 --- a/docs/qualification/tasks.json +++ b/docs/qualification/tasks.json @@ -4,20 +4,20 @@ "prompt_hash_algorithm": "sha256 over the exact UTF-8 prompt text, no trailing newline", "tasks": [ { - "id": "arch-plugin-host-extension-seam", - "name": "Explain the extension architecture and its dependency direction", + "id": "arch-unstorage-driver-seam", + "name": "Explain the driver extension seam and mount resolution", "category": "architecture-understanding", - "target": "qual-plugin-host", + "target": "unstorage", "tiers": [ 1, 2 ], "status": "frozen", "prompt": { - "text": "Describe the extension architecture of this workspace. What is the stable extension surface, what are the module layers and the allowed dependency direction between them, and does any module currently depend in the wrong direction?", - "sha256": "3f4837829b97f6bdd273a9f751ee31141b928e7d77c5bfe850ed1fb2ff05bfd8" + "text": "Describe this library's extension architecture. What is the stable surface a new backend has to implement, how is a backend selected for a given key at call time, and what work does the core do that a backend never sees?", + "sha256": "5c3722d182b5c36478ecabd2d1fb24f4f3eb40eba6e12032da89bebf12ef1d9f" }, - "truth_ref": "truth/arch-plugin-host-extension-seam.json", + "truth_ref": "truth/arch-unstorage-driver-seam.json", "scoring": { "tier1_method": "evidence_obligation_recall", "tier2_method": "blinded_rubric", @@ -33,7 +33,7 @@ "author_role": "benchmark author", "authored_at": "2026-08-12", "derived_from": [ - "fixture source authored in the same change, read directly" + "unjs/unstorage @ e6be6135832f350ca16f9a77432e1d4f0aa85ed7, read directly from the pinned checkout" ], "madar_derived_sources_used": [], "inspected_madar_output_before_freeze": false, @@ -42,20 +42,20 @@ } }, { - "id": "flow-ledger-post-entry", - "name": "Trace the post-entry execution flow to every durable side effect", + "id": "flow-hono-request-dispatch", + "name": "Trace the request lifecycle from entry point to response", "category": "execution-flow-explanation", - "target": "qual-ledger-service", + "target": "hono", "tiers": [ 1, 2 ], "status": "frozen", "prompt": { - "text": "Trace what happens when a client posts a new ledger entry. Follow the path from the HTTP handler through to every durable side effect and every downstream consumer, and say which step each side effect happens in.", - "sha256": "4f727f2d70f0df36aa08fc208a27d6dcf1b1ed0a8b26a05b1e1dde32b57e29aa" + "text": "Trace what happens to an incoming request from the application's entry point until a response is returned. Cover how the path is resolved, how a route is matched, how handlers and middleware are run, and how errors and unmatched routes are handled. Say which steps are skipped in the fast path.", + "sha256": "c276d363f0b10c00b75df925f2e658f3ee7af085872bcbcc64a7a4252c7b3b35" }, - "truth_ref": "truth/flow-ledger-post-entry.json", + "truth_ref": "truth/flow-hono-request-dispatch.json", "scoring": { "tier1_method": "evidence_obligation_recall", "tier2_method": "ordered_path_rubric", @@ -71,7 +71,7 @@ "author_role": "benchmark author", "authored_at": "2026-08-12", "derived_from": [ - "fixture source authored in the same change, read directly" + "honojs/hono @ 26de73133b8552f56ba72e025ecd82b08900d796, read directly from the pinned checkout" ], "madar_derived_sources_used": [], "inspected_madar_output_before_freeze": false, @@ -80,20 +80,20 @@ } }, { - "id": "impact-ledger-drop-outbox-publish", - "name": "Impact of dropping outbox publication on the write path", + "id": "impact-hono-drop-router-fallback", + "name": "Impact of removing the router fallback strategy", "category": "impact-analysis", - "target": "qual-ledger-service", + "target": "hono", "tiers": [ 1, 2 ], "status": "frozen", "prompt": { - "text": "What breaks if the write path stops emitting outbox events after a ledger entry is appended? List every module whose behaviour changes and say whether the failure is loud or silent.", - "sha256": "9ccbea6fc04c64b868676c858d2f34c0e808bca307e3481ccda62e3c9713cb6c" + "text": "What breaks if the default router is replaced by the regular-expression router alone, with no fallback to another router implementation? List every module whose behaviour changes, say which modules are unaffected and why, and say whether each failure is loud or silent.", + "sha256": "4d05391549ee28a142a9f24960e43e415cf6c831c29dce720aa74e72c7ab9ac2" }, - "truth_ref": "truth/impact-ledger-drop-outbox-publish.json", + "truth_ref": "truth/impact-hono-drop-router-fallback.json", "scoring": { "tier1_method": "evidence_obligation_recall", "tier2_method": "affected_set_precision_recall", @@ -109,7 +109,7 @@ "author_role": "benchmark author", "authored_at": "2026-08-12", "derived_from": [ - "fixture source authored in the same change, read directly" + "honojs/hono @ 26de73133b8552f56ba72e025ecd82b08900d796, read directly from the pinned checkout" ], "madar_derived_sources_used": [], "inspected_madar_output_before_freeze": false, @@ -118,20 +118,20 @@ } }, { - "id": "rootcause-ledger-duplicate-entries", - "name": "Root-cause a duplicate entry under retried requests", + "id": "rootcause-hono-middleware-rerun", + "name": "Root-cause downstream middleware running twice", "category": "bug-root-cause-investigation", - "target": "qual-ledger-service", + "target": "hono-seeded-compose", "tiers": [ 1, 2 ], "status": "frozen", "prompt": { - "text": "Under load, a client that retries a request with the same idempotency key sometimes ends up with two ledger entries instead of one. Find the root cause and explain the exact ordering of operations that produces the duplicate.", - "sha256": "3d6f2f70a3b21508c17637cce0d9d94ccbea241adaeb2223192664c8d9856164" + "text": "A middleware that awaits the next function twice used to fail loudly with an error. It now silently runs the rest of the chain a second time, so downstream handlers execute twice for one request. Find the root cause and explain the exact mechanism that produces the second execution.", + "sha256": "637cd655712c2e7d96a3566ed0b311d9d7dcfe0d9cfb8b699f5f1c52a721eaf0" }, - "truth_ref": "truth/rootcause-ledger-duplicate-entries.json", + "truth_ref": "truth/rootcause-hono-middleware-rerun.json", "scoring": { "tier1_method": "evidence_obligation_recall", "tier2_method": "single_root_cause_adjudication", @@ -147,7 +147,7 @@ "author_role": "benchmark author", "authored_at": "2026-08-12", "derived_from": [ - "seeded defect introduced deliberately while authoring the fixture" + "seeded defect deliberately injected into honojs/hono @ 26de73133b8552f56ba72e025ecd82b08900d796 via patches/hono-compose-reentrancy-guard.patch" ], "madar_derived_sources_used": [], "inspected_madar_output_before_freeze": false, @@ -156,20 +156,20 @@ } }, { - "id": "plan-plugin-host-object-storage-plugin", - "name": "Plan a bounded new built-in plugin", + "id": "plan-unstorage-add-driver", + "name": "Plan a bounded new backend implementation", "category": "implementation-planning", - "target": "qual-plugin-host", + "target": "unstorage", "tiers": [ 1, 2 ], "status": "frozen", "prompt": { - "text": "Plan the change needed to add a new built-in export plugin that writes batches to object storage. Do not widen the extension surface and do not change host modules that are unrelated to registering a plugin. List the files you would add or change and why each one is required.", - "sha256": "b031c780c0810bf79463045b1b7494ec431860173514c87a2572918a7603674f" + "text": "Plan the change needed to add a new built-in storage backend for an S3-compatible object store. Do not widen the public extension interface and do not change modules unrelated to adding a backend. List the files you would add or change, the ones you must not hand-edit, and why for each.", + "sha256": "b8a0a2b4a0593346d1333d96ada6c339a0763611df1ddab51079369e0c62c7ff" }, - "truth_ref": "truth/plan-plugin-host-object-storage-plugin.json", + "truth_ref": "truth/plan-unstorage-add-driver.json", "scoring": { "tier1_method": "evidence_obligation_recall", "tier2_method": "blinded_rubric", @@ -190,7 +190,7 @@ "author_role": "benchmark author", "authored_at": "2026-08-12", "derived_from": [ - "fixture source authored in the same change, read directly" + "unjs/unstorage @ e6be6135832f350ca16f9a77432e1d4f0aa85ed7, read directly from the pinned checkout" ], "madar_derived_sources_used": [], "inspected_madar_output_before_freeze": false, @@ -199,20 +199,20 @@ } }, { - "id": "review-ledger-http-authorization", - "name": "Review the ledger HTTP surface for authorization defects", + "id": "review-hono-error-handling", + "name": "Review the framework error path for disclosure defects", "category": "review-security", - "target": "qual-ledger-service", + "target": "hono-seeded-error-disclosure", "tiers": [ 1, 2 ], "status": "frozen", "prompt": { - "text": "Review the ledger HTTP surface for authorization defects before merge. For each defect, name the handler, say what an attacker can do, and cite the code that proves it.", - "sha256": "d9a634a7bc95d610223b7d0d3c827b514d14f7871e1b248cc46285821f37ea73" + "text": "Review this framework's default error-handling path before merge. Identify anything that could expose internal information to a client, name the exact function, say what an attacker learns, and cite the code that proves it.", + "sha256": "722278ed3fa2386f1e407b4fd53a7d4b89272f2adba4297f878ad3fd645d8e47" }, - "truth_ref": "truth/review-ledger-http-authorization.json", + "truth_ref": "truth/review-hono-error-handling.json", "scoring": { "tier1_method": "evidence_obligation_recall", "tier2_method": "seeded_defect_detection", @@ -228,7 +228,7 @@ "author_role": "benchmark author", "authored_at": "2026-08-12", "derived_from": [ - "seeded defect introduced deliberately while authoring the fixture" + "seeded defect deliberately injected into honojs/hono @ 26de73133b8552f56ba72e025ecd82b08900d796 via patches/hono-error-message-disclosure.patch" ], "madar_derived_sources_used": [], "inspected_madar_output_before_freeze": false, diff --git a/docs/qualification/tier1.json b/docs/qualification/tier1.json index 7d264b54..a2a37c32 100644 --- a/docs/qualification/tier1.json +++ b/docs/qualification/tier1.json @@ -1,75 +1,84 @@ { "contract_version": "1.0.0", "frozen_at": "2026-08-12", - "purpose": "The deterministic subset that a pull request can run. It measures whether the evidence needed to answer each frozen task was present in the context artifact, and whether readiness was refused when it should have been. It never scores answer quality and never runs an agent.", + "purpose": "The small deterministic subset that a pull request can run. It measures whether the evidence needed to answer each frozen task was present in the context artifact, and whether readiness was correctly refused on the negative-trust probes. It never scores answer quality and never runs an agent.", "properties": { "deterministic": true, - "requires_network": false, + "requires_network": true, + "requires_network_reason": "Targets are natural external repositories pinned at immutable commits. Preparing a cell means cloning at the pinned SHA and, for the two seeded targets, applying the recorded patch. A local mirror or a warm clone cache satisfies this without changing the result, because the SHA and the patch fix the content exactly.", "requires_model_provider": false, "requires_api_spend": false, "trials_per_cell": 1, "trials_rationale": "One trial is sufficient because no sampling occurs. If a Tier 1 cell is ever observed to be non-deterministic, that is itself a defect and the cell must be reported as invalid, not retried until it passes." }, + "preparation": { + "steps": [ + "For each distinct target, clone the pinned repository and check out the recorded ref.", + "For a git_patched target, apply the recorded patch with `git apply` and fail the cell if the patch does not apply cleanly.", + "Verify each target's cited_blobs against the prepared tree with `git rev-parse :` before the cell runs." + ], + "on_preparation_failure": "The cell is invalid with reason `target_revision_mismatch` (or `incomplete_receipt` if verification could not run). It is never silently skipped and never counted as a pass." + }, "cells": [ { - "task_id": "arch-plugin-host-extension-seam", - "target_id": "qual-plugin-host", + "task_id": "arch-unstorage-driver-seam", + "target_id": "unstorage", "method": "evidence_obligation_recall" }, { - "task_id": "flow-ledger-post-entry", - "target_id": "qual-ledger-service", + "task_id": "flow-hono-request-dispatch", + "target_id": "hono", "method": "evidence_obligation_recall" }, { - "task_id": "impact-ledger-drop-outbox-publish", - "target_id": "qual-ledger-service", + "task_id": "impact-hono-drop-router-fallback", + "target_id": "hono", "method": "evidence_obligation_recall" }, { - "task_id": "rootcause-ledger-duplicate-entries", - "target_id": "qual-ledger-service", + "task_id": "rootcause-hono-middleware-rerun", + "target_id": "hono-seeded-compose", "method": "evidence_obligation_recall" }, { - "task_id": "plan-plugin-host-object-storage-plugin", - "target_id": "qual-plugin-host", + "task_id": "plan-unstorage-add-driver", + "target_id": "unstorage", "method": "evidence_obligation_recall" }, { - "task_id": "review-ledger-http-authorization", - "target_id": "qual-ledger-service", + "task_id": "review-hono-error-handling", + "target_id": "hono-seeded-error-disclosure", "method": "evidence_obligation_recall" } ], "negative_trust_probes": [ { - "id": "neg-ledger-absent-feature", - "target_id": "qual-ledger-service", + "id": "neg-unstorage-absent-encryption", + "target_id": "unstorage", "prompt": { - "text": "How does this service enforce per-currency spending limits before an entry is accepted?", - "sha256": "306a894807d16ea4167cf85826dad35504cf9594a93b1ef286b6682dc278efe2" + "text": "How does this library encrypt values at rest before writing them through a backend, and where are the keys managed?", + "sha256": "cab0b45defcdcaf0b88c5613384f84b237c1d365e0a1f3b3bbd16543fe6db37d" }, - "ground_truth": "No spending-limit logic exists anywhere in the workspace. The only validation on the write path is bearer-token authentication, account membership on the post route, and type checks on amountMinor, currency, and idempotencyKey.", + "ground_truth": "There is no cryptography anywhere in src/. The only transformation applied on write is stringify, and on read destr. No key management exists.", "required_behaviour": [ "The artifact must not report a ready state.", - "The artifact must declare that the requested behaviour was not found rather than presenting the nearest validation code as if it implemented limits.", + "The artifact must declare that the requested behaviour was not found rather than presenting stringify, destr, or a driver's own storage as encryption.", "No fabricated path or symbol may appear in the evidence set." ], "failure_is": "false_ready" }, { - "id": "neg-plugin-host-absent-loader", - "target_id": "qual-plugin-host", + "id": "neg-hono-absent-matcher-persistence", + "target_id": "hono", "prompt": { - "text": "Where does the host load third-party plugins from disk at runtime, and how are they sandboxed?", - "sha256": "bd88479cb5d0b2d190d681cfe5f31123cbe4c72d09c064c153aee1e70c802209" + "text": "Where does this framework persist its compiled route matchers to disk so they survive a process restart, and how is that cache invalidated?", + "sha256": "6a173a57d204d86260380b3f9fedcb4dcc5c90962258e849b508fddaa3e07116" }, - "ground_truth": "There is no dynamic loading and no sandboxing. builtInPlugins() returns a hardcoded array, and registerPlugin accepts an already-constructed object from the caller.", + "ground_truth": "Nothing in the routing path touches the filesystem. The only reuse is in-process: SmartRouter rebinds its match method to the winning router and drops its route buffer after the first match. There is no disk cache and therefore no invalidation.", "required_behaviour": [ "The artifact must not report a ready state.", - "The artifact must state that no dynamic loading or sandboxing mechanism exists.", - "Presenting resolveHostConfig or PluginRegistry as a loader counts as a false-ready failure." + "The artifact must state that no on-disk matcher cache exists.", + "Presenting the in-memory memoization in SmartRouter as a persistent cache counts as a false-ready failure." ], "failure_is": "false_ready" } @@ -82,12 +91,13 @@ "Adding a qualification path, symbol, prompt, or repository name to production retrieval, ranking, or claim logic.", "Relaxing a truth file to match observed output.", "Lowering min_critical_fact_recall to make a cell pass.", - "Marking a failing cell not_measured. not_measured is for runs that could not be measured, never for runs that were measured and failed." + "Marking a failing cell not_measured. not_measured is for runs that could not be measured, never for runs that were measured and failed.", + "Replacing a natural target with a self-authored fixture that is easier to satisfy." ] }, "calibration_status": { "state": "pre_registered_uncalibrated", - "explanation": "The thresholds in the truth files were written from the fixture source before Madar was ever run against these fixtures, and the author did not inspect Madar output before freezing. No cell in this subset has a recorded pass or fail yet, so it is unknown how many currently pass.", + "explanation": "The thresholds in the truth files were written from the pinned repository sources before Madar was ever run against them, and the author did not inspect Madar output before freezing. No cell in this subset has a recorded pass or fail yet, so it is unknown how many currently pass. Pre-registering the threshold before calibrating against observed output is the correct order, not a shortfall — calibrating first would make the threshold a description of current behaviour rather than a test of it.", "consequence": "The first execution of this subset (issue #661) is a measurement, not a regression check. A failing cell on first execution is a product finding to be filed as a linked issue, not a reason to edit this contract." } } diff --git a/docs/qualification/tier2-matrix.json b/docs/qualification/tier2-matrix.json index b06c75a8..314561a1 100644 --- a/docs/qualification/tier2-matrix.json +++ b/docs/qualification/tier2-matrix.json @@ -4,26 +4,32 @@ "status": "planned", "status_meaning": "The matrix shape, arms, repeat count, and reporting rules are frozen now so they cannot be chosen after seeing results. No Tier 2 cell has been executed under this contract.", "blocked_by": [ - "Independent truth for the Tier 2 git targets does not exist (corpus.json marks both as pinned_no_truth).", "Blinded review is unavailable in a single-author context (rubrics.json#/blinding/current_status).", - "The sealed holdout slot is unsatisfied (holdout-policy.md)." + "The sealed holdout slot is unsatisfied (holdout-policy.md).", + "No truth file has been reviewed by a second person; every one carries review_status: unreviewed." ], "dimensions": { - "targets": ["qual-ledger-service", "qual-plugin-host", "qual-unkey", "qual-payload", "qual-sealed-a"], + "targets": [ + "hono", + "unstorage", + "hono-seeded-compose", + "hono-seeded-error-disclosure", + "sealed-holdout-a" + ], "tasks": [ - "arch-plugin-host-extension-seam", - "flow-ledger-post-entry", - "impact-ledger-drop-outbox-publish", - "rootcause-ledger-duplicate-entries", - "plan-plugin-host-object-storage-plugin", - "review-ledger-http-authorization" + "arch-unstorage-driver-seam", + "flow-hono-request-dispatch", + "impact-hono-drop-router-fallback", + "rootcause-hono-middleware-rerun", + "plan-unstorage-add-driver", + "review-hono-error-handling" ], "arms": ["native", "madar"], "cache_modes": ["cold", "warm"], "trials_per_cell": 5 }, "trial_rationale": "Five trials per cell is the smallest count that lets a per-cell median be reported with a visible min/max spread while keeping a full sweep affordable. It is not powered for a small effect size. Any claim that depends on a difference smaller than the observed per-cell spread must be reported as not established, whatever the medians say.", - "pairing": "Both arms of a cell run against the same target revision, the same prompt text, the same tool permission list apart from Madar tools, the same cache mode, and the same trial index. A cell where the two arms differ in any identity field is invalid, not a result.", + "pairing": "Both arms of a cell run against the same prepared target tree — same commit, same applied patch, same verified blob digests — with the same prompt text, the same tool permission list apart from Madar tools, the same cache mode, and the same trial index. A cell where the two arms differ in any identity field is invalid, not a result.", "reporting": { "unit": "one row per target per task per cache mode", "statistics": ["median", "min", "max", "n_valid", "n_invalid"], @@ -39,6 +45,7 @@ "Every task in the sweep has a truth file with review_status other than unreviewed.", "A reviewer who did not author the change under evaluation is available for blinded scoring.", "Isolation mode is active and the environment receipt matches the pinned contract.", + "Every prepared target tree has had its cited_blobs verified against the pinned ref.", "The sealed holdout slot is either filled or the sweep is published with the holdout column explicitly marked unsatisfied." ] } diff --git a/docs/qualification/truth/arch-plugin-host-extension-seam.json b/docs/qualification/truth/arch-plugin-host-extension-seam.json deleted file mode 100644 index d3654820..00000000 --- a/docs/qualification/truth/arch-plugin-host-extension-seam.json +++ /dev/null @@ -1,125 +0,0 @@ -{ - "contract_version": "1.0.0", - "task_id": "arch-plugin-host-extension-seam", - "target": "qual-plugin-host", - "category": "architecture-understanding", - "provenance": { - "authored_by": "madar-655-qualification-agent", - "authored_at": "2026-08-12", - "derived_from": ["fixture source authored in the same change, read directly"], - "madar_derived_sources_used": [], - "inspected_madar_output_before_freeze": false, - "independent_of_production_rule_author": false, - "review_status": "unreviewed" - }, - "critical_facts": [ - { - "id": "extension-surface", - "statement": "The stable extension surface is the ExportPlugin interface in src/contracts/plugin.ts; every plugin implements it.", - "criticality": "critical", - "evidence": [{ "path": "src/contracts/plugin.ts", "symbol": "ExportPlugin" }] - }, - { - "id": "layer-direction", - "statement": "The intended dependency direction is contracts <- plugins and contracts <- host; plugins must not depend on host.", - "criticality": "critical", - "evidence": [ - { "path": "src/contracts/plugin.ts", "symbol": "ExportPlugin" }, - { "path": "src/plugins/csv-export-plugin.ts", "symbol": "CsvExportPlugin" } - ] - }, - { - "id": "composition-root", - "statement": "src/host/plugin-host.ts is the composition root and the only module that imports both host internals and concrete plugins.", - "criticality": "critical", - "evidence": [ - { "path": "src/host/plugin-host.ts", "symbol": "PluginHost" }, - { "path": "src/host/plugin-host.ts", "symbol": "builtInPlugins" } - ] - }, - { - "id": "registry-resolution", - "statement": "PluginRegistry maps names to plugins, preserves registration order, and rejects duplicate names with DuplicatePluginError.", - "criticality": "critical", - "evidence": [{ "path": "src/host/registry.ts", "symbol": "PluginRegistry" }] - }, - { - "id": "lifecycle-ownership", - "statement": "runExportLifecycle owns init -> export -> dispose ordering and, when failFast is false, isolates a failing plugin while still running and disposing the rest.", - "criticality": "critical", - "evidence": [{ "path": "src/host/lifecycle.ts", "symbol": "runExportLifecycle" }] - }, - { - "id": "config-layering", - "statement": "resolveHostConfig layers defaults, then file config, then environment overrides; plugins never read configuration themselves and receive only scoped settings through PluginContext.", - "criticality": "supporting", - "evidence": [ - { "path": "src/host/config.ts", "symbol": "resolveHostConfig" }, - { "path": "src/host/plugin-host.ts", "symbol": "PluginHost.contextFor" } - ] - }, - { - "id": "boundary-violation", - "statement": "src/plugins/webhook-export-plugin.ts imports resolveHostConfig from host/config.ts. This is the only wrong-direction dependency in the workspace.", - "criticality": "critical", - "seeded_defect_id": "seeded-boundary-violation", - "evidence": [{ "path": "src/plugins/webhook-export-plugin.ts", "symbol": "WebhookExportPlugin.init" }] - } - ], - "correct_uncertainty": [ - { - "id": "no-enforcement", - "requirement": "The dependency direction is a convention. Nothing in the workspace enforces it — there is no lint rule, no build boundary, and no runtime check. An answer that presents the direction as enforced is wrong." - }, - { - "id": "static-only", - "requirement": "All of this is read from static imports. No runtime trace exists, so claims about what actually loads at runtime are hypotheses." - } - ], - "unsupported_claim_traps": [ - { - "id": "dynamic-discovery", - "claim": "Plugins are discovered dynamically at runtime from a plugins directory or from configuration.", - "why_false": "builtInPlugins() in src/host/plugin-host.ts returns a hardcoded array. Configuration only selects which already-registered names are enabled." - }, - { - "id": "registry-enforces-boundary", - "claim": "PluginRegistry or PluginHost enforces that plugins only depend on contracts.", - "why_false": "Neither module inspects plugin imports. The boundary is documentation only, which is exactly why the webhook plugin can violate it." - }, - { - "id": "csv-also-violates", - "claim": "CsvExportPlugin also reaches into host modules.", - "why_false": "src/plugins/csv-export-plugin.ts imports from ../contracts/plugin.js only." - }, - { - "id": "failfast-still-disposes", - "claim": "With failFast enabled, already-initialized plugins are still disposed after a failure.", - "why_false": "runExportLifecycle throws the PluginFailure immediately when failFast is true, so the dispose loop is never reached." - } - ], - "tier1_obligations": { - "required_evidence_paths": [ - "src/contracts/plugin.ts", - "src/host/plugin-host.ts", - "src/host/registry.ts", - "src/host/lifecycle.ts", - "src/plugins/webhook-export-plugin.ts" - ], - "required_evidence_symbols": ["ExportPlugin", "PluginHost", "PluginRegistry", "runExportLifecycle"], - "min_critical_fact_recall": 1.0, - "must_not_report_ready_when": [ - "any required_evidence_path is absent from the evidence set", - "the wrong-direction import in src/plugins/webhook-export-plugin.ts is not represented in the graph" - ] - }, - "tier2_scoring": { - "method": "blinded_rubric", - "critical_facts_required_for_pass": [ - "extension-surface", - "layer-direction", - "composition-root", - "boundary-violation" - ] - } -} diff --git a/docs/qualification/truth/arch-unstorage-driver-seam.json b/docs/qualification/truth/arch-unstorage-driver-seam.json new file mode 100644 index 00000000..2279d4ea --- /dev/null +++ b/docs/qualification/truth/arch-unstorage-driver-seam.json @@ -0,0 +1,143 @@ +{ + "contract_version": "1.0.0", + "task_id": "arch-unstorage-driver-seam", + "target": "unstorage", + "category": "architecture-understanding", + "provenance": { + "authored_by": "madar-655-qualification-agent", + "authored_at": "2026-08-12", + "derived_from": [ + "unjs/unstorage @ e6be6135832f350ca16f9a77432e1d4f0aa85ed7, read directly from the pinned checkout" + ], + "madar_derived_sources_used": [], + "inspected_madar_output_before_freeze": false, + "independent_of_production_rule_author": false, + "review_status": "unreviewed" + }, + "critical_facts": [ + { + "id": "driver-interface", + "statement": "The stable extension surface is the Driver interface in src/types.ts. Exactly three members are required — hasItem, getItem, getKeys. Everything else (setItem, setItems, setItemRaw, getItems, getItemRaw, removeItem, getMeta, clear, dispose, watch, name, flags, options, getInstance) is optional.", + "criticality": "critical", + "evidence": [{ "path": "src/types.ts", "symbol": "Driver" }] + }, + { + "id": "driver-factory", + "statement": "Drivers are authored as a DriverFactory — a function taking options and returning a Driver — declared in src/drivers/utils/index.ts and default-exported from each driver module.", + "criticality": "critical", + "evidence": [ + { "path": "src/drivers/utils/index.ts", "symbol": "DriverFactory" }, + { "path": "src/drivers/memory.ts", "symbol": "driver" } + ] + }, + { + "id": "mount-registry", + "statement": "createStorage holds a StorageCTX with mounts (base -> Driver) and mountpoints (the list of bases). The root mount is the empty string and defaults to the in-memory driver when no driver option is supplied.", + "criticality": "critical", + "evidence": [ + { "path": "src/storage.ts", "symbol": "createStorage" }, + { "path": "src/drivers/memory.ts", "symbol": "driver" } + ] + }, + { + "id": "longest-prefix-resolution", + "statement": "getMount resolves a key by scanning mountpoints and returning the first base for which key.startsWith(base). mount() pushes the new base and re-sorts mountpoints by descending length, so the longest matching prefix wins; an unmatched key falls back to the root mount.", + "criticality": "critical", + "evidence": [{ "path": "src/storage.ts", "symbol": "getMount" }] + }, + { + "id": "relative-key-stripping", + "statement": "The core strips the mount base before calling the driver — relativeKey is key.slice(base.length) — so a driver never sees the mountpoint it is mounted under.", + "criticality": "critical", + "evidence": [{ "path": "src/storage.ts", "symbol": "getMount" }] + }, + { + "id": "core-only-work", + "statement": "Work the core does that a driver never sees: key normalization (normalizeKey / normalizeBaseKey / joinKeys in src/utils.ts), value serialization on write (stringify) and deserialization on read (destr), wrapping every driver call in asyncCall, grouping multi-key operations per mount in runBatch, and fanning watch events out to registered listeners through onChange.", + "criticality": "critical", + "evidence": [ + { "path": "src/utils.ts", "symbol": "normalizeKey" }, + { "path": "src/_utils.ts", "symbol": "asyncCall" }, + { "path": "src/storage.ts", "symbol": "runBatch" } + ] + }, + { + "id": "optional-method-fallbacks", + "statement": "Optional driver methods are feature-detected, not required: a driver with no setItem is silently treated as read-only, getItemRaw falls back to getItem plus deserializeRaw, and getItems/setItems fall back to per-item calls.", + "criticality": "critical", + "evidence": [ + { "path": "src/storage.ts", "symbol": "setItem" }, + { "path": "src/storage.ts", "symbol": "getItemRaw" } + ] + }, + { + "id": "generated-driver-index", + "statement": "src/_drivers.ts is generated, not hand-maintained. Its header says \"Auto-generated using scripts/gen-drivers. Do not manually edit!\", and the build script runs gen-drivers before bundling.", + "criticality": "supporting", + "evidence": [ + { "path": "src/_drivers.ts", "symbol": "module header" }, + { "path": "scripts/gen-drivers.ts", "symbol": "driverEntries" } + ] + } + ], + "correct_uncertainty": [ + { + "id": "structural-contract-only", + "requirement": "The Driver contract is enforced only by TypeScript structural typing. Nothing validates a driver object at runtime, which is why a driver missing an optional method degrades silently instead of failing. An answer that presents the contract as runtime-enforced is wrong." + }, + { + "id": "static-hypothesis", + "requirement": "All of this is read from static source. No runtime trace exists, so claims about which driver actually serves a key in a deployed system are hypotheses." + } + ], + "unsupported_claim_traps": [ + { + "id": "exact-mount-match", + "claim": "A key is matched to a mount by exact equality with the mountpoint.", + "why_false": "getMount uses key.startsWith(base) over a list sorted by descending length in mount(), so resolution is longest-prefix, not exact." + }, + { + "id": "readonly-throws", + "claim": "Writing through a driver that does not implement setItem raises an error.", + "why_false": "storage.setItem returns early with a `// Readonly` comment. The write is silently discarded." + }, + { + "id": "core-encrypts", + "claim": "The core encrypts, hashes, or otherwise protects values before handing them to a driver.", + "why_false": "The only transformation on write is stringify; on read it is destr. There is no cryptography anywhere in src/." + }, + { + "id": "hand-edit-driver-index", + "claim": "Registering a new driver means adding an entry to src/_drivers.ts.", + "why_false": "That file is generated by scripts/gen-drivers.ts during the build and is explicitly marked do-not-edit." + }, + { + "id": "root-unmountable", + "claim": "The root mount can be unmounted like any other.", + "why_false": "unmount returns immediately when the normalized base is empty, so the root driver cannot be removed." + } + ], + "tier1_obligations": { + "required_evidence_paths": [ + "src/types.ts", + "src/storage.ts", + "src/drivers/utils/index.ts", + "src/utils.ts" + ], + "required_evidence_symbols": ["Driver", "createStorage", "getMount", "DriverFactory"], + "min_critical_fact_recall": 1.0, + "must_not_report_ready_when": [ + "any required_evidence_path is absent from the evidence set", + "the relationship between createStorage and the Driver interface is neither present in the graph nor declared as unresolved" + ] + }, + "tier2_scoring": { + "method": "blinded_rubric", + "critical_facts_required_for_pass": [ + "driver-interface", + "mount-registry", + "longest-prefix-resolution", + "core-only-work" + ] + } +} diff --git a/docs/qualification/truth/flow-hono-request-dispatch.json b/docs/qualification/truth/flow-hono-request-dispatch.json new file mode 100644 index 00000000..c35f6be2 --- /dev/null +++ b/docs/qualification/truth/flow-hono-request-dispatch.json @@ -0,0 +1,178 @@ +{ + "contract_version": "1.0.0", + "task_id": "flow-hono-request-dispatch", + "target": "hono", + "category": "execution-flow-explanation", + "provenance": { + "authored_by": "madar-655-qualification-agent", + "authored_at": "2026-08-12", + "derived_from": [ + "honojs/hono @ 26de73133b8552f56ba72e025ecd82b08900d796, read directly from the pinned checkout" + ], + "madar_derived_sources_used": [], + "inspected_madar_output_before_freeze": false, + "independent_of_production_rule_author": false, + "review_status": "unreviewed" + }, + "ordered_path": [ + { + "step": 1, + "id": "entrypoint", + "statement": "fetch is the entry point. It forwards to the private #dispatch, passing the execution context and the environment bindings.", + "criticality": "critical", + "evidence": [{ "path": "src/hono-base.ts", "symbol": "Hono.fetch" }] + }, + { + "step": 2, + "id": "head-recursion", + "statement": "A HEAD request is handled by re-entering #dispatch with the method forced to GET and wrapping the result in a new Response with a null body. There is no separate HEAD route matching.", + "criticality": "critical", + "evidence": [{ "path": "src/hono-base.ts", "symbol": "Hono.#dispatch" }] + }, + { + "step": 3, + "id": "path-resolution", + "statement": "The request path is produced by this.getPath, bound at construction to getPath or getPathNoStrict from src/utils/url.ts depending on the strict option.", + "criticality": "critical", + "evidence": [ + { "path": "src/hono-base.ts", "symbol": "Hono.constructor" }, + { "path": "src/utils/url.ts", "symbol": "getPath" } + ] + }, + { + "step": 4, + "id": "route-match", + "statement": "router.match(method, path) returns the match result. With the default SmartRouter the first call replays every buffered route into each candidate router in order, skips any router that throws UnsupportedPathError, then rebinds this.match to the winner and discards the route buffer so later requests go straight to that router.", + "criticality": "critical", + "evidence": [ + { "path": "src/router/smart-router/router.ts", "symbol": "SmartRouter.match" }, + { "path": "src/hono.ts", "symbol": "Hono.constructor" } + ] + }, + { + "step": 5, + "id": "context-construction", + "statement": "A Context is constructed after matching, receiving the resolved path, the match result, the environment, the execution context, and the not-found handler.", + "criticality": "critical", + "evidence": [{ "path": "src/context.ts", "symbol": "Context" }] + }, + { + "step": 6, + "id": "fast-path", + "statement": "When exactly one handler matched, compose is skipped entirely. The handler is invoked directly with a next that assigns the not-found handler's response to c.res, and the promise/synchronous result is normalized inline.", + "criticality": "critical", + "evidence": [{ "path": "src/hono-base.ts", "symbol": "Hono.#dispatch" }] + }, + { + "step": 7, + "id": "compose-chain", + "statement": "With two or more matched handlers, compose builds a recursive dispatch loop. Handler i receives a next closure that calls dispatch(i + 1), and context.req.routeIndex is set to the current index before each handler runs.", + "criticality": "critical", + "evidence": [{ "path": "src/compose.ts", "symbol": "compose" }] + }, + { + "step": 8, + "id": "not-found", + "statement": "Inside the composed chain, when no handler remains and context.finalized is still false, the not-found handler runs and its response becomes the result. The framework default returns the text '404 Not Found' with status 404.", + "criticality": "critical", + "evidence": [ + { "path": "src/compose.ts", "symbol": "compose" }, + { "path": "src/hono-base.ts", "symbol": "notFoundHandler" } + ] + }, + { + "step": 9, + "id": "error-path", + "statement": "A thrown Error inside the composed chain is caught by dispatch, recorded on context.error, and passed to the app error handler; its response is applied even though the context may already be finalized. Outside the chain, #handleError re-throws anything that is not an Error and otherwise delegates to the same handler.", + "criticality": "critical", + "evidence": [ + { "path": "src/compose.ts", "symbol": "compose" }, + { "path": "src/hono-base.ts", "symbol": "Hono.#handleError" } + ] + }, + { + "step": 10, + "id": "finalization-check", + "statement": "After the composed chain resolves, #dispatch throws 'Context is not finalized' if context.finalized is false. That throw is caught by the surrounding try and routed through #handleError, so a forgotten return surfaces as a handled error rather than a hang.", + "criticality": "critical", + "evidence": [{ "path": "src/hono-base.ts", "symbol": "Hono.#dispatch" }] + }, + { + "step": 11, + "id": "response", + "statement": "The response returned is context.res.", + "criticality": "supporting", + "evidence": [{ "path": "src/context.ts", "symbol": "Context" }] + } + ], + "correct_uncertainty": [ + { + "id": "router-not-statically-determined", + "requirement": "Which concrete router serves a match is not decidable from the source alone. SmartRouter selects the first router that does not reject the application's actual route set, at the first match call, and then memoizes it. An answer that states the regular-expression router is always used is over-claiming." + }, + { + "id": "static-hypothesis", + "requirement": "The path is reconstructed from static call sites, not from an observed runtime trace." + } + ], + "unsupported_claim_traps": [ + { + "id": "always-compose", + "claim": "Every request is processed through the middleware composition function.", + "why_false": "#dispatch takes a fast path when the match result contains exactly one handler and never calls compose." + }, + { + "id": "head-separate-route", + "claim": "HEAD requests are matched against their own routes.", + "why_false": "#dispatch recurses with the method rewritten to GET and returns a Response with a null body built from the GET result." + }, + { + "id": "smart-router-per-request", + "claim": "The router strategy is re-evaluated on every request.", + "why_false": "SmartRouter.match rebinds this.match to the winning router and sets its route buffer to undefined after the first successful match." + }, + { + "id": "all-throws-become-500", + "claim": "Anything thrown by a handler is converted into a 500 response.", + "why_false": "#handleError re-throws values that are not Error instances; only Error instances reach the error handler." + }, + { + "id": "env-third-argument", + "claim": "fetch takes the request, the execution context, then the environment.", + "why_false": "fetch is (request, env, executionCtx) and forwards them to #dispatch as (request, rest[1], rest[0], method) — the environment is the second argument." + } + ], + "tier1_obligations": { + "required_evidence_paths": [ + "src/hono-base.ts", + "src/compose.ts", + "src/context.ts", + "src/router/smart-router/router.ts", + "src/utils/url.ts" + ], + "required_evidence_symbols": ["fetch", "compose", "Context", "SmartRouter", "getPath"], + "min_critical_fact_recall": 1.0, + "must_not_report_ready_when": [ + "any required_evidence_path is absent from the evidence set", + "the call from the dispatch entry point into compose is neither present in the graph nor declared as unresolved" + ] + }, + "tier2_scoring": { + "method": "ordered_path_rubric", + "order_sensitive_pairs": [ + ["path-resolution", "route-match"], + ["route-match", "context-construction"], + ["compose-chain", "not-found"], + ["compose-chain", "finalization-check"] + ], + "critical_facts_required_for_pass": [ + "entrypoint", + "path-resolution", + "route-match", + "context-construction", + "fast-path", + "compose-chain", + "error-path" + ] + } +} diff --git a/docs/qualification/truth/flow-ledger-post-entry.json b/docs/qualification/truth/flow-ledger-post-entry.json deleted file mode 100644 index 03b08ea5..00000000 --- a/docs/qualification/truth/flow-ledger-post-entry.json +++ /dev/null @@ -1,174 +0,0 @@ -{ - "contract_version": "1.0.0", - "task_id": "flow-ledger-post-entry", - "target": "qual-ledger-service", - "category": "execution-flow-explanation", - "provenance": { - "authored_by": "madar-655-qualification-agent", - "authored_at": "2026-08-12", - "derived_from": ["fixture source authored in the same change, read directly"], - "madar_derived_sources_used": [], - "inspected_madar_output_before_freeze": false, - "independent_of_production_rule_author": false, - "review_status": "unreviewed" - }, - "ordered_path": [ - { - "step": 1, - "id": "http-entrypoint", - "statement": "POST /accounts/:accountId/entries is handled by postLedgerEntry from createLedgerRouter.", - "criticality": "critical", - "evidence": [{ "path": "src/http/ledger-routes.ts", "symbol": "createLedgerRouter.postLedgerEntry" }] - }, - { - "step": 2, - "id": "authentication", - "statement": "resolveRequestContext parses the bearer token and throws UnauthenticatedError when the token is missing or unknown.", - "criticality": "critical", - "evidence": [{ "path": "src/auth/request-context.ts", "symbol": "resolveRequestContext" }] - }, - { - "step": 3, - "id": "authorization", - "statement": "assertAccountAccess rejects the request with ForbiddenError when the path accountId is not in principal.accountIds.", - "criticality": "critical", - "evidence": [{ "path": "src/auth/request-context.ts", "symbol": "assertAccountAccess" }] - }, - { - "step": 4, - "id": "body-validation", - "statement": "requireNumber and requireString validate amountMinor, currency, and idempotencyKey before the service is called.", - "criticality": "supporting", - "evidence": [{ "path": "src/http/ledger-routes.ts", "symbol": "requireString" }] - }, - { - "step": 5, - "id": "replay-check", - "statement": "LedgerService.postEntry first asks IdempotencyStore.find for a prior reservation and returns the stored entry when one exists.", - "criticality": "critical", - "evidence": [ - { "path": "src/service/ledger-service.ts", "symbol": "LedgerService.postEntry" }, - { "path": "src/store/idempotency-store.ts", "symbol": "IdempotencyStore.find" } - ] - }, - { - "step": 6, - "id": "ledger-append", - "statement": "LedgerStore.appendEntry appends a new immutable entry and allocates the entryId. Durable side effect.", - "criticality": "critical", - "side_effect": "durable", - "evidence": [{ "path": "src/store/ledger-store.ts", "symbol": "LedgerStore.appendEntry" }] - }, - { - "step": 7, - "id": "idempotency-reserve", - "statement": "IdempotencyStore.reserve records the key -> entryId mapping. This happens AFTER the append. Durable side effect.", - "criticality": "critical", - "side_effect": "durable", - "evidence": [{ "path": "src/store/idempotency-store.ts", "symbol": "IdempotencyStore.reserve" }] - }, - { - "step": 8, - "id": "outbox-publish", - "statement": "OutboxPublisher.publish emits ledger.entry.posted synchronously to every subscribed handler. Durable side effect.", - "criticality": "critical", - "side_effect": "durable", - "evidence": [{ "path": "src/outbox/outbox-publisher.ts", "symbol": "OutboxPublisher.publish" }] - }, - { - "step": 9, - "id": "balance-projection", - "statement": "BalanceProjection.applyPosted is the downstream consumer of ledger.entry.posted and updates the derived account balance.", - "criticality": "critical", - "evidence": [{ "path": "src/projections/balance-projection.ts", "symbol": "BalanceProjection.attach" }] - }, - { - "step": 10, - "id": "audit-record", - "statement": "AuditLog.record writes the ledger.entry.posted audit row AFTER the outbox publish. Durable side effect.", - "criticality": "critical", - "side_effect": "durable", - "evidence": [{ "path": "src/audit/audit-log.ts", "symbol": "AuditLog.record" }] - }, - { - "step": 11, - "id": "response", - "statement": "The handler returns 201 with the created entry.", - "criticality": "supporting", - "evidence": [{ "path": "src/http/ledger-routes.ts", "symbol": "createLedgerRouter.postLedgerEntry" }] - } - ], - "durable_side_effects": ["ledger-append", "idempotency-reserve", "outbox-publish", "audit-record"], - "correct_uncertainty": [ - { - "id": "projection-not-wired", - "requirement": "No module in this workspace calls BalanceProjection.attach. The projection consumes ledger.entry.posted only if a composition root subscribes it, and no such composition root exists here. An answer that states the projection is definitely wired is over-claiming; an answer that omits the consumer entirely is incomplete." - }, - { - "id": "static-hypothesis", - "requirement": "The path is reconstructed from static call sites, not from an observed runtime trace." - } - ], - "unsupported_claim_traps": [ - { - "id": "async-outbox", - "claim": "Outbox events are persisted to a table or queue and delivered asynchronously by a worker.", - "why_false": "OutboxPublisher.publish invokes subscribed handlers inline; there is no store, no queue, and no worker." - }, - { - "id": "audit-before-outbox", - "claim": "The audit record is written before the outbox event is published.", - "why_false": "In LedgerService.postEntry the publish call precedes the auditLog.record call." - }, - { - "id": "reserve-before-append", - "claim": "The idempotency key is reserved before the ledger entry is appended.", - "why_false": "reserve is called after appendEntry. This ordering is the seeded defect exercised by rootcause-ledger-duplicate-entries." - }, - { - "id": "store-computes-balance", - "claim": "LedgerStore maintains or returns account balances.", - "why_false": "LedgerStore only appends and reads entries; balances exist only in BalanceProjection." - } - ], - "tier1_obligations": { - "required_evidence_paths": [ - "src/http/ledger-routes.ts", - "src/auth/request-context.ts", - "src/service/ledger-service.ts", - "src/store/ledger-store.ts", - "src/store/idempotency-store.ts", - "src/outbox/outbox-publisher.ts", - "src/audit/audit-log.ts", - "src/projections/balance-projection.ts" - ], - "required_evidence_symbols": [ - "postLedgerEntry", - "LedgerService", - "LedgerStore", - "OutboxPublisher", - "AuditLog", - "BalanceProjection" - ], - "min_critical_fact_recall": 1.0, - "must_not_report_ready_when": [ - "any required_evidence_path is absent from the evidence set", - "the event edge from OutboxPublisher.publish to BalanceProjection is neither present nor declared as unresolved" - ] - }, - "tier2_scoring": { - "method": "ordered_path_rubric", - "order_sensitive_pairs": [ - ["ledger-append", "idempotency-reserve"], - ["outbox-publish", "audit-record"] - ], - "critical_facts_required_for_pass": [ - "http-entrypoint", - "replay-check", - "ledger-append", - "outbox-publish", - "audit-record", - "balance-projection" - ] - } -} diff --git a/docs/qualification/truth/impact-hono-drop-router-fallback.json b/docs/qualification/truth/impact-hono-drop-router-fallback.json new file mode 100644 index 00000000..fcf17161 --- /dev/null +++ b/docs/qualification/truth/impact-hono-drop-router-fallback.json @@ -0,0 +1,140 @@ +{ + "contract_version": "1.0.0", + "task_id": "impact-hono-drop-router-fallback", + "target": "hono", + "category": "impact-analysis", + "provenance": { + "authored_by": "madar-655-qualification-agent", + "authored_at": "2026-08-12", + "derived_from": [ + "honojs/hono @ 26de73133b8552f56ba72e025ecd82b08900d796, read directly from the pinned checkout" + ], + "madar_derived_sources_used": [], + "inspected_madar_output_before_freeze": false, + "independent_of_production_rule_author": false, + "review_status": "unreviewed" + }, + "change_under_analysis": "In src/hono.ts, replace `new SmartRouter({ routers: [new RegExpRouter(), new TrieRouter()] })` with a bare `new RegExpRouter()`.", + "affected_set": [ + { + "id": "construction-site", + "path": "src/hono.ts", + "symbols": ["Hono.constructor"], + "effect": "The change site. The default router becomes a single implementation with no alternative.", + "failure_mode": "n/a", + "criticality": "critical" + }, + { + "id": "smart-router-unused", + "path": "src/router/smart-router/router.ts", + "symbols": ["SmartRouter.match", "SmartRouter.activeRouter"], + "effect": "The catch-and-continue loop over UnsupportedPathError, the memoizing rebind of this.match, and the activeRouter accessor stop being exercised by the default path.", + "failure_mode": "silent", + "criticality": "critical" + }, + { + "id": "regexp-router-throws-escape", + "path": "src/router/reg-exp-router/router.ts", + "symbols": ["RegExpRouter.add", "RegExpRouter.#insertPath"], + "effect": "UnsupportedPathError, raised by #insertPath for a path the trie cannot express, no longer has anything to catch it and propagates to the caller.", + "failure_mode": "loud", + "criticality": "critical" + }, + { + "id": "trie-router-unreachable", + "path": "src/router/trie-router/router.ts", + "symbols": ["TrieRouter"], + "effect": "Stops being reachable as the default fallback. It remains importable and usable if an application passes it explicitly via the router option.", + "failure_mode": "silent", + "criticality": "critical" + } + ], + "timing_fact": { + "statement": "The failure also moves in time. Under SmartRouter, route registration only buffers routes and the adds are replayed inside the first match call, so an unsupported path surfaces on the first request. With a bare RegExpRouter, #addRoute calls router.add immediately, so #insertPath runs during route registration and the error surfaces while the application is being defined.", + "criticality": "critical", + "evidence": [ + { "path": "src/hono-base.ts", "symbol": "Hono.#addRoute" }, + { "path": "src/router/smart-router/router.ts", "symbol": "SmartRouter.add" }, + { "path": "src/router/reg-exp-router/router.ts", "symbol": "RegExpRouter.add" } + ] + }, + "unaffected_set": [ + { + "path": "src/hono-base.ts", + "reason": "#dispatch depends only on the Router interface, not on any implementation. Its own logic is unchanged — though see timing_fact for when #addRoute now raises." + }, + { + "path": "src/compose.ts", + "reason": "Composition runs after matching and never inspects the router." + }, + { + "path": "src/context.ts", + "reason": "The Context receives a match result; the identity of the router that produced it is irrelevant." + }, + { + "path": "src/router.ts", + "reason": "The Router interface and the UnsupportedPathError class are unchanged; only who catches the error changes." + }, + { + "path": "src/request.ts", + "reason": "Request parameter access reads from the match result and is independent of the matcher implementation." + } + ], + "correct_uncertainty": [ + { + "id": "which-paths-are-unsupported", + "requirement": "Which concrete route patterns RegExpRouter rejects is not determined by the modules under analysis — it depends on the trie insertion rules in the reg-exp-router package. A correct answer says that some patterns are rejected and points at #insertPath, without inventing a specific list of rejected patterns it did not read." + } + ], + "unsupported_claim_traps": [ + { + "id": "silent-404", + "claim": "Unsupported route patterns would quietly stop matching and return 404.", + "why_false": "UnsupportedPathError is thrown, not swallowed. Nothing converts it into a not-found response." + }, + { + "id": "handled-as-500", + "claim": "The error would be caught and returned as a 500 by the framework error handler.", + "why_false": "router.match in #dispatch is not inside the try block that guards handler execution, and with a bare RegExpRouter the throw happens during route registration, before any request exists." + }, + { + "id": "performance-only", + "claim": "The change only affects matching performance.", + "why_false": "It removes the only mechanism that recovers from an unsupported path, converting a recoverable condition into a hard failure." + }, + { + "id": "compose-affected", + "claim": "Middleware composition changes because the router changed.", + "why_false": "compose consumes the match result and has no dependency on the router implementation." + } + ], + "tier1_obligations": { + "required_evidence_paths": [ + "src/hono.ts", + "src/router/smart-router/router.ts", + "src/router/reg-exp-router/router.ts", + "src/router/trie-router/router.ts" + ], + "required_evidence_symbols": ["SmartRouter", "RegExpRouter", "TrieRouter", "UnsupportedPathError"], + "min_critical_fact_recall": 1.0, + "must_not_report_ready_when": [ + "the relationship between the constructor in src/hono.ts and the three router implementations is missing from the evidence set and is not declared as unresolved" + ] + }, + "tier2_scoring": { + "method": "affected_set_precision_recall", + "recall_denominator": [ + "construction-site", + "smart-router-unused", + "regexp-router-throws-escape", + "trie-router-unreachable" + ], + "precision_penalty_set": [ + "src/compose.ts", + "src/context.ts", + "src/request.ts", + "src/router.ts" + ], + "critical_facts_required_for_pass": ["regexp-router-throws-escape", "trie-router-unreachable"] + } +} diff --git a/docs/qualification/truth/impact-ledger-drop-outbox-publish.json b/docs/qualification/truth/impact-ledger-drop-outbox-publish.json deleted file mode 100644 index 919abb4f..00000000 --- a/docs/qualification/truth/impact-ledger-drop-outbox-publish.json +++ /dev/null @@ -1,111 +0,0 @@ -{ - "contract_version": "1.0.0", - "task_id": "impact-ledger-drop-outbox-publish", - "target": "qual-ledger-service", - "category": "impact-analysis", - "provenance": { - "authored_by": "madar-655-qualification-agent", - "authored_at": "2026-08-12", - "derived_from": ["fixture source authored in the same change, read directly"], - "madar_derived_sources_used": [], - "inspected_madar_output_before_freeze": false, - "independent_of_production_rule_author": false, - "review_status": "unreviewed" - }, - "change_under_analysis": "Remove the OutboxPublisher.publish calls from LedgerService.postEntry and LedgerService.reverseEntry.", - "affected_set": [ - { - "id": "service-call-sites", - "path": "src/service/ledger-service.ts", - "symbols": ["LedgerService.postEntry", "LedgerService.reverseEntry"], - "effect": "Both call sites disappear; the service no longer emits anything on the write path.", - "failure_mode": "silent", - "criticality": "critical" - }, - { - "id": "publisher-dead", - "path": "src/outbox/outbox-publisher.ts", - "symbols": ["OutboxPublisher.publish"], - "effect": "publish has no remaining callers; subscriptions registered through subscribe never fire.", - "failure_mode": "silent", - "criticality": "critical" - }, - { - "id": "projection-stale", - "path": "src/projections/balance-projection.ts", - "symbols": ["BalanceProjection.applyPosted", "BalanceProjection.applyReversed", "BalanceProjection.balanceFor"], - "effect": "The derived read model stops advancing. balanceFor keeps returning the last value before the change, or undefined, while the ledger keeps growing.", - "failure_mode": "silent", - "criticality": "critical" - } - ], - "unaffected_set": [ - { - "path": "src/store/ledger-store.ts", - "reason": "The append-only entry log is the write model and is unchanged; entries are still created and readable." - }, - { - "path": "src/audit/audit-log.ts", - "reason": "AuditLog.record is called directly by LedgerService and does not depend on the outbox." - }, - { - "path": "src/store/idempotency-store.ts", - "reason": "Retry suppression is independent of event publication." - }, - { - "path": "src/auth/request-context.ts", - "reason": "Authentication and authorization run before the service and are unaffected." - }, - { - "path": "src/http/ledger-routes.ts", - "reason": "Both handlers still return 201 with the created entry; the HTTP contract does not change." - } - ], - "correct_uncertainty": [ - { - "id": "no-wiring-site", - "requirement": "No module in this workspace calls BalanceProjection.attach, so the projection's subscription is inferred from the event names it subscribes to rather than observed at a wiring site. The impact on the projection must be stated as conditional on that wiring." - } - ], - "unsupported_claim_traps": [ - { - "id": "loud-failure", - "claim": "Removing the publish calls causes an error, a failed request, or a visible test failure.", - "why_false": "Nothing checks that an event was published. Every failure mode here is silent divergence between the write model and the read model." - }, - { - "id": "ledger-store-affected", - "claim": "LedgerStore stops recording entries or loses data.", - "why_false": "appendEntry is called before publish and is untouched." - }, - { - "id": "audit-affected", - "claim": "The audit trail is lost.", - "why_false": "AuditLog.record is a direct call from LedgerService, not an outbox subscriber." - } - ], - "tier1_obligations": { - "required_evidence_paths": [ - "src/service/ledger-service.ts", - "src/outbox/outbox-publisher.ts", - "src/projections/balance-projection.ts" - ], - "required_evidence_symbols": ["OutboxPublisher", "BalanceProjection", "LedgerService"], - "min_critical_fact_recall": 1.0, - "must_not_report_ready_when": [ - "the consumer side of ledger.entry.posted is missing from the evidence set and is not declared as unresolved" - ] - }, - "tier2_scoring": { - "method": "affected_set_precision_recall", - "recall_denominator": ["service-call-sites", "publisher-dead", "projection-stale"], - "precision_penalty_set": [ - "src/store/ledger-store.ts", - "src/audit/audit-log.ts", - "src/store/idempotency-store.ts", - "src/auth/request-context.ts", - "src/http/ledger-routes.ts" - ], - "critical_facts_required_for_pass": ["projection-stale"] - } -} diff --git a/docs/qualification/truth/plan-plugin-host-object-storage-plugin.json b/docs/qualification/truth/plan-plugin-host-object-storage-plugin.json deleted file mode 100644 index c7ef0088..00000000 --- a/docs/qualification/truth/plan-plugin-host-object-storage-plugin.json +++ /dev/null @@ -1,129 +0,0 @@ -{ - "contract_version": "1.0.0", - "task_id": "plan-plugin-host-object-storage-plugin", - "target": "qual-plugin-host", - "category": "implementation-planning", - "provenance": { - "authored_by": "madar-655-qualification-agent", - "authored_at": "2026-08-12", - "derived_from": ["fixture source authored in the same change, read directly"], - "madar_derived_sources_used": [], - "inspected_madar_output_before_freeze": false, - "independent_of_production_rule_author": false, - "review_status": "unreviewed" - }, - "reference_plan": { - "files_to_add": [ - { - "new_path": "src/plugins/object-storage-export-plugin.ts", - "why": "New ExportPlugin implementation. Must import from ../contracts/plugin.js only and read bucket/prefix/endpoint from PluginContext.settings." - } - ], - "files_to_change": [ - { - "path": "src/host/plugin-host.ts", - "why": "builtInPlugins() is the single registration point for built-in plugins; the new plugin is constructed and returned there." - } - ], - "files_that_must_not_change": [ - { - "path": "src/contracts/plugin.ts", - "why": "Adding an object-storage-specific method or field widens the extension surface, which the prompt forbids." - }, - { - "path": "src/host/registry.ts", - "why": "Registration is name-based and already generic; a new plugin needs no registry change." - }, - { - "path": "src/host/lifecycle.ts", - "why": "init/export/dispose ordering is already generic." - }, - { - "path": "src/host/config.ts", - "why": "Enablement flows through HostConfig.enabledPlugins and the EXPORT_PLUGINS environment override, both of which are already name-driven." - } - ], - "enablement": "The plugin runs only when its name appears in HostConfig.enabledPlugins or in the EXPORT_PLUGINS environment override. No code change is required to make that possible.", - "optional": [ - "Implement dispose() to flush or close the storage client, since runExportLifecycle calls dispose on every initialized plugin." - ] - }, - "critical_facts": [ - { - "id": "single-registration-point", - "statement": "builtInPlugins() in src/host/plugin-host.ts is the only place a built-in plugin is registered.", - "criticality": "critical", - "evidence": [{ "path": "src/host/plugin-host.ts", "symbol": "builtInPlugins" }] - }, - { - "id": "contract-only-dependency", - "statement": "The new plugin must depend on src/contracts/plugin.ts only, following CsvExportPlugin rather than WebhookExportPlugin.", - "criticality": "critical", - "evidence": [ - { "path": "src/plugins/csv-export-plugin.ts", "symbol": "CsvExportPlugin" }, - { "path": "src/contracts/plugin.ts", "symbol": "ExportPlugin" } - ] - }, - { - "id": "settings-through-context", - "statement": "Plugin configuration arrives through PluginContext.settings, which PluginHost.contextFor scopes per plugin name.", - "criticality": "critical", - "evidence": [{ "path": "src/host/plugin-host.ts", "symbol": "PluginHost.contextFor" }] - }, - { - "id": "no-surface-widening", - "statement": "src/contracts/plugin.ts must not change.", - "criticality": "critical", - "evidence": [{ "path": "src/contracts/plugin.ts", "symbol": "ExportPlugin" }] - } - ], - "correct_uncertainty": [ - { - "id": "no-object-storage-dependency", - "requirement": "The workspace has no object-storage client and no runtime dependencies at all. A plan must either introduce one explicitly as a new dependency or state that the transport is left abstract; silently assuming an SDK is already available is wrong." - }, - { - "id": "no-tests-present", - "requirement": "The workspace contains no test files, so a plan that claims existing tests will cover the change is unsupported." - } - ], - "unsupported_claim_traps": [ - { - "id": "extend-contract", - "claim": "Add an ObjectStorage-specific method, option, or field to the ExportPlugin interface.", - "why_false": "That widens the extension surface, which the prompt forbids, and would force every existing plugin to change." - }, - { - "id": "registry-change", - "claim": "PluginRegistry must be modified to know about the new plugin.", - "why_false": "register() and resolve() are name-based and already generic." - }, - { - "id": "copy-webhook", - "claim": "Follow WebhookExportPlugin and read host configuration directly with resolveHostConfig.", - "why_false": "That is the seeded boundary violation; copying it reintroduces a wrong-direction dependency." - } - ], - "tier1_obligations": { - "required_evidence_paths": [ - "src/contracts/plugin.ts", - "src/host/plugin-host.ts", - "src/plugins/csv-export-plugin.ts" - ], - "required_evidence_symbols": ["ExportPlugin", "builtInPlugins", "PluginContext"], - "min_critical_fact_recall": 1.0, - "must_not_report_ready_when": [ - "src/host/plugin-host.ts is absent from the evidence set" - ] - }, - "tier2_scoring": { - "method": "blinded_rubric", - "critical_facts_required_for_pass": ["single-registration-point", "contract-only-dependency", "no-surface-widening"], - "hidden_acceptance_test": { - "required": true, - "status": "unavailable", - "blocking_reason": "An executable hidden acceptance test must be authored and held by a person other than the production-rule author. In the current single-author context it cannot be produced credibly. Until it exists the bounded-implementation score for this task is not_measured and only the plan rubric is scored.", - "human_action_required": "A second maintainer authors an acceptance test asserting that a new plugin can be added by touching exactly src/plugins/.ts and builtInPlugins(), and stores it outside this repository per holdout-policy.md." - } - } -} diff --git a/docs/qualification/truth/plan-unstorage-add-driver.json b/docs/qualification/truth/plan-unstorage-add-driver.json new file mode 100644 index 00000000..a295e520 --- /dev/null +++ b/docs/qualification/truth/plan-unstorage-add-driver.json @@ -0,0 +1,146 @@ +{ + "contract_version": "1.0.0", + "task_id": "plan-unstorage-add-driver", + "target": "unstorage", + "category": "implementation-planning", + "provenance": { + "authored_by": "madar-655-qualification-agent", + "authored_at": "2026-08-12", + "derived_from": [ + "unjs/unstorage @ e6be6135832f350ca16f9a77432e1d4f0aa85ed7, read directly from the pinned checkout" + ], + "madar_derived_sources_used": [], + "inspected_madar_output_before_freeze": false, + "independent_of_production_rule_author": false, + "review_status": "unreviewed" + }, + "reference_plan": { + "files_to_add": [ + { + "new_path": "src/drivers/.ts", + "why": "One module per driver. Default-export a DriverFactory from ./utils/index.ts, implement the three required members (hasItem, getItem, getKeys) and whichever optional members the backend supports, and export a named options type so the generator can pick it up." + } + ], + "files_to_change": [], + "files_that_must_not_be_hand_edited": [ + { + "path": "src/_drivers.ts", + "why": "Generated by scripts/gen-drivers.ts, which enumerates src/drivers and re-derives the option type imports. The build script runs gen-drivers before bundling, so a hand edit is overwritten." + } + ], + "files_that_must_not_change": [ + { + "path": "src/types.ts", + "why": "Adding a backend-specific member to Driver widens the public extension interface, which the prompt forbids and which would ripple to all 34 existing drivers." + }, + { + "path": "src/storage.ts", + "why": "Mounting and key resolution are driver-agnostic; createStorage needs no knowledge of a new backend." + }, + { + "path": "package.json", + "why": "The exports map already publishes ./drivers/* as a wildcard subpath, so a new driver module is exported without an edit." + } + ], + "conventions_to_follow": [ + "Validate required options with createRequiredError / createError from src/drivers/utils/index.ts so failures carry the [unstorage] [] prefix.", + "Set a DRIVER_NAME constant and expose it as the driver's name, as src/drivers/memory.ts does.", + "Implement dispose when the backend holds a client or timers, because storage.dispose fans out to every mounted driver.", + "Leave setItem off only if the backend is genuinely read-only, and know that the core will then discard writes silently." + ] + }, + "critical_facts": [ + { + "id": "one-module-per-driver", + "statement": "A driver is a single module under src/drivers that default-exports a DriverFactory.", + "criticality": "critical", + "evidence": [ + { "path": "src/drivers/utils/index.ts", "symbol": "DriverFactory" }, + { "path": "src/drivers/memory.ts", "symbol": "driver" } + ] + }, + { + "id": "generated-index", + "statement": "src/_drivers.ts must not be hand-edited; scripts/gen-drivers.ts regenerates it from the contents of src/drivers as part of the build.", + "criticality": "critical", + "evidence": [ + { "path": "src/_drivers.ts", "symbol": "module header" }, + { "path": "scripts/gen-drivers.ts", "symbol": "driverEntries" } + ] + }, + { + "id": "no-interface-widening", + "statement": "The Driver interface in src/types.ts must not change; the three required members plus the existing optional members are sufficient for a new backend.", + "criticality": "critical", + "evidence": [{ "path": "src/types.ts", "symbol": "Driver" }] + }, + { + "id": "wildcard-export", + "statement": "package.json already exposes ./drivers/* as a wildcard subpath export, so publishing the new module requires no packaging change.", + "criticality": "critical", + "evidence": [{ "path": "package.json", "symbol": "exports" }] + }, + { + "id": "option-validation-helpers", + "statement": "Option validation uses createError and createRequiredError from src/drivers/utils/index.ts.", + "criticality": "supporting", + "evidence": [{ "path": "src/drivers/utils/index.ts", "symbol": "createRequiredError" }] + } + ], + "correct_uncertainty": [ + { + "id": "no-s3-client-present", + "requirement": "No S3 or object-store client exists in the repository. A plan must either introduce one explicitly as a new dependency, following how other network-backed drivers declare theirs, or state that the transport is left abstract. Assuming an SDK is already available is unsupported." + }, + { + "id": "generator-reads-type-exports", + "requirement": "The generator discovers the options type by scanning the module's exported type names. A plan that never mentions exporting a named options type leaves the generated entry incomplete, but the exact naming rule is decided by scripts/gen-drivers.ts and should not be invented." + } + ], + "unsupported_claim_traps": [ + { + "id": "edit-driver-index", + "claim": "Add an import and an entry to src/_drivers.ts so the driver is registered.", + "why_false": "That file is generated and explicitly marked do-not-edit; the build regenerates it." + }, + { + "id": "add-exports-entry", + "claim": "Add a new subpath to the exports map in package.json.", + "why_false": "./drivers/* is already a wildcard export." + }, + { + "id": "extend-driver-interface", + "claim": "Add object-store-specific options or methods to the Driver interface.", + "why_false": "That widens the extension surface, which the prompt forbids." + }, + { + "id": "register-in-storage", + "claim": "createStorage or the mount logic must learn about the new driver.", + "why_false": "Drivers are passed in by the caller through the driver option or mount; the core never enumerates driver types." + } + ], + "tier1_obligations": { + "required_evidence_paths": [ + "src/types.ts", + "src/drivers/utils/index.ts", + "src/drivers/memory.ts", + "src/_drivers.ts" + ], + "required_evidence_symbols": ["Driver", "DriverFactory", "createRequiredError"], + "min_critical_fact_recall": 1.0, + "must_not_report_ready_when": [ + "src/drivers/utils/index.ts is absent from the evidence set", + "the generated nature of src/_drivers.ts is neither represented nor declared as unresolved" + ] + }, + "tier2_scoring": { + "method": "blinded_rubric", + "critical_facts_required_for_pass": ["one-module-per-driver", "generated-index", "no-interface-widening"], + "hidden_acceptance_test": { + "required": true, + "status": "unavailable", + "blocking_reason": "An executable hidden acceptance test must be authored and held by a person other than the production-rule author. In the current single-author context it cannot be produced credibly. Until it exists the bounded-implementation score for this task is not_measured and only the plan rubric is scored.", + "human_action_required": "A second maintainer authors an acceptance test asserting that a new driver can be added by adding exactly one module under src/drivers and running the generator, with no edit to src/types.ts, src/storage.ts, src/_drivers.ts, or package.json, and stores it outside this repository per holdout-policy.md." + } + } +} diff --git a/docs/qualification/truth/review-hono-error-handling.json b/docs/qualification/truth/review-hono-error-handling.json new file mode 100644 index 00000000..25277e22 --- /dev/null +++ b/docs/qualification/truth/review-hono-error-handling.json @@ -0,0 +1,118 @@ +{ + "contract_version": "1.0.0", + "task_id": "review-hono-error-handling", + "target": "hono-seeded-error-disclosure", + "category": "review-security", + "provenance": { + "authored_by": "madar-655-qualification-agent", + "authored_at": "2026-08-12", + "derived_from": [ + "seeded defect deliberately injected into honojs/hono @ 26de73133b8552f56ba72e025ecd82b08900d796 via patches/hono-error-message-disclosure.patch" + ], + "madar_derived_sources_used": [], + "inspected_madar_output_before_freeze": false, + "independent_of_production_rule_author": false, + "review_status": "unreviewed" + }, + "seeded_defect": { + "id": "seeded-error-message-disclosure", + "path": "src/hono-base.ts", + "symbol": "errorHandler", + "patch": "patches/hono-error-message-disclosure.patch", + "description": "The framework's default error handler builds the 500 response body from the thrown error's stack, falling back to its message, instead of returning a fixed 'Internal Server Error' string.", + "attacker_capability": "Any request that triggers an unhandled error returns a stack trace to the client. That discloses absolute filesystem paths, the internal module layout, dependency names and versions, and often the failing query, identifier, or credential fragment embedded in the error message. It turns every unhandled error into reconnaissance.", + "proving_evidence": [ + { "path": "src/hono-base.ts", "symbol": "errorHandler", "note": "the patched line builds the body from err.stack ?? err.message" }, + { "path": "src/hono-base.ts", "symbol": "Hono.#handleError", "note": "the non-composed path routes every Error into this same handler" }, + { "path": "src/compose.ts", "symbol": "compose", "note": "inside the composed chain the handler's return value is assigned to context.res even when the context was already finalized" } + ], + "reachability": "Reachable from both dispatch paths: #handleError for the fast path and for post-chain failures, and the onError branch inside compose for errors raised by middleware or handlers." + }, + "critical_facts": [ + { + "id": "default-handler-leaks", + "statement": "The default error handler returns internal error detail in the response body.", + "criticality": "critical", + "evidence": [{ "path": "src/hono-base.ts", "symbol": "errorHandler" }] + }, + { + "id": "reachable-from-both-paths", + "statement": "Both the composed chain's onError branch and #handleError converge on the same default handler, so the disclosure is not confined to one route shape.", + "criticality": "critical", + "evidence": [ + { "path": "src/compose.ts", "symbol": "compose" }, + { "path": "src/hono-base.ts", "symbol": "Hono.#handleError" } + ] + }, + { + "id": "fix-is-fixed-string", + "statement": "The fix is to return a constant message with status 500 and keep the detail in the server-side log that the handler already writes.", + "criticality": "critical", + "evidence": [{ "path": "src/hono-base.ts", "symbol": "errorHandler" }] + } + ], + "acceptable_additional_findings": [ + { + "id": "override-dependency", + "statement": "An application that installs its own onError replaces this handler, so real-world exposure depends on the application not overriding the default.", + "scored_as": "supporting, not required" + }, + { + "id": "server-side-logging", + "statement": "The handler also writes the full error to the server log. That is appropriate, not a client-facing disclosure.", + "scored_as": "supporting, not required" + }, + { + "id": "finalized-override", + "statement": "In compose, the error handler's response is applied even when the context was already finalized, so a partially written response can be replaced by the error body.", + "scored_as": "supporting, not required" + } + ], + "correct_uncertainty": [ + { + "id": "message-content-unknown", + "requirement": "What a stack trace actually reveals depends on the deployment — bundling, source maps, and working directory. A correct answer states the class of information disclosed rather than asserting that specific secrets are always present." + } + ], + "unsupported_claim_traps": [ + { + "id": "http-exception-leaks", + "claim": "The HTTPException branch is the disclosure, because it returns the exception's message to the client.", + "why_false": "HTTPException.getResponse builds its response from a message the application author supplied deliberately. That is intended behaviour and is unchanged from the upstream code.", + "counts_as": "false_positive" + }, + { + "id": "not-found-leaks", + "claim": "The not-found handler discloses the requested path.", + "why_false": "The default not-found handler returns the fixed string '404 Not Found' with status 404 and echoes nothing from the request.", + "counts_as": "false_positive" + }, + { + "id": "compose-is-the-defect", + "claim": "compose is the defect because it passes the raw error onward.", + "why_false": "compose hands the error to whatever onError is configured. Passing the error to a handler is correct; the disclosure is in what the default handler does with it.", + "counts_as": "false_positive" + }, + { + "id": "rethrow-is-a-leak", + "claim": "#handleError re-throwing non-Error values leaks internals to the client.", + "why_false": "A re-thrown non-Error propagates out of fetch to the host runtime; it does not construct a response body.", + "counts_as": "false_positive" + } + ], + "tier1_obligations": { + "required_evidence_paths": ["src/hono-base.ts", "src/compose.ts", "src/http-exception.ts"], + "required_evidence_symbols": ["errorHandler", "compose", "HTTPException"], + "min_critical_fact_recall": 1.0, + "must_not_report_ready_when": [ + "src/hono-base.ts is absent from the evidence set", + "only one of the two error paths is present, since the reachability claim needs both" + ] + }, + "tier2_scoring": { + "method": "seeded_defect_detection", + "required_detections": ["seeded-error-message-disclosure"], + "false_positive_set": ["http-exception-leaks", "not-found-leaks", "compose-is-the-defect", "rethrow-is-a-leak"], + "critical_facts_required_for_pass": ["default-handler-leaks", "reachable-from-both-paths"] + } +} diff --git a/docs/qualification/truth/review-ledger-http-authorization.json b/docs/qualification/truth/review-ledger-http-authorization.json deleted file mode 100644 index f393d8be..00000000 --- a/docs/qualification/truth/review-ledger-http-authorization.json +++ /dev/null @@ -1,118 +0,0 @@ -{ - "contract_version": "1.0.0", - "task_id": "review-ledger-http-authorization", - "target": "qual-ledger-service", - "category": "review-security", - "provenance": { - "authored_by": "madar-655-qualification-agent", - "authored_at": "2026-08-12", - "derived_from": ["seeded defect introduced deliberately while authoring the fixture"], - "madar_derived_sources_used": [], - "inspected_madar_output_before_freeze": false, - "independent_of_production_rule_author": false, - "review_status": "unreviewed" - }, - "seeded_defect": { - "id": "seeded-reversal-authorization", - "path": "src/http/ledger-routes.ts", - "symbol": "createLedgerRouter.reverseLedgerEntry", - "description": "reverseLedgerEntry authenticates the caller but never authorizes them against the account that owns entryId. postLedgerEntry calls assertAccountAccess; the reversal handler does not.", - "attacker_capability": "Any principal holding a valid token can reverse a ledger entry belonging to an account they do not own, which writes a negative entry, emits ledger.entry.reversed, and corrupts the other tenant's derived balance.", - "proving_evidence": [ - { "path": "src/http/ledger-routes.ts", "symbol": "createLedgerRouter.reverseLedgerEntry", "note": "no assertAccountAccess call" }, - { "path": "src/http/ledger-routes.ts", "symbol": "createLedgerRouter.postLedgerEntry", "note": "the same check is present here, showing the omission is asymmetric" }, - { "path": "src/service/ledger-service.ts", "symbol": "LedgerService.reverseEntry", "note": "reads the original entry and its accountId without any principal check" }, - { "path": "src/auth/request-context.ts", "symbol": "assertAccountAccess", "note": "the check that exists and is not applied" } - ], - "corroborating_signal": { - "path": "src/service/ledger-service.ts", - "symbol": "LedgerService.findEntryForAuthorization", - "note": "This helper exists precisely to let a caller resolve the owning account before authorizing, and it has no callers." - } - }, - "critical_facts": [ - { - "id": "missing-authz-on-reversal", - "statement": "reverseLedgerEntry performs authentication but no account-level authorization.", - "criticality": "critical", - "evidence": [{ "path": "src/http/ledger-routes.ts", "symbol": "createLedgerRouter.reverseLedgerEntry" }] - }, - { - "id": "cross-tenant-impact", - "statement": "The consequence is cross-tenant write access: a reversal entry is appended to another account and its balance projection is corrupted.", - "criticality": "critical", - "evidence": [ - { "path": "src/service/ledger-service.ts", "symbol": "LedgerService.reverseEntry" }, - { "path": "src/projections/balance-projection.ts", "symbol": "BalanceProjection.applyReversed" } - ] - }, - { - "id": "check-exists-unused", - "statement": "assertAccountAccess already exists and is applied on the post route, so the fix is to apply it after resolving the entry's owning account.", - "criticality": "critical", - "evidence": [{ "path": "src/auth/request-context.ts", "symbol": "assertAccountAccess" }] - } - ], - "acceptable_additional_findings": [ - { - "id": "unused-authorization-helper", - "statement": "LedgerService.findEntryForAuthorization has no callers.", - "scored_as": "supporting, not required" - }, - { - "id": "unchecked-scopes", - "statement": "Principal.scopes is never read anywhere in the workspace, so scope-based authorization is declared but not enforced.", - "scored_as": "supporting, not required" - }, - { - "id": "generated-request-id", - "statement": "resolveRequestContext generates a random requestId when the x-request-id header is absent, which weakens audit correlation.", - "scored_as": "supporting, not required, and not a security defect on its own" - } - ], - "correct_uncertainty": [ - { - "id": "token-directory-opaque", - "requirement": "TokenDirectory is an interface with no implementation in this workspace. Claims about token strength, expiry, or revocation are unsupported." - } - ], - "unsupported_claim_traps": [ - { - "id": "post-route-also-unauthorized", - "claim": "postLedgerEntry is also missing an authorization check.", - "why_false": "It calls assertAccountAccess with the path accountId before invoking the service.", - "counts_as": "false_positive" - }, - { - "id": "no-authentication", - "claim": "The HTTP surface has no authentication at all.", - "why_false": "Both handlers call resolveRequestContext, which throws UnauthenticatedError on a missing or unknown bearer token.", - "counts_as": "false_positive" - }, - { - "id": "sql-injection", - "claim": "The handlers are vulnerable to injection through request parameters.", - "why_false": "Both stores are in-memory Maps; there is no query language anywhere in the workspace.", - "counts_as": "false_positive" - } - ], - "tier1_obligations": { - "required_evidence_paths": [ - "src/http/ledger-routes.ts", - "src/auth/request-context.ts", - "src/service/ledger-service.ts" - ], - "required_evidence_symbols": ["reverseLedgerEntry", "postLedgerEntry", "assertAccountAccess"], - "min_critical_fact_recall": 1.0, - "must_not_report_ready_when": [ - "src/http/ledger-routes.ts is absent from the evidence set", - "only one of the two route handlers is present, since the defect is only visible by comparing them" - ] - }, - "tier2_scoring": { - "method": "seeded_defect_detection", - "required_detections": ["seeded-reversal-authorization"], - "false_positive_set": ["post-route-also-unauthorized", "no-authentication", "sql-injection"], - "critical_facts_required_for_pass": ["missing-authz-on-reversal", "cross-tenant-impact"] - } -} diff --git a/docs/qualification/truth/rootcause-hono-middleware-rerun.json b/docs/qualification/truth/rootcause-hono-middleware-rerun.json new file mode 100644 index 00000000..32773afe --- /dev/null +++ b/docs/qualification/truth/rootcause-hono-middleware-rerun.json @@ -0,0 +1,99 @@ +{ + "contract_version": "1.0.0", + "task_id": "rootcause-hono-middleware-rerun", + "target": "hono-seeded-compose", + "category": "bug-root-cause-investigation", + "provenance": { + "authored_by": "madar-655-qualification-agent", + "authored_at": "2026-08-12", + "derived_from": [ + "seeded defect deliberately injected into honojs/hono @ 26de73133b8552f56ba72e025ecd82b08900d796 via patches/hono-compose-reentrancy-guard.patch" + ], + "madar_derived_sources_used": [], + "inspected_madar_output_before_freeze": false, + "independent_of_production_rule_author": false, + "review_status": "unreviewed" + }, + "seeded_defect": { + "id": "seeded-compose-reentrancy-guard", + "path": "src/compose.ts", + "symbol": "compose.dispatch", + "patch": "patches/hono-compose-reentrancy-guard.patch", + "root_cause": "The re-entrancy guard at the top of dispatch was weakened from `i <= index` to `i < index`. `index` holds the highest index dispatched so far, so a second call to next() from the same middleware re-enters dispatch with the same argument i+1 while index already equals i+1. Under `<=` that comparison is true and the guard throws; under `<` it is false and the guard is bypassed, so the entire downstream chain runs again.", + "required_ordering_statement": "index is assigned immediately after the guard, so the guard must reject an index that is less than OR equal to the last dispatched index. Restoring `<=` restores the single-entry invariant.", + "observable_condition": "Only reachable when two or more handlers match, because the single-handler fast path in #dispatch never calls compose." + }, + "critical_facts": [ + { + "id": "guard-is-the-cause", + "statement": "The cause is the comparison in the re-entrancy guard inside compose's dispatch function, not the middleware that calls next twice.", + "criticality": "critical", + "evidence": [{ "path": "src/compose.ts", "symbol": "compose.dispatch" }] + }, + { + "id": "index-semantics", + "statement": "index tracks the highest index already dispatched and is written on every entry, so an equal index is exactly the repeated-next case the guard exists to reject.", + "criticality": "critical", + "evidence": [{ "path": "src/compose.ts", "symbol": "compose" }] + }, + { + "id": "next-closure", + "statement": "Each handler is invoked as handler(context, () => dispatch(i + 1)), so calling next twice issues two dispatch calls with the identical argument.", + "criticality": "critical", + "evidence": [{ "path": "src/compose.ts", "symbol": "compose.dispatch" }] + }, + { + "id": "fast-path-exclusion", + "statement": "The single-matched-handler fast path in #dispatch bypasses compose entirely, so the defect cannot manifest on routes with only one handler.", + "criticality": "supporting", + "evidence": [{ "path": "src/hono-base.ts", "symbol": "Hono.#dispatch" }] + } + ], + "correct_uncertainty": [ + { + "id": "no-other-protection", + "requirement": "The guard is the only re-entrancy protection in the composition chain. There is no per-request lock, no idempotency key, and no downstream check. A correct answer must not suggest that some other layer would have caught the second run." + }, + { + "id": "response-assignment-nuance", + "requirement": "Whether the duplicated run is visible in the response depends on context.finalized: the composed result is only assigned when finalized is false or an error was handled. The duplicated side effects still happen either way. An answer that concludes 'no visible effect' because the response is unchanged has missed the defect." + } + ], + "unsupported_claim_traps": [ + { + "id": "middleware-is-buggy", + "claim": "The application middleware is at fault for calling next twice.", + "why_false": "Calling next twice is exactly the condition the framework guard is written to reject with 'next() called multiple times'. The framework silently permitting it is the regression." + }, + { + "id": "finalized-flag", + "claim": "The cause is the context.finalized flag not being set.", + "why_false": "finalized governs whether the composed response is assigned. It never gated entry into dispatch and does not control chain re-entry." + }, + { + "id": "fast-path-cause", + "claim": "The fast path in #dispatch runs the handler an extra time.", + "why_false": "The fast path invokes exactly one handler and is only taken when the match result has length one; it does not enter compose at all." + }, + { + "id": "error-handler-retry", + "claim": "The error handler retries the chain after a failure.", + "why_false": "onError produces a response; it never re-invokes dispatch." + } + ], + "tier1_obligations": { + "required_evidence_paths": ["src/compose.ts", "src/hono-base.ts"], + "required_evidence_symbols": ["compose", "dispatch"], + "min_critical_fact_recall": 1.0, + "must_not_report_ready_when": [ + "src/compose.ts is absent from the evidence set", + "the call from the dispatch entry point into compose is missing from the graph" + ] + }, + "tier2_scoring": { + "method": "single_root_cause_adjudication", + "accepted_root_cause_ids": ["seeded-compose-reentrancy-guard"], + "critical_facts_required_for_pass": ["guard-is-the-cause", "index-semantics", "next-closure"], + "uncertainty_required_for_pass": ["no-other-protection"] + } +} diff --git a/docs/qualification/truth/rootcause-ledger-duplicate-entries.json b/docs/qualification/truth/rootcause-ledger-duplicate-entries.json deleted file mode 100644 index 8030ebfe..00000000 --- a/docs/qualification/truth/rootcause-ledger-duplicate-entries.json +++ /dev/null @@ -1,101 +0,0 @@ -{ - "contract_version": "1.0.0", - "task_id": "rootcause-ledger-duplicate-entries", - "target": "qual-ledger-service", - "category": "bug-root-cause-investigation", - "provenance": { - "authored_by": "madar-655-qualification-agent", - "authored_at": "2026-08-12", - "derived_from": ["seeded defect introduced deliberately while authoring the fixture"], - "madar_derived_sources_used": [], - "inspected_madar_output_before_freeze": false, - "independent_of_production_rule_author": false, - "review_status": "unreviewed" - }, - "seeded_defect": { - "id": "seeded-idempotency-ordering", - "path": "src/service/ledger-service.ts", - "symbol": "LedgerService.postEntry", - "root_cause": "The idempotency key is reserved after the ledger append instead of before it. IdempotencyStore.find and IdempotencyStore.reserve form a check-then-act pair with a state mutation in between, so two concurrent requests carrying the same key can both observe no reservation and both call LedgerStore.appendEntry.", - "required_ordering_statement": "read reservation (find) -> append entry -> write reservation (reserve). The append must move after the reservation, or the reservation and the append must become one atomic operation.", - "also_present_in": [ - { "path": "src/service/ledger-service.ts", "symbol": "LedgerService.reverseEntry" } - ] - }, - "critical_facts": [ - { - "id": "ordering-is-the-cause", - "statement": "The cause is the ordering of reserve relative to appendEntry inside LedgerService.postEntry, not a defect in either store.", - "criticality": "critical", - "evidence": [{ "path": "src/service/ledger-service.ts", "symbol": "LedgerService.postEntry" }] - }, - { - "id": "check-then-act", - "statement": "find and reserve are separate operations with a mutation between them, so the window between them is the race window.", - "criticality": "critical", - "evidence": [ - { "path": "src/store/idempotency-store.ts", "symbol": "IdempotencyStore.find" }, - { "path": "src/store/idempotency-store.ts", "symbol": "IdempotencyStore.reserve" } - ] - }, - { - "id": "append-unconditional", - "statement": "LedgerStore.appendEntry always allocates a new entryId; it has no uniqueness constraint that could reject the second write.", - "criticality": "critical", - "evidence": [{ "path": "src/store/ledger-store.ts", "symbol": "LedgerStore.appendEntry" }] - }, - { - "id": "same-shape-in-reversal", - "statement": "reverseEntry repeats the same ordering, so the fix must cover both commands.", - "criticality": "supporting", - "evidence": [{ "path": "src/service/ledger-service.ts", "symbol": "LedgerService.reverseEntry" }] - } - ], - "correct_uncertainty": [ - { - "id": "single-thread-caveat", - "requirement": "The stores are synchronous in-memory Maps, so within one Node thread the find/append/reserve sequence cannot actually be interleaved. The ordering is still wrong, and it becomes observable as soon as either store performs real I/O or the process is replicated. A correct answer names the ordering defect AND states that the duplicate is not reproducible against the current in-memory implementation without introducing an await or a second process." - } - ], - "unsupported_claim_traps": [ - { - "id": "reserve-is-buggy", - "claim": "IdempotencyStore.reserve is the bug because it overwrites or fails to detect an existing key.", - "why_false": "reserve returns the existing record unchanged when the key is present. It is correct in isolation." - }, - { - "id": "sequence-collision", - "claim": "The duplicate comes from the entryId sequence counter colliding.", - "why_false": "The counter is incremented before each id is built and never reused; the duplicates have distinct entryIds." - }, - { - "id": "missing-validation", - "claim": "The route fails to validate idempotencyKey.", - "why_false": "requireString rejects a missing or empty idempotencyKey before the service runs." - }, - { - "id": "outbox-replay", - "claim": "The outbox republishes the event and the consumer creates a second entry.", - "why_false": "BalanceProjection never writes to LedgerStore; consumers cannot create entries." - } - ], - "tier1_obligations": { - "required_evidence_paths": [ - "src/service/ledger-service.ts", - "src/store/idempotency-store.ts", - "src/store/ledger-store.ts" - ], - "required_evidence_symbols": ["LedgerService", "IdempotencyStore", "LedgerStore"], - "min_critical_fact_recall": 1.0, - "must_not_report_ready_when": [ - "src/service/ledger-service.ts is absent from the evidence set", - "the call from LedgerService.postEntry to IdempotencyStore.reserve is missing from the graph" - ] - }, - "tier2_scoring": { - "method": "single_root_cause_adjudication", - "accepted_root_cause_ids": ["seeded-idempotency-ordering"], - "critical_facts_required_for_pass": ["ordering-is-the-cause", "check-then-act"], - "uncertainty_required_for_pass": ["single-thread-caveat"] - } -} diff --git a/docs/qualification/validity-rules.md b/docs/qualification/validity-rules.md index 31a61cdd..cebbdb35 100644 --- a/docs/qualification/validity-rules.md +++ b/docs/qualification/validity-rules.md @@ -16,7 +16,8 @@ to `validity.invalidation_reasons`: | `missing_attributable_madar_call` | The task contract sets `requires_attributable_madar_call` and the transcript shows no attributable Madar call in the Madar arm. | | `prompt_contract_failure` | The prompt actually delivered to the agent does not hash-match the frozen prompt, or the two arms received different prompts. | | `answer_contract_failure` | The arm produced no answer, a permission request instead of an answer, or a truncated answer. | -| `target_revision_mismatch` | The checked-out target revision or fixture digest differs from `corpus.json`. | +| `target_revision_mismatch` | The checked-out target commit differs from `corpus.json`, or a cited blob digest in the prepared tree does not match the recorded `cited_blobs` entry. | +| `patch_application_failure` | A seeded-defect target's patch did not apply cleanly to the pinned commit, or applied with fuzz. | | `package_revision_mismatch` | The Madar commit, package version, or tarball digest differs from the pinned identity. | | `dependency_lock_mismatch` | The dependency lock digest differs from the pinned identity, or the install used `npm install` rather than `npm ci`. | | `isolation_failure` | `environment.isolation` is false, or the run used a `MADAR_BENCH_CLI_PATH`-style development override. | diff --git a/tests/unit/qualification-contract.test.ts b/tests/unit/qualification-contract.test.ts index 3304042c..9b5bd404 100644 --- a/tests/unit/qualification-contract.test.ts +++ b/tests/unit/qualification-contract.test.ts @@ -1,15 +1,14 @@ import { createHash } from 'node:crypto' import { readFileSync } from 'node:fs' -import { resolve } from 'node:path' - import { createRequire } from 'node:module' +import { resolve } from 'node:path' import { Ajv } from 'ajv' import { describe, expect, it } from 'vitest' -// ajv-formats ships CommonJS whose default export is not callable under -// NodeNext type resolution. Load it the same way the CI validator does so the -// test and `npm run qualify:validate` compile the schema identically. +// ajv-formats ships CommonJS whose default export is not callable under NodeNext type +// resolution. Load it the same way the CI validator does so the test and +// `npm run qualify:validate` compile the schema identically. const addFormats = createRequire(import.meta.url)('ajv-formats') as (ajv: Ajv) => void const ROOT = 'docs/qualification' @@ -25,11 +24,27 @@ function readJson(relativePath: string): T { interface Provenance { authored_by: string authored_at: string + derived_from: string[] madar_derived_sources_used: string[] inspected_madar_output_before_freeze: boolean independent_of_production_rule_author: boolean } +interface Target { + id: string + kind: string + natural?: boolean + status: string + license?: string + holdout_class?: string + prepare?: string[] + patch?: string + base_target?: string + source?: { url: string; ref: string } + cited_blobs?: Record + production_coupling?: { level: string; consequence?: string } +} + interface Task { id: string category: string @@ -43,19 +58,22 @@ interface Task { const corpus = readJson<{ contract_version: string - targets: Array<{ id: string; tier: number; kind: string; status: string; holdout_class?: string; source?: { ref: string } }> + targets: Target[] + proxy_targets: unknown[] + forbidden_target_symbols: Record }>(`${ROOT}/corpus.json`) const tasks = readJson<{ contract_version: string; tasks: Task[] }>(`${ROOT}/tasks.json`) const rubrics = readJson<{ - dimensions: Record + dimensions: Record methods: Record blinding: { current_status: string } }>(`${ROOT}/rubrics.json`) const tier1 = readJson<{ - properties: { deterministic: boolean; requires_network: boolean; requires_api_spend: boolean } + properties: { deterministic: boolean; requires_model_provider: boolean; requires_api_spend: boolean } + preparation: { steps: string[]; on_preparation_failure: string } cells: Array<{ task_id: string; target_id: string }> - negative_trust_probes: Array<{ id: string; prompt: { text: string; sha256: string } }> + negative_trust_probes: Array<{ id: string; target_id: string; prompt: { text: string; sha256: string } }> gate: { forbidden_remedies: string[] } calibration_status: { state: string } }>(`${ROOT}/tier1.json`) @@ -63,27 +81,68 @@ const tier2 = readJson<{ status: string; dimensions: { trials_per_cell: number } const receiptSchema = readJson>(`${ROOT}/receipt-schema.json`) const freeze = readJson<{ contract_version: string; files: Record }>(`${ROOT}/freeze.json`) +const evaluationTargets = corpus.targets.filter((target) => target.kind !== 'sealed') + describe('qualification corpus manifest', () => { - it('pins every target with an immutable revision or a frozen digest', () => { - for (const target of corpus.targets) { - if (target.kind === 'git') { - expect(target.source?.ref).toMatch(/^[0-9a-f]{40}$/) - } - if (target.kind === 'fixture') { - expect(target.status).toBe('frozen') + it('uses only natural externally authored targets, with no fixture proxies', () => { + expect(evaluationTargets.length).toBeGreaterThan(0) + expect(corpus.proxy_targets).toEqual([]) + + for (const target of evaluationTargets) { + expect(target.natural).toBe(true) + expect(target.kind === 'git' || target.kind === 'git_patched').toBe(true) + } + }) + + it('pins every target at an immutable commit with a license and prepare steps', () => { + for (const target of evaluationTargets) { + expect(target.source?.ref).toMatch(/^[0-9a-f]{40}$/) + expect(target.source?.url).toMatch(/^https:\/\//) + expect(target.license).toBeTruthy() + expect(target.prepare?.length).toBeGreaterThan(0) + expect(target.status).toBe('frozen') + } + }) + + it('records a frozen blob digest for every path its truth may cite', () => { + for (const target of evaluationTargets) { + const blobs = Object.entries(target.cited_blobs ?? {}) + + expect(blobs.length).toBeGreaterThan(0) + for (const [, blob] of blobs) { + expect(blob).toMatch(/^[0-9a-f]{40}$/) } } }) - it('keeps Tier 2 git targets marked as having no independent truth yet', () => { - const gitTargets = corpus.targets.filter((target) => target.kind === 'git') + it('seeds defects as patches against the pinned commit of a real repository', () => { + const patched = corpus.targets.filter((target) => target.kind === 'git_patched') + + expect(patched.length).toBeGreaterThan(0) + for (const target of patched) { + const base = corpus.targets.find((candidate) => candidate.id === target.base_target) + expect(base?.source?.ref).toBe(target.source?.ref) + + const patch = readDoc(`${ROOT}/${target.patch}`) + expect(patch.startsWith('diff --git ')).toBe(true) - expect(gitTargets.length).toBeGreaterThan(0) - for (const target of gitTargets) { - expect(target.status).toBe('pinned_no_truth') + const touched = [...patch.matchAll(/^\+\+\+ b\/(.+)$/gm)].map((match) => match[1]) + expect(touched.length).toBeGreaterThan(0) + for (const path of touched) { + expect(Object.keys(target.cited_blobs ?? {})).toContain(path) + } } }) + it('discloses where a target overlaps a shipped framework adapter', () => { + const hono = corpus.targets.find((target) => target.id === 'hono') + const unstorage = corpus.targets.find((target) => target.id === 'unstorage') + + expect(hono?.production_coupling?.level).toBe('declared_framework_adapter') + expect(hono?.production_coupling?.consequence).toContain('not evidence about frameworks that have no adapter') + expect(unstorage?.production_coupling?.level).toBe('none_found') + }) + it('keeps the sealed holdout slot visible and explicitly unsatisfied', () => { const sealed = corpus.targets.filter((target) => target.holdout_class === 'sealed') @@ -112,13 +171,23 @@ describe('qualification task definitions', () => { } }) - it('records a truth owner and asserts no Madar-derived source for every task', () => { + it('never names the coupled framework inside a prompt for that target', () => { + const coupled = tasks.tasks.filter((task) => task.target.startsWith('hono')) + + expect(coupled.length).toBeGreaterThan(0) + for (const task of coupled) { + expect(task.prompt.text.toLowerCase()).not.toContain('hono') + } + }) + + it('records a truth owner, a real derivation source, and no Madar-derived source', () => { for (const task of tasks.tasks) { const truth = readJson<{ provenance: Provenance }>(`${ROOT}/${task.truth_ref}`) for (const provenance of [task.truth_provenance, truth.provenance]) { expect(provenance.authored_by.length).toBeGreaterThan(0) expect(provenance.authored_at.length).toBeGreaterThan(0) + expect(provenance.derived_from.length).toBeGreaterThan(0) expect(provenance.madar_derived_sources_used).toEqual([]) expect(provenance.inspected_madar_output_before_freeze).toBe(false) expect(provenance.independent_of_production_rule_author).toBe(false) @@ -126,6 +195,36 @@ describe('qualification task definitions', () => { } }) + it('cites only paths recorded in the target blob manifest', () => { + for (const task of tasks.tasks) { + const truth = readJson>(`${ROOT}/${task.truth_ref}`) + const target = corpus.targets.find((candidate) => candidate.id === task.target) + const cited = new Set() + + const collect = (node: unknown): void => { + if (Array.isArray(node)) { + node.forEach(collect) + return + } + if (node && typeof node === 'object') { + for (const [key, value] of Object.entries(node)) { + if (key === 'path' && typeof value === 'string') { + cited.add(value) + } else if (key !== 'new_path') { + collect(value) + } + } + } + } + collect(truth) + + expect(cited.size).toBeGreaterThan(0) + for (const path of cited) { + expect(Object.keys(target?.cited_blobs ?? {})).toContain(path) + } + } + }) + it('does not use the same scoring method for every category', () => { const methods = new Set(tasks.tasks.map((task) => task.scoring.tier2_method)) @@ -158,9 +257,18 @@ describe('qualification receipt schema', () => { const validTier1 = readJson>(`${ROOT}/examples/receipt-tier1-valid.json`) const invalidTier2 = readJson>(`${ROOT}/examples/receipt-tier2-invalid-no-madar-call.json`) - it('accepts the published examples', () => { + it('accepts the published examples and ties them to frozen prompts', () => { expect(validate(validTier1)).toBe(true) expect(validate(invalidTier2)).toBe(true) + + for (const receipt of [validTier1, invalidTier2] as Array<{ + task_id: string + identity: { prompts: { user_prompt_sha256: string } } + }>) { + const task = tasks.tasks.find((candidate) => candidate.id === receipt.task_id) + expect(task).toBeTruthy() + expect(receipt.identity.prompts.user_prompt_sha256).toBe(task?.prompt.sha256) + } }) it('keeps every quality dimension not_measured on an invalid run', () => { @@ -191,6 +299,19 @@ describe('qualification receipt schema', () => { expect(validate(mutated)).toBe(false) }) + it('can invalidate a run whose seeded patch did not apply', () => { + const reasons = ( + receiptSchema as { + properties: { + validity: { properties: { invalidation_reasons: { items: { enum: string[] } } } } + } + } + ).properties.validity.properties.invalidation_reasons.items.enum + + expect(reasons).toContain('patch_application_failure') + expect(reasons).toContain('target_revision_mismatch') + }) + it('keeps indexing, context building, and agent cost in separate accounts', () => { const costs = (validTier1 as { costs: Record }).costs @@ -200,27 +321,34 @@ describe('qualification receipt schema', () => { }) describe('qualification Tier 1 subset', () => { - it('is deterministic and runnable in a pull request without spend', () => { + it('is deterministic and runnable in a pull request without model spend', () => { expect(tier1.properties.deterministic).toBe(true) - expect(tier1.properties.requires_network).toBe(false) + expect(tier1.properties.requires_model_provider).toBe(false) expect(tier1.properties.requires_api_spend).toBe(false) }) + it('fails a cell whose target could not be prepared instead of skipping it', () => { + expect(tier1.preparation.steps.length).toBeGreaterThan(0) + expect(tier1.preparation.on_preparation_failure).toContain('never silently skipped') + }) + it('covers every frozen task and freezes each negative-trust probe prompt', () => { expect(tier1.cells.map((cell) => cell.task_id).sort()).toEqual(tasks.tasks.map((task) => task.id).sort()) expect(tier1.negative_trust_probes.length).toBeGreaterThan(0) for (const probe of tier1.negative_trust_probes) { expect(createHash('sha256').update(probe.prompt.text, 'utf8').digest('hex')).toBe(probe.prompt.sha256) + expect(corpus.targets.some((target) => target.id === probe.target_id)).toBe(true) } }) - it('forbids clearing a failure by editing the contract or the production rules', () => { + it('forbids clearing a failure by editing the contract or swapping in a fixture', () => { const remedies = tier1.gate.forbidden_remedies.join('\n') expect(remedies).toContain('Adding a qualification path, symbol, prompt, or repository name to production') expect(remedies).toContain('Relaxing a truth file to match observed output') expect(remedies).toContain('Marking a failing cell not_measured') + expect(remedies).toContain('Replacing a natural target with a self-authored fixture') }) it('states that the thresholds are pre-registered and uncalibrated', () => { @@ -251,16 +379,25 @@ describe('qualification policy documents', () => { expect(policy).toContain('## Current status: unsatisfied') expect(policy).toContain('### Human action required') expect(policy).toContain('sealed holdout unsatisfied; results measure regression only') + expect(policy).toContain('Naturalness and hiddenness are separate properties') }) - it('labels synthetic and package-parity artifacts as non-outcome evidence', () => { + it('separates target naturalness from evidence class', () => { const categories = readDoc(`${ROOT}/evidence-categories.md`) + expect(categories).toContain('## Target naturalness qualifies the evidence') expect(categories).toContain('### E1 — Product outcome evidence') expect(categories).toContain('**Currently held: none.**') - expect(categories).toContain('### E4 — Synthetic or fixture receipts') - expect(categories).toContain('### E5 — Package and parity checks') expect(categories).toContain('E4 proves the reporting pipeline works. It is never agent-outcome evidence.') + expect(categories).toContain('five are in-repo proxies') + expect(categories).toContain('six are git-backed and\ndo pin a URL together with an immutable commit SHA') + }) + + it('records the unenforced retrieval/grader boundary in runtime-proof.json', () => { + const categories = readDoc(`${ROOT}/evidence-categories.md`) + + expect(categories).toContain('Open enforcement gap in E3') + expect(categories).toContain('That isolation is asserted in\nprose. No test, lint rule, or CI check enforces it') }) it('defines transcript and receipt retention', () => { @@ -269,6 +406,7 @@ describe('qualification policy documents', () => { expect(rules).toContain('at least **24 months**') expect(rules).toContain('the raw agent transcript (Tier 2) or the context artifact (Tier 1)') expect(rules).toContain('`not_measured` describes a run that could not be measured') + expect(rules).toContain('`patch_application_failure`') }) it('records which receipt fields v0.32.1 does not emit yet', () => { @@ -280,7 +418,7 @@ describe('qualification policy documents', () => { }) describe('qualification freeze', () => { - it('covers every contract and fixture file with a digest', () => { + it('covers every contract file with a digest', () => { expect(freeze.contract_version).toBe(corpus.contract_version) const paths = Object.keys(freeze.files) @@ -288,8 +426,8 @@ describe('qualification freeze', () => { expect(paths).toContain(`${ROOT}/tasks.json`) expect(paths).toContain(`${ROOT}/rubrics.json`) expect(paths).toContain(`${ROOT}/receipt-schema.json`) - expect(paths).toContain(`${ROOT}/fixtures/ledger-service/src/service/ledger-service.ts`) - expect(paths).toContain(`${ROOT}/fixtures/plugin-host/src/host/plugin-host.ts`) + expect(paths).toContain(`${ROOT}/patches/hono-compose-reentrancy-guard.patch`) + expect(paths).toContain(`${ROOT}/patches/hono-error-message-disclosure.patch`) for (const [path, digest] of Object.entries(freeze.files)) { expect(digest).toBe(createHash('sha256').update(readFileSync(resolve(path))).digest('hex')) From 4aa2370e7e26d91eb6e31c65b022a0cf6a8323ed Mon Sep 17 00:00:00 2001 From: mohammed naji Date: Wed, 12 Aug 2026 10:17:58 +0400 Subject: [PATCH 03/10] docs(qualification): surface the no-measured-evidence limitation (#655) Add a `## Read this first` callout above the corpus description in docs/qualification/README.md stating plainly that the contract has never been executed and currently produces no measured evidence of any kind, followed by the three limits that executing it will not fix: regression only and never generalization while the sealed holdout is unsatisfied, thresholds pre-registered rather than calibrated, and Tier 1's network dependency on cloning the pinned targets. A reviewer should not have to infer this by cross-referencing tier, kind, and status across target entries. A test asserts both the wording and that the callout precedes the corpus description. Also fix three references left stale by the corpus migration: two dead task ids in holdout-policy.md and a `pinned_no_truth` status mention in validity-rules.md that no longer exists in corpus.json. --- docs/qualification/README.md | 23 +++++++++++++++++++++++ docs/qualification/freeze.json | 6 +++--- docs/qualification/holdout-policy.md | 8 ++++---- docs/qualification/validity-rules.md | 2 +- tests/unit/qualification-contract.test.ts | 14 ++++++++++++++ 5 files changed, 45 insertions(+), 8 deletions(-) diff --git a/docs/qualification/README.md b/docs/qualification/README.md index d40d67c9..956359ee 100644 --- a/docs/qualification/README.md +++ b/docs/qualification/README.md @@ -7,6 +7,29 @@ This directory is the independent evaluation contract used to decide whether a r change is safe to ship. It is deliberately separate from [`docs/benchmarks/suite/`](../benchmarks/suite/), which is the product benchmark suite. +## Read this first — what this contract does and does not give you today + +> **This contract has never been executed. It currently produces no measured evidence of +> any kind.** Every target is a real external repository pinned at an immutable commit and +> every task has independent truth, so the corpus *can* produce evidence about natural +> code — but nothing has been run against Madar yet. Executing the Tier 1 subset is +> [#661](https://github.com/mohanagy/madar/issues/661). + +Three further limits apply the moment it *is* executed, and none of them is fixed by +running it: + +1. **Regression only, never generalization.** The sealed holdout slot is `unsatisfied` + because Madar has one author. Every result from this corpus must carry the line + `sealed holdout unsatisfied; results measure regression only`. See + [`holdout-policy.md`](./holdout-policy.md). +2. **Thresholds are pre-registered, not calibrated.** Nobody knows how many Tier 1 cells + currently pass. That is the correct order — a threshold fitted to observed output would + describe current behaviour instead of testing it — but it means the first run is a + measurement, not a pass/fail gate. +3. **Tier 1 needs network access** to clone the pinned targets. It is still deterministic: + the commit SHA and the patch fix the content exactly, and a warm clone cache or local + mirror satisfies it without changing any result. + ## Why a separate corpus exists Two properties are required, and the existing benchmark suite has neither. diff --git a/docs/qualification/freeze.json b/docs/qualification/freeze.json index f7581ed4..67651c76 100644 --- a/docs/qualification/freeze.json +++ b/docs/qualification/freeze.json @@ -4,12 +4,12 @@ "algorithm": "sha256 over raw file bytes", "note": "Regenerate deliberately with `npm run qualify:validate -- --write` and say why in the pull request. A silent digest change is a contract change.", "files": { - "docs/qualification/README.md": "4261326a246885f0fe50fbbd382d845af6b71b63d472fde5b1bd2e0320307020", + "docs/qualification/README.md": "7f4920abbac662f13ec2deea6756edac13bd19090953387209a3df8ee97eea5f", "docs/qualification/corpus.json": "faa4511d5d255357fdc6c0b5b28b7f139a84871456b914c8425cb1cf1688ee03", "docs/qualification/evidence-categories.md": "1f75b04229b805050584eadda201bc67906224d34f562afa2098558296a952cd", "docs/qualification/examples/receipt-tier1-valid.json": "6bb66edfaeaba4ccb97af33b0d9d5fcada2a5c5a686ebd919e46049722f825dc", "docs/qualification/examples/receipt-tier2-invalid-no-madar-call.json": "d20124547ba9839c8254e150d3aa06c8961e423e593b5893cbab36e77e829ad8", - "docs/qualification/holdout-policy.md": "92dee1bc216d69f6a651750cf2c95f7f7736df9b1a9c29cc566d8ffeae1894d0", + "docs/qualification/holdout-policy.md": "f5671c4cd5f5487e21dcb44e34fa98b5520a7b9ec58391ce1f086ebd33e23616", "docs/qualification/patches/hono-compose-reentrancy-guard.patch": "9355c5bbb05cd5ae4d998ace18d6381f0cba4fd080203d4d02579da3dcf6dea4", "docs/qualification/patches/hono-error-message-disclosure.patch": "edb79059b72b4f27f5dc8341ba2d9a3617901c402da9ef1f9daf5503e6528d6f", "docs/qualification/receipt-schema.json": "2266deb3afee2c2bc352dc7ee584b37472f2c240d2e5a233b44fc946ab645994", @@ -24,6 +24,6 @@ "docs/qualification/truth/plan-unstorage-add-driver.json": "e6332e047475f88ff41358b17211a22b3096be992a3f523673362163ea9f37f2", "docs/qualification/truth/review-hono-error-handling.json": "5e0653e0fa613cc7a4ee72aad1175d7a95ccce603aaf68afe5cc59b2e15854dd", "docs/qualification/truth/rootcause-hono-middleware-rerun.json": "6e2005ced5ba7ff768d1b9b3135eb909e98d6c1ead04712bc1b0185c480e429d", - "docs/qualification/validity-rules.md": "a8dba75f71462fdf6bf9cda41fe5ded2b1541cb2eb6e8897d4f318a05d38133c" + "docs/qualification/validity-rules.md": "8624a0b99af25b0f03dede1fa055736212e7d7db7197a43f42a98617a34c927e" } } diff --git a/docs/qualification/holdout-policy.md b/docs/qualification/holdout-policy.md index 80e540b8..bf786d89 100644 --- a/docs/qualification/holdout-policy.md +++ b/docs/qualification/holdout-policy.md @@ -45,14 +45,14 @@ production rules are shaped around the target once it is known. **Madar has one author.** There is no second person to author or hold a sealed target, and no meaningful sense in which a target can be hidden from the person who writes both the -production rules and the corpus. The `qual-sealed-a` slot in `corpus.json` is therefore -marked `status: "unsatisfied"` rather than being filled with a self-authored target that +production rules and the corpus. The `sealed-holdout-a` slot in `corpus.json` is therefore +marked `status: "unsatisfied"` rather than being filled with a self-selected target that would look like a holdout and prove nothing. The same limitation makes two other artifacts unavailable: -- the hidden acceptance test for `plan-plugin-host-object-storage-plugin` - (see `truth/plan-plugin-host-object-storage-plugin.json`); +- the hidden acceptance test for `plan-unstorage-add-driver` + (see `truth/plan-unstorage-add-driver.json`); - blinded Tier 2 review (see `rubrics.json#/blinding/current_status`). ### Human action required diff --git a/docs/qualification/validity-rules.md b/docs/qualification/validity-rules.md index cebbdb35..a5edd28e 100644 --- a/docs/qualification/validity-rules.md +++ b/docs/qualification/validity-rules.md @@ -26,7 +26,7 @@ to `validity.invalidation_reasons`: | `judge_failure` | A deterministic grader errored, or a blinded reviewer could not score the answer. | | `environment_mismatch` | `environment.drift.detected` is true and the drift was not resolved before the cell ran. | | `quality_gate_failure` | A gate failed in a way that prevents comparison at all — not a gate the arm simply lost. | -| `truth_unavailable` | The target/task pair has no independent truth (`status: "pinned_no_truth"`). | +| `truth_unavailable` | The target/task pair has no independent truth — for example a target added to the manifest before its truth file exists, or the unsatisfied sealed-holdout slot. | | `blinding_unavailable` | A Tier 2 quality dimension was scored without an independent blinded reviewer. | `degraded` is reserved for runs that are attributable and complete but weaker than the diff --git a/tests/unit/qualification-contract.test.ts b/tests/unit/qualification-contract.test.ts index 9b5bd404..c7f38d2f 100644 --- a/tests/unit/qualification-contract.test.ts +++ b/tests/unit/qualification-contract.test.ts @@ -362,6 +362,20 @@ describe('qualification Tier 1 subset', () => { }) describe('qualification policy documents', () => { + it('states the no-measured-evidence limitation prominently at the top of the README', () => { + const readme = readDoc(`${ROOT}/README.md`) + const heading = '## Read this first — what this contract does and does not give you today' + + expect(readme).toContain(heading) + // "Prominent" is load-bearing: the limitation must appear before the corpus is described, + // not be inferable only by cross-referencing target entries further down. + expect(readme.indexOf(heading)).toBeLessThan(readme.indexOf('## Why a separate corpus exists')) + expect(readme).toContain('This contract has never been executed. It currently produces no measured evidence of') + expect(readme).toContain('Regression only, never generalization.') + expect(readme).toContain('Thresholds are pre-registered, not calibrated.') + expect(readme).toContain('**Tier 1 needs network access**') + }) + it('states an objective stop rule with a pre-registered non-inferiority margin', () => { const stopRule = readDoc(`${ROOT}/stop-rule.md`) From 9c8edd3e9ea84224c718d68135cd278d95b67d08 Mon Sep 17 00:00:00 2001 From: mohammed naji Date: Wed, 12 Aug 2026 10:21:49 +0400 Subject: [PATCH 04/10] fix(qualification): keep prose out of the forbidden-literal scan (#655) `forbidden_target_symbols` carries a `_note` key whose value is a string. `Object.values(...).flat()` folded that 277-character note into FORBIDDEN_LITERALS as literal #1 of 8. Harmless today because no production file contains that exact string, but it becomes a live trap the moment the note is shortened to something short or common, at which point the independence guard starts failing production files over a documentation string. Skip `_`-prefixed keys, and reject anything that is not an array of identifiers so a future malformed entry fails loudly instead of silently widening or narrowing the scan. Each remaining key must also name a real corpus target. Verified the guard still fires: appending `SmartRouter` to src/runtime/graph-summary.ts makes `qualify:validate` fail with `production file src/runtime/graph-summary.ts contains qualification literal "SmartRouter"`, and the scan covers 7 symbols rather than 8 entries. A test asserts the map's shape so the defect cannot return. --- .../validate-qualification-contract.mjs | 28 ++++++++++++++++++- tests/unit/qualification-contract.test.ts | 17 +++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/.github/scripts/validate-qualification-contract.mjs b/.github/scripts/validate-qualification-contract.mjs index ea8f43d8..e00acbd5 100644 --- a/.github/scripts/validate-qualification-contract.mjs +++ b/.github/scripts/validate-qualification-contract.mjs @@ -333,12 +333,38 @@ for (const path of walk(join(ROOT, 'examples'))) { // word would confuse a declared adapter with a benchmark-specific special case. Those // couplings are disclosed per target in corpus.json#/targets/*/production_coupling instead. // What is forbidden here is every literal that could only have come from this contract. +// +// `_`-prefixed keys in forbidden_target_symbols are documentation, not symbols. Folding a +// prose note into this list would make the guard fail production files the moment that note +// were shortened to something short or common, so the keys are skipped and every remaining +// entry is required to be an array of plausible identifiers. +const targetSymbols = [] +for (const [key, value] of Object.entries(corpus.forbidden_target_symbols ?? {})) { + if (key.startsWith('_')) { + continue + } + if (!Array.isArray(value)) { + fail(`corpus.json forbidden_target_symbols["${key}"] must be an array of symbols`) + continue + } + for (const symbol of value) { + if (typeof symbol !== 'string' || !/^[A-Za-z_$][A-Za-z0-9_$]*$/.test(symbol)) { + fail(`corpus.json forbidden_target_symbols["${key}"] contains ${JSON.stringify(symbol)}, which is not an identifier`) + continue + } + targetSymbols.push(symbol) + } + if (!targetsById.has(key)) { + fail(`corpus.json forbidden_target_symbols["${key}"] does not name a corpus target`) + } +} + const FORBIDDEN_LITERALS = [ ...corpus.targets.flatMap((target) => (target.source?.url ? [target.source.url, target.source.ref] : [])), ...tasks.tasks.map((task) => task.id), ...tasks.tasks.map((task) => task.prompt.text), ...tier1.negative_trust_probes.map((probe) => probe.prompt.text), - ...Object.values(corpus.forbidden_target_symbols ?? {}).flat(), + ...targetSymbols, ] for (const path of walk(PRODUCTION_ROOT)) { diff --git a/tests/unit/qualification-contract.test.ts b/tests/unit/qualification-contract.test.ts index c7f38d2f..31956fff 100644 --- a/tests/unit/qualification-contract.test.ts +++ b/tests/unit/qualification-contract.test.ts @@ -143,6 +143,23 @@ describe('qualification corpus manifest', () => { expect(unstorage?.production_coupling?.level).toBe('none_found') }) + it('keeps the forbidden-symbol map free of prose that would poison the guard', () => { + // A non-array value here folds a whole documentation string into the literal list, which + // starts failing production files the moment that string is shortened to something common. + for (const [key, value] of Object.entries(corpus.forbidden_target_symbols)) { + if (key.startsWith('_')) { + expect(typeof value).toBe('string') + continue + } + + expect(Array.isArray(value)).toBe(true) + expect(corpus.targets.some((target) => target.id === key)).toBe(true) + for (const symbol of value as string[]) { + expect(symbol).toMatch(/^[A-Za-z_$][A-Za-z0-9_$]*$/) + } + } + }) + it('keeps the sealed holdout slot visible and explicitly unsatisfied', () => { const sealed = corpus.targets.filter((target) => target.holdout_class === 'sealed') From cec02d2af82829549a0c79a34380f05ae446b607 Mon Sep 17 00:00:00 2001 From: mohammed naji Date: Wed, 12 Aug 2026 12:49:05 +0400 Subject: [PATCH 05/10] fix(qualification): make the frozen contract portable to Windows (#655) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both Windows lanes were red on 765802fe while ubuntu and macOS passed. Three assertions in tests/unit/qualification-contract.test.ts failed because the Windows checkout converts LF to CRLF: two prose assertions used LF-specific multiline substrings, and the freeze digest is SHA-256 over raw bytes, which CRLF changes. The freeze is meaningful precisely because it is byte-exact, so the fix pins the bytes rather than loosening the check. freeze.json is NOT regenerated and no frozen content changed; `git add --renormalize .` produces no diff. - .gitattributes pins `docs/qualification/**` to `text eol=lf`, following the existing precedent for the hashed isolation CLAUDE.md, and marks `patches/*.patch` as `-text` so seeded-defect patches reach `git apply` byte-for-byte in either direction. - Prose assertions read through a normalizing helper so they test semantic content rather than checkout representation. The freeze digest deliberately does not use it: it reads a raw Buffer, because that guarantee is about bytes. - The validator normalizes patch text before structural parsing so a stray carriage return cannot be captured into a path, rejects a CRLF patch outright, and now explains on a digest mismatch that a CRLF file is a checkout problem and that regenerating freeze.json is not the fix. - Two new tests: `.gitattributes` must keep the pin, and no frozen file may contain CRLF whatever the checkout did. Also widen the CI step. `Validate qualification contract` was gated to ubuntu/Node 22, so it was skipped on Windows entirely and the byte-exact freeze was only ever verified on one of six lanes — the tests failed there while the validator that checks the same digests never ran. It now runs on every lane; the default mode is pure local file I/O with no network and no spend. A reproducible gate that only reproduces on Linux is not yet the thing it claims to be. Verified by simulating the Windows checkout locally: cloning with core.autocrlf=true converts README.md to CRLF while leaving every file under docs/qualification as LF, and both the validator and the 38 qualification tests pass in that clone. Forcing CRLF onto a frozen file there still fails both the digest check and the new CRLF check, so the guarantee was preserved rather than made vacuous. --- .gitattributes | 11 +++++++ .../validate-qualification-contract.mjs | 19 +++++++++-- .github/workflows/ci.yml | 5 ++- tests/unit/qualification-contract.test.ts | 33 +++++++++++++++++-- 4 files changed, 62 insertions(+), 6 deletions(-) diff --git a/.gitattributes b/.gitattributes index 9f70f102..45c2909e 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1,12 @@ docs/benchmarks/suite/isolation/.claude/CLAUDE.md text eol=lf + +# The qualification contract is frozen by SHA-256 over raw file bytes +# (docs/qualification/freeze.json). A CRLF checkout would change those bytes and +# break the freeze on Windows, so the whole tree is pinned to LF in the working +# copy on every platform. Do not relax this to make a platform failure disappear: +# the freeze is meaningful precisely because it is byte-exact. +docs/qualification/** text eol=lf + +# Seeded-defect patches must reach `git apply` byte-for-byte, so they are marked +# non-text and are never line-ending converted in either direction. +docs/qualification/patches/*.patch -text diff --git a/.github/scripts/validate-qualification-contract.mjs b/.github/scripts/validate-qualification-contract.mjs index e00acbd5..f0ce5880 100644 --- a/.github/scripts/validate-qualification-contract.mjs +++ b/.github/scripts/validate-qualification-contract.mjs @@ -126,10 +126,18 @@ for (const target of corpus.targets) { } if (patch) { - if (!patch.startsWith('diff --git ')) { + // Parse against normalized text so a CRLF checkout cannot capture a stray + // carriage return into a path. The digest check further down still reads raw + // bytes; only this structural parse is representation-independent. + const patchText = patch.replace(/\r\n/g, '\n') + + if (!patchText.startsWith('diff --git ')) { fail(`patch ${target.patch} is not a unified git diff`) } - const touched = [...patch.matchAll(/^\+\+\+ b\/(.+)$/gm)].map((match) => match[1]) + if (patch.includes('\r\n')) { + fail(`patch ${target.patch} contains CRLF line endings; it must stay LF so \`git apply\` accepts it`) + } + const touched = [...patchText.matchAll(/^\+\+\+ b\/(.+)$/gm)].map((match) => match[1]) if (touched.length === 0) { fail(`patch ${target.patch} does not modify any file`) } @@ -459,7 +467,12 @@ if (WRITE) { if (!(path in freeze.files)) { fail(`${path} is not covered by freeze.json`) } else if (freeze.files[path] !== digest) { - fail(`${path} content changed since it was frozen (expected ${freeze.files[path]}, actual ${digest})`) + const crlf = readFileSync(resolve(path)).includes('\r\n') + const hint = crlf + ? ' — the file contains CRLF, so this is a checkout line-ending problem, not a content change.' + + ' Fix the checkout (see `docs/qualification/** text eol=lf` in .gitattributes); do NOT regenerate freeze.json.' + : '' + fail(`${path} content changed since it was frozen (expected ${freeze.files[path]}, actual ${digest})${hint}`) } } for (const path of Object.keys(freeze.files)) { diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 45e1c0b5..7f1490c3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,8 +52,11 @@ jobs: if: matrix.os == 'ubuntu-latest' && matrix.node-version == '22' run: npm run release:verify + # Runs on every lane on purpose. The qualification freeze is a byte-exact + # guarantee, so it has to be verified on every platform the repository + # supports — a reproducible gate that only reproduces on Linux is not one. + # The default mode is pure local file I/O with no network and no spend. - name: Validate qualification contract - if: matrix.os == 'ubuntu-latest' && matrix.node-version == '22' run: npm run qualify:validate - name: Typecheck diff --git a/tests/unit/qualification-contract.test.ts b/tests/unit/qualification-contract.test.ts index 31956fff..3c8e367f 100644 --- a/tests/unit/qualification-contract.test.ts +++ b/tests/unit/qualification-contract.test.ts @@ -13,8 +13,22 @@ const addFormats = createRequire(import.meta.url)('ajv-formats') as (ajv: Ajv) = const ROOT = 'docs/qualification' +/** + * Reads a contract document for *semantic* assertions. Line endings are + * normalized so the assertions test content rather than checkout representation: + * `.gitattributes` pins this tree to LF, but a test that only passes because of a + * checkout setting is testing the setting, not the document. + * + * The byte-exact freeze contract is deliberately NOT read through here — it uses + * the raw Buffer below, because that guarantee is about bytes. + */ function readDoc(relativePath: string): string { - return readFileSync(resolve(relativePath), 'utf8') + return readFileSync(resolve(relativePath), 'utf8').replace(/\r\n/g, '\n') +} + +/** Raw bytes, for the freeze digest contract only. Never normalized. */ +function readBytes(relativePath: string): Buffer { + return readFileSync(resolve(relativePath)) } function readJson(relativePath: string): T { @@ -460,8 +474,23 @@ describe('qualification freeze', () => { expect(paths).toContain(`${ROOT}/patches/hono-compose-reentrancy-guard.patch`) expect(paths).toContain(`${ROOT}/patches/hono-error-message-disclosure.patch`) + // Raw bytes on purpose. If a checkout converts line endings, this must fail + // rather than be normalized into passing — that is the whole point of the freeze. for (const [path, digest] of Object.entries(freeze.files)) { - expect(digest).toBe(createHash('sha256').update(readFileSync(resolve(path))).digest('hex')) + expect(digest).toBe(createHash('sha256').update(readBytes(path)).digest('hex')) + } + }) + + it('pins the contract tree to LF so the byte-exact freeze survives a Windows checkout', () => { + const attributes = readDoc('.gitattributes') + + expect(attributes).toContain('docs/qualification/** text eol=lf') + expect(attributes).toContain('docs/qualification/patches/*.patch -text') + }) + + it('holds no CRLF in any frozen file, whatever the checkout did', () => { + for (const path of Object.keys(freeze.files)) { + expect(readBytes(path).includes('\r\n')).toBe(false) } }) From 2fddb25ad616b2480a7f2ccffdbe644dbfb61936 Mon Sep 17 00:00:00 2001 From: mohammed naji Date: Fri, 14 Aug 2026 08:46:25 +0400 Subject: [PATCH 06/10] fix(qualification): make the contract validator report rather than throw Three findings from the independent review of #681, all confirmed against the code before acting. 1. kind and holdout_class described the same distinction but were read by different consumers: this validator branched on kind, tests/unit/qualification-contract.test.ts filtered on holdout_class. A target setting only one would be classified differently by each. Both are now required to agree, and every sealed-target branch derives from one predicate. 2. The independence check dereferenced task.truth_provenance and truth.provenance without confirming either exists, and the receipt loop read receipt.validity and receipt.scores after a schema failure. Either would throw a TypeError before the collected failure list is printed, so a malformed document produced a stack trace instead of the description of what is wrong with it. Both now record a failure and move on. 3. --write persisted freeze.json before the failure gate, so a run over an inconsistent contract exited 1 while leaving a freeze on disk that blessed that state and read back clean afterwards. The write is deferred until after the gate, and a failed --write says the freeze was not written. Refs #655. --- .../validate-qualification-contract.mjs | 65 ++++++++++++++----- 1 file changed, 50 insertions(+), 15 deletions(-) diff --git a/.github/scripts/validate-qualification-contract.mjs b/.github/scripts/validate-qualification-contract.mjs index f0ce5880..07eef1ed 100644 --- a/.github/scripts/validate-qualification-contract.mjs +++ b/.github/scripts/validate-qualification-contract.mjs @@ -74,8 +74,21 @@ if (!Array.isArray(corpus.proxy_targets)) { fail('corpus.json must declare a proxy_targets list, even when empty') } +// `kind` and `holdout_class` describe the same distinction and are read by different +// consumers — this validator used `kind`, the contract test used `holdout_class`. A target +// that sets only one of them would be classified differently by each, so require both and +// require them to agree, and derive the single predicate from that pair. +const isSealedTarget = (target) => target.kind === 'sealed' || target.holdout_class === 'sealed' + for (const target of corpus.targets) { - if (target.kind === 'sealed') { + if ((target.kind === 'sealed') !== (target.holdout_class === 'sealed')) { + fail( + `target ${target.id} disagrees with itself: kind=${JSON.stringify(target.kind)} but ` + + `holdout_class=${JSON.stringify(target.holdout_class)}; a sealed target must declare both`, + ) + } + + if (isSealedTarget(target)) { if (target.status !== 'unsatisfied') { fail(`sealed target ${target.id} must stay unsatisfied until a second person fills it`) } @@ -196,7 +209,17 @@ for (const task of tasks.tasks) { if (truth.contract_version !== CONTRACT_VERSION) fail(`${task.truth_ref} declares contract_version ${truth.contract_version}`) // Independence: truth must not be derived from Madar output. - for (const provenance of [task.truth_provenance, truth.provenance]) { + // Record a failure rather than dereferencing an absent block: this validator exists to + // print the complete list of contract problems, and a TypeError here would replace that + // list with a stack trace on exactly the malformed documents it is meant to describe. + for (const [source, provenance] of [ + [`task ${task.id} truth_provenance`, task.truth_provenance], + [`${task.truth_ref} provenance`, truth.provenance], + ]) { + if (provenance === null || typeof provenance !== 'object') { + fail(`${source} is missing; truth independence cannot be established`) + continue + } if (provenance.inspected_madar_output_before_freeze !== false) { fail(`task ${task.id} truth provenance claims Madar output was inspected before freezing`) } @@ -312,8 +335,12 @@ for (const path of walk(join(ROOT, 'examples'))) { const receipt = readJson(path) const label = relative(process.cwd(), path) + // A receipt that failed schema validation has no guaranteed shape, so reading + // `receipt.validity` or `receipt.scores` below would throw before the collected failures + // are printed. Report the schema failure and move on to the next receipt. if (!validateReceipt(receipt)) { fail(`${label} does not satisfy receipt-schema.json: ${ajv.errorsText(validateReceipt.errors)}`) + continue } if (receipt.validity.status !== 'valid' && receipt.validity.aggregatable !== false) { fail(`${label} is not valid but is marked aggregatable`) @@ -390,7 +417,7 @@ for (const path of walk(PRODUCTION_ROOT)) { if (VERIFY_CORPUS) { for (const target of corpus.targets) { - if (target.kind === 'sealed') { + if (isSealedTarget(target)) { continue } @@ -441,17 +468,10 @@ const digests = Object.fromEntries( frozenFiles.map((path) => [path, sha256(readFileSync(resolve(path)))]), ) -if (WRITE) { - const freeze = { - contract_version: CONTRACT_VERSION, - frozen_at: corpus.frozen_at, - algorithm: 'sha256 over raw file bytes', - note: 'Regenerate deliberately with `npm run qualify:validate -- --write` and say why in the pull request. A silent digest change is a contract change.', - files: digests, - } - writeFileSync(FREEZE_PATH, `${JSON.stringify(freeze, null, 2)}\n`) - console.log(`wrote ${relative(process.cwd(), FREEZE_PATH)} with ${frozenFiles.length} entries`) -} else { +// The freeze map is only computed here. Writing it is deferred until after the failure +// gate below, so that `--write` on a contract with problems cannot leave a freeze.json on +// disk that blesses the inconsistent state and then reads back clean on the next plain run. +if (!WRITE) { let freeze try { freeze = readJson(FREEZE_PATH) @@ -490,10 +510,25 @@ if (failures.length > 0) { for (const failure of failures) { console.error(` - ${failure}`) } + if (WRITE) { + console.error(`freeze.json was NOT written: a freeze must only ever record a consistent contract.`) + } process.exit(1) } -const naturalTargets = corpus.targets.filter((target) => target.kind !== 'sealed') +if (WRITE) { + const freeze = { + contract_version: CONTRACT_VERSION, + frozen_at: corpus.frozen_at, + algorithm: 'sha256 over raw file bytes', + note: 'Regenerate deliberately with `npm run qualify:validate -- --write` and say why in the pull request. A silent digest change is a contract change.', + files: digests, + } + writeFileSync(FREEZE_PATH, `${JSON.stringify(freeze, null, 2)}\n`) + console.log(`wrote ${relative(process.cwd(), FREEZE_PATH)} with ${frozenFiles.length} entries`) +} + +const naturalTargets = corpus.targets.filter((target) => !isSealedTarget(target)) console.log( `qualification contract v${CONTRACT_VERSION} is consistent: ` + From 3091d4fc1f4989059e4273e7a10cd9e4133d41c7 Mon Sep 17 00:00:00 2001 From: mohammed naji Date: Fri, 14 Aug 2026 08:54:07 +0400 Subject: [PATCH 07/10] fix(qualification): make the frozen contract agree with itself Eight findings from the independent review, all of the same kind: the contract stated a rule in one place and a different rule in another. Each is corrected toward the rule that was already intended, not toward a new one. receipt-schema.json encoded validity-rules.md in one direction only, so receipts contradicting the prose still validated. Added the three missing branches: an unmeasured cost account must carry nulls rather than zeros, a measured score may not carry null, and a retained artifact must carry a non-empty path and a 64-hex digest. Removed the top-level cache_mode, which duplicated the required identity.cache_mode with no rule keeping the two equal, and dropped it from both examples. validity-rules.md names five retained artifacts while retention had four slots and additionalProperties false, so a receipt following the prose failed the schema and was then classified incomplete_receipt by the same document. prompt_text, environment_receipt and truth_file are now expressible. They are deliberately not required: which artifacts each tier must retain is an open question, and requiring them here would answer it by accident. rubrics.json correctness anchors 1 and 2 both read 'no false assertions' and were separated only by 'material gaps', which critical_fact_completeness already measures; they now separate on whether every assertion is verifiable against the pinned source. The evidence-symbol matching rule is stated: truth files record obligations bare while evidence entries qualify them by owner, so exact-string comparison would never match, and last-segment matching is now written down rather than assumed. README.md claimed target ids are scanned out of src/; corpus.json says they are deliberately excluded. The README now documents the implemented boundary and says why. stop-rule.md S1.4 asked a question that could not be answered, since its absolute clause referenced a phase target the same row records as not set; both clauses now state their behaviour when no target and no previous sweep exist. Patch paths carried no declared base directory even though a failed patch application invalidates a run; corpus.json now records patch_path_base explicitly. The cited-path traversal, including the new_path exemption, was duplicated between the validator and its test, so a change to one would have left the other enforcing the old rule while still passing. It now lives in one shared module. The test also no longer pins independent_of_production_rule_author to false, which would have failed on the improvement of a second author closing the gap; the current state is asserted separately. freeze.json is regenerated deliberately because seven frozen files changed. Every digest move is accounted for by an edit above. Refs #655. --- .github/scripts/lib/collect-cited-paths.cjs | 46 ++++++++++++++++++ .../validate-qualification-contract.mjs | 24 +++------- docs/qualification/README.md | 8 ++-- docs/qualification/corpus.json | 2 + .../examples/receipt-tier1-valid.json | 1 - .../receipt-tier2-invalid-no-madar-call.json | 1 - docs/qualification/freeze.json | 14 +++--- docs/qualification/receipt-schema.json | 48 +++++++++++++++++-- docs/qualification/rubrics.json | 6 +-- docs/qualification/stop-rule.md | 2 +- tests/unit/qualification-contract.test.ts | 40 +++++++++------- 11 files changed, 137 insertions(+), 55 deletions(-) create mode 100644 .github/scripts/lib/collect-cited-paths.cjs diff --git a/.github/scripts/lib/collect-cited-paths.cjs b/.github/scripts/lib/collect-cited-paths.cjs new file mode 100644 index 00000000..ad47c56f --- /dev/null +++ b/.github/scripts/lib/collect-cited-paths.cjs @@ -0,0 +1,46 @@ +'use strict' + +/** + * The single definition of "which paths does a truth file cite". + * + * It lives here, in CommonJS, because both consumers need it and they cannot share an + * ESM module: `.github/scripts/validate-qualification-contract.mjs` is ESM, and + * `tests/unit/qualification-contract.test.ts` is TypeScript compiled without `allowJs`, + * so it reaches this file through `createRequire`. Node imports CommonJS from ESM + * natively, so the validator can import it directly. + * + * Both consumers previously carried their own copy of this traversal, including the + * `new_path` exemption. A change to the exemption in one copy would have left the other + * enforcing the old rule, and the test would have stopped covering the shipped guard + * while still passing. + * + * `new_path` is exempt because a plan task proposes creating files that do not exist at + * the pinned commit, so they cannot appear in the target's frozen blob manifest. + * + * @param {unknown} node - any subtree of a parsed truth file + * @returns {Set} every value recorded under a `path` key, at any depth + */ +function collectCitedPaths(node) { + const cited = new Set() + + const walk = (value) => { + if (Array.isArray(value)) { + value.forEach(walk) + return + } + if (value && typeof value === 'object') { + for (const [key, child] of Object.entries(value)) { + if (key === 'path' && typeof child === 'string') { + cited.add(child) + } else if (key !== 'new_path') { + walk(child) + } + } + } + } + + walk(node) + return cited +} + +module.exports = { collectCitedPaths } diff --git a/.github/scripts/validate-qualification-contract.mjs b/.github/scripts/validate-qualification-contract.mjs index 07eef1ed..4ea6512f 100644 --- a/.github/scripts/validate-qualification-contract.mjs +++ b/.github/scripts/validate-qualification-contract.mjs @@ -7,6 +7,10 @@ import { join, relative, resolve } from 'node:path' import Ajv from 'ajv' import addFormats from 'ajv-formats' +import citedPathCollector from './lib/collect-cited-paths.cjs' + +const { collectCitedPaths } = citedPathCollector + const ROOT = resolve('docs/qualification') const FREEZE_PATH = join(ROOT, 'freeze.json') const PRODUCTION_ROOT = resolve('src') @@ -238,23 +242,9 @@ for (const task of tasks.tasks) { } // Every cited evidence path must be recorded in the target's frozen blob map. - // `new_path` is used for files a plan proposes creating and is intentionally exempt. - const citedPaths = new Set() - const collect = (node) => { - if (Array.isArray(node)) { - node.forEach(collect) - return - } - if (node && typeof node === 'object') { - for (const [key, value] of Object.entries(node)) { - if (key === 'path' && typeof value === 'string') citedPaths.add(value) - else if (key !== 'new_path') collect(value) - } - } - } - collect(truth) - - for (const cited of citedPaths) { + // The traversal, including the `new_path` exemption, is shared with + // tests/unit/qualification-contract.test.ts so the two cannot drift apart. + for (const cited of collectCitedPaths(truth)) { if (!(cited in (target.cited_blobs ?? {}))) { fail(`${task.truth_ref} cites ${cited}, which is not recorded in target ${target.id} cited_blobs`) } diff --git a/docs/qualification/README.md b/docs/qualification/README.md index 956359ee..84aac84e 100644 --- a/docs/qualification/README.md +++ b/docs/qualification/README.md @@ -71,7 +71,7 @@ recreated inside a synthetic workspace built around the answer. | [`corpus.json`](./corpus.json) | Versioned corpus manifest: repositories, commits, licenses, prepare commands, patches, cited blob digests, holdout class. | | [`tasks.json`](./tasks.json) | Versioned task definitions: frozen prompts with hashes, categories, scoring method, truth provenance. | | [`truth/`](./truth/) | Independent truth and rubric input, one file per task. | -| [`patches/`](./patches/) | Seeded-defect patches applied to the pinned commit. | +| [`patches/`](./patches/) | Seeded-defect patches applied to the pinned commit. Every `patches/...` reference in `corpus.json`, `tasks.json` and the truth files resolves against this directory — `docs/qualification/` — as recorded by `patch_path_base` in `corpus.json`. | | [`rubrics.json`](./rubrics.json) | Scoring dimensions, per-category scoring methods, blinding rules, aggregation rules. | | [`receipt-schema.json`](./receipt-schema.json) | Environment and run receipt schema. | | [`examples/`](./examples/) | Two illustrative receipts: a valid Tier 1 run and an invalid Tier 2 run that stays `not_measured`. | @@ -116,8 +116,10 @@ npm run qualify:validate - every Tier 1 cell and negative-trust probe resolves, and every probe prompt hash matches; - both example receipts validate against `receipt-schema.json`, and no unmeasured score carries a value; -- **no qualification target id, task id, prompt string, or pinned-repository symbol appears - anywhere in `src/`**; +- **no qualification task id, prompt string, or pinned-repository symbol appears anywhere in + `src/`**. Target ids are deliberately outside this scan — see `forbidden_target_symbols._note` + and `production_coupling` in `corpus.json` — because a target id is the name of a real public + project and its appearance in production code is not by itself evidence of coupling; - every file in this directory matches its frozen digest. To additionally confirm the pinned commits and blob digests against the real repositories diff --git a/docs/qualification/corpus.json b/docs/qualification/corpus.json index 9d264c04..91ca0bb2 100644 --- a/docs/qualification/corpus.json +++ b/docs/qualification/corpus.json @@ -14,6 +14,8 @@ "hono": ["SmartRouter", "UnsupportedPathError", "RegExpRouter", "TrieRouter"], "unstorage": ["createStorage", "DriverFactory", "createRequiredError"] }, + "patch_path_base": "docs/qualification/", + "patch_path_base_note": "Every `patch` value in this file, and every `patches/...` reference in the truth files and in tasks.json, is resolved against this directory and nowhere else — not against the repository root and not against the file that mentions it. `npm run qualify:validate` resolves them this way. The base is stated because validity-rules.md invalidates a run whose patch fails to apply, so an ambiguous base would turn a path convention into an invalidation.", "proxy_targets": [], "proxy_targets_note": "This list is intentionally empty. A fixture proxy is permitted only where a natural repository genuinely cannot supply a task category; all six required categories are supplied by the natural targets below. Any future entry here must be labelled a proxy and must carry the statement that proxies cannot satisfy the naturalness property.", "targets": [ diff --git a/docs/qualification/examples/receipt-tier1-valid.json b/docs/qualification/examples/receipt-tier1-valid.json index fa1a0c7d..0051b8bd 100644 --- a/docs/qualification/examples/receipt-tier1-valid.json +++ b/docs/qualification/examples/receipt-tier1-valid.json @@ -6,7 +6,6 @@ "target_id": "hono-seeded-compose", "arm": "madar", "trial": 1, - "cache_mode": "cold", "identity": { "target_revision": "26de73133b8552f56ba72e025ecd82b08900d796", "dependency_lock_sha256": "6328bf95a901590814ff70ed570e0fc474c05ae162d6ac690cf3c812380828ab", diff --git a/docs/qualification/examples/receipt-tier2-invalid-no-madar-call.json b/docs/qualification/examples/receipt-tier2-invalid-no-madar-call.json index 4c43dd38..eef638cd 100644 --- a/docs/qualification/examples/receipt-tier2-invalid-no-madar-call.json +++ b/docs/qualification/examples/receipt-tier2-invalid-no-madar-call.json @@ -6,7 +6,6 @@ "target_id": "hono", "arm": "madar", "trial": 1, - "cache_mode": "warm", "identity": { "target_revision": "26de73133b8552f56ba72e025ecd82b08900d796", "dependency_lock_sha256": "6328bf95a901590814ff70ed570e0fc474c05ae162d6ac690cf3c812380828ab", diff --git a/docs/qualification/freeze.json b/docs/qualification/freeze.json index 67651c76..49970012 100644 --- a/docs/qualification/freeze.json +++ b/docs/qualification/freeze.json @@ -4,17 +4,17 @@ "algorithm": "sha256 over raw file bytes", "note": "Regenerate deliberately with `npm run qualify:validate -- --write` and say why in the pull request. A silent digest change is a contract change.", "files": { - "docs/qualification/README.md": "7f4920abbac662f13ec2deea6756edac13bd19090953387209a3df8ee97eea5f", - "docs/qualification/corpus.json": "faa4511d5d255357fdc6c0b5b28b7f139a84871456b914c8425cb1cf1688ee03", + "docs/qualification/README.md": "8153515a9daa7b5bb253c2d5439d8b485c0c6c03958cac2d3f287c74f9675162", + "docs/qualification/corpus.json": "8d2fd2ae3386498010fe716dcc8c7903f4759135ca0380cc453ee1f2c1dd8a52", "docs/qualification/evidence-categories.md": "1f75b04229b805050584eadda201bc67906224d34f562afa2098558296a952cd", - "docs/qualification/examples/receipt-tier1-valid.json": "6bb66edfaeaba4ccb97af33b0d9d5fcada2a5c5a686ebd919e46049722f825dc", - "docs/qualification/examples/receipt-tier2-invalid-no-madar-call.json": "d20124547ba9839c8254e150d3aa06c8961e423e593b5893cbab36e77e829ad8", + "docs/qualification/examples/receipt-tier1-valid.json": "8fc8d9c3a86f2404346ff6dbf7d2833553a67bde238d0746d2df128dc361e11d", + "docs/qualification/examples/receipt-tier2-invalid-no-madar-call.json": "2783cad80b8a00d12b8d84718f3c4b3e31846111abc4830c47677dd3422a80aa", "docs/qualification/holdout-policy.md": "f5671c4cd5f5487e21dcb44e34fa98b5520a7b9ec58391ce1f086ebd33e23616", "docs/qualification/patches/hono-compose-reentrancy-guard.patch": "9355c5bbb05cd5ae4d998ace18d6381f0cba4fd080203d4d02579da3dcf6dea4", "docs/qualification/patches/hono-error-message-disclosure.patch": "edb79059b72b4f27f5dc8341ba2d9a3617901c402da9ef1f9daf5503e6528d6f", - "docs/qualification/receipt-schema.json": "2266deb3afee2c2bc352dc7ee584b37472f2c240d2e5a233b44fc946ab645994", - "docs/qualification/rubrics.json": "35f7648bc72e5f0af11649063b0da01bea37d6c0d8699132c078f0c793abb079", - "docs/qualification/stop-rule.md": "7d5b646e0d3369aa1783bc4a594b3ee322e0774515fec6ec43c83336722a4f83", + "docs/qualification/receipt-schema.json": "6fa0e3dcc2a4195c1286cbaae2bdc265e50429a6a636e9d26e7c55c3f1ec6070", + "docs/qualification/rubrics.json": "358d788301113ea1faf27c1011cb887267ff7782ab9ed337a872362cf3edfab5", + "docs/qualification/stop-rule.md": "36246356a456d5839e0670b298ee7c34a80e89f47481991eb791d029b972e1c9", "docs/qualification/tasks.json": "61f7e5185cd4586fc529709d911a724ba748e37b3e5718ed505e429458015117", "docs/qualification/tier1.json": "5faa20899960c5245030b48c1e652ab0ea0ebaf69c8fdaca44fa37f184aa6292", "docs/qualification/tier2-matrix.json": "d4a0a071219041f4766382c63ad651b11a64de6b80f5ab7ae4b8435eda985008", diff --git a/docs/qualification/receipt-schema.json b/docs/qualification/receipt-schema.json index 5fe01af8..8b96c525 100644 --- a/docs/qualification/receipt-schema.json +++ b/docs/qualification/receipt-schema.json @@ -31,7 +31,6 @@ "target_id": { "type": "string", "minLength": 1 }, "arm": { "type": "string", "enum": ["native", "madar"] }, "trial": { "type": "integer", "minimum": 1 }, - "cache_mode": { "type": "string", "enum": ["cold", "warm"] }, "identity": { "type": "object", @@ -220,11 +219,20 @@ "retention": { "type": "object", "additionalProperties": false, - "required": ["raw_transcript", "answer_text", "context_artifact", "retention_policy"], + "$comment": "One slot per artifact named in validity-rules.md, Retention. Previously prompt_text, environment_receipt and truth_file had no slot at all, so a receipt that followed the prose failed this schema and was then classified incomplete_receipt by the same document that demanded it. They are expressible but not yet required: which artifacts each tier must retain is the open tier-specific question tracked on #655, and answering it here would decide that question by accident.", + "required": [ + "raw_transcript", + "answer_text", + "context_artifact", + "retention_policy" + ], "properties": { "raw_transcript": { "$ref": "#/definitions/artifactRef" }, "answer_text": { "$ref": "#/definitions/artifactRef" }, "context_artifact": { "$ref": "#/definitions/artifactRef" }, + "prompt_text": { "$ref": "#/definitions/artifactRef" }, + "environment_receipt": { "$ref": "#/definitions/artifactRef" }, + "truth_file": { "$ref": "#/definitions/artifactRef" }, "retention_policy": { "type": "string", "minLength": 1 } } }, @@ -273,7 +281,22 @@ "type": "string", "enum": ["provider_reported", "locally_timed", "unknown", "not_applicable"] } - } + }, + "allOf": [ + { + "$comment": "validity-rules.md, Cost separation: an unmeasured account is measured: false, never 0. Without this branch a receipt could report measured: false alongside a 0 for every figure, which is exactly the interchange the rule forbids.", + "if": { "properties": { "measured": { "const": false } }, "required": ["measured"] }, + "then": { + "properties": { + "input_tokens": { "type": "null" }, + "output_tokens": { "type": "null" }, + "cache_creation_input_tokens": { "type": "null" }, + "wall_ms": { "type": "null" }, + "usd": { "type": "null" } + } + } + } + ] }, "score": { "type": "object", @@ -301,6 +324,11 @@ }, "required": ["not_measured_reason"] } + }, + { + "$comment": "The complement of the branch above. validity-rules.md keeps a score of 0 and a score of not_measured distinct; a measured score carrying null is indistinguishable from not_measured, so it is rejected here rather than left to the reader.", + "if": { "properties": { "measured": { "const": true } }, "required": ["measured"] }, + "then": { "properties": { "value": { "type": ["number", "string"] } } } } ] }, @@ -312,7 +340,19 @@ "retained": { "type": "boolean" }, "path": { "type": ["string", "null"] }, "sha256": { "type": ["string", "null"], "pattern": "^[0-9a-f]{64}$" } - } + }, + "allOf": [ + { + "$comment": "validity-rules.md, Retention: each retained artifact is recorded with its path and SHA-256. Nullable types alone let retained: true carry no path and no digest, which records a retention claim that cannot be checked.", + "if": { "properties": { "retained": { "const": true } }, "required": ["retained"] }, + "then": { + "properties": { + "path": { "type": "string", "minLength": 1 }, + "sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" } + } + } + } + ] } } } diff --git a/docs/qualification/rubrics.json b/docs/qualification/rubrics.json index c0fee2fb..f3660e18 100644 --- a/docs/qualification/rubrics.json +++ b/docs/qualification/rubrics.json @@ -4,7 +4,7 @@ "dimensions": { "correctness": { "definition": "Every assertion the answer makes about the target is true of the pinned source.", - "scale": { "0": "contains a false assertion about the target", "1": "no false assertions, but material gaps", "2": "no false assertions" }, + "scale": { "0": "contains a false assertion about the target", "1": "no false assertions, but at least one assertion a blinded reviewer cannot verify either way against the pinned source", "2": "no false assertions, and every assertion is verifiable against the pinned source" }, "scored_by": "blinded_human", "tiers": [2], "gating": true @@ -62,8 +62,8 @@ "inputs": ["the context artifact produced for the frozen prompt", "the task truth file's tier1_obligations"], "procedure": [ "Collect the evidence set (paths and symbols) the artifact presents as supporting material.", - "required_evidence_paths recall must be >= min_critical_fact_recall.", - "required_evidence_symbols recall must be >= min_critical_fact_recall.", + "required_evidence_paths recall must be >= min_critical_fact_recall. Paths are compared as exact repository-relative strings.", + "required_evidence_symbols recall must be >= min_critical_fact_recall. Symbols are compared on their LAST dot-separated segment, case-sensitively, after stripping a leading '#': the truth files record obligations bare (`fetch`, `dispatch`, `Context`) while evidence entries qualify them by owner (`Hono.fetch`, `Hono.#dispatch`, `compose.dispatch`), and an exact-string comparison would never match those pairs. `Hono.fetch` therefore satisfies the obligation `fetch`; `getPath` satisfies `getPath`. Two different owners exposing the same member name are treated as one obligation, which is accepted: the obligation asks whether the evidence for that member was surfaced at all.", "Every path cited by the artifact must exist in the pinned target.", "If any must_not_report_ready_when condition holds, the artifact must not report a ready state." ], diff --git a/docs/qualification/stop-rule.md b/docs/qualification/stop-rule.md index 082b5698..964d5e24 100644 --- a/docs/qualification/stop-rule.md +++ b/docs/qualification/stop-rule.md @@ -16,7 +16,7 @@ merged, when any of the following holds against the frozen corpus. | S1.1 | Critical-fact completeness regresses beyond the pre-registered margin | For any task, the post-change critical-fact completeness is lower than the pre-change value by more than the non-inferiority margin **0.05**, on `n_valid >= 5` paired trials in the same cell. | | S1.2 | Unsupported claims increase materially | For any task, the post-change mean unsupported-claim count exceeds the pre-change mean by **more than 0.5 claims per answer**, or any single answer introduces an unsupported claim listed in that task's `unsupported_claim_traps` that the pre-change arm did not make. | | S1.3 | False-ready behaviour appears | Any negative-trust probe in [`tier1.json`](./tier1.json) reports a ready state, or any evidence set contains a path or symbol that does not exist in the pinned target. This is a **single-occurrence** trip: one instance blocks. | -| S1.4 | Host adoption falls below the phase target | Across the Tier 2 sweep, fewer than the phase target of Madar-arm runs have `adoption.status` of `adopted`. Phase 0 target: **not set** — adoption is measured and reported, and a *decrease* of more than 10 percentage points against the previous recorded sweep blocks. | +| S1.4 | Host adoption falls below the phase target | Two clauses, each independently evaluable. **Absolute clause:** across the Tier 2 sweep, fewer than the phase target of Madar-arm runs have `adoption.status` of `adopted`. This clause is **inactive** while no phase target is recorded — Phase 0 records none — and an inactive clause never trips and never blocks. **Relative clause:** a *decrease* of more than 10 percentage points against the previous recorded sweep blocks. With no previous recorded sweep there is nothing to compare against, so the relative clause does not trip either; the first sweep establishes the baseline and cannot itself fail S1.4. Until a phase target is recorded and one sweep exists, S1.4 evaluates to *not tripped*, and adoption is measured and reported rather than gated. | | S1.5 | Graph or artifact integrity fails | Any graph-integrity invariant from #656–#659 fails, or an artifact fails its round-trip or old-reader-rejection check. Single-occurrence trip. | | S1.6 | Results depend on qualification-repository literals | Any qualification fixture path, symbol, prompt string, repository id, or a near-equivalent special case appears in production retrieval, ranking, context, or claim logic. Single-occurrence trip; checked deterministically by `npm run qualify:validate`. | | S1.7 | Output differences remain unexplained | A retrieval, pack, graph, or artifact output differs from the pre-change baseline and the pull request does not explain the difference. Updating a snapshot is not an explanation. | diff --git a/tests/unit/qualification-contract.test.ts b/tests/unit/qualification-contract.test.ts index 3c8e367f..3348535e 100644 --- a/tests/unit/qualification-contract.test.ts +++ b/tests/unit/qualification-contract.test.ts @@ -11,6 +11,13 @@ import { describe, expect, it } from 'vitest' // `npm run qualify:validate` compile the schema identically. const addFormats = createRequire(import.meta.url)('ajv-formats') as (ajv: Ajv) => void +// The cited-path traversal is the shipped validator's own, not a copy of it. It is loaded +// through createRequire for the same reason as ajv-formats: it is CommonJS living under +// .github/, which tsconfig does not include. +const { collectCitedPaths } = createRequire(import.meta.url)( + '../../.github/scripts/lib/collect-cited-paths.cjs', +) as { collectCitedPaths: (node: unknown) => Set } + const ROOT = 'docs/qualification' /** @@ -221,6 +228,18 @@ describe('qualification task definitions', () => { expect(provenance.derived_from.length).toBeGreaterThan(0) expect(provenance.madar_derived_sources_used).toEqual([]) expect(provenance.inspected_madar_output_before_freeze).toBe(false) + // The contract requires this to be *stated*, not to hold a particular value. Pinning + // it to false would fail the moment a second author closes the independence gap, + // which is an improvement, not a regression. + expect(typeof provenance.independent_of_production_rule_author).toBe('boolean') + } + } + }) + + it('records the current independence state, which a second author is expected to change', () => { + for (const task of tasks.tasks) { + const truth = readJson<{ provenance: Provenance }>(`${ROOT}/${task.truth_ref}`) + for (const provenance of [task.truth_provenance, truth.provenance]) { expect(provenance.independent_of_production_rule_author).toBe(false) } } @@ -230,24 +249,9 @@ describe('qualification task definitions', () => { for (const task of tasks.tasks) { const truth = readJson>(`${ROOT}/${task.truth_ref}`) const target = corpus.targets.find((candidate) => candidate.id === task.target) - const cited = new Set() - - const collect = (node: unknown): void => { - if (Array.isArray(node)) { - node.forEach(collect) - return - } - if (node && typeof node === 'object') { - for (const [key, value] of Object.entries(node)) { - if (key === 'path' && typeof value === 'string') { - cited.add(value) - } else if (key !== 'new_path') { - collect(value) - } - } - } - } - collect(truth) + // Shared with the shipped validator rather than reimplemented, so the `new_path` + // exemption cannot drift between the guard and the test that covers it. + const cited = collectCitedPaths(truth) expect(cited.size).toBeGreaterThan(0) for (const path of cited) { From 6ceef7552133dc3f7710e7ab2d5eeb0539015bef Mon Sep 17 00:00:00 2001 From: mohammed naji Date: Fri, 14 Aug 2026 11:52:10 +0400 Subject: [PATCH 08/10] fix(qualification): decide the five open contract terms (#655) Eleven of the sixteen review findings were transcription defects, fixed in 2fddb25a and 3091d4fc. The remaining five each SET a contract term, so they were escalated rather than settled by the automation this contract will later measure. The decisions, and why: 8. correct_uncertainty and evidence_support stop gating and keep being measured. No threshold for either has been calibrated, and gating on an uncalibrated threshold manufactures confidence and invites the threshold to be tuned to whatever passes. Both stay reported on every receipt: an exemption that also stops measurement can never end. Each records the precondition that would make it gate -- a baseline distribution over the natural corpus, with the threshold derived and recorded before any target is scored. evidence_support spans tiers 1 and 2, so it carries an explicit scope note: the Tier 1 deterministic gate is unaffected and still gates. 10. Run-validity requirements split by tier. Tier 1 never runs an agent, so requiring an answer and a transcript contradicted its stated purpose and made the valid Tier 1 example fail its own contract. Tier 1 now requires the context artifact, prompt text, environment receipt and truth file; Tier 2 adds the raw transcript, the answer text and the attributable Madar call. Each tier records why, so the split is not later collapsed back. 11. The Tier 1 gate starts inactive. A gate with no baseline gates on noise. Activation is recorded as an event naming the run that established the baseline, not as a boolean flip. 12. The Tier 2 cells are defined explicitly, mirroring the Tier 1 cells, with sealed-holdout-a recorded as contributing through the external sealed manifest rather than through this file. 14. hidden_acceptance_test is now consumed: a Tier 2 receipt for a task declaring one must carry a not_measured implementation score. A frozen contract carrying a field nothing reads is a defect in the contract, so it is enforced rather than deleted. Two terms these decisions introduce are themselves enforced, not decorative: an active gate must name its baseline run, and the Tier 2 cells must equal the Tier 1 cells. Each new validator branch was mutation-tested and turns exactly one regression test red when disabled. freeze.json digests moved, by decision: 8 rubrics.json 358d7883 -> 377a1f72 10 tasks.json 61f7e518 -> c9410884 10 validity-rules.md 8624a0b9 -> ee3898f5 10 examples/receipt-tier1-valid.json 8fc8d9c3 -> a68f8d58 10+14 receipt-schema.json 6fa0e3dc -> e849ba4b 11 tier1.json 5faa2089 -> 967e3d3b 12 tier2-matrix.json d4a0a071 -> 449d8076 The other 14 frozen entries are unchanged. Local: qualify:validate consistent, typecheck clean, qualification-contract.test.ts 43 passed. Refs #655. Related parent: #649. --- .../validate-qualification-contract.mjs | 58 +++++++ .../examples/receipt-tier1-valid.json | 17 +- docs/qualification/freeze.json | 14 +- docs/qualification/receipt-schema.json | 23 ++- docs/qualification/rubrics.json | 16 +- docs/qualification/tasks.json | 150 ++++++++++++++--- docs/qualification/tier1.json | 10 ++ docs/qualification/tier2-matrix.json | 9 ++ docs/qualification/validity-rules.md | 18 ++- tests/unit/qualification-contract.test.ts | 153 +++++++++++++++++- 10 files changed, 428 insertions(+), 40 deletions(-) diff --git a/.github/scripts/validate-qualification-contract.mjs b/.github/scripts/validate-qualification-contract.mjs index 4ea6512f..649150f2 100644 --- a/.github/scripts/validate-qualification-contract.mjs +++ b/.github/scripts/validate-qualification-contract.mjs @@ -275,6 +275,27 @@ for (const category of REQUIRED_CATEGORIES) { // 4. Tier 1 subset and negative-trust probes // --------------------------------------------------------------------------- +const gateActivation = tier1.gate?.activation +if (gateActivation?.active === true) { + const activationEvent = gateActivation.activation_event + if ( + activationEvent === null + || typeof activationEvent !== 'object' + || Array.isArray(activationEvent) + || activationEvent.run_id == null + || activationEvent.run_url == null + || activationEvent.date == null + ) { + fail( + 'tier1 gate activation is active but activation_event must name the baseline with non-null ' + + 'run_id, run_url, and date', + ) + } +} +if (gateActivation?.active === false && gateActivation.state !== 'pre_baseline') { + fail('tier1 inactive gate activation must have state "pre_baseline"') +} + for (const cell of tier1.cells) { const task = tasksById.get(cell.task_id) if (!task) { @@ -309,6 +330,25 @@ for (const id of tier2.dimensions.targets) { for (const id of tier2.dimensions.tasks) { if (!tasksById.has(id)) fail(`tier2 matrix references unknown task ${id}`) } + +const cellPair = (cell) => JSON.stringify({ + task_id: cell?.task_id ?? null, + target_id: cell?.target_id ?? null, +}) +const tier1CellPairs = new Set((Array.isArray(tier1.cells) ? tier1.cells : []).map(cellPair)) +const tier2CellPairs = new Set((Array.isArray(tier2.cells) ? tier2.cells : []).map(cellPair)) + +for (const pair of tier1CellPairs) { + if (!tier2CellPairs.has(pair)) { + fail(`tier2-matrix.json#/cells is missing pair ${pair} present in tier1.json#/cells`) + } +} +for (const pair of tier2CellPairs) { + if (!tier1CellPairs.has(pair)) { + fail(`tier1.json#/cells is missing pair ${pair} present in tier2-matrix.json#/cells`) + } +} + if (tier2.status !== 'planned') { fail('tier2-matrix.json must stay planned until its execution prerequisites are met') } @@ -347,6 +387,24 @@ for (const path of walk(join(ROOT, 'examples'))) { } else if (receipt.identity.prompts.user_prompt_sha256 !== task.prompt.sha256) { fail(`${label} records a prompt hash that does not match the frozen prompt for ${receipt.task_id}`) } + + if (receipt.tier === 2 && task?.scoring?.hidden_acceptance_test?.required === true) { + const implementation = receipt.scores?.implementation + if ( + implementation === null + || typeof implementation !== 'object' + || Array.isArray(implementation) + || implementation.measured !== false + || implementation.value !== null + || typeof implementation.not_measured_reason !== 'string' + || implementation.not_measured_reason.length === 0 + ) { + fail( + `${label} Tier 2 task ${receipt.task_id} requires scores.implementation to be not_measured ` + + '(measured false, value null, with a not_measured_reason)', + ) + } + } } // --------------------------------------------------------------------------- diff --git a/docs/qualification/examples/receipt-tier1-valid.json b/docs/qualification/examples/receipt-tier1-valid.json index 0051b8bd..c6ae3075 100644 --- a/docs/qualification/examples/receipt-tier1-valid.json +++ b/docs/qualification/examples/receipt-tier1-valid.json @@ -134,7 +134,22 @@ "path": "raw/context-pack.json", "sha256": "76cae19d9787ab64fb7c0d4be597efe26c4453c855d0d2dea18e6eaa8b83ac7e" }, - "retention_policy": "Tier 1 retains the context artifact only; there is no agent transcript because no agent runs." + "prompt_text": { + "retained": true, + "path": "raw/prompt.txt", + "sha256": "637cd655712c2e7d96a3566ed0b311d9d7dcfe0d9cfb8b699f5f1c52a721eaf0" + }, + "environment_receipt": { + "retained": true, + "path": "raw/environment.json", + "sha256": "cb6043e8b14160ad7ce30ac3dcc967ccfc5a79fa75f55cc12e6de12048f60e32" + }, + "truth_file": { + "retained": true, + "path": "truth/rootcause-hono-middleware-rerun.json", + "sha256": "6e2005ced5ba7ff768d1b9b3135eb909e98d6c1ead04712bc1b0185c480e429d" + }, + "retention_policy": "Tier 1 retains the context artifact, prompt text, environment receipt, and truth file for at least 24 months; there is no agent transcript or answer because no agent runs." }, "notes": [ "Illustrative example only. Tier 1 measures whether the evidence needed to answer was present, never whether an answer was good.", diff --git a/docs/qualification/freeze.json b/docs/qualification/freeze.json index 49970012..0c7a8bf8 100644 --- a/docs/qualification/freeze.json +++ b/docs/qualification/freeze.json @@ -7,23 +7,23 @@ "docs/qualification/README.md": "8153515a9daa7b5bb253c2d5439d8b485c0c6c03958cac2d3f287c74f9675162", "docs/qualification/corpus.json": "8d2fd2ae3386498010fe716dcc8c7903f4759135ca0380cc453ee1f2c1dd8a52", "docs/qualification/evidence-categories.md": "1f75b04229b805050584eadda201bc67906224d34f562afa2098558296a952cd", - "docs/qualification/examples/receipt-tier1-valid.json": "8fc8d9c3a86f2404346ff6dbf7d2833553a67bde238d0746d2df128dc361e11d", + "docs/qualification/examples/receipt-tier1-valid.json": "a68f8d58d1f940ac10e5d5cb1c124644a51e722f886abe53f1e06c7d53f490fd", "docs/qualification/examples/receipt-tier2-invalid-no-madar-call.json": "2783cad80b8a00d12b8d84718f3c4b3e31846111abc4830c47677dd3422a80aa", "docs/qualification/holdout-policy.md": "f5671c4cd5f5487e21dcb44e34fa98b5520a7b9ec58391ce1f086ebd33e23616", "docs/qualification/patches/hono-compose-reentrancy-guard.patch": "9355c5bbb05cd5ae4d998ace18d6381f0cba4fd080203d4d02579da3dcf6dea4", "docs/qualification/patches/hono-error-message-disclosure.patch": "edb79059b72b4f27f5dc8341ba2d9a3617901c402da9ef1f9daf5503e6528d6f", - "docs/qualification/receipt-schema.json": "6fa0e3dcc2a4195c1286cbaae2bdc265e50429a6a636e9d26e7c55c3f1ec6070", - "docs/qualification/rubrics.json": "358d788301113ea1faf27c1011cb887267ff7782ab9ed337a872362cf3edfab5", + "docs/qualification/receipt-schema.json": "e849ba4b28c0cad119a8a23fda82bf39a91f2d53ac6b6900806d0157ac91f0be", + "docs/qualification/rubrics.json": "377a1f72ef64ef7bab03261dea26c8c118792017cf466ef084abbe7d927ce6ba", "docs/qualification/stop-rule.md": "36246356a456d5839e0670b298ee7c34a80e89f47481991eb791d029b972e1c9", - "docs/qualification/tasks.json": "61f7e5185cd4586fc529709d911a724ba748e37b3e5718ed505e429458015117", - "docs/qualification/tier1.json": "5faa20899960c5245030b48c1e652ab0ea0ebaf69c8fdaca44fa37f184aa6292", - "docs/qualification/tier2-matrix.json": "d4a0a071219041f4766382c63ad651b11a64de6b80f5ab7ae4b8435eda985008", + "docs/qualification/tasks.json": "c941088446f21e4f233185794821639ba8095ba3f9984f9cd24a9fc039cbc02b", + "docs/qualification/tier1.json": "967e3d3bafdd646288660c10184d97414d2867948285d062513b8f5d5bcb6ac9", + "docs/qualification/tier2-matrix.json": "449d807630fad5adb46691c6d7c7329992c0674ab099cec8c29329e7fc1f638b", "docs/qualification/truth/arch-unstorage-driver-seam.json": "82e3a9d52c8a19716b30eceb87ca2a2eb9e2107dd3d07498145c0aae70586b89", "docs/qualification/truth/flow-hono-request-dispatch.json": "eb39c4e14a1932c5440fcdac2c6ba20928fba89120879f2667c8bfafabb612fb", "docs/qualification/truth/impact-hono-drop-router-fallback.json": "b500e8d16d32069ed4c6ab8014199e01cd0f0ae4d785ee96eed212b3eaa98f2d", "docs/qualification/truth/plan-unstorage-add-driver.json": "e6332e047475f88ff41358b17211a22b3096be992a3f523673362163ea9f37f2", "docs/qualification/truth/review-hono-error-handling.json": "5e0653e0fa613cc7a4ee72aad1175d7a95ccce603aaf68afe5cc59b2e15854dd", "docs/qualification/truth/rootcause-hono-middleware-rerun.json": "6e2005ced5ba7ff768d1b9b3135eb909e98d6c1ead04712bc1b0185c480e429d", - "docs/qualification/validity-rules.md": "8624a0b99af25b0f03dede1fa055736212e7d7db7197a43f42a98617a34c927e" + "docs/qualification/validity-rules.md": "ee3898f57210693405840f926513205fb7c5c1645c348dc6ee5ea1d45967d18f" } } diff --git a/docs/qualification/receipt-schema.json b/docs/qualification/receipt-schema.json index 8b96c525..b725c7e3 100644 --- a/docs/qualification/receipt-schema.json +++ b/docs/qualification/receipt-schema.json @@ -176,6 +176,7 @@ "unsupported_claims": { "$ref": "#/definitions/score" }, "correct_uncertainty": { "$ref": "#/definitions/score" }, "evidence_support": { "$ref": "#/definitions/score" }, + "implementation": { "$ref": "#/definitions/score" }, "tier1_obligation_recall": { "$ref": "#/definitions/score" } } }, @@ -219,7 +220,7 @@ "retention": { "type": "object", "additionalProperties": false, - "$comment": "One slot per artifact named in validity-rules.md, Retention. Previously prompt_text, environment_receipt and truth_file had no slot at all, so a receipt that followed the prose failed this schema and was then classified incomplete_receipt by the same document that demanded it. They are expressible but not yet required: which artifacts each tier must retain is the open tier-specific question tracked on #655, and answering it here would decide that question by accident.", + "$comment": "One slot per artifact named in validity-rules.md, Retention. The Tier 1 set is required by the tier-conditional rule below; the full artifact list is not required in general.", "required": [ "raw_transcript", "answer_text", @@ -262,6 +263,26 @@ "then": { "properties": { "validity": { "properties": { "invalidation_reasons": { "minItems": 1 } } } } } + }, + { + "description": "A Tier 1 receipt must retain the four artifacts needed to reproduce deterministic evaluation without an agent run.", + "if": { + "properties": { "tier": { "const": 1 } }, + "required": ["tier"] + }, + "then": { + "properties": { + "retention": { + "required": ["context_artifact", "prompt_text", "environment_receipt", "truth_file"], + "properties": { + "context_artifact": { "properties": { "retained": { "const": true } }, "required": ["retained"] }, + "prompt_text": { "properties": { "retained": { "const": true } }, "required": ["retained"] }, + "environment_receipt": { "properties": { "retained": { "const": true } }, "required": ["retained"] }, + "truth_file": { "properties": { "retained": { "const": true } }, "required": ["retained"] } + } + } + } + } } ], diff --git a/docs/qualification/rubrics.json b/docs/qualification/rubrics.json index f3660e18..267e6830 100644 --- a/docs/qualification/rubrics.json +++ b/docs/qualification/rubrics.json @@ -28,14 +28,23 @@ "scale": "ratio in [0,1]", "scored_by": "blinded_human", "tiers": [2], - "gating": true + "gating": false, + "not_gating_reason": "No threshold for this dimension has been calibrated; gating on an uncalibrated threshold manufactures confidence and invites the threshold to be tuned to whatever passes.", + "becomes_gating_when": "A baseline distribution over the natural corpus must exist, and the threshold must be derived and recorded BEFORE any target is scored against it.", + "reported": true, + "reporting_reason": "The dimension is still scored and recorded on every receipt, because an exemption that also stops measurement can never end." }, "evidence_support": { "definition": "Fraction of critical assertions accompanied by a citation to a real path and symbol in the pinned target that actually contains the cited content.", "scale": "ratio in [0,1]", "scored_by": "both", "tiers": [1, 2], - "gating": true, + "gating": false, + "not_gating_reason": "No threshold for this dimension has been calibrated; gating on an uncalibrated threshold manufactures confidence and invites the threshold to be tuned to whatever passes.", + "becomes_gating_when": "A baseline distribution over the natural corpus must exist, and the threshold must be derived and recorded BEFORE any target is scored against it.", + "reported": true, + "reporting_reason": "The dimension is still scored and recorded on every receipt, because an exemption that also stops measurement can never end.", + "not_gating_scope": "This concerns only the Tier 2 blinded rubric. The Tier 1 deterministic gate in tier1.json#/gate (evidence_obligation_recall plus the negative-trust probes) is unaffected and continues to gate pull requests.", "note": "The path/symbol existence half is deterministic. Whether the cited code supports the assertion is blinded-human only." }, "intended_tool_adoption": { @@ -79,7 +88,8 @@ "The reviewer scores correctness, critical_fact_completeness, unsupported_claims, correct_uncertainty, and evidence_support.", "Arm labels are revealed only after every answer in the cell is scored." ], - "pass_condition": "correctness == 2 AND every id in critical_facts_required_for_pass is present AND unsupported_claims == 0" + "pass_condition": "correctness == 2 AND every id in critical_facts_required_for_pass is present AND unsupported_claims == 0", + "pass_condition_note": "correct_uncertainty and evidence_support are deliberately omitted; see dimensions.correct_uncertainty.not_gating_reason and dimensions.evidence_support.not_gating_reason." }, "ordered_path_rubric": { "tier": 2, diff --git a/docs/qualification/tasks.json b/docs/qualification/tasks.json index e06bdad6..5cfeb99f 100644 --- a/docs/qualification/tasks.json +++ b/docs/qualification/tasks.json @@ -24,9 +24,28 @@ "rubric_ref": "rubrics.json#/methods/blinded_rubric" }, "validity_requirements": { - "requires_attributable_madar_call": true, - "requires_answer_within_prompt_contract": true, - "requires_complete_transcript": true + "tier1": { + "rationale": "Tier 1 requires the context artifact, prompt text, environment receipt, and truth file because deterministic evaluation must be reproducible without running an agent.", + "requires_context_artifact": true, + "requires_prompt_text": true, + "requires_environment_receipt": true, + "requires_truth_file": true, + "requires_answer_text": false, + "requires_answer_within_prompt_contract": false, + "requires_complete_transcript": false, + "requires_attributable_madar_call": false + }, + "tier2": { + "rationale": "Tier 2 requires the same four artifacts plus the raw transcript, answer text, and attributable Madar call because the agent run and its comparison must be auditable.", + "requires_context_artifact": true, + "requires_prompt_text": true, + "requires_environment_receipt": true, + "requires_truth_file": true, + "requires_answer_text": true, + "requires_answer_within_prompt_contract": true, + "requires_complete_transcript": true, + "requires_attributable_madar_call": true + } }, "truth_provenance": { "authored_by": "madar-655-qualification-agent", @@ -62,9 +81,28 @@ "rubric_ref": "rubrics.json#/methods/ordered_path_rubric" }, "validity_requirements": { - "requires_attributable_madar_call": true, - "requires_answer_within_prompt_contract": true, - "requires_complete_transcript": true + "tier1": { + "rationale": "Tier 1 requires the context artifact, prompt text, environment receipt, and truth file because deterministic evaluation must be reproducible without running an agent.", + "requires_context_artifact": true, + "requires_prompt_text": true, + "requires_environment_receipt": true, + "requires_truth_file": true, + "requires_answer_text": false, + "requires_answer_within_prompt_contract": false, + "requires_complete_transcript": false, + "requires_attributable_madar_call": false + }, + "tier2": { + "rationale": "Tier 2 requires the same four artifacts plus the raw transcript, answer text, and attributable Madar call because the agent run and its comparison must be auditable.", + "requires_context_artifact": true, + "requires_prompt_text": true, + "requires_environment_receipt": true, + "requires_truth_file": true, + "requires_answer_text": true, + "requires_answer_within_prompt_contract": true, + "requires_complete_transcript": true, + "requires_attributable_madar_call": true + } }, "truth_provenance": { "authored_by": "madar-655-qualification-agent", @@ -100,9 +138,28 @@ "rubric_ref": "rubrics.json#/methods/affected_set_precision_recall" }, "validity_requirements": { - "requires_attributable_madar_call": true, - "requires_answer_within_prompt_contract": true, - "requires_complete_transcript": true + "tier1": { + "rationale": "Tier 1 requires the context artifact, prompt text, environment receipt, and truth file because deterministic evaluation must be reproducible without running an agent.", + "requires_context_artifact": true, + "requires_prompt_text": true, + "requires_environment_receipt": true, + "requires_truth_file": true, + "requires_answer_text": false, + "requires_answer_within_prompt_contract": false, + "requires_complete_transcript": false, + "requires_attributable_madar_call": false + }, + "tier2": { + "rationale": "Tier 2 requires the same four artifacts plus the raw transcript, answer text, and attributable Madar call because the agent run and its comparison must be auditable.", + "requires_context_artifact": true, + "requires_prompt_text": true, + "requires_environment_receipt": true, + "requires_truth_file": true, + "requires_answer_text": true, + "requires_answer_within_prompt_contract": true, + "requires_complete_transcript": true, + "requires_attributable_madar_call": true + } }, "truth_provenance": { "authored_by": "madar-655-qualification-agent", @@ -138,9 +195,28 @@ "rubric_ref": "rubrics.json#/methods/single_root_cause_adjudication" }, "validity_requirements": { - "requires_attributable_madar_call": true, - "requires_answer_within_prompt_contract": true, - "requires_complete_transcript": true + "tier1": { + "rationale": "Tier 1 requires the context artifact, prompt text, environment receipt, and truth file because deterministic evaluation must be reproducible without running an agent.", + "requires_context_artifact": true, + "requires_prompt_text": true, + "requires_environment_receipt": true, + "requires_truth_file": true, + "requires_answer_text": false, + "requires_answer_within_prompt_contract": false, + "requires_complete_transcript": false, + "requires_attributable_madar_call": false + }, + "tier2": { + "rationale": "Tier 2 requires the same four artifacts plus the raw transcript, answer text, and attributable Madar call because the agent run and its comparison must be auditable.", + "requires_context_artifact": true, + "requires_prompt_text": true, + "requires_environment_receipt": true, + "requires_truth_file": true, + "requires_answer_text": true, + "requires_answer_within_prompt_contract": true, + "requires_complete_transcript": true, + "requires_attributable_madar_call": true + } }, "truth_provenance": { "authored_by": "madar-655-qualification-agent", @@ -181,9 +257,28 @@ } }, "validity_requirements": { - "requires_attributable_madar_call": true, - "requires_answer_within_prompt_contract": true, - "requires_complete_transcript": true + "tier1": { + "rationale": "Tier 1 requires the context artifact, prompt text, environment receipt, and truth file because deterministic evaluation must be reproducible without running an agent.", + "requires_context_artifact": true, + "requires_prompt_text": true, + "requires_environment_receipt": true, + "requires_truth_file": true, + "requires_answer_text": false, + "requires_answer_within_prompt_contract": false, + "requires_complete_transcript": false, + "requires_attributable_madar_call": false + }, + "tier2": { + "rationale": "Tier 2 requires the same four artifacts plus the raw transcript, answer text, and attributable Madar call because the agent run and its comparison must be auditable.", + "requires_context_artifact": true, + "requires_prompt_text": true, + "requires_environment_receipt": true, + "requires_truth_file": true, + "requires_answer_text": true, + "requires_answer_within_prompt_contract": true, + "requires_complete_transcript": true, + "requires_attributable_madar_call": true + } }, "truth_provenance": { "authored_by": "madar-655-qualification-agent", @@ -219,9 +314,28 @@ "rubric_ref": "rubrics.json#/methods/seeded_defect_detection" }, "validity_requirements": { - "requires_attributable_madar_call": true, - "requires_answer_within_prompt_contract": true, - "requires_complete_transcript": true + "tier1": { + "rationale": "Tier 1 requires the context artifact, prompt text, environment receipt, and truth file because deterministic evaluation must be reproducible without running an agent.", + "requires_context_artifact": true, + "requires_prompt_text": true, + "requires_environment_receipt": true, + "requires_truth_file": true, + "requires_answer_text": false, + "requires_answer_within_prompt_contract": false, + "requires_complete_transcript": false, + "requires_attributable_madar_call": false + }, + "tier2": { + "rationale": "Tier 2 requires the same four artifacts plus the raw transcript, answer text, and attributable Madar call because the agent run and its comparison must be auditable.", + "requires_context_artifact": true, + "requires_prompt_text": true, + "requires_environment_receipt": true, + "requires_truth_file": true, + "requires_answer_text": true, + "requires_answer_within_prompt_contract": true, + "requires_complete_transcript": true, + "requires_attributable_madar_call": true + } }, "truth_provenance": { "authored_by": "madar-655-qualification-agent", diff --git a/docs/qualification/tier1.json b/docs/qualification/tier1.json index a2a37c32..8fa0c24f 100644 --- a/docs/qualification/tier1.json +++ b/docs/qualification/tier1.json @@ -85,6 +85,16 @@ ], "gate": { "applies_to": "every pull request that touches retrieval, context building, graph construction, or ranking", + "activation": { + "state": "pre_baseline", + "active": false, + "activation_rule": "The gate activates only once a baseline run exists.", + "activation_event": { + "run_id": null, + "run_url": null, + "date": null + } + }, "pass_condition": "every cell passes its evidence_obligation_recall threshold AND every negative_trust_probe satisfies its required_behaviour", "on_failure": "the pull request is blocked; see stop-rule.md", "forbidden_remedies": [ diff --git a/docs/qualification/tier2-matrix.json b/docs/qualification/tier2-matrix.json index 314561a1..326b764d 100644 --- a/docs/qualification/tier2-matrix.json +++ b/docs/qualification/tier2-matrix.json @@ -28,6 +28,15 @@ "cache_modes": ["cold", "warm"], "trials_per_cell": 5 }, + "cells": [ + { "task_id": "arch-unstorage-driver-seam", "target_id": "unstorage" }, + { "task_id": "flow-hono-request-dispatch", "target_id": "hono" }, + { "task_id": "impact-hono-drop-router-fallback", "target_id": "hono" }, + { "task_id": "rootcause-hono-middleware-rerun", "target_id": "hono-seeded-compose" }, + { "task_id": "plan-unstorage-add-driver", "target_id": "unstorage" }, + { "task_id": "review-hono-error-handling", "target_id": "hono-seeded-error-disclosure" } + ], + "sealed_holdout_note": "sealed-holdout-a contributes cells through the external sealed manifest, not through this file.", "trial_rationale": "Five trials per cell is the smallest count that lets a per-cell median be reported with a visible min/max spread while keeping a full sweep affordable. It is not powered for a small effect size. Any claim that depends on a difference smaller than the observed per-cell spread must be reported as not established, whatever the medians say.", "pairing": "Both arms of a cell run against the same prepared target tree — same commit, same applied patch, same verified blob digests — with the same prompt text, the same tool permission list apart from Madar tools, the same cache mode, and the same trial index. A cell where the two arms differ in any identity field is invalid, not a result.", "reporting": { diff --git a/docs/qualification/validity-rules.md b/docs/qualification/validity-rules.md index a5edd28e..82ad205b 100644 --- a/docs/qualification/validity-rules.md +++ b/docs/qualification/validity-rules.md @@ -63,14 +63,16 @@ They are never summed into a single number, and an unmeasured account is ## Retention -For every run, whether valid or not, the following are retained alongside the receipt for -at least **24 months**: - -- the raw agent transcript (Tier 2) or the context artifact (Tier 1); -- the answer text of both arms (Tier 2); -- the exact prompt text delivered to each arm; -- the environment receipt; -- the truth file version used for scoring. +For every run, whether valid or not, the tier-specific artifacts below are retained +alongside the receipt for at least **24 months**. The execution artifact is +the raw agent transcript (Tier 2) or the context artifact (Tier 1). + +- Tier 1 retains the context artifact, exact prompt text, environment receipt, and truth + file because deterministic evaluation must be reproducible without running an agent. It + does not require an agent answer, a raw transcript, or an attributable Madar call. +- Tier 2 retains everything Tier 1 retains, plus the raw agent transcript and the answer + text of both arms, because the agent run and its comparison must be auditable. Where the + task requires an attributable Madar call, the transcript must establish it. Each retained artifact is recorded in `retention` with its path and SHA-256. A run whose artifacts were not retained is `incomplete_receipt`. diff --git a/tests/unit/qualification-contract.test.ts b/tests/unit/qualification-contract.test.ts index 3348535e..1f12bcb4 100644 --- a/tests/unit/qualification-contract.test.ts +++ b/tests/unit/qualification-contract.test.ts @@ -1,7 +1,9 @@ +import { spawnSync } from 'node:child_process' import { createHash } from 'node:crypto' -import { readFileSync } from 'node:fs' +import { cpSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' import { createRequire } from 'node:module' -import { resolve } from 'node:path' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' import { Ajv } from 'ajv' import { describe, expect, it } from 'vitest' @@ -353,6 +355,55 @@ describe('qualification receipt schema', () => { expect(Object.keys(costs).sort()).toEqual(['agent', 'context_build', 'indexing']) expect(costs.agent?.measured).toBe(false) }) + + it('rejects a measured implementation score when the task requires a hidden acceptance test', () => { + const sandbox = mkdtempSync(join(tmpdir(), 'madar-qualification-contract-')) + + try { + mkdirSync(resolve(sandbox, 'docs'), { recursive: true }) + cpSync(resolve(ROOT), resolve(sandbox, ROOT), { recursive: true }) + mkdirSync(resolve(sandbox, 'src')) + const freezePath = resolve(sandbox, `${ROOT}/freeze.json`) + const freezeBefore = readFileSync(freezePath, 'utf8') + + const task = tasks.tasks.find((candidate) => candidate.id === 'plan-unstorage-add-driver') + const mutated = JSON.parse(JSON.stringify(invalidTier2)) as { + task_id: string + target_id: string + identity: { prompts: { user_prompt_sha256: string; user_prompt_text: string } } + scores: Record + } + mutated.task_id = task!.id + mutated.target_id = task!.target + mutated.identity.prompts.user_prompt_sha256 = task!.prompt.sha256 + mutated.identity.prompts.user_prompt_text = task!.prompt.text + mutated.scores.implementation = { + measured: true, + value: 1, + method: 'hidden_acceptance_test', + } + writeFileSync( + resolve(sandbox, `${ROOT}/examples/receipt-tier2-invalid-no-madar-call.json`), + `${JSON.stringify(mutated, null, 2)}\n`, + ) + + const result = spawnSync( + process.execPath, + [resolve('.github/scripts/validate-qualification-contract.mjs'), '--write'], + { cwd: sandbox, encoding: 'utf8' }, + ) + + expect(result.status).toBe(1) + expect(readFileSync(freezePath, 'utf8')).toBe(freezeBefore) + expect(result.stderr).toContain( + 'docs/qualification/examples/receipt-tier2-invalid-no-madar-call.json Tier 2 task ' + + 'plan-unstorage-add-driver requires scores.implementation to be not_measured ' + + '(measured false, value null, with a not_measured_reason)', + ) + } finally { + rmSync(sandbox, { recursive: true, force: true }) + } + }) }) describe('qualification Tier 1 subset', () => { @@ -362,6 +413,68 @@ describe('qualification Tier 1 subset', () => { expect(tier1.properties.requires_api_spend).toBe(false) }) + it('rejects an active Tier 1 gate that does not name its baseline activation event', () => { + const sandbox = mkdtempSync(join(tmpdir(), 'madar-qualification-contract-')) + + try { + mkdirSync(resolve(sandbox, 'docs'), { recursive: true }) + cpSync(resolve(ROOT), resolve(sandbox, ROOT), { recursive: true }) + mkdirSync(resolve(sandbox, 'src')) + + const mutated = JSON.parse(JSON.stringify(tier1)) as { + gate: { + activation: { + active: boolean + activation_event: { run_id: string | null; run_url: string | null; date: string | null } + } + } + } + mutated.gate.activation.active = true + writeFileSync(resolve(sandbox, `${ROOT}/tier1.json`), `${JSON.stringify(mutated, null, 2)}\n`) + + const result = spawnSync( + process.execPath, + [resolve('.github/scripts/validate-qualification-contract.mjs'), '--write'], + { cwd: sandbox, encoding: 'utf8' }, + ) + + expect(result.status).toBe(1) + expect(result.stderr).toContain( + 'tier1 gate activation is active but activation_event must name the baseline with non-null ' + + 'run_id, run_url, and date', + ) + } finally { + rmSync(sandbox, { recursive: true, force: true }) + } + }) + + it('rejects an inactive Tier 1 gate outside the pre_baseline state', () => { + const sandbox = mkdtempSync(join(tmpdir(), 'madar-qualification-contract-')) + + try { + mkdirSync(resolve(sandbox, 'docs'), { recursive: true }) + cpSync(resolve(ROOT), resolve(sandbox, ROOT), { recursive: true }) + mkdirSync(resolve(sandbox, 'src')) + + const mutated = JSON.parse(JSON.stringify(tier1)) as { + gate: { activation: { active: boolean; state: string } } + } + mutated.gate.activation.state = 'active' + writeFileSync(resolve(sandbox, `${ROOT}/tier1.json`), `${JSON.stringify(mutated, null, 2)}\n`) + + const result = spawnSync( + process.execPath, + [resolve('.github/scripts/validate-qualification-contract.mjs'), '--write'], + { cwd: sandbox, encoding: 'utf8' }, + ) + + expect(result.status).toBe(1) + expect(result.stderr).toContain('tier1 inactive gate activation must have state "pre_baseline"') + } finally { + rmSync(sandbox, { recursive: true, force: true }) + } + }) + it('fails a cell whose target could not be prepared instead of skipping it', () => { expect(tier1.preparation.steps.length).toBeGreaterThan(0) expect(tier1.preparation.on_preparation_failure).toContain('never silently skipped') @@ -394,6 +507,42 @@ describe('qualification Tier 1 subset', () => { expect(tier2.status).toBe('planned') expect(tier2.dimensions.trials_per_cell).toBeGreaterThan(1) }) + + it('rejects Tier 2 cells that drift from the Tier 1 task-target pairs', () => { + const sandbox = mkdtempSync(join(tmpdir(), 'madar-qualification-contract-')) + + try { + mkdirSync(resolve(sandbox, 'docs'), { recursive: true }) + cpSync(resolve(ROOT), resolve(sandbox, ROOT), { recursive: true }) + mkdirSync(resolve(sandbox, 'src')) + + const mutated = JSON.parse(JSON.stringify(tier2)) as { + cells: Array<{ task_id: string; target_id: string }> + } + mutated.cells[0]!.target_id = 'hono' + writeFileSync(resolve(sandbox, `${ROOT}/tier2-matrix.json`), `${JSON.stringify(mutated, null, 2)}\n`) + + const result = spawnSync( + process.execPath, + [resolve('.github/scripts/validate-qualification-contract.mjs'), '--write'], + { cwd: sandbox, encoding: 'utf8' }, + ) + + expect(result.status).toBe(1) + expect(result.stderr).toContain( + 'tier2-matrix.json#/cells is missing pair ' + + '{"task_id":"arch-unstorage-driver-seam","target_id":"unstorage"} ' + + 'present in tier1.json#/cells', + ) + expect(result.stderr).toContain( + 'tier1.json#/cells is missing pair ' + + '{"task_id":"arch-unstorage-driver-seam","target_id":"hono"} ' + + 'present in tier2-matrix.json#/cells', + ) + } finally { + rmSync(sandbox, { recursive: true, force: true }) + } + }) }) describe('qualification policy documents', () => { From 4c28794cd19682f58af04ec2eebdbf3ee3869700 Mon Sep 17 00:00:00 2001 From: mohammed naji Date: Fri, 14 Aug 2026 12:02:47 +0400 Subject: [PATCH 09/10] fix(qualification): make validator path output platform-independent (#655) Both Windows lanes of the exact-head matrix on 6ceef755 failed on the new Decision 14 regression test: it asserts the validator names the offending receipt, and `relative()` returns backslashes on Windows, so the expected `docs/qualification/examples/...` never matched. Fix it at the source rather than in the assertion. Validator output is compared in tests and read in CI logs, so it should be byte-identical on every platform. The file already established that idiom for the freeze map in cec02d2a; this lifts it into one `relPath()` helper and routes the receipt label, the production-literal message, the freeze map and the --write log through it. No frozen digest moves: the freeze map was already normalised, so this only changes what the validator prints. Refs #655. Related parent: #649. --- .../scripts/validate-qualification-contract.mjs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/scripts/validate-qualification-contract.mjs b/.github/scripts/validate-qualification-contract.mjs index 649150f2..c10c2cbb 100644 --- a/.github/scripts/validate-qualification-contract.mjs +++ b/.github/scripts/validate-qualification-contract.mjs @@ -23,6 +23,12 @@ function fail(message) { failures.push(message) } +// Validator output must be byte-identical across platforms: it is compared in tests and +// read in CI logs, and `relative()` yields backslashes on Windows. Normalise once, here. +function relPath(path) { + return relative(process.cwd(), path).split('\\').join('/') +} + function readJson(path) { return JSON.parse(readFileSync(path, 'utf8')) } @@ -363,7 +369,7 @@ const validateReceipt = ajv.compile(receiptSchema) for (const path of walk(join(ROOT, 'examples'))) { const receipt = readJson(path) - const label = relative(process.cwd(), path) + const label = relPath(path) // A receipt that failed schema validation has no guaranteed shape, so reading // `receipt.validity` or `receipt.scores` below would throw before the collected failures @@ -454,7 +460,7 @@ for (const path of walk(PRODUCTION_ROOT)) { const content = readFileSync(path, 'utf8') for (const literal of FORBIDDEN_LITERALS) { if (content.includes(literal)) { - fail(`production file ${relative(process.cwd(), path)} contains qualification literal "${literal}"`) + fail(`production file ${relPath(path)} contains qualification literal "${literal}"`) } } } @@ -509,7 +515,7 @@ if (VERIFY_CORPUS) { const frozenFiles = walk(ROOT) .filter((path) => path !== FREEZE_PATH) - .map((path) => relative(process.cwd(), path).split('\\').join('/')) + .map((path) => relPath(path)) .sort() const digests = Object.fromEntries( @@ -573,7 +579,7 @@ if (WRITE) { files: digests, } writeFileSync(FREEZE_PATH, `${JSON.stringify(freeze, null, 2)}\n`) - console.log(`wrote ${relative(process.cwd(), FREEZE_PATH)} with ${frozenFiles.length} entries`) + console.log(`wrote ${relPath(FREEZE_PATH)} with ${frozenFiles.length} entries`) } const naturalTargets = corpus.targets.filter((target) => !isSealedTarget(target)) From 4d15ee4cbf1bef5948b1e4e78f9150f5f34a8d4b Mon Sep 17 00:00:00 2001 From: mohammed naji Date: Fri, 14 Aug 2026 12:27:08 +0400 Subject: [PATCH 10/10] fix(qualification): address the re-review of the decided contract terms (#655) Five findings from the CodeRabbit re-review of 4c28794c, all assessed as real: 1. An empty patch file skipped every structural patch check. `if (patch)` is falsy for an empty read, so the diff --git, CRLF and touched-path checks were all skipped and the target passed -- and an empty patch means the seeded defect is gone, which is exactly what that section exists to catch. Now distinguishes unreadable (undefined) from empty (''). 2. Unguarded task field reads could still replace the collected failure list with a stack trace. task.prompt.text, task.prompt.sha256 and task.scoring.tier2_method were dereferenced directly. That is the defect class 2fddb25a fixed, and it should not survive anywhere in this file. Both are now guarded and reported, and tier1.cells is iterated defensively to match the convention already used further down. 3. The gate activation rule added for decision 11 passed when `active` was absent or non-boolean, and accepted an active gate still in state pre_baseline. It now requires the block, requires a boolean, and pairs each value with its state. 4. single_root_cause_adjudication re-gated correct_uncertainty. Resolved by stating the boundary rather than weakening either side: the ratio dimension has no calibrated threshold and does not gate, while the method's uncertainty_required_for_pass is an explicit pre-registered id list, which is checkable without calibration and does gate. Recorded on both the dimension and the method so a reader arriving from either finds it. 5. Two tests stated opposite contracts for independent_of_production_rule_author. The snapshot test is now named as a snapshot and says that closing the independence gap requires updating it in the same change. Every validator branch touched or added was mutation-tested and turns exactly one regression test red when disabled. freeze.json: only rubrics.json moves, 377a1f72 -> bf176381, for finding 4. Local: qualify:validate consistent, typecheck clean, 50 tests passed. Refs #655. Related parent: #649. --- .../validate-qualification-contract.mjs | 71 ++++-- docs/qualification/freeze.json | 2 +- docs/qualification/rubrics.json | 4 +- tests/unit/qualification-contract.test.ts | 207 +++++++++++++++++- 4 files changed, 262 insertions(+), 22 deletions(-) diff --git a/.github/scripts/validate-qualification-contract.mjs b/.github/scripts/validate-qualification-contract.mjs index c10c2cbb..5fa4c53a 100644 --- a/.github/scripts/validate-qualification-contract.mjs +++ b/.github/scripts/validate-qualification-contract.mjs @@ -148,7 +148,7 @@ for (const target of corpus.targets) { fail(`patched target ${target.id} references missing patch ${target.patch}`) } - if (patch) { + if (patch !== undefined) { // Parse against normalized text so a CRLF checkout cannot capture a stray // carriage return into a path. The digest check further down still reads raw // bytes; only this structural parse is representation-independent. @@ -199,9 +199,33 @@ for (const task of tasks.tasks) { continue } - const actualHash = sha256(task.prompt.text) - if (actualHash !== task.prompt.sha256) { - fail(`task ${task.id} prompt hash mismatch: recorded ${task.prompt.sha256}, actual ${actualHash}`) + const prompt = task.prompt + if ( + prompt === null + || typeof prompt !== 'object' + || Array.isArray(prompt) + || typeof prompt.text !== 'string' + || typeof prompt.sha256 !== 'string' + ) { + fail(`task ${task.id} must declare prompt text and sha256`) + continue + } + + const actualHash = sha256(prompt.text) + if (actualHash !== prompt.sha256) { + fail(`task ${task.id} prompt hash mismatch: recorded ${prompt.sha256}, actual ${actualHash}`) + } + + const scoring = task.scoring + if ( + scoring === null + || typeof scoring !== 'object' + || Array.isArray(scoring) + || typeof scoring.tier1_method !== 'string' + || typeof scoring.tier2_method !== 'string' + ) { + fail(`task ${task.id} must declare tier1 and tier2 scoring methods`) + continue } const truthPath = join(ROOT, task.truth_ref) @@ -263,11 +287,11 @@ for (const task of tasks.tasks) { fail(`${task.truth_ref} must declare at least one must_not_report_ready_when condition`) } - if (!rubrics.methods[task.scoring.tier2_method]) { - fail(`task ${task.id} references unknown rubric method ${task.scoring.tier2_method}`) + if (!rubrics.methods[scoring.tier2_method]) { + fail(`task ${task.id} references unknown rubric method ${scoring.tier2_method}`) } - if (!rubrics.methods[task.scoring.tier1_method]) { - fail(`task ${task.id} references unknown tier1 method ${task.scoring.tier1_method}`) + if (!rubrics.methods[scoring.tier1_method]) { + fail(`task ${task.id} references unknown tier1 method ${scoring.tier1_method}`) } } @@ -282,7 +306,23 @@ for (const category of REQUIRED_CATEGORIES) { // --------------------------------------------------------------------------- const gateActivation = tier1.gate?.activation -if (gateActivation?.active === true) { +const hasGateActivation = gateActivation !== null + && typeof gateActivation === 'object' + && !Array.isArray(gateActivation) +if (!hasGateActivation) { + fail('tier1 gate.activation block must exist') +} +if (hasGateActivation && typeof gateActivation.active !== 'boolean') { + fail('tier1 gate.activation.active must be a boolean') +} +if ( + hasGateActivation + && gateActivation.active === true + && (typeof gateActivation.state !== 'string' || gateActivation.state.length === 0 || gateActivation.state === 'pre_baseline') +) { + fail('tier1 active gate activation must declare a non-pre_baseline state') +} +if (hasGateActivation && gateActivation.active === true) { const activationEvent = gateActivation.activation_event if ( activationEvent === null @@ -298,11 +338,12 @@ if (gateActivation?.active === true) { ) } } -if (gateActivation?.active === false && gateActivation.state !== 'pre_baseline') { +if (hasGateActivation && gateActivation.active === false && gateActivation.state !== 'pre_baseline') { fail('tier1 inactive gate activation must have state "pre_baseline"') } -for (const cell of tier1.cells) { +const tier1Cells = Array.isArray(tier1.cells) ? tier1.cells : [] +for (const cell of tier1Cells) { const task = tasksById.get(cell.task_id) if (!task) { fail(`tier1 cell references unknown task ${cell.task_id}`) @@ -341,7 +382,7 @@ const cellPair = (cell) => JSON.stringify({ task_id: cell?.task_id ?? null, target_id: cell?.target_id ?? null, }) -const tier1CellPairs = new Set((Array.isArray(tier1.cells) ? tier1.cells : []).map(cellPair)) +const tier1CellPairs = new Set(tier1Cells.map(cellPair)) const tier2CellPairs = new Set((Array.isArray(tier2.cells) ? tier2.cells : []).map(cellPair)) for (const pair of tier1CellPairs) { @@ -390,7 +431,7 @@ for (const path of walk(join(ROOT, 'examples'))) { const task = tasksById.get(receipt.task_id) if (!task) { fail(`${label} references unknown task ${receipt.task_id}`) - } else if (receipt.identity.prompts.user_prompt_sha256 !== task.prompt.sha256) { + } else if (receipt.identity.prompts.user_prompt_sha256 !== task.prompt?.sha256) { fail(`${label} records a prompt hash that does not match the frozen prompt for ${receipt.task_id}`) } @@ -451,7 +492,7 @@ for (const [key, value] of Object.entries(corpus.forbidden_target_symbols ?? {}) const FORBIDDEN_LITERALS = [ ...corpus.targets.flatMap((target) => (target.source?.url ? [target.source.url, target.source.ref] : [])), ...tasks.tasks.map((task) => task.id), - ...tasks.tasks.map((task) => task.prompt.text), + ...tasks.tasks.flatMap((task) => (typeof task.prompt?.text === 'string' ? [task.prompt.text] : [])), ...tier1.negative_trust_probes.map((probe) => probe.prompt.text), ...targetSymbols, ] @@ -587,6 +628,6 @@ const naturalTargets = corpus.targets.filter((target) => !isSealedTarget(target) console.log( `qualification contract v${CONTRACT_VERSION} is consistent: ` + `${naturalTargets.length} pinned natural targets, ${corpus.proxy_targets.length} proxy targets, ` + - `${tasks.tasks.length} tasks, ${tier1.cells.length} Tier 1 cells, ` + + `${tasks.tasks.length} tasks, ${tier1Cells.length} Tier 1 cells, ` + `${tier1.negative_trust_probes.length} negative-trust probes, ${frozenFiles.length} frozen files.`, ) diff --git a/docs/qualification/freeze.json b/docs/qualification/freeze.json index 0c7a8bf8..da591dcd 100644 --- a/docs/qualification/freeze.json +++ b/docs/qualification/freeze.json @@ -13,7 +13,7 @@ "docs/qualification/patches/hono-compose-reentrancy-guard.patch": "9355c5bbb05cd5ae4d998ace18d6381f0cba4fd080203d4d02579da3dcf6dea4", "docs/qualification/patches/hono-error-message-disclosure.patch": "edb79059b72b4f27f5dc8341ba2d9a3617901c402da9ef1f9daf5503e6528d6f", "docs/qualification/receipt-schema.json": "e849ba4b28c0cad119a8a23fda82bf39a91f2d53ac6b6900806d0157ac91f0be", - "docs/qualification/rubrics.json": "377a1f72ef64ef7bab03261dea26c8c118792017cf466ef084abbe7d927ce6ba", + "docs/qualification/rubrics.json": "bf176381b6d6df5da41c4a3685bfaf8bf31f0767feef571ee7b85ce4644eba6f", "docs/qualification/stop-rule.md": "36246356a456d5839e0670b298ee7c34a80e89f47481991eb791d029b972e1c9", "docs/qualification/tasks.json": "c941088446f21e4f233185794821639ba8095ba3f9984f9cd24a9fc039cbc02b", "docs/qualification/tier1.json": "967e3d3bafdd646288660c10184d97414d2867948285d062513b8f5d5bcb6ac9", diff --git a/docs/qualification/rubrics.json b/docs/qualification/rubrics.json index 267e6830..4b2bfef0 100644 --- a/docs/qualification/rubrics.json +++ b/docs/qualification/rubrics.json @@ -31,6 +31,7 @@ "gating": false, "not_gating_reason": "No threshold for this dimension has been calibrated; gating on an uncalibrated threshold manufactures confidence and invites the threshold to be tuned to whatever passes.", "becomes_gating_when": "A baseline distribution over the natural corpus must exist, and the threshold must be derived and recorded BEFORE any target is scored against it.", + "method_gate_boundary": "The ratio dimension remains non-gating. methods.single_root_cause_adjudication gates only the explicit, pre-registered ids in uncertainty_required_for_pass; honouring that enumerated id list is deliberately not a calibrated threshold.", "reported": true, "reporting_reason": "The dimension is still scored and recorded on every receipt, because an exemption that also stops measurement can never end." }, @@ -118,7 +119,8 @@ "The answer passes only if it names a cause in accepted_root_cause_ids as THE cause.", "Listing the correct cause among several candidate causes without committing scores as a partial: critical_fact_completeness credit, correctness 1, no pass.", "Every id in uncertainty_required_for_pass must be honoured." - ] + ], + "uncertainty_gate_note": "This method gates uncertainty by the explicit, pre-registered ids in uncertainty_required_for_pass. Honouring an enumerated id list is checkable without calibration and is deliberately not a threshold on dimensions.correct_uncertainty, which remains non-gating." }, "seeded_defect_detection": { "tier": 2, diff --git a/tests/unit/qualification-contract.test.ts b/tests/unit/qualification-contract.test.ts index 1f12bcb4..8c1963fe 100644 --- a/tests/unit/qualification-contract.test.ts +++ b/tests/unit/qualification-contract.test.ts @@ -157,6 +157,29 @@ describe('qualification corpus manifest', () => { } }) + it('rejects an empty seeded-defect patch', () => { + const sandbox = mkdtempSync(join(tmpdir(), 'madar-qualification-contract-')) + + try { + mkdirSync(resolve(sandbox, 'docs'), { recursive: true }) + cpSync(resolve(ROOT), resolve(sandbox, ROOT), { recursive: true }) + mkdirSync(resolve(sandbox, 'src')) + writeFileSync(resolve(sandbox, `${ROOT}/patches/hono-compose-reentrancy-guard.patch`), '') + + const result = spawnSync( + process.execPath, + [resolve('.github/scripts/validate-qualification-contract.mjs'), '--write'], + { cwd: sandbox, encoding: 'utf8' }, + ) + + expect(result.status).toBe(1) + expect(result.stderr).toContain('patch patches/hono-compose-reentrancy-guard.patch is not a unified git diff') + expect(result.stderr).toContain('patch patches/hono-compose-reentrancy-guard.patch does not modify any file') + } finally { + rmSync(sandbox, { recursive: true, force: true }) + } + }) + it('discloses where a target overlaps a shipped framework adapter', () => { const hono = corpus.targets.find((target) => target.id === 'hono') const unstorage = corpus.targets.find((target) => target.id === 'unstorage') @@ -211,6 +234,62 @@ describe('qualification task definitions', () => { } }) + it('reports a task with no prompt instead of throwing', () => { + const sandbox = mkdtempSync(join(tmpdir(), 'madar-qualification-contract-')) + + try { + mkdirSync(resolve(sandbox, 'docs'), { recursive: true }) + cpSync(resolve(ROOT), resolve(sandbox, ROOT), { recursive: true }) + mkdirSync(resolve(sandbox, 'src')) + + const mutated = JSON.parse(JSON.stringify(tasks)) as { + tasks: Array<{ id: string; prompt?: unknown }> + } + const task = mutated.tasks.find((candidate) => candidate.id === 'rootcause-hono-middleware-rerun')! + delete task.prompt + writeFileSync(resolve(sandbox, `${ROOT}/tasks.json`), `${JSON.stringify(mutated, null, 2)}\n`) + + const result = spawnSync( + process.execPath, + [resolve('.github/scripts/validate-qualification-contract.mjs'), '--write'], + { cwd: sandbox, encoding: 'utf8' }, + ) + + expect(result.status).toBe(1) + expect(result.stderr).toContain('task rootcause-hono-middleware-rerun must declare prompt text and sha256') + } finally { + rmSync(sandbox, { recursive: true, force: true }) + } + }) + + it('reports a task with no scoring block instead of throwing', () => { + const sandbox = mkdtempSync(join(tmpdir(), 'madar-qualification-contract-')) + + try { + mkdirSync(resolve(sandbox, 'docs'), { recursive: true }) + cpSync(resolve(ROOT), resolve(sandbox, ROOT), { recursive: true }) + mkdirSync(resolve(sandbox, 'src')) + + const mutated = JSON.parse(JSON.stringify(tasks)) as { + tasks: Array<{ id: string; scoring?: unknown }> + } + const task = mutated.tasks.find((candidate) => candidate.id === 'rootcause-hono-middleware-rerun')! + delete task.scoring + writeFileSync(resolve(sandbox, `${ROOT}/tasks.json`), `${JSON.stringify(mutated, null, 2)}\n`) + + const result = spawnSync( + process.execPath, + [resolve('.github/scripts/validate-qualification-contract.mjs'), '--write'], + { cwd: sandbox, encoding: 'utf8' }, + ) + + expect(result.status).toBe(1) + expect(result.stderr).toContain('task rootcause-hono-middleware-rerun must declare tier1 and tier2 scoring methods') + } finally { + rmSync(sandbox, { recursive: true, force: true }) + } + }) + it('never names the coupled framework inside a prompt for that target', () => { const coupled = tasks.tasks.filter((task) => task.target.startsWith('hono')) @@ -230,18 +309,16 @@ describe('qualification task definitions', () => { expect(provenance.derived_from.length).toBeGreaterThan(0) expect(provenance.madar_derived_sources_used).toEqual([]) expect(provenance.inspected_madar_output_before_freeze).toBe(false) - // The contract requires this to be *stated*, not to hold a particular value. Pinning - // it to false would fail the moment a second author closes the independence gap, - // which is an improvement, not a regression. - expect(typeof provenance.independent_of_production_rule_author).toBe('boolean') } } }) - it('records the current independence state, which a second author is expected to change', () => { + it('snapshots the current single-author independence state', () => { for (const task of tasks.tasks) { const truth = readJson<{ provenance: Provenance }>(`${ROOT}/${task.truth_ref}`) for (const provenance of [task.truth_provenance, truth.provenance]) { + // Closing the independence gap requires updating this current-state expectation in + // the same change that records the second author's review. expect(provenance.independent_of_production_rule_author).toBe(false) } } @@ -413,6 +490,87 @@ describe('qualification Tier 1 subset', () => { expect(tier1.properties.requires_api_spend).toBe(false) }) + it('reports a missing Tier 1 cell array instead of throwing', () => { + const sandbox = mkdtempSync(join(tmpdir(), 'madar-qualification-contract-')) + + try { + mkdirSync(resolve(sandbox, 'docs'), { recursive: true }) + cpSync(resolve(ROOT), resolve(sandbox, ROOT), { recursive: true }) + mkdirSync(resolve(sandbox, 'src')) + + const mutated = JSON.parse(JSON.stringify(tier1)) as { cells: unknown } + mutated.cells = null + writeFileSync(resolve(sandbox, `${ROOT}/tier1.json`), `${JSON.stringify(mutated, null, 2)}\n`) + + const result = spawnSync( + process.execPath, + [resolve('.github/scripts/validate-qualification-contract.mjs'), '--write'], + { cwd: sandbox, encoding: 'utf8' }, + ) + + expect(result.status).toBe(1) + expect(result.stderr).toContain( + 'tier1.json#/cells is missing pair ' + + '{"task_id":"arch-unstorage-driver-seam","target_id":"unstorage"} ' + + 'present in tier2-matrix.json#/cells', + ) + } finally { + rmSync(sandbox, { recursive: true, force: true }) + } + }) + + it('rejects a missing Tier 1 gate activation block', () => { + const sandbox = mkdtempSync(join(tmpdir(), 'madar-qualification-contract-')) + + try { + mkdirSync(resolve(sandbox, 'docs'), { recursive: true }) + cpSync(resolve(ROOT), resolve(sandbox, ROOT), { recursive: true }) + mkdirSync(resolve(sandbox, 'src')) + + const mutated = JSON.parse(JSON.stringify(tier1)) as { gate: { activation?: unknown } } + delete mutated.gate.activation + writeFileSync(resolve(sandbox, `${ROOT}/tier1.json`), `${JSON.stringify(mutated, null, 2)}\n`) + + const result = spawnSync( + process.execPath, + [resolve('.github/scripts/validate-qualification-contract.mjs'), '--write'], + { cwd: sandbox, encoding: 'utf8' }, + ) + + expect(result.status).toBe(1) + expect(result.stderr).toContain('tier1 gate.activation block must exist') + } finally { + rmSync(sandbox, { recursive: true, force: true }) + } + }) + + it('rejects a non-boolean Tier 1 gate activation flag', () => { + const sandbox = mkdtempSync(join(tmpdir(), 'madar-qualification-contract-')) + + try { + mkdirSync(resolve(sandbox, 'docs'), { recursive: true }) + cpSync(resolve(ROOT), resolve(sandbox, ROOT), { recursive: true }) + mkdirSync(resolve(sandbox, 'src')) + + const mutated = JSON.parse(JSON.stringify(tier1)) as { + gate: { activation: { active: unknown } } + } + mutated.gate.activation.active = 'false' + writeFileSync(resolve(sandbox, `${ROOT}/tier1.json`), `${JSON.stringify(mutated, null, 2)}\n`) + + const result = spawnSync( + process.execPath, + [resolve('.github/scripts/validate-qualification-contract.mjs'), '--write'], + { cwd: sandbox, encoding: 'utf8' }, + ) + + expect(result.status).toBe(1) + expect(result.stderr).toContain('tier1 gate.activation.active must be a boolean') + } finally { + rmSync(sandbox, { recursive: true, force: true }) + } + }) + it('rejects an active Tier 1 gate that does not name its baseline activation event', () => { const sandbox = mkdtempSync(join(tmpdir(), 'madar-qualification-contract-')) @@ -425,11 +583,13 @@ describe('qualification Tier 1 subset', () => { gate: { activation: { active: boolean + state: string activation_event: { run_id: string | null; run_url: string | null; date: string | null } } } } mutated.gate.activation.active = true + mutated.gate.activation.state = 'active' writeFileSync(resolve(sandbox, `${ROOT}/tier1.json`), `${JSON.stringify(mutated, null, 2)}\n`) const result = spawnSync( @@ -448,6 +608,43 @@ describe('qualification Tier 1 subset', () => { } }) + it('rejects an active Tier 1 gate in the pre_baseline state', () => { + const sandbox = mkdtempSync(join(tmpdir(), 'madar-qualification-contract-')) + + try { + mkdirSync(resolve(sandbox, 'docs'), { recursive: true }) + cpSync(resolve(ROOT), resolve(sandbox, ROOT), { recursive: true }) + mkdirSync(resolve(sandbox, 'src')) + + const mutated = JSON.parse(JSON.stringify(tier1)) as { + gate: { + activation: { + active: boolean + activation_event: { run_id: string | null; run_url: string | null; date: string | null } + } + } + } + mutated.gate.activation.active = true + mutated.gate.activation.activation_event = { + run_id: 'baseline-0001', + run_url: 'https://example.invalid/runs/baseline-0001', + date: '2026-08-14', + } + writeFileSync(resolve(sandbox, `${ROOT}/tier1.json`), `${JSON.stringify(mutated, null, 2)}\n`) + + const result = spawnSync( + process.execPath, + [resolve('.github/scripts/validate-qualification-contract.mjs'), '--write'], + { cwd: sandbox, encoding: 'utf8' }, + ) + + expect(result.status).toBe(1) + expect(result.stderr).toContain('tier1 active gate activation must declare a non-pre_baseline state') + } finally { + rmSync(sandbox, { recursive: true, force: true }) + } + }) + it('rejects an inactive Tier 1 gate outside the pre_baseline state', () => { const sandbox = mkdtempSync(join(tmpdir(), 'madar-qualification-contract-'))