diff --git a/.gitattributes b/.gitattributes index 9f70f102..45c2909e 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1,12 @@ docs/benchmarks/suite/isolation/.claude/CLAUDE.md text eol=lf + +# The qualification contract is frozen by SHA-256 over raw file bytes +# (docs/qualification/freeze.json). A CRLF checkout would change those bytes and +# break the freeze on Windows, so the whole tree is pinned to LF in the working +# copy on every platform. Do not relax this to make a platform failure disappear: +# the freeze is meaningful precisely because it is byte-exact. +docs/qualification/** text eol=lf + +# Seeded-defect patches must reach `git apply` byte-for-byte, so they are marked +# non-text and are never line-ending converted in either direction. +docs/qualification/patches/*.patch -text diff --git a/.github/scripts/lib/collect-cited-paths.cjs b/.github/scripts/lib/collect-cited-paths.cjs new file mode 100644 index 00000000..ad47c56f --- /dev/null +++ b/.github/scripts/lib/collect-cited-paths.cjs @@ -0,0 +1,46 @@ +'use strict' + +/** + * The single definition of "which paths does a truth file cite". + * + * It lives here, in CommonJS, because both consumers need it and they cannot share an + * ESM module: `.github/scripts/validate-qualification-contract.mjs` is ESM, and + * `tests/unit/qualification-contract.test.ts` is TypeScript compiled without `allowJs`, + * so it reaches this file through `createRequire`. Node imports CommonJS from ESM + * natively, so the validator can import it directly. + * + * Both consumers previously carried their own copy of this traversal, including the + * `new_path` exemption. A change to the exemption in one copy would have left the other + * enforcing the old rule, and the test would have stopped covering the shipped guard + * while still passing. + * + * `new_path` is exempt because a plan task proposes creating files that do not exist at + * the pinned commit, so they cannot appear in the target's frozen blob manifest. + * + * @param {unknown} node - any subtree of a parsed truth file + * @returns {Set} every value recorded under a `path` key, at any depth + */ +function collectCitedPaths(node) { + const cited = new Set() + + const walk = (value) => { + if (Array.isArray(value)) { + value.forEach(walk) + return + } + if (value && typeof value === 'object') { + for (const [key, child] of Object.entries(value)) { + if (key === 'path' && typeof child === 'string') { + cited.add(child) + } else if (key !== 'new_path') { + walk(child) + } + } + } + } + + walk(node) + return cited +} + +module.exports = { collectCitedPaths } diff --git a/.github/scripts/validate-qualification-contract.mjs b/.github/scripts/validate-qualification-contract.mjs new file mode 100644 index 00000000..5fa4c53a --- /dev/null +++ b/.github/scripts/validate-qualification-contract.mjs @@ -0,0 +1,633 @@ +import { execFileSync } from 'node:child_process' +import { createHash } from 'node:crypto' +import { mkdtempSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join, relative, resolve } from 'node:path' + +import Ajv from 'ajv' +import addFormats from 'ajv-formats' + +import citedPathCollector from './lib/collect-cited-paths.cjs' + +const { collectCitedPaths } = citedPathCollector + +const ROOT = resolve('docs/qualification') +const FREEZE_PATH = join(ROOT, 'freeze.json') +const PRODUCTION_ROOT = resolve('src') +const WRITE = process.argv.includes('--write') +const VERIFY_CORPUS = process.argv.includes('--verify-corpus') + +const failures = [] + +function fail(message) { + failures.push(message) +} + +// Validator output must be byte-identical across platforms: it is compared in tests and +// read in CI logs, and `relative()` yields backslashes on Windows. Normalise once, here. +function relPath(path) { + return relative(process.cwd(), path).split('\\').join('/') +} + +function readJson(path) { + return JSON.parse(readFileSync(path, 'utf8')) +} + +function sha256(value) { + return createHash('sha256').update(value).digest('hex') +} + +function walk(dir) { + const entries = [] + for (const name of readdirSync(dir).sort()) { + const full = join(dir, name) + if (statSync(full).isDirectory()) { + entries.push(...walk(full)) + } else { + entries.push(full) + } + } + return entries +} + +const corpus = readJson(join(ROOT, 'corpus.json')) +const tasks = readJson(join(ROOT, 'tasks.json')) +const rubrics = readJson(join(ROOT, 'rubrics.json')) +const tier1 = readJson(join(ROOT, 'tier1.json')) +const tier2 = readJson(join(ROOT, 'tier2-matrix.json')) +const receiptSchema = readJson(join(ROOT, 'receipt-schema.json')) + +const CONTRACT_VERSION = corpus.contract_version + +// --------------------------------------------------------------------------- +// 1. Contract version agreement +// --------------------------------------------------------------------------- + +for (const [name, doc] of [ + ['tasks.json', tasks], + ['rubrics.json', rubrics], + ['tier1.json', tier1], + ['tier2-matrix.json', tier2], +]) { + if (doc.contract_version !== CONTRACT_VERSION) { + fail(`${name} declares contract_version ${doc.contract_version}, expected ${CONTRACT_VERSION}`) + } +} + +// --------------------------------------------------------------------------- +// 2. Targets must be natural and pinned +// --------------------------------------------------------------------------- + +const targetsById = new Map(corpus.targets.map((target) => [target.id, target])) + +if (!Array.isArray(corpus.proxy_targets)) { + fail('corpus.json must declare a proxy_targets list, even when empty') +} + +// `kind` and `holdout_class` describe the same distinction and are read by different +// consumers — this validator used `kind`, the contract test used `holdout_class`. A target +// that sets only one of them would be classified differently by each, so require both and +// require them to agree, and derive the single predicate from that pair. +const isSealedTarget = (target) => target.kind === 'sealed' || target.holdout_class === 'sealed' + +for (const target of corpus.targets) { + if ((target.kind === 'sealed') !== (target.holdout_class === 'sealed')) { + fail( + `target ${target.id} disagrees with itself: kind=${JSON.stringify(target.kind)} but ` + + `holdout_class=${JSON.stringify(target.holdout_class)}; a sealed target must declare both`, + ) + } + + if (isSealedTarget(target)) { + if (target.status !== 'unsatisfied') { + fail(`sealed target ${target.id} must stay unsatisfied until a second person fills it`) + } + continue + } + + if (target.natural !== true) { + fail(`target ${target.id} is not marked natural; a fixture proxy must be declared in proxy_targets, not in targets`) + } + if (!/^[0-9a-f]{40}$/.test(target.source?.ref ?? '')) { + fail(`target ${target.id} must pin an immutable 40-character commit SHA`) + } + if (!target.source?.url?.startsWith('https://')) { + fail(`target ${target.id} must record an https repository URL`) + } + if (!target.license) { + fail(`target ${target.id} must record a license`) + } + if (!Array.isArray(target.prepare) || target.prepare.length === 0) { + fail(`target ${target.id} must record reproducible prepare steps`) + } + if (!target.dependency_lock) { + fail(`target ${target.id} must record a dependency lock policy`) + } + if (!target.cited_blobs || Object.keys(target.cited_blobs).length === 0) { + fail(`target ${target.id} must record cited_blobs so truth citations can be checked offline`) + } + for (const [path, blob] of Object.entries(target.cited_blobs ?? {})) { + if (!/^[0-9a-f]{40}$/.test(blob)) { + fail(`target ${target.id} cited_blobs["${path}"] is not a git blob SHA`) + } + } + + if (target.kind === 'git_patched') { + const base = targetsById.get(target.base_target) + if (!base) { + fail(`patched target ${target.id} references unknown base_target ${target.base_target}`) + } else if (base.source.ref !== target.source.ref) { + fail(`patched target ${target.id} must pin the same commit as its base target`) + } + + const patchPath = join(ROOT, target.patch ?? '') + let patch + try { + patch = readFileSync(patchPath, 'utf8') + } catch { + fail(`patched target ${target.id} references missing patch ${target.patch}`) + } + + if (patch !== undefined) { + // Parse against normalized text so a CRLF checkout cannot capture a stray + // carriage return into a path. The digest check further down still reads raw + // bytes; only this structural parse is representation-independent. + const patchText = patch.replace(/\r\n/g, '\n') + + if (!patchText.startsWith('diff --git ')) { + fail(`patch ${target.patch} is not a unified git diff`) + } + if (patch.includes('\r\n')) { + fail(`patch ${target.patch} contains CRLF line endings; it must stay LF so \`git apply\` accepts it`) + } + const touched = [...patchText.matchAll(/^\+\+\+ b\/(.+)$/gm)].map((match) => match[1]) + if (touched.length === 0) { + fail(`patch ${target.patch} does not modify any file`) + } + for (const path of touched) { + if (!(path in (target.cited_blobs ?? {}))) { + fail(`patch ${target.patch} touches ${path}, which is not recorded in ${target.id} cited_blobs`) + } + } + } + } +} + +// --------------------------------------------------------------------------- +// 3. Tasks, prompts, truth files +// --------------------------------------------------------------------------- + +const REQUIRED_CATEGORIES = [ + 'architecture-understanding', + 'execution-flow-explanation', + 'impact-analysis', + 'bug-root-cause-investigation', + 'implementation-planning', + 'review-security', +] + +const seenCategories = new Set() +const tasksById = new Map() + +for (const task of tasks.tasks) { + tasksById.set(task.id, task) + seenCategories.add(task.category) + + const target = targetsById.get(task.target) + if (!target) { + fail(`task ${task.id} references unknown target ${task.target}`) + continue + } + + const prompt = task.prompt + if ( + prompt === null + || typeof prompt !== 'object' + || Array.isArray(prompt) + || typeof prompt.text !== 'string' + || typeof prompt.sha256 !== 'string' + ) { + fail(`task ${task.id} must declare prompt text and sha256`) + continue + } + + const actualHash = sha256(prompt.text) + if (actualHash !== prompt.sha256) { + fail(`task ${task.id} prompt hash mismatch: recorded ${prompt.sha256}, actual ${actualHash}`) + } + + const scoring = task.scoring + if ( + scoring === null + || typeof scoring !== 'object' + || Array.isArray(scoring) + || typeof scoring.tier1_method !== 'string' + || typeof scoring.tier2_method !== 'string' + ) { + fail(`task ${task.id} must declare tier1 and tier2 scoring methods`) + continue + } + + const truthPath = join(ROOT, task.truth_ref) + let truth + try { + truth = readJson(truthPath) + } catch { + fail(`task ${task.id} truth file ${task.truth_ref} is missing or unreadable`) + continue + } + + if (truth.task_id !== task.id) fail(`${task.truth_ref} declares task_id ${truth.task_id}, expected ${task.id}`) + if (truth.target !== task.target) fail(`${task.truth_ref} declares target ${truth.target}, expected ${task.target}`) + if (truth.category !== task.category) fail(`${task.truth_ref} declares category ${truth.category}, expected ${task.category}`) + if (truth.contract_version !== CONTRACT_VERSION) fail(`${task.truth_ref} declares contract_version ${truth.contract_version}`) + + // Independence: truth must not be derived from Madar output. + // Record a failure rather than dereferencing an absent block: this validator exists to + // print the complete list of contract problems, and a TypeError here would replace that + // list with a stack trace on exactly the malformed documents it is meant to describe. + for (const [source, provenance] of [ + [`task ${task.id} truth_provenance`, task.truth_provenance], + [`${task.truth_ref} provenance`, truth.provenance], + ]) { + if (provenance === null || typeof provenance !== 'object') { + fail(`${source} is missing; truth independence cannot be established`) + continue + } + if (provenance.inspected_madar_output_before_freeze !== false) { + fail(`task ${task.id} truth provenance claims Madar output was inspected before freezing`) + } + if (!Array.isArray(provenance.madar_derived_sources_used) || provenance.madar_derived_sources_used.length > 0) { + fail(`task ${task.id} truth provenance lists Madar-derived sources: ${JSON.stringify(provenance.madar_derived_sources_used)}`) + } + if (!provenance.authored_by || !provenance.authored_at) { + fail(`task ${task.id} truth provenance must record who authored the truth and when`) + } + if (!Array.isArray(provenance.derived_from) || provenance.derived_from.length === 0) { + fail(`task ${task.id} truth provenance must record what the truth was derived from`) + } + if (!('independent_of_production_rule_author' in provenance)) { + fail(`task ${task.id} truth provenance must state whether the author is independent of the production-rule author`) + } + } + + // Every cited evidence path must be recorded in the target's frozen blob map. + // The traversal, including the `new_path` exemption, is shared with + // tests/unit/qualification-contract.test.ts so the two cannot drift apart. + for (const cited of collectCitedPaths(truth)) { + if (!(cited in (target.cited_blobs ?? {}))) { + fail(`${task.truth_ref} cites ${cited}, which is not recorded in target ${target.id} cited_blobs`) + } + } + + const obligations = truth.tier1_obligations + if (!obligations) { + fail(`${task.truth_ref} has no tier1_obligations block`) + } else if (!Array.isArray(obligations.must_not_report_ready_when) || obligations.must_not_report_ready_when.length === 0) { + fail(`${task.truth_ref} must declare at least one must_not_report_ready_when condition`) + } + + if (!rubrics.methods[scoring.tier2_method]) { + fail(`task ${task.id} references unknown rubric method ${scoring.tier2_method}`) + } + if (!rubrics.methods[scoring.tier1_method]) { + fail(`task ${task.id} references unknown tier1 method ${scoring.tier1_method}`) + } +} + +for (const category of REQUIRED_CATEGORIES) { + if (!seenCategories.has(category)) { + fail(`no frozen task covers required category ${category}`) + } +} + +// --------------------------------------------------------------------------- +// 4. Tier 1 subset and negative-trust probes +// --------------------------------------------------------------------------- + +const gateActivation = tier1.gate?.activation +const hasGateActivation = gateActivation !== null + && typeof gateActivation === 'object' + && !Array.isArray(gateActivation) +if (!hasGateActivation) { + fail('tier1 gate.activation block must exist') +} +if (hasGateActivation && typeof gateActivation.active !== 'boolean') { + fail('tier1 gate.activation.active must be a boolean') +} +if ( + hasGateActivation + && gateActivation.active === true + && (typeof gateActivation.state !== 'string' || gateActivation.state.length === 0 || gateActivation.state === 'pre_baseline') +) { + fail('tier1 active gate activation must declare a non-pre_baseline state') +} +if (hasGateActivation && gateActivation.active === true) { + const activationEvent = gateActivation.activation_event + if ( + activationEvent === null + || typeof activationEvent !== 'object' + || Array.isArray(activationEvent) + || activationEvent.run_id == null + || activationEvent.run_url == null + || activationEvent.date == null + ) { + fail( + 'tier1 gate activation is active but activation_event must name the baseline with non-null ' + + 'run_id, run_url, and date', + ) + } +} +if (hasGateActivation && gateActivation.active === false && gateActivation.state !== 'pre_baseline') { + fail('tier1 inactive gate activation must have state "pre_baseline"') +} + +const tier1Cells = Array.isArray(tier1.cells) ? tier1.cells : [] +for (const cell of tier1Cells) { + const task = tasksById.get(cell.task_id) + if (!task) { + fail(`tier1 cell references unknown task ${cell.task_id}`) + continue + } + if (task.target !== cell.target_id) { + fail(`tier1 cell ${cell.task_id} targets ${cell.target_id} but the task targets ${task.target}`) + } + if (!task.tiers.includes(1)) { + fail(`tier1 cell ${cell.task_id} refers to a task that does not declare tier 1`) + } +} + +for (const probe of tier1.negative_trust_probes) { + const actual = sha256(probe.prompt.text) + if (actual !== probe.prompt.sha256) { + fail(`negative-trust probe ${probe.id} prompt hash mismatch: recorded ${probe.prompt.sha256}, actual ${actual}`) + } + if (!targetsById.has(probe.target_id)) { + fail(`negative-trust probe ${probe.id} references unknown target ${probe.target_id}`) + } +} + +// --------------------------------------------------------------------------- +// 5. Tier 2 matrix references +// --------------------------------------------------------------------------- + +for (const id of tier2.dimensions.targets) { + if (!targetsById.has(id)) fail(`tier2 matrix references unknown target ${id}`) +} +for (const id of tier2.dimensions.tasks) { + if (!tasksById.has(id)) fail(`tier2 matrix references unknown task ${id}`) +} + +const cellPair = (cell) => JSON.stringify({ + task_id: cell?.task_id ?? null, + target_id: cell?.target_id ?? null, +}) +const tier1CellPairs = new Set(tier1Cells.map(cellPair)) +const tier2CellPairs = new Set((Array.isArray(tier2.cells) ? tier2.cells : []).map(cellPair)) + +for (const pair of tier1CellPairs) { + if (!tier2CellPairs.has(pair)) { + fail(`tier2-matrix.json#/cells is missing pair ${pair} present in tier1.json#/cells`) + } +} +for (const pair of tier2CellPairs) { + if (!tier1CellPairs.has(pair)) { + fail(`tier1.json#/cells is missing pair ${pair} present in tier2-matrix.json#/cells`) + } +} + +if (tier2.status !== 'planned') { + fail('tier2-matrix.json must stay planned until its execution prerequisites are met') +} + +// --------------------------------------------------------------------------- +// 6. Receipt schema and examples +// --------------------------------------------------------------------------- + +const ajv = new Ajv({ allErrors: true, strict: false }) +addFormats(ajv) +const validateReceipt = ajv.compile(receiptSchema) + +for (const path of walk(join(ROOT, 'examples'))) { + const receipt = readJson(path) + const label = relPath(path) + + // A receipt that failed schema validation has no guaranteed shape, so reading + // `receipt.validity` or `receipt.scores` below would throw before the collected failures + // are printed. Report the schema failure and move on to the next receipt. + if (!validateReceipt(receipt)) { + fail(`${label} does not satisfy receipt-schema.json: ${ajv.errorsText(validateReceipt.errors)}`) + continue + } + if (receipt.validity.status !== 'valid' && receipt.validity.aggregatable !== false) { + fail(`${label} is not valid but is marked aggregatable`) + } + for (const [name, score] of Object.entries(receipt.scores)) { + if (score.measured === false && score.value !== null) { + fail(`${label} score ${name} is not measured but carries a value`) + } + } + + const task = tasksById.get(receipt.task_id) + if (!task) { + fail(`${label} references unknown task ${receipt.task_id}`) + } else if (receipt.identity.prompts.user_prompt_sha256 !== task.prompt?.sha256) { + fail(`${label} records a prompt hash that does not match the frozen prompt for ${receipt.task_id}`) + } + + if (receipt.tier === 2 && task?.scoring?.hidden_acceptance_test?.required === true) { + const implementation = receipt.scores?.implementation + if ( + implementation === null + || typeof implementation !== 'object' + || Array.isArray(implementation) + || implementation.measured !== false + || implementation.value !== null + || typeof implementation.not_measured_reason !== 'string' + || implementation.not_measured_reason.length === 0 + ) { + fail( + `${label} Tier 2 task ${receipt.task_id} requires scores.implementation to be not_measured ` + + '(measured false, value null, with a not_measured_reason)', + ) + } + } +} + +// --------------------------------------------------------------------------- +// 7. Benchmark independence: no qualification literal may reach production code +// --------------------------------------------------------------------------- + +// Bare target ids are deliberately NOT forbidden. A target id may legitimately equal the +// name of a framework Madar declares generic support for — `hono` is one — and banning the +// word would confuse a declared adapter with a benchmark-specific special case. Those +// couplings are disclosed per target in corpus.json#/targets/*/production_coupling instead. +// What is forbidden here is every literal that could only have come from this contract. +// +// `_`-prefixed keys in forbidden_target_symbols are documentation, not symbols. Folding a +// prose note into this list would make the guard fail production files the moment that note +// were shortened to something short or common, so the keys are skipped and every remaining +// entry is required to be an array of plausible identifiers. +const targetSymbols = [] +for (const [key, value] of Object.entries(corpus.forbidden_target_symbols ?? {})) { + if (key.startsWith('_')) { + continue + } + if (!Array.isArray(value)) { + fail(`corpus.json forbidden_target_symbols["${key}"] must be an array of symbols`) + continue + } + for (const symbol of value) { + if (typeof symbol !== 'string' || !/^[A-Za-z_$][A-Za-z0-9_$]*$/.test(symbol)) { + fail(`corpus.json forbidden_target_symbols["${key}"] contains ${JSON.stringify(symbol)}, which is not an identifier`) + continue + } + targetSymbols.push(symbol) + } + if (!targetsById.has(key)) { + fail(`corpus.json forbidden_target_symbols["${key}"] does not name a corpus target`) + } +} + +const FORBIDDEN_LITERALS = [ + ...corpus.targets.flatMap((target) => (target.source?.url ? [target.source.url, target.source.ref] : [])), + ...tasks.tasks.map((task) => task.id), + ...tasks.tasks.flatMap((task) => (typeof task.prompt?.text === 'string' ? [task.prompt.text] : [])), + ...tier1.negative_trust_probes.map((probe) => probe.prompt.text), + ...targetSymbols, +] + +for (const path of walk(PRODUCTION_ROOT)) { + const content = readFileSync(path, 'utf8') + for (const literal of FORBIDDEN_LITERALS) { + if (content.includes(literal)) { + fail(`production file ${relPath(path)} contains qualification literal "${literal}"`) + } + } +} + +// --------------------------------------------------------------------------- +// 8. Optional network verification of the pinned corpus +// --------------------------------------------------------------------------- + +if (VERIFY_CORPUS) { + for (const target of corpus.targets) { + if (isSealedTarget(target)) { + continue + } + + const dir = mkdtempSync(join(tmpdir(), `qualify-${target.id}-`)) + try { + const git = (...args) => execFileSync('git', ['-C', dir, ...args], { encoding: 'utf8' }).trim() + + execFileSync('git', ['init', '--quiet', dir], { stdio: 'ignore' }) + git('remote', 'add', 'origin', target.source.url) + git('fetch', '--quiet', '--depth', '1', 'origin', target.source.ref) + git('checkout', '--quiet', 'FETCH_HEAD') + + const head = git('rev-parse', 'HEAD') + if (head !== target.source.ref) { + fail(`corpus verification: ${target.id} resolved to ${head}, expected ${target.source.ref}`) + } + + for (const [path, blob] of Object.entries(target.cited_blobs)) { + const actual = git('rev-parse', `HEAD:${path}`) + if (actual !== blob) { + fail(`corpus verification: ${target.id} ${path} blob is ${actual}, expected ${blob}`) + } + } + + if (target.kind === 'git_patched') { + execFileSync('git', ['-C', dir, 'apply', '--check', join(ROOT, target.patch)], { stdio: 'pipe' }) + } + + console.log(`corpus verification: ${target.id} ok`) + } catch (error) { + fail(`corpus verification failed for ${target.id}: ${error instanceof Error ? error.message : String(error)}`) + } finally { + rmSync(dir, { recursive: true, force: true }) + } + } +} + +// --------------------------------------------------------------------------- +// 9. Freeze digests +// --------------------------------------------------------------------------- + +const frozenFiles = walk(ROOT) + .filter((path) => path !== FREEZE_PATH) + .map((path) => relPath(path)) + .sort() + +const digests = Object.fromEntries( + frozenFiles.map((path) => [path, sha256(readFileSync(resolve(path)))]), +) + +// The freeze map is only computed here. Writing it is deferred until after the failure +// gate below, so that `--write` on a contract with problems cannot leave a freeze.json on +// disk that blesses the inconsistent state and then reads back clean on the next plain run. +if (!WRITE) { + let freeze + try { + freeze = readJson(FREEZE_PATH) + } catch { + fail('freeze.json is missing; run `npm run qualify:validate -- --write`') + } + + if (freeze) { + if (freeze.contract_version !== CONTRACT_VERSION) { + fail(`freeze.json declares contract_version ${freeze.contract_version}, expected ${CONTRACT_VERSION}`) + } + for (const [path, digest] of Object.entries(digests)) { + if (!(path in freeze.files)) { + fail(`${path} is not covered by freeze.json`) + } else if (freeze.files[path] !== digest) { + const crlf = readFileSync(resolve(path)).includes('\r\n') + const hint = crlf + ? ' — the file contains CRLF, so this is a checkout line-ending problem, not a content change.' + + ' Fix the checkout (see `docs/qualification/** text eol=lf` in .gitattributes); do NOT regenerate freeze.json.' + : '' + fail(`${path} content changed since it was frozen (expected ${freeze.files[path]}, actual ${digest})${hint}`) + } + } + for (const path of Object.keys(freeze.files)) { + if (!(path in digests)) { + fail(`freeze.json references ${path}, which no longer exists`) + } + } + } +} + +// --------------------------------------------------------------------------- + +if (failures.length > 0) { + console.error(`qualification contract validation failed with ${failures.length} problem(s):`) + for (const failure of failures) { + console.error(` - ${failure}`) + } + if (WRITE) { + console.error(`freeze.json was NOT written: a freeze must only ever record a consistent contract.`) + } + process.exit(1) +} + +if (WRITE) { + const freeze = { + contract_version: CONTRACT_VERSION, + frozen_at: corpus.frozen_at, + algorithm: 'sha256 over raw file bytes', + note: 'Regenerate deliberately with `npm run qualify:validate -- --write` and say why in the pull request. A silent digest change is a contract change.', + files: digests, + } + writeFileSync(FREEZE_PATH, `${JSON.stringify(freeze, null, 2)}\n`) + console.log(`wrote ${relPath(FREEZE_PATH)} with ${frozenFiles.length} entries`) +} + +const naturalTargets = corpus.targets.filter((target) => !isSealedTarget(target)) + +console.log( + `qualification contract v${CONTRACT_VERSION} is consistent: ` + + `${naturalTargets.length} pinned natural targets, ${corpus.proxy_targets.length} proxy targets, ` + + `${tasks.tasks.length} tasks, ${tier1Cells.length} Tier 1 cells, ` + + `${tier1.negative_trust_probes.length} negative-trust probes, ${frozenFiles.length} frozen files.`, +) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ca8ef41e..7f1490c3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,6 +52,13 @@ jobs: if: matrix.os == 'ubuntu-latest' && matrix.node-version == '22' run: npm run release:verify + # Runs on every lane on purpose. The qualification freeze is a byte-exact + # guarantee, so it has to be verified on every platform the repository + # supports — a reproducible gate that only reproduces on Linux is not one. + # The default mode is pure local file I/O with no network and no spend. + - name: Validate qualification contract + run: npm run qualify:validate + - name: Typecheck run: npm run typecheck diff --git a/docs/qualification/README.md b/docs/qualification/README.md new file mode 100644 index 00000000..84aac84e --- /dev/null +++ b/docs/qualification/README.md @@ -0,0 +1,166 @@ +# Qualification contract + +Contract version `1.0.0`, frozen 2026-08-12 for [#655](https://github.com/mohanagy/madar/issues/655) +against Madar commit `06b373a447acfce895412ac10eb4e5228c5df0b7` (`v0.32.1`). + +This directory is the independent evaluation contract used to decide whether a roadmap +change is safe to ship. It is deliberately separate from +[`docs/benchmarks/suite/`](../benchmarks/suite/), which is the product benchmark suite. + +## Read this first — what this contract does and does not give you today + +> **This contract has never been executed. It currently produces no measured evidence of +> any kind.** Every target is a real external repository pinned at an immutable commit and +> every task has independent truth, so the corpus *can* produce evidence about natural +> code — but nothing has been run against Madar yet. Executing the Tier 1 subset is +> [#661](https://github.com/mohanagy/madar/issues/661). + +Three further limits apply the moment it *is* executed, and none of them is fixed by +running it: + +1. **Regression only, never generalization.** The sealed holdout slot is `unsatisfied` + because Madar has one author. Every result from this corpus must carry the line + `sealed holdout unsatisfied; results measure regression only`. See + [`holdout-policy.md`](./holdout-policy.md). +2. **Thresholds are pre-registered, not calibrated.** Nobody knows how many Tier 1 cells + currently pass. That is the correct order — a threshold fitted to observed output would + describe current behaviour instead of testing it — but it means the first run is a + measurement, not a pass/fail gate. +3. **Tier 1 needs network access** to clone the pinned targets. It is still deterministic: + the commit SHA and the patch fix the content exactly, and a warm clone cache or local + mirror satisfies it without changing any result. + +## Why a separate corpus exists + +Two properties are required, and the existing benchmark suite has neither. + +**Independence from Madar output.** Grading Madar with expectations Madar produced tells +you nothing. Today's per-task expectations live in +[`docs/benchmarks/suite/runtime-proof.json`](../benchmarks/suite/runtime-proof.json) as +exact expected symbols and paths, authored alongside the product. + +**Naturalness.** Every row in +[`docs/benchmarks/suite/repos.json`](../benchmarks/suite/repos.json) that is keyed by +`path` is an in-repo proxy — `examples/sample-workspace`, two `tests/fixtures/pack-quality` +workspaces, and two fixture directories under the suite itself. A corpus of self-authored +proxies cannot detect production behaviour drifting toward benchmark-shaped repositories, +because the proxies were shaped by the same hands as the production rules. + +Every target in this corpus is therefore a real, externally authored, permissively licensed +project pinned at an immutable commit. There are no fixture proxies; +`corpus.json#/proxy_targets` is empty and documents the conditions under which an entry +would be permitted. + +## Targets + +| Target | Repository | Commit | License | +| --- | --- | --- | --- | +| `hono` | [honojs/hono](https://github.com/honojs/hono) | `26de73133b8552f56ba72e025ecd82b08900d796` | MIT | +| `unstorage` | [unjs/unstorage](https://github.com/unjs/unstorage) | `e6be6135832f350ca16f9a77432e1d4f0aa85ed7` | MIT | +| `hono-seeded-compose` | same as `hono`, plus [`patches/hono-compose-reentrancy-guard.patch`](./patches/hono-compose-reentrancy-guard.patch) | `26de7313…` | MIT | +| `hono-seeded-error-disclosure` | same as `hono`, plus [`patches/hono-error-message-disclosure.patch`](./patches/hono-error-message-disclosure.patch) | `26de7313…` | MIT | +| `sealed-holdout-a` | undisclosed | — | **unsatisfied**, see [`holdout-policy.md`](./holdout-policy.md) | + +Seeded defects are injected into the real code as patches against the pinned commit, not +recreated inside a synthetic workspace built around the answer. + +## Files + +| File | Deliverable | +| --- | --- | +| [`corpus.json`](./corpus.json) | Versioned corpus manifest: repositories, commits, licenses, prepare commands, patches, cited blob digests, holdout class. | +| [`tasks.json`](./tasks.json) | Versioned task definitions: frozen prompts with hashes, categories, scoring method, truth provenance. | +| [`truth/`](./truth/) | Independent truth and rubric input, one file per task. | +| [`patches/`](./patches/) | Seeded-defect patches applied to the pinned commit. Every `patches/...` reference in `corpus.json`, `tasks.json` and the truth files resolves against this directory — `docs/qualification/` — as recorded by `patch_path_base` in `corpus.json`. | +| [`rubrics.json`](./rubrics.json) | Scoring dimensions, per-category scoring methods, blinding rules, aggregation rules. | +| [`receipt-schema.json`](./receipt-schema.json) | Environment and run receipt schema. | +| [`examples/`](./examples/) | Two illustrative receipts: a valid Tier 1 run and an invalid Tier 2 run that stays `not_measured`. | +| [`validity-rules.md`](./validity-rules.md) | Valid/invalid criteria, the `not_measured` rule, retention, and what today's emitter actually produces. | +| [`holdout-policy.md`](./holdout-policy.md) | Hidden holdout handling — and why the sealed slot is currently unsatisfied. | +| [`stop-rule.md`](./stop-rule.md) | Objective stop, rollback, and publication rule. | +| [`evidence-categories.md`](./evidence-categories.md) | Evidence classes E1–E6 and required labelling. | +| [`tier1.json`](./tier1.json) | The small deterministic subset a pull request can run. | +| [`tier2-matrix.json`](./tier2-matrix.json) | The planned repeated-run matrix. | +| [`freeze.json`](./freeze.json) | SHA-256 of every file above. A silent change to any of them fails validation. | + +## Tiers + +**Tier 1** is deterministic, needs no model provider and no spend. It measures whether the +evidence required to answer each frozen task was present in the context artifact, and +whether readiness was correctly refused on the negative-trust probes. It never scores +answer quality and never runs an agent. It does need network access to clone the pinned +targets; a warm clone cache or a local mirror satisfies that without changing any result, +because the commit and the patch fix the content exactly. + +**Tier 2** runs a real agent on both arms with repeated trials and blinded rubric scoring. +It is frozen but not executed; its prerequisites are in `tier2-matrix.json`. + +## Running it + +```bash +npm ci +npm run qualify:validate +``` + +`qualify:validate` checks, offline and without running Madar: + +- every declared contract version agrees; +- every target pins a 40-character commit SHA and records a license and prepare steps; +- every task references a real target, and every frozen prompt matches its recorded hash; +- every truth file exists, matches its task, and cites only paths recorded in that target's + `cited_blobs` map; +- every truth file records who authored it and asserts no Madar-derived source was used; +- all six required task categories are covered; +- every seeded target names a patch file that exists and is a well-formed unified diff + touching only paths recorded for that target; +- every Tier 1 cell and negative-trust probe resolves, and every probe prompt hash matches; +- both example receipts validate against `receipt-schema.json`, and no unmeasured score + carries a value; +- **no qualification task id, prompt string, or pinned-repository symbol appears anywhere in + `src/`**. Target ids are deliberately outside this scan — see `forbidden_target_symbols._note` + and `production_coupling` in `corpus.json` — because a target id is the name of a real public + project and its appearance in production code is not by itself evidence of coupling; +- every file in this directory matches its frozen digest. + +To additionally confirm the pinned commits and blob digests against the real repositories +— this one needs network access: + +```bash +npm run qualify:validate -- --verify-corpus +``` + +Regenerating the freeze file is deliberate and must be explained in the pull request: + +```bash +npm run qualify:validate -- --write +``` + +Executing the Tier 1 subset against Madar is [#661](https://github.com/mohanagy/madar/issues/661), +not this contract. + +## Independence + +All six truth files were authored on 2026-08-12 by reading the pinned repository sources +directly. Madar was never run against any target, and no Madar retrieval output, context +pack, `implementationGuidance`, Madar-selected file list, or Madar-generated validation +command was consulted before freezing. Each task records this in `truth_provenance`. + +Two consequences follow, and both are stated rather than papered over: + +- **Thresholds are pre-registered, not calibrated.** Nobody knows how many Tier 1 cells + currently pass. Pre-registering before calibrating is the correct order: a threshold + fitted to observed output would describe current behaviour instead of testing it. The + first execution is a measurement, and a failure there is a product finding, not a reason + to edit this contract. +- **The author is not independent of the production-rule author.** Madar has one author, so + `independent_of_production_rule_author` is `false` on every task, blinded review is + unavailable, and the sealed holdout slot is unsatisfied. See + [`holdout-policy.md`](./holdout-policy.md) for the human action that would fix this. Until + then this corpus measures regression, not generalization, and no superiority or + generalization claim may rest on it. + +## Non-goals + +This contract does not run the 480+ public superiority experiment, does not change +production retrieval or context logic, does not tune ranking against any target, and does +not publish any claim. diff --git a/docs/qualification/corpus.json b/docs/qualification/corpus.json new file mode 100644 index 00000000..91ca0bb2 --- /dev/null +++ b/docs/qualification/corpus.json @@ -0,0 +1,195 @@ +{ + "contract_version": "1.0.0", + "frozen_at": "2026-08-12", + "frozen_for_issue": 655, + "frozen_against": { + "madar_commit": "06b373a447acfce895412ac10eb4e5228c5df0b7", + "madar_package_version": "0.32.1", + "dependency_lock": "package-lock.json at the pinned commit; runs must use `npm ci`, never `npm install`" + }, + "support_corridor": "typescript-node", + "naturalness_rule": "Every qualification target is a real, externally authored software project pinned at an immutable commit. Self-authored fixture workspaces are not part of this corpus. A corpus of self-made proxies cannot detect production behaviour drifting toward benchmark-shaped repositories, which is the failure mode this contract exists to catch.", + "forbidden_target_symbols": { + "_note": "Distinctive symbols from the pinned targets. Their appearance in src/ would mean production behaviour had been shaped around a qualification repository. Checked by `npm run qualify:validate`. Target ids are deliberately excluded from this check — see production_coupling below.", + "hono": ["SmartRouter", "UnsupportedPathError", "RegExpRouter", "TrieRouter"], + "unstorage": ["createStorage", "DriverFactory", "createRequiredError"] + }, + "patch_path_base": "docs/qualification/", + "patch_path_base_note": "Every `patch` value in this file, and every `patches/...` reference in the truth files and in tasks.json, is resolved against this directory and nowhere else — not against the repository root and not against the file that mentions it. `npm run qualify:validate` resolves them this way. The base is stated because validity-rules.md invalidates a run whose patch fails to apply, so an ambiguous base would turn a path convention into an invalidation.", + "proxy_targets": [], + "proxy_targets_note": "This list is intentionally empty. A fixture proxy is permitted only where a natural repository genuinely cannot supply a task category; all six required categories are supplied by the natural targets below. Any future entry here must be labelled a proxy and must carry the statement that proxies cannot satisfy the naturalness property.", + "targets": [ + { + "id": "hono", + "name": "Hono web framework", + "tier": 1, + "kind": "git", + "natural": true, + "source": { + "url": "https://github.com/honojs/hono", + "ref": "26de73133b8552f56ba72e025ecd82b08900d796", + "committed_at": "2026-08-10T01:20:29Z" + }, + "license": "MIT", + "language": "typescript", + "shape": "http-framework", + "dependency_lock": "the repository's own lockfile at the pinned ref", + "prepare": [ + "git clone --filter=blob:none --no-checkout https://github.com/honojs/hono.git ", + "git -C checkout 26de73133b8552f56ba72e025ecd82b08900d796" + ], + "install": "not required — every frozen task is answered from source; no build or dependency install is needed to read the code", + "holdout_class": "open", + "status": "frozen", + "selection_rationale": "Real, externally authored, permissively licensed TypeScript with a genuine layered request lifecycle (entry point, router abstraction with multiple implementations, middleware composition, context, error handling). Deliberately absent from docs/benchmarks/suite/repos.json so qualification does not inherit repositories that existing production heuristics or published receipts were shaped around.", + "production_coupling": { + "level": "declared_framework_adapter", + "detail": "Madar ships a generic Hono adapter at src/pipeline/spi/framework-hono.ts and Hono-aware query classification in src/runtime/retrieve.ts (an explicit-Hono token check and hono_route / hono_middleware roles). This is declared support for a framework in the TypeScript/Node corridor, not a repository-specific special case, so it is not a #660 contamination finding.", + "consequence": "A result on this target partly measures the shipped Hono adapter. It is not evidence about frameworks that have no adapter, and it must never be generalized to them.", + "mitigation": "The frozen prompts for this target deliberately never name the framework — they say 'this framework' and 'the application's entry point' — so the query classifier is not handed the framework identity in the prompt text. The unstorage target is a plain library with no corresponding adapter and acts as the uncoupled contrast.", + "verified_at": "2026-08-12 against 06b373a447acfce895412ac10eb4e5228c5df0b7" + }, + "cited_blobs": { + "src/hono.ts": "c9472202a710231b8151e06cc812290bf53bd3ab", + "src/hono-base.ts": "e6a7278dd293aa03a1021d3c639b3bda5bd4c23d", + "src/compose.ts": "b1d4508ffe490fe4eaf1fa25a182c95f486685e5", + "src/context.ts": "3553dd181b4a718177a09874ba584c76f4cd54c5", + "src/router.ts": "ec12588ab651a7ffe0d0178b37099edf21833ee0", + "src/router/smart-router/router.ts": "9ec464da8c25dfc2dacaee9ea13174a7d0020ec6", + "src/router/reg-exp-router/router.ts": "6020851a696ff51b3b65adfe80d3daccf5156f3d", + "src/router/trie-router/router.ts": "65b9978861688858f640f7618161b3c0bb8d009c", + "src/utils/url.ts": "ea92ff9355c64340e4d342831aadfc9bb91d7f64", + "src/http-exception.ts": "8fe9c2bb3d078787cfd233634ef8905b5b5d2ccf", + "src/request.ts": "ae5c04076a178319c8af1304bba6b603d235a8f2" + } + }, + { + "id": "unstorage", + "name": "unstorage key-value abstraction", + "tier": 1, + "kind": "git", + "natural": true, + "source": { + "url": "https://github.com/unjs/unstorage", + "ref": "e6be6135832f350ca16f9a77432e1d4f0aa85ed7", + "committed_at": "2026-06-29T17:46:43Z" + }, + "license": "MIT", + "language": "typescript", + "shape": "driver-based-storage-abstraction", + "dependency_lock": "the repository's own lockfile at the pinned ref", + "prepare": [ + "git clone --filter=blob:none --no-checkout https://github.com/unjs/unstorage.git ", + "git -C checkout e6be6135832f350ca16f9a77432e1d4f0aa85ed7" + ], + "install": "not required — every frozen task is answered from source; no build or dependency install is needed to read the code", + "holdout_class": "open", + "status": "frozen", + "selection_rationale": "Real, externally authored, permissively licensed TypeScript with a genuine extension seam (a Driver interface, 34 shipped drivers, mount-prefix resolution, a code-generated driver index). A different architectural shape from the framework target, and deliberately absent from docs/benchmarks/suite/repos.json.", + "production_coupling": { + "level": "none_found", + "detail": "No framework adapter in src/pipeline/spi corresponds to this library, and no symbol from it appears in src/. It is a plain TypeScript library outside every declared adapter.", + "consequence": "This target is the uncoupled contrast to the framework target: a result here exercises generic retrieval and context building rather than a shipped adapter.", + "verified_at": "2026-08-12 against 06b373a447acfce895412ac10eb4e5228c5df0b7" + }, + "cited_blobs": { + "src/index.ts": "748a6b72cb67bffdcc43a50ed5e9cb73eb42a0d3", + "src/types.ts": "1f11e02f4036312f7e56abcda04ef7fd79e14ddf", + "src/storage.ts": "0c25ad8d0afe9f1ad8e2f44af8ba6b8d5f1f75cd", + "src/utils.ts": "2477c70c5a518de8c5a65302ee2f1a33f8dc7ea8", + "src/_utils.ts": "b7c765ebacfcfb7823b7bf9517584d0da8695393", + "src/_drivers.ts": "c42493aa41b95156edff2bcb938787570a2f49d3", + "src/drivers/utils/index.ts": "e4ec594a87fd96a80d52c37ed3fd2c2da417e431", + "src/drivers/memory.ts": "8a82465046d8da4565e31ed39a5633cc6a3e58a7", + "scripts/gen-drivers.ts": "0d5c1ec69a45f3d19e747cf7327f7f2234315bc0", + "package.json": "be04ccb74412b208d39ee3646a29c4ab165a036d" + } + }, + { + "id": "hono-seeded-compose", + "name": "Hono with a seeded middleware re-entrancy defect", + "tier": 1, + "kind": "git_patched", + "natural": true, + "base_target": "hono", + "source": { + "url": "https://github.com/honojs/hono", + "ref": "26de73133b8552f56ba72e025ecd82b08900d796", + "committed_at": "2026-08-10T01:20:29Z" + }, + "patch": "patches/hono-compose-reentrancy-guard.patch", + "patch_summary": "One-character change to the re-entrancy guard in the middleware dispatch loop of src/compose.ts.", + "license": "MIT", + "language": "typescript", + "shape": "http-framework", + "dependency_lock": "the repository's own lockfile at the pinned ref", + "prepare": [ + "git clone --filter=blob:none --no-checkout https://github.com/honojs/hono.git ", + "git -C checkout 26de73133b8552f56ba72e025ecd82b08900d796", + "git -C apply /patches/hono-compose-reentrancy-guard.patch" + ], + "install": "not required", + "holdout_class": "open", + "status": "frozen", + "selection_rationale": "The issue lists seeded defects as a preferred independent truth source. The defect is injected into real externally authored code at a pinned commit rather than being surrounded by a synthetic workspace built to make it findable.", + "cited_blobs": { + "src/compose.ts": "b1d4508ffe490fe4eaf1fa25a182c95f486685e5", + "src/hono-base.ts": "e6a7278dd293aa03a1021d3c639b3bda5bd4c23d", + "src/context.ts": "3553dd181b4a718177a09874ba584c76f4cd54c5" + }, + "cited_blobs_note": "Blob SHAs are for the UNPATCHED pinned tree. The patch changes src/compose.ts; every other cited blob is unchanged." + }, + { + "id": "hono-seeded-error-disclosure", + "name": "Hono with a seeded error-message disclosure defect", + "tier": 1, + "kind": "git_patched", + "natural": true, + "base_target": "hono", + "source": { + "url": "https://github.com/honojs/hono", + "ref": "26de73133b8552f56ba72e025ecd82b08900d796", + "committed_at": "2026-08-10T01:20:29Z" + }, + "patch": "patches/hono-error-message-disclosure.patch", + "patch_summary": "Changes the framework default error handler in src/hono-base.ts to return the thrown error's stack or message in the 500 response body.", + "license": "MIT", + "language": "typescript", + "shape": "http-framework", + "dependency_lock": "the repository's own lockfile at the pinned ref", + "prepare": [ + "git clone --filter=blob:none --no-checkout https://github.com/honojs/hono.git ", + "git -C checkout 26de73133b8552f56ba72e025ecd82b08900d796", + "git -C apply /patches/hono-error-message-disclosure.patch" + ], + "install": "not required", + "holdout_class": "open", + "status": "frozen", + "selection_rationale": "A security-shaped defect seeded into real externally authored code, so the review task is a review of a natural codebase rather than of a workspace authored around its own answer.", + "cited_blobs": { + "src/hono-base.ts": "e6a7278dd293aa03a1021d3c639b3bda5bd4c23d", + "src/http-exception.ts": "8fe9c2bb3d078787cfd233634ef8905b5b5d2ccf", + "src/context.ts": "3553dd181b4a718177a09874ba584c76f4cd54c5", + "src/compose.ts": "b1d4508ffe490fe4eaf1fa25a182c95f486685e5" + }, + "cited_blobs_note": "Blob SHAs are for the UNPATCHED pinned tree. The patch changes src/hono-base.ts; every other cited blob is unchanged." + }, + { + "id": "sealed-holdout-a", + "name": "Sealed holdout target A", + "tier": 2, + "kind": "sealed", + "natural": true, + "language": "typescript", + "shape": "undisclosed", + "dependency_lock": "recorded in the sealed manifest, not in this repository", + "holdout_class": "sealed", + "status": "unsatisfied", + "unsatisfied_reason": "Requires a second person to select the repository and author its truth. See holdout-policy.md; the slot stays visible and explicitly unsatisfied rather than being filled with a self-selected target." + } + ], + "status_meaning": { + "frozen": "Repository, revision, and (where applicable) patch are pinned, cited blob SHAs are recorded, and independent truth exists. Usable as measurable evidence.", + "unsatisfied": "The slot is specified but cannot be filled in the current single-author context. It must never be counted as evidence, present or absent." + } +} diff --git a/docs/qualification/evidence-categories.md b/docs/qualification/evidence-categories.md new file mode 100644 index 00000000..f9b6cfe1 --- /dev/null +++ b/docs/qualification/evidence-categories.md @@ -0,0 +1,128 @@ +# Evidence categories + +Contract version `1.0.0`, frozen 2026-08-12 for [#655](https://github.com/mohanagy/madar/issues/655). + +Madar's repository already contains several kinds of artifact that look like measurement. +They are not interchangeable. Every published statement must name the category of evidence +it rests on. + +## Target naturalness qualifies the evidence + +An evidence class says how a measurement was produced. It does not say what the measurement +was produced against, and both matter. + +- **Natural target** — a real, externally authored project pinned at an immutable commit, + optionally with a recorded seeded-defect patch. Every target in + [`corpus.json`](./corpus.json) is natural. +- **Proxy target** — a workspace authored inside this repository to stand in for one. + +A result measured against a proxy can support a regression statement and nothing more. It +can never support a statement about behaviour on real repositories, because a proxy is +shaped by the same hands as the production rules it is meant to test. + +Recorded finding, 2026-08-12, measured against +[`docs/benchmarks/suite/repos.json`](../benchmarks/suite/repos.json) at +`06b373a447acfce895412ac10eb4e5228c5df0b7`: of eleven rows, **five are in-repo proxies** +keyed by `path` — `ts-small` (`examples/sample-workspace`), `nestjs-mid` and +`ts-monorepo-large` (both `tests/fixtures/pack-quality/**/workspace`), `python-service` and +`go-service` (both fixture directories under the suite). The other **six are git-backed and +do pin a URL together with an immutable commit SHA** — `documenso`, `formbricks`, `dub`, +`twenty`, `cal-diy`, `novu`. + +So the existing corpus is mixed, not entirely proxy-based. What matters for evidence +labelling is that the five proxy rows are the ones backing the checked-in deterministic +fixture bundles, and any citation of those receipts must be labelled proxy-target as well +as E4. + +## Categories + +### E1 — Product outcome evidence + +A real agent, on a pinned target, answering a frozen prompt, scored against independent +truth by a blinded reviewer, with a valid receipt. + +Only E1 supports a statement about what Madar does for a user. + +**Currently held: none.** No artifact in this repository meets E1. + +### E2 — Context sufficiency evidence + +Deterministic measurement of whether the evidence needed to answer was present in the +context artifact, and whether readiness was correctly refused. No agent runs. + +This is what [`tier1.json`](./tier1.json) produces. E2 supports statements about retrieval +and context quality. It **does not** support any statement about answer quality, token +cost, or user outcome. + +**Currently held: none executed.** The Tier 1 subset is frozen but has never been run; +see `tier1.json#/calibration_status`. + +### E3 — Controlled profile-assisted measurement + +A real agent run where the prompt, the grader, or the retrieval path was assisted by +task-specific expectations authored alongside the product. + +The June 10 2026 receipts under `docs/benchmarks/suite/results/` are E3: the answering +prompts included proof checklists and the checkout could load expected files and functions +from `docs/benchmarks/suite/runtime-proof.json`. They are genuine measurements of the setup +they describe. They are **not** evidence of untuned behaviour, and they are **not** E1. + +Open enforcement gap in E3, recorded 2026-08-12 and not addressed here: +`docs/benchmarks/suite/runtime-proof.json` carries per-repository expected symbols and +paths — for example `sendDocument()` and `server-only/document/send-document.ts` under +`documenso-explain-runtime`. `docs/benchmarks/suite/methodology.md` asserts that this file +is grader input only, that it "is not passed into retrieval", and that its obligation +checklist "is not written into the answering agent's prompt". That isolation is asserted in +prose. No test, lint rule, or CI check enforces it, and nothing fails if a future change +reads the manifest from retrieval or splices its obligations into a prompt. Until an +enforcement check exists, every E3 citation must state that the retrieval/grader boundary +is documented rather than proven. This is a separate linked issue and bears directly on +[#660](https://github.com/mohanagy/madar/issues/660). + +### E4 — Synthetic or fixture receipts + +Checked-in deterministic bundles with fixture-anchored timings and tool-call counts, such +as `docs/benchmarks/suite/results/2026-05-31T12-00-00/`. + +E4 proves the reporting pipeline works. It is never agent-outcome evidence. Identical +counts across trials in an E4 bundle are a property of the fixture, not a finding. + +### E5 — Package and parity checks + +`npm run verify:pack-parity`, `npm pack --dry-run`, Registry validation, release +verification. + +E5 proves that the packed artifact behaves like the checkout and that the release is +well-formed. It says nothing about retrieval quality or agent outcome. + +### E6 — Adoption and instrumentation observations + +Counts of attributable Madar calls, trace availability, tool permission failures, +environment drift. + +E6 explains why a run is invalid. It is reported in its own column. An adoption failure is +**not** a quality loss, and an adoption success is **not** a quality win. The July 15 2026 +receipts are largely E6: four of six rows recorded no attributable Madar call at all. + +## Required labelling + +Every table, README line, or release note derived from this corpus states its category. +The permitted phrasings are: + +- E1 — "measured agent outcome" +- E2 — "context sufficiency, no agent" +- E3 — "controlled, profile-assisted" +- E4 — "synthetic fixture receipt" +- E5 — "package parity check" +- E6 — "adoption observation" + +## Prohibited combinations + +- E2, E3, E4, E5, or E6 must never be described as a product outcome, a win, a loss, or a + superiority result. +- E3 must never be presented without the word *controlled* and a pointer to what assisted it. +- E4 must never appear in the same table as E1 or E3 without a category column. +- An E6 adoption failure must never be aggregated as a quality loss, and its cost figures + must never be cited. +- No category may be upgraded by repetition. Running an E4 bundle a hundred times produces + E4. diff --git a/docs/qualification/examples/receipt-tier1-valid.json b/docs/qualification/examples/receipt-tier1-valid.json new file mode 100644 index 00000000..c6ae3075 --- /dev/null +++ b/docs/qualification/examples/receipt-tier1-valid.json @@ -0,0 +1,160 @@ +{ + "contract_version": "1.0.0", + "run_id": "example-tier1-valid-0001", + "tier": 1, + "task_id": "rootcause-hono-middleware-rerun", + "target_id": "hono-seeded-compose", + "arm": "madar", + "trial": 1, + "identity": { + "target_revision": "26de73133b8552f56ba72e025ecd82b08900d796", + "dependency_lock_sha256": "6328bf95a901590814ff70ed570e0fc474c05ae162d6ac690cf3c812380828ab", + "madar_commit": "06b373a447acfce895412ac10eb4e5228c5df0b7", + "madar_package_version": "0.32.1", + "madar_package_tarball_sha256": null, + "madar_runtime_source": "checkout", + "madar_config_sha256": "c6dcdf9130e3ef5503caead4fe07f1b465b429adfa6e43309166d8f650a33caf", + "agent": { + "host": "none", + "host_version": null, + "model_id": "not-applicable-tier1-deterministic" + }, + "prompts": { + "system_prompt_sha256": null, + "user_prompt_sha256": "637cd655712c2e7d96a3566ed0b311d9d7dcfe0d9cfb8b699f5f1c52a721eaf0", + "user_prompt_text": "A middleware that awaits the next function twice used to fail loudly with an error. It now silently runs the rest of the chain a second time, so downstream handlers execute twice for one request. Find the root cause and explain the exact mechanism that produces the second execution." + }, + "tool_permissions": [], + "cache_mode": "cold" + }, + "environment": { + "isolation": true, + "host_os": "darwin", + "node_version": "v22.0.0", + "claude_code_version": null, + "mcp_servers_active": [], + "skills_loaded": [], + "plugins_active": [], + "user_claude_md_hash": null, + "project_claude_md_hash": null, + "hooks_active": { + "user_prompt_submit": [], + "pre_tool_use": [], + "post_tool_use": [] + }, + "drift": { + "detected": false, + "fields": [] + } + }, + "adoption": { + "status": "not_applicable", + "attributable_madar_calls": 1, + "first_madar_call_tool": "context_pack", + "broad_fallback_operations_after_first_call": 0, + "trace_status": "trace_available" + }, + "costs": { + "indexing": { + "measured": true, + "wall_ms": 380, + "usd": 0, + "source": "locally_timed" + }, + "context_build": { + "measured": true, + "wall_ms": 96, + "usd": 0, + "source": "locally_timed" + }, + "agent": { + "measured": false, + "source": "not_applicable" + } + }, + "scores": { + "correctness": { + "measured": false, + "value": null, + "method": "blinded_rubric", + "not_measured_reason": "Tier 1 does not score answer quality; no agent answer exists." + }, + "critical_fact_completeness": { + "measured": false, + "value": null, + "method": "blinded_rubric", + "not_measured_reason": "Tier 1 does not score answer quality; no agent answer exists." + }, + "unsupported_claims": { + "measured": false, + "value": null, + "method": "blinded_rubric", + "not_measured_reason": "Tier 1 does not score answer quality; no agent answer exists." + }, + "correct_uncertainty": { + "measured": false, + "value": null, + "method": "blinded_rubric", + "not_measured_reason": "Tier 1 does not score answer quality; no agent answer exists." + }, + "evidence_support": { + "measured": true, + "value": 1, + "method": "evidence_obligation_recall", + "scored_by": "deterministic", + "blinded": null, + "truth_version": "1.0.0" + }, + "tier1_obligation_recall": { + "measured": true, + "value": 1, + "method": "evidence_obligation_recall", + "scored_by": "deterministic", + "truth_version": "1.0.0" + } + }, + "validity": { + "status": "valid", + "invalidation_reasons": [], + "aggregatable": true + }, + "retention": { + "raw_transcript": { + "retained": false, + "path": null, + "sha256": null + }, + "answer_text": { + "retained": false, + "path": null, + "sha256": null + }, + "context_artifact": { + "retained": true, + "path": "raw/context-pack.json", + "sha256": "76cae19d9787ab64fb7c0d4be597efe26c4453c855d0d2dea18e6eaa8b83ac7e" + }, + "prompt_text": { + "retained": true, + "path": "raw/prompt.txt", + "sha256": "637cd655712c2e7d96a3566ed0b311d9d7dcfe0d9cfb8b699f5f1c52a721eaf0" + }, + "environment_receipt": { + "retained": true, + "path": "raw/environment.json", + "sha256": "cb6043e8b14160ad7ce30ac3dcc967ccfc5a79fa75f55cc12e6de12048f60e32" + }, + "truth_file": { + "retained": true, + "path": "truth/rootcause-hono-middleware-rerun.json", + "sha256": "6e2005ced5ba7ff768d1b9b3135eb909e98d6c1ead04712bc1b0185c480e429d" + }, + "retention_policy": "Tier 1 retains the context artifact, prompt text, environment receipt, and truth file for at least 24 months; there is no agent transcript or answer because no agent runs." + }, + "notes": [ + "Illustrative example only. Tier 1 measures whether the evidence needed to answer was present, never whether an answer was good.", + "The agent cost account is deliberately not_measured rather than zero." + ], + "started_at": "2026-08-12T09:10:00.000Z", + "completed_at": "2026-08-12T09:10:00.476Z" +} diff --git a/docs/qualification/examples/receipt-tier2-invalid-no-madar-call.json b/docs/qualification/examples/receipt-tier2-invalid-no-madar-call.json new file mode 100644 index 00000000..eef638cd --- /dev/null +++ b/docs/qualification/examples/receipt-tier2-invalid-no-madar-call.json @@ -0,0 +1,147 @@ +{ + "contract_version": "1.0.0", + "run_id": "example-tier2-invalid-0001", + "tier": 2, + "task_id": "flow-hono-request-dispatch", + "target_id": "hono", + "arm": "madar", + "trial": 1, + "identity": { + "target_revision": "26de73133b8552f56ba72e025ecd82b08900d796", + "dependency_lock_sha256": "6328bf95a901590814ff70ed570e0fc474c05ae162d6ac690cf3c812380828ab", + "madar_commit": "06b373a447acfce895412ac10eb4e5228c5df0b7", + "madar_package_version": "0.32.1", + "madar_package_tarball_sha256": "e42a5bc1434e84291ba23ddc9da90a2f9c591e6e2bd163adedf647d0c1f711ba", + "madar_runtime_source": "npm_pack", + "madar_config_sha256": "c6dcdf9130e3ef5503caead4fe07f1b465b429adfa6e43309166d8f650a33caf", + "agent": { + "host": "claude-code", + "host_version": "0.0.0-example", + "model_id": "example-model-id" + }, + "prompts": { + "system_prompt_sha256": "21da86de37b6595841807f54b44945498ad599cf682a40391a36d833dca35132", + "user_prompt_sha256": "c276d363f0b10c00b75df925f2e658f3ee7af085872bcbcc64a7a4252c7b3b35", + "user_prompt_text": "Trace what happens to an incoming request from the application's entry point until a response is returned. Cover how the path is resolved, how a route is matched, how handlers and middleware are run, and how errors and unmatched routes are handled. Say which steps are skipped in the fast path." + }, + "tool_permissions": [ + "Read", + "Grep", + "Glob", + "mcp__madar__retrieve" + ], + "cache_mode": "warm" + }, + "environment": { + "isolation": true, + "host_os": "darwin", + "node_version": "v22.0.0", + "claude_code_version": "0.0.0-example", + "mcp_servers_active": [ + "madar" + ], + "skills_loaded": [], + "plugins_active": [], + "user_claude_md_hash": null, + "project_claude_md_hash": null, + "hooks_active": { + "user_prompt_submit": [], + "pre_tool_use": [], + "post_tool_use": [] + }, + "drift": { + "detected": false, + "fields": [] + } + }, + "adoption": { + "status": "absent", + "attributable_madar_calls": 0, + "first_madar_call_tool": null, + "broad_fallback_operations_after_first_call": 0, + "trace_status": "trace_available" + }, + "costs": { + "indexing": { + "measured": true, + "wall_ms": 4120, + "usd": 0, + "source": "locally_timed" + }, + "context_build": { + "measured": false, + "source": "not_applicable" + }, + "agent": { + "measured": true, + "input_tokens": 41233, + "output_tokens": 1902, + "wall_ms": 61204, + "usd": 0.19, + "source": "provider_reported" + } + }, + "scores": { + "correctness": { + "measured": false, + "value": null, + "method": "blinded_rubric", + "not_measured_reason": "run is invalid: no attributable Madar call" + }, + "critical_fact_completeness": { + "measured": false, + "value": null, + "method": "blinded_rubric", + "not_measured_reason": "run is invalid: no attributable Madar call" + }, + "unsupported_claims": { + "measured": false, + "value": null, + "method": "blinded_rubric", + "not_measured_reason": "run is invalid: no attributable Madar call" + }, + "correct_uncertainty": { + "measured": false, + "value": null, + "method": "blinded_rubric", + "not_measured_reason": "run is invalid: no attributable Madar call" + }, + "evidence_support": { + "measured": false, + "value": null, + "method": "blinded_rubric", + "not_measured_reason": "run is invalid: no attributable Madar call" + } + }, + "validity": { + "status": "invalid", + "invalidation_reasons": [ + "missing_attributable_madar_call" + ], + "aggregatable": false + }, + "retention": { + "raw_transcript": { + "retained": true, + "path": "raw/transcript.jsonl", + "sha256": "8afad96d88f459209acacc66728718b270aa3e2df59fb8e92d5fc85475b3272c" + }, + "answer_text": { + "retained": true, + "path": "raw/answer.txt", + "sha256": "95967dd7cc113d0df15f5dd7b3d2185f7755a3da9b4db4b71805cd646434d7b6" + }, + "context_artifact": { + "retained": false, + "path": null, + "sha256": null + }, + "retention_policy": "Raw transcripts, answers, and context artifacts are retained for at least 24 months alongside the receipt; see validity-rules.md." + }, + "notes": [ + "Illustrative example only. The agent never called Madar, so every quality dimension stays not_measured and the recorded costs may not be cited as a cost comparison.", + "The costs block is still populated because adoption failure must be diagnosable, but aggregatable is false so nothing here can enter a headline." + ], + "started_at": "2026-08-12T09:00:00.000Z", + "completed_at": "2026-08-12T09:01:05.324Z" +} diff --git a/docs/qualification/freeze.json b/docs/qualification/freeze.json new file mode 100644 index 00000000..da591dcd --- /dev/null +++ b/docs/qualification/freeze.json @@ -0,0 +1,29 @@ +{ + "contract_version": "1.0.0", + "frozen_at": "2026-08-12", + "algorithm": "sha256 over raw file bytes", + "note": "Regenerate deliberately with `npm run qualify:validate -- --write` and say why in the pull request. A silent digest change is a contract change.", + "files": { + "docs/qualification/README.md": "8153515a9daa7b5bb253c2d5439d8b485c0c6c03958cac2d3f287c74f9675162", + "docs/qualification/corpus.json": "8d2fd2ae3386498010fe716dcc8c7903f4759135ca0380cc453ee1f2c1dd8a52", + "docs/qualification/evidence-categories.md": "1f75b04229b805050584eadda201bc67906224d34f562afa2098558296a952cd", + "docs/qualification/examples/receipt-tier1-valid.json": "a68f8d58d1f940ac10e5d5cb1c124644a51e722f886abe53f1e06c7d53f490fd", + "docs/qualification/examples/receipt-tier2-invalid-no-madar-call.json": "2783cad80b8a00d12b8d84718f3c4b3e31846111abc4830c47677dd3422a80aa", + "docs/qualification/holdout-policy.md": "f5671c4cd5f5487e21dcb44e34fa98b5520a7b9ec58391ce1f086ebd33e23616", + "docs/qualification/patches/hono-compose-reentrancy-guard.patch": "9355c5bbb05cd5ae4d998ace18d6381f0cba4fd080203d4d02579da3dcf6dea4", + "docs/qualification/patches/hono-error-message-disclosure.patch": "edb79059b72b4f27f5dc8341ba2d9a3617901c402da9ef1f9daf5503e6528d6f", + "docs/qualification/receipt-schema.json": "e849ba4b28c0cad119a8a23fda82bf39a91f2d53ac6b6900806d0157ac91f0be", + "docs/qualification/rubrics.json": "bf176381b6d6df5da41c4a3685bfaf8bf31f0767feef571ee7b85ce4644eba6f", + "docs/qualification/stop-rule.md": "36246356a456d5839e0670b298ee7c34a80e89f47481991eb791d029b972e1c9", + "docs/qualification/tasks.json": "c941088446f21e4f233185794821639ba8095ba3f9984f9cd24a9fc039cbc02b", + "docs/qualification/tier1.json": "967e3d3bafdd646288660c10184d97414d2867948285d062513b8f5d5bcb6ac9", + "docs/qualification/tier2-matrix.json": "449d807630fad5adb46691c6d7c7329992c0674ab099cec8c29329e7fc1f638b", + "docs/qualification/truth/arch-unstorage-driver-seam.json": "82e3a9d52c8a19716b30eceb87ca2a2eb9e2107dd3d07498145c0aae70586b89", + "docs/qualification/truth/flow-hono-request-dispatch.json": "eb39c4e14a1932c5440fcdac2c6ba20928fba89120879f2667c8bfafabb612fb", + "docs/qualification/truth/impact-hono-drop-router-fallback.json": "b500e8d16d32069ed4c6ab8014199e01cd0f0ae4d785ee96eed212b3eaa98f2d", + "docs/qualification/truth/plan-unstorage-add-driver.json": "e6332e047475f88ff41358b17211a22b3096be992a3f523673362163ea9f37f2", + "docs/qualification/truth/review-hono-error-handling.json": "5e0653e0fa613cc7a4ee72aad1175d7a95ccce603aaf68afe5cc59b2e15854dd", + "docs/qualification/truth/rootcause-hono-middleware-rerun.json": "6e2005ced5ba7ff768d1b9b3135eb909e98d6c1ead04712bc1b0185c480e429d", + "docs/qualification/validity-rules.md": "ee3898f57210693405840f926513205fb7c5c1645c348dc6ee5ea1d45967d18f" + } +} diff --git a/docs/qualification/holdout-policy.md b/docs/qualification/holdout-policy.md new file mode 100644 index 00000000..bf786d89 --- /dev/null +++ b/docs/qualification/holdout-policy.md @@ -0,0 +1,85 @@ +# Hidden holdout policy + +Contract version `1.0.0`, frozen 2026-08-12 for [#655](https://github.com/mohanagy/madar/issues/655). + +## Why holdouts exist here + +Everything in [`corpus.json`](./corpus.json) with `holdout_class: "open"` is visible to +whoever writes production retrieval and ranking rules. Open targets are still useful — +they catch regressions — but they cannot detect the failure mode this policy exists for: +production behaviour drifting toward the qualification corpus itself. Only a target the +rule author has never seen can measure that. + +Naturalness and hiddenness are separate properties and neither substitutes for the other. +Every open target in this corpus is a real externally authored repository, which removes +the risk that the target was shaped around its own answer. It does not remove the risk that +production rules are shaped around the target once it is known. + +## Classes + +| Class | Meaning | +| --- | --- | +| `open` | Target, prompts, and truth live in this repository. Anyone may read them. Useful for regression detection; **worthless** as evidence of generalization. | +| `sealed` | Target, prompts, and truth are authored and held by someone who does not write production retrieval, ranking, or claim logic. The rule author never reads them before the sweep. | + +## Rules for a sealed holdout + +1. The target, the prompts, and the truth are authored by a person who has not written and + will not write production retrieval, ranking, or claim logic during the evaluation window. +2. They live outside this repository. Nothing about them — no repository name, no path, no + symbol, no prompt wording — is committed here, discussed in an issue, or pasted into a + pull request. +3. The rule author receives the sweep result only: per-cell pass/fail and the scored + dimensions. Never the answers, never the prompts, never the truth. +4. A sealed target is used at most **once per release line**. After a result is reported + against it, it is burned: it becomes an open target or it is retired. Reusing a sealed + target after its result is known makes it open in everything but name. +5. If a sealed cell fails, the holder may release the failing task to the rule author for + diagnosis. That releases the target permanently. +6. The runner supports this today without new code: pass alternate manifests that live + outside the checkout. The existing + [`docs/benchmarks/suite/holdouts/README.md`](../benchmarks/suite/holdouts/README.md) + documents the equivalent mechanism for the product benchmark suite. + +## Current status: unsatisfied + +**Madar has one author.** There is no second person to author or hold a sealed target, and +no meaningful sense in which a target can be hidden from the person who writes both the +production rules and the corpus. The `sealed-holdout-a` slot in `corpus.json` is therefore +marked `status: "unsatisfied"` rather than being filled with a self-selected target that +would look like a holdout and prove nothing. + +The same limitation makes two other artifacts unavailable: + +- the hidden acceptance test for `plan-unstorage-add-driver` + (see `truth/plan-unstorage-add-driver.json`); +- blinded Tier 2 review (see `rubrics.json#/blinding/current_status`). + +### Human action required + +To satisfy this policy, a person other than the production-rule author must: + +1. select and pin one real, permissively licensed TypeScript/Node repository not named + anywhere in this repository; +2. author two to four task prompts and their independent truth for it, without reading + Madar output; +3. author the hidden acceptance test for the bounded-implementation task; +4. hold all of it outside this repository and run the sweep themselves, returning only + per-cell scores; +5. record their name and the seal date in the sweep receipt. + +Until that happens, **no generalization claim may be made from this corpus**, and any +report derived from it must carry this exact line: + +> sealed holdout unsatisfied; results measure regression only + +## What must never happen + +- A sealed target, prompt, path, or symbol must never appear in production retrieval, + ranking, claim, or configuration code. +- A sealed target must never be added to the repository's test fixtures. +- A sealed slot must never be filled with a self-authored fixture workspace. That would + satisfy neither naturalness nor hiddenness while appearing to satisfy both. +- A failing sealed cell must never be resolved by editing the sealed truth. +- The rule author must never request the sealed prompts "just to check whether they are + fair". Fairness disputes are resolved by the holder retiring the task, not by disclosure. diff --git a/docs/qualification/patches/hono-compose-reentrancy-guard.patch b/docs/qualification/patches/hono-compose-reentrancy-guard.patch new file mode 100644 index 00000000..7f5c6335 --- /dev/null +++ b/docs/qualification/patches/hono-compose-reentrancy-guard.patch @@ -0,0 +1,13 @@ +diff --git a/src/compose.ts b/src/compose.ts +index b1d4508..76f84c3 100644 +--- a/src/compose.ts ++++ b/src/compose.ts +@@ -30,7 +30,7 @@ export const compose = ( + * @returns {Promise} - A promise that resolves to the context. + */ + async function dispatch(i: number): Promise { +- if (i <= index) { ++ if (i < index) { + throw new Error('next() called multiple times') + } + index = i diff --git a/docs/qualification/patches/hono-error-message-disclosure.patch b/docs/qualification/patches/hono-error-message-disclosure.patch new file mode 100644 index 00000000..8459b37d --- /dev/null +++ b/docs/qualification/patches/hono-error-message-disclosure.patch @@ -0,0 +1,13 @@ +diff --git a/src/hono-base.ts b/src/hono-base.ts +index e6a7278..6af89b0 100644 +--- a/src/hono-base.ts ++++ b/src/hono-base.ts +@@ -38,7 +38,7 @@ const errorHandler: ErrorHandler = (err, c) => { + return c.newResponse(res.body, res) + } + console.error(err) +- return c.text('Internal Server Error', 500) ++ return c.text(`Internal Server Error: ${err.stack ?? err.message}`, 500) + } + + type GetPath = (request: Request, options?: { env?: E['Bindings'] }) => string diff --git a/docs/qualification/receipt-schema.json b/docs/qualification/receipt-schema.json new file mode 100644 index 00000000..b725c7e3 --- /dev/null +++ b/docs/qualification/receipt-schema.json @@ -0,0 +1,379 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/mohanagy/madar/docs/qualification/receipt-schema.json", + "title": "Madar qualification run receipt", + "description": "Environment and run receipt for a single qualification cell. One receipt describes one arm of one trial of one task against one target.", + "type": "object", + "additionalProperties": false, + "required": [ + "contract_version", + "run_id", + "tier", + "task_id", + "target_id", + "arm", + "trial", + "identity", + "environment", + "adoption", + "costs", + "scores", + "validity", + "retention", + "started_at", + "completed_at" + ], + "properties": { + "contract_version": { "type": "string", "pattern": "^\\d+\\.\\d+\\.\\d+$" }, + "run_id": { "type": "string", "minLength": 1 }, + "tier": { "type": "integer", "enum": [1, 2] }, + "task_id": { "type": "string", "minLength": 1 }, + "target_id": { "type": "string", "minLength": 1 }, + "arm": { "type": "string", "enum": ["native", "madar"] }, + "trial": { "type": "integer", "minimum": 1 }, + + "identity": { + "type": "object", + "additionalProperties": false, + "description": "The full experimental identity. Every field is required; a missing field invalidates the run.", + "required": [ + "target_revision", + "dependency_lock_sha256", + "madar_commit", + "madar_package_version", + "madar_runtime_source", + "madar_config_sha256", + "agent", + "prompts", + "tool_permissions", + "cache_mode" + ], + "properties": { + "target_revision": { + "type": "string", + "minLength": 1, + "description": "Git SHA for a git target, or the frozen content digest for a fixture target." + }, + "dependency_lock_sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" }, + "madar_commit": { "type": "string", "pattern": "^[0-9a-f]{40}$" }, + "madar_package_version": { "type": "string", "minLength": 1 }, + "madar_package_tarball_sha256": { "type": ["string", "null"], "pattern": "^[0-9a-f]{64}$" }, + "madar_runtime_source": { "type": "string", "enum": ["npm_pack", "checkout"] }, + "madar_config_sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" }, + "agent": { + "type": "object", + "additionalProperties": false, + "required": ["host", "host_version", "model_id"], + "properties": { + "host": { "type": "string", "minLength": 1 }, + "host_version": { "type": ["string", "null"] }, + "model_id": { "type": "string", "minLength": 1 } + } + }, + "prompts": { + "type": "object", + "additionalProperties": false, + "required": ["system_prompt_sha256", "user_prompt_sha256", "user_prompt_text"], + "properties": { + "system_prompt_sha256": { "type": ["string", "null"], "pattern": "^[0-9a-f]{64}$" }, + "user_prompt_sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" }, + "user_prompt_text": { "type": "string", "minLength": 1 } + } + }, + "tool_permissions": { + "type": "array", + "items": { "type": "string" }, + "description": "Exact allowed-tool list given to the agent. Both arms of a cell must record the same list apart from Madar tools." + }, + "cache_mode": { "type": "string", "enum": ["cold", "warm"] } + } + }, + + "environment": { + "type": "object", + "additionalProperties": false, + "required": [ + "isolation", + "host_os", + "node_version", + "mcp_servers_active", + "skills_loaded", + "plugins_active", + "hooks_active", + "drift" + ], + "properties": { + "isolation": { "type": "boolean" }, + "host_os": { "type": "string", "minLength": 1 }, + "node_version": { "type": "string", "minLength": 1 }, + "claude_code_version": { "type": ["string", "null"] }, + "mcp_servers_active": { "type": "array", "items": { "type": "string" } }, + "skills_loaded": { "type": "array", "items": { "type": "string" } }, + "plugins_active": { "type": "array", "items": { "type": "string" } }, + "user_claude_md_hash": { "type": ["string", "null"] }, + "project_claude_md_hash": { "type": ["string", "null"] }, + "hooks_active": { + "type": "object", + "additionalProperties": false, + "required": ["user_prompt_submit", "pre_tool_use", "post_tool_use"], + "properties": { + "user_prompt_submit": { "type": "array", "items": { "type": "string" } }, + "pre_tool_use": { "type": "array", "items": { "type": "string" } }, + "post_tool_use": { "type": "array", "items": { "type": "string" } } + } + }, + "drift": { + "type": "object", + "additionalProperties": false, + "required": ["detected", "fields"], + "properties": { + "detected": { "type": "boolean" }, + "fields": { "type": "array", "items": { "type": "string" } } + } + } + } + }, + + "adoption": { + "type": "object", + "additionalProperties": false, + "description": "Behaviour measurement. Never folded into a quality score.", + "required": ["status", "attributable_madar_calls", "broad_fallback_operations_after_first_call"], + "properties": { + "status": { "type": "string", "enum": ["adopted", "late", "absent", "not_applicable"] }, + "attributable_madar_calls": { "type": "integer", "minimum": 0 }, + "first_madar_call_tool": { "type": ["string", "null"] }, + "broad_fallback_operations_after_first_call": { "type": "integer", "minimum": 0 }, + "trace_status": { "type": "string", "enum": ["trace_available", "trace_partial", "trace_missing"] } + } + }, + + "costs": { + "type": "object", + "additionalProperties": false, + "description": "Indexing, context building, and agent execution are separate accounts and must never be summed into a single headline.", + "required": ["indexing", "context_build", "agent"], + "properties": { + "indexing": { "$ref": "#/definitions/costAccount" }, + "context_build": { "$ref": "#/definitions/costAccount" }, + "agent": { "$ref": "#/definitions/costAccount" } + } + }, + + "scores": { + "type": "object", + "additionalProperties": false, + "required": [ + "correctness", + "critical_fact_completeness", + "unsupported_claims", + "correct_uncertainty", + "evidence_support" + ], + "properties": { + "correctness": { "$ref": "#/definitions/score" }, + "critical_fact_completeness": { "$ref": "#/definitions/score" }, + "unsupported_claims": { "$ref": "#/definitions/score" }, + "correct_uncertainty": { "$ref": "#/definitions/score" }, + "evidence_support": { "$ref": "#/definitions/score" }, + "implementation": { "$ref": "#/definitions/score" }, + "tier1_obligation_recall": { "$ref": "#/definitions/score" } + } + }, + + "validity": { + "type": "object", + "additionalProperties": false, + "required": ["status", "invalidation_reasons", "aggregatable"], + "properties": { + "status": { "type": "string", "enum": ["valid", "degraded", "invalid"] }, + "invalidation_reasons": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "missing_attributable_madar_call", + "prompt_contract_failure", + "answer_contract_failure", + "target_revision_mismatch", + "patch_application_failure", + "package_revision_mismatch", + "dependency_lock_mismatch", + "isolation_failure", + "incomplete_transcript", + "incomplete_receipt", + "judge_failure", + "environment_mismatch", + "quality_gate_failure", + "truth_unavailable", + "blinding_unavailable" + ] + } + }, + "aggregatable": { + "type": "boolean", + "description": "MUST be false whenever status is not valid. A false value forbids this receipt from entering any cost, latency, or quality aggregate." + } + } + }, + + "retention": { + "type": "object", + "additionalProperties": false, + "$comment": "One slot per artifact named in validity-rules.md, Retention. The Tier 1 set is required by the tier-conditional rule below; the full artifact list is not required in general.", + "required": [ + "raw_transcript", + "answer_text", + "context_artifact", + "retention_policy" + ], + "properties": { + "raw_transcript": { "$ref": "#/definitions/artifactRef" }, + "answer_text": { "$ref": "#/definitions/artifactRef" }, + "context_artifact": { "$ref": "#/definitions/artifactRef" }, + "prompt_text": { "$ref": "#/definitions/artifactRef" }, + "environment_receipt": { "$ref": "#/definitions/artifactRef" }, + "truth_file": { "$ref": "#/definitions/artifactRef" }, + "retention_policy": { "type": "string", "minLength": 1 } + } + }, + + "notes": { "type": "array", "items": { "type": "string" } }, + "started_at": { "type": "string", "format": "date-time" }, + "completed_at": { "type": "string", "format": "date-time" } + }, + + "allOf": [ + { + "description": "A run that is not valid can never be aggregatable.", + "if": { + "properties": { "validity": { "properties": { "status": { "enum": ["degraded", "invalid"] } }, "required": ["status"] } }, + "required": ["validity"] + }, + "then": { + "properties": { "validity": { "properties": { "aggregatable": { "const": false } } } } + } + }, + { + "description": "An invalid run must give a reason.", + "if": { + "properties": { "validity": { "properties": { "status": { "const": "invalid" } }, "required": ["status"] } }, + "required": ["validity"] + }, + "then": { + "properties": { "validity": { "properties": { "invalidation_reasons": { "minItems": 1 } } } } + } + }, + { + "description": "A Tier 1 receipt must retain the four artifacts needed to reproduce deterministic evaluation without an agent run.", + "if": { + "properties": { "tier": { "const": 1 } }, + "required": ["tier"] + }, + "then": { + "properties": { + "retention": { + "required": ["context_artifact", "prompt_text", "environment_receipt", "truth_file"], + "properties": { + "context_artifact": { "properties": { "retained": { "const": true } }, "required": ["retained"] }, + "prompt_text": { "properties": { "retained": { "const": true } }, "required": ["retained"] }, + "environment_receipt": { "properties": { "retained": { "const": true } }, "required": ["retained"] }, + "truth_file": { "properties": { "retained": { "const": true } }, "required": ["retained"] } + } + } + } + } + } + ], + + "definitions": { + "costAccount": { + "type": "object", + "additionalProperties": false, + "required": ["measured"], + "properties": { + "measured": { "type": "boolean" }, + "input_tokens": { "type": ["integer", "null"], "minimum": 0 }, + "output_tokens": { "type": ["integer", "null"], "minimum": 0 }, + "cache_creation_input_tokens": { "type": ["integer", "null"], "minimum": 0 }, + "wall_ms": { "type": ["integer", "null"], "minimum": 0 }, + "usd": { "type": ["number", "null"], "minimum": 0 }, + "source": { + "type": "string", + "enum": ["provider_reported", "locally_timed", "unknown", "not_applicable"] + } + }, + "allOf": [ + { + "$comment": "validity-rules.md, Cost separation: an unmeasured account is measured: false, never 0. Without this branch a receipt could report measured: false alongside a 0 for every figure, which is exactly the interchange the rule forbids.", + "if": { "properties": { "measured": { "const": false } }, "required": ["measured"] }, + "then": { + "properties": { + "input_tokens": { "type": "null" }, + "output_tokens": { "type": "null" }, + "cache_creation_input_tokens": { "type": "null" }, + "wall_ms": { "type": "null" }, + "usd": { "type": "null" } + } + } + } + ] + }, + "score": { + "type": "object", + "additionalProperties": false, + "required": ["measured", "value", "method"], + "properties": { + "measured": { "type": "boolean" }, + "value": { + "type": ["number", "string", "null"], + "description": "MUST be null whenever measured is false. A not_measured dimension never carries a number." + }, + "method": { "type": "string", "minLength": 1 }, + "scored_by": { "type": ["string", "null"] }, + "blinded": { "type": ["boolean", "null"] }, + "truth_version": { "type": ["string", "null"] }, + "not_measured_reason": { "type": ["string", "null"] } + }, + "allOf": [ + { + "if": { "properties": { "measured": { "const": false } }, "required": ["measured"] }, + "then": { + "properties": { + "value": { "type": "null" }, + "not_measured_reason": { "type": "string", "minLength": 1 } + }, + "required": ["not_measured_reason"] + } + }, + { + "$comment": "The complement of the branch above. validity-rules.md keeps a score of 0 and a score of not_measured distinct; a measured score carrying null is indistinguishable from not_measured, so it is rejected here rather than left to the reader.", + "if": { "properties": { "measured": { "const": true } }, "required": ["measured"] }, + "then": { "properties": { "value": { "type": ["number", "string"] } } } + } + ] + }, + "artifactRef": { + "type": "object", + "additionalProperties": false, + "required": ["retained", "path", "sha256"], + "properties": { + "retained": { "type": "boolean" }, + "path": { "type": ["string", "null"] }, + "sha256": { "type": ["string", "null"], "pattern": "^[0-9a-f]{64}$" } + }, + "allOf": [ + { + "$comment": "validity-rules.md, Retention: each retained artifact is recorded with its path and SHA-256. Nullable types alone let retained: true carry no path and no digest, which records a retention claim that cannot be checked.", + "if": { "properties": { "retained": { "const": true } }, "required": ["retained"] }, + "then": { + "properties": { + "path": { "type": "string", "minLength": 1 }, + "sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" } + } + } + } + ] + } + } +} diff --git a/docs/qualification/rubrics.json b/docs/qualification/rubrics.json new file mode 100644 index 00000000..4b2bfef0 --- /dev/null +++ b/docs/qualification/rubrics.json @@ -0,0 +1,155 @@ +{ + "contract_version": "1.0.0", + "frozen_at": "2026-08-12", + "dimensions": { + "correctness": { + "definition": "Every assertion the answer makes about the target is true of the pinned source.", + "scale": { "0": "contains a false assertion about the target", "1": "no false assertions, but at least one assertion a blinded reviewer cannot verify either way against the pinned source", "2": "no false assertions, and every assertion is verifiable against the pinned source" }, + "scored_by": "blinded_human", + "tiers": [2], + "gating": true + }, + "critical_fact_completeness": { + "definition": "Fraction of the truth file's critical facts that the answer states. Facts marked supporting do not count in the denominator.", + "scale": "ratio in [0,1] with the per-task critical_facts_required_for_pass set treated as mandatory", + "scored_by": "blinded_human", + "tiers": [2], + "gating": true + }, + "unsupported_claims": { + "definition": "Count of assertions matching the task's unsupported_claim_traps, plus any other assertion a blinded reviewer cannot trace to the pinned source.", + "scale": "non-negative integer; lower is better", + "scored_by": "blinded_human", + "tiers": [2], + "gating": true + }, + "correct_uncertainty": { + "definition": "Fraction of the task's correct_uncertainty requirements the answer honours. Both directions are penalised: asserting something the source cannot support, and hedging something the source states plainly.", + "scale": "ratio in [0,1]", + "scored_by": "blinded_human", + "tiers": [2], + "gating": false, + "not_gating_reason": "No threshold for this dimension has been calibrated; gating on an uncalibrated threshold manufactures confidence and invites the threshold to be tuned to whatever passes.", + "becomes_gating_when": "A baseline distribution over the natural corpus must exist, and the threshold must be derived and recorded BEFORE any target is scored against it.", + "method_gate_boundary": "The ratio dimension remains non-gating. methods.single_root_cause_adjudication gates only the explicit, pre-registered ids in uncertainty_required_for_pass; honouring that enumerated id list is deliberately not a calibrated threshold.", + "reported": true, + "reporting_reason": "The dimension is still scored and recorded on every receipt, because an exemption that also stops measurement can never end." + }, + "evidence_support": { + "definition": "Fraction of critical assertions accompanied by a citation to a real path and symbol in the pinned target that actually contains the cited content.", + "scale": "ratio in [0,1]", + "scored_by": "both", + "tiers": [1, 2], + "gating": false, + "not_gating_reason": "No threshold for this dimension has been calibrated; gating on an uncalibrated threshold manufactures confidence and invites the threshold to be tuned to whatever passes.", + "becomes_gating_when": "A baseline distribution over the natural corpus must exist, and the threshold must be derived and recorded BEFORE any target is scored against it.", + "reported": true, + "reporting_reason": "The dimension is still scored and recorded on every receipt, because an exemption that also stops measurement can never end.", + "not_gating_scope": "This concerns only the Tier 2 blinded rubric. The Tier 1 deterministic gate in tier1.json#/gate (evidence_obligation_recall plus the negative-trust probes) is unaffected and continues to gate pull requests.", + "note": "The path/symbol existence half is deterministic. Whether the cited code supports the assertion is blinded-human only." + }, + "intended_tool_adoption": { + "definition": "Whether the run made an attributable Madar call before broad repository exploration, when the task contract requires one.", + "scale": { "adopted": "attributable Madar call precedes broad exploration", "late": "attributable call occurred after broad exploration", "absent": "no attributable Madar call" }, + "scored_by": "deterministic", + "tiers": [2], + "gating": false, + "note": "Adoption is a behaviour measurement, NOT a context-quality measurement. It is reported in its own column and must never be folded into a quality score. absent makes the run invalid for quality comparison; it is not a quality loss." + }, + "broad_fallback_exploration": { + "definition": "Count of broad repository operations (directory-wide search, unscoped glob, full-file reads outside the evidence set) performed after the first Madar call.", + "scale": "non-negative integer; lower is better", + "scored_by": "deterministic", + "tiers": [2], + "gating": false, + "note": "Reported separately from quality and from cost. A high value with a passing quality score means the context pack was insufficient, not that the answer was bad." + } + }, + "methods": { + "evidence_obligation_recall": { + "tier": 1, + "deterministic": true, + "inputs": ["the context artifact produced for the frozen prompt", "the task truth file's tier1_obligations"], + "procedure": [ + "Collect the evidence set (paths and symbols) the artifact presents as supporting material.", + "required_evidence_paths recall must be >= min_critical_fact_recall. Paths are compared as exact repository-relative strings.", + "required_evidence_symbols recall must be >= min_critical_fact_recall. Symbols are compared on their LAST dot-separated segment, case-sensitively, after stripping a leading '#': the truth files record obligations bare (`fetch`, `dispatch`, `Context`) while evidence entries qualify them by owner (`Hono.fetch`, `Hono.#dispatch`, `compose.dispatch`), and an exact-string comparison would never match those pairs. `Hono.fetch` therefore satisfies the obligation `fetch`; `getPath` satisfies `getPath`. Two different owners exposing the same member name are treated as one obligation, which is accepted: the obligation asks whether the evidence for that member was surfaced at all.", + "Every path cited by the artifact must exist in the pinned target.", + "If any must_not_report_ready_when condition holds, the artifact must not report a ready state." + ], + "outcome": ["pass", "fail", "not_measured"], + "explicitly_not_measured": "Answer quality. Tier 1 measures whether the evidence needed to answer was present, not whether an agent answered well." + }, + "blinded_rubric": { + "tier": 2, + "deterministic": false, + "procedure": [ + "The reviewer receives the answer text, the task prompt, and the truth file.", + "The reviewer does NOT receive the arm label, the token counts, the latency, or the transcript.", + "The reviewer scores correctness, critical_fact_completeness, unsupported_claims, correct_uncertainty, and evidence_support.", + "Arm labels are revealed only after every answer in the cell is scored." + ], + "pass_condition": "correctness == 2 AND every id in critical_facts_required_for_pass is present AND unsupported_claims == 0", + "pass_condition_note": "correct_uncertainty and evidence_support are deliberately omitted; see dimensions.correct_uncertainty.not_gating_reason and dimensions.evidence_support.not_gating_reason." + }, + "ordered_path_rubric": { + "tier": 2, + "deterministic": false, + "extends": "blinded_rubric", + "additional_procedure": [ + "For each pair in order_sensitive_pairs, the answer must place the first element before the second.", + "An answer that names every step but inverts an order_sensitive_pair fails correctness." + ] + }, + "affected_set_precision_recall": { + "tier": 2, + "deterministic": false, + "extends": "blinded_rubric", + "additional_procedure": [ + "recall = |named ∩ recall_denominator| / |recall_denominator|", + "Each member of precision_penalty_set named as affected counts as one unsupported claim.", + "The loud-vs-silent judgement is scored under correctness, not under recall." + ] + }, + "single_root_cause_adjudication": { + "tier": 2, + "deterministic": false, + "extends": "blinded_rubric", + "additional_procedure": [ + "The answer passes only if it names a cause in accepted_root_cause_ids as THE cause.", + "Listing the correct cause among several candidate causes without committing scores as a partial: critical_fact_completeness credit, correctness 1, no pass.", + "Every id in uncertainty_required_for_pass must be honoured." + ], + "uncertainty_gate_note": "This method gates uncertainty by the explicit, pre-registered ids in uncertainty_required_for_pass. Honouring an enumerated id list is checkable without calibration and is deliberately not a threshold on dimensions.correct_uncertainty, which remains non-gating." + }, + "seeded_defect_detection": { + "tier": 2, + "deterministic": false, + "extends": "blinded_rubric", + "additional_procedure": [ + "Every id in required_detections must be reported with a citation that proves it.", + "Each reported finding matching false_positive_set counts as one unsupported claim.", + "Findings in acceptable_additional_findings are neither required nor penalised." + ] + } + }, + "blinding": { + "required_for_tier": 2, + "rules": [ + "Answers are stripped of arm labels, tool traces, and cost data before review.", + "Answers from both arms of a cell are shuffled and reviewed in one pass.", + "The reviewer must not be the person who authored the change under evaluation.", + "Reviewer identity, review date, and the truth file version are recorded on every score." + ], + "current_status": "unsatisfied", + "current_status_reason": "Single-author repository. See holdout-policy.md; Tier 2 scores produced without an independent reviewer must be labelled self_reviewed and are not publishable evidence." + }, + "aggregation": { + "rules": [ + "Quality dimensions are aggregated per task and per repo. There is no blended cross-task headline.", + "Runs whose validity is not valid contribute to no aggregate other than the invalid-run count.", + "Adoption and broad_fallback_exploration are reported as their own columns and never merged into a quality score.", + "Cost and latency are reported only for cells whose gating quality dimensions passed." + ] + } +} diff --git a/docs/qualification/stop-rule.md b/docs/qualification/stop-rule.md new file mode 100644 index 00000000..964d5e24 --- /dev/null +++ b/docs/qualification/stop-rule.md @@ -0,0 +1,65 @@ +# Stop, rollback, and publication rule + +Contract version `1.0.0`, frozen 2026-08-12 for [#655](https://github.com/mohanagy/madar/issues/655). + +This rule exists to be objective enough to block a pull request or a release without a +judgement call. Each condition is written so that a reviewer can answer yes or no from the +receipts alone. + +## S1 — Stop conditions (a change must not ship) + +A roadmap change **must not merge**, and must be rolled back or disabled if already +merged, when any of the following holds against the frozen corpus. + +| Id | Condition | Objective test | +| --- | --- | --- | +| S1.1 | Critical-fact completeness regresses beyond the pre-registered margin | For any task, the post-change critical-fact completeness is lower than the pre-change value by more than the non-inferiority margin **0.05**, on `n_valid >= 5` paired trials in the same cell. | +| S1.2 | Unsupported claims increase materially | For any task, the post-change mean unsupported-claim count exceeds the pre-change mean by **more than 0.5 claims per answer**, or any single answer introduces an unsupported claim listed in that task's `unsupported_claim_traps` that the pre-change arm did not make. | +| S1.3 | False-ready behaviour appears | Any negative-trust probe in [`tier1.json`](./tier1.json) reports a ready state, or any evidence set contains a path or symbol that does not exist in the pinned target. This is a **single-occurrence** trip: one instance blocks. | +| S1.4 | Host adoption falls below the phase target | Two clauses, each independently evaluable. **Absolute clause:** across the Tier 2 sweep, fewer than the phase target of Madar-arm runs have `adoption.status` of `adopted`. This clause is **inactive** while no phase target is recorded — Phase 0 records none — and an inactive clause never trips and never blocks. **Relative clause:** a *decrease* of more than 10 percentage points against the previous recorded sweep blocks. With no previous recorded sweep there is nothing to compare against, so the relative clause does not trip either; the first sweep establishes the baseline and cannot itself fail S1.4. Until a phase target is recorded and one sweep exists, S1.4 evaluates to *not tripped*, and adoption is measured and reported rather than gated. | +| S1.5 | Graph or artifact integrity fails | Any graph-integrity invariant from #656–#659 fails, or an artifact fails its round-trip or old-reader-rejection check. Single-occurrence trip. | +| S1.6 | Results depend on qualification-repository literals | Any qualification fixture path, symbol, prompt string, repository id, or a near-equivalent special case appears in production retrieval, ranking, context, or claim logic. Single-occurrence trip; checked deterministically by `npm run qualify:validate`. | +| S1.7 | Output differences remain unexplained | A retrieval, pack, graph, or artifact output differs from the pre-change baseline and the pull request does not explain the difference. Updating a snapshot is not an explanation. | +| S1.8 | Cost improves only by reducing outcome quality | A token, latency, or cost improvement is reported for a cell whose correctness or critical-fact completeness is not non-inferior under S1.1. | + +## S2 — Rollback + +When a stop condition is discovered after merge: + +1. Disable the change at the narrowest available seam — feature flag, default flip, or + revert of the specific commit — the same day it is confirmed. +2. Do not fix forward on the protected branch while a stop condition is tripped. +3. File the finding as a linked issue with the receipt paths that prove it. +4. Re-run the affected Tier 1 subset after the rollback and attach the receipt showing the + condition cleared. +5. If the change already shipped to npm, the release notes are amended and the affected + claim is withdrawn before anything else ships. + +## S3 — Publication + +A claim derived from this corpus may be published only when **all** of the following hold. +Any single failure means the claim is not published in any weakened form either. + +1. Every cell backing the claim has `validity.status: "valid"` and `aggregatable: true`. +2. `n_invalid` is published beside `n_valid` for every row. +3. Correctness and critical-fact completeness passed **before** any cost or latency figure + is shown. +4. Tier 2 scores were produced by a blinded reviewer who did not author the change. +5. The sealed holdout is satisfied, or the report makes no generalization claim and carries + the line `sealed holdout unsatisfied; results measure regression only`. +6. The claim is narrower than or equal to the evidence: per-target and per-task, never a + blended headline. +7. The evidence class is labelled per [`evidence-categories.md`](./evidence-categories.md). + +## S4 — What may never be used to clear a stop condition + +- Editing a truth file, a rubric threshold, or a prompt after seeing a result. +- Adding a qualification path, symbol, prompt, or repository name to production logic. +- Re-running a failing cell until it passes and reporting the passing run. +- Marking a measured failure as `not_measured`. +- Substituting a different task, target, or prompt for the one that failed. +- Narrowing the corpus so the failing cell is no longer in it. + +Changing the frozen contract is possible, but only by bumping `contract_version`, stating +what changed and why in the pull request, and re-baselining every affected cell. A contract +change never retroactively clears a recorded stop condition. diff --git a/docs/qualification/tasks.json b/docs/qualification/tasks.json new file mode 100644 index 00000000..5cfeb99f --- /dev/null +++ b/docs/qualification/tasks.json @@ -0,0 +1,354 @@ +{ + "contract_version": "1.0.0", + "frozen_at": "2026-08-12", + "prompt_hash_algorithm": "sha256 over the exact UTF-8 prompt text, no trailing newline", + "tasks": [ + { + "id": "arch-unstorage-driver-seam", + "name": "Explain the driver extension seam and mount resolution", + "category": "architecture-understanding", + "target": "unstorage", + "tiers": [ + 1, + 2 + ], + "status": "frozen", + "prompt": { + "text": "Describe this library's extension architecture. What is the stable surface a new backend has to implement, how is a backend selected for a given key at call time, and what work does the core do that a backend never sees?", + "sha256": "5c3722d182b5c36478ecabd2d1fb24f4f3eb40eba6e12032da89bebf12ef1d9f" + }, + "truth_ref": "truth/arch-unstorage-driver-seam.json", + "scoring": { + "tier1_method": "evidence_obligation_recall", + "tier2_method": "blinded_rubric", + "rubric_ref": "rubrics.json#/methods/blinded_rubric" + }, + "validity_requirements": { + "tier1": { + "rationale": "Tier 1 requires the context artifact, prompt text, environment receipt, and truth file because deterministic evaluation must be reproducible without running an agent.", + "requires_context_artifact": true, + "requires_prompt_text": true, + "requires_environment_receipt": true, + "requires_truth_file": true, + "requires_answer_text": false, + "requires_answer_within_prompt_contract": false, + "requires_complete_transcript": false, + "requires_attributable_madar_call": false + }, + "tier2": { + "rationale": "Tier 2 requires the same four artifacts plus the raw transcript, answer text, and attributable Madar call because the agent run and its comparison must be auditable.", + "requires_context_artifact": true, + "requires_prompt_text": true, + "requires_environment_receipt": true, + "requires_truth_file": true, + "requires_answer_text": true, + "requires_answer_within_prompt_contract": true, + "requires_complete_transcript": true, + "requires_attributable_madar_call": true + } + }, + "truth_provenance": { + "authored_by": "madar-655-qualification-agent", + "author_role": "benchmark author", + "authored_at": "2026-08-12", + "derived_from": [ + "unjs/unstorage @ e6be6135832f350ca16f9a77432e1d4f0aa85ed7, read directly from the pinned checkout" + ], + "madar_derived_sources_used": [], + "inspected_madar_output_before_freeze": false, + "independent_of_production_rule_author": false, + "review_status": "unreviewed" + } + }, + { + "id": "flow-hono-request-dispatch", + "name": "Trace the request lifecycle from entry point to response", + "category": "execution-flow-explanation", + "target": "hono", + "tiers": [ + 1, + 2 + ], + "status": "frozen", + "prompt": { + "text": "Trace what happens to an incoming request from the application's entry point until a response is returned. Cover how the path is resolved, how a route is matched, how handlers and middleware are run, and how errors and unmatched routes are handled. Say which steps are skipped in the fast path.", + "sha256": "c276d363f0b10c00b75df925f2e658f3ee7af085872bcbcc64a7a4252c7b3b35" + }, + "truth_ref": "truth/flow-hono-request-dispatch.json", + "scoring": { + "tier1_method": "evidence_obligation_recall", + "tier2_method": "ordered_path_rubric", + "rubric_ref": "rubrics.json#/methods/ordered_path_rubric" + }, + "validity_requirements": { + "tier1": { + "rationale": "Tier 1 requires the context artifact, prompt text, environment receipt, and truth file because deterministic evaluation must be reproducible without running an agent.", + "requires_context_artifact": true, + "requires_prompt_text": true, + "requires_environment_receipt": true, + "requires_truth_file": true, + "requires_answer_text": false, + "requires_answer_within_prompt_contract": false, + "requires_complete_transcript": false, + "requires_attributable_madar_call": false + }, + "tier2": { + "rationale": "Tier 2 requires the same four artifacts plus the raw transcript, answer text, and attributable Madar call because the agent run and its comparison must be auditable.", + "requires_context_artifact": true, + "requires_prompt_text": true, + "requires_environment_receipt": true, + "requires_truth_file": true, + "requires_answer_text": true, + "requires_answer_within_prompt_contract": true, + "requires_complete_transcript": true, + "requires_attributable_madar_call": true + } + }, + "truth_provenance": { + "authored_by": "madar-655-qualification-agent", + "author_role": "benchmark author", + "authored_at": "2026-08-12", + "derived_from": [ + "honojs/hono @ 26de73133b8552f56ba72e025ecd82b08900d796, read directly from the pinned checkout" + ], + "madar_derived_sources_used": [], + "inspected_madar_output_before_freeze": false, + "independent_of_production_rule_author": false, + "review_status": "unreviewed" + } + }, + { + "id": "impact-hono-drop-router-fallback", + "name": "Impact of removing the router fallback strategy", + "category": "impact-analysis", + "target": "hono", + "tiers": [ + 1, + 2 + ], + "status": "frozen", + "prompt": { + "text": "What breaks if the default router is replaced by the regular-expression router alone, with no fallback to another router implementation? List every module whose behaviour changes, say which modules are unaffected and why, and say whether each failure is loud or silent.", + "sha256": "4d05391549ee28a142a9f24960e43e415cf6c831c29dce720aa74e72c7ab9ac2" + }, + "truth_ref": "truth/impact-hono-drop-router-fallback.json", + "scoring": { + "tier1_method": "evidence_obligation_recall", + "tier2_method": "affected_set_precision_recall", + "rubric_ref": "rubrics.json#/methods/affected_set_precision_recall" + }, + "validity_requirements": { + "tier1": { + "rationale": "Tier 1 requires the context artifact, prompt text, environment receipt, and truth file because deterministic evaluation must be reproducible without running an agent.", + "requires_context_artifact": true, + "requires_prompt_text": true, + "requires_environment_receipt": true, + "requires_truth_file": true, + "requires_answer_text": false, + "requires_answer_within_prompt_contract": false, + "requires_complete_transcript": false, + "requires_attributable_madar_call": false + }, + "tier2": { + "rationale": "Tier 2 requires the same four artifacts plus the raw transcript, answer text, and attributable Madar call because the agent run and its comparison must be auditable.", + "requires_context_artifact": true, + "requires_prompt_text": true, + "requires_environment_receipt": true, + "requires_truth_file": true, + "requires_answer_text": true, + "requires_answer_within_prompt_contract": true, + "requires_complete_transcript": true, + "requires_attributable_madar_call": true + } + }, + "truth_provenance": { + "authored_by": "madar-655-qualification-agent", + "author_role": "benchmark author", + "authored_at": "2026-08-12", + "derived_from": [ + "honojs/hono @ 26de73133b8552f56ba72e025ecd82b08900d796, read directly from the pinned checkout" + ], + "madar_derived_sources_used": [], + "inspected_madar_output_before_freeze": false, + "independent_of_production_rule_author": false, + "review_status": "unreviewed" + } + }, + { + "id": "rootcause-hono-middleware-rerun", + "name": "Root-cause downstream middleware running twice", + "category": "bug-root-cause-investigation", + "target": "hono-seeded-compose", + "tiers": [ + 1, + 2 + ], + "status": "frozen", + "prompt": { + "text": "A middleware that awaits the next function twice used to fail loudly with an error. It now silently runs the rest of the chain a second time, so downstream handlers execute twice for one request. Find the root cause and explain the exact mechanism that produces the second execution.", + "sha256": "637cd655712c2e7d96a3566ed0b311d9d7dcfe0d9cfb8b699f5f1c52a721eaf0" + }, + "truth_ref": "truth/rootcause-hono-middleware-rerun.json", + "scoring": { + "tier1_method": "evidence_obligation_recall", + "tier2_method": "single_root_cause_adjudication", + "rubric_ref": "rubrics.json#/methods/single_root_cause_adjudication" + }, + "validity_requirements": { + "tier1": { + "rationale": "Tier 1 requires the context artifact, prompt text, environment receipt, and truth file because deterministic evaluation must be reproducible without running an agent.", + "requires_context_artifact": true, + "requires_prompt_text": true, + "requires_environment_receipt": true, + "requires_truth_file": true, + "requires_answer_text": false, + "requires_answer_within_prompt_contract": false, + "requires_complete_transcript": false, + "requires_attributable_madar_call": false + }, + "tier2": { + "rationale": "Tier 2 requires the same four artifacts plus the raw transcript, answer text, and attributable Madar call because the agent run and its comparison must be auditable.", + "requires_context_artifact": true, + "requires_prompt_text": true, + "requires_environment_receipt": true, + "requires_truth_file": true, + "requires_answer_text": true, + "requires_answer_within_prompt_contract": true, + "requires_complete_transcript": true, + "requires_attributable_madar_call": true + } + }, + "truth_provenance": { + "authored_by": "madar-655-qualification-agent", + "author_role": "benchmark author", + "authored_at": "2026-08-12", + "derived_from": [ + "seeded defect deliberately injected into honojs/hono @ 26de73133b8552f56ba72e025ecd82b08900d796 via patches/hono-compose-reentrancy-guard.patch" + ], + "madar_derived_sources_used": [], + "inspected_madar_output_before_freeze": false, + "independent_of_production_rule_author": false, + "review_status": "unreviewed" + } + }, + { + "id": "plan-unstorage-add-driver", + "name": "Plan a bounded new backend implementation", + "category": "implementation-planning", + "target": "unstorage", + "tiers": [ + 1, + 2 + ], + "status": "frozen", + "prompt": { + "text": "Plan the change needed to add a new built-in storage backend for an S3-compatible object store. Do not widen the public extension interface and do not change modules unrelated to adding a backend. List the files you would add or change, the ones you must not hand-edit, and why for each.", + "sha256": "b8a0a2b4a0593346d1333d96ada6c339a0763611df1ddab51079369e0c62c7ff" + }, + "truth_ref": "truth/plan-unstorage-add-driver.json", + "scoring": { + "tier1_method": "evidence_obligation_recall", + "tier2_method": "blinded_rubric", + "rubric_ref": "rubrics.json#/methods/blinded_rubric", + "hidden_acceptance_test": { + "required": true, + "status": "unavailable", + "reason": "A hidden acceptance test must be authored and held by someone other than the production-rule author. See holdout-policy.md. Until it exists, this task's Tier 2 implementation score is not_measured and only the plan rubric applies." + } + }, + "validity_requirements": { + "tier1": { + "rationale": "Tier 1 requires the context artifact, prompt text, environment receipt, and truth file because deterministic evaluation must be reproducible without running an agent.", + "requires_context_artifact": true, + "requires_prompt_text": true, + "requires_environment_receipt": true, + "requires_truth_file": true, + "requires_answer_text": false, + "requires_answer_within_prompt_contract": false, + "requires_complete_transcript": false, + "requires_attributable_madar_call": false + }, + "tier2": { + "rationale": "Tier 2 requires the same four artifacts plus the raw transcript, answer text, and attributable Madar call because the agent run and its comparison must be auditable.", + "requires_context_artifact": true, + "requires_prompt_text": true, + "requires_environment_receipt": true, + "requires_truth_file": true, + "requires_answer_text": true, + "requires_answer_within_prompt_contract": true, + "requires_complete_transcript": true, + "requires_attributable_madar_call": true + } + }, + "truth_provenance": { + "authored_by": "madar-655-qualification-agent", + "author_role": "benchmark author", + "authored_at": "2026-08-12", + "derived_from": [ + "unjs/unstorage @ e6be6135832f350ca16f9a77432e1d4f0aa85ed7, read directly from the pinned checkout" + ], + "madar_derived_sources_used": [], + "inspected_madar_output_before_freeze": false, + "independent_of_production_rule_author": false, + "review_status": "unreviewed" + } + }, + { + "id": "review-hono-error-handling", + "name": "Review the framework error path for disclosure defects", + "category": "review-security", + "target": "hono-seeded-error-disclosure", + "tiers": [ + 1, + 2 + ], + "status": "frozen", + "prompt": { + "text": "Review this framework's default error-handling path before merge. Identify anything that could expose internal information to a client, name the exact function, say what an attacker learns, and cite the code that proves it.", + "sha256": "722278ed3fa2386f1e407b4fd53a7d4b89272f2adba4297f878ad3fd645d8e47" + }, + "truth_ref": "truth/review-hono-error-handling.json", + "scoring": { + "tier1_method": "evidence_obligation_recall", + "tier2_method": "seeded_defect_detection", + "rubric_ref": "rubrics.json#/methods/seeded_defect_detection" + }, + "validity_requirements": { + "tier1": { + "rationale": "Tier 1 requires the context artifact, prompt text, environment receipt, and truth file because deterministic evaluation must be reproducible without running an agent.", + "requires_context_artifact": true, + "requires_prompt_text": true, + "requires_environment_receipt": true, + "requires_truth_file": true, + "requires_answer_text": false, + "requires_answer_within_prompt_contract": false, + "requires_complete_transcript": false, + "requires_attributable_madar_call": false + }, + "tier2": { + "rationale": "Tier 2 requires the same four artifacts plus the raw transcript, answer text, and attributable Madar call because the agent run and its comparison must be auditable.", + "requires_context_artifact": true, + "requires_prompt_text": true, + "requires_environment_receipt": true, + "requires_truth_file": true, + "requires_answer_text": true, + "requires_answer_within_prompt_contract": true, + "requires_complete_transcript": true, + "requires_attributable_madar_call": true + } + }, + "truth_provenance": { + "authored_by": "madar-655-qualification-agent", + "author_role": "benchmark author", + "authored_at": "2026-08-12", + "derived_from": [ + "seeded defect deliberately injected into honojs/hono @ 26de73133b8552f56ba72e025ecd82b08900d796 via patches/hono-error-message-disclosure.patch" + ], + "madar_derived_sources_used": [], + "inspected_madar_output_before_freeze": false, + "independent_of_production_rule_author": false, + "review_status": "unreviewed" + } + } + ] +} diff --git a/docs/qualification/tier1.json b/docs/qualification/tier1.json new file mode 100644 index 00000000..8fa0c24f --- /dev/null +++ b/docs/qualification/tier1.json @@ -0,0 +1,113 @@ +{ + "contract_version": "1.0.0", + "frozen_at": "2026-08-12", + "purpose": "The small deterministic subset that a pull request can run. It measures whether the evidence needed to answer each frozen task was present in the context artifact, and whether readiness was correctly refused on the negative-trust probes. It never scores answer quality and never runs an agent.", + "properties": { + "deterministic": true, + "requires_network": true, + "requires_network_reason": "Targets are natural external repositories pinned at immutable commits. Preparing a cell means cloning at the pinned SHA and, for the two seeded targets, applying the recorded patch. A local mirror or a warm clone cache satisfies this without changing the result, because the SHA and the patch fix the content exactly.", + "requires_model_provider": false, + "requires_api_spend": false, + "trials_per_cell": 1, + "trials_rationale": "One trial is sufficient because no sampling occurs. If a Tier 1 cell is ever observed to be non-deterministic, that is itself a defect and the cell must be reported as invalid, not retried until it passes." + }, + "preparation": { + "steps": [ + "For each distinct target, clone the pinned repository and check out the recorded ref.", + "For a git_patched target, apply the recorded patch with `git apply` and fail the cell if the patch does not apply cleanly.", + "Verify each target's cited_blobs against the prepared tree with `git rev-parse :` before the cell runs." + ], + "on_preparation_failure": "The cell is invalid with reason `target_revision_mismatch` (or `incomplete_receipt` if verification could not run). It is never silently skipped and never counted as a pass." + }, + "cells": [ + { + "task_id": "arch-unstorage-driver-seam", + "target_id": "unstorage", + "method": "evidence_obligation_recall" + }, + { + "task_id": "flow-hono-request-dispatch", + "target_id": "hono", + "method": "evidence_obligation_recall" + }, + { + "task_id": "impact-hono-drop-router-fallback", + "target_id": "hono", + "method": "evidence_obligation_recall" + }, + { + "task_id": "rootcause-hono-middleware-rerun", + "target_id": "hono-seeded-compose", + "method": "evidence_obligation_recall" + }, + { + "task_id": "plan-unstorage-add-driver", + "target_id": "unstorage", + "method": "evidence_obligation_recall" + }, + { + "task_id": "review-hono-error-handling", + "target_id": "hono-seeded-error-disclosure", + "method": "evidence_obligation_recall" + } + ], + "negative_trust_probes": [ + { + "id": "neg-unstorage-absent-encryption", + "target_id": "unstorage", + "prompt": { + "text": "How does this library encrypt values at rest before writing them through a backend, and where are the keys managed?", + "sha256": "cab0b45defcdcaf0b88c5613384f84b237c1d365e0a1f3b3bbd16543fe6db37d" + }, + "ground_truth": "There is no cryptography anywhere in src/. The only transformation applied on write is stringify, and on read destr. No key management exists.", + "required_behaviour": [ + "The artifact must not report a ready state.", + "The artifact must declare that the requested behaviour was not found rather than presenting stringify, destr, or a driver's own storage as encryption.", + "No fabricated path or symbol may appear in the evidence set." + ], + "failure_is": "false_ready" + }, + { + "id": "neg-hono-absent-matcher-persistence", + "target_id": "hono", + "prompt": { + "text": "Where does this framework persist its compiled route matchers to disk so they survive a process restart, and how is that cache invalidated?", + "sha256": "6a173a57d204d86260380b3f9fedcb4dcc5c90962258e849b508fddaa3e07116" + }, + "ground_truth": "Nothing in the routing path touches the filesystem. The only reuse is in-process: SmartRouter rebinds its match method to the winning router and drops its route buffer after the first match. There is no disk cache and therefore no invalidation.", + "required_behaviour": [ + "The artifact must not report a ready state.", + "The artifact must state that no on-disk matcher cache exists.", + "Presenting the in-memory memoization in SmartRouter as a persistent cache counts as a false-ready failure." + ], + "failure_is": "false_ready" + } + ], + "gate": { + "applies_to": "every pull request that touches retrieval, context building, graph construction, or ranking", + "activation": { + "state": "pre_baseline", + "active": false, + "activation_rule": "The gate activates only once a baseline run exists.", + "activation_event": { + "run_id": null, + "run_url": null, + "date": null + } + }, + "pass_condition": "every cell passes its evidence_obligation_recall threshold AND every negative_trust_probe satisfies its required_behaviour", + "on_failure": "the pull request is blocked; see stop-rule.md", + "forbidden_remedies": [ + "Adding a qualification path, symbol, prompt, or repository name to production retrieval, ranking, or claim logic.", + "Relaxing a truth file to match observed output.", + "Lowering min_critical_fact_recall to make a cell pass.", + "Marking a failing cell not_measured. not_measured is for runs that could not be measured, never for runs that were measured and failed.", + "Replacing a natural target with a self-authored fixture that is easier to satisfy." + ] + }, + "calibration_status": { + "state": "pre_registered_uncalibrated", + "explanation": "The thresholds in the truth files were written from the pinned repository sources before Madar was ever run against them, and the author did not inspect Madar output before freezing. No cell in this subset has a recorded pass or fail yet, so it is unknown how many currently pass. Pre-registering the threshold before calibrating against observed output is the correct order, not a shortfall — calibrating first would make the threshold a description of current behaviour rather than a test of it.", + "consequence": "The first execution of this subset (issue #661) is a measurement, not a regression check. A failing cell on first execution is a product finding to be filed as a linked issue, not a reason to edit this contract." + } +} diff --git a/docs/qualification/tier2-matrix.json b/docs/qualification/tier2-matrix.json new file mode 100644 index 00000000..326b764d --- /dev/null +++ b/docs/qualification/tier2-matrix.json @@ -0,0 +1,60 @@ +{ + "contract_version": "1.0.0", + "frozen_at": "2026-08-12", + "status": "planned", + "status_meaning": "The matrix shape, arms, repeat count, and reporting rules are frozen now so they cannot be chosen after seeing results. No Tier 2 cell has been executed under this contract.", + "blocked_by": [ + "Blinded review is unavailable in a single-author context (rubrics.json#/blinding/current_status).", + "The sealed holdout slot is unsatisfied (holdout-policy.md).", + "No truth file has been reviewed by a second person; every one carries review_status: unreviewed." + ], + "dimensions": { + "targets": [ + "hono", + "unstorage", + "hono-seeded-compose", + "hono-seeded-error-disclosure", + "sealed-holdout-a" + ], + "tasks": [ + "arch-unstorage-driver-seam", + "flow-hono-request-dispatch", + "impact-hono-drop-router-fallback", + "rootcause-hono-middleware-rerun", + "plan-unstorage-add-driver", + "review-hono-error-handling" + ], + "arms": ["native", "madar"], + "cache_modes": ["cold", "warm"], + "trials_per_cell": 5 + }, + "cells": [ + { "task_id": "arch-unstorage-driver-seam", "target_id": "unstorage" }, + { "task_id": "flow-hono-request-dispatch", "target_id": "hono" }, + { "task_id": "impact-hono-drop-router-fallback", "target_id": "hono" }, + { "task_id": "rootcause-hono-middleware-rerun", "target_id": "hono-seeded-compose" }, + { "task_id": "plan-unstorage-add-driver", "target_id": "unstorage" }, + { "task_id": "review-hono-error-handling", "target_id": "hono-seeded-error-disclosure" } + ], + "sealed_holdout_note": "sealed-holdout-a contributes cells through the external sealed manifest, not through this file.", + "trial_rationale": "Five trials per cell is the smallest count that lets a per-cell median be reported with a visible min/max spread while keeping a full sweep affordable. It is not powered for a small effect size. Any claim that depends on a difference smaller than the observed per-cell spread must be reported as not established, whatever the medians say.", + "pairing": "Both arms of a cell run against the same prepared target tree — same commit, same applied patch, same verified blob digests — with the same prompt text, the same tool permission list apart from Madar tools, the same cache mode, and the same trial index. A cell where the two arms differ in any identity field is invalid, not a result.", + "reporting": { + "unit": "one row per target per task per cache mode", + "statistics": ["median", "min", "max", "n_valid", "n_invalid"], + "forbidden": [ + "A blended cross-target or cross-task headline number.", + "Reporting cost or latency for a cell whose gating quality dimensions did not pass.", + "Dropping invalid runs silently; n_invalid is published next to n_valid on every row.", + "Reporting a win for a cell where the Madar arm had adoption status absent." + ] + }, + "ordering_rule": "Quality gates are evaluated first. Cost, token, and latency columns are populated only for cells that already passed correctness and critical-fact completeness. This ordering is part of the contract, not a presentation choice.", + "execution_prerequisites": [ + "Every task in the sweep has a truth file with review_status other than unreviewed.", + "A reviewer who did not author the change under evaluation is available for blinded scoring.", + "Isolation mode is active and the environment receipt matches the pinned contract.", + "Every prepared target tree has had its cited_blobs verified against the pinned ref.", + "The sealed holdout slot is either filled or the sweep is published with the holdout column explicitly marked unsatisfied." + ] +} diff --git a/docs/qualification/truth/arch-unstorage-driver-seam.json b/docs/qualification/truth/arch-unstorage-driver-seam.json new file mode 100644 index 00000000..2279d4ea --- /dev/null +++ b/docs/qualification/truth/arch-unstorage-driver-seam.json @@ -0,0 +1,143 @@ +{ + "contract_version": "1.0.0", + "task_id": "arch-unstorage-driver-seam", + "target": "unstorage", + "category": "architecture-understanding", + "provenance": { + "authored_by": "madar-655-qualification-agent", + "authored_at": "2026-08-12", + "derived_from": [ + "unjs/unstorage @ e6be6135832f350ca16f9a77432e1d4f0aa85ed7, read directly from the pinned checkout" + ], + "madar_derived_sources_used": [], + "inspected_madar_output_before_freeze": false, + "independent_of_production_rule_author": false, + "review_status": "unreviewed" + }, + "critical_facts": [ + { + "id": "driver-interface", + "statement": "The stable extension surface is the Driver interface in src/types.ts. Exactly three members are required — hasItem, getItem, getKeys. Everything else (setItem, setItems, setItemRaw, getItems, getItemRaw, removeItem, getMeta, clear, dispose, watch, name, flags, options, getInstance) is optional.", + "criticality": "critical", + "evidence": [{ "path": "src/types.ts", "symbol": "Driver" }] + }, + { + "id": "driver-factory", + "statement": "Drivers are authored as a DriverFactory — a function taking options and returning a Driver — declared in src/drivers/utils/index.ts and default-exported from each driver module.", + "criticality": "critical", + "evidence": [ + { "path": "src/drivers/utils/index.ts", "symbol": "DriverFactory" }, + { "path": "src/drivers/memory.ts", "symbol": "driver" } + ] + }, + { + "id": "mount-registry", + "statement": "createStorage holds a StorageCTX with mounts (base -> Driver) and mountpoints (the list of bases). The root mount is the empty string and defaults to the in-memory driver when no driver option is supplied.", + "criticality": "critical", + "evidence": [ + { "path": "src/storage.ts", "symbol": "createStorage" }, + { "path": "src/drivers/memory.ts", "symbol": "driver" } + ] + }, + { + "id": "longest-prefix-resolution", + "statement": "getMount resolves a key by scanning mountpoints and returning the first base for which key.startsWith(base). mount() pushes the new base and re-sorts mountpoints by descending length, so the longest matching prefix wins; an unmatched key falls back to the root mount.", + "criticality": "critical", + "evidence": [{ "path": "src/storage.ts", "symbol": "getMount" }] + }, + { + "id": "relative-key-stripping", + "statement": "The core strips the mount base before calling the driver — relativeKey is key.slice(base.length) — so a driver never sees the mountpoint it is mounted under.", + "criticality": "critical", + "evidence": [{ "path": "src/storage.ts", "symbol": "getMount" }] + }, + { + "id": "core-only-work", + "statement": "Work the core does that a driver never sees: key normalization (normalizeKey / normalizeBaseKey / joinKeys in src/utils.ts), value serialization on write (stringify) and deserialization on read (destr), wrapping every driver call in asyncCall, grouping multi-key operations per mount in runBatch, and fanning watch events out to registered listeners through onChange.", + "criticality": "critical", + "evidence": [ + { "path": "src/utils.ts", "symbol": "normalizeKey" }, + { "path": "src/_utils.ts", "symbol": "asyncCall" }, + { "path": "src/storage.ts", "symbol": "runBatch" } + ] + }, + { + "id": "optional-method-fallbacks", + "statement": "Optional driver methods are feature-detected, not required: a driver with no setItem is silently treated as read-only, getItemRaw falls back to getItem plus deserializeRaw, and getItems/setItems fall back to per-item calls.", + "criticality": "critical", + "evidence": [ + { "path": "src/storage.ts", "symbol": "setItem" }, + { "path": "src/storage.ts", "symbol": "getItemRaw" } + ] + }, + { + "id": "generated-driver-index", + "statement": "src/_drivers.ts is generated, not hand-maintained. Its header says \"Auto-generated using scripts/gen-drivers. Do not manually edit!\", and the build script runs gen-drivers before bundling.", + "criticality": "supporting", + "evidence": [ + { "path": "src/_drivers.ts", "symbol": "module header" }, + { "path": "scripts/gen-drivers.ts", "symbol": "driverEntries" } + ] + } + ], + "correct_uncertainty": [ + { + "id": "structural-contract-only", + "requirement": "The Driver contract is enforced only by TypeScript structural typing. Nothing validates a driver object at runtime, which is why a driver missing an optional method degrades silently instead of failing. An answer that presents the contract as runtime-enforced is wrong." + }, + { + "id": "static-hypothesis", + "requirement": "All of this is read from static source. No runtime trace exists, so claims about which driver actually serves a key in a deployed system are hypotheses." + } + ], + "unsupported_claim_traps": [ + { + "id": "exact-mount-match", + "claim": "A key is matched to a mount by exact equality with the mountpoint.", + "why_false": "getMount uses key.startsWith(base) over a list sorted by descending length in mount(), so resolution is longest-prefix, not exact." + }, + { + "id": "readonly-throws", + "claim": "Writing through a driver that does not implement setItem raises an error.", + "why_false": "storage.setItem returns early with a `// Readonly` comment. The write is silently discarded." + }, + { + "id": "core-encrypts", + "claim": "The core encrypts, hashes, or otherwise protects values before handing them to a driver.", + "why_false": "The only transformation on write is stringify; on read it is destr. There is no cryptography anywhere in src/." + }, + { + "id": "hand-edit-driver-index", + "claim": "Registering a new driver means adding an entry to src/_drivers.ts.", + "why_false": "That file is generated by scripts/gen-drivers.ts during the build and is explicitly marked do-not-edit." + }, + { + "id": "root-unmountable", + "claim": "The root mount can be unmounted like any other.", + "why_false": "unmount returns immediately when the normalized base is empty, so the root driver cannot be removed." + } + ], + "tier1_obligations": { + "required_evidence_paths": [ + "src/types.ts", + "src/storage.ts", + "src/drivers/utils/index.ts", + "src/utils.ts" + ], + "required_evidence_symbols": ["Driver", "createStorage", "getMount", "DriverFactory"], + "min_critical_fact_recall": 1.0, + "must_not_report_ready_when": [ + "any required_evidence_path is absent from the evidence set", + "the relationship between createStorage and the Driver interface is neither present in the graph nor declared as unresolved" + ] + }, + "tier2_scoring": { + "method": "blinded_rubric", + "critical_facts_required_for_pass": [ + "driver-interface", + "mount-registry", + "longest-prefix-resolution", + "core-only-work" + ] + } +} diff --git a/docs/qualification/truth/flow-hono-request-dispatch.json b/docs/qualification/truth/flow-hono-request-dispatch.json new file mode 100644 index 00000000..c35f6be2 --- /dev/null +++ b/docs/qualification/truth/flow-hono-request-dispatch.json @@ -0,0 +1,178 @@ +{ + "contract_version": "1.0.0", + "task_id": "flow-hono-request-dispatch", + "target": "hono", + "category": "execution-flow-explanation", + "provenance": { + "authored_by": "madar-655-qualification-agent", + "authored_at": "2026-08-12", + "derived_from": [ + "honojs/hono @ 26de73133b8552f56ba72e025ecd82b08900d796, read directly from the pinned checkout" + ], + "madar_derived_sources_used": [], + "inspected_madar_output_before_freeze": false, + "independent_of_production_rule_author": false, + "review_status": "unreviewed" + }, + "ordered_path": [ + { + "step": 1, + "id": "entrypoint", + "statement": "fetch is the entry point. It forwards to the private #dispatch, passing the execution context and the environment bindings.", + "criticality": "critical", + "evidence": [{ "path": "src/hono-base.ts", "symbol": "Hono.fetch" }] + }, + { + "step": 2, + "id": "head-recursion", + "statement": "A HEAD request is handled by re-entering #dispatch with the method forced to GET and wrapping the result in a new Response with a null body. There is no separate HEAD route matching.", + "criticality": "critical", + "evidence": [{ "path": "src/hono-base.ts", "symbol": "Hono.#dispatch" }] + }, + { + "step": 3, + "id": "path-resolution", + "statement": "The request path is produced by this.getPath, bound at construction to getPath or getPathNoStrict from src/utils/url.ts depending on the strict option.", + "criticality": "critical", + "evidence": [ + { "path": "src/hono-base.ts", "symbol": "Hono.constructor" }, + { "path": "src/utils/url.ts", "symbol": "getPath" } + ] + }, + { + "step": 4, + "id": "route-match", + "statement": "router.match(method, path) returns the match result. With the default SmartRouter the first call replays every buffered route into each candidate router in order, skips any router that throws UnsupportedPathError, then rebinds this.match to the winner and discards the route buffer so later requests go straight to that router.", + "criticality": "critical", + "evidence": [ + { "path": "src/router/smart-router/router.ts", "symbol": "SmartRouter.match" }, + { "path": "src/hono.ts", "symbol": "Hono.constructor" } + ] + }, + { + "step": 5, + "id": "context-construction", + "statement": "A Context is constructed after matching, receiving the resolved path, the match result, the environment, the execution context, and the not-found handler.", + "criticality": "critical", + "evidence": [{ "path": "src/context.ts", "symbol": "Context" }] + }, + { + "step": 6, + "id": "fast-path", + "statement": "When exactly one handler matched, compose is skipped entirely. The handler is invoked directly with a next that assigns the not-found handler's response to c.res, and the promise/synchronous result is normalized inline.", + "criticality": "critical", + "evidence": [{ "path": "src/hono-base.ts", "symbol": "Hono.#dispatch" }] + }, + { + "step": 7, + "id": "compose-chain", + "statement": "With two or more matched handlers, compose builds a recursive dispatch loop. Handler i receives a next closure that calls dispatch(i + 1), and context.req.routeIndex is set to the current index before each handler runs.", + "criticality": "critical", + "evidence": [{ "path": "src/compose.ts", "symbol": "compose" }] + }, + { + "step": 8, + "id": "not-found", + "statement": "Inside the composed chain, when no handler remains and context.finalized is still false, the not-found handler runs and its response becomes the result. The framework default returns the text '404 Not Found' with status 404.", + "criticality": "critical", + "evidence": [ + { "path": "src/compose.ts", "symbol": "compose" }, + { "path": "src/hono-base.ts", "symbol": "notFoundHandler" } + ] + }, + { + "step": 9, + "id": "error-path", + "statement": "A thrown Error inside the composed chain is caught by dispatch, recorded on context.error, and passed to the app error handler; its response is applied even though the context may already be finalized. Outside the chain, #handleError re-throws anything that is not an Error and otherwise delegates to the same handler.", + "criticality": "critical", + "evidence": [ + { "path": "src/compose.ts", "symbol": "compose" }, + { "path": "src/hono-base.ts", "symbol": "Hono.#handleError" } + ] + }, + { + "step": 10, + "id": "finalization-check", + "statement": "After the composed chain resolves, #dispatch throws 'Context is not finalized' if context.finalized is false. That throw is caught by the surrounding try and routed through #handleError, so a forgotten return surfaces as a handled error rather than a hang.", + "criticality": "critical", + "evidence": [{ "path": "src/hono-base.ts", "symbol": "Hono.#dispatch" }] + }, + { + "step": 11, + "id": "response", + "statement": "The response returned is context.res.", + "criticality": "supporting", + "evidence": [{ "path": "src/context.ts", "symbol": "Context" }] + } + ], + "correct_uncertainty": [ + { + "id": "router-not-statically-determined", + "requirement": "Which concrete router serves a match is not decidable from the source alone. SmartRouter selects the first router that does not reject the application's actual route set, at the first match call, and then memoizes it. An answer that states the regular-expression router is always used is over-claiming." + }, + { + "id": "static-hypothesis", + "requirement": "The path is reconstructed from static call sites, not from an observed runtime trace." + } + ], + "unsupported_claim_traps": [ + { + "id": "always-compose", + "claim": "Every request is processed through the middleware composition function.", + "why_false": "#dispatch takes a fast path when the match result contains exactly one handler and never calls compose." + }, + { + "id": "head-separate-route", + "claim": "HEAD requests are matched against their own routes.", + "why_false": "#dispatch recurses with the method rewritten to GET and returns a Response with a null body built from the GET result." + }, + { + "id": "smart-router-per-request", + "claim": "The router strategy is re-evaluated on every request.", + "why_false": "SmartRouter.match rebinds this.match to the winning router and sets its route buffer to undefined after the first successful match." + }, + { + "id": "all-throws-become-500", + "claim": "Anything thrown by a handler is converted into a 500 response.", + "why_false": "#handleError re-throws values that are not Error instances; only Error instances reach the error handler." + }, + { + "id": "env-third-argument", + "claim": "fetch takes the request, the execution context, then the environment.", + "why_false": "fetch is (request, env, executionCtx) and forwards them to #dispatch as (request, rest[1], rest[0], method) — the environment is the second argument." + } + ], + "tier1_obligations": { + "required_evidence_paths": [ + "src/hono-base.ts", + "src/compose.ts", + "src/context.ts", + "src/router/smart-router/router.ts", + "src/utils/url.ts" + ], + "required_evidence_symbols": ["fetch", "compose", "Context", "SmartRouter", "getPath"], + "min_critical_fact_recall": 1.0, + "must_not_report_ready_when": [ + "any required_evidence_path is absent from the evidence set", + "the call from the dispatch entry point into compose is neither present in the graph nor declared as unresolved" + ] + }, + "tier2_scoring": { + "method": "ordered_path_rubric", + "order_sensitive_pairs": [ + ["path-resolution", "route-match"], + ["route-match", "context-construction"], + ["compose-chain", "not-found"], + ["compose-chain", "finalization-check"] + ], + "critical_facts_required_for_pass": [ + "entrypoint", + "path-resolution", + "route-match", + "context-construction", + "fast-path", + "compose-chain", + "error-path" + ] + } +} diff --git a/docs/qualification/truth/impact-hono-drop-router-fallback.json b/docs/qualification/truth/impact-hono-drop-router-fallback.json new file mode 100644 index 00000000..fcf17161 --- /dev/null +++ b/docs/qualification/truth/impact-hono-drop-router-fallback.json @@ -0,0 +1,140 @@ +{ + "contract_version": "1.0.0", + "task_id": "impact-hono-drop-router-fallback", + "target": "hono", + "category": "impact-analysis", + "provenance": { + "authored_by": "madar-655-qualification-agent", + "authored_at": "2026-08-12", + "derived_from": [ + "honojs/hono @ 26de73133b8552f56ba72e025ecd82b08900d796, read directly from the pinned checkout" + ], + "madar_derived_sources_used": [], + "inspected_madar_output_before_freeze": false, + "independent_of_production_rule_author": false, + "review_status": "unreviewed" + }, + "change_under_analysis": "In src/hono.ts, replace `new SmartRouter({ routers: [new RegExpRouter(), new TrieRouter()] })` with a bare `new RegExpRouter()`.", + "affected_set": [ + { + "id": "construction-site", + "path": "src/hono.ts", + "symbols": ["Hono.constructor"], + "effect": "The change site. The default router becomes a single implementation with no alternative.", + "failure_mode": "n/a", + "criticality": "critical" + }, + { + "id": "smart-router-unused", + "path": "src/router/smart-router/router.ts", + "symbols": ["SmartRouter.match", "SmartRouter.activeRouter"], + "effect": "The catch-and-continue loop over UnsupportedPathError, the memoizing rebind of this.match, and the activeRouter accessor stop being exercised by the default path.", + "failure_mode": "silent", + "criticality": "critical" + }, + { + "id": "regexp-router-throws-escape", + "path": "src/router/reg-exp-router/router.ts", + "symbols": ["RegExpRouter.add", "RegExpRouter.#insertPath"], + "effect": "UnsupportedPathError, raised by #insertPath for a path the trie cannot express, no longer has anything to catch it and propagates to the caller.", + "failure_mode": "loud", + "criticality": "critical" + }, + { + "id": "trie-router-unreachable", + "path": "src/router/trie-router/router.ts", + "symbols": ["TrieRouter"], + "effect": "Stops being reachable as the default fallback. It remains importable and usable if an application passes it explicitly via the router option.", + "failure_mode": "silent", + "criticality": "critical" + } + ], + "timing_fact": { + "statement": "The failure also moves in time. Under SmartRouter, route registration only buffers routes and the adds are replayed inside the first match call, so an unsupported path surfaces on the first request. With a bare RegExpRouter, #addRoute calls router.add immediately, so #insertPath runs during route registration and the error surfaces while the application is being defined.", + "criticality": "critical", + "evidence": [ + { "path": "src/hono-base.ts", "symbol": "Hono.#addRoute" }, + { "path": "src/router/smart-router/router.ts", "symbol": "SmartRouter.add" }, + { "path": "src/router/reg-exp-router/router.ts", "symbol": "RegExpRouter.add" } + ] + }, + "unaffected_set": [ + { + "path": "src/hono-base.ts", + "reason": "#dispatch depends only on the Router interface, not on any implementation. Its own logic is unchanged — though see timing_fact for when #addRoute now raises." + }, + { + "path": "src/compose.ts", + "reason": "Composition runs after matching and never inspects the router." + }, + { + "path": "src/context.ts", + "reason": "The Context receives a match result; the identity of the router that produced it is irrelevant." + }, + { + "path": "src/router.ts", + "reason": "The Router interface and the UnsupportedPathError class are unchanged; only who catches the error changes." + }, + { + "path": "src/request.ts", + "reason": "Request parameter access reads from the match result and is independent of the matcher implementation." + } + ], + "correct_uncertainty": [ + { + "id": "which-paths-are-unsupported", + "requirement": "Which concrete route patterns RegExpRouter rejects is not determined by the modules under analysis — it depends on the trie insertion rules in the reg-exp-router package. A correct answer says that some patterns are rejected and points at #insertPath, without inventing a specific list of rejected patterns it did not read." + } + ], + "unsupported_claim_traps": [ + { + "id": "silent-404", + "claim": "Unsupported route patterns would quietly stop matching and return 404.", + "why_false": "UnsupportedPathError is thrown, not swallowed. Nothing converts it into a not-found response." + }, + { + "id": "handled-as-500", + "claim": "The error would be caught and returned as a 500 by the framework error handler.", + "why_false": "router.match in #dispatch is not inside the try block that guards handler execution, and with a bare RegExpRouter the throw happens during route registration, before any request exists." + }, + { + "id": "performance-only", + "claim": "The change only affects matching performance.", + "why_false": "It removes the only mechanism that recovers from an unsupported path, converting a recoverable condition into a hard failure." + }, + { + "id": "compose-affected", + "claim": "Middleware composition changes because the router changed.", + "why_false": "compose consumes the match result and has no dependency on the router implementation." + } + ], + "tier1_obligations": { + "required_evidence_paths": [ + "src/hono.ts", + "src/router/smart-router/router.ts", + "src/router/reg-exp-router/router.ts", + "src/router/trie-router/router.ts" + ], + "required_evidence_symbols": ["SmartRouter", "RegExpRouter", "TrieRouter", "UnsupportedPathError"], + "min_critical_fact_recall": 1.0, + "must_not_report_ready_when": [ + "the relationship between the constructor in src/hono.ts and the three router implementations is missing from the evidence set and is not declared as unresolved" + ] + }, + "tier2_scoring": { + "method": "affected_set_precision_recall", + "recall_denominator": [ + "construction-site", + "smart-router-unused", + "regexp-router-throws-escape", + "trie-router-unreachable" + ], + "precision_penalty_set": [ + "src/compose.ts", + "src/context.ts", + "src/request.ts", + "src/router.ts" + ], + "critical_facts_required_for_pass": ["regexp-router-throws-escape", "trie-router-unreachable"] + } +} diff --git a/docs/qualification/truth/plan-unstorage-add-driver.json b/docs/qualification/truth/plan-unstorage-add-driver.json new file mode 100644 index 00000000..a295e520 --- /dev/null +++ b/docs/qualification/truth/plan-unstorage-add-driver.json @@ -0,0 +1,146 @@ +{ + "contract_version": "1.0.0", + "task_id": "plan-unstorage-add-driver", + "target": "unstorage", + "category": "implementation-planning", + "provenance": { + "authored_by": "madar-655-qualification-agent", + "authored_at": "2026-08-12", + "derived_from": [ + "unjs/unstorage @ e6be6135832f350ca16f9a77432e1d4f0aa85ed7, read directly from the pinned checkout" + ], + "madar_derived_sources_used": [], + "inspected_madar_output_before_freeze": false, + "independent_of_production_rule_author": false, + "review_status": "unreviewed" + }, + "reference_plan": { + "files_to_add": [ + { + "new_path": "src/drivers/.ts", + "why": "One module per driver. Default-export a DriverFactory from ./utils/index.ts, implement the three required members (hasItem, getItem, getKeys) and whichever optional members the backend supports, and export a named options type so the generator can pick it up." + } + ], + "files_to_change": [], + "files_that_must_not_be_hand_edited": [ + { + "path": "src/_drivers.ts", + "why": "Generated by scripts/gen-drivers.ts, which enumerates src/drivers and re-derives the option type imports. The build script runs gen-drivers before bundling, so a hand edit is overwritten." + } + ], + "files_that_must_not_change": [ + { + "path": "src/types.ts", + "why": "Adding a backend-specific member to Driver widens the public extension interface, which the prompt forbids and which would ripple to all 34 existing drivers." + }, + { + "path": "src/storage.ts", + "why": "Mounting and key resolution are driver-agnostic; createStorage needs no knowledge of a new backend." + }, + { + "path": "package.json", + "why": "The exports map already publishes ./drivers/* as a wildcard subpath, so a new driver module is exported without an edit." + } + ], + "conventions_to_follow": [ + "Validate required options with createRequiredError / createError from src/drivers/utils/index.ts so failures carry the [unstorage] [] prefix.", + "Set a DRIVER_NAME constant and expose it as the driver's name, as src/drivers/memory.ts does.", + "Implement dispose when the backend holds a client or timers, because storage.dispose fans out to every mounted driver.", + "Leave setItem off only if the backend is genuinely read-only, and know that the core will then discard writes silently." + ] + }, + "critical_facts": [ + { + "id": "one-module-per-driver", + "statement": "A driver is a single module under src/drivers that default-exports a DriverFactory.", + "criticality": "critical", + "evidence": [ + { "path": "src/drivers/utils/index.ts", "symbol": "DriverFactory" }, + { "path": "src/drivers/memory.ts", "symbol": "driver" } + ] + }, + { + "id": "generated-index", + "statement": "src/_drivers.ts must not be hand-edited; scripts/gen-drivers.ts regenerates it from the contents of src/drivers as part of the build.", + "criticality": "critical", + "evidence": [ + { "path": "src/_drivers.ts", "symbol": "module header" }, + { "path": "scripts/gen-drivers.ts", "symbol": "driverEntries" } + ] + }, + { + "id": "no-interface-widening", + "statement": "The Driver interface in src/types.ts must not change; the three required members plus the existing optional members are sufficient for a new backend.", + "criticality": "critical", + "evidence": [{ "path": "src/types.ts", "symbol": "Driver" }] + }, + { + "id": "wildcard-export", + "statement": "package.json already exposes ./drivers/* as a wildcard subpath export, so publishing the new module requires no packaging change.", + "criticality": "critical", + "evidence": [{ "path": "package.json", "symbol": "exports" }] + }, + { + "id": "option-validation-helpers", + "statement": "Option validation uses createError and createRequiredError from src/drivers/utils/index.ts.", + "criticality": "supporting", + "evidence": [{ "path": "src/drivers/utils/index.ts", "symbol": "createRequiredError" }] + } + ], + "correct_uncertainty": [ + { + "id": "no-s3-client-present", + "requirement": "No S3 or object-store client exists in the repository. A plan must either introduce one explicitly as a new dependency, following how other network-backed drivers declare theirs, or state that the transport is left abstract. Assuming an SDK is already available is unsupported." + }, + { + "id": "generator-reads-type-exports", + "requirement": "The generator discovers the options type by scanning the module's exported type names. A plan that never mentions exporting a named options type leaves the generated entry incomplete, but the exact naming rule is decided by scripts/gen-drivers.ts and should not be invented." + } + ], + "unsupported_claim_traps": [ + { + "id": "edit-driver-index", + "claim": "Add an import and an entry to src/_drivers.ts so the driver is registered.", + "why_false": "That file is generated and explicitly marked do-not-edit; the build regenerates it." + }, + { + "id": "add-exports-entry", + "claim": "Add a new subpath to the exports map in package.json.", + "why_false": "./drivers/* is already a wildcard export." + }, + { + "id": "extend-driver-interface", + "claim": "Add object-store-specific options or methods to the Driver interface.", + "why_false": "That widens the extension surface, which the prompt forbids." + }, + { + "id": "register-in-storage", + "claim": "createStorage or the mount logic must learn about the new driver.", + "why_false": "Drivers are passed in by the caller through the driver option or mount; the core never enumerates driver types." + } + ], + "tier1_obligations": { + "required_evidence_paths": [ + "src/types.ts", + "src/drivers/utils/index.ts", + "src/drivers/memory.ts", + "src/_drivers.ts" + ], + "required_evidence_symbols": ["Driver", "DriverFactory", "createRequiredError"], + "min_critical_fact_recall": 1.0, + "must_not_report_ready_when": [ + "src/drivers/utils/index.ts is absent from the evidence set", + "the generated nature of src/_drivers.ts is neither represented nor declared as unresolved" + ] + }, + "tier2_scoring": { + "method": "blinded_rubric", + "critical_facts_required_for_pass": ["one-module-per-driver", "generated-index", "no-interface-widening"], + "hidden_acceptance_test": { + "required": true, + "status": "unavailable", + "blocking_reason": "An executable hidden acceptance test must be authored and held by a person other than the production-rule author. In the current single-author context it cannot be produced credibly. Until it exists the bounded-implementation score for this task is not_measured and only the plan rubric is scored.", + "human_action_required": "A second maintainer authors an acceptance test asserting that a new driver can be added by adding exactly one module under src/drivers and running the generator, with no edit to src/types.ts, src/storage.ts, src/_drivers.ts, or package.json, and stores it outside this repository per holdout-policy.md." + } + } +} diff --git a/docs/qualification/truth/review-hono-error-handling.json b/docs/qualification/truth/review-hono-error-handling.json new file mode 100644 index 00000000..25277e22 --- /dev/null +++ b/docs/qualification/truth/review-hono-error-handling.json @@ -0,0 +1,118 @@ +{ + "contract_version": "1.0.0", + "task_id": "review-hono-error-handling", + "target": "hono-seeded-error-disclosure", + "category": "review-security", + "provenance": { + "authored_by": "madar-655-qualification-agent", + "authored_at": "2026-08-12", + "derived_from": [ + "seeded defect deliberately injected into honojs/hono @ 26de73133b8552f56ba72e025ecd82b08900d796 via patches/hono-error-message-disclosure.patch" + ], + "madar_derived_sources_used": [], + "inspected_madar_output_before_freeze": false, + "independent_of_production_rule_author": false, + "review_status": "unreviewed" + }, + "seeded_defect": { + "id": "seeded-error-message-disclosure", + "path": "src/hono-base.ts", + "symbol": "errorHandler", + "patch": "patches/hono-error-message-disclosure.patch", + "description": "The framework's default error handler builds the 500 response body from the thrown error's stack, falling back to its message, instead of returning a fixed 'Internal Server Error' string.", + "attacker_capability": "Any request that triggers an unhandled error returns a stack trace to the client. That discloses absolute filesystem paths, the internal module layout, dependency names and versions, and often the failing query, identifier, or credential fragment embedded in the error message. It turns every unhandled error into reconnaissance.", + "proving_evidence": [ + { "path": "src/hono-base.ts", "symbol": "errorHandler", "note": "the patched line builds the body from err.stack ?? err.message" }, + { "path": "src/hono-base.ts", "symbol": "Hono.#handleError", "note": "the non-composed path routes every Error into this same handler" }, + { "path": "src/compose.ts", "symbol": "compose", "note": "inside the composed chain the handler's return value is assigned to context.res even when the context was already finalized" } + ], + "reachability": "Reachable from both dispatch paths: #handleError for the fast path and for post-chain failures, and the onError branch inside compose for errors raised by middleware or handlers." + }, + "critical_facts": [ + { + "id": "default-handler-leaks", + "statement": "The default error handler returns internal error detail in the response body.", + "criticality": "critical", + "evidence": [{ "path": "src/hono-base.ts", "symbol": "errorHandler" }] + }, + { + "id": "reachable-from-both-paths", + "statement": "Both the composed chain's onError branch and #handleError converge on the same default handler, so the disclosure is not confined to one route shape.", + "criticality": "critical", + "evidence": [ + { "path": "src/compose.ts", "symbol": "compose" }, + { "path": "src/hono-base.ts", "symbol": "Hono.#handleError" } + ] + }, + { + "id": "fix-is-fixed-string", + "statement": "The fix is to return a constant message with status 500 and keep the detail in the server-side log that the handler already writes.", + "criticality": "critical", + "evidence": [{ "path": "src/hono-base.ts", "symbol": "errorHandler" }] + } + ], + "acceptable_additional_findings": [ + { + "id": "override-dependency", + "statement": "An application that installs its own onError replaces this handler, so real-world exposure depends on the application not overriding the default.", + "scored_as": "supporting, not required" + }, + { + "id": "server-side-logging", + "statement": "The handler also writes the full error to the server log. That is appropriate, not a client-facing disclosure.", + "scored_as": "supporting, not required" + }, + { + "id": "finalized-override", + "statement": "In compose, the error handler's response is applied even when the context was already finalized, so a partially written response can be replaced by the error body.", + "scored_as": "supporting, not required" + } + ], + "correct_uncertainty": [ + { + "id": "message-content-unknown", + "requirement": "What a stack trace actually reveals depends on the deployment — bundling, source maps, and working directory. A correct answer states the class of information disclosed rather than asserting that specific secrets are always present." + } + ], + "unsupported_claim_traps": [ + { + "id": "http-exception-leaks", + "claim": "The HTTPException branch is the disclosure, because it returns the exception's message to the client.", + "why_false": "HTTPException.getResponse builds its response from a message the application author supplied deliberately. That is intended behaviour and is unchanged from the upstream code.", + "counts_as": "false_positive" + }, + { + "id": "not-found-leaks", + "claim": "The not-found handler discloses the requested path.", + "why_false": "The default not-found handler returns the fixed string '404 Not Found' with status 404 and echoes nothing from the request.", + "counts_as": "false_positive" + }, + { + "id": "compose-is-the-defect", + "claim": "compose is the defect because it passes the raw error onward.", + "why_false": "compose hands the error to whatever onError is configured. Passing the error to a handler is correct; the disclosure is in what the default handler does with it.", + "counts_as": "false_positive" + }, + { + "id": "rethrow-is-a-leak", + "claim": "#handleError re-throwing non-Error values leaks internals to the client.", + "why_false": "A re-thrown non-Error propagates out of fetch to the host runtime; it does not construct a response body.", + "counts_as": "false_positive" + } + ], + "tier1_obligations": { + "required_evidence_paths": ["src/hono-base.ts", "src/compose.ts", "src/http-exception.ts"], + "required_evidence_symbols": ["errorHandler", "compose", "HTTPException"], + "min_critical_fact_recall": 1.0, + "must_not_report_ready_when": [ + "src/hono-base.ts is absent from the evidence set", + "only one of the two error paths is present, since the reachability claim needs both" + ] + }, + "tier2_scoring": { + "method": "seeded_defect_detection", + "required_detections": ["seeded-error-message-disclosure"], + "false_positive_set": ["http-exception-leaks", "not-found-leaks", "compose-is-the-defect", "rethrow-is-a-leak"], + "critical_facts_required_for_pass": ["default-handler-leaks", "reachable-from-both-paths"] + } +} diff --git a/docs/qualification/truth/rootcause-hono-middleware-rerun.json b/docs/qualification/truth/rootcause-hono-middleware-rerun.json new file mode 100644 index 00000000..32773afe --- /dev/null +++ b/docs/qualification/truth/rootcause-hono-middleware-rerun.json @@ -0,0 +1,99 @@ +{ + "contract_version": "1.0.0", + "task_id": "rootcause-hono-middleware-rerun", + "target": "hono-seeded-compose", + "category": "bug-root-cause-investigation", + "provenance": { + "authored_by": "madar-655-qualification-agent", + "authored_at": "2026-08-12", + "derived_from": [ + "seeded defect deliberately injected into honojs/hono @ 26de73133b8552f56ba72e025ecd82b08900d796 via patches/hono-compose-reentrancy-guard.patch" + ], + "madar_derived_sources_used": [], + "inspected_madar_output_before_freeze": false, + "independent_of_production_rule_author": false, + "review_status": "unreviewed" + }, + "seeded_defect": { + "id": "seeded-compose-reentrancy-guard", + "path": "src/compose.ts", + "symbol": "compose.dispatch", + "patch": "patches/hono-compose-reentrancy-guard.patch", + "root_cause": "The re-entrancy guard at the top of dispatch was weakened from `i <= index` to `i < index`. `index` holds the highest index dispatched so far, so a second call to next() from the same middleware re-enters dispatch with the same argument i+1 while index already equals i+1. Under `<=` that comparison is true and the guard throws; under `<` it is false and the guard is bypassed, so the entire downstream chain runs again.", + "required_ordering_statement": "index is assigned immediately after the guard, so the guard must reject an index that is less than OR equal to the last dispatched index. Restoring `<=` restores the single-entry invariant.", + "observable_condition": "Only reachable when two or more handlers match, because the single-handler fast path in #dispatch never calls compose." + }, + "critical_facts": [ + { + "id": "guard-is-the-cause", + "statement": "The cause is the comparison in the re-entrancy guard inside compose's dispatch function, not the middleware that calls next twice.", + "criticality": "critical", + "evidence": [{ "path": "src/compose.ts", "symbol": "compose.dispatch" }] + }, + { + "id": "index-semantics", + "statement": "index tracks the highest index already dispatched and is written on every entry, so an equal index is exactly the repeated-next case the guard exists to reject.", + "criticality": "critical", + "evidence": [{ "path": "src/compose.ts", "symbol": "compose" }] + }, + { + "id": "next-closure", + "statement": "Each handler is invoked as handler(context, () => dispatch(i + 1)), so calling next twice issues two dispatch calls with the identical argument.", + "criticality": "critical", + "evidence": [{ "path": "src/compose.ts", "symbol": "compose.dispatch" }] + }, + { + "id": "fast-path-exclusion", + "statement": "The single-matched-handler fast path in #dispatch bypasses compose entirely, so the defect cannot manifest on routes with only one handler.", + "criticality": "supporting", + "evidence": [{ "path": "src/hono-base.ts", "symbol": "Hono.#dispatch" }] + } + ], + "correct_uncertainty": [ + { + "id": "no-other-protection", + "requirement": "The guard is the only re-entrancy protection in the composition chain. There is no per-request lock, no idempotency key, and no downstream check. A correct answer must not suggest that some other layer would have caught the second run." + }, + { + "id": "response-assignment-nuance", + "requirement": "Whether the duplicated run is visible in the response depends on context.finalized: the composed result is only assigned when finalized is false or an error was handled. The duplicated side effects still happen either way. An answer that concludes 'no visible effect' because the response is unchanged has missed the defect." + } + ], + "unsupported_claim_traps": [ + { + "id": "middleware-is-buggy", + "claim": "The application middleware is at fault for calling next twice.", + "why_false": "Calling next twice is exactly the condition the framework guard is written to reject with 'next() called multiple times'. The framework silently permitting it is the regression." + }, + { + "id": "finalized-flag", + "claim": "The cause is the context.finalized flag not being set.", + "why_false": "finalized governs whether the composed response is assigned. It never gated entry into dispatch and does not control chain re-entry." + }, + { + "id": "fast-path-cause", + "claim": "The fast path in #dispatch runs the handler an extra time.", + "why_false": "The fast path invokes exactly one handler and is only taken when the match result has length one; it does not enter compose at all." + }, + { + "id": "error-handler-retry", + "claim": "The error handler retries the chain after a failure.", + "why_false": "onError produces a response; it never re-invokes dispatch." + } + ], + "tier1_obligations": { + "required_evidence_paths": ["src/compose.ts", "src/hono-base.ts"], + "required_evidence_symbols": ["compose", "dispatch"], + "min_critical_fact_recall": 1.0, + "must_not_report_ready_when": [ + "src/compose.ts is absent from the evidence set", + "the call from the dispatch entry point into compose is missing from the graph" + ] + }, + "tier2_scoring": { + "method": "single_root_cause_adjudication", + "accepted_root_cause_ids": ["seeded-compose-reentrancy-guard"], + "critical_facts_required_for_pass": ["guard-is-the-cause", "index-semantics", "next-closure"], + "uncertainty_required_for_pass": ["no-other-protection"] + } +} diff --git a/docs/qualification/validity-rules.md b/docs/qualification/validity-rules.md new file mode 100644 index 00000000..82ad205b --- /dev/null +++ b/docs/qualification/validity-rules.md @@ -0,0 +1,98 @@ +# Run validity and invalidation rules + +Contract version `1.0.0`, frozen 2026-08-12 for [#655](https://github.com/mohanagy/madar/issues/655). + +A qualification run is either **valid**, **degraded**, or **invalid**. Only a valid run +carries a result. An invalid run is not a loss and not a win — it is a run that did not +happen in a measurable way. + +## Invalidation conditions + +Any one of these sets `validity.status` to `invalid` and adds the matching reason code +to `validity.invalidation_reasons`: + +| Reason code | Condition | +| --- | --- | +| `missing_attributable_madar_call` | The task contract sets `requires_attributable_madar_call` and the transcript shows no attributable Madar call in the Madar arm. | +| `prompt_contract_failure` | The prompt actually delivered to the agent does not hash-match the frozen prompt, or the two arms received different prompts. | +| `answer_contract_failure` | The arm produced no answer, a permission request instead of an answer, or a truncated answer. | +| `target_revision_mismatch` | The checked-out target commit differs from `corpus.json`, or a cited blob digest in the prepared tree does not match the recorded `cited_blobs` entry. | +| `patch_application_failure` | A seeded-defect target's patch did not apply cleanly to the pinned commit, or applied with fuzz. | +| `package_revision_mismatch` | The Madar commit, package version, or tarball digest differs from the pinned identity. | +| `dependency_lock_mismatch` | The dependency lock digest differs from the pinned identity, or the install used `npm install` rather than `npm ci`. | +| `isolation_failure` | `environment.isolation` is false, or the run used a `MADAR_BENCH_CLI_PATH`-style development override. | +| `incomplete_transcript` | The transcript is missing, truncated, or cannot attribute tool calls. | +| `incomplete_receipt` | Any required field in [`receipt-schema.json`](./receipt-schema.json) is absent. | +| `judge_failure` | A deterministic grader errored, or a blinded reviewer could not score the answer. | +| `environment_mismatch` | `environment.drift.detected` is true and the drift was not resolved before the cell ran. | +| `quality_gate_failure` | A gate failed in a way that prevents comparison at all — not a gate the arm simply lost. | +| `truth_unavailable` | The target/task pair has no independent truth — for example a target added to the manifest before its truth file exists, or the unsatisfied sealed-holdout slot. | +| `blinding_unavailable` | A Tier 2 quality dimension was scored without an independent blinded reviewer. | + +`degraded` is reserved for runs that are attributable and complete but weaker than the +contract intends — for example a Madar arm whose first attributable call came only after +broad exploration (`adoption.status: "late"`). A degraded run may be inspected and +discussed; it may not be aggregated. + +## The `not_measured` rule + +1. `validity.aggregatable` **must** be `false` whenever `validity.status` is not `valid`. + The receipt schema enforces this. +2. Every unmeasured score carries `measured: false`, `value: null`, and a + `not_measured_reason`. A score of `0` and a score of `not_measured` are different + things and must never be interchanged. +3. `not_measured` describes a run that could not be measured. A run that **was** measured + and failed is a failure. Relabelling a failure as `not_measured` is a contract + violation, not a reporting choice. +4. Invalid rows stay visible. Every published table prints `n_invalid` beside `n_valid`. + A table that shows only valid rows is not a permitted summary of this corpus. +5. Cost, token, and latency figures from an invalid run may be retained for diagnosis and + must never be cited as a cost or efficiency result. + +## Gate ordering + +Correctness and critical-fact completeness are evaluated before any token, latency, or +cost column is populated. A cost improvement on a cell that failed a quality gate is not +reported as an improvement in any form. + +## Cost separation + +`costs.indexing`, `costs.context_build`, and `costs.agent` are three separate accounts. +They are never summed into a single number, and an unmeasured account is +`measured: false`, never `0`. + +## Retention + +For every run, whether valid or not, the tier-specific artifacts below are retained +alongside the receipt for at least **24 months**. The execution artifact is +the raw agent transcript (Tier 2) or the context artifact (Tier 1). + +- Tier 1 retains the context artifact, exact prompt text, environment receipt, and truth + file because deterministic evaluation must be reproducible without running an agent. It + does not require an agent answer, a raw transcript, or an attributable Madar call. +- Tier 2 retains everything Tier 1 retains, plus the raw agent transcript and the answer + text of both arms, because the agent run and its comparison must be auditable. Where the + task requires an attributable Madar call, the transcript must establish it. + +Each retained artifact is recorded in `retention` with its path and SHA-256. A run whose +artifacts were not retained is `incomplete_receipt`. + +## What today's emitter actually produces + +This schema is a contract, not a description of `v0.32.1` behaviour. Mapping against +`NativeAgentCompareReport` in `src/infrastructure/compare.ts` at the pinned commit: + +| Schema area | Status at `06b373a4` | +| --- | --- | +| `validity.status` | Partially present as `measurement_validity` (`valid` / `degraded` / `invalid`). | +| `validity.invalidation_reasons` | **Not emitted.** Reasons exist only as prose in `benchmark_outcome.evidence`. | +| `validity.aggregatable` | **Not emitted.** | +| `adoption.*` | Partially present as `madar_mcp_call_count` and `trace_status`; there is no `adopted`/`late`/`absent` classification field and no post-first-call broad-exploration counter. | +| `costs.agent` | Present, spread across `reductions`, `prompt_token_source`, and `provider_proof`. | +| `costs.indexing`, `costs.context_build` | **Not emitted.** There is no separate indexing or context-build cost account anywhere in the report. | +| `scores.*` | **Not emitted** in this shape. `answer_quality` carries term-presence checks and a human-review status only. | +| `identity.*` | Partially present via `environment`, `exec_command`, and the isolation launcher; there is no single identity block and no dependency-lock digest. | +| `retention.*` | Paths are emitted in `paths`; digests and a retention policy are **not**. | + +Closing that gap is emitter work and is deliberately out of scope for #655, which must not +modify production or reporting logic. It is a separate linked issue. diff --git a/package.json b/package.json index ef632477..7a048807 100644 --- a/package.json +++ b/package.json @@ -66,7 +66,8 @@ "publish:dry-run": "npm publish --dry-run", "release:verify": "node .github/scripts/verify-release-hygiene.mjs", "verify:pack-parity": "node .github/scripts/verify-packed-retrieval-parity.mjs", - "registry:validate": "node .github/scripts/validate-mcp-registry.mjs" + "registry:validate": "node .github/scripts/validate-mcp-registry.mjs", + "qualify:validate": "node .github/scripts/validate-qualification-contract.mjs" }, "devDependencies": { "@types/node": "^26.0.0", diff --git a/tests/unit/qualification-contract.test.ts b/tests/unit/qualification-contract.test.ts new file mode 100644 index 00000000..8c1963fe --- /dev/null +++ b/tests/unit/qualification-contract.test.ts @@ -0,0 +1,852 @@ +import { spawnSync } from 'node:child_process' +import { createHash } from 'node:crypto' +import { cpSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { createRequire } from 'node:module' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' + +import { Ajv } from 'ajv' +import { describe, expect, it } from 'vitest' + +// ajv-formats ships CommonJS whose default export is not callable under NodeNext type +// resolution. Load it the same way the CI validator does so the test and +// `npm run qualify:validate` compile the schema identically. +const addFormats = createRequire(import.meta.url)('ajv-formats') as (ajv: Ajv) => void + +// The cited-path traversal is the shipped validator's own, not a copy of it. It is loaded +// through createRequire for the same reason as ajv-formats: it is CommonJS living under +// .github/, which tsconfig does not include. +const { collectCitedPaths } = createRequire(import.meta.url)( + '../../.github/scripts/lib/collect-cited-paths.cjs', +) as { collectCitedPaths: (node: unknown) => Set } + +const ROOT = 'docs/qualification' + +/** + * Reads a contract document for *semantic* assertions. Line endings are + * normalized so the assertions test content rather than checkout representation: + * `.gitattributes` pins this tree to LF, but a test that only passes because of a + * checkout setting is testing the setting, not the document. + * + * The byte-exact freeze contract is deliberately NOT read through here — it uses + * the raw Buffer below, because that guarantee is about bytes. + */ +function readDoc(relativePath: string): string { + return readFileSync(resolve(relativePath), 'utf8').replace(/\r\n/g, '\n') +} + +/** Raw bytes, for the freeze digest contract only. Never normalized. */ +function readBytes(relativePath: string): Buffer { + return readFileSync(resolve(relativePath)) +} + +function readJson(relativePath: string): T { + return JSON.parse(readDoc(relativePath)) as T +} + +interface Provenance { + authored_by: string + authored_at: string + derived_from: string[] + madar_derived_sources_used: string[] + inspected_madar_output_before_freeze: boolean + independent_of_production_rule_author: boolean +} + +interface Target { + id: string + kind: string + natural?: boolean + status: string + license?: string + holdout_class?: string + prepare?: string[] + patch?: string + base_target?: string + source?: { url: string; ref: string } + cited_blobs?: Record + production_coupling?: { level: string; consequence?: string } +} + +interface Task { + id: string + category: string + target: string + tiers: number[] + prompt: { text: string; sha256: string } + truth_ref: string + scoring: { tier1_method: string; tier2_method: string } + truth_provenance: Provenance +} + +const corpus = readJson<{ + contract_version: string + targets: Target[] + proxy_targets: unknown[] + forbidden_target_symbols: Record +}>(`${ROOT}/corpus.json`) + +const tasks = readJson<{ contract_version: string; tasks: Task[] }>(`${ROOT}/tasks.json`) +const rubrics = readJson<{ + dimensions: Record + methods: Record + blinding: { current_status: string } +}>(`${ROOT}/rubrics.json`) +const tier1 = readJson<{ + properties: { deterministic: boolean; requires_model_provider: boolean; requires_api_spend: boolean } + preparation: { steps: string[]; on_preparation_failure: string } + cells: Array<{ task_id: string; target_id: string }> + negative_trust_probes: Array<{ id: string; target_id: string; prompt: { text: string; sha256: string } }> + gate: { forbidden_remedies: string[] } + calibration_status: { state: string } +}>(`${ROOT}/tier1.json`) +const tier2 = readJson<{ status: string; dimensions: { trials_per_cell: number } }>(`${ROOT}/tier2-matrix.json`) +const receiptSchema = readJson>(`${ROOT}/receipt-schema.json`) +const freeze = readJson<{ contract_version: string; files: Record }>(`${ROOT}/freeze.json`) + +const evaluationTargets = corpus.targets.filter((target) => target.kind !== 'sealed') + +describe('qualification corpus manifest', () => { + it('uses only natural externally authored targets, with no fixture proxies', () => { + expect(evaluationTargets.length).toBeGreaterThan(0) + expect(corpus.proxy_targets).toEqual([]) + + for (const target of evaluationTargets) { + expect(target.natural).toBe(true) + expect(target.kind === 'git' || target.kind === 'git_patched').toBe(true) + } + }) + + it('pins every target at an immutable commit with a license and prepare steps', () => { + for (const target of evaluationTargets) { + expect(target.source?.ref).toMatch(/^[0-9a-f]{40}$/) + expect(target.source?.url).toMatch(/^https:\/\//) + expect(target.license).toBeTruthy() + expect(target.prepare?.length).toBeGreaterThan(0) + expect(target.status).toBe('frozen') + } + }) + + it('records a frozen blob digest for every path its truth may cite', () => { + for (const target of evaluationTargets) { + const blobs = Object.entries(target.cited_blobs ?? {}) + + expect(blobs.length).toBeGreaterThan(0) + for (const [, blob] of blobs) { + expect(blob).toMatch(/^[0-9a-f]{40}$/) + } + } + }) + + it('seeds defects as patches against the pinned commit of a real repository', () => { + const patched = corpus.targets.filter((target) => target.kind === 'git_patched') + + expect(patched.length).toBeGreaterThan(0) + for (const target of patched) { + const base = corpus.targets.find((candidate) => candidate.id === target.base_target) + expect(base?.source?.ref).toBe(target.source?.ref) + + const patch = readDoc(`${ROOT}/${target.patch}`) + expect(patch.startsWith('diff --git ')).toBe(true) + + const touched = [...patch.matchAll(/^\+\+\+ b\/(.+)$/gm)].map((match) => match[1]) + expect(touched.length).toBeGreaterThan(0) + for (const path of touched) { + expect(Object.keys(target.cited_blobs ?? {})).toContain(path) + } + } + }) + + it('rejects an empty seeded-defect patch', () => { + const sandbox = mkdtempSync(join(tmpdir(), 'madar-qualification-contract-')) + + try { + mkdirSync(resolve(sandbox, 'docs'), { recursive: true }) + cpSync(resolve(ROOT), resolve(sandbox, ROOT), { recursive: true }) + mkdirSync(resolve(sandbox, 'src')) + writeFileSync(resolve(sandbox, `${ROOT}/patches/hono-compose-reentrancy-guard.patch`), '') + + const result = spawnSync( + process.execPath, + [resolve('.github/scripts/validate-qualification-contract.mjs'), '--write'], + { cwd: sandbox, encoding: 'utf8' }, + ) + + expect(result.status).toBe(1) + expect(result.stderr).toContain('patch patches/hono-compose-reentrancy-guard.patch is not a unified git diff') + expect(result.stderr).toContain('patch patches/hono-compose-reentrancy-guard.patch does not modify any file') + } finally { + rmSync(sandbox, { recursive: true, force: true }) + } + }) + + it('discloses where a target overlaps a shipped framework adapter', () => { + const hono = corpus.targets.find((target) => target.id === 'hono') + const unstorage = corpus.targets.find((target) => target.id === 'unstorage') + + expect(hono?.production_coupling?.level).toBe('declared_framework_adapter') + expect(hono?.production_coupling?.consequence).toContain('not evidence about frameworks that have no adapter') + expect(unstorage?.production_coupling?.level).toBe('none_found') + }) + + it('keeps the forbidden-symbol map free of prose that would poison the guard', () => { + // A non-array value here folds a whole documentation string into the literal list, which + // starts failing production files the moment that string is shortened to something common. + for (const [key, value] of Object.entries(corpus.forbidden_target_symbols)) { + if (key.startsWith('_')) { + expect(typeof value).toBe('string') + continue + } + + expect(Array.isArray(value)).toBe(true) + expect(corpus.targets.some((target) => target.id === key)).toBe(true) + for (const symbol of value as string[]) { + expect(symbol).toMatch(/^[A-Za-z_$][A-Za-z0-9_$]*$/) + } + } + }) + + it('keeps the sealed holdout slot visible and explicitly unsatisfied', () => { + const sealed = corpus.targets.filter((target) => target.holdout_class === 'sealed') + + expect(sealed).toHaveLength(1) + expect(sealed[0]?.status).toBe('unsatisfied') + }) +}) + +describe('qualification task definitions', () => { + it('covers every task category named in the issue contract', () => { + const categories = new Set(tasks.tasks.map((task) => task.category)) + + expect([...categories].sort()).toEqual([ + 'architecture-understanding', + 'bug-root-cause-investigation', + 'execution-flow-explanation', + 'impact-analysis', + 'implementation-planning', + 'review-security', + ]) + }) + + it('freezes each prompt against its recorded hash', () => { + for (const task of tasks.tasks) { + expect(createHash('sha256').update(task.prompt.text, 'utf8').digest('hex')).toBe(task.prompt.sha256) + } + }) + + it('reports a task with no prompt instead of throwing', () => { + const sandbox = mkdtempSync(join(tmpdir(), 'madar-qualification-contract-')) + + try { + mkdirSync(resolve(sandbox, 'docs'), { recursive: true }) + cpSync(resolve(ROOT), resolve(sandbox, ROOT), { recursive: true }) + mkdirSync(resolve(sandbox, 'src')) + + const mutated = JSON.parse(JSON.stringify(tasks)) as { + tasks: Array<{ id: string; prompt?: unknown }> + } + const task = mutated.tasks.find((candidate) => candidate.id === 'rootcause-hono-middleware-rerun')! + delete task.prompt + writeFileSync(resolve(sandbox, `${ROOT}/tasks.json`), `${JSON.stringify(mutated, null, 2)}\n`) + + const result = spawnSync( + process.execPath, + [resolve('.github/scripts/validate-qualification-contract.mjs'), '--write'], + { cwd: sandbox, encoding: 'utf8' }, + ) + + expect(result.status).toBe(1) + expect(result.stderr).toContain('task rootcause-hono-middleware-rerun must declare prompt text and sha256') + } finally { + rmSync(sandbox, { recursive: true, force: true }) + } + }) + + it('reports a task with no scoring block instead of throwing', () => { + const sandbox = mkdtempSync(join(tmpdir(), 'madar-qualification-contract-')) + + try { + mkdirSync(resolve(sandbox, 'docs'), { recursive: true }) + cpSync(resolve(ROOT), resolve(sandbox, ROOT), { recursive: true }) + mkdirSync(resolve(sandbox, 'src')) + + const mutated = JSON.parse(JSON.stringify(tasks)) as { + tasks: Array<{ id: string; scoring?: unknown }> + } + const task = mutated.tasks.find((candidate) => candidate.id === 'rootcause-hono-middleware-rerun')! + delete task.scoring + writeFileSync(resolve(sandbox, `${ROOT}/tasks.json`), `${JSON.stringify(mutated, null, 2)}\n`) + + const result = spawnSync( + process.execPath, + [resolve('.github/scripts/validate-qualification-contract.mjs'), '--write'], + { cwd: sandbox, encoding: 'utf8' }, + ) + + expect(result.status).toBe(1) + expect(result.stderr).toContain('task rootcause-hono-middleware-rerun must declare tier1 and tier2 scoring methods') + } finally { + rmSync(sandbox, { recursive: true, force: true }) + } + }) + + it('never names the coupled framework inside a prompt for that target', () => { + const coupled = tasks.tasks.filter((task) => task.target.startsWith('hono')) + + expect(coupled.length).toBeGreaterThan(0) + for (const task of coupled) { + expect(task.prompt.text.toLowerCase()).not.toContain('hono') + } + }) + + it('records a truth owner, a real derivation source, and no Madar-derived source', () => { + for (const task of tasks.tasks) { + const truth = readJson<{ provenance: Provenance }>(`${ROOT}/${task.truth_ref}`) + + for (const provenance of [task.truth_provenance, truth.provenance]) { + expect(provenance.authored_by.length).toBeGreaterThan(0) + expect(provenance.authored_at.length).toBeGreaterThan(0) + expect(provenance.derived_from.length).toBeGreaterThan(0) + expect(provenance.madar_derived_sources_used).toEqual([]) + expect(provenance.inspected_madar_output_before_freeze).toBe(false) + } + } + }) + + it('snapshots the current single-author independence state', () => { + for (const task of tasks.tasks) { + const truth = readJson<{ provenance: Provenance }>(`${ROOT}/${task.truth_ref}`) + for (const provenance of [task.truth_provenance, truth.provenance]) { + // Closing the independence gap requires updating this current-state expectation in + // the same change that records the second author's review. + expect(provenance.independent_of_production_rule_author).toBe(false) + } + } + }) + + it('cites only paths recorded in the target blob manifest', () => { + for (const task of tasks.tasks) { + const truth = readJson>(`${ROOT}/${task.truth_ref}`) + const target = corpus.targets.find((candidate) => candidate.id === task.target) + // Shared with the shipped validator rather than reimplemented, so the `new_path` + // exemption cannot drift between the guard and the test that covers it. + const cited = collectCitedPaths(truth) + + expect(cited.size).toBeGreaterThan(0) + for (const path of cited) { + expect(Object.keys(target?.cited_blobs ?? {})).toContain(path) + } + } + }) + + it('does not use the same scoring method for every category', () => { + const methods = new Set(tasks.tasks.map((task) => task.scoring.tier2_method)) + + expect(methods.size).toBeGreaterThan(1) + for (const task of tasks.tasks) { + expect(rubrics.methods[task.scoring.tier2_method]).toBeTruthy() + expect(rubrics.methods[task.scoring.tier1_method]).toBeTruthy() + } + }) +}) + +describe('qualification rubrics', () => { + it('measures adoption and fallback exploration separately from context quality', () => { + expect(rubrics.dimensions.intended_tool_adoption?.gating).toBe(false) + expect(rubrics.dimensions.broad_fallback_exploration?.gating).toBe(false) + expect(rubrics.dimensions.correctness?.gating).toBe(true) + expect(rubrics.dimensions.critical_fact_completeness?.gating).toBe(true) + }) + + it('declares blinded review unavailable rather than assuming it', () => { + expect(rubrics.blinding.current_status).toBe('unsatisfied') + }) +}) + +describe('qualification receipt schema', () => { + const ajv = new Ajv({ allErrors: true, strict: false }) + addFormats(ajv) + const validate = ajv.compile(receiptSchema) + + const validTier1 = readJson>(`${ROOT}/examples/receipt-tier1-valid.json`) + const invalidTier2 = readJson>(`${ROOT}/examples/receipt-tier2-invalid-no-madar-call.json`) + + it('accepts the published examples and ties them to frozen prompts', () => { + expect(validate(validTier1)).toBe(true) + expect(validate(invalidTier2)).toBe(true) + + for (const receipt of [validTier1, invalidTier2] as Array<{ + task_id: string + identity: { prompts: { user_prompt_sha256: string } } + }>) { + const task = tasks.tasks.find((candidate) => candidate.id === receipt.task_id) + expect(task).toBeTruthy() + expect(receipt.identity.prompts.user_prompt_sha256).toBe(task?.prompt.sha256) + } + }) + + it('keeps every quality dimension not_measured on an invalid run', () => { + const scores = (invalidTier2 as { scores: Record }).scores + const validity = (invalidTier2 as { validity: { status: string; aggregatable: boolean } }).validity + + expect(validity.status).toBe('invalid') + expect(validity.aggregatable).toBe(false) + for (const score of Object.values(scores)) { + expect(score.measured).toBe(false) + expect(score.value).toBeNull() + } + }) + + it('rejects an invalid run that claims to be aggregatable', () => { + const mutated = JSON.parse(JSON.stringify(invalidTier2)) as { validity: { aggregatable: boolean } } + mutated.validity.aggregatable = true + + expect(validate(mutated)).toBe(false) + }) + + it('rejects an unmeasured score that carries a value', () => { + const mutated = JSON.parse(JSON.stringify(invalidTier2)) as { + scores: { correctness: { measured: boolean; value: unknown } } + } + mutated.scores.correctness.value = 2 + + expect(validate(mutated)).toBe(false) + }) + + it('can invalidate a run whose seeded patch did not apply', () => { + const reasons = ( + receiptSchema as { + properties: { + validity: { properties: { invalidation_reasons: { items: { enum: string[] } } } } + } + } + ).properties.validity.properties.invalidation_reasons.items.enum + + expect(reasons).toContain('patch_application_failure') + expect(reasons).toContain('target_revision_mismatch') + }) + + it('keeps indexing, context building, and agent cost in separate accounts', () => { + const costs = (validTier1 as { costs: Record }).costs + + expect(Object.keys(costs).sort()).toEqual(['agent', 'context_build', 'indexing']) + expect(costs.agent?.measured).toBe(false) + }) + + it('rejects a measured implementation score when the task requires a hidden acceptance test', () => { + const sandbox = mkdtempSync(join(tmpdir(), 'madar-qualification-contract-')) + + try { + mkdirSync(resolve(sandbox, 'docs'), { recursive: true }) + cpSync(resolve(ROOT), resolve(sandbox, ROOT), { recursive: true }) + mkdirSync(resolve(sandbox, 'src')) + const freezePath = resolve(sandbox, `${ROOT}/freeze.json`) + const freezeBefore = readFileSync(freezePath, 'utf8') + + const task = tasks.tasks.find((candidate) => candidate.id === 'plan-unstorage-add-driver') + const mutated = JSON.parse(JSON.stringify(invalidTier2)) as { + task_id: string + target_id: string + identity: { prompts: { user_prompt_sha256: string; user_prompt_text: string } } + scores: Record + } + mutated.task_id = task!.id + mutated.target_id = task!.target + mutated.identity.prompts.user_prompt_sha256 = task!.prompt.sha256 + mutated.identity.prompts.user_prompt_text = task!.prompt.text + mutated.scores.implementation = { + measured: true, + value: 1, + method: 'hidden_acceptance_test', + } + writeFileSync( + resolve(sandbox, `${ROOT}/examples/receipt-tier2-invalid-no-madar-call.json`), + `${JSON.stringify(mutated, null, 2)}\n`, + ) + + const result = spawnSync( + process.execPath, + [resolve('.github/scripts/validate-qualification-contract.mjs'), '--write'], + { cwd: sandbox, encoding: 'utf8' }, + ) + + expect(result.status).toBe(1) + expect(readFileSync(freezePath, 'utf8')).toBe(freezeBefore) + expect(result.stderr).toContain( + 'docs/qualification/examples/receipt-tier2-invalid-no-madar-call.json Tier 2 task ' + + 'plan-unstorage-add-driver requires scores.implementation to be not_measured ' + + '(measured false, value null, with a not_measured_reason)', + ) + } finally { + rmSync(sandbox, { recursive: true, force: true }) + } + }) +}) + +describe('qualification Tier 1 subset', () => { + it('is deterministic and runnable in a pull request without model spend', () => { + expect(tier1.properties.deterministic).toBe(true) + expect(tier1.properties.requires_model_provider).toBe(false) + expect(tier1.properties.requires_api_spend).toBe(false) + }) + + it('reports a missing Tier 1 cell array instead of throwing', () => { + const sandbox = mkdtempSync(join(tmpdir(), 'madar-qualification-contract-')) + + try { + mkdirSync(resolve(sandbox, 'docs'), { recursive: true }) + cpSync(resolve(ROOT), resolve(sandbox, ROOT), { recursive: true }) + mkdirSync(resolve(sandbox, 'src')) + + const mutated = JSON.parse(JSON.stringify(tier1)) as { cells: unknown } + mutated.cells = null + writeFileSync(resolve(sandbox, `${ROOT}/tier1.json`), `${JSON.stringify(mutated, null, 2)}\n`) + + const result = spawnSync( + process.execPath, + [resolve('.github/scripts/validate-qualification-contract.mjs'), '--write'], + { cwd: sandbox, encoding: 'utf8' }, + ) + + expect(result.status).toBe(1) + expect(result.stderr).toContain( + 'tier1.json#/cells is missing pair ' + + '{"task_id":"arch-unstorage-driver-seam","target_id":"unstorage"} ' + + 'present in tier2-matrix.json#/cells', + ) + } finally { + rmSync(sandbox, { recursive: true, force: true }) + } + }) + + it('rejects a missing Tier 1 gate activation block', () => { + const sandbox = mkdtempSync(join(tmpdir(), 'madar-qualification-contract-')) + + try { + mkdirSync(resolve(sandbox, 'docs'), { recursive: true }) + cpSync(resolve(ROOT), resolve(sandbox, ROOT), { recursive: true }) + mkdirSync(resolve(sandbox, 'src')) + + const mutated = JSON.parse(JSON.stringify(tier1)) as { gate: { activation?: unknown } } + delete mutated.gate.activation + writeFileSync(resolve(sandbox, `${ROOT}/tier1.json`), `${JSON.stringify(mutated, null, 2)}\n`) + + const result = spawnSync( + process.execPath, + [resolve('.github/scripts/validate-qualification-contract.mjs'), '--write'], + { cwd: sandbox, encoding: 'utf8' }, + ) + + expect(result.status).toBe(1) + expect(result.stderr).toContain('tier1 gate.activation block must exist') + } finally { + rmSync(sandbox, { recursive: true, force: true }) + } + }) + + it('rejects a non-boolean Tier 1 gate activation flag', () => { + const sandbox = mkdtempSync(join(tmpdir(), 'madar-qualification-contract-')) + + try { + mkdirSync(resolve(sandbox, 'docs'), { recursive: true }) + cpSync(resolve(ROOT), resolve(sandbox, ROOT), { recursive: true }) + mkdirSync(resolve(sandbox, 'src')) + + const mutated = JSON.parse(JSON.stringify(tier1)) as { + gate: { activation: { active: unknown } } + } + mutated.gate.activation.active = 'false' + writeFileSync(resolve(sandbox, `${ROOT}/tier1.json`), `${JSON.stringify(mutated, null, 2)}\n`) + + const result = spawnSync( + process.execPath, + [resolve('.github/scripts/validate-qualification-contract.mjs'), '--write'], + { cwd: sandbox, encoding: 'utf8' }, + ) + + expect(result.status).toBe(1) + expect(result.stderr).toContain('tier1 gate.activation.active must be a boolean') + } finally { + rmSync(sandbox, { recursive: true, force: true }) + } + }) + + it('rejects an active Tier 1 gate that does not name its baseline activation event', () => { + const sandbox = mkdtempSync(join(tmpdir(), 'madar-qualification-contract-')) + + try { + mkdirSync(resolve(sandbox, 'docs'), { recursive: true }) + cpSync(resolve(ROOT), resolve(sandbox, ROOT), { recursive: true }) + mkdirSync(resolve(sandbox, 'src')) + + const mutated = JSON.parse(JSON.stringify(tier1)) as { + gate: { + activation: { + active: boolean + state: string + activation_event: { run_id: string | null; run_url: string | null; date: string | null } + } + } + } + mutated.gate.activation.active = true + mutated.gate.activation.state = 'active' + writeFileSync(resolve(sandbox, `${ROOT}/tier1.json`), `${JSON.stringify(mutated, null, 2)}\n`) + + const result = spawnSync( + process.execPath, + [resolve('.github/scripts/validate-qualification-contract.mjs'), '--write'], + { cwd: sandbox, encoding: 'utf8' }, + ) + + expect(result.status).toBe(1) + expect(result.stderr).toContain( + 'tier1 gate activation is active but activation_event must name the baseline with non-null ' + + 'run_id, run_url, and date', + ) + } finally { + rmSync(sandbox, { recursive: true, force: true }) + } + }) + + it('rejects an active Tier 1 gate in the pre_baseline state', () => { + const sandbox = mkdtempSync(join(tmpdir(), 'madar-qualification-contract-')) + + try { + mkdirSync(resolve(sandbox, 'docs'), { recursive: true }) + cpSync(resolve(ROOT), resolve(sandbox, ROOT), { recursive: true }) + mkdirSync(resolve(sandbox, 'src')) + + const mutated = JSON.parse(JSON.stringify(tier1)) as { + gate: { + activation: { + active: boolean + activation_event: { run_id: string | null; run_url: string | null; date: string | null } + } + } + } + mutated.gate.activation.active = true + mutated.gate.activation.activation_event = { + run_id: 'baseline-0001', + run_url: 'https://example.invalid/runs/baseline-0001', + date: '2026-08-14', + } + writeFileSync(resolve(sandbox, `${ROOT}/tier1.json`), `${JSON.stringify(mutated, null, 2)}\n`) + + const result = spawnSync( + process.execPath, + [resolve('.github/scripts/validate-qualification-contract.mjs'), '--write'], + { cwd: sandbox, encoding: 'utf8' }, + ) + + expect(result.status).toBe(1) + expect(result.stderr).toContain('tier1 active gate activation must declare a non-pre_baseline state') + } finally { + rmSync(sandbox, { recursive: true, force: true }) + } + }) + + it('rejects an inactive Tier 1 gate outside the pre_baseline state', () => { + const sandbox = mkdtempSync(join(tmpdir(), 'madar-qualification-contract-')) + + try { + mkdirSync(resolve(sandbox, 'docs'), { recursive: true }) + cpSync(resolve(ROOT), resolve(sandbox, ROOT), { recursive: true }) + mkdirSync(resolve(sandbox, 'src')) + + const mutated = JSON.parse(JSON.stringify(tier1)) as { + gate: { activation: { active: boolean; state: string } } + } + mutated.gate.activation.state = 'active' + writeFileSync(resolve(sandbox, `${ROOT}/tier1.json`), `${JSON.stringify(mutated, null, 2)}\n`) + + const result = spawnSync( + process.execPath, + [resolve('.github/scripts/validate-qualification-contract.mjs'), '--write'], + { cwd: sandbox, encoding: 'utf8' }, + ) + + expect(result.status).toBe(1) + expect(result.stderr).toContain('tier1 inactive gate activation must have state "pre_baseline"') + } finally { + rmSync(sandbox, { recursive: true, force: true }) + } + }) + + it('fails a cell whose target could not be prepared instead of skipping it', () => { + expect(tier1.preparation.steps.length).toBeGreaterThan(0) + expect(tier1.preparation.on_preparation_failure).toContain('never silently skipped') + }) + + it('covers every frozen task and freezes each negative-trust probe prompt', () => { + expect(tier1.cells.map((cell) => cell.task_id).sort()).toEqual(tasks.tasks.map((task) => task.id).sort()) + + expect(tier1.negative_trust_probes.length).toBeGreaterThan(0) + for (const probe of tier1.negative_trust_probes) { + expect(createHash('sha256').update(probe.prompt.text, 'utf8').digest('hex')).toBe(probe.prompt.sha256) + expect(corpus.targets.some((target) => target.id === probe.target_id)).toBe(true) + } + }) + + it('forbids clearing a failure by editing the contract or swapping in a fixture', () => { + const remedies = tier1.gate.forbidden_remedies.join('\n') + + expect(remedies).toContain('Adding a qualification path, symbol, prompt, or repository name to production') + expect(remedies).toContain('Relaxing a truth file to match observed output') + expect(remedies).toContain('Marking a failing cell not_measured') + expect(remedies).toContain('Replacing a natural target with a self-authored fixture') + }) + + it('states that the thresholds are pre-registered and uncalibrated', () => { + expect(tier1.calibration_status.state).toBe('pre_registered_uncalibrated') + }) + + it('keeps the Tier 2 matrix planned with a repeated-run count', () => { + expect(tier2.status).toBe('planned') + expect(tier2.dimensions.trials_per_cell).toBeGreaterThan(1) + }) + + it('rejects Tier 2 cells that drift from the Tier 1 task-target pairs', () => { + const sandbox = mkdtempSync(join(tmpdir(), 'madar-qualification-contract-')) + + try { + mkdirSync(resolve(sandbox, 'docs'), { recursive: true }) + cpSync(resolve(ROOT), resolve(sandbox, ROOT), { recursive: true }) + mkdirSync(resolve(sandbox, 'src')) + + const mutated = JSON.parse(JSON.stringify(tier2)) as { + cells: Array<{ task_id: string; target_id: string }> + } + mutated.cells[0]!.target_id = 'hono' + writeFileSync(resolve(sandbox, `${ROOT}/tier2-matrix.json`), `${JSON.stringify(mutated, null, 2)}\n`) + + const result = spawnSync( + process.execPath, + [resolve('.github/scripts/validate-qualification-contract.mjs'), '--write'], + { cwd: sandbox, encoding: 'utf8' }, + ) + + expect(result.status).toBe(1) + expect(result.stderr).toContain( + 'tier2-matrix.json#/cells is missing pair ' + + '{"task_id":"arch-unstorage-driver-seam","target_id":"unstorage"} ' + + 'present in tier1.json#/cells', + ) + expect(result.stderr).toContain( + 'tier1.json#/cells is missing pair ' + + '{"task_id":"arch-unstorage-driver-seam","target_id":"hono"} ' + + 'present in tier2-matrix.json#/cells', + ) + } finally { + rmSync(sandbox, { recursive: true, force: true }) + } + }) +}) + +describe('qualification policy documents', () => { + it('states the no-measured-evidence limitation prominently at the top of the README', () => { + const readme = readDoc(`${ROOT}/README.md`) + const heading = '## Read this first — what this contract does and does not give you today' + + expect(readme).toContain(heading) + // "Prominent" is load-bearing: the limitation must appear before the corpus is described, + // not be inferable only by cross-referencing target entries further down. + expect(readme.indexOf(heading)).toBeLessThan(readme.indexOf('## Why a separate corpus exists')) + expect(readme).toContain('This contract has never been executed. It currently produces no measured evidence of') + expect(readme).toContain('Regression only, never generalization.') + expect(readme).toContain('Thresholds are pre-registered, not calibrated.') + expect(readme).toContain('**Tier 1 needs network access**') + }) + + it('states an objective stop rule with a pre-registered non-inferiority margin', () => { + const stopRule = readDoc(`${ROOT}/stop-rule.md`) + + for (const id of ['S1.1', 'S1.2', 'S1.3', 'S1.4', 'S1.5', 'S1.6', 'S1.7', 'S1.8']) { + expect(stopRule).toContain(id) + } + expect(stopRule).toContain('non-inferiority margin **0.05**') + expect(stopRule).toContain('Do not fix forward on the protected branch while a stop condition is tripped.') + expect(stopRule).toContain('Marking a measured failure as `not_measured`.') + }) + + it('declares the sealed holdout unsatisfied and names the human action required', () => { + const policy = readDoc(`${ROOT}/holdout-policy.md`) + + expect(policy).toContain('## Current status: unsatisfied') + expect(policy).toContain('### Human action required') + expect(policy).toContain('sealed holdout unsatisfied; results measure regression only') + expect(policy).toContain('Naturalness and hiddenness are separate properties') + }) + + it('separates target naturalness from evidence class', () => { + const categories = readDoc(`${ROOT}/evidence-categories.md`) + + expect(categories).toContain('## Target naturalness qualifies the evidence') + expect(categories).toContain('### E1 — Product outcome evidence') + expect(categories).toContain('**Currently held: none.**') + expect(categories).toContain('E4 proves the reporting pipeline works. It is never agent-outcome evidence.') + expect(categories).toContain('five are in-repo proxies') + expect(categories).toContain('six are git-backed and\ndo pin a URL together with an immutable commit SHA') + }) + + it('records the unenforced retrieval/grader boundary in runtime-proof.json', () => { + const categories = readDoc(`${ROOT}/evidence-categories.md`) + + expect(categories).toContain('Open enforcement gap in E3') + expect(categories).toContain('That isolation is asserted in\nprose. No test, lint rule, or CI check enforces it') + }) + + it('defines transcript and receipt retention', () => { + const rules = readDoc(`${ROOT}/validity-rules.md`) + + expect(rules).toContain('at least **24 months**') + expect(rules).toContain('the raw agent transcript (Tier 2) or the context artifact (Tier 1)') + expect(rules).toContain('`not_measured` describes a run that could not be measured') + expect(rules).toContain('`patch_application_failure`') + }) + + it('records which receipt fields v0.32.1 does not emit yet', () => { + const rules = readDoc(`${ROOT}/validity-rules.md`) + + expect(rules).toContain('## What today\'s emitter actually produces') + expect(rules).toContain('There is no separate indexing or context-build cost account') + }) +}) + +describe('qualification freeze', () => { + it('covers every contract file with a digest', () => { + expect(freeze.contract_version).toBe(corpus.contract_version) + + const paths = Object.keys(freeze.files) + expect(paths).toContain(`${ROOT}/corpus.json`) + expect(paths).toContain(`${ROOT}/tasks.json`) + expect(paths).toContain(`${ROOT}/rubrics.json`) + expect(paths).toContain(`${ROOT}/receipt-schema.json`) + expect(paths).toContain(`${ROOT}/patches/hono-compose-reentrancy-guard.patch`) + expect(paths).toContain(`${ROOT}/patches/hono-error-message-disclosure.patch`) + + // Raw bytes on purpose. If a checkout converts line endings, this must fail + // rather than be normalized into passing — that is the whole point of the freeze. + for (const [path, digest] of Object.entries(freeze.files)) { + expect(digest).toBe(createHash('sha256').update(readBytes(path)).digest('hex')) + } + }) + + it('pins the contract tree to LF so the byte-exact freeze survives a Windows checkout', () => { + const attributes = readDoc('.gitattributes') + + expect(attributes).toContain('docs/qualification/** text eol=lf') + expect(attributes).toContain('docs/qualification/patches/*.patch -text') + }) + + it('holds no CRLF in any frozen file, whatever the checkout did', () => { + for (const path of Object.keys(freeze.files)) { + expect(readBytes(path).includes('\r\n')).toBe(false) + } + }) + + it('is wired into an npm script so a clean checkout can verify it', () => { + const pkg = readJson<{ scripts: Record }>('package.json') + + expect(pkg.scripts['qualify:validate']).toBe('node .github/scripts/validate-qualification-contract.mjs') + }) +})