diff --git a/.github/workflows/bench-live.yml b/.github/workflows/bench-live.yml index 1a2d3af..2f98d58 100644 --- a/.github/workflows/bench-live.yml +++ b/.github/workflows/bench-live.yml @@ -124,33 +124,50 @@ jobs: echo '```' bun -e ' const { parseLiveModelsReport } = await import("./src/livemodels.ts"); - const r = parseLiveModelsReport(await Bun.file(process.env.POSTIL_REPORT_OUT).json()); - const pct = (value) => (value * 100).toFixed(1) + "%"; - const pad = (value, width) => String(value).length >= width - ? String(value) - : String(value) + " ".repeat(width - String(value).length); - console.log([ - pad("profile", 52), pad("block", 8), pad("final", 8), - pad("advisory", 9), pad("overblock", 10), pad("clean FP", 9), - pad("$/review", 12), pad("p95 ms", 8), pad("pass", 5), - ].join(" ")); - for (const a of r.modelAggregates) { + const { parseLiveModelsFailureReport } = await import("./src/run.ts"); + const raw = await Bun.file(process.env.POSTIL_REPORT_OUT).json(); + if (raw?.artifactType === "live-models-failure") { + const failure = parseLiveModelsFailureReport(raw); + const fact = (value) => value === null ? "unknown" : String(value); + console.log("category " + failure.process.category); + console.log("phase " + failure.process.phase); + console.log("provider attempts " + fact(failure.process.providerAttemptCount)); + console.log("identity present " + fact(failure.process.identityPresent)); + console.log("identity matched " + fact(failure.process.identityMatched)); + console.log("usage present " + fact(failure.process.usagePresent)); + console.log("usage complete " + fact(failure.process.usageAccountingComplete)); + console.log("exit " + fact(failure.process.exitCode)); + console.log("signal " + fact(failure.process.signal)); + console.log("killed " + fact(failure.process.killed)); + } else { + const r = parseLiveModelsReport(raw); + const pct = (value) => (value * 100).toFixed(1) + "%"; + const pad = (value, width) => String(value).length >= width + ? String(value) + : String(value) + " ".repeat(width - String(value).length); console.log([ - pad(a.id, 52), pad(pct(a.mustBlockRecall), 8), - pad(pct(a.mustBlockFinalBlockingRate), 8), - pad(pct(a.advisoryDetectionRate), 9), - pad(pct(a.advisoryOverblockRate), 10), - pad(pct(a.cleanFindingFalsePositiveRate), 9), - pad(a.pricingKnown ? "$" + a.meanCostUsdPerReview.toFixed(6) : "unknown", 12), - pad(a.p95DurationMs.toFixed(0), 8), pad(a.passed ? "yes" : "no", 5), + pad("profile", 52), pad("block", 8), pad("final", 8), + pad("advisory", 9), pad("overblock", 10), pad("clean FP", 9), + pad("$/review", 12), pad("p95 ms", 8), pad("pass", 5), ].join(" ")); - for (const failure of a.admissionFailures) console.log(" FAIL: " + failure); + for (const a of r.modelAggregates) { + console.log([ + pad(a.id, 52), pad(pct(a.mustBlockRecall), 8), + pad(pct(a.mustBlockFinalBlockingRate), 8), + pad(pct(a.advisoryDetectionRate), 9), + pad(pct(a.advisoryOverblockRate), 10), + pad(pct(a.cleanFindingFalsePositiveRate), 9), + pad(a.pricingKnown ? "$" + a.meanCostUsdPerReview.toFixed(6) : "unknown", 12), + pad(a.p95DurationMs.toFixed(0), 8), pad(a.passed ? "yes" : "no", 5), + ].join(" ")); + for (const failure of a.admissionFailures) console.log(" FAIL: " + failure); + } + console.log(""); + console.log("Repeats " + r.repeats + " | total $" + r.totalRunCostUsd.toFixed(6)); + console.log("CLI " + r.cliVersion + " | " + r.evaluatorRuntimeIdentity); + console.log("Endpoint " + r.providerEndpointIdentity + " | " + r.apiFormat); + console.log("Manifest " + (r.manifestCandidate ? "emitted" : "not admitted")); } - console.log(""); - console.log("Repeats " + r.repeats + " | total $" + r.totalRunCostUsd.toFixed(6)); - console.log("CLI " + r.cliVersion + " | " + r.evaluatorRuntimeIdentity); - console.log("Endpoint " + r.providerEndpointIdentity + " | " + r.apiFormat); - console.log("Manifest " + (r.manifestCandidate ? "emitted" : "not admitted")); ' echo '```' } >> "$GITHUB_STEP_SUMMARY" diff --git a/bench/src/attribution.test.ts b/bench/src/attribution.test.ts index 3318b44..58e0d13 100644 --- a/bench/src/attribution.test.ts +++ b/bench/src/attribution.test.ts @@ -15,6 +15,7 @@ import { attributionContractSha256, attributionEvidenceSha256, exactRegionOverlap, + parseTerminalDiagnostic, projectedAttributionDecisionCostUsd, replayAttributionEvidence, type AttributionCallEvidence, @@ -228,7 +229,8 @@ console.log(JSON.stringify({ sameDefect: true, reason: "Both identify the same r await writeFile(binary, `#!/usr/bin/env bun process.stderr.write("private-prompt-and-response ".repeat(100)); process.stderr.write("\\npostil: llm response phase=attribution status=200\\n"); -process.stderr.write("postil: error: atomic attribution failed: atomic attribution output invalid after schema repair: invalid reason\\n"); +process.stderr.write('postil:atomic-attribution-terminal:v1:{"version":1,"category":"output-invalid-after-schema-repair","phase":"attribution","providerAttemptCount":1,"identityPresent":null,"identityMatched":null,"usagePresent":null,"usageAccountingComplete":null}\\n'); +process.stderr.write("postil: error: atomic attribution output is invalid after schema repair\\n"); process.exit(1); `); await chmod(binary, 0o700); @@ -246,7 +248,7 @@ process.exit(1); projectedCostUsdDecimal: "0.01", }); expect(result.error).toBe( - "atomic attribution transport failed: category=output-invalid-after-schema-repair exit=1 signal=none killed=false", + "atomic attribution subprocess failed (output-invalid-after-schema-repair)", ); expect(result.error).not.toContain("private-prompt-and-response"); expect(result.error).not.toContain("invalid reason"); @@ -255,6 +257,50 @@ process.exit(1); } }); + test("accepts exactly one strict terminal diagnostic record", () => { + const record = "postil:atomic-attribution-terminal:v1:" + JSON.stringify({ + version: 1, + category: "provider-http-429", + phase: "attribution", + providerAttemptCount: 3, + identityPresent: false, + identityMatched: null, + usagePresent: false, + usageAccountingComplete: false, + }); + expect(parseTerminalDiagnostic(`ordinary log\n${record}\nordinary error`)).toEqual({ + version: 1, + category: "provider-http-429", + phase: "attribution", + providerAttemptCount: 3, + identityPresent: false, + identityMatched: null, + usagePresent: false, + usageAccountingComplete: false, + }); + expect(parseTerminalDiagnostic(`${record}\n${record}`)).toBeNull(); + expect(parseTerminalDiagnostic(`${record} forged-suffix`)).toBeNull(); + expect(parseTerminalDiagnostic(`forged-prefix ${record}`)).toBeNull(); + }); + + test("rejects contradictory or prose-derived terminal facts", () => { + const invalid = (overrides: Record) => + "postil:atomic-attribution-terminal:v1:" + JSON.stringify({ + version: 1, + category: "provider-unclassified", + phase: "attribution", + providerAttemptCount: null, + identityPresent: null, + identityMatched: null, + usagePresent: null, + usageAccountingComplete: null, + ...overrides, + }); + expect(parseTerminalDiagnostic(invalid({ identityPresent: false, identityMatched: true }))).toBeNull(); + expect(parseTerminalDiagnostic(invalid({ usagePresent: false, usageAccountingComplete: true }))).toBeNull(); + expect(parseTerminalDiagnostic("postil: error: provider returned 429 with private detail")).toBeNull(); + }); + test("enforces one global concurrency, provider-call, and spend budget", async () => { const governor = new AttributionGovernor(2, 100, 0.02); let active = 0; diff --git a/bench/src/attribution.ts b/bench/src/attribution.ts index 74a3c47..6e46647 100644 --- a/bench/src/attribution.ts +++ b/bench/src/attribution.ts @@ -16,6 +16,76 @@ import { const execFile = promisify(execFileCallback); +const TERMINAL_DIAGNOSTIC_PREFIX = "postil:atomic-attribution-terminal:v1:"; + +export const atomicAttributionTerminalCategories = [ + "response-identity-missing", + "response-identity-mismatch", + "usage-accounting-incomplete", + "output-invalid-after-schema-repair", + "output-nonterminal-length", + "invalid-output", + "provider-request-too-large", + "provider-timeout", + "provider-deadline", + "provider-transport", + "provider-unclassified", +] as const; + +const subprocessFailureCategories = [ + "subprocess-timeout", + "subprocess-signal", + "subprocess-exit", +] as const; + +const terminalDiagnostic = z.object({ + version: z.literal(1), + category: z.union([ + z.enum(atomicAttributionTerminalCategories), + z.string().regex(/^provider-http-[1-5][0-9]{2}$/u), + ]), + phase: z.enum(["attribution", "unknown"]), + providerAttemptCount: z.number().int().nonnegative().nullable(), + identityPresent: z.boolean().nullable(), + identityMatched: z.boolean().nullable(), + usagePresent: z.boolean().nullable(), + usageAccountingComplete: z.boolean().nullable(), +}).strict().superRefine((diagnostic, context) => { + if (diagnostic.identityMatched === true && diagnostic.identityPresent !== true) { + context.addIssue({ code: z.ZodIssueCode.custom, path: ["identityMatched"], message: "matched identity requires present identity" }); + } + if (diagnostic.usageAccountingComplete === true && diagnostic.usagePresent !== true) { + context.addIssue({ code: z.ZodIssueCode.custom, path: ["usageAccountingComplete"], message: "complete accounting requires present usage" }); + } +}); + +export type AtomicAttributionTerminalDiagnostic = z.infer; +export type AtomicAttributionProcessDiagnostic = Omit & { + category: AtomicAttributionTerminalDiagnostic["category"] | typeof subprocessFailureCategories[number]; +}; + +const atomicAttributionErrorBrand = Symbol("atomic-attribution-transport-error"); + +export class AtomicAttributionTransportError extends Error { + readonly [atomicAttributionErrorBrand] = true; + + constructor( + readonly diagnostic: AtomicAttributionProcessDiagnostic, + readonly exitCode: number | null, + readonly signal: string | null, + readonly killed: boolean | null, + ) { + super(`atomic attribution subprocess failed (${diagnostic.category})`); + this.name = "AtomicAttributionTransportError"; + } +} + +export function atomicAttributionTransportFailure(error: unknown): AtomicAttributionTransportError | null { + return error instanceof AtomicAttributionTransportError && error[atomicAttributionErrorBrand] === true + ? error + : null; +} + export const ATTRIBUTION_CONTRACT_VERSION = 2; export const ATTRIBUTION_MAX_CONCURRENCY = 4; export const ATTRIBUTION_CALL_TIMEOUT_MS = 60_000; @@ -475,7 +545,7 @@ async function runAtomicAttribution( ); stdout = result.stdout; } catch (error) { - throw new Error(`atomic attribution transport failed: ${atomicAttributionProcessFailure(error)}`); + throw atomicAttributionProcessFailure(error); } const output = transportOutput.parse(JSON.parse(stdout)); if (output.model !== options.evaluatorModel || output.provider !== options.expectedProvider || @@ -596,54 +666,44 @@ function boundedError(error: unknown): string { return value.replace(/[\r\n\p{Cc}]+/gu, " ").slice(0, 500); } -function atomicAttributionProcessFailure(error: unknown): string { +function atomicAttributionProcessFailure(error: unknown): AtomicAttributionTransportError { const failure = error !== null && typeof error === "object" ? error as Record : {}; - const stderr = typeof failure.stderr === "string" - ? failure.stderr.slice(-16 * 1024) - : ""; - const terminalDiagnostic = stderr - .split(/\r?\n/u) - .findLast((line) => line.startsWith("postil: error:")) ?? ""; - const category = classifyAtomicAttributionStderr(terminalDiagnostic, error); - const exitCode = typeof failure.code === "number" && Number.isSafeInteger(failure.code) - ? String(failure.code) - : "unknown"; + const exitCode = typeof failure.code === "number" && Number.isSafeInteger(failure.code) && failure.code >= 0 + ? failure.code + : null; const signal = typeof failure.signal === "string" && /^[A-Z0-9]+$/u.test(failure.signal) ? failure.signal - : "none"; - const killed = failure.killed === true ? "true" : "false"; - return `category=${category} exit=${exitCode} signal=${signal} killed=${killed}`; + : null; + const killed = typeof failure.killed === "boolean" ? failure.killed : null; + const diagnostic = parseTerminalDiagnostic(typeof failure.stderr === "string" ? failure.stderr : "") ?? { + version: 1, + category: failureWasTimedOut(error) + ? "subprocess-timeout" + : signal !== null + ? "subprocess-signal" + : "subprocess-exit", + phase: "unknown", + providerAttemptCount: null, + identityPresent: null, + identityMatched: null, + usagePresent: null, + usageAccountingComplete: null, + }; + return new AtomicAttributionTransportError(diagnostic, exitCode, signal, killed); } -function classifyAtomicAttributionStderr(stderr: string, error: unknown): string { - if (/atomic attribution output invalid after schema repair/iu.test(stderr)) { - return "output-invalid-after-schema-repair"; - } - if (/atomic attribution response omitted (?:model|provider) identity/iu.test(stderr)) { - return "response-identity-missing"; - } - if (/atomic attribution response (?:model|provider) identity mismatch/iu.test(stderr)) { - return "response-identity-mismatch"; - } - if (/atomic attribution usage (?:accounting|evidence) is incomplete/iu.test(stderr)) { - return "usage-accounting-incomplete"; - } - if (/atomic attribution provider request exceeds/iu.test(stderr)) { - return "provider-request-too-large"; - } - const httpStatus = stderr.match(/(?:HTTP |status=)([45][0-9]{2})/iu)?.[1]; - if (httpStatus !== undefined) return `provider-http-${httpStatus}`; - if (/request timed out|request timeout|deadline exceeded/iu.test(stderr)) { - return "provider-timeout"; - } - if (/request to model endpoint failed|connection error/iu.test(stderr)) { - return "provider-transport"; +export function parseTerminalDiagnostic(stderr: string): AtomicAttributionTerminalDiagnostic | null { + const framed = stderr + .split(/\r?\n/u) + .filter((line) => line.startsWith(TERMINAL_DIAGNOSTIC_PREFIX)); + if (framed.length !== 1) return null; + try { + return terminalDiagnostic.parse(JSON.parse(framed[0]!.slice(TERMINAL_DIAGNOSTIC_PREFIX.length))); + } catch { + return null; } - if (/atomic attribution failed/iu.test(stderr)) return "attribution-failed"; - if (failureWasTimedOut(error)) return "subprocess-timeout"; - return "subprocess-failed"; } function failureWasTimedOut(error: unknown): boolean { diff --git a/bench/src/livemodels.test.ts b/bench/src/livemodels.test.ts index 5efad15..7870cb5 100644 --- a/bench/src/livemodels.test.ts +++ b/bench/src/livemodels.test.ts @@ -838,7 +838,12 @@ describe("managed admission workflow", () => { expect(workflow).toContain('rm -f "$POSTIL_REPORT_OUT" "$POSTIL_MANIFEST_OUT" "$POSTIL_PRIVATE_EVIDENCE_OUT" "$POSTIL_PRIVATE_EVIDENCE_ENCRYPTED_OUT" "$POSTIL_ATTESTATION_BUNDLE_OUT"'); expect(workflow).toContain('--manifest-out "$POSTIL_MANIFEST_OUT"'); expect(workflow).toContain('--private-evidence-out "$POSTIL_PRIVATE_EVIDENCE_OUT"'); - expect(workflow).toContain('const r = parseLiveModelsReport(await Bun.file(process.env.POSTIL_REPORT_OUT).json());'); + expect(workflow).toContain('const { parseLiveModelsFailureReport } = await import("./src/run.ts");'); + expect(workflow).toContain('if (raw?.artifactType === "live-models-failure")'); + expect(workflow).toContain("const failure = parseLiveModelsFailureReport(raw);"); + expect(workflow).toContain('console.log("category " + failure.process.category);'); + expect(workflow).toContain('console.log("provider attempts " + fact(failure.process.providerAttemptCount));'); + expect(workflow).toContain("const r = parseLiveModelsReport(raw);"); expect(workflow).toContain("secrets.POSTIL_PRIVATE_EVIDENCE_PASSPHRASE"); expect(workflow).toContain("--symmetric --cipher-algo AES256"); expect(workflow).toContain('cmp "$POSTIL_PRIVATE_EVIDENCE_OUT" "$verify_path"'); diff --git a/bench/src/run.test.ts b/bench/src/run.test.ts index 046a222..bf07190 100644 --- a/bench/src/run.test.ts +++ b/bench/src/run.test.ts @@ -2,8 +2,22 @@ import { afterEach, describe, expect, test } from "bun:test"; import { link, mkdir, mkdtemp, readFile, readdir, rm, stat, symlink, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; -import { privateEvidenceSha256, type LiveModelsPrivateEvidenceBundle, type LiveModelsReport } from "./livemodels"; -import { atomicWriteOutput, invalidateExplicitOutputs, prepareExplicitOutputs, writePrivateEvidenceBundle } from "./run"; +import { + parseLiveModelsReport, + parsePrivateEvidenceBundle, + privateEvidenceSha256, + type LiveModelsPrivateEvidenceBundle, + type LiveModelsReport, +} from "./livemodels"; +import { + atomicWriteOutput, + createLiveModelsFailureReport, + invalidateExplicitOutputs, + parseLiveModelsFailureReport, + prepareExplicitOutputs, + writePrivateEvidenceBundle, +} from "./run"; +import { AtomicAttributionTransportError } from "./attribution"; const temporaryDirectories: string[] = []; @@ -60,6 +74,130 @@ function emptyReport(privateEvidenceDigest: string): LiveModelsReport { } describe("benchmark output lifecycle", () => { + test("emits a strict public-only failure artifact rejected by success consumers", async () => { + const report = await createLiveModelsFailureReport( + new AtomicAttributionTransportError( + { + version: 1, + category: "provider-http-503", + phase: "attribution", + providerAttemptCount: 2, + identityPresent: true, + identityMatched: true, + usagePresent: true, + usageAccountingComplete: true, + }, + 1, + null, + false, + ), + { + qualificationSourceSha: "9".repeat(40), + pairs: [{ generatorModel: "deepseek/deepseek-v4-pro", scorerModel: "z-ai/glm-5.2" }], + upstreamProvider: "PublicProvider", + }, + ); + expect(parseLiveModelsFailureReport(report)).toBe(report); + expect(report).toEqual({ + artifactType: "live-models-failure", + qualificationSourceSha: expect.stringMatching(/^[0-9a-f]{40}$/u), + profiles: [{ + id: "deepseek/deepseek-v4-pro [consensus 1] + z-ai/glm-5.2", + generatorModels: ["deepseek/deepseek-v4-pro"], + consensus: 1, + scorerModels: ["z-ai/glm-5.2"], + }], + providerEndpointIdentity: "openrouter:managed-routing", + upstreamProviderIdentity: "PublicProvider", + process: { + category: "provider-http-503", exitCode: 1, signal: null, killed: false, + phase: "attribution", providerAttemptCount: 2, identityPresent: true, identityMatched: true, + usagePresent: true, usageAccountingComplete: true, + }, + }); + const serialized = JSON.stringify(report); + for (const forbidden of ["schemaVersion", "manifestCandidate", "privateEvidence", "secret", "/private/path"]) { + expect(serialized).not.toContain(forbidden); + } + expect(() => parseLiveModelsReport(report)).toThrow("schemaVersion is required"); + expect(() => parsePrivateEvidenceBundle(report)).toThrow("invalid live-models private evidence bundle"); + expect(() => parseLiveModelsFailureReport({ ...report, schemaVersion: 2 })).toThrow( + "unknown or missing fields", + ); + expect(() => parseLiveModelsFailureReport({ + ...report, + process: { ...report.process, detail: "private" }, + })).toThrow("unknown or missing fields"); + }); + + test("does not invent attribution subprocess facts for a preflight failure", async () => { + const report = await createLiveModelsFailureReport( + new Error("qualification projected exposure exceeds the configured cap at /private/path"), + { + qualificationSourceSha: "8".repeat(40), + pairs: [{ generatorModel: "deepseek/deepseek-v4-pro", scorerModel: "z-ai/glm-5.2" }], + upstreamProvider: "PublicProvider", + }, + ); + expect(report.process).toEqual({ + category: "provider-unclassified", + exitCode: null, + signal: null, + killed: null, + phase: "unknown", + providerAttemptCount: null, + identityPresent: null, + identityMatched: null, + usagePresent: null, + usageAccountingComplete: null, + }); + expect(JSON.stringify(report)).not.toContain("private/path"); + }); + + test("does not parse a forged diagnostic tuple from plain error prose", async () => { + const report = await createLiveModelsFailureReport( + new Error("category=provider-http-503 identity_present=true usage_complete=true"), + { + qualificationSourceSha: "7".repeat(40), + pairs: [{ generatorModel: "deepseek/deepseek-v4-pro", scorerModel: "z-ai/glm-5.2" }], + upstreamProvider: "PublicProvider", + }, + ); + expect(report.process).toEqual({ + category: "provider-unclassified", + exitCode: null, + signal: null, + killed: null, + phase: "unknown", + providerAttemptCount: null, + identityPresent: null, + identityMatched: null, + usagePresent: null, + usageAccountingComplete: null, + }); + }); + + test("rejects contradictory failure facts", () => { + const base = { + artifactType: "live-models-failure", + qualificationSourceSha: "6".repeat(40), + profiles: [{ id: "pair", generatorModels: ["generator"], consensus: 1, scorerModels: ["scorer"] }], + providerEndpointIdentity: "openrouter:managed-routing", + upstreamProviderIdentity: "PublicProvider", + process: { + category: "provider-unclassified", exitCode: 1, signal: null, killed: false, + phase: "attribution", providerAttemptCount: 1, + identityPresent: false, identityMatched: true, + usagePresent: false, usageAccountingComplete: false, + }, + }; + expect(() => parseLiveModelsFailureReport(base)).toThrow("invalid live-models failure process facts"); + expect(() => parseLiveModelsFailureReport({ + ...base, + process: { ...base.process, identityMatched: false, usageAccountingComplete: true }, + })).toThrow("invalid live-models failure process facts"); + }); + test("cleans and rejects one path used for both evidence and a candidate", async () => { const directory = await temporaryDirectory(); const path = join(directory, "report.json"); diff --git a/bench/src/run.ts b/bench/src/run.ts index 47c177d..660374e 100644 --- a/bench/src/run.ts +++ b/bench/src/run.ts @@ -52,16 +52,69 @@ import { DEFAULT_LIVE_CONCURRENCY as DEFAULT_LIVE_MODELS_CONCURRENCY, formatLiveModelsReport, liveModelsQualificationExitCode, + MANAGED_OPENROUTER_PROVIDER_IDENTITY, parseLiveModelsReport, parsePrivateEvidenceBundle, parseQualificationPairs, serializePrivateEvidenceBundle, runLiveModels, + resolveQualificationSourceSha, pricingFromFile, verifyPrivateEvidenceBundle, type LiveModelsPrivateEvidenceBundle, type LiveModelsReport, } from "./livemodels"; +import { + qualificationGeneratorModels, + qualificationPairId, + qualificationScorerModels, + type QualificationPair, +} from "./livemodels-score"; +import { atomicAttributionTransportFailure } from "./attribution"; + +const LIVE_MODELS_FAILURE_CATEGORIES = [ + "response-identity-missing", + "response-identity-mismatch", + "usage-accounting-incomplete", + "output-invalid-after-schema-repair", + "output-nonterminal-length", + "invalid-output", + "provider-request-too-large", + "provider-timeout", + "provider-deadline", + "provider-transport", + "provider-unclassified", + "subprocess-timeout", + "subprocess-signal", + "subprocess-exit", +] as const; + +type LiveModelsFailureCategory = typeof LIVE_MODELS_FAILURE_CATEGORIES[number] | `provider-http-${number}`; + +export interface LiveModelsFailureReport { + artifactType: "live-models-failure"; + qualificationSourceSha: string; + profiles: Array<{ + id: string; + generatorModels: string[]; + consensus: number; + scorerModels: string[]; + }>; + providerEndpointIdentity: typeof MANAGED_OPENROUTER_PROVIDER_IDENTITY; + upstreamProviderIdentity: string; + process: { + category: LiveModelsFailureCategory; + exitCode: number | null; + signal: string | null; + killed: boolean | null; + phase: "attribution" | "unknown"; + providerAttemptCount: number | null; + identityPresent: boolean | null; + identityMatched: boolean | null; + usagePresent: boolean | null; + usageAccountingComplete: boolean | null; + }; +} function flagValue(args: string[], flag: string): string | undefined { const index = args.indexOf(flag); @@ -118,17 +171,33 @@ async function main() { if (!upstreamProvider?.trim()) { throw new Error("live-models admission needs POSTIL_BENCH_UPSTREAM_PROVIDER or --upstream-provider"); } - const { report, privateEvidence } = await runLiveModels(cases, { - binary, - pairs, - repeats: repeatsRaw === undefined ? undefined : Number.parseInt(repeatsRaw, 10), - apiBase: process.env.POSTIL_API_BASE, - apiFormat, - upstreamProvider, - pricing: pricingFile === undefined ? undefined : await pricingFromFile(pricingFile), - concurrency, - costCapUsd: costCapRaw, - }); + const qualificationSourceSha = await resolveQualificationSourceSha( + resolve(import.meta.dir, "..", ".."), + ); + let report: LiveModelsReport; + let privateEvidence: LiveModelsPrivateEvidenceBundle; + try { + ({ report, privateEvidence } = await runLiveModels(cases, { + binary, + pairs, + repeats: repeatsRaw === undefined ? undefined : Number.parseInt(repeatsRaw, 10), + apiBase: process.env.POSTIL_API_BASE, + apiFormat, + upstreamProvider, + pricing: pricingFile === undefined ? undefined : await pricingFromFile(pricingFile), + concurrency, + costCapUsd: costCapRaw, + })); + } catch (error) { + await invalidateExplicitOutputs([manifestOut, privateEvidenceOut]); + const failureReport = await createLiveModelsFailureReport(error, { + qualificationSourceSha, + pairs, + upstreamProvider, + }); + await writeLiveModelsReport(jsonOut, JSON.stringify(failureReport, null, 2)); + throw error; + } await writePrivateEvidenceBundle(privateEvidenceOut, privateEvidence, report); await writeLiveModelsReport(jsonOut, JSON.stringify(report, null, 2)); if (manifestOut) { @@ -174,6 +243,124 @@ async function main() { } } +export async function createLiveModelsFailureReport( + error: unknown, + options: { + qualificationSourceSha: string; + pairs: QualificationPair[]; + upstreamProvider: string; + }, +): Promise { + const failure = fixedLiveModelsFailure(error); + return Object.freeze(parseLiveModelsFailureReport({ + artifactType: "live-models-failure", + qualificationSourceSha: options.qualificationSourceSha, + profiles: options.pairs.map((pair) => ({ + id: qualificationPairId(pair), + generatorModels: qualificationGeneratorModels(pair), + consensus: pair.consensus ?? qualificationGeneratorModels(pair).length, + scorerModels: qualificationScorerModels(pair), + })), + providerEndpointIdentity: MANAGED_OPENROUTER_PROVIDER_IDENTITY, + upstreamProviderIdentity: options.upstreamProvider, + process: failure, + })); +} + +export function parseLiveModelsFailureReport(value: unknown): LiveModelsFailureReport { + if (!isRecord(value)) throw new Error("invalid live-models failure artifact"); + assertExactKeys(value, [ + "artifactType", "qualificationSourceSha", "profiles", "providerEndpointIdentity", + "upstreamProviderIdentity", "process", + ]); + if (value.artifactType !== "live-models-failure" || + typeof value.qualificationSourceSha !== "string" || + !/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/u.test(value.qualificationSourceSha) || + value.providerEndpointIdentity !== MANAGED_OPENROUTER_PROVIDER_IDENTITY || + typeof value.upstreamProviderIdentity !== "string" || value.upstreamProviderIdentity.trim() === "" || + !Array.isArray(value.profiles) || value.profiles.length === 0 || + !isRecord(value.process)) { + throw new Error("invalid live-models failure artifact"); + } + for (const profile of value.profiles) { + if (!isRecord(profile)) throw new Error("invalid live-models failure profile"); + assertExactKeys(profile, ["id", "generatorModels", "consensus", "scorerModels"]); + if (typeof profile.id !== "string" || profile.id.trim() === "" || + !Array.isArray(profile.generatorModels) || profile.generatorModels.length === 0 || + !profile.generatorModels.every((model) => typeof model === "string" && model.trim() !== "") || + !Number.isSafeInteger(profile.consensus) || (profile.consensus as number) < 1 || + !Array.isArray(profile.scorerModels) || profile.scorerModels.length === 0 || + !profile.scorerModels.every((model) => typeof model === "string" && model.trim() !== "")) { + throw new Error("invalid live-models failure profile"); + } + } + assertExactKeys(value.process, [ + "category", "exitCode", "signal", "killed", "phase", "providerAttemptCount", + "identityPresent", "identityMatched", "usagePresent", "usageAccountingComplete", + ]); + const category = value.process.category; + const validCategory = typeof category === "string" && ( + (LIVE_MODELS_FAILURE_CATEGORIES as readonly string[]).includes(category) || + /^provider-http-[1-5][0-9]{2}$/u.test(category) + ); + if (!validCategory || + !(value.process.exitCode === null || (Number.isSafeInteger(value.process.exitCode) && (value.process.exitCode as number) >= 0)) || + !(value.process.signal === null || (typeof value.process.signal === "string" && /^[A-Z0-9]+$/u.test(value.process.signal))) || + !isOptionalBoolean(value.process.killed) || + (value.process.phase !== "attribution" && value.process.phase !== "unknown") || + !(value.process.providerAttemptCount === null || + (Number.isSafeInteger(value.process.providerAttemptCount) && (value.process.providerAttemptCount as number) >= 0)) || + !isOptionalBoolean(value.process.identityPresent) || !isOptionalBoolean(value.process.identityMatched) || + !isOptionalBoolean(value.process.usagePresent) || !isOptionalBoolean(value.process.usageAccountingComplete)) { + throw new Error("invalid live-models failure process facts"); + } + if (value.process.identityMatched === true && value.process.identityPresent !== true) { + throw new Error("invalid live-models failure process facts"); + } + if (value.process.usageAccountingComplete === true && value.process.usagePresent !== true) { + throw new Error("invalid live-models failure process facts"); + } + return value as unknown as LiveModelsFailureReport; +} + +function fixedLiveModelsFailure(error: unknown): LiveModelsFailureReport["process"] { + const transport = atomicAttributionTransportFailure(error); + if (transport === null) { + return { + category: "provider-unclassified", exitCode: null, signal: null, killed: null, + phase: "unknown", providerAttemptCount: null, identityPresent: null, identityMatched: null, + usagePresent: null, usageAccountingComplete: null, + }; + } + return { + category: transport.diagnostic.category as LiveModelsFailureCategory, + exitCode: transport.exitCode, + signal: transport.signal, + killed: transport.killed, + phase: transport.diagnostic.phase, + providerAttemptCount: transport.diagnostic.providerAttemptCount, + identityPresent: transport.diagnostic.identityPresent, + identityMatched: transport.diagnostic.identityMatched, + usagePresent: transport.diagnostic.usagePresent, + usageAccountingComplete: transport.diagnostic.usageAccountingComplete, + }; +} + +function isOptionalBoolean(value: unknown): value is boolean | null { + return value === null || typeof value === "boolean"; +} + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +function assertExactKeys(value: Record, fields: readonly string[]): void { + const expected = new Set(fields); + if (Object.keys(value).length !== expected.size || Object.keys(value).some((field) => !expected.has(field))) { + throw new Error("live-models failure artifact has unknown or missing fields"); + } +} + function qualificationApiFormat( value: string | undefined, ): "openai-compatible" | "anthropic" | undefined { diff --git a/src/attribution.rs b/src/attribution.rs index 3c62e27..efc11ec 100644 --- a/src/attribution.rs +++ b/src/attribution.rs @@ -11,6 +11,115 @@ use serde::{Deserialize, Serialize}; use crate::config::Config; use crate::llm::LlmClient; +pub(crate) const TERMINAL_DIAGNOSTIC_PREFIX: &str = "postil:atomic-attribution-terminal:v1:"; + +#[derive(Debug, Clone, Copy)] +pub(crate) enum AttributionTerminalCategory { + ResponseIdentityMissing, + ResponseIdentityMismatch, + UsageAccountingIncomplete, + OutputInvalidAfterSchemaRepair, + OutputNonterminalLength, + InvalidOutput, + ProviderRequestTooLarge, + ProviderHttp { status: u16 }, + ProviderTimeout, + ProviderDeadline, + ProviderTransport, + ProviderUnclassified, +} + +impl AttributionTerminalCategory { + fn as_str(self) -> String { + match self { + Self::ResponseIdentityMissing => "response-identity-missing".to_string(), + Self::ResponseIdentityMismatch => "response-identity-mismatch".to_string(), + Self::UsageAccountingIncomplete => "usage-accounting-incomplete".to_string(), + Self::OutputInvalidAfterSchemaRepair => { + "output-invalid-after-schema-repair".to_string() + } + Self::OutputNonterminalLength => "output-nonterminal-length".to_string(), + Self::InvalidOutput => "invalid-output".to_string(), + Self::ProviderRequestTooLarge => "provider-request-too-large".to_string(), + Self::ProviderHttp { status } => format!("provider-http-{status}"), + Self::ProviderTimeout => "provider-timeout".to_string(), + Self::ProviderDeadline => "provider-deadline".to_string(), + Self::ProviderTransport => "provider-transport".to_string(), + Self::ProviderUnclassified => "provider-unclassified".to_string(), + } + } +} + +#[derive(Debug, Clone, Copy)] +pub(crate) enum AttributionTerminalPhase { + Attribution, + Unknown, +} + +impl AttributionTerminalPhase { + fn as_str(self) -> &'static str { + match self { + Self::Attribution => "attribution", + Self::Unknown => "unknown", + } + } +} + +#[derive(Debug, Clone)] +pub(crate) struct AttributionTerminalDiagnostic { + pub version: u8, + pub category: AttributionTerminalCategory, + pub phase: AttributionTerminalPhase, + pub provider_attempt_count: Option, + pub identity_present: Option, + pub identity_matched: Option, + pub usage_present: Option, + pub usage_accounting_complete: Option, +} + +impl AttributionTerminalDiagnostic { + fn unknown() -> Self { + Self { + version: 1, + category: AttributionTerminalCategory::ProviderUnclassified, + phase: AttributionTerminalPhase::Unknown, + provider_attempt_count: None, + identity_present: None, + identity_matched: None, + usage_present: None, + usage_accounting_complete: None, + } + } + + fn emit(&self) { + let record = serde_json::json!({ + "version": self.version, + "category": self.category.as_str(), + "phase": self.phase.as_str(), + "providerAttemptCount": self.provider_attempt_count, + "identityPresent": self.identity_present, + "identityMatched": self.identity_matched, + "usagePresent": self.usage_present, + "usageAccountingComplete": self.usage_accounting_complete, + }); + eprintln!("{TERMINAL_DIAGNOSTIC_PREFIX}{record}"); + } +} + +#[derive(Debug)] +struct AttributionTerminalError { + message: &'static str, + diagnostic: AttributionTerminalDiagnostic, +} + +impl std::fmt::Display for AttributionTerminalError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str(self.message) + } +} + +impl std::error::Error for AttributionTerminalError {} + pub use crate::config::{ ATTRIBUTION_MAX_INPUT_BYTES as MAX_INPUT_BYTES, ATTRIBUTION_MAX_PROVIDER_REQUEST_BYTES as MAX_PROVIDER_REQUEST_BYTES, @@ -72,6 +181,16 @@ struct AttributionSettings { } pub async fn run(input_path: &Path, config_path: Option<&Path>) -> Result { + match run_inner(input_path, config_path).await { + Ok(code) => Ok(code), + Err(error) => { + terminal_diagnostic(&error).emit(); + Err(error) + } + } +} + +async fn run_inner(input_path: &Path, config_path: Option<&Path>) -> Result { let mut options = OpenOptions::new(); options.read(true); #[cfg(unix)] @@ -136,33 +255,53 @@ pub async fn run(input_path: &Path, config_path: Option<&Path>) -> Result { Duration::from_secs(ATTRIBUTION_TIMEOUT_SECS), ) .await - .map_err(|error| anyhow::anyhow!("atomic attribution failed: {error}"))?; - ensure!( - review.model_used == input.model, - "atomic attribution returned an unexpected model identity" - ); - ensure!( - review.provider_used == input.expected_provider, - "atomic attribution returned an unexpected provider identity" - ); - ensure!( - review.usage_accounting_complete, - "atomic attribution usage accounting is incomplete" - ); - ensure!( - review.model_usage.len() == review.raw_responses.len() - && review.model_usage.iter().all(|usage| { - usage.model == input.model - && usage.accounting_complete - && usage.cost_provider_decimal.is_some() - && matches!( - usage.cost_source, - Some(crate::envelope::ModelUsageCostSource::ProviderReported) - ) - && (usage.prompt_tokens > 0 || usage.completion_tokens > 0) - }), - "atomic attribution usage evidence is incomplete or inconsistent" - ); + .map_err(anyhow::Error::new)?; + if review.model_used != input.model || review.provider_used != input.expected_provider { + return Err(terminal_failure( + "atomic attribution returned an unexpected response identity", + AttributionTerminalDiagnostic { + version: 1, + category: AttributionTerminalCategory::ResponseIdentityMismatch, + phase: AttributionTerminalPhase::Attribution, + provider_attempt_count: Some(review.model_usage.len()), + identity_present: Some(true), + identity_matched: Some(false), + usage_present: None, + usage_accounting_complete: None, + }, + )); + } + let usage_present = review.model_usage.last().map(|usage| { + usage.prompt_tokens > 0 + || usage.completion_tokens > 0 + || usage.cost_provider_decimal.is_some() + }); + let usage_evidence_complete = review.model_usage.len() == review.raw_responses.len() + && review.model_usage.iter().all(|usage| { + usage.model == input.model + && usage.accounting_complete + && usage.cost_provider_decimal.is_some() + && matches!( + usage.cost_source, + Some(crate::envelope::ModelUsageCostSource::ProviderReported) + ) + && (usage.prompt_tokens > 0 || usage.completion_tokens > 0) + }); + if !review.usage_accounting_complete || !usage_evidence_complete { + return Err(terminal_failure( + "atomic attribution usage accounting is incomplete", + AttributionTerminalDiagnostic { + version: 1, + category: AttributionTerminalCategory::UsageAccountingIncomplete, + phase: AttributionTerminalPhase::Attribution, + provider_attempt_count: Some(review.model_usage.len()), + identity_present: Some(true), + identity_matched: Some(true), + usage_present, + usage_accounting_complete: Some(false), + }, + )); + } let output = AttributionOutput { same_defect: review.same_defect, reason: review.reason, @@ -183,6 +322,26 @@ pub async fn run(input_path: &Path, config_path: Option<&Path>) -> Result { Ok(0) } +fn terminal_failure( + message: &'static str, + diagnostic: AttributionTerminalDiagnostic, +) -> anyhow::Error { + anyhow::Error::new(AttributionTerminalError { + message, + diagnostic, + }) +} + +fn terminal_diagnostic(error: &anyhow::Error) -> AttributionTerminalDiagnostic { + if let Some(terminal) = error.downcast_ref::() { + return terminal.diagnostic.clone(); + } + if let Some(model_error) = error.downcast_ref::() { + return model_error.atomic_attribution_terminal_diagnostic(); + } + AttributionTerminalDiagnostic::unknown() +} + fn validate_input(input: &AttributionInput) -> Result<()> { ensure!( input.target.start_line > 0 && input.target.end_line >= input.target.start_line, diff --git a/src/llm.rs b/src/llm.rs index d4bfcb0..30e8fa9 100644 --- a/src/llm.rs +++ b/src/llm.rs @@ -105,6 +105,64 @@ pub struct ModelError { usage_accounting_complete: bool, } +#[derive(Debug, Clone, Copy)] +pub(crate) enum AtomicAttributionIdentityFailure { + Missing, + Mismatch, +} + +impl std::fmt::Display for AtomicAttributionIdentityFailure { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Missing => formatter.write_str("atomic attribution response identity is missing"), + Self::Mismatch => { + formatter.write_str("atomic attribution response identity does not match") + } + } + } +} + +impl std::error::Error for AtomicAttributionIdentityFailure {} + +#[cfg(feature = "qualification-candidate")] +#[derive(Debug)] +struct AtomicAttributionInvalidOutput; + +#[cfg(feature = "qualification-candidate")] +impl std::fmt::Display for AtomicAttributionInvalidOutput { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("atomic attribution output is invalid after schema repair") + } +} + +#[cfg(feature = "qualification-candidate")] +impl std::error::Error for AtomicAttributionInvalidOutput {} + +#[cfg(feature = "qualification-candidate")] +#[derive(Debug)] +struct AtomicAttributionRequestTooLarge; + +#[cfg(feature = "qualification-candidate")] +impl std::fmt::Display for AtomicAttributionRequestTooLarge { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("atomic attribution provider request is too large") + } +} + +#[cfg(feature = "qualification-candidate")] +impl std::error::Error for AtomicAttributionRequestTooLarge {} + +#[derive(Debug)] +struct ProviderHttpFailure(reqwest::StatusCode); + +impl std::fmt::Display for ProviderHttpFailure { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "model endpoint returned {}", self.0) + } +} + +impl std::error::Error for ProviderHttpFailure {} + impl ModelError { fn new(error: anyhow::Error, usage: Usage, usage_accounting_complete: bool) -> Self { Self { @@ -152,6 +210,10 @@ impl ModelError { pub fn is_provider(&self) -> bool { self.error.downcast_ref::().is_some() + || self + .error + .chain() + .any(|cause| cause.downcast_ref::().is_some()) } fn is_timeout(&self) -> bool { @@ -161,12 +223,98 @@ impl ModelError { .is_some_and(reqwest::Error::is_timeout) || cause.downcast_ref::().is_some() || cause.downcast_ref::().is_some() + || cause + .downcast_ref::() + .is_some_and(|failure| timeout_status(failure.0.as_u16())) }) } fn is_deadline_exceeded(&self) -> bool { self.error.downcast_ref::().is_some() } + + #[cfg(feature = "qualification-candidate")] + pub(crate) fn atomic_attribution_terminal_diagnostic( + &self, + ) -> crate::attribution::AttributionTerminalDiagnostic { + use crate::attribution::{ + AttributionTerminalCategory as Category, AttributionTerminalDiagnostic, + }; + + let category = if let Some(status) = self.error.downcast_ref::() { + Category::ProviderHttp { + status: status.0.as_u16(), + } + } else if self + .error + .downcast_ref::() + .is_some() + { + Category::ProviderRequestTooLarge + } else if let Some(identity) = self + .error + .downcast_ref::() + { + match identity { + AtomicAttributionIdentityFailure::Missing => Category::ResponseIdentityMissing, + AtomicAttributionIdentityFailure::Mismatch => Category::ResponseIdentityMismatch, + } + } else if self + .error + .downcast_ref::() + .is_some() + { + Category::OutputInvalidAfterSchemaRepair + } else if self.error.chain().any(|cause| { + cause + .downcast_ref::() + .is_some_and(|response| response.reason == "length") + }) { + Category::OutputNonterminalLength + } else if self.is_deadline_exceeded() { + Category::ProviderDeadline + } else if self.is_timeout() { + Category::ProviderTimeout + } else if self.error.chain().any(|cause| { + cause + .downcast_ref::() + .is_some_and(reqwest::Error::is_connect) + }) { + Category::ProviderTransport + } else if self.error.downcast_ref::().is_some() + || self.error.downcast_ref::().is_some() + || self + .error + .downcast_ref::() + .is_some() + { + Category::InvalidOutput + } else { + Category::ProviderUnclassified + }; + let identity = self + .error + .downcast_ref::(); + let terminal_usage = self.model_usage.last(); + AttributionTerminalDiagnostic { + version: 1, + category, + phase: crate::attribution::AttributionTerminalPhase::Attribution, + provider_attempt_count: Some(self.model_usage.len()), + identity_present: identity + .map(|failure| matches!(failure, AtomicAttributionIdentityFailure::Mismatch)), + identity_matched: identity.and_then(|failure| match failure { + AtomicAttributionIdentityFailure::Missing => None, + AtomicAttributionIdentityFailure::Mismatch => Some(false), + }), + usage_present: terminal_usage.map(|usage| { + usage.prompt_tokens > 0 + || usage.completion_tokens > 0 + || usage.cost_provider_decimal.is_some() + }), + usage_accounting_complete: terminal_usage.map(|usage| usage.accounting_complete), + } + } } impl std::fmt::Display for ModelError { @@ -1910,7 +2058,7 @@ impl LlmClient { let mut response_identities = vec![first_identity]; let verdict = match parse_atomic_attribution(&first.content) { Ok(verdict) => verdict, - Err(first_error) => { + Err(_) => { let repair_system = atomic_attribution_repair_system(system); let repair_user = atomic_attribution_repair_user(user, &first.content); let second = client @@ -1943,11 +2091,9 @@ impl LlmClient { )?; raw_responses.push(second.content.clone()); response_identities.push(second_identity); - parse_atomic_attribution(&second.content).map_err(|second_error| { + parse_atomic_attribution(&second.content).map_err(|_| { let mut result = ModelError::new( - anyhow!( - "atomic attribution output invalid after schema repair: {second_error} (first response: {first_error})" - ), + anyhow::Error::new(AtomicAttributionInvalidOutput), usage, usage_accounting_complete, ); @@ -2573,22 +2719,19 @@ impl LlmClient { .as_ref() .and_then(|identity| identity.0.as_deref()) .ok_or_else(|| { - anyhow!("qualification response omitted model identity") + anyhow::Error::new(AtomicAttributionIdentityFailure::Missing) })?; let returned_provider = actual_identity .as_ref() .and_then(|identity| identity.1.as_deref()) .ok_or_else(|| { - anyhow!("qualification response omitted provider identity") + anyhow::Error::new(AtomicAttributionIdentityFailure::Missing) })?; - ensure!( - returned_model == model, - "qualification response model identity mismatch" - ); - ensure!( - returned_provider == expected_provider, - "qualification response provider identity mismatch" - ); + if returned_model != model || returned_provider != expected_provider { + return Err(anyhow::Error::new( + AtomicAttributionIdentityFailure::Mismatch, + )); + } } eprintln!( "postil: llm response phase={} model={} attempt={} status={} elapsed={} bytes={} request_id={} response_id={} returned_model={} provider={} choices={} finish={} usage={} prompt_tokens={} completion_tokens={} reasoning_tokens={} category={}", @@ -2755,7 +2898,7 @@ impl LlmClient { &summary, response.request_id.as_deref(), ); - return Err(anyhow::Error::new(RequestTimedOut).context(detail)); + return Err(anyhow::Error::new(ProviderHttpFailure(status)).context(detail)); } if retryable_status(status.as_u16()) && retries < TRANSIENT_RETRIES { retries += 1; @@ -2773,11 +2916,13 @@ impl LlmClient { attempt_timeout = self.request_timeout; continue; } - return Err(anyhow!(provider_http_status_detail( - status, - &summary, - response.request_id.as_deref(), - ))); + return Err(anyhow::Error::new(ProviderHttpFailure(status)).context( + provider_http_status_detail( + status, + &summary, + response.request_id.as_deref(), + ), + )); } Err(error) if reqwest_error(&error).is_some_and(reqwest::Error::is_timeout) @@ -2833,11 +2978,9 @@ impl LlmClient { fn ensure_atomic_attribution_request_size(body: &serde_json::Value) -> Result<()> { let bytes = serde_json::to_vec(body).context("serialize atomic attribution provider request")?; - ensure!( - bytes.len() <= crate::attribution::MAX_PROVIDER_REQUEST_BYTES, - "atomic attribution provider request exceeds {} bytes", - crate::attribution::MAX_PROVIDER_REQUEST_BYTES, - ); + if bytes.len() > crate::attribution::MAX_PROVIDER_REQUEST_BYTES { + return Err(anyhow::Error::new(AtomicAttributionRequestTooLarge)); + } Ok(()) } @@ -3373,19 +3516,16 @@ fn require_attribution_identity( let returned_model = success .returned_model .as_deref() - .ok_or_else(|| anyhow!("atomic attribution response omitted model identity"))?; + .ok_or_else(|| anyhow::Error::new(AtomicAttributionIdentityFailure::Missing))?; let returned_provider = success .provider .as_deref() - .ok_or_else(|| anyhow!("atomic attribution response omitted provider identity"))?; - ensure!( - returned_model == expected_model, - "atomic attribution response model identity mismatch" - ); - ensure!( - returned_provider == expected_provider, - "atomic attribution response provider identity mismatch" - ); + .ok_or_else(|| anyhow::Error::new(AtomicAttributionIdentityFailure::Missing))?; + if returned_model != expected_model || returned_provider != expected_provider { + return Err(anyhow::Error::new( + AtomicAttributionIdentityFailure::Mismatch, + )); + } Ok(AtomicAttributionResponseIdentity { model: returned_model.to_string(), provider: returned_provider.to_string(), diff --git a/tests/e2e.rs b/tests/e2e.rs index 15f489b..043406d 100644 --- a/tests/e2e.rs +++ b/tests/e2e.rs @@ -5,6 +5,8 @@ use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; use assert_cmd::Command; +#[cfg(feature = "qualification-candidate")] +use predicates::prelude::PredicateBooleanExt; use serde_json::{Value, json}; use wiremock::matchers::{body_string_contains, header, method, path, path_regex, query_param}; use wiremock::{Mock, MockServer, Request, Respond, ResponseTemplate}; @@ -2856,9 +2858,13 @@ async fn hidden_atomic_attribution_never_expands_an_empty_length_retry() { .arg(&input) .assert() .failure() - .stderr(predicates::str::contains( - "atomic attribution usage evidence is incomplete or inconsistent", - )); + .stderr( + predicates::str::contains("atomic attribution usage accounting is incomplete").and( + predicates::str::contains( + "postil:atomic-attribution-terminal:v1:{\"category\":\"usage-accounting-incomplete\"", + ), + ), + ); assert_eq!(calls.load(Ordering::SeqCst), 2); let requests = server.received_requests().await.unwrap(); assert_eq!(requests.len(), 2); @@ -2872,6 +2878,82 @@ async fn hidden_atomic_attribution_never_expands_an_empty_length_retry() { } } +#[cfg(feature = "qualification-candidate")] +#[tokio::test] +async fn hidden_atomic_attribution_reports_terminal_length_response() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/chat/completions")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "model": "provider/scorer", + "provider": "test-provider", + "choices": [{ + "finish_reason": "length", + "message": {"content": "{\"sameDefect\":true"} + }], + "usage": {"prompt_tokens": 30, "completion_tokens": 180, "cost": 0.000045} + }))) + .mount(&server) + .await; + let dir = tempfile::tempdir().unwrap(); + let (profile, input) = write_atomic_attribution_inputs(dir.path()); + postil() + .current_dir(dir.path()) + .env( + "POSTIL_API_BASE", + postil_cli::config::MANAGED_OPENROUTER_API_BASE, + ) + .env("CI", "true") + .env("GITHUB_API_URL", "http://127.0.0.1:9") + .env("POSTIL_BENCH_REQUIRE_HOSTED_PROVIDER_PRIVACY", "1") + .env("POSTIL_QUALIFICATION_CANDIDATE_PROFILE", &profile) + .env("POSTIL_QUALIFICATION_CAPTURE_API_BASE", server.uri()) + .env("POSTIL_ALLOW_PRIVATE_API_BASE", "1") + .env("REVIEW_SCORER_MODEL", "provider/scorer") + .args(["atomic-attribution", "--input"]) + .arg(&input) + .assert() + .failure() + .stderr(predicates::str::contains( + "postil:atomic-attribution-terminal:v1:{\"category\":\"output-nonterminal-length\"", + )); + assert_eq!(server.received_requests().await.unwrap().len(), 1); +} + +#[cfg(feature = "qualification-candidate")] +#[tokio::test] +async fn hidden_atomic_attribution_reports_terminal_provider_http_status() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/chat/completions")) + .respond_with(ResponseTemplate::new(429).insert_header("Retry-After", "0")) + .mount(&server) + .await; + let dir = tempfile::tempdir().unwrap(); + let (profile, input) = write_atomic_attribution_inputs(dir.path()); + postil() + .current_dir(dir.path()) + .env( + "POSTIL_API_BASE", + postil_cli::config::MANAGED_OPENROUTER_API_BASE, + ) + .env("CI", "true") + .env("GITHUB_API_URL", "http://127.0.0.1:9") + .env("POSTIL_BENCH_REQUIRE_HOSTED_PROVIDER_PRIVACY", "1") + .env("POSTIL_QUALIFICATION_CANDIDATE_PROFILE", &profile) + .env("POSTIL_QUALIFICATION_CAPTURE_API_BASE", server.uri()) + .env("POSTIL_ALLOW_PRIVATE_API_BASE", "1") + .env("REVIEW_SCORER_MODEL", "provider/scorer") + .args(["atomic-attribution", "--input"]) + .arg(&input) + .assert() + .failure() + .stderr(predicates::str::contains( + "postil:atomic-attribution-terminal:v1:{\"category\":\"provider-http-429\"", + )); + assert_eq!(server.received_requests().await.unwrap().len(), 3); +} + #[cfg(feature = "qualification-candidate")] #[tokio::test] async fn hidden_atomic_attribution_rejects_oversized_repair_before_second_provider_call() { @@ -2946,7 +3028,13 @@ async fn hidden_atomic_attribution_rejects_oversized_repair_before_second_provid .arg(&input) .assert() .failure() - .stderr(predicates::str::contains("model provider request failed")); + .stderr( + predicates::str::contains("model provider request failed").and( + predicates::str::contains( + "postil:atomic-attribution-terminal:v1:{\"category\":\"provider-request-too-large\"", + ), + ), + ); assert_eq!(calls.load(Ordering::SeqCst), 1); assert_eq!(server.received_requests().await.unwrap().len(), 1); } @@ -3067,9 +3155,11 @@ async fn hidden_atomic_attribution_rejects_provider_substitution() { .arg(&input) .assert() .failure() - .stderr(predicates::str::contains( - "atomic attribution failed: model provider request failed", - )); + .stderr( + predicates::str::contains("atomic attribution response identity does not match").and( + predicates::str::contains("postil:atomic-attribution-terminal:v1:"), + ), + ); assert_eq!(server.received_requests().await.unwrap().len(), 1); }