Skip to content

Emit structured qualification failure diagnostics #73

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
morgaesis merged 1 commit into from
Jul 16, 2026
Merged

Emit structured qualification failure diagnostics #73

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
65 changes: 41 additions & 24 deletions .github/workflows/bench-live.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -124,33 +124,50 @@ jobs:
echo '```'
bun -e '
const { parseLiveModelsReport } = await import("./src/livemodels.ts");
const r = parseLiveModelsReport(await Bun.file(process.env.POSTIL_REPORT_OUT).json());
const pct = (value) => (value * 100).toFixed(1) + "%";
const pad = (value, width) => String(value).length >= width
? String(value)
: String(value) + " ".repeat(width - String(value).length);
console.log([
pad("profile", 52), pad("block", 8), pad("final", 8),
pad("advisory", 9), pad("overblock", 10), pad("clean FP", 9),
pad("$/review", 12), pad("p95 ms", 8), pad("pass", 5),
].join(" "));
for (const a of r.modelAggregates) {
const { parseLiveModelsFailureReport } = await import("./src/run.ts");
const raw = await Bun.file(process.env.POSTIL_REPORT_OUT).json();
if (raw?.artifactType === "live-models-failure") {
const failure = parseLiveModelsFailureReport(raw);
const fact = (value) => value === null ? "unknown" : String(value);
console.log("category " + failure.process.category);
console.log("phase " + failure.process.phase);
console.log("provider attempts " + fact(failure.process.providerAttemptCount));
console.log("identity present " + fact(failure.process.identityPresent));
console.log("identity matched " + fact(failure.process.identityMatched));
console.log("usage present " + fact(failure.process.usagePresent));
console.log("usage complete " + fact(failure.process.usageAccountingComplete));
console.log("exit " + fact(failure.process.exitCode));
console.log("signal " + fact(failure.process.signal));
console.log("killed " + fact(failure.process.killed));
} else {
const r = parseLiveModelsReport(raw);
const pct = (value) => (value * 100).toFixed(1) + "%";
const pad = (value, width) => String(value).length >= width
? String(value)
: String(value) + " ".repeat(width - String(value).length);
console.log([
pad(a.id, 52), pad(pct(a.mustBlockRecall), 8),
pad(pct(a.mustBlockFinalBlockingRate), 8),
pad(pct(a.advisoryDetectionRate), 9),
pad(pct(a.advisoryOverblockRate), 10),
pad(pct(a.cleanFindingFalsePositiveRate), 9),
pad(a.pricingKnown ? "$" + a.meanCostUsdPerReview.toFixed(6) : "unknown", 12),
pad(a.p95DurationMs.toFixed(0), 8), pad(a.passed ? "yes" : "no", 5),
pad("profile", 52), pad("block", 8), pad("final", 8),
pad("advisory", 9), pad("overblock", 10), pad("clean FP", 9),
pad("$/review", 12), pad("p95 ms", 8), pad("pass", 5),
].join(" "));
for (const failure of a.admissionFailures) console.log(" FAIL: " + failure);
for (const a of r.modelAggregates) {
console.log([
pad(a.id, 52), pad(pct(a.mustBlockRecall), 8),
pad(pct(a.mustBlockFinalBlockingRate), 8),
pad(pct(a.advisoryDetectionRate), 9),
pad(pct(a.advisoryOverblockRate), 10),
pad(pct(a.cleanFindingFalsePositiveRate), 9),
pad(a.pricingKnown ? "$" + a.meanCostUsdPerReview.toFixed(6) : "unknown", 12),
pad(a.p95DurationMs.toFixed(0), 8), pad(a.passed ? "yes" : "no", 5),
].join(" "));
for (const failure of a.admissionFailures) console.log(" FAIL: " + failure);
}
console.log("");
console.log("Repeats " + r.repeats + " | total $" + r.totalRunCostUsd.toFixed(6));
console.log("CLI " + r.cliVersion + " | " + r.evaluatorRuntimeIdentity);
console.log("Endpoint " + r.providerEndpointIdentity + " | " + r.apiFormat);
console.log("Manifest " + (r.manifestCandidate ? "emitted" : "not admitted"));
}
console.log("");
console.log("Repeats " + r.repeats + " | total $" + r.totalRunCostUsd.toFixed(6));
console.log("CLI " + r.cliVersion + " | " + r.evaluatorRuntimeIdentity);
console.log("Endpoint " + r.providerEndpointIdentity + " | " + r.apiFormat);
console.log("Manifest " + (r.manifestCandidate ? "emitted" : "not admitted"));
'
echo '```'
} >> "$GITHUB_STEP_SUMMARY"
Expand Down
50 changes: 48 additions & 2 deletions bench/src/attribution.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import {
attributionContractSha256,
attributionEvidenceSha256,
exactRegionOverlap,
parseTerminalDiagnostic,
projectedAttributionDecisionCostUsd,
replayAttributionEvidence,
type AttributionCallEvidence,
Expand Down Expand Up @@ -228,7 +229,8 @@ console.log(JSON.stringify({ sameDefect: true, reason: "Both identify the same r
await writeFile(binary, `#!/usr/bin/env bun
process.stderr.write("private-prompt-and-response ".repeat(100));
process.stderr.write("\\npostil: llm response phase=attribution status=200\\n");
process.stderr.write("postil: error: atomic attribution failed: atomic attribution output invalid after schema repair: invalid reason\\n");
process.stderr.write('postil:atomic-attribution-terminal:v1:{"version":1,"category":"output-invalid-after-schema-repair","phase":"attribution","providerAttemptCount":1,"identityPresent":null,"identityMatched":null,"usagePresent":null,"usageAccountingComplete":null}\\n');
process.stderr.write("postil: error: atomic attribution output is invalid after schema repair\\n");
process.exit(1);
`);
await chmod(binary, 0o700);
Expand All @@ -246,7 +248,7 @@ process.exit(1);
projectedCostUsdDecimal: "0.01",
});
expect(result.error).toBe(
"atomic attribution transport failed: category=output-invalid-after-schema-repair exit=1 signal=none killed=false",
"atomic attribution subprocess failed (output-invalid-after-schema-repair)",
);
expect(result.error).not.toContain("private-prompt-and-response");
expect(result.error).not.toContain("invalid reason");
Expand All @@ -255,6 +257,50 @@ process.exit(1);
}
});

test("accepts exactly one strict terminal diagnostic record", () => {
const record = "postil:atomic-attribution-terminal:v1:" + JSON.stringify({
version: 1,
category: "provider-http-429",
phase: "attribution",
providerAttemptCount: 3,
identityPresent: false,
identityMatched: null,
usagePresent: false,
usageAccountingComplete: false,
});
expect(parseTerminalDiagnostic(`ordinary log\n${record}\nordinary error`)).toEqual({
version: 1,
category: "provider-http-429",
phase: "attribution",
providerAttemptCount: 3,
identityPresent: false,
identityMatched: null,
usagePresent: false,
usageAccountingComplete: false,
});
expect(parseTerminalDiagnostic(`${record}\n${record}`)).toBeNull();
expect(parseTerminalDiagnostic(`${record} forged-suffix`)).toBeNull();
expect(parseTerminalDiagnostic(`forged-prefix ${record}`)).toBeNull();
});

test("rejects contradictory or prose-derived terminal facts", () => {
const invalid = (overrides: Record<string, unknown>) =>
"postil:atomic-attribution-terminal:v1:" + JSON.stringify({
version: 1,
category: "provider-unclassified",
phase: "attribution",
providerAttemptCount: null,
identityPresent: null,
identityMatched: null,
usagePresent: null,
usageAccountingComplete: null,
...overrides,
});
expect(parseTerminalDiagnostic(invalid({ identityPresent: false, identityMatched: true }))).toBeNull();
expect(parseTerminalDiagnostic(invalid({ usagePresent: false, usageAccountingComplete: true }))).toBeNull();
expect(parseTerminalDiagnostic("postil: error: provider returned 429 with private detail")).toBeNull();
});

test("enforces one global concurrency, provider-call, and spend budget", async () => {
const governor = new AttributionGovernor(2, 100, 0.02);
let active = 0;
Expand Down
142 changes: 101 additions & 41 deletions bench/src/attribution.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,76 @@ import {

const execFile = promisify(execFileCallback);

const TERMINAL_DIAGNOSTIC_PREFIX = "postil:atomic-attribution-terminal:v1:";

export const atomicAttributionTerminalCategories = [
"response-identity-missing",
"response-identity-mismatch",
"usage-accounting-incomplete",
"output-invalid-after-schema-repair",
"output-nonterminal-length",
"invalid-output",
"provider-request-too-large",
"provider-timeout",
"provider-deadline",
"provider-transport",
"provider-unclassified",
] as const;

const subprocessFailureCategories = [
"subprocess-timeout",
"subprocess-signal",
"subprocess-exit",
] as const;

const terminalDiagnostic = z.object({
version: z.literal(1),
category: z.union([
z.enum(atomicAttributionTerminalCategories),
z.string().regex(/^provider-http-[1-5][0-9]{2}$/u),
]),
phase: z.enum(["attribution", "unknown"]),
providerAttemptCount: z.number().int().nonnegative().nullable(),
identityPresent: z.boolean().nullable(),
identityMatched: z.boolean().nullable(),
usagePresent: z.boolean().nullable(),
usageAccountingComplete: z.boolean().nullable(),
}).strict().superRefine((diagnostic, context) => {
if (diagnostic.identityMatched === true && diagnostic.identityPresent !== true) {
context.addIssue({ code: z.ZodIssueCode.custom, path: ["identityMatched"], message: "matched identity requires present identity" });
}
if (diagnostic.usageAccountingComplete === true && diagnostic.usagePresent !== true) {
context.addIssue({ code: z.ZodIssueCode.custom, path: ["usageAccountingComplete"], message: "complete accounting requires present usage" });
}
});

export type AtomicAttributionTerminalDiagnostic = z.infer<typeof terminalDiagnostic>;
export type AtomicAttributionProcessDiagnostic = Omit<AtomicAttributionTerminalDiagnostic, "category"> & {
category: AtomicAttributionTerminalDiagnostic["category"] | typeof subprocessFailureCategories[number];
};

const atomicAttributionErrorBrand = Symbol("atomic-attribution-transport-error");

export class AtomicAttributionTransportError extends Error {
readonly [atomicAttributionErrorBrand] = true;

constructor(
readonly diagnostic: AtomicAttributionProcessDiagnostic,
readonly exitCode: number | null,
readonly signal: string | null,
readonly killed: boolean | null,
) {
super(`atomic attribution subprocess failed (${diagnostic.category})`);
this.name = "AtomicAttributionTransportError";
}
}

export function atomicAttributionTransportFailure(error: unknown): AtomicAttributionTransportError | null {
return error instanceof AtomicAttributionTransportError && error[atomicAttributionErrorBrand] === true
? error
: null;
}

export const ATTRIBUTION_CONTRACT_VERSION = 2;
export const ATTRIBUTION_MAX_CONCURRENCY = 4;
export const ATTRIBUTION_CALL_TIMEOUT_MS = 60_000;
Expand Down Expand Up @@ -475,7 +545,7 @@ async function runAtomicAttribution(
);
stdout = result.stdout;
} catch (error) {
throw new Error(`atomic attribution transport failed: ${atomicAttributionProcessFailure(error)}`);
throw atomicAttributionProcessFailure(error);
}
const output = transportOutput.parse(JSON.parse(stdout));
if (output.model !== options.evaluatorModel || output.provider !== options.expectedProvider ||
Expand Down Expand Up @@ -596,54 +666,44 @@ function boundedError(error: unknown): string {
return value.replace(/[\r\n\p{Cc}]+/gu, " ").slice(0, 500);
}

function atomicAttributionProcessFailure(error: unknown): string {
function atomicAttributionProcessFailure(error: unknown): AtomicAttributionTransportError {
const failure = error !== null && typeof error === "object"
? error as Record<string, unknown>
: {};
const stderr = typeof failure.stderr === "string"
? failure.stderr.slice(-16 * 1024)
: "";
const terminalDiagnostic = stderr
.split(/\r?\n/u)
.findLast((line) => line.startsWith("postil: error:")) ?? "";
const category = classifyAtomicAttributionStderr(terminalDiagnostic, error);
const exitCode = typeof failure.code === "number" && Number.isSafeInteger(failure.code)
? String(failure.code)
: "unknown";
const exitCode = typeof failure.code === "number" && Number.isSafeInteger(failure.code) && failure.code >= 0
? failure.code
: null;
const signal = typeof failure.signal === "string" && /^[A-Z0-9]+$/u.test(failure.signal)
? failure.signal
: "none";
const killed = failure.killed === true ? "true" : "false";
return `category=${category} exit=${exitCode} signal=${signal} killed=${killed}`;
: null;
const killed = typeof failure.killed === "boolean" ? failure.killed : null;
const diagnostic = parseTerminalDiagnostic(typeof failure.stderr === "string" ? failure.stderr : "") ?? {
version: 1,
category: failureWasTimedOut(error)
? "subprocess-timeout"
: signal !== null
? "subprocess-signal"
: "subprocess-exit",
phase: "unknown",
providerAttemptCount: null,
identityPresent: null,
identityMatched: null,
usagePresent: null,
usageAccountingComplete: null,
};
return new AtomicAttributionTransportError(diagnostic, exitCode, signal, killed);
}

function classifyAtomicAttributionStderr(stderr: string, error: unknown): string {
if (/atomic attribution output invalid after schema repair/iu.test(stderr)) {
return "output-invalid-after-schema-repair";
}
if (/atomic attribution response omitted (?:model|provider) identity/iu.test(stderr)) {
return "response-identity-missing";
}
if (/atomic attribution response (?:model|provider) identity mismatch/iu.test(stderr)) {
return "response-identity-mismatch";
}
if (/atomic attribution usage (?:accounting|evidence) is incomplete/iu.test(stderr)) {
return "usage-accounting-incomplete";
}
if (/atomic attribution provider request exceeds/iu.test(stderr)) {
return "provider-request-too-large";
}
const httpStatus = stderr.match(/(?:HTTP |status=)([45][0-9]{2})/iu)?.[1];
if (httpStatus !== undefined) return `provider-http-${httpStatus}`;
if (/request timed out|request timeout|deadline exceeded/iu.test(stderr)) {
return "provider-timeout";
}
if (/request to model endpoint failed|connection error/iu.test(stderr)) {
return "provider-transport";
export function parseTerminalDiagnostic(stderr: string): AtomicAttributionTerminalDiagnostic | null {
const framed = stderr
.split(/\r?\n/u)
.filter((line) => line.startsWith(TERMINAL_DIAGNOSTIC_PREFIX));
if (framed.length !== 1) return null;
try {
return terminalDiagnostic.parse(JSON.parse(framed[0]!.slice(TERMINAL_DIAGNOSTIC_PREFIX.length)));
} catch {
return null;
}
if (/atomic attribution failed/iu.test(stderr)) return "attribution-failed";
if (failureWasTimedOut(error)) return "subprocess-timeout";
return "subprocess-failed";
}

function failureWasTimedOut(error: unknown): boolean {
Expand Down
7 changes: 6 additions & 1 deletion bench/src/livemodels.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -838,7 +838,12 @@ describe("managed admission workflow", () => {
expect(workflow).toContain('rm -f "$POSTIL_REPORT_OUT" "$POSTIL_MANIFEST_OUT" "$POSTIL_PRIVATE_EVIDENCE_OUT" "$POSTIL_PRIVATE_EVIDENCE_ENCRYPTED_OUT" "$POSTIL_ATTESTATION_BUNDLE_OUT"');
expect(workflow).toContain('--manifest-out "$POSTIL_MANIFEST_OUT"');
expect(workflow).toContain('--private-evidence-out "$POSTIL_PRIVATE_EVIDENCE_OUT"');
expect(workflow).toContain('const r = parseLiveModelsReport(await Bun.file(process.env.POSTIL_REPORT_OUT).json());');
expect(workflow).toContain('const { parseLiveModelsFailureReport } = await import("./src/run.ts");');
expect(workflow).toContain('if (raw?.artifactType === "live-models-failure")');
expect(workflow).toContain("const failure = parseLiveModelsFailureReport(raw);");
expect(workflow).toContain('console.log("category " + failure.process.category);');
expect(workflow).toContain('console.log("provider attempts " + fact(failure.process.providerAttemptCount));');
expect(workflow).toContain("const r = parseLiveModelsReport(raw);");
expect(workflow).toContain("secrets.POSTIL_PRIVATE_EVIDENCE_PASSPHRASE");
expect(workflow).toContain("--symmetric --cipher-algo AES256");
expect(workflow).toContain('cmp "$POSTIL_PRIVATE_EVIDENCE_OUT" "$verify_path"');
Expand Down
Loading
Loading