From 9e0fdb687cb3f59502b337f8591e7133634497b4 Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Fri, 11 Sep 2026 02:08:36 +0000 Subject: [PATCH 1/4] Retain bounded scorer failure diagnostics in reports --- bench/src/scorer-eval.test.ts | 107 ++++++++++++++++++++++++++++++++++ bench/src/scorer-eval.ts | 92 +++++++++++++++++++++++++++++ 2 files changed, 199 insertions(+) diff --git a/bench/src/scorer-eval.test.ts b/bench/src/scorer-eval.test.ts index 837630a..cdc9cc5 100644 --- a/bench/src/scorer-eval.test.ts +++ b/bench/src/scorer-eval.test.ts @@ -46,6 +46,7 @@ import { reviewCoverageFailure, safeSegment, scorerCasePasses, + scorerCaseDiagnostics, scorerCostProviderDecimal, scorerEvalRootDir, scorerEvaluatorDigest, @@ -430,6 +431,95 @@ describe("parseRepeatCount", () => { }); }); +describe("scorer case diagnostics", () => { + const accountedAttempt: Parameters[0]["attempts"][number] = { + phase: "adjudication", outcome: "completed", durationMs: 100, + promptTokens: 1125, completionTokens: 396, costUsd: 0.000548064, + costProviderDecimal: "0.000548064", usageValid: true, httpStatus: 200, + modelIdentityPresent: true, providerIdentityPresent: true, usagePresent: true, + errorPresent: false, + }; + const semanticFailure = "postil: finding adjudication validation failed; preserving all generated findings: refuted adjudication must cite candidate-specific contradictory evidence"; + const unavailable = "postil: finding adjudication unavailable; preserving all generated findings"; + const diagnosticInput = { + child: { exitCode: 1, stderr: `${semanticFailure}\n`, timedOut: false }, + attempts: [accountedAttempt], envelope: { findings: [] }, passed: false, + }; + + test("identifies the captured unsupported refutation without treating its accounted call as qualification", () => { + const diagnostics = scorerCaseDiagnostics(diagnosticInput); + expect(diagnostics.failureSignals).toEqual(["unsupportedRefutation"]); + expect(diagnostics.requiredUpstreamAttempts).toBe(2); + expect(diagnostics.adjudication).toEqual({ + attempts: 1, collectedResponses: 1, validUsageAttempts: 1, exactCostAttempts: 1, + modelIdentityPresentAttempts: 1, providerIdentityPresentAttempts: 1, + }); + expect(diagnostics.scorer.attempts).toBe(0); + const original = result({ + id: "clean-comment-only", repeat: 2, scenario: "falseFinding", + scorerModel: null, scorerConfidence: null, scorerKind: null, + upstreamRequests: 1, usageAccountingComplete: true, usageValid: false, + reasonContractValid: false, publicationValid: false, passed: false, + }); + const observed = { ...original, diagnostics }; + const { diagnostics: _diagnostics, ...priorFields } = observed; + expect(priorFields).toEqual(original); + expect(isAdmissionFatalStructuralResult(observed, original.model)).toBe(true); + expect(aggregate(original.model, [observed], 3)).toEqual(aggregate(original.model, [original], 3)); + expect(aggregate(original.model, [observed], 3).passed).toBe(false); + }); + + test("separates native parser and provider failures from unclassified unavailability", () => { + const input = { ...diagnosticInput, child: { ...diagnosticInput.child, stderr: unavailable } }; + expect(scorerCaseDiagnostics({ ...input, envelope: { + findings: [], modelIncidents: [{ phase: "scorer", category: "invalidOutput", recovered: false }], + } }).failureSignals).toEqual(["adjudicationParser"]); + expect(scorerCaseDiagnostics({ ...input, envelope: { + findings: [{ path: ".postil/provider", title: "Model provider unavailable" }], + } }).failureSignals).toEqual(["adjudicationProvider"]); + expect(scorerCaseDiagnostics(input).failureSignals).toEqual(["adjudicationUnavailable"]); + }); + + test("retains timeout and invalid-envelope observations without conflating response collection with success", () => { + const diagnostics = scorerCaseDiagnostics({ + ...diagnosticInput, envelope: undefined, + child: { exitCode: undefined, timedOut: true, stderr: "" }, + attempts: [ + { ...accountedAttempt, outcome: "timedOut", usageValid: false, costProviderDecimal: null, httpStatus: null }, + { ...accountedAttempt, phase: "scorer", httpStatus: 503, errorPresent: true }, + ], + }); + expect(diagnostics.childExitCode).toBeNull(); + expect(diagnostics.failureSignals).toEqual(["childTimeout", "invalidEnvelope", "upstreamTimeout", "upstreamHttp"]); + expect(diagnostics.adjudication.validUsageAttempts).toBe(0); + expect(diagnostics.adjudication.exactCostAttempts).toBe(0); + expect(diagnostics.scorer.collectedResponses).toBe(1); + expect(scorerCaseDiagnostics({ + ...diagnosticInput, child: { exitCode: 0, timedOut: false, stderr: "" }, + attempts: [{ ...accountedAttempt, errorPresent: true }], passed: true, + }).failureSignals).toEqual([]); + }); + + test("emits bounded deduplicated codes without copying arbitrary messages or payload properties", () => { + const marker = "credential=PRIVATE_MARKER Authorization: Bearer PRIVATE_MARKER"; + const diagnostics = scorerCaseDiagnostics({ + ...diagnosticInput, + child: { ...diagnosticInput.child, stderr: `${semanticFailure} ${marker}\n${marker}\n`.repeat(1000) }, + publicationFailureCodes: ["check-run-state", "check-run-state", marker], + }); + expect(diagnostics.failureSignals).toEqual(["adjudicationValidation"]); + expect(diagnostics.publicationFailureCodes).toEqual(["check-run-state"]); + expect(JSON.stringify(diagnostics)).not.toContain("PRIVATE_MARKER"); + expect(JSON.stringify(diagnostics).length).toBeLessThan(1024); + expect(scorerCaseDiagnostics({ ...diagnosticInput, + child: { ...diagnosticInput.child, stderr: marker }, + }).failureSignals).toEqual(["unknown"]); + expect(scorerCaseDiagnostics({ ...diagnosticInput, + child: { ...diagnosticInput.child, stderr: `${semanticFailure}\n`.repeat(1000) }, + }).failureSignals).toEqual(["unsupportedRefutation"]); + }); +}); + describe("scorer run artifacts", () => { test("binds qualification to clean evaluator source bytes", () => { expect(() => assertCleanScorerEvaluatorStatus("")).not.toThrow(); @@ -948,6 +1038,11 @@ describe("scorer proxy and isolated runtime", () => { passed: true, findingPublished: false, gateFailing: false, + diagnostics: { + childExitCode: 0, requiredUpstreamAttempts: 2, failureSignals: [], publicationFailureCodes: [], + adjudication: { attempts: 1, collectedResponses: 1, validUsageAttempts: 1, exactCostAttempts: 1 }, + scorer: { attempts: 1, collectedResponses: 1, validUsageAttempts: 1, exactCostAttempts: 1 }, + }, }); expect(scorerRequests).toHaveLength(1); const scorerRequest = JSON.parse(scorerRequests[0]!) as { @@ -1253,6 +1348,10 @@ describe("scorer evaluation checkpoints", () => { name: "MODEL_BODY_MARKER", reason: "API_SECRET_MARKER", scorerError: "UPSTREAM_RESPONSE_MARKER", + diagnostics: scorerCaseDiagnostics({ + child: { exitCode: undefined, stderr: "API_SECRET_MARKER", timedOut: true }, + attempts: [], passed: false, + }), }); try { await writeScorerEvalCheckpoint(jsonOut, ["scorer/model"], 1, 2, [sensitive]); @@ -1268,6 +1367,7 @@ describe("scorer evaluation checkpoints", () => { }); expect(first.cases).toHaveLength(1); expect(first.cases[0]?.publicationValid).toBe(true); + expect(first.cases[0]?.diagnostics).toEqual(sensitive.diagnostics); expect(firstRaw).not.toContain("MODEL_BODY_MARKER"); expect(firstRaw).not.toContain("API_SECRET_MARKER"); expect(firstRaw).not.toContain("UPSTREAM_RESPONSE_MARKER"); @@ -1646,6 +1746,13 @@ describe("qualification utilities", () => { cases: [], }); expect(qualificationExitCode(report([passing]))).toBe(0); + const observedCases = qualificationCases(1).map((entry) => ({ + ...entry, + diagnostics: scorerCaseDiagnostics({ child: { exitCode: 0, stderr: "", timedOut: false }, + attempts: [], envelope: { findings: [] }, passed: entry.passed }), + })); + expect(aggregate("scorer/model", observedCases, 1)).toEqual(passing); + expect(qualificationExitCode({ ...report([passing]), cases: observedCases })).toBe(0); expect(qualificationExitCode(report([{ ...passing, passed: false }]))).toBe(1); expect(qualificationExitCode({ ...report([passing]), completedCases: 1, matrixComplete: false })).toBe(1); expect(qualificationExitCode(report([]))).toBe(1); diff --git a/bench/src/scorer-eval.ts b/bench/src/scorer-eval.ts index d047e00..ec4891e 100644 --- a/bench/src/scorer-eval.ts +++ b/bench/src/scorer-eval.ts @@ -120,6 +120,94 @@ export interface ScorerEvalCase { completionTokens: number; costUsd: number | null; costProviderDecimal?: string | null; + diagnostics?: ScorerCaseDiagnostics; +} + +type ScorerFailureSignal = "childTimeout" | "invalidEnvelope" | + "unsupportedRefutation" | "adjudicationValidation" | "adjudicationParser" | + "adjudicationProvider" | "adjudicationUnavailable" | "upstreamTimeout" | + "upstreamTransport" | "upstreamHttp" | "unknown"; + +interface ScorerPhaseDiagnostics { + attempts: number; + collectedResponses: number; + validUsageAttempts: number; + exactCostAttempts: number; + modelIdentityPresentAttempts: number; + providerIdentityPresentAttempts: number; +} + +export interface ScorerCaseDiagnostics { + childExitCode: number | null; + requiredUpstreamAttempts: 2; + adjudication: ScorerPhaseDiagnostics; + scorer: ScorerPhaseDiagnostics; + failureSignals: ScorerFailureSignal[]; + publicationFailureCodes: string[]; +} + +// Diagnostics observe native results and never participate in qualification. +// Output contains only fixed codes and counts, including in partial reports. +export function scorerCaseDiagnostics(input: { + child: Pick; + attempts: readonly ScorerAttempt[]; + envelope?: { + findings: readonly { path: string; title: string }[]; + modelIncidents?: readonly { phase: string; category: string; recovered: boolean }[]; + }; + publicationFailureCodes?: readonly string[]; + passed: boolean; +}): ScorerCaseDiagnostics { + const signals = new Set(); + const lines = new Set(input.child.stderr.split(/\r?\n/u)); + const validationPrefix = "postil: finding adjudication validation failed; preserving all generated findings: "; + if (lines.has(`${validationPrefix}refuted adjudication must cite candidate-specific contradictory evidence`)) { + signals.add("unsupportedRefutation"); + } else if ([...lines].some((line) => line.startsWith(validationPrefix))) { + signals.add("adjudicationValidation"); + } + if (lines.has("postil: finding adjudication unavailable; preserving all generated findings")) { + if (input.envelope?.findings.some((finding) => + finding.path === ".postil/provider" && finding.title === "Model provider unavailable" + )) { + signals.add("adjudicationProvider"); + } else if (input.envelope?.modelIncidents?.some((incident) => + incident.phase === "scorer" && incident.category === "invalidOutput" && !incident.recovered + )) { + signals.add("adjudicationParser"); + } else { + signals.add("adjudicationUnavailable"); + } + } + if (input.child.timedOut) signals.add("childTimeout"); + if (input.envelope === undefined) signals.add("invalidEnvelope"); + for (const attempt of input.attempts) { + if (attempt.outcome === "timedOut") signals.add("upstreamTimeout"); + if (attempt.outcome === "failed") signals.add("upstreamTransport"); + if (attempt.httpStatus !== null && attempt.httpStatus >= 400) signals.add("upstreamHttp"); + } + if (!input.passed && signals.size === 0) signals.add("unknown"); + const phaseCounts = (phase: ScorerAttempt["phase"]): ScorerPhaseDiagnostics => { + const attempts = input.attempts.filter((attempt) => attempt.phase === phase); + return { + attempts: attempts.length, + // Completed means the proxy collected a response, not that it was valid. + collectedResponses: attempts.filter((attempt) => attempt.outcome === "completed").length, + validUsageAttempts: attempts.filter((attempt) => attempt.usageValid).length, + exactCostAttempts: attempts.filter((attempt) => attempt.costProviderDecimal !== null).length, + modelIdentityPresentAttempts: attempts.filter((attempt) => attempt.modelIdentityPresent === true).length, + providerIdentityPresentAttempts: attempts.filter((attempt) => attempt.providerIdentityPresent === true).length, + }; + }; + return { + childExitCode: input.child.exitCode ?? null, + requiredUpstreamAttempts: 2, + adjudication: phaseCounts("adjudication"), + scorer: phaseCounts("scorer"), + failureSignals: [...signals], + publicationFailureCodes: ["check-run-state", "review-count", "comment-count", "missing-anchor"] + .filter((code) => input.publicationFailureCodes?.includes(code)), + }; } export interface ScorerEvalAggregate { @@ -895,6 +983,7 @@ export async function runScorerEvalCase( caseTimedOut, ), ...telemetry, + diagnostics: scorerCaseDiagnostics({ child, attempts: proxy.attempts, passed: false }), }; } const parsed = parsedEnvelope.data; @@ -1057,6 +1146,9 @@ export async function runScorerEvalCase( publicationValid, ...telemetry, costProviderDecimal, + diagnostics: scorerCaseDiagnostics({ + child, attempts: proxy.attempts, envelope: parsed, publicationFailureCodes, passed, + }), }; } From b3fee498f77268716a40d8a42f83d5ff9df4c43e Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Fri, 11 Sep 2026 02:14:25 +0000 Subject: [PATCH 2/4] Describe native adjudication invalid-output diagnostics accurately --- bench/src/scorer-eval.test.ts | 4 ++-- bench/src/scorer-eval.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bench/src/scorer-eval.test.ts b/bench/src/scorer-eval.test.ts index cdc9cc5..388a3f0 100644 --- a/bench/src/scorer-eval.test.ts +++ b/bench/src/scorer-eval.test.ts @@ -469,11 +469,11 @@ describe("scorer case diagnostics", () => { expect(aggregate(original.model, [observed], 3).passed).toBe(false); }); - test("separates native parser and provider failures from unclassified unavailability", () => { + test("separates native invalid-output and provider failures from unclassified unavailability", () => { const input = { ...diagnosticInput, child: { ...diagnosticInput.child, stderr: unavailable } }; expect(scorerCaseDiagnostics({ ...input, envelope: { findings: [], modelIncidents: [{ phase: "scorer", category: "invalidOutput", recovered: false }], - } }).failureSignals).toEqual(["adjudicationParser"]); + } }).failureSignals).toEqual(["adjudicationInvalidOutput"]); expect(scorerCaseDiagnostics({ ...input, envelope: { findings: [{ path: ".postil/provider", title: "Model provider unavailable" }], } }).failureSignals).toEqual(["adjudicationProvider"]); diff --git a/bench/src/scorer-eval.ts b/bench/src/scorer-eval.ts index ec4891e..8427a40 100644 --- a/bench/src/scorer-eval.ts +++ b/bench/src/scorer-eval.ts @@ -124,7 +124,7 @@ export interface ScorerEvalCase { } type ScorerFailureSignal = "childTimeout" | "invalidEnvelope" | - "unsupportedRefutation" | "adjudicationValidation" | "adjudicationParser" | + "unsupportedRefutation" | "adjudicationValidation" | "adjudicationInvalidOutput" | "adjudicationProvider" | "adjudicationUnavailable" | "upstreamTimeout" | "upstreamTransport" | "upstreamHttp" | "unknown"; @@ -174,7 +174,7 @@ export function scorerCaseDiagnostics(input: { } else if (input.envelope?.modelIncidents?.some((incident) => incident.phase === "scorer" && incident.category === "invalidOutput" && !incident.recovered )) { - signals.add("adjudicationParser"); + signals.add("adjudicationInvalidOutput"); } else { signals.add("adjudicationUnavailable"); } From cbea3b3edd3473286db906bcc6fe76eee5ef2d62 Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Fri, 11 Sep 2026 02:18:23 +0000 Subject: [PATCH 3/4] Keep proxy failure diagnostics neutral about their cause --- bench/src/scorer-eval.test.ts | 18 ++++++++++++++++++ bench/src/scorer-eval.ts | 4 ++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/bench/src/scorer-eval.test.ts b/bench/src/scorer-eval.test.ts index 388a3f0..f6cdabb 100644 --- a/bench/src/scorer-eval.test.ts +++ b/bench/src/scorer-eval.test.ts @@ -500,6 +500,24 @@ describe("scorer case diagnostics", () => { }).failureSignals).toEqual([]); }); + test("counts proxy failures without assuming a transport cause", () => { + const diagnostics = scorerCaseDiagnostics({ + ...diagnosticInput, child: { ...diagnosticInput.child, stderr: "" }, + attempts: [{ + ...accountedAttempt, outcome: "failed", durationMs: 0, + promptTokens: 0, completionTokens: 0, costUsd: null, costProviderDecimal: null, + usageValid: false, httpStatus: null, modelIdentityPresent: false, + providerIdentityPresent: false, usagePresent: false, errorPresent: false, + }], + }); + expect(diagnostics.failureSignals).toEqual(["upstreamFailure"]); + expect(diagnostics.adjudication).toEqual({ + attempts: 1, collectedResponses: 0, validUsageAttempts: 0, exactCostAttempts: 0, + modelIdentityPresentAttempts: 0, providerIdentityPresentAttempts: 0, + }); + expect(diagnostics.scorer.attempts).toBe(0); + }); + test("emits bounded deduplicated codes without copying arbitrary messages or payload properties", () => { const marker = "credential=PRIVATE_MARKER Authorization: Bearer PRIVATE_MARKER"; const diagnostics = scorerCaseDiagnostics({ diff --git a/bench/src/scorer-eval.ts b/bench/src/scorer-eval.ts index 8427a40..2c078f6 100644 --- a/bench/src/scorer-eval.ts +++ b/bench/src/scorer-eval.ts @@ -126,7 +126,7 @@ export interface ScorerEvalCase { type ScorerFailureSignal = "childTimeout" | "invalidEnvelope" | "unsupportedRefutation" | "adjudicationValidation" | "adjudicationInvalidOutput" | "adjudicationProvider" | "adjudicationUnavailable" | "upstreamTimeout" | - "upstreamTransport" | "upstreamHttp" | "unknown"; + "upstreamFailure" | "upstreamHttp" | "unknown"; interface ScorerPhaseDiagnostics { attempts: number; @@ -183,7 +183,7 @@ export function scorerCaseDiagnostics(input: { if (input.envelope === undefined) signals.add("invalidEnvelope"); for (const attempt of input.attempts) { if (attempt.outcome === "timedOut") signals.add("upstreamTimeout"); - if (attempt.outcome === "failed") signals.add("upstreamTransport"); + if (attempt.outcome === "failed") signals.add("upstreamFailure"); if (attempt.httpStatus !== null && attempt.httpStatus >= 400) signals.add("upstreamHttp"); } if (!input.passed && signals.size === 0) signals.add("unknown"); From 36cfb6a7ef19e7a0bcecde6634a170d85425e102 Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Fri, 11 Sep 2026 03:31:44 +0000 Subject: [PATCH 4/4] Count only present decimal strings in scorer diagnostics --- bench/src/scorer-eval.test.ts | 22 ++++++++++++++++++++++ bench/src/scorer-eval.ts | 2 +- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/bench/src/scorer-eval.test.ts b/bench/src/scorer-eval.test.ts index f6cdabb..2066ca9 100644 --- a/bench/src/scorer-eval.test.ts +++ b/bench/src/scorer-eval.test.ts @@ -446,6 +446,28 @@ describe("scorer case diagnostics", () => { attempts: [accountedAttempt], envelope: { findings: [] }, passed: false, }; + test("counts only present string costs at an untyped input boundary", () => { + const missingCost: typeof accountedAttempt = JSON.parse(JSON.stringify({ + ...accountedAttempt, costProviderDecimal: undefined, + })); + expect(missingCost.costProviderDecimal).toBeUndefined(); + for (const phase of ["adjudication", "scorer"] as const) { + const diagnostics = scorerCaseDiagnostics({ + ...diagnosticInput, + attempts: [ + accountedAttempt, + { ...accountedAttempt, costProviderDecimal: "0" }, + { ...accountedAttempt, costProviderDecimal: null }, + missingCost, + ].map((attempt) => ({ ...attempt, phase })), + }); + expect(diagnostics[phase]).toEqual({ + attempts: 4, collectedResponses: 4, validUsageAttempts: 4, exactCostAttempts: 2, + modelIdentityPresentAttempts: 4, providerIdentityPresentAttempts: 4, + }); + } + }); + test("identifies the captured unsupported refutation without treating its accounted call as qualification", () => { const diagnostics = scorerCaseDiagnostics(diagnosticInput); expect(diagnostics.failureSignals).toEqual(["unsupportedRefutation"]); diff --git a/bench/src/scorer-eval.ts b/bench/src/scorer-eval.ts index 2c078f6..00c838d 100644 --- a/bench/src/scorer-eval.ts +++ b/bench/src/scorer-eval.ts @@ -194,7 +194,7 @@ export function scorerCaseDiagnostics(input: { // Completed means the proxy collected a response, not that it was valid. collectedResponses: attempts.filter((attempt) => attempt.outcome === "completed").length, validUsageAttempts: attempts.filter((attempt) => attempt.usageValid).length, - exactCostAttempts: attempts.filter((attempt) => attempt.costProviderDecimal !== null).length, + exactCostAttempts: attempts.filter((attempt) => typeof attempt.costProviderDecimal === "string").length, modelIdentityPresentAttempts: attempts.filter((attempt) => attempt.modelIdentityPresent === true).length, providerIdentityPresentAttempts: attempts.filter((attempt) => attempt.providerIdentityPresent === true).length, };