diff --git a/bench/src/scorer-eval.test.ts b/bench/src/scorer-eval.test.ts index 837630a..2066ca9 100644 --- a/bench/src/scorer-eval.test.ts +++ b/bench/src/scorer-eval.test.ts @@ -46,6 +46,7 @@ import { reviewCoverageFailure, safeSegment, scorerCasePasses, + scorerCaseDiagnostics, scorerCostProviderDecimal, scorerEvalRootDir, scorerEvaluatorDigest, @@ -430,6 +431,135 @@ describe("parseRepeatCount", () => { }); }); +describe("scorer case diagnostics", () => { + const accountedAttempt: Parameters[0]["attempts"][number] = { + phase: "adjudication", outcome: "completed", durationMs: 100, + promptTokens: 1125, completionTokens: 396, costUsd: 0.000548064, + costProviderDecimal: "0.000548064", usageValid: true, httpStatus: 200, + modelIdentityPresent: true, providerIdentityPresent: true, usagePresent: true, + errorPresent: false, + }; + const semanticFailure = "postil: finding adjudication validation failed; preserving all generated findings: refuted adjudication must cite candidate-specific contradictory evidence"; + const unavailable = "postil: finding adjudication unavailable; preserving all generated findings"; + const diagnosticInput = { + child: { exitCode: 1, stderr: `${semanticFailure}\n`, timedOut: false }, + attempts: [accountedAttempt], envelope: { findings: [] }, passed: false, + }; + + test("counts only present string costs at an untyped input boundary", () => { + const missingCost: typeof accountedAttempt = JSON.parse(JSON.stringify({ + ...accountedAttempt, costProviderDecimal: undefined, + })); + expect(missingCost.costProviderDecimal).toBeUndefined(); + for (const phase of ["adjudication", "scorer"] as const) { + const diagnostics = scorerCaseDiagnostics({ + ...diagnosticInput, + attempts: [ + accountedAttempt, + { ...accountedAttempt, costProviderDecimal: "0" }, + { ...accountedAttempt, costProviderDecimal: null }, + missingCost, + ].map((attempt) => ({ ...attempt, phase })), + }); + expect(diagnostics[phase]).toEqual({ + attempts: 4, collectedResponses: 4, validUsageAttempts: 4, exactCostAttempts: 2, + modelIdentityPresentAttempts: 4, providerIdentityPresentAttempts: 4, + }); + } + }); + + test("identifies the captured unsupported refutation without treating its accounted call as qualification", () => { + const diagnostics = scorerCaseDiagnostics(diagnosticInput); + expect(diagnostics.failureSignals).toEqual(["unsupportedRefutation"]); + expect(diagnostics.requiredUpstreamAttempts).toBe(2); + expect(diagnostics.adjudication).toEqual({ + attempts: 1, collectedResponses: 1, validUsageAttempts: 1, exactCostAttempts: 1, + modelIdentityPresentAttempts: 1, providerIdentityPresentAttempts: 1, + }); + expect(diagnostics.scorer.attempts).toBe(0); + const original = result({ + id: "clean-comment-only", repeat: 2, scenario: "falseFinding", + scorerModel: null, scorerConfidence: null, scorerKind: null, + upstreamRequests: 1, usageAccountingComplete: true, usageValid: false, + reasonContractValid: false, publicationValid: false, passed: false, + }); + const observed = { ...original, diagnostics }; + const { diagnostics: _diagnostics, ...priorFields } = observed; + expect(priorFields).toEqual(original); + expect(isAdmissionFatalStructuralResult(observed, original.model)).toBe(true); + expect(aggregate(original.model, [observed], 3)).toEqual(aggregate(original.model, [original], 3)); + expect(aggregate(original.model, [observed], 3).passed).toBe(false); + }); + + test("separates native invalid-output and provider failures from unclassified unavailability", () => { + const input = { ...diagnosticInput, child: { ...diagnosticInput.child, stderr: unavailable } }; + expect(scorerCaseDiagnostics({ ...input, envelope: { + findings: [], modelIncidents: [{ phase: "scorer", category: "invalidOutput", recovered: false }], + } }).failureSignals).toEqual(["adjudicationInvalidOutput"]); + expect(scorerCaseDiagnostics({ ...input, envelope: { + findings: [{ path: ".postil/provider", title: "Model provider unavailable" }], + } }).failureSignals).toEqual(["adjudicationProvider"]); + expect(scorerCaseDiagnostics(input).failureSignals).toEqual(["adjudicationUnavailable"]); + }); + + test("retains timeout and invalid-envelope observations without conflating response collection with success", () => { + const diagnostics = scorerCaseDiagnostics({ + ...diagnosticInput, envelope: undefined, + child: { exitCode: undefined, timedOut: true, stderr: "" }, + attempts: [ + { ...accountedAttempt, outcome: "timedOut", usageValid: false, costProviderDecimal: null, httpStatus: null }, + { ...accountedAttempt, phase: "scorer", httpStatus: 503, errorPresent: true }, + ], + }); + expect(diagnostics.childExitCode).toBeNull(); + expect(diagnostics.failureSignals).toEqual(["childTimeout", "invalidEnvelope", "upstreamTimeout", "upstreamHttp"]); + expect(diagnostics.adjudication.validUsageAttempts).toBe(0); + expect(diagnostics.adjudication.exactCostAttempts).toBe(0); + expect(diagnostics.scorer.collectedResponses).toBe(1); + expect(scorerCaseDiagnostics({ + ...diagnosticInput, child: { exitCode: 0, timedOut: false, stderr: "" }, + attempts: [{ ...accountedAttempt, errorPresent: true }], passed: true, + }).failureSignals).toEqual([]); + }); + + test("counts proxy failures without assuming a transport cause", () => { + const diagnostics = scorerCaseDiagnostics({ + ...diagnosticInput, child: { ...diagnosticInput.child, stderr: "" }, + attempts: [{ + ...accountedAttempt, outcome: "failed", durationMs: 0, + promptTokens: 0, completionTokens: 0, costUsd: null, costProviderDecimal: null, + usageValid: false, httpStatus: null, modelIdentityPresent: false, + providerIdentityPresent: false, usagePresent: false, errorPresent: false, + }], + }); + expect(diagnostics.failureSignals).toEqual(["upstreamFailure"]); + expect(diagnostics.adjudication).toEqual({ + attempts: 1, collectedResponses: 0, validUsageAttempts: 0, exactCostAttempts: 0, + modelIdentityPresentAttempts: 0, providerIdentityPresentAttempts: 0, + }); + expect(diagnostics.scorer.attempts).toBe(0); + }); + + test("emits bounded deduplicated codes without copying arbitrary messages or payload properties", () => { + const marker = "credential=PRIVATE_MARKER Authorization: Bearer PRIVATE_MARKER"; + const diagnostics = scorerCaseDiagnostics({ + ...diagnosticInput, + child: { ...diagnosticInput.child, stderr: `${semanticFailure} ${marker}\n${marker}\n`.repeat(1000) }, + publicationFailureCodes: ["check-run-state", "check-run-state", marker], + }); + expect(diagnostics.failureSignals).toEqual(["adjudicationValidation"]); + expect(diagnostics.publicationFailureCodes).toEqual(["check-run-state"]); + expect(JSON.stringify(diagnostics)).not.toContain("PRIVATE_MARKER"); + expect(JSON.stringify(diagnostics).length).toBeLessThan(1024); + expect(scorerCaseDiagnostics({ ...diagnosticInput, + child: { ...diagnosticInput.child, stderr: marker }, + }).failureSignals).toEqual(["unknown"]); + expect(scorerCaseDiagnostics({ ...diagnosticInput, + child: { ...diagnosticInput.child, stderr: `${semanticFailure}\n`.repeat(1000) }, + }).failureSignals).toEqual(["unsupportedRefutation"]); + }); +}); + describe("scorer run artifacts", () => { test("binds qualification to clean evaluator source bytes", () => { expect(() => assertCleanScorerEvaluatorStatus("")).not.toThrow(); @@ -948,6 +1078,11 @@ describe("scorer proxy and isolated runtime", () => { passed: true, findingPublished: false, gateFailing: false, + diagnostics: { + childExitCode: 0, requiredUpstreamAttempts: 2, failureSignals: [], publicationFailureCodes: [], + adjudication: { attempts: 1, collectedResponses: 1, validUsageAttempts: 1, exactCostAttempts: 1 }, + scorer: { attempts: 1, collectedResponses: 1, validUsageAttempts: 1, exactCostAttempts: 1 }, + }, }); expect(scorerRequests).toHaveLength(1); const scorerRequest = JSON.parse(scorerRequests[0]!) as { @@ -1253,6 +1388,10 @@ describe("scorer evaluation checkpoints", () => { name: "MODEL_BODY_MARKER", reason: "API_SECRET_MARKER", scorerError: "UPSTREAM_RESPONSE_MARKER", + diagnostics: scorerCaseDiagnostics({ + child: { exitCode: undefined, stderr: "API_SECRET_MARKER", timedOut: true }, + attempts: [], passed: false, + }), }); try { await writeScorerEvalCheckpoint(jsonOut, ["scorer/model"], 1, 2, [sensitive]); @@ -1268,6 +1407,7 @@ describe("scorer evaluation checkpoints", () => { }); expect(first.cases).toHaveLength(1); expect(first.cases[0]?.publicationValid).toBe(true); + expect(first.cases[0]?.diagnostics).toEqual(sensitive.diagnostics); expect(firstRaw).not.toContain("MODEL_BODY_MARKER"); expect(firstRaw).not.toContain("API_SECRET_MARKER"); expect(firstRaw).not.toContain("UPSTREAM_RESPONSE_MARKER"); @@ -1646,6 +1786,13 @@ describe("qualification utilities", () => { cases: [], }); expect(qualificationExitCode(report([passing]))).toBe(0); + const observedCases = qualificationCases(1).map((entry) => ({ + ...entry, + diagnostics: scorerCaseDiagnostics({ child: { exitCode: 0, stderr: "", timedOut: false }, + attempts: [], envelope: { findings: [] }, passed: entry.passed }), + })); + expect(aggregate("scorer/model", observedCases, 1)).toEqual(passing); + expect(qualificationExitCode({ ...report([passing]), cases: observedCases })).toBe(0); expect(qualificationExitCode(report([{ ...passing, passed: false }]))).toBe(1); expect(qualificationExitCode({ ...report([passing]), completedCases: 1, matrixComplete: false })).toBe(1); expect(qualificationExitCode(report([]))).toBe(1); diff --git a/bench/src/scorer-eval.ts b/bench/src/scorer-eval.ts index d047e00..00c838d 100644 --- a/bench/src/scorer-eval.ts +++ b/bench/src/scorer-eval.ts @@ -120,6 +120,94 @@ export interface ScorerEvalCase { completionTokens: number; costUsd: number | null; costProviderDecimal?: string | null; + diagnostics?: ScorerCaseDiagnostics; +} + +type ScorerFailureSignal = "childTimeout" | "invalidEnvelope" | + "unsupportedRefutation" | "adjudicationValidation" | "adjudicationInvalidOutput" | + "adjudicationProvider" | "adjudicationUnavailable" | "upstreamTimeout" | + "upstreamFailure" | "upstreamHttp" | "unknown"; + +interface ScorerPhaseDiagnostics { + attempts: number; + collectedResponses: number; + validUsageAttempts: number; + exactCostAttempts: number; + modelIdentityPresentAttempts: number; + providerIdentityPresentAttempts: number; +} + +export interface ScorerCaseDiagnostics { + childExitCode: number | null; + requiredUpstreamAttempts: 2; + adjudication: ScorerPhaseDiagnostics; + scorer: ScorerPhaseDiagnostics; + failureSignals: ScorerFailureSignal[]; + publicationFailureCodes: string[]; +} + +// Diagnostics observe native results and never participate in qualification. +// Output contains only fixed codes and counts, including in partial reports. +export function scorerCaseDiagnostics(input: { + child: Pick; + attempts: readonly ScorerAttempt[]; + envelope?: { + findings: readonly { path: string; title: string }[]; + modelIncidents?: readonly { phase: string; category: string; recovered: boolean }[]; + }; + publicationFailureCodes?: readonly string[]; + passed: boolean; +}): ScorerCaseDiagnostics { + const signals = new Set(); + const lines = new Set(input.child.stderr.split(/\r?\n/u)); + const validationPrefix = "postil: finding adjudication validation failed; preserving all generated findings: "; + if (lines.has(`${validationPrefix}refuted adjudication must cite candidate-specific contradictory evidence`)) { + signals.add("unsupportedRefutation"); + } else if ([...lines].some((line) => line.startsWith(validationPrefix))) { + signals.add("adjudicationValidation"); + } + if (lines.has("postil: finding adjudication unavailable; preserving all generated findings")) { + if (input.envelope?.findings.some((finding) => + finding.path === ".postil/provider" && finding.title === "Model provider unavailable" + )) { + signals.add("adjudicationProvider"); + } else if (input.envelope?.modelIncidents?.some((incident) => + incident.phase === "scorer" && incident.category === "invalidOutput" && !incident.recovered + )) { + signals.add("adjudicationInvalidOutput"); + } else { + signals.add("adjudicationUnavailable"); + } + } + if (input.child.timedOut) signals.add("childTimeout"); + if (input.envelope === undefined) signals.add("invalidEnvelope"); + for (const attempt of input.attempts) { + if (attempt.outcome === "timedOut") signals.add("upstreamTimeout"); + if (attempt.outcome === "failed") signals.add("upstreamFailure"); + if (attempt.httpStatus !== null && attempt.httpStatus >= 400) signals.add("upstreamHttp"); + } + if (!input.passed && signals.size === 0) signals.add("unknown"); + const phaseCounts = (phase: ScorerAttempt["phase"]): ScorerPhaseDiagnostics => { + const attempts = input.attempts.filter((attempt) => attempt.phase === phase); + return { + attempts: attempts.length, + // Completed means the proxy collected a response, not that it was valid. + collectedResponses: attempts.filter((attempt) => attempt.outcome === "completed").length, + validUsageAttempts: attempts.filter((attempt) => attempt.usageValid).length, + exactCostAttempts: attempts.filter((attempt) => typeof attempt.costProviderDecimal === "string").length, + modelIdentityPresentAttempts: attempts.filter((attempt) => attempt.modelIdentityPresent === true).length, + providerIdentityPresentAttempts: attempts.filter((attempt) => attempt.providerIdentityPresent === true).length, + }; + }; + return { + childExitCode: input.child.exitCode ?? null, + requiredUpstreamAttempts: 2, + adjudication: phaseCounts("adjudication"), + scorer: phaseCounts("scorer"), + failureSignals: [...signals], + publicationFailureCodes: ["check-run-state", "review-count", "comment-count", "missing-anchor"] + .filter((code) => input.publicationFailureCodes?.includes(code)), + }; } export interface ScorerEvalAggregate { @@ -895,6 +983,7 @@ export async function runScorerEvalCase( caseTimedOut, ), ...telemetry, + diagnostics: scorerCaseDiagnostics({ child, attempts: proxy.attempts, passed: false }), }; } const parsed = parsedEnvelope.data; @@ -1057,6 +1146,9 @@ export async function runScorerEvalCase( publicationValid, ...telemetry, costProviderDecimal, + diagnostics: scorerCaseDiagnostics({ + child, attempts: proxy.attempts, envelope: parsed, publicationFailureCodes, passed, + }), }; }