Retain bounded scorer qualification diagnostics

bench/src/scorer-eval.test.ts

@@ -46,6 +46,7 @@ import {
   reviewCoverageFailure,
   safeSegment,
   scorerCasePasses,
+  scorerCaseDiagnostics,
   scorerCostProviderDecimal,
   scorerEvalRootDir,
   scorerEvaluatorDigest,
@@ -430,6 +431,135 @@ describe("parseRepeatCount", () => {
   });
 });
 
+describe("scorer case diagnostics", () => {
+  const accountedAttempt: Parameters<typeof scorerCaseDiagnostics>[0]["attempts"][number] = {
+    phase: "adjudication", outcome: "completed", durationMs: 100,
+    promptTokens: 1125, completionTokens: 396, costUsd: 0.000548064,
+    costProviderDecimal: "0.000548064", usageValid: true, httpStatus: 200,
+    modelIdentityPresent: true, providerIdentityPresent: true, usagePresent: true,
+    errorPresent: false,
+  };
+  const semanticFailure = "postil: finding adjudication validation failed; preserving all generated findings: refuted adjudication must cite candidate-specific contradictory evidence";
+  const unavailable = "postil: finding adjudication unavailable; preserving all generated findings";
+  const diagnosticInput = {
+    child: { exitCode: 1, stderr: `${semanticFailure}\n`, timedOut: false },
+    attempts: [accountedAttempt], envelope: { findings: [] }, passed: false,
+  };
+
+  test("counts only present string costs at an untyped input boundary", () => {
+    const missingCost: typeof accountedAttempt = JSON.parse(JSON.stringify({
+      ...accountedAttempt, costProviderDecimal: undefined,
+    }));
+    expect(missingCost.costProviderDecimal).toBeUndefined();
+    for (const phase of ["adjudication", "scorer"] as const) {
+      const diagnostics = scorerCaseDiagnostics({
+        ...diagnosticInput,
+        attempts: [
+          accountedAttempt,
+          { ...accountedAttempt, costProviderDecimal: "0" },
+          { ...accountedAttempt, costProviderDecimal: null },
+          missingCost,
+        ].map((attempt) => ({ ...attempt, phase })),
+      });
+      expect(diagnostics[phase]).toEqual({
+        attempts: 4, collectedResponses: 4, validUsageAttempts: 4, exactCostAttempts: 2,
+        modelIdentityPresentAttempts: 4, providerIdentityPresentAttempts: 4,
+      });
+    }
+  });
+
+  test("identifies the captured unsupported refutation without treating its accounted call as qualification", () => {
+    const diagnostics = scorerCaseDiagnostics(diagnosticInput);
+    expect(diagnostics.failureSignals).toEqual(["unsupportedRefutation"]);
+    expect(diagnostics.requiredUpstreamAttempts).toBe(2);
+    expect(diagnostics.adjudication).toEqual({
+      attempts: 1, collectedResponses: 1, validUsageAttempts: 1, exactCostAttempts: 1,
+      modelIdentityPresentAttempts: 1, providerIdentityPresentAttempts: 1,
+    });
+    expect(diagnostics.scorer.attempts).toBe(0);
+    const original = result({
+      id: "clean-comment-only", repeat: 2, scenario: "falseFinding",
+      scorerModel: null, scorerConfidence: null, scorerKind: null,
+      upstreamRequests: 1, usageAccountingComplete: true, usageValid: false,
+      reasonContractValid: false, publicationValid: false, passed: false,
+    });
+    const observed = { ...original, diagnostics };
+    const { diagnostics: _diagnostics, ...priorFields } = observed;
+    expect(priorFields).toEqual(original);
+    expect(isAdmissionFatalStructuralResult(observed, original.model)).toBe(true);
+    expect(aggregate(original.model, [observed], 3)).toEqual(aggregate(original.model, [original], 3));
+    expect(aggregate(original.model, [observed], 3).passed).toBe(false);
+  });
+
+  test("separates native invalid-output and provider failures from unclassified unavailability", () => {
+    const input = { ...diagnosticInput, child: { ...diagnosticInput.child, stderr: unavailable } };
+    expect(scorerCaseDiagnostics({ ...input, envelope: {
+      findings: [], modelIncidents: [{ phase: "scorer", category: "invalidOutput", recovered: false }],
+    } }).failureSignals).toEqual(["adjudicationInvalidOutput"]);
+    expect(scorerCaseDiagnostics({ ...input, envelope: {
+      findings: [{ path: ".postil/provider", title: "Model provider unavailable" }],
+    } }).failureSignals).toEqual(["adjudicationProvider"]);
+    expect(scorerCaseDiagnostics(input).failureSignals).toEqual(["adjudicationUnavailable"]);
+  });
+
+  test("retains timeout and invalid-envelope observations without conflating response collection with success", () => {
+    const diagnostics = scorerCaseDiagnostics({
+      ...diagnosticInput, envelope: undefined,
+      child: { exitCode: undefined, timedOut: true, stderr: "" },
+      attempts: [
+        { ...accountedAttempt, outcome: "timedOut", usageValid: false, costProviderDecimal: null, httpStatus: null },
+        { ...accountedAttempt, phase: "scorer", httpStatus: 503, errorPresent: true },
+      ],
+    });
+    expect(diagnostics.childExitCode).toBeNull();
+    expect(diagnostics.failureSignals).toEqual(["childTimeout", "invalidEnvelope", "upstreamTimeout", "upstreamHttp"]);
+    expect(diagnostics.adjudication.validUsageAttempts).toBe(0);
+    expect(diagnostics.adjudication.exactCostAttempts).toBe(0);
+    expect(diagnostics.scorer.collectedResponses).toBe(1);
+    expect(scorerCaseDiagnostics({
+      ...diagnosticInput, child: { exitCode: 0, timedOut: false, stderr: "" },
+      attempts: [{ ...accountedAttempt, errorPresent: true }], passed: true,
+    }).failureSignals).toEqual([]);
+  });
+
+  test("counts proxy failures without assuming a transport cause", () => {
+    const diagnostics = scorerCaseDiagnostics({
+      ...diagnosticInput, child: { ...diagnosticInput.child, stderr: "" },
+      attempts: [{
+        ...accountedAttempt, outcome: "failed", durationMs: 0,
+        promptTokens: 0, completionTokens: 0, costUsd: null, costProviderDecimal: null,
+        usageValid: false, httpStatus: null, modelIdentityPresent: false,
+        providerIdentityPresent: false, usagePresent: false, errorPresent: false,
+      }],
+    });
+    expect(diagnostics.failureSignals).toEqual(["upstreamFailure"]);
+    expect(diagnostics.adjudication).toEqual({
+      attempts: 1, collectedResponses: 0, validUsageAttempts: 0, exactCostAttempts: 0,
+      modelIdentityPresentAttempts: 0, providerIdentityPresentAttempts: 0,
+    });
+    expect(diagnostics.scorer.attempts).toBe(0);
+  });
+
+  test("emits bounded deduplicated codes without copying arbitrary messages or payload properties", () => {
+    const marker = "credential=PRIVATE_MARKER Authorization: Bearer PRIVATE_MARKER";
+    const diagnostics = scorerCaseDiagnostics({
+      ...diagnosticInput,
+      child: { ...diagnosticInput.child, stderr: `${semanticFailure} ${marker}\n${marker}\n`.repeat(1000) },
+      publicationFailureCodes: ["check-run-state", "check-run-state", marker],
+    });
+    expect(diagnostics.failureSignals).toEqual(["adjudicationValidation"]);
+    expect(diagnostics.publicationFailureCodes).toEqual(["check-run-state"]);
+    expect(JSON.stringify(diagnostics)).not.toContain("PRIVATE_MARKER");
+    expect(JSON.stringify(diagnostics).length).toBeLessThan(1024);
+    expect(scorerCaseDiagnostics({ ...diagnosticInput,
+      child: { ...diagnosticInput.child, stderr: marker },
+    }).failureSignals).toEqual(["unknown"]);
+    expect(scorerCaseDiagnostics({ ...diagnosticInput,
+      child: { ...diagnosticInput.child, stderr: `${semanticFailure}\n`.repeat(1000) },
+    }).failureSignals).toEqual(["unsupportedRefutation"]);
+  });
+});
+
 describe("scorer run artifacts", () => {
   test("binds qualification to clean evaluator source bytes", () => {
     expect(() => assertCleanScorerEvaluatorStatus("")).not.toThrow();
@@ -948,6 +1078,11 @@ describe("scorer proxy and isolated runtime", () => {
         passed: true,
         findingPublished: false,
         gateFailing: false,
+        diagnostics: {
+          childExitCode: 0, requiredUpstreamAttempts: 2, failureSignals: [], publicationFailureCodes: [],
+          adjudication: { attempts: 1, collectedResponses: 1, validUsageAttempts: 1, exactCostAttempts: 1 },
+          scorer: { attempts: 1, collectedResponses: 1, validUsageAttempts: 1, exactCostAttempts: 1 },
+        },
       });
       expect(scorerRequests).toHaveLength(1);
       const scorerRequest = JSON.parse(scorerRequests[0]!) as {
@@ -1253,6 +1388,10 @@ describe("scorer evaluation checkpoints", () => {
       name: "MODEL_BODY_MARKER",
       reason: "API_SECRET_MARKER",
       scorerError: "UPSTREAM_RESPONSE_MARKER",
+      diagnostics: scorerCaseDiagnostics({
+        child: { exitCode: undefined, stderr: "API_SECRET_MARKER", timedOut: true },
+        attempts: [], passed: false,
+      }),
     });
     try {
       await writeScorerEvalCheckpoint(jsonOut, ["scorer/model"], 1, 2, [sensitive]);
@@ -1268,6 +1407,7 @@ describe("scorer evaluation checkpoints", () => {
       });
       expect(first.cases).toHaveLength(1);
       expect(first.cases[0]?.publicationValid).toBe(true);
+      expect(first.cases[0]?.diagnostics).toEqual(sensitive.diagnostics);
       expect(firstRaw).not.toContain("MODEL_BODY_MARKER");
       expect(firstRaw).not.toContain("API_SECRET_MARKER");
       expect(firstRaw).not.toContain("UPSTREAM_RESPONSE_MARKER");
@@ -1646,6 +1786,13 @@ describe("qualification utilities", () => {
       cases: [],
     });
     expect(qualificationExitCode(report([passing]))).toBe(0);
+    const observedCases = qualificationCases(1).map((entry) => ({
+      ...entry,
+      diagnostics: scorerCaseDiagnostics({ child: { exitCode: 0, stderr: "", timedOut: false },
+        attempts: [], envelope: { findings: [] }, passed: entry.passed }),
+    }));
+    expect(aggregate("scorer/model", observedCases, 1)).toEqual(passing);
+    expect(qualificationExitCode({ ...report([passing]), cases: observedCases })).toBe(0);
     expect(qualificationExitCode(report([{ ...passing, passed: false }]))).toBe(1);
     expect(qualificationExitCode({ ...report([passing]), completedCases: 1, matrixComplete: false })).toBe(1);
     expect(qualificationExitCode(report([]))).toBe(1);

bench/src/scorer-eval.ts

@@ -120,6 +120,94 @@ export interface ScorerEvalCase {
   completionTokens: number;
   costUsd: number | null;
   costProviderDecimal?: string | null;
+  diagnostics?: ScorerCaseDiagnostics;
+}
+
+type ScorerFailureSignal = "childTimeout" | "invalidEnvelope" |
+  "unsupportedRefutation" | "adjudicationValidation" | "adjudicationInvalidOutput" |
+  "adjudicationProvider" | "adjudicationUnavailable" | "upstreamTimeout" |
+  "upstreamFailure" | "upstreamHttp" | "unknown";
+
+interface ScorerPhaseDiagnostics {
+  attempts: number;
+  collectedResponses: number;
+  validUsageAttempts: number;
+  exactCostAttempts: number;
+  modelIdentityPresentAttempts: number;
+  providerIdentityPresentAttempts: number;
+}
+
+export interface ScorerCaseDiagnostics {
+  childExitCode: number | null;
+  requiredUpstreamAttempts: 2;
+  adjudication: ScorerPhaseDiagnostics;
+  scorer: ScorerPhaseDiagnostics;
+  failureSignals: ScorerFailureSignal[];
+  publicationFailureCodes: string[];
+}
+
+// Diagnostics observe native results and never participate in qualification.
+// Output contains only fixed codes and counts, including in partial reports.
+export function scorerCaseDiagnostics(input: {
+  child: Pick<BoundedChildResult, "exitCode" | "stderr" | "timedOut">;
+  attempts: readonly ScorerAttempt[];
+  envelope?: {
+    findings: readonly { path: string; title: string }[];
+    modelIncidents?: readonly { phase: string; category: string; recovered: boolean }[];
+  };
+  publicationFailureCodes?: readonly string[];
+  passed: boolean;
+}): ScorerCaseDiagnostics {
+  const signals = new Set<ScorerFailureSignal>();
+  const lines = new Set(input.child.stderr.split(/\r?\n/u));
+  const validationPrefix = "postil: finding adjudication validation failed; preserving all generated findings: ";
+  if (lines.has(`${validationPrefix}refuted adjudication must cite candidate-specific contradictory evidence`)) {
+    signals.add("unsupportedRefutation");
+  } else if ([...lines].some((line) => line.startsWith(validationPrefix))) {
+    signals.add("adjudicationValidation");
+  }
+  if (lines.has("postil: finding adjudication unavailable; preserving all generated findings")) {
+    if (input.envelope?.findings.some((finding) =>
+      finding.path === ".postil/provider" && finding.title === "Model provider unavailable"
+    )) {
+      signals.add("adjudicationProvider");
+    } else if (input.envelope?.modelIncidents?.some((incident) =>
+      incident.phase === "scorer" && incident.category === "invalidOutput" && !incident.recovered
+    )) {
+      signals.add("adjudicationInvalidOutput");
+    } else {
+      signals.add("adjudicationUnavailable");
+    }
+  }
+  if (input.child.timedOut) signals.add("childTimeout");
+  if (input.envelope === undefined) signals.add("invalidEnvelope");
+  for (const attempt of input.attempts) {
+    if (attempt.outcome === "timedOut") signals.add("upstreamTimeout");
+    if (attempt.outcome === "failed") signals.add("upstreamFailure");
+    if (attempt.httpStatus !== null && attempt.httpStatus >= 400) signals.add("upstreamHttp");
+  }
+  if (!input.passed && signals.size === 0) signals.add("unknown");
+  const phaseCounts = (phase: ScorerAttempt["phase"]): ScorerPhaseDiagnostics => {
+    const attempts = input.attempts.filter((attempt) => attempt.phase === phase);
+    return {
+      attempts: attempts.length,
+      // Completed means the proxy collected a response, not that it was valid.
+      collectedResponses: attempts.filter((attempt) => attempt.outcome === "completed").length,
+      validUsageAttempts: attempts.filter((attempt) => attempt.usageValid).length,
+      exactCostAttempts: attempts.filter((attempt) => typeof attempt.costProviderDecimal === "string").length,
+      modelIdentityPresentAttempts: attempts.filter((attempt) => attempt.modelIdentityPresent === true).length,
+      providerIdentityPresentAttempts: attempts.filter((attempt) => attempt.providerIdentityPresent === true).length,
+    };
+  };
+  return {
+    childExitCode: input.child.exitCode ?? null,
+    requiredUpstreamAttempts: 2,
+    adjudication: phaseCounts("adjudication"),
+    scorer: phaseCounts("scorer"),
+    failureSignals: [...signals],
+    publicationFailureCodes: ["check-run-state", "review-count", "comment-count", "missing-anchor"]
+      .filter((code) => input.publicationFailureCodes?.includes(code)),
+  };
 }
 
 export interface ScorerEvalAggregate {
@@ -895,6 +983,7 @@ export async function runScorerEvalCase(
         caseTimedOut,
       ),
       ...telemetry,
+      diagnostics: scorerCaseDiagnostics({ child, attempts: proxy.attempts, passed: false }),
     };
   }
   const parsed = parsedEnvelope.data;
@@ -1057,6 +1146,9 @@ export async function runScorerEvalCase(
     publicationValid,
     ...telemetry,
     costProviderDecimal,
+    diagnostics: scorerCaseDiagnostics({
+      child, attempts: proxy.attempts, envelope: parsed, publicationFailureCodes, passed,
+    }),
   };
 }