From 9e0fdb687cb3f59502b337f8591e7133634497b4 Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Fri, 11 Sep 2026 02:08:36 +0000 Subject: [PATCH 1/6] Retain bounded scorer failure diagnostics in reports --- bench/src/scorer-eval.test.ts | 107 ++++++++++++++++++++++++++++++++++ bench/src/scorer-eval.ts | 92 +++++++++++++++++++++++++++++ 2 files changed, 199 insertions(+) diff --git a/bench/src/scorer-eval.test.ts b/bench/src/scorer-eval.test.ts index 837630a..cdc9cc5 100644 --- a/bench/src/scorer-eval.test.ts +++ b/bench/src/scorer-eval.test.ts @@ -46,6 +46,7 @@ import { reviewCoverageFailure, safeSegment, scorerCasePasses, + scorerCaseDiagnostics, scorerCostProviderDecimal, scorerEvalRootDir, scorerEvaluatorDigest, @@ -430,6 +431,95 @@ describe("parseRepeatCount", () => { }); }); +describe("scorer case diagnostics", () => { + const accountedAttempt: Parameters[0]["attempts"][number] = { + phase: "adjudication", outcome: "completed", durationMs: 100, + promptTokens: 1125, completionTokens: 396, costUsd: 0.000548064, + costProviderDecimal: "0.000548064", usageValid: true, httpStatus: 200, + modelIdentityPresent: true, providerIdentityPresent: true, usagePresent: true, + errorPresent: false, + }; + const semanticFailure = "postil: finding adjudication validation failed; preserving all generated findings: refuted adjudication must cite candidate-specific contradictory evidence"; + const unavailable = "postil: finding adjudication unavailable; preserving all generated findings"; + const diagnosticInput = { + child: { exitCode: 1, stderr: `${semanticFailure}\n`, timedOut: false }, + attempts: [accountedAttempt], envelope: { findings: [] }, passed: false, + }; + + test("identifies the captured unsupported refutation without treating its accounted call as qualification", () => { + const diagnostics = scorerCaseDiagnostics(diagnosticInput); + expect(diagnostics.failureSignals).toEqual(["unsupportedRefutation"]); + expect(diagnostics.requiredUpstreamAttempts).toBe(2); + expect(diagnostics.adjudication).toEqual({ + attempts: 1, collectedResponses: 1, validUsageAttempts: 1, exactCostAttempts: 1, + modelIdentityPresentAttempts: 1, providerIdentityPresentAttempts: 1, + }); + expect(diagnostics.scorer.attempts).toBe(0); + const original = result({ + id: "clean-comment-only", repeat: 2, scenario: "falseFinding", + scorerModel: null, scorerConfidence: null, scorerKind: null, + upstreamRequests: 1, usageAccountingComplete: true, usageValid: false, + reasonContractValid: false, publicationValid: false, passed: false, + }); + const observed = { ...original, diagnostics }; + const { diagnostics: _diagnostics, ...priorFields } = observed; + expect(priorFields).toEqual(original); + expect(isAdmissionFatalStructuralResult(observed, original.model)).toBe(true); + expect(aggregate(original.model, [observed], 3)).toEqual(aggregate(original.model, [original], 3)); + expect(aggregate(original.model, [observed], 3).passed).toBe(false); + }); + + test("separates native parser and provider failures from unclassified unavailability", () => { + const input = { ...diagnosticInput, child: { ...diagnosticInput.child, stderr: unavailable } }; + expect(scorerCaseDiagnostics({ ...input, envelope: { + findings: [], modelIncidents: [{ phase: "scorer", category: "invalidOutput", recovered: false }], + } }).failureSignals).toEqual(["adjudicationParser"]); + expect(scorerCaseDiagnostics({ ...input, envelope: { + findings: [{ path: ".postil/provider", title: "Model provider unavailable" }], + } }).failureSignals).toEqual(["adjudicationProvider"]); + expect(scorerCaseDiagnostics(input).failureSignals).toEqual(["adjudicationUnavailable"]); + }); + + test("retains timeout and invalid-envelope observations without conflating response collection with success", () => { + const diagnostics = scorerCaseDiagnostics({ + ...diagnosticInput, envelope: undefined, + child: { exitCode: undefined, timedOut: true, stderr: "" }, + attempts: [ + { ...accountedAttempt, outcome: "timedOut", usageValid: false, costProviderDecimal: null, httpStatus: null }, + { ...accountedAttempt, phase: "scorer", httpStatus: 503, errorPresent: true }, + ], + }); + expect(diagnostics.childExitCode).toBeNull(); + expect(diagnostics.failureSignals).toEqual(["childTimeout", "invalidEnvelope", "upstreamTimeout", "upstreamHttp"]); + expect(diagnostics.adjudication.validUsageAttempts).toBe(0); + expect(diagnostics.adjudication.exactCostAttempts).toBe(0); + expect(diagnostics.scorer.collectedResponses).toBe(1); + expect(scorerCaseDiagnostics({ + ...diagnosticInput, child: { exitCode: 0, timedOut: false, stderr: "" }, + attempts: [{ ...accountedAttempt, errorPresent: true }], passed: true, + }).failureSignals).toEqual([]); + }); + + test("emits bounded deduplicated codes without copying arbitrary messages or payload properties", () => { + const marker = "credential=PRIVATE_MARKER Authorization: Bearer PRIVATE_MARKER"; + const diagnostics = scorerCaseDiagnostics({ + ...diagnosticInput, + child: { ...diagnosticInput.child, stderr: `${semanticFailure} ${marker}\n${marker}\n`.repeat(1000) }, + publicationFailureCodes: ["check-run-state", "check-run-state", marker], + }); + expect(diagnostics.failureSignals).toEqual(["adjudicationValidation"]); + expect(diagnostics.publicationFailureCodes).toEqual(["check-run-state"]); + expect(JSON.stringify(diagnostics)).not.toContain("PRIVATE_MARKER"); + expect(JSON.stringify(diagnostics).length).toBeLessThan(1024); + expect(scorerCaseDiagnostics({ ...diagnosticInput, + child: { ...diagnosticInput.child, stderr: marker }, + }).failureSignals).toEqual(["unknown"]); + expect(scorerCaseDiagnostics({ ...diagnosticInput, + child: { ...diagnosticInput.child, stderr: `${semanticFailure}\n`.repeat(1000) }, + }).failureSignals).toEqual(["unsupportedRefutation"]); + }); +}); + describe("scorer run artifacts", () => { test("binds qualification to clean evaluator source bytes", () => { expect(() => assertCleanScorerEvaluatorStatus("")).not.toThrow(); @@ -948,6 +1038,11 @@ describe("scorer proxy and isolated runtime", () => { passed: true, findingPublished: false, gateFailing: false, + diagnostics: { + childExitCode: 0, requiredUpstreamAttempts: 2, failureSignals: [], publicationFailureCodes: [], + adjudication: { attempts: 1, collectedResponses: 1, validUsageAttempts: 1, exactCostAttempts: 1 }, + scorer: { attempts: 1, collectedResponses: 1, validUsageAttempts: 1, exactCostAttempts: 1 }, + }, }); expect(scorerRequests).toHaveLength(1); const scorerRequest = JSON.parse(scorerRequests[0]!) as { @@ -1253,6 +1348,10 @@ describe("scorer evaluation checkpoints", () => { name: "MODEL_BODY_MARKER", reason: "API_SECRET_MARKER", scorerError: "UPSTREAM_RESPONSE_MARKER", + diagnostics: scorerCaseDiagnostics({ + child: { exitCode: undefined, stderr: "API_SECRET_MARKER", timedOut: true }, + attempts: [], passed: false, + }), }); try { await writeScorerEvalCheckpoint(jsonOut, ["scorer/model"], 1, 2, [sensitive]); @@ -1268,6 +1367,7 @@ describe("scorer evaluation checkpoints", () => { }); expect(first.cases).toHaveLength(1); expect(first.cases[0]?.publicationValid).toBe(true); + expect(first.cases[0]?.diagnostics).toEqual(sensitive.diagnostics); expect(firstRaw).not.toContain("MODEL_BODY_MARKER"); expect(firstRaw).not.toContain("API_SECRET_MARKER"); expect(firstRaw).not.toContain("UPSTREAM_RESPONSE_MARKER"); @@ -1646,6 +1746,13 @@ describe("qualification utilities", () => { cases: [], }); expect(qualificationExitCode(report([passing]))).toBe(0); + const observedCases = qualificationCases(1).map((entry) => ({ + ...entry, + diagnostics: scorerCaseDiagnostics({ child: { exitCode: 0, stderr: "", timedOut: false }, + attempts: [], envelope: { findings: [] }, passed: entry.passed }), + })); + expect(aggregate("scorer/model", observedCases, 1)).toEqual(passing); + expect(qualificationExitCode({ ...report([passing]), cases: observedCases })).toBe(0); expect(qualificationExitCode(report([{ ...passing, passed: false }]))).toBe(1); expect(qualificationExitCode({ ...report([passing]), completedCases: 1, matrixComplete: false })).toBe(1); expect(qualificationExitCode(report([]))).toBe(1); diff --git a/bench/src/scorer-eval.ts b/bench/src/scorer-eval.ts index d047e00..ec4891e 100644 --- a/bench/src/scorer-eval.ts +++ b/bench/src/scorer-eval.ts @@ -120,6 +120,94 @@ export interface ScorerEvalCase { completionTokens: number; costUsd: number | null; costProviderDecimal?: string | null; + diagnostics?: ScorerCaseDiagnostics; +} + +type ScorerFailureSignal = "childTimeout" | "invalidEnvelope" | + "unsupportedRefutation" | "adjudicationValidation" | "adjudicationParser" | + "adjudicationProvider" | "adjudicationUnavailable" | "upstreamTimeout" | + "upstreamTransport" | "upstreamHttp" | "unknown"; + +interface ScorerPhaseDiagnostics { + attempts: number; + collectedResponses: number; + validUsageAttempts: number; + exactCostAttempts: number; + modelIdentityPresentAttempts: number; + providerIdentityPresentAttempts: number; +} + +export interface ScorerCaseDiagnostics { + childExitCode: number | null; + requiredUpstreamAttempts: 2; + adjudication: ScorerPhaseDiagnostics; + scorer: ScorerPhaseDiagnostics; + failureSignals: ScorerFailureSignal[]; + publicationFailureCodes: string[]; +} + +// Diagnostics observe native results and never participate in qualification. +// Output contains only fixed codes and counts, including in partial reports. +export function scorerCaseDiagnostics(input: { + child: Pick; + attempts: readonly ScorerAttempt[]; + envelope?: { + findings: readonly { path: string; title: string }[]; + modelIncidents?: readonly { phase: string; category: string; recovered: boolean }[]; + }; + publicationFailureCodes?: readonly string[]; + passed: boolean; +}): ScorerCaseDiagnostics { + const signals = new Set(); + const lines = new Set(input.child.stderr.split(/\r?\n/u)); + const validationPrefix = "postil: finding adjudication validation failed; preserving all generated findings: "; + if (lines.has(`${validationPrefix}refuted adjudication must cite candidate-specific contradictory evidence`)) { + signals.add("unsupportedRefutation"); + } else if ([...lines].some((line) => line.startsWith(validationPrefix))) { + signals.add("adjudicationValidation"); + } + if (lines.has("postil: finding adjudication unavailable; preserving all generated findings")) { + if (input.envelope?.findings.some((finding) => + finding.path === ".postil/provider" && finding.title === "Model provider unavailable" + )) { + signals.add("adjudicationProvider"); + } else if (input.envelope?.modelIncidents?.some((incident) => + incident.phase === "scorer" && incident.category === "invalidOutput" && !incident.recovered + )) { + signals.add("adjudicationParser"); + } else { + signals.add("adjudicationUnavailable"); + } + } + if (input.child.timedOut) signals.add("childTimeout"); + if (input.envelope === undefined) signals.add("invalidEnvelope"); + for (const attempt of input.attempts) { + if (attempt.outcome === "timedOut") signals.add("upstreamTimeout"); + if (attempt.outcome === "failed") signals.add("upstreamTransport"); + if (attempt.httpStatus !== null && attempt.httpStatus >= 400) signals.add("upstreamHttp"); + } + if (!input.passed && signals.size === 0) signals.add("unknown"); + const phaseCounts = (phase: ScorerAttempt["phase"]): ScorerPhaseDiagnostics => { + const attempts = input.attempts.filter((attempt) => attempt.phase === phase); + return { + attempts: attempts.length, + // Completed means the proxy collected a response, not that it was valid. + collectedResponses: attempts.filter((attempt) => attempt.outcome === "completed").length, + validUsageAttempts: attempts.filter((attempt) => attempt.usageValid).length, + exactCostAttempts: attempts.filter((attempt) => attempt.costProviderDecimal !== null).length, + modelIdentityPresentAttempts: attempts.filter((attempt) => attempt.modelIdentityPresent === true).length, + providerIdentityPresentAttempts: attempts.filter((attempt) => attempt.providerIdentityPresent === true).length, + }; + }; + return { + childExitCode: input.child.exitCode ?? null, + requiredUpstreamAttempts: 2, + adjudication: phaseCounts("adjudication"), + scorer: phaseCounts("scorer"), + failureSignals: [...signals], + publicationFailureCodes: ["check-run-state", "review-count", "comment-count", "missing-anchor"] + .filter((code) => input.publicationFailureCodes?.includes(code)), + }; } export interface ScorerEvalAggregate { @@ -895,6 +983,7 @@ export async function runScorerEvalCase( caseTimedOut, ), ...telemetry, + diagnostics: scorerCaseDiagnostics({ child, attempts: proxy.attempts, passed: false }), }; } const parsed = parsedEnvelope.data; @@ -1057,6 +1146,9 @@ export async function runScorerEvalCase( publicationValid, ...telemetry, costProviderDecimal, + diagnostics: scorerCaseDiagnostics({ + child, attempts: proxy.attempts, envelope: parsed, publicationFailureCodes, passed, + }), }; } From b3fee498f77268716a40d8a42f83d5ff9df4c43e Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Fri, 11 Sep 2026 02:14:25 +0000 Subject: [PATCH 2/6] Describe native adjudication invalid-output diagnostics accurately --- bench/src/scorer-eval.test.ts | 4 ++-- bench/src/scorer-eval.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bench/src/scorer-eval.test.ts b/bench/src/scorer-eval.test.ts index cdc9cc5..388a3f0 100644 --- a/bench/src/scorer-eval.test.ts +++ b/bench/src/scorer-eval.test.ts @@ -469,11 +469,11 @@ describe("scorer case diagnostics", () => { expect(aggregate(original.model, [observed], 3).passed).toBe(false); }); - test("separates native parser and provider failures from unclassified unavailability", () => { + test("separates native invalid-output and provider failures from unclassified unavailability", () => { const input = { ...diagnosticInput, child: { ...diagnosticInput.child, stderr: unavailable } }; expect(scorerCaseDiagnostics({ ...input, envelope: { findings: [], modelIncidents: [{ phase: "scorer", category: "invalidOutput", recovered: false }], - } }).failureSignals).toEqual(["adjudicationParser"]); + } }).failureSignals).toEqual(["adjudicationInvalidOutput"]); expect(scorerCaseDiagnostics({ ...input, envelope: { findings: [{ path: ".postil/provider", title: "Model provider unavailable" }], } }).failureSignals).toEqual(["adjudicationProvider"]); diff --git a/bench/src/scorer-eval.ts b/bench/src/scorer-eval.ts index ec4891e..8427a40 100644 --- a/bench/src/scorer-eval.ts +++ b/bench/src/scorer-eval.ts @@ -124,7 +124,7 @@ export interface ScorerEvalCase { } type ScorerFailureSignal = "childTimeout" | "invalidEnvelope" | - "unsupportedRefutation" | "adjudicationValidation" | "adjudicationParser" | + "unsupportedRefutation" | "adjudicationValidation" | "adjudicationInvalidOutput" | "adjudicationProvider" | "adjudicationUnavailable" | "upstreamTimeout" | "upstreamTransport" | "upstreamHttp" | "unknown"; @@ -174,7 +174,7 @@ export function scorerCaseDiagnostics(input: { } else if (input.envelope?.modelIncidents?.some((incident) => incident.phase === "scorer" && incident.category === "invalidOutput" && !incident.recovered )) { - signals.add("adjudicationParser"); + signals.add("adjudicationInvalidOutput"); } else { signals.add("adjudicationUnavailable"); } From cbea3b3edd3473286db906bcc6fe76eee5ef2d62 Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Fri, 11 Sep 2026 02:18:23 +0000 Subject: [PATCH 3/6] Keep proxy failure diagnostics neutral about their cause --- bench/src/scorer-eval.test.ts | 18 ++++++++++++++++++ bench/src/scorer-eval.ts | 4 ++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/bench/src/scorer-eval.test.ts b/bench/src/scorer-eval.test.ts index 388a3f0..f6cdabb 100644 --- a/bench/src/scorer-eval.test.ts +++ b/bench/src/scorer-eval.test.ts @@ -500,6 +500,24 @@ describe("scorer case diagnostics", () => { }).failureSignals).toEqual([]); }); + test("counts proxy failures without assuming a transport cause", () => { + const diagnostics = scorerCaseDiagnostics({ + ...diagnosticInput, child: { ...diagnosticInput.child, stderr: "" }, + attempts: [{ + ...accountedAttempt, outcome: "failed", durationMs: 0, + promptTokens: 0, completionTokens: 0, costUsd: null, costProviderDecimal: null, + usageValid: false, httpStatus: null, modelIdentityPresent: false, + providerIdentityPresent: false, usagePresent: false, errorPresent: false, + }], + }); + expect(diagnostics.failureSignals).toEqual(["upstreamFailure"]); + expect(diagnostics.adjudication).toEqual({ + attempts: 1, collectedResponses: 0, validUsageAttempts: 0, exactCostAttempts: 0, + modelIdentityPresentAttempts: 0, providerIdentityPresentAttempts: 0, + }); + expect(diagnostics.scorer.attempts).toBe(0); + }); + test("emits bounded deduplicated codes without copying arbitrary messages or payload properties", () => { const marker = "credential=PRIVATE_MARKER Authorization: Bearer PRIVATE_MARKER"; const diagnostics = scorerCaseDiagnostics({ diff --git a/bench/src/scorer-eval.ts b/bench/src/scorer-eval.ts index 8427a40..2c078f6 100644 --- a/bench/src/scorer-eval.ts +++ b/bench/src/scorer-eval.ts @@ -126,7 +126,7 @@ export interface ScorerEvalCase { type ScorerFailureSignal = "childTimeout" | "invalidEnvelope" | "unsupportedRefutation" | "adjudicationValidation" | "adjudicationInvalidOutput" | "adjudicationProvider" | "adjudicationUnavailable" | "upstreamTimeout" | - "upstreamTransport" | "upstreamHttp" | "unknown"; + "upstreamFailure" | "upstreamHttp" | "unknown"; interface ScorerPhaseDiagnostics { attempts: number; @@ -183,7 +183,7 @@ export function scorerCaseDiagnostics(input: { if (input.envelope === undefined) signals.add("invalidEnvelope"); for (const attempt of input.attempts) { if (attempt.outcome === "timedOut") signals.add("upstreamTimeout"); - if (attempt.outcome === "failed") signals.add("upstreamTransport"); + if (attempt.outcome === "failed") signals.add("upstreamFailure"); if (attempt.httpStatus !== null && attempt.httpStatus >= 400) signals.add("upstreamHttp"); } if (!input.passed && signals.size === 0) signals.add("unknown"); From 36cfb6a7ef19e7a0bcecde6634a170d85425e102 Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Fri, 11 Sep 2026 03:31:44 +0000 Subject: [PATCH 4/6] Count only present decimal strings in scorer diagnostics --- bench/src/scorer-eval.test.ts | 22 ++++++++++++++++++++++ bench/src/scorer-eval.ts | 2 +- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/bench/src/scorer-eval.test.ts b/bench/src/scorer-eval.test.ts index f6cdabb..2066ca9 100644 --- a/bench/src/scorer-eval.test.ts +++ b/bench/src/scorer-eval.test.ts @@ -446,6 +446,28 @@ describe("scorer case diagnostics", () => { attempts: [accountedAttempt], envelope: { findings: [] }, passed: false, }; + test("counts only present string costs at an untyped input boundary", () => { + const missingCost: typeof accountedAttempt = JSON.parse(JSON.stringify({ + ...accountedAttempt, costProviderDecimal: undefined, + })); + expect(missingCost.costProviderDecimal).toBeUndefined(); + for (const phase of ["adjudication", "scorer"] as const) { + const diagnostics = scorerCaseDiagnostics({ + ...diagnosticInput, + attempts: [ + accountedAttempt, + { ...accountedAttempt, costProviderDecimal: "0" }, + { ...accountedAttempt, costProviderDecimal: null }, + missingCost, + ].map((attempt) => ({ ...attempt, phase })), + }); + expect(diagnostics[phase]).toEqual({ + attempts: 4, collectedResponses: 4, validUsageAttempts: 4, exactCostAttempts: 2, + modelIdentityPresentAttempts: 4, providerIdentityPresentAttempts: 4, + }); + } + }); + test("identifies the captured unsupported refutation without treating its accounted call as qualification", () => { const diagnostics = scorerCaseDiagnostics(diagnosticInput); expect(diagnostics.failureSignals).toEqual(["unsupportedRefutation"]); diff --git a/bench/src/scorer-eval.ts b/bench/src/scorer-eval.ts index 2c078f6..00c838d 100644 --- a/bench/src/scorer-eval.ts +++ b/bench/src/scorer-eval.ts @@ -194,7 +194,7 @@ export function scorerCaseDiagnostics(input: { // Completed means the proxy collected a response, not that it was valid. collectedResponses: attempts.filter((attempt) => attempt.outcome === "completed").length, validUsageAttempts: attempts.filter((attempt) => attempt.usageValid).length, - exactCostAttempts: attempts.filter((attempt) => attempt.costProviderDecimal !== null).length, + exactCostAttempts: attempts.filter((attempt) => typeof attempt.costProviderDecimal === "string").length, modelIdentityPresentAttempts: attempts.filter((attempt) => attempt.modelIdentityPresent === true).length, providerIdentityPresentAttempts: attempts.filter((attempt) => attempt.providerIdentityPresent === true).length, }; From 099e8d66e6e8d5c3711f6a70398cd3260d076469 Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Fri, 11 Sep 2026 03:31:49 +0000 Subject: [PATCH 5/6] Retain unsupported refutation provenance behind strict review validation --- src/adjudication.rs | 316 ++++++++++++++++++++++++++-------- src/review.rs | 12 +- tests/e2e.rs | 405 +++++++++++++++++++++++++++++--------------- 3 files changed, 527 insertions(+), 206 deletions(-) diff --git a/src/adjudication.rs b/src/adjudication.rs index d6e762f..c04f479 100644 --- a/src/adjudication.rs +++ b/src/adjudication.rs @@ -118,6 +118,7 @@ pub(crate) struct AdjudicationApplication { pub kept: Vec, pub kept_indices: Vec, pub unresolved_indices: Vec, + pub invalid_refutation_indices: Vec, pub resolved_indices: Vec, pub suppressed: Vec, } @@ -127,6 +128,7 @@ enum DeterministicDemotionReason { RepositoryReceipt, CitationFragment, InvalidConfirmation, + InvalidRefutation, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -1165,27 +1167,14 @@ pub(crate) fn validate_results( let claim_verdict = finding.repository_claim.as_ref().map(|claim| { crate::repository_search::claim_verdict(claim, repository_receipt, snapshot_id) }); - let direct_refutation_grounded = evidence_is_refutation_grounded( - &result.evidence, - &result.candidate_id, - corpus, - diff_receipt, - ) || evidence_is_complete_matching_window_refutation( - &result.evidence, + let refutation_grounded = result_has_grounded_refutation( + snapshot_id, finding, - &result.candidate_id, + result, corpus, diff_receipt, + repository_receipt, ); - let repository_refutation_grounded = - finding.repository_claim.as_ref().is_some_and(|claim| { - crate::repository_search::refutation_evidence_is_grounded( - claim, - repository_receipt, - snapshot_id, - &result.evidence, - ) - }); match result.status { AdjudicationStatus::Confirmed => { ensure!( @@ -1228,15 +1217,9 @@ pub(crate) fn validate_results( "refuted adjudication cannot publish revised finding text" ); ensure!( - direct_refutation_grounded || repository_refutation_grounded, + refutation_grounded, "refuted adjudication must cite candidate-specific contradictory evidence" ); - if claim_verdict.is_some() { - ensure!( - direct_refutation_grounded || repository_refutation_grounded, - "repository-dependent finding lacks exact candidate-specific refutation evidence" - ); - } } AdjudicationStatus::Unresolved => ensure!( result.revised_title.is_empty() @@ -1290,12 +1273,20 @@ pub(crate) fn apply_results( let mut kept = Vec::new(); let mut kept_indices = Vec::new(); let mut unresolved_indices = Vec::new(); + let mut invalid_refutation_indices = Vec::new(); let mut resolved_indices = Vec::new(); let mut suppressed = Vec::new(); for (index, (mut finding, id)) in findings.into_iter().zip(candidate_ids).enumerate() { let outcome = by_id .get(&id) .ok_or_else(|| anyhow!("validated adjudication result disappeared"))?; + if outcome.provenance + == AdjudicationProvenance::DeterministicEvidenceReceipt( + DeterministicDemotionReason::InvalidRefutation, + ) + { + invalid_refutation_indices.push(index); + } match (outcome.provenance, outcome.disposition) { (AdjudicationProvenance::Model, AdjudicationDisposition::RetainConfirmed) => { finding @@ -1342,6 +1333,7 @@ pub(crate) fn apply_results( kept, kept_indices, unresolved_indices, + invalid_refutation_indices, resolved_indices, suppressed, }) @@ -1509,11 +1501,50 @@ fn deterministic_demotion_reason( }) { Some(DeterministicDemotionReason::InvalidConfirmation) + } else if matches!(result.status, AdjudicationStatus::Refuted) + && result.revised_title.is_empty() + && result.revised_body.is_empty() + && !result_has_grounded_refutation( + snapshot_id, + finding, + result, + corpus, + receipt, + repository_receipt, + ) + { + Some(DeterministicDemotionReason::InvalidRefutation) } else { None } } +fn result_has_grounded_refutation( + snapshot_id: &str, + finding: &Finding, + result: &AdjudicationResult, + corpus: &str, + receipt: &DiffCorpusReceipt, + repository_receipt: &RepositorySearchReceipt, +) -> bool { + evidence_is_refutation_grounded(&result.evidence, &result.candidate_id, corpus, receipt) + || evidence_is_complete_matching_window_refutation( + &result.evidence, + finding, + &result.candidate_id, + corpus, + receipt, + ) + || finding.repository_claim.as_ref().is_some_and(|claim| { + crate::repository_search::refutation_evidence_is_grounded( + claim, + repository_receipt, + snapshot_id, + &result.evidence, + ) + }) +} + fn result_is_bounded_citation_fragment(result: &AdjudicationResult, finding: &Finding) -> bool { let Some(citation) = finding.evidence.as_deref() else { return false; @@ -1757,7 +1788,7 @@ mod tests { let corpus = "@@ -3 +3 @@\n- uses: action@old\n@@ -69 +74 @@\n+ uses: action@new\n"; let direct = direct_receipt(&snapshot, corpus, &findings, &ids); assert!(direct.rendered_evidence.contains("uses: action@new")); - let error = apply_results( + let applied = apply_results( &snapshot, findings, ids, @@ -1766,12 +1797,10 @@ mod tests { &direct, &unavailable_receipt(), ) - .unwrap_err(); - assert!( - error - .to_string() - .contains("candidate-specific contradictory evidence") - ); + .unwrap(); + assert_eq!(applied.kept.len(), 2); + assert_eq!(applied.unresolved_indices, vec![0, 1]); + assert!(applied.suppressed.is_empty()); } #[test] @@ -2023,6 +2052,137 @@ mod tests { assert_eq!(applied.suppressed.len(), 1); } + #[test] + fn ungrounded_refutation_preserves_the_original_candidate_for_scoring() { + let snapshot = "a".repeat(40); + let mut candidate = finding( + Kind::Risk, + "Clean change breaks runtime behavior", + "This change removes required runtime behavior and will break callers after merge.", + ); + candidate.path = "src/lib/logger.ts".into(); + candidate.line = 11; + candidate.confidence = 0.95; + candidate.evidence = Some(" // keep the log prefix stable for tests".into()); + let findings = vec![candidate]; + let ids = stable_candidate_ids(&snapshot, &findings); + let corpus = "--- a/src/lib/logger.ts\n+++ b/src/lib/logger.ts\n@@ -11 +11 @@\n- // keep the log prefix stable\n+ // keep the log prefix stable for tests\n"; + let receipt = direct_receipt(&snapshot, corpus, &findings, &ids); + for evidence in [ + "+ // keep the log prefix stable for tests", + " // keep the log prefix stable for tests", + "unsupported evidence", + "", + ] { + let result = AdjudicationResult { + candidate_id: ids[0].clone(), + status: AdjudicationStatus::Refuted, + revised_title: String::new(), + revised_body: String::new(), + evidence: evidence.into(), + duplicate_of: None, + }; + assert!( + validate_results( + &snapshot, + &findings, + &ids, + std::slice::from_ref(&result), + corpus, + &receipt, + &unavailable_receipt(), + ) + .is_err() + ); + let applied = apply_results( + &snapshot, + findings.clone(), + ids.clone(), + vec![result], + corpus, + &receipt, + &unavailable_receipt(), + ) + .unwrap(); + assert_eq!( + serde_json::to_value(&applied.kept).unwrap(), + serde_json::to_value(&findings).unwrap() + ); + assert_eq!(applied.kept_indices, vec![0]); + assert_eq!(applied.unresolved_indices, vec![0]); + assert!(applied.resolved_indices.is_empty()); + assert!(applied.suppressed.is_empty()); + } + } + + #[test] + fn ungrounded_refutation_does_not_repair_invalid_structure_or_snapshot() { + let snapshot = "a".repeat(40); + let findings = vec![finding( + Kind::Risk, + "Preserve validation", + "The change removes validation.", + )]; + let ids = stable_candidate_ids(&snapshot, &findings); + let corpus = "+ perform_write(input);\n"; + let receipt = direct_receipt(&snapshot, corpus, &findings, &ids); + let result = AdjudicationResult { + candidate_id: ids[0].clone(), + status: AdjudicationStatus::Refuted, + revised_title: String::new(), + revised_body: String::new(), + evidence: "unsupported evidence".into(), + duplicate_of: None, + }; + for results in [ + vec![], + vec![result.clone(), result.clone()], + vec![AdjudicationResult { + candidate_id: "unknown".into(), + ..result.clone() + }], + vec![AdjudicationResult { + duplicate_of: Some(ids[0].clone()), + ..result.clone() + }], + vec![AdjudicationResult { + revised_title: "invalid publication".into(), + ..result.clone() + }], + vec![AdjudicationResult { + revised_body: " ".into(), + ..result.clone() + }], + ] { + assert!( + apply_results( + &snapshot, + findings.clone(), + ids.clone(), + results, + corpus, + &receipt, + &unavailable_receipt(), + ) + .is_err() + ); + } + assert!( + apply_results( + &"b".repeat(40), + findings, + ids, + vec![result], + corpus, + &receipt, + &unavailable_receipt(), + ) + .unwrap_err() + .to_string() + .contains("snapshot mismatch") + ); + } + #[test] fn candidate_location_cannot_refute_its_own_finding() { let snapshot = "a".repeat(40); @@ -2055,18 +2215,19 @@ mod tests { duplicate_of: None, }; - assert!( - apply_results( - &snapshot, - findings, - ids, - vec![result], - corpus, - &receipt, - &unavailable_receipt(), - ) - .is_err() - ); + let applied = apply_results( + &snapshot, + findings, + ids, + vec![result], + corpus, + &receipt, + &unavailable_receipt(), + ) + .unwrap(); + assert_eq!(applied.kept.len(), 1); + assert_eq!(applied.unresolved_indices, vec![0]); + assert!(applied.suppressed.is_empty()); } #[test] @@ -2099,18 +2260,19 @@ mod tests { &ids[0], &receipt, )); - assert!( - apply_results( - &snapshot, - findings, - ids, - vec![result], - corpus, - &receipt, - &unavailable_receipt(), - ) - .is_err() - ); + let applied = apply_results( + &snapshot, + findings, + ids, + vec![result], + corpus, + &receipt, + &unavailable_receipt(), + ) + .unwrap(); + assert_eq!(applied.kept.len(), 1); + assert_eq!(applied.unresolved_indices, vec![0]); + assert!(applied.suppressed.is_empty()); } #[test] @@ -3078,17 +3240,30 @@ mod tests { ..result }; assert!( - apply_results( + validate_results( &snapshot, - findings, - ids, - vec![lexical_refutation], + &findings, + &ids, + std::slice::from_ref(&lexical_refutation), corpus, &direct, &lexical_match, ) .is_err() ); + let applied = apply_results( + &snapshot, + findings, + ids, + vec![lexical_refutation], + corpus, + &direct, + &lexical_match, + ) + .unwrap(); + assert_eq!(applied.kept.len(), 1); + assert_eq!(applied.unresolved_indices, vec![0]); + assert!(applied.suppressed.is_empty()); } #[test] @@ -3168,7 +3343,7 @@ mod tests { let corpus = "+ image: old-image\n"; let direct = direct_receipt(&snapshot, corpus, &findings, &ids); - let error = apply_results( + let applied = apply_results( &snapshot, findings, ids, @@ -3177,12 +3352,10 @@ mod tests { &direct, &receipt, ) - .unwrap_err(); - assert!( - error - .to_string() - .contains("candidate-specific contradictory evidence") - ); + .unwrap(); + assert_eq!(applied.kept.len(), 1); + assert_eq!(applied.unresolved_indices, vec![0]); + assert!(applied.suppressed.is_empty()); } #[test] @@ -3465,12 +3638,11 @@ mod tests { &receipt, &unavailable_receipt(), ); - if confirmed { - let applied = applied.unwrap(); - assert_eq!(applied.kept.len(), 1); - assert!(applied.suppressed.is_empty()); - } else { - assert!(applied.is_err()); + let applied = applied.unwrap(); + assert_eq!(applied.kept.len(), 1); + assert!(applied.suppressed.is_empty()); + if !confirmed { + assert_eq!(applied.unresolved_indices, vec![0]); } } } diff --git a/src/review.rs b/src/review.rs index 87499ba..7136007 100644 --- a/src/review.rs +++ b/src/review.rs @@ -2540,7 +2540,13 @@ async fn review_diff_at( diff_snapshot.as_str(), &diff_receipt, receipt, - ) { + ).and_then(|application| { + anyhow::ensure!( + application.invalid_refutation_indices.is_empty(), + "refuted adjudication must cite candidate-specific contradictory evidence" + ); + Ok(application) + }) { Ok(application) => { lockfile_platform_policy_allowed = true; application @@ -3504,6 +3510,7 @@ fn preserve_unadjudicated_findings( kept_indices: (0..findings.len()).collect(), kept: findings, unresolved_indices: Vec::new(), + invalid_refutation_indices: Vec::new(), resolved_indices: Vec::new(), suppressed: Vec::new(), } @@ -4777,6 +4784,7 @@ mod tests { ], kept_indices: vec![0, 1, 2], unresolved_indices: vec![0, 1, 2], + invalid_refutation_indices: Vec::new(), resolved_indices: vec![], suppressed: vec![], }; @@ -4847,6 +4855,7 @@ mod tests { ], kept_indices: vec![0, 1, 2, 3], unresolved_indices: vec![0, 1, 2, 3], + invalid_refutation_indices: Vec::new(), resolved_indices: Vec::new(), suppressed: Vec::new(), }; @@ -5008,6 +5017,7 @@ mod tests { kept: vec![ordinary.clone()], kept_indices: vec![1], unresolved_indices: Vec::new(), + invalid_refutation_indices: Vec::new(), resolved_indices: vec![0], suppressed: vec![SuppressedFinding { finding: rejected.clone(), diff --git a/tests/e2e.rs b/tests/e2e.rs index 7e13a00..f945971 100644 --- a/tests/e2e.rs +++ b/tests/e2e.rs @@ -6444,15 +6444,28 @@ async fn compact_pnpm_lockfile_ia32_platform_claim_is_suppressed_but_dependency_ #[tokio::test] async fn compact_lockfile_platform_claim_survives_failed_adjudication() { - let server = MockServer::start().await; - Mock::given(method("POST")) - .and(path("/chat/completions")) - .and(body_string_contains("single finding adjudicator")) - .respond_with(ResponseTemplate::new(503)) - .with_priority(1) - .mount(&server) - .await; - Mock::given(method("POST")) + for refuted in [false, true] { + let server = MockServer::start().await; + if !refuted { + Mock::given(method("POST")) + .and(path("/chat/completions")) + .and(body_string_contains("single finding adjudicator")) + .respond_with(ResponseTemplate::new(503)) + .with_priority(1) + .mount(&server) + .await; + } + if refuted { + Mock::given(method("POST")) + .and(path("/chat/completions")) + .and(body_string_contains("single finding adjudicator")) + .respond_with(AllRefutedAdjudicator) + .with_priority(1) + .expect(1) + .mount(&server) + .await; + } + Mock::given(method("POST")) .and(path("/chat/completions")) .respond_with(|request: &Request| { let evidence = prompt_evidence( @@ -6476,48 +6489,63 @@ async fn compact_lockfile_platform_claim_survives_failed_adjudication() { .mount(&server) .await; - let dir = tempfile::tempdir().unwrap(); - let diff = dir.path().join("pnpm-ia32-adjudication-failure.diff"); - std::fs::write( + let dir = tempfile::tempdir().unwrap(); + let diff = dir.path().join("pnpm-ia32-adjudication-failure.diff"); + std::fs::write( &diff, "diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml\n--- a/pnpm-lock.yaml\n+++ b/pnpm-lock.yaml\n@@ -1,3 +1,3 @@\n lockfileVersion: '9.0'\n packages:\n- '@rollup/rollup-win32-ia32-msvc@4.34.8':\n+ '@rollup/rollup-win32-x64-msvc@4.34.8':\n", ) .unwrap(); - let out = postil() - .current_dir(dir.path()) - .env("POSTIL_API_BASE", server.uri()) - .env("POSTIL_DISABLE_SCORER", "1") - .args(["review", "--diff-file"]) - .arg(&diff) - .args(["--output", "json"]) - .assert() - .code(1); - let envelope: Value = serde_json::from_slice(&out.get_output().stdout).unwrap(); - assert!( - envelope["findings"] - .as_array() - .unwrap() - .iter() - .any(|finding| { finding["title"] == "Preserve Windows IA32 support" }) - ); - assert!( - envelope["findings"] - .as_array() - .unwrap() - .iter() - .any(|finding| { finding["path"] == ".postil/provider" }) - ); - assert!( - envelope["suppressedFindings"] - .as_array() - .is_none_or(|findings| { - !findings - .iter() - .any(|finding| finding["reason"] == "lockfilePlatformEvidenceInsufficient") - }) - ); - assert_eq!(envelope["gate"]["failing"], true); + let out = postil() + .current_dir(dir.path()) + .env("POSTIL_API_BASE", server.uri()) + .env("REVIEW_SCORER_MODEL", "scorer-model") + .args(["review", "--diff-file"]) + .arg(&diff) + .args(["--output", "json"]) + .assert() + .code(1); + let envelope: Value = serde_json::from_slice(&out.get_output().stdout).unwrap(); + assert!( + envelope["findings"] + .as_array() + .unwrap() + .iter() + .any(|finding| { finding["title"] == "Preserve Windows IA32 support" }) + ); + assert!( + envelope["findings"] + .as_array() + .unwrap() + .iter() + .any(|finding| { + finding["path"] + == if refuted { + ".postil/model-output" + } else { + ".postil/provider" + } + }) + ); + assert!( + envelope["suppressedFindings"] + .as_array() + .is_none_or(|findings| { + !findings + .iter() + .any(|finding| finding["reason"] == "lockfilePlatformEvidenceInsufficient") + }) + ); + assert_eq!(envelope["gate"]["failing"], true); + let requests = server.received_requests().await.unwrap(); + assert!( + requests.iter().all(|request| !request_system_contains( + request, + "independent second-model scorer" + )) + ); + } } async fn mock_review_model(server: &MockServer, model: &str, findings: Value) { @@ -13562,50 +13590,97 @@ async fn cross_file_package_existence_refutes_false_absence_claim() { #[tokio::test] async fn fresh_unresolved_repository_claims_are_suppressed() { - for (name, repository, resources, state) in [ - ( - "unavailable", - false, - vec!["widget".to_string()], - "unavailable", - ), - ("exhausted", true, vec!["widget".to_string()], "exhausted"), - ] { - let server = MockServer::start().await; - mock_review( - &server, - json!([{ - "path": "src/auth.rs", "line": 42, "severity": "error", "kind": "risk", - "confidence": 0.99, "title": "Widget dependency is absent", - "body": "The repository does not contain the required widget dependency.", - "evidence": "exec_query(&token);", - "repositoryContext": {"claim": "absence", "resources": resources} - }]), - ) - .await; - let dir = tempfile::tempdir().unwrap(); - if repository { - initialize_staged_repository(dir.path()); - } - let diff = write_diff(dir.path()); - let mut command = postil(); - command - .current_dir(dir.path()) - .env("POSTIL_API_BASE", server.uri()) - .env("POSTIL_DISABLE_SCORER", "1") - .arg("review"); - if repository { - command.arg("--staged"); - } else { - command.arg("--diff-file").arg(&diff); + for refuted in [false, true] { + for (name, repository, resources, state) in [ + ( + "unavailable", + false, + vec!["widget".to_string()], + "unavailable", + ), + ("exhausted", true, vec!["widget".to_string()], "exhausted"), + ] { + let server = MockServer::start().await; + mock_review( + &server, + json!([{ + "path": "src/auth.rs", "line": 42, "severity": "error", "kind": "risk", + "confidence": 0.99, "title": "Widget dependency is absent", + "body": "The repository does not contain the required widget dependency.", + "evidence": "exec_query(&token);", + "repositoryContext": {"claim": "absence", "resources": resources} + }]), + ) + .await; + if refuted { + Mock::given(method("POST")) + .and(path("/chat/completions")) + .and(body_string_contains("single finding adjudicator")) + .respond_with(AllRefutedAdjudicator) + .with_priority(1) + .expect(1) + .mount(&server) + .await; + } + let dir = tempfile::tempdir().unwrap(); + if repository { + initialize_staged_repository(dir.path()); + } + let diff = write_diff(dir.path()); + let mut command = postil(); + command + .current_dir(dir.path()) + .env("POSTIL_API_BASE", server.uri()) + .env("REVIEW_SCORER_MODEL", "scorer-model") + .arg("review"); + if repository { + command.arg("--staged"); + } else { + command.arg("--diff-file").arg(&diff); + } + command.args(["--output", "json"]); + let out = command.assert().code(if refuted { 1 } else { 0 }); + let envelope: Value = serde_json::from_slice(&out.get_output().stdout).unwrap(); + assert_eq!(envelope["repositorySearch"]["state"], state, "{name}"); + if refuted { + let findings = envelope["findings"].as_array().unwrap(); + assert!( + findings + .iter() + .any(|finding| finding["path"] == ".postil/model-output") + ); + assert!( + findings + .iter() + .any(|finding| finding["title"] == "Widget dependency is absent") + ); + assert_eq!(envelope["counts"]["suppressed"], 0); + assert_eq!(envelope["gate"]["failing"], true); + assert!( + envelope["modelIncidents"] + .as_array() + .unwrap() + .iter() + .any(|incident| incident["category"] == "invalidOutput" + && incident["recovered"] == false) + ); + } else { + assert_eq!(envelope["counts"]["suppressed"], 1, "{name}"); + assert_eq!(envelope["findings"], json!([]), "{name}"); + assert_eq!(envelope["gate"]["failing"], false, "{name}"); + assert!( + envelope["modelIncidents"] + .as_array() + .is_none_or(Vec::is_empty) + ); + } + let requests = server.received_requests().await.unwrap(); + assert!(requests.iter().all(|request| !request_system_contains( + request, + "independent second-model scorer" + ))); + assert_eq!(envelope["resolved"], json!([])); } - command.args(["--output", "json"]); - let out = command.assert().code(0); - let envelope: Value = serde_json::from_slice(&out.get_output().stdout).unwrap(); - assert_eq!(envelope["repositorySearch"]["state"], state, "{name}"); - assert_eq!(envelope["counts"]["suppressed"], 1, "{name}"); - assert_eq!(envelope["findings"], json!([]), "{name}"); - assert_eq!(envelope["gate"]["failing"], false, "{name}"); } } @@ -13801,50 +13876,114 @@ async fn oversized_adjudication_payload_preserves_findings_without_aborting_revi #[tokio::test] async fn scorer_cannot_suppress_an_unresolved_full_rereview_baseline() { - let server = MockServer::start().await; - mock_review(&server, json!([])).await; - let directory = tempfile::tempdir().unwrap(); - let diff = write_diff(directory.path()); - let baseline = json!({ - "version": 1, "summary": "", "silent": false, - "findings": [{ - "path": "src/auth.rs", "line": 42, "severity": "error", "kind": "risk", - "confidence": 0.9, "title": "Authorization guard remains bypassed", - "body": "The authorization guard remains bypassed before query execution.", - "evidence": "exec_query(&token);" - }], - "resolved": [], "counts": {"info": 0, "warn": 0, "error": 1, "suppressed": 0}, - "confidenceBuckets": [0,0,0,0,1], - "gate": {"failOn": "error", "failing": true}, - "modelUsed": "model", "usage": {"promptTokens": 0, "completionTokens": 0}, - "baseSha": null, "headSha": null, "sinceSha": null - }); - let baseline_path = directory.path().join("scorer-baseline.json"); - std::fs::write(&baseline_path, baseline.to_string()).unwrap(); - - let output = postil() - .current_dir(directory.path()) - .env("POSTIL_API_BASE", server.uri()) - .env("REVIEW_SCORER_MODEL", "scorer-model") - .args(["review", "--diff-file"]) - .arg(&diff) - .arg("--baseline") - .arg(&baseline_path) - .args(["--output", "json"]) - .assert() - .code(1); - let envelope: Value = serde_json::from_slice(&output.get_output().stdout).unwrap(); - assert_eq!( - envelope["findings"][0]["title"], - "Authorization guard remains bypassed" - ); - assert_eq!(envelope["findings"][0]["confidence"], 0.9); - assert_eq!(envelope["resolved"], json!([])); - assert_eq!(envelope["gate"]["failing"], true); - let requests = server.received_requests().await.unwrap(); - assert!(requests.iter().all(|request| { - !String::from_utf8_lossy(&request.body).contains("independent second-model scorer") - })); + for status in ["unresolved", "refuted"] { + for claim in ["none", "absence"] { + let server = MockServer::start().await; + mock_review(&server, json!([])).await; + Mock::given(method("POST")) + .and(path("/chat/completions")) + .and(body_string_contains("single finding adjudicator")) + .respond_with(move |request: &Request| { + let request: Value = request.body_json().unwrap(); + let payload: Value = serde_json::from_str( + request["messages"].as_array().unwrap().last().unwrap()["content"] + .as_str() + .unwrap(), + ) + .unwrap(); + let results = payload["candidates"] + .as_array() + .unwrap() + .iter() + .map(|candidate| { + json!({ + "candidateId": candidate["candidateId"], "status": status, + "revisedTitle": "", "revisedBody": "", + "evidence": if status == "refuted" { "+exec_query(&token);" } else { "" }, "duplicateOf": null + }) + }) + .collect::>(); + ResponseTemplate::new(200).set_body_json(scorer_content(json!(results))) + }) + .with_priority(1) + .expect(1) + .mount(&server) + .await; + let directory = tempfile::tempdir().unwrap(); + let diff = write_diff(directory.path()); + let baseline = json!({ + "version": 1, "summary": "", "silent": false, + "findings": [{ + "path": "src/auth.rs", "line": 42, "severity": "error", "kind": "risk", + "confidence": 0.9, "title": "Authorization guard remains bypassed", + "body": "The authorization guard remains bypassed before query execution.", + "evidence": "exec_query(&token);", + "repositoryContext": if claim == "none" { Value::Null } else { json!({"claim": claim, "identifiers": ["validate_token"]}) } + }], + "resolved": [], "counts": {"info": 0, "warn": 0, "error": 1, "suppressed": 0}, + "confidenceBuckets": [0,0,0,0,1], + "gate": {"failOn": "error", "failing": true}, + "modelUsed": "model", "usage": {"promptTokens": 0, "completionTokens": 0}, + "baseSha": null, "headSha": null, "sinceSha": null + }); + let baseline_path = directory.path().join("scorer-baseline.json"); + std::fs::write(&baseline_path, baseline.to_string()).unwrap(); + + let output = postil() + .current_dir(directory.path()) + .env("POSTIL_API_BASE", server.uri()) + .env("REVIEW_SCORER_MODEL", "scorer-model") + .args(["review", "--diff-file"]) + .arg(&diff) + .arg("--baseline") + .arg(&baseline_path) + .args(["--output", "json"]) + .assert() + .code(1); + let envelope: Value = serde_json::from_slice(&output.get_output().stdout).unwrap(); + assert_eq!( + envelope["findings"][0]["title"], + "Authorization guard remains bypassed" + ); + assert_eq!(envelope["findings"][0]["confidence"], 0.9); + assert_eq!(envelope["resolved"], json!([])); + assert_eq!(envelope["gate"]["failing"], true); + assert_eq!( + envelope["findings"].as_array().unwrap().len(), + if status == "refuted" { 2 } else { 1 } + ); + assert_eq!(envelope["counts"]["suppressed"], 0); + if status == "refuted" { + assert!( + envelope["findings"] + .as_array() + .unwrap() + .iter() + .any(|finding| finding["path"] == ".postil/model-output") + ); + assert!( + envelope["modelIncidents"] + .as_array() + .unwrap() + .iter() + .any(|incident| incident["category"] == "invalidOutput" + && incident["recovered"] == false) + ); + } else { + assert!( + envelope["modelIncidents"] + .as_array() + .is_none_or(Vec::is_empty) + ); + } + assert_eq!(envelope["usageAccountingComplete"], true); + assert_model_usage_matches_aggregate(&envelope); + let requests = server.received_requests().await.unwrap(); + assert!(requests.iter().all(|request| { + !String::from_utf8_lossy(&request.body).contains("independent second-model scorer") + })); + } + } } #[tokio::test] From 698d7487dc131d48092abe9560320cf6a1c66080 Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Fri, 11 Sep 2026 03:38:57 +0000 Subject: [PATCH 6/6] Recover unsupported refutations only after complete scoring --- ARCHITECTURE.md | 2 + Cargo.lock | 2 +- Cargo.toml | 2 +- src/review.rs | 294 ++++++++++++++++++------- tests/e2e.rs | 556 ++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 775 insertions(+), 81 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 1289795..04c9764 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -130,6 +130,8 @@ acquire diff --> parse supported lockfiles --> parse + index --> bounded evidenc and duplicate primaries. Later and cross-file evidence can refute stale claims. Fresh unresolved repository claims are suppressed, ordinary grounded unresolved findings and prior-ledger claims remain open, and provider or contract failure preserves every candidate. + Unsupported refutations of eligible fresh candidates recover only after validated scoring covers + every affected candidate; otherwise the invalid-output blocker remains. Semantic duplicates collapse across files and kinds only when one established defect remains; distinct defects sharing a line remain separate. - `forge/`: trait + GitHub, GitLab, Bitbucket Cloud, and Azure DevOps implementations, diff --git a/Cargo.lock b/Cargo.lock index ef19c18..d2a3632 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1351,7 +1351,7 @@ checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" [[package]] name = "postil-cli" -version = "0.9.7" +version = "0.9.8" dependencies = [ "aho-corasick", "anyhow", diff --git a/Cargo.toml b/Cargo.toml index ae7c9db..14d69d1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "postil-cli" -version = "0.9.7" +version = "0.9.8" edition = "2024" description = "Postil: a low-noise AI review gate. Silent on clean PRs, hard gate on real risk." license = "Apache-2.0" diff --git a/src/review.rs b/src/review.rs index 7136007..f275bcf 100644 --- a/src/review.rs +++ b/src/review.rs @@ -1395,6 +1395,19 @@ fn scorer_inputs( .collect() } +fn scorer_covers_refutation_recovery( + pending: &[usize], + inputs: &[prompt::ScorerPromptFinding], + scores: &[FindingScore], +) -> bool { + pending.iter().all(|index| { + inputs + .get(*index) + .is_some_and(|input| input.index == *index) + && scores.iter().any(|score| score.index == *index) + }) +} + fn apply_scorer_scores(cfg: &Config, findings: &mut [Finding], scores: Vec) -> u32 { let mut disagreements = 0u32; for score in scores { @@ -2393,6 +2406,9 @@ async fn review_diff_at( }; let mut kept = outcome.kept; let mut preserved_baseline_publications = Vec::new(); + let mut pending_refutation_recovery = Vec::new(); + let mut staged_adjudication = None; + let mut scorer_suppressions = Vec::new(); let resolution = crate::resolve::resolve_uncertainties( cfg, @@ -2456,6 +2472,9 @@ async fn review_diff_at( all_adjudication_candidates.len(), crate::adjudication::MAX_ADJUDICATION_CANDIDATES, ); + let mut fallback_application = preserve_unadjudicated_findings( + all_adjudication_candidates.clone(), + ); let snapshot_id = crate::adjudication::reviewed_snapshot_identity( repository_revision.as_deref(), diff_snapshot.as_str(), @@ -2524,7 +2543,6 @@ async fn review_diff_at( None } }; - let mut lockfile_platform_policy_allowed = false; let mut application = match adjudicated { Some(Ok(adjudicated)) => { add_usage(&mut usage, adjudicated.usage); @@ -2540,17 +2558,27 @@ async fn review_diff_at( diff_snapshot.as_str(), &diff_receipt, receipt, - ).and_then(|application| { + ).and_then(|mut application| { + suppress_fresh_unresolved_repository_claims( + &mut application, + fresh_candidate_count, + ); + suppress_fresh_lockfile_platform_claims( + &mut application, + fresh_candidate_count, + &index, + ); anyhow::ensure!( - application.invalid_refutation_indices.is_empty(), + application.invalid_refutation_indices.iter().all(|candidate_index| { + cfg.scorer_enabled() + && *candidate_index < fresh_candidate_count + && application.kept_indices.contains(candidate_index) + }), "refuted adjudication must cite candidate-specific contradictory evidence" ); Ok(application) }) { - Ok(application) => { - lockfile_platform_policy_allowed = true; - application - } + Ok(application) => application, Err(error) => { adjudication_incomplete = true; review_trust = filter::ReviewTrust::Failed; @@ -2597,46 +2625,18 @@ async fn review_diff_at( preserve_unadjudicated_findings(all_adjudication_candidates) } }; - suppress_fresh_unresolved_repository_claims( + promote_incomplete_adjudication_receipts( &mut application, - fresh_candidate_count, + &candidate_ids, + &diff_receipt, + ); + promote_incomplete_adjudication_receipts( + &mut fallback_application, + &candidate_ids, + &diff_receipt, ); - if lockfile_platform_policy_allowed { - suppress_fresh_lockfile_platform_claims( - &mut application, - fresh_candidate_count, - &index, - ); - } - for (candidate_index, finding) in application - .kept_indices - .iter() - .copied() - .zip(&mut application.kept) - { - let Some(candidate_id) = candidate_ids.get(candidate_index) else { - continue; - }; - let receipt_incomplete = diff_receipt - .candidate_citations - .iter() - .find(|receipt| &receipt.candidate_id == candidate_id) - .is_none_or(|receipt| !receipt.queries_complete); - if receipt_incomplete { - finding.severity = crate::envelope::Severity::Error; - } - } debug_assert!(application.kept_indices.iter().all(|index| *index < fresh_candidate_count + baseline_candidate_indices.len())); - for candidate_index in &application.resolved_indices { - let Some(baseline_offset) = - candidate_index.checked_sub(fresh_candidate_count) - else { - continue; - }; - let baseline_index = baseline_candidate_indices[baseline_offset]; - adjudication_resolved.push(baseline[baseline_index].clone()); - } // A baseline finding leaves the ledger only through // an adjudicator's explicit refutation or duplicate // disposition. Deterministic evidence checks can @@ -2660,31 +2660,28 @@ async fn review_diff_at( ) }); } - suppressed += application.suppressed.len() as u32; - suppressed_findings.extend(application.suppressed); kept.clear(); - for (candidate_index, finding) in - application.kept_indices.into_iter().zip(application.kept) + for (candidate_index, finding) in application + .kept_indices + .iter() + .copied() + .zip(&application.kept) { - if candidate_index >= fresh_candidate_count { - adjudication_preserved_baseline.push(finding.clone()); - preserved_baseline_publications.push(finding); - } else { - kept.push(finding); + if candidate_index < fresh_candidate_count { + if application + .invalid_refutation_indices + .contains(&candidate_index) + { + pending_refutation_recovery.push(kept.len()); + } + kept.push(finding.clone()); } } - if !baseline_candidate_indices.is_empty() { - let removed = baseline_candidate_indices - .into_iter() - .collect::>(); - baseline = baseline - .into_iter() - .enumerate() - .filter_map(|(index, finding)| { - (!removed.contains(&index)).then_some(finding) - }) - .collect(); - } + staged_adjudication = Some(( + application, + baseline_candidate_indices, + fallback_application, + )); } if !kept.is_empty() && cfg.scorer_enabled() && !adjudication_incomplete { let scorer_system = prompt::scorer_system_prompt(cfg, current_utc_date); @@ -2726,26 +2723,37 @@ async fn review_diff_at( .await; match scored { Ok(scored) => { - let disagreements = - apply_scorer_scores(cfg, &mut kept, scored.scores); - let scorer_suppressed = - suppress_below_min_confidence(cfg, &mut kept); - suppressed += scorer_suppressed.len() as u32; - suppressed_findings.extend(scorer_suppressed); + if scorer_covers_refutation_recovery( + &pending_refutation_recovery, + &inputs, + &scored.scores, + ) { + pending_refutation_recovery.clear(); + let disagreements = + apply_scorer_scores(cfg, &mut kept, scored.scores); + scorer_suppressions = + suppress_below_min_confidence(cfg, &mut kept); + scorer_disagreements = Some(disagreements); + sort_findings_for_display(&mut kept); + } scorer_model = Some(scored.model_used); add_usage(&mut usage, scored.usage); model_usage.extend(scored.model_usage); model_incidents.extend(scored.model_incidents); usage_accounting_complete &= scored.usage_accounting_complete; - scorer_disagreements = Some(disagreements); - sort_findings_for_display(&mut kept); } Err(e) => { let detail = format!("{e:#}"); - eprintln!( - "postil: scorer failed open after all scorer models failed" - ); + if pending_refutation_recovery.is_empty() { + eprintln!( + "postil: scorer failed open after all scorer models failed" + ); + } else { + eprintln!( + "postil: scorer failed before unsupported refutation recovery completed" + ); + } let scorer_usage = e.usage(); add_usage(&mut usage, scorer_usage); model_usage.extend_from_slice(e.model_usage()); @@ -2760,10 +2768,12 @@ async fn review_diff_at( } } } - if scorer_failure_blocks_hosted( - crate::config::hosted_runtime_mode(), - scorer_error.is_some(), - ) { + if pending_refutation_recovery.is_empty() + && scorer_failure_blocks_hosted( + crate::config::hosted_runtime_mode(), + scorer_error.is_some(), + ) + { return Err(ReviewFailure { kind: scorer_failure_kind .unwrap_or(ReviewFailureKind::InvalidOutput), @@ -2782,6 +2792,69 @@ async fn review_diff_at( .into()); } } + if !pending_refutation_recovery.is_empty() { + eprintln!( + "postil: unsupported refutation recovery requires complete validated scoring; preserving the invalid-output blocker" + ); + review_trust = filter::ReviewTrust::Failed; + model_incidents.push(ModelIncident { + phase: ModelIncidentPhase::Scorer, + category: ModelIncidentCategory::InvalidOutput, + recovered: false, + recovery: None, + }); + adjudication_failure = Some(fail_closed_finding( + "finding adjudication output did not satisfy its admitted contract", + )); + } + if let Some(( + mut application, + baseline_candidate_indices, + fallback_application, + )) = staged_adjudication + { + let fresh_candidate_count = + fallback_application.kept.len() - baseline_candidate_indices.len(); + if !pending_refutation_recovery.is_empty() { + application = fallback_application; + kept = application.kept[..fresh_candidate_count].to_vec(); + } + for candidate_index in &application.resolved_indices { + let Some(baseline_offset) = + candidate_index.checked_sub(fresh_candidate_count) + else { + continue; + }; + let baseline_index = baseline_candidate_indices[baseline_offset]; + adjudication_resolved.push(baseline[baseline_index].clone()); + } + suppressed += application.suppressed.len() as u32; + suppressed_findings.extend(application.suppressed); + for (candidate_index, finding) in + application.kept_indices.into_iter().zip(application.kept) + { + if candidate_index >= fresh_candidate_count { + adjudication_preserved_baseline.push(finding.clone()); + preserved_baseline_publications.push(finding); + } + } + if !baseline_candidate_indices.is_empty() { + let removed = baseline_candidate_indices + .into_iter() + .collect::>(); + baseline = baseline + .into_iter() + .enumerate() + .filter_map(|(index, finding)| { + (!removed.contains(&index)).then_some(finding) + }) + .collect(); + } + } + if pending_refutation_recovery.is_empty() { + suppressed += scorer_suppressions.len() as u32; + suppressed_findings.extend(scorer_suppressions); + } kept.extend(preserved_baseline_publications); sort_findings_for_display(&mut kept); findings = kept; @@ -3503,6 +3576,31 @@ fn ungrounded_findings_failure(count: u32) -> Finding { )) } +fn promote_incomplete_adjudication_receipts( + application: &mut crate::adjudication::AdjudicationApplication, + candidate_ids: &[String], + receipt: &crate::adjudication::DiffCorpusReceipt, +) { + for (candidate_index, finding) in application + .kept_indices + .iter() + .copied() + .zip(&mut application.kept) + { + let Some(candidate_id) = candidate_ids.get(candidate_index) else { + continue; + }; + let receipt_incomplete = receipt + .candidate_citations + .iter() + .find(|receipt| &receipt.candidate_id == candidate_id) + .is_none_or(|receipt| !receipt.queries_complete); + if receipt_incomplete { + finding.severity = crate::envelope::Severity::Error; + } + } +} + fn preserve_unadjudicated_findings( findings: Vec, ) -> crate::adjudication::AdjudicationApplication { @@ -3849,6 +3947,44 @@ fn error_envelope( mod tests { use super::*; + #[test] + fn refutation_recovery_requires_scores_for_every_pending_input() { + let findings = vec![finding("a.rs", 1, "first"), finding("b.rs", 2, "second")]; + let inputs = scorer_inputs(&[], &[], &findings, 0); + let score = |index| FindingScore { + index, + confidence: 0.99, + kind: crate::envelope::Kind::Risk, + reason: "The cited operation remains unsafe.".into(), + }; + assert!(!scorer_covers_refutation_recovery(&[1], &inputs, &[])); + assert!(!scorer_covers_refutation_recovery( + &[1], + &inputs, + &[score(0)] + )); + assert!(!scorer_covers_refutation_recovery( + &[0, 1], + &inputs, + &[score(1)] + )); + assert!(!scorer_covers_refutation_recovery( + &[1], + &inputs[..1], + &[score(1)] + )); + assert!(scorer_covers_refutation_recovery( + &[1], + &inputs, + &[score(0), score(1)] + )); + assert!(scorer_covers_refutation_recovery( + &[0, 1], + &inputs, + &[score(0), score(1)] + )); + } + #[test] fn exact_serialized_shared_context_admits_local_and_ci_batch_edges() { let cfg = Config { diff --git a/tests/e2e.rs b/tests/e2e.rs index f945971..87f6d11 100644 --- a/tests/e2e.rs +++ b/tests/e2e.rs @@ -3733,6 +3733,452 @@ async fn adjudication_provider_failure_preserves_findings_and_baseline_blocker() })); } +#[tokio::test] +async fn ungrounded_refutation_keeps_candidates_for_the_existing_scorer() { + for (label, true_candidate, malformed) in [ + ("comment", false, false), + ("query", true, false), + ("malformed-publication", false, true), + ("scorer-invalid", false, false), + ("scorer-disabled", false, false), + ] { + let scorer_failure = label == "scorer-invalid"; + let scorer_disabled = label == "scorer-disabled"; + let server = MockServer::start().await; + let (before, after, title, body) = if true_candidate { + ( + "validate(input);", + "exec_query(input);", + "Validate query input", + "The query executes attacker-controlled input without validation.", + ) + } else { + ( + " // keep the log prefix stable", + " // keep the log prefix stable for tests", + "Clean change breaks runtime behavior", + "This change removes required runtime behavior and will break callers after merge.", + ) + }; + mock_review_model( + &server, + "generator-model", + json!([{ + "path": "src/lib/logger.ts", "line": 11, "severity": if true_candidate { "error" } else { "warn" }, + "kind": "risk", "confidence": 0.95, "title": title, "body": body, + "evidence": after, "repositoryContext": {"claim": "none"} + }]), + ) + .await; + Mock::given(method("POST")) + .and(path("/chat/completions")) + .and(body_string_contains("single finding adjudicator")) + .respond_with(move |request: &Request| { + let request: Value = request.body_json().unwrap(); + let payload: Value = serde_json::from_str( + request["messages"].as_array().unwrap().last().unwrap()["content"] + .as_str() + .unwrap(), + ) + .unwrap(); + let candidate = &payload["candidates"][0]; + ResponseTemplate::new(200).set_body_json(scorer_text( + &json!([{ + "candidateId": candidate["candidateId"], "status": "refuted", + "revisedTitle": if malformed { "invalid publication" } else { "" }, + "revisedBody": "", "evidence": format!("+{after}"), "duplicateOf": null + }]) + .to_string(), + )) + }) + .with_priority(1) + .expect(1) + .mount(&server) + .await; + Mock::given(method("POST")) + .and(path("/chat/completions")) + .and(body_string_contains("independent second-model scorer")) + .respond_with(ResponseTemplate::new(200).set_body_json(scorer_content(if scorer_failure { json!([]) } else { json!([{ + "confidence": if true_candidate { 0.99 } else { 0.01 }, + "kind": "risk", + "reason": if true_candidate { "The query executes unvalidated input." } else { "The diff only changes a comment." } + }]) }))) + .with_priority(1) + .expect(if malformed || scorer_disabled { 0 } else if scorer_failure { 2 } else { 1 }) + .mount(&server).await; + let directory = tempfile::tempdir().unwrap(); + let diff = directory.path().join("review.diff"); + std::fs::write(&diff, format!( + "diff --git a/src/lib/logger.ts b/src/lib/logger.ts\n--- a/src/lib/logger.ts\n+++ b/src/lib/logger.ts\n@@ -11 +11 @@\n-{before}\n+{after}\n" + )).unwrap(); + let mut command = postil(); + if scorer_disabled { + command.env("POSTIL_DISABLE_SCORER", "1"); + } + let output = command + .current_dir(directory.path()) + .env("POSTIL_API_BASE", server.uri()) + .env("REVIEW_MODEL", "generator-model") + .env("REVIEW_SCORER_MODEL", "scorer-model") + .args(["review", "--diff-file"]) + .arg(&diff) + .args(["--output", "json"]) + .assert() + .code( + if true_candidate || malformed || scorer_failure || scorer_disabled { + 1 + } else { + 0 + }, + ); + let envelope: Value = serde_json::from_slice(&output.get_output().stdout).unwrap(); + let stderr = String::from_utf8_lossy(&output.get_output().stderr); + assert_eq!( + envelope["usageAccountingComplete"], true, + "{label}: {stderr}" + ); + let requests = server.received_requests().await.unwrap(); + let scorers = requests + .iter() + .filter(|request| request_system_contains(request, "independent second-model scorer")) + .collect::>(); + if malformed || scorer_failure || scorer_disabled { + assert_eq!(scorers.len(), if scorer_failure { 2 } else { 0 }); + let incidents = envelope["modelIncidents"].as_array().unwrap(); + assert!( + incidents + .iter() + .any(|incident| incident["category"] == "invalidOutput" + && incident["recovered"] == false + && incident.get("recovery").is_none()) + ); + if malformed { + assert!( + stderr.contains("refuted adjudication cannot publish revised finding text") + ); + } + if scorer_failure { + assert!( + envelope["scorerError"] + .as_str() + .unwrap() + .contains("scorer output invalid") + ); + } + let findings = envelope["findings"].as_array().unwrap(); + assert!( + findings + .iter() + .any(|finding| finding["path"] == ".postil/model-output") + ); + let original = findings + .iter() + .find(|finding| finding["path"] == "src/lib/logger.ts") + .unwrap(); + assert_eq!(original["confidence"], 0.95); + assert_eq!(original["evidence"], after); + assert_eq!(envelope["counts"]["suppressed"], 0); + assert_eq!(envelope["resolved"], json!([])); + assert_eq!(envelope["gate"]["failing"], true); + } else { + assert!( + envelope["modelIncidents"] + .as_array() + .is_none_or(Vec::is_empty) + ); + assert_eq!(scorers.len(), 1, "{label}: {stderr}"); + assert_eq!(requests.len(), 3); + assert_eq!(envelope["scorerModel"], "scorer-model"); + assert!(!stderr.contains("finding adjudication validation failed")); + let scorer: Value = scorers[0].body_json().unwrap(); + let input = scorer["messages"].as_array().unwrap().last().unwrap()["content"] + .as_str() + .unwrap(); + assert!(input.contains(title)); + assert!(input.contains(after)); + assert_eq!(envelope["gate"]["failing"], true_candidate); + if true_candidate { + assert_eq!(envelope["findings"].as_array().unwrap().len(), 1); + assert_eq!(envelope["findings"][0]["evidence"], after); + assert_eq!(envelope["findings"][0]["scorerConfidence"], 0.99); + } else { + assert!(envelope["findings"].as_array().unwrap().is_empty()); + assert_eq!(envelope["counts"]["suppressed"], 1); + } + } + assert_model_usage_matches_aggregate(&envelope); + } +} + +#[tokio::test] +async fn pending_refutation_recovery_survives_skipped_or_partial_scoring() { + for budget_exceeded in [false, true] { + let server = MockServer::start().await; + let evidence = if budget_exceeded { + format!( + "const payload = \"{}\";", + (0..75) + .map(|index| format!("{index:08x}")) + .collect::() + ) + } else { + "exec_query(input);".to_string() + }; + let findings = if budget_exceeded { + json!((0..18).map(|index| json!({ + "path": format!("src/input-{index}.rs"), "line": 1, "severity": "warn", "kind": "risk", "confidence": 0.95, + "title": if index == 0 { "Validate query input".to_string() } else { format!("Validate query input variant {}", char::from(b'a' + index as u8)) }, + "body": format!("Operation {} accepts unvalidated input. {}", char::from(b'a' + index as u8), "Validation must precede use. ".repeat(40).trim_end()), + "evidence": evidence + })).collect::>()) + } else { + json!([ + {"path": "src/auth.rs", "line": 41, "severity": "warn", "kind": "risk", "confidence": 0.95, + "title": "Check operation status", "body": "The status check ignores failed operations.", "evidence": "check_status();"}, + {"path": "src/auth.rs", "line": 42, "severity": "warn", "kind": "risk", "confidence": 0.95, + "title": "Validate query input", "body": "The operation accepts unvalidated input.", "evidence": evidence} + ]) + }; + // Each source request receives only findings grounded in its own input. + // Short evidence lines remain whole at the generator admission boundary. + Mock::given(method("POST")) + .and(path("/chat/completions")) + .respond_with(move |request: &Request| { + let request: Value = request.body_json().unwrap(); + let input = request["messages"].as_array().unwrap().last().unwrap()["content"] + .as_str() + .unwrap(); + let visible = findings + .as_array() + .unwrap() + .iter() + .filter(|finding| { + input.contains(finding["path"].as_str().unwrap()) + && input.contains(finding["evidence"].as_str().unwrap()) + }) + .cloned() + .collect::>(); + ResponseTemplate::new(200).set_body_json(llm_content(json!(visible))) + }) + .mount(&server) + .await; + Mock::given(method("POST")) + .and(path("/chat/completions")) + .and(body_string_contains("single finding adjudicator")) + .respond_with(|request: &Request| { + let request: Value = request.body_json().unwrap(); + let payload: Value = serde_json::from_str(request["messages"].as_array().unwrap().last().unwrap()["content"].as_str().unwrap()).unwrap(); + let results = payload["candidates"].as_array().unwrap().iter().map(|candidate| { + let refuted = candidate["title"] != "Check operation status"; + json!({"candidateId": candidate["candidateId"], + "status": if refuted { "refuted" } else { "unresolved" }, + "revisedTitle": "", "revisedBody": "", + "evidence": if refuted { candidate["citedEvidence"].clone() } else { json!("") }, + "duplicateOf": null}) + }).collect::>(); + ResponseTemplate::new(200).set_body_json(scorer_text(&json!(results).to_string())) + }) + .with_priority(1).expect(1).mount(&server).await; + Mock::given(method("POST")) + .and(path("/chat/completions")) + .and(body_string_contains("independent second-model scorer")) + .respond_with( + ResponseTemplate::new(200).set_body_json(scorer_content(json!([ + {"confidence": 0.01, "kind": "risk", "reason": "The status check is valid."} + ]))), + ) + .with_priority(1) + .expect(if budget_exceeded { 0 } else { 2 }) + .mount(&server) + .await; + let directory = tempfile::tempdir().unwrap(); + if budget_exceeded { + std::fs::create_dir(directory.path().join(".postil")).unwrap(); + std::fs::write( + directory.path().join(".postil/guardrails.md"), + "Validate input before use.\n".repeat(500), + ) + .unwrap(); + std::fs::write( + directory.path().join(".postil/content-policy.md"), + "State concrete defects.\n".repeat(500), + ) + .unwrap(); + } + let diff = directory.path().join("review.diff"); + let source = if budget_exceeded { + (0..18).map(|index| format!("diff --git a/src/input-{index}.rs b/src/input-{index}.rs\n--- a/src/input-{index}.rs\n+++ b/src/input-{index}.rs\n@@ -1 +1 @@\n-old();\n+{evidence}\n")).collect::() + } else { + format!( + "diff --git a/src/auth.rs b/src/auth.rs\n--- a/src/auth.rs\n+++ b/src/auth.rs\n@@ -40,2 +40,3 @@\n context\n-old();\n+check_status();\n+{evidence}\n" + ) + }; + std::fs::write(&diff, source).unwrap(); + let output = postil() + .current_dir(directory.path()) + .env("POSTIL_API_BASE", server.uri()) + .env("REVIEW_MODEL", "openai/gpt-5.6-luna") + .env("REVIEW_SCORER_MODEL", "openai/gpt-5.6-luna") + .env("POSTIL_CONCISE_FINDINGS", "false") + .args(["review", "--diff-file"]) + .arg(&diff) + .args(["--output", "json"]) + .assert() + .code(1); + let envelope: Value = serde_json::from_slice(&output.get_output().stdout).unwrap(); + let stderr = String::from_utf8_lossy(&output.get_output().stderr); + assert!( + stderr.contains("unsupported refutation recovery requires complete validated scoring"), + "budget_exceeded={budget_exceeded}: {stderr}" + ); + let error = envelope["scorerError"].as_str().unwrap(); + assert!( + error.contains(if budget_exceeded { + "bounded input budget was exceeded" + } else { + "expected 2 score(s), got 1" + }), + "budget_exceeded={budget_exceeded}: {error}" + ); + let findings = envelope["findings"].as_array().unwrap(); + assert_eq!(findings.len(), if budget_exceeded { 19 } else { 3 }); + let original = findings + .iter() + .find(|finding| finding["title"] == "Validate query input") + .unwrap(); + assert_eq!(original["confidence"], 0.95); + assert_eq!(original["evidence"], evidence); + assert!( + findings + .iter() + .any(|finding| finding["path"] == ".postil/model-output") + ); + assert_eq!(envelope["counts"]["suppressed"], 0); + assert_eq!(envelope["resolved"], json!([])); + assert_eq!(envelope["gate"]["failing"], true); + assert!( + envelope["modelIncidents"] + .as_array() + .unwrap() + .iter() + .any(|incident| incident["category"] == "invalidOutput" + && incident["recovered"] == false) + ); + assert_eq!(envelope["usageAccountingComplete"], true); + assert_model_usage_matches_aggregate(&envelope); + } +} + +#[cfg(feature = "qualification-candidate")] +#[tokio::test] +async fn hosted_pending_refutation_failure_preserves_the_strict_blocker() { + for refuted in [false, true] { + let server = MockServer::start().await; + Mock::given(method("POST")).and(path("/chat/completions")) + .respond_with(move |request: &Request| { + let mut response = if request_system_contains(request, "single finding adjudicator") { + let body: Value = request.body_json().unwrap(); + let payload: Value = serde_json::from_str(body["messages"].as_array().unwrap().last().unwrap()["content"].as_str().unwrap()).unwrap(); + scorer_text(&json!([{ + "candidateId": payload["candidates"][0]["candidateId"], + "status": if refuted { "refuted" } else { "unresolved" }, + "revisedTitle": "", "revisedBody": "", + "evidence": if refuted { "exec_query(&token);" } else { "" }, "duplicateOf": null + }]).to_string()) + } else if request_system_contains(request, "independent second-model scorer") { + scorer_content(json!([])) + } else { + llm_content(json!([{ + "path": "src/auth.rs", "line": 42, "severity": "warn", "kind": "risk", "confidence": 0.95, + "title": "Validate query input", "body": "The query executes unvalidated input.", "evidence": "exec_query(&token);" + }])) + }; + response["model"] = json!("openai/gpt-5.6-luna"); + response["provider"] = json!("Azure"); + ResponseTemplate::new(200).set_body_json(response) + }).with_priority(1).mount(&server).await; + let directory = tempfile::tempdir().unwrap(); + let profile = directory.path().join("profile.json"); + std::fs::write(&profile, include_str!("../provisional-models.json")).unwrap(); + let diff = write_diff(directory.path()); + let output = postil() + .current_dir(directory.path()) + .env("CI", "true") + .env("GITHUB_API_URL", "http://127.0.0.1:9") + .env("POSTIL_BENCH_REQUIRE_HOSTED_PROVIDER_PRIVACY", "1") + .env("POSTIL_QUALIFICATION_CANDIDATE_PROFILE", &profile) + .env("POSTIL_QUALIFICATION_CAPTURE_API_BASE", server.uri()) + .args(["review", "--diff-file"]) + .arg(&diff) + .args(["--output", "json"]) + .output() + .unwrap(); + let stderr = String::from_utf8_lossy(&output.stderr); + if refuted { + assert_eq!(output.status.code(), Some(1), "{stderr}"); + let envelope: Value = serde_json::from_slice(&output.stdout).unwrap(); + assert_eq!(envelope["gate"]["failing"], true); + assert!( + envelope["findings"] + .as_array() + .unwrap() + .iter() + .any(|finding| finding["title"] == "Validate query input" + && finding["confidence"] == 0.95) + ); + assert!( + envelope["findings"] + .as_array() + .unwrap() + .iter() + .any(|finding| finding["path"] == ".postil/model-output") + ); + assert!( + envelope["modelIncidents"] + .as_array() + .unwrap() + .iter() + .any(|incident| incident["category"] == "invalidOutput" + && incident["recovered"] == false) + ); + assert!( + stderr.contains( + "unsupported refutation recovery requires complete validated scoring" + ) + ); + assert_eq!(envelope["usageAccountingComplete"], true); + assert_model_usage_matches_aggregate(&envelope); + } else { + assert_eq!(output.status.code(), Some(1), "{stderr}"); + let envelope: Value = serde_json::from_slice(&output.stdout).unwrap(); + assert!( + !envelope["findings"] + .as_array() + .unwrap() + .iter() + .any(|finding| finding["title"] == "Validate query input") + ); + assert!( + !stderr.contains( + "unsupported refutation recovery requires complete validated scoring" + ) + ); + assert!(stderr.contains("hosted scorer could not complete the admitted profile")); + } + let requests = server.received_requests().await.unwrap(); + assert_eq!( + requests + .iter() + .filter(|request| request_system_contains( + request, + "independent second-model scorer" + )) + .count(), + 2 + ); + } +} + #[tokio::test] async fn malformed_adjudication_output_blocks_under_advisory_provider_policy() { let server = MockServer::start().await; @@ -13343,6 +13789,116 @@ async fn full_rereview_preserves_unresolved_baseline_for_unavailable_and_exhaust } } +#[tokio::test] +async fn pending_refutation_recovery_commits_baseline_resolution_only_after_scoring() { + for scorer_succeeds in [false, true] { + let server = MockServer::start().await; + mock_review_model(&server, "generator-model", json!([{ + "path": "src/auth.rs", "line": 1, "severity": "warn", "kind": "risk", "confidence": 0.95, + "title": "Preserve login audit", "body": "The login path omits the required audit event.", + "evidence": "fn login() { authenticate(); }" + }])).await; + Mock::given(method("POST")).and(path("/chat/completions")) + .and(body_string_contains("single finding adjudicator")) + .respond_with(|request: &Request| { + let body: Value = request.body_json().unwrap(); + let payload: Value = serde_json::from_str(body["messages"].as_array().unwrap().last().unwrap()["content"].as_str().unwrap()).unwrap(); + let evidence = payload["repositoryEvidence"].as_array().unwrap().first().unwrap()["source"].as_str().unwrap(); + let results = payload["candidates"].as_array().unwrap().iter().map(|candidate| json!({ + "candidateId": candidate["candidateId"], "status": "refuted", + "revisedTitle": "", "revisedBody": "", "duplicateOf": null, + "evidence": if candidate["repositoryContext"].is_object() { json!(evidence) } else { candidate["citedEvidence"].clone() } + })).collect::>(); + ResponseTemplate::new(200).set_body_json(scorer_text(&json!(results).to_string())) + }).with_priority(1).expect(1).mount(&server).await; + Mock::given(method("POST")).and(path("/chat/completions")) + .and(body_string_contains("independent second-model scorer")) + .respond_with(ResponseTemplate::new(200).set_body_json(scorer_content(if scorer_succeeds { + json!([{"confidence": 0.01, "kind": "risk", "reason": "The audit finding is unsupported."}]) + } else { json!([]) }))) + .with_priority(1).expect(if scorer_succeeds { 1 } else { 2 }).mount(&server).await; + let directory = tempfile::tempdir().unwrap(); + initialize_staged_repository_with_unchanged_caller(directory.path()); + let baseline = json!({ + "version": 1, "summary": "", "silent": false, + "findings": [{"path": "src/auth.rs", "line": 1, "severity": "warn", "kind": "risk", + "confidence": 0.95, "title": "Legacy API has no callers", + "body": format!("The repository has no caller for `legacy_api`; remove it or restore its caller. {}.", (0..180).map(|index| format!("q{index:03}")).collect::>().join(" ")), + "evidence": "fn login() {}", "repositoryContext": {"claim": "absence", "identifiers": ["legacy_api"]}}], + "resolved": [], "counts": {"info": 0, "warn": 1, "error": 0, "suppressed": 0}, + "confidenceBuckets": [0,0,0,0,1], "gate": {"failOn": "error", "failing": true}, + "modelUsed": "model", "usage": {"promptTokens": 0, "completionTokens": 0}, + "baseSha": null, "headSha": null, "sinceSha": null + }); + let baseline_path = directory.path().join("baseline.json"); + std::fs::write(&baseline_path, baseline.to_string()).unwrap(); + let output = postil() + .current_dir(directory.path()) + .env("POSTIL_API_BASE", server.uri()) + .env("REVIEW_MODEL", "generator-model") + .env("REVIEW_SCORER_MODEL", "scorer-model") + .args(["review", "--staged", "--baseline"]) + .arg(&baseline_path) + .args(["--output", "json"]) + .assert() + .code(if scorer_succeeds { 0 } else { 1 }); + let envelope: Value = serde_json::from_slice(&output.get_output().stdout).unwrap(); + assert_eq!(envelope["usageAccountingComplete"], true); + assert_model_usage_matches_aggregate(&envelope); + if scorer_succeeds { + assert_eq!(envelope["resolved"].as_array().unwrap().len(), 1); + assert_eq!( + envelope["resolved"][0]["title"], + "Legacy API has no callers" + ); + assert_eq!(envelope["counts"]["suppressed"], 2); + assert_eq!( + envelope["suppressedFindings"] + .as_array() + .unwrap() + .iter() + .map(|entry| entry["finding"]["title"].as_str().unwrap()) + .collect::>(), + vec!["Legacy API has no callers", "Preserve login audit"] + ); + } else { + assert_eq!(envelope["resolved"], json!([])); + assert_eq!(envelope["counts"]["suppressed"], 0); + let findings = envelope["findings"].as_array().unwrap(); + assert_eq!(findings.len(), 3); + assert!( + findings + .iter() + .any(|finding| finding["title"] == "Legacy API has no callers" + && finding["severity"] == "error") + ); + assert!( + findings + .iter() + .any(|finding| finding["title"] == "Preserve login audit" + && finding["confidence"] == 0.95) + ); + assert!( + findings + .iter() + .any(|finding| finding["path"] == ".postil/model-output") + ); + } + let requests = server.received_requests().await.unwrap(); + for request in requests + .iter() + .filter(|request| request_system_contains(request, "independent second-model scorer")) + { + let body: Value = request.body_json().unwrap(); + let input = body["messages"].as_array().unwrap().last().unwrap()["content"] + .as_str() + .unwrap(); + assert!(input.contains("Preserve login audit")); + assert!(!input.contains("Legacy API has no callers")); + } + } +} + #[tokio::test] async fn full_rereview_resolves_false_absence_from_unchanged_repository_source() { let server = MockServer::start().await;