From e856df01b92519f056fd01a978fae0e4fb6d1ef4 Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Tue, 18 Aug 2026 10:52:03 +0000 Subject: [PATCH 1/4] Report the real cause when a review fails before producing a verdict A review that stopped for any reason other than a recorded scorer error was published as "Model output could not be validated" with the detail "scorer output did not satisfy the admitted contract". That branch is reached by admission limits, configuration problems, and unexpected internal errors, and it discarded the error it was handed. The result named a component that may never have run and hid the only text that identified the fault, so every reader began by investigating the model. Failures carrying a recorded scorer error keep the existing wording. Everything else now reports itself through an operational finding that states the review could not be completed and carries the underlying error, on the same path and with the same fail-closed severity, so gate behaviour and operational metrics are unchanged. --- src/envelope.rs | 34 +++++++++++++++++++++++++++ src/review.rs | 61 +++++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 90 insertions(+), 5 deletions(-) diff --git a/src/envelope.rs b/src/envelope.rs index 505dc11..9f6fd40 100644 --- a/src/envelope.rs +++ b/src/envelope.rs @@ -1735,6 +1735,40 @@ pub(crate) fn incomplete_review_finding(reason: IncompleteReviewReason) -> Findi } } +/// The synthetic finding emitted when a review stopped before producing a +/// verdict for a reason that is not model output: an admission limit, a +/// configuration problem, or an unexpected internal error. Postil still fails +/// closed, but the cause is reported as itself. Attributing such a failure to +/// model output sends every reader looking at the model when the model was +/// never called. +pub fn operational_failure_finding(detail: &str) -> Finding { + let detail = detail.trim(); + let opening = "Postil could not complete this review, so no verdict was issued."; + let body = if detail.is_empty() || prose_exposes_evidence_boundary(detail) { + opening.to_string() + } else { + format!("{opening}\n\nDetail: {detail}") + }; + Finding { + path: OPERATIONAL_PATH.to_string(), + line: 1, + end_line: None, + severity: Severity::Error, + kind: Kind::Uncertainty, + confidence: 1.0, + title: "Review could not be completed".to_string(), + body, + evidence: None, + id: None, + generator_confidence: None, + scorer_confidence: None, + generator_kind: None, + scorer_kind: None, + scorer_reason: None, + repository_claim: None, + } +} + /// The synthetic finding emitted when the model produced unusable output. /// Postil fails closed: a review that could not be trusted is an error, not a pass. pub fn fail_closed_finding(detail: &str) -> Finding { diff --git a/src/review.rs b/src/review.rs index fa7beba..6598d40 100644 --- a/src/review.rs +++ b/src/review.rs @@ -3206,11 +3206,14 @@ fn error_envelope( crate::envelope::IncompleteReviewReason::IncompleteInput, ) } else if invalid_output || !advisory_operational_error { - fail_closed_finding( - review_failure - .and_then(|failure| failure.scorer_error.as_deref()) - .unwrap_or("scorer output did not satisfy the admitted contract"), - ) + // Only a recorded scorer error is genuinely about model output. Every + // other failure here reaches this branch with its cause in `err`, so + // report that instead of asserting a scorer contract that may never + // have been exercised. + match review_failure.and_then(|failure| failure.scorer_error.as_deref()) { + Some(scorer_error) => fail_closed_finding(scorer_error), + None => crate::envelope::operational_failure_finding(&format!("{err:#}")), + } } else { crate::envelope::provider_error_finding(&format!("{err:#}")) }]; @@ -3386,6 +3389,54 @@ mod tests { } } + #[test] + fn non_model_failures_report_their_own_cause() { + let envelope = error_envelope( + &Config::default(), + &anyhow::anyhow!( + "hosted review admission projects 11455328 micro-dollars of provider \ + exposure, exceeding the 1000000 micro-dollar operation cap" + ), + "head", + Some(&pr_meta()), + 1, + ); + let finding = &envelope.findings[0]; + assert_eq!(finding.path, crate::envelope::OPERATIONAL_PATH); + assert_eq!(finding.title, "Review could not be completed"); + assert!( + finding.body.contains("micro-dollar operation cap"), + "the real cause must reach the reader: {}", + finding.body + ); + assert!( + !finding.body.contains("scorer"), + "a failure that never ran the scorer must not blame it: {}", + finding.body + ); + assert!(envelope.gate.failing); + } + + #[test] + fn recorded_scorer_failures_still_report_the_scorer() { + let envelope = error_envelope( + &Config::default(), + &rich_scorer_failure(ReviewFailureKind::InvalidOutput), + "head", + Some(&pr_meta()), + 1, + ); + let finding = &envelope.findings[0]; + assert_eq!(finding.title, "Model output could not be validated"); + assert!( + finding + .body + .contains("scorer output invalid after schema repair"), + "the recorded scorer error must be preserved: {}", + finding.body + ); + } + #[test] fn disabled_gate_keeps_provider_error_envelopes_nonblocking() { let cfg = Config { From da9b76b7c34d2d9402d4a9889f11092458c64ba8 Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Tue, 18 Aug 2026 10:56:38 +0000 Subject: [PATCH 2/4] Take the h2 fix for unbounded empty DATA frames RUSTSEC-2026-0258 reports that h2 0.4.14 accepts unbounded empty DATA frames, which lets a peer hold a connection open and consume processing without advancing a stream. The advisory landed after the current lockfile, so audit scanning fails on every branch until the dependency moves to 0.4.16. --- Cargo.lock | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4e90407..daa9fc5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -47,7 +47,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -58,7 +58,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -379,7 +379,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -421,7 +421,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -596,9 +596,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.14" +version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" +checksum = "a9f37a958b41b3b19ee2707c06439c0e9e547e847223eb791ecb0cb821c65e27" dependencies = [ "atomic-waker", "bytes", @@ -1414,7 +1414,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -1471,7 +1471,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -1685,7 +1685,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -1747,7 +1747,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -2165,7 +2165,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] From b95d8de1db0306252ce2859f17854b01f4529566 Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Tue, 18 Aug 2026 12:14:46 +0000 Subject: [PATCH 3/4] Separate the admission projection ceiling from the operation spend cap Hosted admission prices a worst case rather than an estimate: the full bounded selection of batches, every logical call the watchdog plan allows, and a hostile maximum-size retry payload on each one, at the admitted price bounds. The shipped profile projects about 15.6 dollars that way, while a review actually spends on the order of a cent. Both quantities were compared against one constant, so the projection could not clear it and every hosted review was refused before provider contact regardless of how small the change was. The spend cap keeps its value and its meaning: accumulated provider cost within one operation still stops at the per-review reservation the service holds, so a review cannot consume more of an organization's allowance than was set aside for it. Admission now has its own ceiling above the planning envelope. The existing plan test priced every token at one micro per million, so it could not observe either limit. A new test prices the shipped profile at its real bounds, which fails when the projection outgrows the ceiling instead of letting every production review fail. --- src/config.rs | 21 ++++++++++++++ src/llm.rs | 78 ++++++++++++++++++++++++++++++++++++++++++++++----- tests/e2e.rs | 9 ++++-- 3 files changed, 98 insertions(+), 10 deletions(-) diff --git a/src/config.rs b/src/config.rs index 9feb7a2..3920669 100644 --- a/src/config.rs +++ b/src/config.rs @@ -211,7 +211,28 @@ struct QualificationProfileDigestMaterial<'a> { repeated_runs: u32, } +/// Ceiling on what a hosted operation may actually spend, accumulated across +/// provider attempts as they are reserved. This matches the per-review +/// reservation the service holds, so a review can never consume more of an +/// organization's allowance than was set aside for it. pub const HOSTED_OPERATION_COST_CAP_MICROS: u64 = 1_000_000; + +/// Ceiling on what a hosted review may *project* before it runs. Admission +/// prices a worst case rather than an estimate: the full bounded selection of +/// `MAX_HOSTED_SELECTED_BATCHES` batches, every logical call the watchdog plan +/// allows, and a hostile maximum-size retry payload on each one, at the +/// admitted price bounds. The shipped profile projects roughly 15.6 dollars +/// while a review actually spends on the order of 0.01 dollars, so the two +/// quantities differ by three orders of magnitude and cannot share a limit. +/// +/// Holding the projection to the spend cap rejects every review before provider +/// contact, whatever its size, and surfaces as a model failure rather than a +/// budget decision. Spend stays bounded by +/// `HOSTED_OPERATION_COST_CAP_MICROS` above, which this deliberately does not +/// relax; `hosted_review_plan_is_admitted_at_shipped_price_bounds` prices the +/// shipped profile at its real bounds so a projection that outgrows this +/// ceiling fails a test instead of every production review. +pub const HOSTED_ADMISSION_PROJECTION_CAP_MICROS: u64 = 25_000_000; pub const QUALIFICATION_MAX_AGE_DAYS: u32 = 30; const QUALIFICATION_MAX_AGE_SECONDS: u64 = QUALIFICATION_MAX_AGE_DAYS as u64 * 24 * 60 * 60; const MAX_SAFE_JSON_INTEGER: u64 = 9_007_199_254_740_991; diff --git a/src/llm.rs b/src/llm.rs index 2bfeb6a..7d97673 100644 --- a/src/llm.rs +++ b/src/llm.rs @@ -19,7 +19,10 @@ use time::Date; use crate::adjudication::{AdjudicationResult, MAX_ADJUDICATION_OUTPUT_TOKENS}; use crate::api_key; -use crate::config::{ApiFormat, Config, HOSTED_OPERATION_COST_CAP_MICROS, ModelPriceBound}; +use crate::config::{ + ApiFormat, Config, HOSTED_ADMISSION_PROJECTION_CAP_MICROS, HOSTED_OPERATION_COST_CAP_MICROS, + ModelPriceBound, +}; use crate::envelope::{ Finding, Kind, ModelIncident, ModelIncidentCategory, ModelIncidentPhase, ModelIncidentRecovery, ModelUsage, ModelUsageCostSource, ModelUsagePhase, ModelUsageRole, ProviderCost, @@ -2169,8 +2172,8 @@ impl LlmClient { .collect::>() .join(", "); ensure!( - exposure.projected_cost_micros <= HOSTED_OPERATION_COST_CAP_MICROS, - "hosted {operation} admission projects {} micro-dollars of provider exposure across {} attempts, {} serialized input bytes, and {} output tokens (per-model micro-dollars: {model_costs}), exceeding the {HOSTED_OPERATION_COST_CAP_MICROS} micro-dollar operation cap", + exposure.projected_cost_micros <= HOSTED_ADMISSION_PROJECTION_CAP_MICROS, + "hosted {operation} admission projects {} micro-dollars of provider exposure across {} attempts, {} serialized input bytes, and {} output tokens (per-model micro-dollars: {model_costs}), exceeding the {HOSTED_ADMISSION_PROJECTION_CAP_MICROS} micro-dollar admission projection cap", exposure.projected_cost_micros, exposure.attempts, exposure.input_bytes, @@ -8040,6 +8043,64 @@ mod tests { assert_eq!(client.admission.lock().unwrap().attempts, 0); } + /// The shipped provisional hosted profile must actually clear the operation + /// cost cap. The neighbouring plan test prices every token at one micro per + /// million, so it cannot observe the cap at all; this one uses the real + /// admitted price bounds, which is the only way a cap that rejects every + /// production review shows up as a test failure rather than an outage. + #[test] + fn hosted_review_plan_is_admitted_at_shipped_price_bounds() { + let config = Config { + model: "z-ai/glm-5.2".into(), + scorer_enabled: false, + ..Config::default() + }; + let mut client = LlmClient::build( + &config, + "test-key".into(), + Duration::from_secs(1), + None, + None, + ) + .unwrap(); + client.request_decorations.hosted_price_bounds = Some(Arc::new(HashMap::from([( + "z-ai/glm-5.2".into(), + ModelPriceBound { + model: "z-ai/glm-5.2".into(), + input_micros_per_million_tokens: 1_400_000, + output_micros_per_million_tokens: 4_400_000, + }, + )]))); + let users = + vec!["bounded candidate".to_string(); crate::review::MAX_HOSTED_SELECTED_BATCHES]; + let output_tokens = vec![REVIEW_MAX_TOKENS; crate::review::MAX_HOSTED_SELECTED_BATCHES]; + let manifest = "m".repeat(96_000); + let admission = client + .preflight_review_plan_with_output_limits( + &config, + crate::review::MAX_HOSTED_SELECTED_BATCHES, + "system", + ReviewPreflightPrompts { + first_users: &users, + later_users: &users, + output_tokens: &output_tokens, + scorer_system: "scorer system", + current_utc_date: trusted_date(), + }, + ReviewPlanSchedule { + planner: Some((&manifest, 1)), + batch_concurrency: 1, + }, + ) + .expect("the shipped hosted profile must be admissible"); + assert!( + admission.projected_cost_micros <= HOSTED_ADMISSION_PROJECTION_CAP_MICROS, + "projected {} exceeds the {} admission cap", + admission.projected_cost_micros, + HOSTED_ADMISSION_PROJECTION_CAP_MICROS + ); + } + #[test] fn maximum_hosted_plan_matches_watchdog_and_transport_arithmetic() { let config = Config { @@ -8282,10 +8343,10 @@ mod tests { attempts: 6, input_bytes: 12_345, output_tokens: 678, - projected_cost_micros: HOSTED_OPERATION_COST_CAP_MICROS + 1, + projected_cost_micros: HOSTED_ADMISSION_PROJECTION_CAP_MICROS + 1, model_costs_micros: BTreeMap::from([( "provider/model".to_string(), - HOSTED_OPERATION_COST_CAP_MICROS + 1, + HOSTED_ADMISSION_PROJECTION_CAP_MICROS + 1, )]), }, ) @@ -8294,7 +8355,10 @@ mod tests { assert!(message.contains("provider exposure across 6 attempts")); assert!(message.contains("12345 serialized input bytes")); assert!(message.contains("678 output tokens")); - assert!(message.contains(r#""provider/model"=1000001"#)); + assert!(message.contains(&format!( + r#""provider/model"={}"#, + HOSTED_ADMISSION_PROJECTION_CAP_MICROS + 1 + ))); } #[test] @@ -8332,7 +8396,7 @@ mod tests { ); assert_eq!(exposure.projected_cost_micros, 1_056); - exposure.projected_cost_micros = HOSTED_OPERATION_COST_CAP_MICROS + 1; + exposure.projected_cost_micros = HOSTED_ADMISSION_PROJECTION_CAP_MICROS + 1; let error = LlmClient::build( &Config::default(), "test-key".into(), diff --git a/tests/e2e.rs b/tests/e2e.rs index ccb6388..5e483c4 100644 --- a/tests/e2e.rs +++ b/tests/e2e.rs @@ -2640,10 +2640,13 @@ async fn hosted_cost_rejection_precedes_durable_plan_registration() { "generatorChain": [model], "consensus": 1, "scorerChain": [model], + // Priced well above any admissible profile so the plan is rejected + // on cost. The shipped bounds project inside the admission ceiling, + // so a realistic price here would exercise the happy path instead. "modelPriceBounds": [{ "model": model, - "inputMicrosPerMillionTokens": 1_000_000, - "outputMicrosPerMillionTokens": 1_000_000 + "inputMicrosPerMillionTokens": 100_000_000, + "outputMicrosPerMillionTokens": 100_000_000 }] })) .unwrap(), @@ -2672,7 +2675,7 @@ async fn hosted_cost_rejection_precedes_durable_plan_registration() { .code(2); let stderr = String::from_utf8_lossy(&out.get_output().stderr); assert!(stderr.contains("hosted review admission projects")); - assert!(stderr.contains("operation cap")); + assert!(stderr.contains("admission projection cap")); assert!(out.get_output().stdout.is_empty()); assert!( registration_server From a676500bec8868b4822d090d5dbe9a915e86357f Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Tue, 18 Aug 2026 12:22:01 +0000 Subject: [PATCH 4/4] Price the evaluator exposure fixture above the admission ceiling The runtime-shaped preflight case proved that an over-budget plan is refused before it runs, using bounds that sat above the old shared cap. With admission measured against its own ceiling those bounds are admissible, so the case stopped exercising a refusal. Pricing it an order of magnitude above any admissible profile restores what it was written to check. --- bench/src/livemodels.test.ts | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/bench/src/livemodels.test.ts b/bench/src/livemodels.test.ts index 43b30bf..43bd532 100644 --- a/bench/src/livemodels.test.ts +++ b/bench/src/livemodels.test.ts @@ -613,20 +613,23 @@ describe("pair qualification configuration", () => { generatorModel: "z-ai/glm-5.2", scorerModel: "openai/gpt-5-mini", }; + // Priced an order of magnitude above any admissible profile so the plan is + // refused on projected exposure. Realistic bounds project inside the + // admission ceiling and would exercise the happy path instead. const pricing = new Map([ [runtimePair.generatorModel, { providerIdentity: "PinnedProvider", - promptUsdPerToken: 0.00028434, - completionUsdPerToken: 0.00089364, - inputMicrosPerMillionTokens: 284_340, - outputMicrosPerMillionTokens: 893_640, + promptUsdPerToken: 0.0028434, + completionUsdPerToken: 0.0089364, + inputMicrosPerMillionTokens: 2_843_400, + outputMicrosPerMillionTokens: 8_936_400, }], [runtimePair.scorerModel, { providerIdentity: "PinnedProvider", - promptUsdPerToken: 0.00028434, - completionUsdPerToken: 0.00089364, - inputMicrosPerMillionTokens: 284_340, - outputMicrosPerMillionTokens: 893_640, + promptUsdPerToken: 0.0028434, + completionUsdPerToken: 0.0089364, + inputMicrosPerMillionTokens: 2_843_400, + outputMicrosPerMillionTokens: 8_936_400, }], ]); try { @@ -649,7 +652,7 @@ describe("pair qualification configuration", () => { apiFormat: "openai-compatible", costCapUsdDecimal: "70", upstreamProvider: "PinnedProvider", - })).rejects.toThrow("exceeding the 1000000 micro-dollar operation cap"); + })).rejects.toThrow("micro-dollar admission projection cap"); } finally { if (inheritedModelKey === undefined) delete process.env.MODEL_API_KEY; else process.env.MODEL_API_KEY = inheritedModelKey;