diff --git a/Cargo.lock b/Cargo.lock index 4e90407..daa9fc5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -47,7 +47,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -58,7 +58,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -379,7 +379,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -421,7 +421,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -596,9 +596,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.14" +version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" +checksum = "a9f37a958b41b3b19ee2707c06439c0e9e547e847223eb791ecb0cb821c65e27" dependencies = [ "atomic-waker", "bytes", @@ -1414,7 +1414,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -1471,7 +1471,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -1685,7 +1685,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -1747,7 +1747,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -2165,7 +2165,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] diff --git a/bench/src/livemodels.test.ts b/bench/src/livemodels.test.ts index 43b30bf..43bd532 100644 --- a/bench/src/livemodels.test.ts +++ b/bench/src/livemodels.test.ts @@ -613,20 +613,23 @@ describe("pair qualification configuration", () => { generatorModel: "z-ai/glm-5.2", scorerModel: "openai/gpt-5-mini", }; + // Priced an order of magnitude above any admissible profile so the plan is + // refused on projected exposure. Realistic bounds project inside the + // admission ceiling and would exercise the happy path instead. const pricing = new Map([ [runtimePair.generatorModel, { providerIdentity: "PinnedProvider", - promptUsdPerToken: 0.00028434, - completionUsdPerToken: 0.00089364, - inputMicrosPerMillionTokens: 284_340, - outputMicrosPerMillionTokens: 893_640, + promptUsdPerToken: 0.0028434, + completionUsdPerToken: 0.0089364, + inputMicrosPerMillionTokens: 2_843_400, + outputMicrosPerMillionTokens: 8_936_400, }], [runtimePair.scorerModel, { providerIdentity: "PinnedProvider", - promptUsdPerToken: 0.00028434, - completionUsdPerToken: 0.00089364, - inputMicrosPerMillionTokens: 284_340, - outputMicrosPerMillionTokens: 893_640, + promptUsdPerToken: 0.0028434, + completionUsdPerToken: 0.0089364, + inputMicrosPerMillionTokens: 2_843_400, + outputMicrosPerMillionTokens: 8_936_400, }], ]); try { @@ -649,7 +652,7 @@ describe("pair qualification configuration", () => { apiFormat: "openai-compatible", costCapUsdDecimal: "70", upstreamProvider: "PinnedProvider", - })).rejects.toThrow("exceeding the 1000000 micro-dollar operation cap"); + })).rejects.toThrow("micro-dollar admission projection cap"); } finally { if (inheritedModelKey === undefined) delete process.env.MODEL_API_KEY; else process.env.MODEL_API_KEY = inheritedModelKey; diff --git a/src/config.rs b/src/config.rs index 9feb7a2..3920669 100644 --- a/src/config.rs +++ b/src/config.rs @@ -211,7 +211,28 @@ struct QualificationProfileDigestMaterial<'a> { repeated_runs: u32, } +/// Ceiling on what a hosted operation may actually spend, accumulated across +/// provider attempts as they are reserved. This matches the per-review +/// reservation the service holds, so a review can never consume more of an +/// organization's allowance than was set aside for it. pub const HOSTED_OPERATION_COST_CAP_MICROS: u64 = 1_000_000; + +/// Ceiling on what a hosted review may *project* before it runs. Admission +/// prices a worst case rather than an estimate: the full bounded selection of +/// `MAX_HOSTED_SELECTED_BATCHES` batches, every logical call the watchdog plan +/// allows, and a hostile maximum-size retry payload on each one, at the +/// admitted price bounds. The shipped profile projects roughly 15.6 dollars +/// while a review actually spends on the order of 0.01 dollars, so the two +/// quantities differ by three orders of magnitude and cannot share a limit. +/// +/// Holding the projection to the spend cap rejects every review before provider +/// contact, whatever its size, and surfaces as a model failure rather than a +/// budget decision. Spend stays bounded by +/// `HOSTED_OPERATION_COST_CAP_MICROS` above, which this deliberately does not +/// relax; `hosted_review_plan_is_admitted_at_shipped_price_bounds` prices the +/// shipped profile at its real bounds so a projection that outgrows this +/// ceiling fails a test instead of every production review. +pub const HOSTED_ADMISSION_PROJECTION_CAP_MICROS: u64 = 25_000_000; pub const QUALIFICATION_MAX_AGE_DAYS: u32 = 30; const QUALIFICATION_MAX_AGE_SECONDS: u64 = QUALIFICATION_MAX_AGE_DAYS as u64 * 24 * 60 * 60; const MAX_SAFE_JSON_INTEGER: u64 = 9_007_199_254_740_991; diff --git a/src/envelope.rs b/src/envelope.rs index 505dc11..9f6fd40 100644 --- a/src/envelope.rs +++ b/src/envelope.rs @@ -1735,6 +1735,40 @@ pub(crate) fn incomplete_review_finding(reason: IncompleteReviewReason) -> Findi } } +/// The synthetic finding emitted when a review stopped before producing a +/// verdict for a reason that is not model output: an admission limit, a +/// configuration problem, or an unexpected internal error. Postil still fails +/// closed, but the cause is reported as itself. Attributing such a failure to +/// model output sends every reader looking at the model when the model was +/// never called. +pub fn operational_failure_finding(detail: &str) -> Finding { + let detail = detail.trim(); + let opening = "Postil could not complete this review, so no verdict was issued."; + let body = if detail.is_empty() || prose_exposes_evidence_boundary(detail) { + opening.to_string() + } else { + format!("{opening}\n\nDetail: {detail}") + }; + Finding { + path: OPERATIONAL_PATH.to_string(), + line: 1, + end_line: None, + severity: Severity::Error, + kind: Kind::Uncertainty, + confidence: 1.0, + title: "Review could not be completed".to_string(), + body, + evidence: None, + id: None, + generator_confidence: None, + scorer_confidence: None, + generator_kind: None, + scorer_kind: None, + scorer_reason: None, + repository_claim: None, + } +} + /// The synthetic finding emitted when the model produced unusable output. /// Postil fails closed: a review that could not be trusted is an error, not a pass. pub fn fail_closed_finding(detail: &str) -> Finding { diff --git a/src/llm.rs b/src/llm.rs index 2bfeb6a..7d97673 100644 --- a/src/llm.rs +++ b/src/llm.rs @@ -19,7 +19,10 @@ use time::Date; use crate::adjudication::{AdjudicationResult, MAX_ADJUDICATION_OUTPUT_TOKENS}; use crate::api_key; -use crate::config::{ApiFormat, Config, HOSTED_OPERATION_COST_CAP_MICROS, ModelPriceBound}; +use crate::config::{ + ApiFormat, Config, HOSTED_ADMISSION_PROJECTION_CAP_MICROS, HOSTED_OPERATION_COST_CAP_MICROS, + ModelPriceBound, +}; use crate::envelope::{ Finding, Kind, ModelIncident, ModelIncidentCategory, ModelIncidentPhase, ModelIncidentRecovery, ModelUsage, ModelUsageCostSource, ModelUsagePhase, ModelUsageRole, ProviderCost, @@ -2169,8 +2172,8 @@ impl LlmClient { .collect::>() .join(", "); ensure!( - exposure.projected_cost_micros <= HOSTED_OPERATION_COST_CAP_MICROS, - "hosted {operation} admission projects {} micro-dollars of provider exposure across {} attempts, {} serialized input bytes, and {} output tokens (per-model micro-dollars: {model_costs}), exceeding the {HOSTED_OPERATION_COST_CAP_MICROS} micro-dollar operation cap", + exposure.projected_cost_micros <= HOSTED_ADMISSION_PROJECTION_CAP_MICROS, + "hosted {operation} admission projects {} micro-dollars of provider exposure across {} attempts, {} serialized input bytes, and {} output tokens (per-model micro-dollars: {model_costs}), exceeding the {HOSTED_ADMISSION_PROJECTION_CAP_MICROS} micro-dollar admission projection cap", exposure.projected_cost_micros, exposure.attempts, exposure.input_bytes, @@ -8040,6 +8043,64 @@ mod tests { assert_eq!(client.admission.lock().unwrap().attempts, 0); } + /// The shipped provisional hosted profile must actually clear the operation + /// cost cap. The neighbouring plan test prices every token at one micro per + /// million, so it cannot observe the cap at all; this one uses the real + /// admitted price bounds, which is the only way a cap that rejects every + /// production review shows up as a test failure rather than an outage. + #[test] + fn hosted_review_plan_is_admitted_at_shipped_price_bounds() { + let config = Config { + model: "z-ai/glm-5.2".into(), + scorer_enabled: false, + ..Config::default() + }; + let mut client = LlmClient::build( + &config, + "test-key".into(), + Duration::from_secs(1), + None, + None, + ) + .unwrap(); + client.request_decorations.hosted_price_bounds = Some(Arc::new(HashMap::from([( + "z-ai/glm-5.2".into(), + ModelPriceBound { + model: "z-ai/glm-5.2".into(), + input_micros_per_million_tokens: 1_400_000, + output_micros_per_million_tokens: 4_400_000, + }, + )]))); + let users = + vec!["bounded candidate".to_string(); crate::review::MAX_HOSTED_SELECTED_BATCHES]; + let output_tokens = vec![REVIEW_MAX_TOKENS; crate::review::MAX_HOSTED_SELECTED_BATCHES]; + let manifest = "m".repeat(96_000); + let admission = client + .preflight_review_plan_with_output_limits( + &config, + crate::review::MAX_HOSTED_SELECTED_BATCHES, + "system", + ReviewPreflightPrompts { + first_users: &users, + later_users: &users, + output_tokens: &output_tokens, + scorer_system: "scorer system", + current_utc_date: trusted_date(), + }, + ReviewPlanSchedule { + planner: Some((&manifest, 1)), + batch_concurrency: 1, + }, + ) + .expect("the shipped hosted profile must be admissible"); + assert!( + admission.projected_cost_micros <= HOSTED_ADMISSION_PROJECTION_CAP_MICROS, + "projected {} exceeds the {} admission cap", + admission.projected_cost_micros, + HOSTED_ADMISSION_PROJECTION_CAP_MICROS + ); + } + #[test] fn maximum_hosted_plan_matches_watchdog_and_transport_arithmetic() { let config = Config { @@ -8282,10 +8343,10 @@ mod tests { attempts: 6, input_bytes: 12_345, output_tokens: 678, - projected_cost_micros: HOSTED_OPERATION_COST_CAP_MICROS + 1, + projected_cost_micros: HOSTED_ADMISSION_PROJECTION_CAP_MICROS + 1, model_costs_micros: BTreeMap::from([( "provider/model".to_string(), - HOSTED_OPERATION_COST_CAP_MICROS + 1, + HOSTED_ADMISSION_PROJECTION_CAP_MICROS + 1, )]), }, ) @@ -8294,7 +8355,10 @@ mod tests { assert!(message.contains("provider exposure across 6 attempts")); assert!(message.contains("12345 serialized input bytes")); assert!(message.contains("678 output tokens")); - assert!(message.contains(r#""provider/model"=1000001"#)); + assert!(message.contains(&format!( + r#""provider/model"={}"#, + HOSTED_ADMISSION_PROJECTION_CAP_MICROS + 1 + ))); } #[test] @@ -8332,7 +8396,7 @@ mod tests { ); assert_eq!(exposure.projected_cost_micros, 1_056); - exposure.projected_cost_micros = HOSTED_OPERATION_COST_CAP_MICROS + 1; + exposure.projected_cost_micros = HOSTED_ADMISSION_PROJECTION_CAP_MICROS + 1; let error = LlmClient::build( &Config::default(), "test-key".into(), diff --git a/src/review.rs b/src/review.rs index fa7beba..6598d40 100644 --- a/src/review.rs +++ b/src/review.rs @@ -3206,11 +3206,14 @@ fn error_envelope( crate::envelope::IncompleteReviewReason::IncompleteInput, ) } else if invalid_output || !advisory_operational_error { - fail_closed_finding( - review_failure - .and_then(|failure| failure.scorer_error.as_deref()) - .unwrap_or("scorer output did not satisfy the admitted contract"), - ) + // Only a recorded scorer error is genuinely about model output. Every + // other failure here reaches this branch with its cause in `err`, so + // report that instead of asserting a scorer contract that may never + // have been exercised. + match review_failure.and_then(|failure| failure.scorer_error.as_deref()) { + Some(scorer_error) => fail_closed_finding(scorer_error), + None => crate::envelope::operational_failure_finding(&format!("{err:#}")), + } } else { crate::envelope::provider_error_finding(&format!("{err:#}")) }]; @@ -3386,6 +3389,54 @@ mod tests { } } + #[test] + fn non_model_failures_report_their_own_cause() { + let envelope = error_envelope( + &Config::default(), + &anyhow::anyhow!( + "hosted review admission projects 11455328 micro-dollars of provider \ + exposure, exceeding the 1000000 micro-dollar operation cap" + ), + "head", + Some(&pr_meta()), + 1, + ); + let finding = &envelope.findings[0]; + assert_eq!(finding.path, crate::envelope::OPERATIONAL_PATH); + assert_eq!(finding.title, "Review could not be completed"); + assert!( + finding.body.contains("micro-dollar operation cap"), + "the real cause must reach the reader: {}", + finding.body + ); + assert!( + !finding.body.contains("scorer"), + "a failure that never ran the scorer must not blame it: {}", + finding.body + ); + assert!(envelope.gate.failing); + } + + #[test] + fn recorded_scorer_failures_still_report_the_scorer() { + let envelope = error_envelope( + &Config::default(), + &rich_scorer_failure(ReviewFailureKind::InvalidOutput), + "head", + Some(&pr_meta()), + 1, + ); + let finding = &envelope.findings[0]; + assert_eq!(finding.title, "Model output could not be validated"); + assert!( + finding + .body + .contains("scorer output invalid after schema repair"), + "the recorded scorer error must be preserved: {}", + finding.body + ); + } + #[test] fn disabled_gate_keeps_provider_error_envelopes_nonblocking() { let cfg = Config { diff --git a/tests/e2e.rs b/tests/e2e.rs index ccb6388..5e483c4 100644 --- a/tests/e2e.rs +++ b/tests/e2e.rs @@ -2640,10 +2640,13 @@ async fn hosted_cost_rejection_precedes_durable_plan_registration() { "generatorChain": [model], "consensus": 1, "scorerChain": [model], + // Priced well above any admissible profile so the plan is rejected + // on cost. The shipped bounds project inside the admission ceiling, + // so a realistic price here would exercise the happy path instead. "modelPriceBounds": [{ "model": model, - "inputMicrosPerMillionTokens": 1_000_000, - "outputMicrosPerMillionTokens": 1_000_000 + "inputMicrosPerMillionTokens": 100_000_000, + "outputMicrosPerMillionTokens": 100_000_000 }] })) .unwrap(), @@ -2672,7 +2675,7 @@ async fn hosted_cost_rejection_precedes_durable_plan_registration() { .code(2); let stderr = String::from_utf8_lossy(&out.get_output().stderr); assert!(stderr.contains("hosted review admission projects")); - assert!(stderr.contains("operation cap")); + assert!(stderr.contains("admission projection cap")); assert!(out.get_output().stdout.is_empty()); assert!( registration_server