diff --git a/bench/src/scorer-eval.test.ts b/bench/src/scorer-eval.test.ts index e4af467..71cbd04 100644 --- a/bench/src/scorer-eval.test.ts +++ b/bench/src/scorer-eval.test.ts @@ -253,11 +253,11 @@ describe("parseModels", () => { expect(parseModels(" a/model, ,b/model,a/model ", [])).toEqual(["a/model", "b/model"]); }); - test("does not invent an embedded scorer candidate when scoring is disabled", async () => { + test("uses the embedded Luna scorer qualification candidate", async () => { const defaults = await loadEmbeddedScorerDefaults(); - expect(defaults.enabled).toBe(false); - expect(defaults.qualification_candidates).toEqual([]); - expect(parseModels(undefined, defaults.qualification_candidates)).toEqual([]); + expect(defaults.enabled).toBe(true); + expect(defaults.qualification_candidates).toEqual(["openai/gpt-5.6-luna"]); + expect(parseModels(undefined, defaults.qualification_candidates)).toEqual(["openai/gpt-5.6-luna"]); }); }); diff --git a/bench/src/verify-admission.test.ts b/bench/src/verify-admission.test.ts index d0234ff..61cfdef 100644 --- a/bench/src/verify-admission.test.ts +++ b/bench/src/verify-admission.test.ts @@ -44,8 +44,16 @@ async function temporaryDirectory(): Promise { } describe("admission attestation verification", () => { - test("verifies the repository provisional hosted release profile", async () => { + test("accepts a shared model across provisional generator and scorer roles", async () => { const repositoryRoot = join(import.meta.dir, "..", ".."); + const profile = JSON.parse(await readFile(join(repositoryRoot, "provisional-models.json"), "utf8")) as { + generatorChain: string[]; + scorerChain: string[]; + modelPriceBounds: Array<{ model: string }>; + }; + expect(profile.generatorChain).toEqual(["openai/gpt-5.6-luna"]); + expect(profile.scorerChain).toEqual(["openai/gpt-5.6-luna"]); + expect(profile.modelPriceBounds.map((bound) => bound.model)).toEqual(["openai/gpt-5.6-luna"]); expect(await verifyProvisionalRelease( join(repositoryRoot, "qualified-models.json"), join(repositoryRoot, "config.toml"), @@ -53,6 +61,29 @@ describe("admission attestation verification", () => { )).toBe("provisional"); }); + test("rejects duplicate models within either provisional chain", async () => { + const repositoryRoot = join(import.meta.dir, "..", ".."); + const directory = await temporaryDirectory(); + const manifest = join(directory, "qualified-models.json"); + const config = join(directory, "config.toml"); + const profilePath = join(directory, "provisional-models.json"); + await writeFile(manifest, await readFile(join(repositoryRoot, "qualified-models.json"))); + await writeFile(config, await readFile(join(repositoryRoot, "config.toml"))); + const baseline = JSON.parse( + await readFile(join(repositoryRoot, "provisional-models.json"), "utf8"), + ) as { generatorChain: string[]; scorerChain: string[] }; + + for (const [field, message] of [ + ["generatorChain", "generator chain must not repeat models"], + ["scorerChain", "scorer chain must not repeat models"], + ] as const) { + const profile = structuredClone(baseline); + profile[field] = ["openai/gpt-5.6-luna", "openai/gpt-5.6-luna"]; + await writeFile(profilePath, JSON.stringify(profile)); + await expect(verifyProvisionalRelease(manifest, config, profilePath)).rejects.toThrow(message); + } + }); + test("keeps attestation verification active after a profile is formally admitted", async () => { const directory = await temporaryDirectory(); const manifest = join(directory, "qualified-models.json"); @@ -81,7 +112,7 @@ describe("admission attestation verification", () => { )).toBe("verified"); }); - test("rejects a provisional profile that diverges from embedded defaults", async () => { + test("rejects a price-complete provisional profile that diverges from embedded defaults", async () => { const repositoryRoot = join(import.meta.dir, "..", ".."); const directory = await temporaryDirectory(); const manifest = join(directory, "qualified-models.json"); @@ -91,9 +122,22 @@ describe("admission attestation verification", () => { await writeFile(config, await readFile(join(repositoryRoot, "config.toml"))); const altered = JSON.parse( await readFile(join(repositoryRoot, "provisional-models.json"), "utf8"), - ) as { generatorChain: string[]; modelPriceBounds: Array<{ model: string }> }; + ) as { + generatorChain: string[]; + modelPriceBounds: Array<{ + model: string; + inputMicrosPerMillionTokens: number; + outputMicrosPerMillionTokens: number; + }>; + }; altered.generatorChain = ["other/model"]; - altered.modelPriceBounds[0]!.model = "other/model"; + // Keep both role models fully priced so the assertion reaches the exact + // embedded-default comparison instead of failing an earlier bound check. + altered.modelPriceBounds.push({ ...altered.modelPriceBounds[0]!, model: "other/model" }); + expect(altered.modelPriceBounds.map((bound) => bound.model)).toEqual([ + "openai/gpt-5.6-luna", + "other/model", + ]); await writeFile(profile, JSON.stringify(altered)); await expect(verifyProvisionalRelease(manifest, config, profile)).rejects.toThrow( diff --git a/bench/src/verify-admission.ts b/bench/src/verify-admission.ts index e6d05f9..3f17034 100644 --- a/bench/src/verify-admission.ts +++ b/bench/src/verify-admission.ts @@ -218,11 +218,14 @@ export async function verifyProvisionalRelease( const profile = provisionalProfileSchema.parse( JSON.parse(await readFile(provisionalProfilePath, "utf8")), ); - const expectedModels = [...profile.generatorChain, ...profile.scorerChain].sort(); - const boundedModels = profile.modelPriceBounds.map((bound) => bound.model); - if (new Set(expectedModels).size !== expectedModels.length) { - throw new Error("provisional hosted model chains must be unique"); + if (new Set(profile.generatorChain).size !== profile.generatorChain.length) { + throw new Error("provisional hosted generator chain must not repeat models"); + } + if (new Set(profile.scorerChain).size !== profile.scorerChain.length) { + throw new Error("provisional hosted scorer chain must not repeat models"); } + const expectedModels = [...new Set([...profile.generatorChain, ...profile.scorerChain])].sort(); + const boundedModels = profile.modelPriceBounds.map((bound) => bound.model); if ( boundedModels.length !== expectedModels.length || boundedModels.some((model, index) => model !== expectedModels[index]) diff --git a/config.toml b/config.toml index 67987fd..6b8566b 100644 --- a/config.toml +++ b/config.toml @@ -6,7 +6,7 @@ api_base = "https://openrouter.ai/api/v1" api_format = "openai-compatible" [scorer] -enabled = false -default_model = "" +enabled = true +default_model = "openai/gpt-5.6-luna" fallback = "" -qualification_candidates = [] +qualification_candidates = ["openai/gpt-5.6-luna"] diff --git a/provisional-models.json b/provisional-models.json index 1e0a1c5..d7e248c 100644 --- a/provisional-models.json +++ b/provisional-models.json @@ -5,7 +5,7 @@ "apiFormat": "openai-compatible", "generatorChain": ["openai/gpt-5.6-luna"], "consensus": 1, - "scorerChain": [], + "scorerChain": ["openai/gpt-5.6-luna"], "modelPriceBounds": [ { "model": "openai/gpt-5.6-luna", diff --git a/qualified-models.json b/qualified-models.json index 2da149a..f0b8b50 100644 --- a/qualified-models.json +++ b/qualified-models.json @@ -4,6 +4,6 @@ "qualificationIssuedAtUnixSeconds": null, "qualificationExpiresAtUnixSeconds": null, "qualificationMaxAgeDays": null, - "modelDefaultsSha256": "29d0b4d04b6ea7dda4dc3c9ac1fbd815765cda6f4a44afe89e9045068e55bcf7", + "modelDefaultsSha256": "2a2fc3a8732b833ae25e60e76abe6bdafeb3df9c9f9e283c19f2a94dac5c2df5", "profiles": [] } diff --git a/src/config.rs b/src/config.rs index 9c8d31f..e6604ba 100644 --- a/src/config.rs +++ b/src/config.rs @@ -470,10 +470,25 @@ fn validate_profile_model_price_bounds( for model in scorer_chain { validate_model_id(&format!("{label} scorer chain"), model)?; } - let mut expected_models = generator_chain.to_vec(); - expected_models.extend_from_slice(scorer_chain); - expected_models.sort(); - expected_models.dedup(); + let unique_generators = generator_chain + .iter() + .collect::>(); + anyhow::ensure!( + unique_generators.len() == generator_chain.len(), + "{label} generator chain must not repeat models" + ); + let unique_scorers = scorer_chain + .iter() + .collect::>(); + anyhow::ensure!( + unique_scorers.len() == scorer_chain.len(), + "{label} scorer chain must not repeat models" + ); + let expected_models = generator_chain + .iter() + .chain(scorer_chain) + .cloned() + .collect::>(); let mut previous_model: Option<&str> = None; for bound in model_price_bounds { validate_model_id(&format!("{label} model price bound"), &bound.model)?; @@ -483,7 +498,7 @@ fn validate_profile_model_price_bounds( ); previous_model = Some(&bound.model); anyhow::ensure!( - expected_models.binary_search(&bound.model).is_ok(), + expected_models.contains(&bound.model), "{label} model price bound references an unknown model" ); anyhow::ensure!( @@ -498,6 +513,7 @@ fn validate_profile_model_price_bounds( .iter() .map(|bound| bound.model.clone()) .collect::>(); + let expected_models = expected_models.into_iter().collect::>(); anyhow::ensure!( bounded_models == expected_models, "{label} model price bounds must exactly cover the generator and scorer models" @@ -3238,9 +3254,13 @@ scorer = { enabled = true, default_model = "provider/scorer", fallback = "provid let defaults = model_defaults(); assert_eq!(defaults.default_model, "openai/gpt-5.6-luna"); assert!(defaults.cascade.is_empty()); - assert!(defaults.scorer_model.is_empty()); + assert!(defaults.scorer_enabled); + assert_eq!(defaults.scorer_model, "openai/gpt-5.6-luna"); assert!(defaults.scorer_fallback.is_empty()); - assert!(defaults.scorer_qualification_candidates.is_empty()); + assert_eq!( + defaults.scorer_qualification_candidates, + vec!["openai/gpt-5.6-luna"] + ); } #[test] @@ -3249,9 +3269,9 @@ scorer = { enabled = true, default_model = "provider/scorer", fallback = "provid let config = Config { model: profile.generator_chain[0].clone(), cascade: profile.generator_chain[1..].to_vec(), - scorer: String::new(), + scorer: profile.scorer_chain[0].clone(), scorer_fallback: String::new(), - scorer_enabled: false, + scorer_enabled: true, api_base: profile.api_base.clone(), api_format: profile.api_format, consensus: profile.consensus, @@ -3265,7 +3285,7 @@ scorer = { enabled = true, default_model = "provider/scorer", fallback = "provid ); assert_eq!(profile.upstream_provider_identity, "Azure"); assert_eq!(profile.generator_chain, vec!["openai/gpt-5.6-luna"]); - assert!(profile.scorer_chain.is_empty()); + assert_eq!(profile.scorer_chain, vec!["openai/gpt-5.6-luna"]); assert_eq!( profile.model_price_bounds, vec![ModelPriceBound { @@ -3298,6 +3318,64 @@ scorer = { enabled = true, default_model = "provider/scorer", fallback = "provid } } + #[test] + fn provisional_hosted_profile_allows_one_model_in_generator_and_scorer_roles() { + let profile = parse_provisional_hosted_profile( + r#"{ + "benchmarkProviderIdentity": "openrouter:managed-routing", + "upstreamProviderIdentity": "Azure", + "apiBase": "https://openrouter.ai:443/api/v1", + "apiFormat": "openai-compatible", + "generatorChain": ["provider/shared"], + "consensus": 1, + "scorerChain": ["provider/shared"], + "modelPriceBounds": [{ + "model": "provider/shared", + "inputMicrosPerMillionTokens": 1, + "outputMicrosPerMillionTokens": 1 + }] + }"#, + ) + .unwrap(); + assert_eq!(profile.generator_chain, vec!["provider/shared"]); + assert_eq!(profile.scorer_chain, vec!["provider/shared"]); + assert_eq!(profile.model_price_bounds.len(), 1); + } + + #[test] + fn provisional_hosted_profile_rejects_duplicate_models_within_a_chain() { + let valid = serde_json::json!({ + "benchmarkProviderIdentity": "openrouter:managed-routing", + "upstreamProviderIdentity": "Azure", + "apiBase": "https://openrouter.ai:443/api/v1", + "apiFormat": "openai-compatible", + "generatorChain": ["provider/generator"], + "consensus": 1, + "scorerChain": ["provider/scorer"], + "modelPriceBounds": [ + {"model": "provider/generator", "inputMicrosPerMillionTokens": 1, "outputMicrosPerMillionTokens": 1}, + {"model": "provider/scorer", "inputMicrosPerMillionTokens": 1, "outputMicrosPerMillionTokens": 1} + ] + }); + for (field, expected) in [ + ("generatorChain", "generator chain must not repeat models"), + ("scorerChain", "scorer chain must not repeat models"), + ] { + let mut duplicate = valid.clone(); + duplicate[field] = if field == "generatorChain" { + serde_json::json!(["provider/generator", "provider/generator"]) + } else { + serde_json::json!(["provider/scorer", "provider/scorer"]) + }; + assert!( + parse_provisional_hosted_profile(&duplicate.to_string()) + .unwrap_err() + .to_string() + .contains(expected) + ); + } + } + #[test] fn provisional_hosted_roster_cannot_override_formal_admission() { assert!(provisional_hosted_roster_allowed(true, true)); diff --git a/src/llm.rs b/src/llm.rs index f6be0f0..19ff91e 100644 --- a/src/llm.rs +++ b/src/llm.rs @@ -8210,9 +8210,17 @@ mod tests { /// production review shows up as a test failure rather than an outage. #[test] fn hosted_review_plan_is_admitted_at_shipped_price_bounds() { + let profile: crate::config::QualificationCandidateProfile = + serde_json::from_str(include_str!("../provisional-models.json")).unwrap(); let config = Config { - model: "z-ai/glm-5.2".into(), - scorer_enabled: false, + model: profile.generator_chain[0].clone(), + cascade: profile.generator_chain[1..].to_vec(), + consensus: profile.consensus, + scorer_enabled: !profile.scorer_chain.is_empty(), + scorer: profile.scorer_chain.first().cloned().unwrap_or_default(), + scorer_fallback: profile.scorer_chain.get(1).cloned().unwrap_or_default(), + api_base: profile.api_base, + api_format: profile.api_format, ..Config::default() }; let mut client = LlmClient::build( @@ -8223,14 +8231,13 @@ mod tests { None, ) .unwrap(); - client.request_decorations.hosted_price_bounds = Some(Arc::new(HashMap::from([( - "z-ai/glm-5.2".into(), - ModelPriceBound { - model: "z-ai/glm-5.2".into(), - input_micros_per_million_tokens: 1_400_000, - output_micros_per_million_tokens: 4_400_000, - }, - )]))); + client.request_decorations.hosted_price_bounds = Some(Arc::new( + profile + .model_price_bounds + .into_iter() + .map(|bound| (bound.model.clone(), bound)) + .collect(), + )); let users = vec!["bounded candidate".to_string(); crate::review::MAX_HOSTED_SELECTED_BATCHES]; let output_tokens = vec![REVIEW_MAX_TOKENS; crate::review::MAX_HOSTED_SELECTED_BATCHES]; @@ -9857,8 +9864,8 @@ mod tests { model: "openai/gpt-5.6-luna".into(), cascade: Vec::new(), consensus: 1, - scorer_enabled: false, - scorer: String::new(), + scorer_enabled: true, + scorer: "openai/gpt-5.6-luna".into(), scorer_fallback: String::new(), api_base: "https://openrouter.ai:443/api/v1".into(), api_format: ApiFormat::OpenaiCompatible, diff --git a/tests/e2e.rs b/tests/e2e.rs index 1c2e108..c3b9d9c 100644 --- a/tests/e2e.rs +++ b/tests/e2e.rs @@ -5749,6 +5749,7 @@ fn hosted_config_ignores_repository_model_provider_and_scorer() { assert!(stdout.contains("model.name: ")); assert!(stdout.contains("model.cascade: []")); assert!(stdout.contains("model.scorer: ")); + assert!(!stdout.contains("model.scorer: openai/gpt-5.6-luna")); assert!(stdout.contains("model.apiBase: https://openrouter.ai/api/v1")); assert!(stdout.contains("model.apiFormat: openai-compatible")); assert!(stdout.contains("model.consensus: 1")); @@ -5780,7 +5781,7 @@ fn provisional_hosted_config_uses_only_the_baked_roster() { assert!(stdout.contains("model.name: openai/gpt-5.6-luna")); assert!(stdout.contains("model.cascade: []")); - assert!(stdout.contains("model.scorer: ")); + assert!(stdout.contains("model.scorer: openai/gpt-5.6-luna")); assert!(stdout.contains("model.apiBase: https://openrouter.ai:443/api/v1")); assert!(stdout.contains("model.apiFormat: openai-compatible")); assert!(stdout.contains("model.consensus: 1"));