Prepare Luna scoring for hosted reviews

bench/src/scorer-eval.test.ts

@@ -253,11 +253,11 @@ describe("parseModels", () => {
     expect(parseModels(" a/model, ,b/model,a/model ", [])).toEqual(["a/model", "b/model"]);
   });
 
-  test("does not invent an embedded scorer candidate when scoring is disabled", async () => {
+  test("uses the embedded Luna scorer qualification candidate", async () => {
     const defaults = await loadEmbeddedScorerDefaults();
-    expect(defaults.enabled).toBe(false);
-    expect(defaults.qualification_candidates).toEqual([]);
-    expect(parseModels(undefined, defaults.qualification_candidates)).toEqual([]);
+    expect(defaults.enabled).toBe(true);
+    expect(defaults.qualification_candidates).toEqual(["openai/gpt-5.6-luna"]);
+    expect(parseModels(undefined, defaults.qualification_candidates)).toEqual(["openai/gpt-5.6-luna"]);
   });
 });
 

bench/src/verify-admission.test.ts

@@ -44,15 +44,46 @@ async function temporaryDirectory(): Promise<string> {
 }
 
 describe("admission attestation verification", () => {
-  test("verifies the repository provisional hosted release profile", async () => {
+  test("accepts a shared model across provisional generator and scorer roles", async () => {
     const repositoryRoot = join(import.meta.dir, "..", "..");
+    const profile = JSON.parse(await readFile(join(repositoryRoot, "provisional-models.json"), "utf8")) as {
+      generatorChain: string[];
+      scorerChain: string[];
+      modelPriceBounds: Array<{ model: string }>;
+    };
+    expect(profile.generatorChain).toEqual(["openai/gpt-5.6-luna"]);
+    expect(profile.scorerChain).toEqual(["openai/gpt-5.6-luna"]);
+    expect(profile.modelPriceBounds.map((bound) => bound.model)).toEqual(["openai/gpt-5.6-luna"]);
     expect(await verifyProvisionalRelease(
       join(repositoryRoot, "qualified-models.json"),
       join(repositoryRoot, "config.toml"),
       join(repositoryRoot, "provisional-models.json"),
     )).toBe("provisional");
   });
 
+  test("rejects duplicate models within either provisional chain", async () => {
+    const repositoryRoot = join(import.meta.dir, "..", "..");
+    const directory = await temporaryDirectory();
+    const manifest = join(directory, "qualified-models.json");
+    const config = join(directory, "config.toml");
+    const profilePath = join(directory, "provisional-models.json");
+    await writeFile(manifest, await readFile(join(repositoryRoot, "qualified-models.json")));
+    await writeFile(config, await readFile(join(repositoryRoot, "config.toml")));
+    const baseline = JSON.parse(
+      await readFile(join(repositoryRoot, "provisional-models.json"), "utf8"),
+    ) as { generatorChain: string[]; scorerChain: string[] };
+
+    for (const [field, message] of [
+      ["generatorChain", "generator chain must not repeat models"],
+      ["scorerChain", "scorer chain must not repeat models"],
+    ] as const) {
+      const profile = structuredClone(baseline);
+      profile[field] = ["openai/gpt-5.6-luna", "openai/gpt-5.6-luna"];
+      await writeFile(profilePath, JSON.stringify(profile));
+      await expect(verifyProvisionalRelease(manifest, config, profilePath)).rejects.toThrow(message);
+    }
+  });
+
   test("keeps attestation verification active after a profile is formally admitted", async () => {
     const directory = await temporaryDirectory();
     const manifest = join(directory, "qualified-models.json");
@@ -81,7 +112,7 @@ describe("admission attestation verification", () => {
     )).toBe("verified");
   });
 
-  test("rejects a provisional profile that diverges from embedded defaults", async () => {
+  test("rejects a price-complete provisional profile that diverges from embedded defaults", async () => {
     const repositoryRoot = join(import.meta.dir, "..", "..");
     const directory = await temporaryDirectory();
     const manifest = join(directory, "qualified-models.json");
@@ -91,9 +122,22 @@ describe("admission attestation verification", () => {
     await writeFile(config, await readFile(join(repositoryRoot, "config.toml")));
     const altered = JSON.parse(
       await readFile(join(repositoryRoot, "provisional-models.json"), "utf8"),
-    ) as { generatorChain: string[]; modelPriceBounds: Array<{ model: string }> };
+    ) as {
+      generatorChain: string[];
+      modelPriceBounds: Array<{
+        model: string;
+        inputMicrosPerMillionTokens: number;
+        outputMicrosPerMillionTokens: number;
+      }>;
+    };
     altered.generatorChain = ["other/model"];
-    altered.modelPriceBounds[0]!.model = "other/model";
+    // Keep both role models fully priced so the assertion reaches the exact
+    // embedded-default comparison instead of failing an earlier bound check.
+    altered.modelPriceBounds.push({ ...altered.modelPriceBounds[0]!, model: "other/model" });
+    expect(altered.modelPriceBounds.map((bound) => bound.model)).toEqual([
+      "openai/gpt-5.6-luna",
+      "other/model",
+    ]);
     await writeFile(profile, JSON.stringify(altered));
 
     await expect(verifyProvisionalRelease(manifest, config, profile)).rejects.toThrow(

bench/src/verify-admission.ts

@@ -218,11 +218,14 @@ export async function verifyProvisionalRelease(
   const profile = provisionalProfileSchema.parse(
     JSON.parse(await readFile(provisionalProfilePath, "utf8")),
   );
-  const expectedModels = [...profile.generatorChain, ...profile.scorerChain].sort();
-  const boundedModels = profile.modelPriceBounds.map((bound) => bound.model);
-  if (new Set(expectedModels).size !== expectedModels.length) {
-    throw new Error("provisional hosted model chains must be unique");
+  if (new Set(profile.generatorChain).size !== profile.generatorChain.length) {
+    throw new Error("provisional hosted generator chain must not repeat models");
+  }
+  if (new Set(profile.scorerChain).size !== profile.scorerChain.length) {
+    throw new Error("provisional hosted scorer chain must not repeat models");
   }
+  const expectedModels = [...new Set([...profile.generatorChain, ...profile.scorerChain])].sort();
+  const boundedModels = profile.modelPriceBounds.map((bound) => bound.model);
   if (
     boundedModels.length !== expectedModels.length ||
     boundedModels.some((model, index) => model !== expectedModels[index])

config.toml

@@ -6,7 +6,7 @@ api_base = "https://openrouter.ai/api/v1"
 api_format = "openai-compatible"
 
 [scorer]
-enabled = false
-default_model = ""
+enabled = true
+default_model = "openai/gpt-5.6-luna"
 fallback = ""
-qualification_candidates = []
+qualification_candidates = ["openai/gpt-5.6-luna"]

provisional-models.json

@@ -5,7 +5,7 @@
   "apiFormat": "openai-compatible",
   "generatorChain": ["openai/gpt-5.6-luna"],
   "consensus": 1,
-  "scorerChain": [],
+  "scorerChain": ["openai/gpt-5.6-luna"],
   "modelPriceBounds": [
     {
       "model": "openai/gpt-5.6-luna",

qualified-models.json

@@ -4,6 +4,6 @@
   "qualificationIssuedAtUnixSeconds": null,
   "qualificationExpiresAtUnixSeconds": null,
   "qualificationMaxAgeDays": null,
-  "modelDefaultsSha256": "29d0b4d04b6ea7dda4dc3c9ac1fbd815765cda6f4a44afe89e9045068e55bcf7",
+  "modelDefaultsSha256": "2a2fc3a8732b833ae25e60e76abe6bdafeb3df9c9f9e283c19f2a94dac5c2df5",
   "profiles": []
 }

src/config.rs

@@ -470,10 +470,25 @@ fn validate_profile_model_price_bounds(
     for model in scorer_chain {
         validate_model_id(&format!("{label} scorer chain"), model)?;
     }
-    let mut expected_models = generator_chain.to_vec();
-    expected_models.extend_from_slice(scorer_chain);
-    expected_models.sort();
-    expected_models.dedup();
+    let unique_generators = generator_chain
+        .iter()
+        .collect::<std::collections::HashSet<_>>();
+    anyhow::ensure!(
+        unique_generators.len() == generator_chain.len(),
+        "{label} generator chain must not repeat models"
+    );
+    let unique_scorers = scorer_chain
+        .iter()
+        .collect::<std::collections::HashSet<_>>();
+    anyhow::ensure!(
+        unique_scorers.len() == scorer_chain.len(),
+        "{label} scorer chain must not repeat models"
+    );
+    let expected_models = generator_chain
+        .iter()
+        .chain(scorer_chain)
+        .cloned()
+        .collect::<std::collections::BTreeSet<_>>();
     let mut previous_model: Option<&str> = None;
     for bound in model_price_bounds {
         validate_model_id(&format!("{label} model price bound"), &bound.model)?;
@@ -483,7 +498,7 @@ fn validate_profile_model_price_bounds(
         );
         previous_model = Some(&bound.model);
         anyhow::ensure!(
-            expected_models.binary_search(&bound.model).is_ok(),
+            expected_models.contains(&bound.model),
             "{label} model price bound references an unknown model"
         );
         anyhow::ensure!(
@@ -498,6 +513,7 @@ fn validate_profile_model_price_bounds(
         .iter()
         .map(|bound| bound.model.clone())
         .collect::<Vec<_>>();
+    let expected_models = expected_models.into_iter().collect::<Vec<_>>();
     anyhow::ensure!(
         bounded_models == expected_models,
         "{label} model price bounds must exactly cover the generator and scorer models"
@@ -3238,9 +3254,13 @@ scorer = { enabled = true, default_model = "provider/scorer", fallback = "provid
         let defaults = model_defaults();
         assert_eq!(defaults.default_model, "openai/gpt-5.6-luna");
         assert!(defaults.cascade.is_empty());
-        assert!(defaults.scorer_model.is_empty());
+        assert!(defaults.scorer_enabled);
+        assert_eq!(defaults.scorer_model, "openai/gpt-5.6-luna");
         assert!(defaults.scorer_fallback.is_empty());
-        assert!(defaults.scorer_qualification_candidates.is_empty());
+        assert_eq!(
+            defaults.scorer_qualification_candidates,
+            vec!["openai/gpt-5.6-luna"]
+        );
     }
 
     #[test]
@@ -3249,9 +3269,9 @@ scorer = { enabled = true, default_model = "provider/scorer", fallback = "provid
         let config = Config {
             model: profile.generator_chain[0].clone(),
             cascade: profile.generator_chain[1..].to_vec(),
-            scorer: String::new(),
+            scorer: profile.scorer_chain[0].clone(),
             scorer_fallback: String::new(),
-            scorer_enabled: false,
+            scorer_enabled: true,
             api_base: profile.api_base.clone(),
             api_format: profile.api_format,
             consensus: profile.consensus,
@@ -3265,7 +3285,7 @@ scorer = { enabled = true, default_model = "provider/scorer", fallback = "provid
         );
         assert_eq!(profile.upstream_provider_identity, "Azure");
         assert_eq!(profile.generator_chain, vec!["openai/gpt-5.6-luna"]);
-        assert!(profile.scorer_chain.is_empty());
+        assert_eq!(profile.scorer_chain, vec!["openai/gpt-5.6-luna"]);
         assert_eq!(
             profile.model_price_bounds,
             vec![ModelPriceBound {
@@ -3298,6 +3318,64 @@ scorer = { enabled = true, default_model = "provider/scorer", fallback = "provid
         }
     }
 
+    #[test]
+    fn provisional_hosted_profile_allows_one_model_in_generator_and_scorer_roles() {
+        let profile = parse_provisional_hosted_profile(
+            r#"{
+                "benchmarkProviderIdentity": "openrouter:managed-routing",
+                "upstreamProviderIdentity": "Azure",
+                "apiBase": "https://openrouter.ai:443/api/v1",
+                "apiFormat": "openai-compatible",
+                "generatorChain": ["provider/shared"],
+                "consensus": 1,
+                "scorerChain": ["provider/shared"],
+                "modelPriceBounds": [{
+                    "model": "provider/shared",
+                    "inputMicrosPerMillionTokens": 1,
+                    "outputMicrosPerMillionTokens": 1
+                }]
+            }"#,
+        )
+        .unwrap();
+        assert_eq!(profile.generator_chain, vec!["provider/shared"]);
+        assert_eq!(profile.scorer_chain, vec!["provider/shared"]);
+        assert_eq!(profile.model_price_bounds.len(), 1);
+    }
+
+    #[test]
+    fn provisional_hosted_profile_rejects_duplicate_models_within_a_chain() {
+        let valid = serde_json::json!({
+            "benchmarkProviderIdentity": "openrouter:managed-routing",
+            "upstreamProviderIdentity": "Azure",
+            "apiBase": "https://openrouter.ai:443/api/v1",
+            "apiFormat": "openai-compatible",
+            "generatorChain": ["provider/generator"],
+            "consensus": 1,
+            "scorerChain": ["provider/scorer"],
+            "modelPriceBounds": [
+                {"model": "provider/generator", "inputMicrosPerMillionTokens": 1, "outputMicrosPerMillionTokens": 1},
+                {"model": "provider/scorer", "inputMicrosPerMillionTokens": 1, "outputMicrosPerMillionTokens": 1}
+            ]
+        });
+        for (field, expected) in [
+            ("generatorChain", "generator chain must not repeat models"),
+            ("scorerChain", "scorer chain must not repeat models"),
+        ] {
+            let mut duplicate = valid.clone();
+            duplicate[field] = if field == "generatorChain" {
+                serde_json::json!(["provider/generator", "provider/generator"])
+            } else {
+                serde_json::json!(["provider/scorer", "provider/scorer"])
+            };
+            assert!(
+                parse_provisional_hosted_profile(&duplicate.to_string())
+                    .unwrap_err()
+                    .to_string()
+                    .contains(expected)
+            );
+        }
+    }
+
     #[test]
     fn provisional_hosted_roster_cannot_override_formal_admission() {
         assert!(provisional_hosted_roster_allowed(true, true));

src/llm.rs

@@ -8210,9 +8210,17 @@ mod tests {
     /// production review shows up as a test failure rather than an outage.
     #[test]
     fn hosted_review_plan_is_admitted_at_shipped_price_bounds() {
+        let profile: crate::config::QualificationCandidateProfile =
+            serde_json::from_str(include_str!("../provisional-models.json")).unwrap();
         let config = Config {
-            model: "z-ai/glm-5.2".into(),
-            scorer_enabled: false,
+            model: profile.generator_chain[0].clone(),
+            cascade: profile.generator_chain[1..].to_vec(),
+            consensus: profile.consensus,
+            scorer_enabled: !profile.scorer_chain.is_empty(),
+            scorer: profile.scorer_chain.first().cloned().unwrap_or_default(),
+            scorer_fallback: profile.scorer_chain.get(1).cloned().unwrap_or_default(),
+            api_base: profile.api_base,
+            api_format: profile.api_format,
             ..Config::default()
         };
         let mut client = LlmClient::build(
@@ -8223,14 +8231,13 @@ mod tests {
             None,
         )
         .unwrap();
-        client.request_decorations.hosted_price_bounds = Some(Arc::new(HashMap::from([(
-            "z-ai/glm-5.2".into(),
-            ModelPriceBound {
-                model: "z-ai/glm-5.2".into(),
-                input_micros_per_million_tokens: 1_400_000,
-                output_micros_per_million_tokens: 4_400_000,
-            },
-        )])));
+        client.request_decorations.hosted_price_bounds = Some(Arc::new(
+            profile
+                .model_price_bounds
+                .into_iter()
+                .map(|bound| (bound.model.clone(), bound))
+                .collect(),
+        ));
         let users =
             vec!["bounded candidate".to_string(); crate::review::MAX_HOSTED_SELECTED_BATCHES];
         let output_tokens = vec![REVIEW_MAX_TOKENS; crate::review::MAX_HOSTED_SELECTED_BATCHES];
@@ -9857,8 +9864,8 @@ mod tests {
             model: "openai/gpt-5.6-luna".into(),
             cascade: Vec::new(),
             consensus: 1,
-            scorer_enabled: false,
-            scorer: String::new(),
+            scorer_enabled: true,
+            scorer: "openai/gpt-5.6-luna".into(),
             scorer_fallback: String::new(),
             api_base: "https://openrouter.ai:443/api/v1".into(),
             api_format: ApiFormat::OpenaiCompatible,

tests/e2e.rs

@@ -5749,6 +5749,7 @@ fn hosted_config_ignores_repository_model_provider_and_scorer() {
     assert!(stdout.contains("model.name: "));
     assert!(stdout.contains("model.cascade: []"));
     assert!(stdout.contains("model.scorer: "));
+    assert!(!stdout.contains("model.scorer: openai/gpt-5.6-luna"));
     assert!(stdout.contains("model.apiBase: https://openrouter.ai/api/v1"));
     assert!(stdout.contains("model.apiFormat: openai-compatible"));
     assert!(stdout.contains("model.consensus: 1"));
@@ -5780,7 +5781,7 @@ fn provisional_hosted_config_uses_only_the_baked_roster() {
 
     assert!(stdout.contains("model.name: openai/gpt-5.6-luna"));
     assert!(stdout.contains("model.cascade: []"));
-    assert!(stdout.contains("model.scorer: "));
+    assert!(stdout.contains("model.scorer: openai/gpt-5.6-luna"));
     assert!(stdout.contains("model.apiBase: https://openrouter.ai:443/api/v1"));
     assert!(stdout.contains("model.apiFormat: openai-compatible"));
     assert!(stdout.contains("model.consensus: 1"));