From 31640895b93724e77afb929d96249da37274920f Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Tue, 25 Aug 2026 07:04:31 +0000 Subject: [PATCH] Keep hosted scorer routing compatible --- .github/workflows/release.yml | 19 +++++++++++++++ README.md | 4 ++-- bench/src/livemodels.test.ts | 10 ++++++++ bench/src/verify-admission.test.ts | 7 +++++- config.toml | 2 +- docs/configuration.md | 4 ++-- docs/model-providers.md | 4 ++-- qualified-models.json | 2 +- src/cli.rs | 2 +- src/config.rs | 5 ++-- src/llm.rs | 37 +++++++++++++++++++++++++----- tests/e2e.rs | 12 +++++----- 12 files changed, 84 insertions(+), 24 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b33b24e..294ae51 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -113,6 +113,25 @@ jobs: working-directory: bench run: bun install --frozen-lockfile + - name: Run the embedded hosted scorer gate + working-directory: bench + env: + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + POSTIL_SCORER_EVAL_REPEATS: "3" + POSTIL_SCORER_EVAL_UPSTREAM_PROVIDER: Azure + run: >- + bun run scorer-eval --json-out + "${{ runner.temp }}/scorer-eval-report.json" + + - name: Upload the scorer gate report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: scorer-eval-report + path: ${{ runner.temp }}/scorer-eval-report.json + if-no-files-found: warn + retention-days: 30 + - name: Run diff-file live benchmark working-directory: bench env: diff --git a/README.md b/README.md index ceac62b..be51536 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ postil hook install # add a pre-push review Bare local review selects staged changes first, then committed changes since the current branch's locally known default branch, then tracked working-tree changes, and finally an empty clean diff. Symbolic remote HEAD plus conventional `main`, `master`, and `trunk` refs are recognized without fetching. If no default branch can be resolved for a repository with commits, Postil fails closed and asks for `--base`, `--staged`, or `--diff-file` instead of reporting a partial review as clean. It sends the selected diff to the configured inference endpoint, but does not modify the working tree, index, or refs and does not write comments or checks to a forge unless `--publish` is supplied. -`postil models` answers model-selection questions offline without reading credentials or contacting a provider. No model setting is required: it identifies the embedded default, explains the model-ID contracts for OpenAI-compatible and native Anthropic endpoints, separates local protocol compatibility from hosted qualification, and gives exact `postil doctor` and override commands. The Luna reviewer uses `low` reasoning effort. Local scoring stays disabled until `REVIEW_SCORER_MODEL` names a scorer; the hosted scorer candidate uses `none`. `postil review` exits `0` when the gate passes, `1` when it fails, and `2` when it cannot produce a review envelope. Interactive terminals show concise progress; `--no-progress` or `POSTIL_NO_PROGRESS=1` replaces animation with concise static milestones, while `--verbose` enables complete provider, retry, and batch telemetry. +`postil models` answers model-selection questions offline without reading credentials or contacting a provider. No model setting is required: it identifies the embedded default, explains the model-ID contracts for OpenAI-compatible and native Anthropic endpoints, separates local protocol compatibility from hosted qualification, and gives exact `postil doctor` and override commands. The Luna reviewer and hosted scorer candidate use `low` reasoning effort. Local scoring stays disabled until `REVIEW_SCORER_MODEL` names a scorer. `postil review` exits `0` when the gate passes, `1` when it fails, and `2` when it cannot produce a review envelope. Interactive terminals show concise progress; `--no-progress` or `POSTIL_NO_PROGRESS=1` replaces animation with concise static milestones, while `--verbose` enables complete provider, retry, and batch telemetry. ## Review a pull request @@ -66,7 +66,7 @@ gate: onError: block model: reasoningEffort: low - scorerReasoningEffort: none + scorerReasoningEffort: low ``` Unknown keys are rejected. Repository configuration cannot redirect a deployment credential to another API host unless the operator explicitly permits that behavior. The embedded local preset uses `openai/gpt-5.6-luna`; local scoring is disabled until explicitly configured. Hosted profile selection remains service-controlled. diff --git a/bench/src/livemodels.test.ts b/bench/src/livemodels.test.ts index c7db75f..b22ab86 100644 --- a/bench/src/livemodels.test.ts +++ b/bench/src/livemodels.test.ts @@ -1339,6 +1339,16 @@ describe("managed admission workflow", () => { // still benchmarked the previous one and passed against its old baseline. expect(release).not.toContain("REVIEW_MODEL:"); expect(release).toContain("OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}"); + expect(release).not.toContain("POSTIL_SCORER_EVAL_MODELS:"); + expect(release).toContain('POSTIL_SCORER_EVAL_REPEATS: "3"'); + expect(release).toContain("POSTIL_SCORER_EVAL_UPSTREAM_PROVIDER: Azure"); + expect(release).toContain("bun run scorer-eval --json-out"); + expect(release).toMatch( + /name: Upload the scorer gate report\n\s+if: always\(\)[\s\S]*if-no-files-found: warn[\s\S]*retention-days: 30/u, + ); + expect(release.indexOf("bun run scorer-eval --json-out")).toBeLessThan( + release.indexOf("bun run bench:live --"), + ); expect(release).toContain("bun run bench:live --"); expect(release).toContain("bun run bench:compare --"); expect(release).toMatch(/build:\n\s+needs: \[validate-tag, bench-live\]/u); diff --git a/bench/src/verify-admission.test.ts b/bench/src/verify-admission.test.ts index 5b41b23..00018ff 100644 --- a/bench/src/verify-admission.test.ts +++ b/bench/src/verify-admission.test.ts @@ -64,9 +64,14 @@ describe("admission attestation verification", () => { test("binds reviewer and scorer reasoning efforts to the model-defaults digest", async () => { const repositoryRoot = join(import.meta.dir, "..", ".."); const original = await readFile(join(repositoryRoot, "config.toml"), "utf8"); + const alteredScorer = original.replace( + /(\[scorer\][\s\S]*?reasoning_effort = )"low"/u, + '$1"none"', + ); + expect(alteredScorer).not.toBe(original); for (const altered of [ original.replace('reasoning_effort = "low"', 'reasoning_effort = "high"'), - original.replace('reasoning_effort = "none"', 'reasoning_effort = "low"'), + alteredScorer, ]) { const directory = await temporaryDirectory(); const manifest = join(directory, "qualified-models.json"); diff --git a/config.toml b/config.toml index 1d70b02..2e26e05 100644 --- a/config.toml +++ b/config.toml @@ -9,6 +9,6 @@ api_format = "openai-compatible" [scorer] enabled = true default_model = "openai/gpt-5.6-luna" -reasoning_effort = "none" +reasoning_effort = "low" fallback = "" qualification_candidates = ["openai/gpt-5.6-luna"] diff --git a/docs/configuration.md b/docs/configuration.md index 3163fad..080d7aa 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -38,7 +38,7 @@ model: reasoningEffort: low # max, xhigh, high, medium, low, minimal, or none cascade: [] # qualified fallbacks only # scorer: provider/qualified-scorer - scorerReasoningEffort: none + scorerReasoningEffort: low consensus: 1 ``` @@ -97,4 +97,4 @@ postil doctor `postil config` prints the resolved non-secret configuration and separate provenance for the model, reviewer reasoning effort, and scorer reasoning effort. `postil doctor` validates endpoint reachability, credential acceptance, and repository setup without printing credential values. Both commands identify renewable logins, access expiry, refresh inactivity expiry, and legacy access-only logins. -Use `--reasoning-effort` and `--scorer-reasoning-effort` for one review. These flags override the matching environment variables, which override `model.reasoningEffort` and `model.scorerReasoningEffort`. The built-in reviewer and scorer defaults are `low` and `none`, respectively. Every request carries the resolved value, including retries and repair calls. +Use `--reasoning-effort` and `--scorer-reasoning-effort` for one review. These flags override the matching environment variables, which override `model.reasoningEffort` and `model.scorerReasoningEffort`. The built-in reviewer and scorer defaults are both `low`. Every request carries the resolved value, including retries and repair calls. diff --git a/docs/model-providers.md b/docs/model-providers.md index e023bcd..3506fbd 100644 --- a/docs/model-providers.md +++ b/docs/model-providers.md @@ -4,7 +4,7 @@ Postil speaks either an OpenAI-compatible chat-completions interface or the nati ## Model admission -Local review defaults to the embedded `openai/gpt-5.6-luna` reviewer with no fallback chain or scorer, so no model setting is required. Reviewer requests use `low` reasoning effort. Local scoring is disabled unless `REVIEW_SCORER_MODEL` names a scorer. The embedded hosted scorer candidate is `openai/gpt-5.6-luna` at `none` effort. `postil models` reports these roles, the accepted endpoint model-ID contracts, embedded qualification status, accepted effort values, and exact `postil doctor` and override commands without reading credentials or contacting a provider. Local use has no fixed model-ID allowlist: Postil passes a non-empty model ID unchanged to the configured OpenAI-compatible or native Anthropic endpoint. Endpoint protocol compatibility does not imply hosted qualification. Hosted deployments admit only complete qualified profiles. `REVIEW_SCORER_MODEL_CASCADE` accepts one qualified scorer fallback. +Local review defaults to the embedded `openai/gpt-5.6-luna` reviewer with no fallback chain or scorer, so no model setting is required. Reviewer requests use `low` reasoning effort. Local scoring is disabled unless `REVIEW_SCORER_MODEL` names a scorer. The embedded hosted scorer candidate is `openai/gpt-5.6-luna` at `low` effort. `postil models` reports these roles, the accepted endpoint model-ID contracts, embedded qualification status, accepted effort values, and exact `postil doctor` and override commands without reading credentials or contacting a provider. Local use has no fixed model-ID allowlist: Postil passes a non-empty model ID unchanged to the configured OpenAI-compatible or native Anthropic endpoint. Endpoint protocol compatibility does not imply hosted qualification. Hosted deployments admit only complete qualified profiles. `REVIEW_SCORER_MODEL_CASCADE` accepts one qualified scorer fallback. Hosted deployments admit only complete profiles listed in the embedded `qualified-models.json` qualification manifest. A profile binds the canonical API base, provider API format, ordered generator chain, consensus width, ordered scorer chain, exact Bun evaluator runtime, qualification source commit, a 30-day authority window, at least three complete runs, and SHA-256 digests for the review contract, evaluator contract, fixture set, and benchmark report. The manifest also binds the exact `config.toml` digest, including the reviewer and scorer reasoning defaults. Hosted configuration must match one unexpired profile exactly, so an effort override that differs from those qualified defaults is rejected. The expiry second is excluded. An empty, degraded, stale, or mismatched profile rejects inference. @@ -33,7 +33,7 @@ postil doctor These requests authenticate with `Authorization: Bearer`. -OpenAI-compatible requests carry the [OpenRouter reasoning configuration](https://openrouter.ai/docs/guides/best-practices/reasoning-tokens) as `reasoning.effort`. The accepted values are `max`, `xhigh`, `high`, `medium`, `low`, `minimal`, and `none`. This field is sent to OpenRouter and custom OpenAI-compatible endpoints on every request. Postil does not drop or downgrade it, so custom endpoints must implement the extension. +OpenAI-compatible requests carry the [OpenRouter reasoning configuration](https://openrouter.ai/docs/guides/best-practices/reasoning-tokens) as `reasoning.effort`. The accepted values are `max`, `xhigh`, `high`, `medium`, `low`, `minimal`, and `none`. Scorer requests exclude returned reasoning unless reasoning is disabled. Postil does not drop or downgrade the selected effort, so custom endpoints must implement the extension. ## Anthropic Messages API diff --git a/qualified-models.json b/qualified-models.json index 5602481..e6c6cf7 100644 --- a/qualified-models.json +++ b/qualified-models.json @@ -4,6 +4,6 @@ "qualificationIssuedAtUnixSeconds": null, "qualificationExpiresAtUnixSeconds": null, "qualificationMaxAgeDays": null, - "modelDefaultsSha256": "b581d37af77c48552412c383cebcab55129c1d8c73d3f6dfa00bf3c7ff04ab3f", + "modelDefaultsSha256": "535d68cf62ce90c0fa11451a5411f7f358ba73b989c1559a1374357c1384697f", "profiles": [] } diff --git a/src/cli.rs b/src/cli.rs index 366c55f..b4019d5 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -141,7 +141,7 @@ pub enum Command { /// Reviewer reasoning effort: max|xhigh|high|medium|low|minimal|none (else REVIEW_REASONING_EFFORT, else config, else low). #[arg(long, value_name = "EFFORT")] reasoning_effort: Option, - /// Scorer reasoning effort: max|xhigh|high|medium|low|minimal|none (else REVIEW_SCORER_REASONING_EFFORT, else config, else none). + /// Scorer reasoning effort: max|xhigh|high|medium|low|minimal|none (else REVIEW_SCORER_REASONING_EFFORT, else config, else low). #[arg(long, value_name = "EFFORT")] scorer_reasoning_effort: Option, /// Keep detailed provider, retry, and batch telemetry in interactive terminals. diff --git a/src/config.rs b/src/config.rs index ea40ae5..1570b5e 100644 --- a/src/config.rs +++ b/src/config.rs @@ -2766,7 +2766,7 @@ scorer = { enabled = true, default_model = "provider/scorer", reasoning_effort = assert_eq!(c.model_source, "embedded default"); assert_eq!(c.reasoning_effort, ReasoningEffort::Low); assert_eq!(c.reasoning_effort_source, "embedded default"); - assert_eq!(c.scorer_reasoning_effort, ReasoningEffort::None); + assert_eq!(c.scorer_reasoning_effort, ReasoningEffort::Low); assert_eq!(c.scorer_reasoning_effort_source, "embedded default"); assert!(c.concise_findings); assert!(!c.scorer_enabled); @@ -3058,7 +3058,7 @@ scorer = { enabled = true, default_model = "provider/scorer", reasoning_effort = assert_eq!(c.model, model_defaults().default_model); assert_eq!(c.cascade, model_defaults().cascade); assert_eq!(c.reasoning_effort, ReasoningEffort::Low); - assert_eq!(c.scorer_reasoning_effort, ReasoningEffort::None); + assert_eq!(c.scorer_reasoning_effort, ReasoningEffort::Low); assert!(c.scorer.is_empty()); } @@ -3694,6 +3694,7 @@ scorer = { enabled = true, default_model = "provider/scorer", reasoning_effort = assert!(defaults.cascade.is_empty()); assert!(defaults.scorer_enabled); assert_eq!(defaults.scorer_model, "openai/gpt-5.6-luna"); + assert_eq!(defaults.scorer_reasoning_effort, ReasoningEffort::Low); assert!(defaults.scorer_fallback.is_empty()); assert_eq!( defaults.scorer_qualification_candidates, diff --git a/src/llm.rs b/src/llm.rs index f53a038..4636189 100644 --- a/src/llm.rs +++ b/src/llm.rs @@ -5277,7 +5277,11 @@ fn apply_openrouter_scorer_contract(body: &mut serde_json::Value, expected_len: .as_object_mut() .expect("provider routing configuration is an object") .insert("require_parameters".to_string(), json!(true)); - body["reasoning"]["exclude"] = json!(true); + // A disabled reasoning request has no reasoning payload to exclude. Keep + // that provider contract minimal instead of sending redundant controls. + if body["reasoning"]["effort"] != "none" { + body["reasoning"]["exclude"] = json!(true); + } body["response_format"] = json!({ "type": "json_schema", "json_schema": { @@ -9555,7 +9559,7 @@ mod tests { ); assert_eq!( scorer["reasoning"], - json!({"effort": "none", "exclude": true}) + json!({"effort": "low", "exclude": true}) ); assert!(scorer["reasoning"].get("enabled").is_none()); assert_eq!(scorer["provider"]["require_parameters"], true); @@ -9611,6 +9615,27 @@ mod tests { assert_eq!(generator["reasoning"], json!({"effort": "low"})); assert!(generator.get("response_format").is_none()); assert!(generator["provider"].get("require_parameters").is_none()); + + let no_reasoning_client = LlmClient::build( + &Config { + scorer_reasoning_effort: ReasoningEffort::None, + ..Config::default() + }, + "test-key".into(), + Duration::from_secs(1), + None, + None, + ) + .unwrap(); + let no_reasoning_scorer = no_reasoning_client.request_body( + "provider/scorer", + "system", + "user", + 400, + 0.0, + LlmPhase::Scorer { expected_len: 1 }, + ); + assert_eq!(no_reasoning_scorer["reasoning"], json!({"effort": "none"})); } #[test] @@ -10166,7 +10191,7 @@ mod tests { LlmPhase::Scorer { expected_len: 1 }, ); assert!(openai_body.get("provider").is_none()); - assert_eq!(openai_body["reasoning"], json!({"effort": "none"})); + assert_eq!(openai_body["reasoning"], json!({"effort": "low"})); assert!(openai_body.get("response_format").is_none()); let anthropic = LlmClient::build( @@ -10191,9 +10216,9 @@ mod tests { LlmPhase::Scorer { expected_len: 1 }, ); assert!(anthropic_body.get("provider").is_none()); - assert_eq!(anthropic_body["thinking"], json!({"type": "disabled"})); - assert!(anthropic_body.get("output_config").is_none()); - assert_eq!(anthropic_body["temperature"], 0.0); + assert!(anthropic_body.get("thinking").is_none()); + assert_eq!(anthropic_body["output_config"], json!({"effort": "low"})); + assert!(anthropic_body.get("temperature").is_none()); assert!(anthropic_body.get("response_format").is_none()); assert_eq!(anthropic_body["system"], "system"); } diff --git a/tests/e2e.rs b/tests/e2e.rs index a144e38..47008d4 100644 --- a/tests/e2e.rs +++ b/tests/e2e.rs @@ -2150,7 +2150,7 @@ async fn native_anthropic_findings_use_explicit_native_scorer() { .body_json() .unwrap(); assert_eq!(generator["output_config"], json!({"effort": "low"})); - assert_eq!(scorer["thinking"], json!({"type": "disabled"})); + assert_eq!(scorer["output_config"], json!({"effort": "low"})); } #[tokio::test] @@ -6037,7 +6037,7 @@ fn hosted_config_ignores_repository_model_provider_and_scorer() { assert!(stdout.contains("model.scorer: ")); assert!(!stdout.contains("model.scorer: openai/gpt-5.6-luna")); assert!(stdout.contains("model.reasoningEffort: low")); - assert!(stdout.contains("model.scorerReasoningEffort: none")); + assert!(stdout.contains("model.scorerReasoningEffort: low")); assert!(stdout.contains("model.apiBase: https://openrouter.ai/api/v1")); assert!(stdout.contains("model.apiFormat: openai-compatible")); assert!(stdout.contains("model.consensus: 1")); @@ -6071,7 +6071,7 @@ fn provisional_hosted_config_uses_only_the_baked_roster() { assert!(stdout.contains("model.cascade: []")); assert!(stdout.contains("model.scorer: openai/gpt-5.6-luna")); assert!(stdout.contains("model.reasoningEffort: low")); - assert!(stdout.contains("model.scorerReasoningEffort: none")); + assert!(stdout.contains("model.scorerReasoningEffort: low")); assert!(stdout.contains("model.apiBase: https://openrouter.ai:443/api/v1")); assert!(stdout.contains("model.apiFormat: openai-compatible")); assert!(stdout.contains("model.consensus: 1")); @@ -6129,7 +6129,7 @@ async fn local_review_reports_grounded_finding_and_gates() { let adjudication: Value = requests[1].body_json().unwrap(); assert_eq!(request["max_tokens"], 8_000); assert_eq!(request["reasoning"], json!({"effort": "low"})); - assert_eq!(adjudication["reasoning"], json!({"effort": "none"})); + assert_eq!(adjudication["reasoning"], json!({"effort": "low"})); assert_eq!(request["messages"].as_array().unwrap().len(), 2); let prompt_bytes = request["messages"] .as_array() @@ -6634,7 +6634,7 @@ fn models_and_config_explain_the_embedded_default() { assert!(models.contains("Reviewer reasoning effort: low")); assert!(models.contains("Local scorer: disabled")); assert!(models.contains("Hosted scorer candidate: openai/gpt-5.6-luna")); - assert!(models.contains("Hosted scorer reasoning effort: none")); + assert!(models.contains("Hosted scorer reasoning effort: low")); assert!(models.contains("does not maintain a fixed local model-ID allowlist")); assert!(models.contains("OpenAI-compatible endpoints accept any non-empty endpoint model ID")); assert!(models.contains("OpenRouter commonly uses provider/model")); @@ -6665,7 +6665,7 @@ fn models_and_config_explain_the_embedded_default() { assert!(config.contains("model.reasoningEffort: low")); assert!(config.contains("model.reasoningEffort.source: embedded default")); assert!(config.contains("model.scorer.enabled: false")); - assert!(config.contains("model.scorerReasoningEffort: none")); + assert!(config.contains("model.scorerReasoningEffort: low")); assert!(config.contains("model.scorerReasoningEffort.source: embedded default")); }