diff --git a/Cargo.lock b/Cargo.lock index d5bff32..08df629 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1058,6 +1058,7 @@ dependencies = [ "similar", "tempfile", "tokio", + "toml", "wiremock", ] @@ -1505,6 +1506,15 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_spanned" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" +dependencies = [ + "serde_core", +] + [[package]] name = "serde_yaml" version = "0.9.34+deprecated" @@ -1755,6 +1765,45 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81f3d15e84cbcd896376e6730314d59fb5a87f31e4b038454184435cd57defee" +dependencies = [ + "indexmap", + "serde_core", + "serde_spanned", + "toml_datetime", + "toml_parser", + "toml_writer", + "winnow", +] + +[[package]] +name = "toml_datetime" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_parser" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" +dependencies = [ + "winnow", +] + +[[package]] +name = "toml_writer" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "756daf9b1013ebe47a8776667b466417e2d4c5679d441c26230efd9ef78692db" + [[package]] name = "tower" version = "0.5.3" @@ -2171,6 +2220,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" +[[package]] +name = "winnow" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0592e1c9d151f854e6fd382574c3a0855250e1d9b2f99d9281c6e6391af352f1" + [[package]] name = "wiremock" version = "0.6.5" diff --git a/Cargo.toml b/Cargo.toml index 4354eab..b9ff185 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,6 +26,7 @@ serde_yaml = "0.9.34" sha2 = "0.11.0" similar = "3.1.1" tokio = { version = "1.52.3", features = ["rt-multi-thread", "macros", "process", "time"] } +toml = "1.1.2" [dev-dependencies] assert_cmd = "2.2.2" diff --git a/README.md b/README.md index 20cf9f3..c1b25ac 100644 --- a/README.md +++ b/README.md @@ -193,7 +193,8 @@ postil review --staged --output json Hosted remote reviews use a 540-second total LLM budget so model cascades and scoring finish inside the worker watchdog. Local reviews do not use a total budget unless `POSTIL_LLM_TOTAL_TIMEOUT_SECS` is set. Each model request still -uses `POSTIL_LLM_REQUEST_TIMEOUT_SECS`, which defaults to 420 seconds. +uses `POSTIL_LLM_REQUEST_TIMEOUT_SECS`, which defaults to 480 seconds locally +and 420 seconds for hosted remote reviews. Use the live benchmark harness before standardizing on a model: diff --git a/config.toml b/config.toml new file mode 100644 index 0000000..2fbb0f6 --- /dev/null +++ b/config.toml @@ -0,0 +1,11 @@ +version = 1 +default_model = "deepseek/deepseek-v4-pro" +cascade = [ + "google/gemini-3.1-flash-lite", + "moonshotai/kimi-k2.7-code", + "mistralai/mistral-large-2512", +] + +[scorer] +default_model = "anthropic/claude-haiku-4.5" +fallback = "openai/gpt-5-mini" diff --git a/src/config.rs b/src/config.rs index 152aa20..18cec3b 100644 --- a/src/config.rs +++ b/src/config.rs @@ -9,22 +9,117 @@ //! local setups with a trusted repo). `POSTIL_API_BASE` (env) always applies. use std::path::{Path, PathBuf}; +use std::sync::OnceLock; use anyhow::{Context, Result}; use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; use crate::envelope::{Kind, Severity}; -pub const DEFAULT_MODEL: &str = "deepseek/deepseek-v4-pro"; -pub const DEFAULT_CASCADE: [&str; 3] = [ - "google/gemini-3.1-flash-lite", - "moonshotai/kimi-k2.7-code", - "mistralai/mistral-large-2512", -]; -pub const DEFAULT_SCORER_MODEL: &str = "anthropic/claude-haiku-4.5"; -pub const DEFAULT_SCORER_FALLBACK: &str = "openai/gpt-5-mini"; +const MODEL_DEFAULTS_TOML: &str = include_str!("../config.toml"); pub const DEFAULT_API_BASE: &str = "https://openrouter.ai/api/v1"; +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ModelDefaults { + pub version: u64, + pub source_sha256: String, + pub default_model: String, + pub cascade: Vec, + pub scorer_model: String, + pub scorer_fallback: String, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct ModelDefaultsFile { + version: u64, + default_model: String, + cascade: Vec, + scorer: ScorerDefaultsFile, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct ScorerDefaultsFile { + default_model: String, + fallback: String, +} + +pub fn model_defaults() -> &'static ModelDefaults { + static MODEL_DEFAULTS: OnceLock = OnceLock::new(); + MODEL_DEFAULTS.get_or_init(|| { + parse_model_defaults(MODEL_DEFAULTS_TOML).expect("embedded model defaults must parse") + }) +} + +pub fn default_model() -> &'static str { + model_defaults().default_model.as_str() +} + +pub fn default_cascade() -> &'static [String] { + model_defaults().cascade.as_slice() +} + +pub fn default_scorer_model() -> &'static str { + model_defaults().scorer_model.as_str() +} + +pub fn default_scorer_fallback() -> &'static str { + model_defaults().scorer_fallback.as_str() +} + +fn parse_model_defaults(raw: &str) -> Result { + let file: ModelDefaultsFile = toml::from_str(raw)?; + anyhow::ensure!( + file.version > 0, + "model defaults version must be greater than zero" + ); + validate_model_id("defaultModel", &file.default_model)?; + anyhow::ensure!(!file.cascade.is_empty(), "cascade must not be empty"); + for model in &file.cascade { + validate_model_id("cascade entries", model)?; + } + validate_model_id("scorer.defaultModel", &file.scorer.default_model)?; + validate_model_id("scorer.fallback", &file.scorer.fallback)?; + Ok(ModelDefaults { + version: file.version, + source_sha256: sha256_hex(raw), + default_model: file.default_model, + cascade: file.cascade, + scorer_model: file.scorer.default_model, + scorer_fallback: file.scorer.fallback, + }) +} + +fn validate_model_id(field: &str, value: &str) -> Result<()> { + anyhow::ensure!(!value.trim().is_empty(), "{field} must not be empty"); + anyhow::ensure!( + !value.contains(['\n', '\r']), + "{field} must not contain line breaks" + ); + Ok(()) +} + +fn sha256_hex(raw: &str) -> String { + let digest = Sha256::digest(raw.as_bytes()); + let mut hex = String::with_capacity(digest.len() * 2); + for byte in digest { + use std::fmt::Write as _; + write!(&mut hex, "{byte:02x}").expect("writing to String cannot fail"); + } + hex +} + +fn yaml_scalar(value: &str) -> String { + serde_yaml::to_string(value) + .expect("model default scalar must serialize") + .lines() + .filter(|line| *line != "...") + .collect::>() + .join("\n") +} + #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "lowercase")] pub enum OnClean { @@ -116,6 +211,7 @@ pub struct Config { impl Default for Config { fn default() -> Self { + let defaults = model_defaults(); Config { enabled: true, ignore: Vec::new(), @@ -128,9 +224,9 @@ impl Default for Config { gate_fail_on: GateLevel::Severity(Severity::Error), gate_on_error: OnError::Block, block_on_kinds: vec![Kind::HumanEscalation], - model: DEFAULT_MODEL.to_string(), - cascade: DEFAULT_CASCADE.iter().map(|m| (*m).to_string()).collect(), - scorer: DEFAULT_SCORER_MODEL.to_string(), + model: defaults.default_model.clone(), + cascade: defaults.cascade.clone(), + scorer: defaults.scorer_model.clone(), api_base: DEFAULT_API_BASE.to_string(), consensus: 1, guardrails: None, @@ -449,7 +545,7 @@ impl Config { /// Scorer models to try, in order, deduplicated. pub fn scorer_chain(&self) -> Vec { let mut chain = vec![self.scorer.clone()]; - let fallback = DEFAULT_SCORER_FALLBACK.to_string(); + let fallback = model_defaults().scorer_fallback.clone(); if !chain.contains(&fallback) { chain.push(fallback); } @@ -477,7 +573,7 @@ fn rel_name(path: &Path) -> String { .unwrap_or_else(|| path.display().to_string()) } -pub const STARTER_CONFIG: &str = r#"# Postil review configuration. Docs: https://postil.dev/docs/config +const STARTER_CONFIG_TEMPLATE: &str = r#"# Postil review configuration. Docs: https://postil.dev/docs/config # Every key is optional; unknown keys are rejected so typos fail loudly. # ignore: @@ -506,12 +602,9 @@ gate: # # outage from freezing merges; the review check goes neutral, not green. model: - name: deepseek/deepseek-v4-pro + name: __DEFAULT_MODEL__ cascade: - - google/gemini-3.1-flash-lite - - moonshotai/kimi-k2.7-code - - mistralai/mistral-large-2512 - scorer: anthropic/claude-haiku-4.5 +__DEFAULT_CASCADE__ scorer: __DEFAULT_SCORER_MODEL__ # apiBase: https://openrouter.ai/api/v1 # any OpenAI-compatible endpoint (Ollama, vLLM, Azure). # # Ignored from config by default (a repo could redirect # # the inference credential). Prefer POSTIL_API_BASE; to @@ -525,6 +618,25 @@ model: # enabled: false # See https://postil.dev/docs/content-policy "#; +pub fn starter_config() -> &'static str { + static STARTER_CONFIG: OnceLock = OnceLock::new(); + STARTER_CONFIG.get_or_init(|| { + let defaults = model_defaults(); + let cascade = defaults + .cascade + .iter() + .map(|model| format!(" - {}\n", yaml_scalar(model))) + .collect::(); + STARTER_CONFIG_TEMPLATE + .replace("__DEFAULT_MODEL__", &yaml_scalar(&defaults.default_model)) + .replace("__DEFAULT_CASCADE__", &cascade) + .replace( + "__DEFAULT_SCORER_MODEL__", + &yaml_scalar(&defaults.scorer_model), + ) + }) +} + /// Built-in content-policy baseline, active whenever the dimension is on /// (see [`Config::content_policy`]). Scoped to human-readable prose only: /// comments, docstrings, Markdown, and PR title/body, never code logic, @@ -593,7 +705,138 @@ mod tests { } fn default_cascade() -> Vec { - DEFAULT_CASCADE.iter().map(|m| (*m).to_string()).collect() + model_defaults().cascade.clone() + } + + #[test] + fn embedded_model_defaults_match_root_config_file() { + let parsed = parse_model_defaults(MODEL_DEFAULTS_TOML).unwrap(); + let raw: toml::Value = toml::from_str(MODEL_DEFAULTS_TOML).unwrap(); + assert_eq!(parsed.version, raw["version"].as_integer().unwrap() as u64); + assert_eq!(parsed.source_sha256, sha256_hex(MODEL_DEFAULTS_TOML)); + assert_eq!(parsed.source_sha256.len(), 64); + assert_eq!(parsed.default_model, raw["default_model"].as_str().unwrap()); + assert_eq!( + parsed.cascade, + raw["cascade"] + .as_array() + .unwrap() + .iter() + .map(|value| value.as_str().unwrap().to_string()) + .collect::>() + ); + assert_eq!( + parsed.scorer_model, + raw["scorer"]["default_model"].as_str().unwrap() + ); + assert_eq!( + parsed.scorer_fallback, + raw["scorer"]["fallback"].as_str().unwrap() + ); + assert_eq!(&parsed, model_defaults()); + } + + #[test] + fn model_default_accessors_expose_embedded_values() { + let defaults = model_defaults(); + assert_eq!(default_model(), defaults.default_model); + assert_eq!(super::default_cascade(), defaults.cascade); + assert_eq!(default_scorer_model(), defaults.scorer_model); + assert_eq!(default_scorer_fallback(), defaults.scorer_fallback); + } + + #[test] + fn malformed_model_defaults_fail_loudly() { + let err = parse_model_defaults( + r#"version = 1 +default_model = "example/model" +cascade = ["example/fallback"] +unexpected_key = "typo" + +[scorer] +default_model = "example/scorer" +fallback = "example/scorer-fallback" +"#, + ) + .unwrap_err(); + let message = format!("{err:#}"); + assert!(message.contains("unknown field")); + assert!(message.contains("unexpected_key")); + } + + #[test] + fn malformed_model_defaults_reject_invalid_values() { + let cases = [ + ( + "version = 0\n\ + default_model = \"example/model\"\n\ + cascade = [\"example/fallback\"]\n\ + scorer = { default_model = \"example/scorer\", fallback = \"example/scorer-fallback\" }\n", + "version must be greater than zero", + ), + ( + "version = 1\n\ + default_model = \"\"\n\ + cascade = [\"example/fallback\"]\n\ + scorer = { default_model = \"example/scorer\", fallback = \"example/scorer-fallback\" }\n", + "defaultModel must not be empty", + ), + ( + "version = 1\n\ + default_model = \"example/model\"\n\ + cascade = []\n\ + scorer = { default_model = \"example/scorer\", fallback = \"example/scorer-fallback\" }\n", + "cascade must not be empty", + ), + ( + "version = 1\n\ + default_model = \"example/model\"\n\ + cascade = [\"\"]\n\ + scorer = { default_model = \"example/scorer\", fallback = \"example/scorer-fallback\" }\n", + "cascade entries must not be empty", + ), + ( + "version = 1\n\ + default_model = \"example/model\"\n\ + cascade = [\"example/fallback\"]\n\ + scorer = { default_model = \"\", fallback = \"example/scorer-fallback\" }\n", + "scorer.defaultModel must not be empty", + ), + ( + "version = 1\n\ + default_model = \"example/model\"\n\ + cascade = [\"example/fallback\"]\n\ + scorer = { default_model = \"example/scorer\", fallback = \"\" }\n", + "scorer.fallback must not be empty", + ), + ]; + for (raw, expected) in cases { + let err = parse_model_defaults(raw).unwrap_err(); + assert!( + format!("{err:#}").contains(expected), + "expected {expected:?} in {err:#}" + ); + } + } + + #[test] + fn starter_config_yaml_quotes_model_defaults_when_needed() { + assert_eq!(yaml_scalar("plain/model"), "plain/model"); + assert_eq!( + serde_yaml::from_str::(&yaml_scalar("model: with # yaml")).unwrap(), + "model: with # yaml" + ); + } + + #[test] + fn readme_model_example_mentions_embedded_defaults() { + let readme = include_str!("../README.md"); + let defaults = model_defaults(); + assert!(readme.contains(&defaults.default_model)); + for model in &defaults.cascade { + assert!(readme.contains(model)); + } + assert!(readme.contains(&defaults.scorer_model)); } #[test] @@ -610,12 +853,16 @@ mod tests { #[test] fn defaults_keep_primary_and_retry_roster_order() { let c = Config::default(); - assert_eq!(c.model, DEFAULT_MODEL); + let defaults = model_defaults(); + assert_eq!(c.model, defaults.default_model); assert_eq!(c.cascade, default_cascade()); - assert_eq!(c.scorer, DEFAULT_SCORER_MODEL); + assert_eq!(c.scorer, defaults.scorer_model); assert_eq!( c.scorer_chain(), - vec![DEFAULT_SCORER_MODEL, DEFAULT_SCORER_FALLBACK] + vec![ + defaults.scorer_model.clone(), + defaults.scorer_fallback.clone() + ] ); assert_eq!( c.model_chain(), @@ -665,13 +912,14 @@ mod tests { #[test] fn starter_config_parses() { - let f: FileConfig = serde_yaml::from_str(STARTER_CONFIG).unwrap(); + let f: FileConfig = serde_yaml::from_str(starter_config()).unwrap(); let mut c = Config::default(); c.apply_file(f).unwrap(); + let defaults = model_defaults(); assert_eq!(c.max_findings, 20); - assert_eq!(c.model, DEFAULT_MODEL); + assert_eq!(c.model, defaults.default_model); assert_eq!(c.cascade, default_cascade()); - assert_eq!(c.scorer, DEFAULT_SCORER_MODEL); + assert_eq!(c.scorer, defaults.scorer_model); } #[test] @@ -682,7 +930,7 @@ mod tests { assert_eq!(c.scorer, "custom/scorer"); assert_eq!( c.scorer_chain(), - vec!["custom/scorer", DEFAULT_SCORER_FALLBACK] + vec!["custom/scorer", &model_defaults().scorer_fallback] ); } diff --git a/src/main.rs b/src/main.rs index 1282029..58b06f0 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,7 +1,7 @@ use clap::Parser; use postil_cli::cli::{Cli, Command, ForgeArg, HookAction}; -use postil_cli::config::{Config, STARTER_CONFIG}; +use postil_cli::config::{Config, starter_config}; use postil_cli::review::{ForgeKind, ReviewArgs}; use postil_cli::{doctor, hook, plan, respond, review}; @@ -166,7 +166,7 @@ async fn dispatch(cli: Cli) -> anyhow::Result { path.display() ); } - std::fs::write(&path, STARTER_CONFIG)?; + std::fs::write(&path, starter_config())?; eprintln!("postil: wrote {}", path.display()); Ok(0) }