From 97fa826171d790f5f9d691ccf3716a64633177ca Mon Sep 17 00:00:00 2001 From: Paul Querna Date: Wed, 29 Jul 2026 18:05:54 +0000 Subject: [PATCH] test(opencode): use the config-defined model so tests stay hermetic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The opencode bwrap scenarios invoked '-m openai/o3-mini', which resolves against opencode's live-fetched model catalog (models.dev) — not the sha256-pinned binary. The catalog floated forward and dropped o3-mini, breaking main's integration (bwrap) job with three failure modes of the same dependency (pane.cast evidence from the CI artifacts): - catalog fetch fails -> 'UnknownError: Unexpected server error' right after the first-run DB migration (persists_streamed, server_500) - catalog fetch succeeds -> 'Model not found: openai/o3-mini. Did you mean: gpt-4.1-mini, gpt-4o-mini, gpt-5-mini?' (bash_tool_use) - catalog fetch falls back to the baked catalog -> test passes (streams_chunks, persists_user_prompt) All three failing tests saw zero requests reach the fake server because opencode crashed before its first model call. Fix: '-m openai/fake-model' — the model the scenario's own opencode.json defines under the provider's models map. Resolution never touches the live catalog, and the fake server echoes any model field back without validating it. Verified: cargo check -p agent-tui-integration --features bwrap --tests, cargo clippy --workspace --all-targets -- -D warnings, cargo fmt. The bwrap suite itself needs a container host — CI's integration (bwrap) job is the execution environment (same arrangement as #127). Note: crates/agent-tui/skill-data/ai-cli/SKILL.md still uses o3-mini in its (non-executed) real-world examples — stale for the same reason; left for a follow-up since the right replacement model for user-facing docs is a product call. Co-authored-by: c1-squire-dev[bot] --- .../tests/opencode_fake_inference.rs | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/crates/agent-tui-integration/tests/opencode_fake_inference.rs b/crates/agent-tui-integration/tests/opencode_fake_inference.rs index 4f08109..e639a64 100644 --- a/crates/agent-tui-integration/tests/opencode_fake_inference.rs +++ b/crates/agent-tui-integration/tests/opencode_fake_inference.rs @@ -13,7 +13,7 @@ //! ## Two assertion strategies //! //! Default-format `opencode run` writes only a session header to stdout -//! ("`> build · o3-mini`"); the actual assistant body is persisted to +//! ("`> build · fake-model`"); the actual assistant body is persisted to //! `~/.local/share/opencode/opencode.db` (`SQLite`, WAL mode). We use //! two assertion strategies depending on what we're testing: //! @@ -102,16 +102,23 @@ fn write_opencode_config(scratch: &std::path::Path, server_url: &str) -> Result< /// title-generation request to the model BEFORE the real prompt /// when no title is supplied. That extra call burns the first slot /// of any multi-slot Script. Passing a fixed title skips it. -/// - `-m openai/o3-mini`: we use a built-in model name. The fake -/// server doesn't care what the model is — only the path and -/// shape matter. +/// - `-m openai/fake-model`: use the model our own `opencode.json` +/// defines under the provider's `models` map. Resolution then never +/// touches opencode's live-fetched model catalog (models.dev) — which +/// is NOT hermetic: the catalog floated forward and dropped the +/// previously-used `o3-mini`, and the failure mode depended on the +/// fetch outcome (fetch fails → `UnknownError` after the first-run DB +/// migration; fetch succeeds → `Model not found: openai/o3-mini`; +/// fetch falls back to the baked catalog → test passes). The fake +/// server doesn't care what the model is — it echoes the request's +/// `model` field back — only the path and shape matter. fn opencode_run_cmd(prompt: &str) -> Vec { vec![ "bash".into(), "-c".into(), format!( "cd /work && exec opencode run --pure --dangerously-skip-permissions \ - --title 'fixed test title' -m openai/o3-mini {}", + --title 'fixed test title' -m openai/fake-model {}", shell_quote(prompt), ), ] @@ -171,8 +178,8 @@ async fn opencode_persists_streamed_response_to_session_db() -> Result<()> { // long enough for the streaming response to land, be parsed by // OpenCode, committed to its SQLite session DB, and for the // process to exit. `wait_idle(2000)` is generous but bounded — - // a real opencode/o3-mini call against a localhost fake settles - // in well under a second. + // an opencode call against a localhost fake settles in well under + // a second. s.wait_text(r"build · ").await?; s.wait_idle(2000).await?; @@ -217,7 +224,7 @@ fn opencode_run_json_cmd(prompt: &str) -> Vec { "-c".into(), format!( "cd /work && exec opencode run --pure --dangerously-skip-permissions \ - --title 'fixed test title' --format json -m openai/o3-mini {}", + --title 'fixed test title' --format json -m openai/fake-model {}", shell_quote(prompt), ), ]