From d865d0b1a3bc782738da36f2cd0798c5d0b26f17 Mon Sep 17 00:00:00 2001 From: RECTOR Date: Sat, 29 Aug 2026 11:32:12 +0700 Subject: [PATCH 1/2] feat(subagent): surface unset model-fallback on retryable failures + ARMORY_FLEET_MODEL_FALLBACK=auto MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #58: a retryable provider failure with neither a per-dispatch modelFallback nor the global default silently skipped auto-retry — the operator had no signal. Two changes, per the issue's rec: - Visibility: the tool appends 'no modelFallback configured - pass modelFallback or set ARMORY_FLEET_MODEL_FALLBACK' to a retryable failure's error, exactly when the missing fallback matters. - auto sentinel: ARMORY_FLEET_MODEL_FALLBACK=auto resolves a fallback from the runtime's configured+available snapshot per session — a different provider than the session model preferred, else a different id; unresolvable (single-model) stays off with a one-time warning. resolveAutoFallback is a pure helper (engine/auto-fallback.ts) + tests; wiring in index.ts init (verbatim values) + session_start (auto). --- src/engine/auto-fallback.ts | 19 +++++++++++++++ src/index.ts | 15 +++++++++++- src/tools/subagent.ts | 10 ++++++++ test/auto-fallback.test.mts | 47 +++++++++++++++++++++++++++++++++++++ test/subagent-tool.test.mts | 40 +++++++++++++++++++++++++++++++ 5 files changed, 130 insertions(+), 1 deletion(-) create mode 100644 src/engine/auto-fallback.ts create mode 100644 test/auto-fallback.test.mts diff --git a/src/engine/auto-fallback.ts b/src/engine/auto-fallback.ts new file mode 100644 index 0000000..fd5361a --- /dev/null +++ b/src/engine/auto-fallback.ts @@ -0,0 +1,19 @@ +// src/engine/auto-fallback.ts +// #58: resolve the ARMORY_FLEET_MODEL_FALLBACK=auto sentinel — pick a fallback model from the +// runtime's configured+available snapshot without the operator naming one. Prefers a model from +// a DIFFERENT provider than the session model (a real fallback family, per the "Ollama primary + +// OpenRouter fallback" pattern the feature was named for); if every available model shares the +// session's provider, a different model id on that provider. undefined when nothing differs +// (single-model setups) — the caller keeps auto-retry off and surfaces why. + +export function resolveAutoFallback( + available: ReadonlyArray<{ provider: string; id: string }>, + parentModel: { provider: string; id: string }, +): string | undefined { + const parentProvider = parentModel.provider || ""; + const parentKey = `${parentModel.provider}/${parentModel.id}`; + const differentProvider = available.filter((m) => m.provider !== parentProvider); + const pool = differentProvider.length > 0 ? differentProvider : available; + const pick = pool.find((m) => `${m.provider}/${m.id}` !== parentKey); + return pick ? `${pick.provider}/${pick.id}` : undefined; +} diff --git a/src/index.ts b/src/index.ts index 598b674..cbc05d0 100644 --- a/src/index.ts +++ b/src/index.ts @@ -18,6 +18,7 @@ import { ArmoryMemoryAdapter } from "./memory-hydrate/adapter.ts"; import { ArmoryVisionAdapter } from "./vision/adapter.ts"; import { buildChildLoader } from "./engine/child-loader.ts"; import { withModelFallbackRetry } from "./engine/retry-fallback.ts"; +import { resolveAutoFallback } from "./engine/auto-fallback.ts"; import { createDescribeImageTool } from "./vision/describe-image-tool.ts"; import type { MemoryHydratePort } from "./memory-hydrate/port.ts"; import type { VisionPort } from "./vision/port.ts"; @@ -204,7 +205,10 @@ export default async function (pi: ExtensionAPI): Promise { // #39 tail: global default fallback model (env-driven for now; a settings.json field is a follow-up). // A retryable provider failure (stopReason "error") retries once on this model even without a // per-dispatch `modelFallback`. Per the AGENTS.md "Ollama primary + OpenRouter fallback" pattern. - deps.defaultModelFallback = process.env.ARMORY_FLEET_MODEL_FALLBACK || undefined; + // #39 tail + #58: the global default fallback. Non-"auto" env values are used verbatim; + // "auto" is resolved per-session in session_start (it needs the session model to differ from). + const rawModelFallback = process.env.ARMORY_FLEET_MODEL_FALLBACK || undefined; + deps.defaultModelFallback = rawModelFallback === "auto" ? undefined : rawModelFallback; // SPEC-6-5: cross-cwd dispatch notify hook (wired per-session in session_start below). // Placeholder; the real wiring happens in session_start where ctx is in scope. @@ -309,6 +313,15 @@ export default async function (pi: ExtensionAPI): Promise { refreshLifecycles(ctx); const m = ctx.model; deps.parentModel = m ? { provider: m.provider, id: m.id } : { provider: "", id: "" }; + // #58: ARMORY_FLEET_MODEL_FALLBACK=auto — pick a fallback from the configured+available + // snapshot that differs from the session model (different provider preferred). Unresolvable + // (single-model setup) → stay off + say why once per session. + if (rawModelFallback === "auto") { + deps.defaultModelFallback = resolveAutoFallback(modelRuntime.getAvailableSnapshot(), deps.parentModel); + if (!deps.defaultModelFallback) { + ctx.ui.notify("ARMORY_FLEET_MODEL_FALLBACK=auto, but no alternative configured model is available — auto-retry stays off", "warning"); + } + } deps.parentCwd = ctx.cwd; deps.onNotify = (m, k) => ctx.ui.notify(m, k ?? "info"); // SPEC-5a: build the per-session async runner + scheduler, start firing, scan for interrupted runs. diff --git a/src/tools/subagent.ts b/src/tools/subagent.ts index 766d815..32e2f68 100644 --- a/src/tools/subagent.ts +++ b/src/tools/subagent.ts @@ -240,6 +240,16 @@ export function createSubagentTool(deps: SubagentToolDeps) { error: `primary '${res.model}' failed: ${res.error}; fallback '${retriedWithModel}' failed: ${finalRes.error ?? finalRes.status}`, }; } + // #58: a retryable failure with NO fallback configured (neither per-dispatch nor global) + // means the auto-retry silently didn't fire — surface that, and how to enable it, exactly + // when it matters. Mutually exclusive with the #59 composition above (a retry implies a + // fallback was configured). + if (finalRes.status === "failed" && finalRes.retryable && !fallback) { + finalRes = { + ...finalRes, + error: `${finalRes.error ?? finalRes.status}\n(no modelFallback configured — pass modelFallback or set ARMORY_FLEET_MODEL_FALLBACK to enable one-shot auto-retry)`, + }; + } const isError = finalRes.status === "failed" || finalRes.status === "aborted"; return { content: [{ type: "text" as const, text: isError ? (finalRes.error ?? finalRes.status) : finalRes.finalText }], diff --git a/test/auto-fallback.test.mts b/test/auto-fallback.test.mts new file mode 100644 index 0000000..43cb332 --- /dev/null +++ b/test/auto-fallback.test.mts @@ -0,0 +1,47 @@ +// test/auto-fallback.test.mts — #58: ARMORY_FLEET_MODEL_FALLBACK=auto sentinel resolution. +import { test } from "node:test"; +import { strictEqual } from "node:assert"; +import { resolveAutoFallback } from "../src/engine/auto-fallback.ts"; + +test("#58 auto: prefers the first available model from a DIFFERENT provider than the session model", () => { + const pick = resolveAutoFallback( + [ + { provider: "Ollama", id: "minimax-m3:cloud" }, + { provider: "openrouter", id: "z-ai/glm-5.2" }, + { provider: "openai", id: "gpt-5.2" }, + ], + { provider: "Ollama", id: "glm-5.2:cloud" }, + ); + strictEqual(pick, "openrouter/z-ai/glm-5.2", "first different-provider model wins (family fallback)"); +}); + +test("#58 auto: same-provider different-id when no other provider is available", () => { + const pick = resolveAutoFallback( + [ + { provider: "Ollama", id: "glm-5.2:cloud" }, + { provider: "Ollama", id: "minimax-m3:cloud" }, + ], + { provider: "Ollama", id: "glm-5.2:cloud" }, + ); + strictEqual(pick, "Ollama/minimax-m3:cloud", "falls back to a different model on the same provider"); +}); + +test("#58 auto: single-model setup → undefined (caller keeps auto-retry off + surfaces why)", () => { + const pick = resolveAutoFallback([{ provider: "Ollama", id: "glm-5.2:cloud" }], { provider: "Ollama", id: "glm-5.2:cloud" }); + strictEqual(pick, undefined, "nothing differs from the primary → no auto fallback"); +}); + +test("#58 auto: empty available snapshot → undefined", () => { + strictEqual(resolveAutoFallback([], { provider: "Ollama", id: "glm-5.2:cloud" }), undefined); +}); + +test("#58 auto: never picks the session model itself even when other models exist", () => { + const pick = resolveAutoFallback( + [ + { provider: "Ollama", id: "glm-5.2:cloud" }, + { provider: "Ollama", id: "glm-5.2:cloud" }, + ], + { provider: "Ollama", id: "glm-5.2:cloud" }, + ); + strictEqual(pick, undefined, "an identical duplicate is not a fallback"); +}); diff --git a/test/subagent-tool.test.mts b/test/subagent-tool.test.mts index 5f7985f..aefb905 100644 --- a/test/subagent-tool.test.mts +++ b/test/subagent-tool.test.mts @@ -487,3 +487,43 @@ test("#59: when the fallback retry also fails, the surfaced error includes the P ok(text.includes("openrouter/z-ai/glm-5.2"), `error names the FALLBACK model: ${text}`); ok(text.includes("fallback rate limited too"), `error includes the FALLBACK's failure text: ${text}`); }); + +test("#58: retryable failure with NO fallback configured → surfaces the enable-retry hint", async () => { + // A retryable (stopReason 'error') failure with neither modelFallback nor defaultModelFallback + // means the auto-retry silently didn't fire (#58) — the operator must be told, at the moment + // it matters, how to enable it. + const errorHandlers: Array<(e: any) => void> = []; + const errorChild = { + prompt: async () => { for (const h of errorHandlers) h({ type: "message_end", message: { role: "assistant", stopReason: "error", content: [{ type: "text", text: "rate limited" }] } }); }, + subscribe: (h: any) => { errorHandlers.push(h); return () => {}; }, abort: async () => {}, dispose: () => {}, + }; + const factory: ChildSessionFactory = { create: async () => ({ session: errorChild, model: "Ollama/glm-5.2:cloud" }) }; + const deps = makeDeps(); // no defaultModelFallback set + const reg = new BackendRegistry(); + reg.register({ id: "pi", factory, available: () => true, versionInfo: () => null, hookParity: PI_HOOK_PARITY }); + deps.backendRegistry = reg; + const tool = createSubagentTool(deps); + const out = await tool.execute!("c", { agent: "g", task: "x" } as any, new AbortController().signal, () => {}, {} as any); + strictEqual(out.isError, true); + const text = (out.content as any)[0].text as string; + ok(text.includes("rate limited"), `keeps the original failure: ${text}`); + ok(text.includes("no modelFallback configured"), `surfaces the no-fallback hint: ${text}`); + ok(text.includes("ARMORY_FLEET_MODEL_FALLBACK"), `names the env var: ${text}`); +}); + +test("#58: per-dispatch modelFallback set → NO no-fallback hint (retry already handled it)", async () => { + const errorHandlers: Array<(e: any) => void> = []; + const errorChild = { + prompt: async () => { for (const h of errorHandlers) h({ type: "message_end", message: { role: "assistant", stopReason: "error", content: [{ type: "text", text: "rate limited" }] } }); }, + subscribe: (h: any) => { errorHandlers.push(h); return () => {}; }, abort: async () => {}, dispose: () => {}, + }; + const factory: ChildSessionFactory = { create: async () => ({ session: errorChild, model: "Ollama/glm-5.2:cloud" }) }; + const deps = makeDeps(); + const reg = new BackendRegistry(); + reg.register({ id: "pi", factory, available: () => true, versionInfo: () => null, hookParity: PI_HOOK_PARITY }); + deps.backendRegistry = reg; + const tool = createSubagentTool(deps); + const out = await tool.execute!("c", { agent: "g", task: "x", modelFallback: "openrouter/z-ai/glm-5.2" } as any, new AbortController().signal, () => {}, {} as any); + const text = (out.content as any)[0].text as string; + ok(!text.includes("no modelFallback configured"), `hint absent when a fallback was configured: ${text}`); +}); From 640f8eb5bf29b5199701d7da589078b9c8377eb8 Mon Sep 17 00:00:00 2001 From: RECTOR Date: Sat, 29 Aug 2026 11:36:17 +0700 Subject: [PATCH 2/2] test: boundary cases for the #58 no-fallback hint gate (global fallback set; non-retryable failure) --- test/subagent-tool.test.mts | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/test/subagent-tool.test.mts b/test/subagent-tool.test.mts index aefb905..26820b4 100644 --- a/test/subagent-tool.test.mts +++ b/test/subagent-tool.test.mts @@ -527,3 +527,40 @@ test("#58: per-dispatch modelFallback set → NO no-fallback hint (retry already const text = (out.content as any)[0].text as string; ok(!text.includes("no modelFallback configured"), `hint absent when a fallback was configured: ${text}`); }); + +test("#58: global defaultModelFallback set → NO no-fallback hint", async () => { + const errorHandlers: Array<(e: any) => void> = []; + const errorChild = { + prompt: async () => { for (const h of errorHandlers) h({ type: "message_end", message: { role: "assistant", stopReason: "error", content: [{ type: "text", text: "rate limited" }] } }); }, + subscribe: (h: any) => { errorHandlers.push(h); return () => {}; }, abort: async () => {}, dispose: () => {}, + }; + const factory: ChildSessionFactory = { create: async () => ({ session: errorChild, model: "Ollama/glm-5.2:cloud" }) }; + const deps = makeDeps(); + deps.defaultModelFallback = "openrouter/z-ai/glm-5.2"; + const reg = new BackendRegistry(); + reg.register({ id: "pi", factory, available: () => true, versionInfo: () => null, hookParity: PI_HOOK_PARITY }); + deps.backendRegistry = reg; + const tool = createSubagentTool(deps); + const out = await tool.execute!("c", { agent: "g", task: "x" } as any, new AbortController().signal, () => {}, {} as any); + const text = (out.content as any)[0].text as string; + ok(!text.includes("no modelFallback configured"), `hint absent when the global default is set: ${text}`); +}); + +test("#58: non-retryable failure (prompt threw) → NO hint regardless of fallback config", async () => { + const hs: Array<(e: any) => void> = []; + const crashChild = { + prompt: async () => { for (const h of hs) h({ type: "turn_start", turnIndex: 0 }); throw new Error("child crashed mid-prompt"); }, + subscribe: (h: any) => { hs.push(h); return () => {}; }, abort: async () => {}, dispose: () => {}, + }; + const factory: ChildSessionFactory = { create: async () => ({ session: crashChild, model: "Ollama/glm-5.2:cloud" }) }; + const deps = makeDeps(); // no fallback configured + const reg = new BackendRegistry(); + reg.register({ id: "pi", factory, available: () => true, versionInfo: () => null, hookParity: PI_HOOK_PARITY }); + deps.backendRegistry = reg; + const tool = createSubagentTool(deps); + const out = await tool.execute!("c", { agent: "g", task: "x" } as any, new AbortController().signal, () => {}, {} as any); + strictEqual(out.isError, true); + const text = (out.content as any)[0].text as string; + ok(text.includes("child crashed mid-prompt"), `surfaces the real failure: ${text}`); + ok(!text.includes("no modelFallback configured"), `no hint on a non-retryable failure: ${text}`); +});