From f8eee173a7104974dcc470775747b197d6018a09 Mon Sep 17 00:00:00 2001 From: karim Date: Sat, 17 Oct 2026 22:04:14 +0200 Subject: [PATCH] feat(319): a skill that cannot decide says so before it runs A decision step now requires the `decision_model` setup token. Under Auto a key for any decision model's provider satisfies it; an assigned decision model needs its own provider's key. A skill with no answerable decision step reads needsSetup with a connect-provider remedy instead of reporting ready and failing at its first decision. The provider list is pinned to the catalog's decision models by a contract test. The composer's Models & reasoning menu gains the Decisions row the settings page already had. Co-Authored-By: Claude Opus 5.5 --- ...-second-decision-model-openai-decisions.md | 2 + .../decisionLaneProviders.contract.test.ts | 36 +++++++++++++++++ .../src/__tests__/codePolicyReadiness.test.ts | 40 ++++++++++++++++++- .../src/skillProjectionReconciler.ts | 18 ++++++++- .../src/stagedChange/bundleInstallManifest.ts | 5 ++- .../schemas/src/cybernetic/skillProjection.ts | 15 +++++++ .../cybernetic/AgentModelSettings.tsx | 10 ++++- 7 files changed, 121 insertions(+), 5 deletions(-) create mode 100644 packages/ai-client/src/decisionLaneProviders.contract.test.ts diff --git a/docs/plans/319-a-second-decision-model-openai-decisions.md b/docs/plans/319-a-second-decision-model-openai-decisions.md index 95a8b4ca..0632aaba 100644 --- a/docs/plans/319-a-second-decision-model-openai-decisions.md +++ b/docs/plans/319-a-second-decision-model-openai-decisions.md @@ -32,6 +32,8 @@ Read from the `openai` SDK's published types (7.30+, `client.decisions.create`) **D6 — A space chooses its decision model.** `modelDefaults.decision` in the space directives names the model. Like the Clerk it does not inherit `default` — a chat model cannot answer a decision — so unset resolves to the first catalog decision model whose provider the space holds a key for, passing over models that read no images when the step carries one. The Agent settings page gains a Decisions row: "Auto · first decision model with a key", or a decision model by its alias. +**D7 — A skill that cannot decide says so before it runs.** A decision step requires the `decision_model` setup token, satisfied by a key for any decision model's provider under Auto, and only by the assigned model's provider when the space assigns one. A skill whose decision steps no key can answer reads `needsSetup` with a connect-provider remedy instead of reporting ready and failing at its first decision step. The provider list is pinned to the catalog's decision models by a contract test. The composer's Models & reasoning menu carries the same Decisions row as the settings page. + **D7 — Images when the model reads them.** The step takes `images`: up to 128 stored-media references, the form every media operation already returns — never inline bytes in a step message. The executor resolves them with the media source resolver and the adapter sends them as data URLs. Sending images to a model without `imageContext` is refused by the client, and by skill validity (`decision_model_reads_no_images`) when the skill names that model. **D8 — The two are compared, not assumed equal.** The same labelled decision sets run through both models — accuracy against labels, abstention at a threshold, a reliability curve of stated confidence against observed accuracy, latency and cost per decision — on the eval batch and judge-scorecard machinery. The result picks the recommended default and is re-run when either model changes. diff --git a/packages/ai-client/src/decisionLaneProviders.contract.test.ts b/packages/ai-client/src/decisionLaneProviders.contract.test.ts new file mode 100644 index 00000000..780faa27 --- /dev/null +++ b/packages/ai-client/src/decisionLaneProviders.contract.test.ts @@ -0,0 +1,36 @@ +import { describe, expect, it } from 'vitest'; +import { + DECISION_MODEL_CAPABILITY_ID, + LANE_CAPABILITY_PROVIDERS, + describeMissingCapability, + laneCapabilityTokensForTask, +} from '@aflow/schemas'; +import { createDefaultModelCatalog } from './catalog.js'; + +describe('decision model readiness token', () => { + it('is satisfied by exactly the providers of the catalog decision models', () => { + const catalogProviders = new Set( + createDefaultModelCatalog() + .listModels() + .filter((m) => m.capabilities.decision === true) + .map((m) => m.provider), + ); + expect(new Set(LANE_CAPABILITY_PROVIDERS[DECISION_MODEL_CAPABILITY_ID])).toEqual( + catalogProviders, + ); + }); + + it('is required by a decision step and by nothing else in the ai lane', () => { + expect(laneCapabilityTokensForTask({ operation: 'ai.decision.decide' })).toEqual([ + DECISION_MODEL_CAPABILITY_ID, + ]); + expect(laneCapabilityTokensForTask({ operation: 'ai.text.generate' })).toEqual([]); + }); + + it('names every provider that would close it', () => { + const remedy = describeMissingCapability(DECISION_MODEL_CAPABILITY_ID); + for (const provider of LANE_CAPABILITY_PROVIDERS[DECISION_MODEL_CAPABILITY_ID] ?? []) { + expect(remedy).toContain(provider); + } + }); +}); diff --git a/packages/cybernetic-runtime/src/__tests__/codePolicyReadiness.test.ts b/packages/cybernetic-runtime/src/__tests__/codePolicyReadiness.test.ts index 16bdcc61..9ed2d3fb 100644 --- a/packages/cybernetic-runtime/src/__tests__/codePolicyReadiness.test.ts +++ b/packages/cybernetic-runtime/src/__tests__/codePolicyReadiness.test.ts @@ -42,6 +42,8 @@ interface SpaceFixture { codePolicyColumnMissing: boolean; readyRepos: number; laneProviders: string[]; + /** `spaces.directives.modelDefaults.decision` — `undefined` = Auto. */ + decisionModel: string | undefined; /** `undefined` = no assignment row (role default decides). */ profileAllowedCapabilities: unknown; /** Applied to the assigned profile AND the role defaults. */ @@ -100,7 +102,16 @@ function rowsFor(table: unknown, cols: Record): unknown[] { } return [{ codePolicy: fixture.codePolicy }]; } - return [{ computePolicy: fixture.computePolicy, ownerId: null }]; + return [ + { + computePolicy: fixture.computePolicy, + ownerId: null, + directives: + fixture.decisionModel === undefined + ? null + : { modelDefaults: { decision: fixture.decisionModel } }, + }, + ]; } return []; } @@ -182,6 +193,7 @@ beforeEach(() => { codePolicyColumnMissing: false, readyRepos: 0, laneProviders: [], + decisionModel: undefined, profileAllowedCapabilities: undefined, profileDeniedCapabilities: [], profileAllowedRiskModifiers: SEEDED_RISK_MODIFIERS, @@ -511,3 +523,29 @@ describe('applyCapabilityStatuses — code operations', () => { expect(deps[0]!.status).toBe('ready'); }); }); + +describe('loadAvailableCapabilities — decision model', () => { + it('is unavailable with no key for any decision provider', async () => { + fixture.laneProviders = ['anthropic']; + expect(await load()).not.toContain('decision_model'); + }); + + it('is available under Auto with a key for any decision provider', async () => { + fixture.laneProviders = ['openai']; + expect(await load()).toContain('decision_model'); + }); + + it('requires the assigned decision model provider, not just any', async () => { + fixture.decisionModel = 'jev'; + fixture.laneProviders = ['openai']; + expect(await load()).not.toContain('decision_model'); + fixture.laneProviders = ['typesafe']; + expect(await load()).toContain('decision_model'); + }); + + it('is unavailable when the assigned decision model is not in the catalog', async () => { + fixture.decisionModel = 'no-such-decider'; + fixture.laneProviders = ['typesafe', 'openai']; + expect(await load()).not.toContain('decision_model'); + }); +}); diff --git a/packages/cybernetic-runtime/src/skillProjectionReconciler.ts b/packages/cybernetic-runtime/src/skillProjectionReconciler.ts index c9bf6485..76bf3079 100644 --- a/packages/cybernetic-runtime/src/skillProjectionReconciler.ts +++ b/packages/cybernetic-runtime/src/skillProjectionReconciler.ts @@ -21,6 +21,7 @@ import { SPACE_POLICY_OPERATION_PREFIXES, CODE_REPO_CAPABILITY_ID, CODE_MODEL_CAPABILITY_ID, + DECISION_MODEL_CAPABILITY_ID, LANE_CAPABILITY_PROVIDERS, laneCapabilityTokensForTask, subsumeLaneTokens, @@ -29,6 +30,7 @@ import { foldMissingCapabilitiesForAbsentCodeLane, type CodeLane, } from '@aflow/schemas'; +import { createDefaultModelCatalog } from '@aflow/ai-client'; import { appendEntityEvent } from '@aflow/redis'; import { createTenantContext, @@ -228,7 +230,11 @@ export async function loadAvailableCapabilities(ctx: { .where(and(eq(repoBindings.spaceId, ctx.spaceId), eq(repoBindings.status, 'ready'))) .limit(1); const [spaceRow] = await tx - .select({ computePolicy: spaces.computePolicy, ownerId: spaces.ownerId }) + .select({ + computePolicy: spaces.computePolicy, + ownerId: spaces.ownerId, + directives: spaces.directives, + }) .from(spaces) .where(eq(spaces.id, ctx.spaceId)); // Lane credentials (`code_model`, `search_provider`): kind-level, so the @@ -443,6 +449,16 @@ export async function loadAvailableCapabilities(ctx: { for (const p of LANE_CAPABILITY_PROVIDERS[CODE_MODEL_CAPABILITY_ID] ?? []) { if (lanePresent.has(p)) available.add(`${CODE_MODEL_CAPABILITY_ID}:${p}`); } + // An assigned decision model is the one every decision step runs on, so only + // its provider's key makes them runnable; a key for another decision + // provider would report ready and fail at the first step. + const assignedDecision = spaceRow?.directives?.modelDefaults.decision; + if (assignedDecision !== undefined) { + const provider = createDefaultModelCatalog().getModel(assignedDecision)?.provider; + if (provider === undefined || !lanePresent.has(provider)) { + available.delete(DECISION_MODEL_CAPABILITY_ID); + } + } return { available, codeLane, withheldByProfile, withheldByCeiling, withheldByProfileAlone }; } diff --git a/packages/cybernetic-runtime/src/stagedChange/bundleInstallManifest.ts b/packages/cybernetic-runtime/src/stagedChange/bundleInstallManifest.ts index 8618050d..af1b7c1f 100644 --- a/packages/cybernetic-runtime/src/stagedChange/bundleInstallManifest.ts +++ b/packages/cybernetic-runtime/src/stagedChange/bundleInstallManifest.ts @@ -5,6 +5,7 @@ import { CODE_REPO_CAPABILITY_ID, SPACE_POLICY_OPERATION_PREFIXES, getPlatformOperationPrefixes, + DECISION_MODEL_CAPABILITY_ID, laneProvidersForToken, } from '@aflow/schemas'; import type { CredentialSlot, McpAuthType, PostInstallTask, SkillBundle } from '@aflow/schemas'; @@ -409,7 +410,9 @@ async function generateUnmetCapabilityEntries(opts: { providerOptions: [...providers], description: token.startsWith('code_model') ? `Connect a coding-lane key (${providers.join(' or ')}) so the coding skills can run.` - : 'Connect a Brave Search key so search steps can run.', + : token === DECISION_MODEL_CAPABILITY_ID + ? `Connect a decision model key (${providers.join(' or ')}) so decision steps can run.` + : 'Connect a Brave Search key so search steps can run.', required: true, }); } diff --git a/packages/schemas/src/cybernetic/skillProjection.ts b/packages/schemas/src/cybernetic/skillProjection.ts index 822203e6..1430434f 100644 --- a/packages/schemas/src/cybernetic/skillProjection.ts +++ b/packages/schemas/src/cybernetic/skillProjection.ts @@ -67,14 +67,19 @@ export const CODE_REPO_CAPABILITY_ID = 'code_repo'; * that exact provider (`code_model:zai`); an unpinned task accepts any * coding backend (`code_model`). Search gates only `search.web.search` * (Brave is hard-required by the handler; `search.web.fetch` runs keyless). + * A decision step needs a key for some decision model's provider; which one + * the space must hold depends on its decision assignment, so availability — + * not the token — carries that. */ export const CODE_MODEL_CAPABILITY_ID = 'code_model'; export const SEARCH_PROVIDER_CAPABILITY_ID = 'search_provider'; +export const DECISION_MODEL_CAPABILITY_ID = 'decision_model'; /** Credential providerIds satisfying the generic (unpinned) lane token. */ export const LANE_CAPABILITY_PROVIDERS: Record = { [CODE_MODEL_CAPABILITY_ID]: ['zai', 'anthropic', 'openai'], [SEARCH_PROVIDER_CAPABILITY_ID]: ['brave'], + [DECISION_MODEL_CAPABILITY_ID]: ['typesafe', 'openai'], }; /** Lane tokens a task's operation (+ pinned backend) requires. */ @@ -94,6 +99,7 @@ export function laneCapabilityTokensForTask(task: { tokens.push(pinned ? `${CODE_MODEL_CAPABILITY_ID}:${pinned}` : CODE_MODEL_CAPABILITY_ID); } if (op === 'search.web.search') tokens.push(SEARCH_PROVIDER_CAPABILITY_ID); + if (op === 'ai.decision.decide') tokens.push(DECISION_MODEL_CAPABILITY_ID); return tokens; } @@ -113,6 +119,7 @@ export function isLaneCapabilityToken(token: string): boolean { return ( token === CODE_MODEL_CAPABILITY_ID || token === SEARCH_PROVIDER_CAPABILITY_ID || + token === DECISION_MODEL_CAPABILITY_ID || token.startsWith(`${CODE_MODEL_CAPABILITY_ID}:`) ); } @@ -187,6 +194,14 @@ export function describeMissingCapability(token: string): string { if (token === SEARCH_PROVIDER_CAPABILITY_ID) { return 'No search key (Brave) is connected; connect one under Settings → Credentials.'; } + if (token === DECISION_MODEL_CAPABILITY_ID) { + return ( + 'No decision model can run in this space: its decision steps need a key for the ' + + 'assigned decision model, or under Auto for any of ' + + `${LANE_CAPABILITY_PROVIDERS[DECISION_MODEL_CAPABILITY_ID]?.join(', ') ?? ''}; ` + + 'connect one under Settings → Credentials.' + ); + } return `${token} has no bound integration in this space; bind one under Integrations.`; } diff --git a/packages/web-product/src/ui/components/cybernetic/AgentModelSettings.tsx b/packages/web-product/src/ui/components/cybernetic/AgentModelSettings.tsx index 29e480e9..e4185fdb 100644 --- a/packages/web-product/src/ui/components/cybernetic/AgentModelSettings.tsx +++ b/packages/web-product/src/ui/components/cybernetic/AgentModelSettings.tsx @@ -43,8 +43,13 @@ export function AgentModelSettings({ const storedModelRefs = Object.values(controller.directives?.modelDefaults ?? {}).filter( (v): v is string => typeof v === 'string', ); - const { modelOptions, clerkModelOptions, modelOptionsLoading, modelOptionsError } = - useChatModelOptions(storedModelRefs); + const { + modelOptions, + clerkModelOptions, + decisionModelOptions, + modelOptionsLoading, + modelOptionsError, + } = useChatModelOptions(storedModelRefs); const { readiness } = useSpaceLlmReadiness(open ? spaceId : null); const [connectProvider, setConnectProvider] = useState(null); @@ -84,6 +89,7 @@ export function AgentModelSettings({ reasoningDefaults={directives.reasoningDefaults} modelOptions={modelOptions} clerkModelOptions={clerkModelOptions} + decisionModelOptions={decisionModelOptions} clerkReadiness={readiness?.clerk ?? null} conversationSummaries={conversationSummariesEnabled(directives)} onConversationSummariesChange={(next) => {