Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion client/src/components/cos/tabs/AgentCard.jsx
Original file line number Diff line number Diff line change
Expand Up @@ -521,7 +521,7 @@ export default function AgentCard({ agent, onPause, onKill, onDelete, onResume,
)}
{agent.metadata?.model && (
<span className={`px-2 py-0.5 text-xs rounded shrink-0 ${
agent.metadata.modelTier === 'heavy' ? 'bg-purple-500/20 text-purple-400' :
['heavy', 'ultra'].includes(agent.metadata.modelTier) ? 'bg-purple-500/20 text-purple-400' :
agent.metadata.modelTier === 'light' ? 'bg-green-500/20 text-green-400' :
'bg-blue-500/20 text-blue-400'
}`} title={agent.metadata.modelReason}>
Expand Down
2 changes: 1 addition & 1 deletion client/src/components/cos/tabs/LearningTab.jsx
Original file line number Diff line number Diff line change
Expand Up @@ -797,7 +797,7 @@ export default function LearningTab() {
<div className="grid grid-cols-1 sm:grid-cols-3 gap-3">
{routing.tierOverview.map((tier, idx) => {
const tierLabels = {
'light': 'Haiku', 'medium': 'Sonnet', 'heavy': 'Opus',
'light': 'Light', 'medium': 'Medium', 'heavy': 'Heavy', 'ultra': 'Ultra',
'default': 'Default', 'user-specified': 'User'
};
return (
Expand Down
3 changes: 2 additions & 1 deletion client/src/components/providers/ProviderCard.jsx
Original file line number Diff line number Diff line change
Expand Up @@ -508,12 +508,13 @@ export default function ProviderCard({
</p>
);
})()}
{(provider.lightModel || provider.mediumModel || provider.heavyModel) && (
{(provider.lightModel || provider.mediumModel || provider.heavyModel || provider.ultraModel) && (
<p className="text-xs">
Tiers:
{provider.lightModel && <span className="ml-1 text-port-success">{provider.lightModel}</span>}
{provider.mediumModel && <span className="ml-1 text-port-warning">{provider.mediumModel}</span>}
{provider.heavyModel && <span className="ml-1 text-port-error">{provider.heavyModel}</span>}
{provider.ultraModel && <span className="ml-1 text-purple-400">Ultra: {provider.ultraModel}</span>}
</p>
)}
{provider.headlessArgs?.length > 0 && (
Expand Down
30 changes: 27 additions & 3 deletions client/src/components/providers/ProviderForm.jsx
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ export default function ProviderForm({ provider, onClose, onSave, onEditProvider
lightModel: provider?.lightModel || '',
mediumModel: provider?.mediumModel || '',
heavyModel: provider?.heavyModel || '',
ultraModel: provider?.ultraModel || '',
fallbackProvider: provider?.fallbackProvider || '',
fallbackModel: provider?.fallbackModel || '',
numCtx: provider?.numCtx ?? '',
Expand Down Expand Up @@ -131,6 +132,7 @@ export default function ProviderForm({ provider, onClose, onSave, onEditProvider
formData.lightModel,
formData.mediumModel,
formData.heavyModel,
formData.ultraModel,
].filter((model) => model
&& !isEmbeddingModel(model)
&& !availableModels.includes(model)
Expand Down Expand Up @@ -327,7 +329,7 @@ export default function ProviderForm({ provider, onClose, onSave, onEditProvider
// still spread into `data` and silently persisted on an unrelated edit.
// Clear any embedding value that slipped through so the saved record matches
// what the picker allows.
for (const field of ['defaultModel', 'lightModel', 'mediumModel', 'heavyModel', 'fallbackModel']) {
for (const field of ['defaultModel', 'lightModel', 'mediumModel', 'heavyModel', 'ultraModel', 'fallbackModel']) {
if (isEmbeddingModel(data[field])) data[field] = '';
}
// Effort is meaningful only for providers/models that expose an effort
Expand Down Expand Up @@ -749,7 +751,7 @@ export default function ProviderForm({ provider, onClose, onSave, onEditProvider
{/* Model Tiers */}
<div className="border-t border-port-border pt-4 mt-4">
<h4 className="text-sm font-medium text-gray-300 mb-3">Model Tiers</h4>
<div className="grid grid-cols-1 sm:grid-cols-3 gap-3">
<div className="grid grid-cols-1 sm:grid-cols-2 gap-3">
<FormField labelClassName="block text-xs text-gray-400 mb-1" label={<>
<span className="inline-block w-2 h-2 rounded-full bg-port-success mr-1"></span>
Light (fast)
Expand Down Expand Up @@ -816,10 +818,32 @@ export default function ProviderForm({ provider, onClose, onSave, onEditProvider
/>
)}
</FormField>
<FormField labelClassName="block text-xs text-gray-400 mb-1" label={<>
<span className="inline-block w-2 h-2 rounded-full bg-port-error mr-1"></span>
Ultra (frontier)
</>}>
{availableModels.length > 0 ? (
<select
value={formData.ultraModel}
onChange={(e) => setFormData(prev => ({ ...prev, ultraModel: e.target.value }))}
className="w-full px-2 py-1.5 bg-port-bg border border-port-border rounded-lg text-white text-sm focus:border-port-accent focus:outline-hidden"
>
{modelSelectOptions}
</select>
) : (
<input
type="text"
value={formData.ultraModel}
onChange={(e) => setFormData(prev => ({ ...prev, ultraModel: e.target.value }))}
placeholder="Fable or Astra model ID"
className="w-full px-2 py-1.5 bg-port-bg border border-port-border rounded-lg text-white text-sm focus:border-port-accent focus:outline-hidden"
/>
)}
</FormField>
</div>
<p className="text-xs text-gray-500 mt-2">
{availableModels.length > 0
? 'Used for intelligent model selection based on task requirements'
? 'Capability mappings for tasks and prompt stages. Ultra is explicit opt-in and falls back to Heavy when unset.'
: 'Save provider, then use Test or Refresh to fetch available models'}
</p>
</div>
Expand Down
11 changes: 11 additions & 0 deletions client/src/components/providers/ProviderForm.test.jsx
Original file line number Diff line number Diff line change
Expand Up @@ -81,4 +81,15 @@ describe('ProviderForm', () => {
expect(api.createProvider).not.toHaveBeenCalled();
expect(screen.getByLabelText('Planning Window')).toBeInTheDocument();
});
it('saves an Ultra mapping after switching away from the Models tab', async () => {
renderForm();
fireEvent.change(screen.getByLabelText('Name *'), { target: { value: 'Example Provider' } });
fireEvent.change(screen.getByLabelText('Command *'), { target: { value: 'example-cli' } });
switchTab('Models');
fireEvent.change(screen.getByLabelText('Ultra (frontier)'), { target: { value: 'frontier-model' } });
switchTab('Connection');
fireEvent.click(screen.getByRole('button', { name: 'Create' }));
await waitFor(() => expect(api.createProvider).toHaveBeenCalledWith(expect.objectContaining({ ultraModel: 'frontier-model' })));
});

});
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,7 @@ export default function StagePromptModelPicker({ stageName, label = 'Stage LLM',
<option value="quick">Quick — provider's light/fast model</option>
<option value="coding">Coding — provider's medium model</option>
<option value="heavy">Heavy — provider's heavy model</option>
<option value="ultra">Ultra — provider's frontier model</option>
</select>
) : (
<ProviderModelSelector
Expand Down
2 changes: 2 additions & 0 deletions client/src/pages/PromptManager.jsx
Original file line number Diff line number Diff line change
Expand Up @@ -906,6 +906,7 @@ export default function PromptManager() {
<option value="quick">Quick</option>
<option value="coding">Coding</option>
<option value="heavy">Heavy</option>
<option value="ultra">Ultra</option>
</select>
) : (
<ProviderModelSelector
Expand Down Expand Up @@ -1357,6 +1358,7 @@ export default function PromptManager() {
<option value="quick">Quick</option>
<option value="coding">Coding</option>
<option value="heavy">Heavy</option>
<option value="ultra">Ultra</option>
</select>
) : (
<ProviderModelSelector
Expand Down
8 changes: 6 additions & 2 deletions data.reference/providers.json
Original file line number Diff line number Diff line change
Expand Up @@ -38,11 +38,12 @@
"type": "cli",
"command": "claude",
"args": ["--print"],
"models": ["claude-haiku-4-5", "claude-sonnet-5", "claude-opus-5"],
"models": ["claude-fable-5-1", "claude-haiku-4-5", "claude-sonnet-5", "claude-opus-5"],
"defaultModel": "claude-opus-5",
"lightModel": "claude-haiku-4-5",
"mediumModel": "claude-sonnet-5",
"heavyModel": "claude-opus-5",
"ultraModel": "claude-fable-5-1",
"timeout": 900000,
"enabled": true,
"envVars": {}
Expand Down Expand Up @@ -99,6 +100,7 @@
"lightModel": "gpt-5.6-luna",
"mediumModel": "gpt-5.6-terra",
"heavyModel": "gpt-5.6-sol",
"ultraModel": "gpt-6-astra",
"timeout": 300000,
"enabled": true,
"textTransport": "codex-app-server",
Expand Down Expand Up @@ -131,6 +133,7 @@
"lightModel": "gpt-5.6-luna",
"mediumModel": "gpt-5.6-terra",
"heavyModel": "gpt-5.6-sol",
"ultraModel": "gpt-6-astra",
"timeout": 600000,
"enabled": false,
"textTransport": "codex-app-server",
Expand Down Expand Up @@ -178,11 +181,12 @@
"type": "tui",
"command": "claude",
"args": ["--dangerously-skip-permissions"],
"models": ["claude-haiku-4-5", "claude-sonnet-5", "claude-opus-5"],
"models": ["claude-fable-5-1", "claude-haiku-4-5", "claude-sonnet-5", "claude-opus-5"],
"defaultModel": "claude-opus-5",
"lightModel": "claude-haiku-4-5",
"mediumModel": "claude-sonnet-5",
"heavyModel": "claude-opus-5",
"ultraModel": "claude-fable-5-1",
"timeout": 900000,
"enabled": true,
"envVars": {},
Expand Down
33 changes: 33 additions & 0 deletions docs/MODEL_TIERS.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
# AI provider model tiers

Tiers are active routing configuration, not display-only labels. Each provider maps
Light (mechanical work), Medium (routine work), Heavy (complex work), and Ultra
(exceptional frontier reasoning) to its own model. Configure them in Models →
Providers. Astra and Fable are examples of Ultra choices; use a model supported
by the selected provider and account.

CoS selects Light/Heavy from task heuristics and can learn tier preferences from
outcomes. Explicit thinking levels have their own routing precedence. Prompt
stages resolve Quick to Light, Coding to Medium, and Heavy to Heavy; they also
accept the canonical Light, Medium, and Ultra names. Prompt Manager exposes Ultra.
Dispatch labels communicate capability to planning/claim agents; they are guidance,
not permission to enable providers or spend on a new service.

Use `model:ultra` on an exceptional tracker task. In CoS task metadata, use
`model: "ultra"`; orchestration profiles accept the same tier names in each role's
`model` field. For example, an explicitly configured architect can request Ultra,
with a Medium implementer and Heavy reviewer. Exact model IDs still work and
remain appropriate for model-specific evaluations or compatibility requirements.
The tier resolves on the selected provider, including provider fallback.

Model capability and reasoning effort are independent: Ultra does not imply
maximum effort. Existing task heuristics, thinking levels, learning escalation,
provider defaults, and scheduled jobs do not automatically upgrade to Ultra.
An unset Ultra mapping falls back to Heavy, then the provider default. Existing
installs receive an optional Ultra field; migration offers Fable additively on standard Claude catalogs, selects Astra/Fable
when advertised by the provider, and preserves all explicit Ultra pins.
No migration calls a provider or starts AI work.

Prefer role/stage tier assignments for portable workflows; keep exact pins for
intentional exceptions. Avoid bulk replacing installed stage pins or changing
scheduled tasks without the user's instruction.
35 changes: 35 additions & 0 deletions scripts/migrations/357-provider-ultra-tier.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
/** Add the optional frontier tier without changing existing routing or custom pins. */
import { join } from 'node:path';
import { atomicWrite, readJSONFileStrict } from '../../server/lib/fileUtils.js';

import { makeAdditiveProviderInsertMigration } from './_lib.js';

const offerFable = makeAdditiveProviderInsertMigration({
label: 'Fable Ultra tier',
targets: ['claude-code', 'claude-code-tui'].map(id => ({
id, retired: 'claude-opus-5', current: 'claude-fable-5-1',
})),
});

export default {
async up({ rootDir }) {
await offerFable.up({ rootDir });
const path = join(rootDir, 'data/providers.json');
const { ok, value: data } = await readJSONFileStrict(path, null);
if (!ok || !data?.providers) return { success: true, skipped: 'no readable providers' };
let updated = 0;
for (const provider of Object.values(data.providers)) {
if (!provider || typeof provider !== 'object') continue;
if (Object.hasOwn(provider, 'ultraModel')) continue;
// Only select a model this install already advertises. Custom catalogs
// and intentionally empty pins stay under the user's control.
const models = Array.isArray(provider.models) ? provider.models : [];
const candidates = provider.command === 'codex' ? ['gpt-6-astra']
: provider.command === 'claude' ? ['claude-fable-5-1', 'claude-fable-5', 'fable'] : [];
provider.ultraModel = candidates.find(model => models.includes(model)) || null;
updated++;
}
if (updated) await atomicWrite(path, data);
return { success: true, updated };
},
};
34 changes: 34 additions & 0 deletions scripts/migrations/357-provider-ultra-tier.test.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
import { it, expect } from 'vitest';
import { mkdtemp, mkdir, writeFile, readFile, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import migration from './357-provider-ultra-tier.js';

it('adds supported Ultra mappings, preserves explicit pins and is idempotent', async () => {
const rootDir = await mkdtemp(join(tmpdir(), 'ultra-tier-'));
await mkdir(join(rootDir, 'data'));
const path = join(rootDir, 'data/providers.json');
const providers = {
'claude-code': { command: 'claude', models: ['claude-opus-5'], defaultModel: 'claude-opus-5' },
invalid: null,
codex: { command: 'codex', models: ['gpt-6-astra'], defaultModel: 'old-default' },
claude: { command: 'claude', models: ['claude-fable-5-1'] },
legacy: { command: 'claude', models: ['opus'], heavyModel: 'opus' },
custom: { ultraModel: 'custom-model' },
empty: { ultraModel: null },
};
await writeFile(path, JSON.stringify({ activeProvider: 'legacy', providers }));
await migration.up({ rootDir });
const once = await readFile(path, 'utf8');
await migration.up({ rootDir });
expect(await readFile(path, 'utf8')).toBe(once);
const saved = JSON.parse(once);
expect(saved.providers['claude-code']).toEqual({ command: 'claude', models: ['claude-opus-5', 'claude-fable-5-1'], defaultModel: 'claude-opus-5', ultraModel: 'claude-fable-5-1' });
expect(saved.activeProvider).toBe('legacy');
expect(saved.providers.codex).toMatchObject({ ultraModel: 'gpt-6-astra', defaultModel: 'old-default' });
expect(saved.providers.claude.ultraModel).toBe('claude-fable-5-1');
expect(saved.providers.legacy).toEqual({ ...providers.legacy, ultraModel: null });
expect(saved.providers.custom).toEqual(providers.custom);
expect(saved.providers.empty).toEqual(providers.empty);
await rm(rootDir, { recursive: true });
});
12 changes: 11 additions & 1 deletion server/lib/aiToolkit/constants.js
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,8 @@ export const PROVIDER_TYPES = Object.freeze({
export const MODEL_TIERS = {
LIGHT: 'light',
MEDIUM: 'medium',
HEAVY: 'heavy'
HEAVY: 'heavy',
ULTRA: 'ultra'
};

export const RUN_TYPES = {
Expand Down Expand Up @@ -53,3 +54,12 @@ export const PROVIDER_STATUS_REASONS = {

export const DEFAULT_USAGE_LIMIT_WAIT = 24 * 60 * 60 * 1000;
export const DEFAULT_RATE_LIMIT_WAIT = 5 * 60 * 1000;

/** Resolve a capability request against one provider, preserving legacy defaults. */
export function resolveProviderModelTier(provider, tier) {
if (!Object.values(MODEL_TIERS).includes(tier)) return null;
return provider[`${tier}Model`]
|| (tier === 'ultra' ? provider.heavyModel : null)
|| provider.defaultModel
|| null;
}
8 changes: 6 additions & 2 deletions server/lib/aiToolkit/defaults/providers.sample.json
Original file line number Diff line number Diff line change
Expand Up @@ -38,11 +38,12 @@
"type": "cli",
"command": "claude",
"args": ["--print"],
"models": ["claude-haiku-4-5", "claude-sonnet-5", "claude-opus-5"],
"models": ["claude-fable-5-1", "claude-haiku-4-5", "claude-sonnet-5", "claude-opus-5"],
"defaultModel": "claude-opus-5",
"lightModel": "claude-haiku-4-5",
"mediumModel": "claude-sonnet-5",
"heavyModel": "claude-opus-5",
"ultraModel": "claude-fable-5-1",
"timeout": 300000,
"enabled": true,
"envVars": {},
Expand Down Expand Up @@ -535,6 +536,7 @@
"lightModel": "gpt-5.6-luna",
"mediumModel": "gpt-5.6-terra",
"heavyModel": "gpt-5.6-sol",
"ultraModel": "gpt-6-astra",
"contextWindow": 1000000,
"timeout": 300000,
"enabled": true,
Expand All @@ -554,6 +556,7 @@
"lightModel": "gpt-5.6-luna",
"mediumModel": "gpt-5.6-terra",
"heavyModel": "gpt-5.6-sol",
"ultraModel": "gpt-6-astra",
"contextWindow": 1000000,
"timeout": 600000,
"enabled": false,
Expand Down Expand Up @@ -603,11 +606,12 @@
"type": "tui",
"command": "claude",
"args": ["--dangerously-skip-permissions"],
"models": ["claude-haiku-4-5", "claude-sonnet-5", "claude-opus-5"],
"models": ["claude-fable-5-1", "claude-haiku-4-5", "claude-sonnet-5", "claude-opus-5"],
"defaultModel": "claude-opus-5",
"lightModel": "claude-haiku-4-5",
"mediumModel": "claude-sonnet-5",
"heavyModel": "claude-opus-5",
"ultraModel": "claude-fable-5-1",
"timeout": 600000,
"enabled": true,
"envVars": {},
Expand Down
4 changes: 2 additions & 2 deletions server/lib/aiToolkit/defaults/providersSeedParity.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -31,13 +31,13 @@ const __dirname = dirname(fileURLToPath(import.meta.url));
const REFERENCE_PATH = resolve(__dirname, '../../../../data.reference/providers.json');
const SAMPLE_PATH = resolve(__dirname, 'providers.sample.json');

const MODEL_FIELDS = ['models', 'defaultModel', 'lightModel', 'mediumModel', 'heavyModel'];
const MODEL_FIELDS = ['models', 'defaultModel', 'lightModel', 'mediumModel', 'heavyModel', 'ultraModel'];

// `lmstudio` is the one documented divergence: PortOS's seed names a concrete
// local model it ships guidance for, while the toolkit sample ships an empty
// list because a generic install has no way to know what the user has pulled.
const EXEMPT_IDS = new Set(['lmstudio']);
const MODEL_PIN_FIELDS = ['defaultModel', 'lightModel', 'mediumModel', 'heavyModel'];
const MODEL_PIN_FIELDS = ['defaultModel', 'lightModel', 'mediumModel', 'heavyModel', 'ultraModel'];
const STATIC_CLI_PROVIDER_SENTINELS = new Map([
// Codex's sentinel-only legacy records migrate to real defaults; its fresh
// seeds intentionally point at selectable models instead.
Expand Down
1 change: 1 addition & 0 deletions server/lib/aiToolkit/providers.js
Original file line number Diff line number Diff line change
Expand Up @@ -652,6 +652,7 @@ export function createProviderService(config = {}) {
lightModel: providerData.lightModel || null,
mediumModel: providerData.mediumModel || null,
heavyModel: providerData.heavyModel || null,
ultraModel: providerData.ultraModel || null,
fallbackProvider: providerData.fallbackProvider || null,
fallbackModel: providerData.fallbackModel || null,
numCtx: providerData.numCtx || null,
Expand Down
1 change: 1 addition & 0 deletions server/lib/aiToolkit/providers.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,7 @@ describe('Provider Service', () => {
lightModel: 'model-b',
mediumModel: 'model-a',
heavyModel: 'model-c',
ultraModel: 'model-c',
fallbackProvider: 'fallback-provider-id',
fallbackModel: 'fallback-model-id',
numCtx: 32768,
Expand Down
1 change: 1 addition & 0 deletions server/lib/aiToolkit/validation.js
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,7 @@ export const providerSchema = z.object({
lightModel: z.string().nullable().optional(),
mediumModel: z.string().nullable().optional(),
heavyModel: z.string().nullable().optional(),
ultraModel: z.string().nullable().optional(),
fallbackProvider: z.string().nullable().optional(),
// Model to run on the fallback provider. The UI sends '' when no model is
// pinned (fall back to the fallback provider's own default), so allow empty.
Expand Down
Loading