From 9fecf5c61be3de105a009ab4afea81bcbae1f126 Mon Sep 17 00:00:00 2001 From: UnloopedMido Date: Wed, 27 May 2026 00:54:53 +0300 Subject: [PATCH] feat: per-tier endpoint routing Route each model tier to its own OpenAI-compatible endpoint via SMALLCODE_BASE_URL_* env vars or [models.*] sections in smallcode.toml. Requests resolve model + URL per tier; auth follows the selected endpoint. --- .env.example | 14 ++- CHANGELOG.md | 22 ++++ README.md | 32 ++++- README_zh-CN.md | 29 +++++ bin/commands.js | 2 + bin/config.js | 214 ++++++++++++++++++++++++++++------ bin/model_client.js | 22 ++-- bin/smallcode.js | 90 ++++++++------ smallcode.toml | 14 +++ src/model/adaptive_router.js | 16 ++- src/model/router.js | 16 ++- test/config_normalize.test.js | 93 ++++++++++++++- test/model_routing.test.js | 54 +++++++++ test/provider_compat.test.js | 30 ++++- 14 files changed, 554 insertions(+), 94 deletions(-) create mode 100644 test/model_routing.test.js diff --git a/.env.example b/.env.example index 1c867281..79b51d8a 100644 --- a/.env.example +++ b/.env.example @@ -54,13 +54,23 @@ SMALLCODE_CLASSIC_TUI=false # ─── OpenRouter Example ────────────────────────────────────────────────────── # SMALLCODE_BASE_URL=https://openrouter.ai/api/v1 # SMALLCODE_MODEL=openai/gpt-4o-mini -# OPENAI_API_KEY=sk-or-v1-... +# OPENROUTER_API_KEY=sk-or-v1-... # ─── Multi-Model Routing (optional) ────────────────────────────────────────── -# Auto-pick model based on task complexity +# Auto-pick model based on task complexity. Each tier may use its own endpoint. # SMALLCODE_MODEL_FAST=gemma-4-e4b # SMALLCODE_MODEL_DEFAULT=qwen3-8b # SMALLCODE_MODEL_STRONG=qwen3-14b +# SMALLCODE_BASE_URL_FAST=http://localhost:1234/v1 +# SMALLCODE_BASE_URL_DEFAULT=http://localhost:1234/v1 +# SMALLCODE_BASE_URL_STRONG=https://openrouter.ai/api/v1 +# +# Local default + OpenRouter strong model: +# SMALLCODE_MODEL=qwen3:8b +# SMALLCODE_BASE_URL=http://localhost:11434/v1 +# SMALLCODE_MODEL_STRONG=openai/gpt-4o-mini +# SMALLCODE_BASE_URL_STRONG=https://openrouter.ai/api/v1 +# OPENROUTER_API_KEY=sk-or-v1-... # ─── RTK (Rust Token Killer) ───────────────────────────────────────────────── # RTK auto-rewrites bash commands (git, tests, ls, etc.) for 60-90% token savings. diff --git a/CHANGELOG.md b/CHANGELOG.md index 877747b1..6eb0cbf2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,27 @@ # Changelog +## Unreleased + +### feat: per-tier endpoint routing + +`SMALLCODE_MODEL_FAST`, `SMALLCODE_MODEL_DEFAULT`, `SMALLCODE_MODEL_MEDIUM`, +and `SMALLCODE_MODEL_STRONG` can now be paired with matching +`SMALLCODE_BASE_URL_*` variables. `smallcode.toml` also supports +`[models.fast]`, `[models.default]`, `[models.medium]`, and +`[models.strong]` sections with `name` and `baseUrl`, so users can keep +default work on localhost while routing larger tiers to OpenRouter. + +Complexity-based tier selection and adaptive failure-rate routing now resolve +the matching endpoint per request (`activeModelTarget`). Auth headers follow +the selected URL — OpenRouter keys and headers for cloud tiers, no auth for +local endpoints. Primary `[model]` config remains env-first; TOML tier sections +always merge; env tier vars override TOML. + +### Verification + +- 90/90 unit tests pass (`npm test`) — 7 new in `test/config_normalize.test.js`, + `test/model_routing.test.js`, and `test/provider_compat.test.js` + ## [1.2.1] - 2026-05-24 ### fix: provider compatibility — 7 bugs causing 400 errors on cloud/local LLMs diff --git a/README.md b/README.md index f145223a..bc0b92b4 100644 --- a/README.md +++ b/README.md @@ -76,11 +76,40 @@ SMALLCODE_BASE_URL=http://localhost:1234/v1 # Optional: escalation (auto-fallback to cloud on hard fail) # ANTHROPIC_API_KEY=sk-ant-... # OPENAI_API_KEY=sk-... +# OPENROUTER_API_KEY=sk-or-v1-... # DEEPSEEK_API_KEY=sk-... ``` See `.env.example` for all options. Also supports `smallcode.toml` for backwards compatibility. +SmallCode can route each model tier to a different endpoint. This lets you keep +fast/default work local while sending complex tasks to a larger OpenRouter model: + +```bash +SMALLCODE_MODEL=qwen3:8b +SMALLCODE_BASE_URL=http://localhost:11434/v1 + +SMALLCODE_MODEL_STRONG=openai/gpt-4o-mini +SMALLCODE_BASE_URL_STRONG=https://openrouter.ai/api/v1 +OPENROUTER_API_KEY=sk-or-v1-... +``` + +Equivalent `smallcode.toml`: + +```toml +[model] +provider = "openai" +name = "qwen3:8b" +baseUrl = "http://localhost:11434/v1" + +[models.strong] +name = "openai/gpt-4o-mini" +baseUrl = "https://openrouter.ai/api/v1" +``` + +Tier URLs are optional. If `SMALLCODE_BASE_URL_STRONG` or +`[models.strong].baseUrl` is omitted, that tier uses the primary base URL. + ## Architecture SmallCode is built with a modular architecture: @@ -227,12 +256,13 @@ When 3 or more files are edited in a single agent turn, `coordinateMultiFileEdit When `patch` fails because `old_str` no longer exists in the file, `semanticMerge()` asks the model to merge the intended change into the current file content. Returns the complete corrected file. Replaces the hard error with a recovery attempt. TTL 1 min (content-specific). ### Adaptive Model Select (Rank 8) -`AdaptiveModelRouter` in `src/model/adaptive_router.js` tracks per-model call/fail counts. When the primary model's failure rate exceeds 0.3 (medium) or 0.6 (strong), `chatCompletion` automatically overrides `body.model` with `SMALLCODE_MODEL_MEDIUM` or `SMALLCODE_MODEL_STRONG`. Requires at least 3 calls before routing decisions activate. Reset via `router.reset()`. +`AdaptiveModelRouter` in `src/model/adaptive_router.js` tracks per-model call/fail counts. When the primary model's failure rate exceeds 0.3 (medium) or 0.6 (strong), `chatCompletion` automatically routes to `SMALLCODE_MODEL_MEDIUM` or `SMALLCODE_MODEL_STRONG` and their matching `SMALLCODE_BASE_URL_*` endpoint when configured. Requires at least 3 calls before routing decisions activate. Reset via `router.reset()`. ```bash # Optional: configure fallback models for adaptive routing SMALLCODE_MODEL_MEDIUM=qwen2.5-coder:32b SMALLCODE_MODEL_STRONG=gpt-4o +SMALLCODE_BASE_URL_STRONG=https://openrouter.ai/api/v1 ``` ### Benchmark Harness diff --git a/README_zh-CN.md b/README_zh-CN.md index ad314856..375dfea4 100644 --- a/README_zh-CN.md +++ b/README_zh-CN.md @@ -50,11 +50,40 @@ SMALLCODE_BASE_URL=http://localhost:1234/v1 # 可选：模型升级（硬失败时自动回退到云端） # ANTHROPIC_API_KEY=sk-ant-... # OPENAI_API_KEY=sk-... +# OPENROUTER_API_KEY=sk-or-v1-... # DEEPSEEK_API_KEY=sk-... ``` 查看 `.env.example` 了解所有选项。同时支持 `smallcode.toml` 以保持向后兼容。 +SmallCode 可以为每个模型层级配置不同 endpoint。这样可以让 fast/default +任务继续使用本地模型，而复杂任务使用 OpenRouter 上的更大模型： + +```bash +SMALLCODE_MODEL=qwen3:8b +SMALLCODE_BASE_URL=http://localhost:11434/v1 + +SMALLCODE_MODEL_STRONG=openai/gpt-4o-mini +SMALLCODE_BASE_URL_STRONG=https://openrouter.ai/api/v1 +OPENROUTER_API_KEY=sk-or-v1-... +``` + +等价的 `smallcode.toml`： + +```toml +[model] +provider = "openai" +name = "qwen3:8b" +baseUrl = "http://localhost:11434/v1" + +[models.strong] +name = "openai/gpt-4o-mini" +baseUrl = "https://openrouter.ai/api/v1" +``` + +层级 URL 是可选的。如果省略 `SMALLCODE_BASE_URL_STRONG` 或 +`[models.strong].baseUrl`，该层级会使用主 `baseUrl`。 + ## 架构 SmallCode 采用模块化架构： diff --git a/bin/commands.js b/bin/commands.js index bd455770..c7bb235c 100644 --- a/bin/commands.js +++ b/bin/commands.js @@ -60,6 +60,7 @@ module.exports = function createCommandHandler(config, conversationHistory, impr } else { const newModel = parts.slice(1).join(' '); config.model.name = newModel; + delete config.activeModelTarget; console.log(` ${chalk.green('✓')} Switched to ${chalk.cyan(newModel)}`); } console.log(''); @@ -73,6 +74,7 @@ module.exports = function createCommandHandler(config, conversationHistory, impr console.log(chalk.gray(' Switch: /endpoint http://host:port/v1')); } else { config.model.baseUrl = parts[1]; + delete config.activeModelTarget; console.log(` ${chalk.green('✓')} Endpoint: ${chalk.gray(parts[1])}`); } console.log(''); diff --git a/bin/config.js b/bin/config.js index 8b3e7393..f233159e 100644 --- a/bin/config.js +++ b/bin/config.js @@ -42,41 +42,30 @@ function loadConfig(flags = {}) { }, }; - // Multi-model routing (optional) - if (env.SMALLCODE_MODEL_FAST || env.SMALLCODE_MODEL_STRONG) { - config.models = { - fast: env.SMALLCODE_MODEL_FAST || config.model.name, - default: env.SMALLCODE_MODEL_DEFAULT || config.model.name, - strong: env.SMALLCODE_MODEL_STRONG || config.model.name, - }; - } - - // Legacy: still check smallcode.toml / config.toml for backwards compatibility + // smallcode.toml / config.toml for backwards compatibility and tier routing. const tomlPaths = [ path.join(process.cwd(), 'smallcode.toml'), path.join(process.cwd(), '.smallcode', 'config.toml'), path.join(os.homedir(), '.config', 'smallcode', 'config.toml'), ]; for (const tomlPath of tomlPaths) { - if (fs.existsSync(tomlPath) && !config.model.name) { + if (fs.existsSync(tomlPath)) { try { - const content = fs.readFileSync(tomlPath, 'utf-8'); - const lines = content.split('\n'); - for (const line of lines) { - const m = line.match(/^name\s*=\s*"?([^"#]+)"?/); - if (m) config.model.name = m[1].trim(); - const b = line.match(/^(?:baseUrl|base_url)\s*=\s*"?([^"#]+)"?/); - if (b) config.model.baseUrl = b[1].trim(); - const p = line.match(/^provider\s*=\s*"?([^"#]+)"?/); - if (p) config.model.provider = p[1].trim(); - const to = line.match(/^timeout\s*=\s*(\d+)/); - if (to) config.model.timeout = parseInt(to[1]); - } + const toml = parseTomlConfig(fs.readFileSync(tomlPath, 'utf-8')); + // Primary [model] from TOML only when env did not set SMALLCODE_MODEL. + if (!config.model.name) applyTomlPrimaryConfig(config, toml); + // Tier sections are always merged — env tier vars override below. + applyTomlModelTiers(config, toml); break; } catch {} } } + applyEnvModelTier(config, 'fast', 'SMALLCODE_MODEL_FAST', 'SMALLCODE_BASE_URL_FAST'); + applyEnvModelTier(config, 'default', 'SMALLCODE_MODEL_DEFAULT', 'SMALLCODE_BASE_URL_DEFAULT'); + applyEnvModelTier(config, 'medium', 'SMALLCODE_MODEL_MEDIUM', 'SMALLCODE_BASE_URL_MEDIUM'); + applyEnvModelTier(config, 'strong', 'SMALLCODE_MODEL_STRONG', 'SMALLCODE_BASE_URL_STRONG'); + // CLI flags override everything if (flags.model) config.model.name = flags.model; if (flags.provider) config.model.provider = flags.provider; @@ -90,10 +79,159 @@ function loadConfig(flags = {}) { // SMALLCODE_BASE_URL=http://localhost:11434 used to fail because the // OpenAI-compat path tried .../models instead of .../v1/models. config.model.baseUrl = normalizeBaseUrl(config.model.baseUrl); + normalizeModelTiers(config); return config; } +function stripInlineComment(value) { + let quote = null; + for (let i = 0; i < value.length; i++) { + const ch = value[i]; + if ((ch === '"' || ch === "'") && value[i - 1] !== '\\') { + quote = quote === ch ? null : (quote || ch); + } + if (ch === '#' && !quote) return value.slice(0, i).trim(); + } + return value.trim(); +} + +function parseTomlValue(raw) { + const value = stripInlineComment(raw); + if ((value.startsWith('"') && value.endsWith('"')) || (value.startsWith("'") && value.endsWith("'"))) { + return value.slice(1, -1); + } + if (value === 'true') return true; + if (value === 'false') return false; + if (/^-?\d+$/.test(value)) return parseInt(value, 10); + return value; +} + +function parseTomlConfig(content) { + const out = {}; + let section = []; + for (const rawLine of content.split('\n')) { + const line = rawLine.trim(); + if (!line || line.startsWith('#')) continue; + const sectionMatch = line.match(/^\[([^\]]+)\]$/); + if (sectionMatch) { + section = sectionMatch[1].split('.').map(s => s.trim()).filter(Boolean); + continue; + } + const kv = line.match(/^([A-Za-z0-9_-]+)\s*=\s*(.+)$/); + if (!kv) continue; + let cursor = out; + for (const part of section) { + if (!cursor[part] || typeof cursor[part] !== 'object') cursor[part] = {}; + cursor = cursor[part]; + } + cursor[kv[1]] = parseTomlValue(kv[2]); + } + return out; +} + +function coerceModelEntry(value) { + if (!value) return {}; + if (typeof value === 'string') return { name: value }; + return { + name: value.name || value.model || '', + baseUrl: value.baseUrl || value.base_url || '', + provider: value.provider || '', + }; +} + +function ensureModels(config) { + if (!config.models) config.models = {}; + return config.models; +} + +function mergeModelTier(config, tier, value) { + const entry = coerceModelEntry(value); + if (!entry.name && !entry.baseUrl && !entry.provider) return; + const models = ensureModels(config); + const prev = coerceModelEntry(models[tier]); + models[tier] = { + ...prev, + ...(entry.name ? { name: entry.name } : {}), + ...(entry.baseUrl ? { baseUrl: entry.baseUrl } : {}), + ...(entry.provider ? { provider: entry.provider } : {}), + }; +} + +function applyTomlPrimaryConfig(config, toml) { + if (toml.provider) config.model.provider = toml.provider; + if (toml.name) config.model.name = toml.name; + if (toml.baseUrl || toml.base_url) config.model.baseUrl = toml.baseUrl || toml.base_url; + if (toml.timeout) config.model.timeout = parseInt(toml.timeout, 10) || config.model.timeout; + if (toml.model) { + if (toml.model.provider) config.model.provider = toml.model.provider; + if (toml.model.name) config.model.name = toml.model.name; + if (toml.model.baseUrl || toml.model.base_url) config.model.baseUrl = toml.model.baseUrl || toml.model.base_url; + if (toml.model.timeout) config.model.timeout = parseInt(toml.model.timeout, 10) || config.model.timeout; + } +} + +function applyTomlModelTiers(config, toml) { + if (!toml.models) return; + for (const tier of ['fast', 'default', 'medium', 'strong']) { + if (toml.models[tier]) mergeModelTier(config, tier, toml.models[tier]); + } +} + +function applyEnvModelTier(config, tier, modelEnv, urlEnv) { + const entry = {}; + if (process.env[modelEnv]) entry.name = process.env[modelEnv]; + if (process.env[urlEnv]) entry.baseUrl = process.env[urlEnv]; + mergeModelTier(config, tier, entry); +} + +function normalizeModelTiers(config) { + if (!config.models) return; + for (const tier of Object.keys(config.models)) { + const entry = coerceModelEntry(config.models[tier]); + config.models[tier] = { + name: entry.name || config.model.name, + baseUrl: normalizeBaseUrl(entry.baseUrl || config.model.baseUrl), + provider: entry.provider || config.model.provider, + }; + } +} + +function getModelTarget(config, tier = 'default') { + const entry = config?.models?.[tier] ? coerceModelEntry(config.models[tier]) : {}; + return { + tier, + model: entry.name || config?.model?.name || '', + name: entry.name || config?.model?.name || '', + baseUrl: normalizeBaseUrl(entry.baseUrl || config?.model?.baseUrl || ''), + provider: entry.provider || config?.model?.provider || 'openai', + }; +} + +function getModelTargetForModel(config, modelName, preferredTier = 'default') { + if (config?.models) { + for (const tier of ['fast', 'default', 'medium', 'strong']) { + const entry = coerceModelEntry(config.models[tier]); + if (entry.name && entry.name === modelName) return getModelTarget(config, tier); + } + } + const fallback = getModelTarget(config, preferredTier); + return { ...fallback, model: modelName || fallback.model, name: modelName || fallback.name }; +} + +function withModelTarget(config, target) { + return { + ...config, + model: { + ...config.model, + name: target.model || target.name || config.model.name, + baseUrl: target.baseUrl || config.model.baseUrl, + provider: target.provider || config.model.provider, + }, + activeModelTarget: target, + }; +} + /** * Auto-append `/v1` to known OpenAI-compatible endpoints when the user * didn't include it. Strips trailing slashes. No-op for URLs that already @@ -141,11 +279,7 @@ async function checkEndpoint(config) { // OpenAI-compatible endpoint (LM Studio, vLLM, OpenRouter, etc.) if (config.model.provider === 'openai' || baseUrl.includes('/v1')) { try { - const headers = {}; - const apiKey = process.env.OPENAI_API_KEY || process.env.ANTHROPIC_API_KEY || process.env.DEEPSEEK_API_KEY || config.model.apiKey; - if (apiKey) { - headers['Authorization'] = `Bearer ${apiKey}`; - } + const headers = buildAuthHeaders(config); const response = await fetch(`${baseUrl}/models`, { headers }); if (!response.ok) { console.log(` ⚠ Cannot reach endpoint at ${baseUrl}`); @@ -209,25 +343,26 @@ async function checkEndpoint(config) { */ function buildAuthHeaders(config) { const headers = { 'Content-Type': 'application/json' }; - const baseUrl = (config.model.baseUrl || '').toLowerCase(); + const modelConfig = config.model || config; + const baseUrl = (modelConfig.baseUrl || '').toLowerCase(); // Route key selection based on the target endpoint URL. let apiKey = null; if (baseUrl.includes('api.deepseek.com')) { - apiKey = process.env.DEEPSEEK_API_KEY || process.env.OPENAI_API_KEY || config.model.apiKey; + apiKey = process.env.DEEPSEEK_API_KEY || process.env.OPENAI_API_KEY || modelConfig.apiKey; } else if (baseUrl.includes('api.openai.com')) { - apiKey = process.env.OPENAI_API_KEY || config.model.apiKey; + apiKey = process.env.OPENAI_API_KEY || modelConfig.apiKey; } else if (baseUrl.includes('openrouter.ai')) { - apiKey = process.env.OPENROUTER_API_KEY || process.env.OPENAI_API_KEY || config.model.apiKey; + apiKey = process.env.OPENROUTER_API_KEY || process.env.OPENAI_API_KEY || modelConfig.apiKey; } else if (baseUrl.includes('anthropic.com')) { // Anthropic uses x-api-key, not Bearer — but if someone routes through // an OpenAI-compat proxy, Bearer still works. We use ANTHROPIC_API_KEY // first, then fall back to the generic key. - apiKey = process.env.ANTHROPIC_API_KEY || config.model.apiKey; + apiKey = process.env.ANTHROPIC_API_KEY || modelConfig.apiKey; } else { // Local server or unknown cloud — fall back to any available key. // SMALLCODE_API_KEY is the explicit "my endpoint needs this key" option. - apiKey = process.env.SMALLCODE_API_KEY || process.env.OPENAI_API_KEY || process.env.ANTHROPIC_API_KEY || process.env.DEEPSEEK_API_KEY || config.model.apiKey; + apiKey = process.env.SMALLCODE_API_KEY || process.env.OPENAI_API_KEY || process.env.ANTHROPIC_API_KEY || process.env.DEEPSEEK_API_KEY || modelConfig.apiKey; } if (apiKey) { @@ -240,4 +375,13 @@ function buildAuthHeaders(config) { return headers; } -module.exports = { loadConfig, checkEndpoint, buildAuthHeaders, normalizeBaseUrl }; +module.exports = { + loadConfig, + checkEndpoint, + buildAuthHeaders, + normalizeBaseUrl, + parseTomlConfig, + getModelTarget, + getModelTargetForModel, + withModelTarget, +}; diff --git a/bin/model_client.js b/bin/model_client.js index 5dafb4b1..179bee76 100644 --- a/bin/model_client.js +++ b/bin/model_client.js @@ -7,7 +7,7 @@ const path = require('path'); const fs = require('fs'); -const { buildAuthHeaders } = require('./config'); +const { buildAuthHeaders, getModelTarget, withModelTarget } = require('./config'); const { redactString } = require('../src/security/sanitize'); /** @@ -16,7 +16,9 @@ const { redactString } = require('../src/security/sanitize'); */ async function chatCompletion(ctx) { const { config, conversationHistory, tokenTracker, sessionStore } = ctx; - const baseUrl = config.model.baseUrl; + const target = config.activeModelTarget || getModelTarget(config, 'default'); + const requestConfig = withModelTarget(config, target); + const baseUrl = target.baseUrl; const systemMsg = { role: 'system', @@ -28,13 +30,13 @@ async function chatCompletion(ctx) { const processedMessages = conversationHistory.map(msg => { if (msg.role !== 'user' || typeof msg.content !== 'string') return msg; const images = extractImages(msg.content, process.cwd()); - if (images.length === 0 || !modelSupportsVision(config.model.name)) return msg; + if (images.length === 0 || !modelSupportsVision(target.model)) return msg; return { ...msg, content: [{ type: 'text', text: msg.content }, ...formatImagesForAPI(images)] }; }); const _tools = ctx.getAllTools(config); const body = { - model: config.model.name, + model: target.model, messages: [systemMsg, ...processedMessages], temperature: 0.1, max_tokens: 4096, @@ -45,7 +47,7 @@ async function chatCompletion(ctx) { body.tools = _tools; } - const headers = buildAuthHeaders(config); + const headers = buildAuthHeaders(requestConfig); const response = await fetch(`${baseUrl}/chat/completions`, { method: 'POST', @@ -72,7 +74,7 @@ async function chatCompletion(ctx) { const data = await response.json(); if (tokenTracker && data?.usage) { - tokenTracker.record(data, config.model.name); + tokenTracker.record(data, target.model); } if (sessionStore) { sessionStore.save(conversationHistory, { tokens: tokenTracker ? tokenTracker.stats() : undefined }); @@ -91,19 +93,21 @@ async function chatCompletion(ctx) { */ async function streamFinalResponse(ctx) { const { config, earlyStop, _fullscreenRef } = ctx; - const baseUrl = config.model.baseUrl; + const target = config.activeModelTarget || getModelTarget(config, 'default'); + const requestConfig = withModelTarget(config, target); + const baseUrl = target.baseUrl; const systemMsg = { role: 'system', content: 'You are SmallCode, a coding assistant. Summarize what you just did in 1-2 sentences. Be concise.' }; try { - const headers = buildAuthHeaders(config); + const headers = buildAuthHeaders(requestConfig); const messages = ctx.conversationHistory; const response = await fetch(`${baseUrl}/chat/completions`, { method: 'POST', headers, body: JSON.stringify({ - model: config.model.name, + model: target.model, messages: [systemMsg, ...messages.slice(-6)], stream: true, temperature: 0.1, diff --git a/bin/smallcode.js b/bin/smallcode.js index 47135ad5..51dd5e69 100755 --- a/bin/smallcode.js +++ b/bin/smallcode.js @@ -45,7 +45,14 @@ const readline = require('readline'); const os = require('os'); const tui = require('./tui'); const chalk = tui.chalk; -const { loadConfig: loadConfigModule, checkEndpoint, buildAuthHeaders } = require('./config'); +const { + loadConfig: loadConfigModule, + checkEndpoint, + buildAuthHeaders, + getModelTarget, + getModelTargetForModel, + withModelTarget, +} = require('./config'); const { TOOLS, COMPOUND_TOOLS, getAllTools: _getAllToolsModule } = require('./tools'); const { runValidation: _runValidationModule } = require('./model_client'); const { mcpCall, initCodeGraph, killMCP, getMcpProcess } = require('./mcp_bridge'); @@ -83,7 +90,7 @@ const { resolveReferences, formatReferencesForPrompt } = require('../src/session const { TokenTracker } = require('../src/session/tokens'); const { UndoStack } = require('../src/session/undo'); const { shouldInjectGitContext, getGitDiffContext } = require('../src/session/git_context'); -const { routeModel } = require('../src/model/router'); +const { routeTier } = require('../src/model/router'); // Initialize structured memory (budget-aware-mcp's SQLite + FTS5 store, falls back to JSON) let memoryStore; @@ -754,9 +761,9 @@ async function runAgentLoop(userMessage, config) { // Multi-model routing: pick model based on task complexity (if configured) // Phase C: Marrowscript-compiled coding_router for tier-based dispatch. - // Falls back to hand-rolled routeModel() if compiled router unavailable. + // Falls back to hand-rolled routeTier() if compiled router unavailable. if (config.models || process.env.SMALLCODE_USE_TIER_ROUTING === 'true') { - let selectedModel = null; + let selectedTarget = null; let selectedTier = null; try { const { routeToTier, estimateComplexity, isCompiledCognitionAvailable } = require('./cognition_adapter'); @@ -764,12 +771,11 @@ async function runAgentLoop(userMessage, config) { const complexity = estimateComplexity(userMessage); const route = routeToTier(complexity); if (route) { - // Map model_id back to actual model name from config.models if set, - // otherwise use the SMALLCODE_MODEL env var (already configured per tier). + // Map tier back to a model target when config.models is set. if (config.models) { - if (route.tier === 'trivial') selectedModel = config.models.fast; - else if (route.tier === 'simple') selectedModel = config.models.default; - else selectedModel = config.models.strong; + if (route.tier === 'trivial') selectedTarget = getModelTarget(config, 'fast'); + else if (route.tier === 'simple') selectedTarget = getModelTarget(config, 'default'); + else selectedTarget = getModelTarget(config, 'strong'); } selectedTier = route.tier; } @@ -777,13 +783,17 @@ async function runAgentLoop(userMessage, config) { } catch {} // Fallback: hand-rolled routeModel - if (!selectedModel && config.models) { - selectedModel = routeModel(userMessage, config); + if (!selectedTarget && config.models) { + const tier = routeTier(userMessage); + selectedTarget = getModelTarget(config, tier); + selectedTier = tier; } - if (selectedModel && selectedModel !== config.model.name) { - config.model.name = selectedModel; - if (_fullscreenRef) _fullscreenRef.addTool('router', 'ok', `→ ${selectedModel}${selectedTier ? ' (' + selectedTier + ')' : ''}`); + if (selectedTarget && selectedTarget.model) { + config.activeModelTarget = selectedTarget; + if (_fullscreenRef && selectedTarget.model !== config.model.name) { + _fullscreenRef.addTool('router', 'ok', `→ ${selectedTarget.model}${selectedTier ? ' (' + selectedTier + ')' : ''}`); + } } } @@ -1989,7 +1999,9 @@ function getPluginPrompts() { // Make a chat completion request (non-streaming for tool use, streaming for final response) async function chatCompletion(config, messages) { - const baseUrl = config.model.baseUrl; + let target = config.activeModelTarget || getModelTarget(config, 'default'); + let requestConfig = withModelTarget(config, target); + let baseUrl = target.baseUrl; const systemMsg = { role: 'system', content: buildCompactSystemPrompt(currentTaskType, messages), @@ -2051,7 +2063,7 @@ async function chatCompletion(config, messages) { // Only extract images from the most recent user message if (idx !== lastUserIdx) return msg; const images = extractImages(msg.content, process.cwd()); - if (images.length === 0 || !modelSupportsVision(config.model.name)) return msg; + if (images.length === 0 || !modelSupportsVision(target.model)) return msg; return { ...msg, content: [ @@ -2063,7 +2075,7 @@ async function chatCompletion(config, messages) { const _tools = getAllTools(config, currentToolCategory); const body = { - model: config.model.name, + model: target.model, messages: [systemMsg, ...processedWithImages], temperature: 0.1, max_tokens: parseInt(process.env.SMALLCODE_MAX_OUTPUT_TOKENS) || 8192, @@ -2086,10 +2098,19 @@ async function chatCompletion(config, messages) { try { const { getAdaptiveRouter } = require('../src/model/adaptive_router'); const router = getAdaptiveRouter(); - const selected = router.selectModel(config); + const selected = router.selectModel(requestConfig); if (selected.model && selected.model !== body.model) { if (_fullscreenRef) _fullscreenRef.addTool('adaptive', 'ok', `→ ${selected.model} (high failure rate)`); - body.model = selected.model; + target = { + ...getModelTargetForModel(config, selected.model, selected.tier || target.tier), + tier: selected.tier || target.tier, + model: selected.model, + name: selected.model, + baseUrl: selected.url || target.baseUrl, + }; + requestConfig = withModelTarget(config, target); + baseUrl = target.baseUrl; + body.model = target.model; } } catch {} @@ -2134,12 +2155,7 @@ async function chatCompletion(config, messages) { } catch {} // Build headers — use provider-aware key routing from config.js - const headers = buildAuthHeaders(config); - // OpenRouter requires HTTP-Referer and X-Title headers - if (baseUrl.includes('openrouter.ai')) { - headers['HTTP-Referer'] = 'https://github.com/Doorman11991/smallcode'; - headers['X-Title'] = 'SmallCode'; - } + const headers = buildAuthHeaders(requestConfig); // Timeout: abort if model doesn't respond within configured limit. // Default 300s (5 min) — enough for slow hardware (RK3588, CPU inference). @@ -2247,7 +2263,7 @@ async function chatCompletion(config, messages) { // Track token usage if (tokenTracker && data?.usage) { - tokenTracker.record(data, config.model.name); + tokenTracker.record(data, body.model || config.model.name); } if (data?.usage) { tokenMonitor.recordCall(data.usage.prompt_tokens, data.usage.completion_tokens); @@ -2281,14 +2297,16 @@ async function chatCompletion(config, messages) { // Stream a final text response (no tools, just text output) async function streamFinalResponse(config, messages) { - const baseUrl = config.model.baseUrl; + const target = config.activeModelTarget || getModelTarget(config, 'default'); + const requestConfig = withModelTarget(config, target); + const baseUrl = target.baseUrl; const systemMsg = { role: 'system', content: `You are SmallCode, a coding assistant. Summarize what you just did in 1-2 sentences. Be concise.` }; try { - const headers = buildAuthHeaders(config); + const headers = buildAuthHeaders(requestConfig); // Fix #3: Only include messages that form valid pairs. Strip tool_call // assistant messages that don't have a following tool result (which causes @@ -2321,7 +2339,7 @@ async function streamFinalResponse(config, messages) { method: 'POST', headers, body: JSON.stringify({ - model: config.model.name, + model: target.model, messages: [systemMsg, ...safeMessages.slice(-6)], stream: true, temperature: 0.1, @@ -2389,7 +2407,9 @@ async function streamFinalResponse(config, messages) { // Streaming version for when no tools are needed (direct responses) async function sendToModel(message, config) { - const baseUrl = config.model.baseUrl || process.env.OLLAMA_HOST || 'http://localhost:11434'; + const target = config.activeModelTarget || getModelTarget(config, 'default'); + const requestConfig = withModelTarget(config, target); + const baseUrl = target.baseUrl || process.env.OLLAMA_HOST || 'http://localhost:11434'; const systemPrompt = `You are SmallCode, a coding assistant. You help users by reading, editing, and creating code files. Rules: - Read files before editing them. @@ -2398,15 +2418,15 @@ Rules: - If a task is complex, break it into steps.`; // OpenAI-compatible (LM Studio, vLLM, etc.) - if (config.model.provider === 'openai' || baseUrl.includes('/v1')) { + if (target.provider === 'openai' || baseUrl.includes('/v1')) { try { - const headers = buildAuthHeaders(config); + const headers = buildAuthHeaders(requestConfig); const response = await fetch(`${baseUrl}/chat/completions`, { method: 'POST', headers, body: JSON.stringify({ - model: config.model.name, + model: target.model, messages: [ { role: 'system', content: systemPrompt }, { role: 'user', content: message }, @@ -2457,12 +2477,12 @@ Rules: // Ollama native endpoint try { - const host = process.env.OLLAMA_HOST || 'http://localhost:11434'; + const host = baseUrl.replace(/\/v1$/, '') || process.env.OLLAMA_HOST || 'http://localhost:11434'; const response = await fetch(`${host}/api/chat`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ - model: config.model.name, + model: target.model, messages: [ { role: 'system', content: systemPrompt }, { role: 'user', content: message }, diff --git a/smallcode.toml b/smallcode.toml index 2bead263..e31471cd 100644 --- a/smallcode.toml +++ b/smallcode.toml @@ -9,6 +9,20 @@ baseUrl = "http://localhost:11434/v1" # tool_format = "hermes" # timeout = 300 # seconds to wait for model response (default: 300 / 5 min) +# Optional tier routing. Each tier may use a different OpenAI-compatible +# endpoint. If a tier omits baseUrl, it falls back to [model].baseUrl. +# [models.fast] +# name = "local-small" +# baseUrl = "http://localhost:11434/v1" +# +# [models.default] +# name = "gpt-oss:20b" +# baseUrl = "http://localhost:11434/v1" +# +# [models.strong] +# name = "openai/gpt-4o-mini" +# baseUrl = "https://openrouter.ai/api/v1" + [context] max_budget_pct = 70 working_memory_tokens = 500 diff --git a/src/model/adaptive_router.js b/src/model/adaptive_router.js index acbbebcb..2aa473a0 100644 --- a/src/model/adaptive_router.js +++ b/src/model/adaptive_router.js @@ -78,16 +78,24 @@ class AdaptiveModelRouter { const rate = entry.fails / entry.calls; if (rate > 0.6) { - const strong = process.env.SMALLCODE_MODEL_STRONG; + const strong = config?.models?.strong?.name || process.env.SMALLCODE_MODEL_STRONG; if (strong && strong !== primaryModel) { - return { model: strong, url: primaryUrl }; + return { + model: strong, + url: config?.models?.strong?.baseUrl || process.env.SMALLCODE_BASE_URL_STRONG || primaryUrl, + tier: 'strong', + }; } } if (rate > 0.3) { - const medium = process.env.SMALLCODE_MODEL_MEDIUM; + const medium = config?.models?.medium?.name || process.env.SMALLCODE_MODEL_MEDIUM; if (medium && medium !== primaryModel) { - return { model: medium, url: primaryUrl }; + return { + model: medium, + url: config?.models?.medium?.baseUrl || process.env.SMALLCODE_BASE_URL_MEDIUM || primaryUrl, + tier: 'medium', + }; } } diff --git a/src/model/router.js b/src/model/router.js index 51e35828..f65cc808 100644 --- a/src/model/router.js +++ b/src/model/router.js @@ -41,23 +41,27 @@ function estimateComplexity(message) { return 'default'; } +function routeTier(message) { + return estimateComplexity(message); +} + /** * Pick the model name based on configured tiers and estimated complexity. */ function routeModel(message, config) { const models = config.models; - if (!models || !models.fast) { + if (!models) { // No multi-model config — use the single configured model return config.model.name; } - const complexity = estimateComplexity(message); + const complexity = routeTier(message); switch (complexity) { - case 'fast': return models.fast || models.default || config.model.name; - case 'strong': return models.strong || models.default || config.model.name; - default: return models.default || config.model.name; + case 'fast': return (models.fast?.name || models.fast) || (models.default?.name || models.default) || config.model.name; + case 'strong': return (models.strong?.name || models.strong) || (models.default?.name || models.default) || config.model.name; + default: return (models.default?.name || models.default) || config.model.name; } } -module.exports = { estimateComplexity, routeModel }; +module.exports = { estimateComplexity, routeTier, routeModel }; diff --git a/test/config_normalize.test.js b/test/config_normalize.test.js index b2e8e484..48ccb533 100644 --- a/test/config_normalize.test.js +++ b/test/config_normalize.test.js @@ -10,8 +10,11 @@ const test = require('node:test'); const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); -const { normalizeBaseUrl } = require('../bin/config'); +const { loadConfig, normalizeBaseUrl } = require('../bin/config'); test('Ollama bare host gets /v1 appended', () => { assert.equal(normalizeBaseUrl('http://localhost:11434'), 'http://localhost:11434/v1'); @@ -63,3 +66,91 @@ test('Empty / falsy input is returned unchanged', () => { test('Malformed URL is returned unchanged (no throw)', () => { assert.equal(normalizeBaseUrl('not-a-url'), 'not-a-url'); }); + +function withTempConfig(toml, fn) { + const prevCwd = process.cwd(); + const prevEnv = { ...process.env }; + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'smallcode-config-')); + fs.writeFileSync(path.join(dir, 'smallcode.toml'), toml); + process.chdir(dir); + for (const key of Object.keys(process.env)) { + if (key.startsWith('SMALLCODE_') || key === 'OLLAMA_HOST') delete process.env[key]; + } + try { + return fn(); + } finally { + process.chdir(prevCwd); + process.env = prevEnv; + fs.rmSync(dir, { recursive: true, force: true }); + } +} + +test('smallcode.toml loads per-tier model targets', () => { + withTempConfig(` +[model] +name = "local-default" +baseUrl = "http://localhost:11434" + +[models.strong] +name = "openrouter/large" +baseUrl = "https://openrouter.ai/api/v1" +`, () => { + const config = loadConfig(); + assert.equal(config.model.name, 'local-default'); + assert.equal(config.model.baseUrl, 'http://localhost:11434/v1'); + assert.equal(config.models.strong.name, 'openrouter/large'); + assert.equal(config.models.strong.baseUrl, 'https://openrouter.ai/api/v1'); + }); +}); + +test('environment tier vars override smallcode.toml tier values', () => { + withTempConfig(` +[model] +name = "local-default" +baseUrl = "http://localhost:11434/v1" + +[models.strong] +name = "toml-strong" +baseUrl = "http://localhost:1234/v1" +`, () => { + process.env.SMALLCODE_MODEL_STRONG = 'env-strong'; + process.env.SMALLCODE_BASE_URL_STRONG = 'https://openrouter.ai/api/v1'; + const config = loadConfig(); + assert.equal(config.models.strong.name, 'env-strong'); + assert.equal(config.models.strong.baseUrl, 'https://openrouter.ai/api/v1'); + }); +}); + +test('missing tier URL falls back to primary base URL', () => { + withTempConfig(` +[model] +name = "local-default" +baseUrl = "http://localhost:1234" + +[models.fast] +name = "local-fast" +`, () => { + const config = loadConfig(); + assert.equal(config.models.fast.name, 'local-fast'); + assert.equal(config.models.fast.baseUrl, 'http://localhost:1234/v1'); + }); +}); + +test('env primary model skips TOML [model] but still loads tier sections', () => { + withTempConfig(` +[model] +name = "toml-default" +baseUrl = "http://localhost:11434/v1" + +[models.strong] +name = "openrouter/large" +baseUrl = "https://openrouter.ai/api/v1" +`, () => { + process.env.SMALLCODE_MODEL = 'env-default'; + const config = loadConfig(); + assert.equal(config.model.name, 'env-default'); + assert.equal(config.model.baseUrl, 'http://localhost:1234/v1'); + assert.equal(config.models.strong.name, 'openrouter/large'); + assert.equal(config.models.strong.baseUrl, 'https://openrouter.ai/api/v1'); + }); +}); diff --git a/test/model_routing.test.js b/test/model_routing.test.js new file mode 100644 index 00000000..747e4392 --- /dev/null +++ b/test/model_routing.test.js @@ -0,0 +1,54 @@ +'use strict'; + +const test = require('node:test'); +const assert = require('node:assert/strict'); + +const { getModelTarget } = require('../bin/config'); +const { routeTier } = require('../src/model/router'); +const { AdaptiveModelRouter } = require('../src/model/adaptive_router'); + +test('complexity routing selects model and base URL for strong tier', () => { + const config = { + model: { name: 'local-default', baseUrl: 'http://localhost:11434/v1', provider: 'openai' }, + models: { + default: { name: 'local-default', baseUrl: 'http://localhost:11434/v1', provider: 'openai' }, + strong: { name: 'openrouter/large', baseUrl: 'https://openrouter.ai/api/v1', provider: 'openai' }, + }, + }; + + const tier = routeTier('refactor this architecture across multiple files'); + const target = getModelTarget(config, tier); + assert.equal(tier, 'strong'); + assert.equal(target.model, 'openrouter/large'); + assert.equal(target.baseUrl, 'https://openrouter.ai/api/v1'); +}); + +test('adaptive routing selects medium and strong tier URLs', () => { + const config = { + model: { name: 'local-default', baseUrl: 'http://localhost:11434/v1', provider: 'openai' }, + models: { + medium: { name: 'openrouter/medium', baseUrl: 'https://openrouter.ai/api/v1', provider: 'openai' }, + strong: { name: 'openrouter/large', baseUrl: 'https://openrouter.ai/api/v1', provider: 'openai' }, + }, + }; + + const mediumRouter = new AdaptiveModelRouter(); + mediumRouter.recordCall('local-default', false); + mediumRouter.recordCall('local-default', true); + mediumRouter.recordCall('local-default', true); + assert.deepEqual(mediumRouter.selectModel(config), { + model: 'openrouter/medium', + url: 'https://openrouter.ai/api/v1', + tier: 'medium', + }); + + const strongRouter = new AdaptiveModelRouter(); + strongRouter.recordCall('local-default', false); + strongRouter.recordCall('local-default', false); + strongRouter.recordCall('local-default', false); + assert.deepEqual(strongRouter.selectModel(config), { + model: 'openrouter/large', + url: 'https://openrouter.ai/api/v1', + tier: 'strong', + }); +}); diff --git a/test/provider_compat.test.js b/test/provider_compat.test.js index 9303ca5d..654b2f2c 100644 --- a/test/provider_compat.test.js +++ b/test/provider_compat.test.js @@ -13,7 +13,7 @@ const assert = require('node:assert/strict'); // ─── Bug 7: provider-aware auth routing ───────────────────────────────────── -const { buildAuthHeaders } = require('../bin/config'); +const { buildAuthHeaders, getModelTarget, withModelTarget } = require('../bin/config'); test('buildAuthHeaders picks DEEPSEEK_API_KEY for api.deepseek.com', () => { const prev = { ...process.env }; @@ -95,6 +95,34 @@ test('buildAuthHeaders uses SMALLCODE_API_KEY for local endpoints when set', () } }); +test('per-tier OpenRouter target gets OpenRouter auth without changing local target', () => { + const prev = { ...process.env }; + delete process.env.OPENAI_API_KEY; + delete process.env.SMALLCODE_API_KEY; + delete process.env.ANTHROPIC_API_KEY; + delete process.env.DEEPSEEK_API_KEY; + process.env.OPENROUTER_API_KEY = 'sk-router'; + try { + const config = { + model: { name: 'local', baseUrl: 'http://localhost:11434/v1', provider: 'openai' }, + models: { + strong: { name: 'openrouter/large', baseUrl: 'https://openrouter.ai/api/v1', provider: 'openai' }, + }, + }; + + const localHeaders = buildAuthHeaders(config); + assert.equal(localHeaders.Authorization, undefined); + + const strongTarget = getModelTarget(config, 'strong'); + const strongHeaders = buildAuthHeaders(withModelTarget(config, strongTarget)); + assert.equal(strongHeaders.Authorization, 'Bearer sk-router'); + assert.ok(strongHeaders['HTTP-Referer']); + assert.ok(strongHeaders['X-Title']); + } finally { + process.env = prev; + } +}); + // ─── Bug 2/5: thinking budget provider detection ──────────────────────────── const { applyThinkingBudget } = require('../src/model/thinking_budget');