From 2b7b08f5397d3f13cd3c3de3659187bf6081aee4 Mon Sep 17 00:00:00 2001 From: nathan-t4 <19811248+nathan-t4@users.noreply.github.com> Date: Wed, 24 Jun 2026 18:37:47 +0800 Subject: [PATCH 01/11] Add support for local models --- README.md | 33 +++ local-models.json | 32 +++ local-models.json.example | 12 + package-lock.json | 125 ++++++----- shared/localModels.test.ts | 152 +++++++++++++ shared/localModels.ts | 114 ++++++++++ src/lib/utils.ts | 19 +- src/server/aiChat.ts | 402 ++++++++++++++++++++++++---------- src/server/localChatConfig.ts | 93 ++++++++ 9 files changed, 806 insertions(+), 176 deletions(-) create mode 100644 local-models.json create mode 100644 local-models.json.example create mode 100644 shared/localModels.test.ts create mode 100644 shared/localModels.ts create mode 100644 src/server/localChatConfig.ts diff --git a/README.md b/README.md index 00a10bd3..cd589309 100644 --- a/README.md +++ b/README.md @@ -211,6 +211,39 @@ npm run dev NGROK_URL="" # Optional local Supabase Storage tunnel for provider-readable signed URLs ``` +### 3. Optional Local LLM Models + +To add your own local models (LM Studio, Ollama, LiteLLM, etc.) alongside the cloud models: + +1. Start one or more local OpenAI-compatible servers (for example LM Studio on port `1234`). +2. Copy the example catalog and edit entries to match your servers: + + ```bash + cp local-models.json.example local-models.json + ``` + + Each entry needs an `id`, `name`, `description`, and `baseUrl`. The `id` must match what your server exposes. + + Example: + + ```json + { + "id": "qwen2.5-vl-7b-instruct", + "name": "Qwen2.5 VL 7B", + "description": "Local VLM", + "baseUrl": "http://localhost:1234", + "supportsTools": true, + "supportsVision": true, + "useForAux": true + } + ``` + +3. Optionally set `OPENROUTER_API_KEY="local"` in `.env.local` if your local server expects an API key header (many accept any value). + +4. Restart `npm run dev`. Models with a `baseUrl` appear in the parametric model picker. + +Cloud models (Gemini, Claude, OpenAI, Z-AI) keep using their existing cloud routes. Only ids listed in `local-models.json` with a `baseUrl` are sent to that model's server. + ## 🌐 Setting Up ngrok for Local Development CADAM uses public URLs for provider callbacks and local signed storage URLs: diff --git a/local-models.json b/local-models.json new file mode 100644 index 00000000..5ea7aec5 --- /dev/null +++ b/local-models.json @@ -0,0 +1,32 @@ +[ + { + "id": "openai/gpt-oss-120b", + "name": "GPT-OSS 120B", + "description": "Local LLM", + "baseUrl": "http://localhost:1234", + "supportsTools": true, + "supportsThinking": false, + "supportsVision": false, + "useForAux": false + }, + { + "id": "qwen2.5-vl-7b-instruct", + "name": "Qwen2.5 VL 7B", + "description": "Local VLM", + "baseUrl": "http://localhost:1234", + "supportsTools": true, + "supportsThinking": false, + "supportsVision": true, + "useForAux": false + }, + { + "id": "qwen/qwen3-vl-30b", + "name": "Qwen3 VL 30B", + "description": "Local VLM", + "baseUrl": "http://localhost:8004/v1", + "supportsTools": true, + "supportsThinking": true, + "supportsVision": true, + "useForAux": false + } +] diff --git a/local-models.json.example b/local-models.json.example new file mode 100644 index 00000000..e00941c3 --- /dev/null +++ b/local-models.json.example @@ -0,0 +1,12 @@ +[ + { + "id": "qwen2.5-vl-7b-instruct", + "name": "Qwen2.5 VL 7B", + "description": "Local VLM", + "baseUrl": "http://localhost:1234", + "supportsTools": true, + "supportsThinking": false, + "supportsVision": true, + "useForAux": true + } +] diff --git a/package-lock.json b/package-lock.json index d1143e06..de268d1b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -365,6 +365,7 @@ "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.29.7.tgz", "integrity": "sha512-RgHBCvtjbOK2gXSNBNIkNoEc9qoVEtau3hj8gEqKQuL3HZAibKarWFEI3Lfm6EYKkLalOh8eSrj9b+ch9H/VBA==", "license": "MIT", + "peer": true, "dependencies": { "@babel/code-frame": "^7.29.7", "@babel/generator": "^7.29.7", @@ -672,31 +673,11 @@ "integrity": "sha512-U+HFai5+zmJCkK86QsaJtoITlboZHBqrVketcO2ROv865xfCMSFpELQoz1GkX5GzME8pTa+3kbKrZHQtI0gdbw==", "license": "Apache-2.0" }, - "node_modules/@emnapi/core": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.10.0.tgz", - "integrity": "sha512-yq6OkJ4p82CAfPl0u9mQebQHKPJkY7WrIuk205cTYnYe+k2Z8YBh11FrbRG/H6ihirqcacOgl2BIO8oyMQLeXw==", - "license": "MIT", - "optional": true, - "dependencies": { - "@emnapi/wasi-threads": "1.2.1", - "tslib": "^2.4.0" - } - }, - "node_modules/@emnapi/runtime": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.10.0.tgz", - "integrity": "sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA==", - "license": "MIT", - "optional": true, - "dependencies": { - "tslib": "^2.4.0" - } - }, "node_modules/@emnapi/wasi-threads": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.1.tgz", "integrity": "sha512-uTII7OYF+/Mes/MrcIOYp5yOtSMLBWSIoLPpcgwipoiKbli6k322tcoFsxoIIxPDqW01SQGAgko4EzZi2BNv2w==", + "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -1146,6 +1127,7 @@ "version": "1.1.5", "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.5.tgz", "integrity": "sha512-AWPoBRJ9tsnVhor4sjO7rkni+7p+2IAEFj6cx06UgP10jkQHqay/36uRV/bFkgrh18D9vb4cr8Q0Pthskgzy+Q==", + "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -2759,6 +2741,7 @@ "version": "9.2.0", "resolved": "https://registry.npmjs.org/@react-three/fiber/-/fiber-9.2.0.tgz", "integrity": "sha512-esZe+E9T/aYEM4HlBkirr/yRE8qWTp9WUsLISyHHMCHKlJv85uc5N4wwKw+Ay0QeTSITw6T9Q3Svpu383Q+CSQ==", + "peer": true, "dependencies": { "@babel/runtime": "^7.17.8", "@types/react-reconciler": "^0.28.9", @@ -2811,6 +2794,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -2827,6 +2811,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -2843,6 +2828,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -2859,6 +2845,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -2875,6 +2862,7 @@ "cpu": [ "arm" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -2891,9 +2879,7 @@ "cpu": [ "arm64" ], - "libc": [ - "glibc" - ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -2910,9 +2896,7 @@ "cpu": [ "arm64" ], - "libc": [ - "musl" - ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -2929,9 +2913,7 @@ "cpu": [ "ppc64" ], - "libc": [ - "glibc" - ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -2948,9 +2930,7 @@ "cpu": [ "s390x" ], - "libc": [ - "glibc" - ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -2967,9 +2947,7 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -2986,9 +2964,7 @@ "cpu": [ "x64" ], - "libc": [ - "musl" - ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -3005,6 +2981,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -3021,6 +2998,7 @@ "cpu": [ "wasm32" ], + "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -3039,6 +3017,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -3055,6 +3034,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -3549,9 +3529,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -3566,9 +3543,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -3583,9 +3557,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -3600,9 +3571,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -4265,6 +4233,7 @@ "version": "0.10.2", "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.2.tgz", "integrity": "sha512-RoBvJ2X0wuKlWFIjrwffGw1IqZHKQqzIchKaadZZfnNpsAYp2mM0h36JtPCjNDAHGgYez/15uMBpfGwchhiMgg==", + "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -4575,6 +4544,7 @@ "resolved": "https://registry.npmjs.org/@types/mdast/-/mdast-4.0.4.tgz", "integrity": "sha512-kGaNbPh1k7AFzgpud/gMdvIm5xuECykRR+JnWKQno9TAXVa6WIVCGTPvYGekIDL4uwCZQSYbUxNBSb1aUo79oA==", "license": "MIT", + "peer": true, "dependencies": { "@types/unist": "*" } @@ -4614,6 +4584,7 @@ "resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.1.tgz", "integrity": "sha512-1U5NQWh/GylZQ50ZMnnPjkYHEaGhg6t5i/KI0LDDh3t4E3h3T3vzm+GLY2BRzMfIjSBwzm6tginoZl5z0O/qsA==", "license": "MIT", + "peer": true, "dependencies": { "csstype": "^3.0.2" } @@ -4624,6 +4595,7 @@ "integrity": "sha512-/EEvYBdT3BflCWvTMO7YkYBHVE9Ci6XdqZciZANQgKpaiDRGOLIlRo91jbTNRQjgPFWVaRxcYc0luVNFitz57A==", "devOptional": true, "license": "MIT", + "peer": true, "peerDependencies": { "@types/react": "^19.2.0" } @@ -4651,6 +4623,7 @@ "version": "0.160.0", "resolved": "https://registry.npmjs.org/@types/three/-/three-0.160.0.tgz", "integrity": "sha512-jWlbUBovicUKaOYxzgkLlhkiEQJkhCVvg4W2IYD2trqD2om3VK4DGLpHH5zQHNr7RweZK/5re/4IVhbhvxbV9w==", + "peer": true, "dependencies": { "@types/stats.js": "*", "@types/webxr": "*", @@ -4736,6 +4709,7 @@ "integrity": "sha512-Zhy8HCvBUEfBECzIl1PKqF4p11+d0aUJS1GeUiuqK9WmOug8YCmC4h4bjyBvMyAMI9sbRczmrYL5lKg/YMbrcQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "8.38.0", "@typescript-eslint/types": "8.38.0", @@ -5046,6 +5020,7 @@ "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.16.0.tgz", "integrity": "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==", "license": "MIT", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -5079,6 +5054,7 @@ "resolved": "https://registry.npmjs.org/ai/-/ai-6.0.177.tgz", "integrity": "sha512-1xQtbeWwNcLyyM86ixZhkKvT+WRXc1lvarIKqPVtsyn8F9NDikwUMBqYu+aQKDgMht50SMXh4qboYuU8MeHZZA==", "license": "Apache-2.0", + "peer": true, "dependencies": { "@ai-sdk/gateway": "3.0.112", "@ai-sdk/provider": "3.0.10", @@ -5352,6 +5328,7 @@ "url": "https://github.com/sponsors/ai" } ], + "peer": true, "dependencies": { "caniuse-lite": "^1.0.30001726", "electron-to-chromium": "^1.5.173", @@ -5806,6 +5783,7 @@ "integrity": "sha512-wUR89x/Rw7/8t+vn0CmGDYM9TD6VtARGb0LD5jq2wjtMy1vCP4M+sm6N6TigWeTYvnA8MoW29NqqXD0ep0rfBA==", "devOptional": true, "license": "MIT", + "peer": true, "peerDependencies": { "srvx": ">=0.11.5" }, @@ -5871,6 +5849,7 @@ "resolved": "https://registry.npmjs.org/cytoscape/-/cytoscape-3.33.2.tgz", "integrity": "sha512-sj4HXd3DokGhzZAdjDejGvTPLqlt84vNFN8m7bGsOzDY5DyVcxIb2ejIXat2Iy7HxWhdT/N1oKyheJ5YdpsGuw==", "license": "MIT", + "peer": true, "engines": { "node": ">=0.10" } @@ -6273,6 +6252,7 @@ "resolved": "https://registry.npmjs.org/d3-selection/-/d3-selection-3.0.0.tgz", "integrity": "sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ==", "license": "ISC", + "peer": true, "engines": { "node": ">=12" } @@ -6623,7 +6603,8 @@ "node_modules/embla-carousel": { "version": "8.6.0", "resolved": "https://registry.npmjs.org/embla-carousel/-/embla-carousel-8.6.0.tgz", - "integrity": "sha512-SjWyZBHJPbqxHOzckOfo8lHisEaJWmwd23XppYFYVh10bU66/Pn5tkVkbkCMZVdbUE5eTCI2nD8OyIP4Z+uwkA==" + "integrity": "sha512-SjWyZBHJPbqxHOzckOfo8lHisEaJWmwd23XppYFYVh10bU66/Pn5tkVkbkCMZVdbUE5eTCI2nD8OyIP4Z+uwkA==", + "peer": true }, "node_modules/embla-carousel-react": { "version": "8.6.0", @@ -6751,6 +6732,7 @@ "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.31.0.tgz", "integrity": "sha512-QldCVh/ztyKJJZLr4jXNUByx3gR+TDYZCRXEktiZoUR3PGy4qCmSbkxcIle8GEwGpb5JBZazlaJ/CxLidXdEbQ==", "dev": true, + "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.2.0", "@eslint-community/regexpp": "^4.12.1", @@ -9432,6 +9414,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "@types/debug": "^4.0.0", "debug": "^4.0.0", @@ -10068,7 +10051,8 @@ "url": "https://opencollective.com/unified" } ], - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/micromatch": { "version": "4.0.8", @@ -10302,6 +10286,7 @@ "integrity": "sha512-RiXXi4WaNzPTHEOu8UPQKMooIbqOEyqA1t7Z6MsdxSCeb8iUC9ko3LcmsLmeUt2SM5bctfArZKkRQggKZz7JNw==", "dev": true, "license": "MIT", + "peer": true, "peerDependencies": { "@electric-sql/pglite": "*", "@libsql/client": "*", @@ -10331,6 +10316,21 @@ } } }, + "node_modules/nitro/node_modules/readdirp": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-5.0.0.tgz", + "integrity": "sha512-9u/XQ1pvrQtYyMpZe7DXKv2p5CNvyVwzUB6uhLAnQwHMSgKMBR62lc7AHljaeteeHXn11XTAaLLUVZYVZyuRBQ==", + "dev": true, + "license": "MIT", + "optional": true, + "engines": { + "node": ">= 20.19.0" + }, + "funding": { + "type": "individual", + "url": "https://paulmillr.com/funding/" + } + }, "node_modules/nitro/node_modules/unstorage": { "version": "2.0.0-alpha.7", "resolved": "https://registry.npmjs.org/unstorage/-/unstorage-2.0.0-alpha.7.tgz", @@ -10544,7 +10544,8 @@ "resolved": "https://registry.npmjs.org/ofetch/-/ofetch-2.0.0-alpha.3.tgz", "integrity": "sha512-zpYTCs2byOuft65vI3z43Dd6iSdFbOZZLb9/d21aCpx2rGastVU9dOCv0lu4ykc1Ur1anAYjDi3SUvR0vq50JA==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/ohash": { "version": "2.0.11", @@ -10909,6 +10910,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "nanoid": "^3.3.12", "picocolors": "^1.1.1", @@ -11074,6 +11076,7 @@ "version": "3.6.2", "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.6.2.tgz", "integrity": "sha512-I7AIg5boAr5R0FFtJ6rCfD+LFsWHp81dolrFD8S79U9tb8Az2nGrJncnMSnys+bpQJfRUzqs9hnA81OAA3hCuQ==", + "peer": true, "bin": { "prettier": "bin/prettier.cjs" }, @@ -11280,6 +11283,7 @@ "resolved": "https://registry.npmjs.org/react/-/react-19.2.1.tgz", "integrity": "sha512-DGrYcCWK7tvYMnWh79yrPHt+vdx9tY+1gPZa7nJQtO/p8bLTDaHp4dzwEhQB7pZ4Xe3ok4XKuEPrVuc+wlpkmw==", "license": "MIT", + "peer": true, "engines": { "node": ">=0.10.0" } @@ -11299,6 +11303,7 @@ "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.2.1.tgz", "integrity": "sha512-ibrK8llX2a4eOskq1mXKu/TGZj9qzomO+sNfO98M6d9zIPOEhlBkMkBUBLd1vgS0gQsLDBzA+8jJBVXDnfHmJg==", "license": "MIT", + "peer": true, "dependencies": { "scheduler": "^0.27.0" }, @@ -11330,6 +11335,7 @@ "version": "7.60.0", "resolved": "https://registry.npmjs.org/react-hook-form/-/react-hook-form-7.60.0.tgz", "integrity": "sha512-SBrYOvMbDB7cV8ZfNpaiLcgjH/a1c7aK0lK+aNigpf4xWLO8q+o4tcvVurv3c4EOyzn/3dCsYt4GKD42VvJ/+A==", + "peer": true, "engines": { "node": ">=18.0.0" }, @@ -11997,6 +12003,7 @@ "resolved": "https://registry.npmjs.org/seroval/-/seroval-1.5.4.tgz", "integrity": "sha512-46uFvgrXTVxZcUorgSSRZ4y+ieqLLQRMlG4bnCZKW3qI6BZm7Rg4ntMW4p1mILEEBZWrFlcpp0AyIIlM6jD9iw==", "license": "MIT", + "peer": true, "engines": { "node": ">=10" } @@ -12523,6 +12530,7 @@ "version": "3.4.17", "resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-3.4.17.tgz", "integrity": "sha512-w33E2aCvSDP0tW9RZuNXadXlkHXqFzSkQew/aIa2i/Sj8fThxwovwlXHSPXTbAHwEIhBFXAedUhP2tueAKP8Og==", + "peer": true, "dependencies": { "@alloc/quick-lru": "^5.2.0", "arg": "^5.0.2", @@ -12594,7 +12602,8 @@ "node_modules/three": { "version": "0.160.1", "resolved": "https://registry.npmjs.org/three/-/three-0.160.1.tgz", - "integrity": "sha512-Bgl2wPJypDOZ1stAxwfWAcJ0WQf7QzlptsxkjYiURPz+n5k4RBDLsq+6f9Y75TYxn6aHLcWz+JNmwTOXWrQTBQ==" + "integrity": "sha512-Bgl2wPJypDOZ1stAxwfWAcJ0WQf7QzlptsxkjYiURPz+n5k4RBDLsq+6f9Y75TYxn6aHLcWz+JNmwTOXWrQTBQ==", + "peer": true }, "node_modules/three-mesh-bvh": { "version": "0.8.3", @@ -12690,6 +12699,7 @@ "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -12868,6 +12878,7 @@ "integrity": "sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ==", "dev": true, "license": "Apache-2.0", + "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -12946,6 +12957,7 @@ "resolved": "https://registry.npmjs.org/unified/-/unified-11.0.5.tgz", "integrity": "sha512-xKvGhPWw3k84Qjh8bI3ZeJjqnyadK+GEFtazSfZv/rKeTkTjOJho6mFqh2SM96iIcZokxiOpg78GazTSg8+KHA==", "license": "MIT", + "peer": true, "dependencies": { "@types/unist": "^3.0.0", "bail": "^2.0.0", @@ -13263,6 +13275,7 @@ "integrity": "sha512-h9bXPmJichP5fLmVQo3PyaGSDE2n3aPuomeAlVRm0JLmt4rY6zmPKd59HYI4LNW8oTK7tlTsuC7l/m7awx9Jcw==", "devOptional": true, "license": "MIT", + "peer": true, "dependencies": { "lightningcss": "^1.32.0", "picomatch": "^4.0.4", @@ -13553,6 +13566,7 @@ "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.0.tgz", "integrity": "sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==", "license": "MIT", + "peer": true, "engines": { "node": ">=10.0.0" }, @@ -13621,6 +13635,7 @@ "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", "license": "MIT", + "peer": true, "funding": { "url": "https://github.com/sponsors/colinhacks" } diff --git a/shared/localModels.test.ts b/shared/localModels.test.ts new file mode 100644 index 00000000..480bf1f7 --- /dev/null +++ b/shared/localModels.test.ts @@ -0,0 +1,152 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; +import { + getAuxiliaryLocalModel, + isMissingLocalBaseUrl, + isLocalModelId, + localModelToPickerConfig, + normalizeOpenRouterBaseUrl, + parseLocalModelsJson, + providerFor, +} from './localModels.ts'; + +const sampleLocalModels = parseLocalModelsJson([ + { + id: 'qwen2.5-coder-7b', + name: 'Qwen', + description: 'Local', + baseUrl: 'http://localhost:1234', + }, +]); + +describe('parseLocalModelsJson', () => { + it('returns an empty array for non-array input', () => { + assert.deepEqual(parseLocalModelsJson(null), []); + assert.deepEqual(parseLocalModelsJson({}), []); + }); + + it('parses valid entries and skips invalid ones', () => { + assert.deepEqual( + parseLocalModelsJson([ + { + id: 'qwen2.5-coder-7b', + name: 'Qwen', + description: 'Local coder', + supportsTools: true, + }, + { id: '', name: 'Bad', description: 'x' }, + ]), + [ + { + id: 'qwen2.5-coder-7b', + name: 'Qwen', + description: 'Local coder', + supportsTools: true, + }, + ], + ); + }); +}); + +describe('isLocalModelId', () => { + it('detects catalog ids', () => { + assert.equal(isLocalModelId('qwen2.5-coder-7b', sampleLocalModels), true); + assert.equal(isLocalModelId('openai/gpt-5.5', sampleLocalModels), false); + }); +}); + +describe('getAuxiliaryLocalModel', () => { + it('returns the model marked useForAux', () => { + const models = parseLocalModelsJson([ + { id: 'local-chat', name: 'Chat', description: 'Chat model' }, + { + id: 'local-aux', + name: 'Aux', + description: 'Aux model', + useForAux: true, + }, + ]); + assert.equal(getAuxiliaryLocalModel(models)?.id, 'local-aux'); + }); +}); + +describe('localModelToPickerConfig', () => { + it('maps catalog entries to picker config', () => { + assert.deepEqual(localModelToPickerConfig(sampleLocalModels[0]!), { + id: 'qwen2.5-coder-7b', + name: 'Qwen', + description: 'Local', + provider: 'Local', + supportsTools: true, + supportsThinking: false, + supportsVision: false, + disabled: false, + }); + }); +}); + +describe('normalizeOpenRouterBaseUrl', () => { + it('returns undefined when unset', () => { + assert.equal(normalizeOpenRouterBaseUrl(''), undefined); + assert.equal(normalizeOpenRouterBaseUrl(' '), undefined); + }); + + it('appends /v1 when missing', () => { + assert.equal( + normalizeOpenRouterBaseUrl('http://localhost:1234'), + 'http://localhost:1234/v1', + ); + }); + + it('preserves an existing /v1 suffix', () => { + assert.equal( + normalizeOpenRouterBaseUrl('http://localhost:1234/v1/'), + 'http://localhost:1234/v1', + ); + }); +}); + +describe('providerFor', () => { + it('keeps cloud providers on their direct routes', () => { + assert.equal( + providerFor('anthropic/claude-opus-4.8', sampleLocalModels), + 'anthropic', + ); + assert.equal( + providerFor('google/gemini-3.1-pro-preview', sampleLocalModels), + 'google', + ); + assert.equal( + providerFor('openai/gpt-5.5', sampleLocalModels), + 'openrouter', + ); + }); + + it('routes catalog ids with a baseUrl locally', () => { + assert.equal(providerFor('qwen2.5-coder-7b', sampleLocalModels), 'local'); + }); + + it('falls back to openrouter for catalog ids without a baseUrl', () => { + const noUrlModels = parseLocalModelsJson([ + { id: 'qwen2.5-coder-7b', name: 'Qwen', description: 'Local' }, + ]); + assert.equal(providerFor('qwen2.5-coder-7b', noUrlModels), 'openrouter'); + }); +}); + +describe('isMissingLocalBaseUrl', () => { + it('is true when catalog lists the model but has no baseUrl', () => { + const noUrlModels = parseLocalModelsJson([ + { id: 'qwen2.5-coder-7b', name: 'Qwen', description: 'Local' }, + ]); + assert.equal(isMissingLocalBaseUrl('qwen2.5-coder-7b', noUrlModels), true); + assert.equal(isMissingLocalBaseUrl('openai/gpt-5.5', noUrlModels), false); + }); + + it('is false when the catalog model has a baseUrl', () => { + assert.equal( + isMissingLocalBaseUrl('qwen2.5-coder-7b', sampleLocalModels), + false, + ); + }); +}); diff --git a/shared/localModels.ts b/shared/localModels.ts new file mode 100644 index 00000000..abb2b94a --- /dev/null +++ b/shared/localModels.ts @@ -0,0 +1,114 @@ +/** + * Pure routing logic for the local OpenAI-compatible model catalog + * (`local-models.json`). No env or disk I/O — `localChatConfig.ts` wires + * server env + file reads; `utils.ts` loads the catalog for the client picker. + */ +import { z } from 'zod'; + +// Shape of each entry in `local-models.json`. +const localModelSchema = z.object({ + id: z.string().min(1), + name: z.string().min(1), + description: z.string().min(1), + baseUrl: z.string().optional(), + supportsTools: z.boolean().optional(), + supportsThinking: z.boolean().optional(), + supportsVision: z.boolean().optional(), + useForAux: z.boolean().optional(), +}); + +export type LocalModelConfig = z.infer; + +// UI model-picker shape; cloud entries in `utils.ts` use the same fields. +export type PickerModelConfig = { + id: string; + name: string; + description: string; + provider?: string; + supportsTools?: boolean; + supportsThinking?: boolean; + supportsVision?: boolean; + disabled?: boolean; +}; + +export type ChatProvider = 'anthropic' | 'google' | 'openrouter' | 'local'; + +/** Parse and validate a raw `local-models.json` payload; skip invalid entries. */ +export function parseLocalModelsJson(raw: unknown): LocalModelConfig[] { + if (!Array.isArray(raw)) return []; + return raw.flatMap((entry) => { + const parsed = localModelSchema.safeParse(entry); + return parsed.success ? [parsed.data] : []; + }); +} + +export function isLocalModelId( + id: string, + localModels: LocalModelConfig[], +): boolean { + return localModels.some((model) => model.id === id); +} + +// One catalog entry may be marked `useForAux` for lightweight tasks (conversation +// titles, suggestion pills) when `ANTHROPIC_API_KEY` is unset. +export function getAuxiliaryLocalModel( + localModels: LocalModelConfig[], +): LocalModelConfig | undefined { + return localModels.find((model) => model.useForAux === true); +} + +// Vercel AI SDK OpenRouter provider expects baseURL to include `/v1` (its default +// is `https://openrouter.ai/api/v1`). LM Studio and similar servers often +// advertise the bare host; append `/v1` when missing. +export function normalizeOpenRouterBaseUrl(raw: string): string | undefined { + const trimmed = raw.trim(); + if (!trimmed) return undefined; + const base = trimmed.replace(/\/+$/, ''); + return base.endsWith('/v1') ? base : `${base}/v1`; +} + +// Cloud ids with a provider prefix keep their direct routes; catalog ids with a +// configured baseUrl route to their own OpenAI-compatible server; everything +// else goes through OpenRouter. +export function providerFor( + modelId: string, + localModels: LocalModelConfig[], +): ChatProvider { + if (modelId.startsWith('anthropic/')) return 'anthropic'; + if (modelId.startsWith('google/')) return 'google'; + const localModel = localModels.find((m) => m.id === modelId); + if (localModel?.baseUrl?.trim()) return 'local'; + return 'openrouter'; +} + +// Catalog ids without a baseUrl cannot be routed — callers reject those +// requests with a clear error instead of falling through to OpenRouter. +export function isMissingLocalBaseUrl( + modelId: string, + catalogOnDisk: LocalModelConfig[], +): boolean { + const model = catalogOnDisk.find((m) => m.id === modelId); + return !!model && !model.baseUrl?.trim(); +} + +// Parametric CAD requires tool calling — grey out catalog entries that explicitly +// opt out via `supportsTools: false`. +export function localModelToPickerConfig( + model: LocalModelConfig, +): PickerModelConfig { + return { + id: model.id, + name: model.name, + description: model.description, + provider: 'Local', + supportsTools: model.supportsTools ?? true, + supportsThinking: model.supportsThinking ?? false, + supportsVision: model.supportsVision ?? false, + disabled: model.supportsTools === false, + }; +} + +// Returns parsed, valid catalog entries that have a baseUrl configured. +export function getLocalModels(rawJson: unknown): LocalModelConfig[] { + return parseLocalModelsJson(rawJson).filter((m) => !!m.baseUrl?.trim()); +} diff --git a/src/lib/utils.ts b/src/lib/utils.ts index 3b326ef8..6cae27fb 100644 --- a/src/lib/utils.ts +++ b/src/lib/utils.ts @@ -1,6 +1,7 @@ import { clsx, type ClassValue } from 'clsx'; import { twMerge } from 'tailwind-merge'; import { Parameter } from '@shared/types'; +import { getLocalModels, localModelToPickerConfig } from '@shared/localModels'; import { ModelConfig } from '../types/misc.ts'; export function cn(...inputs: ClassValue[]) { @@ -235,7 +236,7 @@ export function getInitials(fullName: string | null) { return 'U'; } -export const PARAMETRIC_MODELS: ModelConfig[] = [ +const CLOUD_PARAMETRIC_MODELS: ModelConfig[] = [ { id: 'google/gemini-3.1-pro-preview', name: 'Gemini 3.1 Pro', @@ -274,6 +275,22 @@ export const PARAMETRIC_MODELS: ModelConfig[] = [ }, ]; +const localModelsRaw = Object.values( + import.meta.glob('../../local-models.json', { + eager: true, + import: 'default', + }), +)[0] as unknown; + +const LOCAL_PARAMETRIC_MODELS: ModelConfig[] = getLocalModels( + localModelsRaw, +).map(localModelToPickerConfig); + +export const PARAMETRIC_MODELS: ModelConfig[] = [ + ...CLOUD_PARAMETRIC_MODELS, + ...LOCAL_PARAMETRIC_MODELS, +]; + export const CREATIVE_MODELS: ModelConfig[] = [ { id: 'ultra', diff --git a/src/server/aiChat.ts b/src/server/aiChat.ts index 84ae7a15..b814a134 100644 --- a/src/server/aiChat.ts +++ b/src/server/aiChat.ts @@ -2,6 +2,10 @@ import { createAnthropic } from '@ai-sdk/anthropic'; import { createGoogleGenerativeAI } from '@ai-sdk/google'; import { createOpenRouter } from '@openrouter/ai-sdk-provider'; import { chatTools, type AppUIMessage, type AppTools } from '@shared/chatAi'; +import { + getAuxiliaryLocalModel, + normalizeOpenRouterBaseUrl, +} from '@shared/localModels'; import { cleanAssistantText, getParametricText } from '@shared/parametricParts'; import { imageIdFromFilename, imageStoragePath } from '@shared/imageRefs'; import { normalizeConversationSuggestions } from '@shared/suggestions'; @@ -34,6 +38,14 @@ import { resolveDanglingToolParts, } from './chatToolPersistence'; import { handleMeshRequest } from './mesh'; +import { + getActiveLocalModel, + getLocalChatState, + isActiveLocalModel, + isMissingLocalBaseUrl, + providerFor, + type ChatProvider, +} from './localChatConfig'; import { getAnonSupabaseClient } from './supabaseClient'; /** @@ -307,16 +319,9 @@ function jsonResponse(body: unknown, status: number) { const THINKING_BUDGET_TOKENS = 9000; const PARAMETRIC_MAX_OUTPUT_TOKENS = 64000; -type ChatProvider = 'anthropic' | 'google' | 'openrouter'; - -function providerFor(modelId: string): ChatProvider { - if (modelId.startsWith('anthropic/')) return 'anthropic'; - if (modelId.startsWith('google/')) return 'google'; - return 'openrouter'; -} - type AnthropicProvider = ReturnType; type GoogleProvider = ReturnType; +type OpenRouterProvider = ReturnType; // The Vercel AI SDK's Anthropic provider expects ANTHROPIC_BASE_URL to already // include the "/v1" path segment (its built-in default is @@ -335,13 +340,15 @@ function normalizedAnthropicBaseURL(): string | undefined { type ChatProviders = { anthropic: () => AnthropicProvider; google: () => GoogleProvider; - openrouter: () => ReturnType; + openrouter: () => OpenRouterProvider; + local: (baseUrl: string) => OpenRouterProvider; }; function createChatProviders(): ChatProviders { let anthropic: AnthropicProvider | undefined; let google: GoogleProvider | undefined; - let openrouter: ReturnType | undefined; + let openrouter: OpenRouterProvider | undefined; + const localByUrl = new Map(); return { anthropic: () => { if (!anthropic) { @@ -365,6 +372,18 @@ function createChatProviders(): ChatProviders { }); return openrouter; }, + local: (baseUrl: string) => { + let provider = localByUrl.get(baseUrl); + if (!provider) { + provider = createOpenRouter({ + baseURL: baseUrl, + apiKey: env('OPENROUTER_API_KEY').trim() || 'local', + compatibility: 'compatible', + }); + localByUrl.set(baseUrl, provider); + } + return provider; + }, }; } @@ -384,8 +403,18 @@ function buildChatModel( ): { model: LanguageModel; providerOptions?: ProviderOptions } { const hasCappedThinkingBudget = thinking && thinkingBudget !== THINKING_BUDGET_TOKENS; + const provider = providerFor(modelId); + + if (provider === 'local') { + const modelConfig = getActiveLocalModel(modelId); + const rawBaseUrl = modelConfig?.baseUrl; + if (!rawBaseUrl) + throw new Error(`No baseUrl configured for local model ${modelId}`); + const baseUrl = normalizeOpenRouterBaseUrl(rawBaseUrl) ?? rawBaseUrl; + return { model: providers.local(baseUrl).chat(modelId) }; + } - if (providerFor(modelId) === 'openrouter') { + if (provider === 'openrouter') { return { model: providers.openrouter().chat(modelId, { ...(thinking ? { reasoning: { max_tokens: thinkingBudget } } : {}), @@ -454,6 +483,14 @@ function bareModelId(modelId: string): string { return id.replace(/\./g, '-'); } +// For local models, check supportsVision. +// Otherwise, all cloud models support vision. +function chatModelSupportsVision(modelId: string): boolean { + const local = getActiveLocalModel(modelId); + if (local) return local.supportsVision === true; + return true; +} + // The Claude 5 generation swaps the opus/sonnet/haiku tiers for code names // ("claude-fable-5", "claude-mythos-5", …). Match the `claude--5` // shape rather than enumerating code names so future Claude 5 variants @@ -476,11 +513,21 @@ function usesAdaptiveAnthropicThinking(modelId: string) { // The Claude 5 generation rejects forced tool use with "tool_choice forces // tool use is not compatible with this model" — for those we must fall back // to auto tool choice and steer via the system prompt. +// Local OpenAI-compatible servers (LM Studio, etc.) also reject object-style +// tool_choice and only accept the string forms `none` | `auto` | `required`. function supportsForcedToolChoice(modelId: string): boolean { - return !isClaude5Model(modelId); + return !isClaude5Model(modelId) && !isActiveLocalModel(modelId); } function priceFor(modelId: string) { + if (isActiveLocalModel(modelId)) { + return { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + }; + } const entry = MODEL_PRICES[modelId] ?? FALLBACK_MODEL_PRICE; return { input: entry.input, @@ -703,16 +750,16 @@ async function loadBranchFromDb({ } async function generateConversationTitle({ - anthropic, + auxModel, firstMessage, }: { - anthropic: AnthropicProvider; + auxModel: LanguageModel; firstMessage: AppUIMessage; }) { const text = getParametricText(firstMessage.parts) || 'New conversation'; try { const result = await generateText({ - model: anthropic('claude-haiku-4-5'), + model: auxModel, system: 'Generate a short title for a 3D creation conversation. Return only the title.', prompt: text, @@ -735,11 +782,11 @@ async function generateConversationTitle({ * specific assistant turn. */ async function generateConversationSuggestions({ - anthropic, + auxModel, branch, conversationType, }: { - anthropic: AnthropicProvider; + auxModel: LanguageModel; branch: AppUIMessage[]; conversationType: 'parametric' | 'creative'; }): Promise { @@ -757,7 +804,7 @@ async function generateConversationSuggestions({ const summary = `User request: ${firstUserText.slice(0, 400)}\n\nMost recent assistant reply: ${lastAssistantText.slice(0, 400)}`; try { const result = await generateText({ - model: anthropic('claude-haiku-4-5'), + model: auxModel, system: conversationType === 'creative' ? 'Given a 3D mesh design conversation, return an array of exactly 2 follow-up prompts the user might want to send next. Each prompt is a concise instruction of 3 words or fewer, not a question. Return exactly 2 items — no more, no fewer.' @@ -857,6 +904,102 @@ async function sniffImageMediaType(bytes: Uint8Array): Promise { return sniffed && ACCEPTED_IMAGE_MEDIA_TYPES.has(sniffed) ? sniffed : null; } +async function buildHydratedMessages( + messages: AppUIMessage[], + { + supabaseClient, + conversation, + modelId, + provider, + previewPathForToolCall, + }: { + supabaseClient: SupabaseAnon; + conversation: ConversationAccess; + modelId: string; + provider: ChatProvider; + previewPathForToolCall: (toolCallId: string) => string; + }, +): Promise>> { + const isLocal = provider === 'local'; + const result: Array> = []; + + for (const message of messages) { + const hydratedParts = ( + await Promise.all( + message.parts.map(async (part) => { + if ( + part.type !== 'file' || + typeof part.mediaType !== 'string' || + !part.mediaType.startsWith('image/') || + part.url.startsWith('data:') + ) { + return part; + } + const imageId = imageIdFromFilename(part.filename); + if (!imageId) return null; + const downloaded = await downloadAsBase64( + supabaseClient, + 'images', + imageStoragePath(conversation.user_id, conversation.id, imageId), + ); + if (!downloaded) return null; + return { + ...part, + mediaType: downloaded.mediaType, + url: `data:${downloaded.mediaType};base64,${downloaded.base64}`, + }; + }), + ) + ).filter((part): part is NonNullable => part != null); + + result.push({ ...message, parts: hydratedParts }); + + // Local providers: inject image renders as user messages so they + // never end up inside `role: tool` output (strict OpenAI-compat servers + // reject images there). Only applies for parametric conversations where + // the model supports vision. + if ( + isLocal && + conversation.type === 'parametric' && + chatModelSupportsVision(modelId) && + message.role === 'assistant' + ) { + for (const part of message.parts) { + if ( + part.type !== 'tool-build_parametric_model' || + part.state !== 'output-available' || + !('toolCallId' in part) || + typeof part.toolCallId !== 'string' + ) { + continue; + } + const downloaded = await downloadAsBase64( + supabaseClient, + 'images', + previewPathForToolCall(part.toolCallId), + ); + if (!downloaded) continue; + result.push({ + role: 'user', + parts: [ + { + type: 'text', + text: 'Multi-view inspection render from the tool result above.', + }, + { + type: 'file', + mediaType: downloaded.mediaType, + url: `data:${downloaded.mediaType};base64,${downloaded.base64}`, + }, + ], + }); + } + } + } + + return result; +} + async function downloadAsBase64( supabaseClient: SupabaseAnon, bucket: string, @@ -886,10 +1029,13 @@ async function downloadAsBase64( function parametricTools({ previewPathForToolCall, supabaseClient, + modelId, }: { previewPathForToolCall: (toolCallId: string) => string; supabaseClient: SupabaseAnon; + modelId: string; }) { + const isLocal = providerFor(modelId) === 'local'; return { build_parametric_model: { ...chatTools.build_parametric_model, @@ -900,42 +1046,63 @@ function parametricTools({ toolCallId: string; output: AppTools['build_parametric_model']['output']; }) { - // The client uploads a multi-view render of the compiled SCAD to a path - // derived from toolCallId BEFORE sending the tool result (see - // ChatSession's `onToolCall`). If for any reason the upload - // didn't land, `downloadAsBase64` returns null and we fall back - // to text-only — never block the loop on a missing inspection sheet. - const downloaded = await downloadAsBase64( - supabaseClient, - 'images', - previewPathForToolCall(toolCallId), - ); const views = output.inspection?.views.join(', ') ?? 'ISO, FRONT, BACK, LEFT, RIGHT, TOP, BOTTOM'; - const text = `${output.message}\nRendered inspection views: ${views}.\nMulti-view inspection image attached: ${downloaded ? 'yes' : 'no'}.`; - if (downloaded) { - return { - type: 'content' as const, - value: [ - { type: 'text' as const, text }, - { - type: 'image-data' as const, - data: downloaded.base64, - mediaType: downloaded.mediaType, - }, - ], - }; - } + if (!isLocal && chatModelSupportsVision(modelId)) { + // The client uploads a multi-view render of the compiled SCAD to a path + // derived from toolCallId BEFORE sending the tool result (see + // ChatSession's `onToolCall`). If for any reason the upload + // didn't land, `downloadAsBase64` returns null and we fall back + // to text-only — never block the loop on a missing inspection sheet. + const downloaded = await downloadAsBase64( + supabaseClient, + 'images', + previewPathForToolCall(toolCallId), + ); + const text = `${output.message}\nRendered inspection views: ${views}.\nMulti-view inspection image attached: ${downloaded ? 'yes' : 'no'}.`; - return { type: 'text' as const, value: text }; + if (downloaded) { + return { + type: 'content' as const, + value: [ + { type: 'text' as const, text }, + { + type: 'image-data' as const, + data: downloaded.base64, + mediaType: downloaded.mediaType, + }, + ], + }; + } + + return { type: 'text' as const, value: text }; + } else { + // For models that do not support multimodal tools result messages, + // the rendered inspection views are appended as user messages in buildHydratedMessages + const imageAttached = output.inspection?.imageAttached === true; + const text = `${output.message}\nRendered inspection views: ${views}.\nMulti-view inspection image attached: ${imageAttached ? 'yes' : 'no'}.`; + return { type: 'text' as const, value: text }; + } }, }, answer_user: chatTools.answer_user, }; } +function getAuxLanguageModel(providers: ChatProviders): LanguageModel | null { + const { activeCatalog, hasAnthropicApiKey } = getLocalChatState(); + const auxLocal = getAuxiliaryLocalModel(activeCatalog); + if (auxLocal?.baseUrl) { + const baseUrl = + normalizeOpenRouterBaseUrl(auxLocal.baseUrl) ?? auxLocal.baseUrl; + return providers.local(baseUrl).chat(auxLocal.id); + } + if (hasAnthropicApiKey) return providers.anthropic()('claude-haiku-4-5'); + return null; +} + function chatModel(conversation: ConversationAccess, model: Model) { if (conversation.type === 'creative') { return 'anthropic/claude-sonnet-4.5'; @@ -1024,6 +1191,14 @@ export async function handleAiChatRequest(req: Request) { return jsonResponse({ error: 'Billing service unavailable' }, 503); } + const actualModelId = chatModel(conversation, rawBody.model); + if (isMissingLocalBaseUrl(actualModelId)) { + return jsonResponse( + { error: 'baseUrl is not configured for local model' }, + 503, + ); + } + const tools = conversation.type === 'creative' ? creativeTools({ conversation, req, model: rawBody.model }) @@ -1031,6 +1206,7 @@ export async function handleAiChatRequest(req: Request) { supabaseClient, previewPathForToolCall: (toolCallId) => `${user.id}/${conversation.id}/inspection-preview-${toolCallId}`, + modelId: actualModelId, }); let branchMessages: AppUIMessage[]; @@ -1079,6 +1255,11 @@ export async function handleAiChatRequest(req: Request) { // the very first user turn. const isFirstUserTurn = branchMessages.length === 1 && leafRole === 'user'; + // Resolve the actual model ID the request will run against. + const resolvedProvider = providerFor(actualModelId); + const previewPathForToolCall = (toolCallId: string) => + `${user.id}/${conversation.id}/inspection-preview-${toolCallId}`; + // Rehydrate image file parts before handing them to the model. The // persisted `url` is a storage reference (or, for the oldest backfilled // rows, a dead `/public/` path), neither of which the provider can fetch — @@ -1088,38 +1269,19 @@ export async function handleAiChatRequest(req: Request) { // (legacy rows that inlined base64) pass through untouched; anything we // can't resolve is dropped so a missing image never poisons the request // with an unfetchable URL. - const hydratedMessages = await Promise.all( - branchMessages.map(async (message) => ({ - ...message, - parts: ( - await Promise.all( - message.parts.map(async (part) => { - if ( - part.type !== 'file' || - typeof part.mediaType !== 'string' || - !part.mediaType.startsWith('image/') || - part.url.startsWith('data:') - ) { - return part; - } - const imageId = imageIdFromFilename(part.filename); - if (!imageId) return null; - const downloaded = await downloadAsBase64( - supabaseClient, - 'images', - imageStoragePath(conversation.user_id, conversation.id, imageId), - ); - if (!downloaded) return null; - return { - ...part, - mediaType: downloaded.mediaType, - url: `data:${downloaded.mediaType};base64,${downloaded.base64}`, - }; - }), - ) - ).filter((part): part is NonNullable => part != null), - })), - ); + // + // For local providers on parametric conversations, strict OpenAI-compatible + // servers reject images inside `role: tool` messages. We instead insert a + // follow-up `role: user` message carrying the inspection render after each + // resolved build tool call so `convertToModelMessages` never needs to put + // images in tool output. + const hydratedMessages = await buildHydratedMessages(branchMessages, { + supabaseClient, + conversation, + modelId: actualModelId, + provider: resolvedProvider, + previewPathForToolCall, + }); const modelMessages = await convertToModelMessages( hydratedMessages, @@ -1152,13 +1314,6 @@ export async function handleAiChatRequest(req: Request) { }, }, ); - - // Resolve the actual model ID the request will run against. For - // `creative` conversations this is hardcoded to Sonnet regardless of - // what the client picked — billing has to price the model that ran, - // not the one the user requested. - const actualModelId = chatModel(conversation, rawBody.model); - const resolvedProvider = providerFor(actualModelId); const baseLogContext = { userId: user.id, conversationId: conversation.id, @@ -1239,8 +1394,9 @@ export async function handleAiChatRequest(req: Request) { ) { // Restrict the toolset to the build tool on the first step. Turns that // accept a forced tool_choice get it pinned; turns that reject forced - // tool use (Claude 5, or any thinking-enabled turn) fall back to auto - // and rely on the system prompt to call build_parametric_model. + // tool use (Claude 5, local models, or any thinking-enabled turn) fall + // back to auto and rely on the system prompt to call + // build_parametric_model. return { activeTools: ['build_parametric_model' as never], ...(forceBuildToolChoice @@ -1353,14 +1509,17 @@ export async function handleAiChatRequest(req: Request) { execute: async ({ writer }) => { // Title (first user turn only) runs in parallel with the model // stream — fire-and-forget; the assistant doesn't wait on it. - if (isFirstUserTurn && env('ANTHROPIC_API_KEY')) { - void emitConversationTitle({ - writer, - anthropic: providers.anthropic(), - supabaseClient, - conversation, - firstMessage: branchMessages[0], - }); + if (isFirstUserTurn) { + const auxModel = getAuxLanguageModel(providers); + if (auxModel) { + void emitConversationTitle({ + writer, + auxModel, + supabaseClient, + conversation, + firstMessage: branchMessages[0], + }); + } } writer.merge( @@ -1499,28 +1658,31 @@ export async function handleAiChatRequest(req: Request) { // continuation `onFinish` will fire suggestions for the real // final state. Avoids a wasted Haiku call AND prevents // mid-turn placeholder pills. - if (!hasPendingToolCall && env('ANTHROPIC_API_KEY')) { - // MUST be awaited (not `void`). `createUIMessageStream` - // closes the SSE controller as soon as the merged stream - // drains — and the merged stream resolves once this - // `onFinish` returns. A fire-and-forget here would race - // the close, and the `writer.write` inside - // `emitConversationSuggestions` would silently no-op - // because `safeEnqueue` swallows enqueue errors on a - // closed controller (see ai/dist/index.mjs:8264). The - // ~200-500ms Haiku call delays the client's "streaming" - // → "ready" transition by the same amount, which is the - // tradeoff for getting pills delivered. - await emitConversationSuggestions({ - writer, - anthropic: providers.anthropic(), - supabaseClient, - conversation, - branch: [ - ...branchMessages, - { ...responseMessage, parts: finalizedParts }, - ], - }); + if (!hasPendingToolCall) { + const auxModel = getAuxLanguageModel(providers); + if (auxModel) { + // MUST be awaited (not `void`). `createUIMessageStream` + // closes the SSE controller as soon as the merged stream + // drains — and the merged stream resolves once this + // `onFinish` returns. A fire-and-forget here would race + // the close, and the `writer.write` inside + // `emitConversationSuggestions` would silently no-op + // because `safeEnqueue` swallows enqueue errors on a + // closed controller (see ai/dist/index.mjs:8264). The + // ~200-500ms aux-model call delays the client's "streaming" + // → "ready" transition by the same amount, which is the + // tradeoff for getting pills delivered. + await emitConversationSuggestions({ + writer, + auxModel, + supabaseClient, + conversation, + branch: [ + ...branchMessages, + { ...responseMessage, parts: finalizedParts }, + ], + }); + } } }, }), @@ -1545,19 +1707,19 @@ export async function handleAiChatRequest(req: Request) { */ async function emitConversationTitle({ writer, - anthropic, + auxModel, supabaseClient, conversation, firstMessage, }: { writer: UIMessageStreamWriter; - anthropic: AnthropicProvider; + auxModel: LanguageModel; supabaseClient: SupabaseAnon; conversation: ConversationAccess; firstMessage: AppUIMessage; }) { try { - const title = await generateConversationTitle({ anthropic, firstMessage }); + const title = await generateConversationTitle({ auxModel, firstMessage }); await supabaseClient .from('conversations') .update({ title }) @@ -1587,20 +1749,20 @@ async function emitConversationTitle({ */ async function emitConversationSuggestions({ writer, - anthropic, + auxModel, supabaseClient, conversation, branch, }: { writer: UIMessageStreamWriter; - anthropic: AnthropicProvider; + auxModel: LanguageModel; supabaseClient: SupabaseAnon; conversation: ConversationAccess; branch: AppUIMessage[]; }) { try { const suggestions = await generateConversationSuggestions({ - anthropic, + auxModel, branch, conversationType: conversation.type, }); diff --git a/src/server/localChatConfig.ts b/src/server/localChatConfig.ts new file mode 100644 index 00000000..dd9d4e2e --- /dev/null +++ b/src/server/localChatConfig.ts @@ -0,0 +1,93 @@ +/** + * Server-side local-chat configuration: reads `local-models.json` from the + * project root. A model is active when it has a `baseUrl` set in the catalog. + * Pure routing logic lives in `@shared/localModels`; this module wires + * disk I/O for `aiChat`. + */ +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { + isMissingLocalBaseUrl as isMissingLocalBaseUrlCore, + isLocalModelId, + parseLocalModelsJson, + providerFor as providerForCore, + type ChatProvider, + type LocalModelConfig, +} from '@shared/localModels'; +import { env } from './env'; + +export type { ChatProvider }; + +export type LocalChatState = { + enabled: boolean; + /** On-disk catalog, even when local chat is disabled. */ + catalog: LocalModelConfig[]; + /** Models with a configured baseUrl. */ + activeCatalog: LocalModelConfig[]; + hasAnthropicApiKey: boolean; +}; + +const projectRoot = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + '../..', +); + +// Cached in production; re-read on each request in dev so `local-models.json` +// edits take effect without restarting the dev server. +let cachedCatalog: LocalModelConfig[] | undefined; + +function loadCatalogFromDisk(): LocalModelConfig[] { + if (cachedCatalog && import.meta.env.PROD) return cachedCatalog; + + const userPath = path.join(projectRoot, 'local-models.json'); + if (!fs.existsSync(userPath)) { + cachedCatalog = []; + return cachedCatalog; + } + + try { + const raw = JSON.parse(fs.readFileSync(userPath, 'utf8')) as unknown; + cachedCatalog = parseLocalModelsJson(raw); + } catch { + cachedCatalog = []; + } + + return cachedCatalog; +} + +export function getLocalChatState(): LocalChatState { + const catalog = loadCatalogFromDisk(); + const activeCatalog = catalog.filter((m) => m.baseUrl?.trim()); + + return { + enabled: activeCatalog.length > 0, + catalog, + activeCatalog, + hasAnthropicApiKey: env('ANTHROPIC_API_KEY').trim().length > 0, + }; +} + +export function providerFor(modelId: string): ChatProvider { + const { activeCatalog } = getLocalChatState(); + return providerForCore(modelId, activeCatalog); +} + +export function isActiveLocalModel(modelId: string): boolean { + return isLocalModelId(modelId, getLocalChatState().activeCatalog); +} + +export function getActiveLocalModel( + modelId: string, +): LocalModelConfig | undefined { + return getLocalChatState().activeCatalog.find( + (model) => model.id === modelId, + ); +} + +// Client may show catalog ids whose baseUrl is missing on the server — reject +// those requests with a clear error. +export function isMissingLocalBaseUrl(modelId: string): boolean { + const { catalog } = getLocalChatState(); + return isMissingLocalBaseUrlCore(modelId, catalog); +} From 56f8efa6f755adfecd4a4028c4c50d9559f9f9b5 Mon Sep 17 00:00:00 2001 From: nathan-t4 <19811248+nathan-t4@users.noreply.github.com> Date: Wed, 24 Jun 2026 19:04:49 +0800 Subject: [PATCH 02/11] Add local-models.json to gitignore --- .gitignore | 3 +++ local-models.json | 32 -------------------------------- 2 files changed, 3 insertions(+), 32 deletions(-) delete mode 100644 local-models.json diff --git a/.gitignore b/.gitignore index d6ae9214..3bb04cb2 100644 --- a/.gitignore +++ b/.gitignore @@ -33,3 +33,6 @@ dist-ssr # other secret files .env.local .env.langfuse + +# local models config +local-models.json \ No newline at end of file diff --git a/local-models.json b/local-models.json deleted file mode 100644 index 5ea7aec5..00000000 --- a/local-models.json +++ /dev/null @@ -1,32 +0,0 @@ -[ - { - "id": "openai/gpt-oss-120b", - "name": "GPT-OSS 120B", - "description": "Local LLM", - "baseUrl": "http://localhost:1234", - "supportsTools": true, - "supportsThinking": false, - "supportsVision": false, - "useForAux": false - }, - { - "id": "qwen2.5-vl-7b-instruct", - "name": "Qwen2.5 VL 7B", - "description": "Local VLM", - "baseUrl": "http://localhost:1234", - "supportsTools": true, - "supportsThinking": false, - "supportsVision": true, - "useForAux": false - }, - { - "id": "qwen/qwen3-vl-30b", - "name": "Qwen3 VL 30B", - "description": "Local VLM", - "baseUrl": "http://localhost:8004/v1", - "supportsTools": true, - "supportsThinking": true, - "supportsVision": true, - "useForAux": false - } -] From 311056e37703c142cc40aa5b4cce43783cf9d195 Mon Sep 17 00:00:00 2001 From: Nathan <19811248+nathan-t4@users.noreply.github.com> Date: Wed, 24 Jun 2026 19:38:54 +0800 Subject: [PATCH 03/11] Update README.md Co-authored-by: cubic-dev-ai[bot] <191113872+cubic-dev-ai[bot]@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index cd589309..c888023f 100644 --- a/README.md +++ b/README.md @@ -238,7 +238,7 @@ To add your own local models (LM Studio, Ollama, LiteLLM, etc.) alongside the cl } ``` -3. Optionally set `OPENROUTER_API_KEY="local"` in `.env.local` if your local server expects an API key header (many accept any value). +3. Optionally set `OPENROUTER_API_KEY="local"` in `.env.local` if your local server expects an API key header (many accept any value). **Note:** this value is also used for OpenRouter cloud model calls, so only use a dummy value if you are not using any OpenRouter-hosted models. 4. Restart `npm run dev`. Models with a `baseUrl` appear in the parametric model picker. From e02a0ce088c3e43845934929bdf50267951822f1 Mon Sep 17 00:00:00 2001 From: nathan-t4 <19811248+nathan-t4@users.noreply.github.com> Date: Wed, 24 Jun 2026 20:58:27 +0800 Subject: [PATCH 04/11] Fix local model image messages bug, add api key support for third-party providers, and additional bug fixes --- README.md | 11 ++ shared/imageRefs.ts | 8 ++ shared/localModels.test.ts | 85 +++++++++++ shared/localModels.ts | 9 +- src/components/chat/ChatSession.tsx | 7 +- src/server/aiChat.ts | 213 +++++++++++++++------------- src/server/localChatConfig.ts | 5 +- src/server/messageUtils.test.ts | 90 ++++++++++++ src/server/messageUtils.ts | 33 ++++- 9 files changed, 358 insertions(+), 103 deletions(-) create mode 100644 src/server/messageUtils.test.ts diff --git a/README.md b/README.md index cd589309..ed02a31d 100644 --- a/README.md +++ b/README.md @@ -235,6 +235,17 @@ To add your own local models (LM Studio, Ollama, LiteLLM, etc.) alongside the cl "supportsTools": true, "supportsVision": true, "useForAux": true + }, + { + "id": "deepseek-v4-flash", + "name": "DeepSeek v4 Flash", + "description": "Third Party LLM", + "baseUrl": "https://api.deepseek.com", + "apiKey": "", + "supportsTools": true, + "supportsThinking": true, + "supportsVision": false, + "useForAux": false } ``` diff --git a/shared/imageRefs.ts b/shared/imageRefs.ts index 26cef4ed..e9f90dd4 100644 --- a/shared/imageRefs.ts +++ b/shared/imageRefs.ts @@ -29,6 +29,14 @@ export function imageStoragePath( return `${userId}/${conversationId}/${imageId}`; } +export function inspectionPreviewStoragePath( + userId: string, + conversationId: string, + toolCallId: string, +): string { + return `${userId}/${conversationId}/inspection-preview-${toolCallId}`; +} + // Canonical reference persisted in a file part's `url`. Kept in the exact // shape the 2026-05-18 AI-SDK backfill produced so every row in // `messages.parts` carries a single, uniform format. Never fetched directly — diff --git a/shared/localModels.test.ts b/shared/localModels.test.ts index 480bf1f7..2062b624 100644 --- a/shared/localModels.test.ts +++ b/shared/localModels.test.ts @@ -2,6 +2,7 @@ import assert from 'node:assert/strict'; import { describe, it } from 'node:test'; import { getAuxiliaryLocalModel, + getLocalModels, isMissingLocalBaseUrl, isLocalModelId, localModelToPickerConfig, @@ -32,6 +33,7 @@ describe('parseLocalModelsJson', () => { id: 'qwen2.5-coder-7b', name: 'Qwen', description: 'Local coder', + apiKey: 'test-key', supportsTools: true, }, { id: '', name: 'Bad', description: 'x' }, @@ -41,11 +43,94 @@ describe('parseLocalModelsJson', () => { id: 'qwen2.5-coder-7b', name: 'Qwen', description: 'Local coder', + apiKey: 'test-key', supportsTools: true, }, ], ); }); + + it('keeps the first entry when duplicate ids appear', () => { + assert.deepEqual( + parseLocalModelsJson([ + { + id: 'dup-model', + name: 'First', + description: 'First wins', + baseUrl: 'http://localhost:1111', + }, + { + id: 'dup-model', + name: 'Second', + description: 'Dropped', + baseUrl: 'http://localhost:2222', + }, + ]), + [ + { + id: 'dup-model', + name: 'First', + description: 'First wins', + baseUrl: 'http://localhost:1111', + }, + ], + ); + }); +}); + +describe('getLocalModels', () => { + it('returns only entries with a configured baseUrl', () => { + assert.deepEqual( + getLocalModels([ + { + id: 'with-url', + name: 'With URL', + description: 'Has baseUrl', + baseUrl: 'http://localhost:1234', + }, + { + id: 'without-url', + name: 'Without URL', + description: 'Missing baseUrl', + }, + ]), + [ + { + id: 'with-url', + name: 'With URL', + description: 'Has baseUrl', + baseUrl: 'http://localhost:1234', + }, + ], + ); + }); + + it('excludes entries with blank or whitespace-only baseUrl', () => { + assert.deepEqual( + getLocalModels([ + { + id: 'blank-url', + name: 'Blank', + description: 'Blank baseUrl', + baseUrl: ' ', + }, + { + id: 'valid-url', + name: 'Valid', + description: 'Valid baseUrl', + baseUrl: 'http://localhost:1234', + }, + ]), + [ + { + id: 'valid-url', + name: 'Valid', + description: 'Valid baseUrl', + baseUrl: 'http://localhost:1234', + }, + ], + ); + }); }); describe('isLocalModelId', () => { diff --git a/shared/localModels.ts b/shared/localModels.ts index abb2b94a..030b4423 100644 --- a/shared/localModels.ts +++ b/shared/localModels.ts @@ -11,6 +11,7 @@ const localModelSchema = z.object({ name: z.string().min(1), description: z.string().min(1), baseUrl: z.string().optional(), + apiKey: z.string().optional(), supportsTools: z.boolean().optional(), supportsThinking: z.boolean().optional(), supportsVision: z.boolean().optional(), @@ -33,12 +34,16 @@ export type PickerModelConfig = { export type ChatProvider = 'anthropic' | 'google' | 'openrouter' | 'local'; -/** Parse and validate a raw `local-models.json` payload; skip invalid entries. */ +// Parse and validate a raw `local-models.json` payload; skip invalid entries. +// skip second occurence of local model with same model id export function parseLocalModelsJson(raw: unknown): LocalModelConfig[] { if (!Array.isArray(raw)) return []; + const seenIds = new Set(); return raw.flatMap((entry) => { const parsed = localModelSchema.safeParse(entry); - return parsed.success ? [parsed.data] : []; + if (!parsed.success || seenIds.has(parsed.data.id)) return []; + seenIds.add(parsed.data.id); + return [parsed.data]; }); } diff --git a/src/components/chat/ChatSession.tsx b/src/components/chat/ChatSession.tsx index f19d9eae..611819cd 100644 --- a/src/components/chat/ChatSession.tsx +++ b/src/components/chat/ChatSession.tsx @@ -29,6 +29,7 @@ import { } from 'ai'; import Tree from '@shared/Tree'; import { isParametricArtifact } from '@shared/parametricParts'; +import { inspectionPreviewStoragePath } from '@shared/imageRefs'; import type { Conversation, Message, @@ -470,7 +471,11 @@ export function ChatSession({ const inspectionBlob = await fetch(inspectionDataUrl).then( (response) => response.blob(), ); - const inspectionPath = `${user.id}/${conversation.id}/inspection-preview-${toolCall.toolCallId}`; + const inspectionPath = inspectionPreviewStoragePath( + user.id, + conversation.id, + toolCall.toolCallId, + ); const { error: inspectionUploadError } = await supabase.storage .from('images') .upload(inspectionPath, inspectionBlob, { diff --git a/src/server/aiChat.ts b/src/server/aiChat.ts index b814a134..3ee38528 100644 --- a/src/server/aiChat.ts +++ b/src/server/aiChat.ts @@ -7,7 +7,11 @@ import { normalizeOpenRouterBaseUrl, } from '@shared/localModels'; import { cleanAssistantText, getParametricText } from '@shared/parametricParts'; -import { imageIdFromFilename, imageStoragePath } from '@shared/imageRefs'; +import { + imageIdFromFilename, + imageStoragePath, + inspectionPreviewStoragePath, +} from '@shared/imageRefs'; import { normalizeConversationSuggestions } from '@shared/suggestions'; import type { Conversation, Message, MeshFileType, Model } from '@shared/types'; import { @@ -47,6 +51,7 @@ import { type ChatProvider, } from './localChatConfig'; import { getAnonSupabaseClient } from './supabaseClient'; +import { findLatestBuildToolCallId } from './messageUtils'; /** * USD list price per **million** tokens, keyed by the same model IDs the @@ -341,14 +346,14 @@ type ChatProviders = { anthropic: () => AnthropicProvider; google: () => GoogleProvider; openrouter: () => OpenRouterProvider; - local: (baseUrl: string) => OpenRouterProvider; + local: (baseUrl: string, apiKey?: string) => OpenRouterProvider; }; function createChatProviders(): ChatProviders { let anthropic: AnthropicProvider | undefined; let google: GoogleProvider | undefined; let openrouter: OpenRouterProvider | undefined; - const localByUrl = new Map(); + const localByTarget = new Map(); return { anthropic: () => { if (!anthropic) { @@ -372,15 +377,18 @@ function createChatProviders(): ChatProviders { }); return openrouter; }, - local: (baseUrl: string) => { - let provider = localByUrl.get(baseUrl); + local: (baseUrl: string, apiKey?: string) => { + const resolvedApiKey = + apiKey?.trim() || env('OPENROUTER_API_KEY').trim() || 'local'; + const providerKey = `${baseUrl}::${resolvedApiKey}`; + let provider = localByTarget.get(providerKey); if (!provider) { provider = createOpenRouter({ baseURL: baseUrl, - apiKey: env('OPENROUTER_API_KEY').trim() || 'local', + apiKey: resolvedApiKey, compatibility: 'compatible', }); - localByUrl.set(baseUrl, provider); + localByTarget.set(providerKey, provider); } return provider; }, @@ -411,7 +419,9 @@ function buildChatModel( if (!rawBaseUrl) throw new Error(`No baseUrl configured for local model ${modelId}`); const baseUrl = normalizeOpenRouterBaseUrl(rawBaseUrl) ?? rawBaseUrl; - return { model: providers.local(baseUrl).chat(modelId) }; + return { + model: providers.local(baseUrl, modelConfig?.apiKey).chat(modelId), + }; } if (provider === 'openrouter') { @@ -911,90 +921,99 @@ async function buildHydratedMessages( conversation, modelId, provider, - previewPathForToolCall, }: { supabaseClient: SupabaseAnon; conversation: ConversationAccess; modelId: string; provider: ChatProvider; - previewPathForToolCall: (toolCallId: string) => string; }, ): Promise>> { - const isLocal = provider === 'local'; - const result: Array> = []; + const shouldAddInspection = + provider === 'local' && + conversation.type === 'parametric' && + chatModelSupportsVision(modelId); + const latestBuildToolCallId = shouldAddInspection + ? findLatestBuildToolCallId(messages) + : null; + + const hydrated = await Promise.all( + messages.map(async (message) => ({ + ...message, + parts: ( + await Promise.all( + message.parts.map(async (part) => { + if ( + part.type !== 'file' || + typeof part.mediaType !== 'string' || + !part.mediaType.startsWith('image/') || + part.url.startsWith('data:') + ) { + return part; + } + const imageId = imageIdFromFilename(part.filename); + if (!imageId) return null; + const downloaded = await downloadAsBase64( + supabaseClient, + 'images', + imageStoragePath(conversation.user_id, conversation.id, imageId), + ); + if (!downloaded) return null; + return { + ...part, + mediaType: downloaded.mediaType, + url: `data:${downloaded.mediaType};base64,${downloaded.base64}`, + }; + }), + ) + ).filter((part): part is NonNullable => part != null), + })), + ); - for (const message of messages) { - const hydratedParts = ( - await Promise.all( - message.parts.map(async (part) => { - if ( - part.type !== 'file' || - typeof part.mediaType !== 'string' || - !part.mediaType.startsWith('image/') || - part.url.startsWith('data:') - ) { - return part; - } - const imageId = imageIdFromFilename(part.filename); - if (!imageId) return null; - const downloaded = await downloadAsBase64( - supabaseClient, - 'images', - imageStoragePath(conversation.user_id, conversation.id, imageId), - ); - if (!downloaded) return null; - return { - ...part, - mediaType: downloaded.mediaType, - url: `data:${downloaded.mediaType};base64,${downloaded.base64}`, - }; - }), - ) - ).filter((part): part is NonNullable => part != null); + if (!latestBuildToolCallId) { + return hydrated.map(({ id: _id, ...message }) => message); + } - result.push({ ...message, parts: hydratedParts }); + const result: Array> = []; + for (const message of hydrated) { + const { id: _id, ...messageWithoutId } = message; + result.push(messageWithoutId); + + if (message.role !== 'assistant') continue; + + const hasLatestBuild = message.parts.some( + (part) => + part.type === 'tool-build_parametric_model' && + part.state === 'output-available' && + 'toolCallId' in part && + part.toolCallId === latestBuildToolCallId, + ); + if (!hasLatestBuild) continue; - // Local providers: inject image renders as user messages so they - // never end up inside `role: tool` output (strict OpenAI-compat servers - // reject images there). Only applies for parametric conversations where - // the model supports vision. - if ( - isLocal && - conversation.type === 'parametric' && - chatModelSupportsVision(modelId) && - message.role === 'assistant' - ) { - for (const part of message.parts) { - if ( - part.type !== 'tool-build_parametric_model' || - part.state !== 'output-available' || - !('toolCallId' in part) || - typeof part.toolCallId !== 'string' - ) { - continue; - } - const downloaded = await downloadAsBase64( - supabaseClient, - 'images', - previewPathForToolCall(part.toolCallId), - ); - if (!downloaded) continue; - result.push({ - role: 'user', - parts: [ - { - type: 'text', - text: 'Multi-view inspection render from the tool result above.', - }, - { - type: 'file', - mediaType: downloaded.mediaType, - url: `data:${downloaded.mediaType};base64,${downloaded.base64}`, - }, - ], - }); - } - } + const downloaded = await downloadAsBase64( + supabaseClient, + 'images', + inspectionPreviewStoragePath( + conversation.user_id, + conversation.id, + latestBuildToolCallId, + ), + ); + if (!downloaded) continue; + + result.push({ + role: 'user', + parts: [ + { + type: 'text', + text: 'Multi-view inspection render from the tool result above.', + }, + { + type: 'file', + mediaType: downloaded.mediaType, + url: `data:${downloaded.mediaType};base64,${downloaded.base64}`, + }, + ], + }); } return result; @@ -1027,15 +1046,14 @@ async function downloadAsBase64( } function parametricTools({ - previewPathForToolCall, supabaseClient, + conversation, modelId, }: { - previewPathForToolCall: (toolCallId: string) => string; supabaseClient: SupabaseAnon; + conversation: ConversationAccess; modelId: string; }) { - const isLocal = providerFor(modelId) === 'local'; return { build_parametric_model: { ...chatTools.build_parametric_model, @@ -1050,7 +1068,10 @@ function parametricTools({ output.inspection?.views.join(', ') ?? 'ISO, FRONT, BACK, LEFT, RIGHT, TOP, BOTTOM'; - if (!isLocal && chatModelSupportsVision(modelId)) { + if ( + providerFor(modelId) !== 'local' && + chatModelSupportsVision(modelId) + ) { // The client uploads a multi-view render of the compiled SCAD to a path // derived from toolCallId BEFORE sending the tool result (see // ChatSession's `onToolCall`). If for any reason the upload @@ -1059,7 +1080,11 @@ function parametricTools({ const downloaded = await downloadAsBase64( supabaseClient, 'images', - previewPathForToolCall(toolCallId), + inspectionPreviewStoragePath( + conversation.user_id, + conversation.id, + toolCallId, + ), ); const text = `${output.message}\nRendered inspection views: ${views}.\nMulti-view inspection image attached: ${downloaded ? 'yes' : 'no'}.`; @@ -1097,7 +1122,7 @@ function getAuxLanguageModel(providers: ChatProviders): LanguageModel | null { if (auxLocal?.baseUrl) { const baseUrl = normalizeOpenRouterBaseUrl(auxLocal.baseUrl) ?? auxLocal.baseUrl; - return providers.local(baseUrl).chat(auxLocal.id); + return providers.local(baseUrl, auxLocal.apiKey).chat(auxLocal.id); } if (hasAnthropicApiKey) return providers.anthropic()('claude-haiku-4-5'); return null; @@ -1204,8 +1229,7 @@ export async function handleAiChatRequest(req: Request) { ? creativeTools({ conversation, req, model: rawBody.model }) : parametricTools({ supabaseClient, - previewPathForToolCall: (toolCallId) => - `${user.id}/${conversation.id}/inspection-preview-${toolCallId}`, + conversation, modelId: actualModelId, }); @@ -1257,8 +1281,6 @@ export async function handleAiChatRequest(req: Request) { // Resolve the actual model ID the request will run against. const resolvedProvider = providerFor(actualModelId); - const previewPathForToolCall = (toolCallId: string) => - `${user.id}/${conversation.id}/inspection-preview-${toolCallId}`; // Rehydrate image file parts before handing them to the model. The // persisted `url` is a storage reference (or, for the oldest backfilled @@ -1272,15 +1294,14 @@ export async function handleAiChatRequest(req: Request) { // // For local providers on parametric conversations, strict OpenAI-compatible // servers reject images inside `role: tool` messages. We instead insert a - // follow-up `role: user` message carrying the inspection render after each - // resolved build tool call so `convertToModelMessages` never needs to put - // images in tool output. + // follow-up `role: user` message carrying the latest inspection render after + // the most recent build tool call so `convertToModelMessages` never needs to + // put images in tool output. const hydratedMessages = await buildHydratedMessages(branchMessages, { supabaseClient, conversation, modelId: actualModelId, provider: resolvedProvider, - previewPathForToolCall, }); const modelMessages = await convertToModelMessages( diff --git a/src/server/localChatConfig.ts b/src/server/localChatConfig.ts index dd9d4e2e..616b4022 100644 --- a/src/server/localChatConfig.ts +++ b/src/server/localChatConfig.ts @@ -33,12 +33,11 @@ const projectRoot = path.resolve( '../..', ); -// Cached in production; re-read on each request in dev so `local-models.json` -// edits take effect without restarting the dev server. +// Cached after first read; restart the dev server after editing `local-models.json`. let cachedCatalog: LocalModelConfig[] | undefined; function loadCatalogFromDisk(): LocalModelConfig[] { - if (cachedCatalog && import.meta.env.PROD) return cachedCatalog; + if (cachedCatalog) return cachedCatalog; const userPath = path.join(projectRoot, 'local-models.json'); if (!fs.existsSync(userPath)) { diff --git a/src/server/messageUtils.test.ts b/src/server/messageUtils.test.ts new file mode 100644 index 00000000..e6da449a --- /dev/null +++ b/src/server/messageUtils.test.ts @@ -0,0 +1,90 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; +import type { AppUIMessage } from '@shared/chatAi'; +import { findLatestBuildToolCallId } from './messageUtils.ts'; + +function buildToolPart( + toolCallId: string, + state: + | 'output-available' + | 'input-available' + | 'output-error' = 'output-available', +): AppUIMessage['parts'][number] { + return { + type: 'tool-build_parametric_model', + toolCallId, + state, + input: { code: 'cube(1);' }, + ...(state === 'output-available' + ? { + output: { + status: 'success' as const, + message: 'ok', + inspection: { views: ['ISO'] as const, imageAttached: true }, + }, + } + : state === 'output-error' + ? { errorText: 'failed' } + : {}), + } as AppUIMessage['parts'][number]; +} + +describe('findLatestBuildToolCallId', () => { + it('returns null for an empty branch', () => { + assert.equal(findLatestBuildToolCallId([]), null); + }); + + it('returns the only completed build tool call', () => { + const messages: AppUIMessage[] = [ + { + id: 'a1', + role: 'assistant', + parts: [buildToolPart('call-1')], + }, + ]; + assert.equal(findLatestBuildToolCallId(messages), 'call-1'); + }); + + it('returns the latest build across assistant messages', () => { + const messages: AppUIMessage[] = [ + { + id: 'a1', + role: 'assistant', + parts: [buildToolPart('call-1')], + }, + { id: 'u1', role: 'user', parts: [{ type: 'text', text: 'taller' }] }, + { + id: 'a2', + role: 'assistant', + parts: [buildToolPart('call-2')], + }, + ]; + assert.equal(findLatestBuildToolCallId(messages), 'call-2'); + }); + + it('returns the last completed build part within one assistant message', () => { + const messages: AppUIMessage[] = [ + { + id: 'a1', + role: 'assistant', + parts: [buildToolPart('call-1'), buildToolPart('call-2')], + }, + ]; + assert.equal(findLatestBuildToolCallId(messages), 'call-2'); + }); + + it('ignores non-output build tool parts', () => { + const messages: AppUIMessage[] = [ + { + id: 'a1', + role: 'assistant', + parts: [ + buildToolPart('call-error', 'output-error'), + buildToolPart('call-pending', 'input-available'), + buildToolPart('call-done'), + ], + }, + ]; + assert.equal(findLatestBuildToolCallId(messages), 'call-done'); + }); +}); diff --git a/src/server/messageUtils.ts b/src/server/messageUtils.ts index adda5a40..0df23eed 100644 --- a/src/server/messageUtils.ts +++ b/src/server/messageUtils.ts @@ -1,4 +1,5 @@ -import { env } from './env'; +import { env } from './env.ts'; +import type { AppUIMessage } from '@shared/chatAi'; export function reformatSignedUrl(signedUrl: string): string { const supabaseHost = ( @@ -8,3 +9,33 @@ export function reformatSignedUrl(signedUrl: string): string { const url = new URL(signedUrl); return `${supabaseHost}${url.pathname}${url.search}`; } + +/** + * Latest `build_parametric_model` with `output-available` on the branch. + * `buildHydratedMessages` uses this so local parametric only re-downloads + * one inspection preview per request (newest-to-oldest walk). + */ +export function findLatestBuildToolCallId( + messages: AppUIMessage[], +): string | null { + for (let index = messages.length - 1; index >= 0; index--) { + const message = messages[index]; + if (message.role !== 'assistant') continue; + for ( + let partIndex = message.parts.length - 1; + partIndex >= 0; + partIndex-- + ) { + const part = message.parts[partIndex]; + if ( + part.type === 'tool-build_parametric_model' && + part.state === 'output-available' && + 'toolCallId' in part && + typeof part.toolCallId === 'string' + ) { + return part.toolCallId; + } + } + } + return null; +} From 48cd9d7df86f3496cd4388c3dfd8aa8000a7a01a Mon Sep 17 00:00:00 2001 From: nathan-t4 <19811248+nathan-t4@users.noreply.github.com> Date: Wed, 24 Jun 2026 21:31:55 +0800 Subject: [PATCH 05/11] Save local model api keys in .env.local files --- README.md | 6 ++-- local-models.json.example | 13 +++++++- shared/localModels.test.ts | 62 +++++++++++++++++++++++++++++++++-- src/server/aiChat.ts | 12 ++++--- src/server/localChatConfig.ts | 12 +++++++ 5 files changed, 96 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 319df747..ab09c249 100644 --- a/README.md +++ b/README.md @@ -241,7 +241,7 @@ To add your own local models (LM Studio, Ollama, LiteLLM, etc.) alongside the cl "name": "DeepSeek v4 Flash", "description": "Third Party LLM", "baseUrl": "https://api.deepseek.com", - "apiKey": "", + "apiKey": "DEEPSEEK_API_KEY", "supportsTools": true, "supportsThinking": true, "supportsVision": false, @@ -249,7 +249,9 @@ To add your own local models (LM Studio, Ollama, LiteLLM, etc.) alongside the cl } ``` -3. Optionally set `OPENROUTER_API_KEY="local"` in `.env.local` if your local server expects an API key header (many accept any value). **Note:** this value is also used for OpenRouter cloud model calls, so only use a dummy value if you are not using any OpenRouter-hosted models. +3. Set local model API keys in `.env.local`. + - `apiKey` in `local-models.json` should be the env var name, not the secret value. + - Example: `"apiKey": "DEEPSEEK_API_KEY"` and then set `DEEPSEEK_API_KEY=""` in `.env.local`. 4. Restart `npm run dev`. Models with a `baseUrl` appear in the parametric model picker. diff --git a/local-models.json.example b/local-models.json.example index e00941c3..e36b9f6b 100644 --- a/local-models.json.example +++ b/local-models.json.example @@ -8,5 +8,16 @@ "supportsThinking": false, "supportsVision": true, "useForAux": true - } + }, + { + "id": "deepseek-v4-flash", + "name": "DeepSeek v4 Flash", + "description": "Third Party LLM", + "baseUrl": "https://api.deepseek.com", + "apiKey": "DEEPSEEK_API_KEY", + "supportsTools": true, + "supportsThinking": true, + "supportsVision": false, + "useForAux": false + }, ] diff --git a/shared/localModels.test.ts b/shared/localModels.test.ts index 2062b624..c76a8885 100644 --- a/shared/localModels.test.ts +++ b/shared/localModels.test.ts @@ -33,7 +33,7 @@ describe('parseLocalModelsJson', () => { id: 'qwen2.5-coder-7b', name: 'Qwen', description: 'Local coder', - apiKey: 'test-key', + apiKey: 'TEST_ENV_KEY_NAME', supportsTools: true, }, { id: '', name: 'Bad', description: 'x' }, @@ -43,7 +43,30 @@ describe('parseLocalModelsJson', () => { id: 'qwen2.5-coder-7b', name: 'Qwen', description: 'Local coder', - apiKey: 'test-key', + apiKey: 'TEST_ENV_KEY_NAME', + supportsTools: true, + }, + ], + ); + }); + + it('accepts entries without apiKey', () => { + assert.deepEqual( + parseLocalModelsJson([ + { + id: 'qwen3-vl-30b', + name: 'Qwen3 VL 30B', + description: 'Local VLM', + baseUrl: 'http://localhost:8004/v1', + supportsTools: true, + }, + ]), + [ + { + id: 'qwen3-vl-30b', + name: 'Qwen3 VL 30B', + description: 'Local VLM', + baseUrl: 'http://localhost:8004/v1', supportsTools: true, }, ], @@ -131,6 +154,41 @@ describe('getLocalModels', () => { ], ); }); + + it('keeps models with and without apiKey when baseUrl exists', () => { + assert.deepEqual( + getLocalModels([ + { + id: 'with-api-key', + name: 'With API key', + description: 'Remote OpenAI-compatible', + baseUrl: 'https://api.example.com', + apiKey: 'EXAMPLE_API_KEY', + }, + { + id: 'without-api-key', + name: 'Without API key', + description: 'Local server with no auth', + baseUrl: 'http://localhost:1234', + }, + ]), + [ + { + id: 'with-api-key', + name: 'With API key', + description: 'Remote OpenAI-compatible', + baseUrl: 'https://api.example.com', + apiKey: 'EXAMPLE_API_KEY', + }, + { + id: 'without-api-key', + name: 'Without API key', + description: 'Local server with no auth', + baseUrl: 'http://localhost:1234', + }, + ], + ); + }); }); describe('isLocalModelId', () => { diff --git a/src/server/aiChat.ts b/src/server/aiChat.ts index 3ee38528..1172f604 100644 --- a/src/server/aiChat.ts +++ b/src/server/aiChat.ts @@ -47,6 +47,7 @@ import { getLocalChatState, isActiveLocalModel, isMissingLocalBaseUrl, + localApiKeyForModel, providerFor, type ChatProvider, } from './localChatConfig'; @@ -378,8 +379,7 @@ function createChatProviders(): ChatProviders { return openrouter; }, local: (baseUrl: string, apiKey?: string) => { - const resolvedApiKey = - apiKey?.trim() || env('OPENROUTER_API_KEY').trim() || 'local'; + const resolvedApiKey = apiKey?.trim() || 'local'; const providerKey = `${baseUrl}::${resolvedApiKey}`; let provider = localByTarget.get(providerKey); if (!provider) { @@ -420,7 +420,9 @@ function buildChatModel( throw new Error(`No baseUrl configured for local model ${modelId}`); const baseUrl = normalizeOpenRouterBaseUrl(rawBaseUrl) ?? rawBaseUrl; return { - model: providers.local(baseUrl, modelConfig?.apiKey).chat(modelId), + model: providers + .local(baseUrl, localApiKeyForModel(modelConfig)) + .chat(modelId), }; } @@ -1122,7 +1124,9 @@ function getAuxLanguageModel(providers: ChatProviders): LanguageModel | null { if (auxLocal?.baseUrl) { const baseUrl = normalizeOpenRouterBaseUrl(auxLocal.baseUrl) ?? auxLocal.baseUrl; - return providers.local(baseUrl, auxLocal.apiKey).chat(auxLocal.id); + return providers + .local(baseUrl, localApiKeyForModel(auxLocal)) + .chat(auxLocal.id); } if (hasAnthropicApiKey) return providers.anthropic()('claude-haiku-4-5'); return null; diff --git a/src/server/localChatConfig.ts b/src/server/localChatConfig.ts index 616b4022..e508215c 100644 --- a/src/server/localChatConfig.ts +++ b/src/server/localChatConfig.ts @@ -84,6 +84,18 @@ export function getActiveLocalModel( ); } +// Resolve local API key from env without exposing secrets to the client bundle. +// If `apiKey` is set in `local-models.json`, it is interpreted as the env var +// name (for example `DEEPSEEK_API_KEY`), not a raw secret. +export function localApiKeyForModel( + model: LocalModelConfig, +): string | undefined { + const configuredName = model.apiKey?.trim(); + if (!configuredName) return undefined; + const configured = env(configuredName).trim(); + return configured || undefined; +} + // Client may show catalog ids whose baseUrl is missing on the server — reject // those requests with a clear error. export function isMissingLocalBaseUrl(modelId: string): boolean { From f0452d52f9b01bd4ebfc0a4ba3b2c5b4f231c8f0 Mon Sep 17 00:00:00 2001 From: Nathan <19811248+nathan-t4@users.noreply.github.com> Date: Wed, 24 Jun 2026 21:38:56 +0800 Subject: [PATCH 06/11] Update local-models.json.example Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- local-models.json.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/local-models.json.example b/local-models.json.example index e36b9f6b..f79b7d99 100644 --- a/local-models.json.example +++ b/local-models.json.example @@ -19,5 +19,5 @@ "supportsThinking": true, "supportsVision": false, "useForAux": false - }, + } ] From d776fbab6e198859a3fabd98eb21d4463125f900 Mon Sep 17 00:00:00 2001 From: nathan-t4 <19811248+nathan-t4@users.noreply.github.com> Date: Fri, 26 Jun 2026 18:11:24 +0800 Subject: [PATCH 07/11] Local models not loaded client side, check before hydrating local messages --- local-models.json.example | 2 +- shared/localModels.test.ts | 13 ++++++ shared/localModels.ts | 29 ++++++------- src/components/TextAreaChat.tsx | 9 ++-- src/components/chat/MessageBubble.tsx | 6 ++- src/hooks/useLocalModels.ts | 19 +++++++++ src/lib/utils.ts | 24 +++-------- src/routeTree.gen.ts | 21 +++++++++ src/routes/api/local-models.ts | 16 +++++++ src/server/aiChat.ts | 61 ++++++++++++++++++++++++--- 10 files changed, 153 insertions(+), 47 deletions(-) create mode 100644 src/hooks/useLocalModels.ts create mode 100644 src/routes/api/local-models.ts diff --git a/local-models.json.example b/local-models.json.example index e36b9f6b..f79b7d99 100644 --- a/local-models.json.example +++ b/local-models.json.example @@ -19,5 +19,5 @@ "supportsThinking": true, "supportsVision": false, "useForAux": false - }, + } ] diff --git a/shared/localModels.test.ts b/shared/localModels.test.ts index c76a8885..4feae3ae 100644 --- a/shared/localModels.test.ts +++ b/shared/localModels.test.ts @@ -226,6 +226,19 @@ describe('localModelToPickerConfig', () => { disabled: false, }); }); + + it('does not expose apiKey or baseUrl in picker config', () => { + const picker = localModelToPickerConfig({ + id: 'secure-model', + name: 'Secure', + description: 'Server-only key', + baseUrl: 'https://example.local/v1', + apiKey: 'DO_NOT_EXPOSE', + }); + + assert.equal('apiKey' in picker, false); + assert.equal('baseUrl' in picker, false); + }); }); describe('normalizeOpenRouterBaseUrl', () => { diff --git a/shared/localModels.ts b/shared/localModels.ts index 030b4423..9def1b14 100644 --- a/shared/localModels.ts +++ b/shared/localModels.ts @@ -1,36 +1,35 @@ /** * Pure routing logic for the local OpenAI-compatible model catalog * (`local-models.json`). No env or disk I/O — `localChatConfig.ts` wires - * server env + file reads; `utils.ts` loads the catalog for the client picker. + * server env + file reads; the client fetches picker entries from `/api/local-models`. */ import { z } from 'zod'; -// Shape of each entry in `local-models.json`. -const localModelSchema = z.object({ +const modelCoreSchema = z.object({ id: z.string().min(1), name: z.string().min(1), description: z.string().min(1), - baseUrl: z.string().optional(), - apiKey: z.string().optional(), supportsTools: z.boolean().optional(), supportsThinking: z.boolean().optional(), supportsVision: z.boolean().optional(), +}); + +// Shape of each entry in `local-models.json`. +export const localModelSchema = modelCoreSchema.extend({ + baseUrl: z.string().optional(), + apiKey: z.string().optional(), useForAux: z.boolean().optional(), }); export type LocalModelConfig = z.infer; +export const pickerModelSchema = modelCoreSchema.extend({ + provider: z.string().optional(), + disabled: z.boolean().optional(), +}); + // UI model-picker shape; cloud entries in `utils.ts` use the same fields. -export type PickerModelConfig = { - id: string; - name: string; - description: string; - provider?: string; - supportsTools?: boolean; - supportsThinking?: boolean; - supportsVision?: boolean; - disabled?: boolean; -}; +export type PickerModelConfig = z.infer; export type ChatProvider = 'anthropic' | 'google' | 'openrouter' | 'local'; diff --git a/src/components/TextAreaChat.tsx b/src/components/TextAreaChat.tsx index 23a435d5..a950eb59 100644 --- a/src/components/TextAreaChat.tsx +++ b/src/components/TextAreaChat.tsx @@ -20,9 +20,9 @@ import { import { cn, CREATIVE_MODELS, - PARAMETRIC_MODELS, parametricModelSupportsVision, } from '@/lib/utils'; +import { useParametricModels } from '@/hooks/useLocalModels.ts'; import { CreativeModel, MeshFileType, Model } from '@shared/types'; import type { AppUIMessage } from '@shared/chatAi'; import { imageFilePartUrl } from '@shared/imageRefs'; @@ -652,12 +652,13 @@ function TextAreaChat({ }, }; + const parametricModels = useParametricModels(); const memoizedModels = useMemo(() => { if (type === 'creative') { return CREATIVE_MODELS; } - return PARAMETRIC_MODELS; - }, [type]); + return parametricModels; + }, [type, parametricModels]); // ------------------------------------------------------------ // Placeholder – Typed-out Animation @@ -1594,7 +1595,7 @@ function TextAreaChat({
{(type !== 'parametric' || - parametricModelSupportsVision(model)) && ( + parametricModelSupportsVision(model, memoizedModels)) && (
>(new Set()); const lastParametricBuildIndex = useMemo(() => { if (conversation.type !== 'parametric') return -1; diff --git a/src/hooks/useLocalModels.ts b/src/hooks/useLocalModels.ts new file mode 100644 index 00000000..c5ff30da --- /dev/null +++ b/src/hooks/useLocalModels.ts @@ -0,0 +1,19 @@ +import { useMemo } from 'react'; +import { useQuery } from '@tanstack/react-query'; +import { z } from 'zod'; +import { pickerModelSchema } from '@shared/localModels'; +import { PARAMETRIC_MODELS } from '@/lib/utils'; +import { apiJson } from '@/services/api'; +import type { ModelConfig } from '@/types/misc'; + +const localModelsSchema = z.array(pickerModelSchema); + +export function useParametricModels(): ModelConfig[] { + const { data: localModels = [] } = useQuery({ + queryKey: ['local-models'], + queryFn: () => apiJson('local-models', {}, localModelsSchema), + staleTime: 5 * 60 * 1000, + }); + + return useMemo(() => [...PARAMETRIC_MODELS, ...localModels], [localModels]); +} diff --git a/src/lib/utils.ts b/src/lib/utils.ts index 6cae27fb..b05c4df5 100644 --- a/src/lib/utils.ts +++ b/src/lib/utils.ts @@ -1,7 +1,6 @@ import { clsx, type ClassValue } from 'clsx'; import { twMerge } from 'tailwind-merge'; import { Parameter } from '@shared/types'; -import { getLocalModels, localModelToPickerConfig } from '@shared/localModels'; import { ModelConfig } from '../types/misc.ts'; export function cn(...inputs: ClassValue[]) { @@ -275,21 +274,7 @@ const CLOUD_PARAMETRIC_MODELS: ModelConfig[] = [ }, ]; -const localModelsRaw = Object.values( - import.meta.glob('../../local-models.json', { - eager: true, - import: 'default', - }), -)[0] as unknown; - -const LOCAL_PARAMETRIC_MODELS: ModelConfig[] = getLocalModels( - localModelsRaw, -).map(localModelToPickerConfig); - -export const PARAMETRIC_MODELS: ModelConfig[] = [ - ...CLOUD_PARAMETRIC_MODELS, - ...LOCAL_PARAMETRIC_MODELS, -]; +export const PARAMETRIC_MODELS: ModelConfig[] = [...CLOUD_PARAMETRIC_MODELS]; export const CREATIVE_MODELS: ModelConfig[] = [ { @@ -315,7 +300,10 @@ export const CREATIVE_MODELS: ModelConfig[] = [ // Whether the selected parametric model can accept image / STL-render inputs. // Unknown ids (e.g. historical messages tagged with a removed model) fall back // to `true` so older saved rows still render normally. -export function parametricModelSupportsVision(modelId: string): boolean { - const cfg = PARAMETRIC_MODELS.find((m) => m.id === modelId); +export function parametricModelSupportsVision( + modelId: string, + models: ModelConfig[] = PARAMETRIC_MODELS, +): boolean { + const cfg = models.find((m) => m.id === modelId); return cfg?.supportsVision !== false; } diff --git a/src/routeTree.gen.ts b/src/routeTree.gen.ts index ad98721d..0d725953 100644 --- a/src/routeTree.gen.ts +++ b/src/routeTree.gen.ts @@ -23,6 +23,7 @@ import { Route as ApiTitleGeneratorRouteImport } from './routes/api/title-genera import { Route as ApiPromptGeneratorRouteImport } from './routes/api/prompt-generator' import { Route as ApiParametricChatRouteImport } from './routes/api/parametric-chat' import { Route as ApiMeshRouteImport } from './routes/api/mesh' +import { Route as ApiLocalModelsRouteImport } from './routes/api/local-models' import { Route as ApiFalWebhookRouteImport } from './routes/api/fal-webhook' import { Route as ApiDeleteUserRouteImport } from './routes/api/delete-user' import { Route as ApiCreativeChatRouteImport } from './routes/api/creative-chat' @@ -108,6 +109,11 @@ const ApiMeshRoute = ApiMeshRouteImport.update({ path: '/api/mesh', getParentRoute: () => rootRouteImport, } as any) +const ApiLocalModelsRoute = ApiLocalModelsRouteImport.update({ + id: '/api/local-models', + path: '/api/local-models', + getParentRoute: () => rootRouteImport, +} as any) const ApiFalWebhookRoute = ApiFalWebhookRouteImport.update({ id: '/api/fal-webhook', path: '/api/fal-webhook', @@ -201,6 +207,7 @@ export interface FileRoutesByFullPath { '/api/creative-chat': typeof ApiCreativeChatRoute '/api/delete-user': typeof ApiDeleteUserRoute '/api/fal-webhook': typeof ApiFalWebhookRoute + '/api/local-models': typeof ApiLocalModelsRoute '/api/mesh': typeof ApiMeshRoute '/api/parametric-chat': typeof ApiParametricChatRoute '/api/prompt-generator': typeof ApiPromptGeneratorRoute @@ -230,6 +237,7 @@ export interface FileRoutesByTo { '/api/creative-chat': typeof ApiCreativeChatRoute '/api/delete-user': typeof ApiDeleteUserRoute '/api/fal-webhook': typeof ApiFalWebhookRoute + '/api/local-models': typeof ApiLocalModelsRoute '/api/mesh': typeof ApiMeshRoute '/api/parametric-chat': typeof ApiParametricChatRoute '/api/prompt-generator': typeof ApiPromptGeneratorRoute @@ -261,6 +269,7 @@ export interface FileRoutesById { '/api/creative-chat': typeof ApiCreativeChatRoute '/api/delete-user': typeof ApiDeleteUserRoute '/api/fal-webhook': typeof ApiFalWebhookRoute + '/api/local-models': typeof ApiLocalModelsRoute '/api/mesh': typeof ApiMeshRoute '/api/parametric-chat': typeof ApiParametricChatRoute '/api/prompt-generator': typeof ApiPromptGeneratorRoute @@ -293,6 +302,7 @@ export interface FileRouteTypes { | '/api/creative-chat' | '/api/delete-user' | '/api/fal-webhook' + | '/api/local-models' | '/api/mesh' | '/api/parametric-chat' | '/api/prompt-generator' @@ -322,6 +332,7 @@ export interface FileRouteTypes { | '/api/creative-chat' | '/api/delete-user' | '/api/fal-webhook' + | '/api/local-models' | '/api/mesh' | '/api/parametric-chat' | '/api/prompt-generator' @@ -352,6 +363,7 @@ export interface FileRouteTypes { | '/api/creative-chat' | '/api/delete-user' | '/api/fal-webhook' + | '/api/local-models' | '/api/mesh' | '/api/parametric-chat' | '/api/prompt-generator' @@ -382,6 +394,7 @@ export interface RootRouteChildren { ApiCreativeChatRoute: typeof ApiCreativeChatRoute ApiDeleteUserRoute: typeof ApiDeleteUserRoute ApiFalWebhookRoute: typeof ApiFalWebhookRoute + ApiLocalModelsRoute: typeof ApiLocalModelsRoute ApiMeshRoute: typeof ApiMeshRoute ApiParametricChatRoute: typeof ApiParametricChatRoute ApiPromptGeneratorRoute: typeof ApiPromptGeneratorRoute @@ -489,6 +502,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof ApiMeshRouteImport parentRoute: typeof rootRouteImport } + '/api/local-models': { + id: '/api/local-models' + path: '/api/local-models' + fullPath: '/api/local-models' + preLoaderRoute: typeof ApiLocalModelsRouteImport + parentRoute: typeof rootRouteImport + } '/api/fal-webhook': { id: '/api/fal-webhook' path: '/api/fal-webhook' @@ -649,6 +669,7 @@ const rootRouteChildren: RootRouteChildren = { ApiCreativeChatRoute: ApiCreativeChatRoute, ApiDeleteUserRoute: ApiDeleteUserRoute, ApiFalWebhookRoute: ApiFalWebhookRoute, + ApiLocalModelsRoute: ApiLocalModelsRoute, ApiMeshRoute: ApiMeshRoute, ApiParametricChatRoute: ApiParametricChatRoute, ApiPromptGeneratorRoute: ApiPromptGeneratorRoute, diff --git a/src/routes/api/local-models.ts b/src/routes/api/local-models.ts new file mode 100644 index 00000000..ef27024f --- /dev/null +++ b/src/routes/api/local-models.ts @@ -0,0 +1,16 @@ +import { createFileRoute } from '@tanstack/react-router'; +import { localModelToPickerConfig } from '@shared/localModels'; +import { getLocalChatState } from '@/server/localChatConfig'; +import { json, preflight } from '@/server/api'; + +export const Route = createFileRoute('/api/local-models')({ + server: { + handlers: { + OPTIONS: preflight, + GET: async () => { + const { activeCatalog } = getLocalChatState(); + return json(activeCatalog.map(localModelToPickerConfig)); + }, + }, + }, +}); diff --git a/src/server/aiChat.ts b/src/server/aiChat.ts index 1172f604..184c8173 100644 --- a/src/server/aiChat.ts +++ b/src/server/aiChat.ts @@ -991,16 +991,30 @@ async function buildHydratedMessages( ); if (!hasLatestBuild) continue; + const inspectionPath = inspectionPreviewStoragePath( + conversation.user_id, + conversation.id, + latestBuildToolCallId, + ); const downloaded = await downloadAsBase64( supabaseClient, 'images', - inspectionPreviewStoragePath( - conversation.user_id, - conversation.id, - latestBuildToolCallId, - ), + inspectionPath, ); - if (!downloaded) continue; + if (!downloaded) { + logError(new Error('Failed to hydrate inspection preview image'), { + functionName: 'ai-chat', + statusCode: 500, + userId: conversation.user_id, + conversationId: conversation.id, + additionalContext: { + operation: 'inspection_hydration_download', + toolCallId: latestBuildToolCallId, + storagePath: inspectionPath, + }, + }); + continue; + } result.push({ role: 'user', @@ -1047,6 +1061,17 @@ async function downloadAsBase64( return { base64: btoa(binary), mediaType }; } +async function storageObjectExists( + supabaseClient: SupabaseAnon, + bucket: string, + path: string, +): Promise { + const { data, error } = await supabaseClient.storage + .from(bucket) + .createSignedUrl(path, 60); + return !error && !!data?.signedUrl; +} + function parametricTools({ supabaseClient, conversation, @@ -1108,7 +1133,29 @@ function parametricTools({ } else { // For models that do not support multimodal tools result messages, // the rendered inspection views are appended as user messages in buildHydratedMessages - const imageAttached = output.inspection?.imageAttached === true; + + // Text-only models never receive inspection images + if (!chatModelSupportsVision(modelId)) { + return { type: 'text' as const, value: output.message }; + } + + // Local vision models rely on the hydration path for the actual image. + // First check storage prescence to determine output message. + let imageAttached = output.inspection?.imageAttached === true; + if ( + providerFor(modelId) === 'local' && + chatModelSupportsVision(modelId) + ) { + imageAttached = await storageObjectExists( + supabaseClient, + 'images', + inspectionPreviewStoragePath( + conversation.user_id, + conversation.id, + toolCallId, + ), + ); + } const text = `${output.message}\nRendered inspection views: ${views}.\nMulti-view inspection image attached: ${imageAttached ? 'yes' : 'no'}.`; return { type: 'text' as const, value: text }; } From 6f43c9ab3a7fb88945eac98d16f8438a4bccedee Mon Sep 17 00:00:00 2001 From: nathan-t4 <19811248+nathan-t4@users.noreply.github.com> Date: Sat, 27 Jun 2026 06:58:47 +0800 Subject: [PATCH 08/11] Cap max length of model selector dropdown --- src/components/ModelSelector.tsx | 75 ++++++++++++++------------- src/components/chat/MessageBubble.tsx | 26 +++++----- 2 files changed, 54 insertions(+), 47 deletions(-) diff --git a/src/components/ModelSelector.tsx b/src/components/ModelSelector.tsx index ba315c35..30970d22 100644 --- a/src/components/ModelSelector.tsx +++ b/src/components/ModelSelector.tsx @@ -7,6 +7,7 @@ import { DropdownMenuTrigger, } from '@/components/ui/dropdown-menu'; import { Button } from '@/components/ui/button'; +import { ScrollArea } from '@/components/ui/scroll-area'; import { cn } from '@/lib/utils'; import { Model } from '@shared/types'; import { ModelConfig } from '../types/misc.ts'; @@ -174,7 +175,7 @@ export function ModelSelector({ - {models.map((model) => ( - { - onModelChange(model.id); - setIsDropdownOpen(false); - event.stopPropagation(); - }} - disabled={!!model.disabled} - > -
-
- - {model.name} - -
-

+

+ {models.map((model) => ( + { + onModelChange(model.id); + setIsDropdownOpen(false); + event.stopPropagation(); + }} + disabled={!!model.disabled} > - {model.description} -

-
- - ))} +
+
+ + {model.name} + +
+

+ {model.description} +

+
+ + ))} +
+ ); diff --git a/src/components/chat/MessageBubble.tsx b/src/components/chat/MessageBubble.tsx index 2d5b9d31..21c64941 100644 --- a/src/components/chat/MessageBubble.tsx +++ b/src/components/chat/MessageBubble.tsx @@ -838,18 +838,20 @@ function RetryModelDropdown({ align="end" className="w-48 rounded-lg border border-adam-neutral-700 bg-adam-neutral-800 p-1" > - {modelOptions.map((option) => ( - { - onRetry(option.id); - setIsOpen(false); - }} - > - {option.name} - - ))} + + {modelOptions.map((option) => ( + { + onRetry(option.id); + setIsOpen(false); + }} + > + {option.name} + + ))} + ); From 0af5403a02d37fb2c417a487aa8b7cee361edef5 Mon Sep 17 00:00:00 2001 From: nathan-t4 <19811248+nathan-t4@users.noreply.github.com> Date: Sun, 28 Jun 2026 11:20:27 +0800 Subject: [PATCH 09/11] Fix billing tokens for local / third party providers --- src/hooks/useLocalModels.ts | 15 ++++- src/server/aiChat.ts | 110 +++++++++++++++++++----------------- src/views/EditorView.tsx | 4 +- src/views/PromptView.tsx | 6 +- 4 files changed, 78 insertions(+), 57 deletions(-) diff --git a/src/hooks/useLocalModels.ts b/src/hooks/useLocalModels.ts index c5ff30da..8bed6eeb 100644 --- a/src/hooks/useLocalModels.ts +++ b/src/hooks/useLocalModels.ts @@ -1,19 +1,28 @@ import { useMemo } from 'react'; import { useQuery } from '@tanstack/react-query'; import { z } from 'zod'; -import { pickerModelSchema } from '@shared/localModels'; +import { isLocalModelId, pickerModelSchema } from '@shared/localModels'; import { PARAMETRIC_MODELS } from '@/lib/utils'; import { apiJson } from '@/services/api'; import type { ModelConfig } from '@/types/misc'; const localModelsSchema = z.array(pickerModelSchema); -export function useParametricModels(): ModelConfig[] { - const { data: localModels = [] } = useQuery({ +function useLocalModelsQuery() { + return useQuery({ queryKey: ['local-models'], queryFn: () => apiJson('local-models', {}, localModelsSchema), staleTime: 5 * 60 * 1000, }); +} + +export function useIsLocalModel(modelId: string): boolean { + const { data: localModels = [] } = useLocalModelsQuery(); + return isLocalModelId(modelId, localModels); +} + +export function useParametricModels(): ModelConfig[] { + const { data: localModels = [] } = useLocalModelsQuery(); return useMemo(() => [...PARAMETRIC_MODELS, ...localModels], [localModels]); } diff --git a/src/server/aiChat.ts b/src/server/aiChat.ts index 184c8173..4ae5cf8c 100644 --- a/src/server/aiChat.ts +++ b/src/server/aiChat.ts @@ -593,6 +593,7 @@ function billingTokensFromUsage( modelId: string, usage: LanguageModelUsage, ): number { + if (isActiveLocalModel(modelId)) return 0; const usdCost = usdCostFromUsage(modelId, usage) * billingMultiplier(); return Math.max(1, Math.ceil(usdCost / USD_PER_BILLING_TOKEN)); } @@ -1238,35 +1239,6 @@ export async function handleAiChatRequest(req: Request) { ); } - // Pre-flight balance gate. A chat costs at least 1 billing token, so a - // total of 0 means we cannot let the stream start. We don't try to - // estimate the exact cost up front — chat is variable, and the billing - // service drains the remainder to zero if the actual usage exceeds - // what's left (see onFinish below). - try { - const status = await billing.getStatus(user.email); - if (status.tokens.total <= 0) { - return jsonResponse( - { - error: 'insufficient_tokens', - code: 'insufficient_tokens', - tokensRequired: 1, - tokensAvailable: 0, - }, - 402, - ); - } - } catch (error) { - logError(error, { - functionName: 'ai-chat', - statusCode: error instanceof BillingClientError ? error.status : 502, - userId: user.id, - conversationId: conversation.id, - additionalContext: { operation: 'billing_preflight' }, - }); - return jsonResponse({ error: 'Billing service unavailable' }, 503); - } - const actualModelId = chatModel(conversation, rawBody.model); if (isMissingLocalBaseUrl(actualModelId)) { return jsonResponse( @@ -1275,6 +1247,38 @@ export async function handleAiChatRequest(req: Request) { ); } + // Pre-flight balance gate for billable (cloud) models. Local-catalog + // models are billing-exempt. Cloud chat costs at least 1 billing token, + // so a total of 0 means we cannot let the stream start. We don't try to + // estimate the exact cost up front — chat is variable, and the billing + // service drains the remainder to zero if the actual usage exceeds + // what's left (see onFinish below). + if (!isActiveLocalModel(actualModelId)) { + try { + const status = await billing.getStatus(user.email); + if (status.tokens.total <= 0) { + return jsonResponse( + { + error: 'insufficient_tokens', + code: 'insufficient_tokens', + tokensRequired: 1, + tokensAvailable: 0, + }, + 402, + ); + } + } catch (error) { + logError(error, { + functionName: 'ai-chat', + statusCode: error instanceof BillingClientError ? error.status : 502, + userId: user.id, + conversationId: conversation.id, + additionalContext: { operation: 'billing_preflight' }, + }); + return jsonResponse({ error: 'Billing service unavailable' }, 503); + } + } + const tools = conversation.type === 'creative' ? creativeTools({ conversation, req, model: rawBody.model }) @@ -1699,29 +1703,31 @@ export async function handleAiChatRequest(req: Request) { }); } - try { - // Drains the user's remaining balance to zero if the - // request cost more than they had. The billing service - // accepts the partial deduction, writes an audit row as - // `_partial`, and the pre-flight gate above - // will block the next request. Not an error path — - // intentional terminal state. Runs after the persist above so - // its latency never delays the row the client is waiting on. - await billing.consume(user.email!, { - tokens: billingTokens, - operation: - conversation.type === 'creative' ? 'chat' : 'parametric', - referenceId: responseMessage.id, - }); - } catch (error) { - logError(error, { - functionName: 'ai-chat', - statusCode: - error instanceof BillingClientError ? error.status : 502, - userId: user.id, - conversationId: conversation.id, - additionalContext: { operation: 'billing_consume' }, - }); + if (billingTokens > 0) { + try { + // Drains the user's remaining balance to zero if the + // request cost more than they had. The billing service + // accepts the partial deduction, writes an audit row as + // `_partial`, and the pre-flight gate above + // will block the next request. Not an error path — + // intentional terminal state. Runs after the persist above so + // its latency never delays the row the client is waiting on. + await billing.consume(user.email!, { + tokens: billingTokens, + operation: + conversation.type === 'creative' ? 'chat' : 'parametric', + referenceId: responseMessage.id, + }); + } catch (error) { + logError(error, { + functionName: 'ai-chat', + statusCode: + error instanceof BillingClientError ? error.status : 502, + userId: user.id, + conversationId: conversation.id, + additionalContext: { operation: 'billing_consume' }, + }); + } } // Only generate suggestions once the assistant has actually diff --git a/src/views/EditorView.tsx b/src/views/EditorView.tsx index bed7cd7d..d6c4f4e6 100644 --- a/src/views/EditorView.tsx +++ b/src/views/EditorView.tsx @@ -14,6 +14,7 @@ import { OpenSCADPreview } from '@/components/viewer/OpenSCADViewer'; import { MeshPreview } from '@/components/viewer/MeshPreview'; import Loader from '@/components/viewer/Loader'; import { useAuth } from '@/contexts/AuthContext'; +import { useIsLocalModel } from '@/hooks/useLocalModels.ts'; import { ConversationContext } from '@/contexts/ConversationContext'; import { SelectedItemsContext } from '@/contexts/SelectedItemsContext'; import { useConversation } from '@/contexts/ConversationContext'; @@ -194,6 +195,7 @@ function ConversationEditor() { ? 'quality' : 'google/gemini-3.1-pro-preview'), ); + const isLocalModel = useIsLocalModel(model); const [activePreview, setActivePreview] = useState(null); const [parameters, setParameters] = useState([]); const [currentOutput, setCurrentOutput] = useState(); @@ -705,7 +707,7 @@ function ConversationEditor() { initialBranch={initialBranch} model={model} setModel={updateSelectedModel} - isDisabled={totalTokens <= 0} + isDisabled={totalTokens <= 0 && !isLocalModel} onSendParts={handleSendParts} onRetry={handleRetry} onEdit={handleEdit} diff --git a/src/views/PromptView.tsx b/src/views/PromptView.tsx index 1d5d7f95..9a54ce9f 100644 --- a/src/views/PromptView.tsx +++ b/src/views/PromptView.tsx @@ -14,6 +14,7 @@ import { LowPromptsWarningMessage } from '@/components/LowPromptsWarningMessage' import { NewProductBanner } from '@/components/NewProductBanner'; import { FreePlanTrialPill } from '@/components/FreePlanTrialPill'; import { useIsMobile } from '@/hooks/useIsMobile'; +import { useIsLocalModel } from '@/hooks/useLocalModels.ts'; import { cn } from '@/lib/utils'; import { SelectedItemsContext } from '@/contexts/SelectedItemsContext'; import posthog from 'posthog-js'; @@ -65,6 +66,8 @@ export function PromptView() { const [model, setModel] = useState('google/gemini-3.1-pro-preview'); + const isLocalModel = useIsLocalModel(model); + const handleTypeChange = (newType: 'parametric' | 'creative') => { setType(newType); // Reset model to the default for the new type @@ -91,8 +94,9 @@ export function PromptView() { const limitReached = useMemo(() => { if (isLoading) return false; + if (type === 'parametric' && isLocalModel) return false; return totalTokens <= 0; - }, [totalTokens, isLoading]); + }, [totalTokens, isLoading, type, isLocalModel]); // Trigger fade in on mount useEffect(() => { From cbffde41ca69ebe5f750dfd4d3331ef09bbcdb8e Mon Sep 17 00:00:00 2001 From: nathan-t4 <19811248+nathan-t4@users.noreply.github.com> Date: Thu, 9 Jul 2026 17:22:12 +0800 Subject: [PATCH 10/11] Add supportsForcedToolChoice for third party models --- README.md | 1 + local-models.json.example | 1 + shared/localModels.test.ts | 33 ++++++++++++++++++++++++++++++++- shared/localModels.ts | 1 + src/server/aiChat.ts | 28 +++++++++++++++++++--------- 5 files changed, 54 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index ab09c249..72d069ba 100644 --- a/README.md +++ b/README.md @@ -245,6 +245,7 @@ To add your own local models (LM Studio, Ollama, LiteLLM, etc.) alongside the cl "supportsTools": true, "supportsThinking": true, "supportsVision": false, + "supportsForcedToolChoice": false, "useForAux": false } ``` diff --git a/local-models.json.example b/local-models.json.example index f79b7d99..d6f834aa 100644 --- a/local-models.json.example +++ b/local-models.json.example @@ -18,6 +18,7 @@ "supportsTools": true, "supportsThinking": true, "supportsVision": false, + "supportsForcedToolChoice": false, "useForAux": false } ] diff --git a/shared/localModels.test.ts b/shared/localModels.test.ts index 4feae3ae..750e1076 100644 --- a/shared/localModels.test.ts +++ b/shared/localModels.test.ts @@ -227,17 +227,48 @@ describe('localModelToPickerConfig', () => { }); }); - it('does not expose apiKey or baseUrl in picker config', () => { + it('does not expose apiKey, baseUrl, or supportsForcedToolChoice', () => { const picker = localModelToPickerConfig({ id: 'secure-model', name: 'Secure', description: 'Server-only key', baseUrl: 'https://example.local/v1', apiKey: 'DO_NOT_EXPOSE', + supportsForcedToolChoice: false, }); assert.equal('apiKey' in picker, false); assert.equal('baseUrl' in picker, false); + assert.equal('supportsForcedToolChoice' in picker, false); + }); +}); + +describe('supportsForcedToolChoice catalog field', () => { + it('parses supportsForcedToolChoice independently of supportsThinking', () => { + assert.deepEqual( + parseLocalModelsJson([ + { + id: 'no-forced-choice', + name: 'No Forced', + description: 'Rejects forced tool_choice without thinking', + baseUrl: 'http://localhost:1234', + supportsTools: true, + supportsThinking: false, + supportsForcedToolChoice: false, + }, + ]), + [ + { + id: 'no-forced-choice', + name: 'No Forced', + description: 'Rejects forced tool_choice without thinking', + baseUrl: 'http://localhost:1234', + supportsTools: true, + supportsThinking: false, + supportsForcedToolChoice: false, + }, + ], + ); }); }); diff --git a/shared/localModels.ts b/shared/localModels.ts index 9def1b14..60332326 100644 --- a/shared/localModels.ts +++ b/shared/localModels.ts @@ -19,6 +19,7 @@ export const localModelSchema = modelCoreSchema.extend({ baseUrl: z.string().optional(), apiKey: z.string().optional(), useForAux: z.boolean().optional(), + supportsForcedToolChoice: z.boolean().optional(), }); export type LocalModelConfig = z.infer; diff --git a/src/server/aiChat.ts b/src/server/aiChat.ts index 6860891d..36eba748 100644 --- a/src/server/aiChat.ts +++ b/src/server/aiChat.ts @@ -524,15 +524,24 @@ function usesAdaptiveAnthropicThinking(modelId: string) { return match ? Number(match[1]) >= 6 : false; } -// The reasoning-tier Claude 5 models (Fable, Mythos) and third-party thinking models -// reject a forced `tool_choice` outright ("tool_choice forces tool use -// is not compatible with this model"). Other Claude 5 tiers — notably Sonnet 5 — -// accept a forced tool_choice on the first-party API, provided thinking -// is disabled for that step (see the per-step override in the parametric flow). +// The reasoning-tier Claude 5 models (Fable, Mythos) and some local / +// OpenAI-compatible servers reject a forced `tool_choice` outright +// ("tool_choice forces tool use is not compatible with this model"). Other +// Claude 5 tiers — notably Sonnet 5 — accept a forced tool_choice on the +// first-party API, provided thinking is disabled for that step (see the +// per-step override in the parametric flow). +// +// For local catalog entries, `supportsForcedToolChoice` is the explicit +// knob. When unset, fall back to treating `supportsThinking: true` as a +// rejection signal (common for thinking-capable OpenAI-compatible servers). function rejectsForcedToolChoice(modelId: string): boolean { if (/^claude-(?:fable|mythos)\b/.test(bareModelId(modelId))) return true; const local = getActiveLocalModel(modelId); - return local?.supportsThinking === true; + if (!local) return false; + if (local.supportsForcedToolChoice !== undefined) { + return local.supportsForcedToolChoice === false; + } + return local.supportsThinking === true; } // Whether a model accepts a forced `tool_choice` (type: "tool" / "any"). @@ -1479,9 +1488,10 @@ export async function handleAiChatRequest(req: Request) { ) { // Restrict the toolset to the build tool on the first step. Turns that // accept a forced tool_choice get it pinned; turns that reject forced - // tool use (Claude Fable/Mythos, local thinking models, or any - // thinking-enabled Anthropic turn) fall back to auto and rely on the - // system prompt to call build_parametric_model. + // tool use (Claude Fable/Mythos, local models with + // supportsForcedToolChoice: false, or the supportsThinking fallback) + // fall back to auto and rely on the system prompt to call + // build_parametric_model. return { activeTools: ['build_parametric_model' as never], ...(forceBuildToolChoice From 723e5354078722c3a828c5f10f8f348e474b78e9 Mon Sep 17 00:00:00 2001 From: nathan-t4 <19811248+nathan-t4@users.noreply.github.com> Date: Thu, 9 Jul 2026 20:25:58 +0800 Subject: [PATCH 11/11] Add logging for loadCatalogFromDisk --- src/server/localChatConfig.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/server/localChatConfig.ts b/src/server/localChatConfig.ts index e508215c..18fb9c57 100644 --- a/src/server/localChatConfig.ts +++ b/src/server/localChatConfig.ts @@ -48,7 +48,11 @@ function loadCatalogFromDisk(): LocalModelConfig[] { try { const raw = JSON.parse(fs.readFileSync(userPath, 'utf8')) as unknown; cachedCatalog = parseLocalModelsJson(raw); - } catch { + } catch (error) { + console.error( + `Failed to load local-models.json from ${userPath}; using empty catalog.`, + error, + ); cachedCatalog = []; }