diff --git a/.gitignore b/.gitignore index d6ae9214..3bb04cb2 100644 --- a/.gitignore +++ b/.gitignore @@ -33,3 +33,6 @@ dist-ssr # other secret files .env.local .env.langfuse + +# local models config +local-models.json \ No newline at end of file diff --git a/README.md b/README.md index 00a10bd3..72d069ba 100644 --- a/README.md +++ b/README.md @@ -211,6 +211,53 @@ npm run dev NGROK_URL="" # Optional local Supabase Storage tunnel for provider-readable signed URLs ``` +### 3. Optional Local LLM Models + +To add your own local models (LM Studio, Ollama, LiteLLM, etc.) alongside the cloud models: + +1. Start one or more local OpenAI-compatible servers (for example LM Studio on port `1234`). +2. Copy the example catalog and edit entries to match your servers: + + ```bash + cp local-models.json.example local-models.json + ``` + + Each entry needs an `id`, `name`, `description`, and `baseUrl`. The `id` must match what your server exposes. + + Example: + + ```json + { + "id": "qwen2.5-vl-7b-instruct", + "name": "Qwen2.5 VL 7B", + "description": "Local VLM", + "baseUrl": "http://localhost:1234", + "supportsTools": true, + "supportsVision": true, + "useForAux": true + }, + { + "id": "deepseek-v4-flash", + "name": "DeepSeek v4 Flash", + "description": "Third Party LLM", + "baseUrl": "https://api.deepseek.com", + "apiKey": "DEEPSEEK_API_KEY", + "supportsTools": true, + "supportsThinking": true, + "supportsVision": false, + "supportsForcedToolChoice": false, + "useForAux": false + } + ``` + +3. Set local model API keys in `.env.local`. + - `apiKey` in `local-models.json` should be the env var name, not the secret value. + - Example: `"apiKey": "DEEPSEEK_API_KEY"` and then set `DEEPSEEK_API_KEY=""` in `.env.local`. + +4. Restart `npm run dev`. Models with a `baseUrl` appear in the parametric model picker. + +Cloud models (Gemini, Claude, OpenAI, Z-AI) keep using their existing cloud routes. Only ids listed in `local-models.json` with a `baseUrl` are sent to that model's server. + ## 🌐 Setting Up ngrok for Local Development CADAM uses public URLs for provider callbacks and local signed storage URLs: diff --git a/local-models.json.example b/local-models.json.example new file mode 100644 index 00000000..d6f834aa --- /dev/null +++ b/local-models.json.example @@ -0,0 +1,24 @@ +[ + { + "id": "qwen2.5-vl-7b-instruct", + "name": "Qwen2.5 VL 7B", + "description": "Local VLM", + "baseUrl": "http://localhost:1234", + "supportsTools": true, + "supportsThinking": false, + "supportsVision": true, + "useForAux": true + }, + { + "id": "deepseek-v4-flash", + "name": "DeepSeek v4 Flash", + "description": "Third Party LLM", + "baseUrl": "https://api.deepseek.com", + "apiKey": "DEEPSEEK_API_KEY", + "supportsTools": true, + "supportsThinking": true, + "supportsVision": false, + "supportsForcedToolChoice": false, + "useForAux": false + } +] diff --git a/package-lock.json b/package-lock.json index 7e96a05f..11f41c1b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -365,6 +365,7 @@ "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.29.7.tgz", "integrity": "sha512-RgHBCvtjbOK2gXSNBNIkNoEc9qoVEtau3hj8gEqKQuL3HZAibKarWFEI3Lfm6EYKkLalOh8eSrj9b+ch9H/VBA==", "license": "MIT", + "peer": true, "dependencies": { "@babel/code-frame": "^7.29.7", "@babel/generator": "^7.29.7", @@ -672,33 +673,10 @@ "integrity": "sha512-U+HFai5+zmJCkK86QsaJtoITlboZHBqrVketcO2ROv865xfCMSFpELQoz1GkX5GzME8pTa+3kbKrZHQtI0gdbw==", "license": "Apache-2.0" }, - "node_modules/@emnapi/core": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.10.0.tgz", - "integrity": "sha512-yq6OkJ4p82CAfPl0u9mQebQHKPJkY7WrIuk205cTYnYe+k2Z8YBh11FrbRG/H6ihirqcacOgl2BIO8oyMQLeXw==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "@emnapi/wasi-threads": "1.2.1", - "tslib": "^2.4.0" - } - }, - "node_modules/@emnapi/runtime": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.10.0.tgz", - "integrity": "sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "tslib": "^2.4.0" - } - }, "node_modules/@emnapi/wasi-threads": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.1.tgz", - "integrity": "sha512-uTII7OYF+/Mes/MrcIOYp5yOtSMLBWSIoLPpcgwipoiKbli6k322tcoFsxoIIxPDqW01SQGAgko4EzZi2BNv2w==", + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.2.tgz", + "integrity": "sha512-c95qOXkHdydNKhscBTebqEC1CVAZpyqOfVfBzQ1qgzyl3gfeldUjIggDbIZgDKsHLgnsM+igH7TJ/eAasaVuMA==", "dev": true, "license": "MIT", "optional": true, @@ -2763,6 +2741,7 @@ "version": "9.2.0", "resolved": "https://registry.npmjs.org/@react-three/fiber/-/fiber-9.2.0.tgz", "integrity": "sha512-esZe+E9T/aYEM4HlBkirr/yRE8qWTp9WUsLISyHHMCHKlJv85uc5N4wwKw+Ay0QeTSITw6T9Q3Svpu383Q+CSQ==", + "peer": true, "dependencies": { "@babel/runtime": "^7.17.8", "@types/react-reconciler": "^0.28.9", @@ -3031,6 +3010,40 @@ "node": "^20.19.0 || >=22.12.0" } }, + "node_modules/@rolldown/binding-wasm32-wasi/node_modules/@emnapi/core": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.10.0.tgz", + "integrity": "sha512-yq6OkJ4p82CAfPl0u9mQebQHKPJkY7WrIuk205cTYnYe+k2Z8YBh11FrbRG/H6ihirqcacOgl2BIO8oyMQLeXw==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/wasi-threads": "1.2.1", + "tslib": "^2.4.0" + } + }, + "node_modules/@rolldown/binding-wasm32-wasi/node_modules/@emnapi/runtime": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.10.0.tgz", + "integrity": "sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@rolldown/binding-wasm32-wasi/node_modules/@emnapi/wasi-threads": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.1.tgz", + "integrity": "sha512-uTII7OYF+/Mes/MrcIOYp5yOtSMLBWSIoLPpcgwipoiKbli6k322tcoFsxoIIxPDqW01SQGAgko4EzZi2BNv2w==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, "node_modules/@rolldown/binding-win32-arm64-msvc": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.0.3.tgz", @@ -4581,6 +4594,7 @@ "resolved": "https://registry.npmjs.org/@types/mdast/-/mdast-4.0.4.tgz", "integrity": "sha512-kGaNbPh1k7AFzgpud/gMdvIm5xuECykRR+JnWKQno9TAXVa6WIVCGTPvYGekIDL4uwCZQSYbUxNBSb1aUo79oA==", "license": "MIT", + "peer": true, "dependencies": { "@types/unist": "*" } @@ -4615,6 +4629,7 @@ "resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.1.tgz", "integrity": "sha512-1U5NQWh/GylZQ50ZMnnPjkYHEaGhg6t5i/KI0LDDh3t4E3h3T3vzm+GLY2BRzMfIjSBwzm6tginoZl5z0O/qsA==", "license": "MIT", + "peer": true, "dependencies": { "csstype": "^3.0.2" } @@ -4625,6 +4640,7 @@ "integrity": "sha512-/EEvYBdT3BflCWvTMO7YkYBHVE9Ci6XdqZciZANQgKpaiDRGOLIlRo91jbTNRQjgPFWVaRxcYc0luVNFitz57A==", "devOptional": true, "license": "MIT", + "peer": true, "peerDependencies": { "@types/react": "^19.2.0" } @@ -4652,6 +4668,7 @@ "version": "0.160.0", "resolved": "https://registry.npmjs.org/@types/three/-/three-0.160.0.tgz", "integrity": "sha512-jWlbUBovicUKaOYxzgkLlhkiEQJkhCVvg4W2IYD2trqD2om3VK4DGLpHH5zQHNr7RweZK/5re/4IVhbhvxbV9w==", + "peer": true, "dependencies": { "@types/stats.js": "*", "@types/webxr": "*", @@ -4729,6 +4746,7 @@ "integrity": "sha512-Zhy8HCvBUEfBECzIl1PKqF4p11+d0aUJS1GeUiuqK9WmOug8YCmC4h4bjyBvMyAMI9sbRczmrYL5lKg/YMbrcQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "8.38.0", "@typescript-eslint/types": "8.38.0", @@ -5039,6 +5057,7 @@ "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.16.0.tgz", "integrity": "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==", "license": "MIT", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -5072,6 +5091,7 @@ "resolved": "https://registry.npmjs.org/ai/-/ai-6.0.177.tgz", "integrity": "sha512-1xQtbeWwNcLyyM86ixZhkKvT+WRXc1lvarIKqPVtsyn8F9NDikwUMBqYu+aQKDgMht50SMXh4qboYuU8MeHZZA==", "license": "Apache-2.0", + "peer": true, "dependencies": { "@ai-sdk/gateway": "3.0.112", "@ai-sdk/provider": "3.0.10", @@ -5345,6 +5365,7 @@ "url": "https://github.com/sponsors/ai" } ], + "peer": true, "dependencies": { "caniuse-lite": "^1.0.30001726", "electron-to-chromium": "^1.5.173", @@ -5799,6 +5820,7 @@ "integrity": "sha512-wUR89x/Rw7/8t+vn0CmGDYM9TD6VtARGb0LD5jq2wjtMy1vCP4M+sm6N6TigWeTYvnA8MoW29NqqXD0ep0rfBA==", "devOptional": true, "license": "MIT", + "peer": true, "peerDependencies": { "srvx": ">=0.11.5" }, @@ -5864,6 +5886,7 @@ "resolved": "https://registry.npmjs.org/cytoscape/-/cytoscape-3.33.2.tgz", "integrity": "sha512-sj4HXd3DokGhzZAdjDejGvTPLqlt84vNFN8m7bGsOzDY5DyVcxIb2ejIXat2Iy7HxWhdT/N1oKyheJ5YdpsGuw==", "license": "MIT", + "peer": true, "engines": { "node": ">=0.10" } @@ -6266,6 +6289,7 @@ "resolved": "https://registry.npmjs.org/d3-selection/-/d3-selection-3.0.0.tgz", "integrity": "sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ==", "license": "ISC", + "peer": true, "engines": { "node": ">=12" } @@ -6616,7 +6640,8 @@ "node_modules/embla-carousel": { "version": "8.6.0", "resolved": "https://registry.npmjs.org/embla-carousel/-/embla-carousel-8.6.0.tgz", - "integrity": "sha512-SjWyZBHJPbqxHOzckOfo8lHisEaJWmwd23XppYFYVh10bU66/Pn5tkVkbkCMZVdbUE5eTCI2nD8OyIP4Z+uwkA==" + "integrity": "sha512-SjWyZBHJPbqxHOzckOfo8lHisEaJWmwd23XppYFYVh10bU66/Pn5tkVkbkCMZVdbUE5eTCI2nD8OyIP4Z+uwkA==", + "peer": true }, "node_modules/embla-carousel-react": { "version": "8.6.0", @@ -6744,6 +6769,7 @@ "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.31.0.tgz", "integrity": "sha512-QldCVh/ztyKJJZLr4jXNUByx3gR+TDYZCRXEktiZoUR3PGy4qCmSbkxcIle8GEwGpb5JBZazlaJ/CxLidXdEbQ==", "dev": true, + "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.2.0", "@eslint-community/regexpp": "^4.12.1", @@ -9420,6 +9446,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "@types/debug": "^4.0.0", "debug": "^4.0.0", @@ -10056,7 +10083,8 @@ "url": "https://opencollective.com/unified" } ], - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/micromatch": { "version": "4.0.8", @@ -10284,30 +10312,13 @@ } } }, - "node_modules/nitro/node_modules/chokidar": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-5.0.0.tgz", - "integrity": "sha512-TQMmc3w+5AxjpL8iIiwebF73dRDF4fBIieAqGn9RGCWaEVwQ6Fb2cGe31Yns0RRIzii5goJ1Y7xbMwo1TxMplw==", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "dependencies": { - "readdirp": "^5.0.0" - }, - "engines": { - "node": ">= 20.19.0" - }, - "funding": { - "url": "https://paulmillr.com/funding/" - } - }, "node_modules/nitro/node_modules/db0": { "version": "0.3.4", "resolved": "https://registry.npmjs.org/db0/-/db0-0.3.4.tgz", "integrity": "sha512-RiXXi4WaNzPTHEOu8UPQKMooIbqOEyqA1t7Z6MsdxSCeb8iUC9ko3LcmsLmeUt2SM5bctfArZKkRQggKZz7JNw==", "dev": true, "license": "MIT", + "peer": true, "peerDependencies": { "@electric-sql/pglite": "*", "@libsql/client": "*", @@ -10337,18 +10348,6 @@ } } }, - "node_modules/nitro/node_modules/lru-cache": { - "version": "11.5.1", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.5.1.tgz", - "integrity": "sha512-RPimw/7aMdv2oqRrxKwvZXcPfwBrn/JZ2xYcY9Hus/6LaS3VOAKVWKWgNLCFSiOm1ESXinjsDlidVU7JlnCN2A==", - "dev": true, - "license": "BlueOak-1.0.0", - "optional": true, - "peer": true, - "engines": { - "node": "20 || >=22" - } - }, "node_modules/nitro/node_modules/readdirp": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-5.0.0.tgz", @@ -10356,7 +10355,6 @@ "dev": true, "license": "MIT", "optional": true, - "peer": true, "engines": { "node": ">= 20.19.0" }, @@ -10578,7 +10576,8 @@ "resolved": "https://registry.npmjs.org/ofetch/-/ofetch-2.0.0-alpha.3.tgz", "integrity": "sha512-zpYTCs2byOuft65vI3z43Dd6iSdFbOZZLb9/d21aCpx2rGastVU9dOCv0lu4ykc1Ur1anAYjDi3SUvR0vq50JA==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/ohash": { "version": "2.0.11", @@ -10943,6 +10942,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "nanoid": "^3.3.12", "picocolors": "^1.1.1", @@ -11108,6 +11108,7 @@ "version": "3.6.2", "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.6.2.tgz", "integrity": "sha512-I7AIg5boAr5R0FFtJ6rCfD+LFsWHp81dolrFD8S79U9tb8Az2nGrJncnMSnys+bpQJfRUzqs9hnA81OAA3hCuQ==", + "peer": true, "bin": { "prettier": "bin/prettier.cjs" }, @@ -11314,6 +11315,7 @@ "resolved": "https://registry.npmjs.org/react/-/react-19.2.1.tgz", "integrity": "sha512-DGrYcCWK7tvYMnWh79yrPHt+vdx9tY+1gPZa7nJQtO/p8bLTDaHp4dzwEhQB7pZ4Xe3ok4XKuEPrVuc+wlpkmw==", "license": "MIT", + "peer": true, "engines": { "node": ">=0.10.0" } @@ -11333,6 +11335,7 @@ "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.2.1.tgz", "integrity": "sha512-ibrK8llX2a4eOskq1mXKu/TGZj9qzomO+sNfO98M6d9zIPOEhlBkMkBUBLd1vgS0gQsLDBzA+8jJBVXDnfHmJg==", "license": "MIT", + "peer": true, "dependencies": { "scheduler": "^0.27.0" }, @@ -11364,6 +11367,7 @@ "version": "7.60.0", "resolved": "https://registry.npmjs.org/react-hook-form/-/react-hook-form-7.60.0.tgz", "integrity": "sha512-SBrYOvMbDB7cV8ZfNpaiLcgjH/a1c7aK0lK+aNigpf4xWLO8q+o4tcvVurv3c4EOyzn/3dCsYt4GKD42VvJ/+A==", + "peer": true, "engines": { "node": ">=18.0.0" }, @@ -12031,6 +12035,7 @@ "resolved": "https://registry.npmjs.org/seroval/-/seroval-1.5.4.tgz", "integrity": "sha512-46uFvgrXTVxZcUorgSSRZ4y+ieqLLQRMlG4bnCZKW3qI6BZm7Rg4ntMW4p1mILEEBZWrFlcpp0AyIIlM6jD9iw==", "license": "MIT", + "peer": true, "engines": { "node": ">=10" } @@ -12557,6 +12562,7 @@ "version": "3.4.17", "resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-3.4.17.tgz", "integrity": "sha512-w33E2aCvSDP0tW9RZuNXadXlkHXqFzSkQew/aIa2i/Sj8fThxwovwlXHSPXTbAHwEIhBFXAedUhP2tueAKP8Og==", + "peer": true, "dependencies": { "@alloc/quick-lru": "^5.2.0", "arg": "^5.0.2", @@ -12628,7 +12634,8 @@ "node_modules/three": { "version": "0.160.1", "resolved": "https://registry.npmjs.org/three/-/three-0.160.1.tgz", - "integrity": "sha512-Bgl2wPJypDOZ1stAxwfWAcJ0WQf7QzlptsxkjYiURPz+n5k4RBDLsq+6f9Y75TYxn6aHLcWz+JNmwTOXWrQTBQ==" + "integrity": "sha512-Bgl2wPJypDOZ1stAxwfWAcJ0WQf7QzlptsxkjYiURPz+n5k4RBDLsq+6f9Y75TYxn6aHLcWz+JNmwTOXWrQTBQ==", + "peer": true }, "node_modules/three-mesh-bvh": { "version": "0.8.3", @@ -12724,6 +12731,7 @@ "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -12902,6 +12910,7 @@ "integrity": "sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ==", "dev": true, "license": "Apache-2.0", + "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -12980,6 +12989,7 @@ "resolved": "https://registry.npmjs.org/unified/-/unified-11.0.5.tgz", "integrity": "sha512-xKvGhPWw3k84Qjh8bI3ZeJjqnyadK+GEFtazSfZv/rKeTkTjOJho6mFqh2SM96iIcZokxiOpg78GazTSg8+KHA==", "license": "MIT", + "peer": true, "dependencies": { "@types/unist": "^3.0.0", "bail": "^2.0.0", @@ -13297,6 +13307,7 @@ "integrity": "sha512-h9bXPmJichP5fLmVQo3PyaGSDE2n3aPuomeAlVRm0JLmt4rY6zmPKd59HYI4LNW8oTK7tlTsuC7l/m7awx9Jcw==", "devOptional": true, "license": "MIT", + "peer": true, "dependencies": { "lightningcss": "^1.32.0", "picomatch": "^4.0.4", @@ -13655,6 +13666,7 @@ "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", "license": "MIT", + "peer": true, "funding": { "url": "https://github.com/sponsors/colinhacks" } diff --git a/shared/imageRefs.ts b/shared/imageRefs.ts index 26cef4ed..e9f90dd4 100644 --- a/shared/imageRefs.ts +++ b/shared/imageRefs.ts @@ -29,6 +29,14 @@ export function imageStoragePath( return `${userId}/${conversationId}/${imageId}`; } +export function inspectionPreviewStoragePath( + userId: string, + conversationId: string, + toolCallId: string, +): string { + return `${userId}/${conversationId}/inspection-preview-${toolCallId}`; +} + // Canonical reference persisted in a file part's `url`. Kept in the exact // shape the 2026-05-18 AI-SDK backfill produced so every row in // `messages.parts` carries a single, uniform format. Never fetched directly — diff --git a/shared/localModels.test.ts b/shared/localModels.test.ts new file mode 100644 index 00000000..750e1076 --- /dev/null +++ b/shared/localModels.test.ts @@ -0,0 +1,339 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; +import { + getAuxiliaryLocalModel, + getLocalModels, + isMissingLocalBaseUrl, + isLocalModelId, + localModelToPickerConfig, + normalizeOpenRouterBaseUrl, + parseLocalModelsJson, + providerFor, +} from './localModels.ts'; + +const sampleLocalModels = parseLocalModelsJson([ + { + id: 'qwen2.5-coder-7b', + name: 'Qwen', + description: 'Local', + baseUrl: 'http://localhost:1234', + }, +]); + +describe('parseLocalModelsJson', () => { + it('returns an empty array for non-array input', () => { + assert.deepEqual(parseLocalModelsJson(null), []); + assert.deepEqual(parseLocalModelsJson({}), []); + }); + + it('parses valid entries and skips invalid ones', () => { + assert.deepEqual( + parseLocalModelsJson([ + { + id: 'qwen2.5-coder-7b', + name: 'Qwen', + description: 'Local coder', + apiKey: 'TEST_ENV_KEY_NAME', + supportsTools: true, + }, + { id: '', name: 'Bad', description: 'x' }, + ]), + [ + { + id: 'qwen2.5-coder-7b', + name: 'Qwen', + description: 'Local coder', + apiKey: 'TEST_ENV_KEY_NAME', + supportsTools: true, + }, + ], + ); + }); + + it('accepts entries without apiKey', () => { + assert.deepEqual( + parseLocalModelsJson([ + { + id: 'qwen3-vl-30b', + name: 'Qwen3 VL 30B', + description: 'Local VLM', + baseUrl: 'http://localhost:8004/v1', + supportsTools: true, + }, + ]), + [ + { + id: 'qwen3-vl-30b', + name: 'Qwen3 VL 30B', + description: 'Local VLM', + baseUrl: 'http://localhost:8004/v1', + supportsTools: true, + }, + ], + ); + }); + + it('keeps the first entry when duplicate ids appear', () => { + assert.deepEqual( + parseLocalModelsJson([ + { + id: 'dup-model', + name: 'First', + description: 'First wins', + baseUrl: 'http://localhost:1111', + }, + { + id: 'dup-model', + name: 'Second', + description: 'Dropped', + baseUrl: 'http://localhost:2222', + }, + ]), + [ + { + id: 'dup-model', + name: 'First', + description: 'First wins', + baseUrl: 'http://localhost:1111', + }, + ], + ); + }); +}); + +describe('getLocalModels', () => { + it('returns only entries with a configured baseUrl', () => { + assert.deepEqual( + getLocalModels([ + { + id: 'with-url', + name: 'With URL', + description: 'Has baseUrl', + baseUrl: 'http://localhost:1234', + }, + { + id: 'without-url', + name: 'Without URL', + description: 'Missing baseUrl', + }, + ]), + [ + { + id: 'with-url', + name: 'With URL', + description: 'Has baseUrl', + baseUrl: 'http://localhost:1234', + }, + ], + ); + }); + + it('excludes entries with blank or whitespace-only baseUrl', () => { + assert.deepEqual( + getLocalModels([ + { + id: 'blank-url', + name: 'Blank', + description: 'Blank baseUrl', + baseUrl: ' ', + }, + { + id: 'valid-url', + name: 'Valid', + description: 'Valid baseUrl', + baseUrl: 'http://localhost:1234', + }, + ]), + [ + { + id: 'valid-url', + name: 'Valid', + description: 'Valid baseUrl', + baseUrl: 'http://localhost:1234', + }, + ], + ); + }); + + it('keeps models with and without apiKey when baseUrl exists', () => { + assert.deepEqual( + getLocalModels([ + { + id: 'with-api-key', + name: 'With API key', + description: 'Remote OpenAI-compatible', + baseUrl: 'https://api.example.com', + apiKey: 'EXAMPLE_API_KEY', + }, + { + id: 'without-api-key', + name: 'Without API key', + description: 'Local server with no auth', + baseUrl: 'http://localhost:1234', + }, + ]), + [ + { + id: 'with-api-key', + name: 'With API key', + description: 'Remote OpenAI-compatible', + baseUrl: 'https://api.example.com', + apiKey: 'EXAMPLE_API_KEY', + }, + { + id: 'without-api-key', + name: 'Without API key', + description: 'Local server with no auth', + baseUrl: 'http://localhost:1234', + }, + ], + ); + }); +}); + +describe('isLocalModelId', () => { + it('detects catalog ids', () => { + assert.equal(isLocalModelId('qwen2.5-coder-7b', sampleLocalModels), true); + assert.equal(isLocalModelId('openai/gpt-5.5', sampleLocalModels), false); + }); +}); + +describe('getAuxiliaryLocalModel', () => { + it('returns the model marked useForAux', () => { + const models = parseLocalModelsJson([ + { id: 'local-chat', name: 'Chat', description: 'Chat model' }, + { + id: 'local-aux', + name: 'Aux', + description: 'Aux model', + useForAux: true, + }, + ]); + assert.equal(getAuxiliaryLocalModel(models)?.id, 'local-aux'); + }); +}); + +describe('localModelToPickerConfig', () => { + it('maps catalog entries to picker config', () => { + assert.deepEqual(localModelToPickerConfig(sampleLocalModels[0]!), { + id: 'qwen2.5-coder-7b', + name: 'Qwen', + description: 'Local', + provider: 'Local', + supportsTools: true, + supportsThinking: false, + supportsVision: false, + disabled: false, + }); + }); + + it('does not expose apiKey, baseUrl, or supportsForcedToolChoice', () => { + const picker = localModelToPickerConfig({ + id: 'secure-model', + name: 'Secure', + description: 'Server-only key', + baseUrl: 'https://example.local/v1', + apiKey: 'DO_NOT_EXPOSE', + supportsForcedToolChoice: false, + }); + + assert.equal('apiKey' in picker, false); + assert.equal('baseUrl' in picker, false); + assert.equal('supportsForcedToolChoice' in picker, false); + }); +}); + +describe('supportsForcedToolChoice catalog field', () => { + it('parses supportsForcedToolChoice independently of supportsThinking', () => { + assert.deepEqual( + parseLocalModelsJson([ + { + id: 'no-forced-choice', + name: 'No Forced', + description: 'Rejects forced tool_choice without thinking', + baseUrl: 'http://localhost:1234', + supportsTools: true, + supportsThinking: false, + supportsForcedToolChoice: false, + }, + ]), + [ + { + id: 'no-forced-choice', + name: 'No Forced', + description: 'Rejects forced tool_choice without thinking', + baseUrl: 'http://localhost:1234', + supportsTools: true, + supportsThinking: false, + supportsForcedToolChoice: false, + }, + ], + ); + }); +}); + +describe('normalizeOpenRouterBaseUrl', () => { + it('returns undefined when unset', () => { + assert.equal(normalizeOpenRouterBaseUrl(''), undefined); + assert.equal(normalizeOpenRouterBaseUrl(' '), undefined); + }); + + it('appends /v1 when missing', () => { + assert.equal( + normalizeOpenRouterBaseUrl('http://localhost:1234'), + 'http://localhost:1234/v1', + ); + }); + + it('preserves an existing /v1 suffix', () => { + assert.equal( + normalizeOpenRouterBaseUrl('http://localhost:1234/v1/'), + 'http://localhost:1234/v1', + ); + }); +}); + +describe('providerFor', () => { + it('keeps cloud providers on their direct routes', () => { + assert.equal( + providerFor('anthropic/claude-opus-4.8', sampleLocalModels), + 'anthropic', + ); + assert.equal( + providerFor('google/gemini-3.1-pro-preview', sampleLocalModels), + 'google', + ); + assert.equal( + providerFor('openai/gpt-5.5', sampleLocalModels), + 'openrouter', + ); + }); + + it('routes catalog ids with a baseUrl locally', () => { + assert.equal(providerFor('qwen2.5-coder-7b', sampleLocalModels), 'local'); + }); + + it('falls back to openrouter for catalog ids without a baseUrl', () => { + const noUrlModels = parseLocalModelsJson([ + { id: 'qwen2.5-coder-7b', name: 'Qwen', description: 'Local' }, + ]); + assert.equal(providerFor('qwen2.5-coder-7b', noUrlModels), 'openrouter'); + }); +}); + +describe('isMissingLocalBaseUrl', () => { + it('is true when catalog lists the model but has no baseUrl', () => { + const noUrlModels = parseLocalModelsJson([ + { id: 'qwen2.5-coder-7b', name: 'Qwen', description: 'Local' }, + ]); + assert.equal(isMissingLocalBaseUrl('qwen2.5-coder-7b', noUrlModels), true); + assert.equal(isMissingLocalBaseUrl('openai/gpt-5.5', noUrlModels), false); + }); + + it('is false when the catalog model has a baseUrl', () => { + assert.equal( + isMissingLocalBaseUrl('qwen2.5-coder-7b', sampleLocalModels), + false, + ); + }); +}); diff --git a/shared/localModels.ts b/shared/localModels.ts new file mode 100644 index 00000000..60332326 --- /dev/null +++ b/shared/localModels.ts @@ -0,0 +1,119 @@ +/** + * Pure routing logic for the local OpenAI-compatible model catalog + * (`local-models.json`). No env or disk I/O — `localChatConfig.ts` wires + * server env + file reads; the client fetches picker entries from `/api/local-models`. + */ +import { z } from 'zod'; + +const modelCoreSchema = z.object({ + id: z.string().min(1), + name: z.string().min(1), + description: z.string().min(1), + supportsTools: z.boolean().optional(), + supportsThinking: z.boolean().optional(), + supportsVision: z.boolean().optional(), +}); + +// Shape of each entry in `local-models.json`. +export const localModelSchema = modelCoreSchema.extend({ + baseUrl: z.string().optional(), + apiKey: z.string().optional(), + useForAux: z.boolean().optional(), + supportsForcedToolChoice: z.boolean().optional(), +}); + +export type LocalModelConfig = z.infer; + +export const pickerModelSchema = modelCoreSchema.extend({ + provider: z.string().optional(), + disabled: z.boolean().optional(), +}); + +// UI model-picker shape; cloud entries in `utils.ts` use the same fields. +export type PickerModelConfig = z.infer; + +export type ChatProvider = 'anthropic' | 'google' | 'openrouter' | 'local'; + +// Parse and validate a raw `local-models.json` payload; skip invalid entries. +// skip second occurence of local model with same model id +export function parseLocalModelsJson(raw: unknown): LocalModelConfig[] { + if (!Array.isArray(raw)) return []; + const seenIds = new Set(); + return raw.flatMap((entry) => { + const parsed = localModelSchema.safeParse(entry); + if (!parsed.success || seenIds.has(parsed.data.id)) return []; + seenIds.add(parsed.data.id); + return [parsed.data]; + }); +} + +export function isLocalModelId( + id: string, + localModels: LocalModelConfig[], +): boolean { + return localModels.some((model) => model.id === id); +} + +// One catalog entry may be marked `useForAux` for lightweight tasks (conversation +// titles, suggestion pills) when `ANTHROPIC_API_KEY` is unset. +export function getAuxiliaryLocalModel( + localModels: LocalModelConfig[], +): LocalModelConfig | undefined { + return localModels.find((model) => model.useForAux === true); +} + +// Vercel AI SDK OpenRouter provider expects baseURL to include `/v1` (its default +// is `https://openrouter.ai/api/v1`). LM Studio and similar servers often +// advertise the bare host; append `/v1` when missing. +export function normalizeOpenRouterBaseUrl(raw: string): string | undefined { + const trimmed = raw.trim(); + if (!trimmed) return undefined; + const base = trimmed.replace(/\/+$/, ''); + return base.endsWith('/v1') ? base : `${base}/v1`; +} + +// Cloud ids with a provider prefix keep their direct routes; catalog ids with a +// configured baseUrl route to their own OpenAI-compatible server; everything +// else goes through OpenRouter. +export function providerFor( + modelId: string, + localModels: LocalModelConfig[], +): ChatProvider { + if (modelId.startsWith('anthropic/')) return 'anthropic'; + if (modelId.startsWith('google/')) return 'google'; + const localModel = localModels.find((m) => m.id === modelId); + if (localModel?.baseUrl?.trim()) return 'local'; + return 'openrouter'; +} + +// Catalog ids without a baseUrl cannot be routed — callers reject those +// requests with a clear error instead of falling through to OpenRouter. +export function isMissingLocalBaseUrl( + modelId: string, + catalogOnDisk: LocalModelConfig[], +): boolean { + const model = catalogOnDisk.find((m) => m.id === modelId); + return !!model && !model.baseUrl?.trim(); +} + +// Parametric CAD requires tool calling — grey out catalog entries that explicitly +// opt out via `supportsTools: false`. +export function localModelToPickerConfig( + model: LocalModelConfig, +): PickerModelConfig { + return { + id: model.id, + name: model.name, + description: model.description, + provider: 'Local', + supportsTools: model.supportsTools ?? true, + supportsThinking: model.supportsThinking ?? false, + supportsVision: model.supportsVision ?? false, + disabled: model.supportsTools === false, + }; +} + +// Returns parsed, valid catalog entries that have a baseUrl configured. +export function getLocalModels(rawJson: unknown): LocalModelConfig[] { + return parseLocalModelsJson(rawJson).filter((m) => !!m.baseUrl?.trim()); +} diff --git a/src/components/ModelSelector.tsx b/src/components/ModelSelector.tsx index fd5cb3ae..96bb0864 100644 --- a/src/components/ModelSelector.tsx +++ b/src/components/ModelSelector.tsx @@ -7,6 +7,7 @@ import { DropdownMenuTrigger, } from '@/components/ui/dropdown-menu'; import { Button } from '@/components/ui/button'; +import { ScrollArea } from '@/components/ui/scroll-area'; import { cn } from '@/lib/utils'; import { Model } from '@shared/types'; import { ModelConfig } from '../types/misc.ts'; @@ -174,7 +175,7 @@ export function ModelSelector({ - {models.map((model) => ( - { - onModelChange(model.id); - setIsDropdownOpen(false); - event.stopPropagation(); - }} - disabled={!!model.disabled} - > -
- +
+ {models.map((model) => ( + { + onModelChange(model.id); + setIsDropdownOpen(false); + event.stopPropagation(); + }} + disabled={!!model.disabled} > - {model.name} - - {/* Creative modes (Draft/Textureless/Max Quality) rely on the - description to explain the speed/quality/texture tradeoff; - parametric models show names only to keep the list compact. */} - {currentType === 'creative' && model.description && ( -

+

+ + {model.name} + +
+ {/* Creative modes (Draft/Textureless/Max Quality) rely on the + description to explain the speed/quality/texture tradeoff; + parametric models show names only to keep the list compact. */} + {currentType === 'creative' && model.description && ( + + {model.description} + )} - > - {model.description} -

- )} -
- - ))} +
+
+ ))} + +
); diff --git a/src/components/TextAreaChat.tsx b/src/components/TextAreaChat.tsx index 23a435d5..a950eb59 100644 --- a/src/components/TextAreaChat.tsx +++ b/src/components/TextAreaChat.tsx @@ -20,9 +20,9 @@ import { import { cn, CREATIVE_MODELS, - PARAMETRIC_MODELS, parametricModelSupportsVision, } from '@/lib/utils'; +import { useParametricModels } from '@/hooks/useLocalModels.ts'; import { CreativeModel, MeshFileType, Model } from '@shared/types'; import type { AppUIMessage } from '@shared/chatAi'; import { imageFilePartUrl } from '@shared/imageRefs'; @@ -652,12 +652,13 @@ function TextAreaChat({ }, }; + const parametricModels = useParametricModels(); const memoizedModels = useMemo(() => { if (type === 'creative') { return CREATIVE_MODELS; } - return PARAMETRIC_MODELS; - }, [type]); + return parametricModels; + }, [type, parametricModels]); // ------------------------------------------------------------ // Placeholder – Typed-out Animation @@ -1594,7 +1595,7 @@ function TextAreaChat({
{(type !== 'parametric' || - parametricModelSupportsVision(model)) && ( + parametricModelSupportsVision(model, memoizedModels)) && (
response.blob(), ); - const inspectionPath = `${user.id}/${conversation.id}/inspection-preview-${toolCall.toolCallId}`; + const inspectionPath = inspectionPreviewStoragePath( + user.id, + conversation.id, + toolCall.toolCallId, + ); const { error: inspectionUploadError } = await supabase.storage .from('images') .upload(inspectionPath, inspectionBlob, { diff --git a/src/components/chat/MessageBubble.tsx b/src/components/chat/MessageBubble.tsx index d1a13867..21c64941 100644 --- a/src/components/chat/MessageBubble.tsx +++ b/src/components/chat/MessageBubble.tsx @@ -8,7 +8,8 @@ import { DropdownMenuItem, DropdownMenuTrigger, } from '@/components/ui/dropdown-menu'; -import { CREATIVE_MODELS, PARAMETRIC_MODELS } from '@/lib/utils'; +import { CREATIVE_MODELS } from '@/lib/utils'; +import { useParametricModels } from '@/hooks/useLocalModels'; import { Avatar, AvatarImage } from '@/components/ui/avatar'; import { Button } from '@/components/ui/button'; import { ScrollArea } from '@/components/ui/scroll-area'; @@ -430,8 +431,9 @@ function AssistantBubble({ onRestore, }: MessageBubbleProps) { const { conversation } = useConversation(); + const parametricModels = useParametricModels(); const modelOptions = - conversation.type === 'creative' ? CREATIVE_MODELS : PARAMETRIC_MODELS; + conversation.type === 'creative' ? CREATIVE_MODELS : parametricModels; const [expandedTools, setExpandedTools] = useState>(new Set()); const lastParametricBuildIndex = useMemo(() => { if (conversation.type !== 'parametric') return -1; @@ -836,18 +838,20 @@ function RetryModelDropdown({ align="end" className="w-48 rounded-lg border border-adam-neutral-700 bg-adam-neutral-800 p-1" > - {modelOptions.map((option) => ( - { - onRetry(option.id); - setIsOpen(false); - }} - > - {option.name} - - ))} + + {modelOptions.map((option) => ( + { + onRetry(option.id); + setIsOpen(false); + }} + > + {option.name} + + ))} + ); diff --git a/src/hooks/useLocalModels.ts b/src/hooks/useLocalModels.ts new file mode 100644 index 00000000..8bed6eeb --- /dev/null +++ b/src/hooks/useLocalModels.ts @@ -0,0 +1,28 @@ +import { useMemo } from 'react'; +import { useQuery } from '@tanstack/react-query'; +import { z } from 'zod'; +import { isLocalModelId, pickerModelSchema } from '@shared/localModels'; +import { PARAMETRIC_MODELS } from '@/lib/utils'; +import { apiJson } from '@/services/api'; +import type { ModelConfig } from '@/types/misc'; + +const localModelsSchema = z.array(pickerModelSchema); + +function useLocalModelsQuery() { + return useQuery({ + queryKey: ['local-models'], + queryFn: () => apiJson('local-models', {}, localModelsSchema), + staleTime: 5 * 60 * 1000, + }); +} + +export function useIsLocalModel(modelId: string): boolean { + const { data: localModels = [] } = useLocalModelsQuery(); + return isLocalModelId(modelId, localModels); +} + +export function useParametricModels(): ModelConfig[] { + const { data: localModels = [] } = useLocalModelsQuery(); + + return useMemo(() => [...PARAMETRIC_MODELS, ...localModels], [localModels]); +} diff --git a/src/lib/utils.ts b/src/lib/utils.ts index 3f3e0fb8..35cb326f 100644 --- a/src/lib/utils.ts +++ b/src/lib/utils.ts @@ -235,7 +235,7 @@ export function getInitials(fullName: string | null) { return 'U'; } -export const PARAMETRIC_MODELS: ModelConfig[] = [ +const CLOUD_PARAMETRIC_MODELS: ModelConfig[] = [ { id: 'google/gemini-3.1-pro-preview', name: 'Gemini 3.1 Pro', @@ -292,6 +292,8 @@ export const PARAMETRIC_MODELS: ModelConfig[] = [ }, ]; +export const PARAMETRIC_MODELS: ModelConfig[] = [...CLOUD_PARAMETRIC_MODELS]; + export const CREATIVE_MODELS: ModelConfig[] = [ { id: 'ultra', @@ -316,7 +318,10 @@ export const CREATIVE_MODELS: ModelConfig[] = [ // Whether the selected parametric model can accept image / STL-render inputs. // Unknown ids (e.g. historical messages tagged with a removed model) fall back // to `true` so older saved rows still render normally. -export function parametricModelSupportsVision(modelId: string): boolean { - const cfg = PARAMETRIC_MODELS.find((m) => m.id === modelId); +export function parametricModelSupportsVision( + modelId: string, + models: ModelConfig[] = PARAMETRIC_MODELS, +): boolean { + const cfg = models.find((m) => m.id === modelId); return cfg?.supportsVision !== false; } diff --git a/src/routeTree.gen.ts b/src/routeTree.gen.ts index ad98721d..0d725953 100644 --- a/src/routeTree.gen.ts +++ b/src/routeTree.gen.ts @@ -23,6 +23,7 @@ import { Route as ApiTitleGeneratorRouteImport } from './routes/api/title-genera import { Route as ApiPromptGeneratorRouteImport } from './routes/api/prompt-generator' import { Route as ApiParametricChatRouteImport } from './routes/api/parametric-chat' import { Route as ApiMeshRouteImport } from './routes/api/mesh' +import { Route as ApiLocalModelsRouteImport } from './routes/api/local-models' import { Route as ApiFalWebhookRouteImport } from './routes/api/fal-webhook' import { Route as ApiDeleteUserRouteImport } from './routes/api/delete-user' import { Route as ApiCreativeChatRouteImport } from './routes/api/creative-chat' @@ -108,6 +109,11 @@ const ApiMeshRoute = ApiMeshRouteImport.update({ path: '/api/mesh', getParentRoute: () => rootRouteImport, } as any) +const ApiLocalModelsRoute = ApiLocalModelsRouteImport.update({ + id: '/api/local-models', + path: '/api/local-models', + getParentRoute: () => rootRouteImport, +} as any) const ApiFalWebhookRoute = ApiFalWebhookRouteImport.update({ id: '/api/fal-webhook', path: '/api/fal-webhook', @@ -201,6 +207,7 @@ export interface FileRoutesByFullPath { '/api/creative-chat': typeof ApiCreativeChatRoute '/api/delete-user': typeof ApiDeleteUserRoute '/api/fal-webhook': typeof ApiFalWebhookRoute + '/api/local-models': typeof ApiLocalModelsRoute '/api/mesh': typeof ApiMeshRoute '/api/parametric-chat': typeof ApiParametricChatRoute '/api/prompt-generator': typeof ApiPromptGeneratorRoute @@ -230,6 +237,7 @@ export interface FileRoutesByTo { '/api/creative-chat': typeof ApiCreativeChatRoute '/api/delete-user': typeof ApiDeleteUserRoute '/api/fal-webhook': typeof ApiFalWebhookRoute + '/api/local-models': typeof ApiLocalModelsRoute '/api/mesh': typeof ApiMeshRoute '/api/parametric-chat': typeof ApiParametricChatRoute '/api/prompt-generator': typeof ApiPromptGeneratorRoute @@ -261,6 +269,7 @@ export interface FileRoutesById { '/api/creative-chat': typeof ApiCreativeChatRoute '/api/delete-user': typeof ApiDeleteUserRoute '/api/fal-webhook': typeof ApiFalWebhookRoute + '/api/local-models': typeof ApiLocalModelsRoute '/api/mesh': typeof ApiMeshRoute '/api/parametric-chat': typeof ApiParametricChatRoute '/api/prompt-generator': typeof ApiPromptGeneratorRoute @@ -293,6 +302,7 @@ export interface FileRouteTypes { | '/api/creative-chat' | '/api/delete-user' | '/api/fal-webhook' + | '/api/local-models' | '/api/mesh' | '/api/parametric-chat' | '/api/prompt-generator' @@ -322,6 +332,7 @@ export interface FileRouteTypes { | '/api/creative-chat' | '/api/delete-user' | '/api/fal-webhook' + | '/api/local-models' | '/api/mesh' | '/api/parametric-chat' | '/api/prompt-generator' @@ -352,6 +363,7 @@ export interface FileRouteTypes { | '/api/creative-chat' | '/api/delete-user' | '/api/fal-webhook' + | '/api/local-models' | '/api/mesh' | '/api/parametric-chat' | '/api/prompt-generator' @@ -382,6 +394,7 @@ export interface RootRouteChildren { ApiCreativeChatRoute: typeof ApiCreativeChatRoute ApiDeleteUserRoute: typeof ApiDeleteUserRoute ApiFalWebhookRoute: typeof ApiFalWebhookRoute + ApiLocalModelsRoute: typeof ApiLocalModelsRoute ApiMeshRoute: typeof ApiMeshRoute ApiParametricChatRoute: typeof ApiParametricChatRoute ApiPromptGeneratorRoute: typeof ApiPromptGeneratorRoute @@ -489,6 +502,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof ApiMeshRouteImport parentRoute: typeof rootRouteImport } + '/api/local-models': { + id: '/api/local-models' + path: '/api/local-models' + fullPath: '/api/local-models' + preLoaderRoute: typeof ApiLocalModelsRouteImport + parentRoute: typeof rootRouteImport + } '/api/fal-webhook': { id: '/api/fal-webhook' path: '/api/fal-webhook' @@ -649,6 +669,7 @@ const rootRouteChildren: RootRouteChildren = { ApiCreativeChatRoute: ApiCreativeChatRoute, ApiDeleteUserRoute: ApiDeleteUserRoute, ApiFalWebhookRoute: ApiFalWebhookRoute, + ApiLocalModelsRoute: ApiLocalModelsRoute, ApiMeshRoute: ApiMeshRoute, ApiParametricChatRoute: ApiParametricChatRoute, ApiPromptGeneratorRoute: ApiPromptGeneratorRoute, diff --git a/src/routes/api/local-models.ts b/src/routes/api/local-models.ts new file mode 100644 index 00000000..ef27024f --- /dev/null +++ b/src/routes/api/local-models.ts @@ -0,0 +1,16 @@ +import { createFileRoute } from '@tanstack/react-router'; +import { localModelToPickerConfig } from '@shared/localModels'; +import { getLocalChatState } from '@/server/localChatConfig'; +import { json, preflight } from '@/server/api'; + +export const Route = createFileRoute('/api/local-models')({ + server: { + handlers: { + OPTIONS: preflight, + GET: async () => { + const { activeCatalog } = getLocalChatState(); + return json(activeCatalog.map(localModelToPickerConfig)); + }, + }, + }, +}); diff --git a/src/server/aiChat.ts b/src/server/aiChat.ts index 3066bd1c..36eba748 100644 --- a/src/server/aiChat.ts +++ b/src/server/aiChat.ts @@ -2,8 +2,16 @@ import { createAnthropic } from '@ai-sdk/anthropic'; import { createGoogleGenerativeAI } from '@ai-sdk/google'; import { createOpenRouter } from '@openrouter/ai-sdk-provider'; import { chatTools, type AppUIMessage, type AppTools } from '@shared/chatAi'; +import { + getAuxiliaryLocalModel, + normalizeOpenRouterBaseUrl, +} from '@shared/localModels'; import { cleanAssistantText, getParametricText } from '@shared/parametricParts'; -import { imageIdFromFilename, imageStoragePath } from '@shared/imageRefs'; +import { + imageIdFromFilename, + imageStoragePath, + inspectionPreviewStoragePath, +} from '@shared/imageRefs'; import { normalizeConversationSuggestions } from '@shared/suggestions'; import type { Conversation, Message, MeshFileType, Model } from '@shared/types'; import { @@ -34,7 +42,17 @@ import { resolveDanglingToolParts, } from './chatToolPersistence'; import { handleMeshRequest } from './mesh'; +import { + getActiveLocalModel, + getLocalChatState, + isActiveLocalModel, + isMissingLocalBaseUrl, + localApiKeyForModel, + providerFor, + type ChatProvider, +} from './localChatConfig'; import { getAnonSupabaseClient } from './supabaseClient'; +import { findLatestBuildToolCallId } from './messageUtils'; /** * USD list price per **million** tokens, keyed by the same model IDs the @@ -309,16 +327,9 @@ function jsonResponse(body: unknown, status: number) { const THINKING_BUDGET_TOKENS = 9000; const PARAMETRIC_MAX_OUTPUT_TOKENS = 64000; -type ChatProvider = 'anthropic' | 'google' | 'openrouter'; - -function providerFor(modelId: string): ChatProvider { - if (modelId.startsWith('anthropic/')) return 'anthropic'; - if (modelId.startsWith('google/')) return 'google'; - return 'openrouter'; -} - type AnthropicProvider = ReturnType; type GoogleProvider = ReturnType; +type OpenRouterProvider = ReturnType; // The Vercel AI SDK's Anthropic provider expects ANTHROPIC_BASE_URL to already // include the "/v1" path segment (its built-in default is @@ -337,13 +348,15 @@ function normalizedAnthropicBaseURL(): string | undefined { type ChatProviders = { anthropic: () => AnthropicProvider; google: () => GoogleProvider; - openrouter: () => ReturnType; + openrouter: () => OpenRouterProvider; + local: (baseUrl: string, apiKey?: string) => OpenRouterProvider; }; function createChatProviders(): ChatProviders { let anthropic: AnthropicProvider | undefined; let google: GoogleProvider | undefined; - let openrouter: ReturnType | undefined; + let openrouter: OpenRouterProvider | undefined; + const localByTarget = new Map(); return { anthropic: () => { if (!anthropic) { @@ -367,6 +380,20 @@ function createChatProviders(): ChatProviders { }); return openrouter; }, + local: (baseUrl: string, apiKey?: string) => { + const resolvedApiKey = apiKey?.trim() || 'local'; + const providerKey = `${baseUrl}::${resolvedApiKey}`; + let provider = localByTarget.get(providerKey); + if (!provider) { + provider = createOpenRouter({ + baseURL: baseUrl, + apiKey: resolvedApiKey, + compatibility: 'compatible', + }); + localByTarget.set(providerKey, provider); + } + return provider; + }, }; } @@ -386,8 +413,22 @@ function buildChatModel( ): { model: LanguageModel; providerOptions?: ProviderOptions } { const hasCappedThinkingBudget = thinking && thinkingBudget !== THINKING_BUDGET_TOKENS; + const provider = providerFor(modelId); + + if (provider === 'local') { + const modelConfig = getActiveLocalModel(modelId); + const rawBaseUrl = modelConfig?.baseUrl; + if (!rawBaseUrl) + throw new Error(`No baseUrl configured for local model ${modelId}`); + const baseUrl = normalizeOpenRouterBaseUrl(rawBaseUrl) ?? rawBaseUrl; + return { + model: providers + .local(baseUrl, localApiKeyForModel(modelConfig)) + .chat(modelId), + }; + } - if (providerFor(modelId) === 'openrouter') { + if (provider === 'openrouter') { return { model: providers.openrouter().chat(modelId, { ...(thinking ? { reasoning: { max_tokens: thinkingBudget } } : {}), @@ -457,6 +498,14 @@ function bareModelId(modelId: string): string { return id.replace(/\./g, '-'); } +// For local models, check supportsVision. +// Otherwise, all cloud models support vision. +function chatModelSupportsVision(modelId: string): boolean { + const local = getActiveLocalModel(modelId); + if (local) return local.supportsVision === true; + return true; +} + // The Claude 5 generation swaps the opus/sonnet/haiku tiers for code names // ("claude-fable-5", "claude-mythos-5", …). Match the `claude--5` // shape rather than enumerating code names so future Claude 5 variants @@ -475,13 +524,24 @@ function usesAdaptiveAnthropicThinking(modelId: string) { return match ? Number(match[1]) >= 6 : false; } -// The reasoning-tier Claude 5 models (Fable, Mythos) reject a forced -// `tool_choice` outright ("tool_choice forces tool use is not compatible with -// this model"). Other Claude 5 tiers — notably Sonnet 5 — accept a forced -// tool_choice on the first-party API, provided thinking is disabled for that -// step (see the per-step override in the parametric flow). +// The reasoning-tier Claude 5 models (Fable, Mythos) and some local / +// OpenAI-compatible servers reject a forced `tool_choice` outright +// ("tool_choice forces tool use is not compatible with this model"). Other +// Claude 5 tiers — notably Sonnet 5 — accept a forced tool_choice on the +// first-party API, provided thinking is disabled for that step (see the +// per-step override in the parametric flow). +// +// For local catalog entries, `supportsForcedToolChoice` is the explicit +// knob. When unset, fall back to treating `supportsThinking: true` as a +// rejection signal (common for thinking-capable OpenAI-compatible servers). function rejectsForcedToolChoice(modelId: string): boolean { - return /^claude-(?:fable|mythos)\b/.test(bareModelId(modelId)); + if (/^claude-(?:fable|mythos)\b/.test(bareModelId(modelId))) return true; + const local = getActiveLocalModel(modelId); + if (!local) return false; + if (local.supportsForcedToolChoice !== undefined) { + return local.supportsForcedToolChoice === false; + } + return local.supportsThinking === true; } // Whether a model accepts a forced `tool_choice` (type: "tool" / "any"). @@ -490,6 +550,14 @@ function supportsForcedToolChoice(modelId: string): boolean { } function priceFor(modelId: string) { + if (isActiveLocalModel(modelId)) { + return { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + }; + } const entry = MODEL_PRICES[modelId] ?? FALLBACK_MODEL_PRICE; return { input: entry.input, @@ -543,6 +611,7 @@ function billingTokensFromUsage( modelId: string, usage: LanguageModelUsage, ): number { + if (isActiveLocalModel(modelId)) return 0; const usdCost = usdCostFromUsage(modelId, usage) * billingMultiplier(); return Math.max(1, Math.ceil(usdCost / USD_PER_BILLING_TOKEN)); } @@ -712,16 +781,16 @@ async function loadBranchFromDb({ } async function generateConversationTitle({ - anthropic, + auxModel, firstMessage, }: { - anthropic: AnthropicProvider; + auxModel: LanguageModel; firstMessage: AppUIMessage; }) { const text = getParametricText(firstMessage.parts) || 'New conversation'; try { const result = await generateText({ - model: anthropic('claude-haiku-4-5'), + model: auxModel, system: 'Generate a short title for a 3D creation conversation. Return only the title.', prompt: text, @@ -744,11 +813,11 @@ async function generateConversationTitle({ * specific assistant turn. */ async function generateConversationSuggestions({ - anthropic, + auxModel, branch, conversationType, }: { - anthropic: AnthropicProvider; + auxModel: LanguageModel; branch: AppUIMessage[]; conversationType: 'parametric' | 'creative'; }): Promise { @@ -766,7 +835,7 @@ async function generateConversationSuggestions({ const summary = `User request: ${firstUserText.slice(0, 400)}\n\nMost recent assistant reply: ${lastAssistantText.slice(0, 400)}`; try { const result = await generateText({ - model: anthropic('claude-haiku-4-5'), + model: auxModel, system: conversationType === 'creative' ? 'Given a 3D mesh design conversation, return an array of exactly 2 follow-up prompts the user might want to send next. Each prompt is a concise instruction of 3 words or fewer, not a question. Return exactly 2 items — no more, no fewer.' @@ -866,6 +935,125 @@ async function sniffImageMediaType(bytes: Uint8Array): Promise { return sniffed && ACCEPTED_IMAGE_MEDIA_TYPES.has(sniffed) ? sniffed : null; } +async function buildHydratedMessages( + messages: AppUIMessage[], + { + supabaseClient, + conversation, + modelId, + provider, + }: { + supabaseClient: SupabaseAnon; + conversation: ConversationAccess; + modelId: string; + provider: ChatProvider; + }, +): Promise>> { + const shouldAddInspection = + provider === 'local' && + conversation.type === 'parametric' && + chatModelSupportsVision(modelId); + const latestBuildToolCallId = shouldAddInspection + ? findLatestBuildToolCallId(messages) + : null; + + const hydrated = await Promise.all( + messages.map(async (message) => ({ + ...message, + parts: ( + await Promise.all( + message.parts.map(async (part) => { + if ( + part.type !== 'file' || + typeof part.mediaType !== 'string' || + !part.mediaType.startsWith('image/') || + part.url.startsWith('data:') + ) { + return part; + } + const imageId = imageIdFromFilename(part.filename); + if (!imageId) return null; + const downloaded = await downloadAsBase64( + supabaseClient, + 'images', + imageStoragePath(conversation.user_id, conversation.id, imageId), + ); + if (!downloaded) return null; + return { + ...part, + mediaType: downloaded.mediaType, + url: `data:${downloaded.mediaType};base64,${downloaded.base64}`, + }; + }), + ) + ).filter((part): part is NonNullable => part != null), + })), + ); + + if (!latestBuildToolCallId) { + return hydrated.map(({ id: _id, ...message }) => message); + } + + const result: Array> = []; + for (const message of hydrated) { + const { id: _id, ...messageWithoutId } = message; + result.push(messageWithoutId); + + if (message.role !== 'assistant') continue; + + const hasLatestBuild = message.parts.some( + (part) => + part.type === 'tool-build_parametric_model' && + part.state === 'output-available' && + 'toolCallId' in part && + part.toolCallId === latestBuildToolCallId, + ); + if (!hasLatestBuild) continue; + + const inspectionPath = inspectionPreviewStoragePath( + conversation.user_id, + conversation.id, + latestBuildToolCallId, + ); + const downloaded = await downloadAsBase64( + supabaseClient, + 'images', + inspectionPath, + ); + if (!downloaded) { + logError(new Error('Failed to hydrate inspection preview image'), { + functionName: 'ai-chat', + statusCode: 500, + userId: conversation.user_id, + conversationId: conversation.id, + additionalContext: { + operation: 'inspection_hydration_download', + toolCallId: latestBuildToolCallId, + storagePath: inspectionPath, + }, + }); + continue; + } + + result.push({ + role: 'user', + parts: [ + { + type: 'text', + text: 'Multi-view inspection render from the tool result above.', + }, + { + type: 'file', + mediaType: downloaded.mediaType, + url: `data:${downloaded.mediaType};base64,${downloaded.base64}`, + }, + ], + }); + } + + return result; +} + async function downloadAsBase64( supabaseClient: SupabaseAnon, bucket: string, @@ -892,12 +1080,25 @@ async function downloadAsBase64( return { base64: btoa(binary), mediaType }; } +async function storageObjectExists( + supabaseClient: SupabaseAnon, + bucket: string, + path: string, +): Promise { + const { data, error } = await supabaseClient.storage + .from(bucket) + .createSignedUrl(path, 60); + return !error && !!data?.signedUrl; +} + function parametricTools({ - previewPathForToolCall, supabaseClient, + conversation, + modelId, }: { - previewPathForToolCall: (toolCallId: string) => string; supabaseClient: SupabaseAnon; + conversation: ConversationAccess; + modelId: string; }) { return { build_parametric_model: { @@ -909,42 +1110,94 @@ function parametricTools({ toolCallId: string; output: AppTools['build_parametric_model']['output']; }) { - // The client uploads a multi-view render of the compiled SCAD to a path - // derived from toolCallId BEFORE sending the tool result (see - // ChatSession's `onToolCall`). If for any reason the upload - // didn't land, `downloadAsBase64` returns null and we fall back - // to text-only — never block the loop on a missing inspection sheet. - const downloaded = await downloadAsBase64( - supabaseClient, - 'images', - previewPathForToolCall(toolCallId), - ); const views = output.inspection?.views.join(', ') ?? 'ISO, FRONT, BACK, LEFT, RIGHT, TOP, BOTTOM'; - const text = `${output.message}\nRendered inspection views: ${views}.\nMulti-view inspection image attached: ${downloaded ? 'yes' : 'no'}.`; - if (downloaded) { - return { - type: 'content' as const, - value: [ - { type: 'text' as const, text }, - { - type: 'image-data' as const, - data: downloaded.base64, - mediaType: downloaded.mediaType, - }, - ], - }; - } + if ( + providerFor(modelId) !== 'local' && + chatModelSupportsVision(modelId) + ) { + // The client uploads a multi-view render of the compiled SCAD to a path + // derived from toolCallId BEFORE sending the tool result (see + // ChatSession's `onToolCall`). If for any reason the upload + // didn't land, `downloadAsBase64` returns null and we fall back + // to text-only — never block the loop on a missing inspection sheet. + const downloaded = await downloadAsBase64( + supabaseClient, + 'images', + inspectionPreviewStoragePath( + conversation.user_id, + conversation.id, + toolCallId, + ), + ); + const text = `${output.message}\nRendered inspection views: ${views}.\nMulti-view inspection image attached: ${downloaded ? 'yes' : 'no'}.`; + + if (downloaded) { + return { + type: 'content' as const, + value: [ + { type: 'text' as const, text }, + { + type: 'image-data' as const, + data: downloaded.base64, + mediaType: downloaded.mediaType, + }, + ], + }; + } - return { type: 'text' as const, value: text }; + return { type: 'text' as const, value: text }; + } else { + // For models that do not support multimodal tools result messages, + // the rendered inspection views are appended as user messages in buildHydratedMessages + + // Text-only models never receive inspection images + if (!chatModelSupportsVision(modelId)) { + return { type: 'text' as const, value: output.message }; + } + + // Local vision models rely on the hydration path for the actual image. + // First check storage prescence to determine output message. + let imageAttached = output.inspection?.imageAttached === true; + if ( + providerFor(modelId) === 'local' && + chatModelSupportsVision(modelId) + ) { + imageAttached = await storageObjectExists( + supabaseClient, + 'images', + inspectionPreviewStoragePath( + conversation.user_id, + conversation.id, + toolCallId, + ), + ); + } + const text = `${output.message}\nRendered inspection views: ${views}.\nMulti-view inspection image attached: ${imageAttached ? 'yes' : 'no'}.`; + return { type: 'text' as const, value: text }; + } }, }, answer_user: chatTools.answer_user, }; } +function getAuxLanguageModel(providers: ChatProviders): LanguageModel | null { + const { activeCatalog, hasAnthropicApiKey } = getLocalChatState(); + const auxLocal = getAuxiliaryLocalModel(activeCatalog); + if (auxLocal?.baseUrl) { + const baseUrl = + normalizeOpenRouterBaseUrl(auxLocal.baseUrl) ?? auxLocal.baseUrl; + return providers + .local(baseUrl, localApiKeyForModel(auxLocal)) + .chat(auxLocal.id); + } + if (hasAnthropicApiKey) return providers.anthropic()('claude-haiku-4-5'); + return null; +} + function chatModel(conversation: ConversationAccess, model: Model) { if (conversation.type === 'creative') { return 'anthropic/claude-sonnet-4.5'; @@ -1004,33 +1257,44 @@ export async function handleAiChatRequest(req: Request) { ); } - // Pre-flight balance gate. A chat costs at least 1 billing token, so a - // total of 0 means we cannot let the stream start. We don't try to + const actualModelId = chatModel(conversation, rawBody.model); + if (isMissingLocalBaseUrl(actualModelId)) { + return jsonResponse( + { error: 'baseUrl is not configured for local model' }, + 503, + ); + } + + // Pre-flight balance gate for billable (cloud) models. Local-catalog + // models are billing-exempt. Cloud chat costs at least 1 billing token, + // so a total of 0 means we cannot let the stream start. We don't try to // estimate the exact cost up front — chat is variable, and the billing // service drains the remainder to zero if the actual usage exceeds // what's left (see onFinish below). - try { - const status = await billing.getStatus(user.email); - if (status.tokens.total <= 0) { - return jsonResponse( - { - error: 'insufficient_tokens', - code: 'insufficient_tokens', - tokensRequired: 1, - tokensAvailable: 0, - }, - 402, - ); + if (!isActiveLocalModel(actualModelId)) { + try { + const status = await billing.getStatus(user.email); + if (status.tokens.total <= 0) { + return jsonResponse( + { + error: 'insufficient_tokens', + code: 'insufficient_tokens', + tokensRequired: 1, + tokensAvailable: 0, + }, + 402, + ); + } + } catch (error) { + logError(error, { + functionName: 'ai-chat', + statusCode: error instanceof BillingClientError ? error.status : 502, + userId: user.id, + conversationId: conversation.id, + additionalContext: { operation: 'billing_preflight' }, + }); + return jsonResponse({ error: 'Billing service unavailable' }, 503); } - } catch (error) { - logError(error, { - functionName: 'ai-chat', - statusCode: error instanceof BillingClientError ? error.status : 502, - userId: user.id, - conversationId: conversation.id, - additionalContext: { operation: 'billing_preflight' }, - }); - return jsonResponse({ error: 'Billing service unavailable' }, 503); } const tools = @@ -1038,8 +1302,8 @@ export async function handleAiChatRequest(req: Request) { ? creativeTools({ conversation, req, model: rawBody.model }) : parametricTools({ supabaseClient, - previewPathForToolCall: (toolCallId) => - `${user.id}/${conversation.id}/inspection-preview-${toolCallId}`, + conversation, + modelId: actualModelId, }); let branchMessages: AppUIMessage[]; @@ -1088,6 +1352,9 @@ export async function handleAiChatRequest(req: Request) { // the very first user turn. const isFirstUserTurn = branchMessages.length === 1 && leafRole === 'user'; + // Resolve the actual model ID the request will run against. + const resolvedProvider = providerFor(actualModelId); + // Rehydrate image file parts before handing them to the model. The // persisted `url` is a storage reference (or, for the oldest backfilled // rows, a dead `/public/` path), neither of which the provider can fetch — @@ -1097,38 +1364,18 @@ export async function handleAiChatRequest(req: Request) { // (legacy rows that inlined base64) pass through untouched; anything we // can't resolve is dropped so a missing image never poisons the request // with an unfetchable URL. - const hydratedMessages = await Promise.all( - branchMessages.map(async (message) => ({ - ...message, - parts: ( - await Promise.all( - message.parts.map(async (part) => { - if ( - part.type !== 'file' || - typeof part.mediaType !== 'string' || - !part.mediaType.startsWith('image/') || - part.url.startsWith('data:') - ) { - return part; - } - const imageId = imageIdFromFilename(part.filename); - if (!imageId) return null; - const downloaded = await downloadAsBase64( - supabaseClient, - 'images', - imageStoragePath(conversation.user_id, conversation.id, imageId), - ); - if (!downloaded) return null; - return { - ...part, - mediaType: downloaded.mediaType, - url: `data:${downloaded.mediaType};base64,${downloaded.base64}`, - }; - }), - ) - ).filter((part): part is NonNullable => part != null), - })), - ); + // + // For local providers on parametric conversations, strict OpenAI-compatible + // servers reject images inside `role: tool` messages. We instead insert a + // follow-up `role: user` message carrying the latest inspection render after + // the most recent build tool call so `convertToModelMessages` never needs to + // put images in tool output. + const hydratedMessages = await buildHydratedMessages(branchMessages, { + supabaseClient, + conversation, + modelId: actualModelId, + provider: resolvedProvider, + }); const modelMessages = await convertToModelMessages( hydratedMessages, @@ -1161,13 +1408,6 @@ export async function handleAiChatRequest(req: Request) { }, }, ); - - // Resolve the actual model ID the request will run against. For - // `creative` conversations this is hardcoded to Sonnet regardless of - // what the client picked — billing has to price the model that ran, - // not the one the user requested. - const actualModelId = chatModel(conversation, rawBody.model); - const resolvedProvider = providerFor(actualModelId); const baseLogContext = { userId: user.id, conversationId: conversation.id, @@ -1246,14 +1486,12 @@ export async function handleAiChatRequest(req: Request) { leafRole === 'user' && stepNumber === 0 ) { - // Restrict the toolset to the build tool on the first step. Models that - // accept a forced tool_choice get it pinned; the reasoning-tier Claude 5 - // models (Fable/Mythos) reject forced tool use and fall back to auto, - // relying on the system prompt to call build_parametric_model. - // When pinning the tool on a thinking-enabled Anthropic model, thinking - // must be off for this step (Anthropic rejects forced tool use while - // thinking is on) — disable it here only; later steps keep the adaptive - // thinking configured in buildChatModel. + // Restrict the toolset to the build tool on the first step. Turns that + // accept a forced tool_choice get it pinned; turns that reject forced + // tool use (Claude Fable/Mythos, local models with + // supportsForcedToolChoice: false, or the supportsThinking fallback) + // fall back to auto and rely on the system prompt to call + // build_parametric_model. return { activeTools: ['build_parametric_model' as never], ...(forceBuildToolChoice @@ -1373,14 +1611,17 @@ export async function handleAiChatRequest(req: Request) { execute: async ({ writer }) => { // Title (first user turn only) runs in parallel with the model // stream — fire-and-forget; the assistant doesn't wait on it. - if (isFirstUserTurn && env('ANTHROPIC_API_KEY')) { - void emitConversationTitle({ - writer, - anthropic: providers.anthropic(), - supabaseClient, - conversation, - firstMessage: branchMessages[0], - }); + if (isFirstUserTurn) { + const auxModel = getAuxLanguageModel(providers); + if (auxModel) { + void emitConversationTitle({ + writer, + auxModel, + supabaseClient, + conversation, + firstMessage: branchMessages[0], + }); + } } writer.merge( @@ -1488,29 +1729,31 @@ export async function handleAiChatRequest(req: Request) { }); } - try { - // Drains the user's remaining balance to zero if the - // request cost more than they had. The billing service - // accepts the partial deduction, writes an audit row as - // `_partial`, and the pre-flight gate above - // will block the next request. Not an error path — - // intentional terminal state. Runs after the persist above so - // its latency never delays the row the client is waiting on. - await billing.consume(user.email!, { - tokens: billingTokens, - operation: - conversation.type === 'creative' ? 'chat' : 'parametric', - referenceId: responseMessage.id, - }); - } catch (error) { - logError(error, { - functionName: 'ai-chat', - statusCode: - error instanceof BillingClientError ? error.status : 502, - userId: user.id, - conversationId: conversation.id, - additionalContext: { operation: 'billing_consume' }, - }); + if (billingTokens > 0) { + try { + // Drains the user's remaining balance to zero if the + // request cost more than they had. The billing service + // accepts the partial deduction, writes an audit row as + // `_partial`, and the pre-flight gate above + // will block the next request. Not an error path — + // intentional terminal state. Runs after the persist above so + // its latency never delays the row the client is waiting on. + await billing.consume(user.email!, { + tokens: billingTokens, + operation: + conversation.type === 'creative' ? 'chat' : 'parametric', + referenceId: responseMessage.id, + }); + } catch (error) { + logError(error, { + functionName: 'ai-chat', + statusCode: + error instanceof BillingClientError ? error.status : 502, + userId: user.id, + conversationId: conversation.id, + additionalContext: { operation: 'billing_consume' }, + }); + } } // Only generate suggestions once the assistant has actually @@ -1519,28 +1762,31 @@ export async function handleAiChatRequest(req: Request) { // continuation `onFinish` will fire suggestions for the real // final state. Avoids a wasted Haiku call AND prevents // mid-turn placeholder pills. - if (!hasPendingToolCall && env('ANTHROPIC_API_KEY')) { - // MUST be awaited (not `void`). `createUIMessageStream` - // closes the SSE controller as soon as the merged stream - // drains — and the merged stream resolves once this - // `onFinish` returns. A fire-and-forget here would race - // the close, and the `writer.write` inside - // `emitConversationSuggestions` would silently no-op - // because `safeEnqueue` swallows enqueue errors on a - // closed controller (see ai/dist/index.mjs:8264). The - // ~200-500ms Haiku call delays the client's "streaming" - // → "ready" transition by the same amount, which is the - // tradeoff for getting pills delivered. - await emitConversationSuggestions({ - writer, - anthropic: providers.anthropic(), - supabaseClient, - conversation, - branch: [ - ...branchMessages, - { ...responseMessage, parts: finalizedParts }, - ], - }); + if (!hasPendingToolCall) { + const auxModel = getAuxLanguageModel(providers); + if (auxModel) { + // MUST be awaited (not `void`). `createUIMessageStream` + // closes the SSE controller as soon as the merged stream + // drains — and the merged stream resolves once this + // `onFinish` returns. A fire-and-forget here would race + // the close, and the `writer.write` inside + // `emitConversationSuggestions` would silently no-op + // because `safeEnqueue` swallows enqueue errors on a + // closed controller (see ai/dist/index.mjs:8264). The + // ~200-500ms aux-model call delays the client's "streaming" + // → "ready" transition by the same amount, which is the + // tradeoff for getting pills delivered. + await emitConversationSuggestions({ + writer, + auxModel, + supabaseClient, + conversation, + branch: [ + ...branchMessages, + { ...responseMessage, parts: finalizedParts }, + ], + }); + } } }, }), @@ -1565,19 +1811,19 @@ export async function handleAiChatRequest(req: Request) { */ async function emitConversationTitle({ writer, - anthropic, + auxModel, supabaseClient, conversation, firstMessage, }: { writer: UIMessageStreamWriter; - anthropic: AnthropicProvider; + auxModel: LanguageModel; supabaseClient: SupabaseAnon; conversation: ConversationAccess; firstMessage: AppUIMessage; }) { try { - const title = await generateConversationTitle({ anthropic, firstMessage }); + const title = await generateConversationTitle({ auxModel, firstMessage }); await supabaseClient .from('conversations') .update({ title }) @@ -1607,20 +1853,20 @@ async function emitConversationTitle({ */ async function emitConversationSuggestions({ writer, - anthropic, + auxModel, supabaseClient, conversation, branch, }: { writer: UIMessageStreamWriter; - anthropic: AnthropicProvider; + auxModel: LanguageModel; supabaseClient: SupabaseAnon; conversation: ConversationAccess; branch: AppUIMessage[]; }) { try { const suggestions = await generateConversationSuggestions({ - anthropic, + auxModel, branch, conversationType: conversation.type, }); diff --git a/src/server/localChatConfig.ts b/src/server/localChatConfig.ts new file mode 100644 index 00000000..18fb9c57 --- /dev/null +++ b/src/server/localChatConfig.ts @@ -0,0 +1,108 @@ +/** + * Server-side local-chat configuration: reads `local-models.json` from the + * project root. A model is active when it has a `baseUrl` set in the catalog. + * Pure routing logic lives in `@shared/localModels`; this module wires + * disk I/O for `aiChat`. + */ +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { + isMissingLocalBaseUrl as isMissingLocalBaseUrlCore, + isLocalModelId, + parseLocalModelsJson, + providerFor as providerForCore, + type ChatProvider, + type LocalModelConfig, +} from '@shared/localModels'; +import { env } from './env'; + +export type { ChatProvider }; + +export type LocalChatState = { + enabled: boolean; + /** On-disk catalog, even when local chat is disabled. */ + catalog: LocalModelConfig[]; + /** Models with a configured baseUrl. */ + activeCatalog: LocalModelConfig[]; + hasAnthropicApiKey: boolean; +}; + +const projectRoot = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + '../..', +); + +// Cached after first read; restart the dev server after editing `local-models.json`. +let cachedCatalog: LocalModelConfig[] | undefined; + +function loadCatalogFromDisk(): LocalModelConfig[] { + if (cachedCatalog) return cachedCatalog; + + const userPath = path.join(projectRoot, 'local-models.json'); + if (!fs.existsSync(userPath)) { + cachedCatalog = []; + return cachedCatalog; + } + + try { + const raw = JSON.parse(fs.readFileSync(userPath, 'utf8')) as unknown; + cachedCatalog = parseLocalModelsJson(raw); + } catch (error) { + console.error( + `Failed to load local-models.json from ${userPath}; using empty catalog.`, + error, + ); + cachedCatalog = []; + } + + return cachedCatalog; +} + +export function getLocalChatState(): LocalChatState { + const catalog = loadCatalogFromDisk(); + const activeCatalog = catalog.filter((m) => m.baseUrl?.trim()); + + return { + enabled: activeCatalog.length > 0, + catalog, + activeCatalog, + hasAnthropicApiKey: env('ANTHROPIC_API_KEY').trim().length > 0, + }; +} + +export function providerFor(modelId: string): ChatProvider { + const { activeCatalog } = getLocalChatState(); + return providerForCore(modelId, activeCatalog); +} + +export function isActiveLocalModel(modelId: string): boolean { + return isLocalModelId(modelId, getLocalChatState().activeCatalog); +} + +export function getActiveLocalModel( + modelId: string, +): LocalModelConfig | undefined { + return getLocalChatState().activeCatalog.find( + (model) => model.id === modelId, + ); +} + +// Resolve local API key from env without exposing secrets to the client bundle. +// If `apiKey` is set in `local-models.json`, it is interpreted as the env var +// name (for example `DEEPSEEK_API_KEY`), not a raw secret. +export function localApiKeyForModel( + model: LocalModelConfig, +): string | undefined { + const configuredName = model.apiKey?.trim(); + if (!configuredName) return undefined; + const configured = env(configuredName).trim(); + return configured || undefined; +} + +// Client may show catalog ids whose baseUrl is missing on the server — reject +// those requests with a clear error. +export function isMissingLocalBaseUrl(modelId: string): boolean { + const { catalog } = getLocalChatState(); + return isMissingLocalBaseUrlCore(modelId, catalog); +} diff --git a/src/server/messageUtils.test.ts b/src/server/messageUtils.test.ts new file mode 100644 index 00000000..e6da449a --- /dev/null +++ b/src/server/messageUtils.test.ts @@ -0,0 +1,90 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; +import type { AppUIMessage } from '@shared/chatAi'; +import { findLatestBuildToolCallId } from './messageUtils.ts'; + +function buildToolPart( + toolCallId: string, + state: + | 'output-available' + | 'input-available' + | 'output-error' = 'output-available', +): AppUIMessage['parts'][number] { + return { + type: 'tool-build_parametric_model', + toolCallId, + state, + input: { code: 'cube(1);' }, + ...(state === 'output-available' + ? { + output: { + status: 'success' as const, + message: 'ok', + inspection: { views: ['ISO'] as const, imageAttached: true }, + }, + } + : state === 'output-error' + ? { errorText: 'failed' } + : {}), + } as AppUIMessage['parts'][number]; +} + +describe('findLatestBuildToolCallId', () => { + it('returns null for an empty branch', () => { + assert.equal(findLatestBuildToolCallId([]), null); + }); + + it('returns the only completed build tool call', () => { + const messages: AppUIMessage[] = [ + { + id: 'a1', + role: 'assistant', + parts: [buildToolPart('call-1')], + }, + ]; + assert.equal(findLatestBuildToolCallId(messages), 'call-1'); + }); + + it('returns the latest build across assistant messages', () => { + const messages: AppUIMessage[] = [ + { + id: 'a1', + role: 'assistant', + parts: [buildToolPart('call-1')], + }, + { id: 'u1', role: 'user', parts: [{ type: 'text', text: 'taller' }] }, + { + id: 'a2', + role: 'assistant', + parts: [buildToolPart('call-2')], + }, + ]; + assert.equal(findLatestBuildToolCallId(messages), 'call-2'); + }); + + it('returns the last completed build part within one assistant message', () => { + const messages: AppUIMessage[] = [ + { + id: 'a1', + role: 'assistant', + parts: [buildToolPart('call-1'), buildToolPart('call-2')], + }, + ]; + assert.equal(findLatestBuildToolCallId(messages), 'call-2'); + }); + + it('ignores non-output build tool parts', () => { + const messages: AppUIMessage[] = [ + { + id: 'a1', + role: 'assistant', + parts: [ + buildToolPart('call-error', 'output-error'), + buildToolPart('call-pending', 'input-available'), + buildToolPart('call-done'), + ], + }, + ]; + assert.equal(findLatestBuildToolCallId(messages), 'call-done'); + }); +}); diff --git a/src/server/messageUtils.ts b/src/server/messageUtils.ts index adda5a40..0df23eed 100644 --- a/src/server/messageUtils.ts +++ b/src/server/messageUtils.ts @@ -1,4 +1,5 @@ -import { env } from './env'; +import { env } from './env.ts'; +import type { AppUIMessage } from '@shared/chatAi'; export function reformatSignedUrl(signedUrl: string): string { const supabaseHost = ( @@ -8,3 +9,33 @@ export function reformatSignedUrl(signedUrl: string): string { const url = new URL(signedUrl); return `${supabaseHost}${url.pathname}${url.search}`; } + +/** + * Latest `build_parametric_model` with `output-available` on the branch. + * `buildHydratedMessages` uses this so local parametric only re-downloads + * one inspection preview per request (newest-to-oldest walk). + */ +export function findLatestBuildToolCallId( + messages: AppUIMessage[], +): string | null { + for (let index = messages.length - 1; index >= 0; index--) { + const message = messages[index]; + if (message.role !== 'assistant') continue; + for ( + let partIndex = message.parts.length - 1; + partIndex >= 0; + partIndex-- + ) { + const part = message.parts[partIndex]; + if ( + part.type === 'tool-build_parametric_model' && + part.state === 'output-available' && + 'toolCallId' in part && + typeof part.toolCallId === 'string' + ) { + return part.toolCallId; + } + } + } + return null; +} diff --git a/src/views/EditorView.tsx b/src/views/EditorView.tsx index bed7cd7d..d6c4f4e6 100644 --- a/src/views/EditorView.tsx +++ b/src/views/EditorView.tsx @@ -14,6 +14,7 @@ import { OpenSCADPreview } from '@/components/viewer/OpenSCADViewer'; import { MeshPreview } from '@/components/viewer/MeshPreview'; import Loader from '@/components/viewer/Loader'; import { useAuth } from '@/contexts/AuthContext'; +import { useIsLocalModel } from '@/hooks/useLocalModels.ts'; import { ConversationContext } from '@/contexts/ConversationContext'; import { SelectedItemsContext } from '@/contexts/SelectedItemsContext'; import { useConversation } from '@/contexts/ConversationContext'; @@ -194,6 +195,7 @@ function ConversationEditor() { ? 'quality' : 'google/gemini-3.1-pro-preview'), ); + const isLocalModel = useIsLocalModel(model); const [activePreview, setActivePreview] = useState(null); const [parameters, setParameters] = useState([]); const [currentOutput, setCurrentOutput] = useState(); @@ -705,7 +707,7 @@ function ConversationEditor() { initialBranch={initialBranch} model={model} setModel={updateSelectedModel} - isDisabled={totalTokens <= 0} + isDisabled={totalTokens <= 0 && !isLocalModel} onSendParts={handleSendParts} onRetry={handleRetry} onEdit={handleEdit} diff --git a/src/views/PromptView.tsx b/src/views/PromptView.tsx index 1d5d7f95..9a54ce9f 100644 --- a/src/views/PromptView.tsx +++ b/src/views/PromptView.tsx @@ -14,6 +14,7 @@ import { LowPromptsWarningMessage } from '@/components/LowPromptsWarningMessage' import { NewProductBanner } from '@/components/NewProductBanner'; import { FreePlanTrialPill } from '@/components/FreePlanTrialPill'; import { useIsMobile } from '@/hooks/useIsMobile'; +import { useIsLocalModel } from '@/hooks/useLocalModels.ts'; import { cn } from '@/lib/utils'; import { SelectedItemsContext } from '@/contexts/SelectedItemsContext'; import posthog from 'posthog-js'; @@ -65,6 +66,8 @@ export function PromptView() { const [model, setModel] = useState('google/gemini-3.1-pro-preview'); + const isLocalModel = useIsLocalModel(model); + const handleTypeChange = (newType: 'parametric' | 'creative') => { setType(newType); // Reset model to the default for the new type @@ -91,8 +94,9 @@ export function PromptView() { const limitReached = useMemo(() => { if (isLoading) return false; + if (type === 'parametric' && isLocalModel) return false; return totalTokens <= 0; - }, [totalTokens, isLoading]); + }, [totalTokens, isLoading, type, isLocalModel]); // Trigger fade in on mount useEffect(() => {