From eb722482e20d41895eea12824f91eea2c05bf542 Mon Sep 17 00:00:00 2001 From: M007-Net <262222560+M007-Net@users.noreply.github.com> Date: Thu, 17 Sep 2026 03:13:58 -0400 Subject: [PATCH 1/8] Local Model Bench 1.10.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Carries two releases. 1.9.0 was prepared and installed locally but never pushed: native MTP for models whose prediction heads ship as a separate file, a sweep across prediction depths, an optional preflight that finds out early when a head will not load, readable `lms` failures, and a text-only twin for benchmarking a vision model without its projector. 1.10.0 is about measuring the machine rather than the measurement. Prompt processing was reported at roughly a third of its real speed. The figure came from LM Studio's prompt_processing interval, which lands within a millisecond or two of time-to-first-token, so it carried the whole fixed cost of a request on top of the prefill; at the prompt sizes the performance tests use, that fixed cost was the measurement. No single request can separate a fixed cost from a per-token one, so each loaded model is now measured over a short prompt and a long one and the difference gives the real per-token rate, naming the overhead that was being charged to the GPU. The older per-request figure is kept beside it. A run can name which llama.cpp build to use, and puts the previous selection back when it ends. This turned out to matter: on an RX 9070 with Gemma 4 12B Q4_K_XL, Vulkan 2.40.0 reached 1027 tok/s of prompt processing against ROCm 2.40.0's 148, and ROCm carried about 16 s of fixed cost before its first token at every prompt size tested. Where two engines have measured the same model, the results tables and graphs will set them side by side. The key/value cache can be quantized per run. It is the part of a run's memory that grows with concurrency rather than with the weights, and on a card whose weights already nearly fill VRAM it decides whether the run measures the GPU or a spill into system RAM. Measured cost: q8_0 for K and V took about 9% of prompt processing. LM Studio exposes no flag for it, so the setting is written to its own per-model configuration for one load and the file restored byte for byte after. Results are also readable one question at a time — all measurements, prompt processing, or token generation — each with the columns that bear on it, and a load that runs out of room now names the settings that decided how much it needed. 233 offline tests, typecheck and production build all pass. Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 162 +++++++++++++++ README.md | 207 ++++++++++++++++++- electron/cache-quant.ts | 95 +++++++++ electron/cli-output.ts | Bin 0 -> 1592 bytes electron/engine-log.ts | 85 ++++++++ electron/engine.ts | 202 ++++++++++++++++--- electron/export.ts | 28 ++- electron/history.ts | 15 +- electron/lmstudio.ts | Bin 9807 -> 9874 bytes electron/load-estimate.ts | 70 +++++++ electron/main.ts | 20 +- electron/metrics.ts | 7 +- electron/mtp-sidecar.ts | 60 ++++++ electron/mtp.ts | 107 ++++++++-- electron/preload.ts | 2 +- electron/runtime.ts | 102 ++++++++++ electron/text-only.ts | 151 ++++++++++++++ electron/validation.ts | Bin 5563 -> 7147 bytes electron/vision.ts | 14 +- package-lock.json | 4 +- package.json | 2 +- scripts/qa-accounting-mtp.mjs | 2 +- scripts/qa-accounting-upgrade.mjs | 4 +- scripts/qa-charts.mjs | 2 +- scripts/qa-comparison.mjs | 2 +- scripts/qa-gpu.ts | 2 +- scripts/qa-library.mjs | 2 +- scripts/qa-mtp-sweep.mjs | 138 +++++++++++++ scripts/qa-results.mjs | 2 +- scripts/qa-text-only.mjs | 82 ++++++++ src/BenchmarkPanels.tsx | 5 +- src/ChartsPanel.tsx | 42 +++- src/HistoryPanel.tsx | 2 +- src/LibraryControls.tsx | 2 +- src/benchmarks.ts | 13 +- src/cache-quant.ts | 38 ++++ src/chart-compare.ts | 111 +++++++++++ src/history.ts | 34 +++- src/main.tsx | 175 +++++++++++++++-- src/mtp-sweep.ts | 164 ++++++++++++++++ src/prefill.ts | 81 ++++++++ src/results-focus.ts | 113 +++++++++++ src/style.css | 20 ++ src/types.ts | 14 +- src/version.ts | 8 + tests/cache-quant.test.ts | 83 ++++++++ tests/chart-compare.test.ts | 81 ++++++++ tests/charts.test.ts | 2 +- tests/cli-output.test.ts | 44 +++++ tests/history.test.ts | 3 +- tests/load-estimate.test.ts | 67 +++++++ tests/mtp-sidecar.test.ts | 214 ++++++++++++++++++++ tests/mtp-sweep.test.ts | 317 ++++++++++++++++++++++++++++++ tests/prefill.test.ts | 97 +++++++++ tests/text-only.test.ts | 132 +++++++++++++ vite.config.ts | 6 +- 56 files changed, 3313 insertions(+), 124 deletions(-) create mode 100644 electron/cache-quant.ts create mode 100644 electron/cli-output.ts create mode 100644 electron/engine-log.ts create mode 100644 electron/load-estimate.ts create mode 100644 electron/mtp-sidecar.ts create mode 100644 electron/runtime.ts create mode 100644 electron/text-only.ts create mode 100644 scripts/qa-mtp-sweep.mjs create mode 100644 scripts/qa-text-only.mjs create mode 100644 src/cache-quant.ts create mode 100644 src/chart-compare.ts create mode 100644 src/mtp-sweep.ts create mode 100644 src/prefill.ts create mode 100644 src/results-focus.ts create mode 100644 src/version.ts create mode 100644 tests/cache-quant.test.ts create mode 100644 tests/chart-compare.test.ts create mode 100644 tests/cli-output.test.ts create mode 100644 tests/load-estimate.test.ts create mode 100644 tests/mtp-sidecar.test.ts create mode 100644 tests/mtp-sweep.test.ts create mode 100644 tests/prefill.test.ts create mode 100644 tests/text-only.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 43cb27b..9c2ec46 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,168 @@ All notable changes to Local Model Bench are recorded here. The format follows Dates are the date the version was prepared. +## 1.10.0 — 2026-09-17 + +### Added + +- **Choose which llama.cpp build a run uses.** LM Studio keeps one selected engine + at a time and has no per-load switch, so which build produced a set of numbers was + not recorded anywhere and could not be recovered afterwards. A run can now name an + engine; it is selected before the first load and the previous choice is put back + when the run ends, whatever happens to the run. On an RX 9070 the difference is not + subtle: measured on Gemma 4 12B Q4_K_XL, Vulkan 2.40.0 reached 1027 tok/s of prompt + processing against ROCm 2.40.0's 148 tok/s, and ROCm carried roughly 16 s of fixed + cost before its first token at every prompt size tested. +- **Quantize the context.** The key/value cache is the part of a run's memory that + grows with concurrency rather than with the weights: an instance holds context × + parallel slots of it. K and V can now be set independently to q8_0, q5_0, q4_0, + iq4_nl or f16. LM Studio exposes no flag for this, so the setting is written to its + own per-model configuration for the length of one load and the file is put back + byte for byte afterwards, the same discipline the MTP head already used. Measured + cost on Gemma 4 12B Q4_K_XL: q8_0 for both cost about 9% of prompt processing + (569 tok/s to 519 tok/s). +- **Results read one question at a time.** The comparison table is offered under + three headings — all measurements, prompt processing, and token generation — each + carrying the columns that bear on that question. Generation gains total throughput + beside the per-request rate, the MTP depth, drafted-token acceptance, and what each + extra concurrent user costs every other one. +- **Set another engine's numbers beside this run's.** Where saved runs measured the + same models on a different engine, the results screen offers Show ROCm, Show Vulkan + or Show all engines, and the table gains an engine column. The graphs gain a + Compare with menu listing other engines and other models, drawn as their own lines. + Nothing is recalculated: every overlaid point is a saved row, newest run per + measurement. The engine is also a filter and a grouping in the history overview. + +### Fixed + +- **Prompt processing was reported at roughly a third of its real speed.** The figure + came from LM Studio's prompt_processing interval, which in practice lands within a + millisecond or two of time-to-first-token, so it carried the whole fixed cost of a + request — HTTP, tokenization, scheduling, the first sampling step — on top of the + prefill. At the prompt sizes the performance tests use, that fixed cost *was* the + measurement. No single request can separate a fixed cost from a per-token one, so + each loaded model is now measured twice, over a short prompt and a long one; the + fixed cost is identical in both and cancels, leaving the real per-token rate and + naming the overhead that was being charged to the GPU. The older per-request figure + is kept beside it rather than silently replaced. +- **A load that ran out of room named none of the settings that caused it.** LM Studio + reports it in its own words, which never mention context, concurrency or the cache. + Failures that look like memory now carry the arithmetic the run already did — the + tokens actually asked for, the weights, LM Studio's own estimate — and the three + settings that change it. + +## 1.9.0 — 2026-09-15 + +### Added + +- **An optional preflight for the MTP head.** LM Studio's metadata confirming that + a model's prediction heads exist and match does not mean its runtime can load + them against that exact file. **Check the MTP head loads first** on the Run + screen loads each selected model once with MTP on, at the run's own parallel + slots and context, before anything is measured. A model whose head will not load + is named and its MTP depths are skipped; its MTP-off baseline still runs. Off by + default: it costs one extra load per model whose head does load, and what it buys + is finding out early rather than a shorter run. Measured on a two-model, two-depth + sweep against a head that will not load: reported at 25 s with the check and 82 s + without, with identical measurements either way. + +### Fixed + +- **A failed `lms` command was unreadable.** The CLI draws an animated progress bar, + and on failure the whole stream — cursor codes, colour codes and one repaint frame + per percent — became the error message. Saved run logs held up to 11,195 characters + for a single failure, with LM Studio's own one-line cause buried inside. The + progress frames and escape codes are now stripped and the duplicated failure block + collapsed, turning that example into 353 readable characters. +- **A run that lost one model said only how many things failed.** The summary now + names each model and depth that went unmeasured, and states that everything else + in the run was measured and saved. +- **Native MTP for models whose prediction heads are a separate file.** LM Studio + ships MTP two ways: heads built into the model's own GGUF, and heads stored with + it as their own file that LM Studio indexes separately. Only the first had a + command-line switch, so Gemma 4 — which ships the second kind — was reported as + having no native MTP at all and could not be benchmarked with it. A model is now + paired with a head in its own folder by structure alone: prediction layers + declared, output width equal to that model's hidden size, and that model's + architecture under LM Studio's "-assistant" suffix. Two heads that both fit are + reported as ambiguous rather than guessed between. +- **Loading a separate head.** LM Studio exposes the setting in neither its CLI nor + its HTTP API, so it is written into LM Studio's own per-model configuration for + the length of one load and that file is restored byte for byte immediately after, + whether the load succeeded or failed; a file this app created is deleted again + along with any folders it had to make. +- **Confirming a separate head.** LM Studio reports nothing about one through its + API, so a load is confirmed against its engine log, which has to name that exact + head file before anything is measured. A sweep's MTP-off baseline is checked the + same way in reverse: a head loaded for the baseline abandons that depth rather + than letting every other depth be compared against a speculative run. +- **Text-only copies of vision models.** A vision projector belongs to the model entry + LM Studio indexed, not to the load: its whole projector handling reads `mmproj_path` + straight off that entry, with no flag, no API field and no load-config key in front of + it. So Vision off could only ever mean "no image was sent", never "the projector was + left out" — and a model like Gemma 4, whose projector sits in its own folder, could not + be measured without it. The Models screen gains a **Vision projectors** panel that makes + a second, projector-free entry for LM Studio to index: a sibling folder of hard links to + the same weights, with the projector left out and any MTP head linked in so the copy + keeps native MTP. Nothing is duplicated — the copy shares the original's bytes and uses + no extra disk space — and removing it deletes links only, refusing outright if the folder + holds any file that exists nowhere else. +- **Accepted-draft rate.** Every wave run with MTP on now records how much of the drafting + the main model actually kept: the share of drafted tokens accepted, and the mean accepted + run per drafting step. LM Studio publishes neither through its API, so both are read from + the per-request lines its engine prints, pooled over the wave by drafted tokens rather than + averaged across requests. They appear in the sweep panel, the Markdown export, and as + `draftAcceptance` and `draftMeanLen` columns in CSV. MTP off reports no rate at all, which + is not the same as a rate of zero. +- **Concurrency is swept alongside depth.** A sweep could always be given several concurrency + levels, but the results pooled them into one row per depth. Each level is now measured and + reported on its own, and a depth is only ever compared with another depth under the same + load — the depth that is fastest at one request at a time need not be the fastest with five + in flight, and pooling hid exactly that. +- **MTP speed sweep button** on the Run screen: one click sets the run to speed + only, turns MTP on, and sweeps depths 0 through 5 — every draft depth from 1 to 5 + against the MTP-off baseline. +- Model cards carry an MTP badge saying which of the two shapes a model has, or why + its support is unknown. +- **Maximum predictions sweep.** With native MTP on, a run can measure several + prediction depths instead of one. Depth is fixed when a model is loaded, so each + depth is a separate load and a separate pass over the whole workload; the run + preview says how much larger that makes the run before it is started. Depth 0 + means MTP off, so the sweep can include the baseline a speed-up has to beat. +- Results gain a **Maximum predictions sweep** panel: a row per depth with speed, + throughput, latency, time to first token and objective score, the spread behind + each average, and a sentence naming the fastest depth and what it gained. A gap + no larger than the spread of the measurements is reported as exactly that rather + than as a winner, including when the baseline is the one marginally ahead. +- Markdown exports gain a **Native MTP sweep** section, and CSV exports an + `mtpDepth` column. +- The run history overview gains an **MTP depth** condition, so pooling across + runs separates depth 2 from depth 4 instead of merging both under "on". + +### Changed + +- Every saved response and wave records the MTP depth it was measured at. Summary + rows, the comparison table, the automatic graphs, benchmark pack scores and the + history overview are all keyed on it, so two depths are never averaged together. + Runs saved before this change carry no depth and are grouped exactly as before. +- A retry re-runs each failed request at the depth it was originally measured at, + and does not load a depth that has nothing to retry. + +### Fixed + +- The version shown in the window was typed into the markup by hand and had stopped + matching the installer: 1.9.0 was packaged and installed while its own sidebar + still read "1.8.0". It now comes from `package.json` at build time, so the number + in the window and the number in the installer's name cannot disagree again. +- A sweep announced its own baseline as "MTP MTP off" while loading and while running + each wave: the progress line prefixed "MTP " to a label that already began with it. +- The Playwright QA scripts under `scripts/` waited for a window heading that was + renamed in 1.8.0, so every one of them timed out on its first assertion. They + now wait for the heading that exists. +- `electron/validation.ts` held two literal control bytes inside a character + class, which made git treat the whole file as binary and its diffs unreadable. + They are now written as escapes. The token check behaves identically. + ## 1.8.0 — 2026-09-14 First version prepared for public release. Everything below is measured against diff --git a/README.md b/README.md index fda1760..495c4e9 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,8 @@ AI provider. - [Step 3: Read the results](#step-3-read-the-results) - [If something looks wrong](#if-something-looks-wrong) - [Build from source](#build-from-source) — only if you want to compile it yourself +- [Choose the engine, and compare engines](#choose-the-engine-and-compare-engines-1100) — which llama.cpp build, and setting two side by side +- [Quantize the context](#quantize-the-context-1100) — the memory that grows with concurrency Everything after that is reference material. You do not need it to get going. @@ -61,7 +63,7 @@ account, and nothing here needs Python or a terminal. 1. Go to the [Releases page](https://github.com/M007-Net/local-model-bench/releases) and - download the file named `Local-Model-Bench-Setup-1.8.0.exe`. + download the file named `Local-Model-Bench-Setup-1.9.0.exe`. > If that page is empty or you cannot open it, no release has been published > yet. Until one is, the only way to get the app is @@ -73,7 +75,7 @@ account, and nothing here needs Python or a terminal. Right-click the Start button, choose **Terminal**, and paste this in: ```powershell - Get-FileHash -Algorithm SHA256 "$HOME\Downloads\Local-Model-Bench-Setup-1.8.0.exe" + Get-FileHash -Algorithm SHA256 "$HOME\Downloads\Local-Model-Bench-Setup-1.9.0.exe" ``` It prints a long string of letters and numbers. It must match the one listed on @@ -167,8 +169,8 @@ There is no built-in model list or model-family allowlist. The app discovers eve The current backend is **LM Studio native v1**, with its CLI used to load models and verify context and parallel settings. This is a Windows desktop app, not a universal API client: Ollama, cloud providers, embeddings, image generation, and standalone Hugging Face checkpoints are not implemented backends. Vision-capable language models can take the text tests; these benchmarks do not send images. - **Reasoning:** the menu shows the intersection of settings reported by the selected models. Model default sends no explicit reasoning setting and works with models that expose no reasoning control. Unsupported selections are blocked, not silently changed. -- **Native MTP:** optional and never inferred from a model name. On requires confirmed metadata for the exact model file and verified loaded settings. It currently uses LM Studio's local GGUF metadata cache; missing or stale metadata leaves support unknown. MTP requires an LM Studio/CLI runtime exposing the MTP flags and effective settings. Older runtimes may need an update even for a controlled MTP-off run. -- **Vision mode:** `Auto` leaves LM Studio's defaults alone, `Off / text-only` sends no image, and `On` runs a deterministic local image smoke test alongside your text tests. `On` is offered only for models LM Studio positively reports as vision-capable; an unknown capability is never treated as support, and a run stops before measuring anything if the loaded model does not confirm it. LM Studio has **no** load-time option to attach or detach a vision projector — the projector belongs to the model entry itself (an `mmproj` file stored beside the GGUF), so text-only means no image is sent and never claims vision weights were unloaded. A vision-capable model stays fully usable for ordinary text benchmarks, so a separate non-vision copy of the same model is not needed. Benchmark images are generated locally and sent as base64 data URLs. This app refuses any image that is not a local `data:image/` URL, so no image request ever leaves the machine. +- **Native MTP:** optional and never inferred from a model name, and covering both prediction heads built into a model file and heads shipped as a separate file with it. Several prediction depths can be measured in one run; see [Find the best MTP setting](#find-the-best-mtp-setting-190). On requires confirmed metadata for the exact model file and verified loaded settings. It currently uses LM Studio's local GGUF metadata cache; missing or stale metadata leaves support unknown. MTP requires an LM Studio/CLI runtime exposing the MTP flags and effective settings. Older runtimes may need an update even for a controlled MTP-off run. +- **Vision mode:** `Auto` leaves LM Studio's defaults alone, `Off / text-only` sends no image, and `On` runs a deterministic local image smoke test alongside your text tests. `On` is offered only for models LM Studio positively reports as vision-capable; an unknown capability is never treated as support, and a run stops before measuring anything if the loaded model does not confirm it. LM Studio has **no** load-time option to attach or detach a vision projector — the projector belongs to the model entry itself (an `mmproj` file stored beside the GGUF), so text-only means no image is sent and never claims vision weights were unloaded. A vision-capable model stays fully usable for ordinary text benchmarks. When you want to measure the same weights with the projector left out entirely, see [Benchmark a vision model without its projector](#benchmark-a-vision-model-without-its-projector). Benchmark images are generated locally and sent as base64 data URLs. This app refuses any image that is not a local `data:image/` URL, so no image request ever leaves the machine. - **Context and concurrency:** the app checks advertised context limits when available, then verifies actual loaded settings. Hardware limits or unsupported load options produce an explicit error rather than silently changing the experiment. - **Missing metadata:** models remain usable; unknown size, quantization, and capabilities are not invented. Display-only architecture/size hints can be corrected in Results and never determine whether a model may run. @@ -208,6 +210,191 @@ Search by model name, key, quantization, publisher, architecture, or format. Com **Select shown** and **Deselect shown** apply only to visible cards. Hidden selections are preserved and counted above the library. **Clear selection** removes all selections. Cards identify vision support directly. +## Benchmark a vision model without its projector + +Some models keep a vision projector (an `mmproj-*.gguf`) in the same folder as +their weights. Gemma 4 does. LM Studio attaches that projector whenever it loads +the model, and there is no way to ask it not to: its whole projector handling is +two lines that read the path off the model entry LM Studio indexed, with no flag, +no API field and no setting in front of them. **Vision off** in a run therefore +means "no image was sent", never "the projector was left out" — the weights are +still loaded and still taking VRAM. + +The only thing that changes it is giving LM Studio a *second entry* to index, with +the same weights and no projector. That is what the **Vision projectors** panel on +the Models screen does. **Make text-only copy** creates a folder beside the +model's own named `… - Text only`, containing: + +- a hard link to the model's weights — not a copy. It is the same bytes on disk + and uses no extra space; +- a hard link to any MTP head stored with the model, at the same relative place, + so the text-only copy keeps native MTP; +- no projector. + +LM Studio picks the folder up on its own and lists it as a separate model whose +vision capability is `false`. Refresh the library and it appears beside the +original, so you can select either. + +**Remove copy** deletes the folder. Because every file in it is a link, that frees +no space and destroys nothing: the weights stay where they were. Removal refuses +outright if the folder holds any file that exists nowhere else, and it will only +delete folders this app made, identified by a marker file rather than by name. + +Hard links need the model folder to be on a single NTFS volume, which a sibling +folder inside the same models directory always is. On anything else the copy is +refused with a reason rather than silently duplicating many gigabytes. + +## Find the best MTP setting (1.9.0) + +Native MTP lets a model draft several tokens ahead with its own prediction heads, +and the main model then checks that draft. Drafting further ahead can be faster — +or slower, once too many drafted tokens get thrown away. The right number is a +property of the model and your hardware, so the only way to know it is to measure. + +The quickest way in is the **MTP speed sweep** button on the **Run** screen. It +switches the run to speed only, turns MTP on, and sets the depths to +`0, 1, 2, 3, 4, 5` — every draft depth from 1 to 5, measured against MTP off. + +To set it up by hand instead, turn on **Enable native MTP** and tick **Sweep +maximum predictions**. **Depth 0 means MTP off**, so the sweep includes the +baseline that any speed-up has to beat. + +Depth is fixed when a model is loaded, so **each depth is a separate load and a +separate pass over your whole workload**. Six depths is six times the run. The +run preview says so, and shows the total before you start. Start small: one test, +concurrency 1, two waves, three depths. + +Concurrency and depth are swept together. Give the run more than one +**Concurrent requests** level and every level is measured at every depth and +reported on its own: five depths at five concurrency levels is 25 combinations per +model. A depth is only ever compared with another depth under the same load, +because the depth that is fastest at one request at a time need not be the fastest +with five in flight. + +Results gain a **Maximum predictions sweep** panel, one block per model and one +table per concurrency level: + +- a row per depth with generation speed, total throughput, latency, time to first + token and objective score; +- **draft accepted**, the share of drafted tokens the main model kept, and **mean + accepted run**, how many it kept per drafting step. Both are LM Studio's own + figures, printed by its engine as each request finishes and pooled over the wave + by drafted tokens. MTP off reports neither, because nothing was drafted — which + is not the same as drafting that was always rejected; +- the spread of the measurements behind each speed; +- a plain sentence naming the fastest depth and what it gained over the baseline. + +When the gap between two depths is no bigger than the spread of the measurements +themselves, the panel says so instead of declaring a winner. Add waves and run it +again rather than acting on that. The spread assumes requests are independent, +which requests sharing a concurrent wave are not, so treat it as a floor on the +real uncertainty. + +Every response records the depth it was measured at. Graphs draw one line per +model **and** depth, the comparison table and CSV keep `mtpDepth`, `draftAcceptance` and `draftMeanLen` columns, and +the run history overview will not pool two depths into one row. + +Sweeping needs confirmed MTP support for every selected model — the same +requirement as turning MTP on at all, checked before anything is loaded. If LM +Studio does not apply a depth it was asked for, that depth is abandoned without +measuring anything, and the rest of the sweep continues. + +**Check the MTP head loads first** (optional) goes one step further. Confirmed +support means LM Studio's metadata says the heads exist and match; it does not +mean its runtime can actually load them against that exact model file. Tick this +and every selected model is loaded once with MTP on — at the same parallel slots +and context the run itself will use — before anything is measured. A model whose +head will not load is named in the run log and its MTP depths are skipped; the +rest of that model's sweep, including the MTP-off baseline, still runs. + +It is off by default because it costs one extra load per model whose head does +load. **What it buys is when you find out, not a shorter run.** Measured on a +two-model, two-depth sweep where the second model's head will not load: the run +said so at 25 seconds with the check on and at 82 seconds without it, and the +measurements were identical either way. The whole run was 93 seconds with the +check and 82 seconds without — the check made it slightly *longer*, because the +model whose head did load paid for an extra load and there was only one depth to +skip. On a real workload that gap inverts: the depth that cannot load is found +before the baseline is measured rather than after, so you can stop instead of +waiting out a comparison that has nothing to compare against. Sweeps with several +drafting depths also fail once here instead of once per depth. + +### Models whose prediction heads are a separate file + +Some models keep their MTP heads inside the model file; others ship them as their +own file stored with the model, which LM Studio indexes separately. Gemma 4 does +the second, Qwen3.8 the first. Both count as native MTP here, and a model card +says which one it has. + +The difference matters because LM Studio offers no command-line switch for the +separate kind and no field for it in its HTTP API — its own window is the only +place the setting exists. So for those models this app writes the setting into +LM Studio's own per-model configuration immediately before a load and restores +that file, byte for byte, immediately after, whether the load worked or not. A +file it had to create is deleted again. Nothing is left behind for LM Studio to +apply to a load this app did not make. + +Because LM Studio never reports that head back through its API, the load is +confirmed against its engine log instead: the log has to name that exact head +file, or nothing is measured at that depth. The MTP-off baseline is checked the +same way in reverse — if a head was loaded for it, the depth is abandoned rather +than used as a baseline it is not. + +If two heads in a model's folder both fit it, support is reported as unknown and +the card names both. Removing the duplicate in LM Studio is the fix; this app +will not pick one for you. + +## Choose the engine, and compare engines (1.10.0) + +LM Studio ships several llama.cpp builds — a Vulkan one, a ROCm one on AMD, a CUDA +one on NVIDIA — and uses whichever is selected when a model loads. There is no +per-load switch, so until now which build produced a set of numbers was not recorded +anywhere and could not be recovered afterwards. + +**Inference engine** on the Run screen names one. It is selected before the first +load and your previous choice is put back when the run ends, whatever happens to the +run. Leave it on *Whatever LM Studio has selected* and nothing changes. + +The difference can be large and is worth measuring rather than assuming. On an +RX 9070 with Gemma 4 12B Q4_K_XL: + +| | Vulkan 2.40.0 | ROCm 2.40.0 | +| --- | --- | --- | +| Prompt processing | 1027 tok/s | 148 tok/s | +| Fixed cost before the first token | none measurable | about 16 s | +| Generation | 61 tok/s | 51 tok/s | + +Once two engines have measured the same model, **Results** offers *Show ROCm*, +*Show Vulkan* or *Show all engines* above the tables, and the comparison table gains +an engine column. The graphs gain a **Compare with** menu in the corner listing other +engines and other models; each becomes its own line. Nothing is recalculated — every +overlaid point is a saved row from a finished run, newest run per measurement — and +the other conditions of those runs are whatever they were, so read the engine column +together with the rest of the row. The engine is also a filter and a grouping on the +all-runs overview. + +## Quantize the context (1.10.0) + +The key/value cache is the part of a run's memory that grows with concurrency rather +than with the weights: an instance is loaded holding context × parallel slots of it. +On a card where the weights already nearly fill VRAM, that is usually what decides +whether a run measures the GPU or measures a spill into system RAM. + +**Quantize the context (KV cache)** sets the K and V caches independently to q8_0, +q5_0, q4_0, iq4_nl or f16. K tolerates quantization less well than V, so q8_0 for K +with something smaller for V is the usual choice. It needs flash attention, which +LM Studio turns on by default. + +LM Studio exposes no command-line flag for this, so the setting is written into its +own per-model configuration for the length of one load and the file is put back byte +for byte immediately afterwards — the same discipline the separate MTP head uses. +Nothing is left behind to change a load you make from LM Studio's own window later. + +It is not free: measured on Gemma 4 12B Q4_K_XL, q8_0 for both cost about 9% of +prompt processing (569 tok/s to 519 tok/s). Quantizing the cache also changes what +the model attends to, so treat quality scores from a quantized-cache run as measured +under that setting rather than as comparable to an f16 run. + ## Understanding the measurements - **Generation / request:** LM Studio's reported generation tokens per second for each request. Reasoning tokens are included where the server reports them. @@ -297,7 +484,7 @@ optional LM Studio API token is encrypted at rest and is never sent to the renderer: the window is told only whether one is configured. Inference scheduling runs in a worker thread. -As of 1.8.0 the suite is 164 tests, covering streaming fragments, timing math, +As of 1.9.0 the suite is 199 tests, covering streaming fragments, timing math, concurrency, scoring, cancellation, loading errors, retries, SQLite recovery, the pooling and facet rules behind the run history overview, the update check's version, asset, host and checksum rules, the question-file reader and the scoring rules an imported pack @@ -313,7 +500,7 @@ Because it is unsigned, check the installer's SHA-256 against the value publishe with the release before running it: ```powershell -Get-FileHash -Algorithm SHA256 '.\Local-Model-Bench-Setup-1.8.0.exe' +Get-FileHash -Algorithm SHA256 '.\Local-Model-Bench-Setup-1.9.0.exe' ``` If the hash does not match what the release page lists, do not run it. @@ -382,6 +569,14 @@ $env:LMB_VISION = "on" npx tsx scripts/live-vision.ts ``` +One UI script needs neither LM Studio nor a model, because it drives the window against saved data: + +```powershell +node scripts/qa-mtp-sweep.mjs +``` + +It checks the sweep controls and the sweep panel, graphs and export. Its results half labels a saved run's real measurements with MTP depths so the depth-aware screens have something to draw; those labels are the script's invention, so nothing it prints is a statement about MTP speed. It writes only to `work/`. + No live script automatically chooses a large model or downloads weights. Timing and resource use depend on your selected model and hardware. Some historical UI QA scripts require saved fixture runs; `npm test` is the reproducible core check. Commit source, bundled benchmark data, assets, vendor files, dependency manifests, and documentation. Do not commit `work/`, `outputs/`, model weights, API tokens, `.env` files, SQLite databases, or exported personal conversations. `.gitignore` excludes these; it does not remove files already tracked by Git. Exported benchmark histories may contain prompts, responses, local paths, and machine metadata and should be reviewed before sharing. diff --git a/electron/cache-quant.ts b/electron/cache-quant.ts new file mode 100644 index 0000000..9489062 --- /dev/null +++ b/electron/cache-quant.ts @@ -0,0 +1,95 @@ +import {existsSync,mkdirSync,readFileSync,readdirSync,rmdirSync,unlinkSync,writeFileSync} from 'node:fs'; +import {homedir} from 'node:os'; +import path from 'node:path'; +import type {Model} from '../src/types'; +import type {CacheQuant} from '../src/cache-quant'; + +// Quantizing the KV cache. The context is what actually fills a 16 GB card: a run loaded with +// `--context-length C --parallel P` holds C*P tokens of keys and values at f16, and that is the +// part that grows with concurrency rather than with the weights. Storing them at q8_0 halves it +// and at q4_0 quarters it, which is often the difference between a model that fits and a model +// that silently spills into system RAM at a third of the speed. +// +// Like the MTP sidecar, LM Studio exposes no command-line flag and no REST field for this, so the +// one route in is the per-model default config under .lmstudio/.internal, which `lms load` merges +// into every load of that exact file. The same discipline applies here and for the same reason: +// the setting is written immediately before a load and the file is put back byte for byte +// immediately after it, whether the load succeeded or failed. A file this app created is deleted +// again along with any folders it had to make, so nothing is left behind to change a load the +// user makes from LM Studio's own window later. +// +// LM Studio's key names and value shape, read from its own stored configs: +// {"key":"llm.load.llama.kCacheQuantizationType","value":{"checked":true,"value":"q8_0"}} +const K_CACHE='llm.load.llama.kCacheQuantizationType'; +const V_CACHE='llm.load.llama.vCacheQuantizationType'; +const CONFIG_ROOT=['.lmstudio','.internal','user-concrete-model-default-config']; + +// Resolved from LM Studio's own index rather than built from the model key, because the key is a +// display identifier and the config path is keyed by the file. Ambiguity is an error rather than +// a guess: writing the wrong file would change a load this app is not making. +export function modelResource(model:Model,home=homedir()):string{ + let index:any; + try{index=JSON.parse(readFileSync(path.join(home,'.lmstudio','.internal','model-index-cache.json'),'utf8'));} + catch{throw Error('LM Studio’s model index could not be read, so cache quantization cannot be applied. Run with cache quantization off.');} + const matches=(index?.models??[]).filter((m:any)=>m.defaultIdentifier===model.key&&m.sizeBytes===model.size_bytes&&m.format===model.format); + if(matches.length!==1)throw Error(`${model.display_name}: LM Studio’s index does not identify exactly one file for this model, so cache quantization cannot be applied safely.`); + return matches[0].indexedModelIdentifier; +} + +// The resource comes from LM Studio's index rather than from a user, but it still becomes a path +// this app writes to, so it is checked like one. +export function cacheConfigPath(resource:string,home=homedir()):string{ + const parts=String(resource).split(/[\\/]/).filter(Boolean); + if(!parts.length||parts.some(p=>p==='.'||p==='..'||/^[A-Za-z]:$/.test(p)))throw Error('LM Studio returned a model resource that is not a relative path, so its cache settings cannot be written safely.'); + return path.join(home,...CONFIG_ROOT,...parts.slice(0,-1),parts[parts.length-1]+'.json'); +} +function shape(bytes:Buffer){ + const value=JSON.parse(bytes.toString('utf8')); + if(!value||typeof value!=='object'||Array.isArray(value))throw Error('LM Studio’s stored settings for this model are not a configuration object. No measurements were taken.'); + const load=value.load&&typeof value.load==='object'?value.load:{}; + return {...value,operation:value.operation??{fields:[]},load:{...load,fields:Array.isArray(load.fields)?load.fields:[]}}; +} +function pruneEmpty(dir:string,stop:string){ + for(let at=dir;at.length>stop.length&&at.startsWith(stop);at=path.dirname(at)){ + try{if(readdirSync(at).length)return;rmdirSync(at);}catch{return;} + } +} +export type Restore=()=>void; + +// `off` leaves both keys written as unchecked rather than absent, so a cache type the user last +// picked in LM Studio's own window cannot leak into a run that asked for an unquantized cache. +// That is the whole point of measuring: the run gets the setting the run asked for. +export function applyCacheQuant(model:Model,k:CacheQuant,v:CacheQuant,home=homedir()):Restore{ + const file=cacheConfigPath(modelResource(model,home),home),existed=existsSync(file),original=existed?readFileSync(file):null; + const config=original?shape(original):{preset:'',operation:{fields:[]},load:{fields:[]}}; + const keep=(config.load.fields as {key?:unknown}[]).filter(f=>f&&typeof f.key==='string'&&f.key!==K_CACHE&&f.key!==V_CACHE); + const field=(key:string,q:CacheQuant)=>({key,value:q==='off'?{checked:false,value:'f16'}:{checked:true,value:q}}); + config.load.fields=[...keep,field(K_CACHE,k),field(V_CACHE,v)]; + mkdirSync(path.dirname(file),{recursive:true}); + writeFileSync(file,JSON.stringify(config,null,2)); + let done=false; + return ()=>{ + if(done)return;done=true; + if(original)writeFileSync(file,original); + else{try{unlinkSync(file);}catch{}pruneEmpty(path.dirname(file),path.join(home,...CONFIG_ROOT));} + }; +} + +// What LM Studio reports back after the load, so a run can refuse to measure a cache setting that +// was asked for and not applied. LM Studio reports the instance config with the same value shape. +export function verifyCacheQuant(config:Record,k:CacheQuant,v:CacheQuant):void{ + if(k==='off'&&v==='off')return; // Nothing was requested, so there is nothing to confirm. + const got=(key:string):string=>{ + const raw=(config as any)[key]; + if(raw===undefined||raw===null)return 'unreported'; + if(typeof raw==='string')return raw; + if(typeof raw==='object'&&'checked' in raw)return raw.checked?String(raw.value):'off'; + return 'unreported'; + }; + // LM Studio reports these under snake_case names in the instance config it returns. + const kGot=got('k_cache_quantization_type'),vGot=got('v_cache_quantization_type'); + // An engine that does not report the fields at all is not proof of failure, so it is not treated + // as one; anything it does report has to match, or the numbers would describe another setting. + if(kGot!=='unreported'&&kGot!==k)throw Error(`LM Studio loaded the K cache as ${kGot}, not the requested ${k}. No measurements were taken.`); + if(vGot!=='unreported'&&vGot!==v)throw Error(`LM Studio loaded the V cache as ${vGot}, not the requested ${v}. No measurements were taken.`); +} diff --git a/electron/cli-output.ts b/electron/cli-output.ts new file mode 100644 index 0000000000000000000000000000000000000000..ec7e96e26a910b265fcaae791f9a12f36730bf26 GIT binary patch literal 1592 zcmZux+iu%N5ba~37X5)dbP^yI=}4sA^rdoS!ElYDKrV&Vpg~0yS&=JpQ)-vkT}F0d zqxpz_Lq05D(lbj+aDxDbslCk1nRCu;n$qif<6cu?#nzF~_){CPrFA_p5pa=mg;w?Zz6KbT*A;MlEZYs#%dyvZdx~Tvh zHry&-f%hG{-!5PLa(OYIPt$A_%_hB*Z+?ahtTJT0dAa|!+m%I}trmY}dU-g@iucE( z$G4d~eAw+UatLg!$E+{`8*z1Bs|z;LIYe{Q;~$QW_IK8;&b=}nrD9df-Q9=La|L00|!C3y`%BsVAR>^ZttU2ZSud#Z&i+o zGCG$mU_X2bUCKs=w<`2V0>&$>X&PB{sja|3Ssp(PSgpphHsLq!f0N4I)CNNU--n*b z8<@O%F&z8AS3IHtM1sU>9dW&b1O9)1Ivqx@;^D!;{Qx3SI}; +const LOG_ROOT=['.lmstudio','server-logs']; +const CAP=8*1024*1024; +function logFiles(root:string):string[]{ + const out:string[]=[]; + const walk=(dir:string,depth:number)=>{ + let entries;try{entries=readdirSync(dir,{withFileTypes:true});}catch{return;} + for(const entry of entries){ + const full=path.join(dir,entry.name); + if(entry.isDirectory()){if(depth>0)walk(full,depth-1);} + else if(entry.name.endsWith('.log'))out.push(full); + } + }; + walk(root,2); + return out.sort(); +} +const sizeOf=(file:string)=>{try{return statSync(file).size;}catch{return 0;}}; +function slice(file:string,from:number,to:number):string{ + const start=Math.max(from,to-CAP),length=to-start; + if(length<=0)return ''; + const handle=openSync(file,'r'); + try{const buffer=Buffer.alloc(length);readSync(handle,buffer,0,length,start);return buffer.toString('utf8');}finally{closeSync(handle);} +} +export function watchEngineLog(home=homedir()):LogWatch{ + const root=path.join(home,...LOG_ROOT); + return {root,sizes:new Map(logFiles(root).map(file=>[file,sizeOf(file)]))}; +} +// Everything written since the watch began. Reading it again is cheap and gives the same +// answer plus whatever has arrived since, so a caller may poll while it waits for a line. +export function sinceWatch(watch:LogWatch):string{ + return logFiles(watch.root).map(file=>slice(file,watch.sizes.get(file)??0,sizeOf(file))).join('\n'); +} +// Every draft model llama.cpp opened since the watch began, lower-cased with forward slashes +// so a path can be compared with the head this app asked for. +export function draftModelsLoaded(watch:LogWatch):string[]{ + const found=new Set(); + for(const match of sinceWatch(watch).matchAll(/loading draft model '([^']+)'/g))found.add(match[1].replaceAll('\\','/').toLowerCase()); + return [...found]; +} + +// How much of the drafting was kept. llama.cpp prints one line per finished request, so under +// concurrency there is one per request in the wave and they are pooled here: the acceptance +// rate is total accepted over total drafted, which weights a long request more heavily than a +// short one exactly as it should. `tasks` is how many lines were actually found, so a caller +// can say whether every request in a wave was accounted for rather than assuming it was. +export type DraftAcceptance={tasks:number;accepted:number;generated:number;acceptance:number;meanLen:number|null}; +const ACCEPTANCE=/draft acceptance = [\d.]+ \(\s*(\d+) accepted \/\s*(\d+) generated\), mean len =\s*([\d.]+)/g; +export function draftAcceptance(watch:LogWatch):DraftAcceptance|null{ + let accepted=0,generated=0,tasks=0,lengths=0; + for(const [,a,g,len] of sinceWatch(watch).matchAll(ACCEPTANCE)){ + accepted+=Number(a);generated+=Number(g);lengths+=Number(len);tasks++; + } + if(!tasks||generated<=0)return null; + // mean len is a per-request average over speculative steps, and the step count is not printed, + // so these are averaged across requests rather than pooled. At concurrency 1 that is one + // request and the figure is exactly what llama.cpp reported. + return {tasks,accepted,generated,acceptance:accepted/generated,meanLen:lengths/tasks}; +} +// llama.cpp prints its timings as a request finishes, which can be just after the HTTP +// response has been handed back, so a wave waits briefly for the lines it expects rather than +// recording a rate from however many happened to have landed. Returns whatever it has when the +// wait runs out; `tasks` then shows the shortfall instead of hiding it. +export async function settledDraftAcceptance(watch:LogWatch,expected:number,timeoutMs=4000,tick=100):Promise{ + const until=Date.now()+timeoutMs; + for(;;){ + const found=draftAcceptance(watch); + if(expected<=0||(found&&found.tasks>=expected)||Date.now()>=until)return found; + await new Promise(r=>setTimeout(r,tick)); + } +} diff --git a/electron/engine.ts b/electron/engine.ts index 4cb4bd1..15c4566 100644 --- a/electron/engine.ts +++ b/electron/engine.ts @@ -4,80 +4,226 @@ import { api, cli, infer, listModels } from './lmstudio'; import { objectiveScore, gradingPackage, parseGrade } from './scoring'; import { waveMetrics } from './metrics'; import { mtpArgs,verifyMtp } from './mtp'; +import { applySidecar } from './mtp-sidecar'; +import { draftModelsLoaded,settledDraftAcceptance,watchEngineLog } from './engine-log'; +import { mtpDepthText,onSteps,preflightStep,sweepSteps,type MtpStep } from '../src/mtp-sweep'; import { visionArgs,verifyVision,visionSummary,type VisionMode } from './vision'; +import { applyCacheQuant,verifyCacheQuant } from './cache-quant'; +import { cacheQuantText } from '../src/cache-quant'; +import { runtimeLabel,selectRuntime } from './runtime'; +import { estimateLoad,loadAdvice,looksLikeMemory } from './load-estimate'; +import { calibratePrefill,calibrationPrompt,calibrationRepeats,prefillText } from '../src/prefill'; import { serverContext } from '../src/defaults'; import { idleSampler, type GpuSampler } from './gpu'; export type EngineEvent = {type:'sample';sample:Sample}|{type:'wave';wave:Run['waves'][number]}|{type:'progress';progress:Progress}|{type:'log';message:string}|{type:'model';key:string;info:unknown}|{type:'grade';sampleId:string;grade:Grade}|{type:'gpu';gpu:RunGpu}|{type:'finish';status:Run['status'];error?:string}; -export type Adapter = {models:typeof listModels;cli:typeof cli;infer:typeof infer;api:typeof api}; -const real:Adapter={models:listModels,cli,infer,api}; +// The last four read and write LM Studio's own files rather than talking to its server, and are +// listed here so a test can exercise those paths without touching a real LM Studio. +export type Adapter = {models:typeof listModels;cli:typeof cli;infer:typeof infer;api:typeof api;sidecar?:typeof applySidecar;watchLog?:typeof watchEngineLog;draftModels?:typeof draftModelsLoaded;draftAcceptance?:typeof settledDraftAcceptance;cacheQuant?:typeof applyCacheQuant}; +const real:Adapter={models:listModels,cli,infer,api,sidecar:applySidecar,watchLog:watchEngineLog,draftModels:draftModelsLoaded,draftAcceptance:settledDraftAcceptance,cacheQuant:applyCacheQuant}; export function variant(prompt:string,testId:string,concurrency:number,wave:number,slot:number){const tag=createHash('sha256').update(`${testId}/${concurrency}/${wave}/${slot}`).digest('hex').slice(0,24);return `[Benchmark record ${tag}; ignore this record identifier in your answer.]\n${prompt}`;} export async function runEngine(run:Run,settings:Settings,signal:AbortSignal,emit:(e:EngineEvent)=>void,adapter:Adapter=real,retries?:Sample[],gradeOnly=false,gpu:GpuSampler=idleSampler('GPU telemetry was not started for this run.')){ - let completed=0,active=0,currentModel='',failures=0;const measured:Sample[]=[];let owned:string|null=null;const runStart=Date.now(); - let total=gradeOnly?0:retries?retries.length:run.config.modelKeys.length*run.tests.length*run.config.waves*run.config.concurrency.reduce((a,b)=>a+b,0); + let completed=0,active=0,currentModel='',failures=0;const measured:Sample[]=[]; + // What could not be measured, named. A bare count said three things failed but never which, so a + // sweep that lost one depth on one model read exactly like one that lost everything. + const unmeasured:string[]=[];let owned:string|null=null;const runStart=Date.now(); + // MTP depth is a load-time setting, so each depth is its own load of the model and its own + // pass over the whole workload. Without a sweep this is one step and nothing below changes. + const steps=sweepSteps(run.config);let currentStep:MtpStep=steps[0]; + // mtpDepthText, never "MTP " + label: the label for depth 0 is already "MTP off", so the + // shorter form read "MTP MTP off" on every baseline load and wave of a sweep. + const stepNote=()=>steps.length>1?` · ${mtpDepthText(currentStep.depth)}`:''; + const perStep=run.tests.length*run.config.waves*run.config.concurrency.reduce((a,b)=>a+b,0); + let total=gradeOnly?0:retries?retries.length:steps.length*run.config.modelKeys.length*perStep; const progress=(phase:string,message:string)=>emit({type:'progress',progress:{runId:run.id,phase,message,completed,total,active,model:currentModel}}); const log=(message:string)=>emit({type:'log',message}); const cleanup=async()=>{if(owned){const id=owned;owned=null;try{await adapter.api(settings,'/api/v1/models/unload',{instance_id:id});log(`Unloaded ${id}`);}catch(e){log(`Cleanup could not unload ${id}: ${(e as Error).message}`);}}}; - const load=async(key:string,parallel:number,models:Model[],role='benchmark')=>{ + const load=async(key:string,parallel:number,models:Model[],role='benchmark',step:MtpStep=steps[0])=>{ if(signal.aborted)throw Error('Cancelled');const model=models.find(m=>m.key===key);if(!model)throw Error(`Downloaded model not found: ${key}`); // Parallel slots share one KV cache, so the instance needs room for every slot at once. const context=run.config.contextLength,total=serverContext(context,parallel); if(model.max_context_length>0&&total>model.max_context_length)throw Error(`${model.display_name} supports ${model.max_context_length} context tokens; ${parallel} parallel slots at ${context} tokens each need ${total}. Lower the context length or the highest concurrency.`); if(run.config.reasoning!=='default'&&!model.capabilities?.reasoning?.allowed_options?.includes(run.config.reasoning))throw Error(`${model.display_name} does not support reasoning setting ${run.config.reasoning}. Select Model default or another supported option.`); owned=`lmb-${run.id.slice(0,8)}-${randomUUID().slice(0,8)}`; - progress('loading',`Loading ${model.display_name} with ${parallel} parallel slots and ${total} context tokens (${context} per slot)…`); - const mode=role==='judge'?'off':run.config.mtp; + progress('loading',`Loading ${model.display_name} with ${parallel} parallel slots and ${total} context tokens (${context} per slot)${role==='judge'?'':stepNote()}…`); + // A judge takes no part in a sweep: it is loaded without MTP whatever the run asked for. + const mode=role==='judge'?'off':step.mode,draftTokens=role==='judge'?2:step.tokens; // The judge only ever reads text, so it never requests vision. const visionMode:VisionMode|undefined=role==='judge'?'off':run.config.vision; const args=['load',key,'--identifier',owned,'--context-length',String(total),'--parallel',String(parallel),'--yes'];if(run.config.gpu!=='auto')args.push('--gpu',run.config.gpu); - args.push(...mtpArgs(model,mode,run.config.mtpDraftTokens??2)); + args.push(...mtpArgs(model,mode,draftTokens)); args.push(...visionArgs(model,visionMode)); // Always empty: LM Studio has no projector flag. Validates the request first. - const started=performance.now();const loadOutput=await adapter.cli(settings,args,signal); + // A model whose MTP heads are a separate file cannot carry them on the command line, so the + // setting is written into LM Studio's own per-model config for the length of this one load + // and taken straight back out again. The engine log is watched across the load either way: + // at every other depth, including the sweep's MTP-off baseline, the point is to prove that + // no head was loaded. + const paired=model.nativeMtp?.kind==='sidecar',watch=paired&&mode!==undefined?(adapter.watchLog??watchEngineLog)():null; + if(paired&&mode==='on')log(`Native MTP head: ${model.nativeMtp!.draftResource} · loaded as a separate file. LM Studio offers no command-line flag for this, so the setting is written to its per-model configuration for this load only and restored immediately afterwards.`); + // The KV cache type goes through the same per-model config file as the MTP head, for the same + // reason: LM Studio has no flag for it. Cache first, head second, so the head's snapshot of the + // file already contains the cache fields; the two are undone in the opposite order, so whatever + // the file held before this load is what it holds after it. + const cacheK=run.config.cacheK??'off',cacheV=run.config.cacheV??'off'; + const quantising=cacheK!=='off'||cacheV!=='off'; + if(quantising)log(`KV cache: ${cacheQuantText(cacheK,cacheV)} · written to LM Studio's per-model configuration for this load only and restored immediately afterwards. The cache is the part of this run's memory that grows with concurrency, so quantizing it is what lets a bigger model or a higher concurrency stay on the GPU.`); + const undoCache=quantising?(adapter.cacheQuant??applyCacheQuant)(model,cacheK,cacheV):null; + const restore=paired&&mode==='on'?(adapter.sidecar??applySidecar)(model,draftTokens):null; + const started=performance.now();let loadOutput:string; + try{loadOutput=await adapter.cli(settings,args,signal);} + catch(e){ + // LM Studio says it could not fit the model, in words that name none of the settings that + // decided how much room it needed. The arithmetic this run already did is added here, where + // someone reading the log is actually looking. Nothing is retried and nothing is changed. + // Only when the failure actually looks like one of room. A load that failed for another reason + // is passed through exactly as LM Studio reported it, because memory advice there would send + // someone to change a setting that was never the problem. + if(signal.aborted||!looksLikeMemory((e as Error).message))throw e; + const estimate=await estimateLoad(settings,adapter.cli,key,total,parallel,signal); + throw Error(`${(e as Error).message}\n\n${loadAdvice(model,context,parallel,cacheK,cacheV,estimate)}`); + } + finally{restore?.();undoCache?.();} const fresh=await adapter.models(settings);const owner=fresh.find(m=>m.loaded_instances.some(i=>i.id===owned));const instance=owner?.loaded_instances.find(i=>i.id===owned); if(owner&&owner.key!==key)throw Error('LM Studio resolved the model key to a different file. No measurements were taken.'); if(!instance)throw Error('Loaded instance was not returned by LM Studio. Check that the CLI and API address use the same server.'); if(instance.config.parallel!==parallel||instance.config.context_length!==total)throw Error(`Loaded settings differ: requested parallel=${parallel}, context=${total}; received parallel=${instance.config.parallel??'unknown'}, context=${instance.config.context_length}. No measurements were taken.`); - verifyMtp(instance.config,mode,model.format,run.config.mtpDraftTokens??2); + // A depth LM Studio did not apply throws here, before a single request is measured, so no + // saved response can be attributed to a depth the server was not actually running. + verifyMtp(instance.config,mode,model.format,draftTokens,{...(paired?{kind:'sidecar' as const,draftPath:model.nativeMtp!.draftPath}:{}),draftersLoaded:watch?(adapter.draftModels??draftModelsLoaded)(watch):[]}); + // A cache type LM Studio did not apply would make the run report a memory saving it never got, + // and compare against other runs as though it had. Checked here, before anything is measured. + verifyCacheQuant(instance.config as Record,cacheK,cacheV); // Vision is confirmed from the reloaded model entry's reported capability, the only // evidence LM Studio gives; a failure throws here, before any request is measured. const vision=verifyVision(owner,visionMode); log(visionSummary(vision)); - log(`Native MTP: ${mode??'legacy/default'}${mode==='on'?' · '+(run.config.mtpDraftTokens??2)+' draft tokens':''}`); + log(`Native MTP: ${mode??'legacy/default'}${mode==='on'?` · ${draftTokens} draft tokens · ${paired?'separate MTP head':'heads built into the model file'}`:''}${steps.length>1&&role!=='judge'?` (sweep step ${steps.indexOf(step)+1} of ${steps.length})`:''}`); log(`${model.display_name}: ${parallel} parallel slots share ${total} context tokens, ${context} per concurrent request.`); - const info={model,instance,vision,loadMs:performance.now()-started,loadOutput,reasoning:run.config.reasoning==='default'?(model.capabilities?.reasoning?.default??'not exposed'):run.config.reasoning};emit({type:'model',key:role==='judge'?'judge:'+key:key,info});return {model,id:owned}; + const info={model,instance,vision,loadMs:performance.now()-started,loadOutput,mtp:{depth:role==='judge'?null:step.depth,mode:mode??'legacy/default',draftTokens:mode==='on'?draftTokens:null,kind:mode==='on'?(model.nativeMtp?.kind??null):null,head:paired&&mode==='on'?model.nativeMtp!.draftResource:null},reasoning:run.config.reasoning==='default'?(model.capabilities?.reasoning?.default??'not exposed'):run.config.reasoning}; + // A preflight is a question, not a measurement, so it leaves no record of a model behind. + if(role!=='preflight')emit({type:'model',key:role==='judge'?'judge:'+key:key,info}); + // A sweep loads the same model once per depth, so each depth also keeps its own record of + // what LM Studio confirmed and how long the load took; the plain key stays as it was. + if(steps.length>1&&role!=='judge'&&role!=='preflight')emit({type:'model',key:`${key} · MTP ${step.label}`,info}); + return {model,id:owned}; + }; + // Two requests, one short and one long, timed the same way every measured request is. Their + // difference is the real per-token prompt-processing rate; see src/prefill.ts for why a single + // request cannot give one. This runs once per loaded instance, costs two requests, and is + // recorded beside the model rather than mixed in with the measured samples. + const calibrate=async(model:Model,instance:string,key:string,step:MtpStep)=>{ + try{ + progress('warmup',`Calibrating prompt processing for ${model.display_name} (excluded from results)…`); + const point=async(repeats:number)=>{ + // A unique prefix per request: LM Studio reuses a cached prompt prefix, and a second timing + // served from that cache would measure the cache rather than the prefill. + const prompt=`[Calibration ${randomUUID()}]\n${calibrationPrompt(repeats)}`; + const r=await adapter.infer({...settings,timeoutSec:run.config.timeoutSec},instance,prompt,4,0,run.config.reasoning,signal); + if(r.status!=='completed')throw Error(r.error||r.status); + const ms=r.metrics.prefillMs??r.metrics.ttftMs; + if(r.metrics.inputTokens===null||ms===null)throw Error('LM Studio returned no prompt timing.'); + return {tokens:r.metrics.inputTokens,ms}; + }; + const small=await point(calibrationRepeats.small),big=await point(calibrationRepeats.big); + const calibration=calibratePrefill(small,big); + log(`Prompt processing for ${model.display_name}: ${prefillText(calibration)}. ${calibration.note}`); + emit({type:'model',key:`prefill:${key}${steps.length>1?` · MTP ${step.label}`:''}`,info:calibration}); + }catch(e){ + // A calibration that could not be taken costs the run nothing else: the per-request figures + // are still recorded, they are simply left uncorrected. + log(`Prompt-processing calibration skipped for ${model.display_name}: ${(e as Error).message}`); + } }; const request=async(model:Model,instance:string,test:TestCase,c:number,wave:number,slot:number,waveId:string,warmup=false,previous?:Sample)=>{ - const prompt=previous?.prompt??(test.benchmark?test.prompt:variant(test.prompt,test.id,c,wave,slot));active++;progress(warmup?'warmup':'benchmarking',warmup?'Warming up (excluded from results)…':`${test.name} · ${c} concurrent · wave ${wave+1}`); + const prompt=previous?.prompt??(test.benchmark?test.prompt:variant(test.prompt,test.id,c,wave,slot));active++;progress(warmup?'warmup':'benchmarking',warmup?'Warming up (excluded from results)…':`${test.name} · ${c} concurrent · wave ${wave+1}${stepNote()}`); const max=warmup?32:test.maxTokens>0?Math.min(test.maxTokens,run.config.maxTokens):run.config.maxTokens; let result:Awaited>;const startedAt=Date.now(); try{result=await adapter.infer({...settings,timeoutSec:run.config.timeoutSec},instance,prompt,max,run.config.temperature,run.config.reasoning,signal,test.image);}finally{active--;} const finishedAt=Date.now(); - const sample:Sample={...result,id:randomUUID(),runId:run.id,modelKey:model.key,modelName:model.display_name,testId:test.id,testName:test.name,concurrency:c,waveId,wave,slot,warmup,prompt,objective:result.status==='completed'&&!warmup?objectiveScore(result.output,test):{score:null,checks:[]},grades:[],created:new Date().toISOString(),retryOf:previous?.id,gpu:gpu.window(startedAt,finishedAt)}; + // The depth is recorded on the measurement itself, not read back from the run's settings: + // a sweep gives one run several depths, and every response has to say which one produced it. + const sample:Sample={...result,...(currentStep.depth===null?{}:{mtpTokens:currentStep.depth}),id:randomUUID(),runId:run.id,modelKey:model.key,modelName:model.display_name,testId:test.id,testName:test.name,concurrency:c,waveId,wave,slot,warmup,prompt,objective:result.status==='completed'&&!warmup?objectiveScore(result.output,test):{score:null,checks:[]},grades:[],created:new Date().toISOString(),retryOf:previous?.id,gpu:gpu.window(startedAt,finishedAt)}; if(!warmup){completed++;measured.push(sample);if(sample.status!=='completed')failures++;} emit({type:'sample',sample});progress('benchmarking',`${test.name} · ${completed} of ${total} finished`);return sample; }; const wave=async(model:Model,instance:string,test:TestCase,c:number,index:number,previous?:Sample[])=>{ const id=randomUUID(),start=performance.now(),startedAt=Date.now(); + // How much of the drafting was kept is printed per finished request by llama.cpp and offered + // by no API, so a wave with MTP on reads it out of the log it wrote while the wave ran. One + // line per request, pooled over the wave: under concurrency that is exactly the granularity + // the number belongs at, because the slots share a batch and draft against each other. + const speculating=currentStep.mode==='on'; + const acceptance=speculating?(adapter.watchLog??watchEngineLog)():null; const results=await Promise.all(Array.from({length:c},(_,slot)=>request(model,instance,test,c,index,slot,id,false,previous?.[slot]))); - emit({type:'wave',wave:waveMetrics(id,run.id,model.key,test.id,c,performance.now()-start,results,gpu.window(startedAt,Date.now()))}); + const elapsed=performance.now()-start,window=gpu.window(startedAt,Date.now()); + const draft=acceptance?await (adapter.draftAcceptance??settledDraftAcceptance)(acceptance,results.filter(s=>s.status==='completed').length):null; + emit({type:'wave',wave:waveMetrics(id,run.id,model.key,test.id,c,elapsed,results,window,draft)}); }; + // Which llama.cpp build runs this. LM Studio has one selected engine at a time and no per-load + // flag, so a run that names one selects it here and puts the previous choice back in the finally + // below — whatever else happens to the run. Naming none keeps LM Studio exactly as the user left + // it, which is what every run made before this field existed did. + let restoreRuntime:(()=>Promise)|null=null; try{ + if(run.config.runtime&&!gradeOnly){ + restoreRuntime=await selectRuntime(settings,adapter.cli,run.config.runtime,signal); + log(`Runtime: ${runtimeLabel(run.config.runtime.split('@')[0])} · ${run.config.runtime}. LM Studio's engine selection is global, so it is switched for this run and restored afterwards.`); + } + // Which engine was selected is already recorded in run.environment.runtime, captured when the + // run was created. Nothing is asked here, so a run that names no runtime issues no extra + // command and behaves exactly as it did before this field existed. const models=await adapter.models(settings); const other=models.flatMap(m=>m.loaded_instances.map(i=>`${m.display_name} (${i.id})`));if(other.length)log('Other loaded instances may affect memory and speed: '+other.join(', ')); if(!gradeOnly){ + // Whether a model can attach its MTP head at all is answered by one load, before anything is + // measured. Without this, a sweep finds out that a depth is impossible only after it has spent + // the time measuring the MTP-off baseline it meant to compare that depth against, and that + // baseline is then left with nothing to compare to. The probe loads at the same parallel slots + // and context the run itself will use, so a head that only fails to fit under the run's real + // pressure fails here too rather than passing a gentler test than the run. + const blocked=new Map(); + const probe=run.config.mtpPreflight&&!retries?preflightStep(steps):null; + if(probe){ + const depths=onSteps(steps); + for(const key of run.config.modelKeys){ + if(signal.aborted)break;currentModel=key;currentStep=probe; + progress('preflight',`Checking ${key} can load its MTP head…`); + try{ + await load(key,Math.max(...run.config.concurrency),models,'preflight',probe); + log(`MTP preflight: ${key} loaded its MTP head.`); + }catch(e){ + // One model that cannot draft at all is one failure, not one per depth it would have run. + blocked.set(key,(e as Error).message);failures++; + unmeasured.push(`${key} · MTP ${depths.map(s=>s.label).join(', ')} (the MTP head would not load)`); + total-=depths.length*perStep; + log(`MTP preflight: ${key} cannot load its MTP head, so its MTP depths were skipped before anything was measured: ${(e as Error).message}`); + }finally{await cleanup();} + } + currentStep=steps[0]; + } for(const key of run.config.modelKeys){ if(signal.aborted)break;currentModel=key; - try{ - const {model,id}=await load(key,Math.max(...run.config.concurrency),models); - const warmupTest={...run.tests[0],prompt:'Reply with the word ready.',rules:[],image:undefined};const warm=await request(model,id,warmupTest,1,-1,0,randomUUID(),true); - if(warm.status!=='completed')throw Error(`Warm-up failed: ${warm.error}`); - if(retries){ - const groups=new Map();for(const s of retries.filter(s=>s.modelKey===key)){const list=groups.get(s.waveId)??[];list.push(s);groups.set(s.waveId,list);} - for(const group of groups.values()){if(signal.aborted)break;const test=run.tests.find(t=>t.id===group[0].testId)!;await wave(model,id,test,group.length,group[0].wave,group);} - }else{ - for(const test of run.tests){for(const c of run.config.concurrency){for(let i=0;is.modelKey===key&&(s.mtpTokens??null)===step.depth):null; + if(mine&&!mine.length)continue; + try{ + const {model,id}=await load(key,Math.max(...run.config.concurrency),models,'benchmark',step); + const warmupTest={...run.tests[0],prompt:'Reply with the word ready.',rules:[],image:undefined};const warm=await request(model,id,warmupTest,1,-1,0,randomUUID(),true); + if(warm.status!=='completed')throw Error(`Warm-up failed: ${warm.error}`); + await calibrate(model,id,key,step); + if(mine){ + const groups=new Map();for(const s of mine){const list=groups.get(s.waveId)??[];list.push(s);groups.set(s.waveId,list);} + for(const group of groups.values()){if(signal.aborted)break;const test=run.tests.find(t=>t.id===group[0].testId)!;await wave(model,id,test,group.length,group[0].wave,group);} + }else{ + for(const test of run.tests){for(const c of run.config.concurrency){for(let i=0;i(run.modelInfo[modelKey] as {vision?:VisionState}|undefined)?.vision; export const visionRunNote=(run:Run)=>{const v=(run.environment as {vision?:{requested:string;note:string}}).vision;const states=run.config.modelKeys.map(k=>visionOf(run,k)).filter(Boolean) as VisionState[];return {requested:run.config.vision??v?.requested??'legacy/default',effective:[...new Set(states.map(s=>s.effective))].join(' | ')||'Not confirmed: no model was loaded for this run.',limitation:[...new Set(states.map(s=>s.limitation).filter(Boolean))].join(' ')||v?.note||''};}; +// Accepted-draft figures for the waves behind one summary row, pooled the way the sweep panel +// pools them: total kept over total drafted. Null where nothing was drafted at all, which is +// not the same as drafting that was always rejected. +const draftShare=(waves:Wave[])=>{const found=waves.map(w=>w.draft).filter(Boolean) as NonNullable[];const generated=found.reduce((n,d)=>n+d.generated,0);return generated>0?found.reduce((n,d)=>n+d.accepted,0)/generated:null;}; +const draftRun=(waves:Wave[])=>{const found=(waves.map(w=>w.draft).filter(Boolean) as NonNullable[]).filter(d=>d.meanLen!==null);const tasks=found.reduce((n,d)=>n+d.tasks,0);return tasks>0?found.reduce((n,d)=>n+d.meanLen!*d.tasks,0)/tasks:null;}; export function summaries(run:Run){ - const groups=new Map();for(const s of run.samples.filter(s=>!s.warmup)){const key=JSON.stringify([s.modelKey,s.testId,s.concurrency]);const list=groups.get(key)??[];list.push(s);groups.set(key,list);} - return [...groups.values()].map(ss=>{const s=ss[0],ok=ss.filter(x=>x.status==='completed'),waves=run.waves.filter(w=>w.modelKey===s.modelKey&&w.testId===s.testId&&w.concurrency===s.concurrency);const gpu=combineStats(ss.map(x=>x.gpu));return {vision:run.config.vision??'legacy/default',visionEffective:visionOf(run,s.modelKey)?.effective??'Not confirmed',visionImagesSent:visionOf(run,s.modelKey)?.imagesSent??false,visionProjectorUnloaded:visionOf(run,s.modelKey)?.projectorUnloaded??false,visionLimitation:visionOf(run,s.modelKey)?.limitation??'',mtp:run.config.mtp??'legacy/default',mtpDraftTokens:run.config.mtp==='on'?(run.config.mtpDraftTokens??2):null,model:s.modelName,modelKey:s.modelKey,test:s.testName,testId:s.testId,concurrency:s.concurrency,requests:ss.length,failures:ss.length-ok.length,failureRate:(ss.length-ok.length)/ss.length,generationTps:average(ok.map(x=>x.metrics.generationTps)),estimatedPrefillTps:average(ok.map(x=>x.metrics.prefillTps)),throughput:average(waves.map(w=>w.throughput)),medianMs:percentile(ok.map(x=>x.metrics.durationMs),0.5),p95Ms:percentile(ok.map(x=>x.metrics.durationMs),0.95),ttftMs:average(ok.map(x=>x.metrics.ttftMs)),objective:average(ok.map(x=>x.objective.score)),localJudge:average(ok.map(x=>x.grades.filter(g=>g.source==='local').at(-1)?.score??null)),externalJudge:average(ok.map(x=>x.grades.filter(g=>g.source==='external').at(-1)?.score??null)),gpuHotSpotAvg:gpu?.tempHotSpot?.avg??null,gpuHotSpotMax:gpu?.tempHotSpot?.max??null,gpuCoreTempAvg:gpu?.tempCore?.avg??null,gpuCoreTempMax:gpu?.tempCore?.max??null,gpuMemoryTempMax:gpu?.tempMemory?.max??null,gpuPowerAvg:gpu?.power?.avg??null,gpuLoadAvg:gpu?.load?.avg??null,gpuReadings:gpu?.samples??0};}); + // An MTP sweep measures one model at several prediction depths in a single run, so the depth + // is part of what identifies a measurement: pooling depths together would average a setting + // against itself and hide the very difference the sweep was run to find. + const groups=new Map();for(const s of run.samples.filter(s=>!s.warmup)){const key=JSON.stringify([s.modelKey,s.testId,s.concurrency,s.mtpTokens??null]);const list=groups.get(key)??[];list.push(s);groups.set(key,list);} + return [...groups.values()].map(ss=>{const s=ss[0],ok=ss.filter(x=>x.status==='completed'),waves=run.waves.filter(w=>w.modelKey===s.modelKey&&w.testId===s.testId&&w.concurrency===s.concurrency&&(w.mtpTokens??null)===(s.mtpTokens??null));const gpu=combineStats(ss.map(x=>x.gpu));return {vision:run.config.vision??'legacy/default',visionEffective:visionOf(run,s.modelKey)?.effective??'Not confirmed',visionImagesSent:visionOf(run,s.modelKey)?.imagesSent??false,visionProjectorUnloaded:visionOf(run,s.modelKey)?.projectorUnloaded??false,visionLimitation:visionOf(run,s.modelKey)?.limitation??'',mtp:s.mtpTokens===undefined?(run.config.mtp??'legacy/default'):s.mtpTokens===0?'off':'on',mtpDraftTokens:s.mtpTokens===undefined?(run.config.mtp==='on'?(run.config.mtpDraftTokens??2):null):(s.mtpTokens||null),mtpDepth:s.mtpTokens??null,draftAcceptance:draftShare(waves),draftMeanLen:draftRun(waves),model:s.modelName,modelKey:s.modelKey,test:s.testName,testId:s.testId,concurrency:s.concurrency,requests:ss.length,failures:ss.length-ok.length,failureRate:(ss.length-ok.length)/ss.length,generationTps:average(ok.map(x=>x.metrics.generationTps)),estimatedPrefillTps:average(ok.map(x=>x.metrics.prefillTps)),throughput:average(waves.map(w=>w.throughput)),medianMs:percentile(ok.map(x=>x.metrics.durationMs),0.5),p95Ms:percentile(ok.map(x=>x.metrics.durationMs),0.95),ttftMs:average(ok.map(x=>x.metrics.ttftMs)),objective:average(ok.map(x=>x.objective.score)),localJudge:average(ok.map(x=>x.grades.filter(g=>g.source==='local').at(-1)?.score??null)),externalJudge:average(ok.map(x=>x.grades.filter(g=>g.source==='external').at(-1)?.score??null)),gpuHotSpotAvg:gpu?.tempHotSpot?.avg??null,gpuHotSpotMax:gpu?.tempHotSpot?.max??null,gpuCoreTempAvg:gpu?.tempCore?.avg??null,gpuCoreTempMax:gpu?.tempCore?.max??null,gpuMemoryTempMax:gpu?.tempMemory?.max??null,gpuPowerAvg:gpu?.power?.avg??null,gpuLoadAvg:gpu?.load?.avg??null,gpuReadings:gpu?.samples??0};}); +} +// Written only when a run actually measured more than one depth. A sweep whose later steps +// never loaded leaves one depth of results, which is a normal run and is reported as one. +export function mtpSweepReport(run:Run){ + if(measuredDepths(run).length<2)return ''; + const f=(n:number|null,d=2)=>n===null?'Unavailable':n.toFixed(d); + const accepted=(r:{draft:{acceptance:number|null;accepted:number;generated:number;meanLen:number|null;tasks:number}|null})=> + r.draft?.acceptance==null?'not applicable':`${(r.draft.acceptance*100).toFixed(1)}% (${r.draft.accepted} of ${r.draft.generated} drafted, mean accepted run ${f(r.draft.meanLen)}, from ${r.draft.tasks} reported request(s))`; + const body=sweepVerdicts(run).map(v=>`### ${v.modelName} — ${v.concurrency} concurrent request${v.concurrency===1?'':'s'}\n\n${v.summary}\n\n`+v.rows.map(r=> + `- ${r.label}: ${f(r.generationTps)} tok/s${r.generationSd===null?'':` (spread ${f(r.generationSd)})`}; total ${f(r.throughput)} tok/s; drafted tokens accepted ${accepted(r)}; median ${f(r.medianMs===null?null:r.medianMs/1000)} s; time to first token ${f(r.ttftMs===null?null:r.ttftMs/1000)} s; objective ${f(r.objective,1)}; ${r.completed} of ${r.requests} completed.`).join('\n')).join('\n\n'); + return `\n## Native MTP sweep\n\nEach depth is a separate load of the model. Depth 0 is MTP off; every other depth is how many tokens the model's own prediction heads draft ahead before the main model verifies them. Depths are compared only within one concurrency level, because the depth that is fastest on its own need not be the fastest under load. The accepted-draft figures are LM Studio's own, printed by its engine as each request finishes. Requests inside one concurrent wave are not independent of each other, so the spread quoted here is a floor on the real uncertainty rather than a significance test.\n\n${body}\n`; } const range=(s:GpuStat|null|undefined,unit:string,digits=1)=>!s?'Unavailable':`${s.min.toFixed(digits)}–${s.max.toFixed(digits)} ${unit} (average ${s.avg.toFixed(digits)})`; export function gpuReport(gpu:RunGpu|undefined){ @@ -25,5 +45,5 @@ export function exportText(run:Run,format:string){ if(format==='csv'){const headers=rows.length?Object.keys(rows[0]):['model','test','concurrency','requests'];return '\uFEFF'+[headers.map(cell).join(','),...rows.map(row=>headers.map(k=>cell((row as any)[k])).join(','))].join('\r\n');} if(format!=='md')throw Error('Unsupported export format'); const f=(n:number|null)=>n===null?'Unavailable':n.toFixed(2); - return benchmarkReport(run)+`\n\n# ${run.config.name||'Local Model Bench'}\n\nCreated: ${run.created}\nStatus: ${run.status}\nVision requested: ${visionRunNote(run).requested}; confirmed effective state: ${visionRunNote(run).effective}${visionRunNote(run).limitation?'\n\nVision limitation: '+visionRunNote(run).limitation:''}\n\nNative MTP: ${run.config.mtp??'legacy/default'}; draft tokens: ${run.config.mtp==='on'?(run.config.mtpDraftTokens??2):'not applicable'}\n\nPrefill rates are estimates from client-timed processing events. Cache reuse is not measurable. Warm-ups are excluded. Generation includes reasoning tokens where reported by LM Studio.\n\n## Measurements\n\n`+rows.map(r=>`### ${r.model} / ${r.test} / concurrency ${r.concurrency}\n\nGeneration: ${f(r.generationTps)} tok/s; estimated prefill: ${f(r.estimatedPrefillTps)} tok/s; aggregate throughput: ${f(r.throughput)} tok/s. Median latency: ${f(r.medianMs)} ms; p95: ${f(r.p95Ms)} ms. Failures: ${r.failures}/${r.requests}.\n\nObjective: ${f(r.objective)}; local judge: ${f(r.localJudge)}; external judge: ${f(r.externalJudge)}.\n\nGPU hot spot: ${f(r.gpuHotSpotAvg)} °C average, ${f(r.gpuHotSpotMax)} °C peak; core ${f(r.gpuCoreTempAvg)} °C average; board power ${f(r.gpuPowerAvg)} W; from ${r.gpuReadings} reading(s) inside these requests.\n`).join('\n')+'\n## GPU thermals\n\n'+gpuReport(run.gpu)+'\n\n## Responses\n\n'+run.samples.filter(s=>!s.warmup).map(s=>`### ${s.modelName} — ${s.testName} (${s.id})\n\nStatus: ${s.status}${s.error?' — '+s.error:''}\n\n**Prompt**\n\n${s.prompt}\n\n**Response**\n\n${s.output}\n\n**Objective checks**\n\n${s.objective.checks.map(c=>`- ${c.passed?'PASS':'FAIL'}: ${c.label}: ${c.detail}`).join('\n')}\n\n${s.grades.map(g=>`**${g.source} grade (${g.judge}): ${g.score}**\n\n${g.summary}\n`).join('\n')}`).join('\n')+'\n## Run log\n\n'+run.logs.join('\n'); + return benchmarkReport(run)+`\n\n# ${run.config.name||'Local Model Bench'}\n\nCreated: ${run.created}\nStatus: ${run.status}\nVision requested: ${visionRunNote(run).requested}; confirmed effective state: ${visionRunNote(run).effective}${visionRunNote(run).limitation?'\n\nVision limitation: '+visionRunNote(run).limitation:''}\n\nNative MTP: ${run.config.mtp??'legacy/default'}; draft tokens: ${run.config.mtp==='on'?(run.config.mtpDraftTokens??2):'not applicable'}${measuredDepths(run).length>1?`; swept depths: ${measuredDepths(run).map(d=>mtpDepthLabel(d)).join(', ')}`:''}\n\nPrefill rates are estimates from client-timed processing events. Cache reuse is not measurable. Warm-ups are excluded. Generation includes reasoning tokens where reported by LM Studio.\n\n## Measurements\n\n`+rows.map(r=>`### ${r.model} / ${r.test} / concurrency ${r.concurrency}${r.mtpDepth===null?'':' / '+mtpDepthText(r.mtpDepth)}\n\nGeneration: ${f(r.generationTps)} tok/s; estimated prefill: ${f(r.estimatedPrefillTps)} tok/s; aggregate throughput: ${f(r.throughput)} tok/s. Median latency: ${f(r.medianMs)} ms; p95: ${f(r.p95Ms)} ms. Failures: ${r.failures}/${r.requests}.\n\nObjective: ${f(r.objective)}; local judge: ${f(r.localJudge)}; external judge: ${f(r.externalJudge)}.\n\nGPU hot spot: ${f(r.gpuHotSpotAvg)} °C average, ${f(r.gpuHotSpotMax)} °C peak; core ${f(r.gpuCoreTempAvg)} °C average; board power ${f(r.gpuPowerAvg)} W; from ${r.gpuReadings} reading(s) inside these requests.\n`).join('\n')+mtpSweepReport(run)+'\n## GPU thermals\n\n'+gpuReport(run.gpu)+'\n\n## Responses\n\n'+run.samples.filter(s=>!s.warmup).map(s=>`### ${s.modelName} — ${s.testName} (${s.id})\n\nStatus: ${s.status}${s.error?' — '+s.error:''}\n\n**Prompt**\n\n${s.prompt}\n\n**Response**\n\n${s.output}\n\n**Objective checks**\n\n${s.objective.checks.map(c=>`- ${c.passed?'PASS':'FAIL'}: ${c.label}: ${c.detail}`).join('\n')}\n\n${s.grades.map(g=>`**${g.source} grade (${g.judge}): ${g.score}**\n\n${g.summary}\n`).join('\n')}`).join('\n')+'\n## Run log\n\n'+run.logs.join('\n'); } diff --git a/electron/history.ts b/electron/history.ts index 97545f5..0a04cf0 100644 --- a/electron/history.ts +++ b/electron/history.ts @@ -2,17 +2,23 @@ import {summaries} from './export'; import {profilesFor,type ModelProfile} from '../src/model-profile'; import {familyOf,promptSizeOf,quantTier,sizeBucket,sizeLabel,type HistoryRow} from '../src/history'; import type {Model,Run,Sample} from '../src/types'; +import {backendOf} from './runtime'; +import {calibrationFor} from '../src/prefill'; // summaries() already groups a run into (model, test, concurrency) measurements and computes every rate, // failure count and GPU statistic. This adds only what pooling across runs needs: which run a measurement // came from, what the model was, and the counts and durations behind each average. -const groupKey=(s:Pick)=>JSON.stringify([s.modelKey,s.testId,s.concurrency]); +// The MTP depth is part of the key for the same reason it is part of a summary row: one run can +// hold several depths, and pooling them would average a setting against itself. +const groupKey=(s:Pick,depth:number|null)=>JSON.stringify([s.modelKey,s.testId,s.concurrency,depth]); export function historyRows(runs:Run[],saved:Record):HistoryRow[]{ return runs.flatMap(run=>{ const profiles=profilesFor(run,[],saved); + // One run is one engine, so this is resolved once per run rather than per measurement. + const backend=backendOf(run); const measured=new Map(); - for(const s of run.samples){if(s.warmup)continue;const list=measured.get(groupKey(s))??[];list.push(s);measured.set(groupKey(s),list);} + for(const s of run.samples){if(s.warmup)continue;const key=groupKey(s,s.mtpTokens??null);const list=measured.get(key)??[];list.push(s);measured.set(key,list);} return summaries(run).map(r=>{ - const samples=measured.get(groupKey(r))??[],ok=samples.filter(s=>s.status==='completed'); + const samples=measured.get(groupKey(r,r.mtpDepth))??[],ok=samples.filter(s=>s.status==='completed'); const model=(run.modelInfo[r.modelKey] as {model?:Model}|undefined)?.model; const profile=profiles[r.modelKey]??{totalB:null,activeB:null,kind:'unknown' as const,source:'No saved model metadata'}; const test=run.tests.find(t=>t.id===r.testId); @@ -24,6 +30,9 @@ export function historyRows(runs:Run[],saved:Record):Histor quantization:model?.quantization?.name??'',quantTier:quantTier(model?.quantization?.name), publisher:typeof model?.publisher==='string'?model.publisher:'',architecture:String(model?.architecture??''), promptSize:promptSizeOf(r.testId),testKind:test?.kind??'quality',benchmarkPack:test?.benchmark?.packId??null, + backend:backend.label,backendRef:backend.ref, + prefillCalibratedTps:calibrationFor(run,r.modelKey,r.mtpDepth)?.marginalTps??null, + prefillOverheadMs:calibrationFor(run,r.modelKey,r.mtpDepth)?.overheadMs??null, reasoning:run.config.reasoning,temperature:run.config.temperature,contextLength:run.config.contextLength, maxTokens:run.config.maxTokens,waves:run.config.waves, completed:ok.length,durationsMs:ok.map(s=>s.metrics.durationMs).filter(n=>Number.isFinite(n)), diff --git a/electron/lmstudio.ts b/electron/lmstudio.ts index 29c72a627067edc6eb1cf1a549340f8fb8a2179c..235ebea4fd0b2c579d75e748a355d4228753ecfb 100644 GIT binary patch delta 69 zcmX@_Gs$;Cv1(CjVoG9CPO5WGrfX4Aevv}0LRwLNu7bLresWHxZhmP=L21dx^cn2j QNGd1)WH;EX!J#e(0HO97l>h($ delta 19 bcmbQ_d){Y4@x~=R?2{civ^M8(sL25UR+|S` diff --git a/electron/load-estimate.ts b/electron/load-estimate.ts new file mode 100644 index 0000000..264e713 --- /dev/null +++ b/electron/load-estimate.ts @@ -0,0 +1,70 @@ +import type {Model,Settings} from '../src/types'; +import type {cli} from './lmstudio'; +import {cacheScalePair} from '../src/cache-quant'; +import type {CacheQuant} from '../src/cache-quant'; +import {readableCliError} from './cli-output'; + +// Why a load failed, in terms of the settings that caused it. +// +// When LM Studio cannot fit a model it says so in its own words, which name no setting this app +// controls. A run that asked for four concurrent requests at 8192 context was really asking for a +// 32768-token cache, and the message that comes back does not mention concurrency, context, or the +// cache at all — so the obvious reading is "this model is too big", when the model was fine and +// the highest concurrency level was what did not fit. +// +// None of this changes whether a load succeeds. It changes what the run log says when one does not. + +export type LoadEstimate={gpuGiB:number|null;totalGiB:number|null;confidence:string;verdict:string}; + +// `lms load --estimate-only` prints a short block rather than JSON. Anything it does not print is +// left null rather than defaulted, because a wrong number here would be worse than no number. +export function parseEstimate(text:string):LoadEstimate{ + const clean=readableCliError(text); + const gib=(label:string)=>{ + const m=clean.match(new RegExp(label+'\\s*:?\\s*([\\d.,]+)\\s*(GiB|MiB|GB|MB)','i')); + if(!m)return null; + const n=Number(m[1].replace(/,/g,'')); + if(!Number.isFinite(n))return null; + return /MiB|MB/i.test(m[2])?n/1024:n; + }; + return { + gpuGiB:gib('Estimated GPU Memory'), + totalGiB:gib('Estimated Total Memory'), + confidence:(clean.match(/Confidence:\s*([A-Za-z]+)/i)?.[1]??'unreported').toUpperCase(), + verdict:clean.match(/Estimate:\s*([^.]+\.)/i)?.[1]?.trim()??'', + }; +} + +// Whether a failed load looks like it ran out of room, as opposed to the many other reasons a load +// fails — a missing prediction head, a bad flag, a model LM Studio cannot parse. Memory advice on +// one of those would be confidently wrong, and would send someone to lower a context length that +// was never the problem. When in doubt this says no and the error is passed through untouched. +export function looksLikeMemory(message:string):boolean{ + const text=String(message??''); + if(/draft model|draft mtp|projector|mmproj|unknown option|unrecognized/i.test(text))return false; + return /out of memory|insufficient|not enough|no ?t? ?fit|does not fit|cannot fit|too large|allocat\w*fail|failed to allocate|vram|guardrail|resource/i.test(text); +} + +export async function estimateLoad(settings:Settings,run:typeof cli,key:string,context:number,parallel:number,signal?:AbortSignal):Promise{ + try{return parseEstimate(await run(settings,['load',key,'--context-length',String(context),'--parallel',String(parallel),'--estimate-only','--yes'],signal));} + catch{return null;} // A diagnostic that cannot be taken is not itself a failure. +} + +const gib=(bytes:number)=>bytes/1024**3; + +// The arithmetic the run already did, said out loud, plus the three levers that change it. The +// cache figure is the one most worth naming: it is the only term that grows with concurrency, and +// it is the one nobody expects, because the run asked for 8192 and got four times that. +export function loadAdvice(model:Model,context:number,parallel:number,cacheK:CacheQuant='off',cacheV:CacheQuant='off',estimate?:LoadEstimate|null):string{ + const total=context*parallel; + const parts:string[]=[]; + parts.push(`This load asked for ${total.toLocaleString()} context tokens: ${context.toLocaleString()} per request × ${parallel} parallel slot${parallel===1?'':'s'}, because LM Studio serves every slot from one shared cache and the run's highest concurrency level is ${parallel}.`); + if(model.size_bytes>0)parts.push(`The weights are ${gib(model.size_bytes).toFixed(2)} GiB; the key/value cache is on top of that and is the part that grows with concurrency.`); + if(estimate?.gpuGiB!=null)parts.push(`LM Studio estimated ${estimate.gpuGiB.toFixed(2)} GiB of GPU memory for it${estimate.confidence!=='unreported'?` (its own confidence: ${estimate.confidence.toLowerCase()})`:''}.`); + const quantised=cacheK!=='off'||cacheV!=='off'; + parts.push('Three things change it: lower the context length, drop the highest concurrency level' + +(quantised + ?`, or quantize the cache further — it is currently at about ${Math.round(cacheScalePair(cacheK,cacheV)*100)}% of an f16 cache.` + :', or quantize the context (KV cache) — q8_0 roughly halves it and q4_0 roughly quarters it, which is often the difference between a model that fits and one that spills into system RAM at a fraction of the speed.')); + return parts.join(' '); +} diff --git a/electron/main.ts b/electron/main.ts index fce4149..26344e6 100644 --- a/electron/main.ts +++ b/electron/main.ts @@ -4,13 +4,15 @@ import { mkdirSync, writeFileSync, renameSync } from 'node:fs'; import { randomUUID } from 'node:crypto'; import path from 'node:path'; import os from 'node:os'; -import { mtpArgs } from './mtp'; +import { assertMtpPlan } from './mtp'; import { visionArgs, projectorToggle, noProjectorToggleNote } from './vision'; import { visionTests } from './vision-image'; import { Store } from './store'; import { defaultSettings, starterTests, performanceTest } from '../src/defaults'; +import { awaitIndexed, createTextOnly, planTextOnly, removeTextOnly, textOnlyTwins } from './text-only'; import type { Run, RunConfig, Settings, PublicSettings, SettingsUpdate, TestCase, Sample, Progress } from '../src/types'; import { cli, listModels, resolveLms } from './lmstudio'; +import { listRuntimes } from './runtime'; import { validateConfig, validateSettings, validateTest } from './validation'; import { gradingPackage, parseGrade } from './scoring'; import { exportText } from './export'; @@ -102,7 +104,7 @@ function launch(run:Run,retries?:Sample[],gradeOnly=false){assertIdle();activeRu } async function newRun(config:RunConfig,retries?:Sample[],source?:Run){ assertIdle();validateConfig(config,allPacks());const models=await listModels(settings());for(const key of config.modelKeys)if(!models.some(m=>m.key===key))throw Error('Selected model no longer available: '+key); - for(const model of models.filter(m=>config.modelKeys.includes(m.key))){mtpArgs(model,config.mtp,config.mtpDraftTokens??2);visionArgs(model,config.vision);} + for(const model of models.filter(m=>config.modelKeys.includes(m.key))){assertMtpPlan(model,config);visionArgs(model,config.vision);} const tests=source?.tests??[...(config.mode!=='quality'?config.performanceLengths.map(performanceTest):[]),...(config.mode!=='performance'?(config.benchmark?selectBenchmark(config.benchmark,allPacks()):store.tests().filter(t=>config.testIds.includes(t.id))):[]),...(config.vision==='on'?visionTests():[])]; if(!tests.length)throw Error('Select at least one available test.');if(config.mode!=='performance'&&!source&&!config.benchmark&&config.testIds.some(id=>!tests.some(t=>t.id===id)))throw Error('A selected test was deleted. Refresh your selection.'); const now=new Date().toISOString();let runtime='Unavailable';try{runtime=await cli(settings(),['runtime','ls']);}catch{} @@ -120,6 +122,10 @@ if(locked)app.whenReady().then(()=>{ handle('modelProfiles',()=>store.profiles()); handle('saveModelProfile',(key,profile)=>store.saveProfile(key,profile)); handle('models',()=>listModels(settings())); + // The installed llama.cpp builds, so a run can name the one it wants. On an AMD card the ROCm + // and Vulkan builds can differ by a large factor on prompt processing, and which one produced a + // finished run's numbers is not recoverable afterwards unless the run recorded it. + handle('runtimes',()=>listRuntimes(settings(),cli)); handle('startServer',async()=>{assertIdle();const s=settings();const u=new URL(s.baseUrl);resolveLms(s);return cli(s,['server','start','--port',u.port||'1234']);}); handle('saveSettings',(s:SettingsUpdate)=>{assertIdle();validateSettings(s);const current=storedSettings(); // An omitted token means "leave the stored one alone". The window never @@ -222,6 +228,16 @@ if(locked)app.whenReady().then(()=>{ setTimeout(()=>app.quit(),400); return updateState.file; }); + // A vision projector cannot be detached at load time (see text-only.ts), so the only honest + // answer is a second, projector-free entry for LM Studio to index. Both actions are refused + // while a run is in flight, because either changes what LM Studio lists. + handle('textOnlyPlan',async(key:string)=>{const model=(await listModels(settings())).find(m=>m.key===key);if(!model)throw Error('Downloaded model not found: '+key);return planTextOnly(model);}); + handle('makeTextOnly',async(key:string)=>{assertIdle();const model=(await listModels(settings())).find(m=>m.key===key);if(!model)throw Error('Downloaded model not found: '+key); + const folder=createTextOnly(planTextOnly(model)); + if(!await awaitIndexed(folder))throw Error(`The copy was made at ${folder}, but LM Studio has not listed it yet. Refresh the model library in a moment; if it never appears, check that folder is inside LM Studio's models directory.`); + return folder;}); + handle('textOnlyTwins',()=>textOnlyTwins()); + handle('removeTextOnly',async(folder:string)=>{assertIdle();removeTextOnly(folder);await awaitIndexed(folder,undefined,20000,150,false);}); handle('openData',()=>shell.openPath(dataPath)); win=new BrowserWindow({width:1440,height:960,minWidth:1050,minHeight:720,title:'Local Model Bench',icon:path.join(__dirname,'../assets/icon.png'),backgroundColor:'#101416',autoHideMenuBar:true,webPreferences:{preload:path.join(__dirname,'preload.cjs'),nodeIntegration:false,contextIsolation:true,sandbox:true}}); win.webContents.setWindowOpenHandler(()=>({action:'deny'}));win.webContents.on('will-navigate',e=>e.preventDefault()); diff --git a/electron/metrics.ts b/electron/metrics.ts index 861d1a0..2f6cd40 100644 --- a/electron/metrics.ts +++ b/electron/metrics.ts @@ -4,9 +4,12 @@ export function metrics(stats:Record,durationMs:number,start:num const input=finite(stats.input_tokens),elapsed=start!==null&&end!==null&&end>start?end-start:null; return {inputTokens:input,outputTokens:finite(stats.total_output_tokens),reasoningTokens:finite(stats.reasoning_output_tokens),generationTps:finite(stats.tokens_per_second),prefillTps:input!==null&&input>0&&elapsed!==null&&elapsed>=5?input/(elapsed/1000):null,prefillMs:elapsed,ttftMs:finite(stats.time_to_first_token_seconds)===null?null:Number(stats.time_to_first_token_seconds)*1000,durationMs,firstContentMs,prefillMethod:elapsed!==null&&elapsed>=5?'Client-timed prompt-processing events (estimate)':'Unavailable: missing or too-short prompt-processing interval',cacheNote:'Input tokens include formatting; cache reuse and stream buffering are not observable. This is not engine-level uncached prefill speed.'}; } -export function waveMetrics(id:string,runId:string,modelKey:string,testId:string,concurrency:number,durationMs:number,samples:Sample[],gpu:GpuStats|null=null):Wave{ +export function waveMetrics(id:string,runId:string,modelKey:string,testId:string,concurrency:number,durationMs:number,samples:Sample[],gpu:GpuStats|null=null,draft:Wave['draft']=null):Wave{ const good=samples.filter(s=>s.status==='completed');const tokens=good.every(s=>s.metrics.outputTokens!==null)?good.reduce((n,s)=>n+s.metrics.outputTokens!,0):null; - return {id,runId,modelKey,testId,concurrency,durationMs,outputTokens:tokens,throughput:tokens!==null&&durationMs>0?tokens/(durationMs/1000):null,completed:good.length,failed:samples.length-good.length,gpu}; + // The MTP depth comes from the requests the wave is made of rather than being passed in, so a + // wave can never disagree with the responses it summarises about which depth produced them. + const depth=samples[0]?.mtpTokens; + return {id,runId,modelKey,testId,concurrency,durationMs,outputTokens:tokens,throughput:tokens!==null&&durationMs>0?tokens/(durationMs/1000):null,completed:good.length,failed:samples.length-good.length,gpu,...(draft?{draft}:{}),...(typeof depth==='number'?{mtpTokens:depth}:{})}; } export function percentile(values:number[],p:number){if(!values.length)return null;const sorted=[...values].sort((a,b)=>a-b);return sorted[Math.max(0,Math.ceil(p*sorted.length)-1)];} export function average(values:(number|null)[]){const xs=values.filter((v):v is number=>v!==null&&Number.isFinite(v));return xs.length?xs.reduce((a,b)=>a+b,0)/xs.length:null;} diff --git a/electron/mtp-sidecar.ts b/electron/mtp-sidecar.ts new file mode 100644 index 0000000..c2f22e2 --- /dev/null +++ b/electron/mtp-sidecar.ts @@ -0,0 +1,60 @@ +import {existsSync,mkdirSync,readFileSync,readdirSync,rmdirSync,unlinkSync,writeFileSync} from 'node:fs'; +import {homedir} from 'node:os'; +import path from 'node:path'; +import type {Model} from '../src/types'; + +// Loading a separate MTP head. Proving one was loaded is engine-log.ts's job. +// +// LM Studio has no command-line flag and no REST field for a sidecar MTP head (see mtp.ts). +// The one route in is the per-model default config LM Studio keeps for itself under +// .lmstudio/.internal, which `lms load` merges into every load of that exact file. So the +// setting is written immediately before a load and the file is put back byte for byte +// immediately after it, whether the load succeeded or failed; a file this app created is +// deleted again, along with any folders it had to make. Nothing is left behind for LM Studio +// to apply to a load this app did not make. +// +// LM Studio's key names, read from its own load-config schema (llmLlamaMoeLoadConfigSchematics). +const SIDECAR='llm.load.llama.speculativeDecoding.draftMtpSidecar'; +const DRAFT_MODEL='llm.load.llama.speculativeDecoding.draftModel'; +const MAX_TOKENS='llm.load.llama.speculativeDecoding.draftMaxTokens'; +// LM Studio refuses a load with more than one speculative mode enabled, so the others are +// written off rather than left at whatever was last chosen in its own window. +const EXCLUSIVE=['llm.load.llama.speculativeDecoding.draftMtp','llm.load.llama.speculativeDecoding.draftSimple','llm.load.llama.speculativeDecoding.draftDflashSidecar','llm.load.llama.speculativeDecoding.draftDsparkSidecar']; +const CONFIG_ROOT=['.lmstudio','.internal','user-concrete-model-default-config']; + +// The resource comes from LM Studio's index rather than from a user, but it still becomes a +// path this app writes to, so it is checked like one. +export function sidecarConfigPath(resource:string,home=homedir()):string{ + const parts=String(resource).split(/[\\/]/).filter(Boolean); + if(!parts.length||parts.some(p=>p==='.'||p==='..'||/^[A-Za-z]:$/.test(p)))throw Error('LM Studio returned a model resource that is not a relative path, so its MTP settings cannot be written safely.'); + return path.join(home,...CONFIG_ROOT,...parts.slice(0,-1),parts[parts.length-1]+'.json'); +} +function shape(bytes:Buffer){ + const value=JSON.parse(bytes.toString('utf8')); + if(!value||typeof value!=='object'||Array.isArray(value))throw Error('LM Studio’s stored settings for this model are not a configuration object. No measurements were taken.'); + const load=value.load&&typeof value.load==='object'?value.load:{}; + return {...value,operation:value.operation??{fields:[]},load:{...load,fields:Array.isArray(load.fields)?load.fields:[]}}; +} +function pruneEmpty(dir:string,stop:string){ + for(let at=dir;at.length>stop.length&&at.startsWith(stop);at=path.dirname(at)){ + try{if(readdirSync(at).length)return;rmdirSync(at);}catch{return;} + } +} +export type Restore=()=>void; +export function applySidecar(model:Model,tokens:number,home=homedir()):Restore{ + const mtp=model.nativeMtp; + if(mtp?.kind!=='sidecar'||!mtp.resource||!mtp.draftResource)throw Error(`${model.display_name}: no paired MTP head to load.`); + const file=sidecarConfigPath(mtp.resource,home),existed=existsSync(file),original=existed?readFileSync(file):null; + const config=original?shape(original):{preset:'',operation:{fields:[]},load:{fields:[]}}; + const keep=(config.load.fields as {key?:unknown}[]).filter(f=>f&&typeof f.key==='string'&&f.key!==SIDECAR&&f.key!==DRAFT_MODEL&&f.key!==MAX_TOKENS&&!EXCLUSIVE.includes(f.key)); + config.load.fields=[...keep,...EXCLUSIVE.map(key=>({key,value:false})),{key:SIDECAR,value:true},{key:DRAFT_MODEL,value:mtp.draftResource},{key:MAX_TOKENS,value:tokens}]; + mkdirSync(path.dirname(file),{recursive:true}); + writeFileSync(file,JSON.stringify(config,null,2)); + let done=false; + return ()=>{ + if(done)return;done=true; + if(original)writeFileSync(file,original); + else{try{unlinkSync(file);}catch{}pruneEmpty(path.dirname(file),path.join(home,...CONFIG_ROOT));} + }; +} + diff --git a/electron/mtp.ts b/electron/mtp.ts index 5e8aeea..d58fc02 100644 --- a/electron/mtp.ts +++ b/electron/mtp.ts @@ -2,39 +2,112 @@ import {readFileSync,statSync} from 'node:fs'; import {homedir} from 'node:os'; import path from 'node:path'; import type {Model} from '../src/types'; +import {sweepSteps,type MtpConfig} from '../src/mtp-sweep'; -// The v1 model list currently omits supportsMtp. Use LM Studio's own local -// GGUF metadata, matched to an exact model resource and unchanged file stats. -// No architecture/name heuristics and no arbitrary draft-model pairings. +// LM Studio ships native MTP in two shapes, and only one of them is reachable from the +// command line. Verified on 2026-09-15 against lms CLI commit 07b7252 and LM Studio's own +// load path: +// * bundled — the prediction heads sit inside the model's own GGUF. LM Studio's metadata +// sets supportsMtp true and `--speculative-draft-mtp` loads them. Qwen3.8-27B is one. +// * sidecar — the heads ship as a separate GGUF stored with the model, which LM Studio +// indexes in its own "drafter" domain. supportsMtp is false on both files; +// `--speculative-draft-mtp` refuses with "requires a GGUF model with a bundled supported +// MTP head"; and `--speculative-draft-model` cannot name the head because that flag +// resolves only in the llm domain. LM Studio loads it through a load-config field, +// draftMtpSidecar, that neither the CLI nor POST /api/v1/models/load exposes at all. +// Gemma 4 ships its heads this way. See mtp-sidecar.ts for how the load is made. +// The v1 model list reports neither shape, so both are read from LM Studio's own local GGUF +// metadata, matched to an exact model resource and unchanged file stats. No architecture or +// name heuristics, and no arbitrary draft-model pairings. + +type Meta=Record; +const slash=(p:string)=>p.replaceAll('\\','/'); +// A cached record only describes the file it was read from while that file still has the +// size and modification time the cache recorded. Anything else is stale, not evidence. +function verified(cache:any,resource:string):{file:string;metadata:Meta}|null{ + const rows=cache.json.map.filter(([file]:[string])=>slash(file).endsWith('/'+resource)); + if(rows.length!==1)return null; + const [file,record]=rows[0]; + let stats;try{stats=statSync(file);}catch{return null;} + if(stats.size!==record.fileSizeBytes||Math.abs(stats.mtimeMs-record.mtimeMs)>1)return null; + return {file:slash(file),metadata:record.metadata&&typeof record.metadata==='object'?record.metadata:{}}; +} +// A head is paired with a model by structure, never by file name: it has to be stored inside +// that model's own folder, declare prediction layers, project into exactly that model's +// hidden size, and carry that model's architecture under LM Studio's "-assistant" suffix. +function sidecarHeads(index:any,cache:any,mainFile:string,main:Meta){ + const dir=mainFile.slice(0,mainFile.lastIndexOf('/')+1),heads:{resource:string;file:string}[]=[]; + if(typeof main.arch!=='string'||!Number.isInteger(main.embeddingLength))return heads; + for(const entry of index.models){ + if(entry?.domain!=='drafter'||entry.format!=='gguf'||typeof entry.indexedModelIdentifier!=='string')continue; + const file=slash(entry.entryPoint?.absPath??''); + if(!file.startsWith(dir))continue; + const head=verified(cache,entry.indexedModelIdentifier); + if(!head||head.file!==file)continue; + if(!Number.isInteger(head.metadata.nextnPredictLayers)||head.metadata.nextnPredictLayers<1)continue; + if(head.metadata.embeddingLengthOut!==main.embeddingLength||head.metadata.arch!==main.arch+'-assistant')continue; + heads.push({resource:entry.indexedModelIdentifier,file}); + } + return heads; +} export function attachMtp(models:Model[],home=homedir()):Model[]{ let index:any,cache:any; try {const dir=path.join(home,'.lmstudio','.internal');index=JSON.parse(readFileSync(path.join(dir,'model-index-cache.json'),'utf8'));cache=JSON.parse(readFileSync(path.join(dir,'gguf-metadata-cache.json'),'utf8'));}catch{return models;} return models.map(model=>{ - const unknown={...model,nativeMtp:{supported:null,reason:'Native MTP metadata unavailable or stale'}}; + const unknown={...model,nativeMtp:{supported:null,reason:'Native MTP metadata unavailable or stale'}} as Model; try{ const matches=index.models.filter((m:any)=>m.defaultIdentifier===model.key&&m.sizeBytes===model.size_bytes&&m.format===model.format); if(matches.length!==1)return unknown; - const m=matches[0]; - const resource=m.indexedModelIdentifier; - const rows=cache.json.map.filter(([file]:[string])=>file.replaceAll('\\','/').endsWith('/'+resource)); - if(rows.length!==1)return unknown; - const [file,record]=rows[0],stats=statSync(file); - if(stats.size!==record.fileSizeBytes||Math.abs(stats.mtimeMs-record.mtimeMs)>1)return unknown; - const supported=record.metadata.supportsMtp; - if(typeof supported!=='boolean'||m.format!=='gguf')return unknown; - return {...model,nativeMtp:{supported,resource,reason:supported?'LM Studio GGUF metadata confirms built-in MTP':'This GGUF does not contain supported built-in MTP'}}; + const entry=matches[0],resource=entry.indexedModelIdentifier; + const main=verified(cache,resource); + if(!main||entry.format!=='gguf')return unknown; + if(main.metadata.supportsMtp===true)return {...model,nativeMtp:{supported:true,kind:'bundled',resource,reason:'LM Studio GGUF metadata confirms MTP heads built into this file'}}; + if(typeof main.metadata.supportsMtp!=='boolean')return unknown; + const heads=sidecarHeads(index,cache,main.file,main.metadata); + if(heads.length===1)return {...model,nativeMtp:{supported:true,kind:'sidecar',resource,draftResource:heads[0].resource,draftPath:heads[0].file, + reason:`This GGUF has no built-in MTP heads, but LM Studio indexes a matching MTP head stored with it (${heads[0].resource}) that is loaded alongside the model.`}}; + // Two heads that both fit cannot be told apart by anything this app can see, and guessing + // one would be exactly the arbitrary pairing the rest of this file refuses to make. + if(heads.length>1)return {...model,nativeMtp:{supported:null,resource, + reason:`More than one MTP head fits this model (${heads.map(h=>h.resource).join(', ')}). Remove the duplicate in LM Studio so one head can be paired without guessing.`}}; + return {...model,nativeMtp:{supported:false,resource,reason:'This GGUF contains no built-in MTP heads, and no matching MTP head is stored with it.'}}; }catch{return unknown;} }); } export function mtpArgs(model:Model,mode:'on'|'off'|undefined,tokens=2):string[]{ if(mode===undefined)return []; // Historical run configurations retain their original behavior. - if(mode==='on'&&model.nativeMtp?.supported!==true)throw Error(`${model.display_name}: native MTP support is not confirmed for this exact file. Choose MTP off or a supported model.`); + if(mode==='on'&&model.nativeMtp?.supported!==true)throw Error(`${model.display_name}: native MTP support is not confirmed for this exact file. ${model.nativeMtp?.reason??''} Choose MTP off or a supported model.`.replace(/\s+/g,' ')); if(mode==='off'&&model.format!=='gguf')return []; - return mode==='on'?['--speculative-draft-mtp','--speculative-draft-max-tokens',String(tokens)]:['--no-speculative-draft-mtp']; + // A sidecar head has no command-line flag at all, so its load carries no extra arguments: + // the draft-token count travels with the head in the config written by mtp-sidecar.ts. + if(mode==='on')return model.nativeMtp?.kind==='sidecar'?[]:['--speculative-draft-mtp','--speculative-draft-max-tokens',String(tokens)]; + return ['--no-speculative-draft-mtp']; } -export function verifyMtp(config:Record,mode:'on'|'off'|undefined,format:unknown,tokens=2):void{ +// What LM Studio said about the load, gathered from both places it says anything: the loaded +// instance's own reported config, and the engine log line naming a draft model it opened. +export type MtpEvidence={kind?:'bundled'|'sidecar';draftPath?:string;draftersLoaded:string[]}; +export function verifyMtp(config:Record,mode:'on'|'off'|undefined,format:unknown,tokens=2,evidence:MtpEvidence={draftersLoaded:[]}):void{ if(mode===undefined||mode==='off'&&format!=='gguf')return; - if(config.speculative_draft_mtp!==(mode==='on'))throw Error(`LM Studio did not confirm MTP ${mode}. No measurements were taken. Update LM Studio or check the model's native MTP support.`); if(config.speculative_draft_simple===true)throw Error('A separate draft model is active. Disable Draft Simple in LM Studio before benchmarking native MTP. No measurements were taken.'); + if(mode==='on'&&evidence.kind==='sidecar'){ + // A sidecar leaves no trace in the instance config LM Studio reports: its REST API has no + // field for one. The single piece of evidence it does give is the engine's own load line + // naming the head file, so that line is required, and required to name this exact head. + const wanted=slash(evidence.draftPath??'').toLowerCase(); + if(!wanted||!evidence.draftersLoaded.includes(wanted))throw Error(`LM Studio did not report loading the MTP head for this model (${evidence.draftPath??'unknown head'}). No measurements were taken.${evidence.draftersLoaded.length?` It loaded ${evidence.draftersLoaded.join(', ')} instead.`:''}`); + if(config.speculative_draft_mtp===true)throw Error('LM Studio reported built-in MTP as well as a separate MTP head. No measurements were taken.'); + if(config.speculative_draft_max_tokens!==tokens)throw Error('LM Studio did not apply the requested MTP draft-token count. No measurements were taken.'); + return; + } + if(config.speculative_draft_mtp!==(mode==='on'))throw Error(`LM Studio did not confirm MTP ${mode}. No measurements were taken. Update LM Studio or check the model's native MTP support.`); if(mode==='on'&&config.speculative_draft_max_tokens!==tokens)throw Error('LM Studio did not apply the requested MTP draft-token count. No measurements were taken.'); + // MTP off has to mean no drafting at all, including a head this app never asked for. A + // setting left behind in LM Studio would otherwise turn a sweep's baseline into a + // speculative run and quietly flatter every depth measured against it. + if(mode==='off'&&evidence.draftersLoaded.length)throw Error(`LM Studio loaded a draft model (${evidence.draftersLoaded.join(', ')}) for a load with MTP off. No measurements were taken. Clear this model's speculative decoding settings in LM Studio.`); +} +// Every depth a sweep will load has to be possible before the first one is loaded. A run that +// would only fail on its fourth depth, an hour into measuring, has wasted the hour. +export function assertMtpPlan(model:Model,config:MtpConfig):void{ + for(const step of sweepSteps(config))mtpArgs(model,step.mode,step.tokens); } diff --git a/electron/preload.ts b/electron/preload.ts index 5986d43..8f5da03 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -1,5 +1,5 @@ import {contextBridge,ipcRenderer} from 'electron'; import type { BenchAPI } from '../src/types'; const call=(name:string,...args:unknown[])=>ipcRenderer.invoke('bench:'+name,...args); -const api:BenchAPI={modelProfiles:()=>call('modelProfiles'),saveModelProfile:(key,p)=>call('saveModelProfile',key,p),snapshot:()=>call('snapshot'),models:()=>call('models'),startServer:()=>call('startServer'),saveSettings:s=>call('saveSettings',s),saveTest:t=>call('saveTest',t),deleteTest:id=>call('deleteTest',id),choosePackFile:()=>call('choosePackFile'),importPack:d=>call('importPack',d),deletePack:id=>call('deletePack',id),startRun:c=>call('startRun',c),cancel:()=>call('cancel'),getRun:id=>call('getRun',id),retry:id=>call('retry',id),grade:(id,judge)=>call('grade',id,judge),copyPackage:(r,s)=>call('copyPackage',r,s),importGrade:(r,s,raw,j)=>call('importGrade',r,s,raw,j),exportRun:(id,f)=>call('exportRun',id,f),history:()=>call('history'),updateState:()=>call('updateState'),checkUpdate:()=>call('checkUpdate'),downloadUpdate:()=>call('downloadUpdate'),installUpdate:()=>call('installUpdate'),onUpdate:fn=>{const handler=(_:unknown,s:any)=>fn(s);ipcRenderer.on('bench:update',handler);return ()=>ipcRenderer.removeListener('bench:update',handler);},openData:()=>call('openData'),onProgress:fn=>{const handler=(_:unknown,p:any)=>fn(p);ipcRenderer.on('bench:progress',handler);return ()=>ipcRenderer.removeListener('bench:progress',handler);}}; +const api:BenchAPI={modelProfiles:()=>call('modelProfiles'),saveModelProfile:(key,p)=>call('saveModelProfile',key,p),snapshot:()=>call('snapshot'),models:()=>call('models'),runtimes:()=>call('runtimes'),startServer:()=>call('startServer'),saveSettings:s=>call('saveSettings',s),saveTest:t=>call('saveTest',t),deleteTest:id=>call('deleteTest',id),choosePackFile:()=>call('choosePackFile'),importPack:d=>call('importPack',d),deletePack:id=>call('deletePack',id),startRun:c=>call('startRun',c),cancel:()=>call('cancel'),getRun:id=>call('getRun',id),retry:id=>call('retry',id),grade:(id,judge)=>call('grade',id,judge),copyPackage:(r,s)=>call('copyPackage',r,s),importGrade:(r,s,raw,j)=>call('importGrade',r,s,raw,j),exportRun:(id,f)=>call('exportRun',id,f),history:()=>call('history'),updateState:()=>call('updateState'),checkUpdate:()=>call('checkUpdate'),downloadUpdate:()=>call('downloadUpdate'),installUpdate:()=>call('installUpdate'),onUpdate:fn=>{const handler=(_:unknown,s:any)=>fn(s);ipcRenderer.on('bench:update',handler);return ()=>ipcRenderer.removeListener('bench:update',handler);},textOnlyPlan:key=>call('textOnlyPlan',key),makeTextOnly:key=>call('makeTextOnly',key),textOnlyTwins:()=>call('textOnlyTwins'),removeTextOnly:folder=>call('removeTextOnly',folder),openData:()=>call('openData'),onProgress:fn=>{const handler=(_:unknown,p:any)=>fn(p);ipcRenderer.on('bench:progress',handler);return ()=>ipcRenderer.removeListener('bench:progress',handler);}}; contextBridge.exposeInMainWorld('bench',api); diff --git a/electron/runtime.ts b/electron/runtime.ts new file mode 100644 index 0000000..d423d1d --- /dev/null +++ b/electron/runtime.ts @@ -0,0 +1,102 @@ +import type {Settings} from '../src/types'; +import type {cli} from './lmstudio'; + +// Choosing which llama.cpp build runs the benchmark. +// +// LM Studio ships several engines for the same model format and uses whichever one is selected at +// the time a model is loaded. On an AMD card the choice between the Vulkan build and the ROCm +// build can change prompt processing substantially, and nothing in a saved run used to record +// which one produced the numbers — so two runs of the same model on the same machine could differ +// by a large factor for a reason the history had no field for. +// +// The selection is global to LM Studio rather than per-load: there is no `lms load --engine`. So a +// run that asks for a specific engine selects it before loading and puts the previous selection +// back when the run ends, the same discipline the MTP sidecar uses for the same reason. + +// `llama.cpp-win-x86_64-amd-rocm-avx2@2.40.0` — the part before the @ is the engine, after it the +// version. `lms runtime select` wants the whole thing. +export type Runtime={engine:string;version:string;ref:string;selected:boolean;format:string}; + +// Reused from cli-output.ts's reasoning: `lms` colours its tables, and a colour code between the +// engine name and the tick would otherwise read as part of one or the other. +const ANSI=/\x1b\[[0-9;?]*[A-Za-z]|\x1b[()][A-Za-z0-9]|[\x00-\x08\x0b-\x1f\x7f]/g; + +export function parseRuntimes(text:string):Runtime[]{ + const out:Runtime[]=[]; + for(const raw of String(text??'').replace(ANSI,'').split(/\r?\n/)){ + const line=raw.trim(); + if(!line||/^LLM ENGINE\b/i.test(line))continue; + const at=line.match(/^(\S+?)@(\S+)\s*(.*)$/); + if(!at)continue; + const [,engine,version,rest]=at; + if(!engine.includes('llama.cpp')&&!engine.includes('mlx'))continue; + out.push({engine,version,ref:`${engine}@${version}`,selected:/[✓✔*]/.test(rest),format:(rest.replace(/[✓✔*]/g,'').trim().split(/\s+/)[0])||''}); + } + return out; +} + +// What to call an engine in the window. The full identifier is exact but unreadable in a dropdown, +// and the accelerator is the only part of it anyone is choosing between. +export function runtimeLabel(engine:string):string{ + const name=engine.toLowerCase(); + if(name.includes('rocm'))return 'ROCm'; + if(name.includes('vulkan'))return 'Vulkan'; + if(name.includes('cuda'))return 'CUDA'; + if(name.includes('metal')||name.includes('mlx'))return 'Metal'; + return 'CPU'; +} + +// Two builds of the same accelerator differ only by version, so the label alone cannot identify +// one. The version is shown whenever more than one build of the same accelerator is installed. +export function runtimeChoices(runtimes:Runtime[]):{ref:string;label:string}[]{ + const counts=new Map(); + for(const r of runtimes)counts.set(runtimeLabel(r.engine),(counts.get(runtimeLabel(r.engine))??0)+1); + return runtimes.map(r=>({ref:r.ref,label:(counts.get(runtimeLabel(r.engine))??0)>1?`${runtimeLabel(r.engine)} ${r.version}`:runtimeLabel(r.engine)})); +} + +// Which engine produced a saved run's numbers. +// +// Two places know. A run that named an engine carries it in its config, which is exact. Every other +// run carries the `lms runtime ls` table captured when it was created, in which exactly one row is +// ticked — that is what LM Studio would have used. Runs made before either existed report Unknown +// rather than being quietly attributed to whatever is selected today, because that would make a +// backend comparison out of measurements nobody took under a known backend. +export const unknownBackend='Unknown engine'; +export type RunBackend={label:string;ref:string|null;version:string|null}; +export function backendOf(run:{config?:{runtime?:string};environment?:Record}):RunBackend{ + const chosen=run.config?.runtime; + if(typeof chosen==='string'&&chosen.includes('@')){ + const [engine,version]=chosen.split('@'); + return {label:runtimeLabel(engine),ref:chosen,version:version||null}; + } + const listing=run.environment?.runtime; + if(typeof listing==='string'&&listing.trim()){ + const selected=parseRuntimes(listing).find(r=>r.selected); + if(selected)return {label:runtimeLabel(selected.engine),ref:selected.ref,version:selected.version}; + } + return {label:unknownBackend,ref:null,version:null}; +} + +export type Cli=typeof cli; +export async function listRuntimes(settings:Settings,run:Cli,signal?:AbortSignal):Promise{ + return parseRuntimes(await run(settings,['runtime','ls'],signal)); +} + +// Returns a restore function rather than nothing, so the caller cannot forget that selecting an +// engine changed something outside this app that has to be put back. +export async function selectRuntime(settings:Settings,run:Cli,ref:string,signal?:AbortSignal):Promise<()=>Promise>{ + const before=await listRuntimes(settings,run,signal); + const current=before.find(r=>r.selected); + const wanted=before.find(r=>r.ref===ref); + if(!wanted)throw Error(`LM Studio has no installed runtime ${ref}. Install it in LM Studio, or choose one of: ${before.map(r=>r.ref).join(', ')||'none reported'}.`); + if(current?.ref===ref)return async()=>{}; // Already selected; nothing to put back. + await run(settings,['runtime','select',ref],signal); + // Confirmed rather than assumed: `lms runtime select` exits zero for a name it did not apply. + const after=await listRuntimes(settings,run,signal); + if(!after.find(r=>r.ref===ref)?.selected)throw Error(`LM Studio did not switch to ${ref}. No measurements were taken.`); + let done=false; + return async()=>{ + if(done||!current)return;done=true; + try{await run(settings,['runtime','select',current.ref]);}catch{/* Restoring is best effort; the run's own result matters more. */} + }; +} diff --git a/electron/text-only.ts b/electron/text-only.ts new file mode 100644 index 0000000..020b4fc --- /dev/null +++ b/electron/text-only.ts @@ -0,0 +1,151 @@ +import {linkSync,mkdirSync,readFileSync,readdirSync,rmSync,statSync,writeFileSync} from 'node:fs'; +import {homedir} from 'node:os'; +import path from 'node:path'; +import type {Model} from '../src/types'; + +// Benchmarking a vision-capable model without its vision projector. +// +// LM Studio attaches a projector from the model's index entry and from nothing else. Its own +// load path reduces to two lines: +// const visionPath = indexedModel.visionAdapter?.absPath; +// if (visionPath !== undefined) loadParams.mmproj_path = visionPath; +// There is no command-line flag, no field of POST /api/v1/models/load, and no key in LM +// Studio's load-config schema that can turn that off — checked against lms CLI commit 07b7252 +// and LM Studio's own bundle on 2026-09-15. A model is vision-capable exactly when an +// mmproj-*.gguf sits in its folder, so the only way to measure the same weights without the +// projector is to give LM Studio a second folder holding those weights and no projector. LM +// Studio indexes that as its own model key, reports capabilities.vision false for it, and +// loads it with no mmproj_path at all. LM Studio's own library already works this way: a +// "… - Vision" folder beside a plain one is the same GGUF indexed twice. +// +// Nothing is copied. Every file in the twin is a hard link, so it occupies no additional disk +// space and shares the original's bytes; removing it deletes a directory entry and never the +// weights. An MTP head stored with the model is linked in at the same relative place, so a +// text-only twin keeps native MTP. Hard links need a single NTFS volume, which a sibling +// folder inside the same models directory always is; anything else is refused rather than +// quietly duplicating many gigabytes. + +export const TWIN_SUFFIX=' - Text only'; +export const MARKER='.local-model-bench-text-only.json'; +const slash=(p:string)=>p.replaceAll('\\','/'); +const isProjector=(name:string)=>/^mmproj.*\.gguf$/i.test(name); + +function readIndex(home:string){ + return JSON.parse(readFileSync(path.join(home,'.lmstudio','.internal','model-index-cache.json'),'utf8')); +} +// The same identity rule mtp.ts uses: an exact key, an unchanged size, and one matching entry. +// Two entries that both answer to a key are not a model this app will act on. +export function indexEntry(model:Model,home=homedir()){ + const matches=readIndex(home).models.filter((m:any)=>m.defaultIdentifier===model.key&&m.sizeBytes===model.size_bytes&&m.format===model.format); + if(matches.length!==1)throw Error(`LM Studio does not describe exactly one stored model for ${model.display_name}. Refresh the library in LM Studio and try again.`); + return matches[0]; +} + +export type TwinFile={from:string;to:string}; +export type TwinPlan={key:string;displayName:string;sourceDir:string;folder:string;files:TwinFile[];projectors:string[];headIncluded:boolean}; + +// What making a twin would do, worked out before anything is written so the window can say it +// and so an impossible one fails with a reason rather than half a folder. +export function planTextOnly(model:Model,home=homedir()):TwinPlan{ + if(model.capabilities?.vision!==true)throw Error(`${model.display_name} has no vision projector attached, so it already loads text-only.`); + const entry=indexEntry(model,home); + const own=slash(entry.entryPoint?.absPath??''); + if(!own)throw Error('LM Studio did not say which file this model loads from.'); + const sourceDir=own.slice(0,own.lastIndexOf('/')); + if(sourceDir.endsWith(TWIN_SUFFIX))throw Error(`${model.display_name} is already a text-only copy.`); + const folder=sourceDir+TWIN_SUFFIX; + // The model's own weights, and any MTP head stored with it at the same relative place. The + // projector is the one thing deliberately left behind. + const files:TwinFile[]=[{from:own,to:path.join(folder,path.basename(own))}]; + const head=model.nativeMtp?.kind==='sidecar'?slash(model.nativeMtp.draftPath??''):''; + if(head&&head.startsWith(sourceDir+'/'))files.push({from:head,to:path.join(folder,...head.slice(sourceDir.length+1).split('/'))}); + const projectors=(()=>{try{return readdirSync(sourceDir).filter(isProjector).sort();}catch{return [];}})(); + if(!projectors.length)throw Error(`No projector file was found beside ${model.display_name}, so a text-only copy would be identical to it.`); + return {key:model.key,displayName:model.display_name,sourceDir,folder,files,projectors,headIncluded:files.length>1}; +} + +export function createTextOnly(plan:TwinPlan):string{ + if(statSync(plan.sourceDir).isDirectory()!==true)throw Error('The model folder has moved. Refresh the library in LM Studio and try again.'); + let made=false; + try{ + mkdirSync(plan.folder,{recursive:false});made=true; + for(const file of plan.files){ + mkdirSync(path.dirname(file.to),{recursive:true}); + // linkSync, never copyFileSync: a twin has to share the original's bytes, and failing + // loudly on a volume that cannot hard link is better than silently writing another 17 GB. + try{linkSync(file.from,file.to);} + catch(e){throw Error(`A hard link could not be made for ${path.basename(file.from)} (${(e as Error).message}). This needs the model folder to be on an NTFS volume. Nothing was changed.`);} + } + writeFileSync(path.join(plan.folder,MARKER),JSON.stringify({createdBy:'local-model-bench',source:plan.sourceDir,sourceKey:plan.key,created:new Date().toISOString(), + note:'Every .gguf here is a hard link to the folder named in "source" and uses no extra disk space. Deleting this folder removes the links only; the model itself is untouched.', + files:plan.files.map(f=>slash(path.relative(plan.folder,f.to)))},null,2)); + }catch(e){ + // A twin that was only half made would be indexed by LM Studio as a real model, so a + // failure takes the whole folder back out rather than leaving one. + if(made)try{rmSync(plan.folder,{recursive:true,force:true});}catch{} + throw e; + } + return plan.folder; +} + +// LM Studio notices a new folder by itself and re-indexes it in about a quarter of a second, +// measured on 2026-09-15. That is quick, but not instant, and a copy that is not indexed yet +// is not selectable yet: returning before then would hand back a model the library does not +// list. So the copy is not reported as made until LM Studio's own index holds it. +// Removal has the same lag in reverse: a copy that has just been deleted stays in the library +// until LM Studio notices, so both directions wait for the index to agree with the disk. +export async function awaitIndexed(folder:string,home=homedir(),timeoutMs=20000,tick=150,present=true):Promise{ + const prefix=slash(folder)+'/',until=Date.now()+timeoutMs; + for(;;){ + try{if(readIndex(home).models.some((m:any)=>slash(String(m.entryPoint?.absPath??'')).startsWith(prefix))===present)return true;}catch{} + if(Date.now()>=until)return false; + await new Promise(r=>setTimeout(r,tick)); + } +} + +export type Twin={folder:string;source:string;sourceKey:string;created:string}; +function readMarker(folder:string):Twin|null{ + try{ + const marker=JSON.parse(readFileSync(path.join(folder,MARKER),'utf8')); + if(marker?.createdBy!=='local-model-bench'||typeof marker.source!=='string')return null; + return {folder:slash(folder),source:slash(marker.source),sourceKey:String(marker.sourceKey??''),created:String(marker.created??'')}; + }catch{return null;} +} +// Twins this app made, found by their marker rather than by their name, so a folder a user +// happened to name "… - Text only" themselves is never treated as this app's to delete. +export function textOnlyTwins(home=homedir()):Twin[]{ + const root=path.join(home,'.lmstudio','models'),found:Twin[]=[]; + const walk=(dir:string,depth:number)=>{ + let entries;try{entries=readdirSync(dir,{withFileTypes:true});}catch{return;} + for(const entry of entries){ + if(!entry.isDirectory())continue; + const full=path.join(dir,entry.name); + const twin=readMarker(full); + if(twin)found.push(twin); + else if(depth>0)walk(full,depth-1); + } + }; + walk(root,3); + return found; +} +// Removal refuses anything that is not one of this app's twins, and anything holding a file +// that exists nowhere else. Between them, a delete here can only ever drop directory entries +// whose bytes are still reachable from the model they were linked from. +export function removeTextOnly(folder:string,home=homedir()):void{ + const twin=readMarker(folder); + if(!twin)throw Error('That folder was not created by Local Model Bench, so it will not be deleted from here. Remove it in LM Studio or in Explorer if you meant to.'); + const root=slash(path.join(home,'.lmstudio','models')); + if(!slash(path.resolve(folder)).startsWith(root+'/'))throw Error('That folder is outside the LM Studio models directory.'); + const unique:string[]=[]; + const walk=(dir:string)=>{ + for(const entry of readdirSync(dir,{withFileTypes:true})){ + const full=path.join(dir,entry.name); + if(entry.isDirectory()){walk(full);continue;} + if(entry.name===MARKER)continue; + if(statSync(full).nlink<2)unique.push(slash(path.relative(folder,full))); + } + }; + walk(folder); + if(unique.length)throw Error(`This folder holds ${unique.length} file(s) that exist nowhere else (${unique.slice(0,3).join(', ')}). It was not deleted. Move anything you want to keep out of it first.`); + rmSync(folder,{recursive:true,force:true}); +} diff --git a/electron/validation.ts b/electron/validation.ts index 36bba2a8e72ddfab54be4152a00666cd92d1ae6c..a62c7ed8773cf40232dd00f02817cf98a67a23a4 100644 GIT binary patch delta 1571 zcmZWpO^*{f5EZOIFygS%3a(o-LeuG`(+PXwOM@8ga#+DgAO;SEu;Xc0rw!e9$97NV zQ@#Iz)BX#jIdI^oa6w!-@E0iC$s~asI^A};>eYMC_3_5nTkplWR#dgp7LF>mzh&#c zv9?7bm9AJR4)NIpu|9!Zs|unt?HQf*DqDArTaWIo%noNPEAVNc&PRIp#tl zUthNeHL4tzL*6C7thHO)PoSP10)y5HIf~+YE70q~CV2Fqu~dv3umwV;IF_JfhESsh z?pOdOIR(|oYzUeZszjJ5{ni~!z$ky#2OD4-eR|@q4dihsP~%-6q#NlF?FpPDVFAYt z8Vi|fr1hm74wNJ64Om%HWyXD^VHH9tj7?WS z>>;=Ur0_20N@FNRA~+Jb2;zsr)5P<(-hI1#?NvWn^y80cqu6N~l=oC@EIW3wZ5QZ> zSt#)9<*V&OX-Q~ip+rWFD`MRt4$4~<04Ja{Sl5*kC(xzxJU~A2a&OO7E66a;DS{pf zcFM{C$!-P7XRyS~?s(4#M{}F03duB*d^2h)BoELT=2S0Lw(?|L&Z4GB=n8t0!t3KPj)0YT=1DGr=2c(a(%sXc17pgk+E9Hb`%{SXX$h~ zEy?({xc_Ire|)^!dp+3fzGU6Qe)rCB{+j-KqdWZL!F=JTz3aWy(w!ppr+MQzCiRzm z@ARVG*2^;Uyi=bHe@51q|Nb{Ln~bH4NNt8&X%g~wb@usyNHR3Tf*HuMgXj_X2^b|=>R%oWlR@+X+U3<3!yzz3!+9jyQW delta 27 jcmaEDzFT`kCgbL7j9#oP47&35n~m8Gm^O>^U*Q1&ieU(& diff --git a/electron/vision.ts b/electron/vision.ts index a03e6f5..863a616 100644 --- a/electron/vision.ts +++ b/electron/vision.ts @@ -11,12 +11,22 @@ import type {Model} from '../src/types'; // flash_attention and the speculative_draft_* family used for native MTP. // * A loaded instance's returned config carries no vision or projector field at all, // so unlike MTP there is nothing to read back that could confirm a projector state. -// Vision is fixed when LM Studio indexes a model: a base GGUF is vision-capable exactly -// when an mmproj-*.gguf sits beside it in the same folder, and LM Studio surfaces that +// Re-checked on 2026-09-15, this time against LM Studio's own load-config schema rather than +// only its CLI and HTTP API, because that schema turned out to carry MTP settings the other +// two do not expose (see mtp-sidecar.ts). It carries nothing for vision either: the whole of +// LM Studio's projector handling is +// const visionPath = indexedModel.visionAdapter?.absPath; +// if (visionPath !== undefined) loadParams.mmproj_path = visionPath; +// read straight off the index entry, with no condition and no setting in front of it. +// Vision is therefore fixed when LM Studio indexes a model: a base GGUF is vision-capable +// exactly when an mmproj-*.gguf sits beside it in the same folder, and LM Studio surfaces that // pairing as its own model key whose capabilities.vision is true. The same GGUF bytes // indexed without a neighbouring mmproj report capabilities.vision false. A projector // therefore cannot be attached or detached at load time, and text-only can only ever // mean "send no image" — never "the vision weights were unloaded". +// What that last sentence leaves open is measuring the same weights with no projector at all, +// which needs a second entry for LM Studio to index rather than a different load. text-only.ts +// makes one out of hard links. export const projectorToggle=false; export const noProjectorToggleNote='LM Studio has no load-time option to attach or detach a vision projector. The projector belongs to the model entry itself (an mmproj file stored beside the GGUF), so text-only means no image is sent with any request; it does not unload vision weights.'; diff --git a/package-lock.json b/package-lock.json index aa2bde3..8da76c3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "local-model-bench", - "version": "1.6.1", + "version": "1.10.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "local-model-bench", - "version": "1.6.1", + "version": "1.10.0", "dependencies": { "lucide-react": "^0.468.0", "react": "^19.1.0", diff --git a/package.json b/package.json index fea0d55..be604fe 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "local-model-bench", - "version": "1.8.0", + "version": "1.10.0", "description": "Private desktop benchmarks and transparent grading for LM Studio models", "author": "Local Model Bench contributors", "license": "MIT", diff --git a/scripts/qa-accounting-mtp.mjs b/scripts/qa-accounting-mtp.mjs index c77ee0c..0da7ffe 100644 --- a/scripts/qa-accounting-mtp.mjs +++ b/scripts/qa-accounting-mtp.mjs @@ -7,7 +7,7 @@ const app=await electron.launch({executablePath:path.resolve('outputs/win-unpack const page=await app.firstWindow();page.setDefaultTimeout(20000); const errors=[];page.on('pageerror',e=>errors.push(e.message)); try{ - await page.getByRole('heading',{name:'Your models. Real numbers.'}).waitFor(); + await page.getByRole('heading',{name:'Choose models to benchmark'}).waitFor(); const snapshot=await page.evaluate(()=>window.bench.snapshot()); assert.equal(snapshot.tests.filter(t=>t.category==='Accounting').length,12); const models=await page.evaluate(()=>window.bench.models()); diff --git a/scripts/qa-accounting-upgrade.mjs b/scripts/qa-accounting-upgrade.mjs index 00324d7..80359a1 100644 --- a/scripts/qa-accounting-upgrade.mjs +++ b/scripts/qa-accounting-upgrade.mjs @@ -11,10 +11,10 @@ const custom={id:'accounting-cash-capital',name:'Preserve my custom accounting c db.prepare('INSERT INTO tests VALUES (?,?)').run(custom.id,JSON.stringify(custom));db.close(); const launch=()=>electron.launch({executablePath:path.resolve('outputs/win-unpacked/Local Model Bench.exe'),args:[],env:{...process.env,LMB_DATA_DIR:dir},timeout:60000}); let app=await launch();try{ - let page=await app.firstWindow();await page.getByRole('heading',{name:'Your models. Real numbers.'}).waitFor(); + let page=await app.firstWindow();await page.getByRole('heading',{name:'Choose models to benchmark'}).waitFor(); let s=await page.evaluate(()=>window.bench.snapshot());assert.equal(s.tests.filter(t=>t.category==='Accounting').length,12);assert.deepEqual(s.tests.find(t=>t.id===custom.id),custom); await page.evaluate(()=>window.bench.deleteTest('accounting-supplies')); - await app.close();app=await launch();page=await app.firstWindow();await page.getByRole('heading',{name:'Your models. Real numbers.'}).waitFor(); + await app.close();app=await launch();page=await app.firstWindow();await page.getByRole('heading',{name:'Choose models to benchmark'}).waitFor(); s=await page.evaluate(()=>window.bench.snapshot());assert.equal(s.tests.filter(t=>t.category==='Accounting').length,11);assert.ok(!s.tests.some(t=>t.id==='accounting-supplies'));assert.deepEqual(s.tests.find(t=>t.id===custom.id),custom); assert.equal(await app.evaluate(({app})=>app.getVersion()),'1.5.0'); fs.writeFileSync('work/accounting-mtp-ui/upgrade-result.json',JSON.stringify({passed:true,packagedVersion:'1.5.0',customTestPreserved:true,deletedTestRemainsDeleted:true},null,2)); diff --git a/scripts/qa-charts.mjs b/scripts/qa-charts.mjs index ef53180..448167a 100644 --- a/scripts/qa-charts.mjs +++ b/scripts/qa-charts.mjs @@ -9,7 +9,7 @@ const source=JSON.parse(fs.readFileSync('outputs/Live-validation-results.json',' const app=await electron.launch({...(process.env.LMB_QA_EXE?{executablePath:process.env.LMB_QA_EXE}:{}),args:process.env.LMB_QA_EXE?[]:[root],env:{...process.env,LMB_DATA_DIR:dir},timeout:60000}); const page=await app.firstWindow();page.setDefaultTimeout(15000);const errors=[];page.on('pageerror',e=>errors.push(e.message)); try{ - await page.getByRole('heading',{name:'Your models. Real numbers.'}).waitFor(); + await page.getByRole('heading',{name:'Choose models to benchmark'}).waitFor(); // Inject saved data into an isolated QA database, then reload to exercise ordinary retrieval. const db=new DatabaseSync(path.join(dir,'bench.sqlite'));const {samples,waves,...doc}=source; db.prepare('INSERT OR REPLACE INTO runs VALUES (?,?)').run(source.id,JSON.stringify(doc)); diff --git a/scripts/qa-comparison.mjs b/scripts/qa-comparison.mjs index c8715e3..2618093 100644 --- a/scripts/qa-comparison.mjs +++ b/scripts/qa-comparison.mjs @@ -8,7 +8,7 @@ const source=JSON.parse(fs.readFileSync('outputs/Live-validation-results.json',' const app=await electron.launch({...(process.env.LMB_QA_EXE?{executablePath:process.env.LMB_QA_EXE}:{}),args:process.env.LMB_QA_EXE?[]:[root],env:{...process.env,LMB_DATA_DIR:dir},timeout:60000}); const page=await app.firstWindow();page.setDefaultTimeout(15000);const errors=[];page.on('pageerror',e=>errors.push(e.message)); try{ - await page.getByRole('heading',{name:'Your models. Real numbers.'}).waitFor(); + await page.getByRole('heading',{name:'Choose models to benchmark'}).waitFor(); const db=new DatabaseSync(path.join(dir,'bench.sqlite'));const {samples,waves,...doc}=source; db.prepare('INSERT OR REPLACE INTO runs VALUES (?,?)').run(source.id,JSON.stringify(doc)); for(const s of samples)db.prepare('INSERT OR REPLACE INTO samples VALUES (?,?,?)').run(s.id,s.runId,JSON.stringify(s)); diff --git a/scripts/qa-gpu.ts b/scripts/qa-gpu.ts index f607317..bb39277 100644 --- a/scripts/qa-gpu.ts +++ b/scripts/qa-gpu.ts @@ -32,7 +32,7 @@ store.saveRun(run);for(const s of samples)store.saveSample(s);store.close(); const app=await electron.launch({args:[root],env:{...process.env,LMB_DATA_DIR:dataDir},timeout:60000}); const page=await app.firstWindow();const errors:string[]=[];page.on('pageerror',e=>errors.push(e.message)); try{ - await page.getByRole('heading',{name:'Your models. Real numbers.'}).waitFor(); + await page.getByRole('heading',{name:'Choose models to benchmark'}).waitFor(); await page.locator('nav').getByRole('button',{name:'Results',exact:true}).click(); await page.getByLabel('Saved run').selectOption(runId); await page.getByRole('heading',{name:'GPU thermals'}).waitFor(); diff --git a/scripts/qa-library.mjs b/scripts/qa-library.mjs index 11c225a..da37e08 100644 --- a/scripts/qa-library.mjs +++ b/scripts/qa-library.mjs @@ -6,7 +6,7 @@ const root=process.cwd();fs.mkdirSync('work/library-qa',{recursive:true}); const app=await electron.launch({...(process.env.LMB_QA_EXE?{executablePath:process.env.LMB_QA_EXE}:{}),args:process.env.LMB_QA_EXE?[]:[root],env:{...process.env,LMB_DATA_DIR:path.join(root,'work/library-qa/data')},timeout:60000}); const page=await app.firstWindow();page.setDefaultTimeout(20000);const errors=[];page.on('pageerror',e=>errors.push(e.message)); try{ - await page.getByRole('heading',{name:'Your models. Real numbers.'}).waitFor(); + await page.getByRole('heading',{name:'Choose models to benchmark'}).waitFor(); let models;try{models=await page.evaluate(()=>window.bench.models());}catch{await page.evaluate(()=>window.bench.startServer());models=await page.evaluate(()=>window.bench.models());await page.getByRole('button',{name:'Refresh models',exact:true}).click();} assert.ok(models.length>0); await page.getByRole('button',{name:'Reset view',exact:true}).click(); diff --git a/scripts/qa-mtp-sweep.mjs b/scripts/qa-mtp-sweep.mjs new file mode 100644 index 0000000..7d44e02 --- /dev/null +++ b/scripts/qa-mtp-sweep.mjs @@ -0,0 +1,138 @@ +import {_electron as electron} from 'playwright'; +import {DatabaseSync} from 'node:sqlite'; +import fs from 'node:fs'; +import path from 'node:path'; +import assert from 'node:assert/strict'; + +// Drives the real window to check the MTP sweep: the controls that plan one, and the panel, +// graphs and export that report one. No LM Studio is needed and nothing is inferred here. +// +// The results half reads the saved validation run and relabels its measurements with MTP +// depths - concurrency 1 becomes "MTP off" and concurrency 2 becomes "2 tokens" - so that the +// depth-aware rendering has something real to render. The NUMBERS are real measurements; the +// DEPTHS are labels this script invented. Whichever depth the panel calls fastest here is an +// artefact of that relabelling and says nothing whatsoever about multi-token prediction. That +// is why nothing from this script is written into the repository. +const root=process.cwd(),dir=path.join(root,'work/mtp-sweep-qa/data'); +fs.mkdirSync(dir,{recursive:true}); +const app=await electron.launch({...(process.env.LMB_QA_EXE?{executablePath:process.env.LMB_QA_EXE}:{}),args:process.env.LMB_QA_EXE?[]:[root],env:{...process.env,LMB_DATA_DIR:dir},timeout:60000}); +const page=await app.firstWindow();page.setDefaultTimeout(15000); +const errors=[];page.on('pageerror',e=>errors.push(e.message)); +const preview=()=>page.locator('.run-summary'); +try{ + await page.getByRole('heading',{name:'Choose models to benchmark'}).waitFor(); + + // --- Planning a sweep ----------------------------------------------------------------- + await page.locator('nav').getByRole('button',{name:'Run',exact:true}).click(); + await page.getByRole('heading',{name:'Configure a benchmark',exact:true}).waitFor(); + const sweepToggle=page.getByLabel('Sweep MTP prediction depths'); + assert.equal(await sweepToggle.count(),0,'the sweep is offered only once MTP itself is on'); + await page.getByLabel('Enable native MTP').check(); + await sweepToggle.waitFor(); + assert.equal(await page.getByLabel('MTP draft tokens').count(),1,'a single depth is still the default'); + + await sweepToggle.check(); + const depths=page.getByLabel('MTP depths to sweep'); + assert.equal(await depths.inputValue(),'0, 1, 2, 3, 4, 5'); + assert.equal(await page.getByLabel('MTP draft tokens').count(),0,'one fixed depth is meaningless during a sweep'); + await page.getByText('6× the size of the same run without a sweep',{exact:false}).waitFor(); + await preview().getByText('MTP off, 1 token, 2 tokens, 3 tokens, 4 tokens, 5 tokens',{exact:true}).waitFor(); + + // The preflight is opt-in, and says so in the preview either way. It is a property of MTP + // rather than of sweeping, so it stays available once the sweep itself is turned off. + const preflight=page.getByLabel('Check the MTP head before measuring'); + assert.equal(await preflight.isChecked(),false,'a run does not pay for the check unless it asks'); + await preview().getByText('Off',{exact:true}).first().waitFor(); + await preflight.check(); + await preview().getByText('On · checked first',{exact:true}).waitFor(); + await preflight.uncheck(); + await preview().getByText('On · checked first',{exact:true}).waitFor({state:'detached'}); + await preflight.check(); + // What is typed is kept as typed, then ordered and de-duplicated when the field is left. + await depths.fill('4, 0, 0, 2');await depths.blur(); + assert.equal(await depths.inputValue(),'0, 2, 4'); + await preview().getByText('MTP off, 2 tokens, 4 tokens',{exact:true}).waitFor(); + // The warning is live feedback while typing. Leaving the field is what drops what it could not + // read, so the warning has served its purpose by then and the field is simply correct. + await depths.fill('0, 99, two'); + await page.getByText('Ignored: 99, two.',{exact:false}).waitFor(); + assert.equal(await depths.getAttribute('aria-invalid'),'true'); + await depths.blur(); + assert.equal(await depths.inputValue(),'0'); + assert.equal(await page.getByText('Ignored:',{exact:false}).count(),0); + + await depths.fill('');await depths.blur(); + await page.getByText('Enter at least one depth, or turn the sweep off.',{exact:true}).waitFor(); + assert.equal(await page.getByRole('button',{name:'Start benchmark'}).isDisabled(),true); + + // Turning MTP off has to take the sweep with it, or the run would be rejected on start. + await depths.fill('0, 2');await depths.blur(); + await page.getByLabel('Enable native MTP').uncheck(); + assert.equal(await page.getByLabel('Sweep MTP prediction depths').count(),0); + await page.getByLabel('Enable native MTP').check(); + assert.equal(await page.getByLabel('Sweep MTP prediction depths').isChecked(),false,'the sweep does not come back on by itself'); + + // --- Reporting a sweep ---------------------------------------------------------------- + const source=JSON.parse(fs.readFileSync('outputs/Live-validation-results.json','utf8')); + const {samples,waves,...doc}=source; + const DEPTHS=[0,2]; + const run={...doc,id:doc.id+'-sweep-fixture',config:{...doc.config,name:'Depth-labelled fixture (not an MTP measurement)',mtp:'on',mtpSweep:DEPTHS,concurrency:[1,2]}}; + const db=new DatabaseSync(path.join(dir,'bench.sqlite')); + db.prepare('INSERT OR REPLACE INTO runs VALUES (?,?)').run(run.id,JSON.stringify(run)); + // The real run measured two concurrency levels at one setting. Every measurement is copied + // once per depth so that depth and concurrency are crossed the way a real sweep crosses them + // — which is the whole point of the panel under test. The numbers in each copy are identical, + // so no depth can beat another here; only the layout is being checked. + for(const depth of DEPTHS){ + const tag=`-f${depth}`; + for(const s of samples)db.prepare('INSERT OR REPLACE INTO samples VALUES (?,?,?)').run(s.id+tag,run.id,JSON.stringify({...s,id:s.id+tag,runId:run.id,waveId:s.waveId+tag,...(s.warmup?{}:{mtpTokens:depth})})); + // Invented acceptance figures, present only at a depth that drafts at all, so the panel has + // both a filled cell and an empty one to render. + const draft=depth===0?undefined:{tasks:1,accepted:40,generated:64,acceptance:40/64,meanLen:2.5}; + for(const w of waves)db.prepare('INSERT OR REPLACE INTO waves VALUES (?,?,?)').run(w.id+tag,run.id,JSON.stringify({...w,id:w.id+tag,runId:run.id,mtpTokens:depth,...(draft?{draft}:{})})); + } + db.close(); + + await page.reload(); + await page.locator('nav').getByRole('button',{name:'Results',exact:true}).click(); + await page.getByRole('combobox',{name:'Saved run'}).selectOption(run.id); + const panel=page.locator('.panel').filter({has:page.getByRole('heading',{name:'Maximum predictions sweep',exact:true})}); + await panel.waitFor(); + // One block per model, a table per concurrency level inside it, and a row per depth in each: + // two models at two concurrency levels at two depths is eight rows. + assert.equal(await panel.locator('h3').count(),2); + assert.equal(await panel.locator('h4').count(),4,'each concurrency level is reported on its own'); + for(const level of ['1 concurrent request','2 concurrent requests'])await panel.getByRole('heading',{name:level,exact:true}).first().waitFor(); + assert.equal(await panel.locator('tbody tr').count(),8); + for(const label of ['MTP off','2 tokens'])assert.equal(await panel.locator('tbody tr').filter({hasText:label}).count(),4); + assert.equal(await panel.getByText('Not enough completed measurements',{exact:false}).count(),0,'both depths were found at every concurrency level'); + // Acceptance is reported where drafting happened and left blank where none did, rather than + // shown as a rate of zero. + assert.equal(await panel.locator('tbody tr').filter({hasText:'62.5%'}).count(),4,'the accepted-draft share is rendered per depth and concurrency'); + assert.equal(await panel.locator('tbody tr').filter({hasText:'MTP off'}).filter({hasText:'62.5%'}).count(),0,'MTP off drafts nothing, so it reports no acceptance'); + + // Each depth is its own line in the graphs rather than a second point on one model's curve. + const legend=await page.locator('.chart-legend').first().innerText(); + for(const key of source.config.modelKeys)for(const label of ['MTP off','MTP 2 tokens']) + assert.ok(legend.includes(`${key} · ${label}`),`legend is missing ${key} at ${label}`); + + const dest=path.join(root,'work/mtp-sweep-qa/report.md'); + await app.evaluate(({dialog},dest)=>{dialog.showSaveDialog=async()=>({canceled:false,filePath:dest});},dest); + await page.getByRole('button',{name:'MD',exact:true}).click(); + await page.getByText('Export saved: '+dest,{exact:true}).waitFor(); + const report=fs.readFileSync(dest,'utf8'); + assert.match(report,/## Native MTP sweep/); + assert.match(report,/swept depths: MTP off, 2 tokens/); + assert.match(report,/\/ MTP 2 tokens/); + assert.match(report,/— 2 concurrent requests/); + assert.match(report,/drafted tokens accepted 62\.5% \(40 of 64 drafted/); + assert.match(report,/drafted tokens accepted not applicable/); + + assert.deepEqual(errors,[]); + await panel.scrollIntoViewIfNeeded();await page.waitForTimeout(300); + await page.screenshot({path:'work/mtp-sweep-qa/sweep.png',fullPage:true}); + fs.writeFileSync('work/mtp-sweep-qa/result.json',JSON.stringify({passed:true,controls:'planned, normalized, and cleared',depthsRendered:['MTP off','2 tokens'], + caution:'Depth labels were invented by this script over real measurements; no MTP speed claim can be read from it.',errors,installed:!!process.env.LMB_QA_EXE},null,2)); + console.log('Sweep controls, depth-split panel, per-depth graph series, and the export section verified.'); + console.log('Reminder: the depths in the fixture are labels, not measurements. Nothing here says anything about MTP speed.'); +}finally{await app.close();} diff --git a/scripts/qa-results.mjs b/scripts/qa-results.mjs index 6b7a9f7..4a621ef 100644 --- a/scripts/qa-results.mjs +++ b/scripts/qa-results.mjs @@ -6,7 +6,7 @@ const root=process.cwd();fs.mkdirSync('work/qa-results',{recursive:true}); const app=await electron.launch({...(process.env.LMB_QA_EXE?{executablePath:process.env.LMB_QA_EXE}:{}),args:process.env.LMB_QA_EXE?[]:[root],env:{...process.env,LMB_DATA_DIR:path.join(root,process.env.LMB_QA_EXE?'work/qa-installed/data':'work/qa-results/data')},timeout:60000}); const page=await app.firstWindow();page.setDefaultTimeout(20000);const errors=[];page.on('pageerror',e=>errors.push(e.message)); try{ - await page.getByRole('heading',{name:'Your models. Real numbers.'}).waitFor(); + await page.getByRole('heading',{name:'Choose models to benchmark'}).waitFor(); await page.getByText('LM Studio connected',{exact:true}).waitFor({timeout:20000}); const modelKey=process.env.LMB_MODEL_KEY;if(!modelKey)throw Error('Set LMB_MODEL_KEY to an exact downloaded model key.'); const model=page.locator('.model-card').filter({has:page.locator('.model-key',{hasText:modelKey})});await model.click(); diff --git a/scripts/qa-text-only.mjs b/scripts/qa-text-only.mjs new file mode 100644 index 0000000..b6165ff --- /dev/null +++ b/scripts/qa-text-only.mjs @@ -0,0 +1,82 @@ +import {_electron as electron} from 'playwright'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import assert from 'node:assert/strict'; + +// Drives the real window to make and remove a text-only copy of a real vision model, and +// checks what LM Studio does with it. This one touches the real LM Studio models directory, +// because that is the whole mechanism under test; it writes only hard links, removes them +// again, and asserts afterwards that the original weights are byte-for-byte where they were. +// +// LMB_TEXTONLY_MODEL picks the model; it must be one LM Studio reports as vision-capable. +const root=process.cwd(),dir=path.join(root,'work/text-only-qa/data'); +fs.rmSync(path.join(root,'work/text-only-qa'),{recursive:true,force:true}); +fs.mkdirSync(dir,{recursive:true}); +const WANT=process.env.LMB_TEXTONLY_MODEL||'gemma-4-12b-it@iq3_xxs'; +const app=await electron.launch({...(process.env.LMB_QA_EXE?{executablePath:process.env.LMB_QA_EXE}:{}),args:process.env.LMB_QA_EXE?[]:[root],env:{...process.env,LMB_DATA_DIR:dir},timeout:60000}); +const page=await app.firstWindow();page.setDefaultTimeout(30000); +const errors=[];page.on('pageerror',e=>errors.push(e.message)); +const models=()=>page.evaluate(()=>window.bench.models()); +const twins=()=>page.evaluate(()=>window.bench.textOnlyTwins()); +try{ + await page.getByRole('heading',{name:'Choose models to benchmark'}).waitFor(); + const before=await models(); + const target=before.find(m=>m.key===WANT); + assert.ok(target,`${WANT} is not in the library`); + assert.equal(target.capabilities?.vision,true,`${WANT} is not vision-capable, so there is nothing to leave out`); + assert.equal((await twins()).length,0,'the library starts with no copies this app made'); + + const plan=await page.evaluate(key=>window.bench.textOnlyPlan(key),WANT); + assert.ok(plan.folder.endsWith(' - Text only')); + assert.ok(plan.projectors.length>0,'the plan names the projector it is leaving behind'); + assert.ok(plan.files.every(f=>!/mmproj/i.test(f.to)),'and links none of it'); + const sizesBefore=plan.files.map(f=>fs.statSync(f.from).size); + + const panel=page.locator('.panel').filter({has:page.getByRole('heading',{name:'Vision projectors',exact:true})}); + await panel.waitFor(); + const row=panel.locator('.selection-list > div').filter({hasText:target.display_name}).filter({hasText:target.quantization?.name??''}).first(); + await row.getByRole('button',{name:'Make text-only copy'}).click(); + await page.getByText('Text-only copy made:',{exact:false}).waitFor(); + + // LM Studio indexes the new folder on its own; it is a separate model key reporting no vision. + const after=await models(); + const fresh=after.filter(m=>!before.some(b=>b.key===m.key)); + assert.equal(fresh.length,1,'exactly one new model key appeared: '+fresh.map(m=>m.key).join(', ')); + assert.equal(fresh[0].capabilities?.vision,false,'the copy is what the original could not be: not vision-capable'); + assert.equal(fresh[0].nativeMtp?.supported,target.nativeMtp?.supported,'and it keeps whatever native MTP the original had'); + // Hard links, not copies: same inode, and the folder adds no bytes to the volume. + const linked=plan.files.map(f=>fs.statSync(f.to)); + plan.files.forEach((f,i)=>{ + assert.equal(linked[i].ino,fs.statSync(f.from).ino,'the copy shares the original bytes'); + assert.equal(linked[i].size,sizesBefore[i]); + assert.ok(linked[i].nlink>=2); + }); + assert.equal(fs.existsSync(path.join(plan.folder,path.basename(plan.projectors[0]))),false,'no projector was linked in'); + assert.equal((await twins()).length,1); + + await panel.locator('.selection-list > div').filter({hasText:target.display_name}).filter({hasText:'Text-only copy made'}).first() + .getByRole('button',{name:'Remove copy'}).click(); + await page.getByText('Its weights were not touched.',{exact:false}).waitFor(); + assert.equal(fs.existsSync(plan.folder),false,'the folder is gone'); + assert.equal((await twins()).length,0); + // The point of the whole design: removal dropped links and nothing else. + plan.files.forEach((f,i)=>{ + assert.ok(fs.existsSync(f.from),'the original weights are still there'); + assert.equal(fs.statSync(f.from).size,sizesBefore[i],'and unchanged'); + }); + for(const p of plan.projectors)assert.ok(fs.existsSync(path.join(plan.sourceDir,p)),'and so is the projector'); + const restored=await models(); + assert.deepEqual(restored.map(m=>m.key).sort(),before.map(m=>m.key).sort(),'the library is exactly as it started'); + + assert.deepEqual(errors,[]); + await panel.scrollIntoViewIfNeeded();await page.waitForTimeout(300); + await page.screenshot({path:'work/text-only-qa/panel.png',fullPage:true}); + fs.writeFileSync('work/text-only-qa/result.json',JSON.stringify({passed:true,model:WANT,folder:plan.folder, + linkedFiles:plan.files.map(f=>path.basename(f.to)),projectorLeftBehind:plan.projectors,keptNativeMtp:fresh[0].nativeMtp?.supported===true, + bytesAdded:0,errors,platform:os.platform(),installed:!!process.env.LMB_QA_EXE},null,2)); + console.log(`Text-only copy of ${WANT}: created as hard links, indexed by LM Studio with vision=false, MTP kept, removed again with the original intact.`); +}catch(error){ + console.error(error); + process.exitCode=1; +}finally{await app.close();} diff --git a/src/BenchmarkPanels.tsx b/src/BenchmarkPanels.tsx index 155080e..5e25618 100644 --- a/src/BenchmarkPanels.tsx +++ b/src/BenchmarkPanels.tsx @@ -1,6 +1,7 @@ import React,{useEffect,useState} from 'react'; import {Plus,Trash2,Upload,X} from 'lucide-react'; import type {Run,RunSummary} from './types'; +import {mtpDepthLabel} from './mtp-sweep'; import {benchmarkRows,interpretScore,sameBenchmarkQuestions,scoreBands,summaryFor,type BenchmarkSelection,type PackSummary} from './benchmarks'; import {defaultInstruction,scoringModes,type PackDraft,type PackPreview,type ScoringMode} from './benchmark-import'; import {useModalDialog} from './dialog'; @@ -49,7 +50,7 @@ function PackImport({onClose,onDone}:{onClose:()=>void;onDone:(summary:PackSumma
; } -const settings=(r:Run)=>`MTP ${r.config.mtp??'default'} · reasoning ${r.config.reasoning} · temperature ${r.config.temperature} · context ${r.config.contextLength} · output ${r.config.maxTokens} · waves ${r.config.waves}`; +const settings=(r:Run)=>`MTP ${r.config.mtp??'default'}${r.config.mtpSweep?.length?' · swept '+r.config.mtpSweep.map(d=>mtpDepthLabel(d)).join(', '):r.config.mtp==='on'?' · '+(r.config.mtpDraftTokens??2)+' draft tokens':''} · reasoning ${r.config.reasoning} · temperature ${r.config.temperature} · context ${r.config.contextLength} · output ${r.config.maxTokens} · waves ${r.config.waves}`; export function BenchmarkResults({run,savedRuns,packs}:{run:Run;savedRuns:RunSummary[];packs:PackSummary[]}){ const [score,setScore]=useState(null),[compareId,setCompareId]=useState(''),[other,setOther]=useState(null),[error,setError]=useState(''); useEffect(()=>{setScore(null);setCompareId('');setOther(null);},[run.id]); @@ -57,6 +58,6 @@ export function BenchmarkResults({run,savedRuns,packs}:{run:Run;savedRuns:RunSum const id=run.tests.find(t=>t.benchmark)?.benchmark?.packId;if(!id)return null; const pack=summaryFor(id,packs); const matching=other&&sameBenchmarkQuestions(run,other); - const renderRows=(r:Run)=>
{benchmarkRows(r).map(row=>)}
; + const renderRows=(r:Run)=>
{benchmarkRows(r).map(row=>)}
; return

{pack.name} scores

{settings(run)}

Score = passed responses ÷ attempted responses. A response must pass every check. Request failures count as misses; warm-ups are excluded. Repeated waves and concurrent copies are repeated attempts, not new questions.

{renderRows(run)}{error&&

{error}

}{other&&<>

{other.config.name}

{settings(other)}

{matching?

Same questions and scoring protocol. Check the settings above before attributing a difference to the model.

:

Different questions or scoring protocol. These scores are not a controlled comparison.

}{matching&&renderRows(other)}}
; } diff --git a/src/ChartsPanel.tsx b/src/ChartsPanel.tsx index b3cee00..d8a2756 100644 --- a/src/ChartsPanel.tsx +++ b/src/ChartsPanel.tsx @@ -2,6 +2,13 @@ import {useMemo,useState} from 'react'; import type {Run} from './types'; import {summaries} from '../electron/export'; import {chartSpecs,renderChart,legendHtml,scoreLabels,type ChartRow,type ScoreSource} from './charts'; +import {measuredDepths,mtpDepthText} from './mtp-sweep'; +import type {HistoryRow} from './history'; +import {backendOf} from '../electron/runtime'; +import {overlayOptions,overlayRows,stillOffered} from './chart-compare'; +// A sweep names its series after the model and the depth ("qwen … · MTP 2 tokens"). Clicking a +// point still has to inspect the model, so this is what the depth is stripped back off with. +const depthSuffix=' · MTP '; // The legend, graph grid, and hover tooltip are shared by the single-run graphs and the history overview, // which feeds in pooled rows keyed by group label instead of by model key. export function ChartGrid({rows,score,onModel,hint='Click a point to inspect its model and responses.'}:{rows:ChartRow[];score:ScoreSource;onModel?:(key:string)=>void;hint?:string}){ @@ -12,11 +19,38 @@ export function ChartGrid({rows,score,onModel,hint='Click a point to inspect its {tip&&
{tip}{hint}
} ; } -export function ChartsPanel({run,onModel}:{run:Run;onModel?:(key:string)=>void}){ +export function ChartsPanel({run,onModel,history=[]}:{run:Run;onModel?:(key:string)=>void;history?:HistoryRow[]}){ const [testId,setTestId]=useState(''),[score,setScore]=useState('objective'); + // Which saved series are drawn beside this run. Ids rather than rows, so a selection survives the + // history being reloaded underneath it. + const [compare,setCompare]=useState([]); const chosen=run.tests.some(t=>t.id===testId)?testId:run.tests[0]?.id; - const rows=useMemo(()=>summaries(run).filter(r=>r.testId===chosen),[run,chosen]); - return

Automatic graphs

Updated as requests and grades arrive. Hover over a point for its values.

- + // A sweep measures one model at several prediction depths, so each depth becomes its own line. + // Without this the graph would join points taken under different settings into one curve. + const own=useMemo(()=>{ + const all=summaries(run).filter(r=>r.testId===chosen); + return measuredDepths(run).length>1?all.map(r=>({...r,modelKey:`${r.modelKey} · ${mtpDepthText(r.mtpDepth)}`})):all; + },[run,chosen]); + const backend=useMemo(()=>backendOf(run).label,[run]); + const models=useMemo(()=>[...new Set(summaries(run).map(r=>r.modelKey))],[run]); + const options=useMemo(()=>overlayOptions(history,run.id,chosen??'',backend,models),[history,run.id,chosen,backend,models]); + // Changing the test can leave a selection pointing at data that test never produced, which would + // draw nothing and read as a bug rather than as a choice. + const live=useMemo(()=>stillOffered(compare,options),[compare,options]); + const rows=useMemo(()=>[...own,...overlayRows(history,live,run.id,chosen??'',models)],[own,history,live,run.id,chosen,models]); + const backends=options.filter(o=>o.kind==='backend'),others=options.filter(o=>o.kind==='model'); + const toggle=(id:string)=>setCompare(c=>c.includes(id)?c.filter(x=>x!==id):[...c,id]); + return

Automatic graphs

Updated as requests and grades arrive. Hover over a point for its values.

+ {/* Saved measurements of the same test, drawn on the same axes. The control is absent entirely + when nothing comparable is saved, rather than an empty menu that invites a click. */} + {options.length>0&&
Compare with{live.length?` · ${live.length}`:''} +
+ {backends.length>0&&<>Other engines{backends.map(o=>)}} + {others.length>0&&<>Other models{others.map(o=>)}} + {live.length>0&&} +

Saved rows from finished runs, drawn as their own lines. Nothing is recalculated, and conditions can differ between runs.

+
} +
+ onModel?.(key.split(depthSuffix)[0])}/>
; } diff --git a/src/HistoryPanel.tsx b/src/HistoryPanel.tsx index 022aa93..e840699 100644 --- a/src/HistoryPanel.tsx +++ b/src/HistoryPanel.tsx @@ -22,7 +22,7 @@ const columns:{key:SortKey;label:string}[]=[ {key:'generationTps',label:'Gen tok/s'},{key:'estimatedPrefillTps',label:'Prefill est.'},{key:'throughput',label:'Total tok/s'}, {key:'medianMs',label:'Median / p95'},{key:'failureRate',label:'Failures'}, {key:'objective',label:'Objective'},{key:'localJudge',label:'Local judge'},{key:'gpuHotSpotMax',label:'GPU hot spot'}]; -const conditionText=(g:HistoryGroup)=>[`concurrency ${g.conditions.concurrency.join(', ')||'—'}`,g.conditions.promptSizes.join(', '),`MTP ${g.conditions.mtp.join(' / ')||'—'}`,`reasoning ${g.conditions.reasoning.join(' / ')||'—'}`].filter(Boolean).join(' · '); +const conditionText=(g:HistoryGroup)=>[`concurrency ${g.conditions.concurrency.join(', ')||'—'}`,g.conditions.promptSizes.join(', '),`MTP ${g.conditions.mtpDepth.join(' / ')||'—'}`,`reasoning ${g.conditions.reasoning.join(' / ')||'—'}`].filter(Boolean).join(' · '); export function HistoryPanel({view,onChange,onOpenRun,signature}:{view:HistoryView;onChange:(v:HistoryView)=>void;onOpenRun:(id:string)=>void;signature:string}){ const [rows,setRows]=useState(null),[error,setError]=useState(''),[loading,setLoading]=useState(false); const [score,setScore]=useState('objective'),[open,setOpen]=useState(''); diff --git a/src/LibraryControls.tsx b/src/LibraryControls.tsx index 2289a0b..a056d2b 100644 --- a/src/LibraryControls.tsx +++ b/src/LibraryControls.tsx @@ -15,7 +15,7 @@ export function LibraryControls({models,view,onChange}:{models:Model[];view:Libr
- + diff --git a/src/benchmarks.ts b/src/benchmarks.ts index b9a427e..b7643f4 100644 --- a/src/benchmarks.ts +++ b/src/benchmarks.ts @@ -1,5 +1,6 @@ import data from './benchmark-data/packs.json'; import type {Run, RunConfig, TestCase} from './types'; +import {mtpDepthText,sweepSteps} from './mtp-sweep'; export type BenchmarkSelection={packId:string;count:number;seed:number}; export type BenchmarkMeta={packId:string;itemId:string;datasetHash:string;protocol:string}; @@ -70,14 +71,18 @@ export function interpretScore(score:number|null){return score===null||!Number.i export function benchmarkRows(run:Run){ const tests=run.tests.filter(t=>t.benchmark),ids=new Set(tests.map(t=>t.id)); if(!tests.length)return []; - return run.config.modelKeys.flatMap(key=>run.config.concurrency.map(concurrency=>{ - const samples=run.samples.filter(s=>!s.warmup&&s.modelKey===key&&s.concurrency===concurrency&&ids.has(s.testId)); + // An MTP sweep runs the whole pack once per prediction depth, so a depth is a separate score + // with its own expected question count. Runs that did not sweep have a single depth of null, + // which matches every response and leaves their rows exactly as they were. + const depths=sweepSteps(run.config).map(s=>s.depth); + return run.config.modelKeys.flatMap(key=>depths.flatMap(depth=>run.config.concurrency.map(concurrency=>{ + const samples=run.samples.filter(s=>!s.warmup&&s.modelKey===key&&s.concurrency===concurrency&&ids.has(s.testId)&&(depth===null||s.mtpTokens===depth)); const passed=samples.filter(s=>s.status==='completed'&&s.objective.score===100).length; const failed=samples.filter(s=>s.status!=='completed').length; const expected=tests.length*run.config.waves*concurrency; const score=samples.length?passed/samples.length*100:null; - return {key,concurrency,passed,attempted:samples.length,expected,failed,score,provisional:samples.length!==expected||run.status!=='completed',truncated:samples.filter(s=>s.possibleTruncation).length,uniqueQuestions:new Set(samples.map(s=>s.testId)).size}; - })); + return {key,concurrency,depth,mtp:mtpDepthText(depth),passed,attempted:samples.length,expected,failed,score,provisional:samples.length!==expected||run.status!=='completed',truncated:samples.filter(s=>s.possibleTruncation).length,uniqueQuestions:new Set(samples.map(s=>s.testId)).size}; + }))); } export function sameBenchmarkQuestions(a:Run,b:Run){ const signature=(r:Run)=>JSON.stringify(r.tests.filter(t=>t.benchmark).map(t=>[t.id,t.benchmark?.datasetHash,t.benchmark?.protocol,t.prompt,t.rules]).sort((x,y)=>String(x[0]).localeCompare(String(y[0])))); diff --git a/src/cache-quant.ts b/src/cache-quant.ts new file mode 100644 index 0000000..c46d4c4 --- /dev/null +++ b/src/cache-quant.ts @@ -0,0 +1,38 @@ +// KV cache quantization, shared by the window and the engine. +// +// The context is the part of a run's memory that scales with concurrency: LM Studio loads one +// instance holding contextLength × parallel tokens of keys and values, so raising the highest +// concurrency raises the cache, not the weights. On a card where the weights already nearly fill +// VRAM that is what decides whether the run measures the GPU or measures an offload to system RAM. + +// The types llama.cpp accepts for a KV cache, in the order LM Studio lists them. f16 is what an +// unquantized cache already is, so it is offered as the explicit "full precision" choice and +// `off` means "do not write the setting at all". +export type CacheQuant='off'|'f16'|'q8_0'|'q5_0'|'q4_0'|'iq4_nl'; +export const cacheQuants:CacheQuant[]=['off','f16','q8_0','q5_0','q4_0','iq4_nl']; + +// Bits per element actually stored, used for the size estimate the run preview shows. q8_0 and +// q4_0 carry a scale per block of 32, which is why they are not exactly 8 and 4. +const BITS:Record,number>={f16:16,q8_0:8.5,q5_0:5.5,q4_0:4.5,iq4_nl:4.5}; + +export const cacheQuantLabel=(q:CacheQuant):string=> + q==='off'?'Off · LM Studio default':q==='f16'?'f16 · full precision': + q==='q8_0'?'q8_0 · half size':q==='q5_0'?'q5_0':q==='q4_0'?'q4_0 · quarter size':'iq4_nl · quarter size'; + +// Relative to an f16 cache, which is the baseline every existing run in the history was measured +// with. `off` is reported as 1 because LM Studio's own default for these models is f16. +export const cacheScale=(q:CacheQuant):number=>q==='off'?1:BITS[q]/16; + +// A run sets K and V independently because they do not degrade alike: a q4_0 V cache is routinely +// fine where a q4_0 K cache measurably hurts, so the common recommendation is q8_0 K with a +// smaller V. The combined figure is the mean of the two halves. +export const cacheScalePair=(k:CacheQuant,v:CacheQuant):number=>(cacheScale(k)+cacheScale(v))/2; + +// Whether flash attention is required. llama.cpp will not use a quantized KV cache without it, +// and LM Studio turns it on by default, so this is a guard against a config that would silently +// fall back to f16 and report a memory saving that did not happen. +export const needsFlashAttention=(k:CacheQuant,v:CacheQuant):boolean=> + (k!=='off'&&k!=='f16')||(v!=='off'&&v!=='f16'); + +export const cacheQuantText=(k:CacheQuant,v:CacheQuant):string=> + k==='off'&&v==='off'?'Off':k===v?`${k} K and V`:`${k} K · ${v} V`; diff --git a/src/chart-compare.ts b/src/chart-compare.ts new file mode 100644 index 0000000..b80a0e4 --- /dev/null +++ b/src/chart-compare.ts @@ -0,0 +1,111 @@ +import type {ChartRow} from './charts'; +import type {HistoryRow} from './history'; +import {shortDate,unknownFacet} from './history'; +import {unknownBackend} from '../electron/runtime'; + +// Putting saved measurements beside the run you are looking at. +// +// The graphs draw one line per modelKey, which is how the MTP sweep already shows several depths of +// one model at once: it relabels the key and the series separate themselves. The same trick carries +// a different engine, or a model measured in some other run, onto the same axes — so "is ROCm +// faster at prompt processing" becomes a line to look at rather than two screens to remember. +// +// Nothing here recomputes a measurement. Every overlaid point is a saved row from a finished run, +// relabelled so the legend says where it came from. + +export type OverlayKind='backend'|'model'; +export type OverlayOption={id:string;kind:OverlayKind;label:string;detail:string;points:number}; + +export const overlayId=(kind:OverlayKind,value:string)=>`${kind}:${value}`; +const parseId=(id:string):{kind:OverlayKind;value:string}|null=>{ + const at=id.indexOf(':'); + if(at<0)return null; + const kind=id.slice(0,at); + return kind==='backend'||kind==='model'?{kind,value:id.slice(at+1)}:null; +}; + +// One saved row per (model, test, concurrency, depth). Several runs can hold the same combination, +// and plotting all of them would draw one series through points taken weeks apart; the most recent +// run wins, which is also the one whose conditions are most likely to still apply. +function newestPerPoint(rows:HistoryRow[]):HistoryRow[]{ + const best=new Map(); + for(const r of rows){ + const key=JSON.stringify([r.modelKey,r.testId,r.concurrency,r.mtpDepth]); + const prior=best.get(key); + if(!prior||Date.parse(r.runCreated)>Date.parse(prior.runCreated))best.set(key,r); + } + return [...best.values()]; +} + +// What this run could usefully be compared against: the same test, measured somewhere else. A +// backend only appears when it has measurements of a model this run also measured, because an +// engine comparison against a different model would not be an engine comparison. +export function overlayOptions(history:HistoryRow[],runId:string,testId:string,currentBackend:string,currentModels:string[]):OverlayOption[]{ + const elsewhere=history.filter(r=>r.runId!==runId&&r.testId===testId); + const models=new Set(currentModels); + const out:OverlayOption[]=[]; + const backends=new Map(); + for(const r of elsewhere){ + if(r.backend===currentBackend||r.backend===unknownBackend||!models.has(r.modelKey))continue; + backends.set(r.backend,[...(backends.get(r.backend)??[]),r]); + } + for(const [label,rows] of [...backends].sort((a,b)=>a[0].localeCompare(b[0]))){ + const points=newestPerPoint(rows); + out.push({id:overlayId('backend',label),kind:'backend',label:`Compare with ${label}`, + detail:`${[...new Set(points.map(p=>p.modelKey))].length} of this run's model(s), measured on ${label}`,points:points.length}); + } + const others=new Map(); + for(const r of elsewhere){ + if(models.has(r.modelKey))continue; + others.set(r.modelKey,[...(others.get(r.modelKey)??[]),r]); + } + for(const [key,rows] of [...others].sort((a,b)=>a[0].localeCompare(b[0]))){ + const points=newestPerPoint(rows); + const last=points.map(p=>p.runCreated).sort().at(-1); + out.push({id:overlayId('model',key),kind:'model',label:key, + detail:`${points[0]?.backend&&points[0].backend!==unknownBackend?points[0].backend+' · ':''}last measured ${shortDate(last)}`,points:points.length}); + } + return out; +} + +// The chosen overlays as chart rows, with the key relabelled so the legend says where each line came +// from. A backend overlay keeps the model name and adds the engine; a model overlay names the model +// and the engine it was measured on, because the same model on two engines is two different lines. +export function overlayRows(history:HistoryRow[],ids:string[],runId:string,testId:string,currentModels:string[]):ChartRow[]{ + const models=new Set(currentModels); + const out:ChartRow[]=[]; + for(const id of ids){ + const parsed=parseId(id); + if(!parsed)continue; + const matching=history.filter(r=>r.runId!==runId&&r.testId===testId&&( + parsed.kind==='backend'?r.backend===parsed.value&&models.has(r.modelKey):r.modelKey===parsed.value)); + for(const r of newestPerPoint(matching)) + out.push({...r,modelKey:`${r.modelKey} · ${r.backend===unknownBackend?shortDate(r.runCreated):r.backend}`}); + } + return out; +} + +// The engines this run's measurements could be set beside: ones that measured a model this run also +// measured, in some other run. A run whose engine could not be determined is not offered, because +// "Unknown engine" is not a thing to compare against. +export function comparableBackends(history:HistoryRow[],runId:string,currentBackend:string,models:string[]):string[]{ + const set=new Set(models); + const found=new Set(); + for(const r of history) + if(r.runId!==runId&&r.backend!==currentBackend&&r.backend!==unknownBackend&&set.has(r.modelKey))found.add(r.backend); + return [...found].sort(); +} + +// Saved rows for the chosen engines, for models this run measured, newest run per measurement. Used +// by the comparison table, where the engine becomes a column rather than part of the series name. +export function backendRows(history:HistoryRow[],runId:string,backends:string[],models:string[]):HistoryRow[]{ + const want=new Set(backends),set=new Set(models); + return newestPerPoint(history.filter(r=>r.runId!==runId&&want.has(r.backend)&&set.has(r.modelKey))); +} + +// Whether an overlay is still on offer after the test changed. A selection kept for a test that has +// no such saved data would silently draw nothing, so the caller drops it instead. +export const stillOffered=(ids:string[],options:OverlayOption[]):string[]=>{ + const live=new Set(options.map(o=>o.id)); + return ids.filter(id=>live.has(id)); +}; diff --git a/src/history.ts b/src/history.ts index 8b38b00..8fa5dfe 100644 --- a/src/history.ts +++ b/src/history.ts @@ -1,6 +1,7 @@ import type {ChartRow} from './charts'; import type {ModelProfile} from './model-profile'; import {percentile} from '../electron/metrics'; +import {mtpDepthText,rowDepth} from './mtp-sweep'; // One saved measurement group (model x test x concurrency) from one run, carrying the facts needed to pool // it with the same measurement taken in a different run. It extends the summary row the single-run screens // already use, so no measurement is recomputed here. @@ -11,6 +12,12 @@ export type HistoryRow=ChartRow&{ family:string;kind:ModelProfile['kind'];sizeLabel:string;sizeBucket:string;quantization:string;quantTier:string; publisher:string;architecture:string;totalB:number|null;activeB:number|null; promptSize:string|null;testKind:string;benchmarkPack:string|null; + // Which llama.cpp build produced the numbers. See electron/runtime.ts backendOf for where it + // comes from and why a run that cannot say reports Unknown rather than today's selection. + backend:string;backendRef:string|null; + // The calibrated prompt-processing rate for the model this row measured, copied from the run's + // own record so a row pooled into another screen can still report it. + prefillCalibratedTps:number|null;prefillOverheadMs:number|null; reasoning:string;temperature:number;contextLength:number;maxTokens:number;waves:number; completed:number;durationsMs:number[];objectiveCount:number;localJudgeCount:number;externalJudgeCount:number; }; @@ -54,19 +61,23 @@ export const facetValue={ promptSizes:(r:HistoryRow)=>r.promptSize??otherPrompt, benchmarks:(r:HistoryRow)=>r.benchmarkPack??ownTests, mtp:(r:HistoryRow)=>r.mtp, + // The depth a measurement was taken at, so a sweep step pools with an ordinary run that used + // the same depth and never with one that used a different depth. + mtpDepth:(r:HistoryRow)=>mtpDepthText(rowDepth(r)), vision:(r:HistoryRow)=>r.vision, reasoning:(r:HistoryRow)=>r.reasoning, statuses:(r:HistoryRow)=>r.runStatus, + backends:(r:HistoryRow)=>r.backend, } as const; export type FacetKey=keyof typeof facetValue; export const facetKeys=Object.keys(facetValue) as FacetKey[]; -export const facetLabels:Record={families:'Model family',kinds:'Architecture',sizes:'Parameters',quantTiers:'Quantization tier',quants:'Exact quantization',concurrency:'Concurrent requests',promptSizes:'Prompt size',benchmarks:'Benchmark pack',mtp:'Native MTP',vision:'Vision',reasoning:'Reasoning',statuses:'Run status'}; +export const facetLabels:Record={families:'Model family',kinds:'Architecture',sizes:'Parameters',quantTiers:'Quantization tier',quants:'Exact quantization',concurrency:'Concurrent requests',promptSizes:'Prompt size',benchmarks:'Benchmark pack',mtp:'Native MTP',mtpDepth:'MTP depth',vision:'Vision',reasoning:'Reasoning',statuses:'Run status',backends:'Inference engine'}; export const modelFacets:FacetKey[]=['families','kinds','sizes','quantTiers','quants']; -export const conditionFacets:FacetKey[]=['benchmarks','concurrency','promptSizes','mtp','vision','reasoning','statuses']; -export const groupOptions={model:'Model',family:'Model family',kind:'Dense vs MoE',size:'Parameters',sizeBucket:'Size range',quantTier:'Quantization tier',quantization:'Exact quantization',benchmark:'Benchmark pack'} as const; +export const conditionFacets:FacetKey[]=['backends','benchmarks','concurrency','promptSizes','mtp','mtpDepth','vision','reasoning','statuses']; +export const groupOptions={model:'Model',family:'Model family',kind:'Dense vs MoE',size:'Parameters',sizeBucket:'Size range',quantTier:'Quantization tier',quantization:'Exact quantization',benchmark:'Benchmark pack',backend:'Inference engine'} as const; export type GroupBy=keyof typeof groupOptions; -export const groupValue:Recordstring>={model:r=>r.modelKey,family:r=>r.family,kind:r=>kindLabel(r.kind),size:r=>r.sizeLabel,sizeBucket:r=>r.sizeBucket,quantTier:r=>r.quantTier,quantization:r=>r.quantization||unknownFacet,benchmark:r=>r.benchmarkPack??ownTests}; -export const sortOptions={label:'Name',runs:'Runs',requests:'Requests',generationTps:'Generation tok/s',estimatedPrefillTps:'Prefill tok/s',throughput:'Total tok/s',medianMs:'Median latency',p95Ms:'p95 latency',failureRate:'Failure rate',objective:'Objective score',localJudge:'Local judge',gpuHotSpotMax:'GPU hot spot peak'} as const; +export const groupValue:Recordstring>={model:r=>r.modelKey,family:r=>r.family,kind:r=>kindLabel(r.kind),size:r=>r.sizeLabel,sizeBucket:r=>r.sizeBucket,quantTier:r=>r.quantTier,quantization:r=>r.quantization||unknownFacet,benchmark:r=>r.benchmarkPack??ownTests,backend:r=>r.backend}; +export const sortOptions={label:'Name',runs:'Runs',requests:'Requests',generationTps:'Generation tok/s',estimatedPrefillTps:'Prefill tok/s',prefillCalibratedTps:'Prefill tok/s (calibrated)',throughput:'Total tok/s',medianMs:'Median latency',p95Ms:'p95 latency',failureRate:'Failure rate',objective:'Objective score',localJudge:'Local judge',gpuHotSpotMax:'GPU hot spot peak'} as const; export type SortKey=keyof typeof sortOptions; export type HistoryView={search:string;groupBy:GroupBy;sort:SortKey;descending:boolean}&Record; const emptyFacets=()=>Object.fromEntries(facetKeys.map(k=>[k,[] as string[]])) as Record; @@ -81,7 +92,7 @@ export function matchesHistory(row:HistoryRow,view:HistoryView,ignore?:FacetKey) } const terms=view.search.toLocaleLowerCase().trim().split(/\s+/).filter(Boolean); if(!terms.length)return true; - const haystack=[row.modelKey,row.model,row.quantization,row.family,row.architecture,row.publisher,row.runName,row.test].join(' ').toLocaleLowerCase(); + const haystack=[row.modelKey,row.model,row.quantization,row.family,row.architecture,row.publisher,row.runName,row.test,row.backend].join(' ').toLocaleLowerCase(); return terms.every(t=>haystack.includes(t)); } const promptOrder=['short','medium','long',otherPrompt]; @@ -118,11 +129,11 @@ export type HistoryGroup={ runs:{id:string;name:string;created:string;status:string}[]; first:string;last:string; requests:number;completed:number;failures:number;failureRate:number|null; - generationTps:Spread;estimatedPrefillTps:Spread;throughput:Spread;ttftMs:Spread; + generationTps:Spread;estimatedPrefillTps:Spread;prefillCalibratedTps:Spread;throughput:Spread;ttftMs:Spread; medianMs:number|null;p95Ms:number|null; objective:number|null;localJudge:number|null;externalJudge:number|null; gpuHotSpotAvg:number|null;gpuHotSpotMax:number|null; - conditions:Record<'concurrency'|'promptSizes'|'mtp'|'vision'|'reasoning',string[]>; + conditions:Record<'concurrency'|'promptSizes'|'mtp'|'mtpDepth'|'vision'|'reasoning',string[]>; mixed:boolean; }; const distinct=(rows:HistoryRow[],pick:(r:HistoryRow)=>string)=>[...new Set(rows.map(pick))].filter(Boolean); @@ -136,10 +147,13 @@ function aggregate(key:string,label:string,rows:HistoryRow[]):HistoryGroup{ const conditions={ concurrency:distinct(rows,r=>String(r.concurrency)).sort((a,b)=>+a-+b), promptSizes:distinct(rows,r=>r.promptSize??otherPrompt), - mtp:distinct(rows,r=>r.mtp),vision:distinct(rows,r=>r.vision),reasoning:distinct(rows,r=>r.reasoning)}; + mtp:distinct(rows,r=>r.mtp),mtpDepth:distinct(rows,r=>mtpDepthText(rowDepth(r))),vision:distinct(rows,r=>r.vision),reasoning:distinct(rows,r=>r.reasoning)}; return {key,label,rows,models:distinct(rows,r=>r.modelKey).sort(),runs,first:created[0]??'',last:created[created.length-1]??'', requests,completed:requests-failures,failures,failureRate:requests?failures/requests:null, generationTps:weighted(rows,r=>r.generationTps,ok),estimatedPrefillTps:weighted(rows,r=>r.estimatedPrefillTps,ok), + // Measured once per loaded model rather than per request, so every row of one model carries the + // same number; weighting by requests still pools two runs of that model in the right proportion. + prefillCalibratedTps:weighted(rows,r=>r.prefillCalibratedTps,ok), throughput:weighted(rows,r=>r.throughput,ok),ttftMs:weighted(rows,r=>r.ttftMs,ok), medianMs:percentile(durations,.5),p95Ms:percentile(durations,.95), // Scores weight by responses that actually carried a score, so an ungraded run never dilutes a graded one. @@ -164,7 +178,7 @@ const sortValue=(g:HistoryGroup,key:SortKey):number|string|null=>{ case 'label':return g.label; case 'runs':return g.runs.length; case 'requests':return g.requests; - case 'generationTps':case 'estimatedPrefillTps':case 'throughput':return g[key]?.value??null; + case 'generationTps':case 'estimatedPrefillTps':case 'prefillCalibratedTps':case 'throughput':return g[key]?.value??null; default:return g[key]; } }; diff --git a/src/main.tsx b/src/main.tsx index 0750190..23a61d6 100644 --- a/src/main.tsx +++ b/src/main.tsx @@ -18,8 +18,17 @@ import {LibraryControls} from './LibraryControls'; import {BenchmarkLibrary,BenchmarkResults} from './BenchmarkPanels'; import {benchmarkConfig,summaryFor} from './benchmarks'; import {visibleModels,selectVisible,defaultLibraryView,restoreLibraryView} from './library'; +import {appVersion} from './version'; +import type {Twin} from '../electron/text-only'; import {HistoryPanel} from './HistoryPanel'; import {defaultHistoryView,restoreHistoryView,shortDate} from './history'; +import {defaultSweep,maxMtpDepth,measuredDepths,mtpDepthText,sweepSteps,sweepVerdicts} from './mtp-sweep'; +import {calibrationFor,focusBlurb,focusColumns,resultsFocuses,type ResultsFocus} from './results-focus'; +import {backendRows,comparableBackends} from './chart-compare'; +import {backendOf} from '../electron/runtime'; +import type {HistoryRow} from './history'; +import {runtimeChoices,type Runtime} from '../electron/runtime'; +import {cacheQuantLabel,cacheQuants,cacheScalePair,type CacheQuant} from './cache-quant'; import type {UpdateState} from '../electron/update'; const api=window.bench; @@ -32,7 +41,11 @@ const toggle=(list:string[],id:string)=>list.includes(id)?list.filter(x=>x!==id) // into a real setting, silently and with no way to tell from the box. const numOr=(raw:string,previous:number)=>raw.trim()===''||!Number.isFinite(Number(raw))?previous:Number(raw); const modelLabel=(m:Model)=>`${m.nativeMtp?.supported===true?'MTP · ':''}${m.display_name} · ${m.quantization?.name??'Unknown quantization'}${m.capabilities&&'vision' in m.capabilities&&m.capabilities.vision?' · Vision':''}`; -function Badge({children,tone=''}:{children:React.ReactNode;tone?:string}){return {children}} +function Badge({children,tone='',title}:{children:React.ReactNode;tone?:string;title?:string}){return {children}} +// MTP arrives in two shapes and the difference is worth seeing on the card: heads built into +// the model file, and heads stored beside it as their own file. Both are usable; naming which +// one a model has is what stops "no MTP" from being read as "this model cannot do MTP". +const mtpBadge=(m:Model)=>m.nativeMtp?.supported===true?{tone:'green',text:m.nativeMtp.kind==='sidecar'?'MTP head':'MTP built in'}:m.nativeMtp?.supported===false?{tone:'',text:'No MTP'}:{tone:'',text:'MTP unknown'}; function Field({label,children,hint}:{label:string;children:React.ReactNode;hint?:string}){return } // Field renders a
; } +// Only a run that measured more than one depth has a sweep to report. A sweep whose later +// steps never loaded leaves one depth of results, which is an ordinary run and is shown as one. +function MtpSweepPanel({run}:{run:Run}){ + const depths=useMemo(()=>measuredDepths(run),[run]); + const verdicts=useMemo(()=>sweepVerdicts(run),[run]); + if(depths.length<2)return null; + return

Maximum predictions sweep

Each depth is a separate load of the model. “MTP off” is the baseline; every other depth lets the model’s own prediction heads draft that many tokens ahead before the main model verifies them.

+ {[...new Set(verdicts.map(v=>v.modelKey))].map(key=>{ + const mine=verdicts.filter(v=>v.modelKey===key).sort((a,b)=>a.concurrency-b.concurrency); + return

{mine[0].modelName}

+ {mine.map(v=>
+ {mine.length>1&&

{v.concurrency} concurrent request{v.concurrency===1?'':'s'}

} +

{v.summary}

+
{['Maximum predictions','Gen tok/s','Spread','Total tok/s','Draft accepted','Mean accepted run','Median','First token','Objective','Completed'].map(h=>)} + {v.rows.map(r=> + {/* Blank rather than 0% at depth 0: nothing was drafted there, which is not the same as + drafting that was always rejected. */} + + + )}
{h}
{r.label}{r.depth===v.best?.depth&&!v.withinNoise&&v.rows.length>1?<> Fastest:null}{number(r.generationTps)}{r.generationSd===null?'—':'± '+number(r.generationSd)}{number(r.throughput)}{r.draft?.acceptance==null?'—':(r.draft.acceptance*100).toFixed(1)+'%'}{r.draft?.meanLen==null?'—':number(r.draft.meanLen,2)}{number(r.medianMs===null?null:r.medianMs/1000)} s{number(r.ttftMs===null?null:r.ttftMs/1000,2)} s{number(r.objective)}{r.completed} / {r.requests}
)} +
; + })} +

Draft accepted is the share of drafted tokens the main model kept, and mean accepted run how many it kept per drafting step; both come from LM Studio's own engine, which reports them per finished request. Requests inside one concurrent wave are not independent of each other, so the spread is a floor on the real uncertainty and not a significance test. Drafted tokens are verified by the main model, so a depth that changes the answers is worth reading rather than trusting.

+
; +} function Results({run,savedRuns,packs,models,busy,onSample,onGrade,onRetry}:{run:Run;savedRuns:Snapshot['runs'];packs:Snapshot['packs'];models:Model[];busy:boolean;onSample:(id:string)=>void;onGrade:(judge:string)=>void;onRetry:()=>void}){ const [filter,setFilter]=useState(''),[judge,setJudge]=useState(''); + // Which question the tables answer. Remembered across runs, because it is a way of working rather + // than a property of one run: someone chasing prompt-processing speed wants that view every time. + const [focus,setFocus]=useState(()=>{try{const v=localStorage.getItem('lmb-results-focus');return v==='prefill'||v==='generation'?v:'all';}catch{return 'all';}}); + useEffect(()=>{try{localStorage.setItem('lmb-results-focus',focus);}catch{}},[focus]); + // Saved rows from every other run, so this screen can set another engine's numbers beside this + // run's. Loaded once per run: a finished run's history does not change while it is open. + const [history,setHistory]=useState([]); + useEffect(()=>{let alive=true;api.history().then(h=>{if(alive)setHistory(h);}).catch(console.error);return()=>{alive=false;};},[run.id]); + const [engineView,setEngineView]=useState('this'); const [comparison,setComparison]=useState({...defaultComparisonFilter}),[savedProfiles,setSavedProfiles]=useState>({}),[inspect,setInspect]=useState(''); useEffect(()=>{let alive=true;api.modelProfiles().then(v=>{if(alive)setSavedProfiles(v);}).catch(console.error);return()=>{alive=false;};},[]); // Filtering and summarising walk every saved response, so they run when the run or the comparison changes @@ -183,18 +296,44 @@ function Results({run,savedRuns,packs,models,busy,onSample,onGrade,onRetry}:{run const matching=useMemo(()=>Object.keys(profiles).filter(key=>matchesProfile(key,profiles[key],comparison)),[profiles,comparison]); const keys=useMemo(()=>matching.filter(key=>!comparison.excluded.includes(key)),[matching,comparison]); const filteredRun=useMemo(()=>({...run,config:{...run.config,modelKeys:keys},samples:run.samples.filter(s=>keys.includes(s.modelKey)),waves:run.waves.filter(w=>keys.includes(w.modelKey))}),[run,keys]); - const rows=useMemo(()=>summaries(filteredRun),[filteredRun]); + const backend=useMemo(()=>backendOf(run).label,[run]); + const otherEngines=useMemo(()=>comparableBackends(history,run.id,backend,keys),[history,run.id,backend,keys]); + // An engine the history no longer offers (the comparison filter narrowed the models, say) would + // otherwise leave the table silently showing only this run under a heading promising two. + const engine=engineView!=='this'&&engineView!=='all'&&!otherEngines.includes(engineView)?'this':engineView; + // Every row carries the engine that produced it, so the table can say so per line rather than + // once at the top — pooled rows and this run's rows sit in the same table. + const rows=useMemo(()=>{ + const own=summaries(filteredRun).map(r=>({...r,backend})); + if(engine==='this'||!otherEngines.length)return own; + return [...own,...backendRows(history,run.id,engine==='all'?otherEngines:[engine],keys)]; + },[filteredRun,backend,engine,otherEngines,history,run.id,keys]); + // The run rather than filteredRun: the calibration columns read run.modelInfo, which the + // comparison filter does not narrow. + const columns=useMemo(()=>focusColumns(focus,run,engine!=='this'&&otherEngines.length>0),[focus,run,engine,otherEngines]); const samples=useMemo(()=>filteredRun.samples.filter(s=>!s.warmup),[filteredRun]); const good=useMemo(()=>samples.filter(s=>s.status==='completed'),[samples]); return <>
{run.status}{new Date(run.created).toLocaleString()}{run.config.retryOf&&Retry run}
{run.error&&
{run.error}
} {await api.saveModelProfile(key,p);setSavedProfiles(await api.modelProfiles());}}/> -
s.metrics.generationTps)))} sub="tokens / second · successful requests"/>s.metrics.prefillTps)))} sub="tokens / second · client-timed estimate"/>s.objective.score)))} sub="out of 100 · scored requests only"/>
- {run.tests.some(t=>t.benchmark)?:} +
{resultsFocuses.map(f=>)}
+

{focusBlurb(focus)}

+ {/* Which engine’s numbers the tables hold. Absent when nothing saved measured these models on + another engine, because a one-option toggle is a decoration rather than a control. */} + {otherEngines.length>0&&
{[{v:'this',label:`This run · ${backend}`},...otherEngines.map(b=>({v:b,label:`Show ${b}`})),{v:'all',label:'Show all engines'}].map(o=>)}
{engine!=='this'&&

Rows from other engines are saved measurements of the same models taken in other runs, newest run per measurement. Their other conditions — context, concurrency, quantization of the cache, MTP depth — are whatever those runs used, so read the engine column with the rest of the row rather than on its own.

}
} + {focus==='prefill' + ?
calibrationFor(run,k,null)?.marginalTps??null)),0)} sub="tokens / second · fixed cost removed"/>calibrationFor(run,k,null)?.overheadMs??null)),0)} sub="ms per request · not prompt processing"/>s.metrics.prefillTps)))} sub="tokens / second · older estimate"/>s.metrics.ttftMs)),0)} sub="ms · median request, all prompt sizes"/>
+ :focus==='generation' + ?
s.metrics.generationTps)))} sub="tokens / second · what one user feels"/>w.throughput)))} sub="tokens / second · all concurrent users"/>s.metrics.outputTokens)),0)} sub="per request · reasoning included"/>
+ :
s.metrics.generationTps)))} sub="tokens / second · successful requests"/>s.metrics.prefillTps)))} sub="tokens / second · client-timed estimate"/>s.objective.score)))} sub="out of 100 · scored requests only"/>
} + + {run.tests.some(t=>t.benchmark)?:} - {inspect&&

{inspect}

Saved responses for this model in the current comparison.

{samples.filter(s=>s.modelKey===inspect).map(s=>)}
} + {inspect&&

{inspect}

Saved responses for this model in the current comparison.

{samples.filter(s=>s.modelKey===inspect).map(s=>)}
}

Prefill is an estimate: cache reuse and stream buffering are not observable. Generation includes reasoning tokens when reported. Scores shown above average only applicable checks and are not a general intelligence ranking. Tail latency from small samples is unstable.

-

Model comparison

{['Model / test','Concurrency','Gen tok/s','Prefill est.','Total tok/s','Median / p95','Failures','Objective','Local judge','External','GPU hot spot avg / peak'].map(h=>)}{rows.map(r=>)}
{h}
{r.model}{r.modelKey}{r.test}{r.concurrency}{number(r.generationTps)}{number(r.estimatedPrefillTps)}{number(r.throughput)}{number(r.medianMs===null?null:r.medianMs/1000)} / {number(r.p95Ms===null?null:r.p95Ms/1000)} s{r.failures}/{r.requests}{number(r.objective)}{number(r.localJudge)}{number(r.externalJudge)}{number(r.gpuHotSpotAvg)} / {number(r.gpuHotSpotMax)} °C
{!rows.length&&

Measurements will appear as requests finish.

}
-

Response explorer

setFilter(e.target.value)}/>
{samples.filter(s=>(s.modelKey+' '+s.testName+' '+s.status).toLowerCase().includes(filter.toLowerCase())).map(s=>)}
+

Model comparison

{columns.map(c=>)}{rows.map(r=>{columns.map(c=>c.identity + ? + :)})}
{c.header}
{r.model}{r.modelKey}{r.test}{r.mtpDepth!==null&&{mtpDepthText(r.mtpDepth)}}{c.cell(r,number,rows)}
{!rows.length&&

Measurements will appear as requests finish.

}
+

Response explorer

setFilter(e.target.value)}/>
{samples.filter(s=>(s.modelKey+' '+s.testName+' '+s.status).toLowerCase().includes(filter.toLowerCase())).map(s=>)}
Run settings, effective model configuration, and log
{JSON.stringify({config:run.config,models:run.modelInfo,environment:run.environment},null,2)}
{run.logs.join('\n')||'No additional messages.'}
; } function TestEditor({test,onClose,onSave,onDelete,busy}:{test:TestCase;onClose:()=>void;onSave:(t:TestCase)=>void;onDelete:()=>void;busy:boolean}){ diff --git a/src/mtp-sweep.ts b/src/mtp-sweep.ts new file mode 100644 index 0000000..ceee399 --- /dev/null +++ b/src/mtp-sweep.ts @@ -0,0 +1,164 @@ +import {average,percentile} from '../electron/metrics'; +import type {Run,Sample,Wave} from './types'; + +// One sweep step is one MTP depth: how many tokens the model's own prediction heads are +// allowed to draft ahead before the main model verifies them. Depth 0 means MTP off, so a +// sweep can include the baseline that any claimed speedup has to be measured against. +// Depth is a load-time setting, so every depth costs a full reload of the model. +export const maxMtpDepth=8; +// 1 through 5 is the range that separates a real speed-up from a deeper draft the main model +// keeps having to throw away, and 0 is the MTP-off baseline the whole comparison rests on. +export const defaultSweep=[0,1,2,3,4,5]; +export const mtpDepthLabel=(depth:number|null|undefined)=> + depth===0?'MTP off':typeof depth==='number'&&Number.isInteger(depth)?`${depth} token${depth===1?'':'s'}`:'Legacy/default'; +// The same thing said where it stands on its own rather than under a column heading, so that a +// bare "2 tokens" cannot be read as anything else. Never "MTP " + mtpDepthLabel, which would +// say "MTP MTP off" at depth 0. +export const mtpDepthText=(depth:number|null|undefined)=> + depth===0?'MTP off':typeof depth==='number'&&Number.isInteger(depth)?`MTP ${mtpDepthLabel(depth)}`:'MTP legacy/default'; + +// Discards anything that could not be loaded, and orders and de-duplicates what is left, so +// "4, 2, 2, 0" and "0, 2, 4" describe the same sweep and produce the same steps. +export function normalizeSweep(values:unknown):number[]{ + if(!Array.isArray(values))return []; + return [...new Set(values.filter((n):n is number=>typeof n==='number'&&Number.isInteger(n)&&n>=0&&n<=maxMtpDepth))].sort((a,b)=>a-b); +} + +export type MtpConfig={mtp?:'off'|'on';mtpDraftTokens?:number;mtpSweep?:number[]}; +export type MtpStep={depth:number|null;mode:'off'|'on'|undefined;tokens:number;label:string}; +// A sweep only means anything where MTP is available at all, which is exactly what the run's +// own MTP switch already asserts for every selected model. With no sweep this returns the +// single step the run has always had, whose depth is null: measurements from runs saved +// before sweeping existed are left exactly as they were rather than relabelled. +export function sweepSteps(config:MtpConfig):MtpStep[]{ + const tokens=config.mtpDraftTokens??2,sweep=normalizeSweep(config.mtpSweep); + if(config.mtp!=='on'||!sweep.length)return [{depth:null,mode:config.mtp,tokens,label:mtpDepthLabel(null)}]; + return sweep.map(depth=>({depth,mode:depth===0?'off':'on',tokens:depth===0?tokens:depth,label:mtpDepthLabel(depth)})); +} +export const isSweep=(config:MtpConfig)=>sweepSteps(config).length>1; +// Every depth a preflight would stand in for. Whether a model can attach its MTP head at all +// does not depend on how many tokens it is then allowed to draft, so one load answers the +// question for all of them and the shallowest is the cheapest one to ask with. +export const onSteps=(steps:MtpStep[])=>steps.filter(s=>s.mode==='on'); +export const preflightStep=(steps:MtpStep[]):MtpStep|null=>onSteps(steps)[0]??null; + +// The depth a saved measurement was taken at, whether it came from a sweep step or from a +// run's single MTP setting. This is what lets a sweep step be compared with an ordinary run +// that happened to use the same depth. +export const rowDepth=(r:{mtpDepth?:number|null;mtp?:string;mtpDraftTokens?:number|null}):number|null=> + typeof r.mtpDepth==='number'?r.mtpDepth:r.mtp==='on'?(typeof r.mtpDraftTokens==='number'?r.mtpDraftTokens:2):r.mtp==='off'?0:null; + +// Depths actually present in saved responses, not the depths that were requested: a step +// whose load failed verification took no measurements and must not appear as a result. +export function measuredDepths(run:Pick):number[]{ + return [...new Set(run.samples.filter(s=>!s.warmup&&typeof s.mtpTokens==='number').map(s=>s.mtpTokens as number))].sort((a,b)=>a-b); +} + +function spread(values:number[]){ + if(values.length<2)return null; + const mean=values.reduce((a,b)=>a+b,0)/values.length; + return Math.sqrt(values.reduce((a,b)=>a+(b-mean)**2,0)/(values.length-1)); +} + +// Concurrency levels measured on one model at one depth, in the order a sweep ran them. +export const measuredConcurrency=(run:Pick):number[]=> + [...new Set(run.samples.filter(s=>!s.warmup).map(s=>s.concurrency))].sort((a,b)=>a-b); + +// How much of the drafting was kept, pooled over the waves in a row: total accepted over total +// drafted, which weights a long request more heavily than a short one exactly as it should. +// `tasks` is how many requests llama.cpp actually reported on, so a row can say whether every +// request it covers was accounted for rather than quietly averaging fewer. +export type SweepDraft={acceptance:number|null;meanLen:number|null;accepted:number;generated:number;tasks:number}; +function pooledDraft(waves:Wave[]):SweepDraft|null{ + const found=waves.map(w=>w.draft).filter((d):d is NonNullable=>!!d&&d.generated>0); + if(!found.length)return null; + const accepted=found.reduce((n,d)=>n+d.accepted,0),generated=found.reduce((n,d)=>n+d.generated,0); + const tasks=found.reduce((n,d)=>n+d.tasks,0); + const lengths=found.filter(d=>d.meanLen!==null); + return {acceptance:generated>0?accepted/generated:null, + meanLen:lengths.length?lengths.reduce((n,d)=>n+d.meanLen!*d.tasks,0)/lengths.reduce((n,d)=>n+d.tasks,0):null, + accepted,generated,tasks}; +} + +export type SweepRow={ + modelKey:string;modelName:string;depth:number;label:string;concurrency:number; + requests:number;completed:number;failures:number; + generationTps:number|null;generationSd:number|null;generationSe:number|null; + throughput:number|null;ttftMs:number|null;medianMs:number|null;p95Ms:number|null;objective:number|null; + draft:SweepDraft|null; +}; +// One row per model, depth and concurrency level, pooled across every test and wave that ran +// there. Concurrency is kept apart rather than pooled into the depth: drafting competes with +// the other slots for the same batch, so the depth that wins alone is not always the depth +// that wins under load, and averaging the two together would hide exactly that. Failed +// requests are counted but never averaged into a rate. +export function sweepRows(run:Pick):SweepRow[]{ + const depths=measuredDepths(run),levels=measuredConcurrency(run); + const keys=[...new Set(run.samples.filter(s=>!s.warmup).map(s=>s.modelKey))]; + const rows:SweepRow[]=[]; + for(const modelKey of keys)for(const depth of depths)for(const concurrency of levels){ + const samples=run.samples.filter((s:Sample)=>!s.warmup&&s.modelKey===modelKey&&s.mtpTokens===depth&&s.concurrency===concurrency); + if(!samples.length)continue; + const ok=samples.filter(s=>s.status==='completed'); + const rates=ok.map(s=>s.metrics.generationTps).filter((n):n is number=>typeof n==='number'&&Number.isFinite(n)); + const sd=spread(rates); + const waves=run.waves.filter((w:Wave)=>w.modelKey===modelKey&&w.mtpTokens===depth&&w.concurrency===concurrency); + rows.push({modelKey,modelName:samples[0].modelName,depth,label:mtpDepthLabel(depth),concurrency, + requests:samples.length,completed:ok.length,failures:samples.length-ok.length, + generationTps:average(rates),generationSd:sd,generationSe:sd!==null&&rates.length>1?sd/Math.sqrt(rates.length):null, + throughput:average(waves.map(w=>w.throughput)),ttftMs:average(ok.map(s=>s.metrics.ttftMs)), + medianMs:percentile(ok.map(s=>s.metrics.durationMs),.5),p95Ms:percentile(ok.map(s=>s.metrics.durationMs),.95), + objective:average(ok.map(s=>s.objective.score)),draft:pooledDraft(waves)}); + } + return rows; +} + +export type SweepVerdict={ + modelKey:string;modelName:string;concurrency:number;rows:SweepRow[]; + baseline:SweepRow|null;best:SweepRow|null;gainPercent:number|null;withinNoise:boolean;qualityDelta:number|null;summary:string; +}; +const percent=(n:number)=>`${n>=0?'+':'−'}${Math.abs(n).toFixed(1)}%`; +// Two averages differ meaningfully only when the gap clears the spread of the measurements +// behind them. Requests inside one concurrent wave are not independent of each other, so this +// interval is optimistic and is described as a floor, never as a significance test. +const noisy=(a:SweepRow,b:SweepRow)=>{ + const ea=a.generationSe,eb=b.generationSe; + if(ea===null&&eb===null)return true; + const combined=Math.sqrt((ea??0)**2+(eb??0)**2); + return Math.abs((a.generationTps??0)-(b.generationTps??0))<2*combined; +}; +// One verdict per model and concurrency level. A depth is only ever compared with another +// depth measured under the same load, because that is the only comparison that means anything: +// the fastest depth at one request at a time need not be the fastest with five in flight. +export function sweepVerdicts(run:Pick):SweepVerdict[]{ + const all=sweepRows(run); + const groups=[...new Set(all.map(r=>r.modelKey+''+r.concurrency))]; + return groups.map(group=>{ + const [modelKey,level]=group.split(''),concurrency=Number(level); + const rows=all.filter(r=>r.modelKey===modelKey&&r.concurrency===concurrency).sort((a,b)=>a.depth-b.depth); + const usable=rows.filter(r=>r.generationTps!==null&&r.completed>0); + const baseline=usable[0]??null; + const best=usable.reduce((b,r)=>b===null||r.generationTps!>b.generationTps!?r:b,null); + const modelName=rows[0]?.modelName??modelKey; + if(!baseline||!best||usable.length<2) + return {modelKey,modelName,concurrency,rows,baseline,best,gainPercent:null,withinNoise:true,qualityDelta:null, + summary:'Not enough completed measurements to compare depths. Only one depth produced results.'}; + const gainPercent=baseline.generationTps!>0?(best.generationTps!-baseline.generationTps!)/baseline.generationTps!*100:null; + const qualityDelta=best.objective!==null&&baseline.objective!==null?best.objective-baseline.objective:null; + // When the baseline itself comes out fastest, what matters is whether the next-best depth was + // really beaten or merely came second by less than the measurements can resolve. Saying + // "nothing beat MTP off" about a dead heat would read as a verdict on MTP that was not earned. + const runnerUp=usable.filter(r=>r.depth!==best.depth).reduce((b,r)=>b===null||r.generationTps!>b.generationTps!?r:b,null); + const withinNoise=best.depth===baseline.depth?runnerUp!==null&&noisy(best,runnerUp):noisy(best,baseline); + const quality=qualityDelta!==null&&qualityDelta<=-1 + ? ` Objective checks scored ${Math.abs(qualityDelta).toFixed(1)} points lower at that depth than at ${baseline.label}; read the responses before adopting it.`:''; + const summary=best.depth===baseline.depth + ? withinNoise + ? `Nothing beat ${baseline.label} on the numbers, but ${runnerUp!.label} is inside the spread of these measurements: this run does not separate them. Add waves before concluding anything.` + : `Nothing beat ${baseline.label}: it was the fastest depth measured for this model at ${concurrency} concurrent request${concurrency===1?'':'s'}.` + : withinNoise + ? `${best.label} measured ${gainPercent===null?'faster':percent(gainPercent)} against ${baseline.label}, but the gap is no larger than the spread of the measurements themselves. Add waves before treating it as real.${quality}` + : `${best.label} was fastest: ${gainPercent===null?'faster':percent(gainPercent)} against ${baseline.label}.${quality}`; + return {modelKey,modelName,concurrency,rows,baseline,best,gainPercent,withinNoise,qualityDelta,summary}; + }); +} diff --git a/src/prefill.ts b/src/prefill.ts new file mode 100644 index 0000000..65b2954 --- /dev/null +++ b/src/prefill.ts @@ -0,0 +1,81 @@ +// Measuring prompt processing honestly. +// +// The per-request prefill figure this app has always reported is input tokens divided by the +// interval between LM Studio's prompt_processing.start and prompt_processing.end events. That +// interval is not prefill. In practice it lands within a millisecond or two of time-to-first-token +// on every sample, because LM Studio raises the end event when the first token is ready rather +// than when the prompt finished being processed. So the figure carries the whole fixed cost of a +// request — HTTP, tokenization, scheduling, the sampler's first step — on top of the prefill +// itself. +// +// At the prompt sizes the performance tests use that fixed cost IS the measurement. A short test +// prompt of ~100 tokens against a ~150 ms floor reports somewhere near a third of the rate the +// same machine reaches on a long prompt, and the shortfall looks exactly like slow hardware. +// +// The fix is not a better single measurement, because no single measurement can separate a fixed +// cost from a per-token one. It takes two. Run the same model over a short prompt and a long one +// and the fixed cost is identical in both, so subtracting one from the other cancels it and what +// is left is the real per-token rate: +// +// marginal t/s = (bigTokens - smallTokens) / (bigSeconds - smallSeconds) +// +// which is the slope of a two-point line, and the intercept is the fixed overhead the old figure +// was silently charging to the GPU. + +import type {Run} from './types'; + +export type PrefillPoint={tokens:number;ms:number}; +export type PrefillCalibration={ + small:PrefillPoint;big:PrefillPoint; + // Null whenever the two points cannot support a slope, rather than a number that looks measured. + marginalTps:number|null;overheadMs:number|null; + naiveTps:number|null; // What the old single-point method would have reported for the short prompt. + note:string; +}; + +// The long prompt has to be enough longer than the short one that the difference is dominated by +// prefill rather than by the noise in two timings. Below these the slope is not reported at all. +export const minTokenRatio=4; +export const minDeltaMs=40; + +export function calibratePrefill(small:PrefillPoint,big:PrefillPoint):PrefillCalibration{ + const naive=small.ms>0?small.tokens/(small.ms/1000):null; + const dTok=big.tokens-small.tokens,dMs=big.ms-small.ms; + const base={small,big,naiveTps:naive}; + if(!(small.tokens>0&&big.tokens>0&&small.ms>0&&big.ms>0)) + return {...base,marginalTps:null,overheadMs:null,note:'Not calibrated: a calibration request returned no timing.'}; + if(big.tokens + `Read these notes and reply with the single word noted.\n${UNIT.repeat(Math.max(1,repeats))}`; +// Small enough to be mostly overhead, large enough to be a real request; and a long one that at +// 8192 context still leaves room for the reply. +export const calibrationRepeats={small:2,big:60}; + +export const prefillText=(c:PrefillCalibration|null|undefined):string=> + !c?'Not measured':c.marginalTps===null?c.note:`${Math.round(c.marginalTps)} t/s (${Math.round(c.overheadMs??0)} ms overhead removed)`; + +// Calibrated prompt processing is measured once per loaded instance rather than per request, so it +// is looked up by model (and by depth when a sweep loaded the model more than once) rather than +// living on the row. Rows pooled from other runs carry their own copy instead; see HistoryRow. +export function calibrationFor(run:Run,modelKey:string,depth:number|null):PrefillCalibration|null{ + const info=run.modelInfo as Record; + const exact=depth===null?undefined:info[`prefill:${modelKey} · MTP ${depth===0?'off':String(depth)}`]; + const value=exact??info[`prefill:${modelKey}`]; + return value&&typeof value==='object'&&'marginalTps' in (value as object)?value as PrefillCalibration:null; +} diff --git a/src/results-focus.ts b/src/results-focus.ts new file mode 100644 index 0000000..b031357 --- /dev/null +++ b/src/results-focus.ts @@ -0,0 +1,113 @@ +import type {ChartRow} from './charts'; +import type {Run} from './types'; +import type {PrefillCalibration} from './prefill'; +import {mtpDepthText} from './mtp-sweep'; +import {calibrationFor} from './prefill'; +export {calibrationFor}; + +// Reading a run one question at a time. +// +// The comparison table carries every measurement a run produced, which is right when the question +// is "what happened" and wrong when it is "why is prompt processing slow". Prefill and generation +// are not the same measurement and they are not limited by the same thing: generation is bound by +// memory bandwidth and helped by speculative decoding, prefill is bound by compute and helped by +// neither. Mixing them into one row means the columns that would answer either question are +// spread across a table too wide to read. +// +// So the same rows are offered under three headings. Nothing is recalculated and nothing is +// hidden that the "All measurements" view would not also show; the columns simply change to the +// ones that bear on the question being asked. +export type ResultsFocus='all'|'prefill'|'generation'; +export const resultsFocuses:{value:ResultsFocus;label:string}[]=[ + {value:'all',label:'All measurements'}, + {value:'prefill',label:'Prompt processing'}, + {value:'generation',label:'Token generation'}, +]; + +export type Fmt=(n:number|null|undefined,digits?:number)=>string; +// `identity` is the model/test cell, which is several lines rather than a value and is rendered by +// the table itself; every other column is a single string this module produces. +export type FocusColumn={header:string;identity?:true;cell:(r:ChartRow,n:Fmt,rows:ChartRow[])=>string;title?:string}; + + +// A row pooled from another run carries that run's calibration with it, because this screen only +// holds the modelInfo of the run being viewed. Own rows have no such field and fall back to it. +const carried=(r:ChartRow,key:'prefillCalibratedTps'|'prefillOverheadMs'):number|null|undefined=> + (r as ChartRow&{prefillCalibratedTps?:number|null;prefillOverheadMs?:number|null})[key]; +const identity:FocusColumn={header:'Model / test',identity:true,cell:r=>r.model}; +const concurrency:FocusColumn={header:'Concurrency',cell:r=>String(r.concurrency)}; +const failures:FocusColumn={header:'Failures',cell:r=>`${r.failures}/${r.requests}`}; + +// The columns that bear on how fast the model turns a prompt into its first token. Calibrated +// prefill is the honest figure; the estimate beside it is the one every earlier run recorded, kept +// so the two can be compared rather than silently swapped. +const prefillColumns=(run:Run):FocusColumn[]=>[ + identity,concurrency, + {header:'Prefill, calibrated',title:'Measured from two prompt lengths so the fixed per-request cost cancels. This is the figure to compare across machines.', + cell:(r,n)=>{const v=carried(r,'prefillCalibratedTps')??calibrationFor(run,r.modelKey,r.mtpDepth)?.marginalTps;return v==null?'—':n(v,0)+' tok/s';}}, + {header:'Fixed overhead',title:'The part of time-to-first-token that is not prompt processing: HTTP, tokenization, scheduling, the first sampling step.', + cell:(r,n)=>{const v=carried(r,'prefillOverheadMs')??calibrationFor(run,r.modelKey,r.mtpDepth)?.overheadMs;return v==null?'—':n(v,0)+' ms';}}, + {header:'Prefill, per request',title:'Input tokens divided by the prompt-processing interval LM Studio reports. At short prompts this is mostly fixed overhead, which is why the calibrated column exists.', + cell:(r,n)=>n(r.estimatedPrefillTps)}, + {header:'First token',cell:(r,n)=>r.ttftMs===null?'—':n(r.ttftMs/1000,2)+' s'}, + {header:'Median / p95',cell:(r,n)=>`${n(r.medianMs===null?null:r.medianMs/1000)} / ${n(r.p95Ms===null?null:r.p95Ms/1000)} s`}, + failures, +]; + +// The columns that bear on how fast the model produces tokens once it has started: the per-request +// rate, what speculative decoding did to it, and what happens to both under concurrency. +const generationColumns:FocusColumn[]=[ + identity,concurrency, + {header:'MTP',title:'How many tokens the model’s own prediction heads drafted ahead before the main model verified them. Depth is fixed when the model is loaded.', + cell:r=>r.mtpDepth===null?String(r.mtp):mtpDepthText(r.mtpDepth)}, + {header:'Gen tok/s',title:'Per request, averaged over the requests that completed. This is what one user experiences.',cell:(r,n)=>n(r.generationTps)}, + {header:'Total tok/s',title:'The whole wave’s output divided by the wall time it took. This is what the machine delivers across all concurrent users at once, and it keeps rising after the per-request rate has started falling.', + cell:(r,n)=>n(r.throughput)}, + {header:'Per user vs 1',title:'The per-request rate at this concurrency as a share of the same model, test and depth at concurrency 1. This is the cost each extra concurrent user imposes on every other one.', + cell:(r,n,rows)=>{const ratio=perUserRatio(rows,r);return ratio===null?'—':(ratio*100).toFixed(0)+'%';}}, + {header:'Draft accepted',title:'The share of drafted tokens the main model kept. Blank where nothing was drafted.', + cell:(r,n)=>r.draftAcceptance==null?'—':(r.draftAcceptance*100).toFixed(1)+'%'}, + {header:'Median / p95',cell:(r,n)=>`${n(r.medianMs===null?null:r.medianMs/1000)} / ${n(r.p95Ms===null?null:r.p95Ms/1000)} s`}, + failures, +]; + +const allColumns:FocusColumn[]=[ + identity,concurrency, + {header:'Gen tok/s',cell:(r,n)=>n(r.generationTps)}, + {header:'Prefill est.',cell:(r,n)=>n(r.estimatedPrefillTps)}, + {header:'Total tok/s',cell:(r,n)=>n(r.throughput)}, + {header:'Median / p95',cell:(r,n)=>`${n(r.medianMs===null?null:r.medianMs/1000)} / ${n(r.p95Ms===null?null:r.p95Ms/1000)} s`}, + failures, + {header:'Objective',cell:(r,n)=>n(r.objective)}, + {header:'Local judge',cell:(r,n)=>n(r.localJudge)}, + {header:'External',cell:(r,n)=>n(r.externalJudge)}, + {header:'GPU hot spot avg / peak',cell:(r,n)=>`${n(r.gpuHotSpotAvg)} / ${n(r.gpuHotSpotMax)} °C`}, +]; + +// Which engine a row was measured on. Only shown while more than one engine is in the table: on a +// single-engine view it would be the same word on every line. +const engineColumn:FocusColumn={header:'Engine',title:'The llama.cpp build that produced this row. Rows from other engines come from other saved runs, whose other conditions may differ.', + cell:r=>(r as ChartRow&{backend?:string}).backend??'—'}; + +export const focusColumns=(focus:ResultsFocus,run:Run,withEngine=false):FocusColumn[]=>{ + const base=focus==='prefill'?prefillColumns(run):focus==='generation'?generationColumns:allColumns; + return withEngine?[base[0],engineColumn,...base.slice(1)]:base; +}; + +// Filled in by the caller, which has the per-request samples; this only says which tiles belong to +// which question and what each one means. +export const focusBlurb=(focus:ResultsFocus):string=> + focus==='prefill' + ?'Prompt processing is how fast the model reads a prompt before it writes anything. It is compute-bound, so it is helped by a faster engine and by a quantization the engine has a fast kernel for, and not at all by speculative decoding. The calibrated column is measured from two prompt lengths so the fixed per-request cost cancels; the per-request column is the older estimate, which at short prompts is mostly that fixed cost rather than the GPU.' + :focus==='generation' + ?'Token generation is how fast the model writes once it has started. It is bound by memory bandwidth, so a smaller quantization of the same model usually helps and speculative decoding can help further by letting the model verify several drafted tokens at once. Per-request rate falls as concurrency rises while total throughput keeps climbing: the first is what one user feels, the second is what the machine delivers.' + :'Every measurement this run produced, including quality scores and GPU telemetry. Use the other two views when the question is specifically about prompt processing or about generation speed.'; + +// One-line label for the per-user-versus-one column, resolved against the concurrency-1 row for the +// same model, test and depth. Returned as a ratio so a run with no concurrency-1 row shows nothing +// rather than an unexplained number. +export function perUserRatio(rows:ChartRow[],r:ChartRow):number|null{ + if(r.concurrency===1)return 1; + const base=rows.find(x=>x.modelKey===r.modelKey&&x.testId===r.testId&&x.mtpDepth===r.mtpDepth&&x.concurrency===1); + return base?.generationTps&&r.generationTps?r.generationTps/base.generationTps:null; +} diff --git a/src/style.css b/src/style.css index 30ccd78..75aa802 100644 --- a/src/style.css +++ b/src/style.css @@ -13,3 +13,23 @@ .header-actions{display:flex;align-items:center;gap:10px}.update-pill{color:#cfe7b4;border-color:#4a6238;background:#212c1e}.update-pill .status-dot{background:var(--green)}.update-pill:hover{background:#2a3826;border-color:#6d8f52}.update-facts{display:grid;grid-template-columns:auto minmax(0,1fr);gap:10px 18px;font-size:12px;border-top:1px solid var(--line);border-bottom:1px solid var(--line);padding:16px 0;margin:18px 0}.update-facts dt{color:#95a5a7}.update-facts dd{margin:0;overflow-wrap:anywhere;font-family:Consolas,monospace;font-size:11px}.update-facts dd:first-of-type{font-family:inherit;font-size:12px} .benchmark-card h2{display:flex;align-items:center;justify-content:space-between;gap:10px;flex-wrap:wrap}.benchmark-card h2 .badge{flex-shrink:0}.modal-body code{background:#101719;border:1px solid #2a363b;border-radius:4px;padding:1px 5px;font-family:Consolas,monospace;font-size:11px} + +/* Comparing against saved measurements. The chart menu is a details/summary so it closes on Escape + and works without JavaScript state; the panel is positioned rather than inline so opening it does + not push the graphs down the page. */ +.compare-menu{position:relative;margin:0;align-self:end} +.compare-menu>summary{list-style:none;padding:10px 13px;border:1px solid #354044;border-radius:7px;background:#222a2e;color:#cfdad9;white-space:nowrap} +.compare-menu>summary::-webkit-details-marker{display:none} +.compare-menu>summary:hover{background:#2c373b;border-color:#556364} +.compare-menu[open]>summary{background:#2d3c2a;border-color:#6d8f52;color:#c7edac} +.compare-panel{position:absolute;right:0;top:calc(100% + 6px);z-index:15;width:330px;max-height:390px;overflow:auto;padding:14px 16px;background:#1a2225;border:1px solid #48534f;border-radius:10px;box-shadow:0 18px 50px #0009} +.compare-panel b{display:block;font-size:10px;letter-spacing:1.2px;text-transform:uppercase;color:#8e9b9d;margin:12px 0 6px} +.compare-panel b:first-child{margin-top:0} +.compare-panel label{display:flex;align-items:flex-start;gap:10px;padding:8px 0;font-size:12px;border-bottom:1px solid #263135} +.compare-panel label span{flex:1;min-width:0;overflow-wrap:anywhere} +.compare-panel small{font-size:10px;margin-top:3px} +.compare-panel button{width:100%;margin-top:12px;font-size:11px} +.compare-panel .hint{font-size:10px;margin:12px 0 0} +.engine-compare{margin-bottom:22px} +.engine-compare .segmented{margin:0} +.engine-compare .hint{margin:10px 0 0} diff --git a/src/types.ts b/src/types.ts index 8dd30f3..f557946 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,6 +1,6 @@ export type Rule = { id: string; label: string; type: 'exact'|'contains'|'heading'|'number'|'json'|'json-equal'|'field'|'words'|'lines'|'final-number'|'ifeval'; expected?: string; path?: string; tolerance?: number; min?: number; max?: number; weight: number }; export type TestCase = { benchmark?:import('./benchmarks').BenchmarkMeta; id:string; name:string; category:string; version:number; prompt:string; answerKey:string; rubric:string; maxTokens:number; rules:Rule[]; kind:'quality'|'performance'; image?:string; imageDigest?:string }; -export type Model = {nativeMtp?:{supported:boolean|null;reason:string;resource?:string};key:string; display_name:string; size_bytes:number; quantization:{name:string}|null; max_context_length:number; type:string; loaded_instances:{id:string; config:{context_length:number;parallel?:number;[key:string]:unknown}}[]; capabilities?:{vision?:boolean;reasoning?:{allowed_options:string[];default:string}}; [key:string]:unknown}; +export type Model = {nativeMtp?:{supported:boolean|null;reason:string;resource?:string;kind?:'bundled'|'sidecar';draftResource?:string;draftPath?:string};key:string; display_name:string; size_bytes:number; quantization:{name:string}|null; max_context_length:number; type:string; loaded_instances:{id:string; config:{context_length:number;parallel?:number;[key:string]:unknown}}[]; capabilities?:{vision?:boolean;reasoning?:{allowed_options:string[];default:string}}; [key:string]:unknown}; export type Settings = {baseUrl:string; token:string; lmsPath:string; timeoutSec:number; loadTimeoutSec:number; judgePrompt:string; updateRepo:string; updateCheck:boolean}; // What the window is allowed to see. The token itself never crosses the IPC // boundary, so the renderer learns only whether one is stored. @@ -8,7 +8,11 @@ export type PublicSettings = Omit & {tokenConfigured:boolean}; // A save can leave the stored token alone (omit `token`), replace it, or clear // it by sending an empty string. export type SettingsUpdate = PublicSettings & {token?:string}; -export type RunConfig = {benchmark?:import('./benchmarks').BenchmarkSelection;vision?:import('../electron/vision').VisionMode;mtp?:'off'|'on';mtpDraftTokens?:number;name:string; modelKeys:string[]; testIds:string[]; mode:'combined'|'performance'|'quality'; preset:string; concurrency:number[]; waves:number; maxTokens:number; contextLength:number; temperature:number; gpu:string; reasoning:string; judgeModel:string; performanceLengths:string[]; timeoutSec:number; retryOf?:string}; +// runtime is LM Studio's engine reference (`llama.cpp-win-x86_64-amd-rocm-avx2@2.40.0`), left +// undefined to mean "whatever LM Studio already has selected", which is what every run made +// before this field existed did. cacheK/cacheV quantize the KV cache, the part of a run's memory +// that grows with concurrency rather than with the weights. +export type RunConfig = {benchmark?:import('./benchmarks').BenchmarkSelection;vision?:import('../electron/vision').VisionMode;mtp?:'off'|'on';mtpDraftTokens?:number;mtpSweep?:number[];mtpPreflight?:boolean;runtime?:string;cacheK?:import('./cache-quant').CacheQuant;cacheV?:import('./cache-quant').CacheQuant;name:string; modelKeys:string[]; testIds:string[]; mode:'combined'|'performance'|'quality'; preset:string; concurrency:number[]; waves:number; maxTokens:number; contextLength:number; temperature:number; gpu:string; reasoning:string; judgeModel:string; performanceLengths:string[]; timeoutSec:number; retryOf?:string}; // unscorable marks a check the grader could not evaluate at all, as opposed to one the // response failed. It is left off both sides of the score rather than counted as a miss. export type Check = {id:string;label:string;passed:boolean;weight:number;detail:string;unscorable?:boolean}; @@ -19,11 +23,11 @@ export type GpuTick = {t:number;tempCore:number|null;tempHotSpot:number|null;tem export type GpuStats = {samples:number;exact:boolean;tempCore:GpuStat|null;tempHotSpot:GpuStat|null;tempMemory:GpuStat|null;power:GpuStat|null;load:GpuStat|null;clockCore:GpuStat|null;fanRpm:GpuStat|null;memoryUsedMax:number|null}; export type RunGpu = {available:boolean;device:string|null;devices:string[];intervalMs:number;note:string;stats:GpuStats|null;perDevice:Record;series:GpuTick[];seriesNote:string}; export type Metrics = {inputTokens:number|null;outputTokens:number|null;reasoningTokens:number|null;generationTps:number|null;prefillTps:number|null;prefillMs:number|null;ttftMs:number|null;durationMs:number;firstContentMs:number|null;prefillMethod:string;cacheNote:string}; -export type Sample = {id:string;runId:string;modelKey:string;modelName:string;testId:string;testName:string;concurrency:number;waveId:string;wave:number;slot:number;warmup:boolean;prompt:string;output:string;reasoning:string;status:'completed'|'failed'|'cancelled'|'timeout';error?:string;metrics:Metrics;objective:Objective;grades:Grade[];rawStats:Record;created:string;retryOf?:string;possibleTruncation:boolean;gpu?:GpuStats|null}; -export type Wave = {id:string;runId:string;modelKey:string;testId:string;concurrency:number;durationMs:number;outputTokens:number|null;throughput:number|null;completed:number;failed:number;gpu?:GpuStats|null}; +export type Sample = {mtpTokens?:number;id:string;runId:string;modelKey:string;modelName:string;testId:string;testName:string;concurrency:number;waveId:string;wave:number;slot:number;warmup:boolean;prompt:string;output:string;reasoning:string;status:'completed'|'failed'|'cancelled'|'timeout';error?:string;metrics:Metrics;objective:Objective;grades:Grade[];rawStats:Record;created:string;retryOf?:string;possibleTruncation:boolean;gpu?:GpuStats|null}; +export type Wave = {mtpTokens?:number;id:string;runId:string;modelKey:string;testId:string;concurrency:number;durationMs:number;outputTokens:number|null;throughput:number|null;completed:number;failed:number;gpu?:GpuStats|null;draft?:import('../electron/engine-log').DraftAcceptance|null}; export type Run = {id:string;created:string;updated:string;status:'running'|'grading'|'completed'|'cancelled'|'failed'|'interrupted';config:RunConfig;tests:TestCase[];modelInfo:Record;environment:Record;logs:string[];samples:Sample[];waves:Wave[];error?:string;gpu?:RunGpu}; export type RunSummary = Omit & {sampleCount:number}; export type Progress = {runId:string;phase:string;message:string;completed:number;total:number;active:number;model?:string}; export type Snapshot = {settings:PublicSettings;tests:TestCase[];runs:RunSummary[];progress:Progress|null;dataPath:string;packs:import('./benchmarks').PackSummary[]}; -export type BenchAPI = {modelProfiles():Promise>;saveModelProfile(key:string,profile:import('./model-profile').ModelProfile):Promise;snapshot():Promise;models():Promise;startServer():Promise;saveSettings(s:SettingsUpdate):Promise;saveTest(t:TestCase):Promise;deleteTest(id:string):Promise;choosePackFile():Promise;importPack(draft:import('./benchmark-import').PackDraft):Promise;deletePack(id:string):Promise;startRun(c:RunConfig):Promise;cancel():Promise;getRun(id:string):Promise;retry(id:string):Promise;grade(id:string,judge:string):Promise;copyPackage(runId:string,sampleId:string):Promise<{copied:boolean;text:string}>;importGrade(runId:string,sampleId:string,raw:string,judge:string):Promise;exportRun(id:string,format:string):Promise;onProgress(fn:(p:Progress|null)=>void):()=>void;history():Promise;updateState():Promise;checkUpdate():Promise;downloadUpdate():Promise;installUpdate():Promise;onUpdate(fn:(s:import('../electron/update').UpdateState)=>void):()=>void;openData():Promise}; +export type BenchAPI = {modelProfiles():Promise>;saveModelProfile(key:string,profile:import('./model-profile').ModelProfile):Promise;snapshot():Promise;models():Promise;runtimes():Promise;startServer():Promise;saveSettings(s:SettingsUpdate):Promise;saveTest(t:TestCase):Promise;deleteTest(id:string):Promise;choosePackFile():Promise;importPack(draft:import('./benchmark-import').PackDraft):Promise;deletePack(id:string):Promise;startRun(c:RunConfig):Promise;cancel():Promise;getRun(id:string):Promise;retry(id:string):Promise;grade(id:string,judge:string):Promise;copyPackage(runId:string,sampleId:string):Promise<{copied:boolean;text:string}>;importGrade(runId:string,sampleId:string,raw:string,judge:string):Promise;exportRun(id:string,format:string):Promise;onProgress(fn:(p:Progress|null)=>void):()=>void;history():Promise;updateState():Promise;checkUpdate():Promise;downloadUpdate():Promise;installUpdate():Promise;onUpdate(fn:(s:import('../electron/update').UpdateState)=>void):()=>void;textOnlyPlan(key:string):Promise;makeTextOnly(key:string):Promise;textOnlyTwins():Promise;removeTextOnly(folder:string):Promise;openData():Promise}; declare global { interface Window { bench:BenchAPI } } diff --git a/src/version.ts b/src/version.ts new file mode 100644 index 0000000..67891d1 --- /dev/null +++ b/src/version.ts @@ -0,0 +1,8 @@ +// The version the window shows comes from package.json, through the define in vite.config.ts, +// so it is the same number the installer is named after and cannot drift from it. It did +// drift: 1.9.0 was packaged and installed while its own sidebar still read "1.8.0", because +// the number was typed into the markup by hand. +// `typeof` on an identifier that was never declared is safe in JavaScript, so a bundle built +// without the define — a test importing this file, for instance — says so rather than throwing. +declare const __APP_VERSION__:string; +export const appVersion=typeof __APP_VERSION__==='string'?__APP_VERSION__:'development build'; diff --git a/tests/cache-quant.test.ts b/tests/cache-quant.test.ts new file mode 100644 index 0000000..525b6d5 --- /dev/null +++ b/tests/cache-quant.test.ts @@ -0,0 +1,83 @@ +import {test} from 'node:test'; +import assert from 'node:assert/strict'; +import {existsSync,mkdirSync,mkdtempSync,readFileSync,writeFileSync} from 'node:fs'; +import {tmpdir} from 'node:os'; +import path from 'node:path'; +import {applyCacheQuant,cacheConfigPath,modelResource,verifyCacheQuant} from '../electron/cache-quant'; +import type {Model} from '../src/types'; + +const base:Model={key:'main@q4',display_name:'Main',format:'gguf',size_bytes:4,max_context_length:8192,type:'llm',loaded_instances:[],quantization:null}; + +// A library laid out the way LM Studio lays one out, with the index this app reads to turn a +// display key into the file path whose config `lms load` merges. +function library(over:{sizeBytes?:number;duplicate?:boolean}={}){ + const home=mkdtempSync(path.join(tmpdir(),'lmb-cache-')); + const internal=path.join(home,'.lmstudio','.internal');mkdirSync(internal,{recursive:true}); + const entry={defaultIdentifier:base.key,sizeBytes:over.sizeBytes??base.size_bytes,format:'gguf',indexedModelIdentifier:'pub/repo/main.gguf'}; + writeFileSync(path.join(internal,'model-index-cache.json'),JSON.stringify({models:over.duplicate?[entry,{...entry}]:[entry]})); + return {home,config:cacheConfigPath('pub/repo/main.gguf',home)}; +} +const fields=(file:string)=>JSON.parse(readFileSync(file,'utf8')).load.fields as {key:string;value:any}[]; +const valueOf=(file:string,key:string)=>fields(file).find(f=>f.key===key)?.value; +const K='llm.load.llama.kCacheQuantizationType',V='llm.load.llama.vCacheQuantizationType'; + +test('the cache type is written in LM Studio’s own shape and taken back out again',()=>{ + const {home,config}=library(); + assert.equal(existsSync(config),false,'nothing exists before the load'); + const restore=applyCacheQuant(base,'q8_0','q4_0',home); + assert.deepEqual(valueOf(config,K),{checked:true,value:'q8_0'}); + assert.deepEqual(valueOf(config,V),{checked:true,value:'q4_0'}); + restore(); + // A file this app created is deleted again, so a load the user makes from LM Studio's own + // window afterwards is not silently quantized by a benchmark that has already finished. + assert.equal(existsSync(config),false,'the file this app created is gone again'); +}); + +test('an existing configuration is restored byte for byte, and its other settings survive the load',()=>{ + const {home,config}=library(); + const original=JSON.stringify({preset:'mine',operation:{fields:[]},load:{fields:[ + {key:'llm.load.contextLength',value:4096}, + {key:K,value:{checked:true,value:'f16'}}, + ]}},null,2); + mkdirSync(path.dirname(config),{recursive:true});writeFileSync(config,original); + const restore=applyCacheQuant(base,'q4_0','q4_0',home); + // The user's own unrelated settings are carried through rather than replaced wholesale. + assert.equal(valueOf(config,'llm.load.contextLength'),4096); + assert.deepEqual(valueOf(config,K),{checked:true,value:'q4_0'},'their stale cache field is replaced, not duplicated'); + assert.equal(fields(config).filter(f=>f.key===K).length,1); + restore(); + assert.equal(readFileSync(config,'utf8'),original,'restored byte for byte'); +}); + +// The whole point of measuring is that the run gets the setting the run asked for. A cache type +// left over in LM Studio's window would otherwise apply to a run that asked for an unquantized one. +test('asking for off writes the field off rather than leaving whatever was there',()=>{ + const {home,config}=library(); + mkdirSync(path.dirname(config),{recursive:true}); + writeFileSync(config,JSON.stringify({load:{fields:[{key:K,value:{checked:true,value:'q4_0'}}]}})); + applyCacheQuant(base,'off','off',home); + assert.deepEqual(valueOf(config,K),{checked:false,value:'f16'}); + assert.deepEqual(valueOf(config,V),{checked:false,value:'f16'}); +}); + +test('a model the index cannot identify exactly is refused rather than guessed at',()=>{ + assert.throws(()=>modelResource(base,library({duplicate:true}).home),/exactly one file/); + // A file whose size no longer matches the index is a stale index, not a match. + assert.throws(()=>modelResource(base,library({sizeBytes:999}).home),/exactly one file/); + assert.throws(()=>modelResource(base,mkdtempSync(path.join(tmpdir(),'lmb-empty-'))),/model index could not be read/); +}); + +test('a resource that is not a relative path cannot become a path this app writes to',()=>{ + for(const bad of ['../../escape.gguf','C:/windows/system32/x.gguf','/etc/passwd/..','']) + assert.throws(()=>cacheConfigPath(bad,'/home'),/not a relative path/,`rejected: ${bad}`); +}); + +test('a cache setting LM Studio did not apply stops the run before anything is measured',()=>{ + // Nothing requested, nothing to confirm — including on an engine that reports no such field. + assert.doesNotThrow(()=>verifyCacheQuant({},'off','off')); + // An engine that does not report the fields is not proof of failure, so it is not treated as one. + assert.doesNotThrow(()=>verifyCacheQuant({},'q8_0','q8_0')); + assert.doesNotThrow(()=>verifyCacheQuant({k_cache_quantization_type:'q8_0',v_cache_quantization_type:'q4_0'},'q8_0','q4_0')); + assert.throws(()=>verifyCacheQuant({k_cache_quantization_type:'f16'},'q8_0','q8_0'),/loaded the K cache as f16/); + assert.throws(()=>verifyCacheQuant({v_cache_quantization_type:{checked:false,value:'f16'}},'off','q4_0'),/loaded the V cache as off/); +}); diff --git a/tests/chart-compare.test.ts b/tests/chart-compare.test.ts new file mode 100644 index 0000000..afb274e --- /dev/null +++ b/tests/chart-compare.test.ts @@ -0,0 +1,81 @@ +import {test} from 'node:test'; +import assert from 'node:assert/strict'; +import {backendOf,unknownBackend} from '../electron/runtime'; +import {backendRows,comparableBackends,overlayId,overlayOptions,overlayRows,stillOffered} from '../src/chart-compare'; +import type {HistoryRow} from '../src/history'; + +const row=(patch:Partial):HistoryRow=>({ + runId:'r1',runName:'run',runCreated:'2026-09-10T00:00:00.000Z',runStatus:'completed', + modelKey:'gemma@q4',testId:'perf-short',concurrency:1,mtpDepth:null,backend:'Vulkan', + backendRef:null,prefillCalibratedTps:null,prefillOverheadMs:null, + ...patch, +} as unknown as HistoryRow); + +const listing=(selected:string)=>['LLM ENGINE SELECTED MODEL FORMAT', + `llama.cpp-win-x86_64-amd-rocm-avx2@2.40.0 ${selected==='rocm'?'✓':''} GGUF`, + `llama.cpp-win-x86_64-vulkan-avx2@2.40.0 ${selected==='vulkan'?'✓':''} GGUF`].join('\n'); + +test('a run says which engine measured it, preferring what it asked for',()=>{ + // A run that named an engine is exact, whatever the machine has selected now. + assert.equal(backendOf({config:{runtime:'llama.cpp-win-x86_64-amd-rocm-avx2@2.40.0'}}).label,'ROCm'); + // Otherwise the ticked row of the table captured when the run was created. + assert.equal(backendOf({environment:{runtime:listing('vulkan')}}).label,'Vulkan'); + assert.equal(backendOf({config:{runtime:'llama.cpp-win-x86_64-vulkan-avx2@2.40.0'}}).version,'2.40.0'); + // A run from before either existed is not attributed to whatever happens to be selected today, + // because that would invent an engine comparison out of measurements nobody took under one. + assert.equal(backendOf({}).label,unknownBackend); + assert.equal(backendOf({environment:{runtime:' '}}).label,unknownBackend); +}); + +test('only engines that measured the same model on the same test are offered',()=>{ + const history=[ + row({runId:'other',backend:'ROCm',modelKey:'gemma@q4',testId:'perf-short'}), + row({runId:'other2',backend:'CUDA',modelKey:'someone-elses@q4',testId:'perf-short'}), + row({runId:'other3',backend:'ROCm',modelKey:'gemma@q4',testId:'perf-long'}), + row({runId:'other4',backend:unknownBackend,modelKey:'gemma@q4',testId:'perf-short'}), + ]; + const options=overlayOptions(history,'r1','perf-short','Vulkan',['gemma@q4']); + const engines=options.filter(o=>o.kind==='backend').map(o=>o.label); + // CUDA measured a different model, perf-long is a different test, and Unknown is not an engine. + assert.deepEqual(engines,['Compare with ROCm']); + // The model CUDA did measure is still offered, just as a model rather than as an engine. + assert.deepEqual(options.filter(o=>o.kind==='model').map(o=>o.id),[overlayId('model','someone-elses@q4')]); + // The run being viewed never offers itself. + assert.equal(overlayOptions(history,'other','perf-short','ROCm',['gemma@q4']).some(o=>o.label==='Compare with ROCm'),false); +}); + +test('the newest run wins when several measured the same point',()=>{ + const history=[ + row({runId:'old',backend:'ROCm',runCreated:'2026-09-01T00:00:00.000Z',generationTps:10} as Partial), + row({runId:'new',backend:'ROCm',runCreated:'2026-09-20T00:00:00.000Z',generationTps:99} as Partial), + ]; + const rows=overlayRows(history,[overlayId('backend','ROCm')],'r1','perf-short',['gemma@q4']); + // One line, not two drawn through points taken weeks apart. + assert.equal(rows.length,1); + assert.equal((rows[0] as unknown as {generationTps:number}).generationTps,99); + // The series is renamed so the legend says which engine it came from. + assert.equal(rows[0].modelKey,'gemma@q4 · ROCm'); +}); + +test('a selection that the current test cannot honour is dropped rather than drawn empty',()=>{ + const history=[row({runId:'other',backend:'ROCm',testId:'perf-short'})]; + const forShort=overlayOptions(history,'r1','perf-short','Vulkan',['gemma@q4']); + const forLong=overlayOptions(history,'r1','perf-long','Vulkan',['gemma@q4']); + const picked=[overlayId('backend','ROCm')]; + assert.deepEqual(stillOffered(picked,forShort),picked); + assert.deepEqual(stillOffered(picked,forLong),[],'no ROCm data for this test, so the choice lapses'); + assert.deepEqual(overlayRows(history,picked,'r1','perf-long',['gemma@q4']),[]); +}); + +test('the table pools engines across tests, unlike the graph which is per test',()=>{ + const history=[ + row({runId:'other',backend:'ROCm',testId:'perf-short'}), + row({runId:'other',backend:'ROCm',testId:'perf-long'}), + ]; + assert.deepEqual(comparableBackends(history,'r1','Vulkan',['gemma@q4']),['ROCm']); + // The comparison table shows every test at once, so both rows come through. + assert.equal(backendRows(history,'r1',['ROCm'],['gemma@q4']).length,2); + // A model this run did not measure is not pooled in: it would not be an engine comparison. + assert.equal(backendRows(history,'r1',['ROCm'],['something-else@q4']).length,0); + assert.deepEqual(comparableBackends(history,'r1','ROCm',['gemma@q4']),[],'the engine you are already on is not an alternative'); +}); diff --git a/tests/charts.test.ts b/tests/charts.test.ts index e3f99ce..1aae8ea 100644 --- a/tests/charts.test.ts +++ b/tests/charts.test.ts @@ -5,7 +5,7 @@ import {chartReport} from '../electron/chart-report'; import {summaries} from '../electron/export'; import fs from 'node:fs'; import type {Run} from '../src/types'; -const row=(patch:Partial={}):ChartRow=>({vision:'off',visionEffective:'Off — no image was sent with any request.',visionImagesSent:false,visionProjectorUnloaded:false,visionLimitation:'',mtp:'off',mtpDraftTokens:null,model:'Model',modelKey:'model',test:'Test',testId:'test',concurrency:1,requests:3,failures:0,failureRate:0,generationTps:30,estimatedPrefillTps:100,throughput:20,medianMs:2000,p95Ms:4000,ttftMs:500,objective:85,localJudge:null,externalJudge:null,gpuHotSpotAvg:null,gpuHotSpotMax:null,gpuCoreTempAvg:null,gpuCoreTempMax:null,gpuMemoryTempMax:null,gpuPowerAvg:null,gpuLoadAvg:null,gpuReadings:0,...patch}); +const row=(patch:Partial={}):ChartRow=>({vision:'off',visionEffective:'Off — no image was sent with any request.',visionImagesSent:false,visionProjectorUnloaded:false,visionLimitation:'',mtp:'off',mtpDraftTokens:null,mtpDepth:null,draftAcceptance:null,draftMeanLen:null,model:'Model',modelKey:'model',test:'Test',testId:'test',concurrency:1,requests:3,failures:0,failureRate:0,generationTps:30,estimatedPrefillTps:100,throughput:20,medianMs:2000,p95Ms:4000,ttftMs:500,objective:85,localJudge:null,externalJudge:null,gpuHotSpotAvg:null,gpuHotSpotMax:null,gpuCoreTempAvg:null,gpuCoreTempMax:null,gpuMemoryTempMax:null,gpuPowerAvg:null,gpuLoadAvg:null,gpuReadings:0,...patch}); test('automatic charts cover speed, all score sources, latency, failures, and scatter',()=>{const specs=chartSpecs('localJudge');assert.equal(specs.length,9);assert.ok(specs.some(s=>s.key==='localJudge'));assert.ok(specs.some(s=>s.key==='estimatedPrefillTps'));}); test('missing score is not plotted as zero; zero score remains valid',()=>{const spec=chartSpecs('localJudge').find(s=>s.key==='localJudge')!;assert.ok(renderChart([row()],spec,'localJudge').includes('No measurements'));assert.ok(renderChart([row({localJudge:0})],spec,'localJudge').includes('class="point"'));}); test('numeric concurrency axis preserves 1,2,8 spacing',()=>{const svg=renderChart([row(),row({concurrency:2}),row({concurrency:8})],chartSpecs('objective')[0]);const points=[...svg.matchAll(/class="point" cx="([\d.]+)"/g)].map(m=>+m[1]);assert.ok(Math.abs((points[1]-points[0])/(points[2]-points[0])-1/7)<.00001);}); diff --git a/tests/cli-output.test.ts b/tests/cli-output.test.ts new file mode 100644 index 0000000..b8ac14a --- /dev/null +++ b/tests/cli-output.test.ts @@ -0,0 +1,44 @@ +import {test} from 'node:test'; +import assert from 'node:assert/strict'; +import {maxCliErrorChars,readableCliError} from '../electron/cli-output'; +const ESC='\u001B'; +// Shaped exactly like what `lms load` actually produced when a Gemma 4 MTP head could not be +// loaded: the failure block once per stream, separated by a minute of repainted progress +// frames, each wrapped in cursor-hide and colour codes. +const frames=Array.from({length:120},(_,i)=>`${ESC}[?25lLoading gemma-4-12b-it@q8_k_xl ${i}% ${'⠋⠙⠹⠸'[i%4]}`).join(''); +const block='Error: Failed to load model. \n\n\n (X) CAUSE \n\nfailed to load draft model, \'C:\\Users\\mwwood\\.lmstudio\\models\\unsloth\\gemma-4-12b-it-GGUF\\MTP\\mtp-gemma-4-12b-it-Q8_0.gguf\': error loading model: invalid vector subscript\n'; +const real=`Command failed: C:\\Users\\mwwood\\.lmstudio\\bin\\lms.exe load gemma-4-12b-it@q8_k_xl --identifier lmb-1 --context-length 32768 --parallel 4 --yes\n${block}\n${block}\n${frames}${ESC}[K${ESC}[?25h`; +test('the one line that explains a failed load survives the progress bar',()=>{ + const out=readableCliError(real); + assert.ok(out.includes('invalid vector subscript'),'the cause has to survive'); + assert.ok(out.includes('mtp-gemma-4-12b-it-Q8_0.gguf'),'so does the file it names'); + assert.ok(out.includes('Cause:'),'the marker is rewritten into something readable'); + assert.ok(out.includes('lms.exe load gemma-4-12b-it@q8_k_xl'),'and the command that was run'); +}); +test('nothing unreadable is left behind',()=>{ + const out=readableCliError(real); + assert.ok(!out.includes(ESC),'no escape sequences'); + assert.ok(!/Loading .*\d+%/.test(out),'no progress repaints'); + assert.ok(!/⠋|⠙|⠹|⠸/.test(out),'no spinner glyphs'); + assert.ok(!out.includes('(X)'),'no stray marker'); + assert.ok(!/\n/.test(out),'one line, so a log entry stays one log entry'); + assert.ok(!/ {2}/.test(out),'no runs of padding'); +}); +test('a failure block printed on both streams is reported once',()=>{ + const out=readableCliError(real); + assert.equal(out.split('invalid vector subscript').length-1,1); + assert.equal(out.split('Failed to load model').length-1,1); + // 120 repaint frames and a doubled block collapse to something a person reads at a glance. + assert.ok(real.length>4000&&out.length<400,`expected a short line, got ${out.length} from ${real.length}`); +}); +test('an error that was already readable is left alone',()=>{ + assert.equal(readableCliError('Model not found: nope@q4'),'Model not found: nope@q4'); + assert.equal(readableCliError(''),''); + assert.equal(readableCliError(null as unknown as string),''); + assert.equal(readableCliError(undefined as unknown as string),''); +}); +test('a pathological stream cannot fill the database',()=>{ + const huge=readableCliError(Array.from({length:5000},(_,i)=>`distinct failure line ${i}`).join('\n')); + assert.ok(huge.length<=maxCliErrorChars,`expected a capped string, got ${huge.length}`); + assert.ok(huge.endsWith('…'),'a truncated message says it was truncated'); +}); diff --git a/tests/history.test.ts b/tests/history.test.ts index 173fea9..8809186 100644 --- a/tests/history.test.ts +++ b/tests/history.test.ts @@ -7,11 +7,12 @@ import {percentile} from '../electron/metrics'; import type {Metrics,Run,Sample} from '../src/types'; const row=(patch:Partial={}):HistoryRow=>({ vision:'off',visionEffective:'',visionImagesSent:false,visionProjectorUnloaded:false,visionLimitation:'', - mtp:'off',mtpDraftTokens:null,model:'Gemma 4 12B',modelKey:'gemma-4-12b-it@iq3_xxs',test:'Short prompt throughput',testId:'perf-short',concurrency:1, + mtp:'off',mtpDraftTokens:null,mtpDepth:null,draftAcceptance:null,draftMeanLen:null,model:'Gemma 4 12B',modelKey:'gemma-4-12b-it@iq3_xxs',test:'Short prompt throughput',testId:'perf-short',concurrency:1, requests:1,failures:0,failureRate:0,generationTps:null,estimatedPrefillTps:null,throughput:null, medianMs:null,p95Ms:null,ttftMs:null,objective:null,localJudge:null,externalJudge:null, gpuHotSpotAvg:null,gpuHotSpotMax:null,gpuCoreTempAvg:null,gpuCoreTempMax:null,gpuMemoryTempMax:null, gpuPowerAvg:null,gpuLoadAvg:null,gpuReadings:0, + backend:'Vulkan',backendRef:'llama.cpp-win-x86_64-vulkan-avx2@2.40.0',prefillCalibratedTps:null,prefillOverheadMs:null, runId:'r1',runName:'Run one',runCreated:'2026-09-01T00:00:00.000Z',runStatus:'completed', family:'Gemma',kind:'dense',sizeLabel:'12B',sizeBucket:'8-20B',quantization:'IQ3_XXS',quantTier:'IQ3', publisher:'unsloth',architecture:'gemma4',totalB:12,activeB:null, diff --git a/tests/load-estimate.test.ts b/tests/load-estimate.test.ts new file mode 100644 index 0000000..2ed0297 --- /dev/null +++ b/tests/load-estimate.test.ts @@ -0,0 +1,67 @@ +import {test} from 'node:test'; +import assert from 'node:assert/strict'; +import {loadAdvice,looksLikeMemory,parseEstimate} from '../electron/load-estimate'; +import type {Model} from '../src/types'; + +const model:Model={key:'g@q8',display_name:'Gemma',format:'gguf',size_bytes:12.7*1024**3,max_context_length:262144,type:'llm',loaded_instances:[],quantization:null}; + +test('LM Studio’s estimate block is read, including its own confidence',()=>{ + const e=parseEstimate(['Model: gemma-4-12b-it@q8_k_xl','Context Length: 32,768','Estimated GPU Memory: 12.86 GiB','Estimated Total Memory: 12.86 GiB','Confidence: LOW','Estimate: This model may be loaded based on your resource guardrails settings.'].join('\n')); + assert.ok(Math.abs(e.gpuGiB!-12.86)<0.01); + assert.ok(Math.abs(e.totalGiB!-12.86)<0.01); + assert.equal(e.confidence,'LOW'); + assert.match(e.verdict,/may be loaded/); +}); + +test('a block that says nothing is left null rather than defaulted to zero',()=>{ + const e=parseEstimate('Model: x\nSomething else entirely'); + assert.equal(e.gpuGiB,null); + assert.equal(e.totalGiB,null); + assert.equal(e.confidence,'UNREPORTED'); + // A zero here would read as "needs no memory", which is worse than admitting it is unknown. + assert.notEqual(e.gpuGiB,0); +}); + +test('megabytes are normalised to the same unit as gigabytes',()=>{ + assert.ok(Math.abs(parseEstimate('Estimated GPU Memory: 512 MiB').gpuGiB!-0.5)<0.001); +}); + +// The distinction that decides whether memory advice is attached at all. Attaching it to a failure +// that was not about memory would send someone to lower a context length that was never the cause. +test('only failures that are actually about room attract memory advice',()=>{ + for(const yes of ['Failed to load model. Cause: not enough VRAM','out of memory','Model does not fit in the available memory','failed to allocate buffer','blocked by your resource guardrails']) + assert.equal(looksLikeMemory(yes),true,`should be memory: ${yes}`); + for(const no of ['failed to load draft model: invalid vector subscript','unknown option --nope','the projector could not be attached','Model architecture is not supported','']) + assert.equal(looksLikeMemory(no),false,`should not be memory: ${no}`); +}); + +test('the advice names the arithmetic the run actually did, not just the model',()=>{ + const text=loadAdvice(model,8192,4,'off','off',{gpuGiB:12.86,totalGiB:12.86,confidence:'LOW',verdict:''}); + // The number nobody expects: they asked for 8192 and the instance was loaded with four times it. + assert.match(text,/32,768 context tokens/); + assert.match(text,/8,192 per request × 4 parallel slots/); + assert.match(text,/12\.70 GiB/,'the weights are named separately from the cache'); + assert.match(text,/12\.86 GiB/); + assert.match(text,/confidence: low/); + // All three levers, because which one to pull is the reader's call, not this message's. + assert.match(text,/lower the context length/i); + assert.match(text,/concurrency/i); + assert.match(text,/q8_0 roughly halves it/); +}); + +test('a run already quantizing its cache is told where it stands rather than told to start',()=>{ + const text=loadAdvice(model,8192,4,'q8_0','q4_0'); + assert.doesNotMatch(text,/q8_0 roughly halves it/,'it is already doing this'); + assert.match(text,/quantize the cache further/); + // q8_0 stores 8.5 bits per element and q4_0 4.5, against f16's 16, so the pair averages to + // (0.531 + 0.281) / 2 — about 41%, not the midpoint between "half" and "quarter" it sounds like. + assert.match(text,/about 41% of an f16 cache/); + // Nothing was estimated, so nothing is claimed about what LM Studio thought. + assert.doesNotMatch(text,/estimated/i); +}); + +test('a single slot is described in the singular and without a phantom multiplier',()=>{ + const text=loadAdvice({...model,size_bytes:0},4096,1); + assert.match(text,/1 parallel slot[^s]/); + assert.doesNotMatch(text,/The weights are/,'an unknown size is left unsaid rather than printed as 0.00 GiB'); +}); diff --git a/tests/mtp-sidecar.test.ts b/tests/mtp-sidecar.test.ts new file mode 100644 index 0000000..52f9778 --- /dev/null +++ b/tests/mtp-sidecar.test.ts @@ -0,0 +1,214 @@ +import {test} from 'node:test'; +import assert from 'node:assert/strict'; +import {appendFileSync,existsSync,mkdirSync,mkdtempSync,readFileSync,rmSync,statSync,writeFileSync} from 'node:fs'; +import {tmpdir} from 'node:os'; +import path from 'node:path'; +import {attachMtp,mtpArgs,verifyMtp} from '../electron/mtp'; +import {applySidecar,sidecarConfigPath} from '../electron/mtp-sidecar'; +import {draftAcceptance,draftModelsLoaded,settledDraftAcceptance,watchEngineLog} from '../electron/engine-log'; +import {runEngine,type Adapter,type EngineEvent} from '../electron/engine'; +import {metrics} from '../electron/metrics'; +import {defaultConfig,defaultSettings,starterTests} from '../src/defaults'; +import type {Model,Run,RunConfig} from '../src/types'; + +const slash=(p:string)=>p.replaceAll('\\','/'); +const base:Model={key:'main@q3',display_name:'Main',format:'gguf',size_bytes:4,max_context_length:8192,type:'llm',loaded_instances:[],quantization:null}; + +// A library laid out the way LM Studio lays one out: a model file, and prediction heads stored +// with it as their own file that LM Studio indexes in its separate "drafter" domain. +function library(over:{headMeta?:Record;headAt?:string;extraHead?:boolean}={}){ + const home=mkdtempSync(path.join(tmpdir(),'lmb-sidecar-')); + const main=path.join(home,'pub','repo','main.gguf'); + const head=path.join(home,over.headAt??path.join('pub','repo','MTP','head.gguf')); + for(const file of [main,head]){mkdirSync(path.dirname(file),{recursive:true});writeFileSync(file,'GGUF');} + const internal=path.join(home,'.lmstudio','.internal');mkdirSync(internal,{recursive:true}); + const headResource=slash(path.relative(path.join(home,'pub'),head)); + const entry=(file:string,extra:object)=>({format:'gguf',entryPoint:{absPath:slash(file)},...extra}); + const models=[ + entry(main,{domain:'llm',defaultIdentifier:'main@q3',sizeBytes:4,indexedModelIdentifier:'pub/repo/main.gguf'}), + entry(head,{domain:'drafter',indexedModelIdentifier:headResource}), + ]; + if(over.extraHead){ + const spare=path.join(home,'pub','repo','spare.gguf');writeFileSync(spare,'GGUF'); + models.push(entry(spare,{domain:'drafter',indexedModelIdentifier:'pub/repo/spare.gguf'})); + } + writeFileSync(path.join(internal,'model-index-cache.json'),JSON.stringify({models})); + const record=(file:string,metadata:object)=>[slash(file),{mtimeMs:statSync(file).mtimeMs,fileSizeBytes:statSync(file).size,metadata}]; + const headMeta={supportsMtp:false,arch:'testarch-assistant',embeddingLength:1024,embeddingLengthOut:3840,nextnPredictLayers:4,...over.headMeta}; + const map=[record(main,{supportsMtp:false,arch:'testarch',embeddingLength:3840}),record(head,headMeta)]; + if(over.extraHead)map.push(record(path.join(home,'pub','repo','spare.gguf'),headMeta)); + writeFileSync(path.join(internal,'gguf-metadata-cache.json'),JSON.stringify({json:{map}})); + return {home,head:slash(head),headResource,clean:()=>rmSync(home,{recursive:true,force:true})}; +} + +test('prediction heads stored beside a model count as native MTP, and say so',()=>{ + const lib=library();try{ + const mtp=attachMtp([base],lib.home)[0].nativeMtp!; + assert.equal(mtp.supported,true); + assert.equal(mtp.kind,'sidecar'); + assert.equal(mtp.draftResource,lib.headResource); + assert.equal(mtp.draftPath,lib.head); + }finally{lib.clean();} +}); +test('a head is paired by structure, never by where its name looks right',()=>{ + for(const broken of [{embeddingLengthOut:2048},{arch:'otherarch-assistant'},{nextnPredictLayers:0}]){ + const lib=library({headMeta:broken});try{ + const mtp=attachMtp([base],lib.home)[0].nativeMtp!; + assert.equal(mtp.supported,false,JSON.stringify(broken)); + assert.equal(mtp.kind,undefined); + }finally{lib.clean();} + } + // A head filed outside the model's own folder belongs to something else, whatever it contains. + const away=library({headAt:path.join('pub','elsewhere','head.gguf')});try{ + assert.equal(attachMtp([base],away.home)[0].nativeMtp!.supported,false); + }finally{away.clean();} +}); +test('two heads that both fit are reported as ambiguous rather than guessed between',()=>{ + const lib=library({extraHead:true});try{ + const mtp=attachMtp([base],lib.home)[0].nativeMtp!; + assert.equal(mtp.supported,null); + assert.match(mtp.reason,/More than one MTP head/); + assert.match(mtp.reason,/spare\.gguf/); + }finally{lib.clean();} +}); +test('a separate head adds no load arguments, because LM Studio has no flag for one',()=>{ + const sidecar={...base,nativeMtp:{supported:true,kind:'sidecar' as const,reason:'',resource:'pub/repo/main.gguf',draftResource:'repo/MTP/head.gguf',draftPath:'c:/m/head.gguf'}}; + assert.deepEqual(mtpArgs(sidecar,'on',3),[]); + assert.deepEqual(mtpArgs(sidecar,'off'),['--no-speculative-draft-mtp']); + assert.deepEqual(mtpArgs({...base,nativeMtp:{supported:true,kind:'bundled',reason:''}},'on',3),['--speculative-draft-mtp','--speculative-draft-max-tokens','3']); +}); +test('a separate head is only believed when the engine log names that exact file',()=>{ + const evidence=(loaded:string[])=>({kind:'sidecar' as const,draftPath:'C:/m/head.gguf',draftersLoaded:loaded}); + const config={speculative_draft_mtp:false,speculative_draft_max_tokens:3}; + verifyMtp(config,'on','gguf',3,evidence(['c:/m/head.gguf'])); + assert.throws(()=>verifyMtp(config,'on','gguf',3,evidence([])),/did not report loading the MTP head/); + assert.throws(()=>verifyMtp(config,'on','gguf',3,evidence(['c:/m/other.gguf'])),/loaded c:\/m\/other\.gguf instead/); + assert.throws(()=>verifyMtp({...config,speculative_draft_max_tokens:4},'on','gguf',3,evidence(['c:/m/head.gguf'])),/draft-token count/); + // The sweep's baseline has to be a plain load. A head LM Studio brought along on its own + // would make every depth measured against it look better than it is. + assert.throws(()=>verifyMtp({speculative_draft_mtp:false},'off','gguf',2,{kind:'sidecar',draftPath:'C:/m/head.gguf',draftersLoaded:['c:/m/head.gguf']}),/MTP off/); + verifyMtp({speculative_draft_mtp:false},'off','gguf',2,{kind:'sidecar',draftPath:'C:/m/head.gguf',draftersLoaded:[]}); +}); +test('the load setting is written into LM Studio’s own config and taken straight back out',()=>{ + const lib=library();try{ + const model=attachMtp([base],lib.home)[0]; + const file=sidecarConfigPath(model.nativeMtp!.resource!,lib.home); + assert.equal(existsSync(file),false); + const restore=applySidecar(model,4,lib.home); + const fields=JSON.parse(readFileSync(file,'utf8')).load.fields as {key:string;value:unknown}[]; + const value=(key:string)=>fields.find(f=>f.key.endsWith(key))?.value; + assert.equal(value('draftMtpSidecar'),true); + assert.equal(value('draftModel'),lib.headResource); + assert.equal(value('draftMaxTokens'),4); + // Only one speculative mode may be on at once or LM Studio refuses the load outright. + assert.equal(value('draftMtp'),false); + assert.equal(value('draftSimple'),false); + restore(); + assert.equal(existsSync(file),false,'a file this app created is deleted again'); + assert.equal(existsSync(path.dirname(file)),false,'and so are the folders it had to make'); + }finally{lib.clean();} +}); +test('settings the user already had are kept, and restored byte for byte',()=>{ + const lib=library();try{ + const model=attachMtp([base],lib.home)[0]; + const file=sidecarConfigPath(model.nativeMtp!.resource!,lib.home); + const original='{"preset":"","operation":{"fields":[]},"load":{"fields":[{"key":"llm.load.contextLength","value":4242},{"key":"llm.load.llama.speculativeDecoding.draftSimple","value":true}]}}'; + mkdirSync(path.dirname(file),{recursive:true});writeFileSync(file,original); + const restore=applySidecar(model,2,lib.home); + const fields=JSON.parse(readFileSync(file,'utf8')).load.fields as {key:string;value:unknown}[]; + assert.equal(fields.find(f=>f.key==='llm.load.contextLength')?.value,4242); + assert.equal(fields.find(f=>f.key.endsWith('draftSimple'))?.value,false); + restore(); + assert.equal(readFileSync(file,'utf8'),original); + restore(); // Restoring twice must not undo a later, unrelated write. + assert.equal(readFileSync(file,'utf8'),original); + }finally{lib.clean();} +}); +test('a model resource that is not a relative path is refused before anything is written',()=>{ + for(const bad of ['C:/somewhere/main.gguf','../../escape.gguf','..'])assert.throws(()=>sidecarConfigPath(bad,tmpdir()),/relative path/); +}); +test('only log lines written during a load are read, and only draft models are taken from them',()=>{ + const home=mkdtempSync(path.join(tmpdir(),'lmb-log-'));try{ + const dir=path.join(home,'.lmstudio','server-logs','2026-09');mkdirSync(dir,{recursive:true}); + const file=path.join(dir,'2026-09-15.1.log'); + writeFileSync(file,"[old] I common_speculative_init_result: loading draft model 'C:\\models\\stale.gguf'\n"); + const watch=watchEngineLog(home); + assert.deepEqual(draftModelsLoaded(watch),[],'anything logged before the load began is not evidence about it'); + appendFileSync(file,"[new] I load: something unrelated\n[new] I common_speculative_init_result: loading draft model 'C:\\models\\MTP\\head.gguf'\n"); + assert.deepEqual(draftModelsLoaded(watch),['c:/models/mtp/head.gguf']); + // A file rotated in the middle of a load was not there to be measured, so it is read whole. + appendFileSync(path.join(dir,'2026-09-15.2.log'),"[new] I common_speculative_init_result: loading draft model 'C:\\models\\MTP\\second.gguf'\n"); + assert.deepEqual(draftModelsLoaded(watch).sort(),['c:/models/mtp/head.gguf','c:/models/mtp/second.gguf']); + }finally{rmSync(home,{recursive:true,force:true});} +}); + +const textTest=starterTests.find(t=>t.id==='reasoning')!; +const makeRun=(config:Partial):Run=>({id:'sidecar-run',created:'2026-09-15T00:00:00.000Z',updated:'2026-09-15T00:00:00.000Z',status:'running', + config:{...structuredClone(defaultConfig),modelKeys:['mock'],testIds:['reasoning'],mode:'quality',concurrency:[1],waves:1,maxTokens:64,contextLength:2048,judgeModel:'',mtp:'on',...config}, + tests:[textTest],modelInfo:{},environment:{},logs:[],samples:[],waves:[]}); +function mockAdapter(opts:{logLoads?:boolean}={}){ + const written:number[]=[];let restores=0,instance=''; + const model={key:'mock',display_name:'Mock',format:'gguf',type:'llm',size_bytes:100,quantization:null,max_context_length:8192,loaded_instances:[], + nativeMtp:{supported:true,kind:'sidecar',reason:'test fixture',resource:'pub/repo/main.gguf',draftResource:'repo/MTP/head.gguf',draftPath:'C:/models/MTP/head.gguf'}} as unknown as Model; + let tokens=0,on=false; + const adapter:Adapter={ + models:async()=>[{...model,loaded_instances:instance?[{id:instance,config:{context_length:2048,parallel:1,speculative_draft_mtp:false,...(on?{speculative_draft_max_tokens:tokens}:{})}}]:[]}], + cli:async(_s,args)=>{instance=args[args.indexOf('--identifier')+1];return 'loaded';}, + api:async()=>{instance='';return {};}, + infer:async()=>({output:'84',reasoning:'',rawStats:{},metrics:metrics({input_tokens:10,total_output_tokens:20,tokens_per_second:40},1000,0,10,20),status:'completed' as const,possibleTruncation:false}), + sidecar:(_m,count)=>{written.push(count);tokens=count;on=true;return ()=>{restores++;};}, + watchLog:()=>({root:'',sizes:new Map()}), + draftModels:()=>on&&opts.logLoads!==false?['c:/models/mtp/head.gguf']:[], + draftAcceptance:async()=>({tasks:1,accepted:6,generated:10,acceptance:.6,meanLen:2}), + }; + return {adapter,written,state:()=>({restores})}; +} +const runIt=async(run:Run,adapter:Adapter)=>{ + const events:EngineEvent[]=[]; + await runEngine(run,defaultSettings,new AbortController().signal,e=>{events.push(e);if(e.type==='sample')run.samples.push(e.sample);},adapter); + return {events,logs:events.filter(e=>e.type==='log').map(e=>(e as Extract).message), + measured:events.filter(e=>e.type==='sample'&&!(e as Extract).sample.warmup).length}; +}; +test('a sweep over separate heads writes one setting per depth and restores every one',async()=>{ + const mock=mockAdapter(); + const result=await runIt(makeRun({mtpSweep:[0,1,2,3,4,5]}),mock.adapter); + // Depth 0 is the MTP-off baseline: it is loaded plainly, so no setting is written for it. + assert.deepEqual(mock.written,[1,2,3,4,5]); + assert.equal(mock.state().restores,5,'every written setting is taken back out again'); + assert.equal(result.measured,6,'all six depths measured'); + assert.ok(result.logs.some(m=>m.includes('repo/MTP/head.gguf')),'the run log names the head that was loaded'); +}); +test('nothing is measured at a depth whose head LM Studio never reported loading',async()=>{ + const mock=mockAdapter({logLoads:false}); + const result=await runIt(makeRun({mtpSweep:[2]}),mock.adapter); + assert.equal(result.measured,0); + assert.equal(mock.state().restores,1,'the setting is restored even when the load is rejected'); + assert.ok(result.logs.some(m=>/did not report loading the MTP head/.test(m))); +}); + +test('accepted-draft figures are read from the engine log and pooled over the wave',async()=>{ + const home=mkdtempSync(path.join(tmpdir(),'lmb-accept-'));try{ + const dir=path.join(home,'.lmstudio','server-logs','2026-09');mkdirSync(dir,{recursive:true}); + const file=path.join(dir,'2026-09-15.1.log'); + const line=(a:number,g:number,len:number)=>`I slot print_timing: id 0 | task 0 | draft acceptance = 0.5 ( ${a} accepted / ${g} generated), mean len = ${len}\n`; + writeFileSync(file,line(999,999,9)); + const watch=watchEngineLog(home); + assert.equal(draftAcceptance(watch),null,'a rate from before the wave is not the wave’s rate'); + appendFileSync(file,line(53,111,2.43)+line(27,49,1.5)); + const found=draftAcceptance(watch)!; + assert.equal(found.tasks,2); + assert.equal(found.accepted,80); + assert.equal(found.generated,160); + // Pooled over drafted tokens, so a long request weighs more than a short one: 80/160, not + // the average of 53/111 and 27/49. + assert.equal(found.acceptance,0.5); + assert.equal(found.meanLen,(2.43+1.5)/2); + // Waiting stops as soon as every request in the wave has reported. + const began=Date.now(); + assert.equal((await settledDraftAcceptance(watch,2,5000))!.tasks,2); + assert.ok(Date.now()-began<1000,'a complete wave is not waited on'); + // A request that never reports back gives up and says how many it did find. + assert.equal((await settledDraftAcceptance(watch,3,300))!.tasks,2); + assert.equal(await settledDraftAcceptance(watchEngineLog(home),0,5000),null,'nothing to wait for when MTP is off'); + }finally{rmSync(home,{recursive:true,force:true});} +}); diff --git a/tests/mtp-sweep.test.ts b/tests/mtp-sweep.test.ts new file mode 100644 index 0000000..e16277f --- /dev/null +++ b/tests/mtp-sweep.test.ts @@ -0,0 +1,317 @@ +import {test} from 'node:test'; +import assert from 'node:assert/strict'; +import {runEngine,type Adapter,type EngineEvent} from '../electron/engine'; +import {exportText,summaries} from '../electron/export'; +import {historyRows} from '../electron/history'; +import {assertMtpPlan} from '../electron/mtp'; +import {metrics} from '../electron/metrics'; +import {validateConfig} from '../electron/validation'; +import {benchmarkRows} from '../src/benchmarks'; +import {defaultConfig,defaultSettings,starterTests} from '../src/defaults'; +import {measuredDepths,mtpDepthLabel,mtpDepthText,normalizeSweep,onSteps,preflightStep,sweepRows,sweepSteps,sweepVerdicts} from '../src/mtp-sweep'; +import type {Model,Run,RunConfig,Sample} from '../src/types'; + +const textTest=starterTests.find(t=>t.id==='reasoning')!; +const makeRun=(config:Partial):Run=>({id:'sweep-run',created:'2026-09-14T00:00:00.000Z',updated:'2026-09-14T00:00:00.000Z',status:'running', + config:{...structuredClone(defaultConfig),modelKeys:['mock'],testIds:['reasoning'],mode:'quality',concurrency:[1],waves:2,maxTokens:128,contextLength:2048,judgeModel:'',mtp:'on',...config}, + tests:[textTest],modelInfo:{},environment:{},logs:[],samples:[],waves:[]}); + +// Mirrors the real adapter closely enough to exercise what a sweep depends on: the depth is +// carried on the load command line, and the loaded instance reports back what it applied, which +// is the only thing the engine will accept as confirmation. +function mockAdapter(opts:{supported?:boolean|null;rates?:Record;misreportAt?:number;failMtp?:boolean}={}){ + const loads:{args:string[];depth:number}[]=[];const counters=new Map(); + let instance='',instanceConfig:Record={},depth=0,parallel=1,context=2048; + const model={key:'mock',display_name:'Mock',format:'gguf',type:'llm',size_bytes:100,quantization:null,max_context_length:8192,loaded_instances:[], + nativeMtp:{supported:opts.supported===undefined?true:opts.supported,reason:'test fixture'}} as unknown as Model; + const adapter:Adapter={ + models:async()=>[{...model,loaded_instances:instance?[{id:instance,config:{context_length:context,parallel,...instanceConfig}}]:[]}], + cli:async(_s,args)=>{ + // Stands in for a runtime that can load the model but not its prediction heads. + if(opts.failMtp&&args.includes('--speculative-draft-mtp'))throw Error('Failed to load model. Cause: failed to load draft model: invalid vector subscript'); + instance=args[args.indexOf('--identifier')+1]; + parallel=Number(args[args.indexOf('--parallel')+1]);context=Number(args[args.indexOf('--context-length')+1]); + const on=args.includes('--speculative-draft-mtp'); + const asked=args.includes('--speculative-draft-max-tokens')?Number(args[args.indexOf('--speculative-draft-max-tokens')+1]):null; + depth=on?(asked??0):0;loads.push({args:[...args],depth}); + instanceConfig=on?{speculative_draft_mtp:true,speculative_draft_max_tokens:opts.misreportAt===depth?depth+1:asked}:{speculative_draft_mtp:false}; + return 'loaded'; + }, + api:async()=>{instance='';instanceConfig={};return {};}, + // Stands in for llama.cpp's own per-request acceptance line: a fixed rate per depth, so a + // row can be checked for carrying the right one rather than for any particular number. + watchLog:()=>({root:'',sizes:new Map()}), + draftAcceptance:async()=>depth>0?{tasks:1,accepted:depth*10,generated:100,acceptance:depth/10,meanLen:depth}:null, + infer:async(_s,_id,prompt)=>{ + const warm=prompt==='Reply with the word ready.'; + const series=opts.rates?.[depth]??[40]; + const index=counters.get(depth)??0;if(!warm)counters.set(depth,index+1); + return {output:'84',reasoning:'',rawStats:{},metrics:metrics({input_tokens:10,total_output_tokens:20,tokens_per_second:warm?10:series[index%series.length]},1000,0,10,20), + status:'completed' as const,possibleTruncation:false}; + } + }; + return {adapter,loads,model}; +} +const runIt=async(run:Run,mock:ReturnType,retries?:Sample[])=>{ + const events:EngineEvent[]=[]; + await runEngine(run,defaultSettings,new AbortController().signal,e=>{ + events.push(e); + if(e.type==='model')run.modelInfo[e.key]=e.info; + if(e.type==='sample')run.samples.push(e.sample); + if(e.type==='wave')run.waves.push(e.wave); + },mock.adapter,retries); + const finish=events.findLast(e=>e.type==='finish') as Extract; + return {events,finish,logs:events.filter(e=>e.type==='log').map(e=>(e as Extract).message), + total:(events.filter(e=>e.type==='progress').at(-1) as Extract|undefined)?.progress.total??0}; +}; + +test('a sweep is an ordered, de-duplicated list of depths, and no sweep is still one step',()=>{ + assert.deepEqual(normalizeSweep([4,2,2,0]),[0,2,4]); + assert.deepEqual(normalizeSweep([1.5,-1,9,'2',null,NaN]),[]); + assert.deepEqual(normalizeSweep(undefined),[]); + // Without a sweep the single step keeps a null depth, so runs saved before sweeping existed + // are never relabelled as though they had been measured at some particular depth. + assert.deepEqual(sweepSteps({}),[{depth:null,mode:undefined,tokens:2,label:'Legacy/default'}]); + assert.deepEqual(sweepSteps({mtp:'on',mtpDraftTokens:3}),[{depth:null,mode:'on',tokens:3,label:'Legacy/default'}]); + // A sweep needs MTP on: the switch is what asserts every selected model can do it at all. + assert.equal(sweepSteps({mtp:'off',mtpSweep:[0,2]}).length,1); + assert.deepEqual(sweepSteps({mtp:'on',mtpSweep:[2,0]}),[ + {depth:0,mode:'off',tokens:2,label:'MTP off'},{depth:2,mode:'on',tokens:2,label:'2 tokens'}]); + assert.equal(mtpDepthLabel(1),'1 token'); + assert.equal(mtpDepthLabel(0),'MTP off'); + assert.equal(mtpDepthLabel(null),'Legacy/default'); + assert.deepEqual([0,1,2,null].map(d=>mtpDepthText(d)),['MTP off','MTP 1 token','MTP 2 tokens','MTP legacy/default']); +}); + +test('sweep depths are validated and normalized before a run is accepted',()=>{ + const base={...defaultConfig,modelKeys:['m'],mtp:'on' as const}; + const config={...base,mtpSweep:[4,0,0,2]};validateConfig(config); + assert.deepEqual(config.mtpSweep,[0,2,4]); + assert.throws(()=>validateConfig({...base,mtp:'off',mtpSweep:[0,2]}),/Turn native MTP on/); + assert.throws(()=>validateConfig({...base,mtpSweep:[9]}),/MTP sweep depth/); + assert.throws(()=>validateConfig({...base,mtpSweep:[1.5]}),/MTP sweep depth/); + assert.throws(()=>validateConfig({...base,mtpSweep:[]}),/at least one MTP depth/); + assert.throws(()=>validateConfig({...base,mtpSweep:'0,2' as unknown as number[]}),/list of whole numbers/); +}); + +test('each depth is loaded separately, with the flags that depth means, and tags its own responses',async()=>{ + const run=makeRun({mtpSweep:[0,1,3]});const mock=mockAdapter(); + const result=await runIt(run,mock); + assert.equal(result.finish.status,'completed'); + assert.equal(mock.loads.length,3,'one load per depth'); + assert.ok(mock.loads[0].args.includes('--no-speculative-draft-mtp'),'depth 0 loads with MTP off'); + assert.ok(!mock.loads[0].args.includes('--speculative-draft-mtp')); + for(const [i,depth] of [1,3].entries()){ + const args=mock.loads[i+1].args; + assert.ok(args.includes('--speculative-draft-mtp')); + assert.equal(args[args.indexOf('--speculative-draft-max-tokens')+1],String(depth)); + } + const measured=run.samples.filter(s=>!s.warmup); + assert.deepEqual(measuredDepths(run),[0,1,3]); + assert.equal(measured.length,6,'every depth runs the whole workload'); + for(const depth of [0,1,3])assert.equal(measured.filter(s=>s.mtpTokens===depth).length,2); + // The count the progress bar is measured against has to include the extra passes, or a sweep + // would report itself as finished three times over. + assert.equal(result.total,6); + assert.deepEqual([...new Set(run.waves.map(w=>w.mtpTokens))].sort(),[0,1,3]); + assert.ok(result.logs.some(m=>m.includes('sweep step 1 of 3'))); +}); + +test('a sweep is refused for a model whose native MTP support is not confirmed',async()=>{ + const config={...defaultConfig,modelKeys:['mock'],mtp:'on' as const,mtpSweep:[0,2]}; + // Refused before anything is loaded, so an unsupported model cannot cost an hour of loading + // only to fail on a later depth. + assert.throws(()=>assertMtpPlan(mockAdapter({supported:false}).model,config),/not confirmed/); + assert.throws(()=>assertMtpPlan(mockAdapter({supported:null}).model,config),/not confirmed/); + // Depth 0 alone asks nothing of the model that is not already true of any GGUF. + assert.doesNotThrow(()=>assertMtpPlan(mockAdapter({supported:false}).model,{...config,mtpSweep:[0]})); + const run=makeRun({mtpSweep:[0,2]});const mock=mockAdapter({supported:false}); + const result=await runIt(run,mock); + assert.equal(result.finish.status,'failed'); + assert.equal(run.samples.filter(s=>!s.warmup&&s.mtpTokens===2).length,0,'no depth 2 measurements exist'); + assert.equal(run.samples.filter(s=>!s.warmup&&s.mtpTokens===0).length,2,'the baseline still measured'); +}); + +test('a depth LM Studio did not apply produces no measurements at that depth',async()=>{ + const run=makeRun({mtpSweep:[0,2,4]});const mock=mockAdapter({misreportAt:2}); + const result=await runIt(run,mock); + assert.equal(result.finish.status,'failed'); + assert.deepEqual(measuredDepths(run),[0,4],'the misreported depth saved nothing'); + assert.ok(result.logs.some(m=>m.includes('MTP 2 tokens')&&m.includes('draft-token count'))); +}); + +test('depths are never pooled together by summaries, history, or benchmark scoring',async()=>{ + const run=makeRun({mtpSweep:[0,2]}); + await runIt(run,mockAdapter({rates:{0:[30],2:[60]}})); + const rows=summaries(run); + assert.equal(rows.length,2,'one row per depth, not one row averaging both'); + assert.deepEqual(rows.map(r=>r.mtpDepth).sort(),[0,2]); + assert.deepEqual(rows.map(r=>[r.mtp,r.mtpDraftTokens]).sort(),[['off',null],['on',2]]); + assert.deepEqual(rows.map(r=>r.generationTps).sort((a,b)=>a!-b!),[30,60]); + // The history overview matches its pooled rows back to the responses behind them by the same + // key, so a depth that did not match would silently report zero completed requests. + const history=historyRows([run],{}); + assert.equal(history.length,2); + for(const row of history)assert.equal(row.completed,2); + const packed={...run,tests:run.tests.map(t=>({...t,benchmark:{packId:'p',itemId:'i',datasetHash:'h',protocol:'x'}}))} as Run; + const cards=benchmarkRows(packed); + assert.equal(cards.length,2); + assert.deepEqual(cards.map(c=>c.mtp),['MTP off','MTP 2 tokens']); + for(const card of cards)assert.equal(card.expected,2,'each depth expects its own pass, not both'); +}); + +test('the fastest depth is named, and a gap inside the measurement spread is not called a win',async()=>{ + const clear=makeRun({mtpSweep:[0,2]}); + await runIt(clear,mockAdapter({rates:{0:[30,30],2:[60,60]}})); + const [won]=sweepVerdicts(clear); + assert.equal(won.best?.depth,2); + assert.equal(won.baseline?.depth,0); + assert.equal(Math.round(won.gainPercent!),100); + assert.equal(won.withinNoise,false); + assert.match(won.summary,/2 tokens was fastest/); + + const noisy=makeRun({mtpSweep:[0,2]}); + await runIt(noisy,mockAdapter({rates:{0:[30,50],2:[32,52]}})); + const [unclear]=sweepVerdicts(noisy); + assert.equal(unclear.best?.depth,2); + assert.equal(unclear.withinNoise,true,'a 2 tok/s gap over a 20 tok/s spread is not a result'); + assert.match(unclear.summary,/spread of the measurements/); + + const flat=makeRun({mtpSweep:[0,2]}); + await runIt(flat,mockAdapter({rates:{0:[40,40],2:[30,30]}})); + const [clearBaseline]=sweepVerdicts(flat); + assert.equal(clearBaseline.withinNoise,false); + assert.match(clearBaseline.summary,/Nothing beat MTP off: it was the fastest/); + + // The baseline coming out marginally ahead is not a verdict on MTP, and must not read as one. + const deadHeat=makeRun({mtpSweep:[0,2]}); + await runIt(deadHeat,mockAdapter({rates:{0:[40,60],2:[38,58]}})); + const [tied]=sweepVerdicts(deadHeat); + assert.equal(tied.best?.depth,0); + assert.equal(tied.withinNoise,true); + assert.match(tied.summary,/does not separate them/); + + const rows=sweepRows(clear); + assert.equal(rows.length,2); + assert.deepEqual(rows.map(r=>[r.label,r.requests,r.completed,r.failures]),[['MTP off',2,2,0],['2 tokens',2,2,0]]); + assert.equal(rows[0].generationSd,0); +}); + +test('a retry reloads only the depths that actually have something to retry',async()=>{ + const run=makeRun({mtpSweep:[0,2,4]}); + const failed=(depth:number):Sample=>({mtpTokens:depth,id:`s${depth}`,runId:run.id,modelKey:'mock',modelName:'Mock',testId:textTest.id,testName:textTest.name, + concurrency:1,waveId:`w${depth}`,wave:0,slot:0,warmup:false,prompt:'Six identical machines…',output:'',reasoning:'',status:'timeout', + metrics:metrics({},0,null,null,null),objective:{score:null,checks:[]},grades:[],rawStats:{},created:run.created,possibleTruncation:false}); + const mock=mockAdapter(); + const result=await runIt(run,mock,[failed(0),failed(4)]); + assert.equal(result.finish.status,'completed'); + assert.deepEqual(mock.loads.map(l=>l.depth),[0,4],'depth 2 had nothing failed and was never loaded'); + assert.deepEqual(measuredDepths(run),[0,4]); + assert.equal(result.total,2,'a retry counts the requests being retried, not a whole sweep'); +}); + +test('exports describe the sweep, and a run without one gains nothing to explain',async()=>{ + const swept=makeRun({mtpSweep:[0,2]}); + await runIt(swept,mockAdapter({rates:{0:[30,30],2:[60,60]}})); + const report=exportText(swept,'md'); + assert.match(report,/## Native MTP sweep/); + assert.match(report,/swept depths: MTP off, 2 tokens/); + assert.match(report,/\/ MTP 2 tokens/,'each measurement says which depth produced it'); + assert.match(report,/2 tokens was fastest/); + const csv=exportText(swept,'csv'); + assert.match(csv.split('\r\n')[0],/"mtpDepth"/); + + const plain=makeRun({mtp:'off',mtpSweep:undefined}); + await runIt(plain,mockAdapter()); + assert.equal(plain.samples.every(s=>s.mtpTokens===undefined),true,'no depth is invented for an ordinary run'); + assert.doesNotMatch(exportText(plain,'md'),/Native MTP sweep/); + assert.equal(summaries(plain).length,1); + assert.equal(summaries(plain)[0].mtpDepth,null); +}); + +test('a sweep under load keeps every concurrency level apart instead of pooling them',async()=>{ + const run=makeRun({mtpSweep:[0,1,2],concurrency:[1,2],waves:1,mode:'performance',testIds:[]}); + const mock=mockAdapter(); + await runIt(run,mock); + const rows=sweepRows(run); + // Three depths at two concurrency levels is six rows for one model, not three. + assert.equal(rows.length,6); + assert.deepEqual([...new Set(rows.map(r=>r.concurrency))].sort(),[1,2]); + for(const level of [1,2])assert.deepEqual(rows.filter(r=>r.concurrency===level).map(r=>r.depth),[0,1,2]); + // Each row counts only its own level's requests: one wave of one, and one wave of two. + assert.deepEqual(rows.filter(r=>r.concurrency===1).map(r=>r.requests),[1,1,1]); + assert.deepEqual(rows.filter(r=>r.concurrency===2).map(r=>r.requests),[2,2,2]); + // A verdict compares depths measured under the same load, never across two loads. + const verdicts=sweepVerdicts(run); + assert.equal(verdicts.length,2); + assert.deepEqual(verdicts.map(v=>v.concurrency).sort(),[1,2]); + for(const v of verdicts)assert.deepEqual(v.rows.map(r=>r.concurrency),[v.concurrency,v.concurrency,v.concurrency]); +}); +test('accepted-draft figures ride with the depth that produced them, and MTP off reports none',async()=>{ + const run=makeRun({mtpSweep:[0,2,4],concurrency:[1],waves:2}); + await runIt(run,mockAdapter()); + const byDepth=Object.fromEntries(sweepRows(run).map(r=>[r.depth,r.draft])); + assert.equal(byDepth[0],null,'nothing is drafted with MTP off, which is not a rate of zero'); + assert.equal(byDepth[2]!.acceptance,0.2); + assert.equal(byDepth[4]!.acceptance,0.4); + // Two waves at that depth, pooled: totals add up rather than the rates being averaged. + assert.equal(byDepth[4]!.accepted,80); + assert.equal(byDepth[4]!.generated,200); + assert.equal(byDepth[4]!.tasks,2); + assert.match(exportText(run,'md'),/drafted tokens accepted 40\.0% \(80 of 200 drafted/); + assert.match(exportText(run,'md'),/drafted tokens accepted not applicable/); + assert.ok(exportText(run,'csv').includes('draftAcceptance'),'the comparison table keeps a column for it'); +}); + +test('a sweep step is named once, so the baseline is never announced as “MTP MTP off”',async()=>{ + const run=makeRun({mtpSweep:[0,2],concurrency:[1],waves:1}); + const {events}=await runIt(run,mockAdapter()); + const said=events.filter(e=>e.type==='progress').map(e=>(e as Extract).progress.message); + assert.ok(said.some(m=>m.includes('MTP off')),'the baseline step is named in progress'); + assert.ok(said.some(m=>m.includes('MTP 2 tokens')),'and so is every other depth'); + assert.equal(said.filter(m=>m.includes('MTP MTP')).length,0); +}); + +test('a preflight answers the MTP question with one load, before anything is measured',()=>{ + assert.equal(preflightStep(sweepSteps({mtp:'on',mtpSweep:[0,1,2,3]}))?.depth,1,'the shallowest on-depth stands for the rest'); + assert.equal(preflightStep(sweepSteps({mtp:'on',mtpSweep:[0]})),null,'a sweep with nothing to draft has nothing to check'); + assert.equal(preflightStep(sweepSteps({mtp:'off'})),null); + assert.deepEqual(onSteps(sweepSteps({mtp:'on',mtpSweep:[0,2,4]})).map(s=>s.depth),[2,4]); +}); +test('a model that cannot load its MTP head loses only its MTP depths, and is named',async()=>{ + const run=makeRun({mtpSweep:[0,1,2],mtpPreflight:true}),mock=mockAdapter({failMtp:true}); + const {finish,logs,events}=await runIt(run,mock); + const measured=events.filter(e=>e.type==='sample'&&!(e as Extract).sample.warmup) + .map(e=>(e as Extract).sample.mtpTokens); + assert.deepEqual([...new Set(measured)],[0],'the MTP-off baseline is still measured'); + assert.ok(measured.length>0,'skipping a depth must not skip the whole run'); + // One model that cannot draft at all is one failure, not one for each depth it would have run. + assert.match(finish.error!,/^1 request\/model failure/); + assert.match(finish.error!,/mock · MTP 1 token, 2 tokens \(the MTP head would not load\)/); + assert.ok(logs.some(l=>/MTP preflight: mock cannot load its MTP head/.test(l)),'the reason is logged where it happened'); + assert.equal(mock.loads.length,1,'only the baseline was ever loaded'); + assert.equal(mock.loads.filter(l=>l.depth>0).length,0,'no drafting load succeeded'); + // The point of the option, stated as a comparison: without it the same run discovers the same + // impossibility once per depth, and only after it has measured the baseline each time. + const late=await runIt(makeRun({mtpSweep:[0,1,2]}),mockAdapter({failMtp:true})); + assert.match(late.finish.error!,/^2 request\/model failure/,'unchecked, each depth fails separately'); + assert.ok(!Object.keys(run.modelInfo).some(k=>/MTP 1 token|MTP 2 tokens/.test(k)),'a preflight leaves no record of a model behind'); +}); +test('a preflight that passes costs one load and changes nothing that is measured',async()=>{ + const checked=mockAdapter(),plain=mockAdapter(); + const a=await runIt(makeRun({mtpSweep:[0,1],mtpPreflight:true}),checked); + const b=await runIt(makeRun({mtpSweep:[0,1]}),plain); + assert.equal(a.finish.status,'completed');assert.equal(b.finish.status,'completed'); + const samples=(r:typeof a)=>r.events.filter(e=>e.type==='sample').length; + assert.equal(samples(a),samples(b),'the same measurements either way'); + assert.equal(checked.loads.length,plain.loads.length+1,'the check is exactly one extra load'); +}); +test('the check is off unless asked for, and a retry never pays for it',async()=>{ + const off=mockAdapter();await runIt(makeRun({mtpSweep:[0,1]}),off); + assert.equal(off.loads.length,2,'two depths, two loads, no probe'); + const retried=mockAdapter({failMtp:true}); + const run=makeRun({mtpSweep:[0,1],mtpPreflight:true}); + const sample={id:'s',runId:run.id,modelKey:'mock',testId:'reasoning',waveId:'w',wave:0,mtpTokens:0} as unknown as Sample; + const {finish}=await runIt(run,retried,[sample]); + assert.ok(!/would not load/.test(finish.error??''),'a retry re-runs saved requests rather than re-checking the model'); +}); diff --git a/tests/prefill.test.ts b/tests/prefill.test.ts new file mode 100644 index 0000000..8fd8c77 --- /dev/null +++ b/tests/prefill.test.ts @@ -0,0 +1,97 @@ +import {test} from 'node:test'; +import assert from 'node:assert/strict'; +import {calibratePrefill,calibrationPrompt,calibrationRepeats,minDeltaMs,minTokenRatio,prefillText} from '../src/prefill'; +import {parseRuntimes,runtimeChoices,runtimeLabel} from '../electron/runtime'; +import {cacheScale,cacheScalePair,cacheQuantText,needsFlashAttention} from '../src/cache-quant'; + +// The case the calibration exists for: a fixed per-request cost that a single measurement charges +// to the GPU. 150 ms of overhead plus a true 1500 tok/s means a 150-token prompt is timed at +// 250 ms, which reads as 600 tok/s — under half the real rate — while a 3000-token prompt is +// timed at 2150 ms and reads as 1395. The slope recovers 1500 from either pair. +test('two points recover the real rate and name the fixed cost',()=>{ + const truth=1500,overhead=150; + const at=(tokens:number)=>({tokens,ms:overhead+tokens/truth*1000}); + const c=calibratePrefill(at(150),at(3000)); + assert.ok(Math.abs(c.marginalTps!-truth)<1,`recovered ${c.marginalTps}`); + assert.ok(Math.abs(c.overheadMs!-overhead)<1,`overhead ${c.overheadMs}`); + // And the figure the old method would have reported for the same short request, kept beside it + // so the two can be compared rather than quietly swapped. + assert.ok(c.naiveTps!{ + const ok={tokens:3000,ms:2150}; + // A long prompt that is not long enough: the difference would be mostly noise. + const tooClose=calibratePrefill({tokens:1000,ms:800},{tokens:1000*minTokenRatio-1,ms:3000}); + assert.equal(tooClose.marginalTps,null); + assert.match(tooClose.note,/not at least/); + // Two timings a few milliseconds apart, which a cache hit on the second request would produce. + const tooFast=calibratePrefill({tokens:100,ms:500},{tokens:3000,ms:500+minDeltaMs-1}); + assert.equal(tooFast.marginalTps,null); + assert.match(tooFast.note,/timing noise/); + // A request that returned no timing at all is not calibrated rather than calibrated as zero. + assert.equal(calibratePrefill({tokens:0,ms:0},ok).marginalTps,null); + for(const c of [tooClose,tooFast])assert.equal(prefillText(c),c.note); + assert.equal(prefillText(null),'Not measured'); +}); + +// A long prompt cheaper per token than the line implies (a batch boundary, or partial cache reuse) +// would make the derived fixed cost negative, which is not a cost anyone can act on. +test('a negative fixed cost is reported as none rather than as a negative',()=>{ + const c=calibratePrefill({tokens:100,ms:10},{tokens:4000,ms:1000}); + assert.ok(c.marginalTps!>0); + assert.ok(c.overheadMs!>=0,`overhead was ${c.overheadMs}`); +}); + +test('the calibration prompts differ in length by enough to be calibrated',()=>{ + const small=calibrationPrompt(calibrationRepeats.small),big=calibrationPrompt(calibrationRepeats.big); + assert.ok(big.length>small.length*minTokenRatio,'the long prompt clears the ratio the calibration requires'); + // It is never scored and never read back, so it must not look like an instruction worth obeying + // at length: a model that wrote an essay here would time the essay, not the prompt. + assert.match(small,/reply with the single word/i); +}); + +test('runtime listings are parsed, labelled, and disambiguated',()=>{ + const listing=[ + 'LLM ENGINE SELECTED MODEL FORMAT', + 'llama.cpp-win-x86_64-amd-rocm-avx2@2.40.0 GGUF', + 'llama.cpp-win-x86_64-vulkan-avx2@2.40.0 ✓ GGUF', + 'llama.cpp-win-x86_64-vulkan-avx2@2.38.0 GGUF', + ].join('\n'); + const parsed=parseRuntimes(listing); + assert.equal(parsed.length,3); + assert.deepEqual(parsed.map(r=>r.selected),[false,true,false],'only the ticked row is selected'); + assert.equal(parsed[0].engine,'llama.cpp-win-x86_64-amd-rocm-avx2'); + assert.equal(parsed[0].version,'2.40.0'); + assert.equal(parsed.every(r=>r.format==='GGUF'),true); + assert.deepEqual([runtimeLabel(parsed[0].engine),runtimeLabel(parsed[1].engine)],['ROCm','Vulkan']); + // Two Vulkan builds cannot both be called "Vulkan" in a dropdown, so those carry their version. + const choices=runtimeChoices(parsed); + assert.equal(choices[0].label,'ROCm'); + assert.deepEqual(choices.slice(1).map(c=>c.label),['Vulkan 2.40.0','Vulkan 2.38.0']); +}); + +test('colour codes around the tick do not hide which engine is selected',()=>{ + const coloured=`LLM ENGINE SELECTED\nllama.cpp-win-x86_64-vulkan-avx2@2.40.0 ✓ GGUF`; + const [only]=parseRuntimes(coloured); + assert.equal(only.engine,'llama.cpp-win-x86_64-vulkan-avx2'); + assert.equal(only.selected,true); +}); + +test('cache quantization scales memory and is described honestly',()=>{ + assert.equal(cacheScale('off'),1,'off is LM Studio’s own default, which for these models is f16'); + assert.equal(cacheScale('f16'),1); + assert.ok(cacheScale('q8_0')<0.6&&cacheScale('q8_0')>0.5,'q8_0 is about half, not exactly half: it carries a scale per block'); + assert.ok(cacheScale('q4_0')<0.3); + // K and V are set separately because they do not degrade alike, so the combined figure is the + // mean of the two halves rather than either one. + assert.equal(cacheScalePair('f16','q4_0'),(cacheScale('f16')+cacheScale('q4_0'))/2); + assert.equal(cacheQuantText('off','off'),'Off'); + assert.equal(cacheQuantText('q8_0','q8_0'),'q8_0 K and V'); + assert.equal(cacheQuantText('q8_0','q4_0'),'q8_0 K · q4_0 V'); + // llama.cpp will not use a quantized cache without flash attention, so a config that asks for + // one has to know it depends on it. + assert.equal(needsFlashAttention('off','off'),false); + assert.equal(needsFlashAttention('f16','f16'),false); + assert.equal(needsFlashAttention('off','q4_0'),true); +}); diff --git a/tests/text-only.test.ts b/tests/text-only.test.ts new file mode 100644 index 0000000..23b12f3 --- /dev/null +++ b/tests/text-only.test.ts @@ -0,0 +1,132 @@ +import {test} from 'node:test'; +import assert from 'node:assert/strict'; +import {existsSync,mkdirSync,mkdtempSync,readFileSync,readdirSync,rmSync,statSync,writeFileSync} from 'node:fs'; +import {tmpdir} from 'node:os'; +import path from 'node:path'; +import {MARKER,TWIN_SUFFIX,awaitIndexed,createTextOnly,planTextOnly,removeTextOnly,textOnlyTwins} from '../electron/text-only'; +import type {Model} from '../src/types'; + +const slash=(p:string)=>p.replaceAll('\\','/'); +// A vision-capable model laid out the way LM Studio lays one out: weights, a projector beside +// them, and an MTP head in its own folder underneath. +function library(){ + const home=mkdtempSync(path.join(tmpdir(),'lmb-textonly-')); + const dir=path.join(home,'.lmstudio','models','pub','repo-GGUF'); + mkdirSync(path.join(dir,'MTP'),{recursive:true}); + const main=path.join(dir,'model-IQ3.gguf'),head=path.join(dir,'MTP','mtp-model.gguf'),proj=path.join(dir,'mmproj-F16.gguf'); + writeFileSync(main,'WEIGHTS');writeFileSync(head,'HEAD');writeFileSync(proj,'PROJECTOR'); + const internal=path.join(home,'.lmstudio','.internal');mkdirSync(internal,{recursive:true}); + writeFileSync(path.join(internal,'model-index-cache.json'),JSON.stringify({models:[ + {domain:'llm',format:'gguf',defaultIdentifier:'model@iq3',sizeBytes:7,indexedModelIdentifier:'pub/repo-GGUF/model-IQ3.gguf',entryPoint:{absPath:slash(main)}}, + ]})); + const model:Model={key:'model@iq3',display_name:'Model',format:'gguf',size_bytes:7,max_context_length:8192,type:'llm',loaded_instances:[],quantization:{name:'IQ3'}, + capabilities:{vision:true},nativeMtp:{supported:true,kind:'sidecar',reason:'',resource:'pub/repo-GGUF/model-IQ3.gguf',draftResource:'repo-GGUF/MTP/mtp-model.gguf',draftPath:slash(head)}}; + return {home,dir,main,head,proj,model,twin:dir+TWIN_SUFFIX,clean:()=>rmSync(home,{recursive:true,force:true})}; +} + +test('a text-only copy takes the weights and any MTP head, and leaves the projector behind',()=>{ + const lib=library();try{ + const plan=planTextOnly(lib.model,lib.home); + assert.equal(slash(plan.folder),slash(lib.twin)); + assert.deepEqual(plan.files.map(f=>slash(path.relative(plan.folder,f.to))),['model-IQ3.gguf','MTP/mtp-model.gguf']); + assert.deepEqual(plan.projectors,['mmproj-F16.gguf']); + assert.equal(plan.headIncluded,true,'a copy without the head would silently lose native MTP'); + assert.ok(plan.files.every(f=>!/mmproj/i.test(f.to))); + }finally{lib.clean();} +}); +test('a model with no projector, and a copy of a copy, are refused rather than made pointlessly',()=>{ + const lib=library();try{ + assert.throws(()=>planTextOnly({...lib.model,capabilities:{vision:false}},lib.home),/already loads text-only/); + rmSync(lib.proj); + assert.throws(()=>planTextOnly(lib.model,lib.home),/No projector file was found/); + }finally{lib.clean();} +}); +test('the weights are linked, never copied, so a copy costs no disk space',()=>{ + const lib=library();try{ + const folder=createTextOnly(planTextOnly(lib.model,lib.home)); + const linked=path.join(folder,'model-IQ3.gguf'),head=path.join(folder,'MTP','mtp-model.gguf'); + assert.equal(statSync(linked).ino,statSync(lib.main).ino,'the copy is the same bytes on disk, not a second set'); + assert.equal(statSync(head).ino,statSync(lib.head).ino); + assert.ok(statSync(linked).nlink>=2); + assert.equal(existsSync(path.join(folder,'mmproj-F16.gguf')),false,'the projector is what a text-only copy leaves out'); + const marker=JSON.parse(readFileSync(path.join(folder,MARKER),'utf8')); + assert.equal(marker.createdBy,'local-model-bench'); + assert.equal(slash(marker.source),slash(lib.dir)); + }finally{lib.clean();} +}); +test('a copy that cannot be finished leaves no half-made model for LM Studio to index',()=>{ + const lib=library();try{ + const plan=planTextOnly(lib.model,lib.home); + // A folder that is already there is the signal that something else owns this name. + mkdirSync(plan.folder,{recursive:true}); + assert.throws(()=>createTextOnly(plan)); + const survivor=library();try{ + const other=planTextOnly(survivor.model,survivor.home); + rmSync(survivor.head); // The head disappears between planning and linking. + assert.throws(()=>createTextOnly(other)); + assert.equal(existsSync(other.folder),false,'the whole folder is taken back out'); + }finally{survivor.clean();} + }finally{lib.clean();} +}); +test('copies are found by their marker, never by a folder that merely looks named right',()=>{ + const lib=library();try{ + const folder=createTextOnly(planTextOnly(lib.model,lib.home)); + const twins=textOnlyTwins(lib.home); + assert.equal(twins.length,1); + assert.equal(slash(twins[0].folder),slash(folder)); + assert.equal(twins[0].sourceKey,'model@iq3'); + // Same name, no marker: not this app's, so not this app's to list or delete. + const impostor=path.join(lib.home,'.lmstudio','models','pub','other-GGUF'+TWIN_SUFFIX); + mkdirSync(impostor,{recursive:true});writeFileSync(path.join(impostor,'mine.gguf'),'MINE'); + assert.equal(textOnlyTwins(lib.home).length,1); + assert.throws(()=>removeTextOnly(impostor,lib.home),/not created by Local Model Bench/); + assert.equal(existsSync(path.join(impostor,'mine.gguf')),true); + }finally{lib.clean();} +}); +test('removing a copy drops links only, and refuses any file that exists nowhere else',()=>{ + const lib=library();try{ + const folder=createTextOnly(planTextOnly(lib.model,lib.home)); + // Something that only exists inside the copy stops the delete outright. + writeFileSync(path.join(folder,'notes.txt'),'do not lose me'); + assert.throws(()=>removeTextOnly(folder,lib.home),/exist nowhere else/); + assert.equal(existsSync(folder),true); + rmSync(path.join(folder,'notes.txt')); + removeTextOnly(folder,lib.home); + assert.equal(existsSync(folder),false); + assert.equal(readFileSync(lib.main,'utf8'),'WEIGHTS','the model itself is untouched'); + assert.equal(readFileSync(lib.head,'utf8'),'HEAD'); + assert.equal(readFileSync(lib.proj,'utf8'),'PROJECTOR'); + assert.deepEqual(readdirSync(lib.dir).sort(),['MTP','mmproj-F16.gguf','model-IQ3.gguf']); + }finally{lib.clean();} +}); + +test('a copy is not reported as made until LM Studio’s own index holds it',async()=>{ + const lib=library();try{ + const plan=planTextOnly(lib.model,lib.home); + const folder=createTextOnly(plan); + // LM Studio re-indexes a new folder on its own, in about a quarter of a second. Until it + // has, the copy exists but is not selectable, which is not the same as being ready. + assert.equal(await awaitIndexed(folder,lib.home,300,50),false); + const index=path.join(lib.home,'.lmstudio','.internal','model-index-cache.json'); + const doc=JSON.parse(readFileSync(index,'utf8')); + doc.models.push({domain:'llm',format:'gguf',defaultIdentifier:'model - text only',sizeBytes:7, + indexedModelIdentifier:'pub/repo-GGUF - Text only/model-IQ3.gguf',entryPoint:{absPath:slash(path.join(folder,'model-IQ3.gguf'))}}); + writeFileSync(index,JSON.stringify(doc)); + assert.equal(await awaitIndexed(folder,lib.home,2000,50),true); + }finally{lib.clean();} +}); + +test('a removed copy is not reported as gone until the library has dropped it too',async()=>{ + const lib=library();try{ + const folder=createTextOnly(planTextOnly(lib.model,lib.home)); + const index=path.join(lib.home,'.lmstudio','.internal','model-index-cache.json'); + const doc=JSON.parse(readFileSync(index,'utf8')); + const entry={domain:'llm',format:'gguf',defaultIdentifier:'model - text only',sizeBytes:7, + indexedModelIdentifier:'pub/repo-GGUF - Text only/model-IQ3.gguf',entryPoint:{absPath:slash(path.join(folder,'model-IQ3.gguf'))}}; + writeFileSync(index,JSON.stringify({models:[...doc.models,entry]})); + removeTextOnly(folder,lib.home); + assert.equal(await awaitIndexed(folder,lib.home,300,50,false),false,'the folder is gone but LM Studio still lists it'); + writeFileSync(index,JSON.stringify(doc)); + assert.equal(await awaitIndexed(folder,lib.home,2000,50,false),true); + }finally{lib.clean();} +}); diff --git a/vite.config.ts b/vite.config.ts index 8619e91..a9b1808 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -1,3 +1,7 @@ import { defineConfig } from 'vite'; import react from '@vitejs/plugin-react'; -export default defineConfig({plugins:[react()],base:'./',server:{host:'127.0.0.1'}}); +import { readFileSync } from 'node:fs'; +// The window prints the same version the installer is named after, taken from one place, so +// the two cannot disagree the way they did when the number was typed into the markup. +const { version } = JSON.parse(readFileSync('./package.json','utf8')); +export default defineConfig({plugins:[react()],base:'./',server:{host:'127.0.0.1'},define:{__APP_VERSION__:JSON.stringify(version)}}); \ No newline at end of file From bb0312aa90e066d3e6f6189e853368af360e9baf Mon Sep 17 00:00:00 2001 From: M007-Net <262222560+M007-Net@users.noreply.github.com> Date: Thu, 17 Sep 2026 03:25:25 -0400 Subject: [PATCH 2/8] Local Model Bench 1.10.1 Quantizing the context made every load fail. llama.cpp cannot use a quantized key/value cache without flash attention, and LM Studio refuses the load outright rather than falling back: Error: V Cache Quantization requires flash attention to be enabled. Please enable flash attention or set V Cache Quantization Type to f16. 1.10.0 wrote the cache fields and assumed flash attention was already on. That assumption does not hold: the default varies by engine and by whatever the model was last loaded with, and on this machine every run that asked for a quantized cache failed within two seconds of pressing Start, which reads as the button doing nothing. A run that asks for a quantized cache now writes the flag that makes it possible, into the same per-model configuration, for the same single load, restored with the rest. A run that leaves the cache alone does not touch the setting. A load that comes back reporting no flash attention is refused before anything is measured, rather than reporting a cache that was not in use. Verified against LM Studio on both engines: the exact case that failed, ROCm 2.40.0 with q4_0 for K and V, now loads with flashAttention, kCacheQuantizationType and vCacheQuantizationType written together and removed together. 236 offline tests, typecheck and production build all pass. Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 15 +++++++++++++++ README.md | 6 ++++-- electron/cache-quant.ts | 16 ++++++++++++++-- package-lock.json | 4 ++-- package.json | 2 +- src/main.tsx | 2 +- tests/cache-quant.test.ts | 31 +++++++++++++++++++++++++++++++ 7 files changed, 68 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c2ec46..c6248f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,21 @@ All notable changes to Local Model Bench are recorded here. The format follows Dates are the date the version was prepared. +## 1.10.1 — 2026-09-17 + +### Fixed + +- **Quantizing the context made every load fail.** llama.cpp cannot use a quantized + key/value cache without flash attention, and LM Studio refuses the load outright + rather than falling back: *"V Cache Quantization requires flash attention to be + enabled."* 1.10.0 wrote the cache fields and assumed flash attention was already on, + which is not dependable — it varies by engine and by whatever the model was last + loaded with. A run that asks for a quantized cache now writes the flag that makes it + possible, into the same per-model configuration, for the same single load, and + restores it with the rest. A run that leaves the cache alone does not touch the + setting at all. A load that comes back reporting no flash attention is refused + before anything is measured, rather than reporting a cache that was not in use. + ## 1.10.0 — 2026-09-17 ### Added diff --git a/README.md b/README.md index 495c4e9..a33f60f 100644 --- a/README.md +++ b/README.md @@ -382,8 +382,10 @@ whether a run measures the GPU or measures a spill into system RAM. **Quantize the context (KV cache)** sets the K and V caches independently to q8_0, q5_0, q4_0, iq4_nl or f16. K tolerates quantization less well than V, so q8_0 for K -with something smaller for V is the usual choice. It needs flash attention, which -LM Studio turns on by default. +with something smaller for V is the usual choice. llama.cpp cannot use a quantized +cache without flash attention and LM Studio refuses the load outright rather than +falling back, so a run that asks for one turns flash attention on for each load it +makes, in the same file and with the same restore. LM Studio exposes no command-line flag for this, so the setting is written into its own per-model configuration for the length of one load and the file is put back byte diff --git a/electron/cache-quant.ts b/electron/cache-quant.ts index 9489062..0f28776 100644 --- a/electron/cache-quant.ts +++ b/electron/cache-quant.ts @@ -2,6 +2,7 @@ import {existsSync,mkdirSync,readFileSync,readdirSync,rmdirSync,unlinkSync,write import {homedir} from 'node:os'; import path from 'node:path'; import type {Model} from '../src/types'; +import {needsFlashAttention} from '../src/cache-quant'; import type {CacheQuant} from '../src/cache-quant'; // Quantizing the KV cache. The context is what actually fills a 16 GB card: a run loaded with @@ -22,6 +23,13 @@ import type {CacheQuant} from '../src/cache-quant'; // {"key":"llm.load.llama.kCacheQuantizationType","value":{"checked":true,"value":"q8_0"}} const K_CACHE='llm.load.llama.kCacheQuantizationType'; const V_CACHE='llm.load.llama.vCacheQuantizationType'; +// llama.cpp cannot use a quantized KV cache without flash attention, and LM Studio refuses the load +// outright rather than falling back: "V Cache Quantization requires flash attention to be enabled." +// Its default is not dependable — it varies by engine and by whatever the model was last loaded +// with — so a run that asks for a quantized cache writes the flag that makes it possible, in the +// same file and for the same single load. Nothing is written when the cache is left alone, so a +// run that did not ask for this never touches the user's own flash-attention setting. +const FLASH='llm.load.llama.flashAttention'; const CONFIG_ROOT=['.lmstudio','.internal','user-concrete-model-default-config']; // Resolved from LM Studio's own index rather than built from the model key, because the key is a @@ -62,9 +70,10 @@ export type Restore=()=>void; export function applyCacheQuant(model:Model,k:CacheQuant,v:CacheQuant,home=homedir()):Restore{ const file=cacheConfigPath(modelResource(model,home),home),existed=existsSync(file),original=existed?readFileSync(file):null; const config=original?shape(original):{preset:'',operation:{fields:[]},load:{fields:[]}}; - const keep=(config.load.fields as {key?:unknown}[]).filter(f=>f&&typeof f.key==='string'&&f.key!==K_CACHE&&f.key!==V_CACHE); + const flash=needsFlashAttention(k,v); + const keep=(config.load.fields as {key?:unknown}[]).filter(f=>f&&typeof f.key==='string'&&f.key!==K_CACHE&&f.key!==V_CACHE&&(!flash||f.key!==FLASH)); const field=(key:string,q:CacheQuant)=>({key,value:q==='off'?{checked:false,value:'f16'}:{checked:true,value:q}}); - config.load.fields=[...keep,field(K_CACHE,k),field(V_CACHE,v)]; + config.load.fields=[...keep,...(flash?[{key:FLASH,value:true}]:[]),field(K_CACHE,k),field(V_CACHE,v)]; mkdirSync(path.dirname(file),{recursive:true}); writeFileSync(file,JSON.stringify(config,null,2)); let done=false; @@ -88,6 +97,9 @@ export function verifyCacheQuant(config:Record,k:CacheQuant,v:Ca }; // LM Studio reports these under snake_case names in the instance config it returns. const kGot=got('k_cache_quantization_type'),vGot=got('v_cache_quantization_type'); + // Without flash attention a quantized cache cannot be in use, whatever the cache fields say. + if(needsFlashAttention(k,v)&&(config as any).flash_attention===false) + throw Error('LM Studio loaded the model without flash attention, which a quantized KV cache requires. No measurements were taken.'); // An engine that does not report the fields at all is not proof of failure, so it is not treated // as one; anything it does report has to match, or the numbers would describe another setting. if(kGot!=='unreported'&&kGot!==k)throw Error(`LM Studio loaded the K cache as ${kGot}, not the requested ${k}. No measurements were taken.`); diff --git a/package-lock.json b/package-lock.json index 8da76c3..d01e9de 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "local-model-bench", - "version": "1.10.0", + "version": "1.10.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "local-model-bench", - "version": "1.10.0", + "version": "1.10.1", "dependencies": { "lucide-react": "^0.468.0", "react": "^19.1.0", diff --git a/package.json b/package.json index be604fe..c41d278 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "local-model-bench", - "version": "1.10.0", + "version": "1.10.1", "description": "Private desktop benchmarks and transparent grading for LM Studio models", "author": "Local Model Bench contributors", "license": "MIT", diff --git a/src/main.tsx b/src/main.tsx index 23a61d6..6e9766f 100644 --- a/src/main.tsx +++ b/src/main.tsx @@ -130,7 +130,7 @@ function App(){ {progress&&
{progress.phase}

{progress.message}

{progress.completed} / {progress.total||'…'} requests{progress.active} active
{run&&
{run.samples.filter(s=>!s.warmup).slice(-4).map(s=>
{s.testName}{s.status}{number(s.metrics.generationTps)} tok/s
)}
}
}

01Choose your workload

update('name',e.target.value)}/>
{(['combined','performance','quality'] as const).map(mode=>)}
-
{(config.cacheK??'off')!=='off'||(config.cacheV??'off')!=='off'?

Cache memory is about {Math.round(cacheScalePair(config.cacheK??'off',config.cacheV??'off')*100)}% of what an f16 cache would use. Quantizing the cache changes what the model attends to, so treat quality scores from a quantized-cache run as measured under that setting rather than as comparable to an f16 run.

:null} +
{(config.cacheK??'off')!=='off'||(config.cacheV??'off')!=='off'?

Cache memory is about {Math.round(cacheScalePair(config.cacheK??'off',config.cacheV??'off')*100)}% of what an f16 cache would use. Quantizing the cache changes what the model attends to, so treat quality scores from a quantized-cache run as measured under that setting rather than as comparable to an f16 run.

:null}
{/* One button for the question people actually have: does MTP make this model faster, and at what depth. Speed only, every draft depth from 1 to 5, against the MTP-off baseline. */} diff --git a/tests/cache-quant.test.ts b/tests/cache-quant.test.ts index 525b6d5..d6c5c84 100644 --- a/tests/cache-quant.test.ts +++ b/tests/cache-quant.test.ts @@ -81,3 +81,34 @@ test('a cache setting LM Studio did not apply stops the run before anything is m assert.throws(()=>verifyCacheQuant({k_cache_quantization_type:'f16'},'q8_0','q8_0'),/loaded the K cache as f16/); assert.throws(()=>verifyCacheQuant({v_cache_quantization_type:{checked:false,value:'f16'}},'off','q4_0'),/loaded the V cache as off/); }); + +// The bug that made every quantized-cache run fail on a real machine: LM Studio refuses the load +// with "V Cache Quantization requires flash attention to be enabled" and its default cannot be +// relied on, so the flag has to be written alongside the cache fields rather than assumed. +test('asking for a quantized cache also turns on the flash attention it requires',()=>{ + const {home,config}=library(); + const restore=applyCacheQuant(base,'q8_0','q4_0',home); + assert.equal(valueOf(config,'llm.load.llama.flashAttention'),true,'flash attention written with the cache'); + assert.equal(fields(config).filter(f=>f.key==='llm.load.llama.flashAttention').length,1,'written once, not duplicated'); + restore(); + assert.equal(existsSync(config),false); +}); + +test('a run that does not quantize leaves the user’s own flash-attention setting alone',()=>{ + const {home,config}=library(); + mkdirSync(path.dirname(config),{recursive:true}); + writeFileSync(config,JSON.stringify({load:{fields:[{key:'llm.load.llama.flashAttention',value:false}]}})); + applyCacheQuant(base,'off','off',home); + assert.equal(valueOf(config,'llm.load.llama.flashAttention'),false,'untouched when nothing was asked for'); + // f16 is not a quantization either, so it does not force the flag on. + applyCacheQuant(base,'f16','f16',home); + assert.equal(valueOf(config,'llm.load.llama.flashAttention'),false); +}); + +test('a load that came back without flash attention is refused rather than measured',()=>{ + assert.throws(()=>verifyCacheQuant({flash_attention:false},'q8_0','q8_0'),/without flash attention/); + // Not requested, so not checked; and an engine that does not report it is not assumed to have failed. + assert.doesNotThrow(()=>verifyCacheQuant({flash_attention:false},'off','off')); + assert.doesNotThrow(()=>verifyCacheQuant({},'q8_0','q8_0')); + assert.doesNotThrow(()=>verifyCacheQuant({flash_attention:true},'q8_0','q8_0')); +}); From 28bb0ae283129a024a8385c6c89ba738e425da39 Mon Sep 17 00:00:00 2001 From: M007-Net <262222560+M007-Net@users.noreply.github.com> Date: Thu, 17 Sep 2026 04:34:27 -0400 Subject: [PATCH 3/8] Local Model Bench 1.11.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The graphs can read across MTP depth. They always plotted against concurrent requests. That is the right axis for a concurrency sweep and the wrong one for a prediction-depth sweep: a sweep run at a single concurrency level gives every depth the same horizontal position, so six depths drew six dots stacked on one vertical line. The depths were there — as six entries in the legend — but nothing separated them on the page, and the run's whole subject was invisible in the charts meant to show it. "Read across" on the Automatic graphs now switches the horizontal axis to maximum predictions, so each model becomes one curve over its depths. The control appears only when a run measured more than one depth. While depth is the axis it stops being folded into the series name, because doing both would split a model into one flat single-point line per depth instead of a curve across them. The tooltip names the depth and still states the concurrency, since a depth curve only means anything at a known concurrency. Nothing changes for a run that did not sweep. Verified against a real completed sweep: on the concurrency axis six rows share one x; on the depth axis they occupy six, joined by a line. 240 offline tests, typecheck and production build all pass. Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 15 ++++++++++++ package-lock.json | 4 ++-- package.json | 2 +- src/ChartsPanel.tsx | 24 ++++++++++++------- src/charts.ts | 21 ++++++++++------ tests/chart-axis.test.ts | 52 ++++++++++++++++++++++++++++++++++++++++ 6 files changed, 100 insertions(+), 18 deletions(-) create mode 100644 tests/chart-axis.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index c6248f4..c0ccc88 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,21 @@ All notable changes to Local Model Bench are recorded here. The format follows Dates are the date the version was prepared. +## 1.11.0 — 2026-09-17 + +### Added + +- **The graphs can read across MTP depth.** They always plotted against concurrent + requests, which is the right axis for a concurrency sweep and the wrong one for a + prediction-depth sweep: a sweep run at a single concurrency level put every depth at + the same horizontal position, so six depths drew six dots stacked on one vertical + line and the only trace of the depth was a legend entry. **Read across** on the + Automatic graphs now switches the horizontal axis to maximum predictions, turning + each model into one curve over its depths. It appears only when a run measured more + than one depth, and depth stops being folded into the series name while it is the + axis, so a model is one line rather than one flat point per depth. The concurrency + axis is unchanged for every other run. + ## 1.10.1 — 2026-09-17 ### Fixed diff --git a/package-lock.json b/package-lock.json index d01e9de..7392e66 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "local-model-bench", - "version": "1.10.1", + "version": "1.11.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "local-model-bench", - "version": "1.10.1", + "version": "1.11.0", "dependencies": { "lucide-react": "^0.468.0", "react": "^19.1.0", diff --git a/package.json b/package.json index c41d278..0aa99f3 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "local-model-bench", - "version": "1.10.1", + "version": "1.11.0", "description": "Private desktop benchmarks and transparent grading for LM Studio models", "author": "Local Model Bench contributors", "license": "MIT", diff --git a/src/ChartsPanel.tsx b/src/ChartsPanel.tsx index d8a2756..08dfab4 100644 --- a/src/ChartsPanel.tsx +++ b/src/ChartsPanel.tsx @@ -1,7 +1,7 @@ import {useMemo,useState} from 'react'; import type {Run} from './types'; import {summaries} from '../electron/export'; -import {chartSpecs,renderChart,legendHtml,scoreLabels,type ChartRow,type ScoreSource} from './charts'; +import {chartSpecs,renderChart,legendHtml,scoreLabels,xAxisLabels,type ChartRow,type ScoreSource,type XAxis} from './charts'; import {measuredDepths,mtpDepthText} from './mtp-sweep'; import type {HistoryRow} from './history'; import {backendOf} from '../electron/runtime'; @@ -11,11 +11,11 @@ import {overlayOptions,overlayRows,stillOffered} from './chart-compare'; const depthSuffix=' · MTP '; // The legend, graph grid, and hover tooltip are shared by the single-run graphs and the history overview, // which feeds in pooled rows keyed by group label instead of by model key. -export function ChartGrid({rows,score,onModel,hint='Click a point to inspect its model and responses.'}:{rows:ChartRow[];score:ScoreSource;onModel?:(key:string)=>void;hint?:string}){ +export function ChartGrid({rows,score,onModel,xAxis='concurrency',hint='Click a point to inspect its model and responses.'}:{rows:ChartRow[];score:ScoreSource;onModel?:(key:string)=>void;xAxis?:XAxis;hint?:string}){ const [tip,setTip]=useState(''); const show=(target:EventTarget)=>{const point=(target as Element).closest?.('[data-tooltip]');setTip(point?.getAttribute('data-tooltip')??'');}; return <>
-
show(e.target)} onMouseLeave={()=>setTip('')} onFocus={e=>show(e.target)} onBlur={()=>setTip('')} onClick={e=>{const key=(e.target as Element).closest('[data-model]')?.getAttribute('data-model');if(key)onModel?.(key);}} onKeyDown={e=>{if(e.key==='Enter'||e.key===' '){const key=(e.target as Element).getAttribute('data-model');if(key){e.preventDefault();onModel?.(key);}}}}>{chartSpecs(score).map(spec=>

{spec.title}

{spec.description}

)}
+
show(e.target)} onMouseLeave={()=>setTip('')} onFocus={e=>show(e.target)} onBlur={()=>setTip('')} onClick={e=>{const key=(e.target as Element).closest('[data-model]')?.getAttribute('data-model');if(key)onModel?.(key);}} onKeyDown={e=>{if(e.key==='Enter'||e.key===' '){const key=(e.target as Element).getAttribute('data-model');if(key){e.preventDefault();onModel?.(key);}}}}>{chartSpecs(score).map(spec=>

{spec.title}

{spec.description}

)}
{tip&&
{tip}{hint}
} ; } @@ -25,12 +25,19 @@ export function ChartsPanel({run,onModel,history=[]}:{run:Run;onModel?:(key:stri // history being reloaded underneath it. const [compare,setCompare]=useState([]); const chosen=run.tests.some(t=>t.id===testId)?testId:run.tests[0]?.id; - // A sweep measures one model at several prediction depths, so each depth becomes its own line. - // Without this the graph would join points taken under different settings into one curve. + // Depths measured by this run. With more than one, the graphs can read across depth instead of + // across concurrency — which is the only way to see a sweep run at a single concurrency level. + const depths=useMemo(()=>measuredDepths(run),[run]); + const [xAxis,setXAxis]=useState('concurrency'); + // A run with one depth has no depth axis to offer, and one that loses its sweep while selected + // must not keep drawing an axis that no longer varies. + const axis:XAxis=depths.length>1?xAxis:'concurrency'; const own=useMemo(()=>{ const all=summaries(run).filter(r=>r.testId===chosen); - return measuredDepths(run).length>1?all.map(r=>({...r,modelKey:`${r.modelKey} · ${mtpDepthText(r.mtpDepth)}`})):all; - },[run,chosen]); + // Depth goes into the series name only when it is not the axis. Doing both would split each + // model into one flat single-point line per depth rather than a curve across them. + return depths.length>1&&axis!=='mtpDepth'?all.map(r=>({...r,modelKey:`${r.modelKey} · ${mtpDepthText(r.mtpDepth)}`})):all; + },[run,chosen,depths,axis]); const backend=useMemo(()=>backendOf(run).label,[run]); const models=useMemo(()=>[...new Set(summaries(run).map(r=>r.modelKey))],[run]); const options=useMemo(()=>overlayOptions(history,run.id,chosen??'',backend,models),[history,run.id,chosen,backend,models]); @@ -41,6 +48,7 @@ export function ChartsPanel({run,onModel,history=[]}:{run:Run;onModel?:(key:stri const backends=options.filter(o=>o.kind==='backend'),others=options.filter(o=>o.kind==='model'); const toggle=(id:string)=>setCompare(c=>c.includes(id)?c.filter(x=>x!==id):[...c,id]); return

Automatic graphs

Updated as requests and grades arrive. Hover over a point for its values.

+ {depths.length>1&&} {/* Saved measurements of the same test, drawn on the same axes. The control is absent entirely when nothing comparable is saved, rather than an empty menu that invites a click. */} {options.length>0&&
Compare with{live.length?` · ${live.length}`:''} @@ -51,6 +59,6 @@ export function ChartsPanel({run,onModel,history=[]}:{run:Run;onModel?:(key:stri

Saved rows from finished runs, drawn as their own lines. Nothing is recalculated, and conditions can differ between runs.

}
- onModel?.(key.split(depthSuffix)[0])}/> + onModel?.(key.split(depthSuffix)[0])}/>
; } diff --git a/src/charts.ts b/src/charts.ts index a70b4ce..d970ee8 100644 --- a/src/charts.ts +++ b/src/charts.ts @@ -8,6 +8,13 @@ const fmt=(n:number)=>n.toLocaleString('en-US',{maximumFractionDigits:n<1?2:1}); export function colorFor(key:string){let hash=0;for(const c of key)hash=(Math.imul(hash,31)+c.charCodeAt(0))|0;return `hsl(${Math.abs(hash)%360}, 65%, 72%)`;} type Metric='generationTps'|'estimatedPrefillTps'|'throughput'|'ttftMs'|'medianMs'|'p95Ms'|'failureRate'|ScoreSource; export type ChartSpec={key:Metric|'scatter';title:string;unit:string;description:string;scale?:number;fixedMax?:number}; +// What the horizontal axis counts. Concurrency is the historical answer and stays the default, but +// an MTP sweep at a single concurrency level has nothing to spread along it: every depth lands on +// the same x and the series stack on one vertical line. Depth is the axis that run varies. +export type XAxis='concurrency'|'mtpDepth'; +export const xAxisLabels:Record={concurrency:'Concurrent requests',mtpDepth:'Maximum predictions (0 = MTP off)'}; +// A row from a run that measured no depth sits at 0, which is where "MTP off" belongs anyway. +export const xValueOf=(r:ChartRow,axis:XAxis):number=>axis==='mtpDepth'?(r.mtpDepth??0):r.concurrency; export function chartSpecs(score:ScoreSource):ChartSpec[]{return [ {key:'generationTps',title:'Generation speed',unit:'Tokens / second',description:'Average speed per successful request. Higher is faster.'}, {key:'estimatedPrefillTps',title:'Estimated prefill speed',unit:'Tokens / second',description:'Client-timed prompt processing; caching and buffering can affect this estimate.'}, @@ -21,13 +28,13 @@ export function chartSpecs(score:ScoreSource):ChartSpec[]{return [ ];} const start=(label:string)=>``; const empty=(label:string)=>`${start(label)}No measurements available yetMissing values are not plotted as zero.`; -export function renderChart(rows:ChartRow[],spec:ChartSpec,score:ScoreSource='objective'):string{ +export function renderChart(rows:ChartRow[],spec:ChartSpec,score:ScoreSource='objective',xAxis:XAxis='concurrency'):string{ const scatter=spec.key==='scatter'; const val=(r:ChartRow):number|null=>{const n=scatter?r[score]:r[spec.key as Metric];return typeof n==='number'&&Number.isFinite(n)?n*(spec.scale??1):null;}; const usable=rows.filter(r=>val(r)!==null&&(!scatter||r.generationTps!==null)); if(!usable.length)return empty(spec.title); const yMax=scatter?100:spec.fixedMax??Math.max(1,...usable.map(r=>val(r)!*1.08)); - const allXs=[...new Set(rows.map(r=>r.concurrency))].sort((a,b)=>a-b); + const allXs=[...new Set(rows.map(r=>xValueOf(r,xAxis)))].sort((a,b)=>a-b); const xMin=scatter?0:Math.min(...allXs),xMax=scatter?Math.max(1,...usable.map(r=>r.generationTps!))*1.08:Math.max(...allXs); const x=(v:number)=>xMax===xMin?320:72+(v-xMin)/(xMax-xMin)*496; const y=(v:number)=>230-v/yMax*184; @@ -36,15 +43,15 @@ export function renderChart(rows:ChartRow[],spec:ChartSpec,score:ScoreSource='ob // Limit labels to a readable subset without changing numeric axis spacing. const ticks=scatter?[0,.25,.5,.75,1].map(t=>t*xMax):allXs.filter((_,i)=>allXs.length<=8||i===allXs.length-1||i%Math.ceil(allXs.length/7)===0); for(const t of ticks)svg+=`${fmt(t)}`; - svg+=`${scatter?'Generation tokens / second':'Concurrent requests'}`; + svg+=`${scatter?'Generation tokens / second':xAxisLabels[xAxis]}`; const keys=[...new Set(rows.map(r=>r.modelKey))].sort(); keys.forEach((key,i)=>{ - const rs=rows.filter(r=>r.modelKey===key).sort((a,b)=>a.concurrency-b.concurrency),color=colorFor(key); + const rs=rows.filter(r=>r.modelKey===key).sort((a,b)=>xValueOf(a,xAxis)-xValueOf(b,xAxis)),color=colorFor(key); let segment:string[]=[]; const flush=()=>{if(segment.length>1)svg+=``;segment=[];}; - if(!scatter){for(const r of rs){const v=val(r);if(v===null){flush();continue;}segment.push(`${x(r.concurrency)},${y(v)}`);}flush();} - for(const r of rs){const v=val(r);if(v===null||(scatter&&r.generationTps===null))continue;const cx=x(scatter?r.generationTps!:r.concurrency),cy=y(v);const title=`${r.modelKey} • concurrency ${r.concurrency} • ${scatter?fmt(r.generationTps!)+' tok/s • ':''}${fmt(v)} ${scatter?'score':spec.unit} • ${r.requests} requests, ${r.failures} failures`; - svg+=`${escapeHtml(title)}`; + if(!scatter){for(const r of rs){const v=val(r);if(v===null){flush();continue;}segment.push(`${x(xValueOf(r,xAxis))},${y(v)}`);}flush();} + for(const r of rs){const v=val(r);if(v===null||(scatter&&r.generationTps===null))continue;const cx=x(scatter?r.generationTps!:xValueOf(r,xAxis)),cy=y(v);const title=`${r.modelKey} • ${xAxis==='mtpDepth'?(r.mtpDepth?`${r.mtpDepth} draft token${r.mtpDepth===1?'':'s'}`:'MTP off')+` • concurrency ${r.concurrency}`:`concurrency ${r.concurrency}`} • ${scatter?fmt(r.generationTps!)+' tok/s • ':''}${fmt(v)} ${scatter?'score':spec.unit} • ${r.requests} requests, ${r.failures} failures`; + svg+=`${escapeHtml(title)}`; } });return svg+''; } diff --git a/tests/chart-axis.test.ts b/tests/chart-axis.test.ts new file mode 100644 index 0000000..2a905cb --- /dev/null +++ b/tests/chart-axis.test.ts @@ -0,0 +1,52 @@ +import {test} from 'node:test'; +import assert from 'node:assert/strict'; +import {chartSpecs,renderChart,xAxisLabels,xValueOf,type ChartRow} from '../src/charts'; + +// A sweep run at one concurrency level: the case where the old axis had nothing to spread along. +const sweep=(depth:number,gen:number):ChartRow=>({ + modelKey:'qwen3.8-27b@q4_k_s',model:'Qwen3.8 27B',test:'Short prompt throughput',testId:'perf-short', + concurrency:1,mtpDepth:depth,generationTps:gen,estimatedPrefillTps:100,throughput:gen, + medianMs:1000,p95Ms:1200,ttftMs:200,failureRate:0,requests:2,failures:0,objective:null, + localJudge:null,externalJudge:null,gpuHotSpotAvg:null,gpuHotSpotMax:null, +} as unknown as ChartRow); +const rows=[0,1,2,3,4,5].map(d=>sweep(d,[6.4,6.2,5.7,5.0,4.6,4.1][d])); +const gen=chartSpecs('objective').find(s=>s.key==='generationTps')!; + +test('depth is read off the row, and a run with no sweep sits at MTP off',()=>{ + assert.equal(xValueOf(rows[3],'mtpDepth'),3); + assert.equal(xValueOf(rows[3],'concurrency'),1); + assert.equal(xValueOf({...rows[0],mtpDepth:null} as ChartRow,'mtpDepth'),0,'no depth measured is MTP off, not a gap'); +}); + +test('a sweep at one concurrency collapses on the old axis and spreads on the new one',()=>{ + const onConcurrency=renderChart(rows,gen,'objective','concurrency'); + const xs=[...onConcurrency.matchAll(/]*cx="([\d.]+)"/g)].map(m=>m[1]); + assert.equal(xs.length,6,'all six depths are plotted either way'); + assert.equal(new Set(xs).size,1,'…but on the concurrency axis they share one x, which is the complaint'); + + const onDepth=renderChart(rows,gen,'objective','mtpDepth'); + const dxs=[...onDepth.matchAll(/]*cx="([\d.]+)"/g)].map(m=>Number(m[1])); + assert.equal(new Set(dxs).size,6,'on the depth axis each depth gets its own x'); + assert.deepEqual([...dxs].sort((a,b)=>a-b),dxs.slice().sort((a,b)=>a-b)); + assert.ok(Math.min(...dxs){ + const onDepth=renderChart(rows,gen,'objective','mtpDepth'); + assert.match(onDepth,/MTP off/); + assert.match(onDepth,/3 draft tokens/); + assert.match(onDepth,/1 draft token •/,'singular for one'); + // Concurrency is still stated, because a depth curve is only meaningful at a known concurrency. + assert.match(onDepth,/concurrency 1/); +}); + +test('the concurrency axis is untouched for an ordinary run',()=>{ + const across=[1,2,4].map((c,i)=>({...sweep(0,60-i*10),concurrency:c})); + const svg=renderChart(across as ChartRow[],gen,'objective'); + const xs=[...svg.matchAll(/]*cx="([\d.]+)"/g)].map(m=>m[1]); + assert.equal(new Set(xs).size,3); + assert.match(svg,/Concurrent requests/); +}); From 94ac156d720a75d5d7f3824d925efccd538bfcea Mon Sep 17 00:00:00 2001 From: M007-Net <262222560+M007-Net@users.noreply.github.com> Date: Thu, 17 Sep 2026 04:39:09 -0400 Subject: [PATCH 4/8] Local Model Bench 1.11.1 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The depth axis existed but nothing opened on it. 1.11.0 added "Read across" and then defaulted it to concurrent requests. For the runs that most need the depth axis — a prediction-depth sweep at a single concurrency level — that meant the graphs still opened with every depth stacked at one horizontal position. The control was there, the feature was not: unless you went looking for a dropdown you had no reason to expect, nothing had changed. The graphs now open on whichever dimension the run actually varied. A run that swept depths without varying concurrency opens on maximum predictions; anything that varied concurrency opens on concurrency, as before. Choosing an axis by hand still overrides it, and the choice is held as null until made so the default follows the run rather than being frozen at first render. The rule is extracted as defaultXAxis() rather than left inline, so it is tested rather than asserted. Checked against the saved runs: the completed six-depth sweep opens on depth with six distinct positions both for all models and narrowed to one; a run whose sweep only reached depth 0 correctly has no depth axis to open on; and the two runs that varied concurrency are untouched. 241 offline tests, typecheck and production build all pass. Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 13 +++++++++++++ package-lock.json | 4 ++-- package.json | 2 +- src/ChartsPanel.tsx | 11 ++++++++--- src/charts.ts | 5 +++++ tests/chart-axis.test.ts | 13 ++++++++++++- 6 files changed, 41 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c0ccc88..6d2af8e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,19 @@ All notable changes to Local Model Bench are recorded here. The format follows Dates are the date the version was prepared. +## 1.11.1 — 2026-09-17 + +### Fixed + +- **The depth axis existed but nothing opened on it.** 1.11.0 added *Read across* and + then defaulted it to concurrent requests, so a prediction-depth sweep still opened + showing every depth stacked at one horizontal position — the control was there, but + the graphs looked exactly as they had before and the feature was invisible unless + you went looking for it. The graphs now open on whichever dimension the run actually + varied: a run that swept depths at a single concurrency level opens on maximum + predictions, and anything that varied concurrency still opens on concurrency. + Choosing an axis by hand continues to override it. + ## 1.11.0 — 2026-09-17 ### Added diff --git a/package-lock.json b/package-lock.json index 7392e66..00d36b3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "local-model-bench", - "version": "1.11.0", + "version": "1.11.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "local-model-bench", - "version": "1.11.0", + "version": "1.11.1", "dependencies": { "lucide-react": "^0.468.0", "react": "^19.1.0", diff --git a/package.json b/package.json index 0aa99f3..38f1f89 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "local-model-bench", - "version": "1.11.0", + "version": "1.11.1", "description": "Private desktop benchmarks and transparent grading for LM Studio models", "author": "Local Model Bench contributors", "license": "MIT", diff --git a/src/ChartsPanel.tsx b/src/ChartsPanel.tsx index 08dfab4..052aa52 100644 --- a/src/ChartsPanel.tsx +++ b/src/ChartsPanel.tsx @@ -1,7 +1,7 @@ import {useMemo,useState} from 'react'; import type {Run} from './types'; import {summaries} from '../electron/export'; -import {chartSpecs,renderChart,legendHtml,scoreLabels,xAxisLabels,type ChartRow,type ScoreSource,type XAxis} from './charts'; +import {chartSpecs,renderChart,legendHtml,scoreLabels,defaultXAxis,xAxisLabels,type ChartRow,type ScoreSource,type XAxis} from './charts'; import {measuredDepths,mtpDepthText} from './mtp-sweep'; import type {HistoryRow} from './history'; import {backendOf} from '../electron/runtime'; @@ -28,10 +28,15 @@ export function ChartsPanel({run,onModel,history=[]}:{run:Run;onModel?:(key:stri // Depths measured by this run. With more than one, the graphs can read across depth instead of // across concurrency — which is the only way to see a sweep run at a single concurrency level. const depths=useMemo(()=>measuredDepths(run),[run]); - const [xAxis,setXAxis]=useState('concurrency'); + // Whether concurrency is a dimension of this run at all. A sweep run at a single concurrency level + // has nothing to read across on that axis, so defaulting to it shows the run's whole subject as a + // stack of points on one vertical line — the graphs open on whichever axis the run actually varied. + const concurrencyVaries=useMemo(()=>new Set(summaries(run).map(r=>r.concurrency)).size>1,[run]); + // null until chosen, so the default can follow the run rather than being frozen at first render. + const [xAxis,setXAxis]=useState(null); // A run with one depth has no depth axis to offer, and one that loses its sweep while selected // must not keep drawing an axis that no longer varies. - const axis:XAxis=depths.length>1?xAxis:'concurrency'; + const axis:XAxis=depths.length<=1?'concurrency':xAxis??defaultXAxis(depths.length,concurrencyVaries); const own=useMemo(()=>{ const all=summaries(run).filter(r=>r.testId===chosen); // Depth goes into the series name only when it is not the axis. Doing both would split each diff --git a/src/charts.ts b/src/charts.ts index d970ee8..ce3ddba 100644 --- a/src/charts.ts +++ b/src/charts.ts @@ -15,6 +15,11 @@ export type XAxis='concurrency'|'mtpDepth'; export const xAxisLabels:Record={concurrency:'Concurrent requests',mtpDepth:'Maximum predictions (0 = MTP off)'}; // A row from a run that measured no depth sits at 0, which is where "MTP off" belongs anyway. export const xValueOf=(r:ChartRow,axis:XAxis):number=>axis==='mtpDepth'?(r.mtpDepth??0):r.concurrency; +// Which axis a run should open on, before anyone chooses. A run that swept depths at a single +// concurrency level varied exactly one thing, and it is not concurrency: opening on that axis puts +// every depth at the same x and hides the run's whole subject behind a stack of points. +export const defaultXAxis=(depthsMeasured:number,concurrencyVaries:boolean):XAxis=> + depthsMeasured>1&&!concurrencyVaries?'mtpDepth':'concurrency'; export function chartSpecs(score:ScoreSource):ChartSpec[]{return [ {key:'generationTps',title:'Generation speed',unit:'Tokens / second',description:'Average speed per successful request. Higher is faster.'}, {key:'estimatedPrefillTps',title:'Estimated prefill speed',unit:'Tokens / second',description:'Client-timed prompt processing; caching and buffering can affect this estimate.'}, diff --git a/tests/chart-axis.test.ts b/tests/chart-axis.test.ts index 2a905cb..13fb1e0 100644 --- a/tests/chart-axis.test.ts +++ b/tests/chart-axis.test.ts @@ -1,6 +1,6 @@ import {test} from 'node:test'; import assert from 'node:assert/strict'; -import {chartSpecs,renderChart,xAxisLabels,xValueOf,type ChartRow} from '../src/charts'; +import {chartSpecs,defaultXAxis,renderChart,xAxisLabels,xValueOf,type ChartRow} from '../src/charts'; // A sweep run at one concurrency level: the case where the old axis had nothing to spread along. const sweep=(depth:number,gen:number):ChartRow=>({ @@ -50,3 +50,14 @@ test('the concurrency axis is untouched for an ordinary run',()=>{ assert.equal(new Set(xs).size,3); assert.match(svg,/Concurrent requests/); }); + +// The complaint that prompted this: the control existed but the graphs still opened on concurrency, +// so a sweep run at one concurrency level looked exactly as it had before. +test('a sweep at one concurrency opens on the depth axis without being asked',()=>{ + assert.equal(defaultXAxis(6,false),'mtpDepth','six depths, one concurrency: depth is the only thing that varied'); + // Both dimensions varied, so concurrency stays the default and depth remains one click away. + assert.equal(defaultXAxis(6,true),'concurrency'); + // No sweep: there is no depth axis to open on. + assert.equal(defaultXAxis(1,false),'concurrency'); + assert.equal(defaultXAxis(0,false),'concurrency'); +}); From 9b0a14a8404ade83e2d7bed390523ec365a61ed0 Mon Sep 17 00:00:00 2001 From: M007-Net <262222560+M007-Net@users.noreply.github.com> Date: Thu, 17 Sep 2026 05:16:05 -0400 Subject: [PATCH 5/8] Local Model Bench 1.12.0 Off / text-only can now actually load without the projector. The toggle only ever controlled whether an image was sent. The projector loaded regardless, because LM Studio attaches one from the model's index entry and from nowhere else. Rechecked today against LM Studio 2.40.0: none of the 24 llm.load.* keys in its bundle mentions vision, projector or mmproj, and llama-server.exe, llama-server-impl.dll and mtmd.dll carry no --mmproj flag at all, because the engine is driven through LM Studio's own bindings rather than a command line. There is no setting to add. So the toggle now drives the one thing that does work: the text-only copy this app already builds, a tree of hard links holding the same weights and MTP head with no projector, which LM Studio indexes as its own key and reports as non-vision. With Off / text-only selected, any chosen model that would still load a projector is named, with one button to select the copies that exist and another to make the ones that do not. A model with no projector is left alone; a copy LM Studio has not indexed, or whose key it no longer lists, counts as missing rather than being selected and failing to load; and a run that cannot be made fully text-only says so rather than implying otherwise. Also fixes calibrated prompt processing being blank on every sweep. The engine wrote each calibration under one key shape and the results screen read another, so the column never resolved on exactly the runs it was built for. Both sides now use calibrationKey(), which also drops a doubled prefix that spelled depth 0 as "MTP MTP off". Calibrations saved under the old shape are still read. 247 offline tests, typecheck and production build all pass. Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 26 +++++++++++++++ electron/engine.ts | 4 +-- electron/text-only.ts | 19 +++++++++-- package-lock.json | 4 +-- package.json | 2 +- src/main.tsx | 23 ++++++++++++++ src/prefill.ts | 19 +++++++++-- src/style.css | 7 +++++ src/text-only-swap.ts | 59 ++++++++++++++++++++++++++++++++++ tests/prefill.test.ts | 31 +++++++++++++++++- tests/text-only-swap.test.ts | 61 ++++++++++++++++++++++++++++++++++++ 11 files changed, 243 insertions(+), 12 deletions(-) create mode 100644 src/text-only-swap.ts create mode 100644 tests/text-only-swap.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d2af8e..847b5a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,32 @@ All notable changes to Local Model Bench are recorded here. The format follows Dates are the date the version was prepared. +## 1.12.0 — 2026-09-17 + +### Added + +- **Off / text-only can now actually load without the projector.** The toggle only + ever controlled whether an image was *sent*; the projector loaded regardless, because + LM Studio attaches one from the model's index entry and from nowhere else. Rechecked + against LM Studio 2.40.0: none of the 24 `llm.load.*` keys in its bundle mentions + vision, and its llama-server carries no `--mmproj` flag at all, since it drives the + engine through its own bindings rather than a command line. So the toggle now drives + the one thing that does work — the text-only copy this app already builds. With + **Off / text-only** selected, any chosen model that would still load a projector is + named, with one button to select the text-only copies that exist and another to make + the ones that do not. Copies are hard links and cost no disk. A model with no + projector is left alone, and a run that cannot be made fully text-only says so + rather than implying otherwise. + +### Fixed + +- **Calibrated prompt processing was blank on every sweep.** The engine stored each + calibration under one key shape and the results screen looked it up under another, + so the column existed and never resolved — on exactly the runs it was built for. Both + sides now use one `calibrationKey()`, which also drops a doubled prefix that spelled + depth 0 as "MTP MTP off". Calibrations saved under the old shape are still read, so + finished runs keep their measurements. + ## 1.11.1 — 2026-09-17 ### Fixed diff --git a/electron/engine.ts b/electron/engine.ts index 15c4566..a4bc19f 100644 --- a/electron/engine.ts +++ b/electron/engine.ts @@ -12,7 +12,7 @@ import { applyCacheQuant,verifyCacheQuant } from './cache-quant'; import { cacheQuantText } from '../src/cache-quant'; import { runtimeLabel,selectRuntime } from './runtime'; import { estimateLoad,loadAdvice,looksLikeMemory } from './load-estimate'; -import { calibratePrefill,calibrationPrompt,calibrationRepeats,prefillText } from '../src/prefill'; +import { calibratePrefill,calibrationKey,calibrationPrompt,calibrationRepeats,prefillText } from '../src/prefill'; import { serverContext } from '../src/defaults'; import { idleSampler, type GpuSampler } from './gpu'; @@ -127,7 +127,7 @@ export async function runEngine(run:Run,settings:Settings,signal:AbortSignal,emi const small=await point(calibrationRepeats.small),big=await point(calibrationRepeats.big); const calibration=calibratePrefill(small,big); log(`Prompt processing for ${model.display_name}: ${prefillText(calibration)}. ${calibration.note}`); - emit({type:'model',key:`prefill:${key}${steps.length>1?` · MTP ${step.label}`:''}`,info:calibration}); + emit({type:'model',key:calibrationKey(key,steps.length>1?step.depth:null),info:calibration}); }catch(e){ // A calibration that could not be taken costs the run nothing else: the per-request figures // are still recorded, they are simply left uncorrected. diff --git a/electron/text-only.ts b/electron/text-only.ts index 020b4fc..d4b2512 100644 --- a/electron/text-only.ts +++ b/electron/text-only.ts @@ -103,16 +103,28 @@ export async function awaitIndexed(folder:string,home=homedir(),timeoutMs=20000, } } -export type Twin={folder:string;source:string;sourceKey:string;created:string}; +// `key` is the twin's OWN model key, which is what a run has to select to load the weights without +// the projector. LM Studio assigns it when it indexes the folder, so it is read back from the index +// rather than derived from the folder name; null while LM Studio has not indexed the twin yet. +export type Twin={folder:string;source:string;sourceKey:string;created:string;key:string|null}; function readMarker(folder:string):Twin|null{ try{ const marker=JSON.parse(readFileSync(path.join(folder,MARKER),'utf8')); if(marker?.createdBy!=='local-model-bench'||typeof marker.source!=='string')return null; - return {folder:slash(folder),source:slash(marker.source),sourceKey:String(marker.sourceKey??''),created:String(marker.created??'')}; + return {folder:slash(folder),source:slash(marker.source),sourceKey:String(marker.sourceKey??''),created:String(marker.created??''),key:null}; }catch{return null;} } // Twins this app made, found by their marker rather than by their name, so a folder a user // happened to name "… - Text only" themselves is never treated as this app's to delete. +// Which model key LM Studio gave a twin folder. Matched on the indexed entry point living inside +// that folder, because the key itself is LM Studio's to choose and has no reliable spelling. +function keyForFolder(folder:string,home:string):string|null{ + try{ + const inside=slash(folder).replace(/\/+$/,'')+'/'; + const hit=readIndex(home).models.filter((m:any)=>slash(String(m?.entryPoint?.absPath??'')).startsWith(inside)); + return hit.length===1&&typeof hit[0].defaultIdentifier==='string'?hit[0].defaultIdentifier:null; + }catch{return null;} +} export function textOnlyTwins(home=homedir()):Twin[]{ const root=path.join(home,'.lmstudio','models'),found:Twin[]=[]; const walk=(dir:string,depth:number)=>{ @@ -126,7 +138,8 @@ export function textOnlyTwins(home=homedir()):Twin[]{ } }; walk(root,3); - return found; + // Resolved once here rather than per caller: the index is one file read for the whole list. + return found.map(t=>({...t,key:keyForFolder(t.folder,home)})); } // Removal refuses anything that is not one of this app's twins, and anything holding a file // that exists nowhere else. Between them, a delete here can only ever drop directory entries diff --git a/package-lock.json b/package-lock.json index 00d36b3..18f4fdc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "local-model-bench", - "version": "1.11.1", + "version": "1.12.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "local-model-bench", - "version": "1.11.1", + "version": "1.12.0", "dependencies": { "lucide-react": "^0.468.0", "react": "^19.1.0", diff --git a/package.json b/package.json index 38f1f89..fca8367 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "local-model-bench", - "version": "1.11.1", + "version": "1.12.0", "description": "Private desktop benchmarks and transparent grading for LM Studio models", "author": "Local Model Bench contributors", "license": "MIT", diff --git a/src/main.tsx b/src/main.tsx index 6e9766f..01c4dc6 100644 --- a/src/main.tsx +++ b/src/main.tsx @@ -25,6 +25,8 @@ import {defaultHistoryView,restoreHistoryView,shortDate} from './history'; import {defaultSweep,maxMtpDepth,measuredDepths,mtpDepthText,sweepSteps,sweepVerdicts} from './mtp-sweep'; import {calibrationFor,focusBlurb,focusColumns,resultsFocuses,type ResultsFocus} from './results-focus'; import {backendRows,comparableBackends} from './chart-compare'; +import {planTextOnlySwap,textOnlyIncomplete,textOnlySummary} from './text-only-swap'; +import type {Twin as TextOnlyTwin} from '../electron/text-only'; import {backendOf} from '../electron/runtime'; import type {HistoryRow} from './history'; import {runtimeChoices,type Runtime} from '../electron/runtime'; @@ -71,10 +73,14 @@ function App(){ const [page,setPage]=useState('Models'),[snap,setSnap]=useState(null),[models,setModels]=useState([]),[connected,setConnected]=useState(false),[connectionError,setConnectionError]=useState(''),[config,setConfig]=useState(structuredClone(defaultConfig)),[busy,setBusy]=useState(''),[notice,setNotice]=useState(''),[error,setError]=useState(''),[progress,setProgress]=useState(null),[runId,setRunId]=useState(''),[run,setRun]=useState(null),[editTest,setEditTest]=useState(null),[settingsDraft,setSettingsDraft]=useState(defaultPublicSettings),[tokenDraft,setTokenDraft]=useState(''),[clearToken,setClearToken]=useState(false),[sampleId,setSampleId]=useState(''); const refresh=async()=>{const s=await api.snapshot();setSnap(s);setProgress(s.progress);return s;}; const [runtimes,setRuntimes]=useState([]); + // Text-only copies of vision models. The vision toggle needs these to mean anything: LM Studio + // attaches a projector from the model's index entry and offers no way to load without it. + const [twins,setTwins]=useState([]); const discover=async()=>{try{const m=await api.models();setModels(m);setConnected(true);setConnectionError(''); // Which llama.cpp builds LM Studio has. Best effort: an older LM Studio that cannot list them // simply leaves the run on whatever engine is already selected, exactly as before. api.runtimes().then(setRuntimes).catch(()=>setRuntimes([])); + api.textOnlyTwins().then(setTwins).catch(()=>setTwins([])); return m;}catch(e){setConnected(false);setConnectionError((e as Error).message);return [];}}; const act=async(name:string,fn:()=>Promise)=>{setBusy(name);setError('');try{await fn();}catch(e){setError((e as Error).message);}finally{setBusy('');}}; useEffect(()=>{if(!api){setError('Open this app through Local Model Bench.exe. The browser preview cannot access desktop features.');return;}refresh().then(s=>{setSettingsDraft(s.settings);setConfig(c=>({...c,timeoutSec:s.settings.timeoutSec}));if(s.runs[0])setRunId(s.runs[0].id);}).catch(e=>setError(e.message));discover();return api.onProgress(p=>{setProgress(p);if(p)setRunId(p.runId);else{refresh();discover();}});},[]); @@ -98,6 +104,8 @@ function App(){ // Confirmed MTP support is already required of every selected model whenever MTP is on, which a // sweep always is; the only extra way to block a sweep is to have emptied its list of depths. const sweepBlocked=!!config.mtpSweep&&!config.mtpSweep.length; + // What "Off / text-only" would actually take, given what is selected and which copies exist. + const textOnly=useMemo(()=>planTextOnlySwap(config.modelKeys,models,twins),[config.modelKeys,models,twins]); const allowedReasoning=reasoningOptions(models,config.modelKeys); const invalidReasoning=config.reasoning!=='default'&&!allowedReasoning.includes(config.reasoning); const visionMode=config.vision??'auto'; @@ -147,6 +155,21 @@ function App(){ {mtpUnsupported.length>0&&

Native MTP is not confirmed for: {mtpUnsupported.map(m=>`${m.display_name} — ${m.nativeMtp?.reason??'LM Studio reports nothing about MTP for this file.'}`).join(' ')} Deselect them or turn MTP off.

} {mtpSidecar.length>0&&

{mtpSidecar.map(m=>m.display_name).join(', ')} keep{mtpSidecar.length===1?'s':''} MTP heads in a separate file rather than inside the model. LM Studio exposes no command-line switch for those, so this app writes the setting into LM Studio’s own per-model configuration for the length of each load and restores it immediately afterwards. Every load is then checked against LM Studio’s engine log, and nothing is measured unless that log names this exact head.

}}
Vision mode
{(['auto','off','on'] as const).map(mode=>)}
LM Studio has no load-time switch for a vision projector: it belongs to the model entry itself, so this chooses whether images are sent, and On additionally requires LM Studio to report the model as vision-capable.

{visionMode==='on'?'Images are sent as local data URLs only; a deterministic image smoke test runs alongside your text tests. Nothing leaves this machine.':visionMode==='off'?'No image is sent with any request. '+noProjectorToggleNote:'LM Studio’s own defaults; this app sends no image.'}

{visionMode==='on'&&<>{visionCapable.length>0&&}{!!visionUnsupported.length&&

Vision is not confirmed for: {visionUnsupported.map(m=>m.display_name).join(', ')}. LM Studio reports no vision projector for these model entries. Deselect them, or choose Auto or Off to benchmark them as text-only — a separate non-vision copy of the model is not needed.

}}{visionMode!=='on'&&selectedModels.some(m=>visionSupport(m).supported===true)&&

A vision-capable model stays fully usable for text benchmarks; its projector remains loaded either way.

} + {visionMode==='off'&&textOnly.projectored.length>0&&
+

{textOnlySummary(textOnly)}

+
+ {textOnly.swappable.length>0&&} + {textOnly.needTwin.length>0&&} +
+ {textOnlyIncomplete(textOnly)&&

Until then this run still loads those projectors; text-only only stops images being sent.

} +
}
{models.map(m=>)}{!models.length&&

Connect to LM Studio from the Models screen first.

}
 {models.some(m=>m.loaded_instances.length>0)&&

Other models are loaded: {models.filter(m=>m.loaded_instances.length).map(m=>m.display_name).join(', ')}. These stay loaded and can affect memory and measurements.

} {config.mode!=='quality'&&
{['short','medium','long'].map(n=>)}
} diff --git a/src/prefill.ts b/src/prefill.ts index 65b2954..df278eb 100644 --- a/src/prefill.ts +++ b/src/prefill.ts @@ -23,6 +23,7 @@ // was silently charging to the GPU. import type {Run} from './types'; +import {mtpDepthLabel,mtpDepthText} from './mtp-sweep'; export type PrefillPoint={tokens:number;ms:number}; export type PrefillCalibration={ @@ -73,9 +74,21 @@ export const prefillText=(c:PrefillCalibration|null|undefined):string=> // Calibrated prompt processing is measured once per loaded instance rather than per request, so it // is looked up by model (and by depth when a sweep loaded the model more than once) rather than // living on the row. Rows pooled from other runs carry their own copy instead; see HistoryRow. +// The key a calibration is stored under. mtpDepthText, never 'MTP ' + mtpDepthLabel: the label for +// depth 0 is already 'MTP off', so the shorter form spells 'MTP MTP off' — the same trap the sweep +// labels carry a warning about. +export const calibrationKey=(modelKey:string,depth:number|null):string=> + depth===null?`prefill:${modelKey}`:`prefill:${modelKey} · ${mtpDepthText(depth)}`; +// Runs saved before that was fixed used 'MTP ' + the label, doubling the prefix at depth 0. They are +// still read, because a key format is not worth losing a finished run's measurements over. +const legacyKey=(modelKey:string,depth:number):string=>`prefill:${modelKey} · MTP ${mtpDepthLabel(depth)}`; export function calibrationFor(run:Run,modelKey:string,depth:number|null):PrefillCalibration|null{ const info=run.modelInfo as Record; - const exact=depth===null?undefined:info[`prefill:${modelKey} · MTP ${depth===0?'off':String(depth)}`]; - const value=exact??info[`prefill:${modelKey}`]; - return value&&typeof value==='object'&&'marginalTps' in (value as object)?value as PrefillCalibration:null; + const candidates=depth===null?[]:[calibrationKey(modelKey,depth),legacyKey(modelKey,depth)]; + // The undepthed key last: a run with no sweep stores exactly one calibration under it. + for(const key of [...candidates,`prefill:${modelKey}`]){ + const value=info[key]; + if(value&&typeof value==='object'&&'marginalTps' in (value as object))return value as PrefillCalibration; + } + return null; } diff --git a/src/style.css b/src/style.css index 75aa802..d414274 100644 --- a/src/style.css +++ b/src/style.css @@ -33,3 +33,10 @@ .engine-compare{margin-bottom:22px} .engine-compare .segmented{margin:0} .engine-compare .hint{margin:10px 0 0} + +/* Making the vision toggle actionable: shown only when "Off / text-only" is selected and something + in the run would still load a projector. */ +.text-only-swap{margin:-8px 0 20px;padding:14px 16px;border:1px solid #3e4a3a;border-radius:8px;background:#1d2420} +.text-only-swap .hint{margin:0 0 10px} +.text-only-swap .button-row button{font-size:12px} +.text-only-swap .hint.amber{margin:10px 0 0} diff --git a/src/text-only-swap.ts b/src/text-only-swap.ts new file mode 100644 index 0000000..16d22fa --- /dev/null +++ b/src/text-only-swap.ts @@ -0,0 +1,59 @@ +import type {Model} from './types'; +import type {Twin} from '../electron/text-only'; + +// Making "Off / text-only" mean what it says. +// +// The vision toggle only ever controlled whether an image was *sent*. It could not stop the +// projector loading, because LM Studio attaches one from the model's index entry and from nowhere +// else — no command-line flag, no load-config key, no field of the load endpoint. Checked again +// against LM Studio 2.40.0: none of the 24 `llm.load.*` keys in its bundle mentions vision, and its +// llama-server carries no --mmproj flag at all, because it drives the engine through its own +// bindings rather than a command line. +// +// What can be done is give LM Studio a second folder holding the same weights and no projector — +// which this app already builds, as a tree of hard links costing no disk. That produces a separate +// model key that genuinely loads without vision weights. This module is the bridge: it works out, +// for the models a run has selected, which ones still carry a projector and what to select instead. + +export type TextOnlyPlan={ + // Selected models that would still load a projector, and so are not text-only whatever the toggle says. + projectored:Model[]; + // Those of them that already have a twin LM Studio has indexed, with the key to select instead. + swappable:{from:string;to:string;name:string}[]; + // Those with no usable twin yet; a twin has to be made before they can be swapped. + needTwin:Model[]; + // The selection with every available swap applied, ready to hand back to the config. + nextKeys:string[]; +}; + +const isProjectored=(m:Model|undefined):m is Model=>!!m&&m.capabilities?.vision===true; + +export function planTextOnlySwap(selected:string[],models:Model[],twins:Twin[]):TextOnlyPlan{ + const byKey=new Map(models.map(m=>[m.key,m])); + // A twin is only useful once LM Studio has indexed it and given it a key of its own. + const twinFor=new Map(twins.filter(t=>t.key&&t.sourceKey).map(t=>[t.sourceKey,t.key!])); + const projectored:Model[]=[],swappable:TextOnlyPlan['swappable']=[],needTwin:Model[]=[]; + const nextKeys=selected.map(key=>{ + const model=byKey.get(key); + if(!isProjectored(model))return key; + projectored.push(model); + const to=twinFor.get(key); + // A twin whose key is not in the model list is one LM Studio has forgotten about; treat it as + // missing rather than selecting a key that would fail to load. + if(to&&byKey.has(to)){swappable.push({from:key,to,name:model.display_name});return to;} + needTwin.push(model); + return key; + }); + return {projectored,swappable,needTwin,nextKeys}; +} + +// Whether the toggle is making a promise it cannot keep as things stand. +export const textOnlyIncomplete=(plan:TextOnlyPlan):boolean=>plan.needTwin.length>0; + +export function textOnlySummary(plan:TextOnlyPlan):string{ + if(!plan.projectored.length)return 'None of the selected models has a vision projector, so every load is already text-only.'; + const parts:string[]=[]; + if(plan.swappable.length)parts.push(`${plan.swappable.length} can be swapped to a text-only copy now`); + if(plan.needTwin.length)parts.push(`${plan.needTwin.length} need a text-only copy making first`); + return `${plan.projectored.length} selected model${plan.projectored.length===1?'':'s'} would still load a projector: ${parts.join(', ')}.`; +} diff --git a/tests/prefill.test.ts b/tests/prefill.test.ts index 8fd8c77..8bf7efe 100644 --- a/tests/prefill.test.ts +++ b/tests/prefill.test.ts @@ -1,7 +1,8 @@ import {test} from 'node:test'; import assert from 'node:assert/strict'; -import {calibratePrefill,calibrationPrompt,calibrationRepeats,minDeltaMs,minTokenRatio,prefillText} from '../src/prefill'; +import {calibratePrefill,calibrationFor,calibrationKey,calibrationPrompt,calibrationRepeats,minDeltaMs,minTokenRatio,prefillText} from '../src/prefill'; import {parseRuntimes,runtimeChoices,runtimeLabel} from '../electron/runtime'; +import type {Run} from '../src/types'; import {cacheScale,cacheScalePair,cacheQuantText,needsFlashAttention} from '../src/cache-quant'; // The case the calibration exists for: a fixed per-request cost that a single measurement charges @@ -95,3 +96,31 @@ test('cache quantization scales memory and is described honestly',()=>{ assert.equal(needsFlashAttention('f16','f16'),false); assert.equal(needsFlashAttention('off','q4_0'),true); }); + +// The bug this caught on real data: the engine stored calibrations under one key shape and the +// results screen looked them up under another, so the calibrated column was blank on every sweep — +// the exact runs it was built for. Emit and lookup now share calibrationKey(), and the shape saved +// before the fix is still read. +test('a calibration is found under the key the engine actually wrote',()=>{ + const c=(tps:number)=>({marginalTps:tps,overheadMs:0,naiveTps:null,note:'',small:{tokens:1,ms:1},big:{tokens:9,ms:9}}); + assert.equal(calibrationKey('m@q4',0),'prefill:m@q4 · MTP off','depth 0 is "MTP off", never "MTP MTP off"'); + assert.equal(calibrationKey('m@q4',1),'prefill:m@q4 · MTP 1 token'); + assert.equal(calibrationKey('m@q4',2),'prefill:m@q4 · MTP 2 tokens'); + assert.equal(calibrationKey('m@q4',null),'prefill:m@q4','a run with no sweep keeps the plain key'); + + // Round trip through the canonical key, per depth. + const canonical={modelInfo:Object.fromEntries([0,1,2].map(d=>[calibrationKey('m@q4',d),c(100+d)]))} as unknown as Run; + for(const d of [0,1,2]) assert.equal(calibrationFor(canonical,'m@q4',d)?.marginalTps,100+d); + + // Runs saved before the fix used 'MTP ' + label, which doubled the prefix at depth 0. + const legacy={modelInfo:{'prefill:m@q4 · MTP MTP off':c(7),'prefill:m@q4 · MTP 2 tokens':c(9)}} as unknown as Run; + assert.equal(calibrationFor(legacy,'m@q4',0)?.marginalTps,7,'the old depth-0 key is still read'); + assert.equal(calibrationFor(legacy,'m@q4',2)?.marginalTps,9); + + // A non-sweep run stores one calibration under the plain key; a depth still finds it. + const plain={modelInfo:{'prefill:m@q4':c(5)}} as unknown as Run; + assert.equal(calibrationFor(plain,'m@q4',0)?.marginalTps,5); + assert.equal(calibrationFor(plain,'m@q4',null)?.marginalTps,5); + // And nothing is invented when nothing was measured. + assert.equal(calibrationFor({modelInfo:{}} as unknown as Run,'m@q4',1),null); +}); diff --git a/tests/text-only-swap.test.ts b/tests/text-only-swap.test.ts new file mode 100644 index 0000000..e6ec773 --- /dev/null +++ b/tests/text-only-swap.test.ts @@ -0,0 +1,61 @@ +import {test} from 'node:test'; +import assert from 'node:assert/strict'; +import {planTextOnlySwap,textOnlyIncomplete,textOnlySummary} from '../src/text-only-swap'; +import type {Model} from '../src/types'; +import type {Twin} from '../electron/text-only'; + +const model=(key:string,vision:boolean,name=key):Model=>({ + key,display_name:name,size_bytes:12040883104,format:'gguf',max_context_length:8192,type:'llm', + loaded_instances:[],quantization:null,capabilities:{vision}, +} as unknown as Model); +const twin=(sourceKey:string,key:string|null):Twin=>({folder:'/m/x - Text only',source:'/m/x',sourceKey,created:'',key}); + +const vis=model('qwen - vision@iq3_s',true,'Qwen 27B Vision'); +const txt=model('qwen - text only@iq3_s',false,'Qwen 27B Text only'); +const plain=model('qwen@iq3_s',false,'Qwen 27B'); + +test('a selection with no projector is already text-only and nothing is proposed',()=>{ + const p=planTextOnlySwap(['qwen@iq3_s'],[plain],[]); + assert.deepEqual(p.projectored,[]); + assert.deepEqual(p.nextKeys,['qwen@iq3_s'],'the selection is untouched'); + assert.equal(textOnlyIncomplete(p),false); + assert.match(textOnlySummary(p),/already text-only/); +}); + +test('a vision model with an indexed twin is swapped to the twin',()=>{ + const p=planTextOnlySwap(['qwen - vision@iq3_s'],[vis,txt],[twin('qwen - vision@iq3_s','qwen - text only@iq3_s')]); + assert.deepEqual(p.nextKeys,['qwen - text only@iq3_s']); + assert.deepEqual(p.swappable,[{from:'qwen - vision@iq3_s',to:'qwen - text only@iq3_s',name:'Qwen 27B Vision'}]); + assert.deepEqual(p.needTwin,[]); + assert.equal(textOnlyIncomplete(p),false); +}); + +test('a vision model with no twin is named rather than silently left as it was',()=>{ + const p=planTextOnlySwap(['qwen - vision@iq3_s'],[vis],[]); + assert.deepEqual(p.nextKeys,['qwen - vision@iq3_s'],'nothing is swapped to a key that does not exist'); + assert.deepEqual(p.needTwin.map(m=>m.key),['qwen - vision@iq3_s']); + assert.equal(textOnlyIncomplete(p),true,'the toggle cannot keep its promise yet, and says so'); + assert.match(textOnlySummary(p),/need a text-only copy making first/); +}); + +// A twin LM Studio has not indexed has no key to select, and one whose key it has since forgotten +// would fail to load. Both are treated as missing rather than swapped to. +test('a twin without a usable key counts as missing',()=>{ + const unindexed=planTextOnlySwap(['qwen - vision@iq3_s'],[vis],[twin('qwen - vision@iq3_s',null)]); + assert.equal(unindexed.needTwin.length,1); + assert.deepEqual(unindexed.nextKeys,['qwen - vision@iq3_s']); + const stale=planTextOnlySwap(['qwen - vision@iq3_s'],[vis],[twin('qwen - vision@iq3_s','gone@iq3_s')]); + assert.equal(stale.needTwin.length,1,'a key LM Studio no longer lists is not selected'); + assert.deepEqual(stale.nextKeys,['qwen - vision@iq3_s']); +}); + +test('a mixed selection swaps what it can and reports the rest',()=>{ + const other=model('gemma - vision@q4',true,'Gemma Vision'); + const p=planTextOnlySwap(['qwen@iq3_s','qwen - vision@iq3_s','gemma - vision@q4'],[plain,vis,txt,other], + [twin('qwen - vision@iq3_s','qwen - text only@iq3_s')]); + assert.deepEqual(p.nextKeys,['qwen@iq3_s','qwen - text only@iq3_s','gemma - vision@q4'],'order is preserved'); + assert.equal(p.swappable.length,1); + assert.deepEqual(p.needTwin.map(m=>m.key),['gemma - vision@q4']); + assert.match(textOnlySummary(p),/2 selected models would still load a projector/); + assert.match(textOnlySummary(p),/1 can be swapped.*1 need/); +}); From abcdcda854de6f5a1187a59b41b610a7d838db50 Mon Sep 17 00:00:00 2001 From: M007-Net <262222560+M007-Net@users.noreply.github.com> Date: Thu, 17 Sep 2026 06:59:48 -0400 Subject: [PATCH 6/8] Local Model Bench 1.13.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The three starter JSON tests now accept a fenced answer. subnet, facts and json-extract ask for "only JSON" and say nothing about code fences. A model that returns correct JSON wrapped in ```json was being scored zero for a markdown habit the prompt never mentioned. Measured on Gemma 4 26B A4B: all five subnet values right, scored 0; all four extraction values right, scored 20. Those three set allowCodeFence, and stored copies are migrated once on startup so existing installs pick the change up instead of only new ones. A test edited by hand keeps whatever rules it has. The accounting pack is deliberately excluded. Its prompt says "no prose or code fences", which makes the fence part of what that pack tests, and its assertions that a fenced answer scores zero still hold. So does any test that has not opted in, and a fence that never closes, several fenced blocks, or prose around the JSON are all still failures — a fence is markup around the answer, not prose instead of it. This changes what those three tests score, so a quality comparison spanning the change compares two scoring rules as well as two models. Runs already saved keep the numbers they were given. The core test that asserted the old behaviour is updated rather than removed, and now also pins the thing that must not change: a fenced *wrong* answer earns only the valid-JSON check and nothing for the values inside the fence. 253 offline tests, typecheck and production build all pass. Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 38 +++++++++++++++++++ electron/json-answer.ts | 17 +++++++++ electron/main.ts | 9 +++++ electron/scoring.ts | 12 ++++-- gemma26run.ts | 83 +++++++++++++++++++++++++++++++++++++++++ package-lock.json | 4 +- package.json | 2 +- src/defaults.ts | 9 ++++- src/types.ts | 7 +++- tests/core.test.ts | 6 ++- tests/unfence.test.ts | 58 ++++++++++++++++++++++++++++ 11 files changed, 234 insertions(+), 11 deletions(-) create mode 100644 gemma26run.ts create mode 100644 tests/unfence.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 847b5a7..1dbbf97 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,44 @@ All notable changes to Local Model Bench are recorded here. The format follows Dates are the date the version was prepared. +## 1.13.0 — 2026-09-17 + +### Changed + +- **The three starter JSON tests now accept a fenced answer.** subnet, facts and + json-extract ask for "only JSON" and say nothing about code fences, so a model that + wraps correct JSON in ```json was being scored zero for a markdown habit the prompt + never mentioned. Measured on Gemma 4 26B A4B: every subnet value correct, scored 0; + every extraction value correct, scored 20. Those three tests set `allowCodeFence`, + and already-saved copies are migrated once on startup, so existing installs pick the + change up rather than only new ones. A test you have edited yourself keeps whatever + rules it has. + + The accounting pack is deliberately excluded: its prompt says "no prose or code + fences", which makes the fence part of what it tests, and it still fails one. So does + any test that has not opted in, along with a fence that never closes, several fenced + blocks, and prose around the JSON. + + This changes what those three tests score. Runs saved before it keep the numbers they + were given, so a quality comparison that spans the change is comparing two scoring + rules as well as two models. + +## 1.12.1 — 2026-09-17 + +### Fixed + +- **A correct JSON answer inside a code fence scored zero.** Models asked for "only + JSON" very often return it wrapped in a ```json fence. That is markup around the + answer rather than prose instead of it, and this app already unwrapped exactly the + same wrapper when reading a judge's reply — but not when scoring the model being + tested. Measured on Gemma 4 26B A4B: all five subnet values correct, fenced, every + check failing with "Unexpected token '`'", for a score of 0; and the extraction test + scored 20 with all four values right. The JSON-shaped checks now read the answer with + a single complete surrounding fence removed, and say so in the check detail rather + than unwrapping silently. A fence that never closes, a reply holding several fenced + blocks, and prose around the JSON are all still failures, because those are different + things to get wrong. Every other check still sees exactly what the model wrote. + ## 1.12.0 — 2026-09-17 ### Added diff --git a/electron/json-answer.ts b/electron/json-answer.ts index 6b7e8e3..7a503c6 100644 --- a/electron/json-answer.ts +++ b/electron/json-answer.ts @@ -1,3 +1,20 @@ +// A model asked for "only JSON" very often returns it inside a ```json fence. That is markup around +// the answer, not prose instead of it, and this app already unwraps exactly the same wrapper when it +// reads a judge's reply (see parseGrade). Not doing it here scored a perfect answer zero: Gemma 4 +// 26B A4B returned all five subnet values correctly, fenced, and every check failed with +// "Unexpected token '`'". Only a single fence that opens the text and closes it is removed; prose +// around the JSON is still a failure, because that is a different thing to get wrong. +export function unfence(text:string):{json:string;fenced:boolean}{ + const trimmed=text.trim(); + if(!trimmed.startsWith('```'))return {json:trimmed,fenced:false}; + const opener=/^```[A-Za-z0-9_-]*[^\S\r\n]*\r?\n?/; + const closer=/\r?\n?```$/; + // A fence that never closes is not a wrapper, so it is left exactly as it was. + if(!closer.test(trimmed))return {json:trimmed,fenced:false}; + const inner=trimmed.replace(opener,'').replace(closer,''); + // A fence that never closes, or that wraps several blocks, is left exactly as it was. + return inner.includes('```')?{json:trimmed,fenced:false}:{json:inner.trim(),fenced:true}; +} // JSON.parse accepts duplicate keys. Reject those so conflicting answers cannot // silently collapse to whichever value appeared last. export function parseAnswer(text:string):unknown { diff --git a/electron/main.ts b/electron/main.ts index 26344e6..b82cd35 100644 --- a/electron/main.ts +++ b/electron/main.ts @@ -118,6 +118,15 @@ if(locked)app.whenReady().then(()=>{ try{store=Store.open(path.join(dataPath,'bench.sqlite'));store.recover();}catch(e){return fatal('The saved benchmark database could not be opened.',e);} if(!store.get('seeded')){starterTests.forEach(t=>store.saveTest(t));store.set('seeded',true);} if(!store.get('accounting-v1-seeded')){const ids=new Set(store.tests().map(t=>t.id));starterTests.filter(t=>t.category==='Accounting'&&!ids.has(t.id)).forEach(t=>store.saveTest(t));store.set('accounting-v1-seeded',true);} + // Tests are seeded once and then belong to the user, so a change to a starter test's scoring has + // to be migrated rather than assumed. This one carries allowCodeFence onto the stored copies of + // the three JSON tests whose prompts never asked for an unfenced answer; anything the user has + // since edited keeps its own rules and prompt, and the accounting pack is untouched. + if(!store.get('fence-tolerance-v1')){ + const tolerant=new Set(starterTests.filter(t=>t.allowCodeFence).map(t=>t.id)); + for(const t of store.tests())if(tolerant.has(t.id)&&!t.allowCodeFence)store.saveTest({...t,allowCodeFence:true}); + store.set('fence-tolerance-v1',true); + } handle('snapshot',()=>({settings:publicSettings(),tests:store.tests(),runs:store.list(),progress,dataPath,packs:allPacks().map(describePack)})); handle('modelProfiles',()=>store.profiles()); handle('saveModelProfile',(key,profile)=>store.saveProfile(key,profile)); diff --git a/electron/scoring.ts b/electron/scoring.ts index 365d2d4..08bcad2 100644 --- a/electron/scoring.ts +++ b/electron/scoring.ts @@ -1,12 +1,16 @@ import type { TestCase, Objective, Grade, Sample } from '../src/types'; -import { parseAnswer, equalAnswer } from './json-answer'; +import { parseAnswer, equalAnswer, unfence } from './json-answer'; import {finalNumber,scoreInstructions,GraderError} from './benchmark-scoring'; export function objectiveScore(output:string,test:TestCase):Objective { const checks=test.rules.map(r=>{ let passed=false,detail='',unscorable=false; try { const s=output.trim(); + // JSON-shaped checks read the answer with a single surrounding code fence removed; every other + // check still sees exactly what the model wrote. + const {json:j,fenced}=test.allowCodeFence?unfence(s):{json:s,fenced:false}; + const note=fenced?' (unwrapped from a code fence)':''; switch(r.type){ case 'final-number': {const n=finalNumber(s);passed=n!==null&&n===Number(r.expected);detail=passed?'Correct final numeric answer':`Expected ${r.expected}; received ${n??'no valid final number'}`;break;} case 'ifeval': {const result=scoreInstructions(s,JSON.parse(r.expected??'[]'));passed=result.passed;detail=result.detail;break;} @@ -14,9 +18,9 @@ export function objectiveScore(output:string,test:TestCase):Objective { case 'contains': passed=s.toLocaleLowerCase().includes((r.expected??'').toLocaleLowerCase());detail=passed?'Required text present':'Required text missing';break; case 'heading': passed=s.split(/\r?\n/).some(line=>line.trim().replace(/^#{1,6}\s+/, '').replace(/^\*\*(.*?)\*\*$/, '$1').replace(/:$/, '').trim().toLowerCase()===(r.expected??'').trim().toLowerCase());detail=passed?'Standalone heading found':'Expected a standalone plain or Markdown heading';break; case 'number': { const n=Number(s); passed=s!==''&&Number.isFinite(n)&&Math.abs(n-Number(r.expected))<=(r.tolerance??0);detail=passed?'Within tolerance':'Expected a single numeric answer within tolerance';break; } - case 'json-equal': {const mismatch=equalAnswer(parseAnswer(s),parseAnswer(r.expected??''));passed=mismatch===null;detail=mismatch??'All required values match exactly';break;} - case 'json': JSON.parse(s);passed=true;detail='Valid JSON without surrounding prose';break; - case 'field': {let actual:any=JSON.parse(s);for(const p of (r.path??'').split('.')){if(p==='__proto__'||p==='constructor'||p==='prototype')throw Error('Invalid path');actual=actual?.[p];}let expected:any=r.expected;try{expected=JSON.parse(r.expected??'');}catch{}passed=JSON.stringify(actual)===JSON.stringify(expected);detail=`Actual: ${JSON.stringify(actual)??'missing'}; expected: ${JSON.stringify(expected)}`;break;} + case 'json-equal': {const mismatch=equalAnswer(parseAnswer(j),parseAnswer(r.expected??''));passed=mismatch===null;detail=mismatch??'All required values match exactly'+note;break;} + case 'json': JSON.parse(j);passed=true;detail='Valid JSON without surrounding prose'+note;break; + case 'field': {let actual:any=JSON.parse(j);for(const p of (r.path??'').split('.')){if(p==='__proto__'||p==='constructor'||p==='prototype')throw Error('Invalid path');actual=actual?.[p];}let expected:any=r.expected;try{expected=JSON.parse(r.expected??'');}catch{}passed=JSON.stringify(actual)===JSON.stringify(expected);detail=`Actual: ${JSON.stringify(actual)??'missing'}; expected: ${JSON.stringify(expected)}`;break;} case 'words': {const n=s?s.split(/\s+/u).length:0;passed=n>=(r.min??0)&&n<=(r.max??Infinity);detail=`${n} words; expected ${r.min??0}–${r.max??'unlimited'}`;break;} case 'lines': {const lines=s.split(/\r?\n/).filter(x=>x.trim());passed=lines.length>=(r.min??0)&&lines.length<=(r.max??Infinity)&&(!r.expected||lines.every(x=>x.startsWith(r.expected!)));detail=`${lines.length} nonempty lines${r.expected?'; required prefix: '+r.expected:''}`;break;} } diff --git a/gemma26run.ts b/gemma26run.ts new file mode 100644 index 0000000..c901730 --- /dev/null +++ b/gemma26run.ts @@ -0,0 +1,83 @@ +import {randomUUID} from 'node:crypto'; +import {homedir} from 'node:os'; +import path from 'node:path'; +import {runEngine} from './electron/engine'; +import {defaultConfig, defaultSettings, performanceTest, starterTests} from './src/defaults'; +import {listModels} from './electron/lmstudio'; +import {Store} from './electron/store'; +import type {Run, RunConfig} from './src/types'; + +const dataDir = process.argv[2]; +const which = process.argv[3]; // 'quants' | 'mtp' +const engine = process.argv[4] ?? 'llama.cpp-win-x86_64-amd-rocm-avx2@2.40.0'; +const engineName = engine.includes('rocm') ? 'ROCm' : 'Vulkan'; +const settings = {...defaultSettings, lmsPath: path.join(homedir(), '.lmstudio', 'bin', 'lms.exe'), timeoutSec: 600, loadTimeoutSec: 900}; +const store = Store.open(path.join(dataDir, 'bench.sqlite')); + +const IQ3 = 'gemma-4-26b-a4b-it'; // IQ3_XXS, text-only, MTP sidecar +const Q4 = 'gemma-4-26b-a4b-it-qat-ud'; // Q4_K_XL QAT, text-only, no MTP +// Tests whose objective checks are crisp: an exact answer, exact JSON, or an exact two-line reply. +const OBJECTIVE = ['subnet', 'json-extract', 'reasoning', 'instructions']; + +async function main() { + const models = await listModels(settings); + const keys = which === 'mtp' ? [IQ3] : [IQ3, Q4]; + for (const k of keys) { + const m = models.find(x => x.key === k); + if (!m) throw Error(`not in LM Studio: ${k}`); + if (m.capabilities?.vision === true) throw Error(`${k} carries a vision projector; this run is text-only`); + if (which === 'mtp' && m.nativeMtp?.supported !== true) throw Error(`${k}: MTP not confirmed`); + console.log(` ok ${k} · ${(m.size_bytes / 1024 ** 3).toFixed(2)} GiB · ${m.quantization?.name} · text-only`); + } + const tests = [...starterTests.filter(t => OBJECTIVE.includes(t.id)), performanceTest('medium')]; + const config: RunConfig = { + ...defaultConfig, + name: which === 'mtp' ? `Gemma 4 26B A4B IQ3_XXS · MTP sweep · ${engineName} (reasoning off)` : `Gemma 4 26B A4B · quant comparison · ${engineName} (f16 cache, reasoning off)`, + modelKeys: keys, mode: 'combined', preset: 'Custom', + testIds: OBJECTIVE, performanceLengths: ['medium'], + concurrency: [1], waves: 2, maxTokens: 1024, contextLength: 8192, + // Reasoning OFF, deliberately. These tests cap themselves at 256 output tokens, and with + // reasoning on this model spent 253 of them thinking and emitted no answer at all — every + // objective check then failed on an empty string, scoring 0 for reasons that had nothing to + // do with the quantization being compared. + temperature: 0, gpu: 'auto', reasoning: 'off', timeoutSec: 600, + judgeModel: '', // Objective checks only — no judge model. + vision: 'off', + mtp: which === 'mtp' ? 'on' : 'off', + mtpDraftTokens: 2, + ...(which === 'mtp' ? {mtpSweep: [0, 1, 2, 3]} : {mtpSweep: undefined}), + runtime: engine, + // f16, explicitly — not 'off'. 'off' means "leave LM Studio's own setting alone", and this + // machine has a stored q4_0 cache for the QAT model with no flash attention, which LM Studio + // refuses to load. Writing f16 states the condition for the load and restores it afterwards, + // and matches the unquantized cache the MTP run measured under. + cacheK: 'f16', cacheV: 'f16', + }; + const now = new Date().toISOString(); + const run: Run = { + id: randomUUID(), created: now, updated: now, status: 'running', config, tests, + modelInfo: Object.fromEntries(models.filter(m => keys.includes(m.key)).map(m => [m.key, {model: m}])), + environment: {platform: 'win32', appVersion: '1.12.0', runtime: '', note: 'Started headlessly at the user’s request.'}, + logs: [], samples: [], waves: [], + }; + store.saveRun(run); + console.log(`run ${run.id.slice(0, 8)} "${config.name}" — ${keys.length} model(s), ${tests.length} tests`); + const ac = new AbortController(); + await runEngine(run, settings, ac.signal, e => { + if (e.type === 'sample') { + run.samples.push(e.sample); store.saveSample(e.sample); + if (!e.sample.warmup) console.log(` ${e.sample.modelKey.slice(0, 24).padEnd(24)} ${String(e.sample.testId).padEnd(13)} d=${e.sample.mtpTokens ?? '-'} ${e.sample.status} gen=${e.sample.metrics.generationTps?.toFixed(1) ?? '-'} obj=${e.sample.objective.score ?? '-'}`); + } + if (e.type === 'wave') { run.waves.push(e.wave); store.saveWave(e.wave); } + if (e.type === 'model') run.modelInfo[e.key] = e.info; + if (e.type === 'log') { run.logs.push(new Date().toLocaleTimeString() + ' ' + e.message); if (/Prompt processing|Model failed|Runtime:|Native MTP/.test(e.message)) console.log(' log: ' + e.message.slice(0, 140)); } + if (e.type === 'gpu') run.gpu = e.gpu; + if (e.type === 'finish') { run.status = e.status; if (e.error) run.error = e.error; } + run.updated = new Date().toISOString(); + store.saveRun(run); + }); + run.updated = new Date().toISOString(); store.saveRun(run); + const ok = run.samples.filter(s => !s.warmup && s.status === 'completed'); + console.log(`\nDONE ${which} status=${run.status} measured=${ok.length}${run.error ? ' error=' + run.error.slice(0, 200) : ''}`); +} +main().then(() => process.exit(0)).catch(e => { console.error('FAILED:', e.message); process.exit(1); }); diff --git a/package-lock.json b/package-lock.json index 18f4fdc..75c9b24 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "local-model-bench", - "version": "1.12.0", + "version": "1.13.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "local-model-bench", - "version": "1.12.0", + "version": "1.13.0", "dependencies": { "lucide-react": "^0.468.0", "react": "^19.1.0", diff --git a/package.json b/package.json index fca8367..f41c4e1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "local-model-bench", - "version": "1.12.0", + "version": "1.13.0", "description": "Private desktop benchmarks and transparent grading for LM Studio models", "author": "Local Model Bench contributors", "license": "MIT", diff --git a/src/defaults.ts b/src/defaults.ts index 3cd5059..f9b3301 100644 --- a/src/defaults.ts +++ b/src/defaults.ts @@ -14,7 +14,12 @@ const rubric='Score correctness, completeness, clarity, and instruction followin const rule=(id:string,label:string,type:Rule['type'],extra:Partial={}):Rule=>({id,label,type,weight:1,...extra}); const test=(id:string,name:string,category:string,prompt:string,rules:Rule[]=[],answerKey='',maxTokens=1024):TestCase=>({id,name,category,prompt,rules,answerKey,maxTokens,version:1,rubric,kind:'quality'}); import { accountingTests } from './accounting'; -export const starterTests:TestCase[]=[ +// These three ask for "only JSON" and never mention code fences, so a fenced answer is the model +// following the instruction in the usual markdown way rather than disobeying it. The accounting +// pack is deliberately not here: its prompt says "no prose or code fences", which makes the fence +// part of what that pack tests. +const fenceTolerant=new Set(['subnet','facts','json-extract']); +export const starterTests:TestCase[]=([ test('report-general','Explain renewable energy','Report','Write a 350–500 word report comparing solar and wind power from your own knowledge. Use the headings Overview, Tradeoffs, and Recommendation. Explain intermittency, storage, and one appropriate use case for each. State uncertainties; do not invent citations.',[rule('length','350–500 words','words',{min:350,max:500}),...['Overview','Tradeoffs','Recommendation'].map(x=>rule(x,x+' heading','heading',{expected:x}))]), test('report-network','Design a small office network','Networking','Write a 350–500 word report recommending a network for a 30-person office with staff devices, guest Wi-Fi, and an internal file server. Use the headings Design, Security, and Operations. Explain VLANs, DHCP, DNS, firewall rules, backups, and monitoring. Work from your own knowledge; do not invent references.',[rule('length','350–500 words','words',{min:350,max:500}),...['Design','Security','Operations'].map(x=>rule(x,x+' heading','heading',{expected:x}))]), test('report-security','Respond to a phishing incident','Security','Write a concise incident-response report for an employee who entered their password on a phishing page. Use the headings Immediate actions, Investigation, and Prevention. Distinguish password reset, session revocation, MFA, and evidence preservation. Work from your own knowledge.', ['Immediate actions','Investigation','Prevention'].map(x=>rule(x,x+' heading','heading',{expected:x}))), @@ -26,7 +31,7 @@ export const starterTests:TestCase[]=[ test('instructions','Follow exact constraints','Instructions','Return exactly two lines and no other text. First line: TCP is connection-oriented. Second line: UDP is connectionless.',[rule('exact','Exact required response','exact',{expected:'TCP is connection-oriented.\nUDP is connectionless.'})],'TCP is connection-oriented.\nUDP is connectionless.',128), test('debug','Find a boundary bug','Coding','Explain the bug in this JavaScript function and give a corrected version. Include headings Bug, Fix, and Example. Do not execute code.\nfunction total(xs) { let sum = 0; for (let i = 0; i <= xs.length; i++) sum += xs[i]; return sum; }',[...['Bug','Fix','Example'].map(x=>rule(x,x+' heading','heading',{expected:x}))],'The <= condition accesses xs[xs.length], which is undefined, making sum NaN. Use i < xs.length. Empty input should return 0.',768), ...accountingTests -]; +] as TestCase[]).map(t=>fenceTolerant.has(t.id)?{...t,allowCodeFence:true}:t); export function performanceTest(length:string):TestCase { const count=length==='long'?180:length==='medium'?55:4; const content=Array.from({length:count},(_,i)=>`Record ${i+1}: A regional office operates a router, two switches, staff workstations, and a file service. Backups complete nightly. Monitoring tracks latency, availability, capacity, and authentication errors.`).join('\n'); diff --git a/src/types.ts b/src/types.ts index f557946..585eb83 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,5 +1,10 @@ export type Rule = { id: string; label: string; type: 'exact'|'contains'|'heading'|'number'|'json'|'json-equal'|'field'|'words'|'lines'|'final-number'|'ifeval'; expected?: string; path?: string; tolerance?: number; min?: number; max?: number; weight: number }; -export type TestCase = { benchmark?:import('./benchmarks').BenchmarkMeta; id:string; name:string; category:string; version:number; prompt:string; answerKey:string; rubric:string; maxTokens:number; rules:Rule[]; kind:'quality'|'performance'; image?:string; imageDigest?:string }; +// allowCodeFence lets a test accept an answer wrapped in a single ```json fence. It defaults to +// false, because a prompt that says "no prose or code fences" — as the accounting pack's does — +// is testing instruction following as much as content, and a fence there is a real failure. Tests +// whose prompt asks only for "JSON only" set it true: scoring those 0 measures a markdown habit +// the prompt never mentioned. +export type TestCase = { allowCodeFence?:boolean; benchmark?:import('./benchmarks').BenchmarkMeta; id:string; name:string; category:string; version:number; prompt:string; answerKey:string; rubric:string; maxTokens:number; rules:Rule[]; kind:'quality'|'performance'; image?:string; imageDigest?:string }; export type Model = {nativeMtp?:{supported:boolean|null;reason:string;resource?:string;kind?:'bundled'|'sidecar';draftResource?:string;draftPath?:string};key:string; display_name:string; size_bytes:number; quantization:{name:string}|null; max_context_length:number; type:string; loaded_instances:{id:string; config:{context_length:number;parallel?:number;[key:string]:unknown}}[]; capabilities?:{vision?:boolean;reasoning?:{allowed_options:string[];default:string}}; [key:string]:unknown}; export type Settings = {baseUrl:string; token:string; lmsPath:string; timeoutSec:number; loadTimeoutSec:number; judgePrompt:string; updateRepo:string; updateCheck:boolean}; // What the window is allowed to see. The token itself never crosses the IPC diff --git a/tests/core.test.ts b/tests/core.test.ts index c4c90c4..28d04e4 100644 --- a/tests/core.test.ts +++ b/tests/core.test.ts @@ -26,7 +26,11 @@ test('SSE accepts one byte fragments including UTF-8 and CRLF',()=>{const d=new test('SSE handles comments and multiline data',()=>{const d=new SSEDecoder();assert.equal(d.feed(new TextEncoder().encode(': ping\n\nevent: x\ndata: {"a":\ndata: 1}\n\n'))[0].data.a,1);}); test('SSE rejects truncated final event',()=>{const d=new SSEDecoder();d.feed(new TextEncoder().encode('event: x\ndata: {'));assert.throws(()=>d.feed(new Uint8Array(),true),/Incomplete/);}); test('all bundled answer keys pass applicable objective checks',()=>{for(const id of ['subnet','reasoning','facts','json-extract','instructions']){const t=starterTests.find(t=>t.id===id)!;assert.equal(objectiveScore(t.answerKey,t).score,100,id);}}); -test('weighted scoring and strict JSON do not reward invalid answer',()=>{const t=starterTests.find(t=>t.id==='subnet')!;assert.equal(objectiveScore('```json\n'+t.answerKey+'\n```',t).score,0);assert.equal(objectiveScore('{}',t).checks.filter(c=>c.passed).length,1);}); +// The subnet prompt asks for "only JSON" and never mentions code fences, so it opts in to +// allowCodeFence and a fenced correct answer now scores full marks. Strictness is still asserted +// where the prompt actually demands it: the accounting pack (accounting.test.ts) and a test with +// the flag off (unfence.test.ts). What must never change is that an invalid answer earns nothing. +test('weighted scoring accepts a fenced answer but never rewards an invalid one',()=>{const t=starterTests.find(t=>t.id==='subnet')!;assert.equal(t.allowCodeFence,true);assert.equal(objectiveScore('```json\n'+t.answerKey+'\n```',t).score,100);const wrong=objectiveScore('```json\n{\"network\":\"10.0.0.0\"}\n```',t);assert.equal(wrong.checks.filter(c=>c.passed).length,1,'only the valid-JSON check passes');assert.ok(wrong.score!==null&&wrong.score<20,'a fenced wrong answer earns almost nothing');assert.equal(objectiveScore('{}',t).checks.filter(c=>c.passed).length,1);}); test('numeric tolerance and empty answers',()=>{const t=structuredClone(starterTests.find(t=>t.id==='reasoning')!);t.rules[0].tolerance=.1;assert.equal(objectiveScore('84.05',t).score,100);assert.equal(objectiveScore('84.2',t).score,0);assert.equal(objectiveScore('',t).score,0);}); test('heading check requires a standalone heading, not a mention in prose',()=>{const t=structuredClone(starterTests[0]);t.rules=[{id:'h',label:'Overview',type:'heading',expected:'Overview',weight:1}];assert.equal(objectiveScore('## Overview\nDetails',t).score,100);assert.equal(objectiveScore('Here is an overview of the system.',t).score,0);}); const grade=JSON.stringify({criteria:['correctness','completeness','clarity','instruction_following'].map(name=>({name,score:80,reason:'Specific evidence'})),summary:'Good'}); diff --git a/tests/unfence.test.ts b/tests/unfence.test.ts new file mode 100644 index 0000000..d95e2eb --- /dev/null +++ b/tests/unfence.test.ts @@ -0,0 +1,58 @@ +import {test} from 'node:test'; +import assert from 'node:assert/strict'; +import {unfence} from '../electron/json-answer'; +import {objectiveScore} from '../electron/scoring'; +import {starterTests} from '../src/defaults'; +import type {TestCase} from '../src/types'; + +// These two prompts ask for "JSON only" and never mention fences, so they opt in. +const subnet={...starterTests.find(t=>t.id==='subnet')!,allowCodeFence:true} as TestCase; +const extract={...starterTests.find(t=>t.id==='json-extract')!,allowCodeFence:true} as TestCase; +// The exact replies Gemma 4 26B A4B gave, which scored 0 and 20. +const FENCED_SUBNET='```json\n{\n "network": "192.168.10.64",\n "broadcast": "192.168.10.95",\n "first_host": "192.168.10.65",\n "last_host": "192.168.10.94",\n "usable_hosts": 30\n}\n```'; +const FENCED_EXTRACT='```json\n{\n "service": "inventory API",\n "duration_minutes": 18,\n "cause": "failed database migration",\n "data_loss": false\n}\n```'; + +test('a correct answer inside a code fence now scores as correct',()=>{ + const s=objectiveScore(FENCED_SUBNET,subnet); + assert.equal(s.score,100,'every subnet value was right; the fence was the only thing wrong'); + assert.ok(s.checks.every(c=>c.passed)); + assert.match(s.checks.find(c=>c.id==='json')!.detail,/code fence/,'the unwrapping is stated, not silent'); + assert.equal(objectiveScore(FENCED_EXTRACT,extract).score,100); +}); + +test('unfenced JSON is unaffected',()=>{ + const plain='{"network":"192.168.10.64","broadcast":"192.168.10.95","first_host":"192.168.10.65","last_host":"192.168.10.94","usable_hosts":30}'; + const s=objectiveScore(plain,subnet); + assert.equal(s.score,100); + assert.doesNotMatch(s.checks.find(c=>c.id==='json')!.detail,/code fence/); +}); + +test('prose around the JSON is still a failure',()=>{ + // A fence is markup around the answer; a sentence before it is not the same thing. + assert.notEqual(objectiveScore('Here is the JSON:\n'+FENCED_SUBNET,subnet).score,100); + assert.notEqual(objectiveScore('The network is 192.168.10.64.',subnet).score,100); +}); + +test('unfence only unwraps a single complete fence',()=>{ + assert.deepEqual(unfence('{"a":1}'),{json:'{"a":1}',fenced:false}); + assert.deepEqual(unfence('```json\n{"a":1}\n```'),{json:'{"a":1}',fenced:true}); + assert.deepEqual(unfence('```\n{"a":1}\n```'),{json:'{"a":1}',fenced:true},'a fence with no language still unwraps'); + // Never closed: left alone rather than half-stripped into something that parses by accident. + assert.equal(unfence('```json\n{"a":1}').fenced,false); + // Two blocks are prose containing code, not one wrapped answer. + assert.equal(unfence('```\n{"a":1}\n```\n```\n{"b":2}\n```').fenced,false); +}); + +// A prompt that forbids fences — the accounting pack says "no prose or code fences" — keeps failing +// them, because there the fence is the instruction being tested. +test('a test that did not opt in still rejects a fence',()=>{ + const strict={...subnet,allowCodeFence:false} as TestCase; + assert.equal(objectiveScore(FENCED_SUBNET,strict).score,0); +}); + +test('a wrong answer is still wrong once unfenced',()=>{ + const wrong='```json\n{"network":"10.0.0.0","broadcast":"10.0.0.255","first_host":"10.0.0.1","last_host":"10.0.0.254","usable_hosts":254}\n```'; + const s=objectiveScore(wrong,subnet); + assert.ok(s.score!==null&&s.score<100,'unwrapping the fence does not make a wrong answer right'); + assert.ok(s.checks.find(c=>c.id==='json')!.passed,'…though it is at least valid JSON now'); +}); From 77178ee0a516fd292a309ff4ad75d449b5fda0cd Mon Sep 17 00:00:00 2001 From: M007-Net <262222560+M007-Net@users.noreply.github.com> Date: Thu, 17 Sep 2026 07:05:36 -0400 Subject: [PATCH 7/8] Local Model Bench 1.14.0 The server does not have to be this machine. The endpoint was restricted to loopback, so LM Studio on another PC, a box on the LAN, or anything else speaking the same API could not be benchmarked from here. Any http:// or https:// address is now accepted. Still refused is anything that is not an origin: credentials in the URL, which belong in the API token field where they are encrypted at rest and never reach the window, and a path, query or fragment, which would change what the address means. Pointing somewhere else changes what this app can honestly claim, so it says so rather than leaving the old promise on screen. The sidebar reads "Remote endpoint" with the host in place of "Your prompts stay on this PC", and the settings field warns that prompts, responses and any token will leave the machine, noting when a plain http address means they travel unencrypted. The badge follows the saved setting rather than the draft, so it describes where runs actually go. The rules move to src/endpoint.ts: the window needs them, and the LM Studio client reaches for node:fs and node:child_process, which cannot be bundled for the renderer. The core test asserting the old loopback-only boundary is updated rather than removed, and still pins what did not change: credentials and paths are refused. 256 offline tests, typecheck and production build all pass. Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 19 ++++++++++ electron/lmstudio.ts | Bin 9874 -> 9814 bytes gemma26run.ts | 83 ----------------------------------------- package-lock.json | 4 +- package.json | 2 +- src/endpoint.ts | 28 ++++++++++++++ src/main.tsx | 11 +++++- tests/core.test.ts | 6 ++- tests/endpoint.test.ts | 31 +++++++++++++++ 9 files changed, 95 insertions(+), 89 deletions(-) delete mode 100644 gemma26run.ts create mode 100644 src/endpoint.ts create mode 100644 tests/endpoint.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 1dbbf97..780a4cf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,25 @@ All notable changes to Local Model Bench are recorded here. The format follows Dates are the date the version was prepared. +## 1.14.0 — 2026-09-17 + +### Added + +- **The server does not have to be this machine.** The endpoint was restricted to + loopback, so LM Studio on another PC, a box on the LAN, or anything else speaking the + same API could not be benchmarked from here. Any http:// or https:// address is now + accepted. What is still refused is anything that is not an origin: credentials in the + URL, which belong in the API token field where they are encrypted at rest and never + reach the window, and a path, query or fragment, which would change what the address + means. + + Pointing somewhere else changes what this app can honestly claim, so it says so rather + than leaving the old promise on screen. The sidebar reads **Remote endpoint** with the + host instead of *Your prompts stay on this PC*, and the settings field warns that + prompts, responses and any token will leave the machine — noting when a plain http + address means they travel unencrypted. The badge follows the saved setting, not what is + being typed, so it describes where runs actually go. + ## 1.13.0 — 2026-09-17 ### Changed diff --git a/electron/lmstudio.ts b/electron/lmstudio.ts index 235ebea4fd0b2c579d75e748a355d4228753ecfb..c9ca5f82b423b2809956c9b824936bb574f4fef8 100644 GIT binary patch delta 274 zcmaivF>b;@5Jjs<#Al%Rof~BL_yQ?N2M5@A2YJML$J$vi5{hV3=OjoRf`Xp&WC)3r zF5TD6)B7`@ueZhNV{)u(EIR^4KM2Q+v<;*Pyd!XgrndfFRav2eZ1l*{%ZS!RY?bb# z9$BQZ&ySHj`Ck%U}^Ec8v&F@Qw^q%mIvBrSDn0Vz(jo v12EP1Xo%*FqP9#COT~2BV)6Hytd!;6`ldrQM)K6_4 diff --git a/gemma26run.ts b/gemma26run.ts deleted file mode 100644 index c901730..0000000 --- a/gemma26run.ts +++ /dev/null @@ -1,83 +0,0 @@ -import {randomUUID} from 'node:crypto'; -import {homedir} from 'node:os'; -import path from 'node:path'; -import {runEngine} from './electron/engine'; -import {defaultConfig, defaultSettings, performanceTest, starterTests} from './src/defaults'; -import {listModels} from './electron/lmstudio'; -import {Store} from './electron/store'; -import type {Run, RunConfig} from './src/types'; - -const dataDir = process.argv[2]; -const which = process.argv[3]; // 'quants' | 'mtp' -const engine = process.argv[4] ?? 'llama.cpp-win-x86_64-amd-rocm-avx2@2.40.0'; -const engineName = engine.includes('rocm') ? 'ROCm' : 'Vulkan'; -const settings = {...defaultSettings, lmsPath: path.join(homedir(), '.lmstudio', 'bin', 'lms.exe'), timeoutSec: 600, loadTimeoutSec: 900}; -const store = Store.open(path.join(dataDir, 'bench.sqlite')); - -const IQ3 = 'gemma-4-26b-a4b-it'; // IQ3_XXS, text-only, MTP sidecar -const Q4 = 'gemma-4-26b-a4b-it-qat-ud'; // Q4_K_XL QAT, text-only, no MTP -// Tests whose objective checks are crisp: an exact answer, exact JSON, or an exact two-line reply. -const OBJECTIVE = ['subnet', 'json-extract', 'reasoning', 'instructions']; - -async function main() { - const models = await listModels(settings); - const keys = which === 'mtp' ? [IQ3] : [IQ3, Q4]; - for (const k of keys) { - const m = models.find(x => x.key === k); - if (!m) throw Error(`not in LM Studio: ${k}`); - if (m.capabilities?.vision === true) throw Error(`${k} carries a vision projector; this run is text-only`); - if (which === 'mtp' && m.nativeMtp?.supported !== true) throw Error(`${k}: MTP not confirmed`); - console.log(` ok ${k} · ${(m.size_bytes / 1024 ** 3).toFixed(2)} GiB · ${m.quantization?.name} · text-only`); - } - const tests = [...starterTests.filter(t => OBJECTIVE.includes(t.id)), performanceTest('medium')]; - const config: RunConfig = { - ...defaultConfig, - name: which === 'mtp' ? `Gemma 4 26B A4B IQ3_XXS · MTP sweep · ${engineName} (reasoning off)` : `Gemma 4 26B A4B · quant comparison · ${engineName} (f16 cache, reasoning off)`, - modelKeys: keys, mode: 'combined', preset: 'Custom', - testIds: OBJECTIVE, performanceLengths: ['medium'], - concurrency: [1], waves: 2, maxTokens: 1024, contextLength: 8192, - // Reasoning OFF, deliberately. These tests cap themselves at 256 output tokens, and with - // reasoning on this model spent 253 of them thinking and emitted no answer at all — every - // objective check then failed on an empty string, scoring 0 for reasons that had nothing to - // do with the quantization being compared. - temperature: 0, gpu: 'auto', reasoning: 'off', timeoutSec: 600, - judgeModel: '', // Objective checks only — no judge model. - vision: 'off', - mtp: which === 'mtp' ? 'on' : 'off', - mtpDraftTokens: 2, - ...(which === 'mtp' ? {mtpSweep: [0, 1, 2, 3]} : {mtpSweep: undefined}), - runtime: engine, - // f16, explicitly — not 'off'. 'off' means "leave LM Studio's own setting alone", and this - // machine has a stored q4_0 cache for the QAT model with no flash attention, which LM Studio - // refuses to load. Writing f16 states the condition for the load and restores it afterwards, - // and matches the unquantized cache the MTP run measured under. - cacheK: 'f16', cacheV: 'f16', - }; - const now = new Date().toISOString(); - const run: Run = { - id: randomUUID(), created: now, updated: now, status: 'running', config, tests, - modelInfo: Object.fromEntries(models.filter(m => keys.includes(m.key)).map(m => [m.key, {model: m}])), - environment: {platform: 'win32', appVersion: '1.12.0', runtime: '', note: 'Started headlessly at the user’s request.'}, - logs: [], samples: [], waves: [], - }; - store.saveRun(run); - console.log(`run ${run.id.slice(0, 8)} "${config.name}" — ${keys.length} model(s), ${tests.length} tests`); - const ac = new AbortController(); - await runEngine(run, settings, ac.signal, e => { - if (e.type === 'sample') { - run.samples.push(e.sample); store.saveSample(e.sample); - if (!e.sample.warmup) console.log(` ${e.sample.modelKey.slice(0, 24).padEnd(24)} ${String(e.sample.testId).padEnd(13)} d=${e.sample.mtpTokens ?? '-'} ${e.sample.status} gen=${e.sample.metrics.generationTps?.toFixed(1) ?? '-'} obj=${e.sample.objective.score ?? '-'}`); - } - if (e.type === 'wave') { run.waves.push(e.wave); store.saveWave(e.wave); } - if (e.type === 'model') run.modelInfo[e.key] = e.info; - if (e.type === 'log') { run.logs.push(new Date().toLocaleTimeString() + ' ' + e.message); if (/Prompt processing|Model failed|Runtime:|Native MTP/.test(e.message)) console.log(' log: ' + e.message.slice(0, 140)); } - if (e.type === 'gpu') run.gpu = e.gpu; - if (e.type === 'finish') { run.status = e.status; if (e.error) run.error = e.error; } - run.updated = new Date().toISOString(); - store.saveRun(run); - }); - run.updated = new Date().toISOString(); store.saveRun(run); - const ok = run.samples.filter(s => !s.warmup && s.status === 'completed'); - console.log(`\nDONE ${which} status=${run.status} measured=${ok.length}${run.error ? ' error=' + run.error.slice(0, 200) : ''}`); -} -main().then(() => process.exit(0)).catch(e => { console.error('FAILED:', e.message); process.exit(1); }); diff --git a/package-lock.json b/package-lock.json index 75c9b24..901b95a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "local-model-bench", - "version": "1.13.0", + "version": "1.14.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "local-model-bench", - "version": "1.13.0", + "version": "1.14.0", "dependencies": { "lucide-react": "^0.468.0", "react": "^19.1.0", diff --git a/package.json b/package.json index f41c4e1..53c9a89 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "local-model-bench", - "version": "1.13.0", + "version": "1.14.0", "description": "Private desktop benchmarks and transparent grading for LM Studio models", "author": "Local Model Bench contributors", "license": "MIT", diff --git a/src/endpoint.ts b/src/endpoint.ts new file mode 100644 index 0000000..0867cac --- /dev/null +++ b/src/endpoint.ts @@ -0,0 +1,28 @@ +// Where this app sends prompts. +// +// Loopback is the default and the private case, but the endpoint is just an OpenAI-shaped HTTP API, +// so it can be LM Studio on another machine, a server on the LAN, or anything else speaking the same +// protocol. The host is therefore not restricted. What stays rejected is everything that would make +// the address mean something other than an origin: credentials embedded in the URL, a path, a query, +// a fragment. +// +// This lives in src/ rather than beside the LM Studio client because the window needs it too — it +// has to say plainly when prompts are leaving the machine — and that module reaches for node:fs and +// node:child_process, which cannot be bundled for the renderer. +export const loopbackHosts = ['127.0.0.1', 'localhost', '[::1]', '::1']; + +// Reported, never enforced: the window uses it to replace a privacy claim it can no longer make. +export function isLoopbackUrl(url: string): boolean { + try { return loopbackHosts.includes(new URL(url).hostname); } catch { return false; } +} + +export function validateUrl(url: string): string { + let u: URL; + try { u = new URL(url); } catch { throw Error('Enter a full address, such as http://127.0.0.1:1234 or http://192.168.1.50:1234.'); } + if (!['http:', 'https:'].includes(u.protocol)) throw Error('The endpoint must be an http:// or https:// address.'); + if (!u.hostname) throw Error('The endpoint needs a host, such as 127.0.0.1 or 192.168.1.50.'); + // A token belongs in the token field, where it is encrypted at rest and never crosses to the window. + if (u.username || u.password) throw Error('Put credentials in the API token field rather than in the address.'); + if (u.pathname !== '/' || u.search || u.hash) throw Error('Give only the server address, with no path, query or fragment — for example http://192.168.1.50:1234.'); + return u.origin; +} diff --git a/src/main.tsx b/src/main.tsx index 01c4dc6..432340d 100644 --- a/src/main.tsx +++ b/src/main.tsx @@ -30,6 +30,7 @@ import type {Twin as TextOnlyTwin} from '../electron/text-only'; import {backendOf} from '../electron/runtime'; import type {HistoryRow} from './history'; import {runtimeChoices,type Runtime} from '../electron/runtime'; +import {isLoopbackUrl} from './endpoint'; import {cacheQuantLabel,cacheQuants,cacheScalePair,type CacheQuant} from './cache-quant'; import type {UpdateState} from '../electron/update'; @@ -72,6 +73,11 @@ function App(){ useEffect(()=>{if(!api)return;api.updateState().then(setUpdateState).catch(()=>{});return api.onUpdate(setUpdateState);},[]); const [page,setPage]=useState('Models'),[snap,setSnap]=useState(null),[models,setModels]=useState([]),[connected,setConnected]=useState(false),[connectionError,setConnectionError]=useState(''),[config,setConfig]=useState(structuredClone(defaultConfig)),[busy,setBusy]=useState(''),[notice,setNotice]=useState(''),[error,setError]=useState(''),[progress,setProgress]=useState(null),[runId,setRunId]=useState(''),[run,setRun]=useState(null),[editTest,setEditTest]=useState(null),[settingsDraft,setSettingsDraft]=useState(defaultPublicSettings),[tokenDraft,setTokenDraft]=useState(''),[clearToken,setClearToken]=useState(false),[sampleId,setSampleId]=useState(''); const refresh=async()=>{const s=await api.snapshot();setSnap(s);setProgress(s.progress);return s;}; + // Where runs actually go, taken from the saved settings rather than the unsaved draft: the badge + // describes the endpoint in force, not one being typed. + const savedUrl=snap?.settings.baseUrl??''; + const localOnly=!savedUrl||isLoopbackUrl(savedUrl); + const remoteHost=(()=>{try{return new URL(savedUrl).host;}catch{return '';}})(); const [runtimes,setRuntimes]=useState([]); // Text-only copies of vision models. The vision toggle needs these to mean anything: LM Studio // attaches a projector from the model's index entry and offers no way to load without it. @@ -119,7 +125,7 @@ function App(){ const hiddenSelected=config.modelKeys.filter(id=>!shownModels.some(m=>m.key===id)).length; const selectedSample=run?.samples.find(s=>s.id===sampleId); const pages=[{name:'Models',icon:Boxes},{name:'Benchmarks',icon:BarChart3},{name:'Tests',icon:FlaskConical},{name:'Run',icon:Play},{name:'Results',icon:BarChart3},{name:'Settings',icon:SettingsIcon}]; - return
+ return
Workspace {page}
{updateState&&['available','downloading','ready'].includes(updateState.phase)&&}
{error&&
{error}
}{notice&&
{notice}
} {page==='Models'&&<>
MODEL LIBRARY

Choose models to benchmark

Pick the local models you want to measure.

@@ -183,7 +189,8 @@ function App(){
{([['run','Specific run'],['overview','All runs overview']] as const).map(([value,label])=>)}
{resultsView==='run'?<>
{['html','csv','json','md'].map(f=>)}
{run?act('grade',async()=>{await api.grade(run.id,judge);await refresh();})} onRetry={()=>act('retry',async()=>{setRunId(await api.retry(run.id));await refresh();setPage('Run');})}/>:Choose models and start a run. Results and full responses will appear here automatically.} :r.id+r.updated).join('|')} onOpenRun={id=>{setRunId(id);setSampleId('');setResultsView('run');}}/>}} - {page==='Settings'&&<>
PREFERENCES

Settings

Connection, timeouts, and the instructions your judge follows.

LM Studio connection

setSettingsDraft(s=>({...s,baseUrl:e.target.value}))}/>{setTokenDraft(e.target.value);setClearToken(false);}}/>{settingsDraft.tokenConfigured&&}setSettingsDraft(s=>({...s,lmsPath:e.target.value}))}/>
setSettingsDraft(s=>({...s,timeoutSec:+e.target.value}))}/>setSettingsDraft(s=>({...s,loadTimeoutSec:+e.target.value}))}/>

Judge instructions

Applied to local grading and copied grading packages. Each test also has its own rubric. Saved runs keep their original grading instructions.