diff --git a/.clang-format b/.clang-format new file mode 100644 index 00000000..f9116525 --- /dev/null +++ b/.clang-format @@ -0,0 +1,10 @@ +BasedOnStyle: LLVM +IndentWidth: 4 +ColumnLimit: 120 +AllowShortFunctionsOnASingleLine: All +AllowShortIfStatementsOnASingleLine: AllIfsAndElse +AllowShortLoopsOnASingleLine: true +BreakBeforeBraces: Attach +PointerAlignment: Right +SpaceAfterCStyleCast: false +SortIncludes: false diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 00000000..e0acabb3 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,24 @@ +root = true + +[*] +charset = utf-8 +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true +indent_style = space +indent_size = 4 + +[*.{c,h,cu}] +indent_size = 4 + +[*.{ts,tsx,js,json,css}] +indent_size = 2 + +[Makefile] +indent_style = tab + +[*.yml] +indent_size = 2 + +[*.md] +trim_trailing_whitespace = false diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 72b7149e..426ea413 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,8 +12,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Build glm - run: cd c && make glm + - name: Build colibri + run: cd c && make colibri - name: C test suite run: cd c && make test-c @@ -41,6 +41,27 @@ jobs: -Xcompiler=-Wall,-Wextra echo "CUDA syntax check passed" + engine-hip-syntax: + name: HIP syntax check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Free runner disk + # rocm/dev images are large; hosted runners need the headroom freed + # BEFORE the pull (which is why this is not a container: job). + run: sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL + - name: Compile backend_cuda.cu + test binary with hipcc (no GPU) + run: | + # Tag pinned: `latest` outgrew the runner disk once already. The + # gcc-side engine link is skipped: containerized gcc/ld cannot link + # ROCm runtime libs (both 22.04 and 24.04 bases); the hipcc-driver + # link of the test binary below covers the single-source claim, and + # the engine link is exercised on real ROCm hardware (GPU_BACKENDS.md). + docker run --rm -v "$PWD":/src -w /src rocm/dev-ubuntu-24.04:6.2 bash -c ' + apt-get update && apt-get install -y --no-install-recommends make gcc g++ && + make -C c gpu-compile HIP=1 HIP_ARCH=gfx1100' + echo "HIP syntax check passed" + windows-cuda-build: # The Windows CUDA path has no coverage anywhere: `engine-cuda-syntax` above # compiles with nvcc's *GCC* host on Linux, and check.yml's windows job is @@ -105,13 +126,13 @@ jobs: make cuda-dll CUDA_ARCH=sm_80 test -f coli_cuda.dll || { echo "cuda-dll reported success but produced no DLL" >&2; exit 1; } echo "coli_cuda.dll built (MSVC host)" - - name: make glm CUDA_DLL=1 (host links backend_loader, not cudart) + - name: make colibri CUDA_DLL=1 (host links backend_loader, not cudart) shell: msys2 {0} run: | cd c - make glm CUDA_DLL=1 - test -f glm.exe || { echo "glm CUDA_DLL=1 reported success but produced no exe" >&2; exit 1; } - echo "glm.exe built against the DLL loader" + make colibri CUDA_DLL=1 + test -f colibri.exe || { echo "colibri CUDA_DLL=1 reported success but produced no exe" >&2; exit 1; } + echo "colibri.exe built against the DLL loader" web: name: Web UI diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e471fb99..a556b80b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -59,7 +59,12 @@ jobs: run: | TAG=${GITHUB_REF#refs/tags/} mkdir -p dist - cp c/glm${{ matrix.ext }} dist/colibri-${TAG}-${{ matrix.name }}${{ matrix.ext }} + # The engine goes in under its PLAIN name. `coli` locates the engine by looking for + # "colibri"/"colibri.exe" (then "glm") next to itself, so shipping it as + # colibri--.exe made every downloaded archive fail with + # "engine is not built. Run: coli build" -- with the engine sitting right there. + # The version lives in the ARCHIVE name, which is where a downloader looks for it. + cp c/glm${{ matrix.ext }} dist/colibri${{ matrix.ext }} cp c/coli dist/ cp c/version.py dist/ cp c/openai_server.py dist/ @@ -73,10 +78,36 @@ jobs: tar czf colibri-${TAG}-${{ matrix.name }}.tar.gz * fi + # Unpack what we are about to publish, in a clean directory, and prove `coli` finds the + # engine. A packaging mistake is invisible to every other job in this repo: the build is + # green, the tests are green, and the artifact is still unusable. Assert on behaviour. + - name: Verify the packaged archive actually runs + run: | + TAG=${GITHUB_REF#refs/tags/} + mkdir -p verify && cd verify + if [ "${{ matrix.ext }}" = ".exe" ]; then + 7z x ../dist/colibri-${TAG}-${{ matrix.name }}.zip > /dev/null + else + tar xzf ../dist/colibri-${TAG}-${{ matrix.name }}.tar.gz + fi + test -x colibri${{ matrix.ext }} || { echo "engine missing from archive"; ls -la; exit 1; } + out=$(python3 coli info 2>&1 || true) + echo "$out" + case "$out" in + *"engine is not built"*) echo "FAIL: coli cannot find the packaged engine"; exit 1 ;; + esac + echo "$out" | grep -q "colibri" || { echo "FAIL: coli did not run"; exit 1; } + echo "OK: coli locates the engine in the published archive" + + # Only the archives -- never the loose files. `dist/colibri-*.*` used to work by accident + # (the engine was versioned, so it did not match); now that the engine is plainly named, + # be explicit about what gets published. - uses: actions/upload-artifact@v4 with: name: colibri-${{ matrix.name }} - path: dist/colibri-*.* + path: | + dist/colibri-*.zip + dist/colibri-*.tar.gz release: needs: build diff --git a/.gitignore b/.gitignore index 981e7400..f62b3792 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,8 @@ desktop/src-tauri/target/ desktop/src-tauri/gen/ # binari compilati (si rigenerano con make / coli build) +c/colibri +c/colibri.exe c/glm c/glm.exe c/olmoe @@ -23,18 +25,12 @@ c/backend_loader.o c/coli_cuda.dll c/coli_cuda.lib c/coli_cuda.exp -c/tests/test_json -c/tests/test_json.exe -c/tests/test_st -c/tests/test_st.exe -c/tests/test_tier -c/tests/test_tier.exe -c/tests/test_grammar -c/tests/test_grammar.exe -c/tests/test_schema_gbnf -c/tests/test_schema_gbnf.exe -c/tests/test_compat_direct -c/tests/test_compat_direct.exe +c/tests/test_* +!c/tests/test_*.c +!c/tests/test_*.cu +!c/tests/test_*.mm +!c/tests/test_*.py +result # oracoli tiny generati (make_glm_oracle.py) e dati benchmark scaricati c/glm_tiny/ @@ -68,3 +64,7 @@ c/tests/test_decode_batch c/tests/test_i4_acc512 c/tests/test_idot c/tests/test_uring +olmoe_merged/ +olmoe_i4/ +c/olmoe_merged/ +c/olmoe_i4/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 61779660..1b98abed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,83 @@ All notable changes to colibrì are documented here. Format follows [Keep a Changelog](https://keepachangelog.com/). +## [1.1.0] — 2026-07-22 + +A community release. 27 pull requests from more than 20 contributors, 216 commits since +v1.0.0. Most of what follows was found, measured, or fixed by people who do not work on +this project and had nothing to gain from it. + +### Added + +- **AMD GPU support (HIP/ROCm)** (#339) — single-source `backend_gpu_compat.h` with a WMMA + dispatch gate, so one codebase builds for CUDA and HIP. Validated on an RX 9070 XT + (RDNA4, ROCm 7.2): token-exact against CPU on a real fmt=4 gs64 container, with resident + dense *and* with routed experts in VRAM, plus a fail-injection control proving the GPU + actually executed the work. +- **Dual-SSD streaming** (`COLI_MODEL_MIRROR`, #421) — read the model from two drives at + once, roughly doubling streaming bandwidth on a disk-bound host. +- **N-drive shard split** (`COLI_MODEL_DIRS`, #469) — capacity aggregation: run a container + no single drive can hold, spread across several with no duplication. +- **fmt=5 (int3-g64)** (#168) — 3-bit weights with per-64 group scales: measured 3.3x lower + outlier-row error than per-row int4 at 25% fewer bytes. +- **fmt=6 (E8/IQ3 lattice)** (#465) — CPU decode kernel and dispatch; index codec tooling (#458). +- **`tools/try_tool_calling.py`** — dependency-free two-turn tool-calling probe that doubles + as a smoke test. + +### Fixed + +- **Tool calling in coding clients** (#401), root cause found, in two parts: + - **#506** — the engine capped prompt encoding at `CTX-2`, and the tokenizer stops dead at + its limit *without reporting anything*. A prompt longer than the context was therefore + silently truncated to its first `CTX-2` tokens and answered anyway. With the 4096 default + that is 4094 — exactly the `prefill 4094` in the field report. The dropped tail was the + tool instructions and the user's actual turn, so the model emitted a bare `<` and stopped; + and because clients append to the *end* while truncation keeps the *head*, every retry + re-sent a byte-identical prompt. Now refused with a 400 `context_length_exceeded`. + - **#505** — a tool call whose closing `` never arrived was dropped whole, + because the parser required both tags. Now recovered when unambiguous, on both the + streamed and non-streamed paths. + - **#437** — non-EOS role markers were armed as hard stops in serve mode and cut generation + the instant a tool block started. +- **Grouped-int4 (fmt=4) produced garbage output on CUDA** with `CUDA_DENSE=1` (#298) — the + dense and attention kernels applied per-group scales as if they were per-row. Hardware-verified. +- **OpenMP tuning re-exec preserved the CPU affinity mask** (#476), jailing every thread onto + one core when `OMP_PROC_BIND`/`OMP_PLACES` were set: roughly a 20x slowdown. +- **Pilot eviction guard dropped ~100% of speculations** once the cache filled (#497), + collapsing `PILOT_REAL` to a hint-only path. +- **Silent budget clamp** capped the CUDA expert tier at ~109 experts regardless of + `CUDA_EXPERT_GB` (#495). +- fmt=4 guard at the per-row-only CUDA entry points (#464/#470); `COLI_CUDA_MTP=1` and + `COLI_CUDA=0` are now honoured over implicit defaults (#468). + +### Security + +Threat model: model files come from mirrors that are not trusted. + +- **#368** — server hardening, JSON and tokenizer parser hardening, build flags, downloader + and dependency pinning. +- **#413** — the quant layout is resolved *and* validated against the on-disk byte counts + (unknown layouts are refused rather than falling through to int2), shape-product overflow + is rejected, and the olmoe dtype-3 path no longer trusts a crafted `nbytes` (heap overflow). + +### Performance — all byte-identical + +- **#481** +4.7x on the MLA-absorb score and value-mix reductions +- **#477** +13% decode on AVX-512 (`qt_addrow` / `qt_matvec_rows`) +- **#475** +11.6% with opt-in `XEXP=1` (one OpenMP region per expert block at S=1 full residency) +- **#473** +5.5% int4 IDOT at S=1 on AVX-512 VNNI + +### Changed + +- `glm.c` is now `colibri.c` plus header modules (#391); `make glm` remains as an alias. +- Serve stage 2 (#192): `response_format`, per-request grammars, grammar-forced drafts. + +### Upgrade notes + +- **`CTX` still defaults to 4096.** Coding clients send far more than that in a single system + prompt. Use `CTX=32768`. Before this release an over-long prompt was silently truncated; + now you get a clear 400 instead. + ## [1.0.0] — 2026-07-19 First tagged release. The engine has been running in production since late June diff --git a/GPU_BACKENDS.md b/GPU_BACKENDS.md new file mode 100644 index 00000000..be421bfb --- /dev/null +++ b/GPU_BACKENDS.md @@ -0,0 +1,85 @@ +# GPU backends: CUDA and HIP/ROCm + +colibrì's GPU expert backend is **one source file** (`c/backend_cuda.cu`) compiled +for either vendor through `c/backend_gpu_compat.h` — the same one-shim-header +pattern `compat.h` uses for the Windows port. Compiled by nvcc the shim is a +pass-through to `cuda_runtime.h` (the NVIDIA path is byte-identical to the +pre-HIP tree); compiled by hipcc it maps the 14-symbol CUDA runtime surface the +backend uses onto HIP 1:1. The kernels use only shared syntax +(`__global__`, `__shared__`, `__syncthreads__`, `<<<>>>`), no vendor intrinsics. + +**Rule for contributors:** vendor differences go in `backend_gpu_compat.h` +only — never `#ifdef __HIP__` (or CUDA-specific code) in `backend_cuda.cu`. + +## Supported environments + +| backend | platform | toolchain | build | +|---|---|---|---| +| CUDA (`CUDA=1`) | Linux x86-64 | CUDA toolkit (nvcc), `CUDA_HOME=/usr/local/cuda` default | `make -C c glm CUDA=1 [CUDA_ARCH=native\|sm_XX]` | +| HIP (`HIP=1`) | Linux x86-64 | ROCm (hipcc), `ROCM_HOME=/opt/rocm` default; tested on ROCm 7.2 | `make -C c glm HIP=1 [HIP_ARCH=native\|gfxXXXX]` | + +`CUDA=1` and `HIP=1` are mutually exclusive and both opt-in: the default build +remains pure, dependency-free CPU. Both are refused on non-Linux with an early +`$(error)`. `*_ARCH=native` targets the local GPU; pass an explicit arch when +distributing or on machines with an unsupported iGPU visible to the runtime +(and mask iGPUs at runtime with `HIP_VISIBLE_DEVICES=` on ROCm). + +## Runtime configuration (identical for both vendors) + +- `COLI_CUDA=1` + `COLI_GPU=N` (or `COLI_GPUS=0,1,...`) — enable, select devices +- `CUDA_EXPERT_GB=G` — VRAM budget for the expert tier (clamped to free VRAM + minus projected dense set and 2 GB headroom per device) +- `CUDA_RELEASE_HOST=1` — GPU-tier experts drop their host backing after + upload (default on multi-GPU); combined with `PIN=auto`/`PIN_FILL`, VRAM + becomes additional pinned capacity at zero RAM cost. The engine + rematerializes an expert from disk (`expert_host_ensure`) whenever the CPU + path needs one whose host copy was released — validated under total GPU + failure below. +- `CUDA_DENSE=1` — experimental resident-dense path (unchanged) +- `COLI_CUDA_TC_W4A16=1` — opt-in W4A16 tensor-core path. **NVIDIA-only**: + the WMMA kernels are compile-gated (`COLI_GPU_HAS_WMMA` in the compat + header) because gfx GPUs report `compute_major >= 7` and a runtime check + alone would select empty kernel bodies under HIP. On AMD, all compute uses + the portable kernels; rocWMMA matrix-core support is a possible follow-up. + +## Validation + +### Unit tests (run on GPU hardware) + +```sh +make -C c cuda-test [CUDA_ARCH=...] # NVIDIA +make -C c hip-test [HIP_ARCH=...] # AMD (same test source) +``` + +Covers q8/q4/q2/f32 matmul correctness, multi-device placement/stats, and +`tensor_update` — the standard upstream suite, unchanged, compiled by hipcc. +(A companion PR adds failure-path tests for the backend; they are +vendor-neutral and run under `hip-test` identically.) + +### CI (no GPU required) + +The `engine-hip-syntax` job in `.github/workflows/ci.yml` compiles the +backend and its test binary with hipcc (`rocm/dev` container pinned to +`6.2`, `gfx1100`) on every PR, mirroring `engine-cuda-syntax`. Kernel +*execution* is not possible on hosted runners; that is what `hip-test` +on real hardware is for (matrix below). + +### Hardware test matrix (documented results) + +| environment | result | +|---|---| +| AMD RX 9070 XT (gfx1201), ROCm 7.2.4, Linux 7.0 | `hip-test` **pass** (all cases above); GLM-5.2 end-to-end runs (0.32 tok/s @ 61% expert hit with CUDA_RELEASE_HOST=1); benchmark series in PR #112 | +| NVIDIA | compile-verified in CI (`sm_80`); nvcc path is a pass-through include — **runtime run of `make cuda-test` on NVIDIA hardware welcomed**, the test source is vendor-neutral | + +## Known behavior notes + +- GPU float matmuls round differently than the CPU int8-dot (IDOT) kernels: + greedy output is **not token-identical** across backends (consistent with + the shape-dependence documented in #100), and MTP draft acceptance measures + lower on GPU-heavy configs (~40% → ~31% on the PR #112 machine). A + numerics-matched integer GPU kernel is the planned follow-up. +- An earlier revision of this branch carried `CUDA_EXTEND=1` (VRAM tier + holding experts beyond the RAM pin). It was superseded by upstream's + `PIN=auto` + `PIN_FILL` + `CUDA_RELEASE_HOST`, which achieve the same + capacity extension with deeper engine integration; this branch's safety + and validation work now targets that mechanism. diff --git a/README.it.md b/README.it.md new file mode 100644 index 00000000..76e3c21c --- /dev/null +++ b/README.it.md @@ -0,0 +1,260 @@ +

+ colibrì — motore piccolo, modello immenso +

+ +

+ English · 简体中文 · 繁體中文 · Italiano +

+ +**Motore piccolo, modello immenso.** Esegui **GLM-5.2 (744 miliardi di parametri, MoE)** su un computer consumer con ~25 GB di RAM — in C puro, zero dipendenze, caricando gli expert dal disco in streaming. + +Colibrì è un runtime MoE leggero e che preserva la qualità: tratta VRAM, RAM e +disco come un'unica gerarchia di memoria gestita. Se la memoria veloce non basta +il modello rallenta, ma la policy predefinita **non cambia mai silenziosamente la +precisione del modello né la semantica del router**. + +``` +$ ./coli chat + 🐦 colibrì v1.0 — GLM-5.2 · 744B MoE · int4 · streaming CPU + ✓ ready in 32s · resident 9.9 GB + › ciao! + ◆ Ciao! 😊 Come posso aiutarti oggi? +``` + +## Guardalo in azione + +

+ dashboard web di colibrì — metriche live, pannello hardware, livelli degli expert +

+

La dashboard web (./coli web): un modello da 744B a 4 tok/s, TTFT 1.6 s, disco 0 — +residenza completa degli expert su 6× RTX 5090, con metriche token in tempo reale, breakdown dei tempi per turno, +la barra dei livelli VRAM/RAM/disco e il mini-cervello live nell'angolo.

+ +

+ la pagina Brain — 19.456 expert come una corteccia vivente +

+

La pagina Brain: tutti i 19.456 expert come una corteccia vivente — il colore indica +il livello di archiviazione, la luminosità il calore di routing, e ogni expert instradato in un turno +lampeggia bianco. Passando il cursore si vede l'affinità +tematica misurata dell'expert.

+ +

+ la pagina Atlas — l'atlante misurato degli expert come una galassia 3D +

+

La pagina Atlas: l'atlante +misurato degli expert come una galassia 3D — 13.260 expert caratterizzati, 1.041 specialisti +replicabili che si raggruppano per argomento (poesia, legge, cinese, SQL…). La posizione deriva +dall'affinità di routing misurata, non da un embedding appreso. Trascinare per ruotare.

+ +## L'idea + +Un modello Mixture-of-Experts da 744B attiva solo ~40B parametri per token — e +solo ~11 GB di quelli cambiano da un token all'altro (gli expert instradati): + +

+ solo ~5.4% dei parametri è attivo per token +

+ +Il modello non ha bisogno di *stare* in memoria veloce — ha bisogno di essere +**piazzato**: + +- la **parte densa** (attenzione, expert condivisi, embedding — ~17B parametri) + resta **residente in RAM a int4** (~9.9 GB); +- i **19.456 expert instradati** (75 layer MoE × 256 + la testa MTP, ~19 MB + ciascuno a int4) stanno **su disco** (~370 GB) e vengono **caricati on demand + in streaming**, con una cache LRU per layer, un hot-store pinnato che impara, + e un livello VRAM opzionale. + +Il motore è un singolo file C (`c/colibri.c`) più header piccoli. Niente BLAS, +niente Python a runtime, niente GPU obbligatoria. + +## Come funziona + +### Il percorso di ogni token + +

+ instrada → unione → piazza → sovrapponi → impara +

+ +Ogni layer di ogni token percorre gli stessi cinque passi. L'obiettivo +progettuale è che **il piazzamento decide solo la velocità** — le decisioni +del router e la precisione dei pesi sono identiche sia che un expert risponda +dalla VRAM sia dal disco. + +### Una gerarchia di memoria, non un requisito di memoria + +

+ residenza expert a tre livelli: VRAM / RAM / NVMe +

+ +Lo stesso motore copre l'intero spettro: su un portatile da 25 GB tutto viene +caricato dal disco in streaming (lento, ma corretto); su un host grande l'intero +set di expert diventa residente (`CUDA_EXPERT_GB=auto PIN_GB=all`) e il disco +esce completamente dal percorso di decode. Tra i livelli c'è una **cache che +impara**: il motore registra quali expert il *tuo* carico di lavoro instrada +(`.coli_usage`, aggiornato a ogni turno) e fissa automaticamente i più caldi — +colibrì diventa letteralmente più veloce man mano che lo usi. Sugli host +multi-socket, `COLI_NUMA=1` interlaccia i pesi residenti tra i controller di +memoria ([#82](https://github.com/JustVugg/colibri/issues/82)). + +### Mai aspettare il disco due volte + +I miss nella cache costano caro, quindi il motore investe la maggior parte +della sua astuzia per evitarli e sovrapporli: le tre matrici di ogni expert sono +memorizzate contigue e lette con un unico `pread`; un pool I/O asincrono +limitato (`PIPE=1`, attivo per default) carica gli expert mancanti mentre quelli +residenti calcolano; le posizioni in batch leggono ogni expert unico una sola +volta (**batch-union**); un thread di lookahead del router (`PILOT=1`) fa il +prefetch degli expert del layer successivo — il routing è misurabilmente +**prevedibile al 71.6% un layer in anticipo**. Sulle GPU, la pipeline residente +(`COLI_CUDA_PIPE=2`) mantiene il flusso residuo on-device tra i layer, così il +loop CPU degli expert procede senza interruzioni; su Apple Silicon un backend +[Metal](docs/metal.md) sperimentale esegue la matmul batch degli expert sulla +GPU a memoria unificata. + +### Modello fedele, stato compresso + +Il forward pass è validato **token-esatto contro un oracle `transformers`** +(teacher-forcing 32/32). L'attenzione MLA memorizza uno stato KV compresso — 576 +float/token invece di 32.768 (**57× più piccolo**) — e lo persiste tra i +riavvii (`.coli_kv`): le conversazioni riaprono "calde", senza alcun re-prefill, +byte-identiche a una sessione ininterrotta. L'attenzione sparsa DSA (il +lightning indexer di GLM-5.2) è implementata fedelmente e validata forzando la +selezione di tutte le chiavi per riprodurre esattamente l'attenzione densa. + +### Decodifica speculativa, onestamente + +La testa MTP nativa di GLM-5.2 propone token che il modello principale verifica +in un unico forward batch — 2.2–2.8 token/forward quando conviene. Due regole +conquistate a caro prezzo sono i default: la testa MTP deve essere **int8** (le +teste int4 crollano al 0–4% di accettazione, +[#8](https://github.com/JustVugg/colibri/issues/8)), e draft e verifica devono +calcolare **la stessa funzione** — `SPEC_PIN=1` fissa entrambi sulla stessa +famiglia di kernel ([#163](https://github.com/JustVugg/colibri/issues/163) +contiene l'intera indagine forense). I draft forzati da grammatica +([`GRAMMAR=file.gbnf`](docs/grammar-draft.md)) aggiungono accettazione quasi +gratuita sull'output JSON vincolato. Se la speculazione conviene dipende dalla +temperatura della cache — misura, e usa `DRAFT=0` quando non paga. + +## Cosa ottiene + +

+ velocità di decode misurata per classe hardware +

+ +Stesso motore, stesso container int4 — cambia solo dove risiedono gli expert. +Punti salienti dalle [tabelle benchmark complete](docs/benchmarks.md): + +- **6× RTX 5090, residenza completa:** 5.8–6.8 tok/s in decode, TTFT ~13 s + ([log dell'esperimento](docs/experiments/glm52-6x5090-2026-07-12.md)); +- **desktop solo-CPU da 128 GB:** ~1.8 tok/s a cache calda + ([#200](https://github.com/JustVugg/colibri/issues/200)); +- **singola RTX 5070 Ti, classe laptop:** 1.07 tok/s tramite la pipeline + GPU-residente ([#273](https://github.com/JustVugg/colibri/issues/273)); +- **macchina di sviluppo da 25 GB:** 0.05–0.1 tok/s a freddo — il punto di + partenza dimostrato da cui è nato il progetto, e ancora oggi la baseline onesta. + +La qualità è misurata, non presunta: il costo di quantizzazione del container +int4 e le ablazioni su granularità delle scale e rotazione sono in +[docs/benchmarks.md](docs/benchmarks.md#quality-benchmark) e +[#108](https://github.com/JustVugg/colibri/issues/108)/[#81](https://github.com/JustVugg/colibri/issues/81). + +## Per iniziare + +### 1. Scarica il modello + +Un container **GLM-5.2 int4** pre-convertito è su Hugging Face — **usa la +versione con le teste MTP int8**: + +**https://huggingface.co/mateogrgic/GLM-5.2-colibri-int4-with-int8-mtp** + +> ⚠️ Il mirror originale contiene teste MTP int4 → accettazione dei draft allo 0% +> ([#8](https://github.com/JustVugg/colibri/issues/8)). Verifica la tua versione: +> `ls -l /out-mtp-*` — int8 (corretto) è `3527131672 / 5366238584 / 1065950496`. + +Oppure converti tu stesso dalla sorgente FP8 — un unico comando riprendibile che +non richiede mai i 756 GB completi su disco contemporaneamente: + +```bash +cd c && ./setup.sh # verifica gcc/OpenMP, compila, autotest +./coli convert --model /nvme/glm52_i4 # scarica e converti shard per shard (python, una tantum) +``` + +### 2. Esegui + +```bash +COLI_MODEL=/nvme/glm52_i4 ./coli chat # budget RAM, cache e MTP rilevati automaticamente +COLI_MODEL=/nvme/glm52_i4 ./coli plan # mostra il piazzamento pianificato VRAM/RAM/disco +COLI_MODEL=/nvme/glm52_i4 ./coli doctor # controllo di idoneità (sola lettura) +./coli web --model /nvme/glm52_i4 # API + dashboard web sulla stessa porta +./coli serve --model /nvme/glm52_i4 # solo API compatibile OpenAI +``` + +Il motore a runtime è puro C — python si usa solo per il convertitore (una tantum) +e per il gateway API opzionale. + +### 3. Approfondisci + +| argomento | documento | +|---|---| +| Benchmark, dati dalla comunità, misurazioni di qualità | [docs/benchmarks.md](docs/benchmarks.md) | +| Parametri di tuning, policy, cache che impara, prefetch | [docs/tuning.md](docs/tuning.md) | +| Build nativa su Windows 11 (con CUDA DLL) | [docs/windows.md](docs/windows.md) | +| Backend CUDA, livello expert in VRAM, residenza completa | [docs/cuda.md](docs/cuda.md) | +| Backend Metal per Apple Silicon | [docs/metal.md](docs/metal.md) | +| API compatibile OpenAI, KV slot, dashboard web | [docs/api.md](docs/api.md) | +| Draft forzati da grammatica (output strutturato) | [docs/grammar-draft.md](docs/grammar-draft.md) | +| Inventario delle variabili d'ambiente | [docs/ENVIRONMENT.md](docs/ENVIRONMENT.md) | + +## Sostenere il progetto + +colibrì è nato come progetto di una sola persona su un portatile con 12 core +e 25 GB di RAM; oggi i suoi numeri arrivano da una comunità di macchine reali. +Se ti è utile: + +- ⭐ metti una stella al repository e condividilo; +- 🐛 apri issue con i numeri di benchmark del tuo hardware — i datapoint + fanno avanzare questo progetto più di qualsiasi altra cosa; +- 💬 contattaci via GitHub issues per sponsorizzare lo sviluppo o donare hardware. + +## Struttura del repository + +``` +Makefile punto d'ingresso root per build/check +c/ +├── colibri.c motore principale +├── quant.h kernel matmul quantizzati (SIMD multi-architettura) +├── sample.h campionamento, RNG, set di stop +├── kv_persist.h persistenza KV su disco (.coli_kv) +├── telemetry.h protocollo dashboard, statistiche, usage +├── st.h, tok.h, json.h header di runtime +├── backend_cuda.* livello CUDA opzionale +├── Makefile build e check locali +├── coli CLI utente +├── openai_server.py gateway HTTP compatibile OpenAI +├── setup.sh setup locale in un solo comando +├── tools/ conversione offline, fixture e benchmark +├── scripts/ helper per conversioni lunghe +└── tests/ test C e Python senza dipendenze +web/ UI browser (puro client API OpenAI) +desktop/ shell desktop Tauri v2 che racchiude la web UI +docs/ documentazione di riferimento, esperimenti, media +``` + +Il percorso a runtime resta intenzionalmente piatto e leggibile: `colibri.c` +più i suoi header. Dalla radice del repository, `make`, `make check` e +`make clean` delegano al Makefile del motore. + +## Perché "colibrì" + +Il colibrì pesa pochi grammi, sta sospeso nel vuoto e visita un migliaio di +fiori al giorno. Questo motore tiene in vita un gigante da 744 miliardi di +parametri con le razioni di un colibrì: 25 GB di RAM, dodici core CPU e +tanta pazienza col disco. + +Il nome è rimasto in italiano perché questa è la lingua in cui è stato scritto +il primo prototipo — i commenti nel codice lo testimoniano ancora. + +## Licenza + +Apache 2.0. I pesi di GLM-5.2 sono rilasciati da Z.ai sotto licenza MIT. diff --git a/README.md b/README.md index d4fe5217..8b48ee59 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@

Website · - English · 繁體中文 + English · 简体中文 · 繁體中文 · Italiano

**Tiny engine, immense model.** Run **GLM-5.2 (744B-parameter MoE)** on a consumer machine with ~25 GB of RAM — in pure C, with zero dependencies, by streaming experts from disk. @@ -50,6 +50,16 @@ brightness is routing heat, and every expert routed in a turn flashes white. Hov as a 3-D galaxy — 13,260 characterised experts, 1,041 replicated specialists clustering by topic (poetry, law, Chinese, SQL…). Position is measured routing affinity, not a learned embedding. Drag to spin.

+## The vision + +Frontier models should not be sealed inside datacenters. colibrì exists so that +**anyone curious enough can open one up**: run a 744B-parameter mind on hardware +you already own, watch every expert fire in real time, and change the code that +does it. Not renting intelligence behind an API — *holding* it: probing it, +measuring it, improving it. Every optimisation in this project started with +someone measuring something on their own machine; the engine is deliberately +small enough that the next one can come from you. + ## The idea A 744B Mixture-of-Experts model activates only ~40B parameters per token — and @@ -67,6 +77,18 @@ So the model doesn't need to *fit* in fast memory — it needs to be **placed**: at int4) live **on disk** (~370 GB) and are **streamed on demand**, with a per-layer LRU cache, a learned pinned hot-store, and an optional VRAM tier. +Think of the core algorithm as **a JIT, but for weights**. A compiler JIT never +compiles the whole program — it watches what actually runs and compiles the hot +paths, just in time. colibrì makes the same bet about a 744B parameter space: +parameters are not resident state to be held, they are **data to be staged** +across a heterogeneous storage hierarchy (VRAM / RAM / NVMe), exactly when the +router proves they are needed. Measured routing heat decides which experts earn +which tier, the router runs a layer ahead so prefetch hides the staging latency, +and — like a JIT — the engine learns your workload: the more you run, the hotter +the right experts get. It works because routing has measurable structure (see +the [expert atlas](https://github.com/JustVugg/colibri/issues/175)) — and +structure is cacheable. + The engine is a single C file (`c/glm.c`) plus small headers. No BLAS, no Python at runtime, no GPU required. @@ -88,6 +110,22 @@ precision are the same whether an expert answered from VRAM or from disk. VRAM / RAM / NVMe three-tier expert residency

+### Dual-SSD: two copies of the model, twice the read bandwidth + +Decode is disk-bound on most machines, and expert reads are read-only — so if you have a **second SSD**, put a full copy of the model on it and let the engine stream from both drives at once: + +```bash +COLI_MODEL=/fast/glm52_i4 COLI_MODEL_MIRROR=/second/glm52_i4 ./coli chat +COLI_DISK_WEIGHTS=9,3 ... # optional: primary,mirror bandwidth ratio (else measured at startup) +``` + +Each expert is routed to one drive by a deterministic hash, weighted by the two drives' measured (or declared) bandwidth, so readahead/PILOT prefetch and the demand read always hit the same drive and nothing is cached twice. The aggregate bandwidth is the sum of both drives — a 9 GB/s + 3 GB/s pair reads experts ~33% faster than the fast drive alone, and the OMP-parallel pin/warmup load streams from both. Details worth knowing: + +- the mirror is **validated at startup** (per-file size + safetensors header must be byte-identical to the primary); divergent or missing files silently stay on the primary, so a **partial mirror is fine** — a smaller second SSD holding only some shards still helps; +- the mirror is **never written**: `.coli_usage`, `.coli_kv` and all sidecars stay on the primary; +- a read error on the mirror falls back to the primary (one warning, no crash), so unplugging the second drive mid-run degrades instead of killing the server; +- routing never changes tokens — both copies are byte-identical, and the per-run `MIRROR:` stats line shows GB served per drive. + The same engine spans the whole range: on a 25 GB laptop everything streams from disk (slow but correct); on a large host the entire expert set becomes resident (`CUDA_EXPERT_GB=auto PIN_GB=all`) and disk drops out of the decode path @@ -110,6 +148,13 @@ on-device across layers so the CPU expert loop runs uninterrupted; on Apple Silicon an experimental [Metal backend](docs/metal.md) does the batched expert math on the unified-memory GPU. +> **On real NVMe, measure `DIRECT=1`.** O_DIRECT bypasses the page cache and is +> often a large win on drives with DRAM cache and bandwidth headroom (+34% +> decode measured with `PIPE=1` on a Blackwell/Windows box; 4.25→9.69 GB/s in +> iobench on a GB10) — but it is drive-dependent: QLC/DRAM-less or virtualised +> disks can be neutral to negative. Try it first; keep what your hardware +> rewards. + ### Faithful model, compressed state The forward pass is validated **token-exact against a `transformers` oracle** @@ -198,6 +243,17 @@ COLI_MODEL=/nvme/glm52_i4 ./coli doctor # read-only readiness check The engine at runtime is pure C — python is only used by the one-time converter and the optional API gateway. +**On Windows?** You don't need to build. Download the +`colibri--windows-x86_64.zip` from +[Releases](https://github.com/JustVugg/colibri/releases), unzip it, rename +`colibri-*-windows-x86_64.exe` → `glm.exe` (so the `coli` launcher finds the +engine), install [Python 3](https://www.python.org/downloads/), then run +`coli chat`. Full walkthrough in the [Quick Start guide](docs/quickstart.md#windows). + +Prefer a `coli` command on your PATH? From a checkout, `pip install -e .` +registers it (the engine itself still lives in `c/` — this is an editable +install from the clone, not a standalone wheel). + ### 3. Go deeper | topic | doc | @@ -211,6 +267,18 @@ and the optional API gateway. | Grammar-forced drafts (structured output) | [docs/grammar-draft.md](docs/grammar-draft.md) | | Environment variable inventory | [docs/ENVIRONMENT.md](docs/ENVIRONMENT.md) | +## What's next + +- **Algorithmic research is active.** The current hierarchy is LRU + a learned + pin set; the next step is under way — smarter placement and scheduling, + overlap of CPU and GPU expert execution, and routing-aware speculation. + Everything lands the way this project always works: measured, reviewed, and + merged in the open. +- **More open models.** The tiering algorithm is model-agnostic: any MoE with + routed experts can be staged the same way. GLM-5.2 and OLMoE run today; + support for more open-weight families — **Kimi K2** (Moonshot AI), + **Qwen3 MoE** (Alibaba), **MiniMax** — is on the roadmap. + ## Supporting the project colibrì started as a one-person project on a 12-core laptop with 25 GB of RAM; @@ -251,6 +319,14 @@ The hummingbird weighs a few grams, hovers in place, and visits a thousand flowers a day. This engine keeps a 744-billion-parameter giant alive on hummingbird rations: 25 GB of RAM, twelve CPU cores, and a lot of disk patience. +## Acknowledgements + +colibrì is an engine; the minds it runs are a gift. Thank you to the teams +releasing frontier-class weights in the open — **Z.ai** (GLM), **Moonshot AI** +(Kimi), **Alibaba Qwen**, **MiniMax**, and **Allen AI** (OLMoE) — and to every +contributor who benchmarked, bisected, replicated an atlas run, or sent a patch. +This project is proof of what open weights make possible. + ## License Apache 2.0. GLM-5.2 weights are released by Z.ai under MIT. diff --git a/README.zh-CN.md b/README.zh-CN.md new file mode 100644 index 00000000..352de33c --- /dev/null +++ b/README.zh-CN.md @@ -0,0 +1,237 @@ +

+ colibrì——小巧引擎，庞大模型 +

+ +

+ English · 简体中文 · 繁體中文 · Italiano +

+ +**小巧引擎，庞大模型。**只需约 25 GB 内存，就能在消费级电脑上运行 **GLM-5.2（744B 参数的 MoE）**——以零依赖的纯 C 实现，从磁盘流式加载专家。 + +Colibrì 是一套轻量、保持模型质量的 MoE 运行时，将 VRAM、RAM +与存储设备视为统一管理的内存层级。高速内存不足可能降低速度， +但默认策略**绝不会在未告知的情况下改变模型精度或路由语义**。 + +``` +$ ./coli chat + 🐦 colibrì v1.0 — GLM-5.2 · 744B MoE · int4 · streaming CPU + ✓ ready in 32s · resident 9.9 GB + › ciao! + ◆ Ciao! 😊 Come posso aiutarti oggi? +``` + +## 实际运行效果 + +

+ colibrì 网页仪表盘——实时指标、硬件面板与专家存储层级 +

+

网页仪表盘（./coli web）：744B 模型达到 4 tok/s、TTFT 1.6 秒、磁盘读取 0—— +在 6× RTX 5090 上让所有专家常驻，并实时显示 token 指标、每轮耗时明细、 +VRAM／RAM／磁盘层级条，以及角落的实时迷你大脑。

+ +

+ 大脑页面——以实时皮层呈现 19,456 个专家 +

+

大脑（Brain）页面：将全部 19,456 个专家呈现为活的皮层——颜色代表存储层级， +亮度代表路由热度，每轮被路由到的专家都会闪白。将光标停在专家上，即可查看其 +实测主题亲和度。

+ +

+ 图谱页面——以 3D 星系呈现实测专家图谱 +

+

图谱（Atlas）页面：将实测专家图谱 +呈现为 3D 星系——共 13,260 个已分析专家，其中 1,041 个可复现的专门专家会按主题聚集 +（诗歌、法律、中文、SQL……）。位置取自实测路由亲和度，而非学习出的嵌入向量。拖拽即可旋转。

+ +## 核心概念 + +744B 的专家混合（Mixture-of-Experts）模型，每个 token 只会激活约 40B 参数—— +其中每个 token 之间会变动的只有约 11 GB（被路由到的专家）： + +

+ 每个 token 只会激活约 5.4% 的参数 +

+ +所以模型不必完整**装进**高速内存，而是需要正确**放置**： + +- **稠密部分**（注意力、共享专家、嵌入——约 17B 参数）以 int4 + **常驻 RAM**（约 9.9 GB）； +- **19,456 个路由专家**（75 个 MoE 层 × 256，加上 MTP head；每个在 int4 下约 19 MB） + **存放在磁盘**（约 370 GB），并**按需流式加载**，配合逐层 LRU 缓存、 + 会学习的热门专家固定存储区，以及可选的 VRAM 层级。 + +引擎是一个 C 主文件（`c/colibri.c`）加上若干头文件。不需要 BLAS， +运行时不需要 Python，也不需要 GPU。 + +## 工作原理 + +### 每个 token 的处理路径 + +

+ 路由 → 并集 → 放置 → 重叠执行 → 学习 +

+ +每个 token 的每一层都会经过相同的五个步骤。设计目标是让 +**放置只决定速度**——无论专家是从 VRAM 还是磁盘响应，路由器的决策与权重精度都完全相同。 + +### 统一内存层级，取代单一内存门槛 + +

+ VRAM／RAM／NVMe 三层专家常驻架构 +

+ +同一套引擎覆盖完整硬件范围：在 25 GB 笔记本上，一切都从磁盘流式加载 +（慢，但结果正确）；在大内存主机上，则可让整组专家常驻 +（`CUDA_EXPERT_GB=auto PIN_GB=all`），让磁盘完全退出解码路径。 +两端之间有一层**学习型缓存**：引擎会记录*你的*工作负载路由到哪些专家 +（`.coli_usage`，每轮更新），并自动固定最热门的专家——colibrì 确实会越用越快。 +在多路主机上，`COLI_NUMA=1` 会将常驻权重交错分配到各内存控制器 +（[#82](https://github.com/JustVugg/colibri/issues/82)）。 + +### 绝不为同一次磁盘读取等待两遍 + +缓存未命中的代价很高，因此引擎大部分的巧思都用来避免或重叠这些读取： +每个专家的三个矩阵相邻存储，并以一次 `pread` 读取；有界异步 I/O 池 +（`PIPE=1`，默认启用）会在常驻专家计算时加载缺失的专家；批量位置只读取每个 +不重复专家一次（**批量并集**）；路由前瞻线程（`PILOT=1`）则预取下一层专家—— +实测显示，路由结果提前一层时有 **71.6% 的可预测性**。 +在 GPU 上，常驻管线（`COLI_CUDA_PIPE=2`）让残差流跨层保留在设备端， +使 CPU 专家循环不中断；在 Apple Silicon 上，实验性的 +[Metal 后端](docs/metal.md)会用统一内存 GPU 执行批量专家运算。 + +### 忠实模型，压缩状态 + +前向传播已通过 `transformers` oracle 验证为**逐 token 完全一致** +（teacher-forcing 32/32）。MLA 注意力存储压缩后的 KV 状态——每个 token 为 576 个 +浮点数，而非 32,768 个（**缩小 57×**）——并跨重启持久保存 +（`.coli_kv`）：对话可暖启恢复，不需重新 prefill，结果与不中断的会话 +逐字节相同。DSA 稀疏注意力（GLM-5.2 的 lightning indexer）已忠实实现， +并通过强制选取所有 key，验证可精确复现稠密注意力。 + +### 诚实的推测解码 + +GLM-5.2 原生 MTP head 会起草 token，再由主模型以一次批量前向传播验证—— +条件合适时每次 forward 可产生 2.2–2.8 个 token。两条来之不易的规则已成为默认值： +MTP head 必须是 **int8**（int4 head 的接受率会崩塌到 0–4%，见 +[#8](https://github.com/JustVugg/colibri/issues/8)），且草稿与验证必须计算 +**相同函数**——`SPEC_PIN=1` 会把两者固定在同一 kernel family +（完整取证过程见 [#163](https://github.com/JustVugg/colibri/issues/163)）。 +语法强制草稿（[`GRAMMAR=file.gbnf`](docs/grammar-draft.md)）可在受限 JSON 输出中， +以近乎免费的代价提高接受率。推测解码是否带来净收益取决于缓存热度——请实测， +若不划算就使用 `DRAFT=0`。 + +## 实际成果 + +

+ 各硬件级别的实测解码速度 +

+ +同一套引擎、同一个 int4 容器——硬件只会改变专家的存放位置。 +[完整 benchmark 表格](docs/benchmarks.md)中的重点如下： + +- **6× RTX 5090，全部常驻：**解码 5.8–6.8 tok/s，TTFT 约 13 秒 + （[实验记录](docs/experiments/glm52-6x5090-2026-07-12.md)）； +- **128 GB、仅使用 CPU 的台式机：**热缓存后约 1.8 tok/s + （[#200](https://github.com/JustVugg/colibri/issues/200)）； +- **单张 RTX 5070 Ti 的笔记本级主机：**通过 GPU 常驻管线达到 1.07 tok/s + （[#273](https://github.com/JustVugg/colibri/issues/273)）； +- **25 GB 开发机：**冷启动 0.05–0.1 tok/s——这是项目起步时已证实的下限， + 也仍是诚实的基准。 + +质量来自测量，而非假设：int4 容器的量化损失，以及 scale granularity／rotation +消融实验，收录于 [docs/benchmarks.md](docs/benchmarks.md#quality-benchmark)、 +[#108](https://github.com/JustVugg/colibri/issues/108) 与 +[#81](https://github.com/JustVugg/colibri/issues/81)。 + +## 开始使用 + +### 1. 获取模型 + +Hugging Face 上已有预转换的 **GLM-5.2 int4** 容器——请务必使用 +**含 int8 MTP head 的版本**： + +**https://huggingface.co/mateogrgic/GLM-5.2-colibri-int4-with-int8-mtp** + +> ⚠️ 原始镜像使用 int4 MTP head → 草稿接受率为 0% +>（[#8](https://github.com/JustVugg/colibri/issues/8)）。请检查你的版本： +> `ls -l /out-mtp-*`——正确的 int8 大小为 `3527131672 / 5366238584 / 1065950496`。 + +你也可以自行从 FP8 源转换——只需一条可断点续传的命令，且任何时候都不需要 +在磁盘上同时存放完整的 756 GB： + +```bash +cd c && ./setup.sh # 检查 gcc/OpenMP、构建并运行自测 +./coli convert --model /nvme/glm52_i4 # 逐 shard 下载并转换（仅此一次需要 python） +``` + +### 2. 运行 + +```bash +COLI_MODEL=/nvme/glm52_i4 ./coli chat # 自动检测 RAM 预算、缓存与 MTP +COLI_MODEL=/nvme/glm52_i4 ./coli plan # 查看规划的 VRAM／RAM／磁盘配置 +COLI_MODEL=/nvme/glm52_i4 ./coli doctor # 只读就绪检查 +./coli web --model /nvme/glm52_i4 # 在同一端口提供 API 与网页仪表盘 +./coli serve --model /nvme/glm52_i4 # 仅提供 OpenAI 兼容 API +``` + +引擎运行时是纯 C——python 只供一次性转换工具与可选的 API gateway 使用。 + +### 3. 深入了解 + +| 主题 | 文档 | +|---|---| +| Benchmark、社区实测数据、质量测量 | [docs/benchmarks.md](docs/benchmarks.md) | +| 调优选项、策略、学习型缓存、预取 | [docs/tuning.md](docs/tuning.md) | +| Windows 11 原生构建（含 CUDA DLL） | [docs/windows.md](docs/windows.md) | +| CUDA 后端、VRAM 专家层级、全部常驻 | [docs/cuda.md](docs/cuda.md) | +| Apple Silicon Metal 后端 | [docs/metal.md](docs/metal.md) | +| OpenAI 兼容 API、KV slots、网页仪表盘 | [docs/api.md](docs/api.md) | +| 语法强制草稿（结构化输出） | [docs/grammar-draft.md](docs/grammar-draft.md) | +| 环境变量完整清单 | [docs/ENVIRONMENT.md](docs/ENVIRONMENT.md) | + +## 支持项目 + +colibrì 最初由一人使用 12 核心、25 GB RAM 的笔记本开发； +如今它的数据来自社区中各种真实机器。如果这个项目对你有用： + +- ⭐ 为仓库加星并分享； +- 🐛 以 issue 提交你的硬件 benchmark 数据——实测数据比任何其他事都更能推动项目； +- 💬 若想赞助开发或捐赠硬件，请通过 GitHub issues 联系。 + +## 仓库结构 + +``` +Makefile 根目录构建／检查入口 +c/ +├── colibri.c 引擎主文件 +├── quant.h 量化 matmul 内核（SIMD 多架构） +├── sample.h 采样与 stop-set 管理 +├── kv_persist.h .coli_kv 磁盘持久化 +├── telemetry.h 仪表盘协议、统计与用量持久化 +├── st.h, tok.h, json.h 运行时头文件 +├── backend_cuda.* 可选的 CUDA 层级 +├── Makefile 构建与本地检查 +├── coli 用户界面 CLI +├── openai_server.py OpenAI 兼容 HTTP gateway +├── setup.sh 一条命令完成本地设置 +├── tools/ 离线转换、fixtures 与 benchmarks +├── scripts/ 长时间转换辅助工具 +└── tests/ 零依赖的 C 与 Python 测试 +web/ 浏览器 UI（纯 OpenAI API client） +desktop/ 封装网页 UI 的 Tauri v2 桌面 shell +docs/ 参考文档、实验与媒体文件 +``` + +运行时路径刻意保持扁平、易读：`colibri.c` 加上若干头文件。 +在仓库根目录执行 `make`、`make check` 与 `make clean`， +都会转发给引擎的 Makefile。 + +## 为什么叫"colibrì" + +蜂鸟只有几克重，能在原地悬停，并在一天内造访上千朵花。 +这套引擎只用蜂鸟般的配给，就能让 744B 参数的巨人运转： +25 GB RAM、十二个 CPU 核心，以及对磁盘的大量耐心。 + +## 许可证 + +Apache 2.0。GLM-5.2 权重由 Z.ai 以 MIT 许可发布。 diff --git a/README.zh-TW.md b/README.zh-TW.md index 1662617c..2576eba1 100644 --- a/README.zh-TW.md +++ b/README.zh-TW.md @@ -3,7 +3,7 @@

- English · 繁體中文 + English · 简体中文 · 繁體中文 · Italiano

**小巧引擎，龐大模型。**只要約 25 GB 記憶體，就能在消費級電腦上執行 **GLM-5.2（744B 參數的 MoE）**——以零相依套件的純 C 實作，從硬碟串流載入專家。 @@ -60,7 +60,7 @@ VRAM／RAM／硬碟層級長條，以及角落的即時迷你大腦。

**存放在硬碟**（約 370 GB），並**隨需串流載入**，搭配逐層 LRU 快取、 會學習的熱門專家固定儲存區，以及選用的 VRAM 層級。 -引擎由單一 C 檔（`c/glm.c`）與少量標頭檔組成。不需要 BLAS， +引擎由主 C 檔（`c/colibri.c`）與多個標頭檔模組組成。不需要 BLAS， 執行階段不需要 Python，也不需要 GPU。 ## 運作方式 @@ -203,7 +203,11 @@ colibrì 最初是由一人使用 12 核心、25 GB RAM 的筆電開發； ``` Makefile 根目錄建置／檢查入口 c/ -├── glm.c 單檔 GLM 引擎 +├── colibri.c GLM 引擎主檔 +├── quant.h 量化 matmul kernel +├── sample.h 取樣與 stop-set +├── kv_persist.h .coli_kv 磁碟持久化 +├── telemetry.h 儀表板協定、統計 ├── st.h, tok.h, json.h 執行階段標頭檔 ├── backend_cuda.* 選用的 CUDA 層級 ├── Makefile 建置與本機檢查 @@ -218,7 +222,7 @@ desktop/ 包裝網頁 UI 的 Tauri v2 桌面 shell docs/ 參考文件、實驗與媒體檔 ``` -執行階段路徑刻意維持扁平、易讀：`glm.c` 加上少量標頭檔。 +執行階段路徑刻意維持扁平、易讀：`colibri.c` 加上模組化標頭檔。 在儲存庫根目錄執行 `make`、`make check` 與 `make clean`， 都會轉交給引擎的 Makefile。 diff --git a/SUMMARY.md b/SUMMARY.md new file mode 100644 index 00000000..f4de3116 --- /dev/null +++ b/SUMMARY.md @@ -0,0 +1,356 @@ +# E5 — MTLResidencySet over the existing malloc'd slabs (experiment branch) + +Branch: `e5/metal-residency-set` (cut from `origin/dev` @ `caa49f7`, per spec — E4 was cut +from `main` @ `72d3d37`; `backend_metal.mm`/`.h` are byte-identical between the two bases, +confirmed via `git diff 72d3d37 origin/dev -- c/backend_metal.mm c/backend_metal.h`). + +## The hypothesis + +E4 (MTLHeap-backed slabs) proved that batching residency declaration kills the GPU stall +(25.9s → 3.9s at cap16, −85%), but changing the *allocation* (heap sub-buffers instead of +malloc'd host memory) brought a +12–13s expert-disk-load tax, suspected first-touch/lock +contention on CPU-writes into GPU-owned heap pages. E5 decouples the two: keep the exact +same malloc'd slabs and per-slab `newBufferWithBytesNoCopy`-wrapped `MTLBuffer`s, and change +**only** residency bookkeeping — declare residency once, ahead of time, on a set attached to +the command queue, instead of once per command buffer via `useResource:`. If the stall +reduction survives without the load-path tax (malloc pages never change ownership), E5 wins. + +## What changed + +All mechanism code is confined to `c/backend_metal.mm` — +`coli_metal_register`/`coli_metal_unregister`'s existing signatures and every call site in +`colibri.c` (expert_load, uring_load_add, qalloc, kv_alloc, map_of_fd) are untouched; the +residency-set bookkeeping lives entirely inside those two functions' existing bodies. The +`colibri.c`/`backend_metal.h` touches are two, both coordinator-sanctioned: the validator +round-1 instrumentation hook (`coli_metal_resset_stats` + the gate-on-only `METAL-RESSET:` +stats line in `profile_print`) and the ported fslab-OOM unwind fix (see "Validator round 1 +fixes" item 4). Still a smaller diff shape than E4, +which needed a new alloc/free API and four new `glm.c` call-site arms because it changed the +allocation function itself. + +Env-gated `COLI_METAL_RESSET=1`, default OFF, runtime `@available(macOS 15.0, *)` guard with +a one-line stderr fallback when requested on an older OS or when residency-set creation +fails. Gate off ⇒ every new branch is skipped and behavior is byte-for-byte the stock path +(verified by inspection: `g_resset_enabled` starts `false` and nothing sets it except inside +the `COLI_METAL_RESSET` `getenv` branch in `coli_metal_init`, so `resset_add`/`resset_remove`/ +`resset_flush` are no-ops and `moe_submit`'s `useResource:` loop runs unconditionally). + +### Lifecycle (`c/backend_metal.mm`) + +- **Init** (`coli_metal_init`, end of the existing pipeline-setup `@autoreleasepool`): if + `COLI_METAL_RESSET=1` and `@available(macOS 15.0, *)`, create one + `MTLResidencySetDescriptor` (`initialCapacity=4096`, a presize hint only), call + `[g_dev newResidencySetWithDescriptor:desc error:&err]`, and `[g_queue addResidencySet:rs]` + — one set, attached once, for the process lifetime. Failure (old OS or creation error) + prints one stderr line and leaves `g_resset_enabled=false` — stock path. +- **`coli_metal_register`**: after wrapping the buffer exactly as today + (`newBufferWithBytesNoCopy`) and pushing the `g_slabs` entry under `g_slab_mtx` exactly as + today, calls `resset_add(b)` **after dropping `g_slab_mtx`** but before returning. + `resset_add` takes a dedicated `g_resset_mtx` (guarding only the set mutations and the + dirty flag), calls `[rs addAllocation:b]` and sets `g_resset_dirty` — **it does not + commit**. No Metal call ever runs under `g_slab_mtx` (validator round-1 fix; E4's audit + round 2 identified mutex-over-live-Metal-call as the leading suspect for its +12s + expert-disk regression). Re-registering a live base (no in-tree caller does today) drops + the replaced wrapper from the set via `resset_remove(old)` before adding the new one + (hazard-audit defensive fix — the set would otherwise retain the old buffer, and its + pages' residency, forever), keeping set membership an exact mirror of `g_slabs`. +- **`coli_metal_unregister`**: erases the `g_slabs` entry under `g_slab_mtx` (stashing the + buffer), then calls `resset_remove(b)` **outside `g_slab_mtx`**, before returning. + `resset_remove` (under `g_resset_mtx`) calls `[rs removeAllocation:b]` **and commits + immediately** — no batching — because the caller frees the host memory right after the + function returns. See UNCERTAINTIES for why this asymmetry is deliberate. +- **`moe_submit`** (the one function whose `use` list — resolved expert weight/scale slabs — + scales with LRU cache size): calls `resset_flush()` at the top (commits any pending adds + from `resset_add`, under `g_resset_mtx` — it never touches the slab lock), then, if + `g_resset_enabled`, **skips** the + `for(auto&b:use) [e useResource:b usage:MTLResourceUsageRead];` loop entirely — residency + is already guaranteed by the queue-attached set. Every other `useResource:` call site in the + file (`bind_gemv`'s weight/scale buffers, `coli_metal_attn_decode`/`coli_metal_layer_decode`'s + `Lb`/`Rb`/`kvbW`/`kvbS`/`inB`/`pnB`/`rwB`/`rbB`, `coli_metal_gemm`'s `wb`/`sb`) is + **left completely unchanged**, regardless of the flag — see "Why only `moe_submit`" below. +- **Shutdown** (`coli_metal_shutdown`): `[g_queue removeResidencySet:rs]` then clears the + globals, ahead of the existing `g_queue=nil; g_dev=nil;`. + +### Why only `moe_submit` skips `useResource:` + +Apple's `MTLResidencySet` class reference (developer.apple.com, fetched during design on +2026-07-18) is explicit: *"Residency sets don't support hazard tracking, so you need to +account for hazards with fences and events."* The SDK header on this box +(`MTLResidencySet.h`, read directly) is **silent** on hazard tracking — the statement comes +from Apple's online documentation and adoption guide ("Simplifying GPU resource management +with residency sets"), not the header (see UNCERTAINTIES for sourcing). Dropping +`useResource:` therefore risks losing whatever hazard-tracking value those calls provided. Rather than apply the residency set uniformly and +argue *in general* that hazard tracking isn't load-bearing, this diff draws the line at the +one call site the mechanism history actually implicates: + +`moe_submit`'s `use` vector holds only **read-only** (`MTLResourceUsageRead`), **indirectly +referenced** slab buffers — the kernel (`moe_gemv`) never touches them via `setBuffer:`; it +dereferences raw GPU addresses (`waddr[e]`/`saddr[e]`) baked into a separately-bound address +array (`bag`/`bau`/`bad`/`bsg`/`bsu`/`bsd`), which is exactly the "indirect reference" case +`useResource:` exists for. No GPU-side write ever touches these buffers, so there is no +write-after-write/read-after-write hazard for Metal's tracking to have been serializing in +the first place; the one real hazard — a slab unregistered+freed+reused by the CPU while an +async in-flight `moe_block_begin` command buffer still references it via a baked-in GPU +address — is a **CPU-write race that Metal's hazard tracking never protected against anyway** +(hazard tracking only covers GPU-side command dependencies visible through the Metal API; a +raw host-memory write via `pread`/`memcpy` is invisible to it regardless of `useResource:`). +That race is, and always was, the engine's own responsibility (slot/generation lifecycle: a +slab isn't freed while an outstanding async handle still owns it) — unrelated to E5. + +Every other call site (`bind_gemv`, attention K/V cache writes) either doesn't scale with +cache size (fixed per-layer dense tensors — no perf benefit to touching) or has real +GPU-side write traffic in the same encoder (`Lb`/`Rb` are written by `a_copy` and read by +`a_score`/`a_clat` within one encoder — currently ordered by explicit +`memoryBarrierWithScope:MTLBarrierScopeBuffers` calls already present in `encode_attention`, +not by `useResource:`'s hazard tracking, but touching them wasn't needed for the hypothesis +and was judged not worth the added surface area). Leaving them untouched keeps the diff's +blast radius matched to the one seam the fix-plan's v5 finding actually names. + +### Deferred-commit design (`resset_add` batches; `resset_remove` doesn't) + +`coli_metal_register` is called from parallel OpenMP loader threads in tight bursts +("warmup fan-out" — same phrase E4's audit used for the same threads). Committing on every +single `addAllocation:` would reintroduce a per-slab cost on the load path, which is exactly +what E4's own +12s regression looked like (mutex held across a live Metal call, serializing +loader threads). So `resset_add` only marks `g_resset_dirty`; the commit is deferred to the +next `moe_submit` call, which flushes once via `resset_flush()` before it relies on the set +for residency. + +This is correct — not just fast — because of an existing invariant the codebase already +depends on for `resolve()` to work at all: a slab's `coli_metal_register` call always +completes — including its trailing `resset_add`, which runs after `g_slab_mtx` is dropped +but **before the function returns** — before any dispatch that references that slab's +pointer can call `resolve()` for it (the caller in `colibri.c` cannot pass a freshly-loaded +expert's pointer to a dispatch before the load — which registers it — returns). After the +validator round-1 mutex split, the flush's synchronization runs through `g_resset_mtx` +alone: `resset_add`'s set mutation + dirty write and `resset_flush`'s dirty read + commit +are serialized by that one mutex, whose release/acquire pairs provide the memory ordering; +`g_slab_mtx` still orders the slab-table bookkeeping (register-before-resolve) exactly as on +stock. So any slab a given `moe_submit` invocation will resolve was `addAllocation:`-ed (and +marked dirty) strictly before that invocation's `resset_flush()` acquired `g_resset_mtx` — +the flush is guaranteed to cover it, regardless of what other threads are concurrently +registering unrelated slabs. The two mutexes are never held simultaneously anywhere, so no +deadlock ordering exists to maintain. + +`resset_remove`, by contrast, commits synchronously and immediately, with no batching, +because the caller (`colibri.c`, in every one of the four slab-realloc call sites, and in +`kv_alloc`) frees the underlying host memory *right after* `coli_metal_unregister` returns. +An uncommitted-but-still-set-member allocation pointing at memory the host has already freed +is a potential use-after-free the GPU could act on — deferring that removal is not a +performance-vs-safety tradeoff, it's just unsafe, so it isn't deferred. (The spec's own +lifecycle wording backs this reading: "`coli_metal_register` → add allocation + commit +**(batch commits where call pattern allows)**" carries a batching allowance that +"`coli_metal_unregister` → remove + commit" does not.) + +## Instrumentation parity + +No existing counter's semantics changed. `coli_metal_moe_times`/`coli_metal_moe_counts` +(`g_t_setup`, `g_t_gpu`, `g_t_kernel`, `g_t_scatter`, `g_moe_ok`/`g_moe_fb`/`g_moe_experts`) +are computed exactly as before — `resset_flush()` runs *before* `ts_start = mnow()` in +`moe_submit`, so its cost is **outside** `g_t_setup`, keeping the orchestrator's A/B harness +reading the same counters with the same meaning across stock/E4/E5. The flush cost is +surfaced separately (validator round-1 fix — the original design left it invisible, a blind +spot for the battery): a dedicated `g_t_resset_flush` accumulator timed around the flush in +`moe_submit`, exported via `coli_metal_resset_stats()` (`backend_metal.h`) and printed by +`profile_print` as its own `METAL-RESSET: flush N.NNs` line — a **separate line following +the `METAL:` line, mirroring E4's `METAL-HEAP:` convention, so the existing `METAL:` line +the harness parses keeps its exact format** — printed **only when the gate is on** (the +function returns 0 when off), so stock output stays byte-identical. The register-side +`resset_add`/`resset_remove` costs have no dedicated counter: they run inside the engine's +existing expert-load wait accounting (the `t_ewait` window in `colibri.c`), noted in a comment +at `resset_add`, so a load-path regression from set bookkeeping would already show in the +existing disk/wait numbers. `[METAL] residency-set: on` / the two fallback stderr lines from +`coli_metal_init` confirm which path a run took. + +## Validator round 1 fixes + +1. **REQUIRED, Metal calls hoisted out of `g_slab_mtx`** (`backend_metal.mm`): the original + design ran `addAllocation:`/`removeAllocation:`/`commit` while holding `g_slab_mtx`, the + lock the parallel OMP loader threads contend on — structurally identical to the + mutex-over-live-Metal-call shape E4's audit round 2 identified as the leading suspect for + its replicated +12s expert-disk regression, and the SDK header notes commit on a resident + set tries to make resources resident "instantly" (real synchronous work; this set is + resident from startup since it is queue-attached for the process lifetime). Fixed by + introducing a dedicated `g_resset_mtx` guarding only the set mutations + dirty flag; + `g_slabs` push/erase stays under `g_slab_mtx` exactly as stock; the two mutexes are never + held together. The register→flush→resolve happens-before argument is preserved — see the + updated "Deferred-commit design" section and the comment at `resset_add`. +2. **REQUIRED, false citations corrected** (this file + the `moe_submit` commit message, + rewritten pre-push): the original text attributed the hazard-tracking and thread-safety + statements to the SDK header (`MTLResidencySet.h`), which is in fact silent on both + topics. The statements come from Apple's **online** `MTLResidencySet` class reference and + the "Simplifying GPU resource management with residency sets" adoption guide (both + fetched 2026-07-18 during design). All attributions now name the actual source; where a + claim rests on design reasoning rather than documentation, it is labeled as such. +3. **REQUIRED, flush cost made harness-visible**: `g_t_resset_flush` + + `coli_metal_resset_stats()` + the gate-on-only `METAL-RESSET:` line in `profile_print` — + see "Instrumentation parity" above. +4. **Pre-existing fslab OOM-unwind bug — now CARRIED ON THIS BRANCH** (follow-up commit, + coordinator-sanctioned second `colibri.c` change): `expert_load`'s fslab OOM path + (`c/colibri.c`, in `expert_load_impl`) freed `s->slab` via `compat_aligned_free` **without** + `coli_metal_unregister` — on stock that leaves a stale `g_slabs` entry whose GPU + exposure ends with the last command buffer that declared it; under E5 the buffer would + additionally be a **permanent residency-set member** referencing freed host memory until + some later realloc of the same slot unregisters by pointer, a strictly longer-lived + exposure than stock's transient per-CB one. Fixed by porting E4's reference + implementation (`6753225`) to dev's non-heap code shape: `coli_metal_unregister(s->slab)` + before the free. The `uring_load_add` analog (E4's audit round-2 "cheap insurance") is + deliberately NOT carried: that arm is `#ifdef __linux__`-gated and `COLI_METAL` is + macOS-only, so it is dead code on every real build target, and unlike E4 this branch has + no allocation-path reason to touch the function at all. + +## Per-seam differences vs E4 + +| Seam | E4 (`e4/metal-heap`) | E5 (this branch) | +|---|---|---| +| Allocation | New: `MTLHeap` sub-buffers via `coli_metal_heap_alloc` | Unchanged: same `posix_memalign` + `newBufferWithBytesNoCopy` | +| Coordinator C source / `backend_metal.h` | `glm.c` touched (new alloc/free API, 4 call sites + `expert_host_release`) | `colibri.c` + header touched only for instrumentation and the OOM-unwind fix | +| Residency scope | Declared once **per command buffer** (`useHeap:`, still inside `moe_submit`) | Declared once **for the process** (queue-attached set), refreshed incrementally at register/unregister | +| Hazard tracking | Heap sub-buffers forced `MTLHazardTrackingModeUntracked` always (allocation-level) | Untouched at the resource level; `moe_submit` alone stops calling `useResource:` (encoder-level), independent of `COLI_METAL_UNTRACKED` | +| Per-buffer vs per-set skip | `[b heap]` (Metal's own `MTLResource.heap` property) checked per buffer — heterogeneous mixes possible if a slab fell back to malloc | Blanket `if (!g_resset_enabled)` — homogeneous by construction, since every registered slab goes through the same `coli_metal_register` path when the gate is on | +| Availability guard | None needed (`MTLHeap` is old API) | `@available(macOS 15.0, *)`, matching this box's macOS 26.5 but required for portability | +| Known regression | +12–13s expert-disk load at cap16 (suspected first-touch/lock contention on heap pages) | None expected — malloc pages never change ownership; **unverified without a run** | + +## What to measure (orchestrator, cap1/cap16, stock vs E4 vs E5) + +1. **GPU stall** (`coli_metal_moe_times` gpu/kernel breakdown) — success: E5 ≈ E4's + −85%-class reduction vs stock at cap16. +2. **Expert-disk load path** (existing load/service-time counters) — success: E5 ≈ stock, + i.e. **no** repeat of E4's +12–13s tax, since allocation is untouched. +3. **tok/s** — should track (1) and (2) together. +4. **md5 within a fixed dispatch composition** — flag on vs off must be byte-identical at a + given cap (the "Output-invariant by construction" hard constraint); flag-on vs flag-on + across cap1/cap16 may legitimately differ (different dispatch composition, per the + fix-plan's "Determinism side-finding"). +5. **`[METAL] residency-set: on` line present in stderr** at flag-on startup, and absent + (or the OS<15/create-failed fallback line) otherwise — cheap sanity check that a run + actually exercised the intended path before trusting its numbers. Also read the + **`METAL-RESSET: flush` line** (gate-on only): if that number is large, the deferred + set-commit cost is eating the stall win from the dispatch side. +6. If the hypothesis holds (E5 stall ≈ E4, E5 load-path ≈ stock, identical output), E5 becomes + the upstream PR candidate and must include the cap-default recalibration flagged in PR + #386's CURRENT-STATE CALIBRATION markers, per the spec's validation plan. + +## Build + +`cd c && make glm METAL=1` and a separate explicit `-Wall -Wextra` compile of +`backend_metal.mm` (the Makefile's `METALXX` line does not itself pass `-Wall -Wextra`, so +the warning surface was checked with those flags added explicitly; current `dev` contributes +one pre-existing `unused variable 'TG'` warning), plus +`cd c && make glm` (plain, non-Metal — the one `colibri.c` instrumentation touch, the `METAL-RESSET` stats line, +is inside the pre-existing `#ifdef COLI_METAL` arm of `profile_print`, so the plain build +compiles none of it), and +`make metal-test` (existing synthetic kernel-correctness unit test — no model, no +`glm52_i4/`, random weights — run once with `COLI_METAL_RESSET` unset and once with +`COLI_METAL_RESSET=1` to numerically exercise `coli_metal_register`/`moe_submit`'s changed +code path, since the task scope excludes running the real model). Exact results in the final +report, not here (build results belong to the report per the task's deliverable split, and +this file is written before the batched build run, per the scheduling constraint). + +## UNCERTAINTIES + +**Everything below is a judgment call, a seam where the residency-set lifecycle interacts +with the existing queue/command-buffer structure, or something unverifiable without a real +model run — flagged per the task's hard requirement.** + +1. **The central design risk: skipping `useResource:` in `moe_submit` gives up Metal's + automatic hazard tracking for that buffer set.** Sourcing (corrected in validator round + 1): the SDK header on this box + (`/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/.../Headers/MTLResidencySet.h`, + read directly) documents the protocol only in terms of residency and says nothing about + hazard tracking either way; the two operative statements are from Apple's **online** + documentation (fetched 2026-07-18): the "Simplifying GPU resource management with + residency sets" adoption guide — *"You don't need to call `useResource`/`useHeap`... for + allocations in a residency set"* — and the `MTLResidencySet` class reference — + *"Residency sets don't support hazard tracking, so you need to account for hazards with + fences and events."* I reasoned through every + code path that touches `moe_submit`'s `use` buffers (read-only, indirectly referenced, + never concurrently written, freed only after the engine's own slot lifecycle guarantees + no outstanding async reference) and concluded removing `useResource:` there specifically + is safe — but this reasoning is **not the same as having run the model**. If any code + path I didn't trace lets a slab get unregistered while an async `moe_block_begin` handle + is still in flight and reading it, this change removes a mitigation (weak as it may have + been) that existed before. **This is the #1 thing to watch for md5 divergence on**, and + the reason the scope was deliberately narrowed to `moe_submit` alone rather than applied + uniformly. +2. **Residency-set mutations are serialized under a dedicated `g_resset_mtx` (validator + round-1 fix — originally they ran under `g_slab_mtx`, the E4-regression shape; no Metal + call runs under the slab lock anymore).** The serialization itself is kept as required + for correctness: Apple's online `MTLResidencySet` class reference states the set's + *"methods aren't thread-safe"* (the SDK header contains no thread-safety statement either + way — citation corrected in round 1; the online doc is the source). What remains + **unverified without profiling a loaded run** is the *cost* of the calls themselves: + `resset_remove`'s synchronous `commit` runs inside `coli_metal_unregister` on the + loader path (its cost lands in the existing `t_ewait` accounting), and the SDK header + says commit on a resident set tries to make added/removed resources resident/non-resident + *"instantly"* — real synchronous work, since this set is resident from startup + (queue-attached for the process lifetime). If `commit()`/`addAllocation:` turn out + expensive on this hardware/OS build, the load path degrades through set bookkeeping + rather than mutex contention — a different, now-decoupled failure mode, but the same + symptom as E4's regression. Orchestrator: check E5's load-path timing against stock, not + just against E4, and read the new `METAL-RESSET: flush` line for the dispatch-side share. +3. **`resset_flush()`'s cost sits outside `g_t_setup`/the `moe_times` breakdown** (it runs + before `ts_start = mnow()`), by design, to keep the harness's existing counters + meaningful — and, since validator round 1, it is **no longer invisible**: the + `g_t_resset_flush` accumulator surfaces it as the gate-on-only `METAL-RESSET: flush` + line (see "Instrumentation parity"). Residual blind spots: (a) the accumulator is a + plain double written from `moe_submit` on the engine thread, matching the existing + `g_t_setup` convention — if `moe_submit` were ever called from multiple threads + concurrently, both counters would be equally wrong; (b) the register-side + `resset_add`/`resset_remove` costs have no dedicated counter and are only visible + blended into the existing `t_ewait`/disk-wait numbers (comment at `resset_add` says so) + — a fine-grained attribution would need a throwaway probe. +4. **`initialCapacity = 4096` on the `MTLResidencySetDescriptor` is an unverified guess.** + It's documented as a presize hint only (no correctness effect either way), chosen to be + "clearly larger than the permanent-weight-tensor + KV-cache + plausible cap16 LRU-slab + count" without actually counting those registrations precisely. Too small just means + internal array growth; not a correctness concern, flagged only because it's a number I + picked without measuring. +5. **Not calling `requestResidency()` proactively.** Apple's guide frames it as an optional + latency-hiding call ("call ahead of time during non-critical moments... to minimize [first + command buffer] latency"), and Blender's Cycles PR (the spec's cited reference + implementation) doesn't appear to use it either per its PR description. Omitted to keep + the lifecycle minimal and match the reference pattern; if profiling shows a + first-command-buffer-after-a-load-burst latency spike, this is the documented lever to try + next, not implemented here. +6. **The deferred-commit correctness argument (item in "Deferred-commit design" above) rests + on a single-writer-before-single-reader program-order guarantee that is true today by + inspection but is not an invariant enforced anywhere in code** (no assertion, no type-level + guarantee) — it's the same kind of implicit ordering `resolve()` itself already depends on + for correctness (a slab must be registered before any dispatch can resolve its pointer), + so this diff doesn't introduce a new category of fragility, but it's worth naming + explicitly rather than leaving implicit. +7. **Async `moe_block_begin`/`moe_block_end` overlap with concurrent `register()` calls** + (background loader threads registering new/different experts while an unrelated MoE block + is still in flight on the GPU) was reasoned through but never exercised in a real + concurrent stress scenario — the synthetic `metal-test` unit test's `run_moe` calls are + single-threaded and synchronous (`coli_metal_moe_block`, not the async `_begin`/`_end` + pair), so it does **not** cover this interleaving. The real engine's `PILOT`/prefetch and + `moe_block_begin`/`_end` overlap path is exactly the concurrency shape most likely to + expose a bug in this design if one exists, and is untested here by construction (out of + scope: no model runs). +8. **`coli_metal_gemm` (prefill path) and `bind_gemv` (attention path) still call + `useResource:` unconditionally, so they get no CPU-overhead benefit from the residency set + even though their buffers are also set members.** This is deliberate (see "Why only + `moe_submit` skips" above) but means E5's win, if any, is scoped to the decode-path MoE + dispatch loop specifically — prefill and attention timing should be unaffected by the flag, + which is itself a testable prediction the orchestrator's harness can check. +9. **API surface verified against this box's actual SDK headers** + (`MTLResidencySet.h`, `MTLDevice.h`, `MTLCommandQueue.h`, `MTLAllocation.h`, + `MTLResource.h` — all read directly, not from memory) and against Apple's own + "Simplifying GPU resource management with residency sets" guide, so the method names/ + signatures (`newResidencySetWithDescriptor:error:`, `addResidencySet:`, + `removeResidencySet:`, `addAllocation:`, `removeAllocation:`, `commit`) are + high-confidence. What is **not** independently verified is runtime behavior beyond what + the docs state and what the synthetic unit test exercises — no substitute for the + orchestrator's real cap-sweep battery. +10. **Pre-existing fslab OOM-unwind bug — carried on this branch** (follow-up commit; see + "Validator round 1 fixes" item 4 for the full mechanism). The one-line + unregister-before-free fix from E4's `6753225` is ported to dev's non-heap code shape, + so the upstream PR built from E5 inherits it automatically. Residual notes: (a) the fix + is only reachable through the fslab-OOM path (allocation failure mid-load), so it is + untestable without an OOM-injection harness and cannot affect the orchestrator's + controlled A/B runs at sane RAM headroom — carried as correctness insurance, verified by + inspection + clean builds only; (b) the `__linux__`-gated `uring_load_add` analog is + deliberately not carried (dead code on every real build target — rationale in the fixes + section). diff --git a/c/.gitignore b/c/.gitignore index d8e1f882..51695f90 100644 --- a/c/.gitignore +++ b/c/.gitignore @@ -3,3 +3,5 @@ glm_tiny/ olmoe_hf/ olmoe_i4/ .build-config +tests/test_st_mirror +tests/test_st_pread diff --git a/c/Makefile b/c/Makefile index 1f8583b7..1799b6b5 100644 --- a/c/Makefile +++ b/c/Makefile @@ -56,11 +56,16 @@ OMPL = endif CFLAGS = -O3 $(OMPC) -Wall -Wextra -Wno-unused-parameter -Wno-misleading-indentation -Wno-unused-function # Opt-in: ARCH=native appends -mcpu=native (arm64 clang uses -mcpu, not -march), -# which unlocks the i8mm SMMLA int8/int4 dot kernels in glm.c. ARCH unset -> +# which unlocks the i8mm SMMLA int8/int4 dot kernels in colibri.c. ARCH unset -> # no -mcpu, default build byte-identical. Apple clang knows apple-m4 / native. +# For older X86_64 Macs (for example, Mac Pro 2019) we need to use -march ifneq ($(ARCH),) +ifneq (,$(X86_64)) +CFLAGS += -march=$(ARCH) +else CFLAGS += -mcpu=$(ARCH) endif +endif LDFLAGS = -lm $(OMPL) EXE = else ifneq ($(IS_WIN),) @@ -70,7 +75,7 @@ else ifneq ($(IS_WIN),) # ARCH default = x86-64-v3 (portable binary with AVX2). For max speed on THIS # machine use ARCH=native: on AVX-VNNI CPUs (Intel Alder Lake+, Meteor Lake+) # it also unlocks the 128-bit VPDPBUSD int8/int4 dot kernel (dot_i8i8/dot_i4i8), -# which the x86-64-v3 baseline does not define. The #ifdef guards in glm.c mean +# which the x86-64-v3 baseline does not define. The #ifdef guards in colibri.c mean # a v3 build simply compiles out the VNNI path - safe on any x86-64. CC = gcc ARCH ?= x86-64-v3 @@ -143,17 +148,45 @@ else CUDA_HOME ?= /usr/local/cuda NVCC ?= $(CUDA_HOME)/bin/nvcc endif +# CUDA target selection. Default `native` builds SASS for the build machine's GPU +# only (fast local dev). A headless CI/Docker build has no GPU, so `-arch=native` +# there detects nothing and the shipped coli_cuda.dll fails at first kernel dispatch. +# For a portable/release DLL that runs on any Ampere..Blackwell card AND JITs on +# newer archs, build with CUDA_ARCH=portable: it emits SASS for sm_80/86/89/90/120 +# plus a compute_120 PTX fallback (sm_120 / RTX 50-series needs CUDA >= 12.8). CUDA_ARCH ?= native +ifeq ($(CUDA_ARCH),portable) +CUDA_GENCODE = -gencode arch=compute_80,code=sm_80 \ + -gencode arch=compute_86,code=sm_86 \ + -gencode arch=compute_89,code=sm_89 \ + -gencode arch=compute_90,code=sm_90 \ + -gencode arch=compute_120,code=sm_120 \ + -gencode arch=compute_120,code=compute_120 +else +CUDA_GENCODE = -arch=$(CUDA_ARCH) +endif ifneq ($(IS_WIN),) # nvcc's host compiler on Windows is MSVC cl.exe: the GCC-style # -Xcompiler=-Wall,-Wextra is rejected ("D8021 invalid numeric argument # '/Wextra'"), which made `make cuda-dll` unbuildable as shipped. -W3 is # the MSVC warning level (dash form, NOT /W3: MSYS make would mangle the # slash-form into a filesystem path). -NVCCFLAGS ?= -O3 -std=c++17 -arch=$(CUDA_ARCH) -Xcompiler=-W3 +NVCCFLAGS ?= -O3 -std=c++17 $(CUDA_GENCODE) -Xcompiler=-W3 else -NVCCFLAGS ?= -O3 -std=c++17 -arch=$(CUDA_ARCH) -Xcompiler=-Wall,-Wextra +NVCCFLAGS ?= -O3 -std=c++17 $(CUDA_GENCODE) -Xcompiler=-Wall,-Wextra endif +# HIP=1 builds the SAME backend for AMD GPUs via ROCm: backend_cuda.cu is +# compiled unchanged through backend_gpu_compat.h (one source, two vendors, +# like compat.h does for Windows). HIP_ARCH=native targets the GPU in this +# machine; set an explicit arch (e.g. HIP_ARCH=gfx1201) when distributing. +HIP ?= 0 +ROCM_HOME ?= /opt/rocm +HIPCC ?= $(ROCM_HOME)/bin/hipcc +HIP_ARCH ?= native +HIPCCFLAGS ?= -O3 -std=c++17 -x hip --offload-arch=$(HIP_ARCH) -Wall -Wextra +GPUCC = $(NVCC) +GPUFLAGS = $(NVCCFLAGS) +GPUCC_NAME = nvcc (set CUDA_HOME or NVCC) # PYTHON is a HOST tool (clean/test-c/test-python run it on the build machine), # so key it off the host, NOT $(IS_WIN) — which is derived from the *target* # triple ($(CC) -dumpmachine). Otherwise a cross build (make CC=x86_64-w64- @@ -166,7 +199,7 @@ else PYTHON ?= python3 endif CUDA_OBJ = -TEST_BINS = tests/test_json$(EXE) tests/test_st$(EXE) tests/test_st_pread$(EXE) tests/test_tier$(EXE) tests/test_grammar$(EXE) tests/test_schema_gbnf$(EXE) tests/test_decode_batch$(EXE) tests/test_idot$(EXE) tests/test_i4_grouped$(EXE) tests/test_stops$(EXE) tests/test_topp$(EXE) tests/test_sample_nan$(EXE) tests/test_kv_alloc$(EXE) tests/test_i4_acc512$(EXE) tests/test_compat_direct$(EXE) tests/test_dsa_select$(EXE) tests/test_logit_nan$(EXE) +TEST_BINS = tests/test_json$(EXE) tests/test_st$(EXE) tests/test_st_mirror$(EXE) tests/test_st_pread$(EXE) tests/test_tier$(EXE) tests/test_grammar$(EXE) tests/test_schema_gbnf$(EXE) tests/test_decode_batch$(EXE) tests/test_idot$(EXE) tests/test_i4_grouped$(EXE) tests/test_stops$(EXE) tests/test_topp$(EXE) tests/test_sample_nan$(EXE) tests/test_tok_o200k$(EXE) tests/test_kv_alloc$(EXE) tests/test_int3$(EXE) tests/test_int3_load$(EXE) tests/test_i4_acc512$(EXE) tests/test_compat_direct$(EXE) tests/test_dsa_select$(EXE) tests/test_logit_nan$(EXE) tests/test_pipe_block$(EXE) tests/test_e8_kernel$(EXE) ifneq (,$(LINUX)) TEST_BINS += tests/test_uring$(EXE) endif @@ -188,11 +221,26 @@ ifneq ($(IS_WIN),) # On Windows use CUDA_DLL=1 (runtime DLL), not CUDA=1 (direct link). $(error On Windows use: make CUDA_DLL=1 cuda-dll (see backend_loader.c)) endif +ifeq ($(HIP),1) +$(error choose CUDA=1 or HIP=1, not both) +endif CFLAGS += -DCOLI_CUDA LDFLAGS += -L$(CUDA_HOME)/lib64 -Wl,-rpath,$(CUDA_HOME)/lib64 -lcudart -lstdc++ CUDA_OBJ = backend_cuda.o endif +ifeq ($(HIP),1) +ifeq (,$(LINUX)) +$(error HIP=1 is supported only on Linux) +endif +CFLAGS += -DCOLI_CUDA +LDFLAGS += -L$(ROCM_HOME)/lib -Wl,-rpath,$(ROCM_HOME)/lib -lamdhip64 -lstdc++ +CUDA_OBJ = backend_cuda.o +GPUCC = $(HIPCC) +GPUFLAGS = $(HIPCCFLAGS) +GPUCC_NAME = hipcc (set ROCM_HOME or HIPCC) +endif + # METAL=1 adds an opt-in Apple-GPU backend (macOS only). The shader is compiled at # runtime, so no Xcode / offline metal compiler is required. Default build unchanged. METAL ?= 0 @@ -207,27 +255,34 @@ LDFLAGS += -framework Metal -framework Foundation -lc++ METAL_OBJ = backend_metal.o endif -all: glm$(EXE) +all: colibri$(EXE) -# phony 'glm' → 'glm.exe' on Windows (so 'make glm' and 'coli build' work on every platform) -glm: glm$(EXE) +# phony targets — 'glm' kept for backward compatibility +colibri: colibri$(EXE) +glm: colibri$(EXE) # Config stamp: make only tracks file timestamps, not flag changes. Without this, -# `make glm.exe CUDA_DLL=1` after a prior CPU-only build reports "up to date" and -# silently keeps the CPU-only binary (no CUDA loader) — a build that looks like it -# worked but isn't. We record the build-affecting flags in .build-config and rewrite -# it ONLY when they change (evaluated here at parse time, so the file's timestamp -# moves exactly when the config moves). glm.exe and the CUDA/loader objects depend +# `make colibri.exe CUDA_DLL=1` after a prior CPU-only build reports "up to date" +# and silently keeps the CPU-only binary (no CUDA loader) — a build that looks like +# it worked but isn't. We record the build-affecting flags in .build-config and +# rewrite it ONLY when they change (evaluated here at parse time, so the file's +# timestamp moves exactly when the config moves). The binary and CUDA/loader objects depend # on it, so they relink on a config change and stay put otherwise. (#306) -BUILD_CONFIG := $(CC)|$(CFLAGS)|$(LDFLAGS)|CUDA=$(CUDA)|CUDA_DLL=$(CUDA_DLL)|ARCH=$(ARCH)|CUDA_ARCH=$(CUDA_ARCH)|METAL=$(METAL) +BUILD_CONFIG := $(CC)|$(CFLAGS)|$(LDFLAGS)|CUDA=$(CUDA)|CUDA_DLL=$(CUDA_DLL)|ARCH=$(ARCH)|CUDA_ARCH=$(CUDA_ARCH)|METAL=$(METAL)|HIP=$(HIP)|HIP_ARCH=$(HIP_ARCH) BUILD_CONFIG_OLD := $(shell cat .build-config 2>/dev/null) ifneq "$(BUILD_CONFIG)" "$(BUILD_CONFIG_OLD)" -$(shell printf '%s' '$(BUILD_CONFIG)' > .build-config) +# $(file ...) writes via make's own primitive (GNU Make >= 4.0, 2013), NOT by +# spawning a shell. The earlier `$(shell printf '%s' ... > .build-config)` +# required a POSIX `printf` on PATH — present under MSYS2/Git-Bash but NOT under +# cmd.exe, so `make glm.exe` from the VS Native Tools prompt or with scoop MinGW +# (neither ships sh.exe) failed with "'printf' is not recognized" and left the +# stamp stale. $(file ...) works regardless of the shell make falls back to. (#478) +$(file >.build-config,$(BUILD_CONFIG)) endif .build-config: ; -glm$(EXE): glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h $(CUDA_OBJ) $(METAL_OBJ) .build-config - $(CC) $(CFLAGS) glm.c $(CUDA_OBJ) $(METAL_OBJ) -o glm$(EXE) $(LDFLAGS) +colibri$(EXE): colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h quant.h sample.h kv_persist.h telemetry.h $(CUDA_OBJ) $(METAL_OBJ) .build-config + $(CC) $(CFLAGS) colibri.c $(CUDA_OBJ) $(METAL_OBJ) -o colibri$(EXE) $(LDFLAGS) # Windows runtime loader object: resolves coli_cuda_* from coli_cuda.dll. backend_loader.o: backend_loader.c backend_cuda.h compat.h .build-config @@ -245,9 +300,9 @@ cuda-dll: backend_cuda.cu backend_cuda.h -L"$(CUDA_HOME)/lib/x64" -lcudart \ backend_cuda.cu -o coli_cuda.dll -backend_cuda.o: backend_cuda.cu backend_cuda.h .build-config - @command -v "$(NVCC)" >/dev/null 2>&1 || { echo "nvcc not found: set CUDA_HOME or NVCC" >&2; exit 1; } - "$(NVCC)" $(NVCCFLAGS) -c backend_cuda.cu -o $@ +backend_cuda.o: backend_cuda.cu backend_cuda.h backend_gpu_compat.h .build-config + @command -v "$(GPUCC)" >/dev/null 2>&1 || { echo "$(GPUCC_NAME) not found" >&2; exit 1; } + "$(GPUCC)" $(GPUFLAGS) -c backend_cuda.cu -o $@ backend_metal.o: backend_metal.mm backend_metal.h $(METALXX) -c backend_metal.mm -o $@ @@ -256,14 +311,23 @@ metal-test: tests/test_backend_metal.mm backend_metal.mm backend_metal.h $(METALXX) tests/test_backend_metal.mm backend_metal.mm -framework Metal -framework Foundation -o backend_metal_test ./backend_metal_test -cuda-test: backend_cuda.cu backend_cuda.h tests/test_backend_cuda.cu - @command -v "$(NVCC)" >/dev/null 2>&1 || { echo "nvcc not found: set CUDA_HOME or NVCC" >&2; exit 1; } - "$(NVCC)" $(NVCCFLAGS) backend_cuda.cu tests/test_backend_cuda.cu -o backend_cuda_test$(EXE) +cuda-test: backend_cuda.cu backend_cuda.h backend_gpu_compat.h tests/test_backend_cuda.cu + @command -v "$(GPUCC)" >/dev/null 2>&1 || { echo "$(GPUCC_NAME) not found" >&2; exit 1; } + "$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_backend_cuda.cu -o backend_cuda_test$(EXE) ./backend_cuda_test$(EXE) -cuda-bench: backend_cuda.cu backend_cuda.h tests/bench_tensor_core.cu - @command -v "$(NVCC)" >/dev/null 2>&1 || { echo "nvcc not found: set CUDA_HOME or NVCC" >&2; exit 1; } - "$(NVCC)" $(NVCCFLAGS) backend_cuda.cu tests/bench_tensor_core.cu -o backend_cuda_bench$(EXE) +# convenience alias: kernel correctness on AMD (same test, hipcc toolchain) +hip-test: + $(MAKE) cuda-test HIP=1 + +# CI: compile the backend and its test binary WITHOUT executing them (for +# runners with the toolchain but no GPU). Pass CUDA_ARCH/HIP=1+HIP_ARCH. +gpu-compile: backend_cuda.o + "$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_backend_cuda.cu -o backend_cuda_test$(EXE) + +cuda-bench: backend_cuda.cu backend_cuda.h backend_gpu_compat.h tests/bench_tensor_core.cu + @command -v "$(GPUCC)" >/dev/null 2>&1 || { echo "$(GPUCC_NAME) not found" >&2; exit 1; } + "$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/bench_tensor_core.cu -o backend_cuda_bench$(EXE) ./backend_cuda_bench$(EXE) olmoe$(EXE): olmoe.c st.h json.h compat.h @@ -272,8 +336,9 @@ olmoe$(EXE): olmoe.c st.h json.h compat.h # Use a baseline that matches the compiler target. macOS already targets a # portable baseline when ARCH is empty; forcing the x86 value there breaks # Apple Silicon. Unknown targets use native rather than an invalid x86 flag. +# Intel Macs need -march for vector instructions ifneq (,$(DARWIN)) -PORTABLE_ARCH = +PORTABLE_ARCH = $(if $(X86_64),x86-64-v3,) else ifneq (,$(AARCH64)) PORTABLE_ARCH = armv8-a else ifneq (,$(PPC64)) @@ -285,7 +350,7 @@ PORTABLE_ARCH = native endif portable: - $(MAKE) glm$(EXE) ARCH=$(PORTABLE_ARCH) + $(MAKE) colibri$(EXE) ARCH=$(PORTABLE_ARCH) iobench$(EXE): iobench.c compat.h $(CC) $(CFLAGS) iobench.c -o iobench$(EXE) $(LDFLAGS) @@ -293,12 +358,17 @@ iobench$(EXE): iobench.c compat.h tests/test_json$(EXE): tests/test_json.c json.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) +tests/test_tok_o200k$(EXE): tests/test_tok_o200k.c tok.h tok_unicode.h tok_unicode_o200k.h json.h + $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) tests/test_st_pread$(EXE): tests/test_st_pread.c st.h json.h compat.h $(CC) $(CFLAGS) -DST_PREAD_CHUNK=7 $< -o $@ $(LDFLAGS) tests/test_st$(EXE): tests/test_st.c st.h json.h compat.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) +tests/test_st_mirror$(EXE): tests/test_st_mirror.c st.h json.h compat.h + $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) + tests/test_tier$(EXE): tests/test_tier.c tier.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) @@ -311,30 +381,41 @@ tests/test_schema_gbnf$(EXE): tests/test_schema_gbnf.c schema_gbnf.h grammar.h j tests/test_decode_batch$(EXE): tests/test_decode_batch.c decode_batch.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) -tests/test_idot$(EXE): tests/test_idot.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h +tests/test_idot$(EXE): tests/test_idot.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) -tests/test_i4_grouped$(EXE): tests/test_i4_grouped.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h +tests/test_i4_grouped$(EXE): tests/test_i4_grouped.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) -tests/test_stops$(EXE): tests/test_stops.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h +tests/test_stops$(EXE): tests/test_stops.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) -tests/test_topp$(EXE): tests/test_topp.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h +tests/test_topp$(EXE): tests/test_topp.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) # bench_topp is a microbenchmark (old qsort vs new heap partial-select, #335), NOT a test # gate -- intentionally absent from TEST_BINS. Build on demand: make tests/bench_topp -tests/bench_topp$(EXE): tests/bench_topp.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h +tests/bench_topp$(EXE): tests/bench_topp.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) -tests/test_sample_nan$(EXE): tests/test_sample_nan.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h +tests/test_sample_nan$(EXE): tests/test_sample_nan.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) -tests/test_kv_alloc$(EXE): tests/test_kv_alloc.c glm.c st.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h +tests/test_kv_alloc$(EXE): tests/test_kv_alloc.c colibri.c st.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) -tests/test_logit_nan$(EXE): tests/test_logit_nan.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h +# fmt=6 kernel oracle: needs the generated grid table, and its fixture comes from +# the reference codec (tools/make_e8_fixture.py) — regenerate if the layout moves. +tests/test_e8_kernel$(EXE): tests/test_e8_kernel.c quant.h + $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) + +tests/test_int3$(EXE): tests/test_int3.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h + $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) + +tests/test_int3_load$(EXE): tests/test_int3_load.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h + $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) + +tests/test_logit_nan$(EXE): tests/test_logit_nan.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) tests/test_i4_acc512$(EXE): tests/test_i4_acc512.c @@ -343,15 +424,28 @@ tests/test_i4_acc512$(EXE): tests/test_i4_acc512.c tests/test_compat_direct$(EXE): tests/test_compat_direct.c compat.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) -tests/test_dsa_select$(EXE): tests/test_dsa_select.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h +tests/test_dsa_select$(EXE): tests/test_dsa_select.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) # bench_dsa_select is a microbenchmark (old qsort vs new quickselect partial-select, #356), # NOT a test gate -- intentionally absent from TEST_BINS. Build on demand: make tests/bench_dsa_select -tests/bench_dsa_select$(EXE): tests/bench_dsa_select.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h +tests/bench_dsa_select$(EXE): tests/bench_dsa_select.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h + $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) + +# bench_idot: microbenchmark (single-acc vs independent-acc AVX-VNNI idot), NOT a test gate. +# Build on demand on an AVX-VNNI CPU: make tests/bench_idot ARCH=native +tests/bench_idot$(EXE): tests/bench_idot.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h + $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) + +# bench_mla_simd: microbenchmark (scalar vs AVX2/NEON MLA-absorb reductions, #442), +# NOT a test gate. Build on demand: make tests/bench_mla_simd +tests/bench_mla_simd$(EXE): tests/bench_mla_simd.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h + $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) + +tests/test_uring$(EXE): tests/test_uring.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) -tests/test_uring$(EXE): tests/test_uring.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h +tests/test_pipe_block$(EXE): tests/test_pipe_block.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) test-c: $(TEST_BINS) @@ -362,18 +456,41 @@ test-python: test: test-c test-python +# --- Efficiency / regression suite (issue: "test the program for inefficiencies") --- +# The tiny-model assertions live in test_inefficiency.py and run as part of +# test-python (they're discovered by the test_*.py glob). These targets are +# convenience entry points; the opt-in full-model report is NEVER in `make test`. +# +# make efficiency tiny-model asserted regression tests (CPU; part of test-python) +# make efficiency-cuda the CUDA-path tests (requires a CUDA build — see below) +# make efficiency-report opt-in full-model 🟢/🔴 diagnostic, never fails CI +# +# CUDA build (Windows): the CUDA tests need a host built with -DCOLI_CUDA plus +# the runtime DLL. Do this FIRST — note CUDA_DLL=1 on BOTH the host and the +# rule below, or `make glm.exe` will rebuild a CPU-only host and overwrite it: +# make clean && make glm.exe CUDA_DLL=1 && make cuda-dll +# The tests auto-skip with a clear message if the host is CPU-only. +efficiency: test-python + $(PYTHON) -m unittest tests.test_inefficiency -v + +efficiency-cuda: + $(PYTHON) -m unittest tests.test_inefficiency.TinyCudaEfficiencyTest -v + +efficiency-report: + $(PYTHON) tests/test_efficiency_report.py + # Local validation: one portable CPU build and dependency-free tests. check: $(MAKE) clean $(MAKE) portable $(MAKE) test -install: glm$(EXE) olmoe$(EXE) +install: colibri$(EXE) olmoe$(EXE) $(INSTALL) -d $(DESTDIR)$(BINDIR) $(INSTALL) -d $(DESTDIR)$(LIBEXECDIR) $(INSTALL) -d $(DESTDIR)$(LIBEXECDIR)/tools $(INSTALL) -m 755 coli $(DESTDIR)$(BINDIR)/coli - $(INSTALL) -m 755 glm$(EXE) $(DESTDIR)$(LIBEXECDIR)/glm$(EXE) + $(INSTALL) -m 755 colibri$(EXE) $(DESTDIR)$(LIBEXECDIR)/colibri$(EXE) $(INSTALL) -m 755 olmoe$(EXE) $(DESTDIR)$(LIBEXECDIR)/olmoe$(EXE) $(INSTALL) -m 644 resource_plan.py doctor.py openai_server.py $(DESTDIR)$(LIBEXECDIR)/ $(INSTALL) -m 644 tools/*.py $(DESTDIR)$(LIBEXECDIR)/tools/ @@ -387,4 +504,4 @@ clean: bench: iobench$(EXE) @if [ -n "$(ARGS)" ]; then ./iobench$(EXE) $(ARGS); else echo "built iobench$(EXE) — run: ./iobench$(EXE) "; fi -.PHONY: all glm cuda-test cuda-bench cuda-dll portable test-c test-python test check clean install uninstall bench +.PHONY: all colibri glm cuda-test hip-test gpu-compile cuda-bench cuda-dll portable test-c test-python test check clean install uninstall bench diff --git a/c/backend_cuda.cu b/c/backend_cuda.cu index 413e53b4..5c98b1fb 100644 --- a/c/backend_cuda.cu +++ b/c/backend_cuda.cu @@ -1,7 +1,6 @@ #include "backend_cuda.h" -#include -#include +#include "backend_gpu_compat.h" #include #include @@ -20,6 +19,9 @@ struct ColiCudaTensor { float *scales; size_t weight_bytes; int fmt, I, O, device; + int gs; /* quant group size; 0 = per-row scales (#334) */ + int ng; /* number of scale groups per row = ceil(I/gs) for fmt=4 */ + size_t scale_count; /* floats in `scales`: O per-row, O*ng grouped */ int tracked; RaggedKVEntry ragged[512]; int ragged_count; @@ -34,15 +36,18 @@ typedef struct { size_t qx_cap, qscale_cap; float *host_x,*host_y,*host_kv; size_t host_x_cap,host_y_cap,host_kv_cap; float *aq,*al,*ar,*ac; size_t aq_cap,al_cap,ar_cap,ac_cap; - float *pipe_buf[24]; size_t pipe_cap[24]; /* scratch persistenti del resident pipeline */ + float *pipe_buf[27]; size_t pipe_cap[27]; /* scratch persistenti del resident pipeline */ cudaStream_t stream; + cudaEvent_t ev_done; int ev_done_ok; /* resident-group issue completion (#431 PR-C0) */ void *group_desc; size_t group_desc_cap; size_t tensor_count, tensor_bytes; + int group_pending; size_t group_pending_bytes; /* async expert-group in flight (Inc.4) */ } DeviceContext; typedef struct { const void *g,*u,*d; const float *gs,*us,*ds; int gf,uf,df,rows,offset; + int ggs,ugs,dgs; /* per-tensor quant group size; 0 = per-row scales (#334 fmt=4) */ } GroupDesc; static DeviceContext g_ctx[COLI_CUDA_MAX_DEVICES]; @@ -54,6 +59,8 @@ static std::mutex g_group_stats_mu; static int cuda_ok(cudaError_t err, const char *what) { if (err == cudaSuccess) return 1; std::fprintf(stderr, "[CUDA] %s: %s\n", what, cudaGetErrorString(err)); + (void)cudaGetLastError(); /* consume the sticky error: a failed call must + not poison the next launch's error check */ return 0; } @@ -79,8 +86,9 @@ static int select_ctx(DeviceContext *ctx) { __host__ __device__ static size_t row_bytes(int fmt, int I) { if (fmt == 0) return (size_t)I * sizeof(float); if (fmt == 1) return (size_t)I; - if (fmt == 2) return (size_t)(I + 1) / 2; + if (fmt == 2 || fmt == 4) return (size_t)(I + 1) / 2; /* fmt=4: same packed int4 */ if (fmt == 3) return (size_t)(I + 3) / 4; + if (fmt == 4) return (size_t)(I + 1) / 2; /* grouped int4: nibbles like fmt 2 */ return 0; } @@ -89,7 +97,7 @@ __device__ static float weight_at(const void *weights, int fmt, size_t row, int if (fmt == 0) return reinterpret_cast(base)[i]; if (fmt == 1) return static_cast(reinterpret_cast(base)[i]); const uint8_t *q = base; - if (fmt == 2) { + if (fmt == 2 || fmt == 4) { /* fmt=4: same nibble layout */ uint8_t v = q[i >> 1]; int n=(i&1)?(v>>4):(v&15); return static_cast(n&8?n-16:n); } @@ -97,20 +105,45 @@ __device__ static float weight_at(const void *weights, int fmt, size_t row, int return static_cast(((v >> ((i & 3) * 2)) & 3) - 2); } +/* Scale for output `row`, input element `k`. fmt=4 (grouped int4) stores ng + * scales per row at scales[row*ng + k/gs]; every other quantized format has + * one scale per row at scales[row]. Mirrors quant_matmul's fmt==4 branch so the + * attention absorb kernels apply per-group scales instead of the per-row + * (fmt=2) semantic that crashed #298's g64 kv_b. */ +__device__ static float absorb_scale(const float *wscale, int fmt, int gs, int ng, int row, int k) { + if (!fmt) return 1.f; + if (fmt != 4) return wscale[row]; + int g = k / gs; if (g >= ng) g = ng - 1; /* tail of the last (partial) group */ + return wscale[(size_t)row * ng + g]; +} + __global__ static void offset_to_signed_s4(uint8_t *q,size_t n){ size_t i=(size_t)blockIdx.x*blockDim.x+threadIdx.x;if(i= ng) g = ng - 1; /* tail elements in the last (partial) group */ + sum += xs[i] * weight_at(weights, fmt, row, i) * scl[g]; + } + } else { + for (int i = threadIdx.x; i < I; i += blockDim.x) + sum += xs[i] * weight_at(weights, fmt, row, i); + } __shared__ float partial[256]; partial[threadIdx.x] = sum; @@ -120,7 +153,7 @@ __global__ static void quant_matmul(float *y, const float *x, const void *weight __syncthreads(); } if (!threadIdx.x) - y[(size_t)s * O + o] = partial[0] * (fmt ? scales[o] : 1.0f); + y[(size_t)s * O + o] = (fmt && fmt != 4) ? partial[0] * scales[o] : partial[0]; } __global__ static void silu_mul(float *gate, const float *up, size_t n) { @@ -296,13 +329,50 @@ __global__ static void grouped_down_w4(float *y,const float *x,const GroupDesc * if(!threadIdx.x)y[(size_t)(d.offset+s)*D+o]=p[0]*d.ds[o]; } +/* fmt=4 grouped-int4 variants (#334): identical structure to the w4 kernels, + * but the scale varies along the input dimension — sc[o*ng + i/gs], applied + * per element inside the accumulation (gs is even, so a packed byte never + * straddles a group). gs<=0 degrades to per-row (ng=1), so mixed fmt2/fmt4 + * groups run correctly through this one kernel family. */ +__global__ static void grouped_hidden_g4_dual(float *gate,float *up,const float *x, + const GroupDesc *desc,int I,int D){ + int o=blockIdx.x,s=blockIdx.y,c=blockIdx.z;GroupDesc d=desc[c];if(s>=d.rows)return; + const uint8_t *gr=(const uint8_t*)d.g+(size_t)o*((D+1)/2); + const uint8_t *ur=(const uint8_t*)d.u+(size_t)o*((D+1)/2); + int ggs=d.ggs>0?d.ggs:D, ugs=d.ugs>0?d.ugs:D; + const float *gsc=d.gs+(size_t)o*(size_t)((D+ggs-1)/ggs); + const float *usc=d.us+(size_t)o*(size_t)((D+ugs-1)/ugs); + const float *xs=x+(size_t)(d.offset+s)*D;float ga=0,ua=0; + for(int b=threadIdx.x;b<(D+1)/2;b+=blockDim.x){float g0,g1,u0,u1;unpack_s4(gr[b],&g0,&g1);unpack_s4(ur[b],&u0,&u1); + int i=b*2;float gv=gsc[i/ggs],uv=usc[i/ugs]; + ga+=xs[i]*g0*gv;ua+=xs[i]*u0*uv; + if(i+1>=1){if(threadIdx.x=d.rows)return; + const uint8_t *row=(const uint8_t*)d.d+(size_t)o*((I+1)/2); + int dgs=d.dgs>0?d.dgs:I; + const float *dsc=d.ds+(size_t)o*(size_t)((I+dgs-1)/dgs); + const float *xs=x+(size_t)(d.offset+s)*I;float sum=0; + for(int b=threadIdx.x;b<(I+1)/2;b+=blockDim.x){float a,z;unpack_s4(row[b],&a,&z); + int i=b*2;float sv=dsc[i/dgs]; + sum+=xs[i]*a*sv;if(i+1>=1){if(threadIdx.x=S||nt<1)return; extern __shared__ float sm[];float *qa=sm,*cl=qa+K,*scores=cl+K,*red=scores+T; const float *qs=q+((size_t)s*H+h)*(Q+R); for(int k=tid;kqx) cudaFree(ctx->qx); if (ctx->qscale) cudaFree(ctx->qscale); if(ctx->aq)cudaFree(ctx->aq);if(ctx->al)cudaFree(ctx->al);if(ctx->ar)cudaFree(ctx->ar);if(ctx->ac)cudaFree(ctx->ac); - for(int b=0;b<24;b++) if(ctx->pipe_buf[b]) cudaFree(ctx->pipe_buf[b]); + for(int b=0;b<27;b++) if(ctx->pipe_buf[b]) cudaFree(ctx->pipe_buf[b]); if (ctx->host_x) cudaFreeHost(ctx->host_x); if (ctx->host_y) cudaFreeHost(ctx->host_y); if (ctx->host_kv) cudaFreeHost(ctx->host_kv); @@ -503,40 +574,66 @@ extern "C" void coli_cuda_group_stats(uint64_t *calls, uint64_t *experts, uint64 if(d2h_ms) *d2h_ms=g_group_d2h_ms; } +/* group size for the NEXT upload on this thread (fmt=4): routed through a + * thread_local so the widely-wired upload signature (and the Windows DLL ABI) + * stays untouched. pin_load uploads in parallel, hence thread_local. */ +static thread_local int g_upload_gs = 0; +extern "C" int coli_cuda_tensor_upload_g(ColiCudaTensor **tensor, + const void *weights, const float *scales, + int fmt, int I, int O, int device, int gs); extern "C" int coli_cuda_tensor_upload(ColiCudaTensor **tensor, const void *weights, const float *scales, int fmt, int I, int O, int device) { - DeviceContext *ctx = find_ctx(device); - if (!tensor || !weights || I < 1 || O < 1 || !select_ctx(ctx)) return 0; - size_t rb = row_bytes(fmt, I); - if (!rb || (fmt && !scales)) return 0; + if (!tensor) return 0; if (*tensor) { + /* Cached device copy: usable even when the caller's host pointers are + * gone. CUDA_RELEASE_HOST slots null their host pointers after upload, + * and with the old order (!weights checked first) every later matmul + * on such a slot failed here — the GPU tier silently never computed + * for host-released slab experts. */ ColiCudaTensor *t = *tensor; - return t->fmt == fmt && t->I == I && t->O == O && t->device == device; + int want_gs = (fmt==4 && g_upload_gs>0) ? g_upload_gs : 0; + return t->fmt == fmt && t->I == I && t->O == O && t->device == device && t->gs == want_gs; } + DeviceContext *ctx = find_ctx(device); + if (!weights || I < 1 || O < 1 || !select_ctx(ctx)) return 0; + size_t rb = row_bytes(fmt, I); + if (!rb || (fmt && !scales)) return 0; ColiCudaTensor *t = static_cast(std::calloc(1, sizeof(*t))); if (!t) return 0; t->fmt = fmt; t->I = I; t->O = O; t->device = device; t->weight_bytes = rb * (size_t)O; + t->gs = (fmt==4 && g_upload_gs>0) ? g_upload_gs : 0; + t->ng = t->gs ? (I + t->gs - 1) / t->gs : 1; + t->scale_count = t->gs ? (size_t)O * (size_t)t->ng : (size_t)O; if (!cuda_ok(cudaMalloc(&t->weights, t->weight_bytes), "tensor allocation") || !cuda_ok(cudaMemcpy(t->weights, weights, t->weight_bytes, cudaMemcpyHostToDevice), "tensor upload")) { coli_cuda_tensor_free(t); return 0; } - if(fmt==2){offset_to_signed_s4<<<(unsigned)((t->weight_bytes+255)/256),256>>>((uint8_t*)t->weights,t->weight_bytes); + if(fmt==2||fmt==4){ /* same nibble layout: offset-binary -> signed in place */ + offset_to_signed_s4<<<(unsigned)((t->weight_bytes+255)/256),256>>>((uint8_t*)t->weights,t->weight_bytes); if(!cuda_ok(cudaGetLastError(),"int4 weight conversion")){coli_cuda_tensor_free(t);return 0;}} if (fmt) { - if (!cuda_ok(cudaMalloc(&t->scales, (size_t)O * sizeof(float)), "scale allocation") || - !cuda_ok(cudaMemcpy(t->scales, scales, (size_t)O * sizeof(float), cudaMemcpyHostToDevice), "scale upload")) { + if (!cuda_ok(cudaMalloc(&t->scales, t->scale_count * sizeof(float)), "scale allocation") || + !cuda_ok(cudaMemcpy(t->scales, scales, t->scale_count * sizeof(float), cudaMemcpyHostToDevice), "scale upload")) { coli_cuda_tensor_free(t); return 0; } } t->tracked = 1; ctx->tensor_count++; - ctx->tensor_bytes += t->weight_bytes + (fmt ? (size_t)O * sizeof(float) : 0); + ctx->tensor_bytes += t->weight_bytes + (fmt ? t->scale_count * sizeof(float) : 0); *tensor = t; return 1; } +extern "C" int coli_cuda_tensor_upload_g(ColiCudaTensor **tensor, + const void *weights, const float *scales, + int fmt, int I, int O, int device, int gs){ + g_upload_gs = gs>0 ? gs : 0; + int r = coli_cuda_tensor_upload(tensor, weights, scales, fmt, I, O, device); + g_upload_gs = 0; + return r; +} extern "C" int coli_cuda_tensor_update(ColiCudaTensor *tensor, const void *weights, @@ -546,20 +643,40 @@ extern "C" int coli_cuda_tensor_update(ColiCudaTensor *tensor, if (!select_ctx(ctx)) return 0; if (!cuda_ok(cudaMemcpy(tensor->weights,weights,tensor->weight_bytes, cudaMemcpyHostToDevice),"tensor refresh")) return 0; - if(tensor->fmt==2){ + if(tensor->fmt==2||tensor->fmt==4){ offset_to_signed_s4<<<(unsigned)((tensor->weight_bytes+255)/256),256>>>( (uint8_t*)tensor->weights,tensor->weight_bytes); if(!cuda_ok(cudaGetLastError(),"int4 weight refresh")) return 0; } + int ng = tensor->ng > 0 ? tensor->ng : 1; return !tensor->fmt || cuda_ok(cudaMemcpy(tensor->scales,scales, - (size_t)tensor->O*sizeof(float),cudaMemcpyHostToDevice),"scale refresh"); + (tensor->scale_count?tensor->scale_count:(size_t)tensor->O)*sizeof(float), + cudaMemcpyHostToDevice),"scale refresh"); +} + +/* Test hook: COLI_GPU_FAIL_AFTER=N makes every GPU COMPUTE entry point report + * failure after N successful calls (N=0: every call fails), exercising the + * engine's CPU fallbacks and host-rematerialization end-to-end without real + * hardware faults. Uploads/queries are not gated. Unset: no effect. */ +static long g_gpu_calls; +static int fault_injected(void) { + const char *fa = std::getenv("COLI_GPU_FAIL_AFTER"); + return fa && g_gpu_calls++ >= std::atol(fa); } extern "C" int coli_cuda_matmul(ColiCudaTensor **tensor, float *y, const float *x, const void *weights, const float *scales, - int fmt, int S, int I, int O, int device) { - if (S < 1 || !coli_cuda_tensor_upload(tensor, weights, scales, fmt, I, O, device)) return 0; + int fmt, int S, int I, int O, int device, int gs) { + if (fault_injected()) return 0; + /* fmt=4 carries [O, ceil(I/gs)] scales: without the group size the plain + * upload truncates the buffer to O floats and quant_matmul divides by + * gs==0. Callers must come through the gs>0 path (upload_g) or stay on + * the CPU (#298, #334). */ + if (fmt == 4 && gs <= 0) return 0; + if (S < 1) return 0; + if (gs > 0) { if (!coli_cuda_tensor_upload_g(tensor, weights, scales, fmt, I, O, device, gs)) return 0; } + else { if (!coli_cuda_tensor_upload(tensor, weights, scales, fmt, I, O, device)) return 0; } ColiCudaTensor *t = *tensor; DeviceContext *ctx = find_ctx(t->device); if (!select_ctx(ctx)) return 0; @@ -568,7 +685,7 @@ extern "C" int coli_cuda_matmul(ColiCudaTensor **tensor, if (!reserve(&ctx->x, &ctx->x_cap, xb) || !reserve(&ctx->y, &ctx->y_cap, yb)) return 0; if (!cuda_ok(cudaMemcpy(ctx->x, x, xb, cudaMemcpyHostToDevice), "input upload")) return 0; dim3 grid((unsigned)O, (unsigned)S); - quant_matmul<<>>(ctx->y, ctx->x, t->weights, t->scales, fmt, S, I, O, rb); + quant_matmul<<>>(ctx->y, ctx->x, t->weights, t->scales, fmt, S, I, O, rb, t->gs, t->ng); if (!cuda_ok(cudaGetLastError(), "matmul launch") || !cuda_ok(cudaMemcpy(y, ctx->y, yb, cudaMemcpyDeviceToHost), "output download")) return 0; return 1; @@ -577,6 +694,12 @@ extern "C" int coli_cuda_matmul(ColiCudaTensor **tensor, extern "C" int coli_cuda_expert_mlp(ColiCudaTensor *gate, ColiCudaTensor *up, ColiCudaTensor *down, float *y, const float *x, int S) { + if (fault_injected()) return 0; + /* same reason as coli_cuda_matmul: fmt=4 without recorded group info would + * misread the scales (and divide by gs==0 in the kernel). */ + if (gate && ((gate->fmt == 4 && gate->gs <= 0) || + (up && up->fmt == 4 && up->gs <= 0) || + (down && down->fmt == 4 && down->gs <= 0))) return 0; if (!gate || !up || !down || !x || !y || S < 1 || gate->device != up->device || gate->device != down->device || gate->I != up->I || gate->O != up->O || @@ -591,13 +714,13 @@ extern "C" int coli_cuda_expert_mlp(ColiCudaTensor *gate, ColiCudaTensor *up, if (!cuda_ok(cudaMemcpy(ctx->x,x,xb,cudaMemcpyHostToDevice),"expert input upload")) return 0; dim3 hidden_grid((unsigned)I,(unsigned)S), output_grid((unsigned)D,(unsigned)S); quant_matmul<<>>(ctx->gate,ctx->x,gate->weights,gate->scales, - gate->fmt,S,D,I,row_bytes(gate->fmt,D)); + gate->fmt,S,D,I,row_bytes(gate->fmt,D),gate->gs,gate->ng); quant_matmul<<>>(ctx->up,ctx->x,up->weights,up->scales, - up->fmt,S,D,I,row_bytes(up->fmt,D)); + up->fmt,S,D,I,row_bytes(up->fmt,D),up->gs,up->ng); size_t n=(size_t)S*I; silu_mul<<<(unsigned)((n+255)/256),256>>>(ctx->gate,ctx->up,n); quant_matmul<<>>(ctx->y,ctx->gate,down->weights,down->scales, - down->fmt,S,I,D,row_bytes(down->fmt,I)); + down->fmt,S,I,D,row_bytes(down->fmt,I),down->gs,down->ng); if (!cuda_ok(cudaGetLastError(),"expert MLP launch") || !cuda_ok(cudaMemcpy(y,ctx->y,yb,cudaMemcpyDeviceToHost),"expert output download")) return 0; return 1; @@ -605,10 +728,11 @@ extern "C" int coli_cuda_expert_mlp(ColiCudaTensor *gate, ColiCudaTensor *up, extern "C" int coli_cuda_shared_mlp_w4a16(ColiCudaTensor *gate,ColiCudaTensor *up, ColiCudaTensor *down,float *y,const float *x,int S){ + if (fault_injected()) return 0; if(!gate||!up||!down||!x||!y||S<1||gate->fmt!=2||up->fmt!=2||down->fmt!=2|| gate->device!=up->device||gate->device!=down->device||gate->I!=up->I|| gate->O!=up->O||down->I!=gate->O||down->O!=gate->I)return 0; - DeviceContext *ctx=find_ctx(gate->device);if(!select_ctx(ctx)||ctx->compute_major<7)return 0; + DeviceContext *ctx=find_ctx(gate->device);if(!select_ctx(ctx)||!COLI_GPU_HAS_WMMA||ctx->compute_major<7)return 0; int D=gate->I,I=gate->O;size_t xb=(size_t)S*D*sizeof(float),ib=(size_t)S*I*sizeof(float); if(!reserve(&ctx->x,&ctx->x_cap,xb)||!reserve(&ctx->gate,&ctx->gate_cap,ib)|| !reserve(&ctx->up,&ctx->up_cap,ib)||!reserve(&ctx->y,&ctx->y_cap,xb)|| @@ -636,19 +760,24 @@ extern "C" int coli_cuda_expert_group(ColiCudaTensor *const *gates, ColiCudaTensor *const *downs, const int *rows, int count, float *y, const float *x) { + if (fault_injected()) return 0; if (!gates || !ups || !downs || !rows || !x || !y || count < 1) return 0; ColiCudaTensor *first=gates[0]; if (!first) return 0; int device=first->device,D=first->I,I=first->O,total=0,max_rows=0; GroupDesc host[64]; if(count>64) return 0; - int all_s4=1; + int all_s4=1,all_q4=1,any_g4=0; for(int c=0;cdevice!=device||u->device!=device||d->device!=device|| g->I!=D||u->I!=D||g->O!=I||u->O!=I||d->I!=I||d->O!=D) return 0; host[c]={g->weights,u->weights,d->weights,g->scales,u->scales,d->scales, - g->fmt,u->fmt,d->fmt,rows[c],total}; + g->fmt,u->fmt,d->fmt,rows[c],total, + g->gs,u->gs,d->gs}; all_s4&=g->fmt==2&&u->fmt==2&&d->fmt==2; + all_q4&=(g->fmt==2||g->fmt==4)&&(u->fmt==2||u->fmt==4)&&(d->fmt==2||d->fmt==4)&& + !(g->gs&1)&&!(u->gs&1)&&!(d->gs&1); /* even gs: a packed byte never straddles groups */ + any_g4|=g->fmt==4||u->fmt==4||d->fmt==4; total+=rows[c]; if(rows[c]>max_rows) max_rows=rows[c]; } DeviceContext *ctx=find_ctx(device); if(!select_ctx(ctx)) return 0; @@ -665,7 +794,8 @@ extern "C" int coli_cuda_expert_group(ColiCudaTensor *const *gates, if(!cuda_ok(copy_desc,"expert group descriptors"))return 0; int profile=getenv("COLI_CUDA_PROFILE")&&atoi(getenv("COLI_CUDA_PROFILE")); cudaEvent_t ev[4]={}; - if(profile) for(int i=0;i<4;i++) if(!cuda_ok(cudaEventCreate(&ev[i]),"profile event")) profile=0; + if(profile) for(int i=0;i<4;i++) if(!cuda_ok(cudaEventCreate(&ev[i]),"profile event")){ + for(int j=0;jstream); if(async)std::memcpy(ctx->host_x,x,xb); cudaError_t copy_x=async?cudaMemcpyAsync(ctx->x,ctx->host_x,xb,cudaMemcpyHostToDevice,ctx->stream) @@ -688,8 +818,16 @@ extern "C" int coli_cuda_expert_group(ColiCudaTensor *const *gates, silu_mul<<<(unsigned)(((size_t)total*I+255)/256),256,0,ctx->stream>>>(ctx->gate,ctx->up,(size_t)total*I); quantize_s4_rows<<stream>>>(ctx->qx,ctx->qscale,ctx->gate,total,I); grouped_s4_wmma<<stream>>>(ctx->y,ctx->qx,ctx->qscale,dev,I,D,2); - }else if(all_s4&&ctx->compute_major>=7&&getenv("COLI_CUDA_TC_W4A16")&& - atoi(getenv("COLI_CUDA_TC_W4A16"))){ + }else if(all_s4&&COLI_GPU_HAS_WMMA&&ctx->compute_major>=7&&getenv("COLI_CUDA_TC_W4A16")&& + atoi(getenv("COLI_CUDA_TC_W4A16"))&& + [&]{ int tc16_min=getenv("COLI_CUDA_TC_W4A16_MIN")?atoi(getenv("COLI_CUDA_TC_W4A16_MIN")):16; + for(int c=0;c=tc16_min) return 1; + return 0; }()){ + /* At least one expert has enough rows for a Tensor Core tile. Groups + * where EVERY expert is below the threshold (decode: r=1) fall through + * to the grouped-W4 path below — 3 launches for the whole group instead + * of 4 per expert (#431: the launch flood measured at ~981 micro-kernels + * per token came from decode riding this branch's per-expert fallback). */ /* W4A16 Tensor Core per gruppo: attivazioni fp16 per tile (lossless al * contrario del path W4A4), un lancio per expert dentro lo stream — * l'overhead di lancio e' trascurabile rispetto ai GEMM. */ @@ -711,12 +849,12 @@ extern "C" int coli_cuda_expert_group(ColiCudaTensor *const *gates, /* piccoli batch: tile TC quasi vuoti + overhead di lancio — il * kernel naive per-elemento resta piu' veloce (misurato in decode) */ quant_matmul<<stream>>>(g16,x16, - host[c].g,host[c].gs,host[c].gf,r,D,I,row_bytes(host[c].gf,D)); + host[c].g,host[c].gs,host[c].gf,r,D,I,row_bytes(host[c].gf,D),0,1); quant_matmul<<stream>>>(u16,x16, - host[c].u,host[c].us,host[c].uf,r,D,I,row_bytes(host[c].uf,D)); + host[c].u,host[c].us,host[c].uf,r,D,I,row_bytes(host[c].uf,D),0,1); silu_mul<<<(unsigned)(((size_t)r*I+255)/256),256,0,ctx->stream>>>(g16,u16,(size_t)r*I); quant_matmul<<stream>>>(y16,g16, - host[c].d,host[c].ds,host[c].df,r,I,D,row_bytes(host[c].df,I)); + host[c].d,host[c].ds,host[c].df,r,I,D,row_bytes(host[c].df,I),0,1); } off16+=r; } @@ -730,7 +868,18 @@ extern "C" int coli_cuda_expert_group(ColiCudaTensor *const *gates, } silu_mul<<<(unsigned)(((size_t)total*I+255)/256),256,0,ctx->stream>>>(ctx->gate,ctx->up,(size_t)total*I); grouped_down_w4<<stream>>>(ctx->y,ctx->gate,dev,D,I); + }else if(all_q4&&any_g4){ + /* grouped-int4 (fmt=4) present: per-group scales (#334). fmt=2 members + * ride along as the ng=1 special case. */ + dim3 hg((unsigned)I,(unsigned)max_rows,(unsigned)count),og((unsigned)D,(unsigned)max_rows,(unsigned)count); + grouped_hidden_g4_dual<<stream>>>(ctx->gate,ctx->up,ctx->x,dev,I,D); + silu_mul<<<(unsigned)(((size_t)total*I+255)/256),256,0,ctx->stream>>>(ctx->gate,ctx->up,(size_t)total*I); + grouped_down_g4<<stream>>>(ctx->y,ctx->gate,dev,D,I); }else{ + /* generic path decodes fmt 0/1/2/3 only — a fmt=4 group that slipped the + * gates above (odd gs) must NOT be silently decoded as int2 (#334). */ + for(int c=0;cstream>>>(ctx->gate,ctx->x,dev,I,D,0); grouped_hidden<<stream>>>(ctx->up,ctx->x,dev,I,D,1); @@ -757,10 +906,80 @@ extern "C" int coli_cuda_expert_group(ColiCudaTensor *const *gates, return 1; } +/* ---- Async expert group (Inc.4): issue/take split of coli_cuda_expert_group ---- + * The measured cost of the sync call at decode is ~0.45 ms/call of HOST-side wait + * (stream sync + staging), vs ~0.18 ms of actual GPU work — 70% tax, paid ~5x per + * layer because a token's 8 experts scatter across devices. issue() stages and + * launches on the device stream and returns immediately; take() syncs and hands + * back the pinned result rows. One issue may be outstanding per device; moe() + * takes at each layer end, which also orders the next layer's reuse of the ctx + * scratch buffers. Small batches only (decode/spec): bigger totals keep the sync + * path with its TC variants. Numerics are the sync path's small-batch kernels, + * so greedy output is byte-identical by construction. */ +extern "C" int coli_cuda_expert_group_issue(ColiCudaTensor *const *gates, + ColiCudaTensor *const *ups, + ColiCudaTensor *const *downs, + const int *rows, int count, + const float *x) { + if (!gates || !ups || !downs || !rows || !x || count < 1 || count > 64) return 0; + ColiCudaTensor *first=gates[0]; + if (!first) return 0; + int device=first->device,D=first->I,I=first->O,total=0; + GroupDesc host[64]; + for(int c=0;cdevice!=device||u->device!=device||d->device!=device|| + g->I!=D||u->I!=D||g->O!=I||u->O!=I||d->I!=I||d->O!=D) return 0; + host[c]={g->weights,u->weights,d->weights,g->scales,u->scales,d->scales, + g->fmt,u->fmt,d->fmt,rows[c],total, + g->gs,u->gs,d->gs}; + total+=rows[c]; + } + if(total>8) return 0; /* decode-scale only */ + DeviceContext *ctx=find_ctx(device); if(!ctx||ctx->group_pending||!select_ctx(ctx)) return 0; + size_t xb=(size_t)total*D*sizeof(float), ib=(size_t)total*I*sizeof(float); + if(!reserve(&ctx->x,&ctx->x_cap,xb)||!reserve(&ctx->y,&ctx->y_cap,xb)|| + !reserve(&ctx->gate,&ctx->gate_cap,ib)||!reserve(&ctx->up,&ctx->up_cap,ib)|| + !reserve_pinned(&ctx->host_x,&ctx->host_x_cap,xb)|| + !reserve_pinned(&ctx->host_y,&ctx->host_y_cap,xb)) return 0; + std::memcpy(ctx->host_x,x,xb); + if(!cuda_ok(cudaMemcpyAsync(ctx->x,ctx->host_x,xb,cudaMemcpyHostToDevice,ctx->stream), + "expert group issue upload")) return 0; + for(int c=0;cgate+(size_t)host[c].offset*I,*u16=ctx->up+(size_t)host[c].offset*I; + float *x16=ctx->x+(size_t)host[c].offset*D,*y16=ctx->y+(size_t)host[c].offset*D; + quant_matmul<<stream>>>(g16,x16, + host[c].g,host[c].gs,host[c].gf,r,D,I,row_bytes(host[c].gf,D),0,1); + quant_matmul<<stream>>>(u16,x16, + host[c].u,host[c].us,host[c].uf,r,D,I,row_bytes(host[c].uf,D),0,1); + silu_mul<<<(unsigned)(((size_t)r*I+255)/256),256,0,ctx->stream>>>(g16,u16,(size_t)r*I); + quant_matmul<<stream>>>(y16,g16, + host[c].d,host[c].ds,host[c].df,r,I,D,row_bytes(host[c].df,I),0,1); + } + if(!cuda_ok(cudaGetLastError(),"expert group issue launch")|| + !cuda_ok(cudaMemcpyAsync(ctx->host_y,ctx->y,xb,cudaMemcpyDeviceToHost,ctx->stream), + "expert group issue download")) return 0; + ctx->group_pending=1; ctx->group_pending_bytes=xb; + { std::lock_guard lock(g_group_stats_mu); + g_group_calls++; g_group_experts+=(uint64_t)count; g_group_rows+=(uint64_t)total; } + return 1; +} + +extern "C" const float *coli_cuda_expert_group_take(int device) { + DeviceContext *ctx=find_ctx(device); + if(!ctx||!ctx->group_pending) return nullptr; + ctx->group_pending=0; + if(!select_ctx(ctx)) return nullptr; + if(!cuda_ok(cudaStreamSynchronize(ctx->stream),"expert group take")) return nullptr; + return ctx->host_y; +} + extern "C" int coli_cuda_attention_absorb(ColiCudaTensor *w,float *ctx,const float *q, const float *latent,const float *rope,int H,int Q, int R,int V,int K,int T,float scale){ + if (fault_injected()) return 0; if(!w||!ctx||!q||!latent||!rope||H<1||Q<1||R<1||V<1||K<1||K>512||T<1||T>4096|| w->I!=K||w->O!=H*(Q+V))return 0; DeviceContext *dc=find_ctx(w->device);if(!select_ctx(dc))return 0; @@ -773,7 +992,7 @@ extern "C" int coli_cuda_attention_absorb(ColiCudaTensor *w,float *ctx,const flo !cuda_ok(cudaMemcpyAsync(dc->ar,rope,rb,cudaMemcpyHostToDevice,dc->stream),"attention rope upload"))return 0; size_t shared=(size_t)(2*K+T)*sizeof(float); attention_absorb_kernel<<stream>>>(dc->ac,dc->aq,dc->al,dc->ar,w->weights,w->scales, - w->fmt,H,Q,R,V,K,T,scale); + w->fmt,H,Q,R,V,K,T,scale,w->gs,w->ng); if(!cuda_ok(cudaGetLastError(),"attention absorb launch")|| !cuda_ok(cudaMemcpyAsync(ctx,dc->ac,cb,cudaMemcpyDeviceToHost,dc->stream),"attention context download")|| !cuda_ok(cudaStreamSynchronize(dc->stream),"attention synchronize"))return 0; @@ -796,13 +1015,13 @@ static int attention_absorb_batch_run(ColiCudaTensor *w,ColiCudaTensor *proj,flo !cuda_ok(cudaMemcpyAsync(dc->ar,rope,rb,cudaMemcpyHostToDevice,dc->stream),"attention batch rope upload"))return 0; size_t shared=(size_t)(2*K+T+256)*sizeof(float); attention_absorb_batch_kernel<<stream>>>(dc->ac,dc->aq,dc->al, - dc->ar,w->weights,w->scales,w->fmt,S,H,Q,R,V,K,T,scale); + dc->ar,w->weights,w->scales,w->fmt,S,H,Q,R,V,K,T,scale,w->gs,w->ng); if(!cuda_ok(cudaGetLastError(),"attention batch launch"))return 0; const float *src=dc->ac;size_t ob=cb; if(proj){ ob=(size_t)S*proj->O*sizeof(float);if(!reserve(&dc->y,&dc->y_cap,ob))return 0; quant_matmul<<O,S),256,0,dc->stream>>>(dc->y,dc->ac,proj->weights, - proj->scales,proj->fmt,S,proj->I,proj->O,row_bytes(proj->fmt,proj->I)); + proj->scales,proj->fmt,S,proj->I,proj->O,row_bytes(proj->fmt,proj->I),proj->gs,proj->ng); if(!cuda_ok(cudaGetLastError(),"attention o_proj launch"))return 0;src=dc->y; } if(!cuda_ok(cudaMemcpyAsync(out,src,ob,cudaMemcpyDeviceToHost,dc->stream), @@ -814,12 +1033,14 @@ static int attention_absorb_batch_run(ColiCudaTensor *w,ColiCudaTensor *proj,flo extern "C" int coli_cuda_attention_absorb_batch(ColiCudaTensor *w,float *ctx,const float *q, const float *latent,const float *rope,int S,int H,int Q,int R,int V,int K,int T, float scale){ + if (fault_injected()) return 0; return attention_absorb_batch_run(w,nullptr,ctx,q,latent,rope,S,H,Q,R,V,K,T,scale); } extern "C" int coli_cuda_attention_project_batch(ColiCudaTensor *w,ColiCudaTensor *proj, float *out,const float *q,const float *latent,const float *rope,int S,int H,int Q, int R,int V,int K,int T,float scale){ + if (fault_injected()) return 0; return attention_absorb_batch_run(w,proj,out,q,latent,rope,S,H,Q,R,V,K,T,scale); } @@ -900,7 +1121,7 @@ extern "C" int coli_cuda_attention_project_ragged(ColiCudaTensor *w,ColiCudaTens attention_absorb_ragged_kernel<<stream>>>(dc->ac,dc->aq,ddl,ddr, dn,w->weights,w->scales,w->fmt,S,H,Q,R,V,K,T,scale); quant_matmul<<O,S),256,0,dc->stream>>>(dc->y,dc->ac,proj->weights, - proj->scales,proj->fmt,S,proj->I,proj->O,row_bytes(proj->fmt,proj->I)); + proj->scales,proj->fmt,S,proj->I,proj->O,row_bytes(proj->fmt,proj->I),proj->gs,proj->ng); return cuda_ok(cudaGetLastError(),"ragged attention launch")&& cuda_ok(cudaMemcpyAsync(out,dc->y,ob,cudaMemcpyDeviceToHost,dc->stream),"ragged output download")&& cuda_ok(cudaStreamSynchronize(dc->stream),"ragged attention synchronize"); @@ -911,7 +1132,8 @@ extern "C" void coli_cuda_tensor_free(ColiCudaTensor *tensor) { DeviceContext *ctx = find_ctx(tensor->device); if (ctx) select_ctx(ctx); if (tensor->tracked && ctx) { - size_t bytes = tensor->weight_bytes + (tensor->fmt ? (size_t)tensor->O * sizeof(float) : 0); + int ng = tensor->ng > 0 ? tensor->ng : 1; + size_t bytes = tensor->weight_bytes + (tensor->fmt ? (size_t)tensor->O * ng * sizeof(float) : 0); if (ctx->tensor_count) ctx->tensor_count--; if (ctx->tensor_bytes >= bytes) ctx->tensor_bytes -= bytes; } @@ -925,7 +1147,9 @@ extern "C" void coli_cuda_tensor_free(ColiCudaTensor *tensor) { } extern "C" size_t coli_cuda_tensor_bytes(const ColiCudaTensor *tensor) { - return tensor ? tensor->weight_bytes + (tensor->fmt ? (size_t)tensor->O * sizeof(float) : 0) : 0; + if (!tensor) return 0; + int ng = tensor->ng > 0 ? tensor->ng : 1; + return tensor->weight_bytes + (tensor->fmt ? (size_t)tensor->O * ng * sizeof(float) : 0); } extern "C" int coli_cuda_tensor_device(const ColiCudaTensor *tensor) { @@ -986,7 +1210,7 @@ __global__ static void pipe_rows_add(float *x,const float *partial,const int *ro * per layer (78 x ~10 alloc/richiesta erano puro churn). */ extern "C" float *coli_cuda_pipe_scratch(int device,int slot,size_t bytes){ DeviceContext *ctx=find_ctx(device); - if(slot<0||slot>=24||!select_ctx(ctx)) return NULL; + if(slot<0||slot>=27||!select_ctx(ctx)) return NULL; if(!reserve(&ctx->pipe_buf[slot],&ctx->pipe_cap[slot],bytes)) return NULL; return ctx->pipe_buf[slot]; } @@ -1010,6 +1234,7 @@ extern "C" int coli_cuda_pipe_download(int device,const void *src,void *dst,size } extern "C" int coli_cuda_pipe_rmsnorm(int device,float *y_dev,const float *x_dev, const float *w_dev,int S,int D,float eps){ + if (fault_injected()) return 0; DeviceContext *ctx=find_ctx(device); if(S<1||D<1||!select_ctx(ctx)) return 0; pipe_rmsnorm_rows<<>>(y_dev,x_dev,w_dev,D,eps,D,D); @@ -1018,6 +1243,7 @@ extern "C" int coli_cuda_pipe_rmsnorm(int device,float *y_dev,const float *x_dev extern "C" int coli_cuda_pipe_rmsnorm_s(int device,float *y_dev,const float *x_dev, const float *w_dev,int S,int D,float eps, int xstride,int ystride){ + if (fault_injected()) return 0; DeviceContext *ctx=find_ctx(device); if(S<1||D<1||xstride>>(y_dev,x_dev,w_dev,D,eps,xstride,ystride); @@ -1026,6 +1252,7 @@ extern "C" int coli_cuda_pipe_rmsnorm_s(int device,float *y_dev,const float *x_d extern "C" int coli_cuda_pipe_rope(int device,float *v_dev,const int *pos_dev, int rows,int stride,int offset,int R,int heads, float theta){ + if (fault_injected()) return 0; DeviceContext *ctx=find_ctx(device); if(rows<1||R<2||R>256||heads<1||!select_ctx(ctx)) return 0; pipe_rope_rows<<>>(v_dev,pos_dev,0,stride,offset,R,heads,theta); @@ -1033,11 +1260,189 @@ extern "C" int coli_cuda_pipe_rope(int device,float *v_dev,const int *pos_dev, } extern "C" int coli_cuda_pipe_rope_base(int device,float *v_dev,int pos_base,int rows, int stride,int offset,int R,int heads,float theta){ + if (fault_injected()) return 0; DeviceContext *ctx=find_ctx(device); if(rows<1||R<2||R>256||heads<1||!select_ctx(ctx)) return 0; pipe_rope_rows<<>>(v_dev,NULL,pos_base,stride,offset,R,heads,theta); return cuda_ok(cudaGetLastError(),"pipe rope base"); } +/* ---- device router (#431 PR-A) ------------------------------------------- + * Router for one decode row, entirely on the layer's home device: logits GEMV + * (E x D, tiny) + sigmoid, bias-augmented top-K selection, route-level TOPP + * truncation, norm_topk and routed_scale — a float-faithful clone of moe()'s + * plain routing path (colibri.c FASE A). Selection runs single-thread so the + * argmax order, tie-breaking (strict >, lowest index wins) and weight math + * match the CPU reference exactly; only the dot/expf rounding can differ, + * which is the documented kernel-family divergence class (#100/#163). + * Results are packed [idx[K] | w[K] | keff] in one scratch buffer and read + * back with a single tiny D2H. */ +__global__ void pipe_router_logits(const float *__restrict__ x, + const float *__restrict__ W, + const float *__restrict__ bias, + int D, float *logit, float *choice){ + int e = blockIdx.x; + const float *w = W + (size_t)e*D; + float acc = 0.f; + for(int i=threadIdx.x; i>1; s>0; s>>=1){ + if(threadIdx.xbv){bv=choice[e];best=e;} } + idx[kk]=best; w[kk]=logit[best]; + } + int Ke=Ksel; + if(topp>0.f && topp<1.f){ + for(int a=1;a=0 && w[b]=topp*tot){ Ke=kk+1; break; } } + } + if(norm_topk){ float sm=0.f; for(int kk=0;kk4096||Ksel<1||Ksel>64||!select_ctx(ctx)) return 0; + size_t pack=(size_t)Ksel*(sizeof(int)+sizeof(float))+sizeof(int); + float *logit=coli_cuda_pipe_scratch(device,22,(size_t)E*sizeof(float)); + float *chc =coli_cuda_pipe_scratch(device,23,(size_t)E*sizeof(float)); + char *out =(char*)coli_cuda_pipe_scratch(device,24,pack); + if(!logit||!chc||!out) return 0; + pipe_router_logits<<>>(x_dev,(const float*)rw_dev,(const float*)rb_dev,D,logit,chc); + pipe_router_select<<<1,1>>>(logit,chc,E,Ksel,topp,norm_topk,routed_scale,out); + if(!cuda_ok(cudaGetLastError(),"pipe router launch")) return 0; + char buf[64*(sizeof(int)+sizeof(float))+sizeof(int)]; + if(!cuda_ok(cudaMemcpy(buf,out,pack,cudaMemcpyDeviceToHost),"pipe router readback")) return 0; + memcpy(idx_host,buf,(size_t)Ksel*sizeof(int)); + memcpy(w_host,buf+Ksel*sizeof(int),(size_t)Ksel*sizeof(float)); + memcpy(keff_host,buf+Ksel*(sizeof(int)+sizeof(float)),sizeof(int)); + return 1; +} +/* ---- resident expert-group accumulation (#431 PR-C0) ---------------------- + * Decode-time (S=1) expert groups without the host round-trip: the input row + * is P2P'd from the layer's home device, the group runs through the grouped-W4 + * kernels on its own stream, the down-projection outputs are weighted and + * reduced ON DEVICE (fixed expert order), and the device's partial sum is + * peer-pushed into a per-issue slot on the home device. take() makes the home + * legacy stream wait on every issue event and reduces the slots in issue order + * — deterministic, no atomics, no host bytes. The CPU tier overlaps with all + * of it exactly as before. */ +__global__ static void bcast_row(float *dst,const float *src,int count,int D){ + for(int i=blockIdx.x*blockDim.x+threadIdx.x;i64||!x_src_dev||!partial_slot_dev) return 0; + ColiCudaTensor *first=gates[0]; if(!first) return 0; + int device=first->device,D=first->I,I=first->O; + GroupDesc host[64]; + int total=0,all_s4=1; + for(int c=0;cdevice!=device||u->device!=device||d->device!=device|| + g->I!=D||u->I!=D||g->O!=I||u->O!=I||d->I!=I||d->O!=D) return 0; + host[c]={g->weights,u->weights,d->weights,g->scales,u->scales,d->scales, + g->fmt,u->fmt,d->fmt,1,total, + g->gs,u->gs,d->gs}; + all_s4&=g->fmt==2&&u->fmt==2&&d->fmt==2; + total++; + } + if(!all_s4) return 0; /* resident path: per-row int4 only */ + DeviceContext *ctx=find_ctx(device); if(!select_ctx(ctx)) return 0; + if(!ctx->ev_done_ok){ + if(!cuda_ok(cudaEventCreateWithFlags(&ctx->ev_done,cudaEventDisableTiming), + "resident group event")) return 0; + ctx->ev_done_ok=1; + } + /* size for the 64-expert cap, not for `count`: reserve() reallocs on growth, + * and a realloc here could free a buffer the PREVIOUS layer's still-queued + * async work on this stream reads. Fixed caps make re-issue realloc-free. */ + size_t xb=(size_t)64*D*sizeof(float), ib=(size_t)64*I*sizeof(float); + if(!reserve(&ctx->x,&ctx->x_cap,xb)||!reserve(&ctx->y,&ctx->y_cap,xb)|| + !reserve(&ctx->gate,&ctx->gate_cap,ib)||!reserve(&ctx->up,&ctx->up_cap,ib)|| + !reserve(&ctx->ac,&ctx->ac_cap,(size_t)(D+64)*sizeof(float))|| + !reserve_bytes(&ctx->group_desc,&ctx->group_desc_cap,(size_t)64*sizeof(GroupDesc))) + return 0; + float *w_dev=ctx->ac+D, *partial_local=ctx->ac; + if(!cuda_ok(cudaMemcpyAsync(ctx->group_desc,host,(size_t)count*sizeof(GroupDesc), + cudaMemcpyHostToDevice,ctx->stream),"resident group desc")|| + !cuda_ok(cudaMemcpyAsync(w_dev,weights,(size_t)count*sizeof(float), + cudaMemcpyHostToDevice,ctx->stream),"resident group weights")) + return 0; + /* input row: P2P from the home device. The caller guarantees x_src_dev is + * materialized (the pre-moe nrm download already synced the home stream). */ + if(!cuda_ok(cudaMemcpyPeerAsync(ctx->x,device,x_src_dev,home_device, + (size_t)D*sizeof(float),ctx->stream),"resident group x p2p")) + return 0; + bcast_row<<<64,256,0,ctx->stream>>>(ctx->x,ctx->x,count,D); /* row 0 -> rows 1..count-1 (in-place safe: row 0 rewritten with itself) */ + GroupDesc *dev=(GroupDesc*)ctx->group_desc; + dim3 hg((unsigned)I,1,(unsigned)count),og((unsigned)D,1,(unsigned)count); + grouped_hidden_w4_dual<<stream>>>(ctx->gate,ctx->up,ctx->x,dev,I,D); + silu_mul<<<(unsigned)(((size_t)count*I+255)/256),256,0,ctx->stream>>>(ctx->gate,ctx->up,(size_t)count*I); + grouped_down_w4<<stream>>>(ctx->y,ctx->gate,dev,D,I); + weighted_sum_rows<<<48,256,0,ctx->stream>>>(partial_local,ctx->y,w_dev,count,D); + if(!cuda_ok(cudaMemcpyPeerAsync(partial_slot_dev,home_device,partial_local,device, + (size_t)D*sizeof(float),ctx->stream),"resident partial p2p")) + return 0; + if(!cuda_ok(cudaEventRecord(ctx->ev_done,ctx->stream),"resident event record")) return 0; + return cuda_ok(cudaGetLastError(),"resident group launch"); +} +extern "C" int coli_cuda_expert_group_resident_take(int home_device,const int *devices,int n_issued, + float *slots_dev,float *acc_dev,int D){ + if(n_issued<1||!slots_dev||!acc_dev||D<1) return 0; + DeviceContext *home=find_ctx(home_device); if(!select_ctx(home)) return 0; + for(int i=0;iev_done_ok) return 0; + if(!cuda_ok(cudaStreamWaitEvent(0,src->ev_done,0),"resident take wait")) return 0; + } + sum_slots<<<48,256>>>(acc_dev,slots_dev,n_issued,D); /* legacy stream: ordered with pipe_* */ + return cuda_ok(cudaGetLastError(),"resident take reduce"); +} extern "C" int coli_cuda_pipe_copy2d(int device,float *dst,int dpitch,const float *src, int spitch,int width,int height){ DeviceContext *ctx=find_ctx(device); if(!select_ctx(ctx)) return 0; @@ -1050,6 +1455,7 @@ extern "C" int coli_cuda_pipe_copy2d(int device,float *dst,int dpitch,const floa extern "C" int coli_cuda_attention_project_batch_dev(ColiCudaTensor *w,ColiCudaTensor *proj, float *out,const float *q_dev,const float *latent_dev,const float *rope_dev, int S,int H,int Q,int R,int V,int K,int T,float scale){ + if (fault_injected()) return 0; if(!w||!proj||!out||!q_dev||!latent_dev||!rope_dev||S<1||H<1||Q<1||R<1||V<1|| K<1||K>512||T8192||w->I!=K||w->O!=H*(Q+V)|| proj->device!=w->device||proj->I!=H*V)return 0; @@ -1058,12 +1464,12 @@ extern "C" int coli_cuda_attention_project_batch_dev(ColiCudaTensor *w,ColiCudaT if(!reserve(&dc->ac,&dc->ac_cap,cb))return 0; size_t shared=(size_t)(2*K+T+256)*sizeof(float); attention_absorb_batch_kernel<<stream>>>(dc->ac,q_dev,latent_dev, - rope_dev,w->weights,w->scales,w->fmt,S,H,Q,R,V,K,T,scale); + rope_dev,w->weights,w->scales,w->fmt,S,H,Q,R,V,K,T,scale,w->gs,w->ng); if(!cuda_ok(cudaGetLastError(),"pipe attention launch"))return 0; size_t ob=(size_t)S*proj->O*sizeof(float); if(!reserve(&dc->y,&dc->y_cap,ob))return 0; quant_matmul<<O,S),256,0,dc->stream>>>(dc->y,dc->ac,proj->weights, - proj->scales,proj->fmt,S,proj->I,proj->O,row_bytes(proj->fmt,proj->I)); + proj->scales,proj->fmt,S,proj->I,proj->O,row_bytes(proj->fmt,proj->I),proj->gs,proj->ng); if(!cuda_ok(cudaGetLastError(),"pipe o_proj launch"))return 0; if(!cuda_ok(cudaMemcpyAsync(out,dc->y,ob,cudaMemcpyDeviceToHost,dc->stream),"pipe attention download")|| !cuda_ok(cudaStreamSynchronize(dc->stream),"pipe attention sync"))return 0; @@ -1071,17 +1477,20 @@ extern "C" int coli_cuda_attention_project_batch_dev(ColiCudaTensor *w,ColiCudaT } extern "C" int coli_cuda_pipe_silu_mul(int device,float *gate_dev,const float *up_dev, size_t n){ + if (fault_injected()) return 0; DeviceContext *ctx=find_ctx(device); if(!n||!select_ctx(ctx)) return 0; silu_mul<<<(unsigned)((n+255)/256),256>>>(gate_dev,up_dev,n); return cuda_ok(cudaGetLastError(),"pipe silu mul"); } extern "C" int coli_cuda_pipe_add(int device,float *x_dev,const float *t_dev,size_t n){ + if (fault_injected()) return 0; DeviceContext *ctx=find_ctx(device); if(!n||!select_ctx(ctx)) return 0; pipe_add_n<<<(unsigned)((n+255)/256),256>>>(x_dev,t_dev,n); return cuda_ok(cudaGetLastError(),"pipe add"); } extern "C" int coli_cuda_pipe_rows_add(int device,float *x_dev,const float *partial_dev, const int *rows_dev,int nrows,int D){ + if (fault_injected()) return 0; DeviceContext *ctx=find_ctx(device); if(nrows<1||D<1||!select_ctx(ctx)) return 0; pipe_rows_add<<>>(x_dev,partial_dev,rows_dev,D); return cuda_ok(cudaGetLastError(),"pipe rows add"); @@ -1090,11 +1499,12 @@ extern "C" int coli_cuda_pipe_rows_add(int device,float *x_dev,const float *part * coli_cuda_matmul, zero host transfers. */ extern "C" int coli_cuda_pipe_gemm(ColiCudaTensor *t,float *y_dev,const float *x_dev, int S){ + if (fault_injected()) return 0; if(!t||S<1) return 0; DeviceContext *ctx=find_ctx(t->device); if(!select_ctx(ctx)) return 0; dim3 grid((unsigned)t->O,(unsigned)S); quant_matmul<<>>(y_dev,x_dev,t->weights,t->scales,t->fmt,S,t->I,t->O, - row_bytes(t->fmt,t->I)); + row_bytes(t->fmt,t->I),t->gs,t->ng); return cuda_ok(cudaGetLastError(),"pipe gemm"); } /* copia diretta scheda->scheda (P2P se disponibile, altrimenti staging driver) */ @@ -1109,6 +1519,7 @@ extern "C" int coli_cuda_pipe_peer_copy(int dst_dev,float *dst,int src_dev, extern "C" int coli_cuda_attention_project_batch_dev_out(ColiCudaTensor *w,ColiCudaTensor *proj, float *out_dev,const float *q_dev,const float *latent_dev,const float *rope_dev, int S,int H,int Q,int R,int V,int K,int T,float scale){ + if (fault_injected()) return 0; if(!w||!proj||!out_dev||!q_dev||!latent_dev||!rope_dev||S<1||H<1||Q<1||R<1||V<1|| K<1||K>512||T8192||w->I!=K||w->O!=H*(Q+V)|| proj->device!=w->device||proj->I!=H*V)return 0; @@ -1117,10 +1528,10 @@ extern "C" int coli_cuda_attention_project_batch_dev_out(ColiCudaTensor *w,ColiC if(!reserve(&dc->ac,&dc->ac_cap,cb))return 0; size_t shared=(size_t)(2*K+T+256)*sizeof(float); attention_absorb_batch_kernel<<stream>>>(dc->ac,q_dev,latent_dev, - rope_dev,w->weights,w->scales,w->fmt,S,H,Q,R,V,K,T,scale); + rope_dev,w->weights,w->scales,w->fmt,S,H,Q,R,V,K,T,scale,w->gs,w->ng); if(!cuda_ok(cudaGetLastError(),"pipe attention launch (dev out)"))return 0; quant_matmul<<O,S),256,0,dc->stream>>>(out_dev,dc->ac,proj->weights, - proj->scales,proj->fmt,S,proj->I,proj->O,row_bytes(proj->fmt,proj->I)); + proj->scales,proj->fmt,S,proj->I,proj->O,row_bytes(proj->fmt,proj->I),proj->gs,proj->ng); if(!cuda_ok(cudaGetLastError(),"pipe o_proj launch (dev out)"))return 0; return cuda_ok(cudaStreamSynchronize(dc->stream),"pipe attention sync (dev out)"); } @@ -1130,12 +1541,13 @@ extern "C" int coli_cuda_attention_project_batch_dev_out(ColiCudaTensor *w,ColiC extern "C" int coli_cuda_attention_absorb_batch_dev(ColiCudaTensor *w,float *ctx_dev, const float *q_dev,const float *latent_dev,const float *rope_dev, int S,int H,int Q,int R,int V,int K,int T,float scale){ + if (fault_injected()) return 0; if(!w||!ctx_dev||!q_dev||!latent_dev||!rope_dev||S<1||H<1||Q<1||R<1||V<1|| K<1||K>512||T8192||w->I!=K||w->O!=H*(Q+V))return 0; DeviceContext *dc=find_ctx(w->device);if(!select_ctx(dc))return 0; size_t shared=(size_t)(2*K+T+256)*sizeof(float); attention_absorb_batch_kernel<<stream>>>(ctx_dev,q_dev,latent_dev, - rope_dev,w->weights,w->scales,w->fmt,S,H,Q,R,V,K,T,scale); + rope_dev,w->weights,w->scales,w->fmt,S,H,Q,R,V,K,T,scale,w->gs,w->ng); if(!cuda_ok(cudaGetLastError(),"pipe shard attention launch"))return 0; return cuda_ok(cudaStreamSynchronize(dc->stream),"pipe shard attention sync"); } @@ -1144,6 +1556,7 @@ extern "C" int coli_cuda_attention_absorb_batch_dev(ColiCudaTensor *w,float *ctx extern "C" int coli_cuda_attention_absorb_kvdev(ColiCudaTensor *w,float *ctx,const float *q, const float *latent_dev,const float *rope_dev,int H,int Q,int R,int V,int K,int T, float scale){ + if (fault_injected()) return 0; if(!w||!ctx||!q||!latent_dev||!rope_dev||H<1||Q<1||R<1||V<1||K<1||K>512||T<1||T>8192|| w->I!=K||w->O!=H*(Q+V))return 0; DeviceContext *dc=find_ctx(w->device);if(!select_ctx(dc))return 0; @@ -1152,7 +1565,7 @@ extern "C" int coli_cuda_attention_absorb_kvdev(ColiCudaTensor *w,float *ctx,con if(!cuda_ok(cudaMemcpyAsync(dc->aq,q,qb,cudaMemcpyHostToDevice,dc->stream),"kvdev q upload"))return 0; size_t shared=(size_t)(2*K+T+256)*sizeof(float); attention_absorb_batch_kernel<<stream>>>(dc->ac,dc->aq,latent_dev, - rope_dev,w->weights,w->scales,w->fmt,1,H,Q,R,V,K,T,scale); + rope_dev,w->weights,w->scales,w->fmt,1,H,Q,R,V,K,T,scale,w->gs,w->ng); if(!cuda_ok(cudaGetLastError(),"kvdev absorb launch")|| !cuda_ok(cudaMemcpyAsync(ctx,dc->ac,cb,cudaMemcpyDeviceToHost,dc->stream),"kvdev ctx download")|| !cuda_ok(cudaStreamSynchronize(dc->stream),"kvdev absorb sync"))return 0; diff --git a/c/backend_cuda.h b/c/backend_cuda.h index 63c1f113..b83c58a9 100644 --- a/c/backend_cuda.h +++ b/c/backend_cuda.h @@ -36,20 +36,24 @@ COLI_CUDA_DLLEXPORT void coli_cuda_group_stats(uint64_t *calls, uint64_t *expert double *h2d_ms, double *kernel_ms, double *d2h_ms); /* Upload without executing, so capacity failures happen during model startup. */ +COLI_CUDA_DLLEXPORT int coli_cuda_tensor_upload_g(ColiCudaTensor **tensor, + const void *weights, const float *scales, + int fmt, int I, int O, int device, int gs); COLI_CUDA_DLLEXPORT int coli_cuda_tensor_upload(ColiCudaTensor **tensor, const void *weights, const float *scales, int fmt, int I, int O, int device); /* * y[S,O] = x[S,I] @ W[O,I]^T. - * fmt matches QT in glm.c: 0=f32, 1=int8, 2=int4, 3=int2. - * The first successful call uploads W and its row scales; later calls reuse it. + * fmt matches QT in glm.c: 0=f32, 1=int8, 2=int4, 3=int2, 4=grouped int4. + * gs is the group size for fmt=4 (0 for all other formats). + * The first successful call uploads W and its scales; later calls reuse it. * Returns 1 on success and 0 when CUDA is not initialized or the format is invalid. */ COLI_CUDA_DLLEXPORT int coli_cuda_matmul(ColiCudaTensor **tensor, float *y, const float *x, const void *weights, const float *scales, - int fmt, int S, int I, int O, int device); + int fmt, int S, int I, int O, int device, int gs); /* Fused expert pipeline: y = down(silu(gate(x)) * up(x)). All three tensors * must already be resident on one device. Activations cross PCIe once in @@ -67,6 +71,16 @@ COLI_CUDA_DLLEXPORT int coli_cuda_shared_mlp_w4a16(ColiCudaTensor *gate, ColiCud /* Packed group of same-shaped experts. Inputs and outputs contain sum(rows) * consecutive [D] rows in call order. */ +/* Async issue/take split of the group call below (Inc.4): issue launches on the + * device stream and returns; take syncs and returns the pinned result rows (valid + * until the next issue on that device). Small totals only (<=8 rows); one + * outstanding issue per device. */ +COLI_CUDA_DLLEXPORT int coli_cuda_expert_group_issue(ColiCudaTensor *const *gates, + ColiCudaTensor *const *ups, + ColiCudaTensor *const *downs, + const int *rows, int count, const float *x); +COLI_CUDA_DLLEXPORT const float *coli_cuda_expert_group_take(int device); + COLI_CUDA_DLLEXPORT int coli_cuda_expert_group(ColiCudaTensor *const *gates, ColiCudaTensor *const *ups, ColiCudaTensor *const *downs, @@ -126,6 +140,16 @@ COLI_CUDA_DLLEXPORT int coli_cuda_pipe_rmsnorm_s(int device,float *y_dev,const f int xstride,int ystride); COLI_CUDA_DLLEXPORT int coli_cuda_pipe_rope_base(int device,float *v_dev,int pos_base,int rows, int stride,int offset,int R,int heads,float theta); +COLI_CUDA_DLLEXPORT int coli_cuda_expert_group_resident_issue(ColiCudaTensor *const *gates, + ColiCudaTensor *const *ups, ColiCudaTensor *const *downs, + const float *weights, int count, + int home_device, const float *x_src_dev, float *partial_slot_dev); +COLI_CUDA_DLLEXPORT int coli_cuda_expert_group_resident_take(int home_device,const int *devices, + int n_issued,float *slots_dev,float *acc_dev,int D); +COLI_CUDA_DLLEXPORT int coli_cuda_pipe_router(int device,const float *x_dev, + const void *rw_dev,const void *rb_dev,int D,int E,int Ksel, + float topp,int norm_topk,float routed_scale, + int *idx_host,float *w_host,int *keff_host); COLI_CUDA_DLLEXPORT int coli_cuda_pipe_copy2d(int device,float *dst,int dpitch,const float *src, int spitch,int width,int height); COLI_CUDA_DLLEXPORT int coli_cuda_attention_project_batch_dev(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj, diff --git a/c/backend_gpu_compat.h b/c/backend_gpu_compat.h new file mode 100644 index 00000000..990462af --- /dev/null +++ b/c/backend_gpu_compat.h @@ -0,0 +1,66 @@ +/* backend_gpu_compat.h — one GPU backend source, two vendors. + * Same pattern as compat.h for Windows: every platform difference lives in + * this header and backend_cuda.cu stays untouched. Compiled by nvcc this is + * a pass-through to the CUDA runtime (and mma.h for the tensor-core paths); + * compiled by hipcc (ROCm, HIP=1) it maps the CUDA runtime surface + * backend_cuda.cu uses onto HIP 1:1. The kernel language (__global__, + * __shared__, <<<>>>) is shared syntax. + * + * COLI_GPU_HAS_WMMA: the WMMA tensor-core kernels are guarded by + * __CUDA_ARCH__ >= 700 (device side) and by this flag at the host dispatch + * sites. Under HIP the flag is 0: gfx GPUs report compute_major >= 7, so a + * runtime-only check would select empty kernel bodies. Matrix-core support + * via rocWMMA is a possible follow-up; until then HIP always uses the + * portable kernels. */ +#ifndef COLIBRI_BACKEND_GPU_COMPAT_H +#define COLIBRI_BACKEND_GPU_COMPAT_H + +#if defined(__HIP_PLATFORM_AMD__) || defined(__HIP__) +#include +#include +#define COLI_GPU_HAS_WMMA 0 +#define cudaError_t hipError_t +#define cudaSuccess hipSuccess +#define cudaGetErrorString hipGetErrorString +#define cudaGetLastError hipGetLastError +#define cudaSetDevice hipSetDevice +#define cudaGetDeviceCount hipGetDeviceCount +#define cudaDeviceProp hipDeviceProp_t +#define cudaGetDeviceProperties hipGetDeviceProperties +#define cudaMalloc hipMalloc +#define cudaFree hipFree +#define cudaMemcpy hipMemcpy +#define cudaMemcpy2D hipMemcpy2D +#define cudaMemcpyAsync hipMemcpyAsync +#define cudaMemcpyHostToDevice hipMemcpyHostToDevice +#define cudaMemcpyDeviceToHost hipMemcpyDeviceToHost +#define cudaMemGetInfo hipMemGetInfo +#define cudaStream_t hipStream_t +#define cudaStreamCreate hipStreamCreate +#define cudaStreamCreateWithFlags hipStreamCreateWithFlags +#define cudaStreamNonBlocking hipStreamNonBlocking +#define cudaStreamDestroy hipStreamDestroy +#define cudaStreamSynchronize hipStreamSynchronize +#define cudaStreamWaitEvent hipStreamWaitEvent +#define cudaDeviceSynchronize hipDeviceSynchronize +#define cudaEvent_t hipEvent_t +#define cudaEventCreate hipEventCreate +#define cudaEventCreateWithFlags hipEventCreateWithFlags +#define cudaEventDisableTiming hipEventDisableTiming +#define cudaEventDestroy hipEventDestroy +#define cudaEventRecord hipEventRecord +#define cudaEventSynchronize hipEventSynchronize +#define cudaEventElapsedTime hipEventElapsedTime +#define cudaMallocHost hipHostMalloc +#define cudaFreeHost hipHostFree +#define cudaMemcpyDeviceToDevice hipMemcpyDeviceToDevice +#define cudaMemcpyPeer hipMemcpyPeer +#define cudaMemcpyPeerAsync hipMemcpyPeerAsync +#define cudaMemsetAsync hipMemsetAsync +#else +#include +#include +#define COLI_GPU_HAS_WMMA 1 +#endif + +#endif diff --git a/c/backend_loader.c b/c/backend_loader.c index eedbd50b..21be5484 100644 --- a/c/backend_loader.c +++ b/c/backend_loader.c @@ -41,14 +41,20 @@ typedef int (*fn_expert_mlp)(ColiCudaTensor *gate, ColiCudaTensor *up typedef int (*fn_expert_group)(ColiCudaTensor *const *gates, ColiCudaTensor *const *ups, ColiCudaTensor *const *downs, const int *rows, int count, float *y, const float *x); +typedef int (*fn_expert_group_issue)(ColiCudaTensor *const *gates, + ColiCudaTensor *const *ups, + ColiCudaTensor *const *downs, + const int *rows, int count, const float *x); +typedef const float * (*fn_expert_group_take)(int device); typedef int (*fn_attention_absorb)(ColiCudaTensor *kv_b, float *ctx, const float *q, const float *latent, const float *rope, int H, int Q, int R, int V, int K, int T, float attention_scale); typedef int (*fn_tensor_upload)(ColiCudaTensor **tensor, const void *weights, const float *scales, int fmt, int I, int O, int device); +typedef int (*fn_tensor_upload_g)(ColiCudaTensor **tensor, const void *weights, const float *scales, int fmt, int I, int O, int device, int gs); typedef int (*fn_matmul)(ColiCudaTensor **tensor, float *y, const float *x, const void *weights, const float *scales, - int fmt, int S, int I, int O, int device); + int fmt, int S, int I, int O, int device, int gs); typedef void (*fn_tensor_free)(ColiCudaTensor *tensor); typedef size_t (*fn_tensor_bytes)(const ColiCudaTensor *tensor); typedef int (*fn_tensor_device)(const ColiCudaTensor *tensor); @@ -76,6 +82,9 @@ typedef int (*fn_pipe_gemm)(ColiCudaTensor *t,float *y_dev,const float *x_dev,in typedef int (*fn_pipe_peer_copy)(int dst_dev,float *dst,int src_dev, const float *src,size_t bytes); typedef int (*fn_pipe_rmsnorm)(int device,float *y_dev,const float *x_dev, const float *w_dev,int S,int D,float eps); typedef int (*fn_pipe_rmsnorm_s)(int device,float *y_dev,const float *x_dev, const float *w_dev,int S,int D,float eps, int xstride,int ystride); +typedef int (*fn_group_resident_issue)(ColiCudaTensor *const *gates,ColiCudaTensor *const *ups,ColiCudaTensor *const *downs,const float *weights,int count,int home_device,const float *x_src_dev,float *partial_slot_dev); +typedef int (*fn_group_resident_take)(int home_device,const int *devices,int n_issued,float *slots_dev,float *acc_dev,int D); +typedef int (*fn_pipe_router)(int device,const float *x_dev,const void *rw_dev,const void *rb_dev,int D,int E,int Ksel,float topp,int norm_topk,float routed_scale,int *idx_host,float *w_host,int *keff_host); typedef int (*fn_pipe_rope)(int device,float *v_dev,const int *pos_dev,int rows, int stride,int offset,int R,int heads,float theta); typedef int (*fn_pipe_rope_base)(int device,float *v_dev,int pos_base,int rows, int stride,int offset,int R,int heads,float theta); typedef int (*fn_pipe_rows_add)(int device,float *x_dev,const float *partial_dev, const int *rows_dev,int nrows,int D); @@ -100,8 +109,11 @@ static struct { fn_group_stats group_stats; fn_expert_mlp expert_mlp; fn_expert_group expert_group; + fn_expert_group_issue expert_group_issue; + fn_expert_group_take expert_group_take; fn_attention_absorb attention_absorb; fn_tensor_upload tensor_upload; + fn_tensor_upload_g tensor_upload_g; fn_matmul matmul; fn_tensor_free tensor_free; fn_tensor_bytes tensor_bytes; @@ -123,6 +135,9 @@ static struct { fn_pipe_peer_copy pipe_peer_copy; fn_pipe_rmsnorm pipe_rmsnorm; fn_pipe_rmsnorm_s pipe_rmsnorm_s; + fn_group_resident_issue expert_group_resident_issue; + fn_group_resident_take expert_group_resident_take; + fn_pipe_router pipe_router; fn_pipe_rope pipe_rope; fn_pipe_rope_base pipe_rope_base; fn_pipe_rows_add pipe_rows_add; @@ -194,8 +209,11 @@ static int coli_cuda_load(void){ RESOLVE(group_stats, fn_group_stats) RESOLVE(expert_mlp, fn_expert_mlp) RESOLVE(expert_group, fn_expert_group) + RESOLVE(expert_group_issue, fn_expert_group_issue) + RESOLVE(expert_group_take, fn_expert_group_take) RESOLVE(attention_absorb, fn_attention_absorb) RESOLVE(tensor_upload, fn_tensor_upload) + RESOLVE(tensor_upload_g, fn_tensor_upload_g) RESOLVE(matmul, fn_matmul) RESOLVE(tensor_free, fn_tensor_free) RESOLVE(tensor_bytes, fn_tensor_bytes) @@ -217,6 +235,9 @@ static int coli_cuda_load(void){ RESOLVE(pipe_peer_copy, fn_pipe_peer_copy) RESOLVE(pipe_rmsnorm, fn_pipe_rmsnorm) RESOLVE(pipe_rmsnorm_s, fn_pipe_rmsnorm_s) + RESOLVE(expert_group_resident_issue, fn_group_resident_issue) + RESOLVE(expert_group_resident_take, fn_group_resident_take) + RESOLVE(pipe_router, fn_pipe_router) RESOLVE(pipe_rope, fn_pipe_rope) RESOLVE(pipe_rope_base, fn_pipe_rope_base) RESOLVE(pipe_rows_add, fn_pipe_rows_add) @@ -289,6 +310,19 @@ int coli_cuda_expert_group(ColiCudaTensor *const *gates, ColiCudaTensor *const * return g_cuda.expert_group(gates, ups, downs, rows, count, y, x); } +int coli_cuda_expert_group_issue(ColiCudaTensor *const *gates, + ColiCudaTensor *const *ups, + ColiCudaTensor *const *downs, + const int *rows, int count, const float *x){ + if(!g_cuda.available) return 0; + return g_cuda.expert_group_issue(gates, ups, downs, rows, count, x); +} + +const float *coli_cuda_expert_group_take(int device){ + if(!g_cuda.available) return NULL; + return g_cuda.expert_group_take(device); +} + int coli_cuda_attention_absorb(ColiCudaTensor *kv_b, float *ctx, const float *q, const float *latent, const float *rope, int H, int Q, int R, int V, int K, int T, float attention_scale){ @@ -302,11 +336,16 @@ int coli_cuda_tensor_upload(ColiCudaTensor **tensor, const void *weights, return g_cuda.tensor_upload(tensor, weights, scales, fmt, I, O, device); } +int coli_cuda_tensor_upload_g(ColiCudaTensor **tensor, const void *weights, const float *scales, int fmt, int I, int O, int device, int gs){ + if(!g_cuda.available || !g_cuda.tensor_upload_g){ return 0; } + return g_cuda.tensor_upload_g(tensor, weights, scales, fmt, I, O, device, gs); +} + int coli_cuda_matmul(ColiCudaTensor **tensor, float *y, const float *x, const void *weights, const float *scales, - int fmt, int S, int I, int O, int device){ + int fmt, int S, int I, int O, int device, int gs){ if(!g_cuda.available) return 0; - return g_cuda.matmul(tensor, y, x, weights, scales, fmt, S, I, O, device); + return g_cuda.matmul(tensor, y, x, weights, scales, fmt, S, I, O, device, gs); } void coli_cuda_tensor_free(ColiCudaTensor *tensor){ @@ -407,6 +446,21 @@ int coli_cuda_pipe_rmsnorm(int device,float *y_dev,const float *x_dev, const flo return g_cuda.pipe_rmsnorm(device, y_dev, x_dev, w_dev, S, D, eps); } +int coli_cuda_expert_group_resident_issue(ColiCudaTensor *const *gates,ColiCudaTensor *const *ups,ColiCudaTensor *const *downs,const float *weights,int count,int home_device,const float *x_src_dev,float *partial_slot_dev){ + if(!g_cuda.available || !g_cuda.expert_group_resident_issue){ return 0; } + return g_cuda.expert_group_resident_issue(gates, ups, downs, weights, count, home_device, x_src_dev, partial_slot_dev); +} + +int coli_cuda_expert_group_resident_take(int home_device,const int *devices,int n_issued,float *slots_dev,float *acc_dev,int D){ + if(!g_cuda.available || !g_cuda.expert_group_resident_take){ return 0; } + return g_cuda.expert_group_resident_take(home_device, devices, n_issued, slots_dev, acc_dev, D); +} + +int coli_cuda_pipe_router(int device,const float *x_dev,const void *rw_dev,const void *rb_dev,int D,int E,int Ksel,float topp,int norm_topk,float routed_scale,int *idx_host,float *w_host,int *keff_host){ + if(!g_cuda.available || !g_cuda.pipe_router){ return 0; } + return g_cuda.pipe_router(device, x_dev, rw_dev, rb_dev, D, E, Ksel, topp, norm_topk, routed_scale, idx_host, w_host, keff_host); +} + int coli_cuda_pipe_rmsnorm_s(int device,float *y_dev,const float *x_dev, const float *w_dev,int S,int D,float eps, int xstride,int ystride){ if(!g_cuda.available){ return 0; } return g_cuda.pipe_rmsnorm_s(device, y_dev, x_dev, w_dev, S, D, eps, xstride, ystride); diff --git a/c/backend_metal.h b/c/backend_metal.h index 4383b3b6..35c4931c 100644 --- a/c/backend_metal.h +++ b/c/backend_metal.h @@ -84,6 +84,23 @@ int coli_metal_layer_decode(float *x, int coli_metal_gemm(float *y, const float *x, const void *weights, const float *scales, int fmt, int S, int I, int O); /* large-batch sync GEMM; 0 -> CPU */ +/* Parallel top-8 expert selection (r_top8_par): run ONE top-8 selection kernel standalone + * on host arrays — par=0 the serial r_top8, par=1 the parallel exact-match replica gated + * in the engine by COLI_RTOP8 (default ON; COLI_RTOP8=0 opts out to the serial kernel). + * Exists so the metal-test suite (and any battery probe) can prove serial/parallel + * equivalence on the ENGINE build's own compiled shaders, not just in the bench tool. + * sig[S*E], bias[E], idx[S*K], w[S*K], keff[S]. + * Expert-count generality: the parallel kernel's blocked-lane design (ch[8]/32-lane + * threadgroup) is validated correct for arbitrary E<=256, including non-multiples of the + * 32-lane width and small E (see metal-test's E=24/E=168/E=256 cases — 168 is the REAP + * expert-pruned package width from #428/#426). For E>256 (out of contract) this function + * transparently falls back to the serial kernel even when par=1 is requested, and the + * same automatic fallback is wired into the engine dispatch site — "par" is a request, + * never a guarantee, so no caller can reach the unguarded parallel path out of contract. + * Returns 1 on success, 0 if Metal is unavailable. */ +int coli_metal_rtop8(int par, const float *sig, const float *bias, int S, int E, int K, + int Ksel, float topp, int normk, float rscale, + int *idx, float *w, int *keff); void coli_metal_attn_counts(uint64_t *ok, double *wall, double *kernel); void coli_metal_attn_lat(double *ksched, double *gsched); int coli_metal_attn_decode(const float *x, @@ -98,6 +115,10 @@ int coli_metal_attn_decode(const float *x, void coli_metal_moe_counts(uint64_t *ok, uint64_t *fb, uint64_t *experts); void coli_metal_moe_times(double *setup, double *gpu, double *scatter); double coli_metal_moe_kernel_time(void); +/* E5 (COLI_METAL_RESSET=1): returns 1 when the queue-attached residency set is active and + * writes the cumulative seconds moe_submit spent committing pending set adds -- a cost that + * sits OUTSIDE the setup/gpu/scatter breakdown above. Returns 0 (and writes 0) when off. */ +int coli_metal_resset_stats(double *flush_s); /* * Batched routed-expert SwiGLU for one MoE block, in ONE command buffer. diff --git a/c/backend_metal.mm b/c/backend_metal.mm index 51ed8a09..6ec3fb4f 100644 --- a/c/backend_metal.mm +++ b/c/backend_metal.mm @@ -219,6 +219,63 @@ kernel void r_top8(device const float* sig [[buffer(0)]], device const float* bi if(normk){ float sm=0; for(int kk=0;kk' ascending max (lowest index wins +// within a lane, matching the serial ascending scan), then a shuffle-down argmax +// reduction where ties prefer the LOWER index — together exactly the serial kernel's +// first-max-wins order. The topp/normk/rscale tail is the serial code verbatim on lane 0 +// (same ops, same order => bitwise-identical results; metal-test enforces this with +// memcmp). Contract: E<=256 (ch[8]/taken mask sizing: ceil(E/32)<=8) — the defensive +// return below makes an out-of-contract dispatch a visible no-op (idx/w/keff untouched), +// never an OOB write; both call sites (coli_metal_layer_decode's dispatch and the +// standalone coli_metal_rtop8 runner) additionally gate on E<=256 in host code before +// selecting this pipeline at all, so the return here is defense-in-depth, not the only +// guard. Sentinel-per-lane design (ch[j]=-1e30f for e>=E) makes non-multiple-of-32 E +// and small E correct without special-casing — validated for E=24, E=168 (REAP +// expert-pruned packages, see the upstream feature-request thread) and E=256 by metal-test. +// ASSUMES SIMD width 32 (shuffle offsets 16..1, 32-thread threadgroup per row): enforced +// at init — coli_metal_init clears g_rtop8_width_ok (and therefore both call sites' use +// of this pipeline) if threadExecutionWidth != 32. +kernel void r_top8_par(device const float* sig [[buffer(0)]], device const float* bias [[buffer(1)]], + device int* idx [[buffer(2)]], device float* w [[buffer(3)]], + device int* keff [[buffer(4)]], constant int& E [[buffer(5)]], + constant int& K [[buffer(6)]], constant int& Ksel [[buffer(7)]], + constant float& topp [[buffer(8)]], constant int& normk [[buffer(9)]], + constant float& rscale [[buffer(10)]], + uint s [[threadgroup_position_in_grid]], + uint slane [[thread_index_in_simdgroup]]) { + if(E>256) return; + device const float* sg=sig+(long)s*E; + device int* id_=idx+(long)s*K; device float* ww=w+(long)s*K; + int per=(E+31)/32, base=(int)slane*per; + float ch[8]; uint taken=0u; + for(int j=0;jbv){ bv=ch[j]; bi=base+j; } + for(uint off=16;off>0;off>>=1){ + float ov=simd_shuffle_down(bv,off); int oi=simd_shuffle_down(bi,off); + if(ov>bv || (ov==bv && oi=base && bi0.0f && topp<1.0f){ + for(int a=1;a=0 && ww[b]=topp*tot){ Ke=kk+1; break; } } + } + keff[s]=Ke; + if(normk){ float sm=0; for(int kk=0;kk g_queue; static id g_gemv, g_moe_gemv, g_moe_silu; static id g_a_rms, g_a_rope, g_a_copy, g_a_qabs, g_a_score, g_a_smax, g_a_clat, g_a_ctx; -static id g_a_add, g_r_router, g_r_top8; +static id g_a_add, g_r_router, g_r_top8, g_r_top8p; +static int g_rtop8_par = 1; // COLI_RTOP8 (default ON); COLI_RTOP8=0 opts out to the + // serial kernel — see coli_metal_init. +static int g_rtop8_width_ok = 1; // hardware fact, independent of the policy gate above: + // false if this device's threadExecutionWidth != 32. + // Consulted by BOTH the engine dispatch site and the + // standalone coli_metal_rtop8 runner, so no caller can + // reach r_top8_par's 32-lane reduction on an unsafe + // device even by explicitly requesting par=1. static size_t g_tensor_count, g_tensor_bytes; static uint64_t g_moe_ok, g_moe_fb, g_moe_experts; // GPU blocks / CPU-fallback blocks / experts on GPU static double g_t_setup, g_t_gpu, g_t_scatter, g_t_kernel; // per-block time breakdown (seconds) @@ -258,6 +323,73 @@ kernel void r_top8(device const float* sig [[buffer(0)]], device const float* bi struct Slab { void *base; size_t len; id buf; }; static std::vector g_slabs; static std::mutex g_slab_mtx; // expert_load registers slabs from parallel OpenMP threads + +// ---- E5 experiment: COLI_METAL_RESSET=1 -- one persistent MTLResidencySet attached to +// g_queue (macOS 15+) replaces moe_submit's per-command-buffer useResource: loop over +// resolved expert weight/scale slabs. Allocation is untouched (same newBufferWithBytesNoCopy +// wrap as stock); only residency bookkeeping moves off the dispatch hot path -- see +// SUMMARY.md for why skipping useResource: there is safe (read-only, indirectly-referenced +// buffers only; residency sets don't do hazard tracking, but nothing here relied on it). +// g_resset_obj is a bare `id` (holds id) so the global's declared type +// carries no availability annotation -- the protocol name only appears inside +// @available(macOS 15.0, *) guards below, keeping -Wunguarded-availability clean. +static id g_resset_obj; +static bool g_resset_enabled; // COLI_METAL_RESSET=1, macOS 15+, and creation succeeded +static bool g_resset_dirty; // addAllocation: calls pending commit; g_resset_mtx-guarded +// Set mutations + dirty flag get their OWN mutex, never held together with g_slab_mtx: no +// live Metal call may run under the slab lock the parallel OMP loader threads contend on +// (E4's audit round 2 found exactly that shape -- mutex over a live Metal call -- as the +// leading suspect for its +12s expert-disk regression). g_slab_mtx keeps guarding g_slabs +// bookkeeping only, exactly as on stock. +static std::mutex g_resset_mtx; +static double g_t_resset_flush; // sec committing pending adds in moe_submit (gate on only) + +// Add a just-wrapped buffer to the set; commit deferred (an OMP loader burst batches into +// one commit at the next moe_submit instead of one per slab). Called by coli_metal_register +// after it drops g_slab_mtx but before it returns -- and the engine cannot dispatch an +// expert before the load that registers its slab returns, so any slab a given moe_submit +// can resolve() was added (and marked dirty) under g_resset_mtx strictly before that +// moe_submit's resset_flush() acquired the same mutex: the flush covers it. The slab-table +// ordering itself (register-before-resolve) is unchanged and stays under g_slab_mtx. +// Cost lands in the caller's existing expert-load accounting (t_ewait window in colibri.c); +// no separate counter for the add/remove side. +static void resset_add(id b) { + if (!g_resset_enabled) return; + std::lock_guard lk(g_resset_mtx); + if (@available(macOS 15.0, *)) { [(id)g_resset_obj addAllocation:b]; g_resset_dirty = true; } +} +// Remove + commit immediately, NOT deferred: the caller frees the underlying host memory +// right after coli_metal_unregister returns, so the removal must be applied before that -- +// an uncommitted-but-still-resident allocation pointing at freed memory is a use-after-free +// risk the GPU could act on. Also runs outside g_slab_mtx (see g_resset_mtx above). +static void resset_remove(id b) { + if (!g_resset_enabled) return; + std::lock_guard lk(g_resset_mtx); + if (@available(macOS 15.0, *)) { + id rs = (id)g_resset_obj; + [rs removeAllocation:b]; [rs commit]; + } + g_resset_dirty = false; // commit above also flushes any pending adds +} +// Flush pending adds before moe_submit relies on the set alone for residency -- the only +// caller that skips per-buffer useResource: (see moe_submit below). Takes g_resset_mtx +// only, never g_slab_mtx; the happens-before argument lives at resset_add above. +static void resset_flush() { + if (!g_resset_enabled) return; + std::lock_guard lk(g_resset_mtx); + if (!g_resset_dirty) return; + if (@available(macOS 15.0, *)) { [(id)g_resset_obj commit]; } + g_resset_dirty = false; +} +// Harness visibility for the flush cost, which sits OUTSIDE the moe_times setup/gpu +// breakdown (timed around resset_flush in moe_submit, before ts_start). Returns whether +// the set is active so colibri.c prints the METAL-RESSET line only when the gate is on -- +// stock output stays byte-identical. +extern "C" int coli_metal_resset_stats(double *flush_s) { + if (flush_s) *flush_s = g_t_resset_flush; + return g_resset_enabled ? 1 : 0; +} + // Persistent scratch buffers (grow-only) for the MoE pipeline. static id g_gg, g_uu, g_hh, g_xg; static size_t g_gg_cap, g_uu_cap, g_hh_cap, g_xg_cap; static id ensure(id b, size_t *cap, size_t need) { @@ -284,6 +416,8 @@ static size_t fmt_bytes(int fmt, int I, int O) { if (g_dev) return 1; if (getenv("COLI_METAL_UNTRACKED") && atoi(getenv("COLI_METAL_UNTRACKED"))) g_res_opts = MTLResourceStorageModeShared | MTLResourceHazardTrackingModeUntracked; + { const char *e = getenv("COLI_RTOP8"); // default ON; COLI_RTOP8=0 opts out + if (e && atoi(e) == 0) g_rtop8_par = 0; } @autoreleasepool { g_dev = MTLCreateSystemDefaultDevice(); if (!g_dev) return 0; @@ -299,11 +433,45 @@ static size_t fmt_bytes(int fmt, int I, int O) { auto P=[&](const char*n){ return [g_dev newComputePipelineStateWithFunction:[lib newFunctionWithName:@(n)] error:&err]; }; g_a_rms=P("a_rmsnorm"); g_a_rope=P("a_rope"); g_a_copy=P("a_copy"); g_a_qabs=P("a_qabs"); g_a_score=P("a_score"); g_a_smax=P("a_smax"); g_a_clat=P("a_clat"); g_a_ctx=P("a_ctx"); - g_a_add=P("a_add"); g_r_router=P("r_router"); g_r_top8=P("r_top8"); - if(!g_a_add||!g_r_router||!g_r_top8){ fprintf(stderr,"[metal] tail pipelines failed\n"); g_dev=nil; return 0; } + g_a_add=P("a_add"); g_r_router=P("r_router"); g_r_top8=P("r_top8"); g_r_top8p=P("r_top8_par"); + if(!g_a_add||!g_r_router||!g_r_top8||!g_r_top8p){ fprintf(stderr,"[metal] tail pipelines failed\n"); g_dev=nil; return 0; } + // r_top8_par's reduction hardcodes SIMD width 32 (shuffle-down offsets 16..1, one + // 32-thread threadgroup per row). True on all Apple Silicon shipped to date, but a + // non-32-width device would reduce wrongly AND race multiple lane-0 writers, so this + // is a hard safety fact (g_rtop8_width_ok), not just a policy default: it gates BOTH + // the engine dispatch site and the standalone coli_metal_rtop8 runner (degrade-to-safe, + // same pattern as the pool/ring fallbacks elsewhere) — no caller can opt back into an + // unsafe reduction on such a device, even by explicitly requesting par=1. + if ([g_r_top8p threadExecutionWidth] != 32) { + g_rtop8_width_ok = 0; + if (g_rtop8_par) + fprintf(stderr, "[metal] COLI_RTOP8 parallel top-8 disabled: threadExecutionWidth=%lu " + "!= 32 (r_top8_par's reduction assumes 32-lane simdgroups) — serial " + "r_top8 in use\n", (unsigned long)[g_r_top8p threadExecutionWidth]); + g_rtop8_par = 0; + } if (!g_gemv || !g_moe_gemv || !g_moe_silu || !g_a_rms || !g_a_rope || !g_a_copy || !g_a_qabs || !g_a_score || !g_a_smax || !g_a_clat || !g_a_ctx) { fprintf(stderr, "[metal] pipeline failed\n"); g_dev = nil; return 0; } + // E5 experiment: COLI_METAL_RESSET=1 -- see g_resset_obj comment above. + if (getenv("COLI_METAL_RESSET") && atoi(getenv("COLI_METAL_RESSET"))) { + if (@available(macOS 15.0, *)) { + MTLResidencySetDescriptor *rd = [MTLResidencySetDescriptor new]; + rd.initialCapacity = 4096; // hint only (internal array presize), not a hard limit + NSError *rerr = nil; + id rs = [g_dev newResidencySetWithDescriptor:rd error:&rerr]; + if (rs) { + [g_queue addResidencySet:rs]; + g_resset_obj = rs; g_resset_enabled = true; + fprintf(stderr, "[METAL] residency-set: on (macOS 15+, moe_submit skips per-buffer useResource:)\n"); + } else { + fprintf(stderr, "[METAL] residency-set create failed: %s -- stock per-CB residency path\n", + rerr ? [[rerr localizedDescription] UTF8String] : "?"); + } + } else { + fprintf(stderr, "[METAL] COLI_METAL_RESSET=1 requested but OS < macOS 15 -- stock per-CB residency path\n"); + } + } } return 1; } @@ -313,13 +481,29 @@ static size_t fmt_bytes(int fmt, int I, int O) { id b = [g_dev newBufferWithBytesNoCopy:base length:len options:g_res_opts deallocator:nil]; if (!b) return; - std::lock_guard lk(g_slab_mtx); // called from parallel expert_load threads - for (auto &s : g_slabs) if (s.base == base) { s.len = len; s.buf = b; return; } - g_slabs.push_back({base, len, b}); + id old = nil; // E5: replaced wrapper on re-register of a live base (defensive) + { + std::lock_guard lk(g_slab_mtx); // called from parallel expert_load threads + bool found = false; + for (auto &s : g_slabs) if (s.base == base) { old = s.buf; s.len = len; s.buf = b; found = true; break; } + if (!found) g_slabs.push_back({base, len, b}); + } + // E5, outside g_slab_mtx (no Metal call under the slab lock), before returning. Invariant + // defended: set membership mirrors g_slabs exactly -- a re-register of a live base must + // drop the replaced wrapper from the set (ARC releases our reference, but the set retains + // it and keeps its pages resident forever) before adding the new one. No in-tree caller + // re-registers a live base today; defensive. + if (old && old != b) resset_remove(old); + if (old != b) resset_add(b); } extern "C" void coli_metal_unregister(void *base) { - std::lock_guard lk(g_slab_mtx); - for (size_t i=0;i b = nil; + { + std::lock_guard lk(g_slab_mtx); + for (size_t i=0;i resolve(const void *p, uint64_t *addr) { @@ -357,7 +541,14 @@ static size_t fmt_bytes(int fmt, int I, int O) { } extern "C" void coli_metal_spin_stop(void) { g_spin_run.store(false); } -extern "C" void coli_metal_shutdown(void) { coli_metal_spin_stop(); g_gemv=nil; g_queue=nil; g_dev=nil; g_tensor_count=g_tensor_bytes=0; } +extern "C" void coli_metal_shutdown(void) { + coli_metal_spin_stop(); + if (g_resset_enabled) { + if (@available(macOS 15.0, *)) { [g_queue removeResidencySet:(id)g_resset_obj]; } + } + g_resset_obj=nil; g_resset_enabled=false; g_resset_dirty=false; + g_gemv=nil; g_queue=nil; g_dev=nil; g_tensor_count=g_tensor_bytes=0; +} extern "C" int coli_metal_available(void) { return g_dev != nil; } extern "C" void coli_metal_stats(size_t *c, size_t *b) { if(c)*c=g_tensor_count; if(b)*b=g_tensor_bytes; } extern "C" int coli_metal_mem_info(size_t *used, size_t *total) { @@ -597,12 +788,22 @@ static bool resolve_attn(const AttnW *W, float *Lc, float *Rc, // 5) silu(gate)*up + exact top-K select [e setComputePipelineState:g_moe_silu]; [e setBuffer:ash1_ offset:0 atIndex:0]; [e setBuffer:ash2_ offset:0 atIndex:1]; [e dispatchThreads:MTLSizeMake((size_t)S*SI,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; - { [e setComputePipelineState:g_r_top8]; + { // COLI_RTOP8 (default ON) swaps the serial 1-thread-per-row select for the exact- + // match 1-simdgroup-per-row replica (same buffers/args; only pipeline+grid change). + // E<=256 is required by r_top8_par's ch[8]/32-lane blocking contract; this call + // site's E is always 256 today (layer_forward_rows' own architecture-shape gate in + // colibri.c requires c->n_experts==256 to reach coli_metal_layer_decode at all — + // see PR body "Scope statement") but the check is kept here too, defense-in-depth, + // so a future relaxation of that gate (e.g. to admit REAP-pruned E=168 models into + // the fused path) degrades safely to the serial kernel instead of mis-dispatching. + int use_par = g_rtop8_par && g_rtop8_width_ok && E<=256; + [e setComputePipelineState:use_par?g_r_top8p:g_r_top8]; [e setBuffer:asig_ offset:0 atIndex:0]; [e setBuffer:rbB offset:rboff atIndex:1]; [e setBuffer:aidx_ offset:0 atIndex:2]; [e setBuffer:aw_ offset:0 atIndex:3]; [e setBuffer:akeff_ offset:0 atIndex:4]; [e setBytes:&E length:4 atIndex:5]; [e setBytes:&K length:4 atIndex:6]; [e setBytes:&Ksel length:4 atIndex:7]; [e setBytes:&topp length:4 atIndex:8]; [e setBytes:&normk length:4 atIndex:9]; [e setBytes:&rscale length:4 atIndex:10]; - [e dispatchThreads:MTLSizeMake(S,1,1) threadsPerThreadgroup:MTLSizeMake(S,1,1)]; } + if(use_par) [e dispatchThreadgroups:MTLSizeMake(S,1,1) threadsPerThreadgroup:MTLSizeMake(32,1,1)]; + else [e dispatchThreads:MTLSizeMake(S,1,1) threadsPerThreadgroup:MTLSizeMake(S,1,1)]; } BAR(); // 6) shared down bind_gemv(e,shd_w,shd_s,shd_fmt,SI,AH,ash1_,ashout_,S); @@ -651,6 +852,44 @@ static bool resolve_attn(const AttnW *W, float *Lc, float *Rc, return 1; } +// Standalone single-kernel runner for the top-8 select (see backend_metal.h). Fresh +// shared buffers per call (a test/probe path, not a hot path); grids exactly as the +// engine dispatch site: serial = S threads of one S-wide threadgroup, parallel = S +// threadgroups x 32 (one simdgroup per row). "par" is a REQUEST, not a guarantee: same +// E<=256 and SIMD-width-32 host-side checks as the engine dispatch site gate the actual +// pipeline choice, so a caller (including metal-test itself) can never reach the parallel +// kernel out of contract by asking for it — par=1 with E>256, or on a non-32-wide device, +// transparently runs the serial kernel instead and still returns 1 (success). +extern "C" int coli_metal_rtop8(int par, const float *sig, const float *bias, int S, int E, int K, + int Ksel, float topp, int normk, float rscale, + int *idx, float *w, int *keff) { + if (!g_dev || S < 1 || E < 1 || K < 1 || Ksel < 1 || Ksel > K) return 0; + int use_par = par && g_r_top8p && g_rtop8_width_ok && E<=256; + @autoreleasepool { + id bs=[g_dev newBufferWithBytes:sig length:(size_t)S*E*4 options:MTLResourceStorageModeShared]; + id bb=[g_dev newBufferWithBytes:bias length:(size_t)E*4 options:MTLResourceStorageModeShared]; + id bi=[g_dev newBufferWithLength:(size_t)S*K*4 options:MTLResourceStorageModeShared]; + id bw=[g_dev newBufferWithLength:(size_t)S*K*4 options:MTLResourceStorageModeShared]; + id bk=[g_dev newBufferWithLength:(size_t)S*4 options:MTLResourceStorageModeShared]; + if(!bs||!bb||!bi||!bw||!bk) return 0; + memset(bi.contents,0xFF,(size_t)S*K*4); // poison: untouched slots stay visible + id cb=[g_queue commandBuffer]; id e=[cb computeCommandEncoder]; + [e setComputePipelineState:use_par?g_r_top8p:g_r_top8]; + [e setBuffer:bs offset:0 atIndex:0]; [e setBuffer:bb offset:0 atIndex:1]; + [e setBuffer:bi offset:0 atIndex:2]; [e setBuffer:bw offset:0 atIndex:3]; [e setBuffer:bk offset:0 atIndex:4]; + [e setBytes:&E length:4 atIndex:5]; [e setBytes:&K length:4 atIndex:6]; [e setBytes:&Ksel length:4 atIndex:7]; + [e setBytes:&topp length:4 atIndex:8]; [e setBytes:&normk length:4 atIndex:9]; [e setBytes:&rscale length:4 atIndex:10]; + if(use_par) [e dispatchThreadgroups:MTLSizeMake((NSUInteger)S,1,1) threadsPerThreadgroup:MTLSizeMake(32,1,1)]; + else [e dispatchThreads:MTLSizeMake((NSUInteger)S,1,1) threadsPerThreadgroup:MTLSizeMake((NSUInteger)S,1,1)]; + [e endEncoding]; [cb commit]; [cb waitUntilCompleted]; + if(cb.status==MTLCommandBufferStatusError){ fprintf(stderr,"[metal] rtop8 cmdbuf error\n"); return 0; } + memcpy(idx,bi.contents,(size_t)S*K*4); + memcpy(w,bw.contents,(size_t)S*K*4); + memcpy(keff,bk.contents,(size_t)S*4); + } + return 1; +} + extern "C" void coli_metal_tensor_free(ColiMetalTensor *t) { if (!t) return; g_tensor_count--; g_tensor_bytes -= t->wbytes; @@ -668,6 +907,9 @@ static bool resolve_attn(const AttnW *W, float *Lc, float *Rc, const float *xg, const int *xoff, const int *nr, int R, id xg_buf, id gg_buf, id uu_buf, id hh_buf) { if (!g_dev || (fmt != 1 && fmt != 2)) return nil; + if (g_resset_enabled) { // E5: commit any pending slab adds before we may skip useResource: + double t0 = mnow(); resset_flush(); g_t_resset_flush += mnow() - t0; // METAL-RESSET line + } double ts_start = mnow(); std::vector ag(nb),au(nb),ad(nb),sgv(nb),suv(nb),sdv(nb); std::vector> use; use.reserve(nb*2); @@ -689,7 +931,18 @@ static bool resolve_attn(const AttnW *W, float *Lc, float *Rc, memcpy([xg_buf contents], xg, (size_t)R*D*4); id cb=[g_queue commandBuffer]; id e=[cb computeCommandEncoder]; - for(auto&b:use) [e useResource:b usage:MTLResourceUsageRead]; + // E5 (COLI_METAL_RESSET=1): the queue-attached MTLResidencySet already guarantees these + // buffers are resident, so skip the per-buffer declaration whose count scales with LRU + // cache size (mechanism history v5). Residency sets don't do hazard tracking (Apple docs), + // but none was load-bearing here: every buffer in `use` is MTLResourceUsageRead-only and + // referenced only indirectly (moe_gemv dereferences waddr[]/saddr[] baked into bag/bsg's + // contents), so there's no GPU-side write to serialize against; the one real hazard -- a + // slab unregistered+freed+reused while an async in-flight CB still reads it -- is a + // CPU-write race outside Metal's hazard tracking either way, held by the engine's own slot + // lifecycle, not by useResource:. See SUMMARY.md UNCERTAINTIES. + if (!g_resset_enabled) { + for(auto&b:use) [e useResource:b usage:MTLResourceUsageRead]; + } auto gemv=[&](id wa,id sa,id xin,id y,int O,int K,int Kin){ int NT=R*O; [e setComputePipelineState:g_moe_gemv]; diff --git a/c/coli b/c/coli index 803c22b5..133d21e6 100755 --- a/c/coli +++ b/c/coli @@ -15,6 +15,9 @@ Run GLM-5.2 (744B) locally on CPU with roughly 15-26 GB of RAM. Configuration through environment variables or flags (also valid after the subcommand): COLI_MODEL= model directory (default /home/vincenzo/glm52_i4) + COLI_MODEL_MIRROR= second copy of the model on another drive: expert reads + are split across both SSDs (COLI_DISK_WEIGHTS=9,3 sets the + primary,mirror bandwidth ratio; default: measured at startup) --ram N RAM budget in GB (automatically sizes the expert cache) --repin N adapt RAM/VRAM experts every N tokens --topp P adaptive expert top-p --topk N fixed top-k @@ -54,16 +57,20 @@ from version import __version__ as _version # guess is right (e.g. a custom packaging layout). _EXE = ".exe" if sys.platform == "win32" else "" _LIBEXEC = os.path.join(os.path.dirname(HERE), "libexec", "colibri") +_here_colibri = os.path.join(HERE, "colibri" + _EXE) _here_glm = os.path.join(HERE, "glm" + _EXE) if os.environ.get("COLI_ENGINE"): GLM = os.environ["COLI_ENGINE"] TOOLS = os.path.join(os.path.dirname(GLM), "tools") +elif os.path.exists(_here_colibri): + GLM = _here_colibri + TOOLS = os.path.join(HERE, "tools") elif os.path.exists(_here_glm): GLM = _here_glm TOOLS = os.path.join(HERE, "tools") else: - GLM = os.path.join(_LIBEXEC, "glm" + _EXE) + GLM = os.path.join(_LIBEXEC, "colibri" + _EXE) TOOLS = os.path.join(_LIBEXEC, "tools") sys.path.insert(0, _LIBEXEC) # so `import resource_plan`, `doctor`, `openai_server` still resolve @@ -233,6 +240,55 @@ def env_for(a): gpu=f" · VRAM {format_bytes(vt['budget_bytes'])}" if has_cuda and vt["devices"] else " · CPU" print(f" {C.dim}[PLAN] RAM {format_bytes(rt['budget_bytes'])} · cap {rt['cache_slots_per_layer']}/layer{gpu}{C.r}",file=sys.stderr) else: + # Windows: a bare `coli chat` (no --gpu/--vram/--auto-tier) used to ALWAYS + # run CPU-only, even on a CUDA build with a GPU present — cuda_binary() + # returned False on Windows (see above), and nothing set COLI_CUDA without + # an explicit flag. Now that detection works, auto-enable the GPU when one + # is detected so `coli chat` Just Works. Scoped to Windows: Linux already + # has working detection + the explicit-flag UX, and changing bare-chat + # semantics there is out of scope. Falls back to CPU with a warning if + # nvidia-smi is missing (discover_gpus can't size VRAM without it). + # An explicit COLI_CUDA=0 in the environment must win over the implicit + # auto-enable: before this check, a Windows user setting COLI_CUDA=0 for + # a CPU baseline silently got a ~12.6 GB VRAM expert tier anyway (the + # engine's "CPU" rows were GPU-assisted). --gpu none remains the + # canonical hard off-switch (works on every platform, also clears the + # CUDA_* sizing vars). + if (sys.platform == "win32" and a.gpu is None and not a.vram + and e.get("COLI_CUDA") != "0"): + if cuda_binary(): + from resource_plan import discover_gpus, build_plan, environment_for_plan, format_bytes + gpus = discover_gpus() + if gpus: + e["COLI_CUDA"]="1" + e.setdefault("COLI_GPUS", ",".join(str(g["index"]) for g in gpus)) + # Reuse the planner so the expert-tier VRAM budget is the real + # free VRAM minus the 2 GB reserve — not a guess. Same machinery + # as --auto-tier, just without requiring the user to pass it. + ram,ctx,devices,vram_req = resource_request(a, e) + try: + plan=build_plan(a.model,ram,ctx,devices,vram_req,policy=a.policy) + e.update(environment_for_plan(plan,e,cuda_enabled=True)) + vt=plan["tiers"]["vram"] + names=",".join(g["name"].strip() for g in gpus) + print(f" {C.dim}[GPU] auto-enabled CUDA · {names} · " + f"{format_bytes(vt['budget_bytes'])} expert tier{C.r}", file=sys.stderr) + except (OSError,ValueError,json.JSONDecodeError) as error: + # Plan failed (e.g. model dir unreadable): don't block the + # run, just leave the unsized COLI_CUDA=1 and let the engine + # pick its own budget. Engine handles a missing budget. + print(f" {C.yel}[GPU] auto-enable: could not size VRAM ({error}); " + f"using engine default{C.r}", file=sys.stderr) + else: + print(f" {C.yel}[GPU] coli_cuda.dll present but nvidia-smi not found on PATH " + f"(cannot size VRAM); running CPU-only. Add nvidia-smi to PATH or pass " + f"--vram N to enable CUDA.{C.r}", file=sys.stderr) + # else: CPU build (no coli_cuda.dll) — stay silent, CPU is correct. + elif e.get("COLI_CUDA") == "0": + # honoured off-switch: also drop stale device/sizing vars so the + # engine can't be re-enabled by leftovers (same as --gpu none). + e.pop("COLI_GPU",None); e.pop("COLI_GPUS",None) + e.pop("CUDA_EXPERT_GB",None); e.pop("CUDA_DENSE",None) # --gpu/--vram SENZA --auto-tier: prima venivano ignorati in silenzio e il run # partiva CPU-only senza alcun avviso — benchmark "GPU" pubblicati per errore (#121). if a.gpu is not None: @@ -241,13 +297,13 @@ def env_for(a): e["COLI_CUDA"]="0"; e.pop("CUDA_EXPERT_GB",None); e.pop("CUDA_DENSE",None) else: if not cuda_binary(): - sys.exit(f"{C.yel}--gpu needs the CUDA build:{C.r} make glm CUDA=1 (this binary is CPU-only)") + sys.exit(f"{C.yel}--gpu needs the CUDA build:{C.r} make colibri CUDA=1 (this binary is CPU-only)") e["COLI_CUDA"]="1" if a.gpu!="auto": e["COLI_GPUS"]=a.gpu e.setdefault("CUDA_DENSE","1") if a.vram and a.gpu!="none": if not cuda_binary(): - sys.exit(f"{C.yel}--vram needs the CUDA build:{C.r} make glm CUDA=1 (this binary is CPU-only)") + sys.exit(f"{C.yel}--vram needs the CUDA build:{C.r} make colibri CUDA=1 (this binary is CPU-only)") e["COLI_CUDA"]="1"; e["CUDA_EXPERT_GB"]=str(a.vram) return e @@ -421,8 +477,8 @@ def cmd_build(a): banner("build") if not os.path.exists(os.path.join(HERE, "Makefile")): sys.exit(f"{C.yel}coli build{C.r} only works from a source checkout (this is an installed copy).\n" - f" Clone https://github.com/JustVugg/colibri and run ./setup.sh, or make -C c glm.") - sys.exit(subprocess.call(["make","-C",HERE,"glm"])) + f" Clone https://github.com/JustVugg/colibri and run ./setup.sh, or make -C c colibri.") + sys.exit(subprocess.call(["make","-C",HERE,"colibri"])) def cmd_info(a): banner("info") @@ -802,7 +858,7 @@ def cmd_stop(a): if "coli" in cmd and " serve" in cmd and pid!=os.getpid(): if not any(p==pid for p,_ in targets): targets.append((pid,"coli serve (cmdline)")) comm=open(f"/proc/{pd}/comm").read().strip() - if comm in ("glm","exe","olmoe"): + if comm in ("colibri","glm","exe","olmoe"): env=open(f"/proc/{pd}/environ","rb").read().replace(b"\0",b"\n").decode("utf-8","replace") if "SERVE=1" in env: targets.append((pid,f"engine `{comm}` (SERVE=1)")) except (OSError,PermissionError): continue diff --git a/c/glm.c b/c/colibri.c similarity index 76% rename from c/glm.c rename to c/colibri.c index a3e7ab74..fef41369 100644 --- a/c/glm.c +++ b/c/colibri.c @@ -69,25 +69,12 @@ static inline int omp_get_thread_num(void){ return 0; } #include static int g_metal_enabled; static int g_metal_gemm_min=16; /* COLI_METAL_GEMM_MIN: min rows to send a matmul_qt GEMM to GPU */ -/* routing precalcolata dalla GPU (layer CB): moe() la usa e salta la FASE A */ -static const int *g_pre_idx; static const float *g_pre_w; static const int *g_pre_keff; -static const float *g_pre_sh; /* output dello shared expert gia' calcolato su GPU */ -#endif -#ifdef __AVX2__ -#include -static inline float hsum256(__m256 v){ /* somma orizzontale di 8 float */ - __m128 lo=_mm256_castps256_ps128(v), hi=_mm256_extractf128_ps(v,1); - lo=_mm_add_ps(lo,hi); __m128 sh=_mm_movehl_ps(lo,lo); lo=_mm_add_ps(lo,sh); - sh=_mm_shuffle_ps(lo,lo,1); lo=_mm_add_ss(lo,sh); return _mm_cvtss_f32(lo); -} -#elif defined(__ARM_NEON) -#include /* Apple Silicon / aarch64: kernel NEON */ -#elif defined(__VSX__) -#include /* POWER8+ (ppc64le): kernel VSX */ -#undef vector /* igiene: si usano __vector/__bool espliciti */ -#undef pixel -#undef bool +/* output dello shared expert gia' calcolato su GPU (solo Metal layer-CB) */ +static const float *g_pre_sh; #endif +/* routing precalcolata dalla GPU (Metal layer CB o device router CUDA, #431): + * moe() la usa e salta la FASE A. NULL = router su CPU. */ +static const int *g_pre_idx; static const float *g_pre_w; static const int *g_pre_keff; #ifdef __APPLE__ #include /* host_statistics64: MemAvailable di macOS */ #endif @@ -107,7 +94,16 @@ typedef struct { * fmt=1 INT8 -> q8 (1 byte/param) + scala per riga * fmt=2 INT4 -> q4 (2 valori per byte, impacchettati) + scala per riga * INT4 e' cio' che fa stare la densa residente nei 15 GB (0.5 byte/param). */ -/* fmt: 0 F32, 1 INT8, 2 INT4 (2/byte), 3 INT2 (4/byte). q4 ospita sia int4 che int2 packed. */ +/* fmt: 0 F32, 1 INT8, 2 INT4 (2/byte), 3 INT2 (4/byte), 4 INT4-GROUPED, 5 INT3-G64. + * q4 ospita int4/int2/int3 packed. fmt=4 (grouped int4, #242): per-row nibbles + one f32 + * scale per group of `gs` inputs (s has O*ceil(I/gs) entries). + * fmt=6 (E8/IQ3 lattice, #452): 98B per 256 weights = 3.0625 bits/weight, grid + * indices + parity-packed signs + sub-scales + fp16 super-scale, ALL inside q4 — + * `s` is unused for this format (see quant.h E8_* and tools/iq3_pack.py). + * fmt=5 (int3, per-GROUP scales, group=64, see quant.h I3_*): values in [-4,3] stored per + * 64-input group as 24 bytes = 16B low plane (2 bits/val, int2 layout) + 8B high plane + * (1 bit/val), plus ONE f32 scale PER GROUP (s has O*ceil(I/64) entries, not O). 3.5 + * bits/weight effective — the quality/size sweet spot measured in the #132 ablation. */ typedef struct { int fmt; float *qf; int8_t *q8; uint8_t *q4; float *s; int O, I, gs; /* gs=group size (0=per-row, 128=grouped) */ #ifdef COLI_CUDA @@ -123,6 +119,12 @@ static int64_t qt_bytes(const QT *t){ /* byte residenti del tensore */ if(t->fmt==4){ /* int4 grouped: packed nibbles + O*ceil(I/gs) scales */ int ng=(t->I+t->gs-1)/t->gs; return (int64_t)t->O*((t->I+1)/2) + (int64_t)t->O*ng*4; } + if(t->fmt==5){ /* int3-g64: 24B/group weights + one f32 scale per group (I3_* in quant.h, + * included below — keep the arithmetic literal here) */ + int64_t ng=((int64_t)t->I+63)/64; + return (int64_t)t->O*ng*24 + (int64_t)t->O*ng*4; } + if(t->fmt==6) /* E8/IQ3: 98B per 256 weights, scales in-block, .qs is a 4-byte tag */ + return (int64_t)t->O*(((int64_t)t->I+255)/256)*98 + 4; return (int64_t)t->O*((t->I+1)/2) + (int64_t)t->O*4; /* fmt=2 int4 per-row */ } @@ -140,6 +142,10 @@ typedef struct { QT gate_proj, up_proj, down_proj; /* moe (sparse==1) */ float *router, *router_bias; /* router f32 (sensibile) */ +#ifdef COLI_CUDA + void *router_cuda, *router_bias_cuda; /* device router (#431 PR-A), lazy-uploaded */ + int router_cuda_bad; /* upload failed once: stay on the CPU router */ +#endif QT sh_gate, sh_up, sh_down; /* shared expert */ } Layer; @@ -148,7 +154,12 @@ typedef struct { * slab_cap/fslab_cap: capienza allocata — gli slot ws[] sono riusati TRA layer e gli * expert non hanno tutti la stessa taglia (layer MTP int8 = 2x i layer int4). */ typedef struct { int eid; QT g,u,d; uint8_t *slab; float *fslab; - int64_t slab_cap, fslab_cap; uint64_t used; } ESlot; + int64_t slab_cap, fslab_cap; uint64_t used; + /* pin-arena backing (#419): when set, slab/fslab are interior + * slices of a per-layer arena and must never be free()d — + * expert_host_release detaches them, expert_host_ensure + * re-attaches. NULL for every individually-allocated slot. */ + uint8_t *aslab; float *afslab; } ESlot; typedef struct { float **Lc, **Rc, **Ic; @@ -178,11 +189,34 @@ typedef struct { KVState *kv; ESlot **ecache; int *ecn; int ecap; /* LRU expert per-layer */ float **kv_dev_L, **kv_dev_R; int *kv_dev_valid; /* ombra KV su device (decode) */ + float **ln_dev; /* in_ln/post_ln cached on device: [layer*2+{0,1}] (Inc.4) */ ESlot ws[64]; /* working set del layer corrente (load paralleli) */ ESlot **pin; int *npin; /* HOT-STORE: expert pinnati in RAM (mai evicted) */ uint32_t **eusage; /* contatori persistenti (per STATS/PIN) */ uint32_t **eheat; /* calore recente per promotion/demotion live */ uint32_t **elast, eaccess_clock; /* recency per LFRU session-local */ + /* DISK-CLASS: PRIVATE recency state, read only by expert_classify(). Private -- + * not the real elast/eaccess_clock -- kept fully separate so DISK-CLASS's bookkeeping + * can never read from or write into stock eviction state: every DISK-CLASS write lives + * inside its own need_classify/dc_on gate, so "byte-identical with PROF=0" is provable + * by construction instead of by argument. (Historical note: when this was first written, + * the Metal pre-routed FASE A path (g_pre_idx) never bumped the real elast/eaccess_clock + * -- on Metal decode the real clock froze at end of prefill, so REPIN's LRU tie-breaker + * ran on stale recency for the rest of the run. That was an upstream defect; it has since + * been reported and fixed (#417, cfcc742) -- FASE A now bumps the real clock too. The + * private clock is retained anyway: separation from stock state is the stronger property, + * independent of whether the real clock is correct.) elast_dc/eaccess_clock_dc tick in + * BOTH FASE A paths, under the same need_classify gate, at the same rate the real clock + * ticks on the CPU path (one per selected (position,expert)) -- so the + * COLI_DISKCLASS_WINDOW window keeps its meaning in every mode. elast_pre snapshots + * elast_dc just BEFORE this call's own bump (see the touched[] guard in FASE A) -- + * classifying against the live array would read the bump routing just made a few lines + * above the load that needed it, so a giant cold prefill burst would score every expert + * "just accessed" and get called warm. Recency alone (not eheat's access COUNT): a count + * never decays, so an expert hot early in a long session would keep reading "warm" long + * after it dropped out of the working set. Same shape/allocation as elast; NULL for dense + * layers. */ + uint32_t **elast_dc, **elast_pre, eaccess_clock_dc; /* DSA lightning indexer (attivo solo se i pesi out-idx-* sono presenti) */ int has_dsa; QT *ix_wq, *ix_wk, *ix_wp; /* per layer FULL: wq_b, wk, weights_proj */ @@ -218,13 +252,29 @@ typedef struct { uint64_t bytes_mtp, bytes_main; /* byte letti da disco per tipo layer */ } Model; -static void usage_save(Model *m); /* cache che impara: definita accanto a stats_dump */ -static void tiers_emit(Model *m); -static void ehit_mark(Model *m, int layer, int eid); -static void emap_emit(Model *m); -static void hits_emit(Model *m); -static void hwinfo_emit(Model *m); -static int64_t expert_bytes_probe(Model *m, int ebits); /* PROF: tier sizes in the report */ +#include "quant.h" +static int g_no_fused_pair=0; +static int g_spec_pin=1; +static int g_spec_live=0; +static inline int spec_pinned(void){ return g_spec_pin && g_spec_live; } + +static void matmul_qt_ex(float *y, const float *x, QT *w, int S, int allow_idot); +static void matmul_qt(float *y, const float *x, QT *w, int S){ matmul_qt_ex(y,x,w,S,1); } + +/* fmt=4 fused gate+up (defined later, after the quant kernels) */ +static void matmul_i4_grouped_pair(float *yg, float *yu, const float *x, + const uint8_t *qg, const float *sg, + const uint8_t *qu, const float *su, + int S, int I, int O, int gs); + +static void expert_gate_up(float *g,float *u,const float *x,QT *wg,QT *wu,int S){ + if(!g_no_fused_pair&&!spec_pinned()&&S==1&&wg->fmt==2&&wu->fmt==2&&wg->I==wu->I&&wg->O==wu->O) + matmul_i4_pair(g,u,x,wg->q4,wg->s,wu->q4,wu->s,wg->I,wg->O); + else if(!g_no_fused_pair&&S==1&&wg->fmt==4&&wu->fmt==4&&wg->I==wu->I&&wg->O==wu->O&&wg->gs==wu->gs) + matmul_i4_grouped_pair(g,u,x,wg->q4,wg->s,wu->q4,wu->s,S,wg->I,wg->O,wg->gs); + else { matmul_qt(g,x,wg,S); matmul_qt(u,x,wu,S); } +} + static int g_repin; static uint64_t g_last_repin; #ifdef COLI_CUDA @@ -233,6 +283,7 @@ static double g_cuda_expert_gb; static int g_cuda_expert_auto; static int g_cuda_dense; static int g_cuda_release_host; +static double g_cuda_reserve_gb; /* CUDA_RESERVE_GB: VRAM headroom kept free of expert tier (default 2 GB) */ static int g_cuda_devices[COLI_CUDA_MAX_DEVICES], g_cuda_ndev, g_cuda_rr; static int64_t g_cuda_dense_projected[COLI_CUDA_MAX_DEVICES]; static void qt_cuda_reset(QT *t){ @@ -240,8 +291,14 @@ static void qt_cuda_reset(QT *t){ t->cuda_failed=0; } static int qt_cuda_upload(QT *t){ + if(t->fmt==5||t->fmt==6) return 0; /* int3-g64 / E8: no CUDA kernel yet — tensor stays CPU-side */ const void *weights = t->fmt==0 ? (const void*)t->qf : t->fmt==1 ? (const void*)t->q8 : (const void*)t->q4; + if(t->fmt==4) /* grouped int4 (#334): scales are [O, ceil(I/gs)] — the plain + * upload would truncate them to O floats and the group kernels + * would read garbage. An old DLL without the _g symbol returns 0 + * and the tensor simply stays CPU-side. */ + return coli_cuda_tensor_upload_g(&t->cuda,weights,t->s,t->fmt,t->I,t->O,t->cuda_device,t->gs); return coli_cuda_tensor_upload(&t->cuda,weights,t->s,t->fmt,t->I,t->O,t->cuda_device); } static int qt_cuda_update(QT *t){ @@ -249,6 +306,7 @@ static int qt_cuda_update(QT *t){ t->fmt==1?(const void*)t->q8:(const void*)t->q4; return coli_cuda_tensor_update(t->cuda,weights,t->s); } +static double g_ovl_issue,g_ovl_cpu,g_ovl_take,g_ovl_mark; /* Inc.4 overlap-window split (OVL report) */ static void cuda_stats_print(void){ size_t n=0,b=0; coli_cuda_stats(-1,&n,&b); fprintf(stderr,"[CUDA] resident set: %zu tensors, %.2f GB VRAM\n",n,b/1e9); @@ -264,6 +322,9 @@ static void cuda_stats_print(void){ getenv("COLI_CUDA_PROFILE")?"; timing sotto":""); if(calls&&getenv("COLI_CUDA_PROFILE")) fprintf(stderr, "[CUDA] expert groups timing: H2D %.1f ms | kernel %.1f ms | D2H %.1f ms\n",h2d,kernel,d2h); + if(g_ovl_issue+g_ovl_cpu+g_ovl_take>0) fprintf(stderr, + "[CUDA] overlap window: pack+issue %.2fs | cpu-rows %.2fs | take(sync+acc) %.2fs\n", + g_ovl_issue,g_ovl_cpu,g_ovl_take); } static int parse_cuda_devices(const char *list, int *out){ if(!list||!*list) return 0; @@ -305,6 +366,65 @@ static _Atomic int64_t g_prof_io; /* bytes pread()/faulted from e * wait ~ service means the loads block the compute thread. */ static _Atomic int64_t g_edisk_ns; static double edisk_s(void){ return atomic_load_explicit(&g_edisk_ns,memory_order_relaxed)*1e-9; } +/* DISK-CLASS (PROF=1): per-load cold/warm classification against the engine's own + * recency state. Instrumentation only -- it never changes which fd serves a read (see + * expert_classify() and its call site in expert_load_impl; the fd choice expression is + * untouched by this feature). COLI_DISKCLASS_WINDOW is the recency window in ticks of the + * PRIVATE clock (m->eaccess_clock_dc -- NOT the real eaccess_clock; see elast_dc in Model + * for why DISK-CLASS keeps its own clock instead of reading the real one): one tick per + * selected (position,expert) in FASE A while classification is active, the same per-token + * rate the real clock has on the CPU path, so the window's meaning is unchanged. At or + * under the window = warm; 0 (default, unset) derives it from topk*n_layers*8 once the + * model config is known (main(), right after model_init) -- roughly "seen in the last ~8 + * tokens", generous on purpose (conservative-toward-warm: a load the page cache could have + * served that gets labeled cold overstates the cold class, the bucket this line exists to + * size). */ +static uint32_t g_direct_heat_ticks=0; +static int g_direct_heat_explicit=0; /* 1 if COLI_DISKCLASS_WINDOW was set (skip the auto-derive) */ +#define DC_COLD 0 +#define DC_WARM 1 +static _Atomic uint64_t g_dc_n[2]; /* [DC_COLD]/[DC_WARM]: loads classified */ +static _Atomic int64_t g_dc_bytes[2]; /* bytes read (weights + scales, matches g_prof_io) */ +static _Atomic int64_t g_dc_ns[2]; /* wall ns spent reading (thread-seconds, like g_edisk_ns) */ +static _Atomic uint64_t g_dc_direct_n[2]; /* subset of the above ACTUALLY served by the uncached fd */ +/* Busy-wall per class + combined: how much WALL time had >=1 classified load of the + * class in flight (thread-seconds / busy-wall = average concurrency; bytes / busy-wall + * = aggregate GB/s the disk actually delivered for that class -- the quantity the + * thread-second numbers alone can't answer: N slow overlapped reads can beat N fast + * serial ones in aggregate, and only wall-denominated rates see it). Transition scheme: + * 0->1 records a start, 1->0 accumulates (now - start). One dedicated mutex serializes + * the transition bookkeeping -- two short lock/unlock pairs per load against ms-scale + * reads; a CAS scheme would save nothing measurable and be harder to audit + * (correctness over cleverness). Only COMPLETED intervals are in the accumulators: an + * interval still open at report time is not counted (bounded by one read's duration -- + * noise at report granularity). */ +static pthread_mutex_t g_dc_wall_mx=PTHREAD_MUTEX_INITIALIZER; +static int g_dc_inflight[2], g_dc_inflight_all; /* guarded by g_dc_wall_mx */ +static double g_dc_wall_open[2], g_dc_wall_open_all; /* start of the open interval */ +static int64_t g_dc_wall_ns[2], g_dc_wall_all_ns; /* completed busy-wall ns */ +static void dc_wall_enter(int cls, double now){ + pthread_mutex_lock(&g_dc_wall_mx); + if(g_dc_inflight[cls]++==0) g_dc_wall_open[cls]=now; + if(g_dc_inflight_all++==0) g_dc_wall_open_all=now; + pthread_mutex_unlock(&g_dc_wall_mx); +} +static void dc_wall_exit(int cls, double now){ + pthread_mutex_lock(&g_dc_wall_mx); + if(--g_dc_inflight[cls]==0) g_dc_wall_ns[cls]+=(int64_t)((now-g_dc_wall_open[cls])*1e9); + if(--g_dc_inflight_all==0) g_dc_wall_all_ns +=(int64_t)((now-g_dc_wall_open_all)*1e9); + pthread_mutex_unlock(&g_dc_wall_mx); +} +static int dc_needed(void); /* fwd: defined with the classifier (needs g_prof) */ +static void dc_wall_read(int64_t out[2], int64_t *all){ /* mutex-consistent snapshot for the report */ + if(!dc_needed()){ out[0]=out[1]=0; *all=0; return; } /* off for the whole process => accumulators are + * provably zero (only dc_wall_exit writes them, + * only under dc_on): skip the lock, zero work. + * Needed because prof_base runs unconditionally + * at some call sites ("cheap enough to always"). */ + pthread_mutex_lock(&g_dc_wall_mx); + out[0]=g_dc_wall_ns[0]; out[1]=g_dc_wall_ns[1]; *all=g_dc_wall_all_ns; + pthread_mutex_unlock(&g_dc_wall_mx); +} #define PROF_LAT_CAP 32768 static double g_prof_lat[PROF_LAT_CAP]; /* per-forward decode wall clock (ring) */ static uint64_t g_prof_nlat; /* forwards recorded (monotonic) */ @@ -314,6 +434,8 @@ typedef struct { double edisk,ewait,emm,ecpu,egpu,route,p2p,attn,head; int64_t io,cpu_bytes; uint64_t hits,miss,ereq,n_fw,n_emit,nlat,n_p2p,cpu_rows; uint64_t hit_pin,hit_ecache; + uint64_t dc_n[2], dc_direct_n[2]; int64_t dc_bytes[2], dc_ns[2]; /* DISK-CLASS */ + int64_t dc_wall_ns[2], dc_wall_all_ns; /* busy-wall (per class + combined) */ } ProfBase; static void prof_base(Model *m, ProfBase *b){ b->edisk=edisk_s(); b->ewait=m->t_ewait; b->emm=m->t_emm; @@ -324,6 +446,13 @@ static void prof_base(Model *m, ProfBase *b){ b->hit_pin=m->hit_pin; b->hit_ecache=m->hit_ecache; b->n_fw=m->n_fw; b->n_emit=m->n_emit; b->nlat=g_prof_nlat; b->n_p2p=m->n_p2p; b->cpu_bytes=m->cpu_expert_bytes;b->cpu_rows=m->cpu_expert_rows; + for(int i=0;i<2;i++){ + b->dc_n[i]=atomic_load_explicit(&g_dc_n[i],memory_order_relaxed); + b->dc_bytes[i]=atomic_load_explicit(&g_dc_bytes[i],memory_order_relaxed); + b->dc_ns[i]=atomic_load_explicit(&g_dc_ns[i],memory_order_relaxed); + b->dc_direct_n[i]=atomic_load_explicit(&g_dc_direct_n[i],memory_order_relaxed); + } + dc_wall_read(b->dc_wall_ns,&b->dc_wall_all_ns); } static float *falloc(int64_t n){ @@ -332,687 +461,62 @@ static float *falloc(int64_t n){ if(n<0 || (uint64_t)n > SIZE_MAX/sizeof(float)){ fprintf(stderr,"falloc: n=%lld is out of range\n",(long long)n); exit(1); } float *p=malloc((size_t)n*sizeof(float)); if(!p){fprintf(stderr,"OOM\n");exit(1);} return p; } -/* ---- Accumulatore int4->float a 512 bit / 512-bit int4->float accumulator ---- - * Stessa matematica lossless di matmul_i4 (nibble->f32, FMA), ma 32 pesi/iter su - * due catene FMA indipendenti. NON bit-identico al vecchio ordine: la riduzione - * ad albero accumula MENO errore della somma sequenziale (misurato 2-4x più - * vicino all'oracolo double sulle forme reali; perplexity invariata, +4-7% sul - * decode con routing CPU-heavy — vedi docs/experiments/glm52-6x5090-2026-07-12.md). - * EN: same lossless math as matmul_i4, 32 weights/iter on two independent FMA - * chains. Not bit-identical to the old order: tree reduction accumulates LESS - * rounding than sequential summation. I4_ACC512=0 restores the old order (A/B). */ -#if defined(__AVX512F__) && defined(__AVX512BW__) -static int g_i4_acc512=1; -static inline float dot_i4f_avx512(const uint8_t *w,const float *x,int I){ - const __m128i m4=_mm_set1_epi8(0x0F); const __m512i b8=_mm512_set1_epi32(8); - __m512 acc0=_mm512_setzero_ps(),acc1=_mm512_setzero_ps(); int i=0; - for(;i+32<=I;i+=32){ __m128i by=_mm_loadu_si128((const __m128i*)(w+(i>>1))); - __m128i lo=_mm_and_si128(by,m4),hi=_mm_and_si128(_mm_srli_epi16(by,4),m4); - __m128i n0=_mm_unpacklo_epi8(lo,hi),n1=_mm_unpackhi_epi8(lo,hi); - __m512 w0=_mm512_cvtepi32_ps(_mm512_sub_epi32(_mm512_cvtepu8_epi32(n0),b8)); - __m512 w1=_mm512_cvtepi32_ps(_mm512_sub_epi32(_mm512_cvtepu8_epi32(n1),b8)); - acc0=_mm512_fmadd_ps(_mm512_loadu_ps(x+i),w0,acc0); - acc1=_mm512_fmadd_ps(_mm512_loadu_ps(x+i+16),w1,acc1); - } - return _mm512_reduce_add_ps(_mm512_add_ps(acc0,acc1)); -} -/* selftest contro il riferimento scalare (I4_ACC512_TEST=1): copre l'ordine dei - * nibble e ogni multiplo di 32. / selftest vs the scalar reference. */ -static int i4_acc512_selftest(void){ - enum { N=224 }; uint8_t w[(N+1)/2]; float x[N]; - for(int i=0;i>1]=(uint8_t)(q+8); - else w[i>>1]|=(uint8_t)((q+8)<<4); - x[i]=(float)(((i*29+7)%101)-50)/37.f; - } - for(int n=32;n<=N;n+=32){ - float ref=0; for(int i=0;i>1]>>((i&1)*4))&15)-8); - float got=dot_i4f_avx512(w,x,n),tol=2e-5f*(1.f+fabsf(ref)); - if(fabsf(got-ref)>tol){ fprintf(stderr,"AVX512 i4 selftest n=%d: %.9g != %.9g\n",n,got,ref); return 0; } - } - return 1; -} -#endif -/* y[S,O] = x[S,I] @ W^T, W[O,I] f32 */ -static void matmul(float *y, const float *x, const float *W, int S, int I, int O){ - #pragma omp parallel for schedule(static) - for (int o=0;o>1))); /* 8 byte=16 nibble */ - __m128i lo=_mm_and_si128(by,m4), hi=_mm_and_si128(_mm_srli_epi16(by,4),m4); - __m128i nib=_mm_unpacklo_epi8(lo,hi); /* nibble in ordine */ - __m256 w0=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(nib),b8)); - __m256 w1=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(nib,8)),b8)); - acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i), w0, acc); - acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i+8), w1, acc); } - a=hsum256(acc); -#elif defined(__ARM_NEON) - const uint8x8_t m4=vdup_n_u8(0x0F); const int8x8_t b8=vdup_n_s8(8); - float32x4_t ac0=vdupq_n_f32(0), ac1=vdupq_n_f32(0); - for(;i+16<=I;i+=16){ uint8x8_t by=vld1_u8(w+(i>>1)); /* 8 byte=16 nibble */ - uint8x8x2_t z=vzip_u8(vand_u8(by,m4), vshr_n_u8(by,4)); /* nibble in ordine */ - int16x8_t w0=vmovl_s8(vsub_s8(vreinterpret_s8_u8(z.val[0]),b8)); - int16x8_t w1=vmovl_s8(vsub_s8(vreinterpret_s8_u8(z.val[1]),b8)); - ac0=vfmaq_f32(ac0, vld1q_f32(xs+i), vcvtq_f32_s32(vmovl_s16(vget_low_s16(w0)))); - ac1=vfmaq_f32(ac1, vld1q_f32(xs+i+4), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w0)))); - ac0=vfmaq_f32(ac0, vld1q_f32(xs+i+8), vcvtq_f32_s32(vmovl_s16(vget_low_s16(w1)))); - ac1=vfmaq_f32(ac1, vld1q_f32(xs+i+12), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w1)))); } - a=vaddvq_f32(vaddq_f32(ac0,ac1)); -#endif -#if defined(__AVX512F__) && defined(__AVX512BW__) - } -#endif - for(;i+1>1]; int lo=(int)(byte&0xF)-8, hi=(int)(byte>>4)-8; - a += xs[i]*(float)lo + xs[i+1]*(float)hi; } - if(i>1]; int lo=(int)(byte&0xF)-8; a += xs[i]*(float)lo; } - y[(int64_t)s*O+o]=a*sc; } } -} -/* y[S,O] = x[S,I] @ W^T with W int4 packed (2/byte) + per-GROUP scales (fmt=4). - * Same nibble math as matmul_i4, but the scale changes every `gs` elements along I. - * The accumulator resets at each group boundary: dot(x[grp], w[grp]) * scale[grp]. - * gs MUST be a multiple of 16 (the AVX2 vector width). */ -static void matmul_i4_grouped(float *y, const float *x, const uint8_t *q4, const float *scale, - int S, int I, int O, int gs){ + +/* Fused gate+up for grouped int4 (fmt=4): computes both yg[S,O] and yu[S,O] from + * the same x[S,I], reading x once instead of twice — saves ~33% of expert-matmul time at decode. + * The per-group scale logic matches matmul_i4_grouped exactly. */ +static void matmul_i4_grouped_pair(float *yg, float *yu, const float *x, + const uint8_t *qg, const float *sg, + const uint8_t *qu, const float *su, + int S, int I, int O, int gs){ int rb=(I+1)/2; int ng=(I+gs-1)/gs; #pragma omp parallel for schedule(static) for(int o=0;oI) glen=I-base; - float sc=scl[g]; + float scg=sgl[g], scu=sul[g]; int i=base; #ifdef __AVX2__ const __m128i m4=_mm_set1_epi8(0x0F); const __m256i b8=_mm256_set1_epi32(8); - __m256 acc=_mm256_setzero_ps(); - for(; i+16<=base+glen; i+=16){ __m128i by=_mm_loadl_epi64((const __m128i*)(w+(i>>1))); - __m128i lo=_mm_and_si128(by,m4),hi=_mm_and_si128(_mm_srli_epi16(by,4),m4); - __m128i nib=_mm_unpacklo_epi8(lo,hi); - __m256 w0=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(nib),b8)); - __m256 w1=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(nib,8)),b8)); - acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i), w0, acc); - acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i+8), w1, acc); } - a+=hsum256(acc)*sc; -#endif - /* scalar tail for the group remainder */ - for(; i>1]; - a+=(xs[i]*(float)((int)(byte&0xF)-8)+xs[i+1]*(float)((int)(byte>>4)-8))*sc; } - else { uint8_t byte=w[i>>1]; a+=xs[i]*(float)((int)(byte&0xF)-8)*sc; } + __m256 accg=_mm256_setzero_ps(), accu=_mm256_setzero_ps(); + for(; i+16<=base+glen; i+=16){ + __m128i byg=_mm_loadl_epi64((const __m128i*)(wg+(i>>1))); + __m128i log=_mm_and_si128(byg,m4),hig=_mm_and_si128(_mm_srli_epi16(byg,4),m4); + __m128i nibg=_mm_unpacklo_epi8(log,hig); + __m256 w0g=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(nibg),b8)); + __m256 w1g=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(nibg,8)),b8)); + accg=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i), w0g, accg); + accg=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i+8), w1g, accg); + __m128i byu=_mm_loadl_epi64((const __m128i*)(wu2+(i>>1))); + __m128i lou=_mm_and_si128(byu,m4),hiu=_mm_and_si128(_mm_srli_epi16(byu,4),m4); + __m128i nibu=_mm_unpacklo_epi8(lou,hiu); + __m256 w0u=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(nibu),b8)); + __m256 w1u=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(nibu,8)),b8)); + accu=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i), w0u, accu); + accu=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i+8), w1u, accu); } - } - y[(int64_t)s*O+o]=a; - } - } -} -/* Decode hot path for gate+up: same exact q4 dot products as matmul_i4, but one - * OpenMP dispatch covers both matrices. KTransformers uses persistent pools; - * this keeps colibri dependency-free while removing one team launch/expert. */ -static void matmul_i4_pair(float *yg, float *yu, const float *x, - const uint8_t *qg, const float *sg, - const uint8_t *qu, const float *su, int I, int O){ - int rb=(I+1)/2; - #pragma omp parallel for schedule(static) - for(int z=0;z<2*O;z++){ - int o=z>1))); - __m128i lo=_mm_and_si128(by,m4),hi=_mm_and_si128(_mm_srli_epi16(by,4),m4); - __m128i nib=_mm_unpacklo_epi8(lo,hi); - __m256 w0=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(nib),b8)); - __m256 w1=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(nib,8)),b8)); - acc=_mm256_fmadd_ps(_mm256_loadu_ps(x+i),w0,acc); - acc=_mm256_fmadd_ps(_mm256_loadu_ps(x+i+8),w1,acc); } - a=hsum256(acc); -#elif defined(__ARM_NEON) - const uint8x8_t m4=vdup_n_u8(0x0F); const int8x8_t b8=vdup_n_s8(8); - float32x4_t ac0=vdupq_n_f32(0),ac1=vdupq_n_f32(0); - for(;i+16<=I;i+=16){ uint8x8_t by=vld1_u8(w+(i>>1)); - uint8x8x2_t n=vzip_u8(vand_u8(by,m4),vshr_n_u8(by,4)); - int16x8_t w0=vmovl_s8(vsub_s8(vreinterpret_s8_u8(n.val[0]),b8)); - int16x8_t w1=vmovl_s8(vsub_s8(vreinterpret_s8_u8(n.val[1]),b8)); - ac0=vfmaq_f32(ac0,vld1q_f32(x+i),vcvtq_f32_s32(vmovl_s16(vget_low_s16(w0)))); - ac1=vfmaq_f32(ac1,vld1q_f32(x+i+4),vcvtq_f32_s32(vmovl_s16(vget_high_s16(w0)))); - ac0=vfmaq_f32(ac0,vld1q_f32(x+i+8),vcvtq_f32_s32(vmovl_s16(vget_low_s16(w1)))); - ac1=vfmaq_f32(ac1,vld1q_f32(x+i+12),vcvtq_f32_s32(vmovl_s16(vget_high_s16(w1)))); } - a=vaddvq_f32(vaddq_f32(ac0,ac1)); -#endif -#if defined(__AVX512F__) && defined(__AVX512BW__) - } -#endif - for(;i+1>1]; a+=x[i]*(float)((b&15)-8)+x[i+1]*(float)((b>>4)-8); } - if(i>1]&15)-8); - (z=2) non calcolano la STESSA funzione. Tre interruttori dipendono da S: il gate - * int4-IDOT (S>=g_i4s — asimmetrico proprio dove g_i4s>1), la fusione gate+up solo-S==1, - * e la soglia righe del GEMM Metal. Con SPEC_PIN=1 (default) ogni forward emesso mentre - * i draft del modello sono attivi resta sulla famiglia di kernel di S=1: draft e verifica - * coincidono per costruzione. Prefill e decode non speculativo sono intoccati. - * EN: MTP acceptance collapses when the draft (S=1) and verify (S>=2) forwards do not - * compute the SAME function. Three switches are S-dependent: the int4 IDOT gate - * (S>=g_i4s — asymmetric exactly on ISAs where g_i4s>1), the S==1-only gate+up fusion, - * and the Metal GEMM row threshold. SPEC_PIN=1 (default) pins every forward issued - * while model drafts are live to the platform's S=1 kernel family, so draft and verify - * agree by construction; prefill and non-speculative decode are untouched. - * SPEC_PIN=0 restores the S-dependent gates (A/B). */ -static int g_spec_pin=1; -static int g_spec_live=0; /* set by spec_decode while drafts are live */ -static inline int spec_pinned(void){ return g_spec_pin && g_spec_live; } -static void expert_gate_up(float *g,float *u,const float *x,QT *wg,QT *wu,int S){ - if(!g_no_fused_pair&&!spec_pinned()&&S==1&&wg->fmt==2&&wu->fmt==2&&wg->I==wu->I&&wg->O==wu->O) - matmul_i4_pair(g,u,x,wg->q4,wg->s,wu->q4,wu->s,wg->I,wg->O); - else { matmul_qt(g,x,wg,S); matmul_qt(u,x,wu,S); } -} -/* y[S,O] = x[S,I] @ W^T con W int2 impacchettato (4 valori/byte) + scala[O]. nibble 2-bit -> [-2,1]. */ -static void matmul_i2(float *y, const float *x, const uint8_t *q2, const float *scale, int S, int I, int O){ - int rb=(I+3)/4; - #pragma omp parallel for schedule(static) - for (int o=0;o>2))); /* 4 byte=16 valori */ - __m128i p0=_mm_and_si128(by,m2), p1=_mm_and_si128(_mm_srli_epi16(by,2),m2); - __m128i p2=_mm_and_si128(_mm_srli_epi16(by,4),m2), p3=_mm_and_si128(_mm_srli_epi16(by,6),m2); - __m128i lo=_mm_unpacklo_epi8(p0,p1), hi=_mm_unpacklo_epi8(p2,p3); - __m128i nib=_mm_unpacklo_epi16(lo,hi); /* 16 valori in ordine */ - __m256 w0=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(nib),b2)); - __m256 w1=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(nib,8)),b2)); - acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i), w0, acc); - acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i+8), w1, acc); } - a=hsum256(acc); -#elif defined(__ARM_NEON) - const uint8x8_t m2v=vdup_n_u8(3); const int8x8_t b2v=vdup_n_s8(2); - float32x4_t ac0=vdupq_n_f32(0), ac1=vdupq_n_f32(0); - for(;i+16<=I;i+=16){ uint32_t wd; memcpy(&wd, w+(i>>2), 4); /* 4 byte=16 valori */ - uint8x8_t by=vreinterpret_u8_u32(vdup_n_u32(wd)); - uint8x8x2_t z01=vzip_u8(vand_u8(by,m2v), vand_u8(vshr_n_u8(by,2),m2v)); - uint8x8x2_t z23=vzip_u8(vand_u8(vshr_n_u8(by,4),m2v), vshr_n_u8(by,6)); - uint16x4x2_t zz=vzip_u16(vreinterpret_u16_u8(z01.val[0]), vreinterpret_u16_u8(z23.val[0])); - int16x8_t w0=vmovl_s8(vsub_s8(vreinterpret_s8_u16(zz.val[0]),b2v)); /* 16 valori in ordine */ - int16x8_t w1=vmovl_s8(vsub_s8(vreinterpret_s8_u16(zz.val[1]),b2v)); - ac0=vfmaq_f32(ac0, vld1q_f32(xs+i), vcvtq_f32_s32(vmovl_s16(vget_low_s16(w0)))); - ac1=vfmaq_f32(ac1, vld1q_f32(xs+i+4), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w0)))); - ac0=vfmaq_f32(ac0, vld1q_f32(xs+i+8), vcvtq_f32_s32(vmovl_s16(vget_low_s16(w1)))); - ac1=vfmaq_f32(ac1, vld1q_f32(xs+i+12), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w1)))); } - a=vaddvq_f32(vaddq_f32(ac0,ac1)); -#endif - for(;i>2]; int sh=(i&3)*2; a += xs[i]*(float)((int)((byte>>sh)&3)-2); } - y[(int64_t)s*O+o]=a*sc; } } -} -/* ---- KERNEL INTERI (IDOT): attivazioni quantizzate a int8 per riga (absmax/127, - * stile Q8_0), prodotto scalare INTERO via maddubs/madd AVX2 — niente conversione - * f32 dei pesi nel ciclo caldo. ~2-3x sui matmul quantizzati; errore aggiunto ~0.3% - * RMS per matmul (attivazione int8), IDOT=0 torna al percorso f32 esatto. */ -#if defined(__AVX512VNNI__) && defined(__AVX512BW__) -#define IDOT_KERNEL "avx512-vnni" -#elif defined(__AVXVNNI__) && defined(__AVX2__) -#define IDOT_KERNEL "avx-vnni" -#elif defined(__AVX2__) -#define IDOT_KERNEL "avx2" -#elif defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8) -#define IDOT_KERNEL "neon-i8mm" -#elif defined(__ARM_NEON) -#define IDOT_KERNEL "neon" -#elif defined(__VSX__) -#define IDOT_KERNEL "vsx" -#else -#define IDOT_KERNEL "scalar" -#endif -static int g_idot=1; -#if defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD) -static int g_i4s=1; /* SDOT presente: int4 IDOT conviene anche a S=1 (decode). Misurato - * su Apple M-series: +14%%, expert-matmul -16%%. EN: with SDOT, int4 - * IDOT pays even at S=1 (decode); measured on Apple M-series. */ -#elif defined(__VSX__) -static int g_i4s=1; /* POWER8 vec_msum: qui il fallback f32 e' SCALARE, quindi l'IDOT - * int4 conviene anche a S=1. Misurato su POWER8 S824 (vedi PR). - * EN: on VSX the f32 fallback is plain scalar C, so int4 IDOT - * pays even at S=1. Measured on a POWER8 S824 (see PR). */ -#else -static int g_i4s=2; /* senza SDOT / altrove: soglia originale (misura AVX2 dell'autore). - * EN: without SDOT / elsewhere: original threshold (author's AVX2). */ -#endif -static inline float qrow_i8(const float *x, int8_t *q, int I){ - float amax=0; for(int i=0;iamax)amax=a; } - float s=amax/127.f; if(s<1e-12f) s=1e-12f; float inv=1.f/s; - for(int i=0;i s32 directly, 64 bytes/iter, no 16-bit intermediate. - * AVX-512 has no vpsignb: |w| via abs, sign folded into x with a mask-negate - * (w==0 -> product 0 either way). |x|<=127 (qrow_i8), |w|<=128 as u8: each - * s32 lane adds <= 4*128*127, safe up to I=16384 like the AVX2 bound. */ - __m512i acc=_mm512_setzero_si512(); - for(;i+64<=I;i+=64){ - __m512i wv=_mm512_loadu_si512((const void*)(w+i)); - __m512i xv=_mm512_loadu_si512((const void*)(x+i)); - __mmask64 neg=_mm512_movepi8_mask(wv); - __m512i xs=_mm512_mask_sub_epi8(xv,neg,_mm512_setzero_si512(),xv); - acc=_mm512_dpbusd_epi32(acc,_mm512_abs_epi8(wv),xs); - } - sum=_mm512_reduce_add_epi32(acc); -#elif defined(__AVXVNNI__) && defined(__AVX2__) - /* AVX-VNNI 128-bit: vpdpbusd u8*s8 -> s32, 16 byte/iter. Stesso trucco del - * segno della variante 512-bit: |w| via abs, segno piegato in x con maschera - * (w==0 -> product 0). __AVX2__ serve per _mm_sign_epi8 / abs. */ - __m128i acc=_mm_setzero_si128(); - for(;i+16<=I;i+=16){ - __m128i wv=_mm_loadu_si128((const __m128i*)(w+i)); - __m128i xv=_mm_loadu_si128((const __m128i*)(x+i)); - __m128i xs=_mm_sign_epi8(xv,wv); /* x * sign(w); _mm_sign zona __AVX2__ */ - acc=_mm_dpbusd_epi32(acc,_mm_abs_epi8(wv),xs); - } - sum=hsum128_i32(acc); -#elif defined(__AVX2__) - __m256i acc=_mm256_setzero_si256(); const __m256i ones=_mm256_set1_epi16(1); - for(;i+32<=I;i+=32){ - __m256i wv=_mm256_loadu_si256((const __m256i*)(w+i)); - __m256i xv=_mm256_loadu_si256((const __m256i*)(x+i)); - __m256i p=_mm256_maddubs_epi16(_mm256_sign_epi8(wv,wv),_mm256_sign_epi8(xv,wv)); - acc=_mm256_add_epi32(acc,_mm256_madd_epi16(p,ones)); - } - sum=hsum256_i32(acc); -#elif defined(__ARM_NEON) - /* ARM: SDOT nativo se disponibile (Apple Silicon: sempre); altrimenti vmull/vpadal. - * Stesso bound anti-overflow del trucco AVX2: coppie <= 128*127*2 = 32512 < 32767. */ -#if defined(__ARM_FEATURE_DOTPROD) - /* 4 accumulatori indipendenti: SDOT ha latenza ~3-4 cicli, con un solo acc la - * catena seriale strozza il core a ~26 GB/s di pesi; con 4 lane indipendenti il - * dot diventa memory-bound (misurato su M4: 26 -> 63 GB/s per core, 2.4x). */ - int32x4_t a0=vdupq_n_s32(0),a1=vdupq_n_s32(0),a2=vdupq_n_s32(0),a3=vdupq_n_s32(0); - for(;i+64<=I;i+=64){ - a0=vdotq_s32(a0,vld1q_s8(w+i), vld1q_s8(x+i)); - a1=vdotq_s32(a1,vld1q_s8(w+i+16),vld1q_s8(x+i+16)); - a2=vdotq_s32(a2,vld1q_s8(w+i+32),vld1q_s8(x+i+32)); - a3=vdotq_s32(a3,vld1q_s8(w+i+48),vld1q_s8(x+i+48)); - } - int32x4_t acc=vaddq_s32(vaddq_s32(a0,a1),vaddq_s32(a2,a3)); - for(;i+16<=I;i+=16) acc=vdotq_s32(acc,vld1q_s8(w+i),vld1q_s8(x+i)); - sum=vaddvq_s32(acc); -#else - int32x4_t acc=vdupq_n_s32(0); - for(;i+16<=I;i+=16){ - int8x16_t wv=vld1q_s8(w+i), xv=vld1q_s8(x+i); - int16x8_t p=vmull_s8(vget_low_s8(wv),vget_low_s8(xv)); - p=vmlal_s8(p,vget_high_s8(wv),vget_high_s8(xv)); - acc=vpadalq_s16(acc,p); - } - sum=vaddvq_s32(acc); -#endif -#elif defined(__VSX__) - /* POWER8: vec_msum (s8 x u8 -> s32) somma i prodotti byte DIRETTAMENTE in lane - * s32, 16 byte/iter: il bound anti-saturazione a 16 bit di maddubs qui non serve. - * Stesso trucco del segno (|w| u8 per x*sign(w) s8), ma |w| via select+sub MODULO - * e non vec_abs: -128 deve diventare 128 u8, non saturare a 127. - * EN: vec_msum accumulates byte products straight into s32 lanes; |w| is built - * with a modulo subtract select instead of vec_abs so w=-128 wraps to 128 (u8) - * rather than saturating to 127. |x|<=127 from qrow_i8, so x negation is safe. */ - __vector signed int acc=vec_splats(0); - const __vector signed char vz=vec_splats((signed char)0); - for(;i+16<=I;i+=16){ - __vector signed char wv=vec_xl(0,(const signed char*)(w+i)); - __vector signed char xv=vec_xl(0,(const signed char*)(x+i)); - __vector __bool char neg=vec_cmplt(wv,vz); - __vector signed char xs=vec_sel(xv,vec_sub(vz,xv),neg); - __vector unsigned char wa=(__vector unsigned char)vec_sel(wv,vec_sub(vz,wv),neg); - acc=vec_msum(xs,wa,acc); - } - sum=vec_extract(acc,0)+vec_extract(acc,1)+vec_extract(acc,2)+vec_extract(acc,3); -#endif - for(;i int8 [-8,7] al volo, poi stesso trucco */ -static inline int32_t dot_i4i8(const uint8_t *w4, const int8_t *x, int I){ - int32_t sum=0; int i=0; -#if defined(__AVX512VNNI__) && defined(__AVX512BW__) - /* 32 bytes = 64 nibbles -> int8 in [-8,7], one vpdpbusd per 64 values. - * 256-bit unpack leaves values in per-128-lane order [0-15][32-47]/[16-31][48-63]; - * dot pairing is order-invariant, so permute x's 128-bit blocks to match - * instead of re-ordering w (one vpermq per iter, off the critical unpack path). */ - const __m256i m4v=_mm256_set1_epi8(0x0F); - const __m512i b8v=_mm512_set1_epi8(8); - const __m512i xidx=_mm512_setr_epi64(0,1,4,5,2,3,6,7); - __m512i acc=_mm512_setzero_si512(); - for(;i+64<=I;i+=64){ - __m256i by=_mm256_loadu_si256((const __m256i*)(w4+(i>>1))); - __m256i lo=_mm256_and_si256(by,m4v), hi=_mm256_and_si256(_mm256_srli_epi16(by,4),m4v); - __m256i z0=_mm256_unpacklo_epi8(lo,hi), z1=_mm256_unpackhi_epi8(lo,hi); - __m512i wv=_mm512_sub_epi8(_mm512_inserti64x4(_mm512_castsi256_si512(z0),z1,1),b8v); - __m512i xv=_mm512_permutexvar_epi64(xidx,_mm512_loadu_si512((const void*)(x+i))); - __mmask64 neg=_mm512_movepi8_mask(wv); - __m512i xs=_mm512_mask_sub_epi8(xv,neg,_mm512_setzero_si512(),xv); - acc=_mm512_dpbusd_epi32(acc,_mm512_abs_epi8(wv),xs); - } - sum=_mm512_reduce_add_epi32(acc); -#elif defined(__AVXVNNI__) && defined(__AVX2__) - /* AVX-VNNI 128-bit, int4: 16 byte = 32 nibble -> int8 [-8,7] in due half - * (n0/n1), ciascuno alimentato a un vpdpbusd da 16 byte. Stesso unpack - * 128-bit del ramo AVX2 sotto; 32 elementi/iter come li. */ - const __m128i m4=_mm_set1_epi8(0x0F); const __m128i b8=_mm_set1_epi8(8); - __m128i acc=_mm_setzero_si128(); - for(;i+32<=I;i+=32){ - __m128i by=_mm_loadu_si128((const __m128i*)(w4+(i>>1))); /* 16 byte = 32 nibble */ - __m128i lo=_mm_and_si128(by,m4), hi=_mm_and_si128(_mm_srli_epi16(by,4),m4); - __m128i n0=_mm_unpacklo_epi8(lo,hi), n1=_mm_unpackhi_epi8(lo,hi); /* nibble in ordine */ - __m128i w0=_mm_sub_epi8(n0,b8), w1=_mm_sub_epi8(n1,b8); - __m128i x0=_mm_loadu_si128((const __m128i*)(x+i)); - __m128i x1=_mm_loadu_si128((const __m128i*)(x+i+16)); - acc=_mm_dpbusd_epi32(acc,_mm_abs_epi8(w0),_mm_sign_epi8(x0,w0)); - acc=_mm_dpbusd_epi32(acc,_mm_abs_epi8(w1),_mm_sign_epi8(x1,w1)); - } - sum=hsum128_i32(acc); -#elif defined(__AVX2__) - const __m128i m4=_mm_set1_epi8(0x0F); const __m256i b8=_mm256_set1_epi8(8); - const __m256i ones=_mm256_set1_epi16(1); - __m256i acc=_mm256_setzero_si256(); - for(;i+32<=I;i+=32){ - __m128i by=_mm_loadu_si128((const __m128i*)(w4+(i>>1))); /* 16 byte = 32 nibble */ - __m128i lo=_mm_and_si128(by,m4), hi=_mm_and_si128(_mm_srli_epi16(by,4),m4); - __m128i n0=_mm_unpacklo_epi8(lo,hi), n1=_mm_unpackhi_epi8(lo,hi); /* in ordine */ - __m256i wv=_mm256_sub_epi8(_mm256_set_m128i(n1,n0),b8); - __m256i xv=_mm256_loadu_si256((const __m256i*)(x+i)); - __m256i p=_mm256_maddubs_epi16(_mm256_sign_epi8(wv,wv),_mm256_sign_epi8(xv,wv)); - acc=_mm256_add_epi32(acc,_mm256_madd_epi16(p,ones)); - } - sum=hsum256_i32(acc); -#elif defined(__ARM_NEON) - const uint8x16_t m4q=vdupq_n_u8(0x0F); const int8x16_t b8q=vdupq_n_s8(8); -#if defined(__ARM_FEATURE_DOTPROD) - /* 4 accumulatori indipendenti (vedi dot_i8i8): spezza la catena seriale su acc. - * Misurato su M4: 12.4 -> 29.9 GB/s di pesi per core (2.4x). */ - int32x4_t a0=vdupq_n_s32(0),a1=vdupq_n_s32(0),a2=vdupq_n_s32(0),a3=vdupq_n_s32(0); - for(;i+64<=I;i+=64){ - uint8x16_t byA=vld1q_u8(w4+(i>>1)), byB=vld1q_u8(w4+(i>>1)+16); - uint8x16x2_t zA=vzipq_u8(vandq_u8(byA,m4q), vshrq_n_u8(byA,4)); /* nibble in ordine */ - uint8x16x2_t zB=vzipq_u8(vandq_u8(byB,m4q), vshrq_n_u8(byB,4)); - a0=vdotq_s32(a0,vsubq_s8(vreinterpretq_s8_u8(zA.val[0]),b8q),vld1q_s8(x+i)); - a1=vdotq_s32(a1,vsubq_s8(vreinterpretq_s8_u8(zA.val[1]),b8q),vld1q_s8(x+i+16)); - a2=vdotq_s32(a2,vsubq_s8(vreinterpretq_s8_u8(zB.val[0]),b8q),vld1q_s8(x+i+32)); - a3=vdotq_s32(a3,vsubq_s8(vreinterpretq_s8_u8(zB.val[1]),b8q),vld1q_s8(x+i+48)); - } - int32x4_t acc=vaddq_s32(vaddq_s32(a0,a1),vaddq_s32(a2,a3)); - for(;i+32<=I;i+=32){ - uint8x16_t by=vld1q_u8(w4+(i>>1)); /* 16 byte = 32 nibble */ - uint8x16x2_t z=vzipq_u8(vandq_u8(by,m4q), vshrq_n_u8(by,4)); /* nibble in ordine */ - acc=vdotq_s32(acc,vsubq_s8(vreinterpretq_s8_u8(z.val[0]),b8q),vld1q_s8(x+i)); - acc=vdotq_s32(acc,vsubq_s8(vreinterpretq_s8_u8(z.val[1]),b8q),vld1q_s8(x+i+16)); - } - sum=vaddvq_s32(acc); -#else - int32x4_t acc=vdupq_n_s32(0); - for(;i+32<=I;i+=32){ - uint8x16_t by=vld1q_u8(w4+(i>>1)); /* 16 byte = 32 nibble */ - uint8x16x2_t z=vzipq_u8(vandq_u8(by,m4q), vshrq_n_u8(by,4)); /* nibble in ordine */ - int8x16_t w0=vsubq_s8(vreinterpretq_s8_u8(z.val[0]),b8q); - int8x16_t w1=vsubq_s8(vreinterpretq_s8_u8(z.val[1]),b8q); - int8x16_t x0=vld1q_s8(x+i), x1=vld1q_s8(x+i+16); - int16x8_t p=vmull_s8(vget_low_s8(w0),vget_low_s8(x0)); /* |w|<=8: nessun overflow */ - p=vmlal_s8(p,vget_high_s8(w0),vget_high_s8(x0)); - acc=vpadalq_s16(acc,p); - p=vmull_s8(vget_low_s8(w1),vget_low_s8(x1)); - p=vmlal_s8(p,vget_high_s8(w1),vget_high_s8(x1)); - acc=vpadalq_s16(acc,p); - } - sum=vaddvq_s32(acc); -#endif -#elif defined(__VSX__) - /* 16 byte = 32 nibble. vec_mergeh/vec_mergel su ppc64le (GCC) interallacciano come - * unpacklo/unpackhi x86 (verificato empiricamente su POWER8): i nibble escono in - * ordine di memoria. |w|<=8 dopo il -8, quindi stesso trucco del segno di dot_i8i8. - * EN: vec_mergeh/l on ppc64le interleave like x86 unpacklo/hi (verified on POWER8), - * so nibbles come out in memory order; then the same sign trick as dot_i8i8. */ - const __vector unsigned char m4v=vec_splats((unsigned char)0x0F); - const __vector unsigned char sh4=vec_splats((unsigned char)4); - const __vector signed char b8v=vec_splats((signed char)8); - const __vector signed char vz=vec_splats((signed char)0); - __vector signed int acc=vec_splats(0); - for(;i+32<=I;i+=32){ - __vector unsigned char by=vec_xl(0,w4+(i>>1)); /* 16 byte = 32 nibble */ - __vector unsigned char lo=vec_and(by,m4v), hi=vec_sr(by,sh4); - __vector signed char w0=vec_sub((__vector signed char)vec_mergeh(lo,hi),b8v); - __vector signed char w1=vec_sub((__vector signed char)vec_mergel(lo,hi),b8v); - __vector signed char x0=vec_xl(0,(const signed char*)(x+i)); - __vector signed char x1=vec_xl(0,(const signed char*)(x+i+16)); - __vector __bool char n0=vec_cmplt(w0,vz), n1=vec_cmplt(w1,vz); - acc=vec_msum(vec_sel(x0,vec_sub(vz,x0),n0), - (__vector unsigned char)vec_sel(w0,vec_sub(vz,w0),n0),acc); - acc=vec_msum(vec_sel(x1,vec_sub(vz,x1),n1), - (__vector unsigned char)vec_sel(w1,vec_sub(vz,w1),n1),acc); - } - sum=vec_extract(acc,0)+vec_extract(acc,1)+vec_extract(acc,2)+vec_extract(acc,3); -#endif - for(;i+1>1]; sum+=((int)(b&0xF)-8)*x[i]+((int)(b>>4)-8)*x[i+1]; } - if(i>1]; sum+=((int)(b&0xF)-8)*x[i]; } - return sum; -} -#if defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8) -/* SMMLA (i8mm): vmmlaq_s32 vede ogni int8x16_t come matrice 2x8 row-major (byte 0-7 = - * riga 0, byte 8-15 = riga 1) e accumula C += A*B^T nel 2x2 int32: lane0=a0.b0, - * lane1=a0.b1, lane2=a1.b0, lane3=a1.b1. vcombine di due mezze-righe costruisce la - * matrice: A = due righe di peso (o,o+1), B = due righe di attivazione (s,s+1), quindi - * meta' traffico pesi e doppio lavoro per istruzione a S>=2. EN: vmmlaq_s32 treats each - * int8x16_t as a 2x8 row-major matrix and does C += A*B^T on a 2x2 int32 tile; vcombine - * of vget_low/high halves builds the 2-row register from two weight/activation rows. */ -static inline int32x4_t mm_tile16(int32x4_t acc, int8x16_t wo, int8x16_t wo1, - int8x16_t xs, int8x16_t xs1){ - acc=vmmlaq_s32(acc, vcombine_s8(vget_low_s8(wo), vget_low_s8(wo1)), - vcombine_s8(vget_low_s8(xs), vget_low_s8(xs1))); - return vmmlaq_s32(acc, vcombine_s8(vget_high_s8(wo), vget_high_s8(wo1)), - vcombine_s8(vget_high_s8(xs), vget_high_s8(xs1))); -} -static void matmul_q_idot_mm(float *y, const int8_t *xq, const float *sx, const int8_t *q, - const float *scale, int S, int I, int O){ - #pragma omp parallel for schedule(static) - for(int o=0;o<(O&~1);o+=2){ - const int8_t *wo=q+(int64_t)o*I, *wo1=q+(int64_t)(o+1)*I; - float sc0=scale[o], sc1=scale[o+1]; - for(int s=0;s<(S&~1);s+=2){ - const int8_t *xs=xq+(int64_t)s*I, *xs1=xq+(int64_t)(s+1)*I; - /* 4 accumulatori indipendenti: una sola catena vmmla e' latency-bound. - * EN: 4 independent accumulators; a single vmmla chain is latency-bound. */ - int32x4_t a0=vdupq_n_s32(0),a1=vdupq_n_s32(0),a2=vdupq_n_s32(0),a3=vdupq_n_s32(0); int i=0; - for(;i+64<=I;i+=64){ - a0=mm_tile16(a0,vld1q_s8(wo+i), vld1q_s8(wo1+i), vld1q_s8(xs+i), vld1q_s8(xs1+i)); - a1=mm_tile16(a1,vld1q_s8(wo+i+16),vld1q_s8(wo1+i+16),vld1q_s8(xs+i+16),vld1q_s8(xs1+i+16)); - a2=mm_tile16(a2,vld1q_s8(wo+i+32),vld1q_s8(wo1+i+32),vld1q_s8(xs+i+32),vld1q_s8(xs1+i+32)); - a3=mm_tile16(a3,vld1q_s8(wo+i+48),vld1q_s8(wo1+i+48),vld1q_s8(xs+i+48),vld1q_s8(xs1+i+48)); - } - for(;i+16<=I;i+=16) - a0=mm_tile16(a0,vld1q_s8(wo+i),vld1q_s8(wo1+i),vld1q_s8(xs+i),vld1q_s8(xs1+i)); - int32x4_t acc=vaddq_s32(vaddq_s32(a0,a1),vaddq_s32(a2,a3)); - int32_t d00=vgetq_lane_s32(acc,0), d01=vgetq_lane_s32(acc,1); - int32_t d10=vgetq_lane_s32(acc,2), d11=vgetq_lane_s32(acc,3); - for(;i>1)), byo1=vld1q_u8(wo1+(i>>1)); - uint8x16_t cyo=vld1q_u8(wo+(i>>1)+16), cyo1=vld1q_u8(wo1+(i>>1)+16); - uint8x16x2_t zo =vzipq_u8(vandq_u8(byo, m4q), vshrq_n_u8(byo, 4)); - uint8x16x2_t zo1=vzipq_u8(vandq_u8(byo1,m4q), vshrq_n_u8(byo1,4)); - uint8x16x2_t ko =vzipq_u8(vandq_u8(cyo, m4q), vshrq_n_u8(cyo, 4)); - uint8x16x2_t ko1=vzipq_u8(vandq_u8(cyo1,m4q), vshrq_n_u8(cyo1,4)); - a0=mm_tile16(a0, vsubq_s8(vreinterpretq_s8_u8(zo.val[0]),b8q), - vsubq_s8(vreinterpretq_s8_u8(zo1.val[0]),b8q), - vld1q_s8(xs+i), vld1q_s8(xs1+i)); - a1=mm_tile16(a1, vsubq_s8(vreinterpretq_s8_u8(zo.val[1]),b8q), - vsubq_s8(vreinterpretq_s8_u8(zo1.val[1]),b8q), - vld1q_s8(xs+i+16), vld1q_s8(xs1+i+16)); - a2=mm_tile16(a2, vsubq_s8(vreinterpretq_s8_u8(ko.val[0]),b8q), - vsubq_s8(vreinterpretq_s8_u8(ko1.val[0]),b8q), - vld1q_s8(xs+i+32), vld1q_s8(xs1+i+32)); - a3=mm_tile16(a3, vsubq_s8(vreinterpretq_s8_u8(ko.val[1]),b8q), - vsubq_s8(vreinterpretq_s8_u8(ko1.val[1]),b8q), - vld1q_s8(xs+i+48), vld1q_s8(xs1+i+48)); - } - for(;i+32<=I;i+=32){ - uint8x16_t byo=vld1q_u8(wo+(i>>1)), byo1=vld1q_u8(wo1+(i>>1)); - uint8x16x2_t zo =vzipq_u8(vandq_u8(byo, m4q), vshrq_n_u8(byo, 4)); - uint8x16x2_t zo1=vzipq_u8(vandq_u8(byo1,m4q), vshrq_n_u8(byo1,4)); - a0=mm_tile16(a0, vsubq_s8(vreinterpretq_s8_u8(zo.val[0]),b8q), - vsubq_s8(vreinterpretq_s8_u8(zo1.val[0]),b8q), - vld1q_s8(xs+i), vld1q_s8(xs1+i)); - a1=mm_tile16(a1, vsubq_s8(vreinterpretq_s8_u8(zo.val[1]),b8q), - vsubq_s8(vreinterpretq_s8_u8(zo1.val[1]),b8q), - vld1q_s8(xs+i+16), vld1q_s8(xs1+i+16)); + for(; i+1>1], bu=wu2[i>>1]; + ag+=(xs[i]*(float)((int)(bg&0xF)-8)+xs[i+1]*(float)((int)(bg>>4)-8))*scg; + au+=(xs[i]*(float)((int)(bu&0xF)-8)+xs[i+1]*(float)((int)(bu>>4)-8))*scu; + } + if(i>1], bu=wu2[i>>1]; + ag+=xs[i]*(float)((int)(bg&0xF)-8)*scg; + au+=xs[i]*(float)((int)(bu&0xF)-8)*scu; + } } - int32x4_t acc=vaddq_s32(vaddq_s32(a0,a1),vaddq_s32(a2,a3)); - int32_t d00=vgetq_lane_s32(acc,0), d01=vgetq_lane_s32(acc,1); - int32_t d10=vgetq_lane_s32(acc,2), d11=vgetq_lane_s32(acc,3); - for(;i+1>1], bo1=wo1[i>>1]; - int a0=(int)(bo&0xF)-8, a1=(int)(bo>>4)-8, b0=(int)(bo1&0xF)-8, b1=(int)(bo1>>4)-8; - int u0=xs[i],u1=xs[i+1],v0=xs1[i],v1=xs1[i+1]; - d00+=a0*u0+a1*u1; d01+=a0*v0+a1*v1; d10+=b0*u0+b1*u1; d11+=b0*v0+b1*v1; } - if(i>1], bo1=wo1[i>>1]; - int a0=(int)(bo&0xF)-8, b0=(int)(bo1&0xF)-8; - d00+=a0*xs[i]; d01+=a0*xs1[i]; d10+=b0*xs[i]; d11+=b0*xs1[i]; } - y[(int64_t)s*O+o] =(float)d00*sc0*sx[s]; - y[(int64_t)s*O+(o+1)] =(float)d10*sc1*sx[s]; - y[(int64_t)(s+1)*O+o] =(float)d01*sc0*sx[s+1]; - y[(int64_t)(s+1)*O+(o+1)]=(float)d11*sc1*sx[s+1]; + yg[(int64_t)s*O+o]=ag; yu[(int64_t)s*O+o]=au; } - if(S&1){ int s=S-1; const int8_t *xs=xq+(int64_t)s*I; - y[(int64_t)s*O+o] =(float)dot_i4i8(wo, xs,I)*sc0*sx[s]; - y[(int64_t)s*O+(o+1)]=(float)dot_i4i8(wo1,xs,I)*sc1*sx[s]; } - } - if(O&1){ int o=O-1; const uint8_t *w=q4+(int64_t)o*rb; float sc=scale[o]; - #pragma omp parallel for schedule(static) - for(int s=0;s=2){ matmul_q_idot_mm(y,xq,sx,q,scale,S,I,O); return; } -#endif - #pragma omp parallel for schedule(static) - for(int o=0;o=2){ matmul_i4_idot_mm(y,xq,sx,q4,scale,S,I,O); return; } -#endif - #pragma omp parallel for schedule(static) - for(int o=0;og_qscratch.xq_cap){ - int8_t *p=realloc(g_qscratch.xq,xn); - if(!p){ fprintf(stderr,"OOM quant scratch\n"); exit(1); } - g_qscratch.xq=p; g_qscratch.xq_cap=xn; - } - if(sn>g_qscratch.sx_cap){ - float *p=realloc(g_qscratch.sx,sn*sizeof(float)); - if(!p){ fprintf(stderr,"OOM quant scales\n"); exit(1); } - g_qscratch.sx=p; g_qscratch.sx_cap=sn; } - *xq=g_qscratch.xq; *sx=g_qscratch.sx; } /* allow_idot=0: forza il kernel int4/int8 ESATTO (attivazioni f32). Serve alle proiezioni di @@ -1022,48 +526,26 @@ static void quant_scratch(size_t xn, size_t sn, int8_t **xq, float **sx){ * EN: allow_idot=0 forces the EXACT int4/int8 kernel (f32 activations). The attention * projections need it: IDOT's int8 activation quantization costs +0.117 nats/token there * (~+12% perplexity), measured. Every other prefill matmul keeps IDOT as before. */ -static void matmul_qt_ex(float *y, const float *x, QT *w, int S, int allow_idot); -static void matmul_qt(float *y, const float *x, QT *w, int S){ matmul_qt_ex(y,x,w,S,1); } static void matmul_qt_ex(float *y, const float *x, QT *w, int S, int allow_idot){ #ifdef COLI_METAL - /* Large row-batches (prefill: kv_b reconstruction, o_proj, dense MLP, step_all logits) - * amortize Metal's ~5ms submit latency; small-S decode matmuls stay on CPU (NEON wins). - * Weights must be registered (all dense QT allocs are, via qalloc). */ if(g_metal_enabled && S>=g_metal_gemm_min && !spec_pinned() && (w->fmt==1||w->fmt==2) && !omp_in_parallel()){ const void *wp = w->fmt==1 ? (const void*)w->q8 : (const void*)w->q4; if(coli_metal_gemm(y,x,wp,w->s,w->fmt,S,w->I,w->O)) return; } #endif #ifdef COLI_CUDA - /* The CUDA backend owns persistent copies only for model-resident tensors. - * Streaming expert slots are reused for different IDs and must never enter - * this cache. Nested OpenMP calls stay on CPU because each device context - * intentionally owns one synchronous scratch stream in this stage. */ - if(g_cuda_enabled && w->cuda_eligible && !w->cuda_failed && !omp_in_parallel()){ + if(g_cuda_enabled && w->cuda_eligible && !w->cuda_failed && w->fmt!=5 && !omp_in_parallel()){ const void *weights = w->fmt==0 ? (const void*)w->qf : w->fmt==1 ? (const void*)w->q8 : (const void*)w->q4; - if(coli_cuda_matmul(&w->cuda,y,x,weights,w->s,w->fmt,S,w->I,w->O,w->cuda_device)) return; + if(coli_cuda_matmul(&w->cuda,y,x,weights,w->s,w->fmt,S,w->I,w->O,w->cuda_device,w->gs)) return; w->cuda_failed=1; fprintf(stderr,"[CUDA] tensor [%d,%d] on device %d disabled after an error; falling back to CPU\n", w->O,w->I,w->cuda_device); } #endif if(w->fmt==0){ matmul(y,x,w->qf,S,w->I,w->O); return; } - /* fmt=4: grouped int4 — always use the exact grouped kernel (no IDOT approximation, - * since the whole point of grouped scales is better quality). */ if(w->fmt==4){ matmul_i4_grouped(y,x,w->q4,w->s,S,w->I,w->O,w->gs); return; } - /* int8 IDOT vince sempre (1.4-2.5x). int4 IDOT: l'autore su AVX2 trovo' che a S=1 - * non ripaga (soglia S>=2); ma su ARM/SDOT il singolo token CONVIENE (vedi g_i4s / - * PR #9 per il gemello VNNI). Soglia configurabile con I4S. - * EN: int8 IDOT always wins (1.4-2.5x). int4 IDOT: on AVX2 the author found S=1 didn't - * pay (S>=2 gate); on ARM/SDOT single-token DOES pay (see g_i4s / PR #9 for the VNNI - * twin). Threshold configurable via I4S. */ - /* #163: sotto SPEC_PIN il gate int4-IDOT usa la decisione di S=1 per OGNI S, cosi' - * draft e verifica restano sulla stessa famiglia. (CUDA non e' toccato: la sua - * condizione non dipende da S, quindi e' gia' coerente tra draft e verifica.) - * EN: under SPEC_PIN the int4 IDOT gate uses the S=1 decision for EVERY S, so draft - * and verify stay in one family. (CUDA untouched: its condition is S-independent, - * hence already draft/verify-consistent.) */ + if(w->fmt==6){ matmul_e8(y,x,w->q4,NULL,S,w->I,w->O); return; } /* scales live in-block */ if(allow_idot && g_idot && (w->fmt==1 || (w->fmt==2 && (spec_pinned() ? g_i4s<=1 : S>=g_i4s)))){ int I=w->I; int8_t *xq; float *sx; if(S<0 || I<0 || (size_t)S>SIZE_MAX/(size_t)(I?I:1)){ fprintf(stderr,"matmul_qt: shape overflow\n"); exit(1); } @@ -1075,52 +557,10 @@ static void matmul_qt_ex(float *y, const float *x, QT *w, int S, int allow_idot) } if(w->fmt==1) matmul_q(y,x,w->q8,w->s,S,w->I,w->O); else if(w->fmt==3) matmul_i2(y,x,w->q4,w->s,S,w->I,w->O); + else if(w->fmt==5) matmul_i3(y,x,w->q4,w->s,S,w->I,w->O); else matmul_i4(y,x,w->q4,w->s,S,w->I,w->O); } -/* quantizza w[O,I] f32 -> int8 q[O,I] + scala[O] simmetrica per riga */ -static void quantize_rows(const float *w, int8_t *q, float *scale, int O, int I, int bits){ - int qmax=(1<<(bits-1))-1; - #pragma omp parallel for schedule(static) - for(int o=0;oamax)amax=a; } - float s=amax/qmax; if(s<1e-8f)s=1e-8f; scale[o]=s; - int8_t *qr=q+(int64_t)o*I; - for(int i=0;iqmax)v=qmax; if(v<-qmax-1)v=-qmax-1; qr[i]=(int8_t)v; } - } -} -/* quantizza w[O,I] f32 -> int4 impacchettato (2/byte) + scala[O]. - * bits<=4: valori in [-qmax-1,qmax] stanno in un nibble [-8,7]; memorizzati come v+8 (0..15). */ -static void pack_int4(const float *w, uint8_t *q4, float *scale, int O, int I, int bits){ - int qmax=(1<<(bits-1))-1, rb=(I+1)/2; - #pragma omp parallel for schedule(static) - for(int o=0;oamax)amax=a; } - float s=amax/qmax; if(s<1e-8f)s=1e-8f; scale[o]=s; - uint8_t *qr=q4+(int64_t)o*rb; - for(int i=0;iqmax)v0=qmax; if(v0<-8)v0=-8; - int v1=0; if(i+1qmax)v1=qmax; if(v1<-8)v1=-8; } - qr[i>>1] = (uint8_t)((v0+8) | ((v1+8)<<4)); - } - } -} - -/* quantizza w[O,I] f32 -> int2 impacchettato (4/byte) + scala[O]. valori nibble 2-bit in [-2,1]. */ -static void pack_int2(const float *w, uint8_t *q2, float *scale, int O, int I, int bits){ - int qmax=(1<<(bits-1))-1, rb=(I+3)/4; - #pragma omp parallel for schedule(static) - for(int o=0;oamax)amax=a; } - float s=amax/qmax; if(s<1e-8f)s=1e-8f; scale[o]=s; - uint8_t *qr=q2+(int64_t)o*rb; - for(int i=0;iqmax)v=qmax; if(v<-2)v=-2; byte|=(uint8_t)((v+2)<<(k*2)); } - qr[i>>2]=byte; - } - } -} - static int g_nopack=0; /* NOPACK=1 -> tiene i valori <=4bit in contenitore int8 (per validare il packing) */ static int g_drop=0; /* DROP=1 -> scarta le pagine expart dopo l'uso. Default 0: le lascia in * page-cache (buff/cache, NON RSS) come L2 gratuito -> sfrutta lo @@ -1163,12 +603,6 @@ static int g_draft=0; /* metodo E: DRAFT=n token auto-speculati per forward v * parte (#8). MAI un vincolo sul sampling: solo proposte, la verifica batch-union * decide — grammatica sbagliata = draft rifiutati, output identico. * GRAMMAR_DRAFT=n (default 24) limita i token forzati per forward. */ -static Grammar g_gram; static GrState g_gst; -static Tok *g_gr_T=NULL; -static int g_gr_on=0; /* grammatica caricata e walker vivo */ -static int g_gr_armed=0; /* lazy: parte dal primo byte ammesso dalla radice (salta i preamboli) */ -static int g_gr_max=24; -static uint64_t g_gr_prop=0, g_gr_acc=0; static FILE *g_route_fp=NULL; /* ROUTE_TRACE=: dump per-position top-K routing (ids:gates) * per layer — offline co-activation / coupling analysis. Zero * effect on computation; measurement only. */ @@ -1188,6 +622,19 @@ static int g_couple=0, g_couple_k=8, g_couple_d=1; static int16_t *cp_pred=NULL; /* [(L*2+(dL-1))*E + e]*CP_M + j -> target id (-1 none) */ static float *cp_cnt=NULL; static long g_cp_enq=0; +/* All grammar-forced-draft state in one struct so it can become per-request + * in the multiplexed server. Fields (same semantics as the former globals): + * on = grammar loaded and walker alive; armed = lazy start from the first byte + * accepted at the root (skips preambles); max = forced-span cap per forward; + * prop/acc = proposed/accepted forced-draft counters. */ +typedef struct { + Grammar gram; + GrState st; + Tok *T; + int on, armed, max; + uint64_t prop, acc; +} GrDraft; +static GrDraft g_grd={.max=24}; /* process-level instance: PROMPT mode + run_serve keep using this */ static void couple_prefetch(Model *m, int layer, const int *idx, int Ke); static int g_looka=0; /* LOOKA=1: misura (solo contatori, zero effetti) quanto il routing MoE * e' predicibile IN ANTICIPO — la domanda che decide se un prefetch @@ -1206,6 +653,11 @@ static int g_disk_split=0; /* DISK_SPLIT=1: contatori che spezzano i DISK LOAD ( * (int4), con i byte letti. Default OFF: a flag spento gli atomic * non vengono MAI toccati (zero overhead), le righe extra di stats * non vengono stampate. Solo misura: nessun effetto sull'output. */ + +#include "sample.h" +#include "kv_persist.h" +#include "telemetry.h" + /* Aligned allocator for dense QT weights/scales: under METAL, page-align + register so the * GPU reads them zero-copy (no upload duplicate). Plain malloc otherwise. */ /* ---- COLI_NUMA=1 (#82): interleave the expert slabs across NUMA nodes ---- @@ -1216,19 +668,31 @@ static int g_disk_split=0; /* DISK_SPLIT=1: contatori che spezzano i DISK LOAD ( * +40% on a 4-socket (#82). Blanket `numactl --interleave=all` is NOT equivalent: * it also interleaves the CUDA pinned staging buffers and cost a 4-socket GPU host * 10x (#82) — hence per-region mbind here and nothing else. Raw syscall, no libnuma - * dependency; MPOL_MF_MOVE migrates pages of reused heap chunks too. Linux-only, - * silent no-op elsewhere or on single-node hosts. */ + * dependency. Linux-only, silent no-op elsewhere or on single-node hosts. + * + * VMA discipline (#419): every mbind carries its own memory policy, so a bound + * region cannot merge with its neighbours — measured ~2 VMAs per slab, with or + * without MPOL_MF_MOVE. Per-slab binds on a PIN_GB=all load (19,456 experts x + * slab+fslab) cross the default vm.max_map_count=65530 and posix_memalign dies + * with terabytes free. So the bulk (the pinned hot-store) is bound as ONE arena + * per layer (see pin_load), and per-slab mbind remains only for the bounded + * allocations: dense qalloc, the LRU ecache, and GPU-tier staging. No flag: + * every bind here lands before the pread that first-touches the pages, so + * there is nothing to migrate. */ #ifdef __linux__ static int g_numa_nodes=0; /* only touched under __linux__; off-Linux NUMA is a no-op */ +static int g_numa_skip_bind=0; /* raised around the GPU-prefix pin load: those slabs are + * upload staging, freed right after — binding them buys + * nothing and costs ~2 transient VMAs each (#419) */ #endif static void numa_slab_bind(void *p, size_t n){ #ifdef __linux__ - if(g_numa_nodes<2 || !p || !n) return; + if(g_numa_nodes<2 || g_numa_skip_bind || !p || !n) return; unsigned long mask=(1UL<=2) fprintf(stderr,"[NUMA] expert slabs interleaved across %d nodes\n",g_numa_nodes); - else fprintf(stderr,"[NUMA] single node: COLI_NUMA ignored\n"); + if(g_numa_nodes<2){ fprintf(stderr,"[NUMA] single node: COLI_NUMA ignored\n"); return; } + /* Probe mbind once so a constrained container degrades with a message + * instead of silently losing the interleave. The probe page must be + * page-aligned (mbind rejects unaligned addresses with EINVAL) and only + * errno==EPERM disables — any other failure keeps NUMA on. */ + { void *pg=NULL; + if(!posix_memalign(&pg,4096,4096)){ + unsigned long mask=(1UL< old behavior (a speculation evicts the plain LRU). + * Default ON: protect a RESIDENT demand-loaded expert from a speculation only when + * it is genuinely WARM (>=2 accesses) AND clearly hotter than the predicted expert + * by tier_pick_lfru's 25%+4-freq hysteresis; otherwise the speculation may evict the + * plain-LRU victim as before. Cache placement only -> output byte-identical. (#441, #490) */ /* Handshake main<->pilota per il load-vero cross-layer. Invariante di sicurezza in DUE parti: * 1) Percorso MATMUL (moe): il pilota scrive SOLO ecache[layer] con layer > g_cur_moe_layer; * il matmul in moe() legge SOLO ecache[layer]==g_cur_moe_layer, e la barriera a inizio moe() @@ -1276,18 +762,21 @@ static _Atomic int g_cur_moe_layer=-1; /* massimo layer moe in cui il MAIN e' static int g_pilot_inflight[256]; /* protected by g_pilot_mx; URING can load a layer concurrently */ static _Atomic long g_pilot_loads=0; /* load cross-layer VERI completati (banda spesa) */ static _Atomic long g_pilot_drops=0; /* predizioni scartate perche' il main possiede gia' il layer */ -/* sceglie il formato da `bits`: >=16 f32, 5..8 int8, <=4 int4-packed */ +/* format from `bits`: >=16 f32, 5..8 int8, 4 int4-packed, 3 int3-g64 (group scales), <=2 int2 */ static void qt_alloc(QT *t, int O, int I, int bits){ t->O=O; t->I=I; t->qf=NULL; t->q8=NULL; t->q4=NULL; t->s=NULL; if(bits>=16){ t->fmt=0; t->qf=falloc((int64_t)O*I); } else if(bits>=5 || g_nopack){ t->fmt=1; t->q8=qalloc((int64_t)O*I); t->s=qsalloc(O); } - else if(bits>=3){ t->fmt=2; t->q4=qalloc((int64_t)O*((I+1)/2)); t->s=qsalloc(O); } + else if(bits>=4){ t->fmt=2; t->q4=qalloc((int64_t)O*((I+1)/2)); t->s=qsalloc(O); } + else if(bits==3){ t->fmt=5; t->q4=qalloc((int64_t)O*i3_rowbytes(I)); + t->s=(float*)qalloc((size_t)O*i3_groups(I)*sizeof(float)); } else { t->fmt=3; t->q4=qalloc((int64_t)O*((I+3)/4)); t->s=qsalloc(O); } } static void qt_fill(QT *t, const float *w, int bits){ if(t->fmt==0) memcpy(t->qf, w, (int64_t)t->O*t->I*sizeof(float)); else if(t->fmt==1) quantize_rows(w, t->q8, t->s, t->O, t->I, bits); else if(t->fmt==3) pack_int2(w, t->q4, t->s, t->O, t->I, bits); + else if(t->fmt==5) pack_int3_g64(w, t->q4, t->s, t->O, t->I); else pack_int4(w, t->q4, t->s, t->O, t->I, bits); } @@ -1332,11 +821,31 @@ static void rope_interleave(float *v, int pos, const Cfg *c){ } /* ---------- config ---------- */ +/* SEC-9: bounded slurp for untrusted config/oracle JSON. config.json arrives from + * unverified mirrors (see qt_check_fmt threat model); an unbounded ftell->malloc + * gave a hostile file a load-time OOM or, on malloc failure, a NULL deref via + * b[got]=0. Cap the size, NULL-check the alloc, require a full read. Returns a + * malloc'd NUL-terminated buffer, or NULL on any failure. Mirrors tok.h tk_read_file. */ +#define CFG_MAX_BYTES (256ll<<20) /* config/oracle JSON is KB-MB in practice */ +static char* cfg_slurp(const char *path){ + FILE *f=fopen(path,"rb"); if(!f) return NULL; + fseek(f,0,SEEK_END); long n=ftell(f); fseek(f,0,SEEK_SET); + if(n<0 || (long long)n>CFG_MAX_BYTES){ fclose(f); return NULL; } + char *b=malloc((size_t)n+1); if(!b){ fclose(f); return NULL; } + size_t got=fread(b,1,(size_t)n,f); fclose(f); + if((long)got!=n){ free(b); return NULL; } + b[got]=0; return b; +} static jval* cfg_root(const char *snap, char **arena){ char p[2048]; snprintf(p,sizeof(p),"%s/config.json",snap); FILE *f=fopen(p,"rb"); if(!f){perror(p);exit(1);} fseek(f,0,SEEK_END); long n=ftell(f); fseek(f,0,SEEK_SET); - char *b=malloc(n+1); size_t got=fread(b,1,n,f); b[got]=0; fclose(f); + /* SEC: config.json arriva dalla dir modello non fidata. Limita la dimensione + * (un file ostile enorme = OOM al load) e controlla la malloc: senza il NULL + * check, b[got]=0 su malloc fallita era un NULL-deref. */ + if(n<0 || n>(256L<<20)){ fprintf(stderr,"%s: size %ld out of range (0..256 MB)\n",p,n); exit(1); } + char *b=malloc((size_t)n+1); if(!b){ fprintf(stderr,"OOM reading %s (%ld bytes)\n",p,n); exit(1); } + size_t got=fread(b,1,(size_t)n,f); b[got]=0; fclose(f); if((long)got!=n) fprintf(stderr,"warning: short read on %s (%ld of %ld)\n",p,(long)got,n); return json_parse(b,arena); } @@ -1375,8 +884,9 @@ static void load_cfg(Cfg *c, const char *snap){ FILE *gf=fopen(gp,"rb"); /* assente = nessun problema: e' opzionale */ if(gf){ fseek(gf,0,SEEK_END); long gn=ftell(gf); fseek(gf,0,SEEK_SET); - if(gn>0){ - char *gb=malloc(gn+1); size_t gg=fread(gb,1,gn,gf); gb[gg]=0; + char *gb = (gn>0 && gn<=(256L<<20)) ? malloc((size_t)gn+1) : NULL; /* SEC: cap + NULL check */ + if(gb){ + size_t gg=fread(gb,1,(size_t)gn,gf); gb[gg]=0; char *ga=NULL; jval *gr=json_parse(gb,&ga); jval *ge=gr?json_get(gr,"eos_token_id"):NULL; if(ge){ @@ -1445,6 +955,37 @@ static int detect_group_size(int O, int I, int64_t ns){ return 0; } +/* SEC: risolve e VALIDA il formato quantizzato di un tensore [O,I] letto da un + * container non fidato (mirror). L'inferenza precedente (`?1:?2:3`) cadeva su + * int2 per QUALSIASI conteggio byte non riconosciuto: un peso troppo corto + * diventava un int2 valido e il matmul leggeva oltre il buffer (O*I nibble a + * 4/byte). Qui i byte del peso devono corrispondere a un layout noto e i byte + * della scala alla cardinalita' attesa (O per-row, O*ng per-gruppo) — altrimenti + * si termina invece di sforare. Ritorna fmt (1/2/3/4/5) e scrive *gs. */ +static int qt_resolve_fmt(const char *name, int O, int I, int64_t nb, int64_t ns, int *gs){ + int64_t exp_i8=(int64_t)O*I, exp_i4=(int64_t)O*((I+1)/2), exp_i2=(int64_t)O*((I+3)/4); + int64_t exp_i3=(int64_t)O*i3_rowbytes(I); /* int3-g64 (fmt=5): 24B per 64-input group */ + /* fmt=6 (E8/IQ3, #452): scales live inside the 98B super-blocks, so the .qs + * convention is kept with a single-float tag — ns==4 is the discriminator + * (every other format carries at least O floats of real scales). */ + if(ns==4 && nb==(int64_t)O*e8_rowbytes(I)){ *gs=0; return 6; } + /* Row formats take precedence: for tiny I the int3-g64 byte count can coincide with + * a row layout (e.g. [O,48]: ceil(48/2)=24=1*24). For real tensor shapes the counts + * are distinct, and the weight bytes — not the scale size — are the int3 tag, because + * int3-g64 and grouped-int4-at-gs=64 carry the SAME scale cardinality O*ceil(I/64). */ + int fmt = (nb==exp_i8)?1 : (nb==exp_i4)?2 : (nb==exp_i2)?3 : (nb==exp_i3)?5 : 0; + if(!fmt){ + fprintf(stderr,"%s: quantized weight is %lld bytes — no int8/int4/int2/int3-g64 layout for [%d,%d], refusing (untrusted container)\n", + name,(long long)nb,O,I); exit(1); } + *gs=0; + if(fmt==2){ int g=detect_group_size(O,I,ns); if(g>0){ fmt=4; *gs=g; } } + int64_t exp_scale = (fmt==4)? (int64_t)O*((I+*gs-1)/(*gs)) + : (fmt==5)? (int64_t)O*i3_groups(I) : (int64_t)O; /* in FLOAT */ + if(ns != exp_scale*4){ + fprintf(stderr,"%s: scale array is %lld bytes — expected %lld for [%d,%d] fmt=%d, refusing (untrusted container)\n", + name,(long long)ns,(long long)(exp_scale*4),O,I,fmt); exit(1); } + return fmt; +} /* costruisce un QT [O,I] dal disco in `t` (buffer riusabili tra chiamate). * - se esiste `name.qs`: pesi GIA' quantizzati nel container (U8 qdata + F32 scala) -> letti diretti * - altrimenti: tensore pieno (f32/bf16) -> quantizzato a runtime a `bits` (oracolo tiny / pesi pieni) @@ -1454,22 +995,35 @@ static void qt_from_disk(Model *m, const char *name, int O, int I, int bits, int if(st_has(&m->S,sn)){ int64_t nb=st_nbytes(&m->S,name); int64_t ns=st_nbytes(&m->S,sn); /* scale bytes (F32) */ - /* Detect int4-grouped (fmt=4): packed int4 weight bytes BUT scale array is - * larger than O*4 — the group size is derived from the scale-array size. */ - int fmt = (nb==(int64_t)O*I)?1 : (nb==(int64_t)O*((I+1)/2))?2 : 3; + /* fmt=4 int4-grouped: byte int4 ma scala > O*4 — gs deriva dalla scala. + * qt_resolve_fmt valida entrambi i conteggi contro [O,I] e termina se + * non fidati (SEC). */ int gs=0; - if(fmt==2) gs=detect_group_size(O,I,ns); - if(gs>0) fmt=4; + int fmt = qt_resolve_fmt(name,O,I,nb,ns,&gs); if(fmt==1){ if(t->fmt!=1||!t->q8){ t->fmt=1; t->O=O; t->I=I; t->gs=0; t->q8=qalloc(nb); t->s=qsalloc(O); } st_read_raw(&m->S,name,t->q8,drop); } else if(fmt==4){ int ng=(I+gs-1)/gs; if(t->fmt!=4||!t->q4){ t->fmt=4; t->O=O; t->I=I; t->gs=gs; t->q4=qalloc(nb); t->s=falloc((int64_t)O*ng); } st_read_raw(&m->S,name,t->q4,drop); } + else if(fmt==5){ int64_t ng=i3_groups(I); /* int3-g64: 24B/group weights + O*ng group scales */ + if(t->fmt!=5||!t->q4){ t->fmt=5; t->O=O; t->I=I; t->gs=0; t->q4=qalloc(nb); t->s=falloc((int64_t)O*ng); } + st_read_raw(&m->S,name,t->q4,drop); } + else if(fmt==6){ /* E8/IQ3: everything in-block, .qs is the 4-byte tag */ + if(t->fmt!=6||!t->q4){ t->fmt=6; t->O=O; t->I=I; t->gs=0; t->q4=qalloc(nb); t->s=qsalloc(1); } + st_read_raw(&m->S,name,t->q4,drop); } else { if(t->fmt!=fmt||!t->q4){ t->fmt=fmt; t->O=O; t->I=I; t->gs=0; t->q4=qalloc(nb); t->s=qsalloc(O); } st_read_raw(&m->S,name,t->q4,drop); } - st_read_f32(&m->S,sn,t->s,drop); + /* cap MUST match the scale cardinality qt_resolve_fmt already validated and + * the falloc above actually reserved, per format: grouped-int4 (fmt=4) keeps + * O*ceil(I/gs) scales and int3-g64 (fmt=5) keeps O*i3_groups(I); everything + * else is per-row O. Using the per-row bound for a grouped format would + * reject a legitimate container (fmt=5 regressed exactly that way). */ + st_read_f32_cap(&m->S,sn,t->s, + fmt==4 ? (int64_t)O*((I+gs-1)/gs) : + fmt==5 ? (int64_t)O*i3_groups(I) : + fmt==6 ? (int64_t)1 : (int64_t)O, drop); } else { if(!t->qf && !t->q8 && !t->q4) qt_alloc(t,O,I,bits); - if(t->fmt==0) st_read_f32(&m->S,name,t->qf,drop); - else { float *tmp=falloc((int64_t)O*I); st_read_f32(&m->S,name,tmp,drop); qt_fill(t,tmp,bits); free(tmp); } + if(t->fmt==0) st_read_f32_cap(&m->S,name,t->qf,(int64_t)O*I,drop); + else { float *tmp=falloc((int64_t)O*I); st_read_f32_cap(&m->S,name,tmp,(int64_t)O*I,drop); qt_fill(t,tmp,bits); free(tmp); } } } static QT qt_load(Model *m, const char *name, int O, int I, int bits){ @@ -1501,13 +1055,14 @@ static void qt_cuda_colocate(QT *dst,const QT *src){ } static void layer_cuda_shard_kvb(Layer *l,int H,int Q,int V){ if(!g_cuda_enabled||!g_cuda_dense||g_cuda_ndev<2||l->kv_b.fmt==0)return; - int rb=l->kv_b.fmt==1?l->kv_b.I:(l->kv_b.fmt==2?(l->kv_b.I+1)/2:(l->kv_b.I+3)/4); + int rb=l->kv_b.fmt==1?l->kv_b.I: + (l->kv_b.fmt==2||l->kv_b.fmt==4)?(l->kv_b.I+1)/2:(l->kv_b.I+3)/4; const uint8_t *weights=l->kv_b.fmt==1?(const uint8_t*)l->kv_b.q8:l->kv_b.q4; for(int d=0,h0=0;dkv_b.s+(int64_t)h0*(Q+V); - if(!coli_cuda_tensor_upload(&l->kv_b_shard[d],part,scale,l->kv_b.fmt,l->kv_b.I,rows,g_cuda_devices[d]))return; + const float *scale=l->kv_b.s+(int64_t)h0*(Q+V)*(l->kv_b.gs>0?(l->kv_b.I+l->kv_b.gs-1)/l->kv_b.gs:1); + if(!coli_cuda_tensor_upload_g(&l->kv_b_shard[d],part,scale,l->kv_b.fmt,l->kv_b.I,rows,g_cuda_devices[d],l->kv_b.gs))return; l->shard_h0[d]=h0;l->shard_hn[d]=hn;l->n_kv_b_shard++;h0+=hn; } int old=-1;for(int i=0;ikv_b.cuda_device)old=i; @@ -1518,7 +1073,9 @@ static void layer_cuda_shard_kvb(Layer *l,int H,int Q,int V){ static void model_init(Model *m, const char *snap, int cap, int ebits, int dbits){ memset(m,0,sizeof(*m)); m->ebits=ebits; m->dbits=dbits; - load_cfg(&m->c,snap); st_init(&m->S,snap); + load_cfg(&m->c,snap); + { const char *xd=getenv("COLI_MODEL_DIRS"); /* SPLIT: model shards spread across N drives */ + st_init_multi(&m->S,snap,(xd&&*xd)?xd:NULL); } Cfg *c=&m->c; char nm[256]; int H=c->n_heads, D=c->hidden; /* embed e lm_head sono il confine I/O: tenerli ad alta precisione (come i quant dynamic * reali). A bf16 ~1.9GB su GLM reale: trascurabile. dbits>=8 -> qui f32; piu' basso -> dbits. */ @@ -1535,6 +1092,7 @@ static void model_init(Model *m, const char *snap, int cap, int ebits, int dbits m->pin=calloc(NR,sizeof(ESlot*)); m->npin=calloc(NR,sizeof(int)); m->eusage=calloc(NR,sizeof(uint32_t*)); m->eheat=calloc(NR,sizeof(uint32_t*)); m->elast=calloc(NR,sizeof(uint32_t*)); + m->elast_dc=calloc(NR,sizeof(uint32_t*)); m->elast_pre=calloc(NR,sizeof(uint32_t*)); m->kv=calloc(1,sizeof(KVState)); m->kv_start=m->kv->kv_start=calloc(NR,sizeof(int)); for(int i=0;in_layers;i++){ @@ -1579,6 +1137,8 @@ static void model_init(Model *m, const char *snap, int cap, int ebits, int dbits m->eusage[i]=calloc(c->n_experts,sizeof(uint32_t)); m->eheat[i]=calloc(c->n_experts,sizeof(uint32_t)); m->elast[i]=calloc(c->n_experts,sizeof(uint32_t)); + m->elast_dc[i]=calloc(c->n_experts,sizeof(uint32_t)); + m->elast_pre[i]=calloc(c->n_experts,sizeof(uint32_t)); } #undef P } @@ -1591,12 +1151,17 @@ static void model_init(Model *m, const char *snap, int cap, int ebits, int dbits "self_attn.q_a_proj.weight","self_attn.q_b_proj.weight","self_attn.kv_a_proj_with_mqa.weight", "self_attn.kv_b_proj.weight","self_attn.o_proj.weight","mlp.gate.weight", "mlp.shared_experts.gate_proj.weight","mlp.shared_experts.down_proj.weight", - "mlp.experts.0.gate_proj.weight","mlp.experts.255.down_proj.weight"}; + "mlp.experts.0.gate_proj.weight"}; char mn[256]; m->has_mtp=1; for(unsigned q=0;qn_layers,req[q]); if(!st_has(&m->S,mn)){ m->has_mtp=0; break; } } + /* probe the LAST expert by index, not a fixed 255: REAP-pruned + * checkpoints have n_routed_experts < 256 and the MTP set stays complete, + * so a hardcoded expert.255 would spuriously report has_mtp=0 on them. */ + snprintf(mn,sizeof(mn),"model.layers.%d.mlp.experts.%d.down_proj.weight",c->n_layers,c->n_experts-1); + if(!st_has(&m->S,mn)) m->has_mtp=0; if(getenv("MTP") && atoi(getenv("MTP"))==0) m->has_mtp=0; if(m->has_mtp){ int i=c->n_layers; Layer *l=&m->mtpL; @@ -1625,6 +1190,8 @@ static void model_init(Model *m, const char *snap, int cap, int ebits, int dbits m->eusage[i]=calloc(c->n_experts,sizeof(uint32_t)); m->eheat[i]=calloc(c->n_experts,sizeof(uint32_t)); m->elast[i]=calloc(c->n_experts,sizeof(uint32_t)); + m->elast_dc[i]=calloc(c->n_experts,sizeof(uint32_t)); + m->elast_pre[i]=calloc(c->n_experts,sizeof(uint32_t)); m->kv_start[i]=-1; /* KV MTP: parte dalla prima posizione di decode */ #undef PM } @@ -1678,13 +1245,29 @@ static void model_init(Model *m, const char *snap, int cap, int ebits, int dbits /* embed: dequantizza la riga del token (scala per-riga) in x[hidden] */ static void embed_row(Model *m, int tok, float *x){ int D=m->c.hidden; QT *e=&m->embed; + if(tok<0 || tok>=e->O){ memset(x,0,(size_t)D*sizeof(float)); return; } /* #SEC-5: out-of-range token id -> zero row, never OOB */ if(e->fmt==0){ memcpy(x, e->qf+(int64_t)tok*D, D*sizeof(float)); return; } + if(e->fmt==4){ /* grouped int4: per-group scale (embed/lm_head at io_bits, usually fmt 0/1) */ + const uint8_t *q=e->q4+(int64_t)tok*((D+1)/2); int gs=e->gs,ng=(D+gs-1)/gs; + const float *scl=e->s+(int64_t)tok*ng; + for(int g=0;g*gsD)glen=D-base; float s=scl[g]; + for(int i=base;i+1>1]; x[i]=(float)((int)(byte&0xF)-8)*s; + x[i+1]=(float)((int)(byte>>4)-8)*s; } + if(glen&1){ uint8_t byte=q[(base+glen-1)>>1]; x[base+glen-1]=(float)((int)(byte&0xF)-8)*s; } } + return; } if(e->fmt==1){ const int8_t *q=e->q8+(int64_t)tok*D; float s=e->s[tok]; for(int i=0;ifmt==2){ const uint8_t *q=e->q4+(int64_t)tok*((D+1)/2); float s=e->s[tok]; /* int4 */ for(int i=0;i>1]; x[i]=(float)((int)(byte&0xF)-8)*s; if(i+1>4)-8)*s; } return; } + if(e->fmt==5){ const uint8_t *q=e->q4+(int64_t)tok*i3_rowbytes(D); /* int3-g64 */ + const float *sr=e->s+(int64_t)tok*i3_groups(D); int64_t ng=i3_groups(D); + for(int64_t g=0; g>2]>>((k&3)*2))&3)|(((hi[k>>3]>>(k&7))&1)<<2); + x[base+k]=(float)((int)u-4)*sr[g]; } } + return; } const uint8_t *q=e->q4+(int64_t)tok*((D+3)/4); float s=e->s[tok]; /* int2 */ for(int i=0;i>2]; int sh=(i&3)*2; x[i]=(float)((int)((byte>>sh)&3)-2)*s; } } @@ -1725,6 +1308,57 @@ static void *map_of_fd(int fd){ return base; } +/* ==================== DUAL-SSD: two model copies, two drives ==================== + * COLI_MODEL_MIRROR= registers a SECOND (read-only) copy of the model on + * another drive; expert reads are split between the two according to + * COLI_DISK_WEIGHTS=, (relative bandwidth; without the env it + * is measured at startup with the engine's own access pattern). Cold decode is + * disk-bound (~11 GB/token): two NVMe drives reading in parallel add up. */ +static const char *g_mirror_dir=NULL; /* COLI_MODEL_MIRROR / SNAP_MIRROR */ +static int g_mirror=0; /* 1 = mirror active (at least one shard accepted) */ +static int g_mir_share=64; /* expert share routed to the mirror, out of 256 */ +static _Atomic int64_t g_mir_bytes[2]; /* bytes served per drive: [0] primary, [1] mirror */ +static _Atomic int64_t g_mir_nread[2]; + +/* replica of one expert: DETERMINISTIC hash of (layer,eid). Determinism is a + * requirement, not a style choice: the readahead/PILOT WILLNEED and the demand + * pread must hit the same fd/page-cache, and in buffered mode an expert must + * never be cached twice (one copy per drive). */ +static inline int expert_route(int layer,int eid){ + if(!g_mirror) return 0; + uint32_t h=(uint32_t)layer*2654435761u ^ (uint32_t)eid*0x9E3779B9u; + h^=h>>16; h*=0x45d9f3bu; h^=h>>16; + return (int)(h&255) < g_mir_share; +} + +/* buffered fd of the replica, falling back to the primary if the file is not mirrored */ +static inline int rep_bfd(shards *S,int fd,int rep){ + int r=st_fd_rep(S,fd,rep); return r<0?fd:r; +} + +static int pread_full(int fd, void *buf, int64_t n, int64_t off, const char *tag); + +/* pread on the chosen replica with fallback to the primary on error/short-read: + * an unreadable sector (or an unmount) of the mirror must never kill the process + * when the primary can serve the same bytes. Delegates to pread_full so the + * mirror path inherits the short-read/EINTR loop and honest reporting (#236). + * Accounts bytes per drive. Returns 0 = ok, -1 = real error/EOF (like pread_full). */ +static ssize_t mir_pread(shards *S,int fd,int rep,void *buf,int64_t n,int64_t off,const char *tag){ + int rfd = st_fd_rep(S,fd,rep); + int used = rep && rfd>=0; + if(rfd<0) rfd=fd; + int rc=pread_full(rfd,buf,n,off,tag); + if(rc && used){ + static _Atomic int warned; + if(!atomic_exchange(&warned,1)) + fprintf(stderr,"[MIRROR] read error on the mirror copy — falling back to the primary drive\n"); + used=0; rc=pread_full(fd,buf,n,off,tag); + } + if(!rc){ atomic_fetch_add_explicit(&g_mir_bytes[used],n,memory_order_relaxed); + atomic_fetch_add_explicit(&g_mir_nread[used],1,memory_order_relaxed); } + return rc; +} + /* carica un expert nello slot. Container pre-quantizzato: le 3 matrici sono contigue nel * file -> UNA pread coalescente da ~19 MB dentro `slab` (+ le scale in fslab); i QT sono * viste dentro lo slab (zero copie). Fallback per modelli non quantizzati (oracolo tiny). @@ -1759,16 +1393,47 @@ static int pread_full(int fd, void *buf, int64_t n, int64_t off, const char *tag } return 0; } -static int expert_load_impl(Model *m, int layer, int eid, ESlot *s, int fatal){ +/* DISK-CLASS: is classification active right now? Single point of truth shared by + * moe()'s pre-bump snapshot (FASE A, below) and expert_load_impl's classify-and-read -- + * they must agree, or expert_load_impl reads a snapshot moe() never bothered to write. + * PROF=1 is the only trigger: this feature is measurement-only, the verdict never picks + * an fd (kept as one function anyway so a future policy consumer cannot drift out of + * sync with the snapshot writer by construction). */ +static int dc_needed(void){ return g_prof; } +/* DISK-CLASS: cold/warm verdict for ONE demand load, read from the pre-bump snapshot + * (Model's elast_pre) so routing's OWN bump for THIS call can't contaminate the read -- + * prefill is one giant moe() call where every newly-seen expert gets its `last` bumped + * a few lines above the load that made it "new"; classifying off the live, post-bump + * array would call the whole cold burst warm. Ages against the PRIVATE clock + * (eaccess_clock_dc, see its declaration in Model), NEVER the real eaccess_clock: kept + * separate by design (see elast_dc in Model for why DISK-CLASS keeps its own clock + * instead of reading the real one -- before #417/cfcc742 the real one also froze on the + * Metal pre-routed decode path, which would have made every prefill-touched expert warm + * forever and everything else cold forever; that's fixed upstream now, but DISK-CLASS + * still doesn't read the real clock, for the isolation property, not the freeze). + * Conservative-toward-warm on the one genuinely ambiguous input (missing snapshot -- + * defensive only, elast_pre is allocated everywhere elast is); a demonstrable first-ever + * access (last_pre==0) is not a judgment call, it stays cold regardless of that bias. */ +static int expert_classify(Model *m, int layer, int eid){ + if(!m->elast_pre || !m->elast_pre[layer]) return DC_WARM; /* no snapshot: label as the safe class */ + uint32_t last_pre=m->elast_pre[layer][eid]; + if(last_pre==0) return DC_COLD; /* never touched before this call: certain cold */ + uint32_t age=m->eaccess_clock_dc-last_pre; /* ticks since last access, PRE this call's bump */ + return age>g_direct_heat_ticks ? DC_COLD : DC_WARM; /* '>' not '>=': ties lean warm */ +} +static int expert_load_impl(Model *m, int layer, int eid, ESlot *s, int fatal, int demand){ #ifdef COLI_CUDA /* A live REPIN may reuse a GPU-enabled pinned slot for a different expert. * Keep its tier assignment, but invalidate the old device weights. */ if(s->eid!=eid){ qt_cuda_reset(&s->g); qt_cuda_reset(&s->u); qt_cuda_reset(&s->d); } #endif Cfg *c=&m->c; int I=c->moe_inter, D=c->hidden, b=m->ebits; - char nm[3][288]; const char *suf[3]={"gate_proj","up_proj","down_proj"}; + /* suf as a bounded char[][16] (not const char*) lets GCC prove the %s in the + * nm[k]/qn snprintfs can't overflow: worst key is "model.layers..mlp.experts..down_proj.weight" + * = 66 bytes incl NUL, well under nm[288] and qn[320]. See #484. */ + char nm[3][288]; const char suf[3][16]={"gate_proj","up_proj","down_proj"}; for(int k=0;k<3;k++) snprintf(nm[k],sizeof(nm[k]),"model.layers.%d.mlp.experts.%d.%s.weight",layer,eid,suf[k]); - char qn[300]; snprintf(qn,sizeof(qn),"%s.qs",nm[0]); + char qn[320]; snprintf(qn,sizeof(qn),"%s.qs",nm[0]); if(!st_has(&m->S,qn)){ /* fallback: tensori pieni, quantizza a runtime. * Reachable ONLY for unquantized models (no .qs); * GLM always has .qs, so the pilot never hits it. */ @@ -1792,21 +1457,20 @@ static int expert_load_impl(Model *m, int layer, int eid, ESlot *s, int fatal){ else { __atomic_add_fetch(&m->ld_main,1,__ATOMIC_RELAXED); __atomic_add_fetch(&m->bytes_main,(uint64_t)tb,__ATOMIC_RELAXED); } } + int rep=expert_route(layer,eid); /* DUAL-SSD: this expert's replica */ + if(rep && st_fd_rep(&m->S,tw[0]->fd,1)<0) rep=0; /* shard not in the mirror (partial) */ if(g_mmap){ void *bw[3],*bq[3]; int okm=1; for(int k=0;k<3;k++){ - bw[k]=map_of_fd(tw[k]->fd); bq[k]=map_of_fd(tq[k]->fd); + bw[k]=map_of_fd(rep_bfd(&m->S,tw[k]->fd,rep)); bq[k]=map_of_fd(rep_bfd(&m->S,tq[k]->fd,rep)); if(!bw[k]||!bq[k]||((tw[k]->off)&3)||((tq[k]->off)&3)) okm=0; } if(okm){ QT *qt[3]={&s->g,&s->u,&s->d}; int OO[3]={I,I,D}, II[3]={D,D,I}; for(int k=0;k<3;k++){ int64_t nb=tw[k]->nbytes; - int fmt=(nb==(int64_t)OO[k]*II[k])?1:(nb==(int64_t)OO[k]*((II[k]+1)/2))?2:3; - /* detect grouped int4 (fmt=4): int4 weight bytes + larger scale array */ int gs=0; - if(fmt==2) gs=detect_group_size(OO[k],II[k],tq[k]->nbytes); - if(gs>0) fmt=4; + int fmt=qt_resolve_fmt(tw[k]->name,OO[k],II[k],nb,tq[k]->nbytes,&gs); qt[k]->fmt=fmt; qt[k]->O=OO[k]; qt[k]->I=II[k]; qt[k]->gs=gs; qt[k]->qf=NULL; qt[k]->q8=(int8_t*)((char*)bw[k]+tw[k]->off); qt[k]->q4=(uint8_t*)((char*)bw[k]+tw[k]->off); qt[k]->s=(float*)((char*)bq[k]+tq[k]->off); @@ -1833,7 +1497,9 @@ static int expert_load_impl(Model *m, int layer, int eid, ESlot *s, int fatal){ * the final resident set only, after GPU release has already nulled out the * pointers for anything that isn't genuinely RAM-tier. */ atomic_fetch_add_explicit(&g_prof_io,(int64_t)(n+nq),memory_order_relaxed); + atomic_fetch_add_explicit(&g_mir_bytes[rep],tw[k]->nbytes+tq[k]->nbytes,memory_order_relaxed); } + atomic_fetch_add_explicit(&g_mir_nread[rep],1,memory_order_relaxed); s->eid=eid; return 0; } } @@ -1868,6 +1534,11 @@ static int expert_load_impl(Model *m, int layer, int eid, ESlot *s, int fatal){ if(ftot<0 || (uint64_t)ftot > SIZE_MAX/sizeof(float) || posix_memalign((void**)&s->fslab,16384,fb)){ fprintf(stderr,"OOM fslab\n"); if(fatal) exit(1); + /* unregister BEFORE freeing -- a stale g_slabs entry would let resolve() hand + * the GPU a pointer into freed memory (and under COLI_METAL_RESSET=1, leave the + * buffer a permanent residency-set member over it). Ported from e4/metal-heap + * validator fix 6753225; pre-existing gap on main/dev. */ + if(s->slab && g_metal_enabled) coli_metal_unregister(s->slab); compat_aligned_free(s->slab); s->slab=NULL; s->slab_cap=0; /* clean, hidden slot (eid stays -1) */ s->fslab=NULL; s->fslab_cap=0; return -1; } @@ -1889,52 +1560,79 @@ static int expert_load_impl(Model *m, int layer, int eid, ESlot *s, int fatal){ numa_slab_bind(s->fslab,(size_t)ftot*sizeof(float)); #endif } + /* DISK-CLASS: classify before the reads; computed unconditionally at dc_on sites so + * the timer (dc_t0) brackets exactly the read work, matching what the GB/s in the + * DISK-CLASS line describes. dc_on gates ALL of it off demand=0 call sites + * (pilot/repin/pin -- never classified, see the call sites) and off PROF=0 runs + * (dc_needed()) -- zero cost, zero behavior change there. The fd choice below is + * NOT influenced by the verdict: this is measurement only. */ + int dc_on = demand && dc_needed(); + int dc_cls = dc_on ? expert_classify(m,layer,eid) : DC_WARM; + double dc_t0 = dc_on ? now_s() : 0; + if(dc_on) dc_wall_enter(dc_cls,dc_t0); /* busy-wall open; EVERY exit path below must pair it */ int ord[3]={0,1,2}; /* ordina per offset nel file */ for(int a=0;a<3;a++) for(int bb=a+1;bb<3;bb++) if(tw[ord[bb]]->offoff){ int t=ord[a]; ord[a]=ord[bb]; ord[bb]=t; } int contig = tw[ord[0]]->fd==tw[ord[1]]->fd && tw[ord[1]]->fd==tw[ord[2]]->fd && tw[ord[0]]->off+tw[ord[0]]->nbytes==tw[ord[1]]->off && tw[ord[1]]->off+tw[ord[1]]->nbytes==tw[ord[2]]->off; - int64_t pos[3]; int done=0; + int64_t pos[3]; int done=0, dc_direct=0; if(contig){ int64_t off0=tw[ord[0]]->off; - int dfd = g_direct ? st_direct_fd(&m->S, tw[ord[0]]->fd) : -1; + int dfd = g_direct ? st_direct_fd_rep(&m->S, tw[ord[0]]->fd, rep) : -1; if(dfd>=0){ /* O_DIRECT: offset/len allineati a 4K */ int64_t base=off0 & ~4095LL, need=(off0-base)+wtot; int64_t len=(need+4095)&~4095LL; ssize_t r=pread(dfd, s->slab, len, base); if(r>=need){ pos[ord[0]]=off0-base; pos[ord[1]]=pos[ord[0]]+tw[ord[0]]->nbytes; - pos[ord[2]]=pos[ord[1]]+tw[ord[1]]->nbytes; done=1; + pos[ord[2]]=pos[ord[1]]+tw[ord[1]]->nbytes; done=1; dc_direct=1; + atomic_fetch_add_explicit(&g_mir_bytes[rep],(int64_t)r,memory_order_relaxed); + atomic_fetch_add_explicit(&g_mir_nread[rep],1,memory_order_relaxed); } } if(!done){ /* fallback bufferizzato */ - if(pread_full(tw[ord[0]]->fd, s->slab, wtot, off0, "pread expert")){ if(fatal) exit(1); return -1; } + if(mir_pread(&m->S, tw[ord[0]]->fd, rep, s->slab, wtot, off0, "pread expert")){ if(fatal) exit(1); + if(dc_on) dc_wall_exit(dc_cls,now_s()); /* pair the enter on the non-fatal unwind */ + return -1; } pos[ord[0]]=0; pos[ord[1]]=tw[ord[0]]->nbytes; pos[ord[2]]=tw[ord[0]]->nbytes+tw[ord[1]]->nbytes; done=1; } } if(!done){ /* non contigui: 3 pread bufferizzate */ int64_t o=0; for(int a=0;a<3;a++){ int k=ord[a]; - if(pread_full(tw[k]->fd, s->slab+o, tw[k]->nbytes, tw[k]->off, "pread expert")){ if(fatal) exit(1); return -1; } + if(mir_pread(&m->S, tw[k]->fd, rep, s->slab+o, tw[k]->nbytes, tw[k]->off, "pread expert")){ if(fatal) exit(1); + if(dc_on) dc_wall_exit(dc_cls,now_s()); /* pair the enter on the non-fatal unwind */ + return -1; } pos[k]=o; o+=tw[k]->nbytes; } } float *fp[3]; int64_t fo=0; /* scale (piccole) */ for(int k=0;k<3;k++){ - if(pread_full(tq[k]->fd, (char*)(s->fslab+fo), tq[k]->nbytes, tq[k]->off, "pread qs")){ if(fatal) exit(1); return -1; } + if(mir_pread(&m->S, tq[k]->fd, rep, (char*)(s->fslab+fo), tq[k]->nbytes, tq[k]->off, "pread qs")){ if(fatal) exit(1); + if(dc_on) dc_wall_exit(dc_cls,now_s()); /* pair the enter on the non-fatal unwind */ + return -1; } fp[k]=s->fslab+fo; fo+=tq[k]->nbytes/4; } atomic_fetch_add_explicit(&g_prof_io,wtot+fo*4,memory_order_relaxed); + if(dc_on){ /* DISK-CLASS accounting, see dc_needed() */ + double dc_t1=now_s(); /* one clock read for thread-ns AND the wall exit */ + int64_t bytes=wtot+fo*4; + atomic_fetch_add_explicit(&g_dc_n[dc_cls],1,memory_order_relaxed); + atomic_fetch_add_explicit(&g_dc_bytes[dc_cls],bytes,memory_order_relaxed); + atomic_fetch_add_explicit(&g_dc_ns[dc_cls],(int64_t)((dc_t1-dc_t0)*1e9),memory_order_relaxed); + dc_wall_exit(dc_cls,dc_t1); + if(dc_direct) /* which fd ACTUALLY served this class */ + atomic_fetch_add_explicit(&g_dc_direct_n[dc_cls],1,memory_order_relaxed); + } if(g_drop){ /* scarta subito le pagine: evita che la page - * cache in pressione strangoli il throughput */ - posix_fadvise(tw[ord[0]]->fd, tw[ord[0]]->off, wtot, POSIX_FADV_DONTNEED); - for(int k=0;k<3;k++) posix_fadvise(tq[k]->fd, tq[k]->off, tq[k]->nbytes, POSIX_FADV_DONTNEED); + * cache in pressione strangoli il throughput. + * The drop targets the fd of the replica READ. */ + posix_fadvise(rep_bfd(&m->S,tw[ord[0]]->fd,rep), tw[ord[0]]->off, wtot, POSIX_FADV_DONTNEED); + for(int k=0;k<3;k++) posix_fadvise(rep_bfd(&m->S,tq[k]->fd,rep), tq[k]->off, tq[k]->nbytes, POSIX_FADV_DONTNEED); } QT *qt[3]={&s->g,&s->u,&s->d}; int OO[3]={I,I,D}, II[3]={D,D,I}; for(int k=0;k<3;k++){ int64_t nb=tw[k]->nbytes; - int fmt = (nb==(int64_t)OO[k]*II[k])?1 : (nb==(int64_t)OO[k]*((II[k]+1)/2))?2 : 3; int gs=0; - if(fmt==2) gs=detect_group_size(OO[k],II[k],tq[k]->nbytes); - if(gs>0) fmt=4; + int fmt=qt_resolve_fmt(tw[k]->name,OO[k],II[k],nb,tq[k]->nbytes,&gs); qt[k]->fmt=fmt; qt[k]->O=OO[k]; qt[k]->I=II[k]; qt[k]->gs=gs; qt[k]->qf=NULL; qt[k]->q8=(int8_t*)(s->slab+pos[k]); qt[k]->q4=s->slab+pos[k]; qt[k]->s=fp[k]; } @@ -1942,9 +1640,15 @@ static int expert_load_impl(Model *m, int layer, int eid, ESlot *s, int fatal){ } /* Every expert read goes through here: time the whole load (pread/fault + * bookkeeping) on the thread that runs it, into the disk-service counter. */ -static int expert_load(Model *m, int layer, int eid, ESlot *s, int fatal){ +static int expert_load(Model *m, int layer, int eid, ESlot *s, int fatal, int demand){ + /* `demand` marks a routing-driven demand-load (moe()'s PIPE/OMP miss path, where the + * pre-bump elast_pre snapshot moe() just wrote is valid) -- pass 0 from anywhere else + * (pilot speculative loads, repin, startup PIN loading): those never run through THIS + * call's own FASE A, so the snapshot either doesn't apply or was never written for + * them, and DISK-CLASS deliberately leaves them unclassified -- see expert_classify()'s + * call site. */ double t0=now_s(); - int rc=expert_load_impl(m,layer,eid,s,fatal); + int rc=expert_load_impl(m,layer,eid,s,fatal,demand); atomic_fetch_add_explicit(&g_edisk_ns,(int64_t)((now_s()-t0)*1e9),memory_order_relaxed); return rc; } @@ -2000,7 +1704,7 @@ static int uring_load_add(UringBatch *b,Model *m,int layer,int eid,ESlot *s,int int li=b->nload++; UringLoad *l=&b->load[li]; memset(l,0,sizeof(*l)); l->m=m; l->s=s; l->layer=layer; l->eid=eid; l->fatal=fatal; - char nm[3][288],qn[300]; const char *suf[3]={"gate_proj","up_proj","down_proj"}; + char nm[3][288],qn[320]; const char suf[3][16]={"gate_proj","up_proj","down_proj"}; /* bounded suf: see #484 */ for(int k=0;k<3;k++) snprintf(nm[k],sizeof(nm[k]),"model.layers.%d.mlp.experts.%d.%s.weight",layer,eid,suf[k]); snprintf(qn,sizeof(qn),"%s.qs",nm[0]); if(g_mmap || !st_has(&m->S,qn)) @@ -2119,8 +1823,11 @@ static int uring_finalize_load(UringBatch *b,int li,int publish_eid){ for(int k=0;k<3;k++){ fp[k]=s->fslab+fo; fo+=l->tq[k]->nbytes/4; int64_t nb=l->tw[k]->nbytes; - int fmt=(nb==(int64_t)OO[k]*II[k])?1:(nb==(int64_t)OO[k]*((II[k]+1)/2))?2:3; - qt[k]->fmt=fmt; qt[k]->O=OO[k]; qt[k]->I=II[k]; qt[k]->qf=NULL; + /* qt_resolve_fmt like the other two expert paths: the raw ?1:?2:3 inference here + * missed grouped int4 (fmt=4, gs never set) and would mis-tag int3-g64 as int2. */ + int gs=0; + int fmt=qt_resolve_fmt(l->tw[k]->name,OO[k],II[k],nb,l->tq[k]->nbytes,&gs); + qt[k]->fmt=fmt; qt[k]->O=OO[k]; qt[k]->I=II[k]; qt[k]->gs=gs; qt[k]->qf=NULL; qt[k]->q8=(int8_t*)(s->slab+l->pos[k]); qt[k]->q4=s->slab+l->pos[k]; qt[k]->s=fp[k]; } if(publish_eid) s->eid=l->eid; @@ -2165,6 +1872,22 @@ static int g_pipe=0; /* PIPE=1: async expert-load pipeline. Default ON for * the matmul. PIPE=0 opts back into the blocking serial path. */ static int g_pipe_nw=8; /* PIPE_WORKERS=n: I/O worker threads (disk-parallel reads) */ static int g_uring=0; /* URING=1: Linux io_uring load/completion backend; implies PIPE */ +static int g_pipe_block=0;/* COLI_PIPE_BLOCK=1: pipe_wait blocca su una condvar invece dello + * spin sched_yield (default OFF = spin byte-identico). EN: a yield + * storm on the main thread fights the OpenMP team for cycles during + * multi-ms loads; the condvar wake costs ~5us against reads that + * cost 0.5-3ms (#159). Pthread pool only: the URING backend has no + * waiter spin to replace. */ +/* PIPE_WORKERS>0 esplicito nell'env implica PIPE=1: dimensionare il pool + * dichiara l'intento di usarlo (una campagna intera l'ha impostato con la + * pipe spenta senza accorgersene). EN: fires ONLY when PIPE is unset in the + * env AND the platform default left the pipe off (on _WIN32 it already + * defaults to 1) AND PIPE_WORKERS parses positive — the internal default of + * 8 does not count, PIPE_WORKERS=0/empty does not, and an explicit PIPE=0 + * always wins. */ +static int pipe_workers_imply_pipe(const char *pipe_env, const char *pw_env, int pipe_now){ + return !pipe_now && !pipe_env && pw_env && atoi(pw_env)>0; +} typedef struct { _Atomic uint64_t cur; /* (gen<<8)|index; gen main-only, index 0..njobs (≤64) */ _Atomic int njobs; /* current batch job count */ @@ -2172,6 +1895,7 @@ typedef struct { _Atomic int layer; /* current batch layer */ _Atomic int ready[64]; /* per-slot load-done flag */ pthread_mutex_t mx; pthread_cond_t cv; /* ONLY for parking/waking idle workers */ + pthread_cond_t cv_done; /* COLI_PIPE_BLOCK: signals ready[] transitions */ Model *m; pthread_t th[16]; int nw; int started; } PipePool; @@ -2194,8 +1918,13 @@ static void *pipe_worker(void *arg){ memory_order_acq_rel,memory_order_relaxed)){ int L =atomic_load_explicit(&p->layer,memory_order_relaxed); int eid=atomic_load_explicit(&p->eids[i],memory_order_relaxed); /* AFTER winning CAS */ - expert_load(p->m,L,eid,&p->m->ws[i],1); /* needed-now load: fatal on I/O error (matches serial path) */ + expert_load(p->m,L,eid,&p->m->ws[i],1,1); /* needed-now load: fatal on I/O error (matches serial path); demand=1: this IS moe()'s own miss path */ atomic_store_explicit(&p->ready[i],1,memory_order_release); + if(g_pipe_block){ /* wake a main thread parked in pipe_wait */ + pthread_mutex_lock(&p->mx); + pthread_cond_broadcast(&p->cv_done); + pthread_mutex_unlock(&p->mx); + } } /* CAS failed → another worker advanced index (or gen advanced): re-loop */ } @@ -2213,6 +1942,7 @@ static void pipe_init(Model *m){ g_pp.m=m; g_pp.nw=g_pipe_nw; if(g_pp.nw>16) g_pp.nw=16; if(g_pp.nw<1) g_pp.nw=1; atomic_store(&g_pp.cur,0); atomic_store(&g_pp.njobs,0); pthread_mutex_init(&g_pp.mx,NULL); pthread_cond_init(&g_pp.cv,NULL); + pthread_cond_init(&g_pp.cv_done,NULL); for(int i=0;islab); free(s->fslab); s->slab=NULL; s->fslab=NULL; s->slab_cap=s->fslab_cap=0; + if(s->aslab){ s->slab=NULL; s->fslab=NULL; } /* arena slice (#419): detach, keep caps, never free */ + else { compat_aligned_free(s->slab); free(s->fslab); s->slab=NULL; s->fslab=NULL; s->slab_cap=s->fslab_cap=0; } QT *q[3]={&s->g,&s->u,&s->d}; for(int k=0;k<3;k++){ q[k]->qf=NULL; q[k]->q8=NULL; q[k]->q4=NULL; q[k]->s=NULL; } m->resident_bytes-=bytes; if(m->resident_bytes<0) m->resident_bytes=0; } static void expert_host_ensure(Model *m, int layer, ESlot *s){ - if(!s->slab) expert_load(m,layer,s->eid,s,1); + if(s->slab) return; + if(s->aslab){ s->slab=s->aslab; s->fslab=s->afslab; } /* re-attach the arena slice; caps survived release */ + /* re-materializing a GPU-resident expert's host copy, not a routing miss: demand=0 */ + expert_load(m,layer,s->eid,s,1,0); /* rebuild the QT views (release NULLed them) + reload */ } #endif /* prefetch asincrono dei pesi di un expert (e delle sue scale .qs): avvia il readahead - * cosi' le letture sincrone successive trovano la page-cache calda. */ + * cosi' le letture sincrone successive trovano la page-cache calda. + * Sotto g_direct i PESI vengono letti con O_DIRECT (bypassa la page-cache, vedi + * expert_load): il WILLNEED su di essi scalda pagine che la lettura di domanda non + * consuma -> readahead sprecato sul disco, la risorsa piu' scarsa nello streaming. + * Le scale .qs restano SEMPRE bufferizzate (pread sul fd normale), quindi il loro + * WILLNEED resta utile anche con DIRECT=1. fadvise e' solo consultivo: saltarlo non + * cambia mai l'output (bit-identico), riduce solo I/O sprecato. + * The readahead targets the SAME replica that will serve the pread (expert_route + * is deterministic). + * EN: under O_DIRECT the weights bypass the page cache, so their WILLNEED is wasted; + * the .qs scales are always buffered, so keep theirs. Advisory hint -> output-preserving. */ static void expert_prefetch(Model *m, int layer, int eid){ - char nm[300]; + char nm[300]; int rep=expert_route(layer,eid); const char *suf[3]={"gate_proj.weight","up_proj.weight","down_proj.weight"}; for(int k=0;k<3;k++){ - snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.%d.%s",layer,eid,suf[k]); st_prefetch(&m->S,nm); - char qs[320]; snprintf(qs,sizeof(qs),"%s.qs",nm); st_prefetch(&m->S,qs); + snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.%d.%s",layer,eid,suf[k]); + if(!g_direct) st_prefetch_rep(&m->S,nm,rep); + char qs[320]; snprintf(qs,sizeof(qs),"%s.qs",nm); st_prefetch_rep(&m->S,qs,rep); } } @@ -2304,9 +2060,20 @@ static void qt_addrow(const QT *t, int row, float coef, float *acc){ acc[i] +=coef*scl[i/gs] *((int)(b&0xF)-8); acc[i+1]+=coef*scl[(i+1)/gs]*((int)(b>>4)-8); } if(I&1){ uint8_t b=w[I>>1]; acc[I-1]+=coef*scl[(I-1)/gs]*((int)(b&0xF)-8); } return; } + /* fmt=5 likewise before c: int3-g64 scales are per-GROUP [O,ng], not s[row] */ + if(t->fmt==5){ const uint8_t *w=t->q4+(int64_t)row*i3_rowbytes(I); + const float *sr=t->s+(int64_t)row*i3_groups(I); int64_t ng=i3_groups(I); + for(int64_t g=0; g>2]>>((k&3)*2))&3)|(((hi[k>>3]>>(k&7))&1)<<2); + acc[base+k]+=cg*(float)((int)u-4); } } + return; } float c=coef*t->s[row]; if(t->fmt==1){ const int8_t *w=t->q8+(int64_t)row*I; for(int i=0;ifmt==2){ const uint8_t *w=t->q4+(int64_t)row*((I+1)/2); +#if defined(__AVX512F__) && defined(__AVX512BW__) + axpy_i4f_avx512(w,c,acc,I); return; /* bit-identical: one fma per element */ +#endif for(int i=0;i+1>1]; acc[i]+=c*((int)(b&0xF)-8); acc[i+1]+=c*((int)(b>>4)-8); } if(I&1){ uint8_t b=w[I>>1]; acc[I-1]+=c*((int)(b&0xF)-8); } return; } const uint8_t *w=t->q4+(int64_t)row*((I+3)/4); @@ -2317,9 +2084,20 @@ static void qt_matvec_rows(const QT *t, int r0, int n, const float *x, float *y) int I=t->I; for(int j=0;jfmt==0){ const float *w=t->qf+(int64_t)row*I; for(int i=0;ifmt==4){ /* grouped int4: per-group scale */ + const uint8_t *w=t->q4+(int64_t)row*((I+1)/2); int gs=t->gs,ng=(I+gs-1)/gs; + const float *scl=t->s+(int64_t)row*ng; + for(int g=0;g*gsI)glen=I-base; float sc=scl[g]; float acc=0; + for(int i=base;i+1>1]; acc+=((int)(b&0xF)-8)*x[i]+((int)(b>>4)-8)*x[i+1]; } + if(glen&1){ uint8_t b=w[(base+glen-1)>>1]; acc+=((int)(b&0xF)-8)*x[base+glen-1]; } + a+=acc*sc; } } else if(t->fmt==1){ const int8_t *w=t->q8+(int64_t)row*I; float s=t->s[row]; float acc=0; for(int i=0;ifmt==2){ const uint8_t *w=t->q4+(int64_t)row*((I+1)/2); float s=t->s[row]; float acc=0; +#if defined(__AVX512F__) && defined(__AVX512BW__) + /* same accumulation-order tradeoff (and gate) as matmul_i4's I4_ACC512 path */ + if(g_i4_acc512 && !(I&31)){ y[j]=dot_i4f_avx512(w,x,I)*s; continue; } +#endif for(int i=0;i+1>1]; acc+=((int)(b&0xF)-8)*x[i]+((int)(b>>4)-8)*x[i+1]; } if(I&1){ uint8_t b=w[I>>1]; acc+=((int)(b&0xF)-8)*x[I-1]; } a=acc*s; } else if(t->fmt==4){ /* per-gruppo, come matmul_i4_grouped / per-group, as matmul_i4_grouped */ @@ -2329,6 +2107,13 @@ static void qt_matvec_rows(const QT *t, int r0, int n, const float *x, float *y) for(int i=base;i>1]; acc+=(float)((i&1)?((int)(b>>4)-8):((int)(b&0xF)-8))*x[i]; } a+=(double)acc*scl[g]; } } + else if(t->fmt==5){ const uint8_t *w=t->q4+(int64_t)row*i3_rowbytes(I); + const float *sr=t->s+(int64_t)row*i3_groups(I); int64_t ng=i3_groups(I); + for(int64_t g=0; g>2]>>((k&3)*2))&3)|(((hi[k>>3]>>(k&7))&1)<<2); + acc+=(float)((int)u-4)*x[base+k]; } + a+=(double)(acc*sr[g]); } } else { const uint8_t *w=t->q4+(int64_t)row*((I+3)/4); float s=t->s[row]; float acc=0; for(int i=0;i>2]; acc+=((int)((b>>((i&3)*2))&3)-2)*x[i]; } a=acc*s; } y[j]=(float)a; @@ -2337,6 +2122,14 @@ static void qt_matvec_rows(const QT *t, int r0, int n, const float *x, float *y) static int g_absorb=-1; #ifdef COLI_CUDA static int g_cuda_pipe=0; /* COLI_CUDA_PIPE=1: prefill attention chain resident on the layer home device */ +static int g_cuda_router=0; /* COLI_CUDA_ROUTER=1 (#431 PR-A): router on the layer home device at decode */ +static int g_cuda_resid=0; /* COLI_CUDA_RESID=1 (#431 PR-C0): expert-group results stay on device */ +/* pipe -> moe handoff for the resident group path (set per layer by + * pipe_layer_sparse, consumed by moe()'s group dispatch, cleared after take) */ +static int g_pres_home=-1; /* home device, -1 = path off */ +static const float *g_pres_xsrc; /* nrm_d on the home device */ +static float *g_pres_slots; /* [ndev][D] partial slots on home */ +static int g_pres_used[COLI_CUDA_MAX_DEVICES], g_pres_nused; #endif /* ABSORB: -1 auto (decode S<=4), 0 mai, 1 sempre (test) */ static int g_dsa_force=0; /* DSA_FORCE=1: selezione sempre attiva (test: top-min(k,T)=denso) */ static int cmp_fdesc(const void *a,const void *b){ @@ -2792,7 +2585,26 @@ static void attention_rows(Model *m, Layer *l, int layer, float *x, int S, int p for(int jj=0;jjLc[layer],t,kvl); const float *kr=coli_kv_row(ks->Rc[layer],t,c->qk_rope); - float a=0; for(int i=0;iqk_rope;d++) a+=qr[d]*kr[d]; sc[jj]=a*c->attn_scale; } @@ -2800,7 +2612,25 @@ static void attention_rows(Model *m, Layer *l, int layer, float *x, int S, int p float clat[512]; memset(clat,0,kvl*sizeof(float)); for(int jj=0;jjLc[layer],t,kvl); - float a=sc[jj]; for(int i=0;ikv_b, rbase+r0v, vh, clat, ctx+((int64_t)s*H+h)*vh); } } @@ -2880,6 +2710,15 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int pthread_mutex_unlock(&g_pilot_mx); } Cfg *c=&m->c; int D=c->hidden, E=c->n_experts, K=c->topk, I=c->moe_inter; + /* DISK-CLASS: does THIS call need the pre-bump recency snapshot? Must agree with + * dc_needed() in expert_load_impl -- that's what reads what this writes. touched[] + * makes the write once-per-call: an expert routed by more than one position in a big + * batch (prefill's S) must snapshot the state from BEFORE this call started, not from + * an earlier position's bump within the SAME call (which would reintroduce the + * same-call contamination one position later). Unconditional VLA like the FASE B + * `seen[E]` below -- E is small, cost is noise. */ + int need_classify = dc_needed(); + unsigned char touched[E]; if(need_classify) memset(touched,0,(size_t)E); float *choice=falloc(E); int sI=c->moe_inter*c->n_shared; /* Rank buffer for CACHE_ROUTE max-rank selection (up to all E experts). */ @@ -2899,7 +2738,8 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int /* router in UN matmul batch: stessa matematica, via le S chiamate S=1 */ float *logits_all=falloc((int64_t)S*E); int pre_routed=0; (void)pre_routed; -#ifdef COLI_METAL + /* pre-routed shortcut: Metal layer-CB o device router CUDA (#431) — stessa + * contabilita' (#417: recency clock incluso), la selezione arriva dalla GPU */ if(g_pre_idx){ /* routing gia' calcolata dal layer CB (GPU) */ memcpy(idxs,g_pre_idx,(size_t)S*K*sizeof(int)); memcpy(ws,g_pre_w,(size_t)S*K*sizeof(float)); @@ -2909,13 +2749,26 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int for(int kk=0;kkeusage[layer][idxs[(int64_t)s*K+kk]]++; ehit_mark(m,layer,idxs[(int64_t)s*K+kk]); + if(need_classify){ /* DISK-CLASS private recency -- snapshot BEFORE this call's own bump, + * then tick. This path also bumps the REAL elast/eaccess_clock a few + * lines below (#417/cfcc742 fixed the once-missing bump on Metal + * decode) -- DISK-CLASS's own clock stays independent regardless of + * that fix, see elast_dc in Model. */ + int e=idxs[(int64_t)s*K+kk]; + if(!touched[e]){ m->elast_pre[layer][e]=m->elast_dc[layer][e]; touched[e]=1; } + m->elast_dc[layer][e]=++m->eaccess_clock_dc; + } if(m->eheat[layer][idxs[(int64_t)s*K+kk]]eheat[layer][idxs[(int64_t)s*K+kk]]++; + /* #417: la scorciatoia GPU-prerouted deve far avanzare l'orologio di recency + * come il percorso router completo (riga ~3055), altrimenti elast/eaccess_clock + * si congelano a fine prefill e il tie-breaker LFRU di REPIN gira su punteggi + * stantii durante il decode su Metal. */ + m->elast[layer][idxs[(int64_t)s*K+kk]]=++m->eaccess_clock; } for(int d=0;drouter, S, D, E); if(!pre_routed) for(int s=0;seusage[layer][idx[kk]]++; ehit_mark(m,layer,idx[kk]); + if(need_classify){ /* DISK-CLASS private recency -- snapshot BEFORE this call's own bump, + * then tick (same rate as the real clock below: one per (s,kk)) */ + if(!touched[idx[kk]]){ m->elast_pre[layer][idx[kk]]=m->elast_dc[layer][idx[kk]]; touched[idx[kk]]=1; } + m->elast_dc[layer][idx[kk]]=++m->eaccess_clock_dc; + } if(m->eheat[layer][idx[kk]]eheat[layer][idx[kk]]++; m->elast[layer][idx[kk]]=++m->eaccess_clock; } @@ -3136,6 +2994,8 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int } /* ---- FASE C/D: risolvi (pin/cache/disco) e calcola, a blocchi di 64 unici ---- */ float *xg=falloc((int64_t)S*D), *gg=falloc((int64_t)S*I), *uu=falloc((int64_t)S*I), *hh=falloc((int64_t)S*D); + float *xe=NULL; /* fmt=6: x under the rotation Q^T, built once per call — all routed + * experts of the layer share it (the placement rule in quant.h) */ int *rows=malloc(S*sizeof(int)); float *rw=malloc(S*sizeof(float)); #ifdef COLI_CUDA /* PIPE Inc.1b: il batch-union del prefill passa dai gruppi GPU — prima di @@ -3230,7 +3090,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int * are timed as service inside expert_load */ } else { double t0=now_s(); /* ORIGINALE: blocking parallel load */ #pragma omp parallel for schedule(dynamic,1) - for(int q=0;qws[q],1); + for(int q=0;qws[q],1,1); /* demand=1: this IS the miss path */ m->t_ewait += now_s()-t0; } /* compute thread blocked for the whole load */ } /* I/O ASINCRONO: readahead (WILLNEED) del blocco SUCCESSIVO mentre calcoliamo @@ -3247,6 +3107,11 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int } #ifdef COLI_CUDA ESlot *group_e[64]; int group_n[64]; int ngroup=0; + /* Inc.4 overlap stash: pass-1 packing kept for the take phase after the CPU loop */ + ESlot *eg_e[64]; int eg_n[64], eg_row[64][4], eg_npg=0; float eg_w[64][4]; + int dev_nc0[COLI_CUDA_MAX_DEVICES], dev_off0[COLI_CUDA_MAX_DEVICES], + dev_total0[COLI_CUDA_MAX_DEVICES], dev_which0[COLI_CUDA_MAX_DEVICES][64]; + memset(dev_nc0,0,sizeof(dev_nc0)); (void)eg_npg; (void)dev_total0; (void)dev_off0; #endif #ifdef COLI_METAL if(g_metal_enabled){ @@ -3276,8 +3141,149 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int } #undef MB_BUILD #endif - if(!metal_done) +#ifdef COLI_CUDA + /* Inc.4 pass 1: collect the VRAM-resident experts' groups and ISSUE them async + * BEFORE the CPU loop below, so the GPU computes its share while the CPU works + * through the RAM-tier/miss rows — t_emm becomes max(cpu, gpu) instead of the + * sum. Only resident experts are collected (misses are never cuda_eligible), so + * no pipe_wait is needed here; the CPU loop keeps its own waits. Any issue + * failure drops the layer back to the collect-in-loop + sync-group path. */ + int early_issued=0, done_j[64]={0}; + { + static int g_group_async2=-1; + if(g_group_async2<0) g_group_async2=getenv("COLI_GROUP_ASYNC")?atoi(getenv("COLI_GROUP_ASYNC")):0; + if(!metal_done && g_group_async2 && group_enabled && S<=4 && g_cuda_enabled && + g_cuda_ndev>0 && !omp_in_parallel()){ + ESlot *pg_e[64]; int pg_n[64], pg_j[64], npg=0; + int prow[64][4]; float pw[64][4]; + for(int j=0;jg.cuda_eligible&&e->u.cuda_eligible&&e->d.cuda_eligible)) continue; + int nr=0; + for(int s=0;sg.cuda_device==g_cuda_devices[di]) pd_total[di]+=pg_n[q]; + for(int di=1;dig.cuda_device==device){ + int nc=pd_nc[di]++; ESlot *e=pg_e[q]; + pd_g[di][nc]=e->g.cuda; pd_u[di][nc]=e->u.cuda; pd_d[di][nc]=e->d.cuda; + pd_rows[di][nc]=pg_n[q]; pd_which[di][nc]=q; + for(int r=0;rt_emm+=now_s()-tg0; + for(int q=0;qgpu_expert_calls++; + } + } else { + for(int di=0;dig.fmt!=2||e->u.fmt!=2||e->d.fmt!=2||e->g.I!=D||e->g.O!=I||e->d.I!=I||e->d.O!=D){ allq4=0; break; } } + if(allq4){ + double t0=now_s(); + float wj[64]; int okw=1; + for(int j=0;jg.q4, *qu=e->u.q4; + float *gj=GG+(int64_t)j*I, *uj=UU+(int64_t)j*I; + for(int o=c0;og.s[o]*sx0; + for(int o=c0;ou.s[o]*sx0; + for(int o=c0;od.q4; const int8_t *gq=GQ+(int64_t)j*I; + float *hj=HH+(int64_t)j*D; + for(int o=c0;od.s[o]*gsc[j]; + } + } + for(int j=0;jt_emm+=dt; + if(g_prof){ m->t_ecpu+=dt; m->cpu_expert_rows+=(uint64_t)nb; + for(int j=0;jcpu_expert_bytes+=qt_bytes(&use[j]->g)+qt_bytes(&use[j]->u)+qt_bytes(&use[j]->d); } + free(xq8); free(GG); free(UU); free(HH); + xexp_done=1; + } + } + } + if(!metal_done && !xexp_done) for(int j=0;j=1 routing invariant. @@ -3302,7 +3308,12 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int ngroup++; continue; } #endif - for(int r=0;rg.fmt==6){ + if(!xe){ xe=falloc((int64_t)S*D); memcpy(xe,x,(size_t)S*D*sizeof(float)); e8_rot_rows(xe,S,D); } + xsrc=xe; + } + for(int r=0;rg.cuda_eligible && e->u.cuda_eligible && @@ -3316,6 +3327,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int #endif expert_gate_up(gg,uu,xg,&e->g,&e->u,nr); for(int64_t z=0;z<(int64_t)nr*I;z++) gg[z]=siluf(gg[z])*uu[z]; + if(e->d.fmt==6) e8_rot_rows(gg,nr,I); /* down input is per-expert — rotate here */ matmul_qt(hh, gg, &e->d, nr); for(int r=0;rcpu_expert_rows+=(uint64_t)nr;} } #ifdef COLI_CUDA + /* Inc.4 take phase: the CPU loop above ran while the GPU computed the issued + * groups — collect them now. A failed device recomputes its experts on the CPU + * (expert_host_ensure reloads slabs released by CUDA_RELEASE_HOST). */ + if(early_issued){ + double tg1=now_s(); + g_ovl_cpu+=tg1-g_ovl_mark; /* CPU-row window between issue and take */ + for(int di=0;dig,&e->u,nr); + for(int64_t z=0;z<(int64_t)nr*I;z++) gg[z]=siluf(gg[z])*uu[z]; + matmul_qt(hh,gg,&e->d,nr); + for(int r=0;rt_emm+=now_s()-tg1; g_ovl_take+=now_s()-tg1; + } ColiCudaTensor *dev_g[COLI_CUDA_MAX_DEVICES][64],*dev_u[COLI_CUDA_MAX_DEVICES][64]; ColiCudaTensor *dev_d[COLI_CUDA_MAX_DEVICES][64]; int dev_rows[COLI_CUDA_MAX_DEVICES][64],dev_which[COLI_CUDA_MAX_DEVICES][64]; @@ -3345,14 +3387,58 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int } } double tg=now_s(); + /* Inc.4: at decode scale, issue every device's group WITHOUT syncing, then take + * them all — one stream sync per device per layer instead of a full staged + * round-trip per call (measured: ~70% of the sync call is host-side wait). + * Any issue failure drains what was issued and the whole layer falls back to + * the sync path below, which recomputes from group_x (idempotent). */ + int async_done=0; + static int g_group_async=-1; + if(g_group_async<0) g_group_async=getenv("COLI_GROUP_ASYNC")?atoi(getenv("COLI_GROUP_ASYNC")):0; + if(g_group_async && S<=4 && g_cuda_ndev>0){ + int issued[COLI_CUDA_MAX_DEVICES]={0}, all=1; + for(int di=0;di=0 && S==1){ + for(int di=0;di1) schedule(static) - for(int di=0;diecap){ slot=nn; isnew=1; } - else { int lru=0; for(int z=1;z=2 demand accesses) AND clearly hotter than the speculation by tier_pick_lfru's + * 25%+4-freq hysteresis. The original #441 formula tested the speculation's score + * against victim+margin, which — because a speculation is by definition historically + * colder than a just-used demand expert — dropped ~all speculations once the cache + * was full, collapsing the LRU hit share (#490). Cache placement only -> output + * byte-identical (a dropped speculation is demand-loaded later, same value). */ + if(g_pilot_evict_guard && m->eheat && m->elast && Sl[lru].eid>=0){ + int vid=Sl[lru].eid; uint32_t vh=m->eheat[layer][vid]; + if(vh>=2){ + uint64_t vs=tier_lfru_score(vh,m->elast[layer][vid],m->eaccess_clock); + uint64_t cs=tier_lfru_score(m->eheat[layer][eid],m->elast[layer][eid],m->eaccess_clock); + if(vs+(vs>>2)+(4u<<8)>cs){ atomic_fetch_add_explicit(&g_pilot_drops,1,memory_order_relaxed); + pthread_mutex_unlock(&g_pilot_mx); return; } } } } ESlot *dst=&Sl[slot]; dst->eid=-1; /* nascondi dagli scan-hint mentre carica */ g_pilot_inflight[layer]++; pthread_mutex_unlock(&g_pilot_mx); - int rc=expert_load(m,layer,eid,dst,0); /* pread VERO — fuori dal lock, sovrapposto al compute; fatal=0: un errore su una speculazione NON deve uccidere il server */ + int rc=expert_load(m,layer,eid,dst,0,0); /* pread VERO — fuori dal lock, sovrapposto al compute; fatal=0: un errore su una speculazione NON deve uccidere il server; demand=0: speculative, never classified */ pthread_mutex_lock(&g_pilot_mx); if(rc==0){ @@ -3581,6 +3682,19 @@ static void pilot_uring_batch(Model *m){ if(Sl[z].eid< -1) continue; /* URING reservation in flight */ if(slot<0 || Sl[z].used=2 accesses) + * AND clearly hotter (25%+4-freq hysteresis). See pilot_realload for the rationale and + * the #490 regression the original formula caused. */ + if(slot>=0 && Sl[slot].eid>=0 && g_pilot_evict_guard && m->eheat && m->elast){ + int vid=Sl[slot].eid; uint32_t vh=m->eheat[layer][vid]; + if(vh>=2){ + uint64_t vs=tier_lfru_score(vh,m->elast[layer][vid],m->eaccess_clock); + uint64_t cs=tier_lfru_score(m->eheat[layer][eid],m->elast[layer][eid],m->eaccess_clock); + if(vs+(vs>>2)+(4u<<8)>cs){ atomic_fetch_add_explicit(&g_pilot_drops,1,memory_order_relaxed); + pthread_mutex_unlock(&g_pilot_mx); continue; } + } + } } if(slot<0){ atomic_fetch_add_explicit(&g_pilot_drops,1,memory_order_relaxed); pthread_mutex_unlock(&g_pilot_mx); continue; } ESlot *dst=&Sl[slot]; @@ -3791,17 +3905,28 @@ static int pipe_layer_sparse(Model *m, Layer *l, int li, float *x_dev, int S, in if(!l->sh_gate.cuda_eligible||!l->sh_up.cuda_eligible||!l->sh_down.cuda_eligible|| !qt_cuda_upload(&l->sh_gate)||!qt_cuda_upload(&l->sh_up)||!qt_cuda_upload(&l->sh_down)|| l->sh_gate.cuda_device!=dev||l->sh_up.cuda_device!=dev||l->sh_down.cuda_device!=dev) return 0; - float *w_in =coli_cuda_pipe_scratch(dev,8,(size_t)D*4); - float *w_post=coli_cuda_pipe_scratch(dev,9,(size_t)D*4); + /* Inc.4: the layernorm weights are constants — upload once per layer and keep them + * on the layer's device, instead of two synchronous 24 KB uploads per layer per + * token (152 sync H2D/token measured on the profile). */ + if(!m->ln_dev) m->ln_dev=calloc((size_t)(c->n_layers+1)*2,sizeof(float*)); + float *w_in=m->ln_dev[(size_t)li*2], *w_post=m->ln_dev[(size_t)li*2+1]; + if(!w_in){ + w_in=coli_cuda_pipe_alloc(dev,(size_t)D*4); + if(!w_in||!coli_cuda_pipe_upload(dev,w_in,l->in_ln,(size_t)D*4)) return 0; + m->ln_dev[(size_t)li*2]=w_in; + } + if(!w_post){ + w_post=coli_cuda_pipe_alloc(dev,(size_t)D*4); + if(!w_post||!coli_cuda_pipe_upload(dev,w_post,l->post_ln,(size_t)D*4)) return 0; + m->ln_dev[(size_t)li*2+1]=w_post; + } float *nrm_d=coli_cuda_pipe_scratch(dev,10,xb); float *y_d =coli_cuda_pipe_scratch(dev,11,xb); float *sg_d =coli_cuda_pipe_scratch(dev,12,(size_t)S*sI*4); float *su_d =coli_cuda_pipe_scratch(dev,13,(size_t)S*sI*4); float *snap =coli_cuda_pipe_scratch(dev,14,xb); - if(!w_in||!w_post||!nrm_d||!y_d||!sg_d||!su_d||!snap) return 0; + if(!nrm_d||!y_d||!sg_d||!su_d||!snap) return 0; if(!coli_cuda_pipe_peer_copy(dev,snap,dev,x_dev,xb)) return 0; /* snapshot per il fallback */ - if(!coli_cuda_pipe_upload(dev,w_in,l->in_ln,(size_t)D*4)|| - !coli_cuda_pipe_upload(dev,w_post,l->post_ln,(size_t)D*4)) return 0; double ta=now_s(); if(!coli_cuda_pipe_rmsnorm(dev,nrm_d,x_dev,w_in,S,D,c->eps)) return 0; /* DSA: i layer con indexer FULL cachano k_idx dalla nrm pre-attention (CPU, piccolo) */ @@ -3819,6 +3944,36 @@ static int pipe_layer_sparse(Model *m, Layer *l, int li, float *x_dev, int S, in if(!attn_pipe_prefill(m,l,li,nrm_d,1,S,pos_base,NULL,y_d)) return 0; if(!coli_cuda_pipe_add(dev,x_dev,y_d,(size_t)S*D)) return 0; /* prima mutazione */ if(!coli_cuda_pipe_rmsnorm(dev,nrm_d,x_dev,w_post,S,D,c->eps)) return 0; + /* device router (#431 PR-A): route THIS row on the home device while the + * stream is still hot, then hand the selection to moe() through the same + * pre-routed shortcut the Metal layer-CB uses. Any failure (upload, launch, + * feature gate) falls back to the CPU router inside moe() — byte-identical + * behaviour, just slower. Gated to the plain routing path: CACHE_ROUTE / + * ROUTE_P / ROUTE_TRACE keep the CPU ranking they need. */ + static int lr_idx[64]; static float lr_w[64]; static int lr_keff[1]; + int dev_routed=0; + if(g_cuda_router && S==1 && !g_cache_route && g_route_p<=0.f && !g_route_fp + && c->n_experts<=4096 && c->topk<=64 && !l->router_cuda_bad){ + int E=c->n_experts, K=c->topk; + int Ksel = g_topk>0 ? (g_topk0 && g_topp<1.f) ? g_topp : 0.f; + if(!l->router_cuda){ + void *rw=coli_cuda_pipe_alloc(dev,(size_t)E*D*4); + void *rb=coli_cuda_pipe_alloc(dev,(size_t)E*4); + if(rw&&rb&&coli_cuda_pipe_upload(dev,rw,l->router,(size_t)E*D*4) + &&coli_cuda_pipe_upload(dev,rb,l->router_bias,(size_t)E*4)){ + l->router_cuda=rw; l->router_bias_cuda=rb; + } else { + if(rw)coli_cuda_pipe_free(dev,rw); if(rb)coli_cuda_pipe_free(dev,rb); + l->router_cuda_bad=1; + } + } + if(l->router_cuda && + coli_cuda_pipe_router(dev,nrm_d,l->router_cuda,l->router_bias_cuda, + D,E,Ksel,tp,c->norm_topk,c->routed_scale, + lr_idx,lr_w,lr_keff)) + dev_routed=1; + } if(!coli_cuda_pipe_download(dev,nrm_d,nrm_host,xb)) return 0; m->t_attn+=now_s()-ta; /* OVERLAP: issue the shared expert on the GPU BEFORE moe() runs on the CPU. @@ -3849,7 +4004,30 @@ static int pipe_layer_sparse(Model *m, Layer *l, int li, float *x_dev, int S, in if(!coli_cuda_pipe_add(dev,x_dev,y_d,(size_t)S*D)) return 0; /* shared residual (async) */ m->t_emm += now_s()-te; /* shared-expert GPU dispatch only */ /* expert routed su CPU/gruppi GPU come oggi (shared saltata: la fa il device) */ + if(dev_routed){ g_pre_idx=lr_idx; g_pre_w=lr_w; g_pre_keff=lr_keff; } + float *res_acc=NULL; + if(g_cuda_resid && S==1){ + g_pres_slots=coli_cuda_pipe_scratch(dev,25,(size_t)g_cuda_ndev*D*sizeof(float)); + res_acc =coli_cuda_pipe_scratch(dev,26,(size_t)D*sizeof(float)); + if(g_pres_slots && res_acc){ g_pres_home=dev; g_pres_xsrc=nrm_d; g_pres_nused=0; } + else { g_pres_slots=NULL; res_acc=NULL; } + } moe(m,l,li,nrm_host,S,out_host,0); /* self-times its own t_emm */ + if(dev_routed){ g_pre_idx=NULL; g_pre_w=NULL; g_pre_keff=NULL; } + if(g_pres_home>=0){ + int nused=g_pres_nused; g_pres_home=-1; g_pres_xsrc=NULL; + if(nused>0){ + /* partials are in flight toward our slots; take() orders the legacy + * stream behind every issue event and reduces in issue order. A + * failure here would silently drop routed experts — that is a wrong + * answer, not a slow one, so it is fatal by design. */ + if(!coli_cuda_expert_group_resident_take(dev,g_pres_used,nused,g_pres_slots,res_acc,D)|| + !coli_cuda_pipe_add(dev,x_dev,res_acc,(size_t)D)){ + fprintf(stderr,"[CUDA] resident expert take failed — refusing to drop routed experts\n"); + exit(1); + } + } + } te=now_s(); if(!coli_cuda_pipe_upload(dev,y_d,out_host,xb)) return 0; /* sync: waits for moe */ if(!coli_cuda_pipe_add(dev,x_dev,y_d,(size_t)S*D)) return 0; /* routed residual (async) */ @@ -4212,23 +4390,45 @@ static void mtp_absorb(Model *m, const int *next_ids, const float *x, int S, int free(hx); free(cat); free(e); free(hn); free(hf); free(nrm); free(tmp); } -static inline int argmax_v(const float *lo, int V){ - /* skip NaN (x==x is false for NaN) so a poisoned logit can't pin the argmax - * to index 0 — pick the max finite/+Inf entry instead. */ - int b=-1; float bv=-INFINITY; - for(int i=0;ibv){ bv=x; b=i; } } - return b<0?0:b; -} + +static void repin_pass_limit(Model *m,int limit); +static void repin_pass(Model *m){ repin_pass_limit(m,16); } /* ---- METODO F: draft grammaticale (#48) ---- * gr_feed consuma i byte di ogni token EMESSO e tiene il walker in sync con l'output; * grammar_draft propone lo span FORZATO successivo (un solo byte legale per posizione) * gia' tokenizzato. Il confine di tokenizzazione non e' garantito coincidere con quello * del modello: la verifica assorbe la differenza (al peggio l'ultimo draft e' rifiutato). */ -static void grammar_setup(Tok *T){ +/* Compile grammar TEXT into g: raw GBNF, or — if the first non-space byte is '{' + * — a JSON-Schema compiled via schema_gbnf.h. Takes ownership of txt (always + * freed). Fail-soft: returns -1 with g->on=0, engine runs without a grammar. */ +static int grammar_setup_text(GrDraft *g, Tok *T, char *txt, const char *label){ + const char *p=txt; while(*p==' '||*p=='\t'||*p=='\n'||*p=='\r') p++; + if(*p=='{'){ + char serr[160]; + char *gbnf=schema_to_gbnf(txt,serr,sizeof serr); + free(txt); + if(!gbnf){ fprintf(stderr,"[SCHEMA] %s: %s (running without grammar)\n",label,serr); return -1; } + txt=gbnf; + } + if(gr_parse(&g->gram,txt)){ fprintf(stderr,"[GRAMMAR] %s: %s\n",label,g->gram.err); free(txt); return -1; } + free(txt); + gr_state_init(&g->st,&g->gram); + if(!g->st.alive){ fprintf(stderr,"[GRAMMAR] %s: grammar cannot be evaluated (left recursion?)\n",label); return -1; } + if(g->max<1) g->max=24; + if(g->max>48) g->max=48; + g->T=T; g->on=1; g->armed=0; + fprintf(stderr,"[GRAMMAR] %s: %d rules, forced span capped at %d tokens/forward\n",label,g->gram.n,g->max); + return 0; +} +/* Release a per-request grammar so the slot can host the next request (keeps max). */ +static void grammar_teardown(GrDraft *g){ + if(g->gram.n) gr_free(&g->gram); + int max=g->max; memset(g,0,sizeof(*g)); g->max=max; +} +static void grammar_setup(GrDraft *g, Tok *T){ /* GRAMMAR= takes precedence; SCHEMA= compiles a JSON-Schema - * to GBNF (schema_gbnf.h) for the same draft source. Both fail soft: the engine - * runs without a grammar and output is unchanged. */ + * to GBNF. Both fail soft: the engine runs without a grammar, output unchanged. */ const char *gf=getenv("GRAMMAR"); const char *sf=(gf&&*gf)?NULL:getenv("SCHEMA"); if((!gf||!*gf)&&(!sf||!*sf)) return; @@ -4240,59 +4440,45 @@ static void grammar_setup(Tok *T){ if(!txt || fread(txt,1,(size_t)n,f)!=(size_t)n){ fprintf(stderr,"[GRAMMAR] failed to read %s\n",path); fclose(f); free(txt); return; } fclose(f); txt[n]=0; - if(sf){ /* schema -> GBNF, then the same gr_parse as the GRAMMAR path */ - char serr[160]; - char *gbnf=schema_to_gbnf(txt,serr,sizeof serr); - free(txt); - if(!gbnf){ fprintf(stderr,"[SCHEMA] %s: %s (running without grammar)\n",sf,serr); return; } - txt=gbnf; - } - if(gr_parse(&g_gram,txt)){ fprintf(stderr,"[GRAMMAR] %s: %s\n",path,g_gram.err); free(txt); return; } - free(txt); - gr_state_init(&g_gst,&g_gram); - if(!g_gst.alive){ fprintf(stderr,"[GRAMMAR] %s: grammar cannot be evaluated (left recursion?)\n",path); return; } - if(getenv("GRAMMAR_DRAFT")) g_gr_max=atoi(getenv("GRAMMAR_DRAFT")); - if(g_gr_max<1) g_gr_max=1; - if(g_gr_max>48) g_gr_max=48; - g_gr_T=T; g_gr_on=1; - fprintf(stderr,"[GRAMMAR] %s: %d rules, forced span capped at %d tokens/forward\n",path,g_gram.n,g_gr_max); + if(getenv("GRAMMAR_DRAFT")) g->max=atoi(getenv("GRAMMAR_DRAFT")); + grammar_setup_text(g,T,txt,path); } /* stato pulito all'inizio di ogni RISPOSTA (non tra i \x02MORE, che continuano) */ -static void grammar_reset(void){ - if(!g_gr_on) return; - gr_state_init(&g_gst,&g_gram); g_gr_armed=0; - if(!g_gst.alive) g_gr_on=0; +static void grammar_reset(GrDraft *g){ + if(!g->on) return; + gr_state_init(&g->st,&g->gram); g->armed=0; + if(!g->st.alive) g->on=0; } /* consuma i byte di un token emesso. Preambolo (prima dell'arming): ignorato. * Desync dopo l'arming: si riarma in attesa del prossimo inizio valido — al peggio * i draft vengono rifiutati dalla verifica, l'output non cambia MAI. */ -static void gr_feed(int t){ - if(!g_gr_on||!g_gr_T) return; - char b[64]; int n=tok_decode(g_gr_T,&t,1,b,63); +static void gr_feed(GrDraft *g, int t){ + if(!g->on||!g->T) return; + char b[64]; int n=tok_decode(g->T,&t,1,b,63); for(int i=0;ist,(unsigned char)b[i]); + if(r==1){ g->armed=1; continue; } + if(r<0){ g->on=0; return; } /* walker spento: fine dei draft */ + if(!g->armed) continue; /* preambolo: aspetta l'inizio */ + gr_state_init(&g->st,&g->gram); g->armed=0; /* desync: riparti dalla radice */ + if(!g->st.alive){ g->on=0; return; } + if(gr_accept(&g->st,(unsigned char)b[i])==1) g->armed=1; } } /* propone lo span forzato come token (max cap); 0 se la grammatica dirama qui */ -static int grammar_draft(int *draft, int cap){ - if(!g_gr_on||!g_gr_armed||!g_gr_T||cap<1) return 0; - if(g_gr_prop>=32 && g_gr_acc*2on||!g->armed||!g->T||cap<1) return 0; + if(g->prop>=32 && g->acc*2prop){ /* guardia adattiva, come per MTP: acceptance sotto il 50% = tokenizzazione fuori asse, meglio spegnersi */ - g_gr_on=0; + g->on=0; fprintf(stderr,"[GRAMMAR] %.0f%% acceptance after %llu proposals: grammar drafts disabled\n", - 100.0*g_gr_acc/g_gr_prop,(unsigned long long)g_gr_prop); + 100.0*g->acc/g->prop,(unsigned long long)g->prop); return 0; } - char fb[512]; int nb=gr_forced(&g_gst,fb,(int)sizeof fb-1); + char fb[512]; int nb=gr_forced(&g->st,fb,(int)sizeof fb-1); if(nb<=0) return 0; - int g=tok_encode(g_gr_T,fb,nb,draft,cap); - return g>0?g:0; + int nt=tok_encode(g->T,fb,nb,draft,cap); /* renamed local: 'g' is now the state param */ + return nt>0?nt:0; } /* ---- SAMPLING (temperatura + nucleus) con verifica speculativa LOSSLESS ---- @@ -4300,113 +4486,6 @@ static int grammar_draft(int *draft, int cap){ * Rejection sampling di Leviathan: accetta il draft x_d con prob p(x_d); al rifiuto * ricampiona da p con x_d azzerato e rinormalizzato. La distribuzione risultante e' * ESATTAMENTE p: la speculazione resta invisibile all'output anche col sampling. */ -static uint64_t g_rng=0x9E3779B97F4A7C15ULL; -static inline double rndu(void){ g_rng^=g_rng<<13; g_rng^=g_rng>>7; g_rng^=g_rng<<17; - return (double)(g_rng>>11)*(1.0/9007199254740992.0); } -static float *g_pbuf=NULL; static int *g_pidx=NULL; /* buffer riusati (decode single-thread) */ -/* sift-down su max-heap in h[0..n), chiave = g_pbuf[h[i]] (#335: partial top-p select). - * Versione "a buco": porta il valore di radice e lo deposita solo alla fine, cosi' - * heapify e' O(V) e ogni pop e' O(log n) senza qsort sull'intero vocabolario. */ -static void topp_siftdown(int *h, int n, int i){ - int iv=h[i]; float kv=g_pbuf[iv]; - for(;;){ int l=2*i+1; - if(l>=n) break; /* foglia */ - int b=l; if(l+1g_pbuf[h[l]]) b=l+1; /* figlio maggiore */ - if(g_pbuf[h[b]]<=kv) break; /* nessun figlio supera la radice -> ferma */ - h[i]=h[b]; i=b; } - h[i]=iv; -} -/* costruisce in g_pbuf la distribuzione target: softmax(lo/temp) troncata a top-p g_nuc. - * Invariante per dist_sample: g_pbuf resta INDICIZZATO per token-id (mai riordinato); - * la coda troncata va AZZERATA in g_pbuf (dist_sample la legge direttamente per id). */ -static void dist_build(const float *lo, int V){ - if(!g_pbuf){ g_pbuf=falloc(V); g_pidx=malloc(V*sizeof(int)); } - /* Un solo logit NaN/+Inf avvelenava tutto (#369): +Inf diventava mx, NaN/Inf-mx - * -> expf NaN -> s NaN -> ogni prob NaN -> dist_sample cade sul fallback - * `g_pbuf[i]>0` (NaN>0 e' falso ovunque) e ritorna 0 PER SEMPRE, in silenzio. - * Difesa: mx solo sui finiti; un logit non finito contribuisce prob 0; - * se la distribuzione degenera (tutti non finiti / somma non valida) si - * ripiega sull'argmax dei finiti e si avvisa UNA volta, mai in silenzio. */ - int mxi=-1; float mx=0; - for(int i=0;imx)){ mx=lo[i]; mxi=i; } - double s=0; float invt=1.f/(g_temp>1e-4f?g_temp:1e-4f); - if(mxi>=0){ - for(int i=0;i=0)?mxi:0; /* mxi = argmax dei logit FINITI (robusto - * anche se lo[0] e' NaN, dove argmax_v fallirebbe) */ - for(int i=0;i0 && g_nuc<1.f){ - for(int i=0;i=0;i--) topp_siftdown(g_pidx,V,i); /* heapify O(V) */ - /* pop verso la coda: i vincitori (testa top-p) cadono in g_pidx[out..V-1] in ordine - * DECRESCENTE, come il vecchio qsort, quindi s2 accumula nello stesso ordine -> - * head bit-identical sui casi senza pareggi (i pareggi erano gia' non specificati - * sotto il qsort instabile e restano tali). Il prefisso g_pidx[0..out-1) e' la coda. */ - double s2=0, cum=0; int out=V; - do{ int root=g_pidx[0]; /* massimo corrente */ - g_pidx[0]=g_pidx[--out]; g_pidx[out]=root; /* sposta il max in coda */ - s2+=g_pbuf[root]; cum+=g_pbuf[root]; - if(out>0) topp_siftdown(g_pidx,out,0); - } while(cum0); - for(int i=0;i=0 -> quel token e' escluso (rinormalizzando al volo) */ -static int dist_sample(int V, int ban){ - double z = 1.0 - (ban>=0 ? g_pbuf[ban] : 0.0); if(z<=1e-12) z=1e-12; - double u = rndu()*z, cum=0; - for(int i=0;i=u) return i; } - for(int i=V-1;i>=0;i--) if(i!=ban && g_pbuf[i]>0) return i; - return 0; -} -/* prossimo token dai logits: greedy se g_temp<=0, altrimenti sampling. - * ban = token escluso perche' rifiutato dalla verifica speculativa precedente. */ -static int pick_tok(const float *lo, int V, int ban){ - if(g_temp<=0) return argmax_v(lo,V); - dist_build(lo,V); - return dist_sample(V,ban); -} - -/* stop-set attivo (popolato da run_text/run_serve dal config; vuoto in validazione, - * dove si genera un numero fisso di token da confrontare con l'oracolo) */ -static int g_stop[64], g_nstop=0; /* config eos + ogni added-token "special" del tokenizer */ -static void repin_pass_limit(Model *m,int limit); -static void repin_pass(Model *m){ repin_pass_limit(m,16); } -static inline int is_stop(int t){ for(int i=0;i solo gli stop del config (validazione/oracolo, dove il tokenizer non serve). */ -static void stops_arm_tok(const Cfg *c, int tok_eos, Tok *T){ - g_nstop=0; - for(int i=0;in_stop && g_nstop<64;i++) g_stop[g_nstop++]=c->stop_ids[i]; - if(tok_eos>=0 && !is_stop(tok_eos) && g_nstop<64) g_stop[g_nstop++]=tok_eos; - int nsp=0; - /* DIFESA IN PROFONDITA' (woolcoxm, #298): il tokenizer marca "special":true i token di - * CONTROLLO -- <|user|>, <|assistant|>, <|observation|>, , [gMASK], i marker - * image/video/audio. Nessuno di questi e' contenuto legittimo di una risposta: se il - * modello ne emette uno, il turno e' finito (infatti GLM ne elenca tre fra gli eos - * ufficiali). Senza questo, uno di quei token non elencato nel config veniva - * DETOKENIZZATO E STAMPATO IN CHAT come testo, e la generazione proseguiva oltre la - * fine reale -- l'"added stuff on the end" riportato su un checkpoint convertito. - * Fidarsi del config di pesi convertiti da terzi e' precisamente cio' che non - * possiamo controllare; il flag del tokenizer lo possiamo leggere. - * NB: // hanno "special":false e restano contenuto vero. */ - if(T) for(int id=0; idn_ids && g_nstop<64; id++) - if(T->id_special[id] && !is_stop(id)){ g_stop[g_nstop++]=id; nsp++; } - fprintf(stderr,"[stop] %d stop tokens:",g_nstop); - for(int i=0;i=0 && next==eos) || is_stop(next)) break; emit(next,ud); all[kv]=next; emitted++; m->n_emit++; - gr_feed(next); /* il walker segue l'output emesso */ + gr_feed(&g_grd,next); /* il walker segue l'output emesso */ if(emitted>=n_new) break; /* l'ultimo token non serve forwardarlo */ int g = 0, gsrc = 0; /* sorgente: 1=grammatica 2=MTP/n-gram */ - if(g_gr_on){ /* metodo F: prima la grammatica — dove + if(g_grd.on){ /* metodo F: prima la grammatica — dove * forza, l'acceptance e' ~1 (#48) */ - g=grammar_draft(draft,g_gr_max); + g=grammar_draft(&g_grd,draft,g_grd.max); if(g>0) gsrc=1; } if(!g && g_draft>0 && m->has_mtp){ @@ -4484,7 +4563,7 @@ static int spec_decode(Model *m, int *all, int kv, int n_new, int eos, float *lo if(g>n_new-emitted) g=n_new-emitted; if(kv+1+g+1>m->max_t) g=m->max_t-kv-2; if(g<0) g=0; - if(gsrc==1) g_gr_prop+=(uint64_t)g; + if(gsrc==1) g_grd.prop+=(uint64_t)g; int S=1+g; int batch[64]; batch[0]=next; memcpy(batch+1,draft,g*sizeof(int)); double tf0=g_prof?now_s():0; float *lo=step_all(m,batch,S,kv); m->n_fw++; @@ -4500,9 +4579,9 @@ static int spec_decode(Model *m, int *all, int kv, int n_new, int eos, float *lo if(!accept){ if(g_temp>0) carry_ban=draft[k]; break; } if((eos>=0 && draft[k]==eos) || is_stop(draft[k])){ done=1; break; } emit(draft[k],ud); all[kv+1+k]=draft[k]; emitted++; m->n_emit++; - gr_feed(draft[k]); k++; + gr_feed(&g_grd,draft[k]); k++; } - if(gsrc==1) g_gr_acc+=(uint64_t)k; + if(gsrc==1) g_grd.acc+=(uint64_t)k; else if(gsrc==2 && m->has_mtp) m->mtp_acc+=k; if(m->has_mtp && k>=1) mtp_absorb(m, all+kv+1, m->h_all, k, kv); /* KV MTP in sync coi verificati */ /* hlast deve corrispondere all'ultima posizione ACCETTATA (kv+k), non a fine batch */ @@ -4558,18 +4637,6 @@ static void forward_all(Model *m, const int *ids, int S, int *pred){ } /* log-prob (log-softmax) del token target dato il vettore di logit; *am=1 se e' l'argmax */ -static double logprob_target(const float *lo, int V, int target, int *am){ - float mx=lo[0]; int best=0; for(int i=1;imx){mx=lo[i];best=i;} } - double se=0; for(int i=0;i .. " (T=ctxlen+contlen) * output: riga " " per richiesta. @@ -4648,6 +4715,14 @@ static void profile_print(Model *m, double elapsed){ (unsigned long long)m->cpu_expert_rows,m->t_egpu,m->t_route,m->t_p2p,(unsigned long long)m->n_p2p, elapsed-m->t_ewait-m->t_emm-m->t_attn-m->t_head-m->t_route-m->t_p2p>0? elapsed-m->t_ewait-m->t_emm-m->t_attn-m->t_head-m->t_route-m->t_p2p:0); + if(g_mirror){ + double b0=atomic_load_explicit(&g_mir_bytes[0],memory_order_relaxed)/1e9; + double b1=atomic_load_explicit(&g_mir_bytes[1],memory_order_relaxed)/1e9; + printf("MIRROR: primary %.2f GB (%lld reads) | mirror %.2f GB (%lld reads) — %.0f%% of expert bytes from the mirror\n", + b0,(long long)atomic_load_explicit(&g_mir_nread[0],memory_order_relaxed), + b1,(long long)atomic_load_explicit(&g_mir_nread[1],memory_order_relaxed), + b0+b1>0?100.0*b1/(b0+b1):0.0); + } #ifdef COLI_METAL if(g_metal_enabled){ uint64_t ok=0,fb=0,ex=0; double su=0,gp=0,sc=0; coli_metal_moe_counts(&ok,&fb,&ex); coli_metal_moe_times(&su,&gp,&sc); @@ -4655,7 +4730,9 @@ static void profile_print(Model *m, double elapsed){ if(aok){ double ks=0,gs=0; coli_metal_attn_lat(&ks,&gs); printf("METAL-ATTN: layer GPU %llu | gpu-wall %.2fs (kernel %.2fs | cpu-sched %.2fs gpu-sched %.2fs)\n",(unsigned long long)aok,aw,ak,ks,gs); } } printf("METAL: blocchi GPU %llu | fallback CPU %llu | expert su GPU %llu | setup %.2fs gpu-wall %.2fs (kernel %.2fs) scatter %.2fs\n", - (unsigned long long)ok,(unsigned long long)fb,(unsigned long long)ex,su,gp,coli_metal_moe_kernel_time(),sc); } + (unsigned long long)ok,(unsigned long long)fb,(unsigned long long)ex,su,gp,coli_metal_moe_kernel_time(),sc); + { double rsf=0; if(coli_metal_resset_stats(&rsf)) /* E5: printed only when the gate is on */ + printf("METAL-RESSET: flush %.2fs (residency-set commit in moe_submit, outside setup/gpu-wall)\n",rsf); } } #endif } @@ -4706,6 +4783,42 @@ static void prof_report(Model *m, const ProfBase *b, double elapsed, int tokens, g_mmap?"; COLI_MMAP=1: page cache may serve part":"", hitp,(unsigned long long)dhp,(unsigned long long)dhe,(unsigned long long)dm, tokens>0?(double)dq/tokens:0.0, io_svc,io_w); + /* DISK-CLASS: per-load cold/warm classification vs. which fd ACTUALLY served it. + * Three per-class rates, labeled to keep the units unambiguous (ambiguous units + * mislead -- measured lesson): GB/s-thread = bytes / thread-seconds (per-read + * service rate, same convention as the read-service line above); GB/s-wall = + * bytes / busy-wall (aggregate rate the disk actually delivered while >=1 load of + * the class was in flight); avg-conc = thread-seconds / busy-wall (mean overlap + * depth). disk-busy = combined busy-wall (either class in flight) as a share of + * the profile window. */ + { + uint64_t dcn[2], dcdn[2]; int64_t dcbytes[2], dcns[2], dcwall[2], wnow[2], dcwall_all, wall_now; + dc_wall_read(wnow,&wall_now); + for(int i=0;i<2;i++){ + dcn[i]=atomic_load_explicit(&g_dc_n[i],memory_order_relaxed)-b->dc_n[i]; + dcbytes[i]=atomic_load_explicit(&g_dc_bytes[i],memory_order_relaxed)-b->dc_bytes[i]; + dcns[i]=atomic_load_explicit(&g_dc_ns[i],memory_order_relaxed)-b->dc_ns[i]; + dcdn[i]=atomic_load_explicit(&g_dc_direct_n[i],memory_order_relaxed)-b->dc_direct_n[i]; + dcwall[i]=wnow[i]-b->dc_wall_ns[i]; + } + dcwall_all=wall_now-b->dc_wall_all_ns; + if(dcn[DC_COLD]+dcn[DC_WARM]){ + double ct=dcns[DC_COLD]>0?(double)dcbytes[DC_COLD]/dcns[DC_COLD]:0.0; /* bytes/ns = GB/s */ + double wt=dcns[DC_WARM]>0?(double)dcbytes[DC_WARM]/dcns[DC_WARM]:0.0; + double cw=dcwall[DC_COLD]>0?(double)dcbytes[DC_COLD]/dcwall[DC_COLD]:0.0; + double ww=dcwall[DC_WARM]>0?(double)dcbytes[DC_WARM]/dcwall[DC_WARM]:0.0; + double cc=dcwall[DC_COLD]>0?(double)dcns[DC_COLD]/dcwall[DC_COLD]:0.0; + double wc=dcwall[DC_WARM]>0?(double)dcns[DC_WARM]/dcwall[DC_WARM]:0.0; + fprintf(f,"[PROF] DISK-CLASS (recency split): cold %llu x %.2f GB @ %.2f GB/s-thread, wall %.1fs @ %.2f GB/s-wall, avg-conc %.1f (direct %llu/%llu) | " + "warm %llu x %.2f GB @ %.2f GB/s-thread, wall %.1fs @ %.2f GB/s-wall, avg-conc %.1f (direct %llu/%llu) | " + "disk-busy %.1fs (%.0f%% of window)\n", + (unsigned long long)dcn[DC_COLD],dcbytes[DC_COLD]/1e9,ct,dcwall[DC_COLD]/1e9,cw,cc, + (unsigned long long)dcdn[DC_COLD],(unsigned long long)dcn[DC_COLD], + (unsigned long long)dcn[DC_WARM],dcbytes[DC_WARM]/1e9,wt,dcwall[DC_WARM]/1e9,ww,wc, + (unsigned long long)dcdn[DC_WARM],(unsigned long long)dcn[DC_WARM], + dcwall_all/1e9,100.0*(dcwall_all/1e9)/elapsed); + } + } fprintf(f,"[PROF] resident experts: %d pinned (%.1f GB) + %d in LRU (%.1f GB, cap %d/layer)\n", pinned,pinned*eb/1e9,lru,lru*eb/1e9,m->ecap); double emm=m->t_emm-b->emm, ecpu=m->t_ecpu-b->ecpu, egpu=m->t_egpu-b->egpu; @@ -4753,6 +4866,8 @@ static void run_replay(Model *m, const int *full, int nfull, int np){ m->hits=m->miss=m->ereq=m->gpu_expert_calls=0; m->hit_pin=m->hit_ecache=0; profile_reset(m); ProfBase pb; prof_base(m,&pb); + atomic_store(&g_mir_bytes[0],0); atomic_store(&g_mir_bytes[1],0); + atomic_store(&g_mir_nread[0],0); atomic_store(&g_mir_nread[1],0); double t0=now_s(); int steps=0; for(int i=np-1;i"); stops_arm_tok(&m->c, eos, &T); - grammar_setup(&T); /* metodo F: GRAMMAR=file.gbnf (#48) */ + grammar_setup(&g_grd,&T); /* metodo F: GRAMMAR=file.gbnf (#48) */ if(g_temp<0) g_temp=0.7f; /* auto: 0.7, NON l'1.0 ufficiale — la coda della * distribuzione int4 e' rumore di quantizzazione */ - int cap=(int)strlen(prompt)+16; int *pids=malloc(cap*sizeof(int)); + int cap=(int)strlen(prompt)+16; int *pids=malloc((cap+2)*sizeof(int)); int np=tok_encode(&T,prompt,(int)strlen(prompt),pids,cap); if(np<1){ fprintf(stderr,"prompt is empty after tokenization\n"); return; } + /* GLM prefix (#108): every GLM training sequence starts [gMASK]; without it the + * model is out-of-distribution and generates garbage. Serve/SCORE modes prepend it; + * run_text must too. CHAT_TEMPLATE=0 disables (raw prompt, for debugging/non-GLM). + * A prompt that already starts with the prefix passes through intact. */ + int templ=getenv("CHAT_TEMPLATE")?atoi(getenv("CHAT_TEMPLATE")):1; + if(templ){ + int gmask=tok_id_of(&T,"[gMASK]"), sop=tok_id_of(&T,""); + if(gmask>=0 && sop>=0 && (np<2 || pids[0]!=gmask || pids[1]!=sop)){ + memmove(pids+2,pids,np*sizeof(int)); + pids[0]=gmask; pids[1]=sop; np+=2; + } + } printf("prompt: %d tokens | generating up to %d (EOS stop=%d) | n-gram draft=%d\n", np, ngen, eos, g_draft); fputs(prompt,stdout); fflush(stdout); kv_alloc(m, np+ngen+g_draft+2); @@ -4807,7 +4934,7 @@ static void run_text(Model *m, const char *snap, const char *prompt, int ngen){ ProfBase pb; prof_base(m,&pb); double t=now_s(); EmitStream es={&T,m,t,0,0}; - grammar_reset(); + grammar_reset(&g_grd); int produced=spec_decode(m,all,np,ngen,eos,logit,emit_stream,&es,NULL); double dt=now_s()-t; double tot=m->hits+m->miss; @@ -4834,8 +4961,8 @@ static void run_text(Model *m, const char *snap, const char *prompt, int ngen){ m->n_fw?(double)m->n_emit/m->n_fw:1.0, (unsigned long long)m->n_fw, (unsigned long long)m->n_emit, m->mtp_prop?100.0*m->mtp_acc/m->mtp_prop:0.0, (unsigned long long)m->mtp_acc, (unsigned long long)m->mtp_prop); if(g_cp_enq) printf("couple: %ld cross-layer prefetch hints enqueued\n", g_cp_enq); - if(g_gr_prop) printf("grammar: %.0f%% acceptance (%llu/%llu forced drafts)\n", - 100.0*g_gr_acc/g_gr_prop, (unsigned long long)g_gr_acc, (unsigned long long)g_gr_prop); + if(g_grd.prop) printf("grammar: %.0f%% acceptance (%llu/%llu forced drafts)\n", + 100.0*g_grd.acc/g_grd.prop, (unsigned long long)g_grd.acc, (unsigned long long)g_grd.prop); if(g_disk_split) printf("disk-load split: draft %llu + absorb %llu + verify/main %llu misses | " "MTP-layer %llu loads %.2f GB | main-layers %llu loads %.2f GB (MTP %.1f%% of bytes)\n", (unsigned long long)m->miss_draft, (unsigned long long)m->miss_absorb, @@ -4966,16 +5093,21 @@ static void rss_guard(Model *m){ if(lru<0){ pthread_mutex_unlock(&g_pilot_mx); continue; } ESlot *s=&m->ecache[l][lru]; s->eid=-1; /* nascosto: nessun hit/evict altrui */ - pthread_mutex_unlock(&g_pilot_mx); int64_t sb=s->slab_cap + s->fslab_cap*4; #ifdef COLI_METAL if(s->slab && g_metal_enabled) coli_metal_unregister(s->slab); #endif + /* La free resta SOTTO g_pilot_mx. Sbloccando prima, lo slot e' visibile come + * {eid=-1, slab ancora valido}: il pilota (pilot_realload) riusa per primo gli + * slot eid==-1, quindi puo' prenderlo e fare pread dentro lo slab MENTRE lo + * liberiamo -> use-after-free / double-free. Slab valido e slot riusabile + * devono restare mutuamente esclusivi finche' il puntatore non e' NULL. */ compat_aligned_free(s->slab); free(s->fslab); s->slab=NULL; s->fslab=NULL; s->slab_cap=s->fslab_cap=0; QT *q[3]={&s->g,&s->u,&s->d}; for(int k=0;k<3;k++){ q[k]->qf=NULL; q[k]->q8=NULL; q[k]->q4=NULL; q[k]->s=NULL; } s->used=0; /* primo candidato al riuso */ + pthread_mutex_unlock(&g_pilot_mx); freed += sb; dropped++; } if(m->ecap>2) m->ecap--; /* il tetto scende: niente ricrescita */ @@ -5043,7 +5175,7 @@ static void repin_pass_limit(Model *m,int limit){ +(int64_t)coli_cuda_tensor_bytes(s->d.cuda) : 0; #endif double t0=now_s(); - expert_load(m,cd[b].l,cd[b].eid,s,1); /* disk -> RAM, same resident slot */ + expert_load(m,cd[b].l,cd[b].eid,s,1,0); /* disk -> RAM, same resident slot; demand=0: repin, never classified */ const char *tier="RAM"; #ifdef COLI_CUDA if(gpu){ /* refresh the same VRAM slot now, not lazily */ @@ -5075,117 +5207,6 @@ static void repin_pass_limit(Model *m,int limit){ * si appendono SOLO le posizioni nuove e si riscrive nrec per ultimo: un crash a meta' * append lascia nrec vecchio = file coerente. La riga KV del layer MTP non si salva: * al resume kv_start=-1 e la finestra di draft riparte da sola. */ -static int g_kvsave=1; -#define KV_MAGIC "COLIKV1\0" -static void kv_hdr(Model *m, int32_t *h, int nrec){ - Cfg *c=&m->c; int nic=0; - for(int i=0;in_layers;i++) if(m->Ic && m->Ic[i]) nic++; - h[0]=c->n_layers; h[1]=c->kv_lora; h[2]=c->qk_rope; - h[3]=m->has_dsa?c->index_hd:0; h[4]=nic; h[5]=c->vocab; h[6]=nrec; h[7]=0; -} -/* Bytes of one on-disk record: [tok i32][Lc+Rc per layer][Ic per DSA layer]. - * Layout matches what kv_disk_append writes and kv_disk_load reads. */ -static int64_t kv_rec_bytes(Model *m){ - Cfg *c=&m->c; - int64_t rec = 4 + (int64_t)c->n_layers*(c->kv_lora+c->qk_rope)*4; - if(m->has_dsa) for(int i=0;in_layers;i++) if(m->Ic[i]) rec+=(int64_t)c->index_hd*4; - return rec; -} -/* Open the persistent handle lazily; write the header if the file is new. After - * this returns successfully, k->disk_fp is valid for the engine's lifetime and - * positioned at end-of-header (nrec==0 case) or wherever the caller seeks. */ -static int kv_disk_open(Model *m){ - KVState *k=m->kv; - if(k->disk_fp) return 1; - k->disk_fp=fopen(k->disk_path,"r+b"); - if(!k->disk_fp){ /* not there yet -> create + header */ - k->disk_fp=fopen(k->disk_path,"wb"); - if(!k->disk_fp) return 0; - int32_t h[8]; kv_hdr(m,h,0); - fwrite(KV_MAGIC,1,8,k->disk_fp); fwrite(h,4,8,k->disk_fp); - fflush(k->disk_fp); - fclose(k->disk_fp); - k->disk_fp=fopen(k->disk_path,"r+b"); /* reopen r+b for append */ - if(!k->disk_fp) return 0; - } - return 1; -} -static void kv_disk_truncate(Model *m, int nrec){ - if(!g_kvsave) return; - KVState *k=m->kv; - if(k->disk_fp){ fclose(k->disk_fp); k->disk_fp=NULL; } /* drop to shrink on disc */ - FILE *f=fopen(k->disk_path,"r+b"); - if(!f){ k->disk_nrec=0; return; } - k->disk_nrec=nrec; - int32_t nr=nrec; fseek(f,8+6*4,SEEK_SET); fwrite(&nr,4,1,f); - fflush(f); fclose(f); -} -static void kv_disk_reset(Model *m){ kv_disk_truncate(m,0); } -static void kv_disk_append(Model *m, const int *hist, int len){ - KVState *k=m->kv; - if(!g_kvsave || len<=k->disk_nrec) return; - Cfg *c=&m->c; - if(!kv_disk_open(m)) return; - FILE *f=k->disk_fp; - int64_t rec = kv_rec_bytes(m); - /* grow the contiguous staging buffer if the record is larger (#1 batching) */ - if(rec > k->disk_buf_cap){ - uint8_t *nb=realloc(k->disk_buf, rec); - if(!nb) return; /* OOM: skip this turn, retry next */ - k->disk_buf=nb; k->disk_buf_cap=rec; - } - fseek(f, 8+8*4 + (int64_t)k->disk_nrec*rec, SEEK_SET); - for(int p=k->disk_nrec;pdisk_buf; /* pack token + every layer into one record */ - *(int32_t*)b = hist[p]; b+=4; - for(int i=0;in_layers;i++){ - memcpy(b, m->Lc[i]+(int64_t)p*c->kv_lora, (size_t)c->kv_lora*4); b+=c->kv_lora*4; - memcpy(b, m->Rc[i]+(int64_t)p*c->qk_rope,(size_t)c->qk_rope*4); b+=c->qk_rope*4; - } - if(m->has_dsa) for(int i=0;in_layers;i++) if(m->Ic[i]){ - memcpy(b, m->Ic[i]+(int64_t)p*c->index_hd, (size_t)c->index_hd*4); b+=c->index_hd*4; - } - fwrite(k->disk_buf, 1, (size_t)rec, f); /* one fwrite per position (was ~157) */ - } - fflush(f); /* dati prima, contatore poi */ - int32_t nr=len; fseek(f,8+6*4,SEEK_SET); fwrite(&nr,4,1,f); - fflush(f); /* persist the counter too */ - k->disk_nrec=len; -} -static int kv_disk_load(Model *m, int *hist, int maxctx){ - if(!g_kvsave) return 0; - KVState *k=m->kv; - Cfg *c=&m->c; - FILE *f=fopen(k->disk_path,"rb"); if(!f) return 0; - char mg[8]; int32_t h[8], w[8]; kv_hdr(m,w,0); - if(fread(mg,1,8,f)!=8 || memcmp(mg,KV_MAGIC,8) || fread(h,4,8,f)!=8 || - h[0]!=w[0]||h[1]!=w[1]||h[2]!=w[2]||h[3]!=w[3]||h[4]!=w[4]||h[5]!=w[5]){ - fprintf(stderr,"[KV] ignoring .coli_kv from a different model or version\n"); fclose(f); return 0; } - int nrec=h[6]; - if(nrec<1){ fclose(f); return 0; } - if(nrec>=maxctx-8-g_draft){ - fprintf(stderr,"[KV] saved conversation (%d tokens) exceeds the context: starting over\n",nrec); - fclose(f); return 0; } - double t0=now_s(); - for(int p=0;pn_layers;i++){ - if(fread(m->Lc[i]+(int64_t)p*c->kv_lora, 4, c->kv_lora, f)!=(size_t)c->kv_lora || - fread(m->Rc[i]+(int64_t)p*c->qk_rope, 4, c->qk_rope, f)!=(size_t)c->qk_rope){ nrec=p; goto out; } - } - if(m->has_dsa) for(int i=0;in_layers;i++) if(m->Ic[i]) - if(fread(m->Ic[i]+(int64_t)p*c->index_hd, 4, c->index_hd, f)!=(size_t)c->index_hd){ nrec=p; goto out; } - } -out: - fclose(f); - if(nrec>0){ - if(m->has_mtp) m->kv_start[c->n_layers]=-1; /* la finestra MTP riparte da sola */ - fprintf(stderr,"[KV] resumed conversation from disk: %d tokens in %.1fs (no re-prefill)\n", - nrec, now_s()-t0); - } - k->disk_nrec=nrec; - return nrec; -} typedef struct { KVState kv; int *hist, len, first; } ServeCtx; static double kv_pool_bytes(Model *m, int max_ctx); @@ -5261,8 +5282,8 @@ static void mux_done(Model *m, ServeCtx *sc, ServeReq *r){ /* Read and prefill one request. Returns -1 on EOF, 0 for a rejected frame and * 1 for an accepted request. Prefill deliberately remains serial: continuous * batching starts at decode, where every active slot contributes one row. */ -static int mux_submit(Model *m, Tok *T, ServeCtx *ctx, ServeReq *req, int nctx, - int maxctx, int eos){ +static int mux_submit(Model *m, Tok *T, ServeCtx *ctx, ServeReq *req, GrDraft *grd, + int nctx, int maxctx, int eos){ char *line=NULL; size_t cap=0; ssize_t nr=getline(&line,&cap,stdin); if(nr<0){ free(line); return -1; } if(nr && line[nr-1]=='\n') line[--nr]=0; @@ -5283,27 +5304,53 @@ static int mux_submit(Model *m, Tok *T, ServeCtx *ctx, ServeReq *req, int nctx, char *raw=malloc((size_t)sub.bytes+1); if(!raw){ fprintf(stderr,"OOM multiplex payload\n"); exit(1); } if(fread(raw,1,(size_t)sub.bytes,stdin)!=(size_t)sub.bytes){ free(raw); free(line); return -1; } + char *gtxt=NULL; /* optional per-request grammar/schema text */ + if(sub.gbytes){ + gtxt=malloc((size_t)sub.gbytes+1); + if(!gtxt){ fprintf(stderr,"OOM multiplex payload\n"); exit(1); } + if(fread(gtxt,1,(size_t)sub.gbytes,stdin)!=(size_t)sub.gbytes){ + free(gtxt); free(raw); free(line); return -1; } + gtxt[sub.gbytes]=0; + } int delim=fgetc(stdin); if(delim!='\n'){ printf("ERROR %llu BAD_FRAME\n",sub.id); fflush(stdout); - free(raw); free(line); return -1; + free(gtxt); free(raw); free(line); return -1; } raw[sub.bytes]=0; if(sub.slot>=nctx || memchr(raw,0,(size_t)sub.bytes)){ - printf("ERROR %llu BAD_REQUEST\n",sub.id); fflush(stdout); free(raw); free(line); return 0; + printf("ERROR %llu BAD_REQUEST\n",sub.id); fflush(stdout); free(gtxt); free(raw); free(line); return 0; } if(req[sub.slot].active){ - printf("ERROR %llu SLOT_BUSY\n",sub.id); fflush(stdout); free(raw); free(line); return 0; + printf("ERROR %llu SLOT_BUSY\n",sub.id); fflush(stdout); free(gtxt); free(raw); free(line); return 0; } for(int i=0;ikv); int *tmp=malloc(maxctx*sizeof(int)); if(!tmp){ fprintf(stderr,"OOM mux_submit tmp\n"); free(raw); free(line); exit(1); } - int nt=tok_encode(T,raw,(int)sub.bytes,tmp,maxctx-2); + /* Encode with one token of headroom (maxctx-1, buffer is maxctx) purely to DETECT overflow: + * tok_encode stops dead at its cap and returns no signal that it ran out, so encoding at the + * real limit silently truncates an over-long prompt to its first maxctx-2 tokens. That is the + * #401 field failure: a coding client's system prompt alone exceeded CTX=4096, the tail -- + * the tool instructions and the actual user turn -- was dropped, the model saw a prompt cut + * mid-markup and emitted a bare "<". Retries were identical because the surviving *head* + * never changes when a client appends to the end (hence "prefill 0" on every retry). + * Refuse loudly instead; the gateway turns this into a 400 context_length_exceeded. */ + int nt=tok_encode(T,raw,(int)sub.bytes,tmp,maxctx-1); free(raw); free(line); if(nt<1){ free(tmp); printf("ERROR %llu EMPTY_PROMPT\n",sub.id); fflush(stdout); return 0; } + if(nt>maxctx-2){ + free(tmp); + fprintf(stderr,"[API] prompt does not fit: >=%d token, context is %d (CTX=%d). " + "Raise it, e.g. CTX=32768 -- coding clients send large system prompts.\n", + nt,maxctx-2,maxctx); + printf("ERROR %llu CONTEXT_EXCEEDED %d %d\n",sub.id,nt,maxctx-2); + fflush(stdout); return 0; + } int prefix=0; while(prefixlen && prefixhist[prefix]==tmp[prefix]) prefix++; if(prefixlen){ sc->len=prefix; if(m->has_mtp) m->kv_start[m->c.n_layers]=-1; kv_disk_truncate(m,sc->len); } @@ -5323,6 +5370,7 @@ static int mux_submit(Model *m, Tok *T, ServeCtx *ctx, ServeReq *req, int nctx, int next=pick_tok(logit,m->c.vocab,-1); free(logit); if(r->maximum<=0 || next==eos || is_stop(next)){ mux_done(m,sc,r); return 1; } r->pending=next; r->emitted=1; r->active=1; sc->hist[sc->len]=next; m->n_emit++; + if(grd[sub.slot].on){ grammar_reset(&grd[sub.slot]); gr_feed(&grd[sub.slot],next); } mux_data(T,r->id,next); if(r->emitted>=r->maximum) mux_done(m,sc,r); return 1; @@ -5331,14 +5379,18 @@ static int mux_submit(Model *m, Tok *T, ServeCtx *ctx, ServeReq *req, int nctx, static void run_serve_mux(Model *m, const char *snap){ char tkp[2048]; snprintf(tkp,sizeof(tkp),"%s/tokenizer.json",snap); Tok T; tok_load(&T,tkp); int eos=tok_id_of(&T,"<|endoftext|>"); stops_arm_tok(&m->c,eos,&T); - g_draft=0; /* one scheduler owns every forward; MTP/speculation is not ragged-safe */ + g_draft=0; /* one scheduler owns every forward; MTP/n-gram speculation is not ragged-safe. + * Grammar-forced drafts ARE mux-safe (below): a drafting slot leaves the shared + * batch for one forward and runs the proven single-sequence verify path + * (kv_bind + step_all), exactly like prefill already does per submission. */ int maxctx=getenv("CTX")?atoi(getenv("CTX")):4096; int nctx=getenv("KV_SLOTS")?atoi(getenv("KV_SLOTS")):1; if(nctx<1||nctx>512){fprintf(stderr,"KV_SLOTS must be between 1 and 512\n");exit(2);} g_kvsave=getenv("KVSAVE")?atoi(getenv("KVSAVE")):1; KVState *initial=m->kv; free(initial->kv_start); free(initial); ServeCtx *ctx=calloc(nctx,sizeof(*ctx)); ServeReq *req=calloc(nctx,sizeof(*req)); - for(int i=0;i0)?1:0; if(ready) #endif - if(mux_submit(m,&T,ctx,req,nctx,maxctx,eos)<0) eof=1; + if(mux_submit(m,&T,ctx,req,grd,nctx,maxctx,eos)<0) eof=1; } active=0; for(int i=0;ion && r->temp==0) k=grammar_draft(gd,draft,gd->max); + if(k>0 && sc->len+1+klen+1+k>=(int)sc->kv.max_t) k=(int)sc->kv.max_t-sc->len-2; + if(k<1){ rows[S]=(DecodeRow){&sc->kv,r->pending,sc->len}; slots[S++]=i; continue; } + kv_bind(m,&sc->kv); + int seq[50]; seq[0]=r->pending; + memcpy(seq+1,draft,(size_t)k*sizeof(int)); + float *lo=step_all(m,seq,1+k,sc->len); m->n_fw++; + gd->prop+=(uint64_t)k; + int done=0; + for(int j=0;j<=k && !done;j++){ + g_temp=r->temp; g_nuc=r->top_p; + int next=pick_tok(lo+(int64_t)j*m->c.vocab,m->c.vocab,-1); + sc->len++; /* seq[j] joins the committed history */ + if(next==eos || is_stop(next)){ mux_done(m,sc,r); done=1; break; } + r->pending=next; sc->hist[sc->len]=next; r->emitted++; m->n_emit++; + if(gd->on) gr_feed(gd,next); + mux_data(&T,r->id,next); + if(r->emitted>=r->maximum){ mux_done(m,sc,r); done=1; break; } + if(jacc++; + } + } + free(lo); + continue; /* handled outside the shared batch */ + } + rows[S]=(DecodeRow){&sc->kv,r->pending,sc->len}; slots[S++]=i; } + if(S==0) continue; /* every active slot drafted this round */ double tf0=g_prof?now_s():0; float *lo=step_decode_batch(m,rows,S); if(!lo){fprintf(stderr,"decode batch failed\n");break;} m->n_fw++; @@ -5404,13 +5489,15 @@ static void run_serve_mux(Model *m, const char *snap){ int next=pick_tok(lo+(int64_t)s*m->c.vocab,m->c.vocab,-1); if(next==eos || is_stop(next)){mux_done(m,sc,r);continue;} r->pending=next; sc->hist[sc->len]=next; r->emitted++; m->n_emit++; + if(grd[i].on) gr_feed(&grd[i],next); /* walker stays in sync when not drafting */ mux_data(&T,r->id,next); if(r->emitted>=r->maximum) mux_done(m,sc,r); } free(lo); } usage_save(m); - for(int i=0;ikv=NULL; m->Lc=m->Rc=m->Ic=NULL; m->kv_start=NULL; m->max_t=0; } @@ -5434,7 +5521,7 @@ static void run_serve(Model *m, const char *snap){ Tok T; tok_load(&T,tkp); int eos=tok_id_of(&T,"<|endoftext|>"); stops_arm_tok(&m->c, eos, &T); - grammar_setup(&T); /* metodo F: GRAMMAR=file.gbnf (#48) */ + grammar_setup(&g_grd,&T); /* metodo F: GRAMMAR=file.gbnf (#48) */ if(g_temp<0) g_temp=0.7f; /* auto: 0.7, NON l'1.0 ufficiale — la coda della * distribuzione int4 e' rumore di quantizzazione */ int ngen=getenv("NGEN")?atoi(getenv("NGEN")):256; @@ -5553,7 +5640,7 @@ static void run_serve(Model *m, const char *snap){ else logit=step(m,hist+len-1,1,len-1); /* prompt identico/prefisso: rigenera i logits */ EmitStream es={&T,m,now_s(),0,1}; int prod=0; - grammar_reset(); /* nuova risposta = nuovo documento (MORE invece continua) */ + grammar_reset(&g_grd); /* nuova risposta = nuovo documento (MORE invece continua) */ if(cur>0) prod=spec_decode(m,hist,len,cur,eos,logit,emit_stream,&es,&len); else free(logit); double tdt=now_s()-tt0; if(tdt<1e-6) tdt=1e-6; @@ -5596,212 +5683,7 @@ static int *read_arr(jval*o,const char*k,int*n){ if(!r){ fprintf(stderr,"OOM read_arr\n"); exit(1); } for(int i=0;ilen;i++) r[i]=(int)a->kids[i]->num; *n=a->len; return r; } -/* byte residenti di un tensore [O,I] al numero di bit dato (specchio di qt_bytes) */ -static int64_t tbytes(int O,int I,int bits){ - if(bits>=16) return (int64_t)O*I*4; - if(bits>=5) return (int64_t)O*I + (int64_t)O*4; - return (int64_t)O*((I+1)/2) + (int64_t)O*4; -} -/* byte VERI di un expert: dal container se pre-quantizzato, altrimenti stima da ebits */ -static int64_t expert_bytes_probe(Model *m, int ebits){ - Cfg *c=&m->c; int64_t eb=0; char nm[256]; - snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.0.gate_proj.weight",c->first_dense); - if(st_nbytes(&m->S,nm)>0){ - const char *suf[3]={"gate_proj","up_proj","down_proj"}; - for(int k=0;k<3;k++){ - snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.0.%s.weight",c->first_dense,suf[k]); - eb+=st_nbytes(&m->S,nm); - snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.0.%s.weight.qs",c->first_dense,suf[k]); - int64_t q=st_nbytes(&m->S,nm); if(q>0) eb+=q; - } - } - if(eb<=0) eb = tbytes(c->moe_inter,c->hidden,ebits)*2 + tbytes(c->hidden,c->moe_inter,ebits); - return eb; -} - -/* TIERS: fotografia della piramide expert per la dashboard web — - * "TIERS " sul canale di protocollo. - * ram = pinnati non-VRAM + LRU corrente; disk = tutto il resto. */ -/* BRAIN MAP (dashboard): per-turn expert hit bitmap + full residency/heat map. - * g_ehit[layer][eid]=1 quando l'expert viene instradato in questo turno; - * hits_emit lo serializza e lo azzera. emap_emit fotografa tier+heat di TUTTI. */ -static uint8_t **g_ehit; -static void ehit_mark(Model *m, int layer, int eid){ - if(!g_ehit){ Cfg *c=&m->c; - g_ehit=calloc(c->n_layers+1,sizeof(uint8_t*)); - for(int i=0;i<=c->n_layers;i++) g_ehit[i]=calloc(c->n_experts,1); - } - g_ehit[layer][eid]=1; -} - -/* HWINFO: hardware snapshot for the web dashboard — emitted once at READY. */ -/* CPU model + cores + RAM (GB); empty/zero where unavailable. - * Shared by the dashboard HWINFO line and the PROF=1 header. */ -static void hw_probe(char *cpu, size_t cn, int *cores, double *ram_total, double *ram_avail){ - cpu[0]=0; -#ifdef _WIN32 - /* niente /proc su Windows: brand string via CPUID (0x80000002..4), zero - * dipendenze extra. La dashboard mostrava "0 GB RAM / 0 cores" perche' - * tutto questo blocco era solo-Linux mentre il ramo CUDA funzionava. */ -#if defined(__x86_64__) || defined(__i386__) - { unsigned int r[12]={0}; unsigned int *w=r; - for(unsigned int f=0x80000002u; f<=0x80000004u; f++,w+=4) - __get_cpuid(f,&w[0],&w[1],&w[2],&w[3]); - char *b=(char*)r; b[47]=0; while(*b==' ')b++; - snprintf(cpu,cn,"%s",b); } -#endif -#else - FILE *ci=fopen("/proc/cpuinfo","r"); - if(ci){ char ln[256]; - while(fgets(ln,sizeof(ln),ci)) if(!strncmp(ln,"model name",10)){ - char *p=strchr(ln,':'); if(p){ p++; while(*p==' ')p++; - int n=(int)strlen(p); if(n>0&&p[n-1]=='\n')p[--n]=0; - snprintf(cpu,cn,"%s",p); } break; } - fclose(ci); } -#endif - *cores=0; -#ifdef _WIN32 - { SYSTEM_INFO si; GetSystemInfo(&si); *cores=(int)si.dwNumberOfProcessors; } -#elif defined(_SC_NPROCESSORS_ONLN) - *cores=(int)sysconf(_SC_NPROCESSORS_ONLN); -#endif - *ram_total=*ram_avail=0; -#ifdef _WIN32 - compat_meminfo(ram_total,ram_avail); /* GlobalMemoryStatusEx, gia' in compat.h */ -#else - FILE *mi=fopen("/proc/meminfo","r"); - if(mi){ char ln[256]; double mt=0,ma=0; - while(fgets(ln,sizeof(ln),mi)){ - if(sscanf(ln,"MemTotal: %lf",&mt)==1) *ram_total=mt/1e6; - if(sscanf(ln,"MemAvailable: %lf",&ma)==1) *ram_avail=ma/1e6; - } fclose(mi); } -#endif -} - -static void hwinfo_emit(Model *m){ - Cfg *c=&m->c; (void)c; /* silence -Wunused on builds without /proc (#148 report) */ - char cpu[256]; int cores; double ram_total,ram_avail; - hw_probe(cpu,sizeof(cpu),&cores,&ram_total,&ram_avail); - /* GPU */ - int ngpu=0; double vram_total=0; - char gpu_name[128]=""; -#ifdef COLI_CUDA - ngpu=g_cuda_ndev; vram_total=m->gpu_expert_bytes/1e9; - for(int i=0;i0){ - /* We don't have a device-name API; parse from the init log line stored in stderr. - * Simpler: just read the nvidia driver sysfs or use a fixed label. */ - snprintf(gpu_name,sizeof(gpu_name),"CUDA device x%d",g_cuda_ndev); - } -#endif - printf("HWINFO %d %.1f %.1f %d %.1f %s|%s\n", - cores,ram_total,ram_avail,ngpu,vram_total,cpu,gpu_name); - fflush(stdout); -} - -static void tiers_emit(Model *m){ - Cfg *c=&m->c; int nsp=0; - for(int i=0;in_layers;i++) if(m->L[i].sparse) nsp++; - int total=(nsp+(m->has_mtp?1:0))*c->n_experts; - int pinned=0,lru=0; - for(int i=0;i<=c->n_layers;i++){ pinned+=m->npin?m->npin[i]:0; lru+=m->ecn?m->ecn[i]:0; } - int vram=0; double vram_gb=0; -#ifdef COLI_CUDA - vram=m->gpu_expert_count; vram_gb=m->gpu_expert_bytes/1e9; -#endif - int ram=pinned-vram+lru; if(ram<0) ram=0; - int disk=total-vram-ram; if(disk<0) disk=0; - double eb=(double)expert_bytes_probe(m,m->ebits); - printf("TIERS %d %d %d %.2f %.2f\n",vram,ram,disk,vram_gb,ram*eb/1e9); - fflush(stdout); -} - -/* EMAP: 1 byte/expert (2bit tier: 0=disk 1=RAM 2=VRAM | 6bit heat log2-bucket), - * righe = layer sparsi in ordine (+MTP se presente), colonne = n_experts. Hex. */ -static void emap_emit(Model *m){ - Cfg *c=&m->c; - int rows=0; - for(int i=0;in_layers;i++) if(m->L[i].sparse) rows++; - int has_mtp = m->has_mtp && m->eusage[c->n_layers]; - if(has_mtp) rows++; - int cols=c->n_experts; - char *hex=malloc((size_t)rows*cols*2+1); int w=0; - for(int i=0;i<=c->n_layers;i++){ - int is_row = (in_layers && m->L[i].sparse) || (i==c->n_layers && has_mtp); - if(!is_row) continue; - for(int e=0;epin[i]; - for(int z=0;znpin[i];z++) if(P[z].eid==e){ -#ifdef COLI_CUDA - tier = P[z].g.cuda?2:1; -#else - tier = 1; -#endif - break; } - if(!tier && m->ecache && m->ecache[i]) - for(int z=0;zecn[i];z++) if(m->ecache[i][z].eid==e){ tier=1; break; } - uint32_t u = m->eusage[i]?m->eusage[i][e]:0; - int heat=0; while(u){ heat++; u>>=1; } if(heat>63) heat=63; - int b=(tier<<6)|heat; - hex[w++]="0123456789abcdef"[b>>4]; hex[w++]="0123456789abcdef"[b&15]; - } - } - hex[w]=0; - printf("EMAP %d %d %s\n",rows,cols,hex); fflush(stdout); free(hex); -} - -/* HITS: bitmap 1bit/expert (stesso ordine di EMAP), poi azzera per il turno dopo. */ -static void hits_emit(Model *m){ - Cfg *c=&m->c; if(!g_ehit) return; - int rows=0; - for(int i=0;in_layers;i++) if(m->L[i].sparse) rows++; - int has_mtp = m->has_mtp && m->eusage[c->n_layers]; - if(has_mtp) rows++; - int cols=c->n_experts, nb=(rows*cols+7)/8; - uint8_t *bm=calloc(nb,1); int bit=0; - for(int i=0;i<=c->n_layers;i++){ - int is_row = (in_layers && m->L[i].sparse) || (i==c->n_layers && has_mtp); - if(!is_row) continue; - for(int e=0;e>3]|=1<<(bit&7); g_ehit[i][e]=0; } - } - char *hex=malloc((size_t)nb*2+1); int w=0; - for(int b=0;b>4]; hex[w++]="0123456789abcdef"[bm[b]&15]; } - hex[w]=0; - printf("HITS %d %d %s\n",rows,cols,hex); fflush(stdout); free(hex); free(bm); -} - -/* scarica su file l'istogramma d'uso degli expert: righe "layer eid count" (per PIN). - * Include la riga MTP (layer n_layers). Scrittura atomica (tmp+rename): viene chiamata - * anche a ogni turno di serve e il processo puo' morire in qualsiasi momento. */ -static void stats_dump_q(Model *m, const char *path, int quiet){ - char tmp[2100]; snprintf(tmp,sizeof(tmp),"%s.tmp",path); - FILE *f=fopen(tmp,"w"); if(!f){ if(!quiet) perror(tmp); return; } - Cfg *c=&m->c; int64_t tot=0, nz=0; - for(int i=0;i<=c->n_layers;i++){ if(!m->eusage[i]) continue; - for(int e=0;en_experts;e++) if(m->eusage[i][e]){ fprintf(f,"%d %d %u\n",i,e,m->eusage[i][e]); tot+=m->eusage[i][e]; nz++; } } - fclose(f); rename(tmp,path); - if(!quiet) fprintf(stderr,"[STATS] %lld selections across %lld distinct experts -> %s\n",(long long)tot,(long long)nz,path); -} -static void stats_dump(Model *m, const char *path){ stats_dump_q(m,path,0); } - -/* CACHE CHE IMPARA: istogramma d'uso PERSISTENTE in /.coli_usage. - * Caricato all'avvio (i contatori ripartono dalla storia), salvato a ogni turno: - * piu' usi colibri', meglio l'auto-pin conosce i TUOI expert caldi. */ -static char g_usage_path[2100]=""; -static int64_t usage_load(Model *m, const char *path){ - FILE *f=fopen(path,"r"); if(!f) return 0; - Cfg *c=&m->c; int l,e; uint32_t cnt; int64_t tot=0; - while(fscanf(f,"%d %d %u",&l,&e,&cnt)==3) - if(l>=0&&l<=c->n_layers&&e>=0&&en_experts&&m->eusage[l]){ m->eusage[l][e]+=cnt; tot+=cnt; } - fclose(f); return tot; -} -static void usage_save(Model *m){ if(g_usage_path[0]) stats_dump_q(m,g_usage_path,1); } +/* telemetry, stats, usage persistence — moved to telemetry.h */ /* HOT-STORE ("il redis del colibri'"): carica in RAM, UNA VOLTA e per sempre, i top expert * per frequenza d'uso misurata (file STATS di un run precedente), entro un budget in GB. @@ -5902,10 +5784,125 @@ static void pin_wire(Model *m){ "(no compression) in %.0fs\n", wired/1e9, now_s()-t0); } +/* DUAL-SSD: measure one replica's read bandwidth with the engine's own access + * pattern (parallel ~19 MB reads, O_DIRECT twin when available — buffered would + * measure the page cache, see iobench's #86 caveat). Reads the largest shard at + * deterministic spread-out offsets; ~150 MB per drive, a few hundred ms total. */ +static double mirror_probe_bw(shards *S,int rep){ + int big=-1; int64_t bsz=0; + for(int i=0;infd;i++){ + if(rep && S->mfds[i]<0) continue; + int64_t sz=lseek(rep?S->mfds[i]:S->fds[i],0,SEEK_END); + if(sz>bsz){ bsz=sz; big=i; } + } + const int64_t blk=19ll<<20; const int NB=8; + if(big<0 || bszmdfds[big] : S->dfds[big]; + int fd = dfd>=0? dfd : (rep? S->mfds[big] : S->fds[big]); + if(dfd<0) fprintf(stderr,"[MIRROR] no O_DIRECT on %s: the probe may read the page cache — " + "set COLI_DISK_WEIGHTS for an accurate split\n", rep?"the mirror":"the primary"); + double t0=now_s(); int64_t tot=0; + #pragma omp parallel for schedule(dynamic,1) reduction(+:tot) + for(int i=0;i0) tot+=r; + compat_aligned_free(buf); + } + } + double dt=now_s()-t0; + return (dt>0 && tot>0)? tot/1e9/dt : 0; +} + +/* DUAL-SSD setup: register the mirror copy, derive the read split from + * COLI_DISK_WEIGHTS=, or from a startup bandwidth probe. + * Runs after model_init (needs the shard index) and BEFORE any pin/autopin + * load, so the OMP-parallel pin warmup already streams from both drives. */ +static void mirror_setup(Model *m){ + if(!g_mirror_dir) return; + const char *snap=getenv("SNAP"); + if(snap && !strcmp(snap,g_mirror_dir)){ + fprintf(stderr,"[MIRROR] mirror dir equals the model dir — ignored\n"); return; + } + int nf=st_mirror_init(&m->S,g_mirror_dir); + if(nf<=0){ + fprintf(stderr,"[MIRROR] %s: no usable shard (missing or divergent copy) — " + "running on the primary drive only\n",g_mirror_dir); + return; + } + g_mirror=1; + double wp=0,wm=0; const char *w=getenv("COLI_DISK_WEIGHTS"); const char *how="COLI_DISK_WEIGHTS"; + if(w && (sscanf(w," %lf , %lf",&wp,&wm)!=2 || wp<=0 || wm<=0)){ + fprintf(stderr,"[MIRROR] invalid COLI_DISK_WEIGHTS '%s' (want e.g. 9,3) — probing instead\n",w); + wp=wm=0; + } + if(wp<=0||wm<=0){ + wp=mirror_probe_bw(&m->S,0); wm=mirror_probe_bw(&m->S,1); how="measured"; + if(wp>0&&wm>0) fprintf(stderr,"[MIRROR] probe: primary %.2f GB/s, mirror %.2f GB/s\n",wp,wm); + else { wp=wm=1; how="fallback 1:1 (probe failed)"; } + } + int share=(int)(256.0*wm/(wp+wm)+0.5); + g_mir_share = share<1?1 : share>255?255 : share; + fprintf(stderr,"[MIRROR] %s: %d/%d shards | read split primary %.0f%% / mirror %.0f%% (%s)\n", + g_mirror_dir,nf,m->S.nfd,100.0*(256-g_mir_share)/256,100.0*g_mir_share/256,how); +} + typedef struct { int l,e; uint32_t c; } PinRec; static int pin_rec_cmp(const void *a,const void *b){ const PinRec *x=a,*y=b; return x->cc?1:x->c>y->c?-1:0; } + +#ifdef __linux__ +/* #419: bind the pinned hot-store as ONE arena per layer instead of one mbind + * per slab. Per-slab policies cost ~2 unmergeable VMAs each; a PIN_GB=all load + * (19,456 experts x slab+fslab) crosses the default vm.max_map_count=65530 and + * posix_memalign dies with terabytes free. Experts of one layer share a tensor + * shape, so a layer's pins pack into two arenas (weights + scales) at a fixed + * stride: 2 binds and a handful of VMAs per layer instead of ~500. Slices are + * pre-attached to the slots (slab_cap covers expert_load's realloc check, so + * its alloc branch never fires); aslab marks arena ownership for the + * release/ensure paths. Arena-OOM just leaves the slots on the individual path. */ +static void pin_arena_bind(Model *m, PinRec *r, int *slot_of, int from, int to){ + if(g_numa_nodes<2 || g_mmap || from>=to) return; + Cfg *c=&m->c; int NR=c->n_layers+1; + int *cnt=calloc((size_t)NR,sizeof(int)); int *first=malloc((size_t)NR*sizeof(int)); + if(!cnt||!first){ free(cnt); free(first); return; } + for(int i=0;iS,nm), *tq=st_find(&m->S,qn); + if(!tw||!tq) ok=0; else { wtot+=tw->nbytes; qtot+=tq->nbytes; } + } + if(!ok) continue; /* unquantized fallback: individual allocs */ + size_t ws=((size_t)wtot+8192+4095)&~(size_t)4095; + size_t fs=((size_t)(qtot/4)*sizeof(float)+4095)&~(size_t)4095; + uint8_t *aw=NULL; float *af=NULL; + if(posix_memalign((void**)&aw,4096,(size_t)cnt[l]*ws)) continue; + if(posix_memalign((void**)&af,4096,(size_t)cnt[l]*fs)){ free(aw); continue; } + numa_slab_bind(aw,(size_t)cnt[l]*ws); + numa_slab_bind(af,(size_t)cnt[l]*fs); + int i=0; + for(int a=from;apin[l][slot_of[a]]; + s->slab=aw+(size_t)i*ws; s->slab_cap=(int64_t)ws; s->aslab=s->slab; + s->fslab=(float*)((uint8_t*)af+(size_t)i*fs); + s->fslab_cap=(int64_t)(fs/sizeof(float)); s->afslab=s->fslab; + i++; + } + } + free(cnt); free(first); +} +#endif static double expert_avail(Model *m, double ram_gb, int ebits, int max_ctx); /* def. sotto */ static void pin_load(Model *m, const char *statspath, double gb){ FILE *f=fopen(statspath,"r"); if(!f){ perror(statspath); return; } @@ -5944,6 +5941,34 @@ static void pin_load(Model *m, const char *statspath, double gb){ double avail=expert_avail(m,ram_env,m->ebits,est_ctx); npin=avail>0?(int)(avail/eb):0; } else npin=(int)(gb*1e9/eb); +#ifdef COLI_CUDA + /* The VRAM budget must be known BEFORE npin is finalized: with + * CUDA_RELEASE_HOST the VRAM-ranked prefix's host slabs are freed right + * after upload, so those slots must NOT consume the RAM pin budget. + * Before this, a 6-GPU host lost its top ~9k ranked experts from the RAM + * count and pinned only the leftovers (measured: 9,280 VRAM + 1,721 RAM + * on a box whose RAM fits ~11k — the cold tail then paid disk forever). */ + double remaining[COLI_CUDA_MAX_DEVICES]={0}, placed_b[COLI_CUDA_MAX_DEVICES]={0}; + int placed_n[COLI_CUDA_MAX_DEVICES]={0}, gpu_prefix=0, prefix_est=0; + double budget=g_cuda_expert_gb*1e9, safe_total=0; + if(g_cuda_enabled&&(g_cuda_expert_gb>0||g_cuda_expert_auto)) for(int i=0;i0){ + prefix_est=(int)(budget/eb)+g_cuda_ndev; + npin+=prefix_est; /* additive: prefix RAM is returned after upload */ + } +#endif if(npin>n) npin=n; if(npin<1){ free(r); return; } int *cnt_l=calloc(c->n_layers+1,sizeof(int)); /* +1: riga MTP */ @@ -5954,26 +5979,26 @@ static void pin_load(Model *m, const char *statspath, double gb){ for(int i=0;i<=c->n_layers;i++) m->npin[i]=cnt_l[i]; double t0=now_s(); #ifdef COLI_CUDA - double remaining[COLI_CUDA_MAX_DEVICES]={0}, placed_b[COLI_CUDA_MAX_DEVICES]={0}; - int placed_n[COLI_CUDA_MAX_DEVICES]={0}, gpu_prefix=0; - double budget=g_cuda_expert_gb*1e9, safe_total=0; - if(g_cuda_enabled&&(g_cuda_expert_gb>0||g_cuda_expert_auto)) for(int i=0;isafe_total) budget=safe_total; - if(g_cuda_enabled&&g_cuda_release_host&&budget>0){ gpu_prefix=(int)(budget/eb)+g_cuda_ndev; if(gpu_prefix>npin)gpu_prefix=npin; } + if(prefix_est>0){ gpu_prefix=prefix_est; if(gpu_prefix>npin) gpu_prefix=npin; } #else int gpu_prefix=0; +#endif +#ifdef __linux__ + /* CPU-resident pins only: the GPU prefix stays on individual allocs (its + * host backing is released after upload; arena slices are never freed). */ + pin_arena_bind(m,r,slot_of,gpu_prefix,npin); #endif /* Load the VRAM-ranked prefix first. Once uploaded its host backing is * released before the disjoint RAM-ranked suffix is allocated. */ +#ifdef __linux__ + if(gpu_prefix>0) g_numa_skip_bind=1; /* prefix slabs = transient upload staging: don't bind (#419) */ +#endif #pragma omp parallel for schedule(dynamic,1) for(int a=0;a<(gpu_prefix?gpu_prefix:npin);a++) - expert_load(m,r[a].l,r[a].e,&m->pin[r[a].l][slot_of[a]],1); + expert_load(m,r[a].l,r[a].e,&m->pin[r[a].l][slot_of[a]],1,0); /* startup pin load; demand=0, never classified */ +#ifdef __linux__ + g_numa_skip_bind=0; +#endif m->resident_bytes+=(int64_t)(gpu_prefix?gpu_prefix:npin)*eb; #ifdef COLI_CUDA if(g_cuda_enabled && budget>0){ @@ -6008,8 +6033,11 @@ static void pin_load(Model *m, const char *statspath, double gb){ } } } - fprintf(stderr,"[CUDA] hot expert tier: %d/%d experts, VRAM %.2f GB (total budget %.1f GB)\n", - m->gpu_expert_count,npin,m->gpu_expert_bytes/1e9,g_cuda_expert_gb); + fprintf(stderr,"[CUDA] hot expert tier: %d/%d experts, VRAM %.2f GB (budget %.1f GB%s, reserve %.1f GB)\n", + m->gpu_expert_count,npin,m->gpu_expert_bytes/1e9, + g_cuda_expert_auto?safe_total/1e9:budget/1e9, + g_cuda_expert_auto?", auto":"", + g_cuda_reserve_gb); for(int i=0;i0&&gpu_prefixpin[r[a].l][slot_of[a]],1); + expert_load(m,r[a].l,r[a].e,&m->pin[r[a].l][slot_of[a]],1,0); /* startup pin load; demand=0, never classified */ m->resident_bytes+=(int64_t)(npin-gpu_prefix)*eb; } fprintf(stderr,"[PIN] placement: %d VRAM + %d RAM expert (%.1f GB warm) in %.0fs da %s\n", @@ -6250,6 +6278,25 @@ int main(int argc, char **argv){ setenv("COLI_OMP_TUNED","1",1); #ifdef __linux__ fprintf(stderr,"[OMP] hot-thread tuning: re-exec once (COLI_NO_OMP_TUNE=1 to skip)\n"); + /* #471: execv PRESERVES the CPU affinity mask. If the user exported + * OMP_PROC_BIND/OMP_PLACES, libgomp's constructor already bound THIS thread to + * place 0 (one core's SMT siblings) before main() ran; the re-exec'd image would + * inherit that 1-core mask, enumerate OMP_PLACES=cores inside it, and jail the + * whole team on one core (measured ~20x slowdown). Reset to all online CPUs so + * the fresh libgomp binds from the full set — the user's OMP_* env still wins. */ + /* CPU_SETSIZE is only exposed when _GNU_SOURCE was defined before the first + * system header. The standalone engine build defines it at the top of this + * file so the reset is active where it matters; test TUs that #include this + * file after / get it too late, so guard for them (a test + * never re-execs — skipping the reset there is harmless). */ +#ifdef CPU_SETSIZE + { cpu_set_t all; CPU_ZERO(&all); + long ncpu = sysconf(_SC_NPROCESSORS_ONLN); + if(ncpu > CPU_SETSIZE) ncpu = CPU_SETSIZE; + for(long i = 0; i < ncpu; i++) CPU_SET((int)i, &all); + if(sched_setaffinity(0, sizeof(all), &all) != 0) + perror("[OMP] sched_setaffinity pre-reexec (continuing)"); } +#endif execv("/proc/self/exe", argv); /* returns only on failure -> fall through and run untuned */ perror("[OMP] execv self-reexec failed, running untuned"); #endif @@ -6339,6 +6386,7 @@ int main(int argc, char **argv){ if(g_pilot_real) g_pilot=1; /* PILOT_REAL implica il pilota attivo */ g_pilot_two = getenv("PILOT_TWO")?atoi(getenv("PILOT_TWO")):0; /* 1 = two-step: shared-expert-corrected router prediction (+2.3% recall, 3 extra matmuls) */ if(g_pilot_two) g_pilot=1; /* PILOT_TWO implies PILOT active */ + g_pilot_evict_guard = getenv("PILOT_EVICT_GUARD")?atoi(getenv("PILOT_EVICT_GUARD")):1; /* 0 = old LRU eviction (A/B) */ /* Default K: hint-only PILOT keeps 8 (WILLNEED hints are free, no eviction). * Under PILOT_REAL the speculative loads are REAL and create LRU eviction * pressure, so at ~28% mispredict a large K thrashes the cache — default to 6 @@ -6353,9 +6401,16 @@ int main(int argc, char **argv){ 0 #endif ; + if(pipe_workers_imply_pipe(getenv("PIPE"),getenv("PIPE_WORKERS"),g_pipe)){ + g_pipe=1; + fprintf(stderr,"[PIPE] PIPE_WORKERS is set — enabling the async pipe (PIPE=1 implied; set PIPE=0 to override)\n"); + } g_pipe_nw = getenv("PIPE_WORKERS")?atoi(getenv("PIPE_WORKERS")):8; /* I/O worker threads */ if(g_pipe_nw<1) g_pipe_nw=1; + g_pipe_block = getenv("COLI_PIPE_BLOCK")?atoi(getenv("COLI_PIPE_BLOCK")):0; /* blocking pipe_wait (default: spin) */ g_direct = getenv("DIRECT")?atoi(getenv("DIRECT")):0; + { const char *dh=getenv("COLI_DISKCLASS_WINDOW"); /* DISK-CLASS recency window, see its declaration */ + if(dh){ g_direct_heat_ticks=(uint32_t)strtoul(dh,NULL,10); g_direct_heat_explicit=1; } } g_uring = getenv("URING")?atoi(getenv("URING")):0; if(g_uring){ #ifdef __linux__ @@ -6375,6 +6430,9 @@ int main(int argc, char **argv){ fprintf(stderr,"URING=1 is supported only on Linux\n"); return 2; #endif } + g_mirror_dir = getenv("COLI_MODEL_MIRROR"); /* DUAL-SSD: second model copy */ + if(!g_mirror_dir||!*g_mirror_dir) g_mirror_dir = getenv("SNAP_MIRROR"); + if(g_mirror_dir&&!*g_mirror_dir) g_mirror_dir = NULL; g_idot = getenv("IDOT")?atoi(getenv("IDOT")):1; /* 0 = kernel f32 esatti (A/B) */ g_spec_pin = getenv("SPEC_PIN")?atoi(getenv("SPEC_PIN")):1; /* #163: 0 = gate S-dipendenti storici / legacy S-dependent gates */ if(getenv("ROUTE_TRACE")&&*getenv("ROUTE_TRACE")){ @@ -6390,6 +6448,7 @@ int main(int argc, char **argv){ * EN: matmul_qt documents the int4 IDOT threshold as "configurable via I4S", but the * getenv was missing, so the knob did nothing. I4S= -> int4 IDOT only for S>=n. */ if(getenv("I4S")) g_i4s=atoi(getenv("I4S")); + if(getenv("XEXP")) g_xexp=atoi(getenv("XEXP"))!=0; g_temp = getenv("TEMP")?atof(getenv("TEMP")):-1; /* -1 = auto (1.0 chat/testo, greedy altrove) */ g_nuc = getenv("NUCLEUS")?atof(getenv("NUCLEUS")):0.90f; /* piu' stretto dell'ufficiale 0.95: la coda int4 e' rumore */ if(getenv("SEED")) g_rng = (uint64_t)atoll(getenv("SEED"))*0x9E3779B97F4A7C15ULL+1; @@ -6415,9 +6474,12 @@ int main(int argc, char **argv){ } g_cuda_dense=getenv("CUDA_DENSE")?atoi(getenv("CUDA_DENSE")):0; g_cuda_pipe=getenv("COLI_CUDA_PIPE")?atoi(getenv("COLI_CUDA_PIPE")):0; + g_cuda_router=getenv("COLI_CUDA_ROUTER")?atoi(getenv("COLI_CUDA_ROUTER")):0; + g_cuda_resid=getenv("COLI_CUDA_RESID")?atoi(getenv("COLI_CUDA_RESID")):0; const char *cuda_expert=getenv("CUDA_EXPERT_GB"); g_cuda_expert_auto=cuda_expert&&!strcmp(cuda_expert,"auto"); g_cuda_expert_gb=cuda_expert&&!g_cuda_expert_auto?atof(cuda_expert):0; + g_cuda_reserve_gb=getenv("CUDA_RESERVE_GB")?atof(getenv("CUDA_RESERVE_GB")):2.0; if(!getenv("REPIN")&&g_cuda_expert_auto&&getenv("PIN_GB")&& !strcmp(getenv("PIN_GB"),"all")) g_repin=16; g_cuda_release_host=getenv("CUDA_RELEASE_HOST")?atoi(getenv("CUDA_RELEASE_HOST")):(g_cuda_ndev>1); @@ -6454,6 +6516,19 @@ int main(int argc, char **argv){ printf("== GLM C engine (glm_moe_dsa), cache=%d experts/layer | experts@%d-bit dense@%d-bit | idot: " IDOT_KERNEL " ==\n", cap, ebits, dbits); g_mem_avail_boot = mem_available_gb(); Model m; double t0=now_s(); model_init(&m,snap,cap,ebits,dbits); + if(!g_direct_heat_explicit){ /* COLI_DISKCLASS_WINDOW default, needs m.c (topk/n_layers) */ + /* CURRENT-STATE CALIBRATION: the "8" multiplier (recency window ~= the last 8 + * tokens' worth of routing) is a measured-config constant, not a derived truth. + * Coordinates: measured 2026-07, macOS 26.5, M5 Max 128 GB, base caa49f7, + * GLM-5.2 int4 (topk=8, n_layers=78). On that setup it splits the classes + * cleanly (decode cold share ~71% of classified bytes, 1061.85/1491.49 GB); + * other models, page-cache pressures, or disks may want a different window -- + * COLI_DISKCLASS_WINDOW overrides, and the DISK-CLASS line itself is the + * tuning instrument. */ + uint32_t nl=(uint32_t)(m.c.n_layers>0?m.c.n_layers:1), k=(uint32_t)(m.c.topk>0?m.c.topk:1); + g_direct_heat_ticks = k*nl*8u; /* ~last 8 tokens' worth of routing, see the declaration */ + if(!g_direct_heat_ticks) g_direct_heat_ticks=1; + } if(g_draft<0){ #ifdef COLI_CUDA /* MTP is disabled under CUDA by default: cold (streaming) experts still @@ -6497,6 +6572,9 @@ int main(int argc, char **argv){ " Keep it on a native Linux fs (ext4/xfs/zfs) for memory efficiency and speed.\n", snap); } #endif + /* DUAL-SSD: register the mirror copy BEFORE any pin/autopin load, so the + * OMP-parallel pin warmup already streams from both drives. */ + mirror_setup(&m); /* HOT-STORE: PIN= [PIN_GB=g] -> top expert per frequenza fissi in RAM. * Va PRIMA di cap_for_ram: i pinnati contano nel residente. */ if(getenv("PIN")){ @@ -6574,10 +6652,8 @@ int main(int argc, char **argv){ /* altrimenti: validazione contro l'oracolo (ref_glm.json) */ const char *refpath=getenv("REF")?getenv("REF"):"ref_glm.json"; - FILE *f=fopen(refpath,"rb"); if(!f){perror(refpath);return 1;} - fseek(f,0,SEEK_END); long n=ftell(f); fseek(f,0,SEEK_SET); - char *b=malloc(n+1); size_t got=fread(b,1,n,f); b[got]=0; fclose(f); - if((long)got!=n) fprintf(stderr,"warning: short read on %s (%ld of %ld)\n",refpath,(long)got,n); + char *b=cfg_slurp(refpath); + if(!b){ fprintf(stderr,"%s: cannot read oracle file (missing, unreadable, short, or > %lld bytes)\n",refpath,(long long)CFG_MAX_BYTES); return 1; } char *ar=NULL; jval *ref=json_parse(b,&ar); int np=0,nfull=0; int *prompt=read_arr(ref,"prompt_ids",&np); int *full=read_arr(ref,"full_ids",&nfull); if(!prompt||!full||np<1||nfullid, &s->slot, + if (!line || !s) return 0; + s->gbytes = 0; + if (sscanf(line, "SUBMIT %llu %d %llu %d %f %f %llu %c", &s->id, &s->slot, &s->bytes, &s->max_tokens, &s->temperature, &s->top_p, - &tail) != 6) - return 0; - return s->id > 0 && s->bytes <= (16u << 20) && s->slot >= 0 && s->max_tokens >= 1 && + &s->gbytes, &tail) != 7) { + s->gbytes = 0; + if (sscanf(line, "SUBMIT %llu %d %llu %d %f %f %c", &s->id, &s->slot, + &s->bytes, &s->max_tokens, &s->temperature, &s->top_p, + &tail) != 6) + return 0; + } + return s->id > 0 && s->bytes <= (16u << 20) && s->gbytes <= (1u << 20) && + s->slot >= 0 && s->max_tokens >= 1 && isfinite(s->temperature) && isfinite(s->top_p) && s->temperature >= 0 && s->temperature <= 2 && s->top_p > 0 && s->top_p <= 1; diff --git a/c/download_fp8.py b/c/download_fp8.py index fc8e2b23..dd4e96db 100644 --- a/c/download_fp8.py +++ b/c/download_fp8.py @@ -53,14 +53,14 @@ def download_file_ms(fn): model_id=REPO_MS, file_path=fn, local_dir=DEST, - revision="master", + revision=os.environ.get("GLM_MS_REVISION", "master"), # pin to a commit for supply-chain integrity ) def download_file_hf(fn): """Download a single file from HuggingFace with hf_transfer.""" os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1" from huggingface_hub import hf_hub_download - hf_hub_download(REPO_HF, fn, local_dir=DEST) + hf_hub_download(REPO_HF, fn, local_dir=DEST, revision=os.environ.get("GLM_HF_REVISION", "main")) def download_file_curl(fn, base_url, expected_size): """Fallback: download with curl to a .part file with resume.""" diff --git a/c/json.h b/c/json.h index 8a35e244..ef590536 100644 --- a/c/json.h +++ b/c/json.h @@ -55,10 +55,12 @@ static jval *j_parse_val(jparser *p); static char *j_parse_str_raw(jparser *p) { /* assume *p->s == '"' */ p->s++; - const char *start = p->s; - /* trova la fine gestendo gli escape, poi copia decodificando i casi base */ - char tmp[1 << 16]; int n = 0; - #define J_PUT(ch) do{ if (n < (int)sizeof(tmp)-1) tmp[n++] = (char)(ch); }while(0) + /* buffer su heap che CRESCE: niente troncamento silenzioso a 64KB (le stringhe + * lunghe di tokenizer.json/config venivano tagliate) e niente 64KB di stack. */ + size_t cap = 64, n = 0; char *tmp = (char *)malloc(cap); + if (!tmp) { fprintf(stderr, "OOM parsing JSON string\n"); exit(1); } + #define J_PUT(ch) do{ if (n + 1 >= cap) { cap *= 2; tmp = (char *)realloc(tmp, cap); \ + if (!tmp) { fprintf(stderr, "OOM parsing JSON string\n"); exit(1); } } tmp[n++] = (char)(ch); }while(0) while (*p->s && *p->s != '"') { char c = *p->s++; if (c == '\\' && *p->s) { @@ -69,9 +71,11 @@ static char *j_parse_str_raw(jparser *p) { case 'f': c = '\f'; break; case '/': c = '/'; break; case '\\': c = '\\'; break; case '"': c = '"'; break; case 'u': { /* \uXXXX -> codepoint UTF-8 (con coppie surrogate) */ + if (!p->s[0]||!p->s[1]||!p->s[2]||!p->s[3]) { c='?'; break; } /* \u troncato: non leggere oltre il NUL */ unsigned cp = (unsigned)strtoul((char[]){p->s[0],p->s[1],p->s[2],p->s[3],0}, NULL, 16); p->s += 4; - if (cp >= 0xD800 && cp <= 0xDBFF && p->s[0]=='\\' && p->s[1]=='u') { + if (cp >= 0xD800 && cp <= 0xDBFF && p->s[0]=='\\' && p->s[1]=='u' + && p->s[2] && p->s[3] && p->s[4] && p->s[5]) { unsigned lo = (unsigned)strtoul((char[]){p->s[2],p->s[3],p->s[4],p->s[5],0}, NULL, 16); if (lo >= 0xDC00 && lo <= 0xDFFF) { cp = 0x10000 + ((cp-0xD800)<<10) + (lo-0xDC00); p->s += 6; } } @@ -88,8 +92,8 @@ static char *j_parse_str_raw(jparser *p) { } #undef J_PUT if (*p->s == '"') p->s++; - (void)start; - return j_dup(p, tmp, n); + char *out = j_dup(p, tmp, (int)n); free(tmp); + return out; } static jval *j_parse_val(jparser *p) { @@ -135,9 +139,9 @@ static jval *j_parse_val(jparser *p) { p->depth--; return v; } - if (c == 't') { p->s += 4; jval *v = j_new(J_BOOL); v->boolean = 1; return v; } - if (c == 'f') { p->s += 5; jval *v = j_new(J_BOOL); v->boolean = 0; return v; } - if (c == 'n') { p->s += 4; return j_new(J_NULL); } + if (c == 't' && !strncmp(p->s, "true", 4)) { p->s += 4; jval *v = j_new(J_BOOL); v->boolean = 1; return v; } + if (c == 'f' && !strncmp(p->s, "false", 5)) { p->s += 5; jval *v = j_new(J_BOOL); v->boolean = 0; return v; } + if (c == 'n' && !strncmp(p->s, "null", 4)) { p->s += 4; return j_new(J_NULL); } /* numero */ { char *end; double d = strtod(p->s, &end); p->s = end; jval *v = j_new(J_NUM); v->num = d; return v; } } diff --git a/c/kv_persist.h b/c/kv_persist.h new file mode 100644 index 00000000..51289a89 --- /dev/null +++ b/c/kv_persist.h @@ -0,0 +1,121 @@ +/* kv_persist.h — .coli_kv on-disk KV cache persistence. + * Conversations reopen warm across engine restarts: the compressed MLA KV-cache + * is appended incrementally after every turn, crash-safe (nrec written last). + * Include after Model/KVState/Cfg are defined; requires now_s() and g_draft. */ +#ifndef KV_PERSIST_H +#define KV_PERSIST_H + +static int g_kvsave=1; +#define KV_MAGIC "COLIKV1\0" + +static void kv_hdr(Model *m, int32_t *h, int nrec){ + Cfg *c=&m->c; int nic=0; + for(int i=0;in_layers;i++) if(m->Ic && m->Ic[i]) nic++; + h[0]=c->n_layers; h[1]=c->kv_lora; h[2]=c->qk_rope; + h[3]=m->has_dsa?c->index_hd:0; h[4]=nic; h[5]=c->vocab; h[6]=nrec; h[7]=0; +} + +static int64_t kv_rec_bytes(Model *m){ + Cfg *c=&m->c; + int64_t rec = 4 + (int64_t)c->n_layers*(c->kv_lora+c->qk_rope)*4; + if(m->has_dsa) for(int i=0;in_layers;i++) if(m->Ic[i]) rec+=(int64_t)c->index_hd*4; + return rec; +} + +static int kv_disk_open(Model *m){ + KVState *k=m->kv; + if(k->disk_fp) return 1; + k->disk_fp=fopen(k->disk_path,"r+b"); + if(!k->disk_fp){ + k->disk_fp=fopen(k->disk_path,"wb"); + if(!k->disk_fp) return 0; + int32_t h[8]; kv_hdr(m,h,0); + fwrite(KV_MAGIC,1,8,k->disk_fp); fwrite(h,4,8,k->disk_fp); + fflush(k->disk_fp); + fclose(k->disk_fp); + k->disk_fp=fopen(k->disk_path,"r+b"); + if(!k->disk_fp) return 0; + } + return 1; +} + +static void kv_disk_truncate(Model *m, int nrec){ + if(!g_kvsave) return; + KVState *k=m->kv; + if(k->disk_fp){ fclose(k->disk_fp); k->disk_fp=NULL; } + FILE *f=fopen(k->disk_path,"r+b"); + if(!f){ k->disk_nrec=0; return; } + k->disk_nrec=nrec; + int32_t nr=nrec; fseek(f,8+6*4,SEEK_SET); fwrite(&nr,4,1,f); + fflush(f); fclose(f); +} + +static void kv_disk_reset(Model *m){ kv_disk_truncate(m,0); } + +static void kv_disk_append(Model *m, const int *hist, int len){ + KVState *k=m->kv; + if(!g_kvsave || len<=k->disk_nrec) return; + Cfg *c=&m->c; + if(!kv_disk_open(m)) return; + FILE *f=k->disk_fp; + int64_t rec = kv_rec_bytes(m); + if(rec > k->disk_buf_cap){ + uint8_t *nb=realloc(k->disk_buf, rec); + if(!nb) return; + k->disk_buf=nb; k->disk_buf_cap=rec; + } + fseek(f, 8+8*4 + (int64_t)k->disk_nrec*rec, SEEK_SET); + for(int p=k->disk_nrec;pdisk_buf; + *(int32_t*)b = hist[p]; b+=4; + for(int i=0;in_layers;i++){ + memcpy(b, m->Lc[i]+(int64_t)p*c->kv_lora, (size_t)c->kv_lora*4); b+=c->kv_lora*4; + memcpy(b, m->Rc[i]+(int64_t)p*c->qk_rope,(size_t)c->qk_rope*4); b+=c->qk_rope*4; + } + if(m->has_dsa) for(int i=0;in_layers;i++) if(m->Ic[i]){ + memcpy(b, m->Ic[i]+(int64_t)p*c->index_hd, (size_t)c->index_hd*4); b+=c->index_hd*4; + } + fwrite(k->disk_buf, 1, (size_t)rec, f); + } + fflush(f); + int32_t nr=len; fseek(f,8+6*4,SEEK_SET); fwrite(&nr,4,1,f); + fflush(f); + k->disk_nrec=len; +} + +static int kv_disk_load(Model *m, int *hist, int maxctx){ + if(!g_kvsave) return 0; + KVState *k=m->kv; + Cfg *c=&m->c; + FILE *f=fopen(k->disk_path,"rb"); if(!f) return 0; + char mg[8]; int32_t h[8], w[8]; kv_hdr(m,w,0); + if(fread(mg,1,8,f)!=8 || memcmp(mg,KV_MAGIC,8) || fread(h,4,8,f)!=8 || + h[0]!=w[0]||h[1]!=w[1]||h[2]!=w[2]||h[3]!=w[3]||h[4]!=w[4]||h[5]!=w[5]){ + fprintf(stderr,"[KV] ignoring .coli_kv from a different model or version\n"); fclose(f); return 0; } + int nrec=h[6]; + if(nrec<1){ fclose(f); return 0; } + if(nrec>=maxctx-8-g_draft){ + fprintf(stderr,"[KV] saved conversation (%d tokens) exceeds the context: starting over\n",nrec); + fclose(f); return 0; } + double t0=now_s(); + for(int p=0;pn_layers;i++){ + if(fread(m->Lc[i]+(int64_t)p*c->kv_lora, 4, c->kv_lora, f)!=(size_t)c->kv_lora || + fread(m->Rc[i]+(int64_t)p*c->qk_rope, 4, c->qk_rope, f)!=(size_t)c->qk_rope){ nrec=p; goto out; } + } + if(m->has_dsa) for(int i=0;in_layers;i++) if(m->Ic[i]) + if(fread(m->Ic[i]+(int64_t)p*c->index_hd, 4, c->index_hd, f)!=(size_t)c->index_hd){ nrec=p; goto out; } + } +out: + fclose(f); + if(nrec>0){ + if(m->has_mtp) m->kv_start[c->n_layers]=-1; + fprintf(stderr,"[KV] resumed conversation from disk: %d tokens in %.1fs (no re-prefill)\n", + nrec, now_s()-t0); + } + k->disk_nrec=nrec; + return nrec; +} + +#endif /* KV_PERSIST_H */ diff --git a/c/olmoe.c b/c/olmoe.c index cda5913d..d9460d40 100644 --- a/c/olmoe.c +++ b/c/olmoe.c @@ -5,6 +5,15 @@ * Densa (embed, attn, router, norme, lm_head) residente in RAM (float32). * Expert letti dal disco on-demand via pread+fadvise(DONTNEED), cache LRU per-layer. * Matmul multi-thread con OpenMP (niente BLAS). + * + * ENV VARS: + * PILOT=0/1/2/3 : 0=no prefetch, 1=1-layer lookahead, 2=2-layer, 3=3-layer lookahead + * HOT=N : pin top-N hot experts per layer permanently (never evict) + * WARMUP=N : tokens before hot pinning activates (default 5) + * WIDE=N : prefetch top-K*N candidates (default 1, try 2 or 3) + * SMOOTH=F : EMA coefficient for routing momentum (default 0.3, range 0.0-0.95) + * CONF_LIMIT=F : cumulative gate probability threshold for prefetch cutoff (default 0.92) + * (expert queue is sorted by eid for SSD read locality) */ #define _GNU_SOURCE #include @@ -12,11 +21,22 @@ #include #include #include +#include #if defined(__APPLE__) || defined(__linux__) || defined(__FreeBSD__) #include +#include #endif #include "st.h" +#ifdef _WIN32 +#include +#define sleep_ms(ms) Sleep(ms) +#else +#define sleep_ms(ms) usleep((ms) * 1000) +#endif + + + /* ---------- config ---------- */ typedef struct { int hidden, n_layers, n_heads, n_kv_heads, head_dim; @@ -33,22 +53,61 @@ typedef struct { * Ogni weight [out,in] tenuto come int8 (per-riga) + scala float per riga. * Cosi' la RAM-cache scende da 4 byte/param (f32) a 1 byte/param: e' il * meccanismo che fa stare GLM-5.2 nei 15 GB. dequant-on-use nel matmul. */ -typedef struct { int eid; int8_t *g, *u, *d; float *gs, *us, *ds; uint64_t used; } Slot; +/* pinned=1 means this slot is strongly preferred to keep (hot expert); it will + * not be evicted during normal LRU eviction, but may be displaced under extreme + * cache pressure when all slots are pinned or in-flight. */ +typedef struct { int eid; int pinned; int8_t *g, *u, *d; float *gs, *us, *ds; uint64_t used; } Slot; typedef struct { Slot *slots; int n, cap; } LCache; typedef struct { Cfg c; shards S; - int quant_bits; /* bit di quantizzazione degli expert (2..8); storage int8, niente f32 (#134) */ + int quant_bits; float *embed, *lm_head, *final_norm; Layer *L; LCache *cache; /* [n_layers] */ uint64_t clock, hits, miss; - /* kv-cache per-layer: K,V come [H * maxT * head_dim] */ float **K, **V; int kv_len, max_t; double dense_load_s; + /* IMPROVEMENT 2: expert frequency heatmap */ + uint32_t *freq; + int freq_token_count, hot_pinned, hot_n, warmup_tokens; + int token_count; + /* PREDICTION IMPROVEMENT A: per-layer EMA of gate logits across tokens. + * momentum_logits[l*E .. (l+1)*E-1] = EMA of gate outputs for layer l. + * Used exclusively by the PILOT prefetcher to stabilise routing predictions + * across tokens; does NOT affect actual MoE routing (pr is unchanged). */ + float *momentum_logits; /* [n_layers * n_experts], EMA of gate logits */ + float pilot_smooth; /* SMOOTH env: EMA coefficient 0.0-0.9 (default 0.3) */ + uint8_t *is_pinned; /* [n_layers * n_experts], 1 if expert is globally pinned */ + uint8_t *is_queued; /* [n_layers * n_experts], 1 if expert is currently in the prefetch queue */ + float pilot_conf_limit; /* CONF_LIMIT env: cumulative gate probability threshold (e.g. 0.92) */ } Model; +static pthread_mutex_t g_pilot_mx = PTHREAD_MUTEX_INITIALIZER; +static struct { int l, e; } pilot_q[4096]; +static volatile unsigned pilot_r = 0, pilot_w = 0; +static Model *pilot_m = NULL; +static int g_pilot = 0; +static int g_wide = 1; /* IMPROVEMENT 4: top-K * g_wide candidates prefetched */ + +static void pilot_prefetch(Model *m, int lnext, const float *x, int S); +static void *pilot_worker(void *arg); +static void ensure_pilot_worker_started(Model *m); +static void slot_ensure_allocated(Model *m, Slot *s); + +static void ensure_pilot_worker_started(Model *m) { + if (!pilot_m) { + pilot_m = m; + pthread_t t; + if (pthread_create(&t, NULL, pilot_worker, NULL) != 0) { + fprintf(stderr, "Error: Failed to create pilot prefetch worker thread\n"); + exit(1); + } + pthread_detach(t); + } +} + /* ---------- utility ---------- */ static double now_s(void) { struct timespec t; clock_gettime(CLOCK_MONOTONIC, &t); return t.tv_sec + t.tv_nsec*1e-9; } #if defined(__APPLE__) @@ -157,20 +216,35 @@ static void softmax_row(float *x, int n) { } /* ---------- caricamento ---------- */ +/* config.json arrives from an untrusted mirror: a missing key was a NULL-deref + * (json_get(...)->num), so require each dimension present + numeric. */ +static double req_num(jval *r, const char *k){ + jval *v=json_get(r,k); + if(!v||v->t!=J_NUM){ fprintf(stderr,"config.json: missing or non-numeric \"%s\"\n",k); exit(1); } + return v->num; +} static void load_cfg(Cfg *c, const char *snap) { char path[2048]; snprintf(path, sizeof(path), "%s/config.json", snap); FILE *f = fopen(path, "rb"); if(!f){perror(path);exit(1);} fseek(f,0,SEEK_END); long n=ftell(f); fseek(f,0,SEEK_SET); - char *buf = malloc(n+1); if(fread(buf,1,n,f)!=(size_t)n){} buf[n]=0; fclose(f); + if(n<0 || n>(256L<<20)){ fprintf(stderr,"%s: config.json missing or larger than 256 MB\n",path); exit(1); } /* SEC-9 */ + char *buf = malloc((size_t)n+1); if(!buf){ fprintf(stderr,"OOM reading %s\n",path); exit(1); } + if(fread(buf,1,(size_t)n,f)!=(size_t)n){ fprintf(stderr,"%s: short read\n",path); exit(1); } buf[n]=0; fclose(f); char *arena=NULL; jval *r = json_parse(buf, &arena); - c->hidden = (int)json_get(r,"hidden_size")->num; - c->n_layers = (int)json_get(r,"num_hidden_layers")->num; - c->n_heads = (int)json_get(r,"num_attention_heads")->num; - c->n_kv_heads= (int)json_get(r,"num_key_value_heads")->num; - c->n_experts = (int)json_get(r,"num_experts")->num; - c->topk = (int)json_get(r,"num_experts_per_tok")->num; - c->inter = (int)json_get(r,"intermediate_size")->num; - c->vocab = (int)json_get(r,"vocab_size")->num; + c->hidden = (int)req_num(r,"hidden_size"); + c->n_layers = (int)req_num(r,"num_hidden_layers"); + c->n_heads = (int)req_num(r,"num_attention_heads"); + c->n_kv_heads= (int)req_num(r,"num_key_value_heads"); + c->n_experts = (int)req_num(r,"num_experts"); + c->topk = (int)req_num(r,"num_experts_per_tok"); + c->inter = (int)req_num(r,"intermediate_size"); + c->vocab = (int)req_num(r,"vocab_size"); + /* range-check so bad dims can't drive a later malloc(inter*hidden) / div-by-zero */ + if(c->hidden<1||c->hidden>(1<<20) || c->n_heads<1||c->n_heads>(1<<16) || + c->inter<1||c->inter>(1<<24) || c->vocab<1||c->vocab>(1<<24) || + c->n_layers<1||c->n_layers>4096 || c->n_experts<1||c->n_experts>(1<<20) || + c->n_kv_heads<1 || c->topk<1||c->topk>c->n_experts){ + fprintf(stderr,"config.json: dimension out of range\n"); exit(1); } c->head_dim = c->hidden / c->n_heads; jval *th = json_get(r,"rope_theta"); c->theta = th ? (float)th->num : 10000.f; jval *ep = json_get(r,"rms_norm_eps"); c->eps = ep ? (float)ep->num : 1e-5f; @@ -210,51 +284,243 @@ static void model_init(Model *m, const char *snap, int cap, int bits) { #undef LD } m->cache = calloc(c->n_layers, sizeof(LCache)); - for (int i = 0; i < c->n_layers; i++) { m->cache[i].cap = cap; m->cache[i].slots = calloc(cap, sizeof(Slot)); } + for (int i = 0; i < c->n_layers; i++) { + m->cache[i].cap = cap; + m->cache[i].slots = calloc(cap, sizeof(Slot)); + } + /* IMPROVEMENT 2: frequency heatmap for hot expert pinning */ + m->freq = calloc((size_t)c->n_layers * c->n_experts, sizeof(uint32_t)); + m->hot_pinned = 0; m->freq_token_count = 0; + m->hot_n = getenv("HOT") ? atoi(getenv("HOT")) : 0; + m->warmup_tokens = getenv("WARMUP") ? atoi(getenv("WARMUP")) : 5; + m->token_count = 0; + /* PREDICTION A: routing momentum — EMA of gate logits across tokens. + * Initialized to zero; first token sets EMA = fresh logits. */ + m->momentum_logits = calloc((size_t)c->n_layers * c->n_experts, sizeof(float)); + float sv = getenv("SMOOTH") ? (float)atof(getenv("SMOOTH")) : 0.3f; + if (sv < 0.f) sv = 0.f; if (sv > 0.95f) sv = 0.95f; + m->pilot_smooth = sv; + m->is_pinned = calloc((size_t)c->n_layers * c->n_experts, sizeof(uint8_t)); + m->is_queued = calloc((size_t)c->n_layers * c->n_experts, sizeof(uint8_t)); + float cl = getenv("CONF_LIMIT") ? (float)atof(getenv("CONF_LIMIT")) : 0.92f; + if (cl < 0.1f) cl = 0.1f; if (cl > 1.0f) cl = 1.0f; + m->pilot_conf_limit = cl; m->dense_load_s = now_s() - t0; + + // Persistent Hot Pinning: try to load hot_pinned.bin + char pinpath[512]; + snprintf(pinpath, sizeof(pinpath), "%s/hot_pinned.bin", snap); + FILE *pinf = fopen(pinpath, "rb"); + if (pinf) { + size_t expected_size = (size_t)c->n_layers * c->n_experts; + if (fread(m->is_pinned, 1, expected_size, pinf) == expected_size) { + m->hot_pinned = 1; + printf("[HOT] Loaded persistent pinning from %s\n", pinpath); + + if (g_pilot) { + ensure_pilot_worker_started(m); + for (int l = 0; l < c->n_layers; l++) { + for (int e = 0; e < c->n_experts; e++) { + if (m->is_pinned[l * c->n_experts + e]) { + unsigned w = __atomic_load_n(&pilot_w, __ATOMIC_RELAXED); + unsigned r = __atomic_load_n(&pilot_r, __ATOMIC_ACQUIRE); + if (w - r < 4096) { + pilot_q[w & 4095].l = l; pilot_q[w & 4095].e = e; + pthread_mutex_lock(&g_pilot_mx); + m->is_queued[l * c->n_experts + e] = 1; + pthread_mutex_unlock(&g_pilot_mx); + __atomic_store_n(&pilot_w, w + 1, __ATOMIC_RELEASE); + } + } + } + } + printf("[HOT] Pre-loading pinned experts into cache...\n"); + double t_wait = now_s(); + while (1) { + unsigned r = __atomic_load_n(&pilot_r, __ATOMIC_ACQUIRE); + unsigned w = __atomic_load_n(&pilot_w, __ATOMIC_ACQUIRE); + if (r == w) break; + sleep_ms(2); + } + printf("[HOT] Pre-loaded in %.1fs!\n", now_s() - t_wait); + } + } + fclose(pinf); + } } -/* legge un weight dal disco (streaming) e lo quantizza in q[O,I]+scale[O]. - * Container pre-quantizzato (convert_olmoe.py: int8 + scale f32 in "name.qs"): - * lettura raw diretta — meta' I/O e zero quantize_rows a runtime. Prima di - * questa patch il container int8 causava SIGBUS (st_read_f32 su tensori I8). */ -static void load_expert_w(Model *m, const char *name, int8_t *q, float *scale, int O, int I, float *tmp) { - st_tensor *t = st_find(&m->S, name); - if (t && t->dtype == 3) { /* I8/U8: container colibri */ - char qs[300]; snprintf(qs, sizeof(qs), "%s.qs", name); - st_read_raw(&m->S, name, q, 1); - st_read_f32(&m->S, qs, scale, 1); - return; +static void slot_ensure_allocated(Model *m, Slot *s) { + if (s->g) return; + Cfg *c = &m->c; + int64_t ng = (int64_t)c->inter * c->hidden; + int64_t nd = (int64_t)c->hidden * c->inter; + int8_t *w_block = malloc(ng + ng + nd); + if (!w_block) { + fprintf(stderr, "Error: Out of memory allocating slot weights block\n"); + exit(1); } - st_read_f32(&m->S, name, tmp, 1); /* pread + fadvise DONTNEED */ - quantize_rows(tmp, q, scale, O, I, m->quant_bits); + s->g = w_block; + s->u = w_block + ng; + s->d = w_block + ng + ng; + float *s_block = falloc(c->inter + c->inter + c->hidden); + s->gs = s_block; + s->us = s_block + c->inter; + s->ds = s_block + c->inter + c->inter; + s->pinned = 0; +} + +static void load_expert_merged(Model *m, int layer, int eid, Slot *s) { + char nm[256], qsnm[256]; + snprintf(nm, sizeof(nm), "model.layers.%d.mlp.experts.%d.merged_weight", layer, eid); + snprintf(qsnm, sizeof(qsnm), "model.layers.%d.mlp.experts.%d.qs", layer, eid); + /* SEC: st_init skips its numel*esz==nbytes cross-check for dtype-3 (U8/I8) + * tensors, so a crafted header from an untrusted mirror can declare nbytes + * far larger than the config-sized destination and st_read_raw would write + * past w_block (heap overflow); an oversized .qs likewise overruns s->gs. + * slot_ensure_allocated sizes those buffers exactly ng+ng+nd bytes and + * inter+inter+hidden floats, so require an exact match. Reject, never + * repair — same contract as qt_resolve_fmt in the GLM engine. */ + Cfg *cc = &m->c; + int64_t ng = (int64_t)cc->inter * cc->hidden, nd = (int64_t)cc->hidden * cc->inter; + int64_t want_w = ng + ng + nd; + int64_t want_s = (int64_t)cc->inter + cc->inter + cc->hidden; + st_tensor *tw = st_find(&m->S, nm), *ts = st_find(&m->S, qsnm); + if (!tw || tw->nbytes != want_w) { + fprintf(stderr, "%s: expert weight is %lld bytes — expected %lld for [inter=%d,hidden=%d], " + "refusing (untrusted container)\n", nm, (long long)(tw ? tw->nbytes : -1), + (long long)want_w, cc->inter, cc->hidden); exit(1); } + if (!ts || ts->numel != want_s) { + fprintf(stderr, "%s: scale array is %lld elems — expected %lld, refusing (untrusted container)\n", + qsnm, (long long)(ts ? ts->numel : -1), (long long)want_s); exit(1); } + st_read_raw(&m->S, nm, s->g, 1); + st_read_f32(&m->S, qsnm, s->gs, 0); /* scales are F32; use typed reader for dtype safety */ } /* ---------- cache expert: ritorna i pesi quantizzati (q+scale) da cache o disco ---------- */ static void expert_get(Model *m, int layer, int eid, Slot **out) { LCache *lc = &m->cache[layer]; + pthread_mutex_lock(&g_pilot_mx); for (int i = 0; i < lc->n; i++) if (lc->slots[i].eid == eid) { - m->hits++; lc->slots[i].used = ++m->clock; *out = &lc->slots[i]; return; + m->hits++; lc->slots[i].used = ++m->clock; *out = &lc->slots[i]; + pthread_mutex_unlock(&g_pilot_mx); + return; } m->miss++; Cfg *c = &m->c; - int64_t ng = (int64_t)c->inter * c->hidden, nd = (int64_t)c->hidden * c->inter; Slot *s; if (lc->n < lc->cap) { s = &lc->slots[lc->n++]; - s->g = malloc(ng); s->u = malloc(ng); s->d = malloc(nd); - s->gs = falloc(c->inter); s->us = falloc(c->inter); s->ds = falloc(c->hidden); - } else { int lru = 0; for (int i = 1; i < lc->n; i++) if (lc->slots[i].used < lc->slots[lru].used) lru = i; s = &lc->slots[lru]; } - float *tmp = falloc(ng > nd ? ng : nd); - char nm[256]; - snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.%d.gate_proj.weight",layer,eid); load_expert_w(m,nm,s->g,s->gs,c->inter,c->hidden,tmp); - snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.%d.up_proj.weight", layer,eid); load_expert_w(m,nm,s->u,s->us,c->inter,c->hidden,tmp); - snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.%d.down_proj.weight",layer,eid); load_expert_w(m,nm,s->d,s->ds,c->hidden,c->inter,tmp); - free(tmp); - s->eid = eid; s->used = ++m->clock; + slot_ensure_allocated(m, s); + } else { + /* LRU eviction — skip pinned and in-flight (eid==-1) slots */ + int lru = -1; + for (int i = 0; i < lc->n; i++) { + if (lc->slots[i].pinned || lc->slots[i].eid < 0) continue; + if (lru < 0 || lc->slots[i].used < lc->slots[lru].used) lru = i; + } + if (lru < 0) { + /* All slots are pinned or in-flight; find oldest non-in-flight slot + * (may be pinned, but never select one currently being loaded). */ + for (int i = 0; i < lc->n; i++) { + if (lc->slots[i].eid < 0) continue; /* never evict in-flight */ + if (lru < 0 || lc->slots[i].used < lc->slots[lru].used) lru = i; + } + } + if (lru < 0) lru = 0; /* absolute last resort: all in-flight, evict slot 0 */ + s = &lc->slots[lru]; + s->pinned = 0; + } + s->eid = -1; + s->used = ++m->clock; + pthread_mutex_unlock(&g_pilot_mx); + + load_expert_merged(m, layer, eid, s); + + pthread_mutex_lock(&g_pilot_mx); + s->eid = eid; + s->pinned = m->is_pinned[layer * c->n_experts + eid]; + s->used = ++m->clock; *out = s; + pthread_mutex_unlock(&g_pilot_mx); } +/* ---------- IMPROVEMENT 2: pin top-N hot experts per layer ---------- */ +static void pin_hot_experts(Model *m) { + Cfg *c = &m->c; + if (m->hot_n <= 0 || m->hot_pinned) return; + m->hot_pinned = 1; + + int is_dynamic = (m->hot_n >= 100); + double thresh = is_dynamic ? (double)m->hot_n / 1000.0 : 0.0; + + int pinned_total = 0; + for (int l = 0; l < c->n_layers; l++) { + uint32_t *freq_l = m->freq + (int64_t)l * c->n_experts; + + uint64_t layer_total = 0; + for (int e = 0; e < c->n_experts; e++) layer_total += freq_l[e]; + if (layer_total == 0) continue; + + int max_pin = m->cache[l].cap - 8; + if (max_pin < 4) max_pin = 4; + + int hn = is_dynamic ? max_pin : (m->hot_n < c->n_experts ? m->hot_n : c->n_experts); + if (hn > 256) hn = 256; + int hot_eids[256]; + int actual_hn = 0; + + for (int k = 0; k < hn; k++) { + int best = -1; uint32_t bv = 0; + for (int e = 0; e < c->n_experts; e++) { + int already = 0; + for (int j = 0; j < k; j++) if (hot_eids[j] == e) { already = 1; break; } + if (!already && freq_l[e] > bv) { bv = freq_l[e]; best = e; } + } + if (best < 0 || bv == 0) break; + if (is_dynamic && bv < thresh * layer_total) break; + hot_eids[k] = best; + actual_hn++; + } + + for (int k = 0; k < actual_hn; k++) { + int eid = hot_eids[k]; + m->is_pinned[l * c->n_experts + eid] = 1; + + LCache *lc = &m->cache[l]; + int found = 0; + pthread_mutex_lock(&g_pilot_mx); + for (int i = 0; i < lc->n; i++) { + if (lc->slots[i].eid == eid) { lc->slots[i].pinned = 1; found = 1; break; } + } + pthread_mutex_unlock(&g_pilot_mx); + if (!found && g_pilot > 0) { + /* Only enqueue when the prefetch worker is active (PILOT>0). */ + ensure_pilot_worker_started(m); + unsigned w = __atomic_load_n(&pilot_w, __ATOMIC_RELAXED); + unsigned r = __atomic_load_n(&pilot_r, __ATOMIC_ACQUIRE); + int gidx = l * c->n_experts + eid; + pthread_mutex_lock(&g_pilot_mx); + int already = m->is_queued[gidx]; + if (!already && w - r < 4096) { + pilot_q[w & 4095].l = l; pilot_q[w & 4095].e = eid; + m->is_queued[gidx] = 1; + __atomic_store_n(&pilot_w, w + 1, __ATOMIC_RELEASE); + } + pthread_mutex_unlock(&g_pilot_mx); + } + pinned_total++; + } + } + if (is_dynamic) { + printf("[HOT] Dynamic Pinned %d experts total (thresh=%.1f%%) after %d warmup tokens\n", + pinned_total, thresh * 100.0, m->freq_token_count); + } else { + printf("[HOT] Pinned %d experts (top-%d/layer) after %d warmup tokens\n", + pinned_total, m->hot_n, m->freq_token_count); + } +} + + /* ---------- RoPE su un vettore di una testa (head_dim) a posizione assoluta pos ---------- */ static void rope_head(float *x, int pos, const Cfg *c) { int h = c->head_dim / 2; @@ -325,6 +591,19 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out) { float *g = falloc(I), *u = falloc(I), *hh = falloc(D); for (int s = 0; s < S; s++) { float *pr = logits + (int64_t)s*E; + if (m->momentum_logits && m->pilot_smooth > 0.f) { + float *ema = m->momentum_logits + (int64_t)layer * E; + int is_zero = 1; + for (int e = 0; e < E; e++) { if (ema[e] != 0.f) { is_zero = 0; break; } } + if (is_zero) { + for (int e = 0; e < E; e++) ema[e] = pr[e]; + } else { + for (int e = 0; e < E; e++) { + ema[e] = (1.f - m->pilot_smooth) * pr[e] + m->pilot_smooth * ema[e]; + } + } + } + softmax_row(pr, E); /* top-K indici (selezione parziale) */ int idx[64]; float val[64]; @@ -337,6 +616,11 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out) { idx[kk] = best; val[kk] = bv; } if (c->norm_topk) { float sm=0; for(int kk=0;kkhot_pinned && m->freq) { + uint32_t *freq_l = m->freq + (int64_t)layer * E; + for (int kk = 0; kk < K; kk++) if (idx[kk] >= 0) freq_l[idx[kk]]++; + } const float *xs = x + (int64_t)s*D; for (int kk = 0; kk < K; kk++) { Slot *e; expert_get(m, layer, idx[kk], &e); @@ -352,9 +636,20 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out) { free(logits); free(g); free(u); free(hh); } -/* un passo: token nuovi ids[S] a posizione pos_base. Ritorna logits dell'ultimo token (malloc'd). */ static float *step(Model *m, const int *ids, int S, int pos_base) { Cfg *c = &m->c; int D = c->hidden; + if (g_pilot && m->token_count > 0) { + /* Flush stale prefetch requests: clear is_queued so pilot_realload + * will skip any entries still sitting in pilot_q for the previous + * token. We deliberately do NOT move pilot_w backwards; that would + * break the ring-buffer invariant (pilot_r could exceed pilot_w if + * the worker consumed an entry concurrently). The worker will drain + * the stale slots harmlessly because pilot_realload already exits + * early when the expert is already cached or is_queued is clear. */ + pthread_mutex_lock(&g_pilot_mx); + memset(m->is_queued, 0, (size_t)c->n_layers * c->n_experts); + pthread_mutex_unlock(&g_pilot_mx); + } float *x = falloc((int64_t)S*D); for (int s = 0; s < S; s++) memcpy(x + (int64_t)s*D, m->embed + (int64_t)ids[s]*D, D*sizeof(float)); float *nrm = falloc((int64_t)S*D), *tmp = falloc((int64_t)S*D); @@ -363,12 +658,26 @@ static float *step(Model *m, const int *ids, int S, int pos_base) { for (int s = 0; s < S; s++) rmsnorm_row(nrm + (int64_t)s*D, x + (int64_t)s*D, l->in_ln, D, c->eps); attention(m, l, i, nrm, S, pos_base, tmp); for (int64_t j = 0; j < (int64_t)S*D; j++) x[j] += tmp[j]; + /* IMPROVEMENT 1: PILOT=1 -> 1-layer lookahead */ + if (g_pilot >= 1 && S <= 8 && i + 1 < c->n_layers) + pilot_prefetch(m, i + 1, x, S); for (int s = 0; s < S; s++) rmsnorm_row(nrm + (int64_t)s*D, x + (int64_t)s*D, l->post_ln, D, c->eps); moe(m, l, i, nrm, S, tmp); for (int64_t j = 0; j < (int64_t)S*D; j++) x[j] += tmp[j]; + + /* PREDICTION IMPROVEMENT C (Residual gate trick): + * PILOT=2 -> prefetch layer i+2 using completed state x (containing MoE residual). */ + if (g_pilot >= 2 && S <= 8 && i + 2 < c->n_layers) + pilot_prefetch(m, i + 2, x, S); + if (g_pilot >= 3 && S <= 8 && i + 3 < c->n_layers) + pilot_prefetch(m, i + 3, x, S); + } + /* count actual tokens processed (S>1 during prefill) */ + m->token_count += S; m->freq_token_count += S; + if (!m->hot_pinned && m->hot_n > 0 && m->freq_token_count >= m->warmup_tokens) + pin_hot_experts(m); m->kv_len = pos_base + S; - /* solo l'ultimo token -> logits */ float *last = falloc(D); rmsnorm_row(last, x + (int64_t)(S-1)*D, m->final_norm, D, c->eps); float *logit = falloc(c->vocab); @@ -377,6 +686,192 @@ static float *step(Model *m, const int *ids, int S, int pos_base) { return logit; } +static void pilot_realload(Model *m, int layer, int eid) { + LCache *lc = &m->cache[layer]; + Cfg *c = &m->c; + + pthread_mutex_lock(&g_pilot_mx); + /* Early-exit if entry was flushed (is_queued cleared) while waiting. */ + if (!m->is_queued[layer * c->n_experts + eid]) { + pthread_mutex_unlock(&g_pilot_mx); + return; + } + for (int i = 0; i < lc->n; i++) { + if (lc->slots[i].eid == eid) { + m->is_queued[layer * c->n_experts + eid] = 0; + pthread_mutex_unlock(&g_pilot_mx); + return; + } + } + Slot *s; + if (lc->n < lc->cap) { + s = &lc->slots[lc->n++]; + slot_ensure_allocated(m, s); + } else { + /* LRU eviction — skip pinned and in-flight (eid==-1) slots */ + int lru = -1; + for (int i = 0; i < lc->n; i++) { + if (lc->slots[i].pinned || lc->slots[i].eid < 0) continue; + if (lru < 0 || lc->slots[i].used < lc->slots[lru].used) lru = i; + } + if (lru < 0) { + m->is_queued[layer * c->n_experts + eid] = 0; + pthread_mutex_unlock(&g_pilot_mx); + return; /* all pinned/in-flight, skip */ + } + s = &lc->slots[lru]; s->pinned = 0; + } + s->eid = -1; s->used = ++m->clock; + pthread_mutex_unlock(&g_pilot_mx); + + load_expert_merged(m, layer, eid, s); + + pthread_mutex_lock(&g_pilot_mx); + s->eid = eid; + s->pinned = m->is_pinned[layer * c->n_experts + eid]; + s->used = ++m->clock; + m->is_queued[layer * c->n_experts + eid] = 0; + pthread_mutex_unlock(&g_pilot_mx); +} + +static void *pilot_worker(void *arg) { + (void)arg; + while (1) { + unsigned r = __atomic_load_n(&pilot_r, __ATOMIC_ACQUIRE); + unsigned w = __atomic_load_n(&pilot_w, __ATOMIC_ACQUIRE); + if (r == w) { + sleep_ms(1); + continue; + } + int layer = pilot_q[r & 4095].l; + int eid = pilot_q[r & 4095].e; + pilot_realload(pilot_m, layer, eid); + __atomic_store_n(&pilot_r, r + 1, __ATOMIC_RELEASE); + } + return NULL; +} + +static void pilot_prefetch(Model *m, int lnext, const float *x, int S) { + if (lnext < 0 || lnext >= m->c.n_layers) return; + Cfg *c = &m->c; int D = c->hidden, E = c->n_experts; + ensure_pilot_worker_started(m); + float *logits = falloc((int64_t)S * E); + Layer *l = &m->L[lnext]; + + // PREDICTION IMPROVEMENT B: Apply RMSNorm to x using destination layer's post_ln + // This scales inputs to the distribution expected by l->gate. + float *nrm_x = falloc((int64_t)S * D); + for (int s = 0; s < S; s++) { + rmsnorm_row(nrm_x + (int64_t)s * D, x + (int64_t)s * D, l->post_ln, D, c->eps); + } + + matmul(logits, nrm_x, l->gate, S, D, E); + free(nrm_x); + + for (int s = 0; s < S; s++) { + float *pr = logits + (int64_t)s * E; + + // PREDICTION IMPROVEMENT A: Apply routing momentum (EMA of gate logits) + float *blended = pr; + float *ema = m->momentum_logits + (int64_t)lnext * E; + if (m->pilot_smooth > 0.f) { + blended = falloc(E); + int is_zero = 1; + for (int e = 0; e < E; e++) { if (ema[e] != 0.f) { is_zero = 0; break; } } + if (is_zero) { + for (int e = 0; e < E; e++) { + ema[e] = pr[e]; + blended[e] = pr[e]; + } + } else { + for (int e = 0; e < E; e++) { + blended[e] = (1.f - m->pilot_smooth) * pr[e] + m->pilot_smooth * ema[e]; + ema[e] = blended[e]; // update EMA + } + } + } + + int cand = 0; + int idx[128]; + + float max_logit = -1e30f; + for (int e = 0; e < E; e++) { if (blended[e] > max_logit) max_logit = blended[e]; } + float *exps = falloc(E); + float sum_exps = 0.f; + for (int e = 0; e < E; e++) { + exps[e] = expf(blended[e] - max_logit); + sum_exps += exps[e]; + } + + float cum_sum = 0.f; + int min_cand = c->topk; + int max_cand = c->topk * g_wide; + if (max_cand < min_cand) max_cand = min_cand; + if (max_cand > 128) max_cand = 128; /* idx[] buffer bound */ + if (max_cand > E) max_cand = E; + + for (int kk = 0; kk < max_cand; kk++) { + int best = -1; float bv = -1.f; + for (int e = 0; e < E; e++) { + int taken = 0; for (int j = 0; j < kk; j++) if (idx[j] == e) { taken=1; break; } + if (!taken && exps[e] > bv) { bv = exps[e]; best = e; } + } + if (best < 0) break; + idx[kk] = best; + cum_sum += bv; + cand++; + if (cum_sum >= m->pilot_conf_limit * sum_exps && cand >= min_cand) { + break; + } + } + free(exps); + + if (blended != pr) free(blended); + + /* IMPROVEMENT 5: sort candidates by eid for sequential SSD read locality */ + for (int a = 0; a < cand-1; a++) + for (int b = a+1; b < cand; b++) + if (idx[b] >= 0 && (idx[a] < 0 || idx[a] > idx[b])) { int t = idx[a]; idx[a] = idx[b]; idx[b] = t; } + + for (int kk = 0; kk < cand; kk++) { + int eid = idx[kk]; + if (eid < 0) continue; + int found = 0; + pthread_mutex_lock(&g_pilot_mx); + LCache *lc = &m->cache[lnext]; + for (int z = 0; z < lc->n; z++) { + if (lc->slots[z].eid == eid) { found = 1; break; } + } + pthread_mutex_unlock(&g_pilot_mx); + if (!found) { + int gidx = lnext * E + eid; + pthread_mutex_lock(&g_pilot_mx); + int already_queued = m->is_queued[gidx]; + if (!already_queued) { + m->is_queued[gidx] = 1; + } + pthread_mutex_unlock(&g_pilot_mx); + + if (!already_queued) { + unsigned w2 = __atomic_load_n(&pilot_w, __ATOMIC_RELAXED); + unsigned r2 = __atomic_load_n(&pilot_r, __ATOMIC_ACQUIRE); + if (w2 - r2 < 4096) { + pilot_q[w2 & 4095].l = lnext; + pilot_q[w2 & 4095].e = eid; + __atomic_store_n(&pilot_w, w2 + 1, __ATOMIC_RELEASE); + } else { + pthread_mutex_lock(&g_pilot_mx); + m->is_queued[gidx] = 0; + pthread_mutex_unlock(&g_pilot_mx); + } + } + } + } + } + free(logits); +} + + /* generazione greedy. prompt[np] -> riempie out[np+n_new] */ static void generate(Model *m, const int *prompt, int np, int n_new, int *out) { Cfg *c = &m->c; @@ -442,22 +937,32 @@ static int *read_int_array(jval *o, const char *key, int *n_out) { int main(int argc, char **argv) { const char *snap = getenv("SNAP"); if (!snap) { fprintf(stderr, "set SNAP=\n"); return 1; } - int cap = argc > 1 ? atoi(argv[1]) : 16; - int bits = argc > 2 ? atoi(argv[2]) : 8; - if (bits < 2 || bits > 8) { /* expert storage is int8_t: bits>8 truncates in quantize_rows (#134). f32 mode is not implemented here — int8 is already token-exact vs the oracle. */ - fprintf(stderr, "quant_bits must be 2..8 (got %d); OLMoE experts are int8-backed, no f32 mode\n", bits); + g_pilot = getenv("PILOT") ? atoi(getenv("PILOT")) : 0; + g_wide = getenv("WIDE") ? atoi(getenv("WIDE")) : 1; + if (g_wide < 1) g_wide = 1; + if (g_wide > 4) g_wide = 4; + int hot_n = getenv("HOT") ? atoi(getenv("HOT")) : 0; + int cap = argc > 1 ? atoi(argv[1]) : 16; + int bits = argc > 2 ? atoi(argv[2]) : 8; + if (bits < 2 || bits > 8) { + fprintf(stderr, "quant_bits must be 2..8 (got %d)\n", bits); return 1; } const char *refpath = argc > 3 ? argv[3] : "ref.json"; - FILE *f = fopen(refpath, "rb"); if(!f){perror(refpath);return 1;} + float smooth = getenv("SMOOTH") ? (float)atof(getenv("SMOOTH")) : 0.3f; + float conf = getenv("CONF_LIMIT") ? (float)atof(getenv("CONF_LIMIT")) : 0.92f; + + printf("== Streaming C engine v2.2 | cache=%d/layer bits=%d pilot=%d wide=%d hot=%d smooth=%.2f conf=%.2f ==\n", + cap, bits, g_pilot, g_wide, hot_n, smooth, conf); + + FILE *f = fopen(refpath, "rb"); if (!f) { perror(refpath); return 1; } fseek(f,0,SEEK_END); long n=ftell(f); fseek(f,0,SEEK_SET); - char *buf=malloc(n+1); if(fread(buf,1,n,f)!=(size_t)n){} buf[n]=0; fclose(f); + char *buf=malloc(n+1); if (fread(buf,1,n,f)!=(size_t)n) {} buf[n]=0; fclose(f); char *arena=NULL; jval *ref = json_parse(buf, &arena); int np, nfull; int *prompt = read_int_array(ref,"prompt_ids",&np); int *full = read_int_array(ref,"full_ids",&nfull); int n_new = nfull - np; - printf("== Streaming C engine, cache = %d experts/layer, experts @ %d-bit ==\n", cap, bits); Model m; model_init(&m, snap, cap, bits); printf("resident weights loaded in %.1fs | RSS after load: %.2f GB\n", m.dense_load_s, rss_gb()); @@ -487,6 +992,26 @@ int main(int argc, char **argv) { printf("\nPEAK RSS: %.2f GB\n", rss_gb()); printf("Expert cache hit rate: %.1f%% (hit=%llu miss=%llu)\n", tot?100.0*m.hits/tot:0.0, (unsigned long long)m.hits, (unsigned long long)m.miss); + + + // Persistent Hot Pinning: save dynamic pinning if newly created + if (m.hot_pinned) { + char pinpath[512]; + snprintf(pinpath, sizeof(pinpath), "%s/hot_pinned.bin", snap); + FILE *pinf_chk = fopen(pinpath, "rb"); + if (!pinf_chk) { + FILE *pinf_save = fopen(pinpath, "wb"); + if (pinf_save) { + size_t expected_size = (size_t)m.c.n_layers * m.c.n_experts; + fwrite(m.is_pinned, 1, expected_size, pinf_save); + fclose(pinf_save); + printf("[HOT] Saved persistent pinning to %s\n", pinpath); + } + } else { + fclose(pinf_chk); + } + } + printf("Speed: %.2f tok/s (%.1fs for %d tokens)\n", n_new/dt, dt, n_new); free(buf); free(arena); return 0; diff --git a/c/openai_server.py b/c/openai_server.py index 3900ba6a..50afe2f0 100644 --- a/c/openai_server.py +++ b/c/openai_server.py @@ -59,6 +59,24 @@ def error_object(error): "param": error.param, "code": error.code}} +def _engine_error(fields, message): + """Turn an engine ERROR frame into the right exception type. + + CONTEXT_EXCEEDED is a client mistake, not a server fault: the prompt is longer than the + engine's context. Report it the way every OpenAI-compatible server does, so clients that + know how to compact a conversation actually get the chance to (previously the engine + silently truncated the prompt instead, which is #401).""" + if fields and fields[0] == "CONTEXT_EXCEEDED": + limit = fields[2] if len(fields) > 2 else "the context" + used = fields[1] if len(fields) > 1 else "?" + return APIError(400, + f"This model's maximum context length is {limit} tokens, however your " + f"messages resulted in at least {used} tokens. Please shorten the " + f"conversation, or restart the server with a larger CTX.", + "messages", "context_length_exceeded") + return RuntimeError(message) + + class GenerationScheduler: """Bounded FIFO admission for the engine's independent KV contexts.""" @@ -86,6 +104,7 @@ def __init__(self, max_queue=8, queue_timeout=300, capacity=1): @contextlib.contextmanager def admit(self, cancelled=None, slot=None): ticket = object() + entry = (ticket, slot) # (#B2) remember each waiter's target slot for fair, per-slot admission queued_at = time.monotonic() with self.condition: if self.closed: @@ -95,31 +114,46 @@ def admit(self, cancelled=None, slot=None): self.rejected += 1 raise APIError(429, "The inference queue is full.", None, "queue_full", "rate_limit_error", {"Retry-After": "1"}) - self.queue.append(ticket) + self.queue.append(entry) deadline = queued_at + self.queue_timeout while True: if self.closed: - self.queue.remove(ticket) + self.queue.remove(entry) self.condition.notify_all() raise APIError(503, "The inference scheduler is shutting down.", None, "scheduler_closed", "server_error") available = min(self.free_slots) if slot is None and self.free_slots else slot - if self.queue[0] is ticket and available in self.free_slots: + # (#B2) Admit as soon as our target slot is free AND no strictly-earlier + # waiter also wants it (an earlier waiter "wants" it if it is any-slot or + # pinned to the same slot). This replaces the old strict FIFO-head rule, + # which let a head pinned to a busy slot block every request behind it — + # even ones targeting a currently-free slot (head-of-line blocking). + # ponytail: O(queue) scan per wakeup — negligible at the default max_queue; + # switch to per-slot wait sets if max_queue is ever raised to thousands. + can_admit = available in self.free_slots + if can_admit: + for t2, s2 in self.queue: + if t2 is ticket: + break + if s2 is None or s2 == available: + can_admit = False + break + if can_admit: break if cancelled and cancelled(): - self.queue.remove(ticket) + self.queue.remove(entry) self.cancelled += 1 self.condition.notify_all() raise ClientCancelled() remaining = deadline - time.monotonic() if remaining <= 0: - self.queue.remove(ticket) + self.queue.remove(entry) self.timed_out += 1 self.condition.notify_all() raise APIError(429, "Timed out waiting for the inference engine.", None, "queue_timeout", "rate_limit_error", {"Retry-After": "1"}) self.condition.wait(min(remaining, 0.25)) - self.queue.popleft() + self.queue.remove(entry) self.free_slots.remove(available) self.active += 1 self.admitted += 1 @@ -180,6 +214,8 @@ def content_text(content, param): _ARG_RE = re.compile(r"([^<]*)(.*?)", re.DOTALL) _NAME_RE = re.compile(r"\s*([A-Za-z0-9_.\-]+)") _TAG_RE = re.compile(r"|") +# A closing tag the model started but never finished ("K structure. Default OFF (never rewrites well-formed output). @@ -241,14 +277,41 @@ def _coerce_arg(value, declared): return parsed +def _unclosed_tail(reply, tools): + """Body of a trailing that was never closed, or None. + + Only returned when the recovery is unambiguous, so ordinary prose that merely mentions + "" can never be turned into a call. Both conditions must hold: + * the last BOX_START is not followed by a BOX_END (a closed box is the strict parser's job); + * the tail carries a complete .. pair, OR it is exactly the name of a + tool the client declared (the zero-argument case). + """ + start = reply.rfind(BOX_START) + if start < 0 or BOX_END in reply[start:]: + return None + inner = _PARTIAL_END_RE.sub("", reply[start + len(BOX_START):]) + if _ARG_RE.search(inner): + return inner + declared = {(t.get("function", t) if isinstance(t, dict) else {}).get("name") + for t in (tools or []) if isinstance(t, dict)} + return inner if inner.strip() in declared else None + + def parse_tool_calls(reply, tools=None): """Return (content, tool_calls). Strict GLM parse; optional de-mangler (COLI_TOOL_SALVAGE=1) rescues malformed int4 output by mapping a lone payload onto the tool's primary parameter.""" param_order = _tool_param_order(tools) param_types = _tool_param_types(tools) calls, salvaged = [], [] - for match in _BOX_RE.finditer(reply): - inner = match.group(1) + # #401: a box the model opened but never closed -- it ran out of budget, or the closing tag + # came out mangled (" " + ", ".join(salvaged)) if dm else "")) sys.stderr.flush() return text.strip(), calls @@ -370,10 +437,46 @@ def render_chat(messages, enable_thinking=False, reasoning_effort=None, tools=No return "".join(prompt) +# Generic whitespace-tolerant JSON grammar for response_format {"type": "json_object"}. +# Draft-source semantics: positions with one legal byte draft; jws points just keep +# the walker alive through the model's own spacing (see docs/grammar-draft.md). +GENERIC_JSON_GBNF = ( + 'root ::= jws jval jws\n' + 'jval ::= jobj | jarr | jstr | jnum | "true" | "false" | "null"\n' + 'jobj ::= "{" jws ( jstr jws ":" jws jval jws ( "," jws jstr jws ":" jws jval jws )* )? "}"\n' + 'jarr ::= "[" jws ( jval jws ( "," jws jval jws )* )? "]"\n' + 'jstr ::= "\\"" jchar* "\\""\n' + 'jchar ::= [^"\\\\\\x00-\\x1f] | "\\\\" ( ["\\\\/bfnrt] | "u" jhex jhex jhex jhex )\n' + 'jhex ::= [0-9a-fA-F]\n' + 'jnum ::= "-"? ( "0" | [1-9] [0-9]* ) ( "." [0-9]+ )? ( ( "e" | "E" ) ( "+" | "-" )? [0-9]+ )?\n' + 'jws ::= ( " " | "\\t" | "\\n" | "\\r" )*\n' +) + def generation_options(body, limit): if body.get("n", 1) != 1: raise APIError(400, "Colibri currently supports `n=1` only.", "n", "unsupported_value") # `tools`/`functions` are handled by render_chat (declaration) + parse_tool_calls (output). + # Validate tools/functions structure early so malformed input fails with a clear error. + tools_raw = body.get("tools") or body.get("functions") + if tools_raw is not None: + if not isinstance(tools_raw, list): + raise APIError(400, "`tools` must be a non-empty array.", "tools", "invalid_value") + if not tools_raw: + raise APIError(400, "`tools` must be a non-empty array.", "tools", "invalid_value") + for idx, tool in enumerate(tools_raw): + if not isinstance(tool, dict): + raise APIError(400, f"Each tool must be an object, got {type(tool).__name__} at index {idx}.", + f"tools.{idx}", "invalid_value") + fn = tool.get("function", tool) if isinstance(tool, dict) else {} + if not isinstance(fn, dict): + raise APIError(400, f"Tool function must be an object at index {idx}.", + f"tools.{idx}.function", "invalid_value") + if not fn.get("name"): + raise APIError(400, f"Each tool must have a `name` at index {idx}.", + f"tools.{idx}.function.name", "invalid_value") + if not isinstance(fn["name"], str): + raise APIError(400, f"Tool `name` must be a string at index {idx}.", + f"tools.{idx}.function.name", "invalid_value") choice = body.get("tool_choice") if choice is not None: if isinstance(choice, str): @@ -403,10 +506,38 @@ def generation_options(body, limit): raise APIError(400, "Token penalties are not supported yet.", None, "unsupported_parameter") if body.get("seed") is not None: raise APIError(400, "Per-request seeds are not supported yet.", "seed", "unsupported_parameter") + # response_format -> optional per-request grammar for the engine's grammar-forced + # draft source (#70/#148). NEVER a sampling constraint: drafts are verified, so a + # schema the engine cannot compile degrades to "no speedup", not to an error and + # not to changed output. json_schema payloads are forwarded as-is (the engine + # compiles them via schema_gbnf.h); {"type": "gbnf"} is a raw-GBNF extension. + grammar = None response_format = body.get("response_format") - if response_format not in (None, {"type": "text"}): - raise APIError(400, "Only the default text response format is supported.", - "response_format", "unsupported_parameter") + if response_format is not None and response_format != {"type": "text"}: + if not isinstance(response_format, dict) or "type" not in response_format: + raise APIError(400, "`response_format` must be an object with a `type`.", + "response_format", "invalid_value") + ftype = response_format["type"] + if ftype == "json_object": + grammar = GENERIC_JSON_GBNF + elif ftype == "json_schema": + schema = (response_format.get("json_schema") or {}).get("schema") + if not isinstance(schema, dict): + raise APIError(400, "`response_format.json_schema.schema` must be an object.", + "response_format", "invalid_value") + grammar = json.dumps(schema) + elif ftype == "gbnf": + grammar = response_format.get("grammar") + if not isinstance(grammar, str) or not grammar.strip(): + raise APIError(400, "`response_format.grammar` must be a non-empty GBNF string.", + "response_format", "invalid_value") + else: + raise APIError(400, "`response_format.type` must be \"text\", \"json_object\", " + "\"json_schema\" or \"gbnf\".", + "response_format", "unsupported_value") + if grammar is not None and len(grammar.encode("utf-8")) > (1 << 20): + raise APIError(400, "`response_format` grammar/schema exceeds 1 MiB.", + "response_format", "invalid_value") maximum = body.get("max_completion_tokens") maximum_param = "max_completion_tokens" @@ -433,7 +564,7 @@ def generation_options(body, limit): if (isinstance(top_p, bool) or not isinstance(top_p, (int, float)) or not math.isfinite(top_p) or not 0 < top_p <= 1): raise APIError(400, "`top_p` must be greater than 0 and at most 1.", "top_p") - return maximum, float(temperature), float(top_p) + return maximum, float(temperature), float(top_p), grammar def read_engine_turn(stream, sentinel, on_bytes): @@ -588,7 +719,7 @@ def _dispatch_stdout(self): with self.pending_lock: events = self.pending.pop(request_id, None) if events is not None: - events.put(("error", RuntimeError(message))) + events.put(("error", _engine_error(fields[2:], message))) else: raise RuntimeError(f"invalid engine response: {' '.join(fields)}") except Exception as error: @@ -597,12 +728,15 @@ def _dispatch_stdout(self): self._fail_pending(error) def generate(self, prompt, max_tokens, temperature, top_p, on_text, cache_slot=0, - cancelled=None): + cancelled=None, grammar=None): if isinstance(cache_slot, bool) or not isinstance(cache_slot, int) or not 0 <= cache_slot < self.kv_slots: raise APIError(400, "Invalid cache slot.", "cache_slot") payload = prompt.encode("utf-8") if b"\0" in payload: raise APIError(400, "NUL bytes are not supported in prompts.", "messages") + gpayload = grammar.encode("utf-8") if grammar else b"" + if b"\0" in gpayload: + raise APIError(400, "NUL bytes are not supported in grammars.", "response_format") decoder = codecs.getincrementaldecoder("utf-8")("replace") def decode(data): @@ -622,12 +756,13 @@ def decode(data): self.next_request_id += 1 self.pending[request_id] = events header = (f"SUBMIT {request_id} {cache_slot} {len(payload)} {max_tokens} " - f"{temperature:.8g} {top_p:.8g}\n").encode() + f"{temperature:.8g} {top_p:.8g}" + + (f" {len(gpayload)}" if gpayload else "") + "\n").encode() try: with self.write_lock: if self.process.poll() is not None: raise RuntimeError("colibri engine is not running") - self.process.stdin.write(header + payload + b"\n") + self.process.stdin.write(header + payload + gpayload + b"\n") self.process.stdin.flush() except Exception: with self.pending_lock: @@ -695,6 +830,8 @@ def __init__(self, address, engine, model_id, api_key=None, max_tokens=1024, class APIHandler(BaseHTTPRequestHandler): protocol_version = "HTTP/1.1" + timeout = 30 # per-request socket timeout: a slowloris client that dribbles its + # request line/body can't pin a worker thread (and a slot) forever server_version = "colibri" def log_message(self, fmt, *args): @@ -726,14 +863,40 @@ def send_cors_headers(self): if "*" not in self.server.cors_origins: self.send_header("Vary", "Origin") + LOOPBACK_HOSTS = {"127.0.0.1", "localhost", "::1", ""} + + def _is_authed(self): + """True if no key is configured, or a correct Bearer key was presented.""" + if not self.server.api_key: + return True + import hmac + provided = self.headers.get("Authorization", "") + return hmac.compare_digest(provided, f"Bearer {self.server.api_key}") + def require_auth(self): - if self.server.api_key: - import hmac - provided = self.headers.get("Authorization", "") - expected = f"Bearer {self.server.api_key}" - if not hmac.compare_digest(provided, expected): - raise APIError(401, "Invalid or missing API key.", None, "invalid_api_key", - "authentication_error") + if not self._is_authed(): + raise APIError(401, "Invalid or missing API key.", None, "invalid_api_key", + "authentication_error") + + def _check_host(self): + """DNS-rebinding guard: a web page can resolve a hostname to 127.0.0.1 and + drive this local server unless we pin the Host header to loopback / the bind + address. Rejects requests whose Host is anything else. (#SEC-7)""" + host = self.headers.get("Host", "") + if host.startswith("["): + name = host[1:].split("]", 1)[0] # [ipv6]:port + elif host.count(":") == 1: + name = host.rsplit(":", 1)[0] # host:port / ipv4:port + else: + name = host # bare host / bracketless ipv6 + name = name.strip().lower() + allowed = set(self.LOOPBACK_HOSTS) + try: + allowed.add(str(self.server.server_address[0]).strip("[]").lower()) + except Exception: + pass + if name not in allowed: + raise APIError(403, "Host header not allowed.", None, "forbidden") def read_json(self): try: @@ -791,20 +954,26 @@ def serve_static(self, path): def do_GET(self): request_id = "req_" + uuid.uuid4().hex try: + self._check_host() path = urlsplit(self.path).path if path == "/health": - payload = {"status": "ok", "scheduler": self.server.scheduler.snapshot(), - "kv_slots": self.server.kv_slots} - tiers = getattr(self.server.engine, "tiers", None) if self.server.engine else None - if tiers: payload["tiers"] = tiers - hwinfo = getattr(self.server.engine, "hwinfo", None) if self.server.engine else None - if hwinfo: payload["hwinfo"] = hwinfo + # Liveness is always public; hardware/scheduler internals only when a + # request is authed (or no key set), so a configured key isn't leaked + # past a bare 200 to an unauthenticated probe. (#SEC-8) + payload = {"status": "ok"} + if self._is_authed(): + payload["scheduler"] = self.server.scheduler.snapshot() + payload["kv_slots"] = self.server.kv_slots + tiers = getattr(self.server.engine, "tiers", None) if self.server.engine else None + if tiers: payload["tiers"] = tiers + hwinfo = getattr(self.server.engine, "hwinfo", None) if self.server.engine else None + if hwinfo: payload["hwinfo"] = hwinfo self.send_json(200, payload, request_id) return if path == "/experts": - eng = self.server.engine payload = {"rows": 0, "cols": 0, "map": "", "hits": "", "seq": 0} - if eng and getattr(eng, "emap", None): + eng = self.server.engine + if self._is_authed() and eng and getattr(eng, "emap", None): # (#SEC-8) hide routing telemetry unless authed payload.update(eng.emap) payload["hits"] = eng.hits or "" payload["seq"] = eng.hits_seq @@ -830,6 +999,13 @@ def do_GET(self): self.send_json(error.status, error_object(error), request_id, error.headers) def do_OPTIONS(self): + try: # (#SEC-7) apply the Host guard uniformly, incl. CORS preflight + self._check_host() + except APIError: + self.send_response(403) + self.send_header("Content-Length", "0") + self.end_headers() + return self.send_response(204) self.send_header("Content-Length", "0") self.send_cors_headers() @@ -838,6 +1014,7 @@ def do_OPTIONS(self): def do_POST(self): request_id = "req_" + uuid.uuid4().hex try: + self._check_host() self.require_auth() body = self.read_json() self.check_model(body) @@ -863,7 +1040,7 @@ def do_POST(self): except OSError: pass - def generation(self, body, prompt, request_id, chat): + def generation(self, body, prompt, request_id, chat, tools=None, tool_choice=None): # COLI_DEBUG tees the engine transaction to stderr: 1 = decoded output stream only, # 2 = both sides (rendered prompt + output). render_chat already folds prior turns and # tool results into `prompt`, so level 2 is the full conversation the engine saw. @@ -874,9 +1051,9 @@ def generation(self, body, prompt, request_id, chat): if dbg >= 2: sys.stderr.write(f"\n===== PROMPT [{request_id}] =====\n{prompt}\n===== OUTPUT [{request_id}] =====\n") sys.stderr.flush() - maximum, temperature, top_p = generation_options(body, self.server.max_tokens) - tools = (body.get("tools") or body.get("functions") or None) if chat else None - if body.get("tool_choice") == "none": + maximum, temperature, top_p, grammar = generation_options(body, self.server.max_tokens) + # tools and tool_choice come from chat_completion() already processed/filtered + if chat and tool_choice == "none": tools = None # client forbade tools: never surface tool_calls cache_slot = body.get("cache_slot") if (cache_slot is not None and @@ -903,7 +1080,7 @@ def generation(self, body, prompt, request_id, chat): output = [] stats = self.server.engine.generate( prompt, maximum, temperature, top_p, output.append, cache_slot, - self.client_disconnected) + self.client_disconnected, grammar=grammar) text = "".join(output) length_finish = "length" if stats["length_limited"] else "stop" if chat and tools: @@ -1010,7 +1187,7 @@ def emit_tools(chunk): sp["buf"] = sp["buf"][flush:] stats = self.server.engine.generate( prompt, maximum, temperature, top_p, emit_tools, cache_slot, - lambda: not connected) + lambda: not connected, grammar=grammar) if not sp["tool"] and sp["buf"]: emit(sp["buf"]) # no tool call happened: flush held tail _content, calls = parse_tool_calls("".join(raw), tools) @@ -1027,7 +1204,7 @@ def emit_plain(chunk): emit(chunk) stats = self.server.engine.generate( prompt, maximum, temperature, top_p, emit_plain, cache_slot, - lambda: not connected) + lambda: not connected, grammar=grammar) finish = "length" if stats["length_limited"] else "stop" ka_stop.set() # generation done: stop the keepalive pump ka_thread.join(timeout=2) @@ -1038,11 +1215,12 @@ def emit_plain(chunk): if include_usage: event([], self.usage(stats)) if connected: - try: - self.wfile.write(b"data: [DONE]\n\n") - self.wfile.flush() - except OSError: - pass + with ka_lock: # (#B9) share the pump's lock so [DONE] can't interleave a keepalive write + try: + self.wfile.write(b"data: [DONE]\n\n") + self.wfile.flush() + except OSError: + pass self.close_connection = True def client_disconnected(self): @@ -1078,9 +1256,10 @@ def chat_completion(self, body, request_id): if not isinstance(enable_thinking, bool): raise APIError(400, "`enable_thinking` must be a boolean.", "enable_thinking") tools = body.get("tools") or body.get("functions") or None + tool_choice = body.get("tool_choice") prompt = render_chat(body.get("messages"), enable_thinking, reasoning_effort, tools, - body.get("tool_choice")) - self.generation(body, prompt, request_id, True) + tool_choice) + self.generation(body, prompt, request_id, True, tools, tool_choice) def completion(self, body, request_id): prompt = body.get("prompt") @@ -1105,7 +1284,15 @@ def serve(model, host="127.0.0.1", port=8000, model_id="glm-5.2-colibri", api_ke if not 1 <= kv_slots <= 16: raise ValueError("kv_slots must be between 1 and 16") if host not in ("127.0.0.1", "localhost", "::1") and not api_key: - print("WARNING: API is listening beyond localhost without COLI_API_KEY", file=sys.stderr) + # (#SEC-6) Fail closed: an unauthenticated engine on a non-loopback bind exposes + # a compute-heavy API to the network. Refuse unless explicitly overridden. + if os.environ.get("COLI_ALLOW_INSECURE_BIND") == "1": + print("WARNING: binding %s beyond localhost with NO auth (COLI_ALLOW_INSECURE_BIND=1)" % host, + file=sys.stderr) + else: + print("refusing to bind %s beyond localhost without COLI_API_KEY set " + "(set COLI_ALLOW_INSECURE_BIND=1 to override)" % host, file=sys.stderr) + sys.exit(1) origins = DEFAULT_CORS_ORIGINS if cors_origins is None else tuple(cors_origins) # Bind before starting the 744B engine. A stale/occupied port must fail in # milliseconds rather than loading hundreds of GB and leaking a child. diff --git a/c/quant.h b/c/quant.h new file mode 100644 index 00000000..505141f5 --- /dev/null +++ b/c/quant.h @@ -0,0 +1,1219 @@ +/* quant.h — quantized matmul kernels (header-only, all functions static). + * Multi-architecture SIMD: AVX2 / AVX-512 / AVX-VNNI / ARM NEON / NEON-SDOT / + * NEON-i8mm / POWER VSX. Pure compute — no Model or QT dependency. */ +#ifndef COLI_QUANT_H +#define COLI_QUANT_H + +#include +#include +#include +#include +#include + +#ifdef _OPENMP +#include +#endif + +/* ---- SIMD includes -------------------------------------------------------- */ +#ifdef __AVX2__ +#include +static inline float hsum256(__m256 v){ + __m128 lo=_mm256_castps256_ps128(v), hi=_mm256_extractf128_ps(v,1); + lo=_mm_add_ps(lo,hi); __m128 sh=_mm_movehl_ps(lo,lo); lo=_mm_add_ps(lo,sh); + sh=_mm_shuffle_ps(lo,lo,1); lo=_mm_add_ss(lo,sh); return _mm_cvtss_f32(lo); +} +static inline int hsum256_i32(__m256i v){ + __m128i lo=_mm256_castsi256_si128(v), hi=_mm256_extracti128_si256(v,1); + lo=_mm_add_epi32(lo,hi); lo=_mm_hadd_epi32(lo,lo); lo=_mm_hadd_epi32(lo,lo); + return _mm_cvtsi128_si32(lo); +} +#endif +#if defined(__AVXVNNI__) && defined(__AVX2__) +static inline int hsum128_i32(__m128i v){ + v=_mm_hadd_epi32(v,v); v=_mm_hadd_epi32(v,v); return _mm_cvtsi128_si32(v); +} +#endif +#ifdef __ARM_NEON +#include +#endif +#ifdef __VSX__ +#include +#undef vector +#undef pixel +#undef bool +#endif + +/* ---- AVX-512 int4->float accumulator -------------------------------------- */ +#if defined(__AVX512F__) && defined(__AVX512BW__) +static int g_i4_acc512=1; +static inline float dot_i4f_avx512(const uint8_t *w,const float *x,int I){ + const __m128i m4=_mm_set1_epi8(0x0F); const __m512i b8=_mm512_set1_epi32(8); + __m512 acc0=_mm512_setzero_ps(),acc1=_mm512_setzero_ps(); int i=0; + for(;i+32<=I;i+=32){ __m128i by=_mm_loadu_si128((const __m128i*)(w+(i>>1))); + __m128i lo=_mm_and_si128(by,m4),hi=_mm_and_si128(_mm_srli_epi16(by,4),m4); + __m128i n0=_mm_unpacklo_epi8(lo,hi),n1=_mm_unpackhi_epi8(lo,hi); + __m512 w0=_mm512_cvtepi32_ps(_mm512_sub_epi32(_mm512_cvtepu8_epi32(n0),b8)); + __m512 w1=_mm512_cvtepi32_ps(_mm512_sub_epi32(_mm512_cvtepu8_epi32(n1),b8)); + acc0=_mm512_fmadd_ps(_mm512_loadu_ps(x+i),w0,acc0); + acc1=_mm512_fmadd_ps(_mm512_loadu_ps(x+i+16),w1,acc1); + } + return _mm512_reduce_add_ps(_mm512_add_ps(acc0,acc1)); +} +/* acc[0..I) += coef * dequant(int4 row) — the axpy twin of dot_i4f_avx512, for the + * MLA absorption path (qt_addrow). Each acc[i] receives exactly ONE fma per call + * (no cross-element accumulation), so this is bit-identical to the scalar loop + * (gcc -O3 -ffp-contract already emits scalar fma there). Tail handled scalar. */ +static inline void axpy_i4f_avx512(const uint8_t *w,float coef,float *acc,int I){ + const __m128i m4=_mm_set1_epi8(0x0F); const __m512i b8=_mm512_set1_epi32(8); + const __m512 cv=_mm512_set1_ps(coef); int i=0; + for(;i+32<=I;i+=32){ __m128i by=_mm_loadu_si128((const __m128i*)(w+(i>>1))); + __m128i lo=_mm_and_si128(by,m4),hi=_mm_and_si128(_mm_srli_epi16(by,4),m4); + __m128i n0=_mm_unpacklo_epi8(lo,hi),n1=_mm_unpackhi_epi8(lo,hi); + __m512 w0=_mm512_cvtepi32_ps(_mm512_sub_epi32(_mm512_cvtepu8_epi32(n0),b8)); + __m512 w1=_mm512_cvtepi32_ps(_mm512_sub_epi32(_mm512_cvtepu8_epi32(n1),b8)); + _mm512_storeu_ps(acc+i, _mm512_fmadd_ps(cv,w0,_mm512_loadu_ps(acc+i))); + _mm512_storeu_ps(acc+i+16,_mm512_fmadd_ps(cv,w1,_mm512_loadu_ps(acc+i+16))); + } + for(;i+1>1]; acc[i]+=coef*(float)((int)(b&0xF)-8); acc[i+1]+=coef*(float)((int)(b>>4)-8); } + if(i>1]; acc[i]+=coef*(float)((int)(b&0xF)-8); } +} +static int i4_acc512_selftest(void){ + enum { N=224 }; uint8_t w[(N+1)/2]; float x[N]; + for(int i=0;i>1]=(uint8_t)(q+8); + else w[i>>1]|=(uint8_t)((q+8)<<4); + x[i]=(float)(((i*29+7)%101)-50)/37.f; + } + for(int n=32;n<=N;n+=32){ + float ref=0; for(int i=0;i>1]>>((i&1)*4))&15)-8); + float got=dot_i4f_avx512(w,x,n),tol=2e-5f*(1.f+fabsf(ref)); + if(fabsf(got-ref)>tol){ fprintf(stderr,"AVX512 i4 selftest n=%d: %.9g != %.9g\n",n,got,ref); return 0; } + } + return 1; +} +#endif + +/* ---- y[S,O] = x[S,I] @ W^T, W[O,I] f32 ---------------------------------- */ +static void matmul(float *y, const float *x, const float *W, int S, int I, int O){ + #pragma omp parallel for schedule(static) + for (int o=0;o>1))); + __m128i lo=_mm_and_si128(by,m4), hi=_mm_and_si128(_mm_srli_epi16(by,4),m4); + __m128i nib=_mm_unpacklo_epi8(lo,hi); + __m256 w0=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(nib),b8)); + __m256 w1=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(nib,8)),b8)); + acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i), w0, acc); + acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i+8), w1, acc); } + a=hsum256(acc); +#elif defined(__ARM_NEON) + const uint8x8_t m4=vdup_n_u8(0x0F); const int8x8_t b8=vdup_n_s8(8); + float32x4_t ac0=vdupq_n_f32(0), ac1=vdupq_n_f32(0); + for(;i+16<=I;i+=16){ uint8x8_t by=vld1_u8(w+(i>>1)); + uint8x8x2_t z=vzip_u8(vand_u8(by,m4), vshr_n_u8(by,4)); + int16x8_t w0=vmovl_s8(vsub_s8(vreinterpret_s8_u8(z.val[0]),b8)); + int16x8_t w1=vmovl_s8(vsub_s8(vreinterpret_s8_u8(z.val[1]),b8)); + ac0=vfmaq_f32(ac0, vld1q_f32(xs+i), vcvtq_f32_s32(vmovl_s16(vget_low_s16(w0)))); + ac1=vfmaq_f32(ac1, vld1q_f32(xs+i+4), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w0)))); + ac0=vfmaq_f32(ac0, vld1q_f32(xs+i+8), vcvtq_f32_s32(vmovl_s16(vget_low_s16(w1)))); + ac1=vfmaq_f32(ac1, vld1q_f32(xs+i+12), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w1)))); } + a=vaddvq_f32(vaddq_f32(ac0,ac1)); +#endif +#if defined(__AVX512F__) && defined(__AVX512BW__) + } +#endif + for(;i+1>1]; int lo=(int)(byte&0xF)-8, hi=(int)(byte>>4)-8; + a += xs[i]*(float)lo + xs[i+1]*(float)hi; } + if(i>1]; int lo=(int)(byte&0xF)-8; a += xs[i]*(float)lo; } + y[(int64_t)s*O+o]=a*sc; } } +} + +/* ---- y[S,O] = x[S,I] @ W^T, W int4 packed + per-GROUP scales (fmt=4) ----- */ +static void matmul_i4_grouped(float *y, const float *x, const uint8_t *q4, const float *scale, + int S, int I, int O, int gs){ + int rb=(I+1)/2; int ng=(I+gs-1)/gs; + #pragma omp parallel for schedule(static) + for(int o=0;oI) glen=I-base; + float sc=scl[g]; + int i=base; +#ifdef __AVX2__ + const __m128i m4=_mm_set1_epi8(0x0F); const __m256i b8=_mm256_set1_epi32(8); + __m256 acc=_mm256_setzero_ps(); + for(; i+16<=base+glen; i+=16){ __m128i by=_mm_loadl_epi64((const __m128i*)(w+(i>>1))); + __m128i lo=_mm_and_si128(by,m4),hi=_mm_and_si128(_mm_srli_epi16(by,4),m4); + __m128i nib=_mm_unpacklo_epi8(lo,hi); + __m256 w0=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(nib),b8)); + __m256 w1=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(nib,8)),b8)); + acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i), w0, acc); + acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i+8), w1, acc); } + a+=hsum256(acc)*sc; +#endif + for(; i>1]; + a+=(xs[i]*(float)((int)(byte&0xF)-8)+xs[i+1]*(float)((int)(byte>>4)-8))*sc; } + else { uint8_t byte=w[i>>1]; a+=xs[i]*(float)((int)(byte&0xF)-8)*sc; } + } + } + y[(int64_t)s*O+o]=a; + } + } +} + +/* ---- fused gate+up: one OMP dispatch for both matrices -------------------- */ +static void matmul_i4_pair(float *yg, float *yu, const float *x, + const uint8_t *qg, const float *sg, + const uint8_t *qu, const float *su, int I, int O){ + int rb=(I+1)/2; + #pragma omp parallel for schedule(static) + for(int z=0;z<2*O;z++){ + int o=z>1))); + __m128i lo=_mm_and_si128(by,m4),hi=_mm_and_si128(_mm_srli_epi16(by,4),m4); + __m128i nib=_mm_unpacklo_epi8(lo,hi); + __m256 w0=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(nib),b8)); + __m256 w1=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(nib,8)),b8)); + acc=_mm256_fmadd_ps(_mm256_loadu_ps(x+i),w0,acc); + acc=_mm256_fmadd_ps(_mm256_loadu_ps(x+i+8),w1,acc); } + a=hsum256(acc); +#elif defined(__ARM_NEON) + const uint8x8_t m4=vdup_n_u8(0x0F); const int8x8_t b8=vdup_n_s8(8); + float32x4_t ac0=vdupq_n_f32(0),ac1=vdupq_n_f32(0); + for(;i+16<=I;i+=16){ uint8x8_t by=vld1_u8(w+(i>>1)); + uint8x8x2_t n=vzip_u8(vand_u8(by,m4),vshr_n_u8(by,4)); + int16x8_t w0=vmovl_s8(vsub_s8(vreinterpret_s8_u8(n.val[0]),b8)); + int16x8_t w1=vmovl_s8(vsub_s8(vreinterpret_s8_u8(n.val[1]),b8)); + ac0=vfmaq_f32(ac0,vld1q_f32(x+i),vcvtq_f32_s32(vmovl_s16(vget_low_s16(w0)))); + ac1=vfmaq_f32(ac1,vld1q_f32(x+i+4),vcvtq_f32_s32(vmovl_s16(vget_high_s16(w0)))); + ac0=vfmaq_f32(ac0,vld1q_f32(x+i+8),vcvtq_f32_s32(vmovl_s16(vget_low_s16(w1)))); + ac1=vfmaq_f32(ac1,vld1q_f32(x+i+12),vcvtq_f32_s32(vmovl_s16(vget_high_s16(w1)))); } + a=vaddvq_f32(vaddq_f32(ac0,ac1)); +#endif +#if defined(__AVX512F__) && defined(__AVX512BW__) + } +#endif + for(;i+1>1]; a+=x[i]*(float)((b&15)-8)+x[i+1]*(float)((b>>4)-8); } + if(i>1]&15)-8); + (z>2))); + __m128i p0=_mm_and_si128(by,m2), p1=_mm_and_si128(_mm_srli_epi16(by,2),m2); + __m128i p2=_mm_and_si128(_mm_srli_epi16(by,4),m2), p3=_mm_and_si128(_mm_srli_epi16(by,6),m2); + __m128i lo=_mm_unpacklo_epi8(p0,p1), hi=_mm_unpacklo_epi8(p2,p3); + __m128i nib=_mm_unpacklo_epi16(lo,hi); + __m256 w0=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(nib),b2)); + __m256 w1=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(nib,8)),b2)); + acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i), w0, acc); + acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i+8), w1, acc); } + a=hsum256(acc); +#elif defined(__ARM_NEON) + const uint8x8_t m2v=vdup_n_u8(3); const int8x8_t b2v=vdup_n_s8(2); + float32x4_t ac0=vdupq_n_f32(0), ac1=vdupq_n_f32(0); + for(;i+16<=I;i+=16){ uint32_t wd; memcpy(&wd, w+(i>>2), 4); + uint8x8_t by=vreinterpret_u8_u32(vdup_n_u32(wd)); + uint8x8x2_t z01=vzip_u8(vand_u8(by,m2v), vand_u8(vshr_n_u8(by,2),m2v)); + uint8x8x2_t z23=vzip_u8(vand_u8(vshr_n_u8(by,4),m2v), vshr_n_u8(by,6)); + uint16x4x2_t zz=vzip_u16(vreinterpret_u16_u8(z01.val[0]), vreinterpret_u16_u8(z23.val[0])); + int16x8_t w0=vmovl_s8(vsub_s8(vreinterpret_s8_u16(zz.val[0]),b2v)); + int16x8_t w1=vmovl_s8(vsub_s8(vreinterpret_s8_u16(zz.val[1]),b2v)); + ac0=vfmaq_f32(ac0, vld1q_f32(xs+i), vcvtq_f32_s32(vmovl_s16(vget_low_s16(w0)))); + ac1=vfmaq_f32(ac1, vld1q_f32(xs+i+4), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w0)))); + ac0=vfmaq_f32(ac0, vld1q_f32(xs+i+8), vcvtq_f32_s32(vmovl_s16(vget_low_s16(w1)))); + ac1=vfmaq_f32(ac1, vld1q_f32(xs+i+12), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w1)))); } + a=vaddvq_f32(vaddq_f32(ac0,ac1)); +#endif + for(;i>2]; int sh=(i&3)*2; a += xs[i]*(float)((int)((byte>>sh)&3)-2); } + y[(int64_t)s*O+o]=a*sc; } } +} + +/* ---- int3-g64 (fmt=5): 3-bit weights with ONE f32 scale per 64-input group - + * Per group: 16B low plane (2 bits/val, int2 layout) + 8B high plane (1 bit/val), + * values in [-4,3] stored v+4. 3.5 bits/weight effective — the quality/size point + * the #132 OLMoE ablation measured BEATING per-row int4. */ +#define I3_GROUP 64 +#define I3_GBYTES 24 /* 16B low plane + 8B high plane per group */ +static inline int64_t i3_groups(int I){ return ((int64_t)I + I3_GROUP - 1) / I3_GROUP; } +static inline int64_t i3_rowbytes(int I){ return i3_groups(I) * I3_GBYTES; } + +/* Dequant-on-use with PER-GROUP scale. Exact f32 path only (no IDOT in v1: int8 + * activations don't compose with per-group accumulation without a kernel + * restructure — follow-up). NEON: low plane = matmul_i2's unpack, high plane + * expanded via vtst on bit masks; x86 stays scalar for now (follow-up). */ +static void matmul_i3(float *y, const float *x, const uint8_t *q3, const float *scale, int S, int I, int O){ + int64_t ng=i3_groups(I), rb=i3_rowbytes(I); + #pragma omp parallel for schedule(static) + for(int o=0;o>2), 4); /* 4 bytes = 16 low-plane values */ + uint8x8_t by=vreinterpret_u8_u32(vdup_n_u32(wd)); + uint8x8x2_t z01=vzip_u8(vand_u8(by,m2v), vand_u8(vshr_n_u8(by,2),m2v)); + uint8x8x2_t z23=vzip_u8(vand_u8(vshr_n_u8(by,4),m2v), vshr_n_u8(by,6)); + uint16x4x2_t zz=vzip_u16(vreinterpret_u16_u8(z01.val[0]), vreinterpret_u16_u8(z23.val[0])); + uint8x16_t lov=vcombine_u8(vreinterpret_u8_u16(zz.val[0]), vreinterpret_u8_u16(zz.val[1])); + uint8x16_t hv=vcombine_u8(vdup_n_u8(hi[k>>3]), vdup_n_u8(hi[(k>>3)+1])); + uint8x16_t hb=vandq_u8(vtstq_u8(hv,bitm), fourq); /* 4 where high bit set */ + int8x16_t wq=vsubq_s8(vreinterpretq_s8_u8(vaddq_u8(lov,hb)), b4q); /* [-4,3] in order */ + int16x8_t w0=vmovl_s8(vget_low_s8(wq)), w1=vmovl_s8(vget_high_s8(wq)); + ac0=vfmaq_f32(ac0, vld1q_f32(xs+base+k), vcvtq_f32_s32(vmovl_s16(vget_low_s16(w0)))); + ac1=vfmaq_f32(ac1, vld1q_f32(xs+base+k+4), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w0)))); + ac0=vfmaq_f32(ac0, vld1q_f32(xs+base+k+8), vcvtq_f32_s32(vmovl_s16(vget_low_s16(w1)))); + ac1=vfmaq_f32(ac1, vld1q_f32(xs+base+k+12), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w1)))); + } + a=vaddvq_f32(vaddq_f32(ac0,ac1)); + } +#endif + for(;k>2]>>((k&3)*2))&3) | (((hi[k>>3]>>(k&7))&1)<<2); + a += xs[base+k]*(float)((int)u-4); + } + acc += a*srow[g]; + } + y[(int64_t)s*O+o]=acc; + } + } +} + +/* ---- IDOT: integer dot kernels (int8-quantized activations) --------------- */ +#if defined(__AVX512VNNI__) && defined(__AVX512BW__) +#define IDOT_KERNEL "avx512-vnni" +#elif defined(__AVXVNNI__) && defined(__AVX2__) +#define IDOT_KERNEL "avx-vnni" +#elif defined(__AVX2__) +#define IDOT_KERNEL "avx2" +#elif defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8) +#define IDOT_KERNEL "neon-i8mm" +#elif defined(__ARM_NEON) +#define IDOT_KERNEL "neon" +#elif defined(__VSX__) +#define IDOT_KERNEL "vsx" +#else +#define IDOT_KERNEL "scalar" +#endif +static int g_idot=1; +#if defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD) +static int g_i4s=1; +#elif defined(__VSX__) +static int g_i4s=1; +#elif defined(__AVX512VNNI__) && defined(__AVX512BW__) +static int g_i4s=1; /* AVX-512 VNNI: come SDOT, l'IDOT int4 conviene anche a S=1. Misurato su + * 2x Xeon 8370C (48 core, GLM-5.2 int4 tutto residente, TEMP=0 DRAFT=0, + * 256 token): 3.65 -> 3.85 tok/s (+5.5%), expert-matmul 67.8 -> 89.5 GB/s. + * EN: with AVX-512 VNNI, like SDOT, int4 IDOT pays at S=1 too. Measured on + * a 2-socket Ice Lake (config above): +5.5% end-to-end greedy decode. */ +#else +static int g_i4s=2; +#endif +static int g_xexp=0; /* XEXP=1 (opt-in): S==1 decode, all-resident int4 block -> ONE OpenMP + * region across all experts of the batch-union block instead of ~2 + * fork/joins per expert. Engages only with the int4-IDOT S=1 family + * (g_i4s<=1) and off the speculation window (spec_pinned): output is + * byte-identical to that family (same dot_i4i8 per row, same silu, + * same requant, same accumulation order into out). Measured on a + * 2-socket Ice Lake 48C (GLM-5.2 int4 fully resident, TEMP=0 DRAFT=0, + * 256 tok greedy, ABAB 3 prompts x 2 reps): 4.20 -> 4.68 tok/s + * (+11.6% mean, worst prompt +11.3%), expert-matmul effective + * 89.5 -> 131.9 GB/s. A similar restructuring was NEUTRAL/negative on + * a 24-core box (docs/experiments/glm52-6x5090-2026-07-12.md) - hence + * opt-in; measure on your host. */ + +static inline float qrow_i8(const float *x, int8_t *q, int I){ + float amax=0; for(int i=0;iamax)amax=a; } + float s=amax/127.f; if(s<1e-12f) s=1e-12f; float inv=1.f/s; + for(int i=0;i bit-identico. Stessa struttura + * dei 4 accumulatori del ramo NEON piu' sotto. + * EN: four independent accumulators break the serial vpdpbusd->acc chain; integer + * adds are associative, so the result is bit-identical (mirrors the NEON path). */ + __m128i a0=_mm_setzero_si128(),a1=_mm_setzero_si128(),a2=_mm_setzero_si128(),a3=_mm_setzero_si128(); + for(;i+64<=I;i+=64){ + __m128i w0=_mm_loadu_si128((const __m128i*)(w+i)), x0=_mm_loadu_si128((const __m128i*)(x+i)); + __m128i w1=_mm_loadu_si128((const __m128i*)(w+i+16)), x1=_mm_loadu_si128((const __m128i*)(x+i+16)); + __m128i w2=_mm_loadu_si128((const __m128i*)(w+i+32)), x2=_mm_loadu_si128((const __m128i*)(x+i+32)); + __m128i w3=_mm_loadu_si128((const __m128i*)(w+i+48)), x3=_mm_loadu_si128((const __m128i*)(x+i+48)); + a0=_mm_dpbusd_epi32(a0,_mm_abs_epi8(w0),_mm_sign_epi8(x0,w0)); + a1=_mm_dpbusd_epi32(a1,_mm_abs_epi8(w1),_mm_sign_epi8(x1,w1)); + a2=_mm_dpbusd_epi32(a2,_mm_abs_epi8(w2),_mm_sign_epi8(x2,w2)); + a3=_mm_dpbusd_epi32(a3,_mm_abs_epi8(w3),_mm_sign_epi8(x3,w3)); + } + __m128i acc=_mm_add_epi32(_mm_add_epi32(a0,a1),_mm_add_epi32(a2,a3)); + for(;i+16<=I;i+=16){ + __m128i wv=_mm_loadu_si128((const __m128i*)(w+i)); + __m128i xv=_mm_loadu_si128((const __m128i*)(x+i)); + acc=_mm_dpbusd_epi32(acc,_mm_abs_epi8(wv),_mm_sign_epi8(xv,wv)); + } + sum=hsum128_i32(acc); +#elif defined(__AVX2__) + __m256i acc=_mm256_setzero_si256(); const __m256i ones=_mm256_set1_epi16(1); + for(;i+32<=I;i+=32){ + __m256i wv=_mm256_loadu_si256((const __m256i*)(w+i)); + __m256i xv=_mm256_loadu_si256((const __m256i*)(x+i)); + __m256i p=_mm256_maddubs_epi16(_mm256_sign_epi8(wv,wv),_mm256_sign_epi8(xv,wv)); + acc=_mm256_add_epi32(acc,_mm256_madd_epi16(p,ones)); + } + sum=hsum256_i32(acc); +#elif defined(__ARM_NEON) +#if defined(__ARM_FEATURE_DOTPROD) + int32x4_t a0=vdupq_n_s32(0),a1=vdupq_n_s32(0),a2=vdupq_n_s32(0),a3=vdupq_n_s32(0); + for(;i+64<=I;i+=64){ + a0=vdotq_s32(a0,vld1q_s8(w+i), vld1q_s8(x+i)); + a1=vdotq_s32(a1,vld1q_s8(w+i+16),vld1q_s8(x+i+16)); + a2=vdotq_s32(a2,vld1q_s8(w+i+32),vld1q_s8(x+i+32)); + a3=vdotq_s32(a3,vld1q_s8(w+i+48),vld1q_s8(x+i+48)); + } + int32x4_t acc=vaddq_s32(vaddq_s32(a0,a1),vaddq_s32(a2,a3)); + for(;i+16<=I;i+=16) acc=vdotq_s32(acc,vld1q_s8(w+i),vld1q_s8(x+i)); + sum=vaddvq_s32(acc); +#else + int32x4_t acc=vdupq_n_s32(0); + for(;i+16<=I;i+=16){ + int8x16_t wv=vld1q_s8(w+i), xv=vld1q_s8(x+i); + int16x8_t p=vmull_s8(vget_low_s8(wv),vget_low_s8(xv)); + p=vmlal_s8(p,vget_high_s8(wv),vget_high_s8(xv)); + acc=vpadalq_s16(acc,p); + } + sum=vaddvq_s32(acc); +#endif +#elif defined(__VSX__) + __vector signed int acc=vec_splats(0); + const __vector signed char vz=vec_splats((signed char)0); + for(;i+16<=I;i+=16){ + __vector signed char wv=vec_xl(0,(const signed char*)(w+i)); + __vector signed char xv=vec_xl(0,(const signed char*)(x+i)); + __vector __bool char neg=vec_cmplt(wv,vz); + __vector signed char xs=vec_sel(xv,vec_sub(vz,xv),neg); + __vector unsigned char wa=(__vector unsigned char)vec_sel(wv,vec_sub(vz,wv),neg); + acc=vec_msum(xs,wa,acc); + } + sum=vec_extract(acc,0)+vec_extract(acc,1)+vec_extract(acc,2)+vec_extract(acc,3); +#endif + for(;i>1))); + __m256i lo=_mm256_and_si256(by,m4v), hi=_mm256_and_si256(_mm256_srli_epi16(by,4),m4v); + __m256i z0=_mm256_unpacklo_epi8(lo,hi), z1=_mm256_unpackhi_epi8(lo,hi); + __m512i wv=_mm512_sub_epi8(_mm512_inserti64x4(_mm512_castsi256_si512(z0),z1,1),b8v); + __m512i xv=_mm512_permutexvar_epi64(xidx,_mm512_loadu_si512((const void*)(x+i))); + __mmask64 neg=_mm512_movepi8_mask(wv); + __m512i xs=_mm512_mask_sub_epi8(xv,neg,_mm512_setzero_si512(),xv); + acc=_mm512_dpbusd_epi32(acc,_mm512_abs_epi8(wv),xs); + } + sum=_mm512_reduce_add_epi32(acc); +#elif defined(__AVXVNNI__) && defined(__AVX2__) + /* 4 accumulatori indipendenti (64 elementi = 32 byte packed/iter): un solo acc + * incatena i vpdpbusd (latenza-bound ~5c). Somme intere associative -> bit-identico. + * Stessa struttura dei 4 accumulatori del ramo NEON piu' sotto. + * EN: four independent accumulators break the serial vpdpbusd->acc chain; integer + * adds are associative, so the result is bit-identical (mirrors the NEON path). */ + const __m128i m4=_mm_set1_epi8(0x0F); const __m128i b8=_mm_set1_epi8(8); + __m128i a0=_mm_setzero_si128(),a1=_mm_setzero_si128(),a2=_mm_setzero_si128(),a3=_mm_setzero_si128(); + for(;i+64<=I;i+=64){ + __m128i by0=_mm_loadu_si128((const __m128i*)(w4+(i>>1))); /* elem i..i+31 */ + __m128i by1=_mm_loadu_si128((const __m128i*)(w4+(i>>1)+16)); /* elem i+32..i+63 */ + __m128i lo0=_mm_and_si128(by0,m4), hi0=_mm_and_si128(_mm_srli_epi16(by0,4),m4); + __m128i lo1=_mm_and_si128(by1,m4), hi1=_mm_and_si128(_mm_srli_epi16(by1,4),m4); + __m128i w0=_mm_sub_epi8(_mm_unpacklo_epi8(lo0,hi0),b8), w1=_mm_sub_epi8(_mm_unpackhi_epi8(lo0,hi0),b8); + __m128i w2=_mm_sub_epi8(_mm_unpacklo_epi8(lo1,hi1),b8), w3=_mm_sub_epi8(_mm_unpackhi_epi8(lo1,hi1),b8); + __m128i x0=_mm_loadu_si128((const __m128i*)(x+i)), x1=_mm_loadu_si128((const __m128i*)(x+i+16)); + __m128i x2=_mm_loadu_si128((const __m128i*)(x+i+32)), x3=_mm_loadu_si128((const __m128i*)(x+i+48)); + a0=_mm_dpbusd_epi32(a0,_mm_abs_epi8(w0),_mm_sign_epi8(x0,w0)); + a1=_mm_dpbusd_epi32(a1,_mm_abs_epi8(w1),_mm_sign_epi8(x1,w1)); + a2=_mm_dpbusd_epi32(a2,_mm_abs_epi8(w2),_mm_sign_epi8(x2,w2)); + a3=_mm_dpbusd_epi32(a3,_mm_abs_epi8(w3),_mm_sign_epi8(x3,w3)); + } + __m128i acc=_mm_add_epi32(_mm_add_epi32(a0,a1),_mm_add_epi32(a2,a3)); + for(;i+32<=I;i+=32){ /* 32-nibble remainder: 2 dpbusd, same unpack */ + __m128i by=_mm_loadu_si128((const __m128i*)(w4+(i>>1))); + __m128i lo=_mm_and_si128(by,m4), hi=_mm_and_si128(_mm_srli_epi16(by,4),m4); + __m128i w0=_mm_sub_epi8(_mm_unpacklo_epi8(lo,hi),b8), w1=_mm_sub_epi8(_mm_unpackhi_epi8(lo,hi),b8); + __m128i x0=_mm_loadu_si128((const __m128i*)(x+i)); + __m128i x1=_mm_loadu_si128((const __m128i*)(x+i+16)); + acc=_mm_dpbusd_epi32(acc,_mm_abs_epi8(w0),_mm_sign_epi8(x0,w0)); + acc=_mm_dpbusd_epi32(acc,_mm_abs_epi8(w1),_mm_sign_epi8(x1,w1)); + } + sum=hsum128_i32(acc); +#elif defined(__AVX2__) + const __m128i m4=_mm_set1_epi8(0x0F); const __m256i b8=_mm256_set1_epi8(8); + const __m256i ones=_mm256_set1_epi16(1); + __m256i acc=_mm256_setzero_si256(); + for(;i+32<=I;i+=32){ + __m128i by=_mm_loadu_si128((const __m128i*)(w4+(i>>1))); + __m128i lo=_mm_and_si128(by,m4), hi=_mm_and_si128(_mm_srli_epi16(by,4),m4); + __m128i n0=_mm_unpacklo_epi8(lo,hi), n1=_mm_unpackhi_epi8(lo,hi); + __m256i wv=_mm256_sub_epi8(_mm256_set_m128i(n1,n0),b8); + __m256i xv=_mm256_loadu_si256((const __m256i*)(x+i)); + __m256i p=_mm256_maddubs_epi16(_mm256_sign_epi8(wv,wv),_mm256_sign_epi8(xv,wv)); + acc=_mm256_add_epi32(acc,_mm256_madd_epi16(p,ones)); + } + sum=hsum256_i32(acc); +#elif defined(__ARM_NEON) + const uint8x16_t m4q=vdupq_n_u8(0x0F); const int8x16_t b8q=vdupq_n_s8(8); +#if defined(__ARM_FEATURE_DOTPROD) + int32x4_t a0=vdupq_n_s32(0),a1=vdupq_n_s32(0),a2=vdupq_n_s32(0),a3=vdupq_n_s32(0); + for(;i+64<=I;i+=64){ + uint8x16_t byA=vld1q_u8(w4+(i>>1)), byB=vld1q_u8(w4+(i>>1)+16); + uint8x16x2_t zA=vzipq_u8(vandq_u8(byA,m4q), vshrq_n_u8(byA,4)); + uint8x16x2_t zB=vzipq_u8(vandq_u8(byB,m4q), vshrq_n_u8(byB,4)); + a0=vdotq_s32(a0,vsubq_s8(vreinterpretq_s8_u8(zA.val[0]),b8q),vld1q_s8(x+i)); + a1=vdotq_s32(a1,vsubq_s8(vreinterpretq_s8_u8(zA.val[1]),b8q),vld1q_s8(x+i+16)); + a2=vdotq_s32(a2,vsubq_s8(vreinterpretq_s8_u8(zB.val[0]),b8q),vld1q_s8(x+i+32)); + a3=vdotq_s32(a3,vsubq_s8(vreinterpretq_s8_u8(zB.val[1]),b8q),vld1q_s8(x+i+48)); + } + int32x4_t acc=vaddq_s32(vaddq_s32(a0,a1),vaddq_s32(a2,a3)); + for(;i+32<=I;i+=32){ + uint8x16_t by=vld1q_u8(w4+(i>>1)); + uint8x16x2_t z=vzipq_u8(vandq_u8(by,m4q), vshrq_n_u8(by,4)); + acc=vdotq_s32(acc,vsubq_s8(vreinterpretq_s8_u8(z.val[0]),b8q),vld1q_s8(x+i)); + acc=vdotq_s32(acc,vsubq_s8(vreinterpretq_s8_u8(z.val[1]),b8q),vld1q_s8(x+i+16)); + } + sum=vaddvq_s32(acc); +#else + int32x4_t acc=vdupq_n_s32(0); + for(;i+32<=I;i+=32){ + uint8x16_t by=vld1q_u8(w4+(i>>1)); + uint8x16x2_t z=vzipq_u8(vandq_u8(by,m4q), vshrq_n_u8(by,4)); + int8x16_t w0=vsubq_s8(vreinterpretq_s8_u8(z.val[0]),b8q); + int8x16_t w1=vsubq_s8(vreinterpretq_s8_u8(z.val[1]),b8q); + int8x16_t x0=vld1q_s8(x+i), x1=vld1q_s8(x+i+16); + int16x8_t p=vmull_s8(vget_low_s8(w0),vget_low_s8(x0)); + p=vmlal_s8(p,vget_high_s8(w0),vget_high_s8(x0)); + acc=vpadalq_s16(acc,p); + p=vmull_s8(vget_low_s8(w1),vget_low_s8(x1)); + p=vmlal_s8(p,vget_high_s8(w1),vget_high_s8(x1)); + acc=vpadalq_s16(acc,p); + } + sum=vaddvq_s32(acc); +#endif +#elif defined(__VSX__) + const __vector unsigned char m4v=vec_splats((unsigned char)0x0F); + const __vector unsigned char sh4=vec_splats((unsigned char)4); + const __vector signed char b8v=vec_splats((signed char)8); + const __vector signed char vz=vec_splats((signed char)0); + __vector signed int acc=vec_splats(0); + for(;i+32<=I;i+=32){ + __vector unsigned char by=vec_xl(0,w4+(i>>1)); + __vector unsigned char lo=vec_and(by,m4v), hi=vec_sr(by,sh4); + __vector signed char w0=vec_sub((__vector signed char)vec_mergeh(lo,hi),b8v); + __vector signed char w1=vec_sub((__vector signed char)vec_mergel(lo,hi),b8v); + __vector signed char x0=vec_xl(0,(const signed char*)(x+i)); + __vector signed char x1=vec_xl(0,(const signed char*)(x+i+16)); + __vector __bool char n0=vec_cmplt(w0,vz), n1=vec_cmplt(w1,vz); + acc=vec_msum(vec_sel(x0,vec_sub(vz,x0),n0), + (__vector unsigned char)vec_sel(w0,vec_sub(vz,w0),n0),acc); + acc=vec_msum(vec_sel(x1,vec_sub(vz,x1),n1), + (__vector unsigned char)vec_sel(w1,vec_sub(vz,w1),n1),acc); + } + sum=vec_extract(acc,0)+vec_extract(acc,1)+vec_extract(acc,2)+vec_extract(acc,3); +#endif + for(;i+1>1]; sum+=((int)(b&0xF)-8)*x[i]+((int)(b>>4)-8)*x[i+1]; } + if(i>1]; sum+=((int)(b&0xF)-8)*x[i]; } + return sum; +} + +/* ---- ARM i8mm SMMLA tiled kernels ---------------------------------------- */ +#if defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8) +static inline int32x4_t mm_tile16(int32x4_t acc, int8x16_t wo, int8x16_t wo1, + int8x16_t xs, int8x16_t xs1){ + acc=vmmlaq_s32(acc, vcombine_s8(vget_low_s8(wo), vget_low_s8(wo1)), + vcombine_s8(vget_low_s8(xs), vget_low_s8(xs1))); + return vmmlaq_s32(acc, vcombine_s8(vget_high_s8(wo), vget_high_s8(wo1)), + vcombine_s8(vget_high_s8(xs), vget_high_s8(xs1))); +} +static void matmul_q_idot_mm(float *y, const int8_t *xq, const float *sx, const int8_t *q, + const float *scale, int S, int I, int O){ + #pragma omp parallel for schedule(static) + for(int o=0;o<(O&~1);o+=2){ + const int8_t *wo=q+(int64_t)o*I, *wo1=q+(int64_t)(o+1)*I; + float sc0=scale[o], sc1=scale[o+1]; + for(int s=0;s<(S&~1);s+=2){ + const int8_t *xs=xq+(int64_t)s*I, *xs1=xq+(int64_t)(s+1)*I; + int32x4_t a0=vdupq_n_s32(0),a1=vdupq_n_s32(0),a2=vdupq_n_s32(0),a3=vdupq_n_s32(0); int i=0; + for(;i+64<=I;i+=64){ + a0=mm_tile16(a0,vld1q_s8(wo+i), vld1q_s8(wo1+i), vld1q_s8(xs+i), vld1q_s8(xs1+i)); + a1=mm_tile16(a1,vld1q_s8(wo+i+16),vld1q_s8(wo1+i+16),vld1q_s8(xs+i+16),vld1q_s8(xs1+i+16)); + a2=mm_tile16(a2,vld1q_s8(wo+i+32),vld1q_s8(wo1+i+32),vld1q_s8(xs+i+32),vld1q_s8(xs1+i+32)); + a3=mm_tile16(a3,vld1q_s8(wo+i+48),vld1q_s8(wo1+i+48),vld1q_s8(xs+i+48),vld1q_s8(xs1+i+48)); + } + for(;i+16<=I;i+=16) + a0=mm_tile16(a0,vld1q_s8(wo+i),vld1q_s8(wo1+i),vld1q_s8(xs+i),vld1q_s8(xs1+i)); + int32x4_t acc=vaddq_s32(vaddq_s32(a0,a1),vaddq_s32(a2,a3)); + int32_t d00=vgetq_lane_s32(acc,0), d01=vgetq_lane_s32(acc,1); + int32_t d10=vgetq_lane_s32(acc,2), d11=vgetq_lane_s32(acc,3); + for(;i>1)), byo1=vld1q_u8(wo1+(i>>1)); + uint8x16_t cyo=vld1q_u8(wo+(i>>1)+16), cyo1=vld1q_u8(wo1+(i>>1)+16); + uint8x16x2_t zo =vzipq_u8(vandq_u8(byo, m4q), vshrq_n_u8(byo, 4)); + uint8x16x2_t zo1=vzipq_u8(vandq_u8(byo1,m4q), vshrq_n_u8(byo1,4)); + uint8x16x2_t ko =vzipq_u8(vandq_u8(cyo, m4q), vshrq_n_u8(cyo, 4)); + uint8x16x2_t ko1=vzipq_u8(vandq_u8(cyo1,m4q), vshrq_n_u8(cyo1,4)); + a0=mm_tile16(a0, vsubq_s8(vreinterpretq_s8_u8(zo.val[0]),b8q), + vsubq_s8(vreinterpretq_s8_u8(zo1.val[0]),b8q), + vld1q_s8(xs+i), vld1q_s8(xs1+i)); + a1=mm_tile16(a1, vsubq_s8(vreinterpretq_s8_u8(zo.val[1]),b8q), + vsubq_s8(vreinterpretq_s8_u8(zo1.val[1]),b8q), + vld1q_s8(xs+i+16), vld1q_s8(xs1+i+16)); + a2=mm_tile16(a2, vsubq_s8(vreinterpretq_s8_u8(ko.val[0]),b8q), + vsubq_s8(vreinterpretq_s8_u8(ko1.val[0]),b8q), + vld1q_s8(xs+i+32), vld1q_s8(xs1+i+32)); + a3=mm_tile16(a3, vsubq_s8(vreinterpretq_s8_u8(ko.val[1]),b8q), + vsubq_s8(vreinterpretq_s8_u8(ko1.val[1]),b8q), + vld1q_s8(xs+i+48), vld1q_s8(xs1+i+48)); + } + for(;i+32<=I;i+=32){ + uint8x16_t byo=vld1q_u8(wo+(i>>1)), byo1=vld1q_u8(wo1+(i>>1)); + uint8x16x2_t zo =vzipq_u8(vandq_u8(byo, m4q), vshrq_n_u8(byo, 4)); + uint8x16x2_t zo1=vzipq_u8(vandq_u8(byo1,m4q), vshrq_n_u8(byo1,4)); + a0=mm_tile16(a0, vsubq_s8(vreinterpretq_s8_u8(zo.val[0]),b8q), + vsubq_s8(vreinterpretq_s8_u8(zo1.val[0]),b8q), + vld1q_s8(xs+i), vld1q_s8(xs1+i)); + a1=mm_tile16(a1, vsubq_s8(vreinterpretq_s8_u8(zo.val[1]),b8q), + vsubq_s8(vreinterpretq_s8_u8(zo1.val[1]),b8q), + vld1q_s8(xs+i+16), vld1q_s8(xs1+i+16)); + } + int32x4_t acc=vaddq_s32(vaddq_s32(a0,a1),vaddq_s32(a2,a3)); + int32_t d00=vgetq_lane_s32(acc,0), d01=vgetq_lane_s32(acc,1); + int32_t d10=vgetq_lane_s32(acc,2), d11=vgetq_lane_s32(acc,3); + for(;i+1>1], bo1=wo1[i>>1]; + int a0=(int)(bo&0xF)-8, a1=(int)(bo>>4)-8, b0=(int)(bo1&0xF)-8, b1=(int)(bo1>>4)-8; + int u0=xs[i],u1=xs[i+1],v0=xs1[i],v1=xs1[i+1]; + d00+=a0*u0+a1*u1; d01+=a0*v0+a1*v1; d10+=b0*u0+b1*u1; d11+=b0*v0+b1*v1; } + if(i>1], bo1=wo1[i>>1]; + int a0=(int)(bo&0xF)-8, b0=(int)(bo1&0xF)-8; + d00+=a0*xs[i]; d01+=a0*xs1[i]; d10+=b0*xs[i]; d11+=b0*xs1[i]; } + y[(int64_t)s*O+o] =(float)d00*sc0*sx[s]; + y[(int64_t)s*O+(o+1)] =(float)d10*sc1*sx[s]; + y[(int64_t)(s+1)*O+o] =(float)d01*sc0*sx[s+1]; + y[(int64_t)(s+1)*O+(o+1)]=(float)d11*sc1*sx[s+1]; + } + if(S&1){ int s=S-1; const int8_t *xs=xq+(int64_t)s*I; + y[(int64_t)s*O+o] =(float)dot_i4i8(wo, xs,I)*sc0*sx[s]; + y[(int64_t)s*O+(o+1)]=(float)dot_i4i8(wo1,xs,I)*sc1*sx[s]; } + } + if(O&1){ int o=O-1; const uint8_t *w=q4+(int64_t)o*rb; float sc=scale[o]; + #pragma omp parallel for schedule(static) + for(int s=0;s=2){ matmul_q_idot_mm(y,xq,sx,q,scale,S,I,O); return; } +#endif + #pragma omp parallel for schedule(static) + for(int o=0;o=2){ matmul_i4_idot_mm(y,xq,sx,q4,scale,S,I,O); return; } +#endif + #pragma omp parallel for schedule(static) + for(int o=0;og_qscratch.xq_cap){ + int8_t *p=realloc(g_qscratch.xq,xn); + if(!p){ fprintf(stderr,"OOM quant scratch\n"); exit(1); } + g_qscratch.xq=p; g_qscratch.xq_cap=xn; + } + if(sn>g_qscratch.sx_cap){ + float *p=realloc(g_qscratch.sx,sn*sizeof(float)); + if(!p){ fprintf(stderr,"OOM quant scales\n"); exit(1); } + g_qscratch.sx=p; g_qscratch.sx_cap=sn; + } + *xq=g_qscratch.xq; *sx=g_qscratch.sx; +} + +/* ---- f32 -> quantized packing --------------------------------------------- */ +static void quantize_rows(const float *w, int8_t *q, float *scale, int O, int I, int bits){ + int qmax=(1<<(bits-1))-1; + #pragma omp parallel for schedule(static) + for(int o=0;oamax)amax=a; } + float s=amax/qmax; if(s<1e-8f)s=1e-8f; scale[o]=s; + int8_t *qr=q+(int64_t)o*I; + for(int i=0;iqmax)v=qmax; if(v<-qmax-1)v=-qmax-1; qr[i]=(int8_t)v; } + } +} +static void pack_int4(const float *w, uint8_t *q4, float *scale, int O, int I, int bits){ + int qmax=(1<<(bits-1))-1, rb=(I+1)/2; + #pragma omp parallel for schedule(static) + for(int o=0;oamax)amax=a; } + float s=amax/qmax; if(s<1e-8f)s=1e-8f; scale[o]=s; + uint8_t *qr=q4+(int64_t)o*rb; + for(int i=0;iqmax)v0=qmax; if(v0<-8)v0=-8; + int v1=0; if(i+1qmax)v1=qmax; if(v1<-8)v1=-8; } + qr[i>>1] = (uint8_t)((v0+8) | ((v1+8)<<4)); + } + } +} +/* quantize w[O,I] f32 -> int3-g64 (fmt=5): per 64-input group, symmetric absmax + * (qmax=3, clamp [-4,3], stored v+4), 16B low plane + 8B high plane, ONE f32 scale + * per group. Same math as tools/quant_ablation.py `_quant_last_dim(bits=3, group=64)` + * (#132), here with real bit packing. */ +static void pack_int3_g64(const float *w, uint8_t *q3, float *scale, int O, int I){ + int64_t ng=i3_groups(I), rb=i3_rowbytes(I); + #pragma omp parallel for schedule(static) + for(int o=0;oamax)amax=a; } + float s=amax/3.f; if(s<1e-8f)s=1e-8f; sr[g]=s; + uint8_t *lo=qr+g*I3_GBYTES, *hi=lo+16; + memset(lo,0,I3_GBYTES); + for(int k=0;k3)v=3; if(v<-4)v=-4; + unsigned u=(unsigned)(v+4); /* 0..7 */ + lo[k>>2] |= (uint8_t)((u&3)<<((k&3)*2)); + hi[k>>3] |= (uint8_t)(((u>>2)&1)<<(k&7)); + } + } + } +} + +static void pack_int2(const float *w, uint8_t *q2, float *scale, int O, int I, int bits){ + int qmax=(1<<(bits-1))-1, rb=(I+3)/4; + #pragma omp parallel for schedule(static) + for(int o=0;oamax)amax=a; } + float s=amax/qmax; if(s<1e-8f)s=1e-8f; scale[o]=s; + uint8_t *qr=q2+(int64_t)o*rb; + for(int i=0;iqmax)v=qmax; if(v<-2)v=-2; byte|=(uint8_t)((v+2)<<(k*2)); } + qr[i>>2]=byte; + } + } +} + +/* ---- fmt=6: E8/IQ3 lattice container (#452) -------------------------------- + * 98 bytes per 256 weights = 3.0625 bits/weight. Per super-block: + * [ 0..63] uint8 grid index per 4-dim magnitude block + * [64..95] uint32 x8 - four 7-bit sign words + 4-bit sub-scale, per 32 weights + * [96..97] fp16 super-scale + * value = d * (0.5 + code) * 0.5 * grid[idx][j] * sign, with the 8th sign of every + * eight derived from odd parity (that is what buys the 8th bit back). + * Byte layout and arithmetic mirror tools/iq3_pack.py exactly - that codec is the + * oracle this kernel is tested against. + * + * Decode strategy: expand one 32-weight sub-block into a stack buffer, then FMA it + * against the activations. Per-weight table lookups would dominate; per-sub-block + * expansion keeps the grid rows (16 bytes each) hot in L1 and lets the compiler + * vectorize the multiply-accumulate. */ +#define E8_QK 256 /* weights per super-block */ +#define E8_SUB 32 /* weights per sign/scale word */ +#define E8_BBYTES 98 /* bytes per super-block */ +static inline int64_t e8_blocks(int I){ return ((int64_t)I + E8_QK - 1) / E8_QK; } +static inline int64_t e8_rowbytes(int I){ return e8_blocks(I) * E8_BBYTES; } + +/* The published 256x4 magnitude grid, stored doubled (4,12,..,62 mean 2,6,..,31). + * From ggml-common.h (MIT); tools/iq3xxs_grid.json is the same table for the + * Python codec, and tests/test_e8_kernel.c checks the two agree through the + * fixture. Header-local like the rest of quant.h - the engine is a single + * translation unit and the tests include this header directly. */ +static const uint8_t e8_grid[256][4] = { + { 4, 4, 4, 4}, + { 20, 4, 4, 4}, + { 36, 4, 4, 4}, + { 12, 12, 4, 4}, + { 28, 12, 4, 4}, + { 62, 12, 4, 4}, + { 4, 20, 4, 4}, + { 20, 20, 4, 4}, + { 12, 28, 4, 4}, + { 20, 36, 4, 4}, + { 28, 62, 4, 4}, + { 44, 62, 4, 4}, + { 12, 4, 12, 4}, + { 28, 4, 12, 4}, + { 4, 12, 12, 4}, + { 20, 12, 12, 4}, + { 12, 20, 12, 4}, + { 44, 20, 12, 4}, + { 4, 28, 12, 4}, + { 20, 28, 12, 4}, + { 12, 36, 12, 4}, + { 36, 44, 12, 4}, + { 4, 62, 12, 4}, + { 4, 4, 20, 4}, + { 20, 4, 20, 4}, + { 36, 4, 20, 4}, + { 12, 12, 20, 4}, + { 4, 20, 20, 4}, + { 20, 20, 20, 4}, + { 12, 28, 20, 4}, + { 28, 28, 20, 4}, + { 62, 28, 20, 4}, + { 12, 44, 20, 4}, + { 62, 44, 20, 4}, + { 44, 62, 20, 4}, + { 12, 4, 28, 4}, + { 62, 4, 28, 4}, + { 4, 12, 28, 4}, + { 20, 12, 28, 4}, + { 44, 20, 28, 4}, + { 4, 62, 28, 4}, + { 28, 12, 36, 4}, + { 62, 28, 36, 4}, + { 36, 36, 36, 4}, + { 62, 44, 36, 4}, + { 28, 62, 36, 4}, + { 44, 62, 36, 4}, + { 12, 4, 44, 4}, + { 62, 4, 44, 4}, + { 20, 28, 44, 4}, + { 20, 44, 44, 4}, + { 44, 28, 52, 4}, + { 36, 52, 52, 4}, + { 4, 12, 62, 4}, + { 36, 12, 62, 4}, + { 52, 12, 62, 4}, + { 28, 36, 62, 4}, + { 12, 52, 62, 4}, + { 12, 4, 4, 12}, + { 28, 4, 4, 12}, + { 4, 12, 4, 12}, + { 20, 12, 4, 12}, + { 12, 20, 4, 12}, + { 28, 20, 4, 12}, + { 4, 28, 4, 12}, + { 20, 28, 4, 12}, + { 36, 28, 4, 12}, + { 62, 36, 4, 12}, + { 4, 44, 4, 12}, + { 4, 4, 12, 12}, + { 20, 4, 12, 12}, + { 12, 12, 12, 12}, + { 4, 20, 12, 12}, + { 20, 20, 12, 12}, + { 12, 4, 20, 12}, + { 28, 4, 20, 12}, + { 4, 12, 20, 12}, + { 20, 12, 20, 12}, + { 12, 20, 20, 12}, + { 4, 28, 20, 12}, + { 20, 62, 20, 12}, + { 4, 4, 28, 12}, + { 20, 4, 28, 12}, + { 4, 20, 28, 12}, + { 12, 28, 28, 12}, + { 52, 36, 28, 12}, + { 52, 52, 28, 12}, + { 12, 4, 36, 12}, + { 44, 4, 36, 12}, + { 4, 44, 36, 12}, + { 4, 20, 44, 12}, + { 36, 20, 44, 12}, + { 52, 36, 44, 12}, + { 12, 62, 44, 12}, + { 44, 4, 52, 12}, + { 20, 20, 62, 12}, + { 4, 36, 62, 12}, + { 4, 4, 4, 20}, + { 20, 4, 4, 20}, + { 12, 12, 4, 20}, + { 28, 12, 4, 20}, + { 4, 20, 4, 20}, + { 20, 20, 4, 20}, + { 52, 20, 4, 20}, + { 12, 28, 4, 20}, + { 20, 36, 4, 20}, + { 12, 4, 12, 20}, + { 28, 4, 12, 20}, + { 44, 4, 12, 20}, + { 4, 12, 12, 20}, + { 20, 12, 12, 20}, + { 12, 20, 12, 20}, + { 4, 28, 12, 20}, + { 28, 52, 12, 20}, + { 62, 52, 12, 20}, + { 4, 62, 12, 20}, + { 4, 4, 20, 20}, + { 20, 4, 20, 20}, + { 12, 12, 20, 20}, + { 62, 12, 20, 20}, + { 4, 20, 20, 20}, + { 20, 20, 20, 20}, + { 62, 28, 20, 20}, + { 4, 36, 20, 20}, + { 44, 44, 20, 20}, + { 12, 4, 28, 20}, + { 4, 12, 28, 20}, + { 36, 12, 28, 20}, + { 4, 62, 28, 20}, + { 36, 62, 28, 20}, + { 44, 28, 36, 20}, + { 28, 44, 36, 20}, + { 28, 4, 44, 20}, + { 62, 20, 44, 20}, + { 12, 36, 44, 20}, + { 36, 62, 44, 20}, + { 12, 4, 62, 20}, + { 28, 4, 62, 20}, + { 52, 12, 62, 20}, + { 44, 36, 62, 20}, + { 12, 4, 4, 28}, + { 4, 12, 4, 28}, + { 20, 12, 4, 28}, + { 12, 20, 4, 28}, + { 28, 20, 4, 28}, + { 4, 44, 4, 28}, + { 44, 52, 4, 28}, + { 20, 62, 4, 28}, + { 4, 4, 12, 28}, + { 20, 4, 12, 28}, + { 4, 20, 12, 28}, + { 12, 28, 12, 28}, + { 36, 36, 12, 28}, + { 52, 36, 12, 28}, + { 12, 4, 20, 28}, + { 28, 4, 20, 28}, + { 4, 12, 20, 28}, + { 44, 20, 20, 28}, + { 20, 44, 20, 28}, + { 20, 62, 20, 28}, + { 12, 12, 28, 28}, + { 28, 28, 28, 28}, + { 4, 28, 36, 28}, + { 62, 36, 36, 28}, + { 20, 62, 36, 28}, + { 4, 4, 44, 28}, + { 52, 4, 44, 28}, + { 20, 20, 44, 28}, + { 44, 44, 44, 28}, + { 36, 12, 52, 28}, + { 52, 28, 52, 28}, + { 28, 52, 52, 28}, + { 28, 28, 62, 28}, + { 4, 52, 62, 28}, + { 36, 4, 4, 36}, + { 62, 12, 4, 36}, + { 44, 28, 4, 36}, + { 62, 28, 4, 36}, + { 28, 44, 4, 36}, + { 62, 44, 4, 36}, + { 36, 62, 12, 36}, + { 4, 20, 20, 36}, + { 62, 28, 20, 36}, + { 4, 36, 20, 36}, + { 4, 52, 20, 36}, + { 52, 52, 20, 36}, + { 62, 4, 28, 36}, + { 44, 36, 28, 36}, + { 36, 4, 36, 36}, + { 12, 44, 36, 36}, + { 36, 52, 36, 36}, + { 44, 20, 44, 36}, + { 28, 36, 44, 36}, + { 4, 62, 44, 36}, + { 44, 4, 62, 36}, + { 4, 12, 62, 36}, + { 20, 12, 62, 36}, + { 4, 28, 62, 36}, + { 20, 12, 4, 44}, + { 12, 36, 4, 44}, + { 4, 62, 4, 44}, + { 4, 4, 12, 44}, + { 52, 4, 12, 44}, + { 52, 20, 12, 44}, + { 44, 44, 12, 44}, + { 36, 12, 20, 44}, + { 20, 28, 20, 44}, + { 20, 62, 20, 44}, + { 20, 4, 28, 44}, + { 28, 44, 28, 44}, + { 4, 12, 36, 44}, + { 28, 20, 36, 44}, + { 62, 20, 36, 44}, + { 20, 62, 36, 44}, + { 20, 4, 44, 44}, + { 12, 28, 44, 44}, + { 4, 44, 52, 44}, + { 36, 20, 62, 44}, + { 20, 36, 62, 44}, + { 36, 20, 4, 52}, + { 36, 36, 4, 52}, + { 52, 36, 4, 52}, + { 36, 52, 4, 52}, + { 12, 20, 12, 52}, + { 12, 52, 12, 52}, + { 62, 12, 20, 52}, + { 36, 52, 20, 52}, + { 4, 28, 28, 52}, + { 52, 28, 28, 52}, + { 36, 36, 36, 52}, + { 44, 4, 44, 52}, + { 20, 44, 44, 52}, + { 28, 28, 52, 52}, + { 28, 4, 62, 52}, + { 12, 20, 62, 52}, + { 28, 4, 4, 62}, + { 44, 4, 4, 62}, + { 62, 4, 4, 62}, + { 4, 12, 4, 62}, + { 20, 28, 4, 62}, + { 20, 44, 4, 62}, + { 52, 20, 12, 62}, + { 4, 36, 12, 62}, + { 20, 12, 20, 62}, + { 44, 36, 20, 62}, + { 20, 44, 20, 62}, + { 4, 4, 28, 62}, + { 44, 12, 28, 62}, + { 28, 28, 28, 62}, + { 4, 52, 28, 62}, + { 12, 20, 36, 62}, + { 12, 36, 36, 62}, + { 4, 4, 44, 62}, + { 20, 4, 44, 62}, + { 36, 20, 44, 62}, + { 4, 28, 52, 62} +}; + +static inline float e8_fp16_to_f32(uint16_t h){ + uint32_t sign=(uint32_t)(h>>15)<<31, exp=(h>>10)&0x1F, man=h&0x3FF, bits; + if(!exp) bits = man ? (sign | ((127-15+1)<<23) | (man<<13)) : sign; /* subnormal->approx */ + else if(exp==0x1F) bits = sign | 0x7F800000u | (man<<13); + else bits = sign | ((exp+112)<<23) | (man<<13); + float f; memcpy(&f,&bits,4); return f; +} + +/* Expand one 32-weight sub-block. `out` must hold 32 floats. */ +static inline void e8_expand_sub(const uint8_t *blk, int ib, float d, float *out){ + uint32_t word; memcpy(&word, blk + E8_QK/4 + ib*4, 4); + float db = d * (0.5f + (float)((word>>28)&0xF)) * 0.5f; + const uint8_t *idx = blk + ib*8; + for(int l=0;l<4;l++){ + uint32_t seven=(word>>(7*l))&0x7F; + const uint8_t *g0=e8_grid[idx[l*2+0]], *g1=e8_grid[idx[l*2+1]]; + int par=0; + for(int j=0;j<8;j++){ + int neg = j<7 ? (int)((seven>>j)&1) : 0; + if(j<7) par^=neg; else neg=par; /* odd parity closes the block */ + float mag = (j<4 ? (float)g0[j] : (float)g1[j-4]) * 0.5f; + out[l*8+j] = neg ? -mag*db : mag*db; + } + } +} + +/* Fast Walsh-Hadamard transform with the per-tensor sign flip: y = Q^T x for + * Q = D*H/sqrt(n). fmt=6 stores W@Q, so activations must be transformed before + * the matmul (#452). Placement is the engine's job and it matters: all routed + * experts of a layer share one input row, so ONE transform per (layer, + * projection group) costs ~1.4 ms/token on GLM dims, while doing it per expert + * costs ~11 ms. n must be a power of two >= the real dim; the tail is zero-pad. + * Self-inverse up to the sign flip, so the same routine serves both directions. */ +static inline void e8_fwht(float *a, int n, const uint8_t *signbits){ + if(signbits) for(int i=0;i>3]>>(i&7)&1) a[i]=-a[i]; + for(int len=1;len>12; s^=s<<25; s^=s>>27; + bits[i]=(uint8_t)((s*2685821657736338717ULL)>>56); + } +} +/* Apply the fmt=6 activation rotation Q^T in place, row-major [nr,dim]. + * Non-power-of-two dims tile block-diagonally; each block is the largest power + * of two dividing the remainder (= its lowest set bit): 6144 -> 2048+4096, + * 1536 -> 512+1024. Blocks over 32768 halve until they fit the sign buffer. + * The converter rotates weight rows with this exact routine (W@Q and Q^T x are + * the same transform: Q is symmetric-orthogonal up to the sign flip). */ +static inline void e8_rot_rows(float *rows, int nr, int dim){ + int off=0; + while(off32768) b>>=1; + uint8_t bits[32768/8]; + e8_signs(bits,b); + for(int r=0;r=I) break; + float w[E8_SUB]; + e8_expand_sub(blk, ib, d, w); + int n = I-off < E8_SUB ? I-off : E8_SUB; + float a=0; + for(int k=0;k skipped. + # + # Column layout robustness: `lscpu -p=` emits *exactly* the + # requested columns (no CPU prefix), while bare `lscpu -p` prepends + # CPU. We requested two columns, but take the LAST TWO fields so the + # parser stays correct whether or not a CPU column is present + # (JustVugg review: the previous fields[1]/fields[2] indexing assumed + # a 3-column layout and regressed 2-column output to the logical + # count -- the opposite of the fix). result = subprocess.run(["lscpu", "-p=core,socket"], text=True, capture_output=True, check=True, timeout=5) - cores = {tuple(map(int, line.split(","))) for line in result.stdout.splitlines() - if line and not line.startswith("#")} + cores = set() + for line in result.stdout.splitlines(): + if not line or line.startswith("#"): + continue + fields = line.split(",") + if len(fields) < 2: + continue + try: + core, socket = int(fields[-2]), int(fields[-1]) + except ValueError: + continue # "-" for an offline core/socket + cores.add((core, socket)) if cores: return len(cores) - except (OSError, ValueError, subprocess.SubprocessError): - pass - return os.cpu_count() or 1 + except (OSError, ValueError, subprocess.SubprocessError) as error: + _physical_cores_warn(f"lscpu core probe failed: {error}") + logical = os.cpu_count() + if not logical: + _physical_cores_warn( + "could not detect any CPU cores; falling back to 1. " + "Set OMP_NUM_THREADS manually to fix single-core decode (#325).") + return 1 + _physical_cores_warn( + f"physical-core probes unavailable; using {logical} logical CPUs " + f"(SMT may over-subscribe). Set OMP_NUM_THREADS to physical cores if slow.") + return logical + + +def _resolve_physical_cores(physical_cpus): + """Coerce the build_plan() physical-core argument to a sane positive int. + + A None/0/None-ish value reaching here means physical_cpu_count() already + warned; clamp to 1 (so the engine always gets a positive team size) but keep + that clamp visible rather than silently masking it as the old ``max(1, int())`` + did (#325).""" + try: + count = int(physical_cpus or 0) + except (TypeError, ValueError): + count = 0 + if count < 1: + _physical_cores_warn( + "physical core count resolved to 0; defaulting to 1. " + "Set OMP_NUM_THREADS to fix single-core decode (#325).") + return 1 + return count def cpu_socket_count(): @@ -219,6 +302,63 @@ def cpu_socket_count(): return 1 +def _auto_tune(bottleneck_class, projected_hit, gpus, cpu_sockets, plan_has_metal): + """Derive tuning knobs from the bottleneck classification.""" + tune = {} + has_gpu = bool(gpus) + n_gpu = len(gpus) + + # MTP: costs more than it saves when compute-bound (#389 measured 42% loss) + # or streaming-bound (#467 measured 32% loss under CUDA at 85% hit). + # EXCEPTION: an explicit COLI_CUDA_MTP=1 in the environment is a documented + # opt-in to test speculation under CUDA (glm.c resolves DRAFT=-1 -> 3 only + # when it sees the var). Exporting DRAFT=0 here preempted that auto path, + # so the opt-in was silently inert on the Windows bare-run/auto-tier flows + # (#467): respect it and let the engine's auto path take over. Unset still + # gets DRAFT=0 -> MTP off, which is the measured-correct default. + if os.environ.get("COLI_CUDA_MTP") == "1": + pass # explicit opt-in: leave DRAFT to the engine's auto resolution + elif bottleneck_class == "compute": + tune["DRAFT"] = {"value": "0", + "reason": "compute-bound: MTP batch overhead exceeds yield"} + elif bottleneck_class == "disk" and projected_hit < 0.90: + tune["DRAFT"] = {"value": "0", + "reason": "low hit rate: MTP widens expert union, adds disk reads"} + # otherwise leave DRAFT unset (engine default: auto) + + # PIPE: resident pipeline mode depends on GPU count + if has_gpu and n_gpu == 1: + tune["COLI_CUDA_PIPE"] = {"value": "1", + "reason": "single GPU: S=1 pipeline gate"} + elif has_gpu and n_gpu > 1: + tune["COLI_CUDA_PIPE"] = {"value": "2", + "reason": "multi-GPU: residual stays on-device across layers"} + elif not has_gpu and bottleneck_class == "disk": + tune["PIPE"] = {"value": "1", + "reason": "overlap disk reads with resident expert compute"} + + # NUMA: selective interleave for GPU hosts, blanket hint for CPU-only + if cpu_sockets > 1 and has_gpu: + tune["COLI_NUMA"] = {"value": "1", + "reason": "multi-socket + GPU: interleave expert slabs, protect DMA buffers"} + elif cpu_sockets > 1 and not has_gpu: + tune["COLI_NUMA"] = {"value": "1", + "reason": "multi-socket CPU-only: interleave expert slabs across nodes"} + tune["_numa_hint"] = "numactl --interleave=all may perform better on CPU-only hosts" + + # OMP: kill hot-thread spin when GPU/Metal owns the power budget + if plan_has_metal: + tune["COLI_NO_OMP_TUNE"] = {"value": "1", + "reason": "Metal: OMP spin-wait steals GPU power budget"} + + # PIN: fully resident if RAM allows and no GPU tier competes + if projected_hit >= 0.99 and not has_gpu: + tune["PIN_GB"] = {"value": "all", + "reason": "enough RAM for full expert residency"} + + return tune + + POLICIES = { "quality": {"preserve_quantization": True, "preserve_router": True}, "balanced": {"preserve_quantization": True, "preserve_router": True}, @@ -290,19 +430,35 @@ def build_plan(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0, if cold_bytes: warnings.append("cold expert misses may reach disk; normal decode speed depends on hit rate") + total_expert = info["expert_bytes"] + resident_expert = hot_bytes + warm_bytes + projected_hit = resident_expert / total_expert if total_expert else 1.0 + if cold_bytes: bottleneck = "disk expert misses" - elif warm_bytes: - bottleneck = "CPU expert compute and RAM bandwidth" + bottleneck_class = "disk" + elif warm_bytes and gpus: + bottleneck = "CPU expert tail and GPU compute" + bottleneck_class = "mixed" + elif projected_hit >= 0.99: + if gpus: + bottleneck = "GPU compute and interconnect" + else: + bottleneck = "CPU expert compute (fully resident)" + bottleneck_class = "compute" else: - bottleneck = "GPU compute and interconnect" + bottleneck = "CPU expert compute and RAM bandwidth" + bottleneck_class = "memory" + + tune = _auto_tune(bottleneck_class, projected_hit, gpus, cpu_sockets, + plan_has_metal=False) return { "version": 2, "policy": {"name": policy, **POLICIES[policy], "quality_preserving": policy != "experimental-fast"}, "model": {key: value for key, value in info.items() if key != "config"}, - "cpu": {"physical_cores": max(1, int(physical_cpus)), + "cpu": {"physical_cores": _resolve_physical_cores(physical_cpus), "sockets": max(1, int(cpu_sockets)), "thread_policy": "physical-cores"}, "tiers": { @@ -317,6 +473,9 @@ def build_plan(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0, "expert_capacity": vram_experts, "requires_host_backing": False}, }, "expected_bottleneck": bottleneck, + "bottleneck_class": bottleneck_class, + "projected_hit_rate": round(projected_hit, 4), + "tune": tune, "decisions": [ {"target": "VRAM", "reason": "profile-ranked hot experts"}, {"target": "RAM", "reason": "warm experts execute on CPU without quality loss"}, @@ -331,15 +490,23 @@ def environment_for_plan(plan, env=None, cuda_enabled=True): result = dict(env or {}) result.setdefault("COLI_POLICY", plan["policy"]["name"]) result.setdefault("OMP_NUM_THREADS", str(plan["cpu"]["physical_cores"])) - if sys.platform != "win32": - # la libgomp di MinGW non supporta l'affinity su Windows - # ("Affinity not supported on this configuration"): non impostarle li'. - result.setdefault("OMP_PROC_BIND", "spread") - result.setdefault("OMP_PLACES", "cores") - if sys.platform.startswith("linux") and plan["cpu"].get("sockets", 1) > 1: - # Selectively interleave large expert/dense slabs across memory controllers. - # Unlike blanket numactl interleave, this leaves CUDA staging buffers local. - result.setdefault("COLI_NUMA", "1") + # NOTE: we intentionally do NOT set OMP_PROC_BIND / OMP_PLACES here. + # The engine's own hot-thread tuning (glm.c main(), the COLI_OMP_TUNED + # self-exec) sets OMP_PROC_BIND=close with overwrite=0 -- it prefers + # packing the team onto adjacent cores for the tiny back-to-back per-expert + # matmuls. Pre-setting OMP_PROC_BIND=spread here ran first and won (the + # engine's overwrite=0 setenv could not override an already-set var), and + # spread + OMP_PLACES=cores collapsed the team to one CPU on some libgomp / + # multi-socket topologies (#325: --auto-tier pinned decode to 1 core on a + # 64-core box even with OMP_NUM_THREADS=64). Leaving affinity to the engine + # makes --auto-tier match the plain (working) path. A user who wants a + # specific policy can still set OMP_PROC_BIND/OMP_PLACES in the environment + # themselves -- setdefault above only covers OMP_NUM_THREADS. + tune = plan.get("tune", {}) + for key, entry in tune.items(): + if key.startswith("_"): + continue + result.setdefault(key, entry["value"]) if plan["policy"]["name"] == "balanced": result.setdefault("REPIN", "64") ram = plan["tiers"]["ram"] @@ -386,5 +553,18 @@ def format_plan(plan): else: lines.append("VRAM no NVIDIA device detected · CPU path") lines.append(f"limit {plan['expected_bottleneck']}") + hit = plan.get("projected_hit_rate", 0) + lines.append(f"hit {hit:.0%} projected expert residency") + tune = plan.get("tune", {}) + if tune: + lines.append("") + lines.append("auto-tune:") + for key, entry in tune.items(): + if key.startswith("_"): + continue + lines.append(f" {key}={entry['value']:12s} {entry['reason']}") + hint = tune.get("_numa_hint") + if hint: + lines.append(f" hint: {hint}") lines.extend(f"warn {warning}" for warning in plan["warnings"]) return "\n".join(lines) diff --git a/c/sample.h b/c/sample.h new file mode 100644 index 00000000..414c6210 --- /dev/null +++ b/c/sample.h @@ -0,0 +1,153 @@ +/* sample.h — sampling (temperature + nucleus) and stop-set management. + * Header-only: all functions are static — include from the main engine file. */ +#ifndef SAMPLE_H +#define SAMPLE_H + +#include +#include +#include +#include "tok.h" + +/* ---- RNG (xorshift64*) -------------------------------------------------- */ +static uint64_t g_rng = 0x9E3779B97F4A7C15ULL; +static inline double rndu(void){ + g_rng ^= g_rng << 13; g_rng ^= g_rng >> 7; g_rng ^= g_rng << 17; + return (double)(g_rng >> 11) * (1.0 / 9007199254740992.0); +} + +/* ---- argmax over a float vector ----------------------------------------- */ +static inline int argmax_v(const float *lo, int V){ + int b=-1; float bv=-INFINITY; + for(int i=0;ibv){ bv=x; b=i; } } + return b<0?0:b; +} + +/* ---- distribution buffers (reused, single-threaded decode) --------------- */ +static float *g_pbuf = NULL; +static int *g_pidx = NULL; + +/* sift-down on max-heap in h[0..n), key = g_pbuf[h[i]] (#335: partial top-p). + * "hole" variant: carries the root value and deposits only at the end, so + * heapify is O(V) and each pop is O(log n) without qsort on the full vocab. */ +static void topp_siftdown(int *h, int n, int i){ + int iv = h[i]; float kv = g_pbuf[iv]; + for (;;) { + int l = 2*i + 1; + if (l >= n) break; + int b = l; if (l+1 < n && g_pbuf[h[l+1]] > g_pbuf[h[l]]) b = l+1; + if (g_pbuf[h[b]] <= kv) break; + h[i] = h[b]; i = b; + } + h[i] = iv; +} + +/* build the target distribution in g_pbuf: softmax(lo/temp) truncated to + * top-p g_nuc. Invariant: g_pbuf stays indexed by token-id (never reordered); + * the truncated tail is zeroed (dist_sample reads by id directly). + * Requires: g_temp, g_nuc, falloc() — declared in the main engine file. */ +static void dist_build(const float *lo, int V){ + if (!g_pbuf) { g_pbuf = falloc(V); g_pidx = malloc(V * sizeof(int)); } + int mxi = -1; float mx = 0; + for (int i = 0; i < V; i++) + if (isfinite(lo[i]) && (mxi < 0 || lo[i] > mx)) { mx = lo[i]; mxi = i; } + double s = 0; float invt = 1.f / (g_temp > 1e-4f ? g_temp : 1e-4f); + if (mxi >= 0) { + for (int i = 0; i < V; i++) { + g_pbuf[i] = isfinite(lo[i]) ? expf((lo[i] - mx) * invt) : 0.f; + s += g_pbuf[i]; + } + } + if (mxi < 0 || !isfinite(s) || s <= 0.0) { + static int warned = 0; + if (!warned) { warned = 1; fprintf(stderr, + "[SAMPLE] warning: non-finite logits (NaN/Inf) — falling back to argmax; " + "output may be degraded. This usually means a numerical blow-up upstream.\n"); } + int a = (mxi >= 0) ? mxi : 0; + for (int i = 0; i < V; i++) g_pbuf[i] = 0.f; + g_pbuf[a] = 1.f; + return; + } + for (int i = 0; i < V; i++) g_pbuf[i] /= (float)s; + if (g_nuc > 0 && g_nuc < 1.f) { + for (int i = 0; i < V; i++) g_pidx[i] = i; + for (int i = V/2-1; i >= 0; i--) topp_siftdown(g_pidx, V, i); + double s2 = 0, cum = 0; int out = V; + do { + int root = g_pidx[0]; + g_pidx[0] = g_pidx[--out]; g_pidx[out] = root; + s2 += g_pbuf[root]; cum += g_pbuf[root]; + if (out > 0) topp_siftdown(g_pidx, out, 0); + } while (cum < g_nuc && out > 0); + for (int i = 0; i < out; i++) g_pbuf[g_pidx[i]] = 0; + float s2f = (float)s2; + for (int i = out; i < V; i++) g_pbuf[g_pidx[i]] /= s2f; + } +} + +/* sample from g_pbuf; ban>=0 excludes that token (renormalizing on the fly) */ +static int dist_sample(int V, int ban){ + double z = 1.0 - (ban >= 0 ? g_pbuf[ban] : 0.0); + if (z <= 1e-12) z = 1e-12; + double u = rndu() * z, cum = 0; + for (int i = 0; i < V; i++) { if (i == ban) continue; cum += g_pbuf[i]; if (cum >= u) return i; } + for (int i = V-1; i >= 0; i--) if (i != ban && g_pbuf[i] > 0) return i; + return 0; +} + +/* next token from logits: greedy if g_temp<=0, sampling otherwise. + * ban = token excluded because it was rejected by speculative verification. */ +static int pick_tok(const float *lo, int V, int ban){ + if (g_temp <= 0) return argmax_v(lo, V); + dist_build(lo, V); + return dist_sample(V, ban); +} + +/* ---- stop set ----------------------------------------------------------- */ +static int g_stop[64], g_nstop = 0; +static inline int is_stop(int t){ + for (int i = 0; i < g_nstop; i++) if (t == g_stop[i]) return 1; + return 0; +} +/* T=NULL -> config stops only (validation/oracle, where the tokenizer is not needed). */ +static void stops_arm_tok(const Cfg *c, int tok_eos, Tok *T){ + g_nstop = 0; + for (int i = 0; i < c->n_stop && g_nstop < 64; i++) g_stop[g_nstop++] = c->stop_ids[i]; + if (tok_eos >= 0 && !is_stop(tok_eos) && g_nstop < 64) g_stop[g_nstop++] = tok_eos; + int nsp = 0; + if (T) for (int id = 0; id < T->n_ids && g_nstop < 64; id++) + if (T->id_special[id] && !is_stop(id)) { g_stop[g_nstop++] = id; nsp++; } + /* #401: in serve mode keep ONLY <|endoftext|>. Role markers <|user|>/<|observation|> + * (config stops + tokenizer special set) are boundaries the Python server owns; as + * hard stops they cut generation the moment the model opens a block, + * because int4 argmax noise picks a stop-token ID over the correct '<' token. */ + if (getenv("SERVE") && tok_eos >= 0) { + int kept = 0; + for (int i = 0; i < g_nstop; i++) if (g_stop[i] == tok_eos) g_stop[kept++] = g_stop[i]; + if (kept < g_nstop) fprintf(stderr, "[stop] serve mode: filtered %d non-EOS stop tokens (tool-call safety, #401)\n", g_nstop - kept); + g_nstop = kept; nsp = 0; + } + fprintf(stderr, "[stop] %d stop tokens:", g_nstop); + for (int i = 0; i < g_nstop; i++) fprintf(stderr, " %d", g_stop[i]); + if (nsp) fprintf(stderr, " (%d from the tokenizer's special set)", nsp); + fprintf(stderr, "\n"); +} +static void stops_arm(const Cfg *c, int tok_eos){ stops_arm_tok(c, tok_eos, NULL); } + +/* ---- log-prob of a target token given the logit vector ------------------- */ +static double logprob_target(const float *lo, int V, int target, int *am){ + float mx = lo[0]; int best = 0; + for (int i = 1; i < V; i++) if (lo[i] > mx) { mx = lo[i]; best = i; } + double se = 0; + for (int i = 0; i < V; i++) se += exp((double)lo[i] - mx); + if (am) *am = (best == target); + return (double)(lo[target] - mx) - log(se); +} + +/* "glm" in model_type, case-insensitive */ +static int mt_is_glm(const char *s){ + if (s) for (; *s; s++) + if ((s[0]|32) == 'g' && (s[1]|32) == 'l' && (s[2]|32) == 'm') return 1; + return 0; +} + +#endif /* SAMPLE_H */ diff --git a/c/setup.sh b/c/setup.sh index e31e60ab..dcb851d5 100755 --- a/c/setup.sh +++ b/c/setup.sh @@ -32,11 +32,11 @@ esac # 2) build: nativa (veloce, per QUESTA macchina). Per un binario da distribuire: make portable echo " building (ARCH=${ARCH:-native})…" -make -s glm ARCH="${ARCH:-native}" +make -s colibri ARCH="${ARCH:-native}" # 3) self-test sull'oracolo tiny, se presente if [ -d glm_tiny ] && [ -f ref_glm.json ]; then - r=$(SNAP=./glm_tiny TF=1 ./glm 64 16 16 2>/dev/null | grep -oE "[0-9]+/[0-9]+ positions" || true) + r=$(SNAP=./glm_tiny TF=1 ./colibri 64 16 16 2>/dev/null | grep -oE "[0-9]+/[0-9]+ positions" || true) echo " engine self-test: ${r:-?} (expected 32/32)" fi diff --git a/c/st.h b/c/st.h index efa6e5a0..ab259ae2 100644 --- a/c/st.h +++ b/c/st.h @@ -40,6 +40,9 @@ typedef struct { int dfds[512]; /* gemelli O_DIRECT (aperti pigramente): -2 = non ancora provato */ char *paths[512]; int nfd; + int mfds[512]; /* MIRROR: fds of the second model copy (dual-SSD), -1 = absent */ + int mdfds[512]; /* O_DIRECT twins of the second copy, -1 = absent */ + int nmirror; /* files accepted into the mirror (0 = mirror inactive) */ int *hidx; /* hash map nome->indice (open addressing): con ~120k tensori * (GLM: 256 expert x 78 layer x 3 x 2) la scansione lineare * costava decine di secondi/token (misurato sul primo run reale) */ @@ -99,10 +102,80 @@ static int st_open_fd(shards *S, const char *path) { /* fd gemello O_DIRECT dello stesso file (bypassa la page cache: il buffered read su * ext4-in-VHDX si strozza a ~0.8 GB/s, O_DIRECT arriva a 2.3+; misurato). -1 se non disponibile. */ -static int st_direct_fd(shards *S, int fd) { - for (int i = 0; i < S->nfd; i++) if (S->fds[i] == fd) return S->dfds[i]; +static int st_fidx(shards *S, int fd) { + for (int i = 0; i < S->nfd; i++) if (S->fds[i] == fd) return i; return -1; } +static int st_direct_fd(shards *S, int fd) { + int i = st_fidx(S, fd); return i < 0 ? -1 : S->dfds[i]; +} + +/* ---- MIRROR (dual-SSD): second read-only copy of the model on another drive ---- + * st_fd_rep/st_direct_fd_rep: fd of replica `rep` (0 = primary, 1 = mirror) for + * the SAME file identified by its primary fd. -1 if that replica is absent. */ +static int st_fd_rep(shards *S, int fd, int rep) { + if (!rep) return fd; + if (!S->nmirror) return -1; + int i = st_fidx(S, fd); return i < 0 ? -1 : S->mfds[i]; +} +static int st_direct_fd_rep(shards *S, int fd, int rep) { + if (!rep) return st_direct_fd(S, fd); + if (!S->nmirror) return -1; + int i = st_fidx(S, fd); return i < 0 ? -1 : S->mdfds[i]; +} + +/* Registers / as a read replica of every already-indexed shard. + * A file is accepted ONLY if its size and safetensors header are byte-identical + * to the primary: the data_offsets then match by construction, so every pread + * is valid on either copy. Missing or divergent files simply stay on the + * primary (the mirror may be partial, e.g. a smaller SSD holding only the + * expert shards). Returns the number of accepted files. The mirror is NEVER + * written to: .coli_usage/.coli_kv keep deriving from the primary alone. */ +static int st_mirror_init(shards *S, const char *dir) { + if (S->nmirror) for (int i = 0; i < S->nfd; i++) { /* re-init: drop the old replica */ + if (S->mfds[i] >= 0) close(S->mfds[i]); + if (S->mdfds[i] >= 0) close(S->mdfds[i]); + } + for (int i = 0; i < ST_MAX_SHARDS; i++) { S->mfds[i] = -1; S->mdfds[i] = -1; } + S->nmirror = 0; + for (int i = 0; i < S->nfd; i++) { + const char *base = strrchr(S->paths[i], '/'); +#ifdef _WIN32 + const char *b2 = strrchr(S->paths[i], '\\'); + if (b2 && (!base || b2 > base)) base = b2; +#endif + base = base ? base + 1 : S->paths[i]; + char mp[2048]; snprintf(mp, sizeof(mp), "%s/%s", dir, base); + int mfd = open(mp, COMPAT_O_RDONLY); + if (mfd < 0) continue; /* partial mirror: this shard stays on the primary */ + int64_t sza = lseek(S->fds[i], 0, SEEK_END), szb = lseek(mfd, 0, SEEK_END); + if (sza != szb) { + fprintf(stderr, "[MIRROR] %s: size differs from the primary copy — file skipped\n", mp); + close(mfd); continue; + } + uint64_t ha = 0, hb = 0; int ok = 1; /* identical header => identical data_offsets */ + if (pread(S->fds[i], &ha, 8, 0) != 8 || pread(mfd, &hb, 8, 0) != 8 || + ha != hb || ha == 0 || ha > (uint64_t)256 << 20 || (int64_t)(8 + ha) > sza) ok = 0; + if (ok) { + char *ba = malloc(ha), *bb = malloc(ha); + if (!ba || !bb || pread(S->fds[i], ba, ha, 8) != (ssize_t)ha || + pread(mfd, bb, ha, 8) != (ssize_t)ha || memcmp(ba, bb, ha)) ok = 0; + free(ba); free(bb); + } + if (!ok) { + fprintf(stderr, "[MIRROR] %s: header differs from the primary copy — file skipped\n", mp); + close(mfd); continue; + } + S->mfds[i] = mfd; +#ifdef O_DIRECT + S->mdfds[i] = open(mp, COMPAT_O_RDONLY | O_DIRECT); +#elif defined(__APPLE__) + S->mdfds[i] = compat_open_direct(mp); +#endif + S->nmirror++; + } + return S->nmirror; +} /* indicizza tutti i model-*.safetensors in snap_dir */ /* pread completo: chunk-loop (una singola pread si ferma a ~2^31 byte su Linux @@ -137,21 +210,66 @@ static void st_pread_full(int fd, void *buf, int64_t n, int64_t off, const char } } -static void st_init(shards *S, const char *snap_dir) { - memset(S, 0, sizeof(*S)); - S->cap = 4096; S->t = calloc(S->cap, sizeof(st_tensor)); - /* raccoglie ordinatamente i nomi dei file shard */ - static char files[ST_MAX_SHARDS][1024]; int nf = 0; - DIR *d = opendir(snap_dir); struct dirent *e; - if (!d) { perror(snap_dir); exit(1); } +/* Scan one directory for *.safetensors shards, appending to files[] (dedup by + * basename, so a list of directories acts as a SEARCH PATH: the same shard + * present on two drives is taken from the first-listed one only). *added + * returns how many shards this dir contributed. */ +static void st_scan_dir(const char *dir, char files[][1024], int *nf, int *added) { + DIR *d = opendir(dir); struct dirent *e; + if (!d) { perror(dir); exit(1); } + int base_n = *nf; while ((e = readdir(d))) { const char *dot = strrchr(e->d_name, '.'); if (dot && !strcmp(dot, ".safetensors")) { /* model.safetensors o model-0000N-of-... */ - if (nf >= ST_MAX_SHARDS) { fprintf(stderr, "too many shards (>%d): raise ST_MAX_SHARDS\n", ST_MAX_SHARDS); exit(1); } - snprintf(files[nf++], 1024, "%s/%s", snap_dir, e->d_name); + int dup = 0; + for (int i = 0; i < *nf; i++) { + const char *b = strrchr(files[i], '/'); +#ifdef _WIN32 + const char *b2 = strrchr(files[i], '\\'); if (b2 && (!b || b2 > b)) b = b2; +#endif + b = b ? b + 1 : files[i]; + if (!strcmp(b, e->d_name)) { dup = 1; break; } /* already taken from a higher-priority drive */ + } + if (dup) continue; + if (*nf >= ST_MAX_SHARDS) { fprintf(stderr, "too many shards (>%d): raise ST_MAX_SHARDS\n", ST_MAX_SHARDS); exit(1); } + snprintf(files[(*nf)++], 1024, "%s/%s", dir, e->d_name); } } closedir(d); + if (added) *added = *nf - base_n; +} + +/* Index shards from snap_dir, optionally SPLIT across extra drives listed in + * extra_dirs (';' or ',' separated). Each shard lives on exactly ONE drive + * (no duplication — unlike the dual-SSD mirror); a demand pread hits whichever + * drive holds that shard, so concurrent expert loads parallelise across drives + * and combined capacity is used. Scales to N drives. Metadata (config / + * tokenizer / .coli_usage / .coli_kv) is read from snap_dir only. */ +static void st_init_multi(shards *S, const char *snap_dir, const char *extra_dirs) { + memset(S, 0, sizeof(*S)); + S->cap = 4096; S->t = calloc(S->cap, sizeof(st_tensor)); + /* raccoglie ordinatamente i nomi dei file shard */ + static char files[ST_MAX_SHARDS][1024]; int nf = 0; + int c0 = 0; st_scan_dir(snap_dir, files, &nf, &c0); + int ndir = 1; + if (extra_dirs && *extra_dirs) { + char buf[4096]; snprintf(buf, sizeof(buf), "%s", extra_dirs); + char *p = buf; + while (p && *p) { + char *sep = p; while (*sep && *sep != ';' && *sep != ',') sep++; + int last = (*sep == 0); *sep = 0; + while (*p == ' ') p++; + size_t plen = strlen(p); while (plen > 0 && p[plen-1] == ' ') p[--plen] = 0; + if (*p) { + int cN = 0; st_scan_dir(p, files, &nf, &cN); + fprintf(stderr, "[SPLIT] +%s -> %d shard(s)\n", p, cN); + ndir++; + } + p = last ? NULL : sep + 1; + } + fprintf(stderr, "[SPLIT] model across %d dir(s): %d shard(s) total (primary %s -> %d shard(s)), no duplication\n", + ndir, nf, snap_dir, c0); + } for (int a = 0; a < nf; a++) for (int b = a+1; b < nf; b++) if (strcmp(files[a], files[b]) > 0) { char tmp[1024]; strcpy(tmp, files[a]); strcpy(files[a], files[b]); strcpy(files[b], tmp); } @@ -200,11 +318,32 @@ static void st_init(shards *S, const char *snap_dir) { if (a0 < 0 || b0 < a0 || data_start + b0 > fsz) { fprintf(stderr, "%s: tensor '%s' data_offsets [%lld,%lld] out of file bounds (%lld)\n", files[fi], name, (long long)a0, (long long)b0, (long long)fsz); exit(1); } - int64_t numel = 1; for (int k = 0; k < shp->len; k++) numel *= (int64_t)shp->kids[k]->num; + /* SEC: lo shape viene da un file non fidato (mirror). Senza il guard + * di overflow, uno shape tipo [65535,65535,65535,...] fa avvolgere + * numel a un valore piccolo/negativo che poi passerebbe il cross-check + * numel*esz==nbytes in st_read_f32, riaprendo l'OOB. */ + int64_t numel = 1; int bad_shape = 0; + for (int k = 0; k < shp->len; k++) { + int64_t d = (int64_t)shp->kids[k]->num; + if (d < 0 || (d != 0 && numel > INT64_MAX / d)) { bad_shape = 1; break; } + numel *= d; + } + if (bad_shape) { + fprintf(stderr, "%s: tensor '%s' shape overflows int64 — refusing (hostile or corrupt file)\n", + files[fi], name); exit(1); } if (S->n == S->cap) { S->cap *= 2; S->t = realloc(S->t, S->cap*sizeof(st_tensor)); } st_tensor *t = &S->t[S->n++]; t->name = strdup(name); t->fd = fd; t->off = data_start + a0; t->nbytes = b0 - a0; t->dtype = st_dtype_code(dt->str); t->numel = numel; + /* cross-check the declared element count against the byte span for FLOAT + * dtypes: st_read_f32 writes `numel` floats (BF16/F16 loop or F32 memcpy) + * into a caller-sized buffer, so a header with numel != nbytes/esz is an + * OOB write primitive. U8/I8 (raw quant bytes) are read by byte count, so + * their numel is unused by the read path and legitimately may differ. */ + { int esz = t->dtype==2 ? 4 : (t->dtype==3 ? 1 : 2); + if (t->dtype != 3 && t->nbytes != numel * (int64_t)esz) { + fprintf(stderr, "%s: tensor '%s' numel %lld disagrees with byte span %lld (esz %d)\n", + files[fi], name, (long long)numel, (long long)t->nbytes, esz); exit(1); } } } free(arena); /* i jval restano leakati: ok, una tantum all'avvio */ free(hdr); @@ -220,6 +359,9 @@ static void st_init(shards *S, const char *snap_dir) { } } +/* backward-compatible single-directory entry point */ +static void st_init(shards *S, const char *snap_dir) { st_init_multi(S, snap_dir, NULL); } + static st_tensor *st_find(shards *S, const char *name) { if (S->hidx) { uint64_t h = st_hash(name) & (S->hcap - 1); @@ -244,11 +386,30 @@ static void st_prefetch(shards *S, const char *name) { if (t) posix_fadvise(t->fd, t->off, t->nbytes, POSIX_FADV_WILLNEED); } +/* like st_prefetch, but on replica `rep`'s drive: the WILLNEED must warm the + * page cache of the SAME fd the later demand pread will hit. */ +static void st_prefetch_rep(shards *S, const char *name, int rep) { + st_tensor *t = st_find(S, name); + if (!t) return; + int fd = st_fd_rep(S, t->fd, rep); + if (fd < 0) fd = t->fd; + posix_fadvise(fd, t->off, t->nbytes, POSIX_FADV_WILLNEED); +} + /* legge un tensore in un buffer float32 fornito dal chiamante (numel float). * drop=1 -> consiglia al kernel di scartare le pagine (per gli expert in streaming). */ static int64_t st_read_f32(shards *S, const char *name, float *out, int drop) { st_tensor *t = st_find(S, name); if (!t) { fprintf(stderr, "missing tensor: %s\n", name); exit(1); } + /* SEC: numel viene dallo shape, nbytes dagli offset — due campi indipendenti + * del file. Se non concordano, la memcpy F32 (nbytes) o i loop BF16/F16 + * (numel elementi da un raw di soli nbytes) sforano il buffer del chiamante, + * che e' dimensionato sul config, non sul file. Il chiamante che alloca su + * st_numel resta coerente; questo blocca l'ingresso ostile a monte. */ + int esz = (t->dtype == 2) ? 4 : 2; + if (t->numel < 0 || t->numel > t->nbytes / esz || t->numel * (int64_t)esz != t->nbytes) { + fprintf(stderr, "%s: tensor '%s' shape/bytes mismatch (numel %lld, %lld bytes, dtype %d) — refusing (hostile or corrupt file)\n", + name, name, (long long)t->numel, (long long)t->nbytes, t->dtype); exit(1); } void *raw = malloc(t->nbytes); if (!raw) { fprintf(stderr, "malloc %lld bytes for tensor %s failed\n", (long long)t->nbytes, name); exit(1); } st_pread_full(t->fd, raw, t->nbytes, t->off, "pread data"); @@ -264,6 +425,20 @@ static int64_t st_read_f32(shards *S, const char *name, float *out, int drop) { return t->numel; } +/* like st_read_f32 but refuses to write more than `cap` floats into `out`. + * Callers that size `out` from CONFIG dims (qt_alloc's O*I, per-row scales) must + * use this: a crafted header whose tensor holds more elements than the config + * shape would otherwise overrun `out`. Callers that size `out` from st_numel are + * self-consistent and may keep using st_read_f32. */ +static int64_t st_read_f32_cap(shards *S, const char *name, float *out, int64_t cap, int drop) { + st_tensor *t = st_find(S, name); + if (!t) { fprintf(stderr, "missing tensor: %s\n", name); exit(1); } + if (t->numel > cap) { + fprintf(stderr, "tensor %s: numel %lld exceeds destination capacity %lld\n", + name, (long long)t->numel, (long long)cap); exit(1); } + return st_read_f32(S, name, out, drop); +} + static int64_t st_numel(shards *S, const char *name) { st_tensor *t = st_find(S, name); return t ? t->numel : -1; } @@ -287,9 +462,13 @@ static void st_read_slice_f32(shards *S, const char *name, int64_t elem_off, int st_tensor *t = st_find(S, name); if (!t) { fprintf(stderr, "missing tensor: %s\n", name); exit(1); } int esz = (t->dtype == 2) ? 4 : 2; + if (elem_off < 0 || n_elems < 0 || elem_off + n_elems > t->numel) { /* keep the slice inside the tensor */ + fprintf(stderr, "slice %s [%lld,+%lld) out of tensor bounds (numel %lld)\n", + name, (long long)elem_off, (long long)n_elems, (long long)t->numel); exit(1); } int64_t boff = t->off + elem_off * esz, nb = n_elems * esz; void *raw = malloc(nb); - st_pread_full(t->fd, raw, nb, boff, "pread slice"); + if (!raw) { fprintf(stderr, "malloc %lld bytes for slice %s failed\n", (long long)nb, name); exit(1); } + st_pread_full(t->fd, raw, nb, boff, "pread slice"); /* dev #331: chunked + EINTR + honest short-read */ if (t->dtype == 2) memcpy(out, raw, nb); else if (t->dtype == 0) { uint16_t *p = raw; for (int64_t i = 0; i < n_elems; i++) out[i] = bf16_to_f32(p[i]); } else { uint16_t *p = raw; for (int64_t i = 0; i < n_elems; i++) out[i] = f16_to_f32(p[i]); } diff --git a/c/telemetry.h b/c/telemetry.h new file mode 100644 index 00000000..3c3fe855 --- /dev/null +++ b/c/telemetry.h @@ -0,0 +1,189 @@ +/* telemetry.h — dashboard protocol lines, stats/usage persistence, hardware probe. + * Include after Model/Cfg/QT/ESlot/shards and st.h are defined; requires + * qt_bytes(), now_s(), rss_gb(), edisk_s(), and the g_cuda_* globals (ifdef). */ +#ifndef TELEMETRY_H +#define TELEMETRY_H + +static int64_t tbytes(int O,int I,int bits){ + if(bits>=16) return (int64_t)O*I*4; + if(bits>=5) return (int64_t)O*I + (int64_t)O*4; + return (int64_t)O*((I+1)/2) + (int64_t)O*4; +} + +static int64_t expert_bytes_probe(Model *m, int ebits){ + Cfg *c=&m->c; int64_t eb=0; char nm[256]; + snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.0.gate_proj.weight",c->first_dense); + if(st_nbytes(&m->S,nm)>0){ + const char *suf[3]={"gate_proj","up_proj","down_proj"}; + for(int k=0;k<3;k++){ + snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.0.%s.weight",c->first_dense,suf[k]); + eb+=st_nbytes(&m->S,nm); + snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.0.%s.weight.qs",c->first_dense,suf[k]); + int64_t q=st_nbytes(&m->S,nm); if(q>0) eb+=q; + } + } + if(eb<=0) eb = tbytes(c->moe_inter,c->hidden,ebits)*2 + tbytes(c->hidden,c->moe_inter,ebits); + return eb; +} + +/* BRAIN MAP: per-turn expert hit bitmap for the dashboard. */ +static uint8_t **g_ehit; +static void ehit_mark(Model *m, int layer, int eid){ + if(!g_ehit){ Cfg *c=&m->c; + g_ehit=calloc(c->n_layers+1,sizeof(uint8_t*)); + for(int i=0;i<=c->n_layers;i++) g_ehit[i]=calloc(c->n_experts,1); + } + g_ehit[layer][eid]=1; +} + +/* CPU model + cores + RAM (GB); empty/zero where unavailable. */ +static void hw_probe(char *cpu, size_t cn, int *cores, double *ram_total, double *ram_avail){ + cpu[0]=0; +#ifdef _WIN32 +#if defined(__x86_64__) || defined(__i386__) + { unsigned int r[12]={0}; unsigned int *w=r; + for(unsigned int f=0x80000002u; f<=0x80000004u; f++,w+=4) + __get_cpuid(f,&w[0],&w[1],&w[2],&w[3]); + char *b=(char*)r; b[47]=0; while(*b==' ')b++; + snprintf(cpu,cn,"%s",b); } +#endif +#else + FILE *ci=fopen("/proc/cpuinfo","r"); + if(ci){ char ln[256]; + while(fgets(ln,sizeof(ln),ci)) if(!strncmp(ln,"model name",10)){ + char *p=strchr(ln,':'); if(p){ p++; while(*p==' ')p++; + int n=(int)strlen(p); if(n>0&&p[n-1]=='\n')p[--n]=0; + snprintf(cpu,cn,"%s",p); } break; } + fclose(ci); } +#endif + *cores=0; +#ifdef _WIN32 + { SYSTEM_INFO si; GetSystemInfo(&si); *cores=(int)si.dwNumberOfProcessors; } +#elif defined(_SC_NPROCESSORS_ONLN) + *cores=(int)sysconf(_SC_NPROCESSORS_ONLN); +#endif + *ram_total=*ram_avail=0; +#ifdef _WIN32 + compat_meminfo(ram_total,ram_avail); +#else + FILE *mi=fopen("/proc/meminfo","r"); + if(mi){ char ln[256]; double mt=0,ma=0; + while(fgets(ln,sizeof(ln),mi)){ + if(sscanf(ln,"MemTotal: %lf",&mt)==1) *ram_total=mt/1e6; + if(sscanf(ln,"MemAvailable: %lf",&ma)==1) *ram_avail=ma/1e6; + } fclose(mi); } +#endif +} + +static void hwinfo_emit(Model *m){ + Cfg *c=&m->c; (void)c; + char cpu[256]; int cores; double ram_total,ram_avail; + hw_probe(cpu,sizeof(cpu),&cores,&ram_total,&ram_avail); + int ngpu=0; double vram_total=0; + char gpu_name[128]=""; +#ifdef COLI_CUDA + ngpu=g_cuda_ndev; vram_total=m->gpu_expert_bytes/1e9; + for(int i=0;i0) + snprintf(gpu_name,sizeof(gpu_name),"CUDA device x%d",g_cuda_ndev); +#endif + printf("HWINFO %d %.1f %.1f %d %.1f %s|%s\n", + cores,ram_total,ram_avail,ngpu,vram_total,cpu,gpu_name); + fflush(stdout); +} + +static void tiers_emit(Model *m){ + Cfg *c=&m->c; int nsp=0; + for(int i=0;in_layers;i++) if(m->L[i].sparse) nsp++; + int total=(nsp+(m->has_mtp?1:0))*c->n_experts; + int pinned=0,lru=0; + for(int i=0;i<=c->n_layers;i++){ pinned+=m->npin?m->npin[i]:0; lru+=m->ecn?m->ecn[i]:0; } + int vram=0; double vram_gb=0; +#ifdef COLI_CUDA + vram=m->gpu_expert_count; vram_gb=m->gpu_expert_bytes/1e9; +#endif + int ram=pinned-vram+lru; if(ram<0) ram=0; + int disk=total-vram-ram; if(disk<0) disk=0; + double eb=(double)expert_bytes_probe(m,m->ebits); + printf("TIERS %d %d %d %.2f %.2f\n",vram,ram,disk,vram_gb,ram*eb/1e9); + fflush(stdout); +} + +static void emap_emit(Model *m){ + Cfg *c=&m->c; + int rows=0; + for(int i=0;in_layers;i++) if(m->L[i].sparse) rows++; + int has_mtp = m->has_mtp && m->eusage[c->n_layers]; + if(has_mtp) rows++; + int cols=c->n_experts; + char *hex=malloc((size_t)rows*cols*2+1); int w=0; + for(int i=0;i<=c->n_layers;i++){ + int is_row = (in_layers && m->L[i].sparse) || (i==c->n_layers && has_mtp); + if(!is_row) continue; + for(int e=0;epin[i]; + for(int z=0;znpin[i];z++) if(P[z].eid==e){ +#ifdef COLI_CUDA + tier = P[z].g.cuda?2:1; +#else + tier = 1; +#endif + break; } + if(!tier && m->ecache && m->ecache[i]) + for(int z=0;zecn[i];z++) if(m->ecache[i][z].eid==e){ tier=1; break; } + uint32_t u = m->eusage[i]?m->eusage[i][e]:0; + int heat=0; while(u){ heat++; u>>=1; } if(heat>63) heat=63; + int b=(tier<<6)|heat; + hex[w++]="0123456789abcdef"[b>>4]; hex[w++]="0123456789abcdef"[b&15]; + } + } + hex[w]=0; + printf("EMAP %d %d %s\n",rows,cols,hex); fflush(stdout); free(hex); +} + +static void hits_emit(Model *m){ + Cfg *c=&m->c; if(!g_ehit) return; + int rows=0; + for(int i=0;in_layers;i++) if(m->L[i].sparse) rows++; + int has_mtp = m->has_mtp && m->eusage[c->n_layers]; + if(has_mtp) rows++; + int cols=c->n_experts, nb=(rows*cols+7)/8; + uint8_t *bm=calloc(nb,1); int bit=0; + for(int i=0;i<=c->n_layers;i++){ + int is_row = (in_layers && m->L[i].sparse) || (i==c->n_layers && has_mtp); + if(!is_row) continue; + for(int e=0;e>3]|=1<<(bit&7); g_ehit[i][e]=0; } + } + char *hex=malloc((size_t)nb*2+1); int w=0; + for(int b=0;b>4]; hex[w++]="0123456789abcdef"[bm[b]&15]; } + hex[w]=0; + printf("HITS %d %d %s\n",rows,cols,hex); fflush(stdout); free(hex); free(bm); +} + +static void stats_dump_q(Model *m, const char *path, int quiet){ + char tmp[2100]; snprintf(tmp,sizeof(tmp),"%s.tmp",path); + FILE *f=fopen(tmp,"w"); if(!f){ if(!quiet) perror(tmp); return; } + Cfg *c=&m->c; int64_t tot=0, nz=0; + for(int i=0;i<=c->n_layers;i++){ if(!m->eusage[i]) continue; + for(int e=0;en_experts;e++) if(m->eusage[i][e]){ fprintf(f,"%d %d %u\n",i,e,m->eusage[i][e]); tot+=m->eusage[i][e]; nz++; } } + fclose(f); rename(tmp,path); + if(!quiet) fprintf(stderr,"[STATS] %lld selections across %lld distinct experts -> %s\n",(long long)tot,(long long)nz,path); +} +static void stats_dump(Model *m, const char *path){ stats_dump_q(m,path,0); } + +static char g_usage_path[2100]=""; +static int64_t usage_load(Model *m, const char *path){ + FILE *f=fopen(path,"r"); if(!f) return 0; + Cfg *c=&m->c; int l,e; uint32_t cnt; int64_t tot=0; + while(fscanf(f,"%d %d %u",&l,&e,&cnt)==3) + if(l>=0&&l<=c->n_layers&&e>=0&&en_experts&&m->eusage[l]){ m->eusage[l][e]+=cnt; tot+=cnt; } + fclose(f); return tot; +} +static void usage_save(Model *m){ if(g_usage_path[0]) stats_dump_q(m,g_usage_path,1); } + +#endif /* TELEMETRY_H */ diff --git a/c/tests/README_efficiency.md b/c/tests/README_efficiency.md new file mode 100644 index 00000000..1f8d339a --- /dev/null +++ b/c/tests/README_efficiency.md @@ -0,0 +1,98 @@ +# Efficiency suite — regression tests + optimization dossier + +Two layers: + +1. **`test_inefficiency.py`** — tiny-model *asserted* regression tests. Fast + (~0.15s/run), gate CI, catch breakage. Run as part of `make test`. +2. **`test_efficiency_report.py`** — an *opt-in optimization dossier* for a real + model. Runs every instrumentation flag, prints a 9-section report answering + *what is doing what, when, with what, is it inefficient, how to improve*. + Never fails CI (it's a report, not a gate). + +## The dossier (what you run when optimizing) + +```bash +# CPU-only (safe, fast to validate): +COLI_EFFICIENCY_MODEL=../glm52_i4_g64 make efficiency-report + +# CUDA (dense + expert tiers — needs a CUDA build, see below): +COLI_EFFICIENCY_MODEL=../glm52_i4_g64 COLI_EFFICIENCY_CUDA=1 make efficiency-report +``` + +It turns ON every observability flag the engine supports — `PROF=1`, +`COLI_CUDA_PROFILE=1`, `CACHE_ROUTE=1` (auto-unlocks `route_agree`/`route_kl`), +`DISK_SPLIT=1`, `LOOKA=1` — so nothing the engine can tell you is left dark. +None of these change the computed output; they only add telemetry. + +The 9 sections, and the question each answers: + +| § | section | answers | +|---|---|---| +| 1 | PROVENANCE | what is running, on what CPU/backend, with what effective config | +| 2 | THROUGHPUT | tok/s + forward-latency p50/p90/p99/max (is the tail healthy?) | +| 3 | WHERE TIME GOES | the 5 PROFILE phases as % of decode + absolute seconds + verdict | +| 3a | ATTENTION BREAKDOWN | attention split into projection/RoPE, score-softmax-value, output | +| 4 | EXPERT CACHE | hit %, experts-loaded/token vs baseline topk | +| 5 | DISK I/O | GB fetched, MB/token, GB/s, read-service vs felt-wait, phase split | +| 5a | DISK-LOAD SPLIT | loads by decode phase (draft/absorb/verify) + MTP-vs-main bytes | +| 6 | ROUTING QUALITY | route_agree %, route_kl, cache swaps | +| 6a | ROUTING PREDICTABILITY | LOOKAHEAD recall per predictor (which prefetch wins) | +| 7 | SPECULATION | tokens/forward, MTP acceptance % | +| 8 | GPU TIERS | resident tensors, expert tier (count/GB/calls), H2D/kernel/D2H ms | + +Every line that crosses an advisory threshold is marked `[FLAG]` with the +concrete lever to pull (raise RAM_GB, add PIN_GB, try DIRECT=1, lower CTX, …), +and all flags repeat in a summary at the end. + +## Tunable thresholds + +The `IS IT INEFFICIENT?` lines are advisory constants at the top of +`test_efficiency_report.py`: + +| constant | default | meaning | +|---|---|---| +| `DISK_WAIT_DOMINANT` | 0.40 | >40% decode waiting on expert reads → I/O-bound | +| `LOW_HIT_RATE` | 0.30 | <30% cache hit → thrashing | +| `LOW_ROUTE_AGREE` | 0.80 | <80% routing overlap → prefetch guessing wrong | +| `HIGH_TAIL_RATIO` | 3.0 | p99 > 3× p50 → decode stalls | +| `LOW_MTP_ACCEPT` | 0.20 | <20% MTP acceptance → draft decoder is dead weight | + +The tiny-model asserted floors live in `tools/efficiency.py` (`TINY_TOK_S_FLOOR`, +`MAX_DISK_WAIT_SHARE`, `MIN_CPU_CUDA_AGREEMENT`). + +## The regression tests (what gates CI) + +`test_inefficiency.py` runs on the bundled `glm_tiny` model and asserts: + +- telemetry parses (no format drift) +- tiny tok/s ≥ floor (throughput regression) +- PROFILE phases present and non-negative (accounting sanity) +- disk-wait not dominant on a resident model (I/O-path regression) +- CPU determinism (two greedy runs agree) +- **CUDA** (skip unless CUDA built): init path, dense uploads VRAM, CPU-vs-CUDA + argmax agreement ≥ 70% (kernel-correctness guard) + +```bash +make efficiency # tiny CPU tests +make efficiency-cuda # tiny CUDA tests (needs CUDA build) +``` + +## CUDA build prerequisite + +The default `make glm.exe` builds **without** CUDA. The CUDA tests and the CUDA +dossier need a host built with `-DCOLI_CUDA` plus the runtime DLL: + +```bash +make clean && make glm.exe CUDA_DLL=1 && make cuda-dll +``` + +`make efficiency-cuda` auto-skips with a clear message if the host is CPU-only +(it scans the binary for the "CPU-only" marker the engine embeds). + +## Files + +- `tools/efficiency.py` — shared harness: `parse_run()` (captures every + telemetry signal), `run_engine()`, thresholds. Reuses `PROFILE_RE`/`SPEED_RE` + from `tools/benchmark_cuda_fixture.py`. +- `tests/test_inefficiency.py` — tiny-model asserted tests (CPU + CUDA). +- `tests/test_efficiency_report.py` — the opt-in optimization dossier. diff --git a/c/tests/bench_dsa_select.c b/c/tests/bench_dsa_select.c index a1f20149..676e4c27 100644 --- a/c/tests/bench_dsa_select.c +++ b/c/tests/bench_dsa_select.c @@ -24,7 +24,7 @@ * Run: make tests/bench_dsa_select && ./tests/bench_dsa_select (not in TEST_BINS) */ #define main coli_glm_main_unused -#include "../glm.c" +#include "../colibri.c" #undef main #include diff --git a/c/tests/bench_idot.c b/c/tests/bench_idot.c new file mode 100644 index 00000000..37b0b30a --- /dev/null +++ b/c/tests/bench_idot.c @@ -0,0 +1,92 @@ +/* Microbenchmark: old (single-accumulator) vs new (independent-accumulator) AVX-VNNI + * int8/int4 dot kernels (quant.h). NOT a unit test -- test_idot.c proves correctness. + * This measures the headline claim: breaking the serial vpdpbusd->acc chain lifts + * per-core kernel throughput, the same win the NEON path already took ("26->63 GB/s, 2.4x"). + * + * It re-implements the OLD single-acc AVX-VNNI kernels inline and calls the REAL (new) + * ones via the include-colibri.c pattern -- one process, identical frozen inputs, warm + * caches. Reports median ns/call + GB/s-of-weights + new/old ratio. Measures the kernel's + * compute ceiling (warm caches), NOT end-to-end tok/s. + * + * Run: make tests/bench_idot ARCH=native && ./tests/bench_idot (not in TEST_BINS -- not a gate) + */ +#define main coli_glm_main_unused +#include "../colibri.c" +#undef main +#include +#include + +static uint32_t rs=0x2545F491u; +static uint32_t xr(void){ rs^=rs<<13; rs^=rs>>17; rs^=rs<<5; return rs; } + +/* ---- OLD kernels: verbatim copies of the pre-change __AVXVNNI__ branches (single acc) ---- */ +#if defined(__AVXVNNI__) && defined(__AVX2__) +static int32_t dot_i8i8_old(const int8_t *w, const int8_t *x, int I){ + int32_t sum=0; int i=0; + __m128i acc=_mm_setzero_si128(); + for(;i+16<=I;i+=16){ + __m128i wv=_mm_loadu_si128((const __m128i*)(w+i)); + __m128i xv=_mm_loadu_si128((const __m128i*)(x+i)); + __m128i xs=_mm_sign_epi8(xv,wv); + acc=_mm_dpbusd_epi32(acc,_mm_abs_epi8(wv),xs); + } + sum=hsum128_i32(acc); + for(;i>1))); + __m128i lo=_mm_and_si128(by,m4), hi=_mm_and_si128(_mm_srli_epi16(by,4),m4); + __m128i n0=_mm_unpacklo_epi8(lo,hi), n1=_mm_unpackhi_epi8(lo,hi); + __m128i w0=_mm_sub_epi8(n0,b8), w1=_mm_sub_epi8(n1,b8); + __m128i x0=_mm_loadu_si128((const __m128i*)(x+i)); + __m128i x1=_mm_loadu_si128((const __m128i*)(x+i+16)); + acc=_mm_dpbusd_epi32(acc,_mm_abs_epi8(w0),_mm_sign_epi8(x0,w0)); + acc=_mm_dpbusd_epi32(acc,_mm_abs_epi8(w1),_mm_sign_epi8(x1,w1)); + } + sum=hsum128_i32(acc); + for(;i>1]; int v=(i&1)?((int)(b>>4)-8):((int)(b&0xF)-8); sum+=v*x[i]; } + return sum; +} +#else +#error "bench_idot requires an AVX-VNNI build: make tests/bench_idot ARCH=native on an AVX-VNNI CPU" +#endif + +#define I_DIM 6144 +#define N_REPEAT 20000 +static int cmp_d(const void*a,const void*b){ double x=*(const double*)a,y=*(const double*)b; return xy?1:0; } + +int main(void){ + static int8_t w8[I_DIM], x8[I_DIM]; static uint8_t w4[I_DIM/2]; + for(int i=0;i +#include + +static uint32_t rs=0x2545F491u; +static uint32_t xr(void){ rs^=rs<<13; rs^=rs>>17; rs^=rs<<5; return rs; } + +/* ---- OLD: verbatim scalar reductions (pre-#442 colibri.c:2459, 2467) ---- */ +static void score_scalar(const float *qabs, const float *Lt, int kvl, float *out){ + float a=0; + for(int i=0;imax_err)max_err=e; } + printf("correctness: score max abs diff over 5 = %.3e, vmix max abs diff = %.3e\n", + fabsf(sc_old[0]-sc_new[0]), max_err); + printf("(score reassociates -> small diff expected; vmix is lane-wise -> ~0 diff)\n\n"); + + /* bench: H * (nt score calls + nt vmix calls), as in one attention forward */ + int reps=200; + double t0=now_s(); + for(int r=0;r diff --git a/c/tests/fixtures/e8_case.bin b/c/tests/fixtures/e8_case.bin new file mode 100644 index 00000000..aa9573df Binary files /dev/null and b/c/tests/fixtures/e8_case.bin differ diff --git a/c/tests/test_backend_cuda.cu b/c/tests/test_backend_cuda.cu index 5550bdef..1a826b8f 100644 --- a/c/tests/test_backend_cuda.cu +++ b/c/tests/test_backend_cuda.cu @@ -49,11 +49,57 @@ int main(int argc, char **argv) { if (!coli_cuda_tensor_upload(&t8, q8, s8, 1, 4, 2, d0)) return 1; if (coli_cuda_tensor_upload(&t8, q8, s8, 1, 5, 2, d0)) return 1; if (ndev > 1 && coli_cuda_tensor_upload(&t8, q8, s8, 1, 4, 2, d1)) return 1; - if (!coli_cuda_matmul(&t8, got, x, q8, s8, 1, 2, 4, 2, d0) || !close_enough(got, want8, 4)) return 1; + if (!coli_cuda_matmul(&t8, got, x, q8, s8, 1, 2, 4, 2, d0, 0) || !close_enough(got, want8, 4)) return 1; + /* Cached tensor must stay callable without live host pointers + * (CUDA_RELEASE_HOST slots null theirs after upload) — including + * SUSTAINED reuse, not just the first call. */ + for (int rep = 0; rep < 64; rep++) + if (!coli_cuda_matmul(&t8, got, x, nullptr, nullptr, 1, 2, 4, 2, d0, 0) || + !close_enough(got, want8, 4)) return 1; + /* A tensor uploaded from a TEMPORARY host buffer must survive the buffer + * being scribbled and freed (the release-host lifecycle). */ + { + int8_t *tmpw = static_cast(std::malloc(8)); + float *tmps = static_cast(std::malloc(2 * sizeof(float))); + if (!tmpw || !tmps) return 2; + for (int i = 0; i < 8; i++) tmpw[i] = q8[i]; + tmps[0] = s8[0]; tmps[1] = s8[1]; + ColiCudaTensor *tt = nullptr; + if (!coli_cuda_tensor_upload(&tt, tmpw, tmps, 1, 4, 2, d0)) return 1; + for (int i = 0; i < 8; i++) tmpw[i] = 99; + std::free(tmpw); std::free(tmps); + if (!coli_cuda_matmul(&tt, got, x, nullptr, nullptr, 1, 2, 4, 2, d0, 0) || + !close_enough(got, want8, 4)) return 1; + coli_cuda_tensor_free(tt); + } + /* Upload failures must be graceful and must not corrupt accounting — + * and must not poison LATER healthy launches (sticky-error regression). */ + { + size_t c0 = 0, b0 = 0, c1 = 0, b1 = 0; + coli_cuda_stats(-1, &c0, &b0); + ColiCudaTensor *bad = nullptr; + if (coli_cuda_tensor_upload(&bad, q8, s8, 1, 4, 2, 9999)) return 1; + if (coli_cuda_tensor_upload(&bad, q8, s8, 7, 4, 2, d0)) return 1; + if (coli_cuda_tensor_upload(&bad, q8, nullptr, 1, 4, 2, d0)) return 1; + if (coli_cuda_tensor_upload(&bad, nullptr, s8, 1, 4, 2, d0)) return 1; + if (coli_cuda_tensor_upload(&bad, q8, s8, 1, 1 << 20, 1 << 24, d0)) return 1; /* ~16 TB */ + if (bad) return 1; + coli_cuda_stats(-1, &c1, &b1); + if (c0 != c1 || b0 != b1) return 1; + /* healthy launch immediately after the failed allocation */ + if (!coli_cuda_matmul(&t8, got, x, nullptr, nullptr, 1, 2, 4, 2, d0, 0) || + !close_enough(got, want8, 4)) return 1; + } + /* Fault injection hook: on/off, restores cleanly. */ + if (setenv("COLI_GPU_FAIL_AFTER", "0", 1)) return 2; + if (coli_cuda_matmul(&t8, got, x, nullptr, nullptr, 1, 2, 4, 2, d0, 0)) return 1; + if (unsetenv("COLI_GPU_FAIL_AFTER")) return 2; + if (!coli_cuda_matmul(&t8, got, x, nullptr, nullptr, 1, 2, 4, 2, d0, 0) || + !close_enough(got, want8, 4)) return 1; const int8_t q8b[8]={-1,-2,-3,-4, 1,-2,3,-4}; const float s8b[2]={1.f,.5f},want8b[4]={10.f,15.f,-3.f,-2.5f}; if(!coli_cuda_tensor_update(t8,q8b,s8b)|| - !coli_cuda_matmul(&t8,got,x,q8b,s8b,1,2,4,2,d0)|| + !coli_cuda_matmul(&t8,got,x,q8b,s8b,1,2,4,2,d0,0)|| !close_enough(got,want8b,4))return 1; /* Rows [-8,-1,0,7] and [1,2,3,4], packed low nibble first. */ @@ -61,26 +107,26 @@ int main(int argc, char **argv) { const float s4[2] = {1.0f, 0.25f}; const float want4[2] = {-34.0f, -2.5f}; ColiCudaTensor *t4 = nullptr; - if (!coli_cuda_matmul(&t4, got, x, q4, s4, 2, 1, 4, 2, d1) || !close_enough(got, want4, 2)) return 1; + if (!coli_cuda_matmul(&t4, got, x, q4, s4, 2, 1, 4, 2, d1, 0) || !close_enough(got, want4, 2)) return 1; const uint8_t q2[2] = {0xe4, 0x1b}; const float s2[2] = {0.5f, 2.0f}; const float want2[2] = {-2.0f, 12.0f}; ColiCudaTensor *t2 = nullptr; - if (!coli_cuda_matmul(&t2, got, x, q2, s2, 3, 1, 4, 2, d1) || !close_enough(got, want2, 2)) return 1; + if (!coli_cuda_matmul(&t2, got, x, q2, s2, 3, 1, 4, 2, d1, 0) || !close_enough(got, want2, 2)) return 1; const float wf[8] = {1, 0, -1, 2, 0.5f, 0.5f, 0.5f, 0.5f}; const float wantf[2] = {-10.0f, -1.0f}; ColiCudaTensor *tf = nullptr; - if (!coli_cuda_matmul(&tf, got, x, wf, nullptr, 0, 1, 4, 2, d0) || !close_enough(got, wantf, 2)) return 1; + if (!coli_cuda_matmul(&tf, got, x, wf, nullptr, 0, 1, 4, 2, d0, 0) || !close_enough(got, wantf, 2)) return 1; const float eg[8] = {1,0,0,0, 0,1,0,0}; const float eu[8] = {1,0,0,0, 0,1,0,0}; const float ed[8] = {1,0, 0,1, 1,1, 1,-1}; ColiCudaTensor *tg=nullptr,*tu=nullptr,*td=nullptr; - if (!coli_cuda_tensor_upload(&tg,eg,nullptr,0,4,2,d0) || - !coli_cuda_tensor_upload(&tu,eu,nullptr,0,4,2,d0) || - !coli_cuda_tensor_upload(&td,ed,nullptr,0,2,4,d0)) return 1; + if (!coli_cuda_tensor_upload_g(&tg,eg,nullptr,0,4,2,d0,0) || + !coli_cuda_tensor_upload_g(&tu,eu,nullptr,0,4,2,d0,0) || + !coli_cuda_tensor_upload_g(&td,ed,nullptr,0,2,4,d0,0)) return 1; float expert[8], want_expert[8]; for(int s=0;s<2;s++){ float a=x[s*4], b=x[s*4+1]; @@ -98,7 +144,7 @@ int main(int argc, char **argv) { const float aw[16]={1,0,0,0, 0,1,0,0, 0,0,1,0, 0,0,0,1}; const float aq[4]={1,2,.5f,-.5f},al[12]={1,0,0,0, 0,1,0,0, 0,0,1,0}; const float ar[6]={1,0, 0,1, 1,1};float actx[2],aref[2]; - ColiCudaTensor *at=nullptr;if(!coli_cuda_tensor_upload(&at,aw,nullptr,0,4,4,d0))return 1; + ColiCudaTensor *at=nullptr;if(!coli_cuda_tensor_upload_g(&at,aw,nullptr,0,4,4,d0,0))return 1; float score[3];for(int t=0;t<3;t++)score[t]=aq[0]*al[t*4]+aq[1]*al[t*4+1]+aq[2]*ar[t*2]+aq[3]*ar[t*2+1]; float mx=score[0],z=0;for(int t=1;t<3;t++)mx=score[t]>mx?score[t]:mx; for(int t=0;t<3;t++){score[t]=std::exp(score[t]-mx);z+=score[t];}for(int t=0;t<3;t++)score[t]/=z; @@ -117,9 +163,9 @@ int main(int argc, char **argv) { for(int i=0;i<32;i++)ws4[i]=0.01f+(i%5)*0.002f; for(int i=0;i<64;i++)gx4[i]=std::sin((float)(i+1)*0.17f)*2.f; ColiCudaTensor *g4=nullptr,*u4=nullptr,*d4=nullptr; - if(!coli_cuda_tensor_upload(&g4,w4,ws4,2,32,32,d0)|| - !coli_cuda_tensor_upload(&u4,w4,ws4,2,32,32,d0)|| - !coli_cuda_tensor_upload(&d4,w4,ws4,2,32,32,d0))return 1; + if(!coli_cuda_tensor_upload_g(&g4,w4,ws4,2,32,32,d0,0)|| + !coli_cuda_tensor_upload_g(&u4,w4,ws4,2,32,32,d0,0)|| + !coli_cuda_tensor_upload_g(&d4,w4,ws4,2,32,32,d0,0))return 1; ColiCudaTensor *gg4[2]={g4,g4},*ug4[2]={u4,u4},*dg4[2]={d4,d4}; if(!coli_cuda_expert_group(gg4,ug4,dg4,group_rows,2,scalar4,gx4))return 1; setenv("COLI_CUDA_TC_INT4","1",1); diff --git a/c/tests/test_backend_metal.mm b/c/tests/test_backend_metal.mm index ffdd39be..bff0444b 100644 --- a/c/tests/test_backend_metal.mm +++ b/c/tests/test_backend_metal.mm @@ -177,6 +177,68 @@ static int run_attn(int S, int pos_base, const char* name){ return pass?0:1; } +// serial r_top8 vs parallel r_top8_par on the ENGINE build's own compiled shaders — the +// exact-match contract (same indices, same order, same weights bitwise, same keff) +// enforced with memcmp, per adversarial input family. `mode` selects the input +// construction; see the inventory at the call sites in main(). E is a parameter (not +// hardcoded 256) so the same helper drives both the original E=256 fuzz and the +// expert-count-generality cases (E=24 <32-lane-width, E=168 REAP-pruned, E=200 +// lane-straddling boundary, E=257 out-of-contract auto-serial-fallback proof). +static int run_rtop8(int mode, int S, int E, float topp, int normk, float rscale, const char *name) { + const int K=8, Ksel=8; + std::vector sig((size_t)S*E), bias(E); + srand(4242+mode*17+S+E); + for (int e=0;e=E-4)?1.0f:(float)(rand()%10000)/10000.f; break; + default: *v=(float)(rand()%10000)/10000.f; break; + } + } + if (mode==1) for (int e=0;e is((size_t)S*K), ip((size_t)S*K); std::vector ws((size_t)S*K), wp((size_t)S*K); + std::vector ks(S), kp(S); + if (!coli_metal_rtop8(0,sig.data(),bias.data(),S,E,K,Ksel,topp,normk,rscale,is.data(),ws.data(),ks.data()) || + !coli_metal_rtop8(1,sig.data(),bias.data(),S,E,K,Ksel,topp,normk,rscale,ip.data(),wp.data(),kp.data())) { + printf(" %-34s FAIL (rtop8 runner returned 0)\n", name); return 1; } + int ok = memcmp(is.data(),ip.data(),(size_t)S*K*4)==0 && + memcmp(ws.data(),wp.data(),(size_t)S*K*4)==0 && // bitwise: same ops, same order + memcmp(ks.data(),kp.data(),(size_t)S*4)==0; + if (mode==5 && ok) { + // Don't just trust the input design -- confirm the straddling lane's valid segment + // (E-4..E-1) was actually selected, in EVERY row, so this case can't silently + // degrade into an unrelated pass if the input construction above ever changes. + for (int s=0;s=E-4 && ip[(size_t)s*K+k]256, auto-serial-fallback)"); printf(fail? "metal backend tests: FAILED\n" : "metal backend tests: ok\n"); coli_metal_shutdown(); return fail; diff --git a/c/tests/test_cuda_env.py b/c/tests/test_cuda_env.py new file mode 100644 index 00000000..3cfbf427 --- /dev/null +++ b/c/tests/test_cuda_env.py @@ -0,0 +1,249 @@ +"""Integration tests for COLI_CUDA auto-enable and CUDA_EXPERT_GB auto-size. + +These tests run the compiled ``glm`` binary (via ``coli run``) and verify +the three COLI_CUDA modes (unset / auto-detect, 0 / forced-CPU, 1 / hard-fail) +and the CUDA_EXPERT_GB auto-size behavior. + +Prerequisites (tests skip gracefully when unmet): +- Compiled ``coli`` / ``glm`` binary (any build — CUDA or CPU-only) +- nvidia-smi (for GPU-specific tests) +""" + +import json +import os +import struct +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + + +HERE = Path(__file__).resolve().parent.parent +GLM = HERE / ("glm.exe" if sys.platform == "win32" else "glm") + + +def _binary_has_cuda(): + """Check whether the ``glm`` binary was compiled with CUDA support.""" + if not GLM.exists(): + return False + try: + result = subprocess.run( + [str(GLM), "1"], + env={**os.environ, "COLI_CUDA": "1", + "SNAP": str(HERE)}, + cwd=str(HERE), text=True, capture_output=True, timeout=10, + ) + return "CPU-only" not in (result.stderr or "") + except (OSError, subprocess.SubprocessError): + return False + + +def _gpu_available(): + """Return True if at least one CUDA-capable GPU is visible to nvidia-smi.""" + try: + result = subprocess.run( + ["nvidia-smi", "--query-gpu=name", "--format=csv,noheader"], + text=True, capture_output=True, timeout=5, + ) + return result.returncode == 0 and bool(result.stdout.strip()) + except (OSError, subprocess.SubprocessError): + return False + + +_HAS_GLM = GLM.exists() +_HAS_CUDA_BINARY = _HAS_GLM and _binary_has_cuda() +_HAS_GPU = _gpu_available() + + +def _write_shard(path, tensors): + """Write a minimal safetensors file to *path*.""" + offset = 0 + header = {} + payload = b"" + for name, size in tensors: + header[name] = {"dtype": "U8", "shape": [size], + "data_offsets": [offset, offset + size]} + payload += b"\0" * size + offset += size + raw = json.dumps(header).encode() + path.write_bytes(struct.pack(" diff --git a/c/tests/test_e8_kernel.c b/c/tests/test_e8_kernel.c new file mode 100644 index 00000000..9d7b15d0 --- /dev/null +++ b/c/tests/test_e8_kernel.c @@ -0,0 +1,127 @@ +/* fmt=6 (E8/IQ3 lattice) CPU kernel oracle — #452 ladder step 3. + * + * tools/iq3_pack.py is the format's reference implementation; this checks that + * matmul_e8 reads the same bytes to the same numbers. The fixture is generated + * by that codec (tests/fixtures/e8_case.bin, written by tools/make_e8_fixture.py) + * and carries, for one [O,I] tensor: the packed bytes, the codec's own dequant of + * them, an activation row, and the reference y = dequant @ x computed in float64. + * + * Two properties are checked: + * 1. dequant agreement — expanding each sub-block must reproduce the codec's + * floats bit-for-bit (same grid, same parity rule, same scale arithmetic) + * 2. matmul agreement — y within float rounding of the float64 reference + * + * Build: cc -O2 -fopenmp tests/test_e8_kernel.c e8_grid.c -o tests/test_e8_kernel -lm + */ +#include +#include +#include +#include +#include + +#include "../quant.h" + +static void *slurp(const char *path, size_t *n) { + FILE *f = fopen(path, "rb"); + if (!f) { fprintf(stderr, "cannot open %s — run tools/make_e8_fixture.py\n", path); exit(77); } + fseek(f, 0, SEEK_END); long sz = ftell(f); fseek(f, 0, SEEK_SET); + void *p = malloc((size_t)sz); + if (fread(p, 1, (size_t)sz, f) != (size_t)sz) { fprintf(stderr, "short read\n"); exit(1); } + fclose(f); *n = (size_t)sz; return p; +} + +int main(void) { + size_t nbytes; + uint8_t *raw = slurp("tests/fixtures/e8_case.bin", &nbytes); + /* header: int32 O, I, then packed[O*rowbytes], deq[O*I], x[I], yref[O] */ + int32_t O, I; + memcpy(&O, raw, 4); memcpy(&I, raw + 4, 4); + size_t rb = (size_t)e8_rowbytes(I); + const uint8_t *packed = raw + 8; + const float *deq = (const float *)(packed + (size_t)O * rb); + const float *x = deq + (size_t)O * I; + const float *yref = x + I; + + /* 1. dequant agreement, sub-block by sub-block */ + int mism = 0; + float w[E8_SUB]; + for (int o = 0; o < O && mism < 5; o++) { + const uint8_t *row = packed + (size_t)o * rb; + for (int64_t b = 0; b < e8_blocks(I); b++) { + const uint8_t *blk = row + b * E8_BBYTES; + uint16_t dh; memcpy(&dh, blk + 96, 2); + float d = e8_fp16_to_f32(dh); + for (int ib = 0; ib < E8_QK / E8_SUB; ib++) { + int off = (int)(b * E8_QK) + ib * E8_SUB; + if (off >= I) break; + e8_expand_sub(blk, ib, d, w); + int n = I - off < E8_SUB ? I - off : E8_SUB; + for (int k = 0; k < n; k++) { + float got = w[k], want = deq[(size_t)o * I + off + k]; + if (fabsf(got - want) > 1e-6f * (fabsf(want) + 1e-6f)) { + if (mism < 5) + fprintf(stderr, "dequant mismatch o=%d i=%d: got %.9g want %.9g\n", + o, off + k, got, want); + mism++; + } + } + } + } + } + if (mism) { printf("FAIL: %d dequant mismatches\n", mism); return 1; } + + /* 2. matmul agreement */ + float *y = malloc((size_t)O * sizeof(float)); + matmul_e8(y, x, packed, NULL, 1, I, O); + double worst = 0; + for (int o = 0; o < O; o++) { + double d = fabs((double)y[o] - (double)yref[o]); + double rel = d / (fabs((double)yref[o]) + 1e-6); + if (rel > worst) worst = rel; + } + printf("e8 kernel oracle: O=%d I=%d, dequant exact, matmul worst rel %.2e\n", O, I, worst); + if (worst > 1e-5) { printf("FAIL\n"); return 1; } + + /* 3. rotation agreement (converter step, #452): the fixture carries a + * rotated-weights section — packed W@Q from the Python side, the raw + * activation, Python's Q^T x, and the float64 reference product. The C + * side must reproduce the rotation (regenerated signs + block tiling + + * FWHT) and then the matmul, or every rotated container decodes wrong. */ + const uint8_t *sec = (const uint8_t *)(yref + O); + if ((size_t)(sec - raw) < nbytes) { + int32_t O2, I2; + memcpy(&O2, sec, 4); memcpy(&I2, sec + 4, 4); + size_t rb2 = (size_t)e8_rowbytes(I2); + const uint8_t *packed2 = sec + 8; + const float *x2 = (const float *)(packed2 + (size_t)O2 * rb2); + const float *xrot2 = x2 + I2; + const float *y2ref = xrot2 + I2; + float *xr = malloc((size_t)I2 * sizeof(float)); + memcpy(xr, x2, (size_t)I2 * sizeof(float)); + e8_rot_rows(xr, 1, I2); + double wrot = 0; + for (int i = 0; i < I2; i++) { + double d = fabs((double)xr[i] - (double)xrot2[i]); + double rel = d / (fabs((double)xrot2[i]) + 1e-6); + if (rel > wrot) wrot = rel; + } + float *y2 = malloc((size_t)O2 * sizeof(float)); + matmul_e8(y2, xr, packed2, NULL, 1, I2, O2); + double wmm = 0; + for (int o = 0; o < O2; o++) { + double d = fabs((double)y2[o] - (double)y2ref[o]); + double rel = d / (fabs((double)y2ref[o]) + 1e-6); + if (rel > wmm) wmm = rel; + } + printf("e8 rotation oracle: O2=%d I2=%d, rot worst rel %.2e, matmul worst rel %.2e\n", + O2, I2, wrot, wmm); + if (wrot > 1e-5 || wmm > 1e-5) { printf("FAIL\n"); return 1; } + free(xr); free(y2); + } else { + fprintf(stderr, "fixture has no rotated section — regenerate with tools/make_e8_fixture.py\n"); + return 1; + } + printf("OK\n"); + free(y); free(raw); + return 0; +} diff --git a/c/tests/test_efficiency_report.py b/c/tests/test_efficiency_report.py new file mode 100644 index 00000000..ba01b131 --- /dev/null +++ b/c/tests/test_efficiency_report.py @@ -0,0 +1,315 @@ +#!/usr/bin/env python3 +"""Exhaustive optimization dossier for a colibri engine run. + +This is NOT a pass/fail test. It runs the engine with every instrumentation flag +on (PROF, COLI_CUDA_PROFILE, CACHE_ROUTE, DISK_SPLIT, LOOKA) and prints a section- +by-section report answering, for each subsystem: + + WHAT is doing it — which phase/kernel/tier + WHEN it is doing it — how much of decode wall-time it owns + WITH WHAT — the config/weights/tier it used + IS IT INEFFICIENT? — a verdict, with the threshold + HOW TO IMPROVE — the concrete knob, named + +Activation (opt-in only — NOT in `make test`): + COLI_EFFICIENCY_MODEL= python tests/test_efficiency_report.py + +Optional env: + COLI_EFFICIENCY_CUDA=1 also exercise the CUDA dense/expert tiers + COLI_EFFICIENCY_NGEN=N decode tokens (default 24) + COLI_EFFICIENCY_RAM_GB=N RAM budget (default 28) + COLI_EFFICIENCY_VRAM_GB=N CUDA expert-tier budget GB (default 4) + COLI_EFFICIENCY_PROMPT=... prompt (default: a code-gen prompt) + +Exit code is always 0 (it's a dossier, not a gate). Lines marked FLAG point at +the most likely lever to move tok/s for the observed bottleneck. +""" +import os +import sys +import time +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +from tools.efficiency import run_engine, disk_wait_share # noqa: E402 + + +# --- advisory thresholds (the "IS IT INEFFICIENT?" lines) --- +DISK_WAIT_DOMINANT = 0.40 # >40% decode waiting on expert reads -> I/O-bound +LOW_HIT_RATE = 0.30 # <30% cache hit -> thrashing (cap too small) +LOW_ROUTE_AGREE = 0.80 # <80% routing overlap -> prefetch is guessing wrong +HIGH_TAIL_RATIO = 3.0 # p99 > 3x p50 -> decode stalls (I/O hiccups / KV grow) +LOW_MTP_ACCEPT = 0.20 # <20% MTP acceptance -> draft decoder is dead weight +VRAM_WASTE_CALLS = 0 # experts pinned in VRAM but 0 calls served + + +def _flag(ok): return "OK " if ok else "FLAG" + + +def _bar(frac, width=24): + """A simple ASCII bar for share visualization.""" + n = max(0, min(width, round(frac * width))) + return "#" * n + "." * (width - n) + + +def _line(label, value, flag=None, note=""): + tag = f" [{flag}]" if flag else "" + print(f" {label:<22} {value}{tag} {note}" if note else f" {label:<22} {value}{tag}") + + +def main() -> int: + model = os.environ.get("COLI_EFFICIENCY_MODEL") + if not model: + print(__doc__) + print("\nNot activated: set COLI_EFFICIENCY_MODEL= to run.") + return 0 + model = str(Path(model).resolve()) + if not Path(model).is_dir(): + print(f"ERROR: {model} is not a directory", file=sys.stderr) + return 0 + + ngen = int(os.environ.get("COLI_EFFICIENCY_NGEN", "24")) + ram_gb = os.environ.get("COLI_EFFICIENCY_RAM_GB", "28") + vram_gb = os.environ.get("COLI_EFFICIENCY_VRAM_GB", "4") + prompt = os.environ.get( + "COLI_EFFICIENCY_PROMPT", + "Write a Python function that computes the factorial of a number. " + "Include error handling and a docstring.") + use_cuda = os.environ.get("COLI_EFFICIENCY_CUDA") == "1" + + # Turn ON every instrumentation flag so the dossier has maximum detail. + # These are all observability toggles (PROF/COLI_CUDA_PROFILE/CACHE_ROUTE/ + # DISK_SPLIT/LOOKA); none change the computed output. + overlay = dict( + NGEN=str(ngen), TEMP="0", RAM_GB=ram_gb, PROMPT=prompt, + PROF="1", CACHE_ROUTE="1", DISK_SPLIT="1", LOOKA="1", ROUTE_AGREE="1", + ) + if use_cuda: + overlay.update(COLI_CUDA="1", COLI_GPU="0", CUDA_DENSE="1", + COLI_CUDA_PROFILE="1", CUDA_EXPERT_GB=vram_gb) + + print("=" * 78) + print(f"OPTIMIZATION DOSSIER — {Path(model).name}") + print(f" mode : {'CUDA (dense+expert tiers)' if use_cuda else 'CPU-only'} " + f"ngen : {ngen} ram : {ram_gb} GB" + + (f" vram : {vram_gb} GB" if use_cuda else "")) + print("=" * 78) + + t0 = time.time() + t, proc = run_engine(overlay, snap=model, timeout=3600.0) + wall = time.time() - t0 + flags = [] # collected FLAG lines for the summary + + print(f"\n[0] RUN") + _line("wall clock", f"{wall:.0f}s") + _line("exit code", proc.returncode, + None if proc.returncode == 0 else "FLAG", + "" if proc.returncode == 0 else "non-zero exit") + if proc.returncode != 0: + print(" stderr tail:") + for ln in proc.stderr.strip().splitlines()[-8:]: + print(f" {ln}") + return 0 + + # ---------------------------------------------------------------- [1] WHO ---- + print(f"\n[1] PROVENANCE — what is running, on what, with what config") + if t.get("machine"): + m = t["machine"] + _line("CPU", m["cpu"]) + _line("cores / omp", f"{m['cores']} cores") + _line("backend", m["backend"]) + if t.get("load"): + ld = t["load"] + _line("model load time", f"{ld['load_s']:.2f}s") + _line("resident dense", f"{ld['resident_dense_mb']:.1f} MB") + _line("layers / experts", f"{ld['layers']} layers, {ld['experts']} experts") + _line("MTP", f"{ld['mtp_status']} (draft={ld['draft']})") + if t.get("config_str"): + _line("resolved config", t["config_str"]) + print(" (this is the EFFECTIVE config after auto-budgeting — not your env verbatim)") + + # ---------------------------------------------------------------- [2] SPEED -- + print(f"\n[2] THROUGHPUT — is it fast, is the tail healthy") + if t.get("tok_s") is not None: + _line("tok/s", f"{t['tok_s']:.3f}") + else: + flags.append("throughput line missing — engine output format may have changed") + if t.get("latency"): + la = t["latency"] + _line("decode forwards", f"{int(la['forwards'])}") + _line("p50 / p90", f"{la['p50_ms']:.1f} / {la['p90_ms']:.1f} ms") + _line("p99 / max", f"{la['p99_ms']:.1f} / {la['max_ms']:.1f} ms") + tail_ok = la["p99_ms"] <= HIGH_TAIL_RATIO * la["p50_ms"] + _line("tail ratio (p99/p50)", f"{la['p99_ms']/max(la['p50_ms'],1e-9):.2f}x", + _flag(tail_ok), + "high tail = decode stalls (I/O hiccups, KV growth, re-pin)") + if not tail_ok: + flags.append(f"tail latency p99={la['p99_ms']:.1f}ms >> p50={la['p50_ms']:.1f}ms " + "(look for REPIN swaps or disk stalls)") + + # ---------------------------------------------------------------- [3] TIME --- + print(f"\n[3] WHERE TIME GOES — what is doing it, when (share of decode)") + ts = t.get("time_shares") + prof = t.get("profile") + if ts: + order = [("io", "expert-disk I/O", DISK_WAIT_DOMINANT, "the cache is too small / disk is slow"), + ("matmul", "expert matmul", 0.40, "compute-bound; more cores or a GPU expert tier"), + ("attention", "attention", 0.35, "context length is the cost; lower CTX"), + ("head", "lm_head", 0.10, "vocab projection; unusual to dominate"), + ("other", "other", 0.30, "scheduling / KV bookkeeping overhead")] + for key, name, thresh, lever in order: + f = ts[key] + ok = f < thresh + _line(name, f"{f:5.0%} {_bar(f)}", _flag(ok), + "" if ok else f"->{lever}") + if not ok: + flags.append(f"{name} dominates ({f:.0%}) -> {lever}") + if t.get("verdict"): + print(f" engine verdict : {t['verdict']}") + elif prof: + print(" (no [PROF] time shares — set PROF=1 for phase percentages)") + if prof: + print(" absolute seconds :") + for k in ("disk", "expert_matmul", "attention", "lm_head", "other"): + _line(k, f"{prof[k]:.3f}s") + + # attention sub-breakdown: how is attention being read + ab = t.get("attn_breakdown") + if ab: + print(f"\n[3a] ATTENTION BREAKDOWN — how the attention phase is spent") + atot = sum(ab.values()) or 1.0 + for k, label in (("proj_rope", "projection + RoPE"), + ("score_sm_value", "score-softmax-value"), + ("out_proj", "output projection")): + _line(label, f"{ab[k]:.3f}s ({ab[k]/atot:.0%} of attn)") + + # ---------------------------------------------------------------- [4] CACHE -- + print(f"\n[4] EXPERT CACHE — is the cache efficient") + hit = t.get("hit_pct") + if hit is not None: + ok = hit >= LOW_HIT_RATE * 100 + _line("hit rate", f"{hit:.1f}%", _flag(ok), + "" if ok else "<30% = thrashing; raise RAM_GB or cap") + if not ok: + flags.append(f"cache hit {hit:.1f}% is low -> raise RAM_GB (or cap), add PIN_GB") + el = t.get("experts_loaded") + if el: + per_tok = el["per_tok"] + _line("experts loaded/token", f"{per_tok:.1f}") + _line(" per-layer", f"{el['per_layer']:.2f} across {el['n_sparse_layers']} sparse layers") + _line(" baseline", f"topk={el['baseline_topk']} active experts/token") + base_topk = el["baseline_topk"] + if base_topk > 0 and per_tok > 2 * base_topk: + flags.append(f"loading {per_tok:.0f} experts/token vs topk={base_topk} " + "-> redundant I/O; cache is re-fetching evicted experts") + + # ---------------------------------------------------------------- [5] DISK --- + print(f"\n[5] DISK I/O — is I/O the bottleneck, and where") + eio = t.get("expert_io") + if eio: + _line("total fetched", f"{eio['gb_fetched']:.3f} GB") + _line("per token", f"{eio['mb_per_tok']:.1f} MB/token") + _line("disk throughput", f"{eio['gb_per_s']:.2f} GB/s over the run") + _line("read service", f"{eio['read_service_s']:.2f}s (on I/O threads)") + _line("felt wait", f"{eio['felt_wait_s']:.2f}s (stall compute felt)") + if eio["felt_wait_s"] > eio["read_service_s"] * 0.5 and eio["read_service_s"] > 0: + flags.append("felt wait is a large fraction of read service -> PIPE=1 may not be " + "overlapping fully, or DIRECT=1 on NVMe") + ds = t.get("disk_split") + if ds: + print(f"\n[5a] DISK-LOAD SPLIT — which decode phase reads the bytes") + _line("draft phase", f"{ds['draft']} loads") + _line("absorb phase", f"{ds['absorb']} loads") + _line("verify/main", f"{ds['verify_main']} loads") + _line("MTP-layer bytes", f"{ds['mtp_loads']} loads, {ds['mtp_gb']:.2f} GB") + _line("main-layer bytes", f"{ds['main_loads']} loads, {ds['main_gb']:.2f} GB") + if ds.get("mtp_bytes_pct") is not None: + _line("MTP share of bytes", f"{ds['mtp_bytes_pct']:.1f}%") + share = disk_wait_share(t) + if share is not None: + ok = share < DISK_WAIT_DOMINANT + _line("disk-wait share", f"{share:.0%}", _flag(ok), + "" if ok else "I/O-bound (see levers in [3])") + + # ---------------------------------------------------------------- [6] ROUTE -- + print(f"\n[6] ROUTING QUALITY — is the router / prefetch accurate") + ra = t.get("route_agree") + if ra: + ok = ra["agree_pct"] >= LOW_ROUTE_AGREE * 100 + _line("route_agree", f"{ra['agree_pct']:.1f}% overlap with true top-K", + _flag(ok), + "" if ok else "prefetch is guessing wrong; CACHE_ROUTE params may need tuning") + _line("route_kl", f"{ra['kl']:.4f} mean KL (lower = closer to true routing)") + if not ok: + flags.append(f"route_agree {ra['agree_pct']:.1f}% low -> tune ROUTE_J/M/P, " + "or prefetch is hurting more than helping") + sw = t.get("swap") + if sw: + _line("cache swaps", f"{sw['swaps']}/{sw['slots']} ({sw['pct']:.1f}%)", + None, "high swap = churn between turns") + la = t.get("lookahead") + if la: + print(f"\n[6a] ROUTING PREDICTABILITY — recall of true experts in predicted top-8") + print(" (which predictor should drive prefetch? highest recall wins)") + for row in la: + _line(row["predictor"][:34], f"{row['pct']:5.1f}% ({row['hit']}/{row['tot']})") + + # ---------------------------------------------------------------- [7] SPEC --- + print(f"\n[7] SPECULATION — is the draft decoder pulling weight") + sp = t.get("speculation") + if sp: + _line("tokens/forward", f"{sp['tok_per_fw']:.2f} (>1.0 means speculation helps)") + _line("forwards/tokens", f"{sp['forwards']} forwards for {sp['tokens']} tokens") + # Speculation helps only if acceptance is high enough that tok/forward > 1. + # tok_per_fw already == 1.0 when nothing verifies, so judge by acceptance. + ok = sp["mtp_accept_pct"] >= LOW_MTP_ACCEPT * 100 + _line("MTP acceptance", f"{sp['mtp_accept_pct']:.0f}%", _flag(ok), + "" if ok else "<20% -> drafts rarely verify; DRAFT=0 may be faster") + if not ok: + flags.append(f"MTP acceptance {sp['mtp_accept_pct']:.0f}% low -> " + "drafts cost more I/O than they save; try DRAFT=0") + + # ---------------------------------------------------------------- [8] GPU ---- + print(f"\n[8] GPU TIERS — is the GPU actually used") + cuda = t.get("cuda") or {} + if not cuda.get("enabled"): + print(" (CUDA not enabled — CPU-only run)") + else: + if cuda.get("resident_tensors") is not None: + _line("resident dense tensors", f"{cuda['resident_tensors']} tensors, " + f"{cuda['resident_gb']:.2f} GB") + if cuda.get("expert_count") is not None: + waste = cuda["calls_served"] <= VRAM_WASTE_CALLS + _line("expert tier", f"{cuda['expert_count']} experts pinned " + f"({cuda['expert_gb']:.2f} GB)", _flag(not waste)) + _line(" calls served", f"{cuda['calls_served']} from VRAM", + _flag(not waste), + "" if not waste else "WASTE: pinned but never routed -> lower CUDA_EXPERT_GB") + if waste: + flags.append("VRAM expert tier has 0 calls served -> experts pinned but unused; " + "PIN stats may not match this workload") + if cuda.get("groups"): + g = cuda["groups"] + _line("expert groups", f"{g['calls']} calls, {g['experts']} experts, " + f"{g['rows']} rows ({g['experts_per_call']:.1f} experts/call)") + if cuda.get("groups_timing"): + gt = cuda["groups_timing"] + _line("GPU timing", f"H2D {gt['h2d_ms']:.1f} ms | kernel {gt['kernel_ms']:.1f} ms | " + f"D2H {gt['d2h_ms']:.1f} ms") + if gt["h2d_ms"] + gt["d2h_ms"] > gt["kernel_ms"]: + flags.append("CUDA H2D+D2H > kernel time -> transfer-bound; " + "consider larger expert tier to keep weights resident") + + # ---------------------------------------------------------------- summary ---- + print("\n" + "=" * 78) + if flags: + print(f" {len(flags)} FLAG(s) — the most likely levers to move tok/s:") + for f in flags: + print(f" - {f}") + else: + print(" no flags — every measured subsystem is within advisory thresholds.") + print("=" * 78) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/c/tests/test_env_defaults.py b/c/tests/test_env_defaults.py index 12a3c999..3bb08fc8 100644 --- a/c/tests/test_env_defaults.py +++ b/c/tests/test_env_defaults.py @@ -32,9 +32,16 @@ def args(**over): class EnvDefaultsTest(unittest.TestCase): - def env_for_with(self, environ, platform): + def env_for_with(self, environ, platform, cuda=False): + """Run env_for on a bare-chat args() under a faked env + platform. + + cuda=False by default so the existing default-I/O tests stay + deterministic: the Windows auto-enable branch calls cuda_binary() and + (if True) discover_gpus(), both of which reach the real machine — faking + False keeps these tests independent of the host's GPU.""" with mock.patch.dict(os.environ, environ, clear=True), \ - mock.patch.object(sys, "platform", platform): + mock.patch.object(sys, "platform", platform), \ + mock.patch.object(coli, "cuda_binary", return_value=cuda): return coli.env_for(args()) def test_win32_sets_measured_defaults(self): @@ -64,5 +71,80 @@ def test_non_windows_untouched(self): self.assertNotIn(k, e) +class CudaAutoEnableTest(unittest.TestCase): + """Windows bare `coli chat` (no --gpu/--vram/--auto-tier) used to ALWAYS run + CPU-only even on a CUDA build with a GPU present. env_for now auto-enables + CUDA on win32 when cuda_binary() is True and a GPU is discoverable; falls + back to CPU with a warning if nvidia-smi (discover_gpus) is missing; stays + silent on a CPU build; and never touches the Linux path.""" + + def _env_for(self, platform, cuda, gpus, plan=None): + # Patch discover_gpus / build_plan / environment_for_plan at the + # resource_plan module (env_for imports them lazily on each call, so the + # patches are live when those imports run). Stubbing the planner keeps + # the test independent of a real model dir (args().model == "X"). + import resource_plan + a = args() + GPB = 1024 ** 3 + if plan is None: + plan = {"tiers": {"ram": {"budget_bytes": 16 * GPB, "cache_slots_per_layer": 4}, + "vram": {"budget_bytes": int(8.0 * GPB), "devices": gpus}}} + + def fake_environment_for_plan(p, env, cuda_enabled=True): + # Mirror the real contract: size CUDA_EXPERT_GB from the plan's VRAM + # budget (this is the value env_for propagates into the engine env). + r = dict(env) + if cuda_enabled and p["tiers"]["vram"]["devices"] and p["tiers"]["vram"]["budget_bytes"] > 0: + r["CUDA_EXPERT_GB"] = f"{p['tiers']['vram']['budget_bytes'] / GPB:.3f}" + return r + + with mock.patch.dict(os.environ, {}, clear=True), \ + mock.patch.object(sys, "platform", platform), \ + mock.patch.object(coli, "cuda_binary", return_value=cuda), \ + mock.patch.object(resource_plan, "discover_gpus", return_value=gpus), \ + mock.patch.object(resource_plan, "build_plan", return_value=plan), \ + mock.patch.object(resource_plan, "environment_for_plan", + side_effect=fake_environment_for_plan): + return coli.env_for(a) + + def _fake_gpu(self, index=0, name="NVIDIA GeForce RTX 5070 Ti", + total_mib=16384, free_mib=15000): + return {"index": index, "name": name, + "total_bytes": total_mib * 1024 * 1024, + "free_bytes": free_mib * 1024 * 1024} + + def test_win32_auto_enables_cuda_when_gpu_present(self): + e = self._env_for("win32", cuda=True, gpus=[self._fake_gpu()]) + self.assertEqual(e["COLI_CUDA"], "1") + self.assertEqual(e["COLI_GPUS"], "0") + # VRAM budget is sized from free VRAM by build_plan (real minus reserve), + # so it must be present and positive — never a guess or zero. + self.assertIn("CUDA_EXPERT_GB", e) + self.assertGreater(float(e["CUDA_EXPERT_GB"]), 0.0) + # Dense offload is an explicit opt-in (matches --auto-tier): not set here. + self.assertNotIn("CUDA_DENSE", e) + + def test_win32_falls_back_to_cpu_when_nvidia_smi_missing(self): + # coli_cuda.dll present (cuda=True) but nvidia-smi absent (no GPUs found) + # -> warn + CPU-only, never crash, never set COLI_CUDA. + e = self._env_for("win32", cuda=True, gpus=[]) + self.assertNotIn("COLI_CUDA", e) + self.assertNotIn("COLI_GPUS", e) + self.assertNotIn("CUDA_EXPERT_GB", e) + + def test_win32_cpu_build_stays_silent(self): + # No coli_cuda.dll (cuda=False) -> CPU build, nothing GPU-related emitted. + e = self._env_for("win32", cuda=False, gpus=[self._fake_gpu()]) + self.assertNotIn("COLI_CUDA", e) + self.assertNotIn("COLI_GPUS", e) + + def test_linux_bare_chat_not_auto_enabled(self): + # The auto-enable is scoped to win32: a Linux bare chat with a GPU + # present must NOT turn CUDA on (Linux keeps the explicit-flag UX). + e = self._env_for("linux", cuda=True, gpus=[self._fake_gpu()]) + self.assertNotIn("COLI_CUDA", e) + self.assertNotIn("CUDA_EXPERT_GB", e) + + if __name__ == "__main__": unittest.main() diff --git a/c/tests/test_grouped_g4_cuda.cu b/c/tests/test_grouped_g4_cuda.cu new file mode 100644 index 00000000..b370cdda --- /dev/null +++ b/c/tests/test_grouped_g4_cuda.cu @@ -0,0 +1,108 @@ +/* Grouped-int4 (fmt=4) CUDA kernel oracle (#334). + * + * Feeds random offset-binary nibble weights + [O, ng] group scales through + * grouped_hidden_g4_dual / grouped_down_g4 and checks against a CPU reference + * that replicates matmul_i4_grouped's semantics (value = nibble - 8, per-group + * partial dot x scale). Covers gs=64, a non-divisible tail group, and a + * per-row (gs=0) member riding in the same launch — the fmt=2-compat case. + * + * The device buffers get the same XOR 0x88 offset->signed conversion the + * upload path applies, so the kernels are exercised exactly as deployed. + * + * Build: nvcc -O2 -std=c++17 -arch=native tests/test_grouped_g4_cuda.cu -o tests/test_grouped_g4 + */ +#include +#include +#include +#include +#include + +#include "../backend_cuda.cu" + +static void cpu_gemv_g4(const uint8_t *q,const float *sc,int K,int O,int gs, + const float *x,float *y){ + int rb=(K+1)/2, ng=gs>0?(K+gs-1)/gs:1, egs=gs>0?gs:K; + for(int o=0;oK) glen=K-base; + double p=0; + for(int i=base;i>1]; int n=(i&1)?(v>>4):(v&15); + p+=(double)x[i]*(n-8); + } + a+=p*scl[g]; + } + y[o]=(float)a; + } +} + +int main(void){ + srand(7); + const int D=200, I=96, gs=64; /* tail group: 200 % 64 = 8 */ + const int COUNT=3; /* expert 0,1: fmt4 gs=64; expert 2: per-row (gs=0) */ + const int rbD=(D+1)/2, rbI=(I+1)/2; + const int ngD=(D+gs-1)/gs, ngI=(I+gs-1)/gs; + int trials=50, bad=0; + for(int t=0;t>>(qg[c],(size_t)I*rbD); + offset_to_signed_s4<<<64,256>>>(qu[c],(size_t)I*rbD); + offset_to_signed_s4<<<64,256>>>(qd[c],(size_t)D*rbI); + cudaMemcpy(sg[c],hgs[c],(size_t)I*cngD*4,cudaMemcpyHostToDevice); + cudaMemcpy(su[c],hus[c],(size_t)I*cngD*4,cudaMemcpyHostToDevice); + cudaMemcpy(sd[c],hds[c],(size_t)D*cngI*4,cudaMemcpyHostToDevice); + host[c]={qg[c],qu[c],qd[c],sg[c],su[c],sd[c],4,4,4,1,c,cgs,cgs,cgs}; + } + for(size_t i=0;i<(size_t)COUNT*D;i++) xs[i]=(rand()/(float)RAND_MAX-.5f)*2.f; + GroupDesc *ddesc; cudaMalloc(&ddesc,sizeof(host)); + cudaMemcpy(ddesc,host,sizeof(host),cudaMemcpyHostToDevice); + dim3 hgd((unsigned)I,1,(unsigned)COUNT),ogd((unsigned)D,1,(unsigned)COUNT); + grouped_hidden_g4_dual<<>>(gate,up,xs,ddesc,I,D); + grouped_down_g4<<>>(y,gate,ddesc,D,I); + if(cudaDeviceSynchronize()!=cudaSuccess){ printf("FAIL cuda\n"); return 1; } + for(int c=0;c1e-3f*(fabsf(rg[o])+1e-3f)|| + fabsf(up[(size_t)c*I+o]-ru[o])>1e-3f*(fabsf(ru[o])+1e-3f)) bad++; + } + cpu_gemv_g4(hd[c],hds[c],I,D,cgs,(float*)gate+(size_t)c*I,ry); + for(int o=0;o1e-3f*(fabsf(ry[o])+1e-3f)) bad++; + } + for(int c=0;c bool: + """True iff BOTH the built engine AND the tiny fixture are available. + + These tests need glm.exe (a build artifact) AND glm_tiny/ (a generated + fixture, gitignored). CI runs `make check` = "dependency-free tests, no + model downloads" (workflow .github/workflows/check.yml, by design #140), so + neither is present there and these tests must SKIP rather than fail. They + run locally after `make glm.exe` (glm_tiny ships alongside the source, or + is regenerated by tools/make_glm_oracle.py). + """ + return ENGINE.exists() and (TINY / "config.json").exists() + + +def _skip_reason() -> str: + """Name exactly which prerequisite is missing, so the skip is actionable.""" + if not ENGINE.exists(): + return "glm.exe not built (run: make glm.exe)" + if not (TINY / "config.json").exists(): + return "glm_tiny fixture absent (gitignored; ship it locally or run tools/make_glm_oracle.py)" + return "" + + +def _cuda_available() -> bool: + """True iff the engine binary has the CUDA loader compiled in AND the DLL is present. + + The host is built with -DCOLI_CUDA only when CUDA_DLL=1 (Makefile). A binary + built without it embeds the string "this binary is CPU-only; rebuild" and + exits 2 on any CUDA env var — so we detect the CPU-only build by scanning + the binary for that marker (avoids a slow ldd/strings on every import; we + only read enough to find it). On Windows the DLL is also required + (backend_loader.c loads it at runtime); on Linux it's direct-linked. + """ + if not _engine_present(): + return False + try: + # Read once; the marker is near the read-only string table. 256 KB is + # plenty for this string and avoids loading a 1 MB binary into memory. + with open(ENGINE, "rb") as f: + blob = f.read(2 * 1024 * 1024) + if b"this binary is CPU-only" in blob: + return False # CPU-only build: COLI_CUDA=1 would exit 2. + except OSError: + pass + # Windows: DLL required at runtime. Linux: direct-linked (no DLL). + if os.name == "nt": + return (C_DIR / "coli_cuda.dll").exists() + import subprocess + try: + out = subprocess.run(["ldd", str(ENGINE)], capture_output=True, text=True) + return "libcudart" in out.stdout + except (FileNotFoundError, OSError): + return False + + +@unittest.skipUnless(_engine_present(), _skip_reason() or "glm.exe + glm_tiny required") +class TinyEfficiencyTest(unittest.TestCase): + """Asserted regression tests on the resident tiny model. Gates CI.""" + + def _run(self, **overlay): + return run_engine(overlay, engine=str(ENGINE), snap=str(TINY))[0] + + # -- telemetry contract --------------------------------------------------- + + def test_telemetry_parses(self): + """A REPLAY run must emit the throughput + PROFILE lines the suite keys on. + + If this fails, either the engine changed its output format (update the + parsers in tools/efficiency.py) or the run crashed early.""" + t = self._run(REPLAY="1", TEMP="0", NGEN="4") + self.assertEqual(t["returncode"], 0, f"engine exited non-zero:\n{t['stderr']}") + self.assertIn("tok_s", t["parsed"], f"missing tok/s line:\n{t['stderr']}") + self.assertIn("profile", t["parsed"], f"missing PROFILE line:\n{t['stderr']}") + self.assertIn("hit_pct", t["parsed"], f"missing expert hit line:\n{t['stderr']}") + self.assertIsNotNone(t["tok_s"]) + self.assertIsNotNone(t["profile"]) + + # -- throughput floor ----------------------------------------------------- + + def test_tiny_tok_s_floor(self): + """Tiny decode must beat TINY_TOK_S_FLOOR. + + The tiny model is fully resident and runs ~200 tok/s; the 20 tok/s + default floor is a 10x margin that catches broken builds or a pathological + config cascade (the cap=1 trap from ISSUE_new_model_resource_regression.md) + without flapping on machine noise.""" + t = self._run(REPLAY="1", TEMP="0", NGEN="8") + self.assertEqual(t["returncode"], 0, f"engine exited non-zero:\n{t['stderr']}") + self.assertGreaterEqual( + t["tok_s"], TINY_TOK_S_FLOOR, + f"tok/s {t['tok_s']:.1f} below floor {TINY_TOK_S_FLOOR} " + f"(regression, or a config cascade starving the cache)", + ) + + # -- accounting sanity ---------------------------------------------------- + + def test_profile_phases_present_and_nonneg(self): + """Every PROFILE phase must be present and non-negative. + + `other` can go slightly negative from timer overhead (the engine allows + it), but a large negative means the timers are double-counting.""" + t = self._run(REPLAY="1", TEMP="0", NGEN="4") + p = t["profile"] + for phase in ("disk", "expert_matmul", "attention", "lm_head"): + self.assertGreaterEqual(p[phase], -0.01, f"{phase} went negative: {p}") + # 'other' is a residual; allow a small negative from timer overlap. + self.assertGreaterEqual(p["other"], -0.05, f"other too negative (double-count): {p}") + + # -- disk-wait not dominant on a resident model --------------------------- + + def test_disk_wait_not_dominant(self): + """A fully-resident tiny model must NOT be I/O-bound. + + Everything fits in RAM; the expert-disk wait share should be ~0. If it + exceeds MAX_DISK_WAIT_SHARE, the cache/I/O path regressed — on a real + model this same regression would make decode I/O-bound (the exact + failure mode the [PROF] verdict flags).""" + t = self._run(REPLAY="1", TEMP="0", NGEN="8", PROF="1") + self.assertIn("time_shares", t["parsed"], f"missing [PROF] time shares:\n{t['stderr']}") + share = disk_wait_share(t) + self.assertIsNotNone(share) + self.assertLess( + share, MAX_DISK_WAIT_SHARE, + f"expert-I/O share {share:.0%} on a resident model — I/O path regressed", + ) + + # -- determinism ---------------------------------------------------------- + + def test_cpu_vs_cpu_determinism(self): + """Two greedy REPLAY runs with the same seed produce identical telemetry. + + TEMP=0 = greedy (no sampling), so tok/s and hit-rate must be reproducible. + A drift here means non-determinism crept into the decode path (stray + threading, uninitialized state) — which on a real model would make A/B + comparisons meaningless.""" + a = self._run(REPLAY="1", TEMP="0", NGEN="8", SEED="1") + b = self._run(REPLAY="1", TEMP="0", NGEN="8", SEED="1") + self.assertEqual(a["returncode"], 0) + self.assertEqual(b["returncode"], 0) + self.assertEqual(a["hit_pct"], b["hit_pct"], "greedy hit-rate drifted between runs") + # tok/s within 25% — exact equality is too strict across scheduler noise. + self.assertLess(abs(a["tok_s"] - b["tok_s"]) / max(a["tok_s"], b["tok_s"]), 0.25) + + +@unittest.skipUnless(_cuda_available(), + _skip_reason() or "CUDA build not present (run: make clean && make glm.exe CUDA_DLL=1 && make cuda-dll)") +class TinyCudaEfficiencyTest(unittest.TestCase): + """CUDA-path regression tests on the tiny model. Skip unless CUDA built. + + glm_tiny is small and fully resident, so CUDA here is fast and exercises the + real GPU code path (init, dense upload, kernel correctness) without the + long load time or memory pressure of the full model. These guard the + silent-failure modes that are otherwise invisible: + - COLI_CUDA=1 silently falling back to CPU (loader/DLL missing) + - CUDA_DENSE=1 uploading nothing + - a CUDA kernel producing different argmax than CPU on identical inputs + """ + + def _run(self, **overlay): + return run_engine(overlay, engine=str(ENGINE), snap=str(TINY))[0] + + def test_cuda_init_path(self): + """COLI_CUDA=1 must initialize the device and NOT exit 2. + + Exit 2 is the engine's "requested backend is unavailable" path + (glm.c: g_cuda_enabled check). A clean init prints the [CUDA] device + banner to stderr. If this fails, the DLL is broken or the loader can't + resolve symbols (ABI drift between backend_cuda.h and the dll).""" + t = self._run(COLI_CUDA="1", COLI_GPU="0", REPLAY="1", TEMP="0", NGEN="4") + self.assertNotEqual(t["returncode"], 2, + f"engine refused CUDA backend:\n{t['stderr']}") + self.assertTrue(t["cuda"]["enabled"], + f"no [CUDA] device banner on stderr:\n{t['stderr']}") + + def test_cuda_dense_uses_vram(self): + """CUDA_DENSE=1 must actually upload dense tensors to VRAM. + + The minimal GPU-exercising config (per backend_loader.c analysis): + COLI_CUDA=1 + CUDA_DENSE=1. Without CUDA_DENSE the dense path stays on + CPU and [CUDA] resident set reports 0 tensors — a silent no-op. This + catches that regression: after the run, resident_tensors > 0.""" + t = self._run(COLI_CUDA="1", COLI_GPU="0", CUDA_DENSE="1", + REPLAY="1", TEMP="0", NGEN="4") + self.assertEqual(t["returncode"], 0, f"engine exited non-zero:\n{t['stderr']}") + rt = t["cuda"]["resident_tensors"] + self.assertIsNotNone(rt, f"no [CUDA] resident set line:\n{t['stderr']}") + self.assertGreater(rt, 0, + f"CUDA_DENSE=1 but {rt} tensors resident — silent CPU fallback") + + def test_cpu_vs_cuda_tf_match(self): + """CPU and CUDA teacher-forcing must agree on most positions (DIRECTLY). + + Both paths prefill the SAME oracle sequence on the SAME weights, so their + argmaxes should match position-for-position — but not exactly: the two + backends accumulate dot-products in different orders (x86 SIMD vs CUDA + kernel), so a few near-tied logits flip. That divergence is expected + numeric behavior, not a kernel bug. A *catastrophic* kernel regression + (wrong GEMM, wrong scale, wrong fmt) would collapse agreement toward + random (~1/vocab = ~4%); the MIN_CPU_CUDA_AGREEMENT floor (default 70%) + catches that while tolerating harmless drift. + + We compare CPU-vs-CUDA *directly* (not via the oracle match-counts), + because both backends differ from the oracle at different positions and + the summary line can't tell "CPU≠CUDA" from "CPU≠oracle".""" + import json + ref = json.loads((C_DIR / "ref_glm.json").read_text()) + oracle = ref["tf_pred"] + + cpu = self._run(TF="1", TEMP="0") + cuda = self._run(COLI_CUDA="1", COLI_GPU="0", CUDA_DENSE="1", TF="1", TEMP="0") + self.assertEqual(cpu["returncode"], 0, f"CPU TF run failed:\n{cpu['stderr']}") + self.assertEqual(cuda["returncode"], 0, f"CUDA TF run failed:\n{cuda['stderr']}") + self.assertIn("tf_mismatches", cpu["parsed"], "CPU run missing per-position mismatches") + self.assertIn("tf_mismatches", cuda["parsed"], "CUDA run missing per-position mismatches") + + agree, diff = tf_agreement(cpu, cuda, oracle) + self.assertGreaterEqual( + agree, MIN_CPU_CUDA_AGREEMENT, + f"CPU-vs-CUDA argmax agreement {agree:.0%} below floor " + f"{MIN_CPU_CUDA_AGREEMENT:.0%} — CUDA kernel likely regressed. " + f"Differing positions: {diff[:10]}{'...' if len(diff)>10 else ''}", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/c/tests/test_int3.c b/c/tests/test_int3.c new file mode 100644 index 00000000..9fa7c73f --- /dev/null +++ b/c/tests/test_int3.c @@ -0,0 +1,147 @@ +/* int3-g64 (fmt=5) tests: pack layout, dequant round-trip vs plain-C reference, + * matmul_i3 (NEON + scalar tail) vs reference dequant-matmul, per-row helpers, + * the .qs-size format tag, and the quality claim in miniature (per-group int3 + * beats per-row int4 on rows with outliers — the #132 result this format ships). */ +#define main coli_glm_main_unused +#include "../colibri.c" +#undef main + +#include +#include +#include + +static int fails = 0; +#define CHECK(c) do{ if(!(c)){ printf("FAIL %s:%d: %s\n", __FILE__, __LINE__, #c); fails++; } }while(0) + +static uint64_t rng = 0x9E3779B97F4A7C15ull; +static float rndf(void){ rng ^= rng << 13; rng ^= rng >> 7; rng ^= rng << 17; + return ((int64_t)(rng & 0xFFFFF) - 0x80000) / (float)0x80000; } + +/* reference: quantize like pack_int3_g64 but keep dequantized f32 (mirrors + * quant_ablation._quant_last_dim(bits=3, group=64)) */ +static void ref_i3_dequant(const float *w, float *dq, int O, int I){ + int64_t ng=i3_groups(I); + for(int o=0;oamax)amax=a; } + float s=amax/3.f; if(s<1e-8f)s=1e-8f; + for(int k=0;k3)v=3; if(v<-4)v=-4; + dq[(int64_t)o*I+base+k]=(float)v*s; + } + } +} +static void unpack_i3(const uint8_t *q3, const float *s, float *dq, int O, int I){ + int64_t ng=i3_groups(I), rb=i3_rowbytes(I); + for(int o=0;o>2]>>((k&3)*2))&3)|(((hi[k>>3]>>(k&7))&1)<<2); + dq[(int64_t)o*I+base+k]=(float)((int)u-4)*s[(int64_t)o*ng+g]; + } + } +} + +int main(void){ + const int Is[]={64,128,192,100,65,7168}; /* incl. short tail groups and one real GLM dim */ + enum { O=7, MAXI=7168 }; + static float w[(int64_t)O*MAXI], dq_ref[(int64_t)O*MAXI], dq_pk[(int64_t)O*MAXI]; + static float x[4*MAXI], y_ref[4*O], y_ker[4*O]; + static uint8_t q3[(int64_t)O*(MAXI/64+1)*24]; + static float sc[(int64_t)O*(MAXI/64+1)]; + + for(unsigned c=0;c unpack == reference quantize-dequantize, bit for bit */ + pack_int3_g64(w, q3, sc, O, I); + ref_i3_dequant(w, dq_ref, O, I); + unpack_i3(q3, sc, dq_pk, O, I); + int bad=0; + for(int64_t i=0;i<(int64_t)O*I;i++) if(dq_pk[i]!=dq_ref[i]) bad++; + CHECK(bad==0); + + /* 2. matmul_i3 == matmul over the dequantized reference (fp tolerance: + * NEON fma order differs from the scalar reference loop) */ + for(int S=1;S<=4;S+=3){ + for(int64_t i=0;i<(int64_t)S*I;i++) x[i]=rndf(); + matmul_i3(y_ker, x, q3, sc, S, I, O); + for(int s=0;s1?fabsf(y_ref[i]):1; + if(d/m>2e-4f){ CHECK(!"matmul_i3 mismatch"); break; } + } + } + + /* 3. QT plumbing: qt_alloc(bits=3) -> qt_fill -> matmul_qt & qt_bytes & helpers */ + QT t; qt_alloc(&t, O, I, 3); + CHECK(t.fmt==5); + qt_fill(&t, w, 3); + CHECK(qt_bytes(&t)==(int64_t)O*i3_rowbytes(I)+(int64_t)O*i3_groups(I)*4); + matmul_qt(y_ker, x, &t, 1); + for(int o=0;o1?fabsf((float)a):1; + CHECK(d/m<=2e-4f); + } + float acc[MAXI]; memset(acc,0,I*sizeof(float)); + qt_addrow(&t, 2, 0.5f, acc); + for(int i=0;i1?fabsf((float)a):1; + CHECK(d/m<=2e-4f); + } + + /* 4. format resolution through the #413 gate: fmt=5 is tagged by its distinct + * WEIGHT byte count. int3-g64 and grouped-int4-at-gs=64 carry the SAME scale + * cardinality O*ceil(I/64), so the pair (weight bytes, scale bytes) must + * disambiguate: same scales, int4 weights -> fmt=4/gs=64; int3 weights -> fmt=5. + * Only well-posed for I > 256: below that, O row scales legitimately match a + * 1-group grouped layout too (detect_group_size probes gs up to 256), so + * per-row vs grouped is not distinguishable from byte counts alone. */ + if(I>256){ int gs=-1; + int64_t ns_g64=(int64_t)O*i3_groups(I)*4, ns_row=(int64_t)O*4; + CHECK(qt_resolve_fmt("t.i3", O, I, (int64_t)O*i3_rowbytes(I), ns_g64, &gs)==5); + CHECK(gs==0); + CHECK(qt_resolve_fmt("t.i8", O, I, (int64_t)O*I, ns_row, &gs)==1); + CHECK(qt_resolve_fmt("t.i4", O, I, (int64_t)O*((I+1)/2), ns_row, &gs)==2); + CHECK(qt_resolve_fmt("t.i4g", O, I, (int64_t)O*((I+1)/2), ns_g64, &gs)==4); + CHECK(gs==64); + CHECK(qt_resolve_fmt("t.i2", O, I, (int64_t)O*((I+3)/4), ns_row, &gs)==3); } + free(t.q4); free(t.s); + } + + /* 5. quality in miniature: on rows with outliers, per-group int3 must beat + * per-row int4 on reconstruction RMS (the #132 finding this format ships). */ + { + int I=1024; + for(int64_t i=0;i<(int64_t)O*I;i++) w[i]=rndf()*0.02f; + for(int o=0;o>1]; + int v=(i&1)?((int)(b>>4)-8):((int)(b&0xF)-8); w4=(float)v*t4.s[o]; } + double d3=w[(int64_t)o*I+i]-dq_ref[(int64_t)o*I+i], d4=w[(int64_t)o*I+i]-w4; + e3+=d3*d3; e4+=d4*d4; + } + CHECK(e3 < e4); + printf(" outlier-rows RMS: int3-g64 %.3e < int4-row %.3e (ratio %.2f)\n", + sqrt(e3/((double)O*I)), sqrt(e4/((double)O*I)), sqrt(e4/e3)); + free(t4.q4); free(t4.s); + } + + if(fails){ printf("int3-g64 tests: %d FAILED\n", fails); return 1; } + printf("int3-g64 tests: ok\n"); + return 0; +} diff --git a/c/tests/test_int3_convert.py b/c/tests/test_int3_convert.py new file mode 100644 index 00000000..2eaa7f79 --- /dev/null +++ b/c/tests/test_int3_convert.py @@ -0,0 +1,70 @@ +"""quant_int3_g64 (tools/convert_fp8_to_int4.py): pack layout + round-trip. + +Decodes the packed bytes with an independent NumPy decoder implementing the +fmt=5 spec (16B low plane / 8B high plane per 64-group, v+4, per-group f32 +scale) and checks the dequantized result equals the reference +quantize-dequantize (same math as quant_ablation._quant_last_dim(3, 64)). +The C side of the same layout is covered by tests/test_int3.c. +""" +import os, sys, unittest +try: + import numpy as np +except ImportError: + raise unittest.SkipTest("numpy not installed") + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "tools")) +from convert_fp8_to_int4 import quant_int3_g64 + + +def decode(packed, scales, O, I, group=64): + ng = (I + group - 1) // group + b = packed.reshape(O, ng, 24) + lo, hi = b[:, :, :16], b[:, :, 16:] + k = np.arange(group) + lov = (lo[:, :, k >> 2] >> ((k & 3) * 2)[None, None, :]) & 3 + hiv = (hi[:, :, k >> 3] >> (k & 7)[None, None, :]) & 1 + v = (lov | (hiv << 2)).astype(np.int64) - 4 + dq = v.astype(np.float64) * scales.reshape(O, ng, 1).astype(np.float64) + return dq.reshape(O, ng * group)[:, :I] + + +def reference(w, group=64): + """same math as quant_int3_g64 (which works in f32), replayed exactly, then + dequantized in f64 so it matches decode() bit for bit""" + O, I = w.shape + ng = (I + group - 1) // group + pad = ng * group - I + wp = np.pad(w, ((0, 0), (0, pad))) if pad else w + g = wp.reshape(O, ng, group) + s = np.maximum(np.abs(g).max(axis=2, keepdims=True) / 3.0, 1e-8).astype(np.float32) + q = np.clip(np.rint(g / s), -4, 3).astype(np.int64) + return (q.astype(np.float64) * s.astype(np.float64)).reshape(O, ng * group)[:, :I] + + +class Int3ConvertTest(unittest.TestCase): + def test_round_trip(self): + rng = np.random.default_rng(7) + for I in (64, 128, 100, 65, 7168): + w = (rng.standard_normal((5, I)) * 0.05).astype(np.float32) + w[0, 3] = 1.7; w[2, min(5, I - 1)] = -2.2 + packed, scales = quant_int3_g64(w) + ng = (I + 63) // 64 + self.assertEqual(packed.size, 5 * ng * 24) + self.assertEqual(scales.size, 5 * ng) + np.testing.assert_allclose(decode(packed, scales, 5, I), + reference(w), rtol=0, atol=0) + + def test_outliers_beat_row_int4(self): + rng = np.random.default_rng(11) + w = (rng.standard_normal((8, 1024)) * 0.02).astype(np.float32) + for o in range(8): w[o, (o * 37) % 1024] = 1.5 + packed, scales = quant_int3_g64(w) + e3 = float(((decode(packed, scales, 8, 1024) - w) ** 2).mean()) + s4 = np.maximum(np.abs(w).max(axis=1, keepdims=True) / 7.0, 1e-8) + w4 = np.clip(np.rint(w / s4), -8, 7) * s4 + e4 = float(((w4 - w) ** 2).mean()) + self.assertLess(e3, e4) + + +if __name__ == "__main__": + unittest.main() diff --git a/c/tests/test_int3_load.c b/c/tests/test_int3_load.c new file mode 100644 index 00000000..57634c01 --- /dev/null +++ b/c/tests/test_int3_load.c @@ -0,0 +1,103 @@ +/* Loader-seam test for fmt=5: writes a real .safetensors file containing an + * int3-g64 tensor (U8 payload + per-GROUP .qs) next to an int4 control tensor + * (per-row .qs), indexes it with st_init, loads both through qt_from_disk, and + * checks the byte-count/.qs-size format inference picks fmt=5 vs fmt=2 correctly + * and the loaded weights dequantize identically to pack_int3_g64's output. */ +#define main coli_glm_main_unused +#include "../colibri.c" +#undef main + +#include +#include +#include +#include + +static int fails = 0; +#define CHECK(c) do{ if(!(c)){ printf("FAIL %s:%d: %s\n", __FILE__, __LINE__, #c); fails++; } }while(0) + +static uint64_t rng = 0xA5A5A5A55A5A5A5Aull; +static float rndf(void){ rng ^= rng << 13; rng ^= rng >> 7; rng ^= rng << 17; + return ((int64_t)(rng & 0xFFFFF) - 0x80000) / (float)0x80000; } + +static void deq4(const QT *t, float *dq){ + for(int o=0;oO;o++) for(int i=0;iI;i++){ + if(t->fmt==5){ + int64_t g=i/I3_GROUP; const uint8_t *lo=t->q4+(int64_t)o*i3_rowbytes(t->I)+g*I3_GBYTES, *hi=lo+16; + int k=i%I3_GROUP; + unsigned u=((lo[k>>2]>>((k&3)*2))&3)|(((hi[k>>3]>>(k&7))&1)<<2); + dq[(int64_t)o*t->I+i]=(float)((int)u-4)*t->s[(int64_t)o*i3_groups(t->I)+g]; + } else { /* fmt2 */ + uint8_t b=t->q4[(int64_t)o*((t->I+1)/2)+(i>>1)]; + int v=(i&1)?((int)(b>>4)-8):((int)(b&0xF)-8); + dq[(int64_t)o*t->I+i]=(float)v*t->s[o]; + } + } +} + +int main(void){ + enum { O=5, I=320 }; /* 5 groups per row; I > 256 so the per-row int4 + * control stays fmt=2 (detect_group_size probes + * gs up to 256: any smaller I would make O row + * scales match a legitimate 1-group layout) */ + int64_t ng=i3_groups(I), rb=i3_rowbytes(I); + static float w[O*I]; + for(int i=0;i> 28) & 0xF)) * 0.5 + for l in range(4): + seven = (word >> (7 * l)) & 0x7F + bits = [(seven >> j) & 1 for j in range(7)] + bits.append(sum(bits) & 1) # odd parity closes the 8th + idx = int(rows[r, base + ib * 8 + l * 2 + 0]) + idx2 = int(rows[r, base + ib * 8 + l * 2 + 1]) + mags = list(g[idx]) + list(g[idx2]) + for j in range(8): + pos = sb * P.QK + ib * P.SUB + l * 8 + j + out[r, pos] = mags[j] * db * (-1.0 if bits[j] else 1.0) + return out + + +@unittest.skipIf(P is None, "numpy not available (offline-tooling test)") +class TestIq3Pack(unittest.TestCase): + def setUp(self): + np.random.seed(4242) + self.x = (np.random.randn(6, 1024) * 0.05).astype(np.float32) + + def test_byte_budget(self): + self.assertEqual(P.BLOCK_BYTES, 98) + self.assertAlmostEqual(P.bpw(), 3.0625, places=6) + packed = P.encode(self.x) + self.assertEqual(packed.shape, (6, 1024 // P.QK * 98)) + self.assertEqual(packed.dtype, np.uint8) + + def test_encode_is_deterministic(self): + self.assertTrue(np.array_equal(P.encode(self.x), P.encode(self.x))) + + def test_decode_matches_spec_reader(self): + packed = P.encode(self.x) + fast = P.decode(packed, 1024) + slow = ref_decode(packed, 1024) + self.assertTrue(np.allclose(fast, slow, rtol=1e-6, atol=1e-8), + f"max |Δ| = {np.abs(fast - slow).max()}") + + def test_sign_parity_closes(self): + """Every 8-weight block must have an even number of negatives — that is + what lets the 8th sign be derived instead of stored.""" + y = P.decode(P.encode(self.x), 1024) + neg = (y < 0).reshape(-1, 8).sum(-1) + self.assertTrue(np.all(neg % 2 == 0), "a block stored odd negatives") + + def test_reconstruction_quality(self): + y = P.decode(P.encode(self.x), 1024) + rel = np.sqrt(((y - self.x) ** 2).mean()) / np.sqrt((self.x ** 2).mean()) + # the torch model that won the #453 A/B measures ~0.195 on this input class + self.assertLess(rel, 0.25, f"rel-RMSE {rel:.4f} — worse than the chosen scheme") + self.assertGreater(rel, 0.05, f"rel-RMSE {rel:.4f} — implausibly good, check the test") + + def test_shape_guard(self): + with self.assertRaises(ValueError): + P.encode(np.zeros((2, 300), dtype=np.float32)) + + def test_signs_spec_frozen(self): + """First bytes of the n=256 stream, computed once by hand from the + xorshift64* spec. If this fails, the PRNG drifted and every rotated + container in the wild decodes to garbage — the constants are the spec.""" + s = P.signs(256) + self.assertEqual(s.shape, (256,)) + self.assertTrue(set(np.unique(s)) <= {-1.0, 1.0}) + self.assertEqual(list((s[:16] < 0).astype(int)), + [0, 1, 1, 1, 1, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 0]) + self.assertFalse(np.array_equal(P.signs(256), P.signs(512)[:256]), + "streams for different sizes must differ (seed = 417+n)") + + def test_rot_blocks_tiling(self): + self.assertEqual(P.rot_blocks(2048), [2048]) + self.assertEqual(P.rot_blocks(6144), [2048, 4096]) + self.assertEqual(P.rot_blocks(1536), [512, 1024]) + for dim in (768, 1536, 6144): + self.assertEqual(sum(P.rot_blocks(dim)), dim) + + def test_rotation_is_orthogonal(self): + """Q preserves norms and cancels between W@Q and Q^T x: the rotated + matmul must reproduce the unrotated one to float precision.""" + w = (np.random.randn(8, 1536) * 0.05).astype(np.float32) + x = np.random.randn(1536).astype(np.float32) + wr, xr = P.rotate_rows(w), P.rotate_rows(x[None, :])[0] + self.assertTrue(np.allclose(np.linalg.norm(wr, axis=1), + np.linalg.norm(w, axis=1), rtol=1e-5)) + self.assertTrue(np.allclose(wr @ xr, w @ x, rtol=1e-4, atol=1e-5), + f"max |Δ| = {np.abs(wr @ xr - w @ x).max()}") + + def test_unrotate_inverts_rotate(self): + w = (np.random.randn(4, 768) * 0.05).astype(np.float32) + back = P.unrotate_rows(P.rotate_rows(w)) + self.assertTrue(np.allclose(back, w, rtol=1e-5, atol=1e-7), + f"max |Δ| = {np.abs(back - w).max()}") + + +if __name__ == "__main__": + unittest.main() diff --git a/c/tests/test_kv_alloc.c b/c/tests/test_kv_alloc.c index 40ec00dd..d080c08c 100644 --- a/c/tests/test_kv_alloc.c +++ b/c/tests/test_kv_alloc.c @@ -4,7 +4,7 @@ * arrays twice on the second call -> allocator abort. No model file needed: * the CPU path of kv_alloc only reads c->n_layers/kv_lora/qk_rope. */ #define main coli_glm_main_unused -#include "../glm.c" +#include "../colibri.c" #undef main int main(void){ diff --git a/c/tests/test_logit_nan.c b/c/tests/test_logit_nan.c index 981d74d3..de6dc2dd 100644 --- a/c/tests/test_logit_nan.c +++ b/c/tests/test_logit_nan.c @@ -15,7 +15,7 @@ #include #include #define main coli_glm_main_unused -#include "../glm.c" +#include "../colibri.c" #undef main static int approx1(double x){ return x > 0.999 && x < 1.001; } diff --git a/c/tests/test_makefile_platform.py b/c/tests/test_makefile_platform.py index 3c728802..50b4c7af 100644 --- a/c/tests/test_makefile_platform.py +++ b/c/tests/test_makefile_platform.py @@ -35,7 +35,7 @@ def test_windows_nt_without_uname_selects_mingw_build(self): env["PATH"] = "" result = subprocess.run( - [MAKE, "--no-print-directory", "-B", "-n", "glm"], + [MAKE, "--no-print-directory", "-B", "-n", "colibri"], cwd=C_DIR, env=env, text=True, @@ -43,7 +43,7 @@ def test_windows_nt_without_uname_selects_mingw_build(self): check=True, ) - self.assertIn("-o glm.exe", result.stdout) + self.assertIn("-o colibri.exe", result.stdout) self.assertIn("-fopenmp", result.stdout) self.assertIn("-static", result.stdout) diff --git a/c/tests/test_openai_server.py b/c/tests/test_openai_server.py index 4cda9ef3..98da4c18 100644 --- a/c/tests/test_openai_server.py +++ b/c/tests/test_openai_server.py @@ -11,7 +11,7 @@ from pathlib import Path from openai_server import (APIError, APIHandler, APIServer, ClientCancelled, END, GenerationScheduler, - READY, Engine, generation_options, parse_tool_calls, + READY, Engine, _engine_error, generation_options, parse_tool_calls, read_engine_turn, render_chat, serve) @@ -20,8 +20,8 @@ def __init__(self): self.calls = [] def generate(self, prompt, maximum, temperature, top_p, on_text, cache_slot=0, - cancelled=None): - self.calls.append((prompt, maximum, temperature, top_p, cache_slot)) + cancelled=None, grammar=None): + self.calls.append((prompt, maximum, temperature, top_p, cache_slot, grammar)) on_text("Hé") on_text("llo") return {"prompt_tokens": 7, "completion_tokens": 2, "length_limited": False} @@ -34,7 +34,7 @@ def __init__(self): self.release = threading.Event() def generate(self, prompt, maximum, temperature, top_p, on_text, cache_slot=0, - cancelled=None): + cancelled=None, grammar=None): self.entered.set() self.release.wait(2) return super().generate(prompt, maximum, temperature, top_p, on_text, cache_slot, @@ -71,11 +71,11 @@ def test_renders_thinking_prefix(self): def test_validates_generation_limits(self): self.assertEqual(generation_options({"max_tokens": 4, "temperature": 0, "top_p": 1}, 8), - (4, 0.0, 1.0)) + (4, 0.0, 1.0, None)) # max_tokens above the server cap is clamped, not rejected (#260): OpenAI # clients default to large values; erroring breaks them. self.assertEqual(generation_options({"max_tokens": 9, "temperature": 0, "top_p": 1}, 8), - (8, 0.0, 1.0)) + (8, 0.0, 1.0, None)) # non-positive / non-int max_tokens is still a hard error with self.assertRaises(APIError): generation_options({"max_tokens": 0}, 8) @@ -84,7 +84,31 @@ def test_validates_generation_limits(self): with self.assertRaises(APIError): generation_options({"top_p": math.inf}, 8) self.assertEqual(generation_options({"temperature": None, "top_p": None}, 8), - (8, 0.7, 0.9)) + (8, 0.7, 0.9, None)) + # response_format -> grammar plumbing (draft source, never a constraint) + opts = generation_options({"max_tokens": 4, "response_format": {"type": "json_object"}}, 8) + self.assertIn("root ::=", opts[3]) + schema = {"type": "object", "properties": {"a": {"type": "string"}}, "required": ["a"]} + opts = generation_options({"max_tokens": 4, "response_format": + {"type": "json_schema", "json_schema": {"schema": schema}}}, 8) + self.assertEqual(json.loads(opts[3]), schema) + opts = generation_options({"max_tokens": 4, "response_format": + {"type": "gbnf", "grammar": 'root ::= "x"'}}, 8) + self.assertEqual(opts[3], 'root ::= "x"') + with self.assertRaises(APIError): + generation_options({"response_format": {"type": "yaml"}}, 8) + with self.assertRaises(APIError): + generation_options({"response_format": {"type": "json_schema", "json_schema": {}}}, 8) + with self.assertRaises(APIError): # non-dict response_format + generation_options({"response_format": "json"}, 8) + with self.assertRaises(APIError): # empty gbnf + generation_options({"response_format": {"type": "gbnf", "grammar": " "}}, 8) + with self.assertRaises(APIError): # oversized grammar (> 1 MiB pre-check) + generation_options({"response_format": {"type": "gbnf", "grammar": "x" * ((1 << 20) + 1)}}, 8) + # malformed GBNF passes the gateway by design: the ENGINE fail-softs it + # (draft source only — bad grammar costs the speedup, never the request) + opts = generation_options({"response_format": {"type": "gbnf", "grammar": "not a grammar ::="}}, 8) + self.assertEqual(opts[3], "not a grammar ::=") class ProtocolTest(unittest.TestCase): @@ -658,6 +682,91 @@ def test_unknown_parameter_keeps_permissive_decoding(self): self.assertEqual(args["extra"], 7) +class EngineErrorFrameTest(unittest.TestCase): + """#401: an over-long prompt used to be silently truncated to the first CTX-2 tokens, so the + model answered from a mutilated prompt and the client got HTTP 200 with junk. The engine now + refuses, and the refusal has to reach the client as a 400 it can act on -- not a 500.""" + + def test_context_exceeded_becomes_a_400_the_client_can_act_on(self): + err = _engine_error(["CONTEXT_EXCEEDED", "8321", "4094"], "CONTEXT_EXCEEDED 8321 4094") + self.assertIsInstance(err, APIError) + self.assertEqual(err.status, 400) + self.assertEqual(err.code, "context_length_exceeded") + self.assertEqual(err.param, "messages") + self.assertIn("4094", err.message) + self.assertIn("8321", err.message) + + def test_other_engine_errors_stay_runtime_errors(self): + for frame in (["SLOT_BUSY"], ["BAD_REQUEST"], []): + err = _engine_error(frame, " ".join(frame) or "engine request failed") + self.assertIsInstance(err, RuntimeError) + self.assertNotIsInstance(err, APIError) + + def test_malformed_context_frame_does_not_crash_the_dispatcher(self): + err = _engine_error(["CONTEXT_EXCEEDED"], "CONTEXT_EXCEEDED") + self.assertIsInstance(err, APIError) + self.assertEqual(err.status, 400) +class UnclosedToolCallTest(unittest.TestCase): + """#401: the model opens , emits a well-formed call, then stops without the + closing tag (budget ran out, or quantization mangled it). The strict regex needs both tags, + so the client used to get zero tool_calls -- a total failure from a recoverable output.""" + + NO_ARG_TOOL = ORDER_TOOL + [{"type": "function", + "function": {"name": "list_orders", "parameters": {}}}] + + def _calls(self, reply, tools=ORDER_TOOL): + return parse_tool_calls(reply, tools) + + def test_unclosed_box_is_recovered(self): + content, calls = self._calls("lookup_order" + "order_idA-1") + self.assertEqual(len(calls), 1) + self.assertEqual(json.loads(calls[0]["function"]["arguments"]), {"order_id": "A-1"}) + self.assertEqual(content, "") + + def test_mangled_closing_tag_is_recovered(self): + _, calls = self._calls("lookup_order" + "order_idA-1lookup_order" + "order_idA-1") + self.assertEqual(len(calls), 1) + self.assertEqual(content, "Let me check.") + + def test_closed_call_followed_by_an_unclosed_one(self): + _, calls = self._calls("lookup_order" + "order_idA-1" + "lookup_order" + "order_idB-2") + self.assertEqual([json.loads(c["function"]["arguments"])["order_id"] for c in calls], + ["A-1", "B-2"]) + + def test_bare_declared_name_recovers_a_zero_argument_call(self): + _, calls = self._calls("list_orders", self.NO_ARG_TOOL) + self.assertEqual(len(calls), 1) + self.assertEqual(calls[0]["function"]["name"], "list_orders") + self.assertEqual(json.loads(calls[0]["function"]["arguments"]), {}) + + def test_prose_mentioning_the_marker_does_not_fabricate_a_call(self): + content, calls = self._calls("To call a tool, write and then the name.") + self.assertEqual(calls, []) + self.assertIn("", content) + + def test_undeclared_name_without_arguments_is_not_recovered(self): + _, calls = self._calls("drop_all_tables") + self.assertEqual(calls, []) + + def test_well_formed_output_is_untouched(self): + content, calls = self._calls("Done.lookup_order" + "order_idA-1" + "") + self.assertEqual(len(calls), 1) + self.assertEqual(content, "Done.") + + class ToolChoiceTest(unittest.TestCase): def test_none_does_not_offer_the_tools(self): prompt = render_chat([{"role": "user", "content": "hi"}], tools=ORDER_TOOL, diff --git a/c/tests/test_openai_tools_e2e.py b/c/tests/test_openai_tools_e2e.py index d921843b..c6c43e69 100644 --- a/c/tests/test_openai_tools_e2e.py +++ b/c/tests/test_openai_tools_e2e.py @@ -51,6 +51,10 @@ def reply(rid, text, chunks=1): elif "weather in Rome" in prompt: reply(rid, "get_weathercity" "Rome") + elif "weather in Naples" in prompt: + # #401: a well-formed call whose closing tag never arrives (budget ran out, or + # quantization mangled it). Strict parsing loses the whole call. + reply(rid, "get_weathercityNaples") elif "weather in Milan" in prompt: # split across many tiny DATA chunks: streamed marker suppression must # hold even when a marker straddles a chunk boundary @@ -170,6 +174,36 @@ def test_tool_result_round_trip(self): self.assertIn("# Tools", rendered) self.assertIn('"get_weather"', rendered) + def test_unclosed_tool_call_still_reaches_the_client(self): + """#401: the closing never arrives. Before the recovery the client got + finish_reason=stop with the raw markers dumped into content -- a total failure from + an otherwise perfectly well-formed call.""" + r = self.post({"model": MODEL_ID, "tools": TOOLS, + "messages": [{"role": "user", + "content": "What is the weather in Naples?"}]}) + choice = r["choices"][0] + self.assertEqual(choice["finish_reason"], "tool_calls") + calls = choice["message"]["tool_calls"] + self.assertEqual(len(calls), 1) + self.assertEqual(calls[0]["function"]["name"], "get_weather") + self.assertEqual(json.loads(calls[0]["function"]["arguments"]), {"city": "Naples"}) + self.assertNotIn("", choice["message"].get("content") or "") + + def test_unclosed_tool_call_streamed(self): + """Same recovery on the streaming path -- this is the shape coding clients use.""" + events = self.post({"model": MODEL_ID, "tools": TOOLS, "stream": True, + "messages": [{"role": "user", + "content": "What is the weather in Naples?"}]}, + stream=True) + deltas = [e["choices"][0]["delta"] for e in events if e["choices"]] + self.assertNotIn("", "".join(d.get("content") or "" for d in deltas)) + calls = [d["tool_calls"] for d in deltas if d.get("tool_calls")] + self.assertEqual(len(calls), 1) + self.assertEqual(json.loads(calls[0][0]["function"]["arguments"]), {"city": "Naples"}) + finish = [e["choices"][0]["finish_reason"] for e in events + if e["choices"] and e["choices"][0].get("finish_reason")] + self.assertEqual(finish, ["tool_calls"]) + def test_no_tools_plain_text(self): r = self.post({"model": MODEL_ID, "messages": [{"role": "user", "content": "Hi!"}]}) diff --git a/c/tests/test_pipe_block.c b/c/tests/test_pipe_block.c new file mode 100644 index 00000000..85716957 --- /dev/null +++ b/c/tests/test_pipe_block.c @@ -0,0 +1,169 @@ +/* COLI_PIPE_BLOCK: the pipe pool's condvar waiter must be observably + * equivalent to the sched_yield spin it replaces — same bytes land in the + * same ws[] slots, and no interleaving loses a wakeup (the worker RELEASE- + * stores ready[] BEFORE taking mx to broadcast; the waiter re-checks under + * the lock, so a flag set between its fast-path check and the wait cannot + * be missed). Both waiters are exercised against the same on-disk fixture, + * alternating parked waits (wait issued before the load finishes) with + * fast-path waits (load already done), across enough generations to cycle + * the pool's gen-tagged cursor. + * + * Also pins the PIPE_WORKERS => PIPE implication table: fires ONLY when + * PIPE is unset in the env AND the platform default left the pipe off AND + * PIPE_WORKERS parses positive (PIPE_WORKERS=0/empty/negative must NOT + * silently enable a clamped 1-worker pipe). */ +#include +#include +#include +#include +#include +#include +#define main coli_glm_main_unused +#include "../colibri.c" +#undef main + +static int fail(const char *s){ fprintf(stderr,"FAIL: %s\n",s); return 1; } + +enum { NE=8, LAYER=1 }; /* experts 0..NE-1 on one MoE layer */ +/* per-expert file image: [gate 12][up 12][down 12][gate.qs 12][up.qs 12][down.qs 16] */ +enum { WB=12, QS_G=12, QS_U=12, QS_D=16, ESZ=3*WB+QS_G+QS_U+QS_D }; + +static unsigned char wbyte(int e,int j){ return (unsigned char)(e*31+j+1); } +static float scale(int e,int i){ return (float)(e*8+i)+0.5f; } + +#define TMPF "test_pipe_block.tmp" + +static int write_fixture(void){ + FILE *w=fopen(TMPF,"wb"); if(!w) return fail("create temp"); + for(int e=0;ec.hidden=4; m->c.moe_inter=3; m->ebits=8; + m->S.n=NE*6; m->S.cap=NE*6; m->S.t=calloc(NE*6,sizeof(st_tensor)); + if(!m->S.t) return fail("tensor metadata allocation"); + const char *proj[3]={"gate_proj","up_proj","down_proj"}; + int sbytes[3]={QS_G,QS_U,QS_D}; + for(int e=0;eS.t[e*6+k]=(st_tensor){strdup(name),fd,wo,WB,3,WB}; wo+=WB; + size_t n=strlen(name); memcpy(name+n,".qs",4); + m->S.t[e*6+3+k]=(st_tensor){strdup(name),fd,so,sbytes[k],2,sbytes[k]/4}; so+=sbytes[k]; + } + } + return 0; +} + +static int check_slot(ESlot *s,int e){ + if(s->eid!=e || s->g.fmt!=1 || s->u.fmt!=1 || s->d.fmt!=1){ + fprintf(stderr," slot: eid=%d (want %d) fmt g/u/d=%d/%d/%d (want 1/1/1)\n", + s->eid,e,s->g.fmt,s->u.fmt,s->d.fmt); + return 1; + } + const unsigned char *g=(const unsigned char*)s->g.q8, + *u=(const unsigned char*)s->u.q8, + *d=(const unsigned char*)s->d.q8; /* q8 is int8_t; compare raw bytes */ + for(int j=0;jg.s[i]!=scale(e,i) || s->u.s[i]!=scale(e,3+i)){ + fprintf(stderr," slot e=%d scale %d: g=%g/%g u=%g/%g (got/want)\n",e,i, + (double)s->g.s[i],(double)scale(e,i),(double)s->u.s[i],(double)scale(e,3+i)); + return 1; + } + for(int i=0;i<4;i++) + if(s->d.s[i]!=scale(e,6+i)){ + fprintf(stderr," slot e=%d scale %d: d=%g/%g (got/want)\n",e,i, + (double)s->d.s[i],(double)scale(e,6+i)); + return 1; + } + return 0; +} + +static int run_generations(Model *m,int block,int gens){ + g_pipe_block=block; + for(int gen=0;genws[q],eids[q])) return fail(block?"slot contents (block)":"slot contents (spin)"); + } + } + return 0; +} + +static int test_implication_table(void){ + struct { const char *pipe_env,*pw_env; int pipe_now,want; } T[]={ + {NULL,"4",0,1}, /* pool sized, pipe off, PIPE unset → imply */ + {NULL,"16",0,1}, + {NULL,"0",0,0}, /* PIPE_WORKERS=0 must NOT enable a clamped pipe */ + {NULL,"",0,0}, + {NULL,"-3",0,0}, + {"0","4",0,0}, /* explicit PIPE=0 always wins */ + {"1","4",1,0}, /* explicit PIPE=1: nothing to imply */ + {NULL,"4",1,0}, /* platform default already ON (win32) */ + {NULL,NULL,0,0}, + }; + for(size_t i=0;i",T[i].pw_env?T[i].pw_env:"",T[i].pipe_now); + return 1; + } + return 0; +} + +int main(void){ + if(test_implication_table()) return 1; + + /* Relative to the CWD, like test_compat_direct's TMPF — NOT "/tmp/...": + * the windows job builds native .exe files and "/tmp" is not a Windows + * path. fwrite then reopen read-only: Windows compat has pread, not pwrite. */ + if(write_fixture()) return 1; + int fd=open(TMPF,COMPAT_O_RDONLY); + if(fd<0) return fail("open temp"); + + static Model m; /* zeroed: buffered pread path, no mmap/cuda */ + if(build_fixture(&m,fd)){ close(fd); remove(TMPF); return 1; } + + g_pipe=1; g_pipe_nw=4; + pipe_init(&m); + + /* spin waiter first (control), then the condvar waiter under the same + * dispatch pattern; 200 generations each cycles the gen-tagged cursor + * and alternates parked/fast-path waits. */ + if(run_generations(&m,0,200) || run_generations(&m,1,200)){ close(fd); remove(TMPF); return 1; } + + for(int q=0;q 24 threads, 12 physical cores. + + # The parser must return 12 physical cores under BOTH lscpu layouts: + # - 2-col: `lscpu -p=core,socket` emits exactly [core,socket] (this is + # what the probe actually requests; the previous fields[1]/[2] + # indexing skipped every line here and fell through to the + # logical count -> the regression JustVugg caught). + # - 3-col: bare `lscpu -p` prepends a CPU column -> [cpu,core,socket]. + # Taking the last two fields is correct in both cases. + layouts = { + "2-col (-p=core,socket)": ( + "# core,socket\n" + + "\n".join(f"{core},0" for core in range(12) for _ in range(2))), + "3-col (bare -p, CPU prefix)": ( + "# CPU,Core,Socket\n" + + "\n".join(f"{cpu},{core},0" for core in range(12) for cpu in range(2))), + } + for label, blob in layouts.items(): + with mock.patch("resource_plan.subprocess.run", + return_value=lscpu(blob)), \ + mock.patch.object(sys, "platform", "linux"): + plan = build_plan(self.model, available_memory=16 * GB, + available_disk=1, gpus=[]) + env = environment_for_plan(plan) + self.assertEqual(plan["cpu"]["physical_cores"], 12, label) + self.assertEqual(env["OMP_NUM_THREADS"], "12", label) + + def test_plan_does_not_set_omp_affinity_vars(self): + # The real #325 regression: --auto-tier set OMP_PROC_BIND=spread + + # OMP_PLACES=cores, which ran before the engine's overwrite=0 setenv and + # so won, collapsing the OpenMP team to one CPU on the reporter's 64-core + # Linux box even though OMP_NUM_THREADS was correct. The plan must leave + # affinity to the engine's own hot-thread tuning (which prefers 'close'). + plan = build_plan(self.model, available_memory=16 * GB, + available_disk=1, gpus=[], physical_cpus=64) + env = environment_for_plan(plan) + self.assertEqual(env["OMP_NUM_THREADS"], "64") + self.assertNotIn("OMP_PROC_BIND", env) + self.assertNotIn("OMP_PLACES", env) + + def test_plan_conserves_budget_and_experts_above_256gb(self): + # Regression for #325's reporter: a 512 GB machine loading the whole + # model into RAM. Verify the budget math stays exact at large RAM sizes + # (no integer truncation, no over-allocation, no experts lost between + # tiers). Checked at 256/512/800 GB to bracket the reporter's box. + for ram_gb in (256, 512, 800): + plan = build_plan(self.model, ram_gb=ram_gb, available_disk=1, + gpus=[], physical_cpus=64) + ram = plan["tiers"]["ram"] + # RAM budget never over-allocated: dense + runtime + cache <= budget. + allocated = (ram["dense_bytes"] + ram["runtime_bytes"] + + ram["expert_cache_bytes"]) + self.assertLessEqual(allocated, ram["budget_bytes"], + f"over-allocated RAM at {ram_gb} GB") + # Every expert byte is accounted for exactly once across the tiers. + tiers = plan["tiers"] + tiered = (tiers["vram"]["hot_expert_bytes"] + + ram["warm_expert_bytes"] + + tiers["disk"]["cold_expert_bytes"]) + self.assertEqual(tiered, plan["model"]["expert_bytes"], + f"expert bytes lost/duplicated at {ram_gb} GB") + # A positive RAM budget yields a non-negative cache and a sensible cap. + self.assertGreaterEqual(ram["expert_cache_bytes"], 0) + self.assertGreaterEqual(ram["cache_slots_per_layer"], 0) + def test_filters_requested_devices(self): gpus = [{"index": 0, "name": "a", "total_bytes": 8 * GB, "free_bytes": 8 * GB}] plan = build_plan(self.model, available_memory=16 * GB, available_disk=1, @@ -117,13 +192,13 @@ def test_applies_plan_without_overriding_explicit_settings(self): self.assertEqual(env["COLI_CUDA"], "1") self.assertEqual(env["COLI_GPUS"], "1") self.assertEqual(env["OMP_NUM_THREADS"], str(plan["cpu"]["physical_cores"])) - if sys.platform == "win32": - # MinGW libgomp: niente affinity su Windows, le chiavi non vanno emesse - self.assertNotIn("OMP_PROC_BIND", env) - self.assertNotIn("OMP_PLACES", env) - else: - self.assertEqual(env["OMP_PROC_BIND"], "spread") - self.assertEqual(env["OMP_PLACES"], "cores") + # The plan must NOT set OMP_PROC_BIND / OMP_PLACES on any platform: + # the engine's own hot-thread tuning owns affinity (it prefers + # OMP_PROC_BIND=close for the back-to-back per-expert matmuls). Setting + # spread + cores here ran before the engine's overwrite=0 setenv and so + # won, collapsing the team to one CPU on some libgomp topologies (#325). + self.assertNotIn("OMP_PROC_BIND", env) + self.assertNotIn("OMP_PLACES", env) self.assertEqual(env["PIN_GB"], env["CUDA_EXPERT_GB"]) explicit_threads = environment_for_plan(plan, {"OMP_NUM_THREADS": "7", @@ -140,6 +215,81 @@ def test_single_socket_plan_does_not_enable_numa(self): plan = build_plan(self.model, available_memory=16 * GB, available_disk=1, gpus=[], physical_cpus=8, cpu_sockets=1) self.assertNotIn("COLI_NUMA", environment_for_plan(plan)) + + def test_auto_tune_mtp_off_when_compute_bound(self): + # Tiny model with 64 GB RAM and no GPU: all experts fit in RAM with no + # warm tier, so the plan classifies as compute-bound. + plan = build_plan(self.model, ram_gb=64, available_memory=64 * GB, + available_disk=100 * GB, gpus=[], physical_cpus=24, + cpu_sockets=2) + # With such a small model fully in RAM and no GPU, bottleneck is compute + self.assertEqual(plan["bottleneck_class"], "compute") + self.assertIn("DRAFT", plan["tune"]) + self.assertEqual(plan["tune"]["DRAFT"]["value"], "0") + env = environment_for_plan(plan) + self.assertEqual(env["DRAFT"], "0") + explicit = environment_for_plan(plan, {"DRAFT": "3"}) + self.assertEqual(explicit["DRAFT"], "3") + + def test_auto_tune_mtp_off_when_disk_low_hit(self): + # Use a model large enough that 8 GB RAM can't hold all experts. + big = tempfile.TemporaryDirectory() + bigmodel = Path(big.name) + (bigmodel / "config.json").write_text(json.dumps({ + "num_hidden_layers": 2, "n_routed_experts": 4, + "kv_lora_rank": 4, "qk_rope_head_dim": 2, + "qk_nope_head_dim": 3, "v_head_dim": 5, "num_attention_heads": 2, + })) + expert_size = 3 * GB # each expert 3 GB → 12 GB total, won't fit in 8 GB budget + write_shard(bigmodel / "out-00000.safetensors", [ + ("model.embed_tokens.weight", 100), + ("model.layers.0.self_attn.q_a_proj.weight", 200), + ]) + for i in range(4): + write_shard(bigmodel / f"out-{i+1:05d}.safetensors", [ + (f"model.layers.1.mlp.experts.{i}.gate_proj.weight", expert_size), + ]) + plan = build_plan(bigmodel, ram_gb=0, available_memory=4 * GB, + available_disk=100 * GB, gpus=[], physical_cpus=8, + cpu_sockets=1) + big.cleanup() + self.assertEqual(plan["bottleneck_class"], "disk") + self.assertLess(plan["projected_hit_rate"], 0.90) + self.assertEqual(plan["tune"]["DRAFT"]["value"], "0") + + def test_auto_tune_pipe_multi_gpu(self): + gpus = [ + {"index": 0, "name": "a", "total_bytes": 32 * GB, "free_bytes": 30 * GB}, + {"index": 1, "name": "b", "total_bytes": 32 * GB, "free_bytes": 30 * GB}, + ] + plan = build_plan(self.model, ram_gb=16, available_memory=32 * GB, + available_disk=1, gpus=gpus, cpu_sockets=2) + self.assertEqual(plan["tune"]["COLI_CUDA_PIPE"]["value"], "2") + env = environment_for_plan(plan) + self.assertEqual(env["COLI_CUDA_PIPE"], "2") + + def test_auto_tune_pipe_single_gpu(self): + gpus = [{"index": 0, "name": "a", "total_bytes": 12 * GB, "free_bytes": 10 * GB}] + plan = build_plan(self.model, ram_gb=16, available_memory=32 * GB, + available_disk=1, gpus=gpus, cpu_sockets=1) + self.assertEqual(plan["tune"]["COLI_CUDA_PIPE"]["value"], "1") + + def test_auto_tune_numa_hint_for_cpu_only(self): + plan = build_plan(self.model, ram_gb=64, available_memory=64 * GB, + available_disk=1, gpus=[], physical_cpus=64, cpu_sockets=2) + self.assertIn("_numa_hint", plan["tune"]) + self.assertIn("numactl", plan["tune"]["_numa_hint"]) + self.assertIn("auto-tune", format_plan(plan)) + + def test_format_plan_shows_tune_and_hit_rate(self): + plan = build_plan(self.model, ram_gb=64, available_memory=64 * GB, + available_disk=100 * GB, gpus=[], physical_cpus=24, + cpu_sockets=1) + text = format_plan(plan) + self.assertIn("hit", text) + self.assertIn("auto-tune", text) + self.assertIn("DRAFT", text) + def test_cpu_binary_does_not_apply_gpu_tier(self): plan = build_plan(self.model, available_memory=16 * GB, available_disk=1, gpus=[{"index": 0, "name": "a", "total_bytes": 8 * GB, @@ -178,5 +328,73 @@ def test_plan_explains_hot_warm_and_cold_placement(self): self.assertIn("expected_bottleneck", plan) +class PhysicalCpuCountTest(unittest.TestCase): + """Regression for #325: --auto-tier pinned decode to one core because + physical_cpu_count() silently returned 1. + + Two root causes this locks down: + 1. lscpu -p prepends a CPU column, so `-p=core,socket` emits + CPU,Core,Socket; counting rows counted logical SMT siblings. + 2. any probe failure fell through to ``os.cpu_count() or 1`` and the + ``or 1`` could pin a constrained/cgroup'd box to a single core. + """ + + def _lscpu(self, stdout): + return subprocess.CompletedProcess(args=[], returncode=0, + stdout=stdout, stderr="") + + def _lscpu_topology(self, sockets, cores_per_socket, threads_per_core): + # Real lscpu shape: socket-local core IDs repeat across sockets; the + # CPU column (always prepended) is a unique logical-CPU index. + rows, cpu = [], 0 + for sock in range(sockets): + for core in range(cores_per_socket): + for _ in range(threads_per_core): + rows.append(f"{cpu},{core},{sock}") + cpu += 1 + return "# CPU,Core,Socket\n" + "\n".join(rows) + + def test_counts_physical_cores_not_smt_threads(self): + blob = self._lscpu_topology(sockets=2, cores_per_socket=16, threads_per_core=2) + with mock.patch("resource_plan.subprocess.run", return_value=self._lscpu(blob)), \ + mock.patch.object(sys, "platform", "linux"): + self.assertEqual(physical_cpu_count(), 32) + + def test_single_socket_no_smt(self): + blob = self._lscpu_topology(sockets=1, cores_per_socket=8, threads_per_core=1) + with mock.patch("resource_plan.subprocess.run", return_value=self._lscpu(blob)), \ + mock.patch.object(sys, "platform", "linux"): + self.assertEqual(physical_cpu_count(), 8) + + def test_skips_offline_core_socket_fields(self): + # VMs / large NUMA boxes emit "-" for offline core or socket IDs; that + # used to raise ValueError, discard the whole parse, and fall through + # to the single-core fallback. + blob = "# CPU,Core,Socket\n0,0,0\n1,-,0\n2,1,0\n3,1,0\n" + with mock.patch("resource_plan.subprocess.run", return_value=self._lscpu(blob)), \ + mock.patch.object(sys, "platform", "linux"): + self.assertEqual(physical_cpu_count(), 2) + + def test_lscpu_missing_falls_back_to_logical_not_silent_one(self): + # The bug: lscpu absent -> os.cpu_count() or 1. On a constrained box + # os.cpu_count() can be 1. We still must never silently pick 1 without + # a warning, and when logical cores exist they must be used. + import os + with mock.patch("resource_plan.subprocess.run", side_effect=FileNotFoundError), \ + mock.patch.object(sys, "platform", "linux"), \ + mock.patch("resource_plan.os.cpu_count", return_value=16), \ + mock.patch("sys.stderr"): + self.assertEqual(physical_cpu_count(), 16) + + def test_zero_logical_cores_warns_and_returns_one(self): + # The genuine degenerate case: no probe works and os.cpu_count() is + # None/1. Must return 1 (engine needs a positive team size) but warn. + with mock.patch("resource_plan.subprocess.run", side_effect=FileNotFoundError), \ + mock.patch.object(sys, "platform", "linux"), \ + mock.patch("resource_plan.os.cpu_count", return_value=None), \ + mock.patch("sys.stderr"): + self.assertEqual(physical_cpu_count(), 1) + + if __name__ == "__main__": unittest.main() diff --git a/c/tests/test_router_cuda.cu b/c/tests/test_router_cuda.cu new file mode 100644 index 00000000..4e714a6a --- /dev/null +++ b/c/tests/test_router_cuda.cu @@ -0,0 +1,79 @@ +/* Device-router kernel oracle (#431 PR-A). + * + * Feeds random activations/router weights through pipe_router_logits + + * pipe_router_select and checks against a CPU reference that replicates + * moe()'s plain routing path verbatim (sigmoid -> bias-augmented top-K by + * `choice`, weights from raw `logit`, route-level TOPP truncation, norm_topk, + * routed_scale). The dot/expf rounding may differ from libm at ~1e-6 rel, so + * a handful of near-tie index flips across trials is tolerated; the weight + * math itself must agree to 1e-4 rel on matching selections. + * + * Build: nvcc -O2 -std=c++17 -arch=native tests/test_router_cuda.cu -o tests/test_router_cuda + */ +#include +#include +#include +#include +#include + +/* pull in the kernel definitions (same idiom as the CPU tests' #include "../colibri.c") */ +#include "../backend_cuda.cu" + +static void cpu_ref(const float *x,const float *W,const float *bias,int D,int E, + int Ksel,float topp,int norm_topk,float rscale, + int *idx,float *w,int *keff){ + float *logit=(float*)malloc(E*sizeof(float)),*choice=(float*)malloc(E*sizeof(float)); + for(int e=0;ebv){bv=choice[e];best=e;} } + idx[kk]=best; w[kk]=logit[best]; } + int Ke=Ksel; + if(topp>0.f && topp<1.f){ + for(int a=1;a=0 && w[b]=topp*tot){ Ke=kk+1; break; } } } + if(norm_topk){ float sm=0; for(int kk=0;kk>>(x,W,b,D,lg,ch); + pipe_router_select<<<1,1>>>(lg,ch,E,K,topp,nt,rs,out); + char pack[K*8+4]; + if(cudaMemcpy(pack,out,sizeof(pack),cudaMemcpyDeviceToHost)!=cudaSuccess){ + printf("FAIL cuda\n"); return 1; } + int gidx[K],gkeff; float gw[K]; + memcpy(gidx,pack,K*4); memcpy(gw,pack+K*4,K*4); memcpy(&gkeff,pack+K*8,4); + int ridx[K],rkeff; float rw[K]; + cpu_ref(x,W,b,D,E,K,topp,nt,rs,ridx,rw,&rkeff); + int mism=0; for(int k2=0;k21e-4f*(fabsf(ref)+1e-6f)+1e-6f){ bad++; break; } + } + } + printf("router oracle: %d trials, %d near-tie flips, %d weight mismatches\n",TRIALS,flips,bad); + if(flips>4||bad){ printf("FAIL\n"); return 1; } + printf("OK\n"); return 0; +} diff --git a/c/tests/test_sample_nan.c b/c/tests/test_sample_nan.c index fc332683..4f31def9 100644 --- a/c/tests/test_sample_nan.c +++ b/c/tests/test_sample_nan.c @@ -6,7 +6,7 @@ * iniettato il token scelto e' l'argmax dei FINITI (mai 0 per default), su ogni posizione * del NaN inclusa lo[0]; (c) nessun NaN sopravvive in g_pbuf. */ #define main coli_glm_main_unused -#include "../glm.c" +#include "../colibri.c" #undef main #include diff --git a/c/tests/test_st_mirror.c b/c/tests/test_st_mirror.c new file mode 100644 index 00000000..769c99d6 --- /dev/null +++ b/c/tests/test_st_mirror.c @@ -0,0 +1,109 @@ +/* Dual-SSD mirror (st_mirror_init & friends): a second read-only model copy is + * accepted only when byte-identical in size and safetensors header, reads on + * either replica return the same bytes, and divergent/missing copies degrade + * to the primary instead of being trusted. Fixture dirs are created in the + * current working directory and removed on exit. */ +#include +#include +#include + +#include "../st.h" + +#ifdef _WIN32 +#include +#define MKDIR(p) _mkdir(p) +#else +#include +#define MKDIR(p) mkdir(p, 0777) +#endif + +#define CHECK(condition) do { \ + if (!(condition)) { \ + fprintf(stderr, "%s:%d: check failed: %s\n", __FILE__, __LINE__, #condition); \ + return 1; \ + } \ +} while (0) + +#define DIR_A "tmp_mirror_a" /* primary */ +#define DIR_B "tmp_mirror_b" /* identical copy */ +#define DIR_C "tmp_mirror_c" /* same size, divergent header */ +#define DIR_D "tmp_mirror_d" /* different size */ +#define DIR_E "tmp_mirror_e" /* empty (missing file) */ + +/* one-tensor safetensors file; flip lets us corrupt one header byte and + * pad lets us grow the payload, both without changing anything else */ +static int write_model(const char *dir, int flip, int pad) { + char hdr[128], path[256]; + int hl = snprintf(hdr, sizeof(hdr), + "{\"t0\":{\"dtype\":\"F32\",\"shape\":[8],\"data_offsets\":[0,32]}}"); + if (flip) hdr[hl - 2] = ' '; /* inside the JSON header, same length */ + snprintf(path, sizeof(path), "%s/model.safetensors", dir); + FILE *f = fopen(path, "wb"); + if (!f) { perror(path); return -1; } + uint64_t n = (uint64_t)hl; + fwrite(&n, 8, 1, f); + fwrite(hdr, 1, (size_t)hl, f); + float data[8] = {1, -2, 3.5f, 0, 42, -0.5f, 7, 8}; + fwrite(data, 4, 8, f); + for (int i = 0; i < pad; i++) fputc(0, f); + fclose(f); + return 0; +} + +static void cleanup(void) { + const char *dirs[] = {DIR_A, DIR_B, DIR_C, DIR_D, DIR_E}; + for (int i = 0; i < 5; i++) { + char path[256]; + snprintf(path, sizeof(path), "%s/model.safetensors", dirs[i]); + remove(path); + remove(dirs[i]); + } +} + +int main(void) { + cleanup(); + CHECK(MKDIR(DIR_A) == 0 && MKDIR(DIR_B) == 0 && MKDIR(DIR_C) == 0 && + MKDIR(DIR_D) == 0 && MKDIR(DIR_E) == 0); + CHECK(write_model(DIR_A, 0, 0) == 0); + CHECK(write_model(DIR_B, 0, 0) == 0); + CHECK(write_model(DIR_C, 1, 0) == 0); + CHECK(write_model(DIR_D, 0, 64) == 0); + + shards S; + st_init(&S, DIR_A); + CHECK(S.n == 1 && S.nfd == 1); + st_tensor *t = st_find(&S, "t0"); + CHECK(t != NULL); + + /* without a mirror: replica 0 is the identity, replica 1 is absent */ + CHECK(st_fd_rep(&S, t->fd, 0) == t->fd); + CHECK(st_fd_rep(&S, t->fd, 1) == -1); + + /* identical copy: accepted, and both replicas serve the same bytes */ + CHECK(st_mirror_init(&S, DIR_B) == 1); + int mfd = st_fd_rep(&S, t->fd, 1); + CHECK(mfd >= 0 && mfd != t->fd); + float a[8], b[8]; + CHECK(pread(t->fd, a, t->nbytes, t->off) == t->nbytes); + CHECK(pread(mfd, b, t->nbytes, t->off) == t->nbytes); + CHECK(memcmp(a, b, sizeof(a)) == 0); + st_prefetch_rep(&S, "t0", 1); /* smoke: WILLNEED on the mirror fd */ + st_prefetch_rep(&S, "t0", 0); + + /* divergent header, same size: rejected */ + CHECK(st_mirror_init(&S, DIR_C) == 0); + CHECK(st_fd_rep(&S, t->fd, 1) == -1); + + /* different size: rejected */ + CHECK(st_mirror_init(&S, DIR_D) == 0); + + /* missing file: rejected (partial mirror with zero shards) */ + CHECK(st_mirror_init(&S, DIR_E) == 0); + + /* unknown fd never maps to a replica */ + CHECK(st_fd_rep(&S, 987654, 1) == -1); + + cleanup(); + puts("safetensors mirror tests: ok"); + return 0; +} diff --git a/c/tests/test_stops.c b/c/tests/test_stops.c index 00f922bf..5e7e9d2a 100644 --- a/c/tests/test_stops.c +++ b/c/tests/test_stops.c @@ -20,7 +20,7 @@ * Defense 2 is what makes this robust against checkpoints we don't control: * even with BOTH configs mutilated, a control token cannot leak into a reply. */ #define main coli_glm_main_unused -#include "../glm.c" +#include "../colibri.c" #undef main static const char *TOKJSON = diff --git a/c/tests/test_tok_o200k.c b/c/tests/test_tok_o200k.c new file mode 100644 index 00000000..fe628908 --- /dev/null +++ b/c/tests/test_tok_o200k.c @@ -0,0 +1,64 @@ +/* o200k pre-tokenizer validation against HF-tokenizers-generated expectations. + * Self-contained for the test-c harness: loads tests/tok_o200k_tiny.json (a + * synthetic byte-level BPE whose Split regex is the o200k pattern — a few KB, + * no model download) and scores tests/tok_o200k_cases.txt, whose expected ids + * were produced by HF `tokenizers` on the same file. Guards the case-aware + * letter matcher, contractions, digit groups, the [\r\n/]* punctuation tail, + * whitespace branches, and added-token atomicity; round-trips every case. + * The cl100k path is untouched by construction (dispatch requires \p{Lu} in + * the tokenizer's own Split pattern) and stays covered by the GLM oracle. */ +#define _GNU_SOURCE +#include "../tok.h" + +int main(void) { + Tok T; + tok_load(&T, "tests/tok_o200k_tiny.json"); + if (!T.o200k) { fprintf(stderr, "test_tok_o200k: o200k pattern not detected\n"); return 1; } + FILE *f = fopen("tests/tok_o200k_cases.txt", "rb"); + if (!f) { perror("tests/tok_o200k_cases.txt"); return 1; } + /* fgets, not getline: MinGW's UCRT lacks getline and this must run on + * the windows job. Case lines are short; 8 KB is generous. */ + char line[8192]; + int pass = 0, tot = 0, dpass = 0; + while (fgets(line, sizeof(line), f)) { + size_t nr = strlen(line); + while (nr > 0 && (line[nr-1] == '\n' || line[nr-1] == '\r')) line[--nr] = 0; + if (nr == 0) continue; + char *tab = strchr(line, '\t'); if (!tab) continue; + *tab = 0; + const char *text = line, *idstr = tab + 1; + char tbuf[4096]; int tn = 0; + for (const char *q = text; *q && tn < 4095; q++) { + if (q[0]=='\\' && q[1]=='n') { tbuf[tn++]='\n'; q++; } + else if (q[0]=='\\' && q[1]=='t') { tbuf[tn++]='\t'; q++; } + else if (q[0]=='\\' && q[1]=='r') { tbuf[tn++]='\r'; q++; } + else if (q[0]=='\\' && q[1]=='\\') { tbuf[tn++]='\\'; q++; } + else tbuf[tn++] = *q; + } + tbuf[tn] = 0; + int exp[512], ne = 0; + for (const char *q = idstr; *q; ) { + while (*q == ',' || *q == ' ') q++; + if (!*q) break; + exp[ne++] = atoi(q); + while (*q && *q != ',') q++; + } + int got[512]; int ng = tok_encode(&T, tbuf, tn, got, 512); + int ok = (ng == ne); + for (int i = 0; i < ng && ok; i++) ok = (got[i] == exp[i]); + tot++; if (ok) pass++; + char dec[8192]; int dn = tok_decode(&T, got, ng, dec, 8191); + int drt = (dn == tn) && !memcmp(dec, tbuf, tn); + if (drt) dpass++; + if (!ok || !drt) { + fprintf(stderr, "MISMATCH text=%s\n exp(%d):", text, ne); + for (int i = 0; i < ne; i++) fprintf(stderr, " %d", exp[i]); + fprintf(stderr, "\n got(%d):", ng); + for (int i = 0; i < ng; i++) fprintf(stderr, " %d", got[i]); + fprintf(stderr, "\n decode_ok=%d\n", drt); + } + } + fclose(f); + printf("test_tok_o200k: ENCODE %d/%d DECODE %d/%d\n", pass, tot, dpass, tot); + return (pass == tot && dpass == tot) ? 0 : 2; +} diff --git a/c/tests/test_topp.c b/c/tests/test_topp.c index 9276507d..49f45d61 100644 --- a/c/tests/test_topp.c +++ b/c/tests/test_topp.c @@ -24,7 +24,7 @@ * No scratch files: the test runs entirely in memory (no mkdtemp), so it builds clean on * the Windows MinGW CI job without the unmerged compat shim (#352). */ #define main coli_glm_main_unused -#include "../glm.c" +#include "../colibri.c" #undef main #include diff --git a/c/tests/test_uring.c b/c/tests/test_uring.c index e64b9f2f..625c79bb 100644 --- a/c/tests/test_uring.c +++ b/c/tests/test_uring.c @@ -6,7 +6,7 @@ #include #include #define main coli_glm_main_unused -#include "../glm.c" +#include "../colibri.c" #undef main static int fail(const char *s){ fprintf(stderr,"FAIL: %s\n",s); return 1; } diff --git a/c/tests/tok_o200k_cases.txt b/c/tests/tok_o200k_cases.txt new file mode 100644 index 00000000..38e37fa5 --- /dev/null +++ b/c/tests/tok_o200k_cases.txt @@ -0,0 +1,40 @@ +hello world 259,32,119,111,114,263 +HelloWorld 72,101,257,111,262 +XMLHttpRequest 88,77,76,72,116,116,112,82,101,113,117,101,115,116 +helloWORLDhello 259,87,79,82,76,68,259 +dog's 100,111,103,270 +DON'T 68,79,78,39,84 +don't 100,111,110,39,116 +I'll've 73,39,257,39,118,101 +O'Brien 79,39,66,114,105,101,110 +the theatre 265,32,116,256,97,116,114,101 +12345 268,52,53 +a1b22c333d4444 97,49,98,50,50,99,51,51,51,100,52,52,52,52 +3.14 51,46,49,52 +http://x.com/a/b 104,116,116,112,58,47,47,120,46,99,111,109,47,97,47,98 +path/to/file 112,97,264,47,116,111,47,102,105,108,101 +a//b///c 97,47,47,98,47,47,47,99 +!!\r\n//x 33,33,13,10,47,47,120 +one\ntwo\r\nthree 111,110,101,10,116,119,111,13,10,264,114,101,101 + \n x 32,32,10,32,32,120 + 32,32,32 +a b c 97,32,32,98,32,32,32,99 +tab\there 116,269,9,256,114,101 +Café 67,97,102,101,204,129 +naiveBayes 110,97,105,118,101,66,97,121,101,115 +Éclair 195,137,99,108,97,105,114 +北京大学 229,140,151,228,186,172,229,164,167,229,173,166 +ΑΒαβ 206,145,206,146,206,177,206,178 +Иван 208,152,208,178,208,176,208,189 +ẞßscharf 225,186,158,195,159,115,99,104,97,114,102 +i̇stanbul 105,204,135,115,116,97,110,98,117,108 +hello<|endoftext|>world 259,274,119,111,114,263 +<|message_user|>hi 275,104,105 +mixedCASEand123 109,105,120,101,100,67,65,83,69,97,110,100,268 +'s 270 + 's 32,39,115 +A 65 +aB 97,66 +Ab 65,98 +AB 65,66 +ab 269 diff --git a/c/tests/tok_o200k_tiny.json b/c/tests/tok_o200k_tiny.json new file mode 100644 index 00000000..829301ec --- /dev/null +++ b/c/tests/tok_o200k_tiny.json @@ -0,0 +1 @@ +{"version": "1.0", "truncation": null, "padding": null, "added_tokens": [{"id": 274, "content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 275, "content": "<|message_user|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}], "normalizer": null, "pre_tokenizer": {"type": "Sequence", "pretokenizers": [{"type": "Split", "pattern": {"Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"}, "behavior": "Isolated", "invert": false}, {"type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false}]}, "post_processor": null, "decoder": {"type": "ByteLevel", "add_prefix_space": true, "trim_offsets": true, "use_regex": true}, "model": {"type": "BPE", "dropout": null, "unk_token": null, "continuing_subword_prefix": null, "end_of_word_suffix": null, "fuse_unk": false, "byte_fallback": false, "ignore_merges": true, "vocab": {"Ā": 0, "ā": 1, "Ă": 2, "ă": 3, "Ą": 4, "ą": 5, "Ć": 6, "ć": 7, "Ĉ": 8, "ĉ": 9, "Ċ": 10, "ċ": 11, "Č": 12, "č": 13, "Ď": 14, "ď": 15, "Đ": 16, "đ": 17, "Ē": 18, "ē": 19, "Ĕ": 20, "ĕ": 21, "Ė": 22, "ė": 23, "Ę": 24, "ę": 25, "Ě": 26, "ě": 27, "Ĝ": 28, "ĝ": 29, "Ğ": 30, "ğ": 31, "Ġ": 32, "!": 33, "\"": 34, "#": 35, "$": 36, "%": 37, "&": 38, "'": 39, "(": 40, ")": 41, "*": 42, "+": 43, ",": 44, "-": 45, ".": 46, "/": 47, "0": 48, "1": 49, "2": 50, "3": 51, "4": 52, "5": 53, "6": 54, "7": 55, "8": 56, "9": 57, ":": 58, ";": 59, "<": 60, "=": 61, ">": 62, "?": 63, "@": 64, "A": 65, "B": 66, "C": 67, "D": 68, "E": 69, "F": 70, "G": 71, "H": 72, "I": 73, "J": 74, "K": 75, "L": 76, "M": 77, "N": 78, "O": 79, "P": 80, "Q": 81, "R": 82, "S": 83, "T": 84, "U": 85, "V": 86, "W": 87, "X": 88, "Y": 89, "Z": 90, "[": 91, "\\": 92, "]": 93, "^": 94, "_": 95, "`": 96, "a": 97, "b": 98, "c": 99, "d": 100, "e": 101, "f": 102, "g": 103, "h": 104, "i": 105, "j": 106, "k": 107, "l": 108, "m": 109, "n": 110, "o": 111, "p": 112, "q": 113, "r": 114, "s": 115, "t": 116, "u": 117, "v": 118, "w": 119, "x": 120, "y": 121, "z": 122, "{": 123, "|": 124, "}": 125, "~": 126, "ġ": 127, "Ģ": 128, "ģ": 129, "Ĥ": 130, "ĥ": 131, "Ħ": 132, "ħ": 133, "Ĩ": 134, "ĩ": 135, "Ī": 136, "ī": 137, "Ĭ": 138, "ĭ": 139, "Į": 140, "į": 141, "İ": 142, "ı": 143, "IJ": 144, "ij": 145, "Ĵ": 146, "ĵ": 147, "Ķ": 148, "ķ": 149, "ĸ": 150, "Ĺ": 151, "ĺ": 152, "Ļ": 153, "ļ": 154, "Ľ": 155, "ľ": 156, "Ŀ": 157, "ŀ": 158, "Ł": 159, "ł": 160, "¡": 161, "¢": 162, "£": 163, "¤": 164, "¥": 165, "¦": 166, "§": 167, "¨": 168, "©": 169, "ª": 170, "«": 171, "¬": 172, "Ń": 173, "®": 174, "¯": 175, "°": 176, "±": 177, "²": 178, "³": 179, "´": 180, "µ": 181, "¶": 182, "·": 183, "¸": 184, "¹": 185, "º": 186, "»": 187, "¼": 188, "½": 189, "¾": 190, "¿": 191, "À": 192, "Á": 193, "Â": 194, "Ã": 195, "Ä": 196, "Å": 197, "Æ": 198, "Ç": 199, "È": 200, "É": 201, "Ê": 202, "Ë": 203, "Ì": 204, "Í": 205, "Î": 206, "Ï": 207, "Ð": 208, "Ñ": 209, "Ò": 210, "Ó": 211, "Ô": 212, "Õ": 213, "Ö": 214, "×": 215, "Ø": 216, "Ù": 217, "Ú": 218, "Û": 219, "Ü": 220, "Ý": 221, "Þ": 222, "ß": 223, "à": 224, "á": 225, "â": 226, "ã": 227, "ä": 228, "å": 229, "æ": 230, "ç": 231, "è": 232, "é": 233, "ê": 234, "ë": 235, "ì": 236, "í": 237, "î": 238, "ï": 239, "ð": 240, "ñ": 241, "ò": 242, "ó": 243, "ô": 244, "õ": 245, "ö": 246, "÷": 247, "ø": 248, "ù": 249, "ú": 250, "û": 251, "ü": 252, "ý": 253, "þ": 254, "ÿ": 255, "he": 256, "ll": 257, "hell": 258, "hello": 259, "Wo": 260, "Wor": 261, "World": 262, "ld": 263, "th": 264, "the": 265, "Ġthe": 266, "12": 267, "123": 268, "ab": 269, "'s": 270, "Ġa": 271, "./": 272, "ĊĊ": 273}, "merges": [["h", "e"], ["l", "l"], ["he", "ll"], ["hell", "o"], ["W", "o"], ["Wo", "r"], ["Wor", "ld"], ["l", "d"], ["t", "h"], ["th", "e"], ["Ġ", "the"], ["1", "2"], ["12", "3"], ["a", "b"], ["'", "s"], ["Ġ", "a"], [".", "/"], ["Ċ", "Ċ"]]}} \ No newline at end of file diff --git a/c/tok.h b/c/tok.h index 7d1140bb..48ebba90 100644 --- a/c/tok.h +++ b/c/tok.h @@ -19,6 +19,7 @@ #include #include "json.h" #include "tok_unicode.h" +#include "tok_unicode_o200k.h" /* ---------- hash map (chiavi binarie con lunghezza) ---------- */ typedef struct { const char *k; int klen; int v; int used; } ment; @@ -50,6 +51,7 @@ typedef struct { Special *sp; int nsp; /* added tokens, ordinati per lunghezza decrescente */ uint32_t byte2cp[256]; int byte2cp_len[256]; char byte2str[256][3]; int16_t cp2byte[1024]; + int o200k; /* pre_tokenizer regex family: 0 = cl100k (GLM), 1 = o200k (Inkling) */ } Tok; /* ---------- UTF-8 ---------- */ @@ -88,8 +90,14 @@ static void tk_build_bytemap(Tok *T){ /* ---------- caricamento tokenizer.json ---------- */ static char *tk_read_file(const char *path, long *out_n){ FILE *f=fopen(path,"rb"); if(!f){ perror(path); exit(1); } - fseek(f,0,SEEK_END); long n=ftell(f); fseek(f,0,SEEK_SET); - char *b=malloc(n+1); if(fread(b,1,n,f)!=(size_t)n){} b[n]=0; fclose(f); if(out_n)*out_n=n; return b; + if(fseek(f,0,SEEK_END)!=0){ perror(path); exit(1); } + long n=ftell(f); + if(n<0){ perror(path); exit(1); } + if(n>(1L<<30)){ fprintf(stderr,"%s: file too large (%ld bytes)\n",path,n); exit(1); } /* sanity cap vs a hostile size */ + if(fseek(f,0,SEEK_SET)!=0){ perror(path); exit(1); } + char *b=malloc((size_t)n+1); if(!b){ fprintf(stderr,"OOM reading %s (%ld bytes)\n",path,n); exit(1); } + if(fread(b,1,(size_t)n,f)!=(size_t)n){ fprintf(stderr,"%s: short read\n",path); exit(1); } + b[n]=0; fclose(f); if(out_n)*out_n=n; return b; } static int cmp_sp_len(const void *a, const void *b){ return ((const Special*)b)->len - ((const Special*)a)->len; } @@ -104,14 +112,28 @@ static void tok_load(Tok *T, const char *path){ jval *added=json_get(root,"added_tokens"); if(!vocab||!merges){ fprintf(stderr,"tokenizer.json: missing model.vocab/merges\n"); exit(1); } - /* id massimo per dimensionare id2str */ + /* id massimo per dimensionare id2str. Gli id vengono da un tokenizer.json di + * mirror non fidato: un id NEGATIVO indicizzerebbe id2str[id] SOTTO l'allocazione + * (OOB write) e un added_token privo di "id"/"content" darebbe NULL-deref. */ int maxid=0; - for(int i=0;ilen;i++){ int id=(int)vocab->kids[i]->num; if(id>maxid)maxid=id; } - if(added) for(int i=0;ilen;i++){ int id=(int)json_get(added->kids[i],"id")->num; if(id>maxid)maxid=id; } + for(int i=0;ilen;i++){ + if(vocab->kids[i]->t!=J_NUM){ fprintf(stderr,"tokenizer.json: non-numeric vocab id at %d\n",i); exit(1); } + int id=(int)vocab->kids[i]->num; + if(id<0){ fprintf(stderr,"tokenizer.json: negative vocab id %d\n",id); exit(1); } + if(id>maxid)maxid=id; } + if(added) for(int i=0;ilen;i++){ + jval *ji=json_get(added->kids[i],"id"); + if(!ji||ji->t!=J_NUM){ fprintf(stderr,"tokenizer.json: added_token missing numeric id\n"); exit(1); } + int id=(int)ji->num; + if(id<0){ fprintf(stderr,"tokenizer.json: negative added id %d\n",id); exit(1); } + if(id>maxid)maxid=id; } + /* an id near INT_MAX would overflow n_ids=maxid+1 (UB) and calloc multi-GB */ + if(maxid > (1<<21)){ fprintf(stderr,"tokenizer.json: implausible max vocab id %d\n",maxid); exit(1); } T->n_ids=maxid+1; T->id2str=calloc(T->n_ids,sizeof(char*)); T->id_added=calloc(T->n_ids,sizeof(int)); T->id_special=calloc(T->n_ids,sizeof(int)); + if(!T->id2str||!T->id_added||!T->id_special){ fprintf(stderr,"tokenizer.json: OOM sizing %d ids\n",T->n_ids); exit(1); } /* vocab: stringa -> id (capacita' potenza di 2, ~2-3x) */ int vc=1; while(vc < vocab->len*2) vc<<=1; @@ -126,6 +148,9 @@ static void tok_load(Tok *T, const char *path){ hm_init(&T->merges, mc); for(int i=0;ilen;i++){ jval *pr=merges->kids[i]; + if(!pr||pr->t!=J_ARR||pr->len<2||!pr->kids[0]||!pr->kids[1]|| + pr->kids[0]->t!=J_STR||pr->kids[1]->t!=J_STR){ + fprintf(stderr,"tokenizer.json: malformed merge entry %d\n",i); exit(1); } const char *l=pr->kids[0]->str, *r=pr->kids[1]->str; int ll=(int)strlen(l), rl=(int)strlen(r); char *key=malloc(ll+1+rl); memcpy(key,l,ll); key[ll]=0; memcpy(key+ll+1,r,rl); @@ -136,7 +161,10 @@ static void tok_load(Tok *T, const char *path){ T->nsp=added->len; T->sp=calloc(T->nsp,sizeof(Special)); for(int i=0;ilen;i++){ jval *a=added->kids[i]; - char *content=json_get(a,"content")->str; int id=(int)json_get(a,"id")->num; + jval *jc=json_get(a,"content"), *ji=json_get(a,"id"); + if(!jc||jc->t!=J_STR||!jc->str||!ji||ji->t!=J_NUM){ fprintf(stderr,"tokenizer.json: malformed added_token\n"); exit(1); } + char *content=jc->str; int id=(int)ji->num; + if(id<0||id>maxid){ fprintf(stderr,"tokenizer.json: added id %d out of range\n",id); exit(1); } T->sp[i].str=content; T->sp[i].len=(int)strlen(content); T->sp[i].id=id; T->id2str[id]=content; T->id_added[id]=1; jval *sf=json_get(a,"special"); /* "special": true/false */ @@ -144,6 +172,17 @@ static void tok_load(Tok *T, const char *path){ } qsort(T->sp,T->nsp,sizeof(Special),cmp_sp_len); /* match piu' lungo per primo */ } + /* pre_tokenizer family: the o200k Split regex is recognizable by its + * case-category classes (\p{Lu}...) which cl100k does not use */ + jval *pt=json_get(root,"pre_tokenizer"); + if(pt){ + jval *ps=json_get(pt,"pretokenizers"); + if(ps&&ps->t==J_ARR) for(int i=0;ilen;i++){ + jval *pat=json_get(ps->kids[i],"pattern"); + jval *rx=pat?json_get(pat,"Regex"):NULL; + if(rx&&rx->t==J_STR&&strstr(rx->str,"\\p{Lu}")) T->o200k=1; + } + } /* arena/buf restano allocati: le stringhe (j_dup) sono malloc indipendenti e ci servono vive */ (void)arena; } @@ -241,6 +280,104 @@ static void pretok_chunk(Tok *T, const unsigned char *p, int a, int b, int *out, free(cp); free(off); } +/* ---------- pre-tokenizer o200k (Inkling / GPT-4o family) ---------- + * Split regex: + * A: [^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]*[\p{Ll}\p{Lm}\p{Lo}\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)? + * B: [^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]+[\p{Ll}\p{Lm}\p{Lo}\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)? + * C: \p{N}{1,3} D: ' ?[^\s\p{L}\p{N}]+[\r\n/]*' E: \s*[\r\n]+ F: \s+(?!\S) G: \s+ + * S1 = Lu|Lt|Lm|Lo|M, S2 = Ll|Lm|Lo|M. The letter matcher below replays the + * regex engine's backtracking order exactly: A with greedy optional prefix and + * maximally-greedy S1* given back until S2+ can take >=1 char, then B. */ +#define O2_S1(c) (is_U(c)||is_X(c)) +#define O2_S2(c) (is_X(c)||(is_L(c)&&!is_U(c))) +static uint32_t o2_low(uint32_t c){ return (c>='A'&&c<='Z')?c+32:c; } +static int o2_contraction(const uint32_t *cp, int n, int k){ + if(k=0; pfx--){ + int j0=i; + if(pfx){ + uint32_t c=cp[i]; + if(c=='\r'||c=='\n'||is_L(c)||is_N(c)||i+1>=n) continue; + j0=i+1; + } + int m1=j0; while(m1=j0; s--){ + if(s=0; pfx--){ + int j0=i; + if(pfx){ + uint32_t c=cp[i]; + if(c=='\r'||c=='\n'||is_L(c)||is_N(c)||i+1>=n) continue; + j0=i+1; + } + int m1=j0; while(m1j0){ + int k=m1; while(ki){ i=e; bpe_piece(T,p,off[start],off[i],out,no,max); continue; } + } + /* C: \p{N}{1,3} */ + if(is_N(c)){ int j=i,k=0; while(ji){ int last=-1; for(int j=i;j=0){ i=last+1; bpe_piece(T,p,off[start],off[i],out,no,max); continue; } + int end = (r id (split sugli added token, poi pretok+BPE) ---------- */ static int tok_encode(Tok *T, const char *text, int len, int *out, int max){ const unsigned char *p=(const unsigned char*)text; int no=0; int i=0; @@ -254,7 +391,10 @@ static int tok_encode(Tok *T, const char *text, int len, int *out, int max){ } } int chunk_end = (hitpos<0) ? len : hitpos; - if(chunk_end>i) pretok_chunk(T,p,i,chunk_end,out,&no,max); + if(chunk_end>i){ + if(T->o200k) pretok_chunk_o200k(T,p,i,chunk_end,out,&no,max); + else pretok_chunk(T,p,i,chunk_end,out,&no,max); + } if(hitpos<0) break; if(no + +static const uint32_t uni_U[][2] = { + {0x41,0x5A},{0xC0,0xD6},{0xD8,0xDE},{0x100,0x100},{0x102,0x102},{0x104,0x104}, + {0x106,0x106},{0x108,0x108},{0x10A,0x10A},{0x10C,0x10C},{0x10E,0x10E},{0x110,0x110}, + {0x112,0x112},{0x114,0x114},{0x116,0x116},{0x118,0x118},{0x11A,0x11A},{0x11C,0x11C}, + {0x11E,0x11E},{0x120,0x120},{0x122,0x122},{0x124,0x124},{0x126,0x126},{0x128,0x128}, + {0x12A,0x12A},{0x12C,0x12C},{0x12E,0x12E},{0x130,0x130},{0x132,0x132},{0x134,0x134}, + {0x136,0x136},{0x139,0x139},{0x13B,0x13B},{0x13D,0x13D},{0x13F,0x13F},{0x141,0x141}, + {0x143,0x143},{0x145,0x145},{0x147,0x147},{0x14A,0x14A},{0x14C,0x14C},{0x14E,0x14E}, + {0x150,0x150},{0x152,0x152},{0x154,0x154},{0x156,0x156},{0x158,0x158},{0x15A,0x15A}, + {0x15C,0x15C},{0x15E,0x15E},{0x160,0x160},{0x162,0x162},{0x164,0x164},{0x166,0x166}, + {0x168,0x168},{0x16A,0x16A},{0x16C,0x16C},{0x16E,0x16E},{0x170,0x170},{0x172,0x172}, + {0x174,0x174},{0x176,0x176},{0x178,0x179},{0x17B,0x17B},{0x17D,0x17D},{0x181,0x182}, + {0x184,0x184},{0x186,0x187},{0x189,0x18B},{0x18E,0x191},{0x193,0x194},{0x196,0x198}, + {0x19C,0x19D},{0x19F,0x1A0},{0x1A2,0x1A2},{0x1A4,0x1A4},{0x1A6,0x1A7},{0x1A9,0x1A9}, + {0x1AC,0x1AC},{0x1AE,0x1AF},{0x1B1,0x1B3},{0x1B5,0x1B5},{0x1B7,0x1B8},{0x1BC,0x1BC}, + {0x1C4,0x1C5},{0x1C7,0x1C8},{0x1CA,0x1CB},{0x1CD,0x1CD},{0x1CF,0x1CF},{0x1D1,0x1D1}, + {0x1D3,0x1D3},{0x1D5,0x1D5},{0x1D7,0x1D7},{0x1D9,0x1D9},{0x1DB,0x1DB},{0x1DE,0x1DE}, + {0x1E0,0x1E0},{0x1E2,0x1E2},{0x1E4,0x1E4},{0x1E6,0x1E6},{0x1E8,0x1E8},{0x1EA,0x1EA}, + {0x1EC,0x1EC},{0x1EE,0x1EE},{0x1F1,0x1F2},{0x1F4,0x1F4},{0x1F6,0x1F8},{0x1FA,0x1FA}, + {0x1FC,0x1FC},{0x1FE,0x1FE},{0x200,0x200},{0x202,0x202},{0x204,0x204},{0x206,0x206}, + {0x208,0x208},{0x20A,0x20A},{0x20C,0x20C},{0x20E,0x20E},{0x210,0x210},{0x212,0x212}, + {0x214,0x214},{0x216,0x216},{0x218,0x218},{0x21A,0x21A},{0x21C,0x21C},{0x21E,0x21E}, + {0x220,0x220},{0x222,0x222},{0x224,0x224},{0x226,0x226},{0x228,0x228},{0x22A,0x22A}, + {0x22C,0x22C},{0x22E,0x22E},{0x230,0x230},{0x232,0x232},{0x23A,0x23B},{0x23D,0x23E}, + {0x241,0x241},{0x243,0x246},{0x248,0x248},{0x24A,0x24A},{0x24C,0x24C},{0x24E,0x24E}, + {0x370,0x370},{0x372,0x372},{0x376,0x376},{0x37F,0x37F},{0x386,0x386},{0x388,0x38A}, + {0x38C,0x38C},{0x38E,0x38F},{0x391,0x3A1},{0x3A3,0x3AB},{0x3CF,0x3CF},{0x3D2,0x3D4}, + {0x3D8,0x3D8},{0x3DA,0x3DA},{0x3DC,0x3DC},{0x3DE,0x3DE},{0x3E0,0x3E0},{0x3E2,0x3E2}, + {0x3E4,0x3E4},{0x3E6,0x3E6},{0x3E8,0x3E8},{0x3EA,0x3EA},{0x3EC,0x3EC},{0x3EE,0x3EE}, + {0x3F4,0x3F4},{0x3F7,0x3F7},{0x3F9,0x3FA},{0x3FD,0x42F},{0x460,0x460},{0x462,0x462}, + {0x464,0x464},{0x466,0x466},{0x468,0x468},{0x46A,0x46A},{0x46C,0x46C},{0x46E,0x46E}, + {0x470,0x470},{0x472,0x472},{0x474,0x474},{0x476,0x476},{0x478,0x478},{0x47A,0x47A}, + {0x47C,0x47C},{0x47E,0x47E},{0x480,0x480},{0x48A,0x48A},{0x48C,0x48C},{0x48E,0x48E}, + {0x490,0x490},{0x492,0x492},{0x494,0x494},{0x496,0x496},{0x498,0x498},{0x49A,0x49A}, + {0x49C,0x49C},{0x49E,0x49E},{0x4A0,0x4A0},{0x4A2,0x4A2},{0x4A4,0x4A4},{0x4A6,0x4A6}, + {0x4A8,0x4A8},{0x4AA,0x4AA},{0x4AC,0x4AC},{0x4AE,0x4AE},{0x4B0,0x4B0},{0x4B2,0x4B2}, + {0x4B4,0x4B4},{0x4B6,0x4B6},{0x4B8,0x4B8},{0x4BA,0x4BA},{0x4BC,0x4BC},{0x4BE,0x4BE}, + {0x4C0,0x4C1},{0x4C3,0x4C3},{0x4C5,0x4C5},{0x4C7,0x4C7},{0x4C9,0x4C9},{0x4CB,0x4CB}, + {0x4CD,0x4CD},{0x4D0,0x4D0},{0x4D2,0x4D2},{0x4D4,0x4D4},{0x4D6,0x4D6},{0x4D8,0x4D8}, + {0x4DA,0x4DA},{0x4DC,0x4DC},{0x4DE,0x4DE},{0x4E0,0x4E0},{0x4E2,0x4E2},{0x4E4,0x4E4}, + {0x4E6,0x4E6},{0x4E8,0x4E8},{0x4EA,0x4EA},{0x4EC,0x4EC},{0x4EE,0x4EE},{0x4F0,0x4F0}, + {0x4F2,0x4F2},{0x4F4,0x4F4},{0x4F6,0x4F6},{0x4F8,0x4F8},{0x4FA,0x4FA},{0x4FC,0x4FC}, + {0x4FE,0x4FE},{0x500,0x500},{0x502,0x502},{0x504,0x504},{0x506,0x506},{0x508,0x508}, + {0x50A,0x50A},{0x50C,0x50C},{0x50E,0x50E},{0x510,0x510},{0x512,0x512},{0x514,0x514}, + {0x516,0x516},{0x518,0x518},{0x51A,0x51A},{0x51C,0x51C},{0x51E,0x51E},{0x520,0x520}, + {0x522,0x522},{0x524,0x524},{0x526,0x526},{0x528,0x528},{0x52A,0x52A},{0x52C,0x52C}, + {0x52E,0x52E},{0x531,0x556},{0x10A0,0x10C5},{0x10C7,0x10C7},{0x10CD,0x10CD},{0x13A0,0x13F5}, + {0x1C90,0x1CBA},{0x1CBD,0x1CBF},{0x1E00,0x1E00},{0x1E02,0x1E02},{0x1E04,0x1E04},{0x1E06,0x1E06}, + {0x1E08,0x1E08},{0x1E0A,0x1E0A},{0x1E0C,0x1E0C},{0x1E0E,0x1E0E},{0x1E10,0x1E10},{0x1E12,0x1E12}, + {0x1E14,0x1E14},{0x1E16,0x1E16},{0x1E18,0x1E18},{0x1E1A,0x1E1A},{0x1E1C,0x1E1C},{0x1E1E,0x1E1E}, + {0x1E20,0x1E20},{0x1E22,0x1E22},{0x1E24,0x1E24},{0x1E26,0x1E26},{0x1E28,0x1E28},{0x1E2A,0x1E2A}, + {0x1E2C,0x1E2C},{0x1E2E,0x1E2E},{0x1E30,0x1E30},{0x1E32,0x1E32},{0x1E34,0x1E34},{0x1E36,0x1E36}, + {0x1E38,0x1E38},{0x1E3A,0x1E3A},{0x1E3C,0x1E3C},{0x1E3E,0x1E3E},{0x1E40,0x1E40},{0x1E42,0x1E42}, + {0x1E44,0x1E44},{0x1E46,0x1E46},{0x1E48,0x1E48},{0x1E4A,0x1E4A},{0x1E4C,0x1E4C},{0x1E4E,0x1E4E}, + {0x1E50,0x1E50},{0x1E52,0x1E52},{0x1E54,0x1E54},{0x1E56,0x1E56},{0x1E58,0x1E58},{0x1E5A,0x1E5A}, + {0x1E5C,0x1E5C},{0x1E5E,0x1E5E},{0x1E60,0x1E60},{0x1E62,0x1E62},{0x1E64,0x1E64},{0x1E66,0x1E66}, + {0x1E68,0x1E68},{0x1E6A,0x1E6A},{0x1E6C,0x1E6C},{0x1E6E,0x1E6E},{0x1E70,0x1E70},{0x1E72,0x1E72}, + {0x1E74,0x1E74},{0x1E76,0x1E76},{0x1E78,0x1E78},{0x1E7A,0x1E7A},{0x1E7C,0x1E7C},{0x1E7E,0x1E7E}, + {0x1E80,0x1E80},{0x1E82,0x1E82},{0x1E84,0x1E84},{0x1E86,0x1E86},{0x1E88,0x1E88},{0x1E8A,0x1E8A}, + {0x1E8C,0x1E8C},{0x1E8E,0x1E8E},{0x1E90,0x1E90},{0x1E92,0x1E92},{0x1E94,0x1E94},{0x1E9E,0x1E9E}, + {0x1EA0,0x1EA0},{0x1EA2,0x1EA2},{0x1EA4,0x1EA4},{0x1EA6,0x1EA6},{0x1EA8,0x1EA8},{0x1EAA,0x1EAA}, + {0x1EAC,0x1EAC},{0x1EAE,0x1EAE},{0x1EB0,0x1EB0},{0x1EB2,0x1EB2},{0x1EB4,0x1EB4},{0x1EB6,0x1EB6}, + {0x1EB8,0x1EB8},{0x1EBA,0x1EBA},{0x1EBC,0x1EBC},{0x1EBE,0x1EBE},{0x1EC0,0x1EC0},{0x1EC2,0x1EC2}, + {0x1EC4,0x1EC4},{0x1EC6,0x1EC6},{0x1EC8,0x1EC8},{0x1ECA,0x1ECA},{0x1ECC,0x1ECC},{0x1ECE,0x1ECE}, + {0x1ED0,0x1ED0},{0x1ED2,0x1ED2},{0x1ED4,0x1ED4},{0x1ED6,0x1ED6},{0x1ED8,0x1ED8},{0x1EDA,0x1EDA}, + {0x1EDC,0x1EDC},{0x1EDE,0x1EDE},{0x1EE0,0x1EE0},{0x1EE2,0x1EE2},{0x1EE4,0x1EE4},{0x1EE6,0x1EE6}, + {0x1EE8,0x1EE8},{0x1EEA,0x1EEA},{0x1EEC,0x1EEC},{0x1EEE,0x1EEE},{0x1EF0,0x1EF0},{0x1EF2,0x1EF2}, + {0x1EF4,0x1EF4},{0x1EF6,0x1EF6},{0x1EF8,0x1EF8},{0x1EFA,0x1EFA},{0x1EFC,0x1EFC},{0x1EFE,0x1EFE}, + {0x1F08,0x1F0F},{0x1F18,0x1F1D},{0x1F28,0x1F2F},{0x1F38,0x1F3F},{0x1F48,0x1F4D},{0x1F59,0x1F59}, + {0x1F5B,0x1F5B},{0x1F5D,0x1F5D},{0x1F5F,0x1F5F},{0x1F68,0x1F6F},{0x1F88,0x1F8F},{0x1F98,0x1F9F}, + {0x1FA8,0x1FAF},{0x1FB8,0x1FBC},{0x1FC8,0x1FCC},{0x1FD8,0x1FDB},{0x1FE8,0x1FEC},{0x1FF8,0x1FFC}, + {0x2102,0x2102},{0x2107,0x2107},{0x210B,0x210D},{0x2110,0x2112},{0x2115,0x2115},{0x2119,0x211D}, + {0x2124,0x2124},{0x2126,0x2126},{0x2128,0x2128},{0x212A,0x212D},{0x2130,0x2133},{0x213E,0x213F}, + {0x2145,0x2145},{0x2183,0x2183},{0x2C00,0x2C2E},{0x2C60,0x2C60},{0x2C62,0x2C64},{0x2C67,0x2C67}, + {0x2C69,0x2C69},{0x2C6B,0x2C6B},{0x2C6D,0x2C70},{0x2C72,0x2C72},{0x2C75,0x2C75},{0x2C7E,0x2C80}, + {0x2C82,0x2C82},{0x2C84,0x2C84},{0x2C86,0x2C86},{0x2C88,0x2C88},{0x2C8A,0x2C8A},{0x2C8C,0x2C8C}, + {0x2C8E,0x2C8E},{0x2C90,0x2C90},{0x2C92,0x2C92},{0x2C94,0x2C94},{0x2C96,0x2C96},{0x2C98,0x2C98}, + {0x2C9A,0x2C9A},{0x2C9C,0x2C9C},{0x2C9E,0x2C9E},{0x2CA0,0x2CA0},{0x2CA2,0x2CA2},{0x2CA4,0x2CA4}, + {0x2CA6,0x2CA6},{0x2CA8,0x2CA8},{0x2CAA,0x2CAA},{0x2CAC,0x2CAC},{0x2CAE,0x2CAE},{0x2CB0,0x2CB0}, + {0x2CB2,0x2CB2},{0x2CB4,0x2CB4},{0x2CB6,0x2CB6},{0x2CB8,0x2CB8},{0x2CBA,0x2CBA},{0x2CBC,0x2CBC}, + {0x2CBE,0x2CBE},{0x2CC0,0x2CC0},{0x2CC2,0x2CC2},{0x2CC4,0x2CC4},{0x2CC6,0x2CC6},{0x2CC8,0x2CC8}, + {0x2CCA,0x2CCA},{0x2CCC,0x2CCC},{0x2CCE,0x2CCE},{0x2CD0,0x2CD0},{0x2CD2,0x2CD2},{0x2CD4,0x2CD4}, + {0x2CD6,0x2CD6},{0x2CD8,0x2CD8},{0x2CDA,0x2CDA},{0x2CDC,0x2CDC},{0x2CDE,0x2CDE},{0x2CE0,0x2CE0}, + {0x2CE2,0x2CE2},{0x2CEB,0x2CEB},{0x2CED,0x2CED},{0x2CF2,0x2CF2},{0xA640,0xA640},{0xA642,0xA642}, + {0xA644,0xA644},{0xA646,0xA646},{0xA648,0xA648},{0xA64A,0xA64A},{0xA64C,0xA64C},{0xA64E,0xA64E}, + {0xA650,0xA650},{0xA652,0xA652},{0xA654,0xA654},{0xA656,0xA656},{0xA658,0xA658},{0xA65A,0xA65A}, + {0xA65C,0xA65C},{0xA65E,0xA65E},{0xA660,0xA660},{0xA662,0xA662},{0xA664,0xA664},{0xA666,0xA666}, + {0xA668,0xA668},{0xA66A,0xA66A},{0xA66C,0xA66C},{0xA680,0xA680},{0xA682,0xA682},{0xA684,0xA684}, + {0xA686,0xA686},{0xA688,0xA688},{0xA68A,0xA68A},{0xA68C,0xA68C},{0xA68E,0xA68E},{0xA690,0xA690}, + {0xA692,0xA692},{0xA694,0xA694},{0xA696,0xA696},{0xA698,0xA698},{0xA69A,0xA69A},{0xA722,0xA722}, + {0xA724,0xA724},{0xA726,0xA726},{0xA728,0xA728},{0xA72A,0xA72A},{0xA72C,0xA72C},{0xA72E,0xA72E}, + {0xA732,0xA732},{0xA734,0xA734},{0xA736,0xA736},{0xA738,0xA738},{0xA73A,0xA73A},{0xA73C,0xA73C}, + {0xA73E,0xA73E},{0xA740,0xA740},{0xA742,0xA742},{0xA744,0xA744},{0xA746,0xA746},{0xA748,0xA748}, + {0xA74A,0xA74A},{0xA74C,0xA74C},{0xA74E,0xA74E},{0xA750,0xA750},{0xA752,0xA752},{0xA754,0xA754}, + {0xA756,0xA756},{0xA758,0xA758},{0xA75A,0xA75A},{0xA75C,0xA75C},{0xA75E,0xA75E},{0xA760,0xA760}, + {0xA762,0xA762},{0xA764,0xA764},{0xA766,0xA766},{0xA768,0xA768},{0xA76A,0xA76A},{0xA76C,0xA76C}, + {0xA76E,0xA76E},{0xA779,0xA779},{0xA77B,0xA77B},{0xA77D,0xA77E},{0xA780,0xA780},{0xA782,0xA782}, + {0xA784,0xA784},{0xA786,0xA786},{0xA78B,0xA78B},{0xA78D,0xA78D},{0xA790,0xA790},{0xA792,0xA792}, + {0xA796,0xA796},{0xA798,0xA798},{0xA79A,0xA79A},{0xA79C,0xA79C},{0xA79E,0xA79E},{0xA7A0,0xA7A0}, + {0xA7A2,0xA7A2},{0xA7A4,0xA7A4},{0xA7A6,0xA7A6},{0xA7A8,0xA7A8},{0xA7AA,0xA7AE},{0xA7B0,0xA7B4}, + {0xA7B6,0xA7B6},{0xA7B8,0xA7B8},{0xA7BA,0xA7BA},{0xA7BC,0xA7BC},{0xA7BE,0xA7BE},{0xA7C2,0xA7C2}, + {0xA7C4,0xA7C7},{0xA7C9,0xA7C9},{0xA7F5,0xA7F5},{0xFF21,0xFF3A},{0x10400,0x10427},{0x104B0,0x104D3}, + {0x10C80,0x10CB2},{0x118A0,0x118BF},{0x16E40,0x16E5F},{0x1D400,0x1D419},{0x1D434,0x1D44D},{0x1D468,0x1D481}, + {0x1D49C,0x1D49C},{0x1D49E,0x1D49F},{0x1D4A2,0x1D4A2},{0x1D4A5,0x1D4A6},{0x1D4A9,0x1D4AC},{0x1D4AE,0x1D4B5}, + {0x1D4D0,0x1D4E9},{0x1D504,0x1D505},{0x1D507,0x1D50A},{0x1D50D,0x1D514},{0x1D516,0x1D51C},{0x1D538,0x1D539}, + {0x1D53B,0x1D53E},{0x1D540,0x1D544},{0x1D546,0x1D546},{0x1D54A,0x1D550},{0x1D56C,0x1D585},{0x1D5A0,0x1D5B9}, + {0x1D5D4,0x1D5ED},{0x1D608,0x1D621},{0x1D63C,0x1D655},{0x1D670,0x1D689},{0x1D6A8,0x1D6C0},{0x1D6E2,0x1D6FA}, + {0x1D71C,0x1D734},{0x1D756,0x1D76E},{0x1D790,0x1D7A8},{0x1D7CA,0x1D7CA},{0x1E900,0x1E921}, +}; +static const int uni_U_n = 641; + +static const uint32_t uni_X[][2] = { + {0xAA,0xAA},{0xBA,0xBA},{0x1BB,0x1BB},{0x1C0,0x1C3},{0x294,0x294},{0x2B0,0x2C1}, + {0x2C6,0x2D1},{0x2E0,0x2E4},{0x2EC,0x2EC},{0x2EE,0x2EE},{0x300,0x36F},{0x374,0x374}, + {0x37A,0x37A},{0x483,0x489},{0x559,0x559},{0x591,0x5BD},{0x5BF,0x5BF},{0x5C1,0x5C2}, + {0x5C4,0x5C5},{0x5C7,0x5C7},{0x5D0,0x5EA},{0x5EF,0x5F2},{0x610,0x61A},{0x620,0x65F}, + {0x66E,0x6D3},{0x6D5,0x6DC},{0x6DF,0x6E8},{0x6EA,0x6EF},{0x6FA,0x6FC},{0x6FF,0x6FF}, + {0x710,0x74A},{0x74D,0x7B1},{0x7CA,0x7F5},{0x7FA,0x7FA},{0x7FD,0x7FD},{0x800,0x82D}, + {0x840,0x85B},{0x860,0x86A},{0x8A0,0x8B4},{0x8B6,0x8C7},{0x8D3,0x8E1},{0x8E3,0x963}, + {0x971,0x983},{0x985,0x98C},{0x98F,0x990},{0x993,0x9A8},{0x9AA,0x9B0},{0x9B2,0x9B2}, + {0x9B6,0x9B9},{0x9BC,0x9C4},{0x9C7,0x9C8},{0x9CB,0x9CE},{0x9D7,0x9D7},{0x9DC,0x9DD}, + {0x9DF,0x9E3},{0x9F0,0x9F1},{0x9FC,0x9FC},{0x9FE,0x9FE},{0xA01,0xA03},{0xA05,0xA0A}, + {0xA0F,0xA10},{0xA13,0xA28},{0xA2A,0xA30},{0xA32,0xA33},{0xA35,0xA36},{0xA38,0xA39}, + {0xA3C,0xA3C},{0xA3E,0xA42},{0xA47,0xA48},{0xA4B,0xA4D},{0xA51,0xA51},{0xA59,0xA5C}, + {0xA5E,0xA5E},{0xA70,0xA75},{0xA81,0xA83},{0xA85,0xA8D},{0xA8F,0xA91},{0xA93,0xAA8}, + {0xAAA,0xAB0},{0xAB2,0xAB3},{0xAB5,0xAB9},{0xABC,0xAC5},{0xAC7,0xAC9},{0xACB,0xACD}, + {0xAD0,0xAD0},{0xAE0,0xAE3},{0xAF9,0xAFF},{0xB01,0xB03},{0xB05,0xB0C},{0xB0F,0xB10}, + {0xB13,0xB28},{0xB2A,0xB30},{0xB32,0xB33},{0xB35,0xB39},{0xB3C,0xB44},{0xB47,0xB48}, + {0xB4B,0xB4D},{0xB55,0xB57},{0xB5C,0xB5D},{0xB5F,0xB63},{0xB71,0xB71},{0xB82,0xB83}, + {0xB85,0xB8A},{0xB8E,0xB90},{0xB92,0xB95},{0xB99,0xB9A},{0xB9C,0xB9C},{0xB9E,0xB9F}, + {0xBA3,0xBA4},{0xBA8,0xBAA},{0xBAE,0xBB9},{0xBBE,0xBC2},{0xBC6,0xBC8},{0xBCA,0xBCD}, + {0xBD0,0xBD0},{0xBD7,0xBD7},{0xC00,0xC0C},{0xC0E,0xC10},{0xC12,0xC28},{0xC2A,0xC39}, + {0xC3D,0xC44},{0xC46,0xC48},{0xC4A,0xC4D},{0xC55,0xC56},{0xC58,0xC5A},{0xC60,0xC63}, + {0xC80,0xC83},{0xC85,0xC8C},{0xC8E,0xC90},{0xC92,0xCA8},{0xCAA,0xCB3},{0xCB5,0xCB9}, + {0xCBC,0xCC4},{0xCC6,0xCC8},{0xCCA,0xCCD},{0xCD5,0xCD6},{0xCDE,0xCDE},{0xCE0,0xCE3}, + {0xCF1,0xCF2},{0xD00,0xD0C},{0xD0E,0xD10},{0xD12,0xD44},{0xD46,0xD48},{0xD4A,0xD4E}, + {0xD54,0xD57},{0xD5F,0xD63},{0xD7A,0xD7F},{0xD81,0xD83},{0xD85,0xD96},{0xD9A,0xDB1}, + {0xDB3,0xDBB},{0xDBD,0xDBD},{0xDC0,0xDC6},{0xDCA,0xDCA},{0xDCF,0xDD4},{0xDD6,0xDD6}, + {0xDD8,0xDDF},{0xDF2,0xDF3},{0xE01,0xE3A},{0xE40,0xE4E},{0xE81,0xE82},{0xE84,0xE84}, + {0xE86,0xE8A},{0xE8C,0xEA3},{0xEA5,0xEA5},{0xEA7,0xEBD},{0xEC0,0xEC4},{0xEC6,0xEC6}, + {0xEC8,0xECD},{0xEDC,0xEDF},{0xF00,0xF00},{0xF18,0xF19},{0xF35,0xF35},{0xF37,0xF37}, + {0xF39,0xF39},{0xF3E,0xF47},{0xF49,0xF6C},{0xF71,0xF84},{0xF86,0xF97},{0xF99,0xFBC}, + {0xFC6,0xFC6},{0x1000,0x103F},{0x1050,0x108F},{0x109A,0x109D},{0x10FC,0x10FC},{0x1100,0x1248}, + {0x124A,0x124D},{0x1250,0x1256},{0x1258,0x1258},{0x125A,0x125D},{0x1260,0x1288},{0x128A,0x128D}, + {0x1290,0x12B0},{0x12B2,0x12B5},{0x12B8,0x12BE},{0x12C0,0x12C0},{0x12C2,0x12C5},{0x12C8,0x12D6}, + {0x12D8,0x1310},{0x1312,0x1315},{0x1318,0x135A},{0x135D,0x135F},{0x1380,0x138F},{0x1401,0x166C}, + {0x166F,0x167F},{0x1681,0x169A},{0x16A0,0x16EA},{0x16F1,0x16F8},{0x1700,0x170C},{0x170E,0x1714}, + {0x1720,0x1734},{0x1740,0x1753},{0x1760,0x176C},{0x176E,0x1770},{0x1772,0x1773},{0x1780,0x17D3}, + {0x17D7,0x17D7},{0x17DC,0x17DD},{0x180B,0x180D},{0x1820,0x1878},{0x1880,0x18AA},{0x18B0,0x18F5}, + {0x1900,0x191E},{0x1920,0x192B},{0x1930,0x193B},{0x1950,0x196D},{0x1970,0x1974},{0x1980,0x19AB}, + {0x19B0,0x19C9},{0x1A00,0x1A1B},{0x1A20,0x1A5E},{0x1A60,0x1A7C},{0x1A7F,0x1A7F},{0x1AA7,0x1AA7}, + {0x1AB0,0x1AC0},{0x1B00,0x1B4B},{0x1B6B,0x1B73},{0x1B80,0x1BAF},{0x1BBA,0x1BF3},{0x1C00,0x1C37}, + {0x1C4D,0x1C4F},{0x1C5A,0x1C7D},{0x1CD0,0x1CD2},{0x1CD4,0x1CFA},{0x1D2C,0x1D6A},{0x1D78,0x1D78}, + {0x1D9B,0x1DF9},{0x1DFB,0x1DFF},{0x2071,0x2071},{0x207F,0x207F},{0x2090,0x209C},{0x20D0,0x20F0}, + {0x2135,0x2138},{0x2C7C,0x2C7D},{0x2CEF,0x2CF1},{0x2D30,0x2D67},{0x2D6F,0x2D6F},{0x2D7F,0x2D96}, + {0x2DA0,0x2DA6},{0x2DA8,0x2DAE},{0x2DB0,0x2DB6},{0x2DB8,0x2DBE},{0x2DC0,0x2DC6},{0x2DC8,0x2DCE}, + {0x2DD0,0x2DD6},{0x2DD8,0x2DDE},{0x2DE0,0x2DFF},{0x2E2F,0x2E2F},{0x3005,0x3006},{0x302A,0x302F}, + {0x3031,0x3035},{0x303B,0x303C},{0x3041,0x3096},{0x3099,0x309A},{0x309D,0x309F},{0x30A1,0x30FA}, + {0x30FC,0x30FF},{0x3105,0x312F},{0x3131,0x318E},{0x31A0,0x31BF},{0x31F0,0x31FF},{0x3400,0x4DBF}, + {0x4E00,0x9FFC},{0xA000,0xA48C},{0xA4D0,0xA4FD},{0xA500,0xA60C},{0xA610,0xA61F},{0xA62A,0xA62B}, + {0xA66E,0xA672},{0xA674,0xA67D},{0xA67F,0xA67F},{0xA69C,0xA6E5},{0xA6F0,0xA6F1},{0xA717,0xA71F}, + {0xA770,0xA770},{0xA788,0xA788},{0xA78F,0xA78F},{0xA7F7,0xA7F9},{0xA7FB,0xA827},{0xA82C,0xA82C}, + {0xA840,0xA873},{0xA880,0xA8C5},{0xA8E0,0xA8F7},{0xA8FB,0xA8FB},{0xA8FD,0xA8FF},{0xA90A,0xA92D}, + {0xA930,0xA953},{0xA960,0xA97C},{0xA980,0xA9C0},{0xA9CF,0xA9CF},{0xA9E0,0xA9EF},{0xA9FA,0xA9FE}, + {0xAA00,0xAA36},{0xAA40,0xAA4D},{0xAA60,0xAA76},{0xAA7A,0xAAC2},{0xAADB,0xAADD},{0xAAE0,0xAAEF}, + {0xAAF2,0xAAF6},{0xAB01,0xAB06},{0xAB09,0xAB0E},{0xAB11,0xAB16},{0xAB20,0xAB26},{0xAB28,0xAB2E}, + {0xAB5C,0xAB5F},{0xAB69,0xAB69},{0xABC0,0xABEA},{0xABEC,0xABED},{0xAC00,0xD7A3},{0xD7B0,0xD7C6}, + {0xD7CB,0xD7FB},{0xF900,0xFA6D},{0xFA70,0xFAD9},{0xFB1D,0xFB28},{0xFB2A,0xFB36},{0xFB38,0xFB3C}, + {0xFB3E,0xFB3E},{0xFB40,0xFB41},{0xFB43,0xFB44},{0xFB46,0xFBB1},{0xFBD3,0xFD3D},{0xFD50,0xFD8F}, + {0xFD92,0xFDC7},{0xFDF0,0xFDFB},{0xFE00,0xFE0F},{0xFE20,0xFE2F},{0xFE70,0xFE74},{0xFE76,0xFEFC}, + {0xFF66,0xFFBE},{0xFFC2,0xFFC7},{0xFFCA,0xFFCF},{0xFFD2,0xFFD7},{0xFFDA,0xFFDC},{0x10000,0x1000B}, + {0x1000D,0x10026},{0x10028,0x1003A},{0x1003C,0x1003D},{0x1003F,0x1004D},{0x10050,0x1005D},{0x10080,0x100FA}, + {0x101FD,0x101FD},{0x10280,0x1029C},{0x102A0,0x102D0},{0x102E0,0x102E0},{0x10300,0x1031F},{0x1032D,0x10340}, + {0x10342,0x10349},{0x10350,0x1037A},{0x10380,0x1039D},{0x103A0,0x103C3},{0x103C8,0x103CF},{0x10450,0x1049D}, + {0x10500,0x10527},{0x10530,0x10563},{0x10600,0x10736},{0x10740,0x10755},{0x10760,0x10767},{0x10800,0x10805}, + {0x10808,0x10808},{0x1080A,0x10835},{0x10837,0x10838},{0x1083C,0x1083C},{0x1083F,0x10855},{0x10860,0x10876}, + {0x10880,0x1089E},{0x108E0,0x108F2},{0x108F4,0x108F5},{0x10900,0x10915},{0x10920,0x10939},{0x10980,0x109B7}, + {0x109BE,0x109BF},{0x10A00,0x10A03},{0x10A05,0x10A06},{0x10A0C,0x10A13},{0x10A15,0x10A17},{0x10A19,0x10A35}, + {0x10A38,0x10A3A},{0x10A3F,0x10A3F},{0x10A60,0x10A7C},{0x10A80,0x10A9C},{0x10AC0,0x10AC7},{0x10AC9,0x10AE6}, + {0x10B00,0x10B35},{0x10B40,0x10B55},{0x10B60,0x10B72},{0x10B80,0x10B91},{0x10C00,0x10C48},{0x10D00,0x10D27}, + {0x10E80,0x10EA9},{0x10EAB,0x10EAC},{0x10EB0,0x10EB1},{0x10F00,0x10F1C},{0x10F27,0x10F27},{0x10F30,0x10F50}, + {0x10FB0,0x10FC4},{0x10FE0,0x10FF6},{0x11000,0x11046},{0x1107F,0x110BA},{0x110D0,0x110E8},{0x11100,0x11134}, + {0x11144,0x11147},{0x11150,0x11173},{0x11176,0x11176},{0x11180,0x111C4},{0x111C9,0x111CC},{0x111CE,0x111CF}, + {0x111DA,0x111DA},{0x111DC,0x111DC},{0x11200,0x11211},{0x11213,0x11237},{0x1123E,0x1123E},{0x11280,0x11286}, + {0x11288,0x11288},{0x1128A,0x1128D},{0x1128F,0x1129D},{0x1129F,0x112A8},{0x112B0,0x112EA},{0x11300,0x11303}, + {0x11305,0x1130C},{0x1130F,0x11310},{0x11313,0x11328},{0x1132A,0x11330},{0x11332,0x11333},{0x11335,0x11339}, + {0x1133B,0x11344},{0x11347,0x11348},{0x1134B,0x1134D},{0x11350,0x11350},{0x11357,0x11357},{0x1135D,0x11363}, + {0x11366,0x1136C},{0x11370,0x11374},{0x11400,0x1144A},{0x1145E,0x11461},{0x11480,0x114C5},{0x114C7,0x114C7}, + {0x11580,0x115B5},{0x115B8,0x115C0},{0x115D8,0x115DD},{0x11600,0x11640},{0x11644,0x11644},{0x11680,0x116B8}, + {0x11700,0x1171A},{0x1171D,0x1172B},{0x11800,0x1183A},{0x118FF,0x11906},{0x11909,0x11909},{0x1190C,0x11913}, + {0x11915,0x11916},{0x11918,0x11935},{0x11937,0x11938},{0x1193B,0x11943},{0x119A0,0x119A7},{0x119AA,0x119D7}, + {0x119DA,0x119E1},{0x119E3,0x119E4},{0x11A00,0x11A3E},{0x11A47,0x11A47},{0x11A50,0x11A99},{0x11A9D,0x11A9D}, + {0x11AC0,0x11AF8},{0x11C00,0x11C08},{0x11C0A,0x11C36},{0x11C38,0x11C40},{0x11C72,0x11C8F},{0x11C92,0x11CA7}, + {0x11CA9,0x11CB6},{0x11D00,0x11D06},{0x11D08,0x11D09},{0x11D0B,0x11D36},{0x11D3A,0x11D3A},{0x11D3C,0x11D3D}, + {0x11D3F,0x11D47},{0x11D60,0x11D65},{0x11D67,0x11D68},{0x11D6A,0x11D8E},{0x11D90,0x11D91},{0x11D93,0x11D98}, + {0x11EE0,0x11EF6},{0x11FB0,0x11FB0},{0x12000,0x12399},{0x12480,0x12543},{0x13000,0x1342E},{0x14400,0x14646}, + {0x16800,0x16A38},{0x16A40,0x16A5E},{0x16AD0,0x16AED},{0x16AF0,0x16AF4},{0x16B00,0x16B36},{0x16B40,0x16B43}, + {0x16B63,0x16B77},{0x16B7D,0x16B8F},{0x16F00,0x16F4A},{0x16F4F,0x16F87},{0x16F8F,0x16F9F},{0x16FE0,0x16FE1}, + {0x16FE3,0x16FE4},{0x16FF0,0x16FF1},{0x17000,0x187F7},{0x18800,0x18CD5},{0x18D00,0x18D08},{0x1B000,0x1B11E}, + {0x1B150,0x1B152},{0x1B164,0x1B167},{0x1B170,0x1B2FB},{0x1BC00,0x1BC6A},{0x1BC70,0x1BC7C},{0x1BC80,0x1BC88}, + {0x1BC90,0x1BC99},{0x1BC9D,0x1BC9E},{0x1D165,0x1D169},{0x1D16D,0x1D172},{0x1D17B,0x1D182},{0x1D185,0x1D18B}, + {0x1D1AA,0x1D1AD},{0x1D242,0x1D244},{0x1DA00,0x1DA36},{0x1DA3B,0x1DA6C},{0x1DA75,0x1DA75},{0x1DA84,0x1DA84}, + {0x1DA9B,0x1DA9F},{0x1DAA1,0x1DAAF},{0x1E000,0x1E006},{0x1E008,0x1E018},{0x1E01B,0x1E021},{0x1E023,0x1E024}, + {0x1E026,0x1E02A},{0x1E100,0x1E12C},{0x1E130,0x1E13D},{0x1E14E,0x1E14E},{0x1E2C0,0x1E2EF},{0x1E800,0x1E8C4}, + {0x1E8D0,0x1E8D6},{0x1E944,0x1E94B},{0x1EE00,0x1EE03},{0x1EE05,0x1EE1F},{0x1EE21,0x1EE22},{0x1EE24,0x1EE24}, + {0x1EE27,0x1EE27},{0x1EE29,0x1EE32},{0x1EE34,0x1EE37},{0x1EE39,0x1EE39},{0x1EE3B,0x1EE3B},{0x1EE42,0x1EE42}, + {0x1EE47,0x1EE47},{0x1EE49,0x1EE49},{0x1EE4B,0x1EE4B},{0x1EE4D,0x1EE4F},{0x1EE51,0x1EE52},{0x1EE54,0x1EE54}, + {0x1EE57,0x1EE57},{0x1EE59,0x1EE59},{0x1EE5B,0x1EE5B},{0x1EE5D,0x1EE5D},{0x1EE5F,0x1EE5F},{0x1EE61,0x1EE62}, + {0x1EE64,0x1EE64},{0x1EE67,0x1EE6A},{0x1EE6C,0x1EE72},{0x1EE74,0x1EE77},{0x1EE79,0x1EE7C},{0x1EE7E,0x1EE7E}, + {0x1EE80,0x1EE89},{0x1EE8B,0x1EE9B},{0x1EEA1,0x1EEA3},{0x1EEA5,0x1EEA9},{0x1EEAB,0x1EEBB},{0x20000,0x2A6DD}, + {0x2A700,0x2B734},{0x2B740,0x2B81D},{0x2B820,0x2CEA1},{0x2CEB0,0x2EBE0},{0x2F800,0x2FA1D},{0x30000,0x3134A}, + {0xE0100,0xE01EF}, +}; +static const int uni_X_n = 595; + +static inline int is_U(uint32_t c){ return uni_in(uni_U,uni_U_n,c); } +static inline int is_X(uint32_t c){ return uni_in(uni_X,uni_X_n,c); } +#endif diff --git a/c/tools/convert_fp8_to_int4.py b/c/tools/convert_fp8_to_int4.py index eac51f95..6647e071 100644 --- a/c/tools/convert_fp8_to_int4.py +++ b/c/tools/convert_fp8_to_int4.py @@ -83,6 +83,45 @@ def quant_int4_grouped(w, bits, gs=128): s_flat = s[:, :, 0].astype(np.float32).reshape(-1) return out.reshape(-1), s_flat +def quant_int3_g64(w, bits=3, group=64): # -> (qbytes U8 [O*ceil(I/64)*24], scales f32 [O*ceil(I/64)]) + """int3 with PER-GROUP scales (fmt=5 in colibri.c): per 64-input group, symmetric absmax + (qmax=3, clamp [-4,3], stored v+4), packed as 16B low plane (2 bits/val, int2 layout) + + 8B high plane (1 bit/val). Same math as quant_ablation._quant_last_dim(bits=3, + group=64) (#132), here with real packing. 3.5 bits/weight effective.""" + O, I = w.shape + ng = (I + group - 1) // group + pad = ng * group - I + wp = np.pad(w, ((0, 0), (0, pad))) if pad else w + g = wp.reshape(O, ng, group) + amax = np.abs(g).max(axis=2, keepdims=True) + s = np.maximum(amax / 3.0, 1e-8) + q = (np.clip(np.rint(g / s), -4, 3).astype(np.int32) + 4).astype(np.uint8) # 0..7 + if pad: q[:, -1, group - pad:] = 4 # pad packs as 0 after -4 + lo = np.zeros((O, ng, 16), np.uint8) + for k in range(4): + lo |= ((q[:, :, k::4] & 3) << (k * 2)).astype(np.uint8) + hi = np.zeros((O, ng, 8), np.uint8) + for b in range(8): + hi |= (((q[:, :, b::8] >> 2) & 1) << b).astype(np.uint8) + out = np.concatenate([lo, hi], axis=2) # [O, ng, 24] + return out.reshape(-1), s[:, :, 0].astype(np.float32).reshape(-1) + +E8 = "e8" # CLI/bits-plumbing marker for fmt=6 (not a bit width) + +def quant_e8(w): # -> (qbytes U8 [O*ceil(I/256)*98], tag f32 [1]) + """E8/IQ3 lattice (fmt=6 in colibri.c, #452): rotate the rows first (W@Q, + block-diagonal FWHT with regenerated signs — iq3_pack.rotate_rows mirrors + quant.h e8_rot_rows), then pack with the iq3 codec: 98B per 256 weights, + 3.0625 bpw, every scale in-block. The .qs companion is a single float, + the engine's fmt=6 discriminator — not a scale.""" + sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + import iq3_pack + O, I = w.shape + if I % 256: + raise SystemExit(f"e8: input dim {I} is not a multiple of 256") + packed = iq3_pack.encode(iq3_pack.rotate_rows(np.asarray(w, dtype=np.float32))) + return packed.reshape(-1), np.array([6.0], dtype=np.float32) + def quant_int2(w, bits): # -> (qbytes U8 [O*ceil(I/4)], scale f32 [O]); 4/byte O, I = w.shape qmax = (1 << (bits - 1)) - 1 # bits=2 -> qmax=1, valori [-2,1] @@ -214,6 +253,14 @@ def dequant(f, name, keys): return (w * sc).numpy() return f.get_tensor(name).to(torch.float32).numpy() +# Per-projection bit overrides for ROUTED experts (gate_proj/up_proj/down_proj), set from +# --up-bits/--gate-bits/--down-bits in main(). Empty = uniform xbits. Motivated by the +# measured result that up_proj tolerates int3-g64 at ~zero quality cost while int2 craters +# (OLMoE ablation, PR #168 comment): up-only int3 drops ~8% of expert bytes for free. +# NB: the resume manifests (check_or_record_params and the --indir progress file) already +# record dict(PROJ_BITS) — this global is the definition those sites depend on. +PROJ_BITS = {} + def convert_shard(path, out_dict, n_layers, ebits, io_bits, xbits, keep_mtp=False, keep_idx=False, group_size=0, bits_map=None): from safetensors import safe_open @@ -235,9 +282,19 @@ def convert_shard(path, out_dict, n_layers, ebits, io_bits, xbits, # Any unknown kind that fell through classify as "q" if bits_map and kind not in bits_map and kind not in ("io", "x", "sh", "o", "kvb", "attn", "dmlp"): bits = ebits + # Per-projection override for routed experts, applied on top of the type-level bits. + if kind == "x" and PROJ_BITS: # e.g. up_proj -> 3 (int3-g64) while gate/down stay 4 + for proj, pb in PROJ_BITS.items(): + if f".{proj}.weight" in name: bits = pb; break if w.ndim != 2: # es. bias 1D non previsto come 'q' -> tienilo f32 out_dict[name] = w.astype(np.float32); continue - if group_size > 0 and bits <= 4: + if bits == E8: + # fmt=6 E8/IQ3 — routed-expert projections only, enforced in main(). + q, s = quant_e8(w) + elif bits == 3: + # int3-g64 (fmt=5): inherently group-64, distinct from grouped-int4. + q, s = quant_int3_g64(w) + elif group_size > 0 and bits <= 4: q, s = quant_int4_grouped(w, bits, group_size) else: q, s = (quant_int2(w, bits) if bits <= 2 else @@ -247,6 +304,35 @@ def convert_shard(path, out_dict, n_layers, ebits, io_bits, xbits, def free_gb(p): return shutil.disk_usage(p).free / 1e9 +def check_or_record_params(outdir, prefix, params): + """#383-class guard, mirrored onto the --repo download loops from the --indir + path's resume manifest (below): a resumed run with DIFFERENT conversion + parameters (bits, group size, PROJ_BITS, ...) must not silently mix bit-widths + across shards in the same outdir -- the #355 failure mode (a second pass with + changed flags overwriting/interleaving with a finished container in silence). + Unlike the --indir manifest this doesn't need to track per-shard completion: + the --repo loops already do that via out-NNNNN.safetensors existence, since + shard index maps directly to output filename there. Only whether the params + used SO FAR match this run's needs checking. Returns False (caller should + abort) on a mismatch, True otherwise; records params on first use.""" + path = os.path.join(outdir, f".{prefix}params.json") + if os.path.exists(path): + try: prev = json.loads(open(path).read()) + except (OSError, ValueError): prev = None + if prev is not None and prev != params: + print(f"ERROR: {path} records a conversion with {prev};\n" + f" this run uses {params}. Refusing to mix conversions in the " + f"same outdir — use a fresh --outdir (or delete {path} and the " + f"{prefix}*.safetensors shards to redo).") + return False + tmp = path + ".tmp" + with open(tmp, "w") as f: json.dump(params, f, indent=1) # atomic write, same reasoning as the --indir manifest + os.replace(tmp, path) + return True + +def _bits(v): # "e8" -> fmt=6 marker; anything else an int width + return E8 if v == E8 else int(v) + def main(): ap = argparse.ArgumentParser() ap.add_argument("--repo", default=None) @@ -254,7 +340,7 @@ def main(): ap.add_argument("--outdir", required=False) ap.add_argument("--ebits", type=int, default=None) # bit residenti (default 4; 8 per --mtp/--indexer) ap.add_argument("--io-bits", type=int, default=8) # bit di embed/lm_head - ap.add_argument("--xbits", type=int, default=None) # bit degli expert ROUTED (streaming); default=ebits + ap.add_argument("--xbits", type=_bits, default=None) # bit degli expert ROUTED (streaming), o "e8" (fmt=6); default=ebits # Mixed-precision: per-tensor-type bit overrides. Default = ebits (all same). # Set these higher to protect sensitive tensors from quantization error. ap.add_argument("--shared-bits", type=int, default=None, @@ -269,6 +355,13 @@ def main(): help="bits for dense MLP (first 3 layers). Default=ebits") ap.add_argument("--group-size", type=int, default=0, # 0 = per-row (backward compat); 128 = group-scaled help="group size for int4 scales: 0=per-row (default), 128=one scale per 128 elements (much better quality)") + # Per-projection bit overrides for routed experts (orthogonal to the type-level flags above). + ap.add_argument("--up-bits", type=_bits, default=None, + help="bits for up_proj in routed experts (e.g. 3 = int3-g64). Default=xbits") + ap.add_argument("--gate-bits", type=_bits, default=None, + help="bits for gate_proj in routed experts. Default=xbits") + ap.add_argument("--down-bits", type=_bits, default=None, + help="bits for down_proj in routed experts. Default=xbits") ap.add_argument("--n-layers", type=int, default=78) ap.add_argument("--min-free-gb", type=float, default=20.0) ap.add_argument("--selftest", action="store_true") @@ -298,6 +391,17 @@ def main(): "embedding half -> MTP acceptance ~0% (issue #8). Use the default --ebits 8, " "or add --group-size 128 for group-scaled int4.") if a.xbits is None: a.xbits = a.ebits + for proj, val in (("gate_proj", a.gate_bits), ("up_proj", a.up_bits), ("down_proj", a.down_bits)): + if val is not None: PROJ_BITS[proj] = val + if PROJ_BITS: + print(f"[per-projection expert bits] {PROJ_BITS} (others -> xbits={a.xbits})") + # fmt=6 is all-or-nothing across the three expert projections: gate and up + # share one rotated input row in the engine (the placement rule in quant.h), + # so a mixed layout would need two gather buffers for zero measured benefit. + eff = [PROJ_BITS.get(p, a.xbits) for p in ("gate_proj", "up_proj", "down_proj")] + if any(b == E8 for b in eff) and not all(b == E8 for b in eff): + raise SystemExit(f"e8 covers all three expert projections or none (got {eff}); " + "use --xbits e8, or none of the e8 flags") # Build per-type bits map. If a type-specific arg is set, use it; otherwise the # converter falls back to ebits for that type. @@ -440,7 +544,8 @@ def get_slice(s, n): return None # EN: resume skips only what matches, and different parameters on the same # EN: outdir are refused instead of mixing containers (the #355 failure mode). params = {"ebits": a.ebits, "io_bits": a.io_bits, "xbits": a.xbits, - "group_size": a.group_size, "n_layers": a.n_layers, "bits_map": bits_map} + "group_size": a.group_size, "n_layers": a.n_layers, "bits_map": bits_map, + "proj_bits": dict(PROJ_BITS)} prog_path = os.path.join(a.outdir, f".{prefix}progress.json") prog = {} if os.path.exists(prog_path): @@ -587,8 +692,9 @@ def worker(t): s0, s1 = segs[t] while done[t] < s1 - s0 and not stopfail: pos = s0 + done[t] - req = urllib.request.Request(url, headers={"User-Agent": "colibri-convert", - "Range": f"bytes={pos}-{s1-1}"}) + _hdrs = {"User-Agent": "colibri-convert", "Range": f"bytes={pos}-{s1-1}"} + if os.environ.get("HF_TOKEN"): _hdrs["Authorization"] = f"Bearer {os.environ['HF_TOKEN']}" + req = urllib.request.Request(url, headers=_hdrs) try: with urllib.request.urlopen(req, timeout=8) as r: if r.status != 206: # Range ignorato: multi-stream impossibile @@ -651,6 +757,7 @@ def _download_single(url, fn, out, part, expected): have0 = have req = urllib.request.Request(url, headers={"User-Agent": "colibri-convert"}) if have: req.add_header("Range", f"bytes={have}-") + if os.environ.get("HF_TOKEN"): req.add_header("Authorization", f"Bearer {os.environ['HF_TOKEN']}") try: with urllib.request.urlopen(req, timeout=8) as r: if have and r.status == 200: # server ha ignorato il Range: riparti pulito @@ -718,6 +825,10 @@ def _download_single(url, fn, out, part, expected): except Exception: pass tmp = os.path.join(a.outdir, "_inflight"); os.makedirs(tmp, exist_ok=True) if a.mtp: + params = {"ebits": a.ebits, "io_bits": a.io_bits, "xbits": a.xbits, + "group_size": a.group_size, "n_layers": a.n_layers, "bits_map": bits_map, + "proj_bits": dict(PROJ_BITS)} + if not check_or_record_params(a.outdir, "out-mtp-", params): return import urllib.request idx = json.loads(urllib.request.urlopen( f"https://huggingface.co/{a.repo}/resolve/main/model.safetensors.index.json", timeout=30).read())["weight_map"] @@ -737,6 +848,10 @@ def _download_single(url, fn, out, part, expected): print(f" -> {os.path.basename(outp)} ({os.path.getsize(outp)/1e9:.2f} GB, {len(out)} tensors)", flush=True) shutil.rmtree(tmp, ignore_errors=True); print("[MTP] DONE."); return if a.indexer: + params = {"ebits": a.ebits, "io_bits": a.io_bits, "xbits": a.xbits, + "group_size": a.group_size, "n_layers": a.n_layers, "bits_map": bits_map, + "proj_bits": dict(PROJ_BITS)} + if not check_or_record_params(a.outdir, "out-idx-", params): return import urllib.request idx = json.loads(urllib.request.urlopen( f"https://huggingface.co/{a.repo}/resolve/main/model.safetensors.index.json", timeout=30).read())["weight_map"] @@ -756,6 +871,10 @@ def _download_single(url, fn, out, part, expected): if os.path.isfile(blob): os.remove(blob) print(f" -> {os.path.basename(outp)} ({len(out)} tensors)", flush=True) shutil.rmtree(tmp, ignore_errors=True); print("[IDX] DONE."); return + params = {"ebits": a.ebits, "io_bits": a.io_bits, "xbits": a.xbits, + "group_size": a.group_size, "n_layers": a.n_layers, "bits_map": bits_map, + "proj_bits": dict(PROJ_BITS)} + if not check_or_record_params(a.outdir, "out-", params): return for i, sh in enumerate(shards): if free_gb(a.outdir) < a.min_free_gb: print(f"STOP: free space is below {a.min_free_gb} GB. Free space and rerun to resume."); break diff --git a/c/tools/convert_olmoe_merged.py b/c/tools/convert_olmoe_merged.py new file mode 100644 index 00000000..883b8df2 --- /dev/null +++ b/c/tools/convert_olmoe_merged.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +"""Convert OLMoE HuggingFace checkpoint to colibri merged int8 format. + +Consolidates gate_proj, up_proj, and down_proj into a single merged tensor per expert. +This allows olmoe.c to load an expert in a single disk read call instead of 3. + +Usage: + python tools/convert_olmoe_merged.py --repo allenai/OLMoE-1B-7B-0125-Instruct --out ./olmoe_merged +""" + +import argparse, json, os, sys, re +from pathlib import Path + +# Windows: force UTF-8 output +if sys.platform == "win32": + for s in (sys.stdout, sys.stderr): + try: s.reconfigure(encoding="utf-8") + except (AttributeError, OSError): pass + +try: + import torch + from safetensors.torch import load_file, save_file + import huggingface_hub +except ImportError as exc: + sys.exit(f"Missing dependencies: {exc}. Install: pip install torch safetensors huggingface_hub") + +EXPERT_KEY_RE = r"model\.layers\.(\d+)\.mlp\.experts\.(\d+)\.(gate_proj|up_proj|down_proj)\.weight" + +def quantize_row(w: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + """Row-wise int8 quantization. Returns (int8_weights, float32_scales).""" + w_f32 = w.float() + row_max = w_f32.abs().amax(dim=1, keepdim=True).clamp(min=1e-12) + scales = row_max / 127.0 + q = (w_f32 / scales).round().clamp(-128, 127).to(torch.int8) + return q, scales.squeeze(1) + +def main(): + ap = argparse.ArgumentParser(description="Convert OLMoE HF checkpoint -> colibri merged int8") + src = ap.add_mutually_exclusive_group(required=True) + src.add_argument("--repo", help="HuggingFace repo ID") + src.add_argument("--model", help="Local HF checkpoint directory") + ap.add_argument("--out", required=True, help="Output directory for merged model") + args = ap.parse_args() + + if args.repo: + from huggingface_hub import snapshot_download + from huggingface_hub.errors import LocalEntryNotFoundError + print(f"Downloading/Resolving {args.repo}...") + try: + src_dir = snapshot_download(args.repo, local_files_only=True, max_workers=4) + except LocalEntryNotFoundError: + src_dir = None + if src_dir is None or not any(Path(src_dir).glob("*.safetensors")): + print("Downloading safetensors...") + src_dir = snapshot_download(args.repo, max_workers=4) + else: + src_dir = args.model + + src = Path(src_dir) + if not src.is_dir(): + sys.exit(f"Model directory not found: {src}") + if not (src / "config.json").is_file(): + sys.exit(f"config.json missing in {src}") + + out = Path(args.out) + out.mkdir(parents=True, exist_ok=True) + + # Copy config.json + import shutil + shutil.copy2(src / "config.json", out / "config.json") + print(f"config.json -> {out}") + + # Process safetensors + shards = sorted(src.glob("*.safetensors")) + if not shards: + sys.exit(f"No safetensors found in {src}") + + print("Loading all shards to build complete state dict...") + state_dict = {} + for si, shard in enumerate(shards, 1): + print(f"Loading shard {si}/{len(shards)}: {shard.name}...") + tensors = load_file(str(shard)) + state_dict.update(tensors) + + # Gather experts + experts = {} + for name in list(state_dict.keys()): + m = re.match(EXPERT_KEY_RE, name) + if m: + layer_idx, expert_idx, proj = m.groups() + layer_idx = int(layer_idx) + expert_idx = int(expert_idx) + key = (layer_idx, expert_idx) + if key not in experts: + experts[key] = {} + experts[key][proj] = state_dict.pop(name) + + print(f"Found {len(experts)} experts to merge.") + + # Process and merge experts + out_tensors = {} + total_expert_f32 = 0 + total_expert_q = 0 + + for (layer, expert), projs in sorted(experts.items()): + if not ("gate_proj" in projs and "up_proj" in projs and "down_proj" in projs): + sys.exit(f"Missing projection for layer {layer} expert {expert}!") + + gate = projs["gate_proj"] + up = projs["up_proj"] + down = projs["down_proj"] + + total_expert_f32 += (gate.numel() + up.numel() + down.numel()) * gate.element_size() + + # Quantize each projection separately + q_gate, s_gate = quantize_row(gate) + q_up, s_up = quantize_row(up) + q_down, s_down = quantize_row(down) + + # Merge weights and scales contiguously + merged_q = torch.cat([q_gate.flatten(), q_up.flatten(), q_down.flatten()]) + merged_scales = torch.cat([s_gate, s_up, s_down]) + + total_expert_q += merged_q.numel() * 1 + merged_scales.numel() * 4 + + # Save to output + out_tensors[f"model.layers.{layer}.mlp.experts.{expert}.merged_weight"] = merged_q + out_tensors[f"model.layers.{layer}.mlp.experts.{expert}.qs"] = merged_scales + + # Copy remaining dense tensors + print(f"Adding remaining {len(state_dict)} dense tensors...") + out_tensors.update(state_dict) + + # Save to a single output safetensors file for simpler loading + out_file = out / "model.safetensors" + print(f"Saving merged safetensors model to {out_file}...") + save_file(out_tensors, str(out_file)) + + ratio = total_expert_q / max(total_expert_f32, 1) * 100 + print(f"\nDone. {len(experts)} experts successfully merged and saved.") + print(f"Expert storage: {total_expert_f32/1e9:.1f} GB -> {total_expert_q/1e9:.1f} GB ({ratio:.0f}%)") + print(f"Model ready at: {out}") + +if __name__ == "__main__": + main() diff --git a/c/tools/diag_harness.py b/c/tools/diag_harness.py new file mode 100644 index 00000000..56a1b225 --- /dev/null +++ b/c/tools/diag_harness.py @@ -0,0 +1,704 @@ +#!/usr/bin/env python3 +""" +diag_harness.py — Comprehensive model diagnostic harness for the colibri GLM-5.2 engine. + +Runs a full campaign of tests against any model snapshot: + Phase 0 (system): startup telemetry — GPU, RAM, cache cap, load time, MTP, idot kernel + Phase 1 (smoke): correctness — curated prompts, coherence checks, corruption detection + Phase 2 (diagnostic): deep x-ray — full PROFILE breakdown, routing, MTP, disk-split, CUDA tier + Phase 3 (quality): benchmark accuracy — hellaswag/arc_challenge/mmlu via eval_glm.py SCORE + Phase 4 (throughput): tok/s with and without MTP speculation + Phase 5 (report): structured JSON + human-readable Markdown summary + +Usage: + python tools/diag_harness.py --snap /path/to/model --phase all + python tools/diag_harness.py --snap /path/to/model --phase smoke --ngen 64 + python tools/diag_harness.py --snap /path/to/model --phase quality --quality-limit 200 + +Output goes to --out (default ./diag_results//). Each phase writes a raw log +(_.log) and all metrics are collected into report.json + report.md. + +Design notes: + - stdout and stderr are captured separately via subprocess.PIPE. The engine streams + generated text to stdout (interleaved with prompt + PROFILE stats), but TOKENS=1 dumps + clean token-id lists to stderr — that is the primary text-capture path. + - Every regex is anchored to the exact printf format strings in glm.c (verified against + profile_print line 3853, run_text line 3948, the banner line 5299, etc.). + - Subprocess calls have a hard timeout (default 600s) and are killed cleanly on expiry. + - A single phase can be run standalone; results accumulate in the output dir. +""" +import os, sys, re, json, time, argparse, subprocess, signal, traceback +from datetime import datetime +from pathlib import Path + +# --------------------------------------------------------------------------- +# PROMPT SUITE — curated across categories. "expect" is a case-insensitive +# substring checked against the generated text (None = coherence-only check). +# --------------------------------------------------------------------------- +PROMPTS = [ + {"id":"fact_capital", "cat":"factual", "prompt":"The capital of France is", + "expect":"Paris", "note":"basic world knowledge"}, + {"id":"fact_boiling", "cat":"factual", "prompt":"What is the boiling point of water in Celsius?", + "expect":"100", "note":"basic science"}, + {"id":"fact_planet", "cat":"factual", "prompt":"What planet is closest to the Sun?", + "expect":"Mercury","note":"astronomy fact"}, + {"id":"math_mult", "cat":"math", "prompt":"What is 15 times 12?", + "expect":"180", "note":"2-digit multiplication"}, + {"id":"math_add", "cat":"math", "prompt":"What is 847 plus 153?", + "expect":"1000", "note":"3-digit addition"}, + {"id":"reason_train", "cat":"reasoning", "prompt":"If a train travels 60 mph for 2.5 hours, how far does it go?", + "expect":"150", "note":"rate-time-distance"}, + {"id":"code_factorial","cat":"code", "prompt":"Write a Python function that computes the factorial of a number.", + "expect":"def", "note":"code generation"}, + {"id":"explain_nn", "cat":"explanation","prompt":"Explain what a neural network is in one sentence.", + "expect":None, "note":"coherence check"}, + {"id":"creative_story","cat":"creative", "prompt":"Write a one-sentence story about a lighthouse.", + "expect":None, "note":"creative coherence"}, + {"id":"edge_hello", "cat":"edge", "prompt":"Hello, how are you today?", + "expect":None, "note":"conversational opener"}, + {"id":"edge_the", "cat":"edge", "prompt":"The weather today is", + "expect":None, "note":"simple continuation"}, + {"id":"edge_repeat", "cat":"edge", "prompt":"The quick brown fox jumps over the lazy dog. The quick brown fox", + "expect":None, "note":"repetition-bait"}, +] + +# --------------------------------------------------------------------------- +# METRIC EXTRACTION — each parser is (name, compiled_regex, group_index, cast). +# Regexes match the EXACT printf format strings in glm.c. +# --------------------------------------------------------------------------- +def _f1(g): return float(g) +def _i1(g): return int(g) + +# Build parsers as (regex, lambda(match)->value) for clarity +RX = { + # stdout: run_text summary line (line 3948) + "decode_toks": (re.compile(r"decode (\d+) tokens in"), lambda m: int(m.group(1))), + "decode_secs": (re.compile(r"decode \d+ tokens in ([\d.]+)s"), lambda m: float(m.group(1))), + "decode_tps": (re.compile(r"decode \d+ tokens in [\d.]+s \(([\d.]+) tok/s\)"), lambda m: float(m.group(1))), + "prefill_toks": (re.compile(r"prefill (\d+) tokens in"), lambda m: int(m.group(1))), + "prefill_secs": (re.compile(r"prefill \d+ tokens in ([\d.]+)s"), lambda m: float(m.group(1))), + "hit_rate": (re.compile(r"expert hit rate ([\d.]+)%"), lambda m: float(m.group(1))), + "rss_gb": (re.compile(r"RSS ([\d.]+) GB"), lambda m: float(m.group(1))), + "experts_per_tok":(re.compile(r"experts loaded/token: ([\d.]+)"), lambda m: float(m.group(1))), + "mtp_accept": (re.compile(r"MTP acceptance ([\d.]+)%"), lambda m: float(m.group(1))), + "mtp_acc_cnt": (re.compile(r"MTP acceptance \d+% \((\d+)/(\d+)\)"), lambda m: (int(m.group(1)), int(m.group(2)))), + "spec_tok_per_fw":(re.compile(r"speculation: ([\d.]+) tokens/forward"),lambda m: float(m.group(1))), + # stdout: PROFILE lines (profile_print, line 3853-3864) + "prof_expert_disk": (re.compile(r"expert-disk ([\d.]+)s service"), lambda m: float(m.group(1))), + "prof_expert_wait": (re.compile(r"service / ([\d.]+)s wait"), lambda m: float(m.group(1))), + "prof_expert_mm": (re.compile(r"expert-matmul ([\d.]+)s"), lambda m: float(m.group(1))), + "prof_attention": (re.compile(r"\| attention ([\d.]+)s"), lambda m: float(m.group(1))), + "prof_kvb": (re.compile(r"including kvb ([\d.]+)s"), lambda m: float(m.group(1))), + "prof_lm_head": (re.compile(r"lm_head ([\d.]+)s"), lambda m: float(m.group(1))), + "prof_other": (re.compile(r"\| other ([\d.]+)s"), lambda m: float(m.group(1))), + # stdout: banner (line 5299) + "load_secs": (re.compile(r"loaded in ([\d.]+)s"), lambda m: float(m.group(1))), + "resident_mb": (re.compile(r"resident dense: ([\d.]+) MB"), lambda m: float(m.group(1))), + "mtp_status": (re.compile(r"MTP (ACTIVE|absent)"), lambda m: m.group(1)), + "n_layers": (re.compile(r"layers=(\d+) experts=(\d+)"), lambda m: int(m.group(1))), + "n_experts": (re.compile(r"layers=(\d+) experts=(\d+)"), lambda m: int(m.group(2))), + # stdout: banner idot kernel + "idot_kernel": (re.compile(r"idot: (\S+) =="), lambda m: m.group(1)), + "cache_cap": (re.compile(r"cache=(\d+) experts/layer"), lambda m: int(m.group(1))), + # stderr: RAM_GB (line 5086-5112) + "ram_budget": (re.compile(r"\[RAM_GB=([\d.]+)"), lambda m: float(m.group(1))), + "cap_lowered": (re.compile(r"cap lowered (\d+)->(\d+)"), lambda m: (int(m.group(1)), int(m.group(2)))), + "cap_raised": (re.compile(r"cap raised (\d+)->(\d+)"), lambda m: (int(m.group(1)), int(m.group(2)))), + "cap_ok": (re.compile(r"cap=(\d+) ok"), lambda m: int(m.group(1))), + # stderr: CUDA (backend_cuda.cu:389) + "cuda_device": (re.compile(r"\[CUDA\] device (\d+): (.*?), ([\d.]+) GB VRAM, sm_(\d)(\d)"), + lambda m: {"id":int(m.group(1)),"name":m.group(2).strip(),"vram_gb":float(m.group(3)), + "sm":f"{m.group(4)}.{m.group(5)}"}), + "cuda_mode": (re.compile(r"\[CUDA\] mode: (.+)"), lambda m: m.group(1)), + "cuda_tier": (re.compile(r"CUDA expert tier: (\d+) resident experts \(([\d.]+) GB\)"), + lambda m: {"resident":int(m.group(1)),"vram_gb":float(m.group(2))}), + # stderr: TOKENS dump (line 4010-4012) + "tokens_dump": (re.compile(r"^\[TOKENS\] (\d+) generated:(.*)$", re.MULTILINE), + lambda m: [int(x) for x in m.group(2).split()]), + # stderr: per-16-token progress (emit_stream line 3742) + "progress_tps": (re.compile(r"t=(\d+)\s+RSS ([\d.]+) GB\s+hit ([\d.]+)%\s+([\d.]+) tok/s\s+([\d.]+) tok/fw"), + lambda m: {"tok":int(m.group(1)),"rss":float(m.group(2)),"hit":float(m.group(3)), + "tps":float(m.group(4)),"tpf":float(m.group(5))}), + # stderr: DSA, USAGE, KV startup lines + "usage_loaded": (re.compile(r"\[USAGE\].*?(\d+) selections"), lambda m: int(m.group(1))), + "kv_slots": (re.compile(r"\[KV\].*?(\d+) context slots"), lambda m: int(m.group(1))), +} + +def extract_metrics(stdout: str, stderr: str) -> dict: + """Parse all metrics from the engine's stdout+stderr output.""" + text_out = stdout or "" + text_err = stderr or "" + metrics = {} + # For most parsers we search BOTH streams (engine is inconsistent about which + # channel a given line lands on). Some are stream-specific (noted below). + combined = text_out + "\n" + text_err + for name, (rx, fn) in RX.items(): + m = rx.search(combined) + if m: + try: metrics[name] = fn(m) + except (ValueError, IndexError): pass + # TOKENS dump is stderr-only and may appear once; grab it explicitly + m = RX["tokens_dump"][0].search(text_err) + if m: + try: metrics["tokens_dump"] = RX["tokens_dump"][1](m) + except: pass + # Multiple CUDA devices — collect all + cuda_devs = [] + for m in re.finditer(r"\[CUDA\] device (\d+): (.*?), ([\d.]+) GB VRAM, sm_(\d)(\d)", text_err): + cuda_devs.append({"id":int(m.group(1)),"name":m.group(2).strip(), + "vram_gb":float(m.group(3)),"sm":f"{m.group(4)}.{m.group(5)}"}) + if cuda_devs: metrics["cuda_devices"] = cuda_devs + # Multiple progress checkpoints — collect the full curve + progress = [] + for m in RX["progress_tps"][0].finditer(text_err): + try: + progress.append(RX["progress_tps"][1](m)) + except: pass + if progress: metrics["progress_curve"] = progress + # Multiple PROFILE lines — there are two (prefill + decode). Keep both. + prof_lines = [(i, line) for i, line in enumerate(text_out.splitlines()) if line.startswith("PROFILE:")] + for idx, (line_no, line) in enumerate(prof_lines): + label = "prefill_profile" if idx == 0 else "decode_profile" + p = {} + for pname, (rx, fn) in RX.items(): + if not pname.startswith("prof_"): continue + mm = rx.search(line) + if mm: + try: p[pname] = fn(mm) + except: pass + if p: metrics[label] = p + return metrics + + +# --------------------------------------------------------------------------- +# ENGINE RUNNER — subprocess wrapper with timeout, signal handling, logging. +# --------------------------------------------------------------------------- +class EngineRunner: + def __init__(self, glm_path, snap, out_dir, default_env=None, timeout=600): + self.glm = str(glm_path) + self.snap = str(snap) + self.out_dir = Path(out_dir) + self.out_dir.mkdir(parents=True, exist_ok=True) + self.default_env = default_env or {} + self.timeout = timeout + self.default_cap = 75 # production default (matches bench_full.sh) + + def run_prompt(self, prompt, ngen=64, env_extra=None, log_name=None, cap=None): + """Run the engine in PROMPT mode and return (stdout, stderr, returncode, elapsed).""" + env = dict(os.environ, SNAP=self.snap, PROMPT=prompt, NGEN=str(ngen)) + env.update(self.default_env) + if env_extra: env.update(env_extra) + env["TOKENS"] = "1" # always capture token ids for reliable text extraction + cmd = [self.glm, str(cap if cap is not None else self.default_cap)] + return self._exec(cmd, env, log_name) + + def run_score(self, score_file, cap=None, env_extra=None, log_name=None): + """Run the engine in SCORE (log-likelihood) mode.""" + env = dict(os.environ, SNAP=self.snap, SCORE=score_file) + env.update(self.default_env) + if env_extra: env.update(env_extra) + cmd = [self.glm, str(cap if cap is not None else self.default_cap)] + return self._exec(cmd, env, log_name) + + def _exec(self, cmd, env, log_name): + t0 = time.time() + log_path = self.out_dir / (log_name or f"run_{int(t0)}.log") + try: + proc = subprocess.Popen(cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + text=True, encoding="utf-8", errors="replace") + try: + stdout, stderr = proc.communicate(timeout=self.timeout) + rc = proc.returncode + except subprocess.TimeoutExpired: + proc.kill() + stdout, stderr = proc.communicate() + rc = -1 + stderr = (stderr or "") + f"\n[TIMEOUT after {self.timeout}s]\n" + except Exception as e: + stdout, stderr, rc = "", f"[EXCEPTION] {e}\n{traceback.format_exc()}", -2 + elapsed = time.time() - t0 + # Write raw log (both streams, clearly delimited) + with open(log_path, "w", encoding="utf-8") as f: + f.write(f"=== CMD: {' '.join(cmd)}\n=== ELAPSED: {elapsed:.1f}s\n=== RC: {rc}\n\n") + f.write("--- STDOUT ---\n"); f.write(stdout or ""); f.write("\n") + f.write("--- STDERR ---\n"); f.write(stderr or ""); f.write("\n") + return stdout or "", stderr or "", rc, elapsed + + +# --------------------------------------------------------------------------- +# TEXT RECOVERY — decode the [TOKENS] ids back to text using the tokenizer. +# Falls back to stdout extraction if tokenizer/tokens unavailable. +# --------------------------------------------------------------------------- +def recover_text(stdout, stderr, prompt, tokenizer=None): + """Extract generated text. Primary: TOKENS dump decoded. Fallback: stdout parse.""" + # Method 1: decode TOKENS ids via tokenizer + if tokenizer: + m = re.search(r"^\[TOKENS\] \d+ generated:(.*)$", stderr, re.MULTILINE) + if m: + ids = [int(x) for x in m.group(1).split()] + if ids: + try: + return tokenizer.decode(ids), "tokens" + except Exception: pass + # Method 2: stdout — text is between the prompt string and "PROFILO" or "\n---" + text = stdout or "" + # Find the prompt in stdout, take everything after it up to PROFILO + idx = text.find(prompt) + if idx >= 0: + after = text[idx + len(prompt):] + cut = after.find("PROFILO") + if cut >= 0: + return after[:cut].strip(), "stdout" + cut = after.find("\n---") + if cut >= 0: + return after[:cut].strip(), "stdout" + return "", "none" + + +# --------------------------------------------------------------------------- +# CORRUPTION / COHERENCE CHECKS +# --------------------------------------------------------------------------- +def check_repetition(token_ids): + """Detect degenerate repetition: same token 3+ consecutive times.""" + if not token_ids or len(token_ids) < 6: + return False, 0 + max_run = 1; cur = 1 + for i in range(1, len(token_ids)): + if token_ids[i] == token_ids[i-1]: cur += 1; max_run = max(max_run, cur) + else: cur = 1 + return max_run >= 3, max_run + +def check_expected(text, expect): + """Check if expected substring appears case-insensitively in the first 200 chars.""" + if not expect or not text: return None + return expect.lower() in text[:200].lower() + + +# --------------------------------------------------------------------------- +# PHASES +# --------------------------------------------------------------------------- +class DiagnosticHarness: + def __init__(self, args): + self.args = args + self.snap = args.snap + self.glm = args.glm + ts = datetime.now().strftime("%Y%m%d_%H%M%S") + self.out_dir = Path(args.out or f"./diag_results/{ts}") + self.out_dir.mkdir(parents=True, exist_ok=True) + # Base env for all runs + base_env = {"TEMP": "0", "PIPE": "1", "PIPE_WORKERS": "8", "DIRECT": "1"} + if args.ram: base_env["RAM_GB"] = str(args.ram) + if args.cuda: + base_env.update({ + "COLI_CUDA": "1", + "COLI_GPU": str(args.gpu) if args.gpu is not None else "0", + "CUDA_DENSE": "1", + "COLI_CUDA_ATTN": "1", + "COLI_CUDA_PIPE": "2", + "COLI_CUDA_PIPE_S_MIN": "1", + }) + self.runner = EngineRunner(self.glm, self.snap, self.out_dir, base_env, timeout=args.timeout) + self.runner.default_cap = args.cap + # Load tokenizer for text recovery + self.tokenizer = None + tok_path = os.path.join(self.snap, "tokenizer.json") + if os.path.exists(tok_path): + try: + from tokenizers import Tokenizer + self.tokenizer = Tokenizer.from_file(tok_path) + except Exception as e: + print(f"[warn] could not load tokenizer: {e}", file=sys.stderr) + self.results = {"meta": {"snap": self.snap, "glm": self.glm, + "timestamp": ts, "args": vars(args)}, + "phases": {}} + + def phase_system(self): + """Phase 0: minimal run to capture startup telemetry.""" + print("\n" + "="*60 + "\nPHASE 0: SYSTEM PROBE\n" + "="*60) + stdout, stderr, rc, elapsed = self.runner.run_prompt( + "Hello", ngen=1, log_name="system_probe.log") + metrics = extract_metrics(stdout, stderr) + result = { + "rc": rc, "elapsed": elapsed, + "load_secs": metrics.get("load_secs"), + "resident_mb": metrics.get("resident_mb"), + "n_layers": metrics.get("n_layers"), + "n_experts": metrics.get("n_experts"), + "mtp_status": metrics.get("mtp_status"), + "idot_kernel": metrics.get("idot_kernel"), + "cache_cap_requested": metrics.get("cache_cap"), + "ram_budget_gb": metrics.get("ram_budget"), + "cap_lowered": metrics.get("cap_lowered"), + "cap_raised": metrics.get("cap_raised"), + "cap_final": metrics.get("cap_ok"), + "cuda_devices": metrics.get("cuda_devices", []), + "cuda_mode": metrics.get("cuda_mode"), + } + # Print summary + print(f" load time: {result['load_secs']:.2f}s" if result['load_secs'] else " load time: FAILED") + print(f" resident: {result['resident_mb']:.0f} MB" if result['resident_mb'] else "") + print(f" layers/exp: {result['n_layers']}/{result['n_experts']}" if result['n_layers'] else "") + print(f" MTP: {result['mtp_status']}") + print(f" idot kernel: {result['idot_kernel']}") + print(f" RAM budget: {result['ram_budget_gb']} GB" if result['ram_budget_gb'] else " RAM budget: auto") + if result['cap_lowered']: + print(f" cache cap: {result['cap_lowered'][0]} -> {result['cap_lowered'][1]} (RAM-lowered)") + elif result['cap_final']: + print(f" cache cap: {result['cap_final']} (ok)") + else: + print(f" cache cap: {result['cache_cap_requested']}") + for d in result['cuda_devices']: + print(f" GPU {d['id']}: {d['name']}, {d['vram_gb']:.1f} GB, sm_{d['sm']}") + if rc != 0 and rc != -1: + print(f" WARNING: engine returned rc={rc}") + self.results["phases"]["system"] = result + return result + + def phase_smoke(self): + """Phase 1: correctness smoke test across curated prompts.""" + print("\n" + "="*60 + "\nPHASE 1: CORRECTNESS SMOKE TEST\n" + "="*60) + ngen = self.args.ngen + prompt_results = [] + pass_count = 0; total = len(PROMPTS) + for p in PROMPTS: + stdout, stderr, rc, elapsed = self.runner.run_prompt( + p["prompt"], ngen=ngen, log_name=f"smoke_{p['id']}.log") + metrics = extract_metrics(stdout, stderr) + token_ids = metrics.get("tokens_dump", []) + text, method = recover_text(stdout, stderr, p["prompt"], self.tokenizer) + is_rep, max_run = check_repetition(token_ids) + expect_ok = check_expected(text, p.get("expect")) + # Verdict: pass if text is non-empty, no severe repetition, and (if factual) expected found + has_text = bool(text.strip()) + ok = has_text and not is_rep + if p.get("expect") and expect_ok is False: ok = False + if ok: pass_count += 1 + # Diagnose failure mode for reporting + if not has_text: + fail_reason = "no tokens generated (immediate EOS)" + elif is_rep: + fail_reason = f"repetition loop (max run={max_run})" + elif p.get("expect") and expect_ok is False: + fail_reason = f"expected '{p['expect']}' not found" + else: + fail_reason = "" + prompt_results.append({ + "id": p["id"], "cat": p["cat"], "prompt": p["prompt"], + "expect": p.get("expect"), "generated": text[:300], + "expect_match": expect_ok, "repetition": is_rep, "max_run": max_run, + "toks_generated": len(token_ids), "decode_tps": metrics.get("decode_tps"), + "hit_rate": metrics.get("hit_rate"), "rc": rc, "elapsed": elapsed, + "text_method": method, "pass": ok, "fail_reason": fail_reason, + }) + status = "PASS" if ok else "FAIL" + extra = f" expect={'Y' if expect_ok else ('N' if expect_ok is False else '-')}" if p.get("expect") else "" + tps = f" {metrics.get('decode_tps',0):.2f}t/s" if metrics.get("decode_tps") else "" + print(f" [{status}] {p['id']:<22} ({p['cat']:<11}) rep={'Y' if is_rep else 'N'}{extra}{tps}") + if text: + preview = text.replace("\n", " ")[:80] + print(f" -> {preview}") + elif rc != 0: + print(f" -> [engine rc={rc}]") + else: + print(f" -> [{fail_reason}]") + result = {"prompts": prompt_results, "pass": pass_count, "total": total, + "pass_rate": 100.0 * pass_count / total if total else 0} + print(f"\n SMOKE SUMMARY: {pass_count}/{total} passed ({result['pass_rate']:.0f}%)") + self.results["phases"]["smoke"] = result + return result + + def phase_diagnostic(self): + """Phase 2: single deep-instrumented run — full PROFILE + routing + MTP.""" + print("\n" + "="*60 + "\nPHASE 2: FULL SYSTEM DIAGNOSTIC\n" + "="*60) + diag_env = { + "LOOKA": "1", "DISK_SPLIT": "1", "ROUTE_AGREE": "1", + } + if self.args.cuda: + diag_env["COLI_CUDA_PROFILE"] = "1" + prompt = ("Write a short paragraph explaining how photosynthesis works. " + "Include the roles of sunlight, water, and carbon dioxide.") + stdout, stderr, rc, elapsed = self.runner.run_prompt( + prompt, ngen=self.args.ngen, env_extra=diag_env, log_name="diagnostic.log") + metrics = extract_metrics(stdout, stderr) + text, _ = recover_text(stdout, stderr, prompt, self.tokenizer) + result = { + "rc": rc, "elapsed": elapsed, + "generated_text": text[:500], + "prefill_profile": metrics.get("prefill_profile", {}), + "decode_profile": metrics.get("decode_profile", {}), + "decode_tps": metrics.get("decode_tps"), + "prefill_secs": metrics.get("prefill_secs"), + "hit_rate": metrics.get("hit_rate"), + "rss_gb": metrics.get("rss_gb"), + "experts_per_tok": metrics.get("experts_per_tok"), + "mtp_accept": metrics.get("mtp_accept"), + "mtp_counts": metrics.get("mtp_acc_cnt"), + "spec_tok_per_fw": metrics.get("spec_tok_per_fw"), + "cuda_tier": metrics.get("cuda_tier"), + "progress_curve": metrics.get("progress_curve", []), + } + # Print the PROFILE breakdown + dp = result["decode_profile"] + print(f"\n DECODE TIMING BREAKDOWN (per-bucket, seconds):") + print(f" expert-disk: {dp.get('prof_expert_disk', '?'):>8}") + print(f" expert-matmul: {dp.get('prof_expert_mm', '?'):>8}") + print(f" attention: {dp.get('prof_attention', '?'):>8}") + print(f" lm_head: {dp.get('prof_lm_head', '?'):>8}") + print(f" other: {dp.get('prof_other', '?'):>8}") + print(f"\n PERFORMANCE:") + print(f" prefill: {result['prefill_secs']:.2f}s" if result['prefill_secs'] else " prefill: ?") + print(f" decode: {result['decode_tps']:.3f} tok/s" if result['decode_tps'] else " decode: ?") + print(f" hit rate: {result['hit_rate']:.1f}%" if result.get('hit_rate') is not None else " hit rate: ?") + print(f" RSS: {result['rss_gb']:.1f} GB" if result.get('rss_gb') else " RSS: ?") + print(f" exp/tok: {result['experts_per_tok']:.1f}" if result.get('experts_per_tok') else " exp/tok: ?") + print(f" MTP: {result['mtp_accept']:.0f}% accept" if result.get('mtp_accept') is not None else " MTP: ?") + if result.get('cuda_tier'): + ct = result['cuda_tier'] + print(f" CUDA tier: {ct['resident']} experts, {ct['vram_gb']:.1f} GB VRAM") + # Show generated text preview + if text: + print(f"\n GENERATED TEXT (first 200 chars):") + print(f" {text[:200].replace(chr(10), ' ')}") + else: + print(f"\n GENERATED TEXT: [none recovered]") + self.results["phases"]["diagnostic"] = result + return result + + def phase_quality(self): + """Phase 3: benchmark accuracy via eval_glm.py SCORE mode.""" + print("\n" + "="*60 + "\nPHASE 3: QUALITY BENCHMARKS\n" + "="*60) + eval_script = os.path.join(os.path.dirname(__file__), "eval_glm.py") + bench_dir = os.path.join(os.path.dirname(os.path.dirname(__file__)), "bench") + if not os.path.exists(eval_script): + print(f" [SKIP] eval_glm.py not found at {eval_script}") + self.results["phases"]["quality"] = {"error": "eval_glm.py not found"} + return None + tasks = ["hellaswag", "arc_challenge", "mmlu"] + missing = [t for t in tasks if not os.path.exists(os.path.join(bench_dir, f"{t}.jsonl"))] + if missing: + print(f" [SKIP] benchmark data missing: {missing}") + print(f" run: python tools/fetch_benchmarks.py --out {bench_dir}") + self.results["phases"]["quality"] = {"error": f"missing benchmark data: {missing}"} + return None + limit = self.args.quality_limit + py = sys.executable + cmd = [py, eval_script, "--snap", self.snap, "--glm", self.glm, + "--data", bench_dir, "--tasks", ",".join(tasks), "--limit", str(limit)] + env = dict(os.environ) + if self.args.ram: env["RAM_GB"] = str(self.args.ram) + print(f" Running eval_glm.py (tasks={tasks}, n={limit})...") + print(f" This takes ~{limit*3*4/0.05:.0f}s at 0.05 tok/s (worst case)...") + t0 = time.time() + log_path = self.out_dir / "quality_eval.log" + try: + with open(log_path, "w", encoding="utf-8") as logf: + proc = subprocess.run(cmd, env=env, capture_output=True, text=True, timeout=self.args.timeout*3, + encoding="utf-8", errors="replace") + logf.write(proc.stdout); logf.write("\n--- STDERR ---\n"); logf.write(proc.stderr) + elapsed = time.time() - t0 + # Parse the acc/acc_norm table from eval_glm.py output + scores = {} + for line in proc.stdout.splitlines(): + # lines like: "hellaswag 40 45.0% 50.0%" + m = re.match(r"(\w+)\s+(\d+)\s+([\d.]+)%\s+([\d.]+)%", line.strip()) + if m: + scores[m.group(1)] = {"n": int(m.group(2)), "acc": float(m.group(3)), + "acc_norm": float(m.group(4))} + mean_m = re.search(r"MEAN acc_norm:\s*([\d.]+)%", proc.stdout) + result = {"scores": scores, "mean_acc_norm": float(mean_m.group(1)) if mean_m else None, + "elapsed": elapsed, "rc": proc.returncode} + print(f" Completed in {elapsed:.0f}s\n") + print(f" {'task':<18} {'n':>4} {'acc':>7} {'acc_norm':>9}") + for t, s in scores.items(): + print(f" {t:<18} {s['n']:>4} {s['acc']:>6.1f}% {s['acc_norm']:>8.1f}%") + if result["mean_acc_norm"] is not None: + print(f"\n MEAN acc_norm: {result['mean_acc_norm']:.1f}%") + except subprocess.TimeoutExpired: + result = {"error": f"eval timed out after {self.args.timeout*3}s"} + print(f" [TIMEOUT] eval_glm.py exceeded {self.args.timeout*3}s") + except Exception as e: + result = {"error": str(e)} + print(f" [ERROR] {e}") + self.results["phases"]["quality"] = result + return result + + def phase_throughput(self): + """Phase 4: tok/s comparison — MTP on vs off.""" + print("\n" + "="*60 + "\nPHASE 4: THROUGHPUT BENCHMARK\n" + "="*60) + prompt = "Summarize the plot of Romeo and Juliet in three sentences." + ngen = self.args.ngen + results = {} + # Run 1: with MTP (default) + print(f" [1/2] MTP ON (draft=3)...") + stdout, stderr, rc, t = self.runner.run_prompt( + prompt, ngen=ngen, log_name="throughput_mtp_on.log") + m_on = extract_metrics(stdout, stderr) + results["mtp_on"] = {"tps": m_on.get("decode_tps"), "hit": m_on.get("hit_rate"), + "mtp_accept": m_on.get("mtp_accept"), "elapsed": t} + print(f" {m_on.get('decode_tps',0):.3f} tok/s | hit {m_on.get('hit_rate',0):.1f}% | " + f"MTP {m_on.get('mtp_accept',0):.0f}%") + # Run 2: MTP off + print(f" [2/2] MTP OFF (MTP=0)...") + stdout, stderr, rc, t = self.runner.run_prompt( + prompt, ngen=ngen, env_extra={"MTP": "0"}, log_name="throughput_mtp_off.log") + m_off = extract_metrics(stdout, stderr) + results["mtp_off"] = {"tps": m_off.get("decode_tps"), "hit": m_off.get("hit_rate"), + "elapsed": t} + print(f" {m_off.get('decode_tps',0):.3f} tok/s | hit {m_off.get('hit_rate',0):.1f}%") + # Compute speedup + if results["mtp_on"]["tps"] and results["mtp_off"]["tps"] and results["mtp_off"]["tps"] > 0: + sp = results["mtp_on"]["tps"] / results["mtp_off"]["tps"] + results["mtp_speedup"] = sp + print(f"\n MTP speedup: {sp:.2f}x ({'MTP helps' if sp > 1.05 else 'MTP hurts' if sp < 0.95 else 'no effect'})") + else: + print(f"\n MTP speedup: (insufficient data)") + self.results["phases"]["throughput"] = results + return results + + def write_report(self): + """Write report.json and report.md.""" + ts = self.results["meta"]["timestamp"] + json_path = self.out_dir / "report.json" + md_path = self.out_dir / "report.md" + # JSON + with open(json_path, "w", encoding="utf-8") as f: + json.dump(self.results, f, indent=2, default=str) + # Markdown + lines = [ + f"# Diagnostic Report — {ts}", + f"", + f"**Model:** `{self.snap}`", + f"**Engine:** `{self.glm}`", + f"**Date:** {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}", + f"", + ] + phases = self.results["phases"] + # System + if "system" in phases: + s = phases["system"] + lines += ["## Phase 0: System Probe", ""] + lines += [f"- Load time: **{s.get('load_secs','?')}s**"] + lines += [f"- Layers/experts: {s.get('n_layers','?')}/{s.get('n_experts','?')}"] + lines += [f"- MTP: {s.get('mtp_status','?')}"] + lines += [f"- idot kernel: `{s.get('idot_kernel','?')}`"] + lines += [f"- RAM budget: {s.get('ram_budget_gb','auto')} GB"] + if s.get("cap_lowered"): + lines += [f"- Cache cap: {s['cap_lowered'][0]}→{s['cap_lowered'][1]} (RAM-lowered)"] + elif s.get("cap_final"): + lines += [f"- Cache cap: {s['cap_final']}"] + for d in s.get("cuda_devices", []): + lines += [f"- GPU {d['id']}: {d['name']}, {d['vram_gb']:.1f} GB, sm_{d['sm']}"] + lines.append("") + # Smoke + if "smoke" in phases: + sm = phases["smoke"] + lines += [f"## Phase 1: Correctness Smoke", "", + f"**{sm['pass']}/{sm['total']} prompts passed ({sm['pass_rate']:.0f}%)**", "", + "| ID | Category | Pass | Expect | Repetition | tok/s | Generated (first 80 chars) |", + "|---|---|---|---|---|---|---|"] + for p in sm["prompts"]: + gen = p["generated"][:80].replace("|", "\\|").replace("\n", " ") if p["generated"] else "" + exp = "Y" if p.get("expect_match") else ("N" if p.get("expect_match") is False else "-") + tps = f"{p.get('decode_tps',0):.2f}" if p.get("decode_tps") else "-" + lines.append(f"| {p['id']} | {p['cat']} | {'✅' if p['pass'] else '❌'} | {exp} | " + f"{'⚠️' if p['repetition'] else '-'} (run={p.get('max_run',0)}) | {tps} | {gen} |") + lines.append("") + # Diagnostic + if "diagnostic" in phases: + d = phases["diagnostic"] + lines += ["## Phase 2: Full System Diagnostic", ""] + dp = d.get("decode_profile", {}) + lines += ["### Decode Timing Breakdown", "", + "| Bucket | Seconds |", "|---|---|"] + for k, label in [("prof_expert_disk","expert-disk"),("prof_expert_mm","expert-matmul"), + ("prof_attention","attention"),("prof_lm_head","lm_head"), + ("prof_other","other")]: + v = dp.get(k, "?") + lines.append(f"| {label} | {v} |") + lines += [f"", f"### Performance", f"- Decode: **{d.get('decode_tps','?')} tok/s**", + f"- Prefill: {d.get('prefill_secs','?')}s", + f"- Expert hit rate: {d.get('hit_rate','?')}%", + f"- RSS: {d.get('rss_gb','?')} GB", + f"- Experts/token: {d.get('experts_per_tok','?')}", + f"- MTP acceptance: {d.get('mtp_accept','?')}%", ""] + if d.get("generated_text"): + lines += [f"### Generated Text", f"```\n{d['generated_text'][:300]}\n```", ""] + # Quality + if "quality" in phases: + q = phases["quality"] + lines += ["## Phase 3: Quality Benchmarks", ""] + if "error" in q: + lines += [f"⚠️ {q['error']}", ""] + else: + lines += [f"**Mean acc_norm: {q.get('mean_acc_norm','?')}%**", "", + "| Task | n | acc | acc_norm |", "|---|---|---|---|"] + for t, s in q.get("scores", {}).items(): + lines.append(f"| {t} | {s['n']} | {s['acc']:.1f}% | {s['acc_norm']:.1f}% |") + lines.append("") + # Throughput + if "throughput" in phases: + th = phases["throughput"] + lines += ["## Phase 4: Throughput", "", + "| Mode | tok/s | hit% | MTP accept |", "|---|---|---|---|"] + on = th.get("mtp_on", {}); off = th.get("mtp_off", {}) + lines.append(f"| MTP ON | {on.get('tps','?')} | {on.get('hit','?')}% | {on.get('mtp_accept','?')}% |") + lines.append(f"| MTP OFF | {off.get('tps','?')} | {off.get('hit','?')}% | — |") + if th.get("mtp_speedup"): + lines.append(f"\n**MTP speedup: {th['mtp_speedup']:.2f}x**") + lines.append("") + with open(md_path, "w", encoding="utf-8") as f: + f.write("\n".join(lines)) + print(f"\n{'='*60}") + print(f"Report written:") + print(f" JSON: {json_path}") + print(f" MD: {md_path}") + print(f" Logs: {self.out_dir}/*.log") + print(f"{'='*60}") + + def run(self): + phase = self.args.phase + if phase == "all": + self.phase_system() + self.phase_smoke() + self.phase_diagnostic() + self.phase_quality() + self.phase_throughput() + elif phase == "system": self.phase_system() + elif phase == "smoke": self.phase_smoke() + elif phase == "diagnostic": self.phase_diagnostic() + elif phase == "quality": self.phase_quality() + elif phase == "throughput": self.phase_throughput() + else: + print(f"Unknown phase: {phase}", file=sys.stderr); sys.exit(1) + self.write_report() + + +def main(): + ap = argparse.ArgumentParser(description="Comprehensive model diagnostic harness for colibri GLM-5.2") + ap.add_argument("--snap", required=True, help="model snapshot directory") + ap.add_argument("--glm", default=None, help="engine binary path (default: ./glm.exe or ./glm)") + ap.add_argument("--phase", default="all", + choices=["all","system","smoke","diagnostic","quality","throughput"], + help="which test phase to run") + ap.add_argument("--out", default=None, help="output directory (default: ./diag_results/)") + ap.add_argument("--ngen", type=int, default=64, help="generation length for smoke/throughput (default 64)") + ap.add_argument("--quality-limit", type=int, default=40, help="questions per benchmark task (default 40)") + ap.add_argument("--ram", type=float, default=0, help="RAM_GB override (0=auto)") + ap.add_argument("--cuda", action="store_true", help="enable COLI_CUDA GPU tier") + ap.add_argument("--gpu", type=int, default=None, help="GPU device ordinal (with --cuda)") + ap.add_argument("--cap", type=int, default=75, help="experts-per-layer cache cap (default 75)") + ap.add_argument("--timeout", type=int, default=600, help="per-run timeout in seconds (default 600)") + a = ap.parse_args() + # Auto-detect engine binary + if a.glm is None: + for cand in ["./glm.exe", "./glm", "../glm.exe", os.path.join(os.path.dirname(__file__), "..", "glm.exe")]: + if os.path.exists(cand): a.glm = os.path.abspath(cand); break + if a.glm is None: + print("ERROR: could not find glm/glm.exe. Specify with --glm", file=sys.stderr); sys.exit(1) + if not os.path.isdir(a.snap): + print(f"ERROR: snapshot dir does not exist: {a.snap}", file=sys.stderr); sys.exit(1) + harness = DiagnosticHarness(a) + harness.run() + +if __name__ == "__main__": + main() diff --git a/c/tools/download_glm52.py b/c/tools/download_glm52.py index aa0c791a..ebd3b2fe 100644 --- a/c/tools/download_glm52.py +++ b/c/tools/download_glm52.py @@ -18,12 +18,15 @@ from huggingface_hub import snapshot_download, HfApi REPO = "zai-org/GLM-5.2-FP8" +# Pin the model revision for supply-chain integrity: set GLM_REVISION to a commit SHA +# so a mutated/compromised upstream can't silently swap the weights you fetch. +REVISION = os.environ.get("GLM_REVISION", "main") DEST = os.environ.get("GLM_DIR", "/home/vincenzo/glm52") # su ext4 (/dev/sdd), MAI su /mnt/c def human(n): return f"{n/1e9:.0f} GB" def check(): - info = HfApi().repo_info(REPO, files_metadata=True) + info = HfApi().repo_info(REPO, revision=REVISION, files_metadata=True) tot = sum((s.size or 0) for s in info.siblings) sts = [s for s in info.siblings if s.rfilename.endswith(".safetensors")] free = shutil.disk_usage(os.path.dirname(DEST) or "/").free @@ -40,6 +43,7 @@ def download(): # resume_download e' implicito; in caso di interruzione, rilancia e riprende. snapshot_download( repo_id=REPO, + revision=REVISION, local_dir=DEST, allow_patterns=["*.safetensors", "*.json", "*.txt", "*.model"], max_workers=8, diff --git a/c/tools/efficiency.py b/c/tools/efficiency.py new file mode 100644 index 00000000..775e8cf0 --- /dev/null +++ b/c/tools/efficiency.py @@ -0,0 +1,458 @@ +"""Efficiency / regression harness for the colibri engine. + +The engine already emits rich telemetry (REPLAY tok/s, PROFILE phase timings, +[PROF] time shares + verdict, CUDA expert-tier utilization). Until now every +consumer of that telemetry — `benchmark_cuda_fixture.py`, `bench_full.sh`, +`bench_ux.sh` — has only *printed* it for a human to eyeball. This module turns +each signal into a parseable field so tests can assert on it. + +Design: + - Reuses SPEED_RE / PROFILE_RE from tools.benchmark_cuda_fixture (no drift). + - parse_run() is pure: stdout+stderr in, dict out. Easy to unit-test against + captured strings (like the existing test_benchmark_cuda_fixture does). + - run_engine() is the subprocess wrapper. Captures stdout and stderr + separately, because the engine splits them: PROFILE/REPLAY/CUDA-tier go to + stdout, the [CUDA]/[PROF]/[prefill] banners go to stderr. + - Floor defaults are module constants (tunable in one place, not scattered). + +No model file is required to import this module; only run_engine() invokes the +binary. parse_run() works on any captured text, so most test surface is covered +by string fixtures without spinning the engine at all. +""" +from __future__ import annotations + +import os +import re +import subprocess +from pathlib import Path +from typing import Optional + +# Reuse the validated regexes from the existing A/B benchmark harness so the +# PROFILE field order (disk, expert_matmul, attention, lm_head, other) and the +# tok/s capture stay identical. Drift here would silently break every consumer. +from tools.benchmark_cuda_fixture import SPEED_RE as _SPEED_RE_REPLAY, PROFILE_RE, PROFILE_KEYS + +# SPEED_RE (from benchmark_cuda_fixture) matches the REPLAY-mode line only: +# "REPLAY decode: ... | 12.34 tok/s | ..." +# run_text / PROMPT mode uses a DIFFERENT format (glm.c:4682): +# "decode N tokens in X.XXs (12.34 tok/s) | expert hit rate ..." +# This alt regex catches the parenthesized form so the full-model report (which +# uses PROMPT mode) gets a real tok/s instead of reporting it missing. +SPEED_RE_TEXT = re.compile(r"decode \d+ tokens in [0-9.]+s \(([0-9.]+) tok/s\)") + + +def _first_speed(stdout: str): + """Find tok/s in whichever run-mode format the engine used.""" + for rx in (_SPEED_RE_REPLAY, SPEED_RE_TEXT): + m = rx.search(stdout) + if m: + return m + return None + + +# Public alias so existing imports keep working (tests reference SPEED_RE). +SPEED_RE = _SPEED_RE_REPLAY + + +# --- additional parsers (formats verified against glm.c printf strings) --- + +# "expert hit rate 88.1%" (summary line) | "expert hit 95.0%" (REPLAY line) +HIT_RE = re.compile(r"expert hit(?:\s+rate)?\s+([0-9.]+)%") + +# "[PROF] time shares: expert-I/O 3% | expert-matmul 12% | attention 56% | lm_head 2% | other 27%" +SHARES_RE = re.compile( + r"\[PROF\] time shares: expert-I/O\s+([0-9.]+)%\s*\|\s*expert-matmul\s+([0-9.]+)%\s*" + r"\|\s*attention\s+([0-9.]+)%\s*\|\s*lm_head\s+([0-9.]+)%\s*\|\s*other\s+([0-9.]+)%" +) + +# "[PROF] verdict: I/O-bound — 60% of the time ..." (also compute-bound / attention-bound / balanced) +VERDICT_RE = re.compile(r"\[PROF\] verdict:\s*(I/O-bound|compute-bound|attention-bound|balanced)") + +# "[PROF] expert I/O: ... hit 95.0% (76 hit / 4 load) | 4.0 loads/token" +LOADS_PER_TOK_RE = re.compile(r"\|\s*([0-9.]+)\s+loads/token") + +# "CUDA expert tier: 111 resident experts (2.36 GB) | 5400 calls served from VRAM" (stdout) +CUDA_TIER_RE = re.compile( + r"CUDA expert tier:\s+(\d+)\s+resident experts\s+\(([0-9.]+)\s+GB\)\s*\|\s+(\d+)\s+calls served from VRAM" +) + +# "[CUDA] device 0: NVIDIA ..., 14.4 GB VRAM, sm_120" (stderr, per device at init) +CUDA_DEVICE_RE = re.compile(r"\[CUDA\] device\s+\d+:") + +# "[CUDA] resident set: 12 tensors, 0.45 GB VRAM" (stderr, cuda_stats_print) +CUDA_RESIDENT_RE = re.compile( + r"\[CUDA\] resident set:\s+(\d+)\s+tensors,\s+([0-9.]+)\s+GB\s+VRAM" +) + +# "PREFILL (teacher-forcing) C vs oracle: 11/32 positions | 1700.4 pos/s" (TF=1 mode, stdout) +TF_MATCH_RE = re.compile(r"PREFILL \(teacher-forcing\).*:\s+(\d+)/(\d+)\s+positions") + +# --- the six deeper signals (added after "are you gathering everything?" audit) --- + +# "ATTENTION: projection/RoPE 0.050s | score-softmax-value 0.009s | output projection 0.011s" +# Sub-breakdown of the attention phase — answers "how is attention being read". +ATTN_BREAKDOWN_RE = re.compile( + r"ATTENTION: projection/RoPE\s+([0-9.]+)s\s*\|\s*score-softmax-value\s+([0-9.]+)s\s*" + r"\|\s*output projection\s+([0-9.]+)s" +) + +# "[PROF] decode forwards: 20 | latency p50 7.3 ms | p90 8.0 ms | p99 8.1 ms | max 8.2 ms | 1.00 tok/forward" +# Per-forward tail latency — a p99 >> p50 means decode stalls (I/O hiccups, KV grow). +LATENCY_RE = re.compile( + r"\[PROF\] decode forwards:\s+(\d+)\s*\| latency p50\s+([0-9.]+)\s*ms\s*\| " + r"p90\s+([0-9.]+)\s*ms\s*\|\s*p99\s+([0-9.]+)\s*ms\s*\|\s*max\s+([0-9.]+)\s*ms" +) + +# "[PROF] expert I/O: 0.004 GB fetched (0.2 MB/token, 0.03 GB/s over the run) | hit 95.0% ... | +# 4.0 loads/token | 0.0s read service / 0.0s felt wait" +# Absolute disk throughput + the felt-wait split (the [PROF] version, more detailed than PROFILE). +EXPERT_IO_RE = re.compile( + r"\[PROF\] expert I/O:\s+([0-9.]+)\s+GB fetched\s+\(([0-9.]+)\s+MB/token,\s*([0-9.]+)\s+GB/s" + r".*?\|\s*([0-9.]+)\s+loads/token\s*\|\s*([0-9.]+)s\s+read service\s*/\s*([0-9.]+)s\s+felt wait" +) + +# "speculation: 1.05 tokens/forward (19 forwards per 20 tokens) | MTP acceptance 44% (7/16)" +# Draft efficiency — is the speculative decoder pulling weight or dead overhead? +SPECULATION_RE = re.compile( + r"speculation:\s+([0-9.]+)\s+tokens/forward\s+\((\d+)\s+forwards per\s+(\d+)\s+tokens\)" + r"\s*\|\s*MTP acceptance\s+([0-9.]+)%" +) + +# "experts loaded/token: 450.0 (per-layer 56.25 across 8; baseline topk=8) | TOPK=0 TOPP=0.00" +# Fuller than loads_per_tok: includes the per-layer spread + the active topk/topp. +EXPERTS_LOADED_RE = re.compile( + r"experts loaded/token:\s+([0-9.]+)\s+\(per-layer\s+([0-9.]+)\s+across\s+(\d+);\s*baseline topk=(\d+)\)" +) + +# "[PROF] machine: Intel(...) | 22 cores (22 omp threads) | RAM 34.1 GB total, 27.1 GB available | backend CUDA" +# Provenance — makes a report reproducible across machines/runs. +MACHINE_RE = re.compile(r"\[PROF\] machine:\s*(.*)\|\s*(\d+)\s+cores.*backend\s+(\S+)") + +# "[PROF] config: RAM_GB=auto 23.9 CTX=4096 | expert cache cap 8/layer ... | DRAFT=0 PIPE=1 DIRECT=0 ..." +# Effective resolved config (after auto-budgeting). Answers "what config actually ran". +CONFIG_RE = re.compile(r"\[PROF\] config:\s*(.*)") + +# "[ORACLE] mismatch pos=7 expected=197 got=22" (TF=1 mode, stderr, per position) +# Captures the engine's *actual* argmax at each position, so two backends can be +# compared DIRECTLY (independent of how each relates to the oracle). +TF_MISMATCH_RE = re.compile(r"\[ORACLE\] mismatch pos=(\d+) expected=\d+ got=(\d+)") + +# --- routing-quality + disk-split + cuda-groups (the deep signals) --- + +# summary suffix: " | swap 3.1% (12/384)" (CACHE_ROUTE inline) +SWAP_RE = re.compile(r"swap\s+([0-9.]+)%\s+\((\d+)/(\d+)\)") + +# summary suffix: " | route_agree 85.0% | route_kl 0.0123" (CACHE_ROUTE/ROUTE_AGREE) +ROUTE_AGREE_RE = re.compile(r"route_agree\s+([0-9.]+)%\s*\|\s*route_kl\s+([0-9.]+)") + +# "disk-load split: draft 8 + absorb 0 + verify/main 3 misses | MTP-layer 0 loads 0.00 GB | +# main-layers 11 loads 0.01 GB (MTP 0.0% of bytes)" (DISK_SPLIT=1) +DISK_SPLIT_RE = re.compile( + r"disk-load split: draft\s+(\d+)\s+\+\s+absorb\s+(\d+)\s+\+\s+verify/main\s+(\d+)\s+misses" + r"\s*\|\s*MTP-layer\s+(\d+)\s+loads\s+([0-9.]+)\s+GB\s*\|\s*main-layers\s+(\d+)\s+loads\s+([0-9.]+)\s+GB" + r"(?:\s+\(MTP\s+([0-9.]+)%\s+of bytes\))?" +) + +# "[CUDA] expert groups: 120 call, 840 expert, 1200 righe (7.00 expert/call)" +CUDA_GROUPS_RE = re.compile( + r"\[CUDA\] expert groups:\s+(\d+)\s+call,\s+(\d+)\s+expert,\s+(\d+)\s+righe\s+\(([0-9.]+)\s+expert/call\)" +) + +# "[CUDA] expert groups timing: H2D 12.3 ms | kernel 45.6 ms | D2H 7.8 ms" (COLI_CUDA_PROFILE=1) +CUDA_GROUPS_TIME_RE = re.compile( + r"\[CUDA\] expert groups timing: H2D\s+([0-9.]+)\s+ms\s*\|\s*kernel\s+([0-9.]+)\s+ms\s*\|\s*D2H\s+([0-9.]+)\s+ms" +) + +# LOOKAHEAD recall block — 4 named rows. (name, pct, hit, tot) +LOOKAHEAD_RE = re.compile( + r"^\s*(.+?)\s+([0-9.]+)%\s+\((\d+)/(\d+)\)\s*$", re.MULTILINE +) + +# "loaded in 0.02s | resident dense: 0.21 MB | layers=5 experts=8 | MTP absent (draft=0)" +LOAD_BANNER_RE = re.compile( + r"loaded in\s+([0-9.]+)s\s*\|\s*resident dense:\s+([0-9.]+)\s+MB\s*\|" + r"\s*layers=(\d+)\s+experts=(\d+)\s*\|\s*MTP\s+(\w+)\s+\(draft=(\d+)\)" +) + + +# --- tunable floors ----------------------------------------------------------- +# These are deliberately generous so they catch *regressions* (broken builds, +# pathological configs, telemetry accounting bugs) without flapping on machine +# noise. The tiny model is fully resident at ~200 tok/s, so a 20 tok/s floor is +# a 10x margin. Tune per-host via env if needed (documented in README). +TINY_TOK_S_FLOOR = float(os.environ.get("COLI_TINY_TOK_S_FLOOR", "20.0")) +# On a fully-resident tiny model the expert-disk wait share should be tiny. +# If it exceeds this, something regressed in the I/O accounting or cache path. +MAX_DISK_WAIT_SHARE = float(os.environ.get("COLI_MAX_DISK_WAIT_SHARE", "0.20")) +# decode wall-time should be roughly the sum of PROFILE phases (other = residual). +PROFILE_SUM_TOLERANCE = float(os.environ.get("COLI_PROFILE_SUM_TOL", "0.05")) +# Minimum direct CPU-vs-CUDA argmax agreement on the tiny TF fixture. The two +# backends use different accumulation orders (SIMD dot vs CUDA kernel), so a +# few near-tied positions flip argmax — that's expected numeric divergence, not +# a kernel bug. Measured baseline ~84% (27/32) on this fixture; the 70% floor +# leaves headroom for machine noise while still catching a catastrophic kernel +# regression (e.g. a wrong GEMM would drop this to ~random = ~4%). +MIN_CPU_CUDA_AGREEMENT = float(os.environ.get("COLI_MIN_CPU_CUDA_AGREE", "0.70")) + + +def parse_run(stdout: str, stderr: str = "") -> dict: + """Parse one engine run's output into a telemetry dict. + + Returns keys: tok_s, hit_pct, profile (dict, seconds), profile_sum, + time_shares (dict, fractions 0..1), verdict, loads_per_tok, cuda (dict), + tf_match (tuple or None), parsed (set of field names found). + + Raises RuntimeError only if the core throughput line is missing — everything + else is optional and absent on some run modes (e.g. [PROF] needs PROF=1, + CUDA tier needs gpu_expert_count>0). + """ + out = dict( + tok_s=None, hit_pct=None, profile=None, profile_sum=None, + time_shares=None, verdict=None, loads_per_tok=None, + cuda=None, tf_match=None, stderr=stderr, + ) + parsed = set() + blob = stdout + "\n" + stderr # [PROF]/[CUDA] live on stderr; scan both. + + m = _first_speed(stdout) + if m: + out["tok_s"] = float(m.group(1)); parsed.add("tok_s") + + m = HIT_RE.search(blob) + if m: + out["hit_pct"] = float(m.group(1)); parsed.add("hit_pct") + + m = PROFILE_RE.search(stdout) + if m: + service, wait, emm, attn, head, other = (float(x) for x in m.groups()) + disk = service + (wait or 0.0) + out["profile"] = dict(zip(PROFILE_KEYS, (disk, emm, attn, head, other))) + out["profile_sum"] = disk + emm + attn + head + other + parsed.add("profile") + + # ATTENTION sub-breakdown: projection/RoPE | score-softmax-value | output. + m = ATTN_BREAKDOWN_RE.search(stdout) + if m: + out["attn_breakdown"] = dict(zip( + ("proj_rope", "score_sm_value", "out_proj"), + (float(x) for x in m.groups()))) + parsed.add("attn_breakdown") + + m = SHARES_RE.search(blob) + if m: + io, emm, attn, head, other = (float(x) / 100.0 for x in m.groups()) + out["time_shares"] = dict(io=io, matmul=emm, attention=attn, head=head, other=other) + parsed.add("time_shares") + + m = VERDICT_RE.search(blob) + if m: + out["verdict"] = m.group(1); parsed.add("verdict") + + # [PROF] decode forwards + latency p50/p90/p99/max. + m = LATENCY_RE.search(blob) + if m: + out["latency"] = dict(zip( + ("forwards", "p50_ms", "p90_ms", "p99_ms", "max_ms"), + (float(x) for x in m.groups()))) + parsed.add("latency") + + # [PROF] expert I/O throughput: GB fetched, MB/token, GB/s, service vs felt wait. + m = EXPERT_IO_RE.search(blob) + if m: + out["expert_io"] = dict(zip( + ("gb_fetched", "mb_per_tok", "gb_per_s", "loads_per_tok", + "read_service_s", "felt_wait_s"), + (float(x) for x in m.groups()))) + parsed.add("expert_io") + + m = LOADS_PER_TOK_RE.search(blob) + if m: + out["loads_per_tok"] = float(m.group(1)); parsed.add("loads_per_tok") + + # experts loaded/token with per-layer spread + baseline topk (run_text summary). + m = EXPERTS_LOADED_RE.search(stdout) + if m: + out["experts_loaded"] = dict( + per_tok=float(m.group(1)), per_layer=float(m.group(2)), + n_sparse_layers=int(m.group(3)), baseline_topk=int(m.group(4))) + parsed.add("experts_loaded") + + # speculation: tokens/forward, forwards, tokens, MTP acceptance%. + m = SPECULATION_RE.search(stdout) + if m: + out["speculation"] = dict(zip( + ("tok_per_fw", "forwards", "tokens", "mtp_accept_pct"), + (float(m.group(1)), int(m.group(2)), int(m.group(3)), float(m.group(4))))) + parsed.add("speculation") + + # routing quality (CACHE_ROUTE inline suffixes on the summary line). + m = SWAP_RE.search(stdout) + if m: + out["swap"] = dict(pct=float(m.group(1)), swaps=int(m.group(2)), slots=int(m.group(3))) + parsed.add("swap") + m = ROUTE_AGREE_RE.search(stdout) + if m: + out["route_agree"] = dict(agree_pct=float(m.group(1)), kl=float(m.group(2))) + parsed.add("route_agree") + + # disk-load split by decode phase (DISK_SPLIT=1). + m = DISK_SPLIT_RE.search(stdout) + if m: + out["disk_split"] = dict(zip( + ("draft", "absorb", "verify_main", "mtp_loads", "mtp_gb", + "main_loads", "main_gb", "mtp_bytes_pct"), + (int(m.group(1)), int(m.group(2)), int(m.group(3)), int(m.group(4)), + float(m.group(5)), int(m.group(6)), float(m.group(7)), + float(m.group(8)) if m.group(8) else None))) + parsed.add("disk_split") + + # provenance: machine + resolved config. + m = MACHINE_RE.search(blob) + if m: + out["machine"] = dict(cpu=m.group(1).strip(), cores=int(m.group(2)), backend=m.group(3)) + parsed.add("machine") + m = CONFIG_RE.search(blob) + if m: + out["config_str"] = m.group(1).strip(); parsed.add("config") + + # load banner: load time, resident dense MB, layers, experts, MTP status. + m = LOAD_BANNER_RE.search(stdout) + if m: + out["load"] = dict(zip( + ("load_s", "resident_dense_mb", "layers", "experts", "mtp_status", "draft"), + (float(m.group(1)), float(m.group(2)), int(m.group(3)), + int(m.group(4)), m.group(5), int(m.group(6))))) + parsed.add("load") + + # LOOKAHEAD routing-recall block (LOOKA=1): list of {predictor, pct, hit, tot}. + la_block = re.search( + r"LOOKAHEAD routing.*?recall.*?:\n((?:^\s+.+?\s+[0-9.]+%\s+\(\d+/\d+\)\s*$\n?)+)", + blob, re.MULTILINE) + if la_block: + out["lookahead"] = [] + for row in LOOKAHEAD_RE.finditer(la_block.group(1)): + out["lookahead"].append(dict( + predictor=row.group(1).strip(), pct=float(row.group(2)), + hit=int(row.group(3)), tot=int(row.group(4)))) + parsed.add("lookahead") + + cuda = dict(enabled=False, expert_count=None, expert_gb=None, + calls_served=None, resident_tensors=None, resident_gb=None, + groups=None, groups_timing=None) + if CUDA_DEVICE_RE.search(stderr): + cuda["enabled"] = True + m = CUDA_TIER_RE.search(stdout) + if m: + cuda["expert_count"] = int(m.group(1)) + cuda["expert_gb"] = float(m.group(2)) + cuda["calls_served"] = int(m.group(3)) + m = CUDA_RESIDENT_RE.search(stderr) + if m: + cuda["resident_tensors"] = int(m.group(1)) + cuda["resident_gb"] = float(m.group(2)) + m = CUDA_GROUPS_RE.search(stderr) + if m: + cuda["groups"] = dict(zip( + ("calls", "experts", "rows", "experts_per_call"), + (int(m.group(1)), int(m.group(2)), int(m.group(3)), float(m.group(4))))) + m = CUDA_GROUPS_TIME_RE.search(stderr) + if m: + cuda["groups_timing"] = dict(zip( + ("h2d_ms", "kernel_ms", "d2h_ms"), + (float(m.group(1)), float(m.group(2)), float(m.group(3))))) + out["cuda"] = cuda + + m = TF_MATCH_RE.search(stdout) + if m: + out["tf_match"] = (int(m.group(1)), int(m.group(2))); parsed.add("tf_match") + + # Capture per-position argmax divergences from the oracle, keyed by position. + mismatches = {int(mm.group(1)): int(mm.group(2)) + for mm in TF_MISMATCH_RE.finditer(blob)} + if out["tf_match"] is not None: + out["tf_mismatches"] = mismatches + parsed.add("tf_mismatches") + + out["parsed"] = parsed + return out + + +def run_engine( + env_overlay: dict, + *, + engine: Optional[str] = None, + cap: int = 4, + ebits: int = 4, + dbits: int = 4, + timeout: float = 600.0, + snap: Optional[str] = None, +) -> tuple[dict, subprocess.CompletedProcess]: + """Run the engine with an env overlay; return (parsed_telemetry, proc). + + `engine` defaults to ./glm.exe (colibri's Windows host). `snap` defaults to + the bundled tiny model (glm_tiny) so callers can omit it for fast tests. + The positional argv is `cap ebits dbits`, matching the engine's main(). + """ + if engine is None: + engine = str(Path(__file__).resolve().parent.parent / "glm.exe") + env = os.environ.copy() + # Strip CUDA vars by default so a CPU run isn't accidentally GPU-accelerated + # by a leftover env; callers opt in by passing them in env_overlay. + for k in ("COLI_CUDA", "COLI_GPU", "COLI_GPUS", "CUDA_DENSE", "CUDA_EXPERT_GB"): + env.pop(k, None) + env.update(env_overlay) + if snap is not None: + env["SNAP"] = snap + elif "SNAP" not in env: + env["SNAP"] = str(Path(__file__).resolve().parent.parent / "glm_tiny") + + proc = subprocess.run( + [engine, str(cap), str(ebits), str(dbits)], + env=env, capture_output=True, text=True, timeout=timeout, + ) + telemetry = parse_run(proc.stdout, proc.stderr) + telemetry["returncode"] = proc.returncode + telemetry["env"] = {k: env_overlay[k] for k in env_overlay} + return telemetry, proc + + +def disk_wait_share(t: dict) -> Optional[float]: + """Fraction of decode wall-time spent waiting on expert disk reads. + + Preferred source: [PROF] time_shares (the engine's own accounting, which + separates felt-wait from read-service). Falls back to PROFILE disk / sum if + [PROF] wasn't emitted (PROF=0 runs). None if neither is available. + """ + if t.get("time_shares"): + return t["time_shares"]["io"] + if t.get("profile") and t.get("profile_sum"): + return t["profile"]["disk"] / t["profile_sum"] + return None + + +def tf_agreement(cpu: dict, cuda: dict, oracle: list[int]) -> tuple[float, list[int]]: + """Direct CPU-vs-CUDA argmax agreement on the TF fixture. + + Both runs prefilled the SAME oracle sequence; tf_mismatches holds each + backend's actual argmax where it diverged from the oracle. Where a backend + is ABSENT from the mismatch map, its prediction equals the oracle token at + that position. So the reconstructed per-position prediction is: + oracle[i] if i not in mismatches else mismatches[i] + and agreement is the fraction of positions where CPU and CUDA predictions + are identical — independent of how each relates to the oracle. + + `oracle` is ref_glm.json's tf_pred (the per-position oracle argmax). Pass + n_positions = len(oracle). + + Returns (agreement_fraction, list_of_differing_positions). + """ + cm = cpu.get("tf_mismatches") or {} + gm = cuda.get("tf_mismatches") or {} + diff = [] + for i, orc in enumerate(oracle): + cpu_tok = cm.get(i, orc) # matched oracle => oracle token + cuda_tok = gm.get(i, orc) + if cpu_tok != cuda_tok: + diff.append(i) + agree = (len(oracle) - len(diff)) / len(oracle) if oracle else 0.0 + return agree, diff diff --git a/c/tools/eval_glm.py b/c/tools/eval_glm.py index 3aa568e4..55e12d6d 100644 --- a/c/tools/eval_glm.py +++ b/c/tools/eval_glm.py @@ -22,7 +22,7 @@ # leve di ricerca: passate al motore via env TOPP=0.9 python3 tools/eval_glm.py --snap /home/vincenzo/glm52_i4 --data ./bench --tasks mmlu --ram 15 """ -import os, sys, subprocess, argparse, random, json, tempfile, time +import os, sys, subprocess, argparse, random, json, tempfile, time, threading # mini-set OFFLINE per testare la meccanica (NON misura qualita': domande banali) SMOKE = [ @@ -114,6 +114,7 @@ def main(): ap.add_argument("--seed", type=int, default=1234) ap.add_argument("--dry", action="store_true", help="build requests and stop without running the engine") ap.add_argument("--selftest", action="store_true", help="verify the scoring calculations") + ap.add_argument("--out", default="", help="write incremental results CSV here (one row per request, flushed as it lands)") a = ap.parse_args() if a.selftest: # acc/acc_norm con logprob sintetici @@ -143,15 +144,62 @@ def main(): if a.ram: env["RAM_GB"] = str(a.ram) cmd = [a.glm, str(a.cap)] + a.bits.split() print("running:", " ".join(cmd), file=sys.stderr) + + # Stream results line-by-line so a crash at request N keeps 1..N-1 and shows + # exactly where it stopped. The engine prints " " per + # request to stdout and "[score N req | ...]" progress to stderr; buffering + # both until exit (the old subprocess.run) wastes the whole run on a crash. + out_f = open(a.out, "a") if a.out else None + if out_f: + out_f.write(f"# eval_glm snap={a.snap} tasks={a.tasks} limit={a.limit} seed={a.seed} started={time.strftime('%Y-%m-%dT%H:%M:%S')}\n") + out_f.write("req_idx,task,qi,oi,contlen,contchars,gold,logprob,greedy\n") + out_f.flush() t0 = time.time() - proc = subprocess.run(cmd, env=env, capture_output=True, text=True) - if proc.returncode != 0: - print("ENGINE ERROR:\n", proc.stderr[-2000:], file=sys.stderr); sys.exit(1) - lines = [l for l in proc.stdout.strip().splitlines() if l and l[0] in "-0123456789"] - if len(lines) != len(reqs): - print(f"WARNING: {len(lines)} outputs for {len(reqs)} requests", file=sys.stderr) - lp = [float(l.split()[0]) for l in lines] - print(f"(engine: {time.time()-t0:.0f}s){proc.stderr.strip().splitlines()[-1] if proc.stderr.strip() else ''}", file=sys.stderr) + proc = subprocess.Popen(cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + text=True, bufsize=1) # line-buffered + lp = [None] * len(reqs) + n_done = 0 + # Drain stderr (engine progress lines) to console live on a background thread + # so the [score N req] heartbeat is visible while stdout is consumed below. + def _drain_stderr(): + for line in proc.stderr: + print(f" [engine] {line.rstrip()}", file=sys.stderr) + threading.Thread(target=_drain_stderr, daemon=True).start() + for line in proc.stdout: + line = line.strip() + if not line or line[0] not in "-0123456789": continue + parts = line.split() + if n_done >= len(reqs): break + try: logprob = float(parts[0]) + except (ValueError, IndexError): continue + lp[n_done] = logprob + greedy = parts[2] if len(parts) > 2 else "?" + t, qi, oi, clen, cchars, gold = meta[n_done] + if out_f: + out_f.write(f"{n_done},{t},{qi},{oi},{clen},{cchars},{gold},{logprob:.6f},{greedy}\n") + out_f.flush() + n_done += 1 + if n_done % 5 == 0 or n_done == len(reqs): + elapsed = time.time() - t0 + rate = n_done / elapsed if elapsed > 0 else 0 + eta = (len(reqs) - n_done) / rate if rate > 0 else 0 + print(f"[progress] {n_done}/{len(reqs)} requests scored | {elapsed:.0f}s elapsed | " + f"{rate:.2f} req/s | ETA {eta:.0f}s | last: {t} q{qi} opt{oi} lp={logprob:.3f}", + file=sys.stderr) + proc.wait() + elapsed = time.time() - t0 + if out_f: + out_f.write(f"# finished: {n_done}/{len(reqs)} in {elapsed:.0f}s, exit={proc.returncode}\n") + out_f.close() + if proc.returncode != 0 and n_done == 0: + print(f"ENGINE ERROR (exit {proc.returncode})", file=sys.stderr); sys.exit(1) + if n_done != len(reqs): + print(f"WARNING: only {n_done}/{len(reqs)} requests scored (engine exited {proc.returncode}); " + f"scoring partial results.", file=sys.stderr) + # Fill any unscored slots with -inf so argmax never picks them + for i in range(len(lp)): + if lp[i] is None: lp[i] = float("-inf") + print(f"(engine: {elapsed:.0f}s, {n_done}/{len(reqs)} scored, exit {proc.returncode})", file=sys.stderr) score_accuracy(tasks, meta, perq, lp) print("\nNOTE: compare acc_norm with GLM-5.2's PUBLISHED model-card score. A close result" "\n indicates that int4 quantization preserved quality. (Fill REFERENCE in tools/eval_glm.py.)") diff --git a/c/tools/iq3_pack.py b/c/tools/iq3_pack.py new file mode 100644 index 00000000..c95a343a --- /dev/null +++ b/c/tools/iq3_pack.py @@ -0,0 +1,249 @@ +#!/usr/bin/env python3 +"""fmt=6 (E8/IQ3 lattice container) index codec — #452 ladder step 2. + +Note: fmt=5 is taken by the int3 dual-plane container (#132); this lattice +container is fmt=6. + +The ablation (#453) proved the SCHEME: an IQ3_XXS-style codebook plus rotation +matches our simulated E8 ball (51.5% vs 51.5% on OLMoE). That code quantizes to +lattice points and keeps floats. This module produces the DEPLOYABLE bytes and +reads them back, so the container, the converter and the decode kernels all +agree on one layout. + +Layout - one 256-weight super-block, 98 bytes, 3.0625 bpw: + + [0 .. 63] uint8 grid index per 4-dim magnitude block (64 blocks) + [64 .. 95] uint32 x8, one per 32-weight sub-block: + bits 0..20 three 7-bit sign words (8 weights each, + bit i set => weight i negative; the 8th sign + is implied by odd parity) + bits 21..27 the fourth 7-bit sign word + bits 28..31 4-bit sub-scale code + [96 .. 97] fp16 super-scale d + + value(w) = d * (0.5 + code) * 0.5 * grid[idx][j] * 0.5 * sign + +The last 0.5 is the half-unit convention of the published grid (magnitudes are +stored doubled: 4,12,...,62 mean 2.0,6.0,...,31.0). + +Odd-parity signs: llama.cpp stores 7 of every 8 signs and derives the 8th so the +product of the eight is +1. The encoder therefore flips the smallest-magnitude +weight of any block whose true signs violate that — the same cost the ablation +priced in, now applied for real. +""" +import json +import os +import numpy as np + +QK = 256 # weights per super-block +SUB = 32 # weights per sub-block (one uint32 of signs+scale) +BLOCK_BYTES = QK // 4 + (QK // SUB) * 4 + 2 # 64 + 32 + 2 = 98 + +_GRID = None + + +def grid(): + """[256,4] float32 magnitudes in weight units (published table is doubled).""" + global _GRID + if _GRID is None: + path = os.path.join(os.path.dirname(__file__), "iq3xxs_grid.json") + _GRID = np.asarray(json.load(open(path)), dtype=np.float32) * 0.5 + return _GRID + + +def _nearest(mag4): + """[N,4] magnitudes -> [N] grid indices, argmin ||m-g||^2 without cdist.""" + g = grid() + g2 = (g * g).sum(1) + out = np.empty(len(mag4), dtype=np.uint8) + for i in range(0, len(mag4), 1 << 16): # bounded working set + c = mag4[i:i + (1 << 16)] + out[i:i + len(c)] = np.argmin(g2 - 2.0 * (c @ g.T), axis=1).astype(np.uint8) + return out + + +def encode(x): + """float32 [..., K] (K % 256 == 0) -> packed uint8 [..., K//256 * 98].""" + x = np.ascontiguousarray(x, dtype=np.float32) + K = x.shape[-1] + if K % QK: + raise ValueError(f"fmt=5 needs K % {QK} == 0, got {K}") + rows = x.reshape(-1, K) + nsb = K // QK + out = np.empty((len(rows), nsb * BLOCK_BYTES), dtype=np.uint8) + + for sb in range(nsb): + blk = rows[:, sb * QK:(sb + 1) * QK] # [R,256] + sign = np.where(blk < 0, -1.0, 1.0).astype(np.float32) + mag = np.abs(blk) + + # parity fix: flip the smallest magnitude of every 8 whose product is -1 + s8 = sign.reshape(len(rows), QK // 8, 8) + m8 = mag.reshape(len(rows), QK // 8, 8) + viol = s8.prod(-1) < 0 # [R,32] + amin = m8.argmin(-1) + r, b = np.nonzero(viol) + s8[r, b, amin[r, b]] *= -1.0 + sign = s8.reshape(len(rows), QK) + + base = sb * BLOCK_BYTES + # super-scale: RMS anchor, same statistic the ablation searches around + d = np.sqrt((mag * mag).mean(-1, keepdims=True)) / 20.0 + 1e-12 + out[:, base + 96:base + 98] = d.astype(np.float16).view(np.uint8) + d = d.astype(np.float16).astype(np.float32) # encode what we store + + g = grid() + for ib in range(QK // SUB): + m = mag[:, ib * SUB:(ib + 1) * SUB] # [R,32] + best_err = None + best = None + for code in range(16): + db = d * (0.5 + code) * 0.5 + q = (m / np.maximum(db, 1e-20)).reshape(-1, 4) + idx = _nearest(q) + rec = g[idx].reshape(len(rows), SUB) * db + err = ((rec - m) ** 2).sum(-1, keepdims=True) + if best_err is None: + best_err, best = err, (idx.reshape(len(rows), SUB // 4), code) + else: + take = (err < best_err)[:, 0] + if take.any(): + keep_idx, keep_code = best + ni = idx.reshape(len(rows), SUB // 4) + keep_idx = np.where(take[:, None], ni, keep_idx) + # per-row code: store alongside, resolved below + keep_code = np.where(take, code, keep_code) if isinstance( + keep_code, np.ndarray) else np.where( + take, code, np.full(len(rows), keep_code)) + best = (keep_idx, keep_code) + best_err = np.where(take[:, None], err, best_err) + bidx, bcode = best + if not isinstance(bcode, np.ndarray): + bcode = np.full(len(rows), bcode) + out[:, base + ib * 8:base + (ib + 1) * 8] = bidx.astype(np.uint8) + + # signs: four 7-bit words for this sub-block + the 4-bit code + s = sign[:, ib * SUB:(ib + 1) * SUB].reshape(len(rows), 4, 8) + neg = (s < 0).astype(np.uint32) + word = np.zeros(len(rows), dtype=np.uint32) + for l in range(4): + seven = np.zeros(len(rows), dtype=np.uint32) + for j in range(7): + seven |= neg[:, l, j] << j + word |= seven << (7 * l) + word |= (bcode.astype(np.uint32) & 0xF) << 28 + off = base + QK // 4 + ib * 4 + out[:, off:off + 4] = word.view(np.uint8).reshape(len(rows), 4) if False else \ + np.ascontiguousarray(word).view(np.uint8).reshape(len(rows), 4) + return out.reshape(*x.shape[:-1], nsb * BLOCK_BYTES) + + +def decode(packed, K): + """packed uint8 [..., K//256*98] -> float32 [..., K]. The kernels' reference.""" + packed = np.ascontiguousarray(packed, dtype=np.uint8) + nsb = K // QK + rows = packed.reshape(-1, nsb * BLOCK_BYTES) + out = np.empty((len(rows), K), dtype=np.float32) + g = grid() + for sb in range(nsb): + base = sb * BLOCK_BYTES + d = rows[:, base + 96:base + 98].copy().view(np.float16).astype(np.float32) + for ib in range(QK // SUB): + idx = rows[:, base + ib * 8:base + (ib + 1) * 8] # [R,8] + off = base + QK // 4 + ib * 4 + word = np.ascontiguousarray(rows[:, off:off + 4]).view(np.uint32).reshape(-1) + code = (word >> 28) & 0xF + db = d[:, 0] * (0.5 + code) * 0.5 # [R] + mag = g[idx].reshape(len(rows), SUB) # [R,32] + sgn = np.ones((len(rows), 4, 8), dtype=np.float32) + for l in range(4): + seven = (word >> (7 * l)) & 0x7F + par = 0 + for j in range(7): + bit = (seven >> j) & 1 + sgn[:, l, j] = np.where(bit == 1, -1.0, 1.0) + par ^= bit + sgn[:, l, 7] = np.where(par == 1, -1.0, 1.0) # odd parity closes the block + out[:, sb * QK + ib * SUB:sb * QK + (ib + 1) * SUB] = \ + mag * sgn.reshape(len(rows), SUB) * db[:, None] + return out.reshape(*packed.shape[:-1], K) + + +def bpw(): + return BLOCK_BYTES * 8 / QK + + +# ---- rotation (converter side of quant.h e8_signs / e8_rot_rows) ------------- +# +# Q = D @ H / sqrt(n) per power-of-two block; W@Q on weight rows here equals +# Q^T x on activations in the engine (sign-flip then FWHT — one routine both +# sides). The sign diagonal D is REGENERATED from the block size, never stored: +# both sides draw the same xorshift64* stream seeded 417+n, and the kernel +# fixture (make_e8_fixture.py) pins the agreement. + +def signs(n): + """[n] float32 in {+1,-1} — bit-exact mirror of quant.h e8_signs().""" + s = np.uint64(417 + n) + out = np.empty((n + 7) // 8, dtype=np.uint8) + with np.errstate(over="ignore"): + for i in range(len(out)): + s ^= s >> np.uint64(12) + s ^= (s << np.uint64(25)) & np.uint64(0xFFFFFFFFFFFFFFFF) + s ^= s >> np.uint64(27) + out[i] = np.uint8(((s * np.uint64(2685821657736338717)) & + np.uint64(0xFFFFFFFFFFFFFFFF)) >> np.uint64(56)) + bits = np.unpackbits(out, bitorder="little")[:n] + return np.where(bits == 1, -1.0, 1.0).astype(np.float32) + + +def rot_blocks(dim): + """Block tiling: each block is the largest power of two dividing the + remainder (its lowest set bit) — 6144 -> [2048, 4096], 1536 -> [512, 1024]. + Blocks over 32768 halve, mirroring the engine's sign-buffer cap.""" + out, rem = [], dim + while rem: + b = rem & (-rem) + while b > 32768: + b >>= 1 + out.append(b) + rem -= b + return out + + +def _fwht_rows(blk, b): + """In-place-style FWHT over rows of [..., b] (b a power of two).""" + h = 1 + while h < b: + blk = blk.reshape(-1, b // (2 * h), 2, h) + u, v = blk[:, :, 0, :].copy(), blk[:, :, 1, :].copy() + blk[:, :, 0, :] = u + v + blk[:, :, 1, :] = u - v + blk = blk.reshape(-1, b) + h <<= 1 + return blk / np.sqrt(b, dtype=np.float32) + + +def rotate_rows(x): + """Apply the rotation to rows of float32 [..., dim] (weights W@Q or + activations Q^T x — the transform is the same). Returns a new array.""" + x = np.ascontiguousarray(x, dtype=np.float32) + dim = x.shape[-1] + rows = x.reshape(-1, dim).copy() + off = 0 + for b in rot_blocks(dim): + rows[:, off:off + b] = _fwht_rows(rows[:, off:off + b] * signs(b)[None, :], b) + off += b + return rows.reshape(x.shape) + + +def unrotate_rows(x): + """Inverse of rotate_rows (Q v back to v): FWHT first, then the sign flip. + Eval-side only — the engine always works in the rotated space.""" + x = np.ascontiguousarray(x, dtype=np.float32) + dim = x.shape[-1] + rows = x.reshape(-1, dim).copy() + off = 0 + for b in rot_blocks(dim): + rows[:, off:off + b] = _fwht_rows(rows[:, off:off + b].copy(), b) * signs(b)[None, :] + off += b + return rows.reshape(x.shape) diff --git a/c/tools/iq3xxs_grid.json b/c/tools/iq3xxs_grid.json new file mode 100644 index 00000000..3cde1a28 --- /dev/null +++ b/c/tools/iq3xxs_grid.json @@ -0,0 +1 @@ +[[4, 4, 4, 4], [20, 4, 4, 4], [36, 4, 4, 4], [12, 12, 4, 4], [28, 12, 4, 4], [62, 12, 4, 4], [4, 20, 4, 4], [20, 20, 4, 4], [12, 28, 4, 4], [20, 36, 4, 4], [28, 62, 4, 4], [44, 62, 4, 4], [12, 4, 12, 4], [28, 4, 12, 4], [4, 12, 12, 4], [20, 12, 12, 4], [12, 20, 12, 4], [44, 20, 12, 4], [4, 28, 12, 4], [20, 28, 12, 4], [12, 36, 12, 4], [36, 44, 12, 4], [4, 62, 12, 4], [4, 4, 20, 4], [20, 4, 20, 4], [36, 4, 20, 4], [12, 12, 20, 4], [4, 20, 20, 4], [20, 20, 20, 4], [12, 28, 20, 4], [28, 28, 20, 4], [62, 28, 20, 4], [12, 44, 20, 4], [62, 44, 20, 4], [44, 62, 20, 4], [12, 4, 28, 4], [62, 4, 28, 4], [4, 12, 28, 4], [20, 12, 28, 4], [44, 20, 28, 4], [4, 62, 28, 4], [28, 12, 36, 4], [62, 28, 36, 4], [36, 36, 36, 4], [62, 44, 36, 4], [28, 62, 36, 4], [44, 62, 36, 4], [12, 4, 44, 4], [62, 4, 44, 4], [20, 28, 44, 4], [20, 44, 44, 4], [44, 28, 52, 4], [36, 52, 52, 4], [4, 12, 62, 4], [36, 12, 62, 4], [52, 12, 62, 4], [28, 36, 62, 4], [12, 52, 62, 4], [12, 4, 4, 12], [28, 4, 4, 12], [4, 12, 4, 12], [20, 12, 4, 12], [12, 20, 4, 12], [28, 20, 4, 12], [4, 28, 4, 12], [20, 28, 4, 12], [36, 28, 4, 12], [62, 36, 4, 12], [4, 44, 4, 12], [4, 4, 12, 12], [20, 4, 12, 12], [12, 12, 12, 12], [4, 20, 12, 12], [20, 20, 12, 12], [12, 4, 20, 12], [28, 4, 20, 12], [4, 12, 20, 12], [20, 12, 20, 12], [12, 20, 20, 12], [4, 28, 20, 12], [20, 62, 20, 12], [4, 4, 28, 12], [20, 4, 28, 12], [4, 20, 28, 12], [12, 28, 28, 12], [52, 36, 28, 12], [52, 52, 28, 12], [12, 4, 36, 12], [44, 4, 36, 12], [4, 44, 36, 12], [4, 20, 44, 12], [36, 20, 44, 12], [52, 36, 44, 12], [12, 62, 44, 12], [44, 4, 52, 12], [20, 20, 62, 12], [4, 36, 62, 12], [4, 4, 4, 20], [20, 4, 4, 20], [12, 12, 4, 20], [28, 12, 4, 20], [4, 20, 4, 20], [20, 20, 4, 20], [52, 20, 4, 20], [12, 28, 4, 20], [20, 36, 4, 20], [12, 4, 12, 20], [28, 4, 12, 20], [44, 4, 12, 20], [4, 12, 12, 20], [20, 12, 12, 20], [12, 20, 12, 20], [4, 28, 12, 20], [28, 52, 12, 20], [62, 52, 12, 20], [4, 62, 12, 20], [4, 4, 20, 20], [20, 4, 20, 20], [12, 12, 20, 20], [62, 12, 20, 20], [4, 20, 20, 20], [20, 20, 20, 20], [62, 28, 20, 20], [4, 36, 20, 20], [44, 44, 20, 20], [12, 4, 28, 20], [4, 12, 28, 20], [36, 12, 28, 20], [4, 62, 28, 20], [36, 62, 28, 20], [44, 28, 36, 20], [28, 44, 36, 20], [28, 4, 44, 20], [62, 20, 44, 20], [12, 36, 44, 20], [36, 62, 44, 20], [12, 4, 62, 20], [28, 4, 62, 20], [52, 12, 62, 20], [44, 36, 62, 20], [12, 4, 4, 28], [4, 12, 4, 28], [20, 12, 4, 28], [12, 20, 4, 28], [28, 20, 4, 28], [4, 44, 4, 28], [44, 52, 4, 28], [20, 62, 4, 28], [4, 4, 12, 28], [20, 4, 12, 28], [4, 20, 12, 28], [12, 28, 12, 28], [36, 36, 12, 28], [52, 36, 12, 28], [12, 4, 20, 28], [28, 4, 20, 28], [4, 12, 20, 28], [44, 20, 20, 28], [20, 44, 20, 28], [20, 62, 20, 28], [12, 12, 28, 28], [28, 28, 28, 28], [4, 28, 36, 28], [62, 36, 36, 28], [20, 62, 36, 28], [4, 4, 44, 28], [52, 4, 44, 28], [20, 20, 44, 28], [44, 44, 44, 28], [36, 12, 52, 28], [52, 28, 52, 28], [28, 52, 52, 28], [28, 28, 62, 28], [4, 52, 62, 28], [36, 4, 4, 36], [62, 12, 4, 36], [44, 28, 4, 36], [62, 28, 4, 36], [28, 44, 4, 36], [62, 44, 4, 36], [36, 62, 12, 36], [4, 20, 20, 36], [62, 28, 20, 36], [4, 36, 20, 36], [4, 52, 20, 36], [52, 52, 20, 36], [62, 4, 28, 36], [44, 36, 28, 36], [36, 4, 36, 36], [12, 44, 36, 36], [36, 52, 36, 36], [44, 20, 44, 36], [28, 36, 44, 36], [4, 62, 44, 36], [44, 4, 62, 36], [4, 12, 62, 36], [20, 12, 62, 36], [4, 28, 62, 36], [20, 12, 4, 44], [12, 36, 4, 44], [4, 62, 4, 44], [4, 4, 12, 44], [52, 4, 12, 44], [52, 20, 12, 44], [44, 44, 12, 44], [36, 12, 20, 44], [20, 28, 20, 44], [20, 62, 20, 44], [20, 4, 28, 44], [28, 44, 28, 44], [4, 12, 36, 44], [28, 20, 36, 44], [62, 20, 36, 44], [20, 62, 36, 44], [20, 4, 44, 44], [12, 28, 44, 44], [4, 44, 52, 44], [36, 20, 62, 44], [20, 36, 62, 44], [36, 20, 4, 52], [36, 36, 4, 52], [52, 36, 4, 52], [36, 52, 4, 52], [12, 20, 12, 52], [12, 52, 12, 52], [62, 12, 20, 52], [36, 52, 20, 52], [4, 28, 28, 52], [52, 28, 28, 52], [36, 36, 36, 52], [44, 4, 44, 52], [20, 44, 44, 52], [28, 28, 52, 52], [28, 4, 62, 52], [12, 20, 62, 52], [28, 4, 4, 62], [44, 4, 4, 62], [62, 4, 4, 62], [4, 12, 4, 62], [20, 28, 4, 62], [20, 44, 4, 62], [52, 20, 12, 62], [4, 36, 12, 62], [20, 12, 20, 62], [44, 36, 20, 62], [20, 44, 20, 62], [4, 4, 28, 62], [44, 12, 28, 62], [28, 28, 28, 62], [4, 52, 28, 62], [12, 20, 36, 62], [12, 36, 36, 62], [4, 4, 44, 62], [20, 4, 44, 62], [36, 20, 44, 62], [4, 28, 52, 62]] \ No newline at end of file diff --git a/c/tools/make_e8_fixture.py b/c/tools/make_e8_fixture.py new file mode 100644 index 00000000..1a146cff --- /dev/null +++ b/c/tools/make_e8_fixture.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +"""Generate the fmt=6 kernel fixture from the reference codec (#452). + +tests/test_e8_kernel.c checks the C kernel against these bytes, so the fixture +must come from iq3_pack.py — the format's reference implementation — and never +from the kernel itself. Regenerate whenever the layout changes: + + python3 tools/make_e8_fixture.py +""" +import os +import struct + +import numpy as np + +import iq3_pack as P + +O, I = 24, 512 +SEED = 20260721 + + +def main(): + rng = np.random.default_rng(SEED) + w = (rng.standard_normal((O, I)) * 0.05).astype(np.float32) + packed = P.encode(w) + deq = P.decode(packed, I).astype(np.float32) + x = (rng.standard_normal(I) * 1.0).astype(np.float32) + # float64 reference so the C float path is compared against something better + yref = (deq.astype(np.float64) @ x.astype(np.float64)).astype(np.float32) + + # Rotated section (#452 converter step): the container stores W@Q; the + # engine rotates activations with e8_rot_rows. I2=768 tiles as 256+512, + # exercising the multi-block path and the regenerated signs on both sides. + O2, I2 = 16, 768 + w2 = (rng.standard_normal((O2, I2)) * 0.05).astype(np.float32) + packed2 = P.encode(P.rotate_rows(w2)) + x2 = (rng.standard_normal(I2) * 1.0).astype(np.float32) + xrot2 = P.rotate_rows(x2[None, :])[0].astype(np.float32) + y2ref = (P.decode(packed2, I2).astype(np.float64) + @ xrot2.astype(np.float64)).astype(np.float32) + + out = os.path.join(os.path.dirname(__file__), "..", "tests", "fixtures") + os.makedirs(out, exist_ok=True) + path = os.path.join(out, "e8_case.bin") + with open(path, "wb") as f: + f.write(struct.pack(" 0), -s8, s8) + s = s8.reshape(-1, 32) + # sub-scale search: db candidates from the 4-bit code, super d from block RMS + d = m.pow(2).mean(-1, keepdim=True).sqrt() / 20.0 + 1e-12 # rough anchor + best = None + for code in range(16): + db = d * (0.5 + code) * 0.5 + q = m / db # [N,32] target magnitudes + q4 = q.reshape(-1, 4) # 4-dim grid blocks + # chunked argmin ||q-g||^2 = argmin(|g|^2 - 2 q.g): a full cdist on a + # 100M-param tensor materializes tens of GB — this stays at ~256 MB. + g2 = grid.pow(2).sum(-1) + idx = torch.empty(q4.shape[0], dtype=torch.long, device=q4.device) + CH = 1 << 18 + for i0 in range(0, q4.shape[0], CH): + cc = q4[i0:i0+CH] + idx[i0:i0+CH] = (g2 - 2.0 * (cc @ grid.T)).argmin(-1) + hit = grid[idx].reshape(-1, 8, 4) + rec = (hit.reshape(-1, 32) * db) + err = (rec - m).pow(2).sum(-1, keepdim=True) + if best is None: + best = (err, rec) + else: + take = err < best[0] + best = (torch.where(take, err, best[0]), torch.where(take, rec, best[1])) + out[:, sb*32:(sb+1)*32] = best[1] * s + return out.reshape(orig) + + def _rot_quant(x, bits, group, e8=""): """W -> Qn(W@Q) @ Q^T along the last (input) dim — see rotation() above.""" q = rotation(x.shape[-1], x.device) @@ -206,13 +274,30 @@ def _quant_e8(x, group, bits, ball): return best_out.reshape(shp) -SCHEME_RE = re.compile(r"^int(2|3|4|8)(?:-g(\d+))?(-e8u?)?(-rot)?(-nohead)?$") +SCHEME_RE = re.compile(r"^int(2|3|4|8)(?:-g(\d+))?(-e8u?|-iq3)?(-rot)?(-nohead)?$") + + +def _quant_iq3_pack(x): + """Bytes-true fmt=6 (#452 step 2): the tensor goes through tools/iq3_pack — + real 98-byte blocks, the parity sign cost, fp16 super-scales, regenerated + rotation signs — and comes back to model space via the inverse rotation. + This is the number a SHIPPED container gets, where `int3-iq3-rot` above is + the float simulation that chose the scheme. The rotation sign diagonal + differs from the simulation's torch PRNG (xorshift spec, quant.h e8_signs); + statistically equivalent, and it is what the engine regenerates.""" + import iq3_pack as IP + w = x.detach().cpu().float().numpy() + packed = IP.encode(IP.rotate_rows(w.reshape(-1, w.shape[-1]))) + deq = IP.decode(packed, w.shape[-1]) + return torch.from_numpy(IP.unrotate_rows(deq).reshape(w.shape)).to(x.device) def parse_scheme(name): """'int4-g128-nohead' -> (bits, group, e8, skip_head...). 'fp16' -> None.""" if name == "fp16": return None + if name == "iq3-pack": + return "PACK" m = SCHEME_RE.match(name) if not m: raise SystemExit(f"bad scheme '{name}' (expected fp16 | int{{2,3,4,8}}[-g][-e8|-e8u][-rot][-nohead])") @@ -236,8 +321,19 @@ def apply_scheme(model, scheme): spec = parse_scheme(scheme) if spec is None: return 0, 0, total - bits, group, e8, rot, skip_head = spec n = qp = 0 + if spec == "PACK": + with torch.no_grad(): + for name, p in model.named_parameters(): + if p.ndim < 2 or is_router(name): + continue + if p.shape[-1] % 256: # fmt=6 needs I % 256 == 0 — same rule the converter enforces + continue + p.data.copy_(_quant_iq3_pack(p.data).to(p.dtype)) + n += 1 + qp += p.numel() + return n, qp, total + bits, group, e8, rot, skip_head = spec with torch.no_grad(): for name, p in model.named_parameters(): if p.ndim < 2 or is_router(name): diff --git a/c/tools/requirements.txt b/c/tools/requirements.txt new file mode 100644 index 00000000..3d0087c6 --- /dev/null +++ b/c/tools/requirements.txt @@ -0,0 +1,15 @@ +# Python dependencies for the colibri offline converter / oracle / download tools +# (c/tools/*.py and c/*.py). Minimum-version pins; for a fully reproducible env, +# freeze exact versions into your own lockfile with `pip freeze > locked.txt`. +torch>=2.4 +safetensors>=0.4 +huggingface-hub>=0.24 +numpy>=1.26 +tokenizers>=0.20 +transformers>=5.11 # <5.11 applies split-half Llama RoPE and drifts vs the engine's interleaved MLA (issue #281) +datasets>=2.20 +requests>=2.31 + +# Optional download-fallback backends (only needed for download_fp8.py's alt paths): +# modelscope>=1.9 +# hf_transfer>=0.1 diff --git a/c/tools/test_olmoe_real.py b/c/tools/test_olmoe_real.py new file mode 100644 index 00000000..1abf64d0 --- /dev/null +++ b/c/tools/test_olmoe_real.py @@ -0,0 +1,91 @@ +"""Bootstrap ref_olmoe_real.json by running olmoe.exe once and capturing output. + +Step 1: Creates a temp ref with only prompt_ids (no full_ids). +Step 2: Runs olmoe.exe, parses the generated IDs from stdout. +Step 3: Saves {prompt_ids, full_ids} as ref_olmoe_real.json. +Step 4: Runs olmoe.exe again against the saved ref to verify determinism. + +No RAM loading of the full model -- the engine streams from SSD as designed. +""" +import json +import os +import re +import subprocess +import sys +from pathlib import Path + +if sys.platform == "win32": + for s in (sys.stdout, sys.stderr): + try: + s.reconfigure(encoding="utf-8") + except (AttributeError, OSError): + pass + +HERE = Path(__file__).resolve().parent.parent +ext = ".exe" if sys.platform == "win32" else "" +ENGINE = HERE / f"olmoe{ext}" +SNAP = os.getenv("SNAP", str(HERE.parent / "olmoe_merged")) +REF_OUT = HERE / "ref_olmoe_real.json" +BOOTSTRAP_REF = HERE / "ref_olmoe_bootstrap.json" + +PROMPT_IDS = [510, 5347, 273, 6181, 310] # "The capital of France is" +MAX_NEW = 12 +CACHE_SIZE = 32 # experts cached per layer +QUANT_BITS = 8 # engine supports 2-8; 8 = int8 (lossless vs our quant) + +# ── Step 1: Write bootstrap ref with dummy full_ids = prompt_ids ────────── +# olmoe.exe needs full_ids to know how many tokens to generate (nfull - np). +# We extend with MAX_NEW zeros so the engine generates MAX_NEW tokens. +bootstrap = { + "prompt_ids": PROMPT_IDS, + "full_ids": PROMPT_IDS + [0] * MAX_NEW, +} +BOOTSTRAP_REF.write_text(json.dumps(bootstrap)) +print(f"Bootstrap ref written to {BOOTSTRAP_REF}") + +env = {**os.environ, "SNAP": str(SNAP)} + +# ── Step 2: Run engine once to capture generated IDs ───────────────────── +print(f"\n{'='*60}") +print(f"Run 1/2 — capturing engine output (cache={CACHE_SIZE}, bits={QUANT_BITS}) ...") +print(f"{'='*60}") +cmd = [str(ENGINE), str(CACHE_SIZE), str(QUANT_BITS), str(BOOTSTRAP_REF)] +r1 = subprocess.run(cmd, env=env, capture_output=True, text=True, cwd=str(HERE)) +print(r1.stdout) +if r1.returncode != 0: + print("STDERR:", r1.stderr, file=sys.stderr) + sys.exit(r1.returncode) + +# Parse "C engine : ..." line +m = re.search(r"C engine\s*:\s*([\d ]+)", r1.stdout) +if not m: + sys.exit("Could not parse 'C engine :' line from output") +gen_ids = [int(x) for x in m.group(1).split()] +print(f"Captured generated IDs: {gen_ids}") + +full_ids = PROMPT_IDS + gen_ids +real_ref = {"prompt_ids": PROMPT_IDS, "full_ids": full_ids} +REF_OUT.write_text(json.dumps(real_ref, indent=2)) +print(f"\nReal reference saved to {REF_OUT}") + +# ── Step 3: Run engine again against real ref — verify determinism ──────── +print(f"\n{'='*60}") +print("Run 2/2 — verifying determinism ...") +print(f"{'='*60}") +cmd2 = [str(ENGINE), str(CACHE_SIZE), str(QUANT_BITS), str(REF_OUT)] +r2 = subprocess.run(cmd2, env=env, capture_output=True, text=True, cwd=str(HERE)) +print(r2.stdout) +if r2.returncode != 0: + print("STDERR:", r2.stderr, file=sys.stderr) + sys.exit(r2.returncode) + +if "Matching tokens: 12/12" in r2.stdout or f"Matching tokens: {MAX_NEW}/{MAX_NEW}" in r2.stdout: + print("✓ Engine is DETERMINISTIC — same output on both runs!") +else: + m2 = re.search(r"Matching tokens: (\d+)/(\d+)", r2.stdout) + if m2: + print(f"⚠ Partial match: {m2.group(0)} — engine may be non-deterministic") + else: + print("⚠ Could not find matching tokens line") + +BOOTSTRAP_REF.unlink(missing_ok=True) diff --git a/c/tools/try_tool_calling.py b/c/tools/try_tool_calling.py new file mode 100644 index 00000000..c1e46958 --- /dev/null +++ b/c/tools/try_tool_calling.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +"""Manual end-to-end tool-calling probe against a running colibri server (#401). + +Runs a real two-turn loop against the OpenAI-compatible endpoint: declare a tool, let the +model call it, execute it locally, feed the result back, and check the model uses it. +No client library and no dependencies -- stdlib only. + + python3 tools/try_tool_calling.py # default http://127.0.0.1:8080 + python3 tools/try_tool_calling.py --url http://host:port + python3 tools/try_tool_calling.py --raw # also dump the raw model text + +Exit status is 0 only if every stage passed, so it can be used as a smoke test. +""" +import argparse +import json +import sys +import urllib.error +import urllib.request + +WEATHER_TOOL = { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather in a given city.", + "parameters": { + "type": "object", + "properties": { + "city": {"type": "string", "description": "City name, e.g. Rome"}, + "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, + }, + "required": ["city"], + }, + }, +} + +# What the tool "really" returns, so the second turn has a fact the model could not invent. +FAKE_WEATHER = {"city": "Rome", "temp_c": 31, "conditions": "sunny"} + + +def discover_model(url, timeout): + """Ask the server which model it is serving, so the caller never has to guess the id.""" + with urllib.request.urlopen(url + "/v1/models", timeout=timeout) as resp: + data = json.loads(resp.read()).get("data") or [] + if not data: + raise RuntimeError("server reports no models") + return data[0]["id"] + + +def post(url, body, timeout): + req = urllib.request.Request(url + "/v1/chat/completions", + data=json.dumps(body).encode(), + headers={"Content-Type": "application/json"}) + with urllib.request.urlopen(req, timeout=timeout) as resp: + return json.loads(resp.read()) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--url", default="http://127.0.0.1:8080", help="server base URL") + ap.add_argument("--timeout", type=int, default=1800, help="seconds per request") + ap.add_argument("--raw", action="store_true", help="print raw message objects") + ap.add_argument("--tool-choice", default=None, help='e.g. "required" to force a call') + ap.add_argument("--model", default=None, help="model id (default: ask the server)") + args = ap.parse_args() + url = args.url.rstrip("/") + + try: + model = args.model or discover_model(url, 30) + except (urllib.error.URLError, OSError) as e: + print(f"FAIL: cannot reach {url} -- is the server running? ({e})") + return 2 + print(f"server: {url} model: {model}") + + messages = [{"role": "user", "content": "What's the weather in Rome right now? " + "Use the tool, don't guess."}] + body = {"model": model, "messages": messages, "tools": [WEATHER_TOOL], + "temperature": 0, "max_tokens": 256} + if args.tool_choice: + body["tool_choice"] = (args.tool_choice if args.tool_choice in ("auto", "none", "required") + else {"type": "function", "function": {"name": args.tool_choice}}) + + print("== turn 1: asking the model to call the tool ==") + try: + out = post(url, body, args.timeout) + except urllib.error.URLError as e: + print(f"FAIL: cannot reach {url} -- is the server running? ({e})") + return 2 + msg = out["choices"][0]["message"] + if args.raw: + print(json.dumps(msg, indent=2, ensure_ascii=False)) + + calls = msg.get("tool_calls") or [] + if not calls: + print("FAIL: the model returned no tool_calls.") + print(f" content: {(msg.get('content') or '')[:400]!r}") + print(" If the content shows markers, the parse is the problem -- " + "check the server's stderr line starting with '[api]'.") + return 1 + + call = calls[0] + name = call["function"]["name"] + try: + parsed = json.loads(call["function"]["arguments"]) + except json.JSONDecodeError: + print(f"FAIL: arguments are not valid JSON: {call['function']['arguments']!r}") + return 1 + print(f" tool_calls: {len(calls)} name={name} arguments={parsed}") + + if name != "get_weather": + print(f"FAIL: model called {name!r}, not the declared tool.") + return 1 + if "city" not in parsed: + print(f"FAIL: required parameter 'city' missing from {parsed}.") + return 1 + if "rom" not in str(parsed["city"]).lower(): + print(f"WARN: city is {parsed['city']!r}, expected Rome -- continuing anyway.") + + print("== turn 2: feeding the tool result back ==") + messages.append({"role": "assistant", "content": msg.get("content"), "tool_calls": calls}) + messages.append({"role": "tool", "tool_call_id": call["id"], "name": name, + "content": json.dumps(FAKE_WEATHER)}) + body["messages"] = messages + body.pop("tool_choice", None) + out2 = post(url, body, args.timeout) + reply = (out2["choices"][0]["message"].get("content") or "").strip() + if args.raw: + print(json.dumps(out2["choices"][0]["message"], indent=2, ensure_ascii=False)) + print(f" reply: {reply[:400]!r}") + + if "31" not in reply: + print("FAIL: the reply does not mention 31 -- the model ignored the tool result.") + return 1 + print("\nPASS: tool declared -> called with valid arguments -> result consumed in the answer.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/c/version.py b/c/version.py index 9951ca18..ce018d8e 100644 --- a/c/version.py +++ b/c/version.py @@ -1,3 +1,3 @@ """Single source of truth for the colibrì version number.""" -__version__ = "1.0.0" +__version__ = "1.1.0" diff --git a/colibri/__init__.py b/colibri/__init__.py new file mode 100644 index 00000000..77f70615 --- /dev/null +++ b/colibri/__init__.py @@ -0,0 +1,5 @@ +"""colibrì — tiny engine, immense model.""" + +from colibri._version import __version__ + +__all__ = ["__version__"] diff --git a/colibri/_version.py b/colibri/_version.py new file mode 100644 index 00000000..174442aa --- /dev/null +++ b/colibri/_version.py @@ -0,0 +1,23 @@ +"""Version accessor for the pip package. + +The single source of truth is c/version.py (#394): coli --version and the +GitHub Release workflow read it, so the pip metadata must read the SAME file +instead of carrying a second literal that would drift on the first bump. + +From a checkout (the supported install: `pip install -e .`) the file is read +directly. From an installed wheel c/ is not on disk, so fall back to the +package metadata that setuptools baked at build time from that same file. +""" +from pathlib import Path + +try: + _ns = {} + exec((Path(__file__).resolve().parent.parent / "c" / "version.py").read_text(), _ns) + __version__ = _ns["__version__"] +except OSError: + from importlib.metadata import PackageNotFoundError, version + + try: + __version__ = version("colibri-engine") + except PackageNotFoundError: + __version__ = "0.0.0+unknown" diff --git a/colibri/cli.py b/colibri/cli.py new file mode 100644 index 00000000..3f192344 --- /dev/null +++ b/colibri/cli.py @@ -0,0 +1,30 @@ +"""Entry point for `coli` when installed via pip. + +Delegates to the original c/coli script which handles all subcommands. +This wrapper exists so `pip install colibri-engine` creates a `coli` console +script that works without the user having to add c/ to PATH manually. +""" + +import os +import sys +import runpy + + +def main(): + here = os.path.dirname(os.path.abspath(__file__)) + engine_dir = os.path.join(os.path.dirname(here), "c") + coli_script = os.path.join(engine_dir, "coli") + + if not os.path.exists(coli_script): + sys.exit( + "colibri engine directory not found.\n" + "Install from source: git clone + pip install -e ." + ) + + sys.path.insert(0, engine_dir) + sys.argv[0] = coli_script + runpy.run_path(coli_script, run_name="__main__") + + +if __name__ == "__main__": + main() diff --git a/docker/README.IT.md b/docker/README.IT.md index 43c3e312..302f0180 100644 --- a/docker/README.IT.md +++ b/docker/README.IT.md @@ -9,23 +9,23 @@ Una guida semplice per eseguire **Colibrì**, un motore di inferenza locale basa ## 📋 Sommario -- [Cosa è Colibrì?](#cosa-è-colibrì) -- [Cosa serve](#cosa-serve) - - [Hardware](#hardware) - - [Software](#software) -- [Come iniziare](#come-iniziare) - - [Passo 1: Scarica il modello](#passo-1-scarica-il-modello) - - [Passo 2: Scarica il Dockerfile di Colibrì](#passo-2-scarica-il-dockerfile-di-colibrì) - - [Passo 3: Compila l'immagine Docker](#passo-3-compila-limmagine-docker) - - [Passo 4: Avvia Colibrì](#passo-4-avvia-colibr%C3%AC) - - [Cosa significa quel comando?](#cosa-significa-quel-comando) - - [Usare Colibrì](#usare-colibrì) -- [Entrare in una console Linux dentro il container](#entrare-in-una-console-linux-dentro-il-container) -- [Risoluzione dei problemi](#risoluzione-dei-problemi) -- [Note tecniche](#note-tecniche) -- [Domande frequenti](#domande-frequenti) -- [Supporto e contributi](#supporto-e-contributi) -- [Test sul mio PC](#test-sul-mio-pc) +- [Cosa è Colibrì?](https://github.com/JustVugg/colibri/blob/main/docker/README.IT.md#cosa-%C3%A8-colibr%C3%AC) +- [Cosa serve](https://github.com/JustVugg/colibri/blob/main/docker/README.IT.md#cosa-serve) + - [Hardware](https://github.com/JustVugg/colibri/blob/main/docker/README.IT.md#hardware) + - [Software](https://github.com/JustVugg/colibri/blob/main/docker/README.IT.md#software) +- [Come iniziare](https://github.com/JustVugg/colibri/blob/main/docker/README.IT.md#come-iniziare) + - [Passo 1: Scarica il modello](https://github.com/JustVugg/colibri/blob/main/docker/README.IT.md#passo-1-scarica-il-modello) + - [Passo 2: Scarica il Dockerfile di Colibrì](https://github.com/JustVugg/colibri/blob/main/docker/README.IT.md#passo-2-scarica-il-dockerfile-di-colibr%C3%AC) + - [Passo 3: Compila l'immagine Docker](https://github.com/JustVugg/colibri/blob/main/docker/README.IT.md#passo-3-compila-limmagine-docker) + - [Passo 4: Avvia Colibrì](https://github.com/JustVugg/colibri/blob/main/docker/README.IT.md#passo-4-avvia-colibr%C3%AC) + - [Cosa significa quel comando?](https://github.com/JustVugg/colibri/blob/main/docker/README.IT.md#cosa-significa-quel-comando) + - [Usare Colibrì](https://github.com/JustVugg/colibri/blob/main/docker/README.IT.md#usare-colibr%C3%AC) +- [Entrare nel container](https://github.com/JustVugg/colibri/blob/main/docker/README.IT.md#entrare-nel-container) +- [Risoluzione dei problemi](https://github.com/JustVugg/colibri/blob/main/docker/README.IT.md#risoluzione-dei-problemi) +- [Note tecniche](https://github.com/JustVugg/colibri/blob/main/docker/README.IT.md#note-tecniche) +- [Domande frequenti](https://github.com/JustVugg/colibri/blob/main/docker/README.IT.md#domande-frequenti) +- [Supporto e contributi](https://github.com/JustVugg/colibri/blob/main/docker/README.IT.md#supporto-e-contributi) +- [Testing on a low resource PC](https://github.com/JustVugg/colibri/blob/main/docker/README.IT.md#test-su-un-pc-con-poche-risorse) --- @@ -196,7 +196,7 @@ Il modello capisce **italiano, inglese, cinese e altre lingue**, anche se è ott --- -## Entrare in una console Linux dentro il container +## Entrare nel container Se vuoi esplorare il container come fosse una macchina Linux normale: @@ -408,7 +408,7 @@ Buon divertimento! 🐦 --- -## Test sul mio PC +## Test su un PC con poche risorse Nel primo caso ho fatto una domanda in italiano, nel secondo in giapponese, e nel terzo ho rifatto la domanda in giapponese ma ho richiesto una risposta in italiano. diff --git a/docker/README.md b/docker/README.md index 44d8a193..0fc19f1f 100644 --- a/docker/README.md +++ b/docker/README.md @@ -9,23 +9,23 @@ A simple guide to running **Colibrì**, a local inference engine based on GLM 5. ## 📋 Table of Contents -* [What is Colibrì?](https://www.google.com/search?q=#what-is-colibr%C3%AC) -* [Requirements](https://www.google.com/search?q=#requirements) - * [Hardware](https://www.google.com/search?q=#hardware) - * [Software](https://www.google.com/search?q=#software) -* [How to get started](https://www.google.com/search?q=#how-to-get-started) - * [Step 1: Download the model](https://www.google.com/search?q=#step-1-download-the-model) - * [Step 2: Download the Colibrì Dockerfile](https://www.google.com/search?q=#step-2-download-the-colibr%C3%AC-dockerfile) - * [Step 3: Build the Docker image](https://www.google.com/search?q=#step-3-build-the-docker-image) - * [Step 4: Start Colibrì](https://www.google.com/search?q=#step-4-start-colibr%C3%AC) - * [What does that command mean?](https://www.google.com/search?q=#what-does-that-command-mean) - * [Using Colibrì](https://www.google.com/search?q=#using-colibr%C3%AC) -* [Entering a Linux console inside the container](https://www.google.com/search?q=#entering-a-linux-console-inside-the-container) -* [Troubleshooting](https://www.google.com/search?q=#troubleshooting) -* [Technical notes](https://www.google.com/search?q=#technical-notes) -* [Frequently Asked Questions](https://www.google.com/search?q=#frequently-asked-questions) -* [Support and contributions](https://www.google.com/search?q=#support-and-contributions) -* [Tests on my PC](https://www.google.com/search?q=#tests-on-my-pc) +* [What is Colibrì?](https://github.com/JustVugg/colibri/blob/main/docker/README.md#what-is-colibr%C3%AC) +* [Requirements](https://github.com/JustVugg/colibri/blob/main/docker/README.md#requirements) + * [Hardware](https://github.com/JustVugg/colibri/blob/main/docker/README.md#hardware) + * [Software](https://github.com/JustVugg/colibri/blob/main/docker/README.md#software) +* [How to get started](https://github.com/JustVugg/colibri/blob/main/docker/README.md#how-to-get-started) + * [Step 1: Download the model](https://github.com/JustVugg/colibri/blob/main/docker/README.md#step-2-download-the-colibr%C3%AC-dockerfile) + * [Step 2: Download the Colibrì Dockerfile](https://github.com/JustVugg/colibri/blob/main/docker/README.md#step-2-download-the-colibr%C3%AC-dockerfile) + * [Step 3: Build the Docker image](https://github.com/JustVugg/colibri/blob/main/docker/README.md#step-3-build-the-docker-image) + * [Step 4: Start Colibrì](https://github.com/JustVugg/colibri/blob/main/docker/README.md#step-4-start-colibr%C3%AC) + * [What does that command mean?](https://github.com/JustVugg/colibri/blob/main/docker/README.md#what-does-that-command-mean) + * [Using Colibrì](https://github.com/JustVugg/colibri/blob/main/docker/README.md#using-colibr%C3%AC) +* [Enter the container](https://github.com/JustVugg/colibri/blob/main/docker/README.md#entering-a-linux-console-inside-the-container) +* [Troubleshooting](https://github.com/JustVugg/colibri/blob/main/docker/README.md#troubleshooting) +* [Technical notes](https://github.com/JustVugg/colibri/blob/main/docker/README.md#technical-notes) +* [Frequently Asked Questions](https://github.com/JustVugg/colibri/blob/main/docker/README.md#frequently-asked-questions) +* [Support and contributions](https://github.com/JustVugg/colibri/blob/main/docker/README.md#support-and-contributions) +* [Testing on a low resource PC](https://github.com/JustVugg/colibri/blob/main/docker/README.md#testing-on-a-low-resource-pc) --- @@ -206,7 +206,7 @@ The model understands **Italian, English, Chinese, and other languages**, althou --- -## Entering a Linux console inside the container +## Enter the container If you want to explore the container as if it were a normal Linux machine: @@ -439,7 +439,7 @@ Have fun! 🐦 --- -## Tests on my PC +## Testing on a low resource PC In the first case, I asked a question in Italian; in the second, in Japanese; and in the third, I repeated the question in Japanese but requested an answer in Italian. diff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.md index 9f0d7490..2684753f 100644 --- a/docs/ENVIRONMENT.md +++ b/docs/ENVIRONMENT.md @@ -41,7 +41,7 @@ Format: `VAR` — default — effect. | `PIPE` | `0` (off) | Overlap expert disk-load with matmul via I/O worker threads. Byte-identical output; reorders I/O. `PIPE=1` opts in. | | `PIPE_WORKERS` | `8` | Number of pthread loaders when `PIPE=1`, or the io-wq worker maximum per ring when `URING=1` (capped at 64). Tune to SSD queue depth and available cores. | | `URING` | `0` (off) | Linux-only queued expert I/O. `URING=1` implies `PIPE=1`, forces cold reads through io-wq (`IOSQE_ASYNC`), replaces blocking loader pthreads and spin waits with batched SQEs/CQEs, and batches `PILOT_REAL` loads on a separate ring. Use `DIRECT=1` for cold NVMe to avoid page-cache copy/readahead limits. Fails clearly if the kernel denies io_uring; incompatible with `COLI_MMAP=1`. | -| `DIRECT` | `0` (off) | Use `O_DIRECT`/unbuffered reads for expert slabs. Helps sustained NVMe; keeps the zero-copy GPU path. | +| `DIRECT` | `0` (off) | Use `O_DIRECT`/unbuffered reads for expert slabs. **Drive-dependent — measure it on your hardware.** On real NVMe with DRAM cache and headroom it is often a large win (measured +34% decode with `PIPE=1` on a Blackwell/Windows box, and 4.25→9.69 GB/s in iobench on a GB10); on QLC/DRAM-less drives or slow/virtualised disks it can be neutral to negative. Helps sustained NVMe; keeps the zero-copy GPU path. | | `COLI_NO_OMP_TUNE` | off | **Kill-switch** for the OpenMP hot-thread tuning (`OMP_WAIT_POLICY=active` spin + proc-bind). Set `=1` when the CPU is mostly waiting on the GPU (Metal) so spin doesn't steal the shared power budget. | | `COLI_NUMA` | auto in generated plans on multi-socket Linux; otherwise off | `COLI_NUMA=1` selectively interleaves large expert and dense slabs across NUMA nodes via `mbind` (raw syscall, no libnuma). Helps multi-socket hosts (+7–40% expert matmul); silent no-op on single-node or non-Linux. Explicit `COLI_NUMA=0` overrides the generated plan. | | `MLOCK` | `-1` (auto: on for macOS) | Wire the streamed expert cache into physical RAM (`mlock`) to dodge the memory compressor. `0` off, `1` force. | @@ -78,11 +78,21 @@ Format: `VAR` — default — effect. --- +## Dual-SSD streaming + +| Variable | Default | Effect | +|---|---|---| +| `COLI_MODEL_DIRS` | unset | SPLIT the model across 2+ drives: a `;`/`,`-separated list of extra directories, each holding a **distinct** subset of the `.safetensors` shards (no duplication). Shards act as a search path — every shard is read from whichever drive holds it, so concurrent expert loads parallelise across drives and combined capacity is used. Scales to N drives. Metadata (config/tokenizer/`.coli_usage`) stays in the primary `COLI_MODEL` dir. Pairs well with `PIPE=1` (concurrent loaders) + `DIRECT=1`. Distinct from — and composable with — `COLI_MODEL_MIRROR`: the mirror is matched per-shard by basename against the merged (split) index, so a mirror dir may hold a copy of any subset of the split's shards. | +| `COLI_MODEL_MIRROR` | unset | Path to a second, byte-identical (read-only) copy of the model on another drive; expert reads are split across both. Partial mirrors work (only the shards present are used). | +| `COLI_DISK_WEIGHTS` | unset (startup bandwidth probe) | Split ratio `,` (e.g. `1,1` for 50/50, `9,3` for a fast+slow pair). Unset = probe both drives with the engine's own access pattern at startup. | + +Per-drive byte counts are reported in a `MIRROR:` stats line. Combine with `DIRECT=1` so the two copies never compete for page cache. + ## CUDA (NVIDIA) | Variable | Default | Effect | |---|---|---| -| `COLI_CUDA` | off | Enable the CUDA backend. Requires a CUDA build. | +| `COLI_CUDA` | off | Enable the CUDA backend. Requires a CUDA build. An explicit `COLI_CUDA=0` disables it **and suppresses the Windows bare-run auto-enable** (before this, Windows "CPU" runs with `COLI_CUDA=0` silently got a VRAM expert tier). The CLI flag `--gpu none` is the canonical hard off-switch on every platform. | | `COLI_GPU` / `COLI_GPUS` | unset | Device selection (`auto`, `none`, or a list like `0,1`). Requires `COLI_CUDA=1`. | | `CUDA_DENSE` | `0` | Place dense (non-expert) matmuls on the GPU. | | `CUDA_EXPERT_GB` | `0` | VRAM budget (GB) for caching experts on the GPU. | @@ -93,7 +103,7 @@ Format: `VAR` — default — effect. | `COLI_CUDA_PIPE` | `0` (off) | `1` engages the multi-step attention pipeline; `2` enables the pipe2 path. | | `COLI_CUDA_PIPE_SHARD` | off | `=1` runs the multi-device P2P head-shard attention path (opt-in for NVLink topologies; serializes ~95 MB/layer over a star PCIe topology). | | `COLI_CUDA_PIPE_S_MIN` | `1` single-GPU, `8` multi-GPU | Minimum prefill batch S to engage the pipe2 CUDA path. | -| `COLI_CUDA_MTP` | `0` (off) | `=1` opts into MTP speculation under CUDA (off by default: cold streaming experts run on CPU where the fused-pair/IDOT kernels diverge in FP order, collapsing draft acceptance, #163/#292). | +| `COLI_CUDA_MTP` | `0` (off) | `=1` opts into MTP speculation under CUDA (off by default: cold streaming experts run on CPU where the fused-pair/IDOT kernels diverge in FP order, collapsing draft acceptance, #163/#292 — though #467 measured acceptance holding at 49% on sm_120). When set explicitly, the resource planner skips its `DRAFT=0` export so the engine's auto path can engage draft=3 — no need to also set `DRAFT`. Note the measured trade-off (#467): at ~85% hit the widened S=4 expert union costs more than speculation saves (−32%); the opt-in pays only near-full residency (~99% hit). | | `COLI_CUDA_ASYNC` | on | `=0` forces synchronous `cudaMemcpy` instead of async + pinned host staging. | | `COLI_CUDA_DUAL_PROJ` | on | `=0` issues gate+up as two separate launches instead of one fused `grouped_hidden_w4_dual`. | | `COLI_CUDA_W4_PACKED` | on | `=0` disables the grouped packed-int4 path. | @@ -105,6 +115,15 @@ Format: `VAR` — default — effect. | `COLI_CUDA_SHARED_W4A16_MIN_ROWS` | `32` | Min row count to engage the shared-MLP W4A16 kernel. | | `COLI_METAL_UNTRACKED` | off (Metal only) | `=1` sets `MTLResourceHazardTrackingModeUntracked` on Metal buffers (reduces hazard-tracking overhead). | +> **Windows note.** On Windows, a bare `coli chat` / `coli run` / `coli serve` +> (no `--gpu`/`--vram`/`--auto-tier`) **auto-enables the GPU** when it detects a +> CUDA build (`coli_cuda.dll` next to the engine) and at least one GPU via +> `nvidia-smi`. The expert-tier VRAM budget is then sized automatically from the +> card's free VRAM (same computation as `--auto-tier`). If `nvidia-smi` is not on +> `PATH` the run falls back to CPU with a warning — pass `--vram N` (or add +> `nvidia-smi` to `PATH`) to enable CUDA in that case. `--gpu none` forces +> CPU-only. (Linux/macOS behaviour is unchanged: pass a flag to enable CUDA.) + --- ## Advanced / experimental / debug diff --git a/docs/grammar-draft.md b/docs/grammar-draft.md index 707fe5c6..6140cbbb 100644 --- a/docs/grammar-draft.md +++ b/docs/grammar-draft.md @@ -83,3 +83,27 @@ this implementation adds: forced spans as a **draft source verified in the targe own forward** (lossless even under a wrong grammar, composes with MTP/n-gram in one union batch), deployed where the win is denominated in expert I/O rather than forward passes. + +## Server usage: `response_format` (OpenAI API) + +The gateway (`openai_server.py`) turns `response_format` into a **per-request** +grammar carried to the engine over the `SUBMIT` protocol (optional 7th header +field, `gbytes`; 6-field headers from older clients remain valid — the field is +additive and back-compatible in both directions): + +```jsonc +{"response_format": {"type": "json_object"}} // generic JSON grammar +{"response_format": {"type": "json_schema", + "json_schema": {"schema": { ... }}}} // compiled by schema_gbnf.h +{"response_format": {"type": "gbnf", "grammar": "root ::= ..."}} // raw GBNF (extension) +``` + +Semantics are identical to `GRAMMAR=`/`SCHEMA=`: a **draft source, never a +sampling constraint**. A schema outside the supported subset, or malformed GBNF, +costs the speedup — never the request, never the output. Drafting engages for +greedy requests (`temperature: 0`); sampled requests run undrafted. Compile +overhead is negligible: ~8 µs/request for a typical schema, ~18 µs at the +32-level nesting cap (measured, M3 Max). Grammar payloads are capped at 1 MiB. +As with MTP (#100), a drafted greedy run may differ from an undrafted one in +near-tie tokens (the verify forward has a different batch shape); each output +is a valid greedy stream of its own forward shapes. diff --git a/docs/tuning.md b/docs/tuning.md index c95eca8d..ff1e7850 100644 --- a/docs/tuning.md +++ b/docs/tuning.md @@ -40,6 +40,13 @@ Auto-tier plans size OpenMP from physical cores and bind workers across cores. Memory-bound quantized kernels can regress sharply when SMT siblings compete for limited memory channels; explicit `OMP_*` settings always take precedence. +> Note (#471): exporting `OMP_PROC_BIND`/`OMP_PLACES` used to interact badly with +> the engine's one-time OpenMP tuning re-exec on Linux — the re-exec'd image +> inherited the first image's place-0 thread binding and the whole team landed on +> one core (~20× slowdown). The engine now resets its affinity to all online CPUs +> right before the re-exec, so explicit `OMP_*` pinning works as documented. +> `COLI_OMP_TUNED=1` remains the escape hatch that skips the re-exec entirely. + ```bash coli plan --model /models/glm52_i4 --policy quality coli run --auto-tier --policy quality "Explain MoE offloading" diff --git a/docs/windows.md b/docs/windows.md index ecb81e70..ee4944d7 100644 --- a/docs/windows.md +++ b/docs/windows.md @@ -8,6 +8,8 @@ A start-to-finish, reproducible path from a fresh Windows 11 machine to GLM-5.2 |---|---|---| | git, Python 3 | clone + `coli` launcher | winget / python.org | | MinGW-w64 gcc + make | builds the engine (MSVC can't) | `scoop install mingw-winlibs`, MSYS2, or portable **w64devkit** (no admin, unzip and go) | + +> **scoop MinGW caveat (#478):** `scoop install mingw-winlibs` ships `gcc` + `make` but **no `sh.exe`** — the Makefile's recipes use POSIX shell idioms (`command -v`, `{ ...; }`, redirect to `/dev/null`) that GNU make runs through `/bin/sh`. Without sh.exe on PATH, make falls back to `cmd.exe` and the build fails with `'printf' is not recognized` / `The system cannot find the path specified`. Two fixes: install **MSYS2** (recommended — it's what the recipes target), or `set PATH=%PATH%;C:\msys64\usr\bin` in any shell you build from. The portable **w64devkit** bundle includes sh.exe and works as-is. | CUDA Toolkit ≥ 12.8 | GPU tier; ≥12.8 required for Blackwell/sm_120 | `winget install Nvidia.CUDA` | | MSVC Build Tools (C++ workload) | nvcc's host compiler for the CUDA DLL | `winget install Microsoft.VisualStudio.2022.BuildTools` + "Desktop development with C++" | | ~400 GB free on a local NVMe | the int4 model (~370–384 GB) | NTFS is fine; **never** a network mount | @@ -54,6 +56,12 @@ This is not Defender and not Mark-of-the-Web — SAC blocks *all* unsigned, unkn nvcc needs MSVC as host compiler, so this one step must run from a shell with the MSVC environment: open **"x64 Native Tools Command Prompt for VS 2022"** from the Start menu (plain PowerShell will fail the `cl` check). Then: +> **The VS prompt has no `sh.exe` (#478):** that prompt is a `cmd.exe` shell, and the `cuda-dll` recipe uses POSIX idioms (`command -v`, `{ ...; }`) that need `/bin/sh`. Run this once in the VS prompt before building: +> ```cmd +> set PATH=%PATH%;C:\msys64\usr\bin +> ``` +> (adjust the path if you installed MSYS2 elsewhere). If you skipped MSYS2 in favor of w64devkit or scoop MinGW, point this at wherever `sh.exe` lives. + ```cmd make cuda-dll CUDA_ARCH=sm_120 # match your GPU: sm_120 Blackwell, sm_89 Ada, ... make glm.exe CUDA_DLL=1 ARCH=native # relink host with the runtime loader @@ -91,6 +99,7 @@ Size `CUDA_EXPERT_GB` so dense (~10 GB) + experts + working set stays under your | Symptom | Cause | Fix | |---|---|---| +| `'printf' is not recognized` / `The system cannot find the path specified` during `make glm.exe` | scoop MinGW has no `sh.exe`; make fell back to cmd.exe (#478) | §0 — use MSYS2/w64devkit, or `set PATH=%PATH%;C:\msys64\usr\bin` | | `An Application Control policy has blocked this file` | Smart App Control | §2 — turn SAC off + **reboot** | | `cuda-dll ... Error 1` immediately | old tree: spaced CUDA_HOME / MSVC rejects `-Wextra` | update to current `dev` (#314) | | `glm.exe is up to date` but GPU never engages | old tree: stale CPU-only binary | update to `dev`, or delete `glm.exe` and rebuild | diff --git a/flake.lock b/flake.lock index 9b788fa6..a89029e8 100644 --- a/flake.lock +++ b/flake.lock @@ -20,11 +20,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1784160687, - "narHash": "sha256-iYL/bixrb6FlHFu/gIuBYzq6c6lM5AAXsXNSWXtIgQc=", + "lastModified": 1784280462, + "narHash": "sha256-DtoqIqM7VkR6NxAkcLpMwmi02USwWb3JdmNGLyhthc0=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "4382ed2b7a6839d4280a9b386db49cbc5907414d", + "rev": "293d6abedf0478e681a4dfcfcb35b30fc796a32f", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 584a0886..d67aa1ce 100644 --- a/flake.nix +++ b/flake.nix @@ -1,42 +1,57 @@ { description = "colibrì — run GLM-5.2 (744B MoE) on a consumer machine with ~25 GB RAM"; + # Reproducibility: these inputs track a branch, so torch/numpy/etc. float across + # rebuilds. For deterministic builds run `nix flake lock` once and COMMIT the + # generated flake.lock (it pins each input to a commit SHA). (#D2) inputs = { nixpkgs.url = "github:NixOS/nixpkgs/nixos-26.05"; flake-utils.url = "github:numtide/flake-utils"; }; - outputs = { self, nixpkgs, flake-utils }: - flake-utils.lib.eachDefaultSystem (system: - let - pkgs = import nixpkgs { inherit system; }; + outputs = { + self, + nixpkgs, + flake-utils, + }: + flake-utils.lib.eachDefaultSystem ( + system: let + pkgs = import nixpkgs {inherit system;}; # Python with the packages needed by the offline converter tools - pythonEnv = pkgs.python3.withPackages (ps: with ps; [ - torch - safetensors - huggingface-hub - numpy - tokenizers - datasets - ]); + pythonEnv = pkgs.python3.withPackages ( + ps: + with ps; [ + torch + safetensors + huggingface-hub + numpy + tokenizers + datasets + ] + ); colibri = pkgs.stdenv.mkDerivation { pname = "colibri"; version = "1.0"; src = ./.; - # python3 is needed by checkPhase: `make test-c` shells out to - # `python3 tools/run_tests.py` (see c/Makefile, PYTHON ?= python3). - nativeBuildInputs = [ pkgs.makeWrapper pkgs.python3 ]; + nativeBuildInputs = with pkgs; [makeWrapper]; - buildInputs = [ - pkgs.gcc - pkgs.gmp + buildInputs = with pkgs; [ + gcc + gmp ]; + # python3 is needed by checkPhase: `make test-c` shells out to + # `python3 tools/run_tests.py` (see c/Makefile, PYTHON ?= python3). + nativeCheckInputs = with pkgs; [python3]; + # Use x86-64-v3 (AVX2) for a portable binary; override with ARCH=native for local builds - ARCH = "x86-64-v3"; + ARCH = + if pkgs.stdenv.hostPlatform.isx86_64 + then "x86-64-v3" + else "native"; buildPhase = '' runHook preBuild @@ -56,7 +71,8 @@ cp c/glm $out/lib/colibri/glm cp c/coli $out/lib/colibri/coli chmod +x $out/lib/colibri/coli - cp c/openai_server.py c/resource_plan.py c/doctor.py $out/lib/colibri/ + cp c/openai_server.py c/resource_plan.py c/doctor.py c/version.py \ + $out/lib/colibri/ cp -r c/tools/* $out/lib/colibri/tools/ # $out/bin holds the user-facing entry points. @@ -86,12 +102,11 @@ description = "Run GLM-5.2 (744B MoE) on a consumer machine with ~25 GB RAM"; homepage = "https://github.com/JustVugg/colibri"; license = licenses.asl20; - platforms = platforms.linux; - mainProgram = "glm"; + platforms = with platforms; linux ++ darwin; + mainProgram = "coli"; }; }; - in - rec { + in { packages = { default = colibri; inherit colibri; @@ -100,23 +115,25 @@ apps = { default = { type = "app"; - program = "${colibri}/bin/glm"; + program = pkgs.lib.getExe colibri; }; - coli = { + glm = { type = "app"; - program = "${colibri}/bin/coli"; + program = "${colibri}/share/colibri/glm"; }; }; + formatter = pkgs.alejandra; + devShells.default = pkgs.mkShell { - inputsFrom = [ colibri ]; + inputsFrom = [colibri]; - packages = [ + packages = with pkgs; [ pythonEnv - pkgs.gcc - pkgs.gnumake - pkgs.clang-tools # clangd / clang-tidy for IDE support - pkgs.pkg-config + gcc + gnumake + clang-tools # clangd / clang-tidy for IDE support + pkg-config ]; shellHook = '' diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..18f8f858 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,53 @@ +[build-system] +requires = ["setuptools>=68.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "colibri-engine" +dynamic = ["version"] +description = "Tiny engine, immense model — run GLM-5.2 (744B MoE) locally" +readme = "README.md" +license = "Apache-2.0" +requires-python = ">=3.10" +authors = [ + {name = "JustVugg"}, +] +classifiers = [ + "Development Status :: 4 - Beta", + "Environment :: Console", + "Intended Audience :: Science/Research", + "Operating System :: POSIX :: Linux", + "Operating System :: MacOS", + "Operating System :: Microsoft :: Windows", + "Programming Language :: Python :: 3", + "Topic :: Scientific/Engineering :: Artificial Intelligence", +] + +[project.optional-dependencies] +convert = [ + "numpy", + "huggingface_hub", +] +oracle = [ + "torch>=2.0", + "transformers>=4.40", + "safetensors", +] +bench = [ + "tokenizers", + "datasets", +] + +[project.scripts] +coli = "colibri.cli:main" + +[project.urls] +Homepage = "https://github.com/JustVugg/colibri" +Issues = "https://github.com/JustVugg/colibri/issues" + +[tool.setuptools.dynamic] +version = {attr = "colibri._version.__version__"} + +[tool.setuptools.packages.find] +where = ["."] +include = ["colibri*"] diff --git a/web/src/App.tsx b/web/src/App.tsx index a33cc880..39c9f3b0 100644 --- a/web/src/App.tsx +++ b/web/src/App.tsx @@ -9,6 +9,7 @@ import { Database, Feather, Gauge, + Globe, HardDrive, KeyRound, Layers, @@ -34,6 +35,7 @@ import { Brain } from "./Brain" import { Profiling } from "./Profiling" import { persistPublicSettings, stored } from "@/lib/storage" import { cn } from "@/lib/utils" +import { useLocale } from "./i18n" const message = (role: ChatMessage["role"], content: string): ChatMessage => { let id: string @@ -42,15 +44,12 @@ const message = (role: ChatMessage["role"], content: string): ChatMessage => { } export default function App() { - // When the page is served by the engine itself (coli web), same-origin is the - // right default: no CORS, no manual endpoint editing. The Vite dev server - // (port 5173) keeps the classic default. + const { t, locale, setLocale, locales } = useLocale() + const servedByEngine = typeof window !== "undefined" && window.location.port !== "5173" && window.location.protocol.startsWith("http") const defaultBase = servedByEngine ? `${window.location.origin}/v1` : "http://127.0.0.1:8000/v1" const [baseUrl, setBaseUrl] = useState(() => { const saved = stored(localStorage, "colibri.baseUrl", defaultBase) - // migrate: a stored FACTORY default pointing at another origin would trip CORS - // when the page is engine-served — upgrade it to same-origin once. if (servedByEngine && saved === "http://127.0.0.1:8000/v1" && defaultBase !== saved) return defaultBase return saved }) @@ -120,7 +119,7 @@ export default function App() { const result = await getHealth(baseUrl, apiKey) if (!disposed) { setHealth(result); setHealthError("") } } catch (cause) { - if (!disposed) setHealthError(cause instanceof Error ? cause.message : "Runtime metrics unavailable") + if (!disposed) setHealthError(cause instanceof Error ? cause.message : "status.runtimeUnavailable") } } const timer = window.setInterval(() => void poll(), 5000) @@ -155,13 +154,13 @@ export default function App() { } catch (cause) { if (!controller.signal.aborted) { setHealth(null) - setHealthError(cause instanceof Error ? cause.message : "Runtime metrics unavailable") + setHealthError(cause instanceof Error ? cause.message : "status.runtimeUnavailable") } } } catch (cause) { if (controller.signal.aborted) return setConnected(false) - setError(cause instanceof Error ? cause.message : "Could not reach the server.") + setError(cause instanceof Error ? cause.message : "status.serverError") } finally { if (probeRef.current === controller) { probeRef.current = null; setConnecting(false) } } @@ -227,7 +226,7 @@ export default function App() { if (controller.signal.aborted) { updateMessages((current) => current.filter((item) => item.id !== assistant.id || item.content)) } else { - setError(cause instanceof Error ? cause.message : "Generation failed.") + setError(cause instanceof Error ? cause.message : "status.generationFailed") updateMessages((current) => current.filter((item) => item.id !== assistant.id || item.content)) } } finally { @@ -241,22 +240,22 @@ export default function App() {
-
ACTIVE MODEL{model}
+
{t("topbar.activeModel")}{model}
- - - + + +
- {loading && tokenCount > 0 ? {tokenCount} tokens : null} - {!loading && tokPerSec != null ? {tokPerSec.toFixed(1)} tok/s : null} + {loading && tokenCount > 0 ? {t("topbar.tokens", { n: tokenCount })} : null} + {!loading && tokPerSec != null ? {t("topbar.tokPerSec", { n: tokPerSec.toFixed(1) })} : null} {!loading && ttft != null ? TTFT {(ttft/1000).toFixed(1)}s : null} {!loading && lastRun?.usage ? {lastRun.usage.prompt_tokens}→{lastRun.usage.completion_tokens} : null} {lastRun?.queueWaitMs != null ? queue {Math.round(lastRun.queueWaitMs)}ms : null} - slot {cacheSlot + 1} - + {t("topbar.slot", { n: cacheSlot + 1 })} +
@@ -335,11 +342,11 @@ export default function App() { {!messages.length ? (
- COLIBRÌ ENGINE -

Ask the giant.
Keep the machine yours.

-

Connect to a local colibrì server and stream responses directly from your hardware. Nothing leaves the endpoint you choose.

+ {t("hero.title")} +

{t("hero.subtitle")}
{t("hero.tagline")}

+

{t("hero.description")}

- {["Explain how expert routing works", "Write a small C benchmark", "Compare RAM and VRAM caching"].map((item) => )} + {[t("prompts.routing"), t("prompts.benchmark"), t("prompts.caching")].map((item) => )}
) : ( @@ -347,7 +354,7 @@ export default function App() { {messages.map((item) => (
{item.role === "user" ? "Y" : }
-
{item.role === "user" ? "You" : "colibrì"}
{item.content || }
+
{item.role === "user" ? t("chat.you") : t("chat.colibri")}
{item.content || }
))}
@@ -356,10 +363,10 @@ export default function App() {
- {error &&
{error}
} + {error &&
{t(error)}
}
-