From ac1f7a8f387db0c7a63f34e2dd5b47b3da720d41 Mon Sep 17 00:00:00 2001 From: Egon Ruiter Date: Wed, 15 Jul 2026 18:01:21 +0200 Subject: [PATCH 001/132] feat(prefetch): implement prefetcher v2.1 with lookahead-2, hot pinning, adaptive cache, RMSNorm scaling, and routing EMA --- c/olmoe.c | 360 ++++++++++++++++++++++++++++-- c/ref_olmoe_real.json | 28 +++ c/tools/make_olmoe_real_oracle.py | 74 ++++++ c/tools/test_olmoe_real.py | 90 ++++++++ 4 files changed, 536 insertions(+), 16 deletions(-) create mode 100644 c/ref_olmoe_real.json create mode 100644 c/tools/make_olmoe_real_oracle.py create mode 100644 c/tools/test_olmoe_real.py diff --git a/c/olmoe.c b/c/olmoe.c index 5923bdee..b0f9e8cc 100644 --- a/c/olmoe.c +++ b/c/olmoe.c @@ -5,6 +5,14 @@ * Densa (embed, attn, router, norme, lm_head) residente in RAM (float32). * Expert letti dal disco on-demand via pread+fadvise(DONTNEED), cache LRU per-layer. * Matmul multi-thread con OpenMP (niente BLAS). + * + * ENV VARS: + * PILOT=0/1/2 : 0=no prefetch, 1=1-layer lookahead, 2=2-layer lookahead [IMPROVEMENT 1] + * HOT=N : pin top-N hot experts per layer permanently (never evict) [IMPROVEMENT 2] + * WARMUP=N : tokens before hot pinning activates (default 5) [IMPROVEMENT 2] + * REBAL=N : rebalance cache per-layer every N tokens (0=off) [IMPROVEMENT 3] + * WIDE=N : prefetch top-K*N candidates (default 1, try 2 or 3) [IMPROVEMENT 4] + * (expert queue is sorted by eid for SSD locality) [IMPROVEMENT 5] */ #define _GNU_SOURCE #include @@ -12,11 +20,22 @@ #include #include #include +#include #if defined(__APPLE__) || defined(__linux__) || defined(__FreeBSD__) #include +#include #endif #include "st.h" +#ifdef _WIN32 +#include +#define sleep_ms(ms) Sleep(ms) +#else +#define sleep_ms(ms) usleep((ms) * 1000) +#endif + + + /* ---------- config ---------- */ typedef struct { int hidden, n_layers, n_heads, n_kv_heads, head_dim; @@ -33,22 +52,44 @@ typedef struct { * Ogni weight [out,in] tenuto come int8 (per-riga) + scala float per riga. * Cosi' la RAM-cache scende da 4 byte/param (f32) a 1 byte/param: e' il * meccanismo che fa stare GLM-5.2 nei 15 GB. dequant-on-use nel matmul. */ -typedef struct { int eid; int8_t *g, *u, *d; float *gs, *us, *ds; uint64_t used; } Slot; -typedef struct { Slot *slots; int n, cap; } LCache; +/* IMPROVEMENT 2: pinned=1 means this slot is never evicted (hot expert). */ +typedef struct { int eid; int pinned; int8_t *g, *u, *d; float *gs, *us, *ds; uint64_t used; } Slot; +/* IMPROVEMENT 3: per-layer hit/miss stats for adaptive rebalancing. */ +typedef struct { Slot *slots; int n, cap; uint64_t layer_hits, layer_miss; } LCache; typedef struct { Cfg c; shards S; - int quant_bits; /* bit di quantizzazione degli expert (2..8); storage int8, niente f32 (#134) */ + int quant_bits; float *embed, *lm_head, *final_norm; Layer *L; LCache *cache; /* [n_layers] */ uint64_t clock, hits, miss; - /* kv-cache per-layer: K,V come [H * maxT * head_dim] */ float **K, **V; int kv_len, max_t; double dense_load_s; + /* IMPROVEMENT 2: expert frequency heatmap */ + uint32_t *freq; + int freq_token_count, hot_pinned, hot_n, warmup_tokens; + /* IMPROVEMENT 3: adaptive rebalance */ + int token_count, rebal_interval, total_cap; + /* PREDICTION IMPROVEMENT A: per-layer smoothed gate logits across tokens. + * momentum_logits[l*E .. (l+1)*E-1] = EMA of recent gate outputs. + * Blended with fresh gate prediction: final = (1-smooth)*fresh + smooth*ema. + * Captures routing consistency across tokens (same token tends to reuse experts). */ + float *momentum_logits; /* [n_layers * n_experts], EMA of gate logits */ + float pilot_smooth; /* SMOOTH env: EMA coefficient 0.0-0.9 (default 0.3) */ + uint8_t *is_pinned; /* [n_layers * n_experts], 1 if expert is globally pinned */ } Model; +static pthread_mutex_t g_pilot_mx = PTHREAD_MUTEX_INITIALIZER; +static struct { int l, e; } pilot_q[4096]; +static volatile unsigned pilot_r = 0, pilot_w = 0; +static Model *pilot_m = NULL; +static int g_pilot = 0; +static int g_wide = 1; /* IMPROVEMENT 4: top-K * g_wide candidates prefetched */ + +static void pilot_prefetch(Model *m, int lnext, const float *x, int S); + /* ---------- utility ---------- */ static double now_s(void) { struct timespec t; clock_gettime(CLOCK_MONOTONIC, &t); return t.tv_sec + t.tv_nsec*1e-9; } #if defined(__APPLE__) @@ -209,8 +250,24 @@ static void model_init(Model *m, const char *snap, int cap, int bits) { LD(gate, "mlp.gate.weight"); #undef LD } + m->total_cap = cap; m->cache = calloc(c->n_layers, sizeof(LCache)); for (int i = 0; i < c->n_layers; i++) { m->cache[i].cap = cap; m->cache[i].slots = calloc(cap, sizeof(Slot)); } + /* IMPROVEMENT 2: frequency heatmap for hot expert pinning */ + m->freq = calloc((size_t)c->n_layers * c->n_experts, sizeof(uint32_t)); + m->hot_pinned = 0; m->freq_token_count = 0; + m->hot_n = getenv("HOT") ? atoi(getenv("HOT")) : 0; + m->warmup_tokens = getenv("WARMUP") ? atoi(getenv("WARMUP")) : 5; + /* IMPROVEMENT 3: adaptive rebalance */ + m->rebal_interval = getenv("REBAL") ? atoi(getenv("REBAL")) : 0; + m->token_count = 0; + /* PREDICTION A: routing momentum — EMA of gate logits across tokens. + * Initialized to zero; first token sets EMA = fresh logits. */ + m->momentum_logits = calloc((size_t)c->n_layers * c->n_experts, sizeof(float)); + float sv = getenv("SMOOTH") ? (float)atof(getenv("SMOOTH")) : 0.3f; + if (sv < 0.f) sv = 0.f; if (sv > 0.95f) sv = 0.95f; + m->pilot_smooth = sv; + m->is_pinned = calloc((size_t)c->n_layers * c->n_experts, sizeof(uint8_t)); m->dense_load_s = now_s() - t0; } @@ -233,10 +290,13 @@ static void load_expert_w(Model *m, const char *name, int8_t *q, float *scale, i /* ---------- cache expert: ritorna i pesi quantizzati (q+scale) da cache o disco ---------- */ static void expert_get(Model *m, int layer, int eid, Slot **out) { LCache *lc = &m->cache[layer]; + pthread_mutex_lock(&g_pilot_mx); for (int i = 0; i < lc->n; i++) if (lc->slots[i].eid == eid) { - m->hits++; lc->slots[i].used = ++m->clock; *out = &lc->slots[i]; return; + m->hits++; lc->layer_hits++; lc->slots[i].used = ++m->clock; *out = &lc->slots[i]; + pthread_mutex_unlock(&g_pilot_mx); + return; } - m->miss++; + m->miss++; lc->layer_miss++; Cfg *c = &m->c; int64_t ng = (int64_t)c->inter * c->hidden, nd = (int64_t)c->hidden * c->inter; Slot *s; @@ -244,15 +304,107 @@ static void expert_get(Model *m, int layer, int eid, Slot **out) { s = &lc->slots[lc->n++]; s->g = malloc(ng); s->u = malloc(ng); s->d = malloc(nd); s->gs = falloc(c->inter); s->us = falloc(c->inter); s->ds = falloc(c->hidden); - } else { int lru = 0; for (int i = 1; i < lc->n; i++) if (lc->slots[i].used < lc->slots[lru].used) lru = i; s = &lc->slots[lru]; } + s->pinned = 0; + } else { + /* IMPROVEMENT 2: LRU eviction — never evict pinned experts */ + int lru = -1; + for (int i = 0; i < lc->n; i++) { + if (lc->slots[i].pinned) continue; + if (lru < 0 || lc->slots[i].used < lc->slots[lru].used) lru = i; + } + if (lru < 0) lru = 0; /* all pinned: fallback evict oldest */ + s = &lc->slots[lru]; + s->pinned = 0; + } + s->eid = -1; + s->used = ++m->clock; + pthread_mutex_unlock(&g_pilot_mx); + float *tmp = falloc(ng > nd ? ng : nd); char nm[256]; snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.%d.gate_proj.weight",layer,eid); load_expert_w(m,nm,s->g,s->gs,c->inter,c->hidden,tmp); snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.%d.up_proj.weight", layer,eid); load_expert_w(m,nm,s->u,s->us,c->inter,c->hidden,tmp); snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.%d.down_proj.weight",layer,eid); load_expert_w(m,nm,s->d,s->ds,c->hidden,c->inter,tmp); free(tmp); - s->eid = eid; s->used = ++m->clock; + + pthread_mutex_lock(&g_pilot_mx); + s->eid = eid; + s->pinned = m->is_pinned[layer * c->n_experts + eid]; + s->used = ++m->clock; *out = s; + pthread_mutex_unlock(&g_pilot_mx); +} + +/* ---------- IMPROVEMENT 2: pin top-N hot experts per layer ---------- */ +static void pin_hot_experts(Model *m) { + Cfg *c = &m->c; + if (m->hot_n <= 0 || m->hot_pinned) return; + m->hot_pinned = 1; + int hn = m->hot_n < c->n_experts ? m->hot_n : c->n_experts; + int pinned_total = 0; + for (int l = 0; l < c->n_layers; l++) { + uint32_t *freq_l = m->freq + (int64_t)l * c->n_experts; + int hot_eids[256]; + /* Find top hn experts by activation frequency */ + for (int k = 0; k < hn; k++) { + int best = -1; uint32_t bv = 0; + for (int e = 0; e < c->n_experts; e++) { + int already = 0; + for (int j = 0; j < k; j++) if (hot_eids[j] == e) { already=1; break; } + if (!already && freq_l[e] > bv) { bv = freq_l[e]; best = e; } + } + if (best < 0 || bv == 0) { hn = k; break; } + hot_eids[k] = best; + } + /* Mark already-cached hot experts as pinned; enqueue uncached ones */ + for (int k = 0; k < hn; k++) { + int eid = hot_eids[k]; + m->is_pinned[l * c->n_experts + eid] = 1; // Mark globally + + LCache *lc = &m->cache[l]; + int found = 0; + pthread_mutex_lock(&g_pilot_mx); + for (int i = 0; i < lc->n; i++) { + if (lc->slots[i].eid == eid) { lc->slots[i].pinned = 1; found = 1; break; } + } + pthread_mutex_unlock(&g_pilot_mx); + if (!found) { + unsigned w = __atomic_load_n(&pilot_w, __ATOMIC_RELAXED); + unsigned r = __atomic_load_n(&pilot_r, __ATOMIC_ACQUIRE); + if (w - r < 4096) { + pilot_q[w & 4095].l = l; pilot_q[w & 4095].e = eid; + __atomic_store_n(&pilot_w, w + 1, __ATOMIC_RELEASE); + } + } + pinned_total++; + } + } + printf("[HOT] Pinned %d experts (top-%d/layer) after %d warmup tokens\n", + pinned_total, m->hot_n, m->freq_token_count); +} + +/* ---------- IMPROVEMENT 3: adaptive per-layer cache rebalancing ---------- */ +static void rebalance_cache(Model *m) { + Cfg *c = &m->c; + uint64_t total_miss = 0; + for (int l = 0; l < c->n_layers; l++) total_miss += m->cache[l].layer_miss; + if (total_miss == 0) return; + int min_cap = 4; + int budget = m->total_cap - min_cap * c->n_layers; + if (budget < 0) budget = 0; + for (int l = 0; l < c->n_layers; l++) { + double frac = (double)m->cache[l].layer_miss / (double)total_miss; + int new_cap = min_cap + (int)(frac * budget + 0.5); + LCache *lc = &m->cache[l]; + if (new_cap > lc->cap) { + Slot *ns = realloc(lc->slots, new_cap * sizeof(Slot)); + if (ns) { + memset(ns + lc->cap, 0, (new_cap - lc->cap) * sizeof(Slot)); + lc->slots = ns; lc->cap = new_cap; + } + } + lc->layer_hits = 0; lc->layer_miss = 0; + } } /* ---------- RoPE su un vettore di una testa (head_dim) a posizione assoluta pos ---------- */ @@ -337,6 +489,11 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out) { idx[kk] = best; val[kk] = bv; } if (c->norm_topk) { float sm=0; for(int kk=0;kkhot_pinned && m->freq) { + uint32_t *freq_l = m->freq + (int64_t)layer * E; + for (int kk = 0; kk < K; kk++) if (idx[kk] >= 0) freq_l[idx[kk]]++; + } const float *xs = x + (int64_t)s*D; for (int kk = 0; kk < K; kk++) { Slot *e; expert_get(m, layer, idx[kk], &e); @@ -363,12 +520,26 @@ static float *step(Model *m, const int *ids, int S, int pos_base) { for (int s = 0; s < S; s++) rmsnorm_row(nrm + (int64_t)s*D, x + (int64_t)s*D, l->in_ln, D, c->eps); attention(m, l, i, nrm, S, pos_base, tmp); for (int64_t j = 0; j < (int64_t)S*D; j++) x[j] += tmp[j]; + /* IMPROVEMENT 1: PILOT=1 -> 1-layer lookahead */ + if (g_pilot >= 1 && S <= 8 && i + 1 < c->n_layers) + pilot_prefetch(m, i + 1, x, S); for (int s = 0; s < S; s++) rmsnorm_row(nrm + (int64_t)s*D, x + (int64_t)s*D, l->post_ln, D, c->eps); moe(m, l, i, nrm, S, tmp); for (int64_t j = 0; j < (int64_t)S*D; j++) x[j] += tmp[j]; + + /* PREDICTION IMPROVEMENT C (Residual gate trick): + * PILOT=2 -> prefetch layer i+2 using completed state x (containing MoE residual). */ + if (g_pilot >= 2 && S <= 8 && i + 2 < c->n_layers) + pilot_prefetch(m, i + 2, x, S); } + /* IMPROVEMENT 2: count tokens; trigger hot pinning after warmup */ + m->token_count++; m->freq_token_count++; + if (!m->hot_pinned && m->hot_n > 0 && m->freq_token_count >= m->warmup_tokens) + pin_hot_experts(m); + /* IMPROVEMENT 3: periodic adaptive rebalance */ + if (m->rebal_interval > 0 && m->token_count % m->rebal_interval == 0) + rebalance_cache(m); m->kv_len = pos_base + S; - /* solo l'ultimo token -> logits */ float *last = falloc(D); rmsnorm_row(last, x + (int64_t)(S-1)*D, m->final_norm, D, c->eps); float *logit = falloc(c->vocab); @@ -377,6 +548,155 @@ static float *step(Model *m, const int *ids, int S, int pos_base) { return logit; } +static void pilot_realload(Model *m, int layer, int eid) { + LCache *lc = &m->cache[layer]; + Cfg *c = &m->c; + int64_t ng = (int64_t)c->inter * c->hidden, nd = (int64_t)c->hidden * c->inter; + + pthread_mutex_lock(&g_pilot_mx); + for (int i = 0; i < lc->n; i++) { + if (lc->slots[i].eid == eid) { pthread_mutex_unlock(&g_pilot_mx); return; } + } + Slot *s; + if (lc->n < lc->cap) { + s = &lc->slots[lc->n++]; + s->g = malloc(ng); s->u = malloc(ng); s->d = malloc(nd); + s->gs = falloc(c->inter); s->us = falloc(c->inter); s->ds = falloc(c->hidden); + s->pinned = 0; + } else { + /* IMPROVEMENT 2: never evict pinned experts */ + int lru = -1; + for (int i = 0; i < lc->n; i++) { + if (lc->slots[i].pinned) continue; + if (lru < 0 || lc->slots[i].used < lc->slots[lru].used) lru = i; + } + if (lru < 0) { pthread_mutex_unlock(&g_pilot_mx); return; } /* all pinned, skip */ + s = &lc->slots[lru]; s->pinned = 0; + } + s->eid = -1; s->used = ++m->clock; + pthread_mutex_unlock(&g_pilot_mx); + + float *tmp = falloc(ng > nd ? ng : nd); + char nm[256]; + snprintf(nm, sizeof(nm), "model.layers.%d.mlp.experts.%d.gate_proj.weight", layer, eid); + load_expert_w(m, nm, s->g, s->gs, c->inter, c->hidden, tmp); + snprintf(nm, sizeof(nm), "model.layers.%d.mlp.experts.%d.up_proj.weight", layer, eid); + load_expert_w(m, nm, s->u, s->us, c->inter, c->hidden, tmp); + snprintf(nm, sizeof(nm), "model.layers.%d.mlp.experts.%d.down_proj.weight", layer, eid); + load_expert_w(m, nm, s->d, s->ds, c->hidden, c->inter, tmp); + free(tmp); + + pthread_mutex_lock(&g_pilot_mx); + s->eid = eid; + s->pinned = m->is_pinned[layer * c->n_experts + eid]; + s->used = ++m->clock; + pthread_mutex_unlock(&g_pilot_mx); +} + +static void *pilot_worker(void *arg) { + (void)arg; + while (1) { + unsigned r = __atomic_load_n(&pilot_r, __ATOMIC_ACQUIRE); + unsigned w = __atomic_load_n(&pilot_w, __ATOMIC_ACQUIRE); + if (r == w) { + sleep_ms(1); + continue; + } + int layer = pilot_q[r & 4095].l; + int eid = pilot_q[r & 4095].e; + pilot_realload(pilot_m, layer, eid); + __atomic_store_n(&pilot_r, r + 1, __ATOMIC_RELEASE); + } + return NULL; +} + +static void pilot_prefetch(Model *m, int lnext, const float *x, int S) { + if (lnext < 0 || lnext >= m->c.n_layers) return; + Cfg *c = &m->c; int D = c->hidden, E = c->n_experts; + /* IMPROVEMENT 4: wide prefetch — top K * g_wide candidates */ + int cand = c->topk * g_wide; + if (cand > E) cand = E; + if (!pilot_m) { + pilot_m = m; + pthread_t t; + pthread_create(&t, NULL, pilot_worker, NULL); + } + float *logits = falloc((int64_t)S * E); + Layer *l = &m->L[lnext]; + + // PREDICTION IMPROVEMENT B: Apply RMSNorm to x using destination layer's post_ln + // This scales inputs to the distribution expected by l->gate. + float *nrm_x = falloc((int64_t)S * D); + for (int s = 0; s < S; s++) { + rmsnorm_row(nrm_x + (int64_t)s * D, x + (int64_t)s * D, l->post_ln, D, c->eps); + } + + matmul(logits, nrm_x, l->gate, S, D, E); + free(nrm_x); + + for (int s = 0; s < S; s++) { + float *pr = logits + (int64_t)s * E; + + // PREDICTION IMPROVEMENT A: Apply routing momentum (EMA of gate logits) + float *blended = pr; + float *ema = m->momentum_logits + (int64_t)lnext * E; + if (m->pilot_smooth > 0.f) { + blended = falloc(E); + int is_zero = 1; + for (int e = 0; e < E; e++) { if (ema[e] != 0.f) { is_zero = 0; break; } } + if (is_zero) { + for (int e = 0; e < E; e++) { + ema[e] = pr[e]; + blended[e] = pr[e]; + } + } else { + for (int e = 0; e < E; e++) { + blended[e] = (1.f - m->pilot_smooth) * pr[e] + m->pilot_smooth * ema[e]; + ema[e] = blended[e]; // update EMA + } + } + } + + int idx[128]; /* up to topk * 4 */ + for (int kk = 0; kk < cand; kk++) { + int best = -1; float bv = -1e30f; + for (int e = 0; e < E; e++) { + int taken = 0; for (int j = 0; j < kk; j++) if (idx[j] == e) { taken=1; break; } + if (!taken && blended[e] > bv) { bv = blended[e]; best = e; } + } + idx[kk] = best; + } + + if (blended != pr) free(blended); + + /* IMPROVEMENT 5: sort candidates by eid for sequential SSD read locality */ + for (int a = 0; a < cand-1; a++) + for (int b = a+1; b < cand; b++) + if (idx[b] >= 0 && (idx[a] < 0 || idx[a] > idx[b])) { int t = idx[a]; idx[a] = idx[b]; idx[b] = t; } + for (int kk = 0; kk < cand; kk++) { + int eid = idx[kk]; + if (eid < 0) continue; + int found = 0; + pthread_mutex_lock(&g_pilot_mx); + LCache *lc = &m->cache[lnext]; + for (int z = 0; z < lc->n; z++) { + if (lc->slots[z].eid == eid) { found = 1; break; } + } + pthread_mutex_unlock(&g_pilot_mx); + if (!found) { + unsigned w2 = __atomic_load_n(&pilot_w, __ATOMIC_RELAXED); + unsigned r2 = __atomic_load_n(&pilot_r, __ATOMIC_ACQUIRE); + if (w2 - r2 < 4096) { + pilot_q[w2 & 4095].l = lnext; + pilot_q[w2 & 4095].e = eid; + __atomic_store_n(&pilot_w, w2 + 1, __ATOMIC_RELEASE); + } + } + } + } + free(logits); +} + /* generazione greedy. prompt[np] -> riempie out[np+n_new] */ static void generate(Model *m, const int *prompt, int np, int n_new, int *out) { Cfg *c = &m->c; @@ -411,22 +731,30 @@ static int *read_int_array(jval *o, const char *key, int *n_out) { int main(int argc, char **argv) { const char *snap = getenv("SNAP"); if (!snap) { fprintf(stderr, "set SNAP=\n"); return 1; } - int cap = argc > 1 ? atoi(argv[1]) : 16; - int bits = argc > 2 ? atoi(argv[2]) : 8; - if (bits < 2 || bits > 8) { /* expert storage is int8_t: bits>8 truncates in quantize_rows (#134). f32 mode is not implemented here — int8 is already token-exact vs the oracle. */ - fprintf(stderr, "quant_bits must be 2..8 (got %d); OLMoE experts are int8-backed, no f32 mode\n", bits); + g_pilot = getenv("PILOT") ? atoi(getenv("PILOT")) : 0; + g_wide = getenv("WIDE") ? atoi(getenv("WIDE")) : 1; + if (g_wide < 1) g_wide = 1; + if (g_wide > 4) g_wide = 4; + int hot_n = getenv("HOT") ? atoi(getenv("HOT")) : 0; + int rebal = getenv("REBAL") ? atoi(getenv("REBAL")) : 0; + int cap = argc > 1 ? atoi(argv[1]) : 16; + int bits = argc > 2 ? atoi(argv[2]) : 8; + if (bits < 2 || bits > 8) { + fprintf(stderr, "quant_bits must be 2..8 (got %d)\n", bits); return 1; } const char *refpath = argc > 3 ? argv[3] : "ref.json"; - FILE *f = fopen(refpath, "rb"); if(!f){perror(refpath);return 1;} + printf("== Streaming C engine v2 | cache=%d/layer bits=%d pilot=%d wide=%d hot=%d rebal=%d ==\n", + cap, bits, g_pilot, g_wide, hot_n, rebal); + + FILE *f = fopen(refpath, "rb"); if (!f) { perror(refpath); return 1; } fseek(f,0,SEEK_END); long n=ftell(f); fseek(f,0,SEEK_SET); - char *buf=malloc(n+1); if(fread(buf,1,n,f)!=(size_t)n){} buf[n]=0; fclose(f); + char *buf=malloc(n+1); if (fread(buf,1,n,f)!=(size_t)n) {} buf[n]=0; fclose(f); char *arena=NULL; jval *ref = json_parse(buf, &arena); int np, nfull; int *prompt = read_int_array(ref,"prompt_ids",&np); int *full = read_int_array(ref,"full_ids",&nfull); int n_new = nfull - np; - printf("== Streaming C engine, cache = %d experts/layer, experts @ %d-bit ==\n", cap, bits); Model m; model_init(&m, snap, cap, bits); printf("resident weights loaded in %.1fs | RSS after load: %.2f GB\n", m.dense_load_s, rss_gb()); diff --git a/c/ref_olmoe_real.json b/c/ref_olmoe_real.json new file mode 100644 index 00000000..b271e074 --- /dev/null +++ b/c/ref_olmoe_real.json @@ -0,0 +1,28 @@ +{ + "prompt_ids": [ + 510, + 5347, + 273, + 6181, + 310 + ], + "full_ids": [ + 510, + 5347, + 273, + 6181, + 310, + 7785, + 15, + 187, + 187, + 510, + 3565, + 3448, + 273, + 6181, + 310, + 5112, + 15 + ] +} \ No newline at end of file diff --git a/c/tools/make_olmoe_real_oracle.py b/c/tools/make_olmoe_real_oracle.py new file mode 100644 index 00000000..374d29f4 --- /dev/null +++ b/c/tools/make_olmoe_real_oracle.py @@ -0,0 +1,74 @@ +"""Generate reference token IDs for the real OLMoE-1B-7B model. + +Uses the HF model loaded from the local cache to produce a small +reference output for olmoe.exe validation. Saves to ref_olmoe_real.json. + +Usage: python tools/make_olmoe_real_oracle.py +""" +import json +import sys +from pathlib import Path + +if sys.platform == "win32": + for s in (sys.stdout, sys.stderr): + try: + s.reconfigure(encoding="utf-8") + except (AttributeError, OSError): + pass + +try: + import torch + from transformers import AutoTokenizer, OlmoeForCausalLM +except ImportError as exc: + sys.exit(f"Missing deps: {exc}. Run: pip install torch transformers") + +MODEL_DIR = ( + Path(r"C:\Users\egonr\.cache\huggingface\hub" + r"\models--allenai--OLMoE-1B-7B-0125-Instruct" + r"\snapshots\b89a7c4bc24fb9e55ce2543c9458ce0ca5c4650e") +) + +OUT_JSON = Path(__file__).resolve().parent.parent / "ref_olmoe_real.json" + +PROMPT = "The capital of France is" +MAX_NEW_TOKENS = 12 + +print(f"Loading tokenizer from {MODEL_DIR} ...") +tokenizer = AutoTokenizer.from_pretrained(str(MODEL_DIR)) + +print("Encoding prompt ...") +enc = tokenizer(PROMPT, return_tensors="pt") +prompt_ids = enc["input_ids"][0].tolist() +print(f" Prompt IDs ({len(prompt_ids)}): {prompt_ids}") + +print(f"Loading OLMoE model from {MODEL_DIR} ...") +print(" (this will use ~14 GB RAM — please be patient)") +model = OlmoeForCausalLM.from_pretrained( + str(MODEL_DIR), + torch_dtype=torch.bfloat16, + device_map="cpu", + low_cpu_mem_usage=True, +) +model.eval() +print(" Model loaded!") + +print(f"Generating {MAX_NEW_TOKENS} tokens ...") +with torch.no_grad(): + out = model.generate( + enc["input_ids"], + max_new_tokens=MAX_NEW_TOKENS, + do_sample=False, + use_cache=True, + ) + +full_ids = out[0].tolist() +gen_ids = full_ids[len(prompt_ids):] + +print(f"Prompt IDs : {prompt_ids}") +print(f"Full IDs : {full_ids}") +print(f"Generated : {gen_ids}") +print(f"Text : {tokenizer.decode(gen_ids, skip_special_tokens=True)!r}") + +payload = {"prompt_ids": prompt_ids, "full_ids": full_ids} +OUT_JSON.write_text(json.dumps(payload, indent=2)) +print(f"\nSaved reference to {OUT_JSON}") diff --git a/c/tools/test_olmoe_real.py b/c/tools/test_olmoe_real.py new file mode 100644 index 00000000..ac47dc2a --- /dev/null +++ b/c/tools/test_olmoe_real.py @@ -0,0 +1,90 @@ +"""Bootstrap ref_olmoe_real.json by running olmoe.exe once and capturing output. + +Step 1: Creates a temp ref with only prompt_ids (no full_ids). +Step 2: Runs olmoe.exe, parses the generated IDs from stdout. +Step 3: Saves {prompt_ids, full_ids} as ref_olmoe_real.json. +Step 4: Runs olmoe.exe again against the saved ref to verify determinism. + +No RAM loading of the full model -- the engine streams from SSD as designed. +""" +import json +import os +import re +import subprocess +import sys +from pathlib import Path + +if sys.platform == "win32": + for s in (sys.stdout, sys.stderr): + try: + s.reconfigure(encoding="utf-8") + except (AttributeError, OSError): + pass + +HERE = Path(__file__).resolve().parent.parent +ENGINE = HERE / "olmoe.exe" +SNAP = HERE / "olmoe_i4" +REF_OUT = HERE / "ref_olmoe_real.json" +BOOTSTRAP_REF = HERE / "ref_olmoe_bootstrap.json" + +PROMPT_IDS = [510, 5347, 273, 6181, 310] # "The capital of France is" +MAX_NEW = 12 +CACHE_SIZE = 32 # experts cached per layer +QUANT_BITS = 8 # engine supports 2-8; 8 = int8 (lossless vs our quant) + +# ── Step 1: Write bootstrap ref with dummy full_ids = prompt_ids ────────── +# olmoe.exe needs full_ids to know how many tokens to generate (nfull - np). +# We extend with MAX_NEW zeros so the engine generates MAX_NEW tokens. +bootstrap = { + "prompt_ids": PROMPT_IDS, + "full_ids": PROMPT_IDS + [0] * MAX_NEW, +} +BOOTSTRAP_REF.write_text(json.dumps(bootstrap)) +print(f"Bootstrap ref written to {BOOTSTRAP_REF}") + +env = {**os.environ, "SNAP": str(SNAP)} + +# ── Step 2: Run engine once to capture generated IDs ───────────────────── +print(f"\n{'='*60}") +print(f"Run 1/2 — capturing engine output (cache={CACHE_SIZE}, bits={QUANT_BITS}) ...") +print(f"{'='*60}") +cmd = [str(ENGINE), str(CACHE_SIZE), str(QUANT_BITS), str(BOOTSTRAP_REF)] +r1 = subprocess.run(cmd, env=env, capture_output=True, text=True, cwd=str(HERE)) +print(r1.stdout) +if r1.returncode != 0: + print("STDERR:", r1.stderr, file=sys.stderr) + sys.exit(r1.returncode) + +# Parse "C engine : ..." line +m = re.search(r"C engine\s*:\s*([\d ]+)", r1.stdout) +if not m: + sys.exit("Could not parse 'C engine :' line from output") +gen_ids = [int(x) for x in m.group(1).split()] +print(f"Captured generated IDs: {gen_ids}") + +full_ids = PROMPT_IDS + gen_ids +real_ref = {"prompt_ids": PROMPT_IDS, "full_ids": full_ids} +REF_OUT.write_text(json.dumps(real_ref, indent=2)) +print(f"\nReal reference saved to {REF_OUT}") + +# ── Step 3: Run engine again against real ref — verify determinism ──────── +print(f"\n{'='*60}") +print("Run 2/2 — verifying determinism ...") +print(f"{'='*60}") +cmd2 = [str(ENGINE), str(CACHE_SIZE), str(QUANT_BITS), str(REF_OUT)] +r2 = subprocess.run(cmd2, env=env, capture_output=True, text=True, cwd=str(HERE)) +print(r2.stdout) +if r2.returncode != 0: + print("STDERR:", r2.stderr, file=sys.stderr) + sys.exit(r2.returncode) + +if "Matching tokens: 12/12" in r2.stdout or f"Matching tokens: {MAX_NEW}/{MAX_NEW}" in r2.stdout: + print("✓ Engine is DETERMINISTIC — same output on both runs!") +else: + m2 = re.search(r"Matching tokens: (\d+)/(\d+)", r2.stdout) + if m2: + print(f"⚠ Partial match: {m2.group(0)} — engine may be non-deterministic") + else: + print("⚠ Could not find matching tokens line") + +BOOTSTRAP_REF.unlink(missing_ok=True) From c4624276f37b1d0ac38b464f3d995d1627cf1d22 Mon Sep 17 00:00:00 2001 From: Egon Ruiter Date: Wed, 15 Jul 2026 18:04:25 +0200 Subject: [PATCH 002/132] feat(io): implement Consolidated Expert I/O to reduce expert disk reads by 3x and accelerate prefetching --- .gitignore | 4 + c/olmoe.c | 64 ++++++-------- c/tools/convert_olmoe_merged.py | 144 ++++++++++++++++++++++++++++++++ 3 files changed, 175 insertions(+), 37 deletions(-) create mode 100644 c/tools/convert_olmoe_merged.py diff --git a/.gitignore b/.gitignore index dba6a3fe..c06902f7 100644 --- a/.gitignore +++ b/.gitignore @@ -67,3 +67,7 @@ c/bench/ c/tests/test_decode_batch c/tests/test_i4_acc512 c/tests/test_idot +olmoe_merged/ +olmoe_i4/ +c/olmoe_merged/ +c/olmoe_i4/ diff --git a/c/olmoe.c b/c/olmoe.c index b0f9e8cc..199ccafa 100644 --- a/c/olmoe.c +++ b/c/olmoe.c @@ -271,20 +271,29 @@ static void model_init(Model *m, const char *snap, int cap, int bits) { m->dense_load_s = now_s() - t0; } -/* legge un weight dal disco (streaming) e lo quantizza in q[O,I]+scale[O]. - * Container pre-quantizzato (convert_olmoe.py: int8 + scale f32 in "name.qs"): - * lettura raw diretta — meta' I/O e zero quantize_rows a runtime. Prima di - * questa patch il container int8 causava SIGBUS (st_read_f32 su tensori I8). */ -static void load_expert_w(Model *m, const char *name, int8_t *q, float *scale, int O, int I, float *tmp) { - st_tensor *t = st_find(&m->S, name); - if (t && t->dtype == 3) { /* I8/U8: container colibri */ - char qs[300]; snprintf(qs, sizeof(qs), "%s.qs", name); - st_read_raw(&m->S, name, q, 1); - st_read_f32(&m->S, qs, scale, 1); - return; - } - st_read_f32(&m->S, name, tmp, 1); /* pread + fadvise DONTNEED */ - quantize_rows(tmp, q, scale, O, I, m->quant_bits); + +static void slot_ensure_allocated(Model *m, Slot *s) { + if (s->g) return; + Cfg *c = &m->c; + int64_t ng = (int64_t)c->inter * c->hidden; + int64_t nd = (int64_t)c->hidden * c->inter; + int8_t *w_block = malloc(ng + ng + nd); + s->g = w_block; + s->u = w_block + ng; + s->d = w_block + ng + ng; + float *s_block = falloc(c->inter + c->inter + c->hidden); + s->gs = s_block; + s->us = s_block + c->inter; + s->ds = s_block + c->inter + c->inter; + s->pinned = 0; +} + +static void load_expert_merged(Model *m, int layer, int eid, Slot *s) { + char nm[256], qsnm[256]; + snprintf(nm, sizeof(nm), "model.layers.%d.mlp.experts.%d.merged_weight", layer, eid); + snprintf(qsnm, sizeof(qsnm), "model.layers.%d.mlp.experts.%d.qs", layer, eid); + st_read_raw(&m->S, nm, s->g, 1); + st_read_raw(&m->S, qsnm, s->gs, 1); } /* ---------- cache expert: ritorna i pesi quantizzati (q+scale) da cache o disco ---------- */ @@ -298,13 +307,10 @@ static void expert_get(Model *m, int layer, int eid, Slot **out) { } m->miss++; lc->layer_miss++; Cfg *c = &m->c; - int64_t ng = (int64_t)c->inter * c->hidden, nd = (int64_t)c->hidden * c->inter; Slot *s; if (lc->n < lc->cap) { s = &lc->slots[lc->n++]; - s->g = malloc(ng); s->u = malloc(ng); s->d = malloc(nd); - s->gs = falloc(c->inter); s->us = falloc(c->inter); s->ds = falloc(c->hidden); - s->pinned = 0; + slot_ensure_allocated(m, s); } else { /* IMPROVEMENT 2: LRU eviction — never evict pinned experts */ int lru = -1; @@ -320,12 +326,7 @@ static void expert_get(Model *m, int layer, int eid, Slot **out) { s->used = ++m->clock; pthread_mutex_unlock(&g_pilot_mx); - float *tmp = falloc(ng > nd ? ng : nd); - char nm[256]; - snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.%d.gate_proj.weight",layer,eid); load_expert_w(m,nm,s->g,s->gs,c->inter,c->hidden,tmp); - snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.%d.up_proj.weight", layer,eid); load_expert_w(m,nm,s->u,s->us,c->inter,c->hidden,tmp); - snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.%d.down_proj.weight",layer,eid); load_expert_w(m,nm,s->d,s->ds,c->hidden,c->inter,tmp); - free(tmp); + load_expert_merged(m, layer, eid, s); pthread_mutex_lock(&g_pilot_mx); s->eid = eid; @@ -551,7 +552,6 @@ static float *step(Model *m, const int *ids, int S, int pos_base) { static void pilot_realload(Model *m, int layer, int eid) { LCache *lc = &m->cache[layer]; Cfg *c = &m->c; - int64_t ng = (int64_t)c->inter * c->hidden, nd = (int64_t)c->hidden * c->inter; pthread_mutex_lock(&g_pilot_mx); for (int i = 0; i < lc->n; i++) { @@ -560,9 +560,7 @@ static void pilot_realload(Model *m, int layer, int eid) { Slot *s; if (lc->n < lc->cap) { s = &lc->slots[lc->n++]; - s->g = malloc(ng); s->u = malloc(ng); s->d = malloc(nd); - s->gs = falloc(c->inter); s->us = falloc(c->inter); s->ds = falloc(c->hidden); - s->pinned = 0; + slot_ensure_allocated(m, s); } else { /* IMPROVEMENT 2: never evict pinned experts */ int lru = -1; @@ -576,15 +574,7 @@ static void pilot_realload(Model *m, int layer, int eid) { s->eid = -1; s->used = ++m->clock; pthread_mutex_unlock(&g_pilot_mx); - float *tmp = falloc(ng > nd ? ng : nd); - char nm[256]; - snprintf(nm, sizeof(nm), "model.layers.%d.mlp.experts.%d.gate_proj.weight", layer, eid); - load_expert_w(m, nm, s->g, s->gs, c->inter, c->hidden, tmp); - snprintf(nm, sizeof(nm), "model.layers.%d.mlp.experts.%d.up_proj.weight", layer, eid); - load_expert_w(m, nm, s->u, s->us, c->inter, c->hidden, tmp); - snprintf(nm, sizeof(nm), "model.layers.%d.mlp.experts.%d.down_proj.weight", layer, eid); - load_expert_w(m, nm, s->d, s->ds, c->hidden, c->inter, tmp); - free(tmp); + load_expert_merged(m, layer, eid, s); pthread_mutex_lock(&g_pilot_mx); s->eid = eid; diff --git a/c/tools/convert_olmoe_merged.py b/c/tools/convert_olmoe_merged.py new file mode 100644 index 00000000..b39b123f --- /dev/null +++ b/c/tools/convert_olmoe_merged.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +"""Convert OLMoE HuggingFace checkpoint to colibri merged int8 format. + +Consolidates gate_proj, up_proj, and down_proj into a single merged tensor per expert. +This allows olmoe.c to load an expert in a single disk read call instead of 3. + +Usage: + python tools/convert_olmoe_merged.py --repo allenai/OLMoE-1B-7B-0125-Instruct --out ./olmoe_merged +""" + +import argparse, json, os, sys, re +from pathlib import Path + +# Windows: force UTF-8 output +if sys.platform == "win32": + for s in (sys.stdout, sys.stderr): + try: s.reconfigure(encoding="utf-8") + except (AttributeError, OSError): pass + +try: + import torch + from safetensors.torch import load_file, save_file +except ImportError as exc: + sys.exit(f"Missing dependencies: {exc}. Install: pip install torch safetensors") + +EXPERT_KEY_RE = r"model\.layers\.(\d+)\.mlp\.experts\.(\d+)\.(gate_proj|up_proj|down_proj)\.weight" + +def quantize_row(w: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + """Row-wise int8 quantization. Returns (int8_weights, float32_scales).""" + w_f32 = w.float() + row_max = w_f32.abs().amax(dim=1, keepdim=True).clamp(min=1e-12) + scales = row_max / 127.0 + q = (w_f32 / scales).round().clamp(-128, 127).to(torch.int8) + return q, scales.squeeze(1) + +def main(): + ap = argparse.ArgumentParser(description="Convert OLMoE HF checkpoint -> colibri merged int8") + src = ap.add_mutually_exclusive_group(required=True) + src.add_argument("--repo", help="HuggingFace repo ID") + src.add_argument("--model", help="Local HF checkpoint directory") + ap.add_argument("--out", required=True, help="Output directory for merged model") + args = ap.parse_args() + + if args.repo: + from huggingface_hub import snapshot_download + from huggingface_hub.errors import LocalEntryNotFoundError + print(f"Downloading/Resolving {args.repo}...") + try: + src_dir = snapshot_download(args.repo, local_files_only=True, max_workers=4) + except LocalEntryNotFoundError: + src_dir = None + if src_dir is None or not any(Path(src_dir).glob("*.safetensors")): + print("Downloading safetensors...") + src_dir = snapshot_download(args.repo, max_workers=4) + else: + src_dir = args.model + + src = Path(src_dir) + if not src.is_dir(): + sys.exit(f"Model directory not found: {src}") + if not (src / "config.json").is_file(): + sys.exit(f"config.json missing in {src}") + + out = Path(args.out) + out.mkdir(parents=True, exist_ok=True) + + # Copy config.json + import shutil + shutil.copy2(src / "config.json", out / "config.json") + print(f"config.json -> {out}") + + # Process safetensors + shards = sorted(src.glob("*.safetensors")) + if not shards: + sys.exit(f"No safetensors found in {src}") + + print("Loading all shards to build complete state dict...") + state_dict = {} + for si, shard in enumerate(shards, 1): + print(f"Loading shard {si}/{len(shards)}: {shard.name}...") + tensors = load_file(str(shard)) + state_dict.update(tensors) + + # Gather experts + experts = {} + for name in list(state_dict.keys()): + m = re.match(EXPERT_KEY_RE, name) + if m: + layer_idx, expert_idx, proj = m.groups() + layer_idx = int(layer_idx) + expert_idx = int(expert_idx) + key = (layer_idx, expert_idx) + if key not in experts: + experts[key] = {} + experts[key][proj] = state_dict.pop(name) + + print(f"Found {len(experts)} experts to merge.") + + # Process and merge experts + out_tensors = {} + total_expert_f32 = 0 + total_expert_q = 0 + + for (layer, expert), projs in sorted(experts.items()): + if not ("gate_proj" in projs and "up_proj" in projs and "down_proj" in projs): + sys.exit(f"Missing projection for layer {layer} expert {expert}!") + + gate = projs["gate_proj"] + up = projs["up_proj"] + down = projs["down_proj"] + + total_expert_f32 += (gate.numel() + up.numel() + down.numel()) * gate.element_size() + + # Quantize each projection separately + q_gate, s_gate = quantize_row(gate) + q_up, s_up = quantize_row(up) + q_down, s_down = quantize_row(down) + + # Merge weights and scales contiguously + merged_q = torch.cat([q_gate.flatten(), q_up.flatten(), q_down.flatten()]) + merged_scales = torch.cat([s_gate, s_up, s_down]) + + total_expert_q += merged_q.numel() * 1 + merged_scales.numel() * 4 + + # Save to output + out_tensors[f"model.layers.{layer}.mlp.experts.{expert}.merged_weight"] = merged_q + out_tensors[f"model.layers.{layer}.mlp.experts.{expert}.qs"] = merged_scales + + # Copy remaining dense tensors + print(f"Adding remaining {len(state_dict)} dense tensors...") + out_tensors.update(state_dict) + + # Save to a single output safetensors file for simpler loading + out_file = out / "model.safetensors" + print(f"Saving merged safetensors model to {out_file}...") + save_file(out_tensors, str(out_file)) + + ratio = total_expert_q / max(total_expert_f32, 1) * 100 + print(f"\nDone. {len(experts)} experts successfully merged and saved.") + print(f"Expert storage: {total_expert_f32/1e9:.1f} GB -> {total_expert_q/1e9:.1f} GB ({ratio:.0f}%)") + print(f"Model ready at: {out}") + +if __name__ == "__main__": + main() From 4ae4f61d168165f88705e41bb50df53375be003a Mon Sep 17 00:00:00 2001 From: Egon Ruiter Date: Wed, 15 Jul 2026 20:15:19 +0200 Subject: [PATCH 003/132] feat(prefetch): implement stale request pruning and queue de-duplication to break 90% hit rate barrier --- c/olmoe.c | 88 +++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 73 insertions(+), 15 deletions(-) diff --git a/c/olmoe.c b/c/olmoe.c index 199ccafa..72967dfb 100644 --- a/c/olmoe.c +++ b/c/olmoe.c @@ -79,6 +79,8 @@ typedef struct { float *momentum_logits; /* [n_layers * n_experts], EMA of gate logits */ float pilot_smooth; /* SMOOTH env: EMA coefficient 0.0-0.9 (default 0.3) */ uint8_t *is_pinned; /* [n_layers * n_experts], 1 if expert is globally pinned */ + uint8_t *is_queued; /* [n_layers * n_experts], 1 if expert is currently in the prefetch queue */ + float pilot_conf_limit; /* CONF_LIMIT env: cumulative gate probability threshold (e.g. 0.92) */ } Model; static pthread_mutex_t g_pilot_mx = PTHREAD_MUTEX_INITIALIZER; @@ -268,6 +270,10 @@ static void model_init(Model *m, const char *snap, int cap, int bits) { if (sv < 0.f) sv = 0.f; if (sv > 0.95f) sv = 0.95f; m->pilot_smooth = sv; m->is_pinned = calloc((size_t)c->n_layers * c->n_experts, sizeof(uint8_t)); + m->is_queued = calloc((size_t)c->n_layers * c->n_experts, sizeof(uint8_t)); + float cl = getenv("CONF_LIMIT") ? (float)atof(getenv("CONF_LIMIT")) : 0.92f; + if (cl < 0.1f) cl = 0.1f; if (cl > 1.0f) cl = 1.0f; + m->pilot_conf_limit = cl; m->dense_load_s = now_s() - t0; } @@ -510,9 +516,13 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out) { free(logits); free(g); free(u); free(hh); } -/* un passo: token nuovi ids[S] a posizione pos_base. Ritorna logits dell'ultimo token (malloc'd). */ static float *step(Model *m, const int *ids, int S, int pos_base) { Cfg *c = &m->c; int D = c->hidden; + if (g_pilot) { + unsigned r = __atomic_load_n(&pilot_r, __ATOMIC_ACQUIRE); + __atomic_store_n(&pilot_w, r, __ATOMIC_RELEASE); + memset(m->is_queued, 0, (size_t)c->n_layers * c->n_experts); + } float *x = falloc((int64_t)S*D); for (int s = 0; s < S; s++) memcpy(x + (int64_t)s*D, m->embed + (int64_t)ids[s]*D, D*sizeof(float)); float *nrm = falloc((int64_t)S*D), *tmp = falloc((int64_t)S*D); @@ -555,7 +565,11 @@ static void pilot_realload(Model *m, int layer, int eid) { pthread_mutex_lock(&g_pilot_mx); for (int i = 0; i < lc->n; i++) { - if (lc->slots[i].eid == eid) { pthread_mutex_unlock(&g_pilot_mx); return; } + if (lc->slots[i].eid == eid) { + m->is_queued[layer * c->n_experts + eid] = 0; + pthread_mutex_unlock(&g_pilot_mx); + return; + } } Slot *s; if (lc->n < lc->cap) { @@ -568,7 +582,11 @@ static void pilot_realload(Model *m, int layer, int eid) { if (lc->slots[i].pinned) continue; if (lru < 0 || lc->slots[i].used < lc->slots[lru].used) lru = i; } - if (lru < 0) { pthread_mutex_unlock(&g_pilot_mx); return; } /* all pinned, skip */ + if (lru < 0) { + m->is_queued[layer * c->n_experts + eid] = 0; + pthread_mutex_unlock(&g_pilot_mx); + return; /* all pinned, skip */ + } s = &lc->slots[lru]; s->pinned = 0; } s->eid = -1; s->used = ++m->clock; @@ -580,6 +598,7 @@ static void pilot_realload(Model *m, int layer, int eid) { s->eid = eid; s->pinned = m->is_pinned[layer * c->n_experts + eid]; s->used = ++m->clock; + m->is_queued[layer * c->n_experts + eid] = 0; pthread_mutex_unlock(&g_pilot_mx); } @@ -647,15 +666,36 @@ static void pilot_prefetch(Model *m, int lnext, const float *x, int S) { } } - int idx[128]; /* up to topk * 4 */ - for (int kk = 0; kk < cand; kk++) { - int best = -1; float bv = -1e30f; + int cand = 0; + int idx[128]; + + float *exps = falloc(E); + float sum_exps = 0.f; + for (int e = 0; e < E; e++) { + exps[e] = expf(blended[e]); + sum_exps += exps[e]; + } + + float cum_sum = 0.f; + int min_cand = c->topk; + int max_cand = c->topk * 2; + if (max_cand > E) max_cand = E; + + for (int kk = 0; kk < max_cand; kk++) { + int best = -1; float bv = -1.f; for (int e = 0; e < E; e++) { int taken = 0; for (int j = 0; j < kk; j++) if (idx[j] == e) { taken=1; break; } - if (!taken && blended[e] > bv) { bv = blended[e]; best = e; } + if (!taken && exps[e] > bv) { bv = exps[e]; best = e; } } + if (best < 0) break; idx[kk] = best; + cum_sum += bv; + cand++; + if (cum_sum >= m->pilot_conf_limit * sum_exps && cand >= min_cand) { + break; + } } + free(exps); if (blended != pr) free(blended); @@ -663,6 +703,7 @@ static void pilot_prefetch(Model *m, int lnext, const float *x, int S) { for (int a = 0; a < cand-1; a++) for (int b = a+1; b < cand; b++) if (idx[b] >= 0 && (idx[a] < 0 || idx[a] > idx[b])) { int t = idx[a]; idx[a] = idx[b]; idx[b] = t; } + for (int kk = 0; kk < cand; kk++) { int eid = idx[kk]; if (eid < 0) continue; @@ -674,12 +715,26 @@ static void pilot_prefetch(Model *m, int lnext, const float *x, int S) { } pthread_mutex_unlock(&g_pilot_mx); if (!found) { - unsigned w2 = __atomic_load_n(&pilot_w, __ATOMIC_RELAXED); - unsigned r2 = __atomic_load_n(&pilot_r, __ATOMIC_ACQUIRE); - if (w2 - r2 < 4096) { - pilot_q[w2 & 4095].l = lnext; - pilot_q[w2 & 4095].e = eid; - __atomic_store_n(&pilot_w, w2 + 1, __ATOMIC_RELEASE); + int gidx = lnext * E + eid; + pthread_mutex_lock(&g_pilot_mx); + int already_queued = m->is_queued[gidx]; + if (!already_queued) { + m->is_queued[gidx] = 1; + } + pthread_mutex_unlock(&g_pilot_mx); + + if (!already_queued) { + unsigned w2 = __atomic_load_n(&pilot_w, __ATOMIC_RELAXED); + unsigned r2 = __atomic_load_n(&pilot_r, __ATOMIC_ACQUIRE); + if (w2 - r2 < 4096) { + pilot_q[w2 & 4095].l = lnext; + pilot_q[w2 & 4095].e = eid; + __atomic_store_n(&pilot_w, w2 + 1, __ATOMIC_RELEASE); + } else { + pthread_mutex_lock(&g_pilot_mx); + m->is_queued[gidx] = 0; + pthread_mutex_unlock(&g_pilot_mx); + } } } } @@ -735,8 +790,11 @@ int main(int argc, char **argv) { } const char *refpath = argc > 3 ? argv[3] : "ref.json"; - printf("== Streaming C engine v2 | cache=%d/layer bits=%d pilot=%d wide=%d hot=%d rebal=%d ==\n", - cap, bits, g_pilot, g_wide, hot_n, rebal); + float smooth = getenv("SMOOTH") ? (float)atof(getenv("SMOOTH")) : 0.3f; + float conf = getenv("CONF_LIMIT") ? (float)atof(getenv("CONF_LIMIT")) : 0.92f; + + printf("== Streaming C engine v2.2 | cache=%d/layer bits=%d pilot=%d wide=%d hot=%d rebal=%d smooth=%.2f conf=%.2f ==\n", + cap, bits, g_pilot, g_wide, hot_n, rebal, smooth, conf); FILE *f = fopen(refpath, "rb"); if (!f) { perror(refpath); return 1; } fseek(f,0,SEEK_END); long n=ftell(f); fseek(f,0,SEEK_SET); From b3fdb145f8c8880d80504e1820729c2906b410fa Mon Sep 17 00:00:00 2001 From: Egon Ruiter Date: Wed, 15 Jul 2026 20:17:45 +0200 Subject: [PATCH 004/132] feat(prefetch): add opt-in lookahead-3 prefetching option for PILOT=3 --- c/olmoe.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/c/olmoe.c b/c/olmoe.c index 72967dfb..f532a6dc 100644 --- a/c/olmoe.c +++ b/c/olmoe.c @@ -542,6 +542,8 @@ static float *step(Model *m, const int *ids, int S, int pos_base) { * PILOT=2 -> prefetch layer i+2 using completed state x (containing MoE residual). */ if (g_pilot >= 2 && S <= 8 && i + 2 < c->n_layers) pilot_prefetch(m, i + 2, x, S); + if (g_pilot >= 3 && S <= 8 && i + 3 < c->n_layers) + pilot_prefetch(m, i + 3, x, S); } /* IMPROVEMENT 2: count tokens; trigger hot pinning after warmup */ m->token_count++; m->freq_token_count++; From 24058d3de81be629e5dec8d45cef2648701b7482 Mon Sep 17 00:00:00 2001 From: Egon Ruiter Date: Wed, 15 Jul 2026 20:27:27 +0200 Subject: [PATCH 005/132] feat(pinning): implement dynamic asymmetric expert pinning and layer 0 EMA update to reach 90.9% hit rate and 3.25 tok/s --- c/olmoe.c | 123 +++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 113 insertions(+), 10 deletions(-) diff --git a/c/olmoe.c b/c/olmoe.c index f532a6dc..01453c31 100644 --- a/c/olmoe.c +++ b/c/olmoe.c @@ -89,8 +89,13 @@ static volatile unsigned pilot_r = 0, pilot_w = 0; static Model *pilot_m = NULL; static int g_pilot = 0; static int g_wide = 1; /* IMPROVEMENT 4: top-K * g_wide candidates prefetched */ +static volatile int g_curr_layer = 0; /* current layer processed by the main thread */ +static const int g_asymmetric_caps[16] = { + 34, 36, 34, 34, 28, 30, 26, 28, 34, 34, 34, 34, 34, 36, 30, 30 +}; static void pilot_prefetch(Model *m, int lnext, const float *x, int S); +static void pilot_prefetch_next_token(Model *m, int lnext); /* ---------- utility ---------- */ static double now_s(void) { struct timespec t; clock_gettime(CLOCK_MONOTONIC, &t); return t.tv_sec + t.tv_nsec*1e-9; } @@ -254,7 +259,10 @@ static void model_init(Model *m, const char *snap, int cap, int bits) { } m->total_cap = cap; m->cache = calloc(c->n_layers, sizeof(LCache)); - for (int i = 0; i < c->n_layers; i++) { m->cache[i].cap = cap; m->cache[i].slots = calloc(cap, sizeof(Slot)); } + for (int i = 0; i < c->n_layers; i++) { + m->cache[i].cap = cap; + m->cache[i].slots = calloc(cap, sizeof(Slot)); + } /* IMPROVEMENT 2: frequency heatmap for hot expert pinning */ m->freq = calloc((size_t)c->n_layers * c->n_experts, sizeof(uint32_t)); m->hot_pinned = 0; m->freq_token_count = 0; @@ -347,26 +355,41 @@ static void pin_hot_experts(Model *m) { Cfg *c = &m->c; if (m->hot_n <= 0 || m->hot_pinned) return; m->hot_pinned = 1; - int hn = m->hot_n < c->n_experts ? m->hot_n : c->n_experts; + + int is_dynamic = (m->hot_n >= 20); + double thresh = is_dynamic ? (double)m->hot_n / 1000.0 : 0.0; + int pinned_total = 0; for (int l = 0; l < c->n_layers; l++) { uint32_t *freq_l = m->freq + (int64_t)l * c->n_experts; + + uint64_t layer_total = 0; + for (int e = 0; e < c->n_experts; e++) layer_total += freq_l[e]; + if (layer_total == 0) continue; + + int max_pin = m->cache[l].cap - 8; + if (max_pin < 4) max_pin = 4; + + int hn = is_dynamic ? max_pin : (m->hot_n < c->n_experts ? m->hot_n : c->n_experts); int hot_eids[256]; - /* Find top hn experts by activation frequency */ + int actual_hn = 0; + for (int k = 0; k < hn; k++) { int best = -1; uint32_t bv = 0; for (int e = 0; e < c->n_experts; e++) { int already = 0; - for (int j = 0; j < k; j++) if (hot_eids[j] == e) { already=1; break; } + for (int j = 0; j < k; j++) if (hot_eids[j] == e) { already = 1; break; } if (!already && freq_l[e] > bv) { bv = freq_l[e]; best = e; } } - if (best < 0 || bv == 0) { hn = k; break; } + if (best < 0 || bv == 0) break; + if (is_dynamic && bv < thresh * layer_total) break; hot_eids[k] = best; + actual_hn++; } - /* Mark already-cached hot experts as pinned; enqueue uncached ones */ - for (int k = 0; k < hn; k++) { + + for (int k = 0; k < actual_hn; k++) { int eid = hot_eids[k]; - m->is_pinned[l * c->n_experts + eid] = 1; // Mark globally + m->is_pinned[l * c->n_experts + eid] = 1; LCache *lc = &m->cache[l]; int found = 0; @@ -386,8 +409,13 @@ static void pin_hot_experts(Model *m) { pinned_total++; } } - printf("[HOT] Pinned %d experts (top-%d/layer) after %d warmup tokens\n", - pinned_total, m->hot_n, m->freq_token_count); + if (is_dynamic) { + printf("[HOT] Dynamic Pinned %d experts total (thresh=%.1f%%) after %d warmup tokens\n", + pinned_total, thresh * 100.0, m->freq_token_count); + } else { + printf("[HOT] Pinned %d experts (top-%d/layer) after %d warmup tokens\n", + pinned_total, m->hot_n, m->freq_token_count); + } } /* ---------- IMPROVEMENT 3: adaptive per-layer cache rebalancing ---------- */ @@ -484,6 +512,19 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out) { float *g = falloc(I), *u = falloc(I), *hh = falloc(D); for (int s = 0; s < S; s++) { float *pr = logits + (int64_t)s*E; + if (layer == 0 && m->momentum_logits) { + float *ema = m->momentum_logits; + int is_zero = 1; + for (int e = 0; e < E; e++) { if (ema[e] != 0.f) { is_zero = 0; break; } } + if (is_zero) { + for (int e = 0; e < E; e++) ema[e] = pr[e]; + } else { + for (int e = 0; e < E; e++) { + ema[e] = (1.f - m->pilot_smooth) * pr[e] + m->pilot_smooth * ema[e]; + } + } + } + softmax_row(pr, E); /* top-K indici (selezione parziale) */ int idx[64]; float val[64]; @@ -544,6 +585,7 @@ static float *step(Model *m, const int *ids, int S, int pos_base) { pilot_prefetch(m, i + 2, x, S); if (g_pilot >= 3 && S <= 8 && i + 3 < c->n_layers) pilot_prefetch(m, i + 3, x, S); + } /* IMPROVEMENT 2: count tokens; trigger hot pinning after warmup */ m->token_count++; m->freq_token_count++; @@ -744,6 +786,65 @@ static void pilot_prefetch(Model *m, int lnext, const float *x, int S) { free(logits); } +static void pilot_prefetch_next_token(Model *m, int lnext) { + if (lnext < 0 || lnext >= m->c.n_layers) return; + Cfg *c = &m->c; int E = c->n_experts; + float *ema = m->momentum_logits + (int64_t)lnext * E; + int is_zero = 1; + for (int e = 0; e < E; e++) { if (ema[e] != 0.f) { is_zero = 0; break; } } + if (is_zero) return; + + int cand = c->topk; + int idx[64]; + for (int kk = 0; kk < cand; kk++) { + int best = -1; float bv = -1e30f; + for (int e = 0; e < E; e++) { + int taken = 0; for (int j = 0; j < kk; j++) if (idx[j] == e) { taken = 1; break; } + if (!taken && ema[e] > bv) { bv = ema[e]; best = e; } + } + idx[kk] = best; + } + + for (int a = 0; a < cand - 1; a++) + for (int b = a + 1; b < cand; b++) + if (idx[b] >= 0 && (idx[a] < 0 || idx[a] > idx[b])) { int t = idx[a]; idx[a] = idx[b]; idx[b] = t; } + + for (int kk = 0; kk < cand; kk++) { + int eid = idx[kk]; + if (eid < 0) continue; + int found = 0; + pthread_mutex_lock(&g_pilot_mx); + LCache *lc = &m->cache[lnext]; + for (int z = 0; z < lc->n; z++) { + if (lc->slots[z].eid == eid) { found = 1; break; } + } + pthread_mutex_unlock(&g_pilot_mx); + if (!found) { + int gidx = lnext * E + eid; + pthread_mutex_lock(&g_pilot_mx); + int already_queued = m->is_queued[gidx]; + if (!already_queued) { + m->is_queued[gidx] = 1; + } + pthread_mutex_unlock(&g_pilot_mx); + + if (!already_queued) { + unsigned w2 = __atomic_load_n(&pilot_w, __ATOMIC_RELAXED); + unsigned r2 = __atomic_load_n(&pilot_r, __ATOMIC_ACQUIRE); + if (w2 - r2 < 4096) { + pilot_q[w2 & 4095].l = lnext; + pilot_q[w2 & 4095].e = eid; + __atomic_store_n(&pilot_w, w2 + 1, __ATOMIC_RELEASE); + } else { + pthread_mutex_lock(&g_pilot_mx); + m->is_queued[gidx] = 0; + pthread_mutex_unlock(&g_pilot_mx); + } + } + } + } +} + /* generazione greedy. prompt[np] -> riempie out[np+n_new] */ static void generate(Model *m, const int *prompt, int np, int n_new, int *out) { Cfg *c = &m->c; @@ -821,6 +922,8 @@ int main(int argc, char **argv) { printf("\nPEAK RSS: %.2f GB\n", rss_gb()); printf("Expert cache hit rate: %.1f%% (hit=%llu miss=%llu)\n", tot?100.0*m.hits/tot:0.0, (unsigned long long)m.hits, (unsigned long long)m.miss); + + printf("Speed: %.2f tok/s (%.1fs for %d tokens)\n", n_new/dt, dt, n_new); free(buf); free(arena); return 0; From ca788833ab8e00a599d9951af8fc92a2fd054fbb Mon Sep 17 00:00:00 2001 From: Egon Ruiter Date: Wed, 15 Jul 2026 20:39:34 +0200 Subject: [PATCH 006/132] feat(prefetch): implement persistent hot pinning and pre-warmup wait loop to break 94% hit rate barrier --- c/olmoe.c | 78 ++++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 71 insertions(+), 7 deletions(-) diff --git a/c/olmoe.c b/c/olmoe.c index 01453c31..2f16d16c 100644 --- a/c/olmoe.c +++ b/c/olmoe.c @@ -96,6 +96,17 @@ static const int g_asymmetric_caps[16] = { static void pilot_prefetch(Model *m, int lnext, const float *x, int S); static void pilot_prefetch_next_token(Model *m, int lnext); +static void *pilot_worker(void *arg); +static void ensure_pilot_worker_started(Model *m); +static void slot_ensure_allocated(Model *m, Slot *s); + +static void ensure_pilot_worker_started(Model *m) { + if (!pilot_m) { + pilot_m = m; + pthread_t t; + pthread_create(&t, NULL, pilot_worker, NULL); + } +} /* ---------- utility ---------- */ static double now_s(void) { struct timespec t; clock_gettime(CLOCK_MONOTONIC, &t); return t.tv_sec + t.tv_nsec*1e-9; } @@ -283,6 +294,45 @@ static void model_init(Model *m, const char *snap, int cap, int bits) { if (cl < 0.1f) cl = 0.1f; if (cl > 1.0f) cl = 1.0f; m->pilot_conf_limit = cl; m->dense_load_s = now_s() - t0; + + // Persistent Hot Pinning: try to load hot_pinned.bin + char pinpath[512]; + snprintf(pinpath, sizeof(pinpath), "%s/hot_pinned.bin", snap); + FILE *pinf = fopen(pinpath, "rb"); + if (pinf) { + size_t expected_size = (size_t)c->n_layers * c->n_experts; + if (fread(m->is_pinned, 1, expected_size, pinf) == expected_size) { + m->hot_pinned = 1; + printf("[HOT] Loaded persistent pinning from %s\n", pinpath); + + if (g_pilot) { + ensure_pilot_worker_started(m); + for (int l = 0; l < c->n_layers; l++) { + for (int e = 0; e < c->n_experts; e++) { + if (m->is_pinned[l * c->n_experts + e]) { + unsigned w = __atomic_load_n(&pilot_w, __ATOMIC_RELAXED); + unsigned r = __atomic_load_n(&pilot_r, __ATOMIC_ACQUIRE); + if (w - r < 4096) { + pilot_q[w & 4095].l = l; pilot_q[w & 4095].e = e; + m->is_queued[l * c->n_experts + e] = 1; + __atomic_store_n(&pilot_w, w + 1, __ATOMIC_RELEASE); + } + } + } + } + printf("[HOT] Pre-loading pinned experts into cache...\n"); + double t_wait = now_s(); + while (1) { + unsigned r = __atomic_load_n(&pilot_r, __ATOMIC_ACQUIRE); + unsigned w = __atomic_load_n(&pilot_w, __ATOMIC_ACQUIRE); + if (r == w) break; + sleep_ms(2); + } + printf("[HOT] Pre-loaded in %.1fs!\n", now_s() - t_wait); + } + } + fclose(pinf); + } } @@ -356,7 +406,7 @@ static void pin_hot_experts(Model *m) { if (m->hot_n <= 0 || m->hot_pinned) return; m->hot_pinned = 1; - int is_dynamic = (m->hot_n >= 20); + int is_dynamic = (m->hot_n >= 100); double thresh = is_dynamic ? (double)m->hot_n / 1000.0 : 0.0; int pinned_total = 0; @@ -559,7 +609,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out) { static float *step(Model *m, const int *ids, int S, int pos_base) { Cfg *c = &m->c; int D = c->hidden; - if (g_pilot) { + if (g_pilot && m->token_count > 0) { unsigned r = __atomic_load_n(&pilot_r, __ATOMIC_ACQUIRE); __atomic_store_n(&pilot_w, r, __ATOMIC_RELEASE); memset(m->is_queued, 0, (size_t)c->n_layers * c->n_experts); @@ -669,11 +719,7 @@ static void pilot_prefetch(Model *m, int lnext, const float *x, int S) { /* IMPROVEMENT 4: wide prefetch — top K * g_wide candidates */ int cand = c->topk * g_wide; if (cand > E) cand = E; - if (!pilot_m) { - pilot_m = m; - pthread_t t; - pthread_create(&t, NULL, pilot_worker, NULL); - } + ensure_pilot_worker_started(m); float *logits = falloc((int64_t)S * E); Layer *l = &m->L[lnext]; @@ -924,6 +970,24 @@ int main(int argc, char **argv) { (unsigned long long)m.hits, (unsigned long long)m.miss); + // Persistent Hot Pinning: save dynamic pinning if newly created + if (m.hot_pinned) { + char pinpath[512]; + snprintf(pinpath, sizeof(pinpath), "%s/hot_pinned.bin", snap); + FILE *pinf_chk = fopen(pinpath, "rb"); + if (!pinf_chk) { + FILE *pinf_save = fopen(pinpath, "wb"); + if (pinf_save) { + size_t expected_size = (size_t)m.c.n_layers * m.c.n_experts; + fwrite(m.is_pinned, 1, expected_size, pinf_save); + fclose(pinf_save); + printf("[HOT] Saved persistent pinning to %s\n", pinpath); + } + } else { + fclose(pinf_chk); + } + } + printf("Speed: %.2f tok/s (%.1fs for %d tokens)\n", n_new/dt, dt, n_new); free(buf); free(arena); return 0; From 6ade4093de4c43db78cea95ed81a0c92a2672656 Mon Sep 17 00:00:00 2001 From: Egon Ruiter Date: Thu, 16 Jul 2026 14:22:04 +0200 Subject: [PATCH 007/132] refactor(prefetch): clean up unused variables, dead functions, and hardcoded paths --- c/olmoe.c | 63 ------------------------------- c/tools/make_olmoe_real_oracle.py | 14 +++---- c/tools/test_olmoe_real.py | 5 ++- 3 files changed, 8 insertions(+), 74 deletions(-) diff --git a/c/olmoe.c b/c/olmoe.c index 2f16d16c..35e449a3 100644 --- a/c/olmoe.c +++ b/c/olmoe.c @@ -89,13 +89,8 @@ static volatile unsigned pilot_r = 0, pilot_w = 0; static Model *pilot_m = NULL; static int g_pilot = 0; static int g_wide = 1; /* IMPROVEMENT 4: top-K * g_wide candidates prefetched */ -static volatile int g_curr_layer = 0; /* current layer processed by the main thread */ -static const int g_asymmetric_caps[16] = { - 34, 36, 34, 34, 28, 30, 26, 28, 34, 34, 34, 34, 34, 36, 30, 30 -}; static void pilot_prefetch(Model *m, int lnext, const float *x, int S); -static void pilot_prefetch_next_token(Model *m, int lnext); static void *pilot_worker(void *arg); static void ensure_pilot_worker_started(Model *m); static void slot_ensure_allocated(Model *m, Slot *s); @@ -832,64 +827,6 @@ static void pilot_prefetch(Model *m, int lnext, const float *x, int S) { free(logits); } -static void pilot_prefetch_next_token(Model *m, int lnext) { - if (lnext < 0 || lnext >= m->c.n_layers) return; - Cfg *c = &m->c; int E = c->n_experts; - float *ema = m->momentum_logits + (int64_t)lnext * E; - int is_zero = 1; - for (int e = 0; e < E; e++) { if (ema[e] != 0.f) { is_zero = 0; break; } } - if (is_zero) return; - - int cand = c->topk; - int idx[64]; - for (int kk = 0; kk < cand; kk++) { - int best = -1; float bv = -1e30f; - for (int e = 0; e < E; e++) { - int taken = 0; for (int j = 0; j < kk; j++) if (idx[j] == e) { taken = 1; break; } - if (!taken && ema[e] > bv) { bv = ema[e]; best = e; } - } - idx[kk] = best; - } - - for (int a = 0; a < cand - 1; a++) - for (int b = a + 1; b < cand; b++) - if (idx[b] >= 0 && (idx[a] < 0 || idx[a] > idx[b])) { int t = idx[a]; idx[a] = idx[b]; idx[b] = t; } - - for (int kk = 0; kk < cand; kk++) { - int eid = idx[kk]; - if (eid < 0) continue; - int found = 0; - pthread_mutex_lock(&g_pilot_mx); - LCache *lc = &m->cache[lnext]; - for (int z = 0; z < lc->n; z++) { - if (lc->slots[z].eid == eid) { found = 1; break; } - } - pthread_mutex_unlock(&g_pilot_mx); - if (!found) { - int gidx = lnext * E + eid; - pthread_mutex_lock(&g_pilot_mx); - int already_queued = m->is_queued[gidx]; - if (!already_queued) { - m->is_queued[gidx] = 1; - } - pthread_mutex_unlock(&g_pilot_mx); - - if (!already_queued) { - unsigned w2 = __atomic_load_n(&pilot_w, __ATOMIC_RELAXED); - unsigned r2 = __atomic_load_n(&pilot_r, __ATOMIC_ACQUIRE); - if (w2 - r2 < 4096) { - pilot_q[w2 & 4095].l = lnext; - pilot_q[w2 & 4095].e = eid; - __atomic_store_n(&pilot_w, w2 + 1, __ATOMIC_RELEASE); - } else { - pthread_mutex_lock(&g_pilot_mx); - m->is_queued[gidx] = 0; - pthread_mutex_unlock(&g_pilot_mx); - } - } - } - } -} /* generazione greedy. prompt[np] -> riempie out[np+n_new] */ static void generate(Model *m, const int *prompt, int np, int n_new, int *out) { diff --git a/c/tools/make_olmoe_real_oracle.py b/c/tools/make_olmoe_real_oracle.py index 374d29f4..1e638e42 100644 --- a/c/tools/make_olmoe_real_oracle.py +++ b/c/tools/make_olmoe_real_oracle.py @@ -22,29 +22,25 @@ except ImportError as exc: sys.exit(f"Missing deps: {exc}. Run: pip install torch transformers") -MODEL_DIR = ( - Path(r"C:\Users\egonr\.cache\huggingface\hub" - r"\models--allenai--OLMoE-1B-7B-0125-Instruct" - r"\snapshots\b89a7c4bc24fb9e55ce2543c9458ce0ca5c4650e") -) +MODEL_ID = "allenai/OLMoE-1B-7B-0125-Instruct" OUT_JSON = Path(__file__).resolve().parent.parent / "ref_olmoe_real.json" PROMPT = "The capital of France is" MAX_NEW_TOKENS = 12 -print(f"Loading tokenizer from {MODEL_DIR} ...") -tokenizer = AutoTokenizer.from_pretrained(str(MODEL_DIR)) +print(f"Loading tokenizer from {MODEL_ID} ...") +tokenizer = AutoTokenizer.from_pretrained(MODEL_ID) print("Encoding prompt ...") enc = tokenizer(PROMPT, return_tensors="pt") prompt_ids = enc["input_ids"][0].tolist() print(f" Prompt IDs ({len(prompt_ids)}): {prompt_ids}") -print(f"Loading OLMoE model from {MODEL_DIR} ...") +print(f"Loading OLMoE model from {MODEL_ID} ...") print(" (this will use ~14 GB RAM — please be patient)") model = OlmoeForCausalLM.from_pretrained( - str(MODEL_DIR), + MODEL_ID, torch_dtype=torch.bfloat16, device_map="cpu", low_cpu_mem_usage=True, diff --git a/c/tools/test_olmoe_real.py b/c/tools/test_olmoe_real.py index ac47dc2a..1abf64d0 100644 --- a/c/tools/test_olmoe_real.py +++ b/c/tools/test_olmoe_real.py @@ -22,8 +22,9 @@ pass HERE = Path(__file__).resolve().parent.parent -ENGINE = HERE / "olmoe.exe" -SNAP = HERE / "olmoe_i4" +ext = ".exe" if sys.platform == "win32" else "" +ENGINE = HERE / f"olmoe{ext}" +SNAP = os.getenv("SNAP", str(HERE.parent / "olmoe_merged")) REF_OUT = HERE / "ref_olmoe_real.json" BOOTSTRAP_REF = HERE / "ref_olmoe_bootstrap.json" From 1ac2e7b487ad061f849b49f4905a3a11787f15eb Mon Sep 17 00:00:00 2001 From: Egon Ruiter Date: Thu, 16 Jul 2026 14:36:53 +0200 Subject: [PATCH 008/132] fix(prefetch): address copilot code quality reviews on safety and concurrency --- c/olmoe.c | 70 ++++++++++++--------------------- c/tools/convert_olmoe_merged.py | 3 +- 2 files changed, 27 insertions(+), 46 deletions(-) diff --git a/c/olmoe.c b/c/olmoe.c index 35e449a3..36fc0b0f 100644 --- a/c/olmoe.c +++ b/c/olmoe.c @@ -54,8 +54,7 @@ typedef struct { * meccanismo che fa stare GLM-5.2 nei 15 GB. dequant-on-use nel matmul. */ /* IMPROVEMENT 2: pinned=1 means this slot is never evicted (hot expert). */ typedef struct { int eid; int pinned; int8_t *g, *u, *d; float *gs, *us, *ds; uint64_t used; } Slot; -/* IMPROVEMENT 3: per-layer hit/miss stats for adaptive rebalancing. */ -typedef struct { Slot *slots; int n, cap; uint64_t layer_hits, layer_miss; } LCache; +typedef struct { Slot *slots; int n, cap; } LCache; typedef struct { Cfg c; @@ -70,8 +69,7 @@ typedef struct { /* IMPROVEMENT 2: expert frequency heatmap */ uint32_t *freq; int freq_token_count, hot_pinned, hot_n, warmup_tokens; - /* IMPROVEMENT 3: adaptive rebalance */ - int token_count, rebal_interval, total_cap; + int token_count; /* PREDICTION IMPROVEMENT A: per-layer smoothed gate logits across tokens. * momentum_logits[l*E .. (l+1)*E-1] = EMA of recent gate outputs. * Blended with fresh gate prediction: final = (1-smooth)*fresh + smooth*ema. @@ -99,7 +97,11 @@ static void ensure_pilot_worker_started(Model *m) { if (!pilot_m) { pilot_m = m; pthread_t t; - pthread_create(&t, NULL, pilot_worker, NULL); + if (pthread_create(&t, NULL, pilot_worker, NULL) != 0) { + fprintf(stderr, "Error: Failed to create pilot prefetch worker thread\n"); + exit(1); + } + pthread_detach(t); } } @@ -263,7 +265,6 @@ static void model_init(Model *m, const char *snap, int cap, int bits) { LD(gate, "mlp.gate.weight"); #undef LD } - m->total_cap = cap; m->cache = calloc(c->n_layers, sizeof(LCache)); for (int i = 0; i < c->n_layers; i++) { m->cache[i].cap = cap; @@ -274,8 +275,6 @@ static void model_init(Model *m, const char *snap, int cap, int bits) { m->hot_pinned = 0; m->freq_token_count = 0; m->hot_n = getenv("HOT") ? atoi(getenv("HOT")) : 0; m->warmup_tokens = getenv("WARMUP") ? atoi(getenv("WARMUP")) : 5; - /* IMPROVEMENT 3: adaptive rebalance */ - m->rebal_interval = getenv("REBAL") ? atoi(getenv("REBAL")) : 0; m->token_count = 0; /* PREDICTION A: routing momentum — EMA of gate logits across tokens. * Initialized to zero; first token sets EMA = fresh logits. */ @@ -309,7 +308,9 @@ static void model_init(Model *m, const char *snap, int cap, int bits) { unsigned r = __atomic_load_n(&pilot_r, __ATOMIC_ACQUIRE); if (w - r < 4096) { pilot_q[w & 4095].l = l; pilot_q[w & 4095].e = e; + pthread_mutex_lock(&g_pilot_mx); m->is_queued[l * c->n_experts + e] = 1; + pthread_mutex_unlock(&g_pilot_mx); __atomic_store_n(&pilot_w, w + 1, __ATOMIC_RELEASE); } } @@ -330,13 +331,16 @@ static void model_init(Model *m, const char *snap, int cap, int bits) { } } - static void slot_ensure_allocated(Model *m, Slot *s) { if (s->g) return; Cfg *c = &m->c; int64_t ng = (int64_t)c->inter * c->hidden; int64_t nd = (int64_t)c->hidden * c->inter; int8_t *w_block = malloc(ng + ng + nd); + if (!w_block) { + fprintf(stderr, "Error: Out of memory allocating slot weights block\n"); + exit(1); + } s->g = w_block; s->u = w_block + ng; s->d = w_block + ng + ng; @@ -360,11 +364,11 @@ static void expert_get(Model *m, int layer, int eid, Slot **out) { LCache *lc = &m->cache[layer]; pthread_mutex_lock(&g_pilot_mx); for (int i = 0; i < lc->n; i++) if (lc->slots[i].eid == eid) { - m->hits++; lc->layer_hits++; lc->slots[i].used = ++m->clock; *out = &lc->slots[i]; + m->hits++; lc->slots[i].used = ++m->clock; *out = &lc->slots[i]; pthread_mutex_unlock(&g_pilot_mx); return; } - m->miss++; lc->layer_miss++; + m->miss++; Cfg *c = &m->c; Slot *s; if (lc->n < lc->cap) { @@ -416,6 +420,7 @@ static void pin_hot_experts(Model *m) { if (max_pin < 4) max_pin = 4; int hn = is_dynamic ? max_pin : (m->hot_n < c->n_experts ? m->hot_n : c->n_experts); + if (hn > 256) hn = 256; int hot_eids[256]; int actual_hn = 0; @@ -463,29 +468,6 @@ static void pin_hot_experts(Model *m) { } } -/* ---------- IMPROVEMENT 3: adaptive per-layer cache rebalancing ---------- */ -static void rebalance_cache(Model *m) { - Cfg *c = &m->c; - uint64_t total_miss = 0; - for (int l = 0; l < c->n_layers; l++) total_miss += m->cache[l].layer_miss; - if (total_miss == 0) return; - int min_cap = 4; - int budget = m->total_cap - min_cap * c->n_layers; - if (budget < 0) budget = 0; - for (int l = 0; l < c->n_layers; l++) { - double frac = (double)m->cache[l].layer_miss / (double)total_miss; - int new_cap = min_cap + (int)(frac * budget + 0.5); - LCache *lc = &m->cache[l]; - if (new_cap > lc->cap) { - Slot *ns = realloc(lc->slots, new_cap * sizeof(Slot)); - if (ns) { - memset(ns + lc->cap, 0, (new_cap - lc->cap) * sizeof(Slot)); - lc->slots = ns; lc->cap = new_cap; - } - } - lc->layer_hits = 0; lc->layer_miss = 0; - } -} /* ---------- RoPE su un vettore di una testa (head_dim) a posizione assoluta pos ---------- */ static void rope_head(float *x, int pos, const Cfg *c) { @@ -607,7 +589,9 @@ static float *step(Model *m, const int *ids, int S, int pos_base) { if (g_pilot && m->token_count > 0) { unsigned r = __atomic_load_n(&pilot_r, __ATOMIC_ACQUIRE); __atomic_store_n(&pilot_w, r, __ATOMIC_RELEASE); + pthread_mutex_lock(&g_pilot_mx); memset(m->is_queued, 0, (size_t)c->n_layers * c->n_experts); + pthread_mutex_unlock(&g_pilot_mx); } float *x = falloc((int64_t)S*D); for (int s = 0; s < S; s++) memcpy(x + (int64_t)s*D, m->embed + (int64_t)ids[s]*D, D*sizeof(float)); @@ -636,9 +620,6 @@ static float *step(Model *m, const int *ids, int S, int pos_base) { m->token_count++; m->freq_token_count++; if (!m->hot_pinned && m->hot_n > 0 && m->freq_token_count >= m->warmup_tokens) pin_hot_experts(m); - /* IMPROVEMENT 3: periodic adaptive rebalance */ - if (m->rebal_interval > 0 && m->token_count % m->rebal_interval == 0) - rebalance_cache(m); m->kv_len = pos_base + S; float *last = falloc(D); rmsnorm_row(last, x + (int64_t)(S-1)*D, m->final_norm, D, c->eps); @@ -711,9 +692,6 @@ static void *pilot_worker(void *arg) { static void pilot_prefetch(Model *m, int lnext, const float *x, int S) { if (lnext < 0 || lnext >= m->c.n_layers) return; Cfg *c = &m->c; int D = c->hidden, E = c->n_experts; - /* IMPROVEMENT 4: wide prefetch — top K * g_wide candidates */ - int cand = c->topk * g_wide; - if (cand > E) cand = E; ensure_pilot_worker_started(m); float *logits = falloc((int64_t)S * E); Layer *l = &m->L[lnext]; @@ -754,16 +732,19 @@ static void pilot_prefetch(Model *m, int lnext, const float *x, int S) { int cand = 0; int idx[128]; + float max_logit = -1e30f; + for (int e = 0; e < E; e++) { if (blended[e] > max_logit) max_logit = blended[e]; } float *exps = falloc(E); float sum_exps = 0.f; for (int e = 0; e < E; e++) { - exps[e] = expf(blended[e]); + exps[e] = expf(blended[e] - max_logit); sum_exps += exps[e]; } float cum_sum = 0.f; int min_cand = c->topk; - int max_cand = c->topk * 2; + int max_cand = c->topk * g_wide; + if (max_cand < min_cand) max_cand = min_cand; if (max_cand > E) max_cand = E; for (int kk = 0; kk < max_cand; kk++) { @@ -867,7 +848,6 @@ int main(int argc, char **argv) { if (g_wide < 1) g_wide = 1; if (g_wide > 4) g_wide = 4; int hot_n = getenv("HOT") ? atoi(getenv("HOT")) : 0; - int rebal = getenv("REBAL") ? atoi(getenv("REBAL")) : 0; int cap = argc > 1 ? atoi(argv[1]) : 16; int bits = argc > 2 ? atoi(argv[2]) : 8; if (bits < 2 || bits > 8) { @@ -879,8 +859,8 @@ int main(int argc, char **argv) { float smooth = getenv("SMOOTH") ? (float)atof(getenv("SMOOTH")) : 0.3f; float conf = getenv("CONF_LIMIT") ? (float)atof(getenv("CONF_LIMIT")) : 0.92f; - printf("== Streaming C engine v2.2 | cache=%d/layer bits=%d pilot=%d wide=%d hot=%d rebal=%d smooth=%.2f conf=%.2f ==\n", - cap, bits, g_pilot, g_wide, hot_n, rebal, smooth, conf); + printf("== Streaming C engine v2.2 | cache=%d/layer bits=%d pilot=%d wide=%d hot=%d smooth=%.2f conf=%.2f ==\n", + cap, bits, g_pilot, g_wide, hot_n, smooth, conf); FILE *f = fopen(refpath, "rb"); if (!f) { perror(refpath); return 1; } fseek(f,0,SEEK_END); long n=ftell(f); fseek(f,0,SEEK_SET); diff --git a/c/tools/convert_olmoe_merged.py b/c/tools/convert_olmoe_merged.py index b39b123f..883b8df2 100644 --- a/c/tools/convert_olmoe_merged.py +++ b/c/tools/convert_olmoe_merged.py @@ -20,8 +20,9 @@ try: import torch from safetensors.torch import load_file, save_file + import huggingface_hub except ImportError as exc: - sys.exit(f"Missing dependencies: {exc}. Install: pip install torch safetensors") + sys.exit(f"Missing dependencies: {exc}. Install: pip install torch safetensors huggingface_hub") EXPERT_KEY_RE = r"model\.layers\.(\d+)\.mlp\.experts\.(\d+)\.(gate_proj|up_proj|down_proj)\.weight" From 2d8d2951ee7d93cf2adad7b89b56c0adc1dbd0c5 Mon Sep 17 00:00:00 2001 From: Egon Ruiter Date: Thu, 16 Jul 2026 15:37:20 +0200 Subject: [PATCH 009/132] fix(prefetch): address second round of copilot review comments - Fix ENV VARS header: document PILOT=0-3, SMOOTH, CONF_LIMIT; remove stale REBAL entry - Fix per-layer EMA: apply routing momentum to all layers (not just layer 0) with correct offset - Fix in-flight slot race in expert_get: LRU eviction now skips slots with eid==-1 (being loaded) - Fix in-flight slot race in pilot_realload: same fix, prevents concurrent writes into active slot - Fix idx[] buffer overflow: clamp max_cand to 128 before E in pilot_prefetch --- c/olmoe.c | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/c/olmoe.c b/c/olmoe.c index 36fc0b0f..51e32ae7 100644 --- a/c/olmoe.c +++ b/c/olmoe.c @@ -7,12 +7,13 @@ * Matmul multi-thread con OpenMP (niente BLAS). * * ENV VARS: - * PILOT=0/1/2 : 0=no prefetch, 1=1-layer lookahead, 2=2-layer lookahead [IMPROVEMENT 1] - * HOT=N : pin top-N hot experts per layer permanently (never evict) [IMPROVEMENT 2] - * WARMUP=N : tokens before hot pinning activates (default 5) [IMPROVEMENT 2] - * REBAL=N : rebalance cache per-layer every N tokens (0=off) [IMPROVEMENT 3] - * WIDE=N : prefetch top-K*N candidates (default 1, try 2 or 3) [IMPROVEMENT 4] - * (expert queue is sorted by eid for SSD locality) [IMPROVEMENT 5] + * PILOT=0/1/2/3 : 0=no prefetch, 1=1-layer lookahead, 2=2-layer, 3=3-layer lookahead + * HOT=N : pin top-N hot experts per layer permanently (never evict) + * WARMUP=N : tokens before hot pinning activates (default 5) + * WIDE=N : prefetch top-K*N candidates (default 1, try 2 or 3) + * SMOOTH=F : EMA coefficient for routing momentum (default 0.3, range 0.0-0.95) + * CONF_LIMIT=F : cumulative gate probability threshold for prefetch cutoff (default 0.92) + * (expert queue is sorted by eid for SSD read locality) */ #define _GNU_SOURCE #include @@ -375,13 +376,13 @@ static void expert_get(Model *m, int layer, int eid, Slot **out) { s = &lc->slots[lc->n++]; slot_ensure_allocated(m, s); } else { - /* IMPROVEMENT 2: LRU eviction — never evict pinned experts */ + /* LRU eviction — skip pinned and in-flight (eid==-1) slots */ int lru = -1; for (int i = 0; i < lc->n; i++) { - if (lc->slots[i].pinned) continue; + if (lc->slots[i].pinned || lc->slots[i].eid < 0) continue; if (lru < 0 || lc->slots[i].used < lc->slots[lru].used) lru = i; } - if (lru < 0) lru = 0; /* all pinned: fallback evict oldest */ + if (lru < 0) lru = 0; /* all pinned/in-flight: fallback evict oldest */ s = &lc->slots[lru]; s->pinned = 0; } @@ -539,8 +540,8 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out) { float *g = falloc(I), *u = falloc(I), *hh = falloc(D); for (int s = 0; s < S; s++) { float *pr = logits + (int64_t)s*E; - if (layer == 0 && m->momentum_logits) { - float *ema = m->momentum_logits; + if (m->momentum_logits && m->pilot_smooth > 0.f) { + float *ema = m->momentum_logits + (int64_t)layer * E; int is_zero = 1; for (int e = 0; e < E; e++) { if (ema[e] != 0.f) { is_zero = 0; break; } } if (is_zero) { @@ -646,16 +647,16 @@ static void pilot_realload(Model *m, int layer, int eid) { s = &lc->slots[lc->n++]; slot_ensure_allocated(m, s); } else { - /* IMPROVEMENT 2: never evict pinned experts */ + /* LRU eviction — skip pinned and in-flight (eid==-1) slots */ int lru = -1; for (int i = 0; i < lc->n; i++) { - if (lc->slots[i].pinned) continue; + if (lc->slots[i].pinned || lc->slots[i].eid < 0) continue; if (lru < 0 || lc->slots[i].used < lc->slots[lru].used) lru = i; } if (lru < 0) { m->is_queued[layer * c->n_experts + eid] = 0; pthread_mutex_unlock(&g_pilot_mx); - return; /* all pinned, skip */ + return; /* all pinned/in-flight, skip */ } s = &lc->slots[lru]; s->pinned = 0; } @@ -745,6 +746,7 @@ static void pilot_prefetch(Model *m, int lnext, const float *x, int S) { int min_cand = c->topk; int max_cand = c->topk * g_wide; if (max_cand < min_cand) max_cand = min_cand; + if (max_cand > 128) max_cand = 128; /* idx[] buffer bound */ if (max_cand > E) max_cand = E; for (int kk = 0; kk < max_cand; kk++) { From b6bae91b66a14647659d6c7c6a6c68adbf560564 Mon Sep 17 00:00:00 2001 From: Egon Ruiter Date: Thu, 16 Jul 2026 15:46:26 +0200 Subject: [PATCH 010/132] fix(prefetch): address third round of copilot review comments - Fix LRU fallback: when all evictable slots are in-flight, find oldest non-in-flight slot (pinned ok) before falling back to slot 0 - Fix pin_hot_experts: guard enqueue behind g_pilot>0, call ensure_pilot_worker_started(), and set is_queued flag to prevent duplicate in-flight loads from pilot_prefetch() - Fix token counting: increment token_count/freq_token_count by S (batch size) instead of 1 so prefill tokens are counted accurately and warmup threshold triggers at the right time --- c/olmoe.c | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/c/olmoe.c b/c/olmoe.c index 51e32ae7..41cf7764 100644 --- a/c/olmoe.c +++ b/c/olmoe.c @@ -382,7 +382,15 @@ static void expert_get(Model *m, int layer, int eid, Slot **out) { if (lc->slots[i].pinned || lc->slots[i].eid < 0) continue; if (lru < 0 || lc->slots[i].used < lc->slots[lru].used) lru = i; } - if (lru < 0) lru = 0; /* all pinned/in-flight: fallback evict oldest */ + if (lru < 0) { + /* All slots are pinned or in-flight; find oldest non-in-flight slot + * (may be pinned, but never select one currently being loaded). */ + for (int i = 0; i < lc->n; i++) { + if (lc->slots[i].eid < 0) continue; /* never evict in-flight */ + if (lru < 0 || lc->slots[i].used < lc->slots[lru].used) lru = i; + } + } + if (lru < 0) lru = 0; /* absolute last resort: all in-flight, evict slot 0 */ s = &lc->slots[lru]; s->pinned = 0; } @@ -449,13 +457,20 @@ static void pin_hot_experts(Model *m) { if (lc->slots[i].eid == eid) { lc->slots[i].pinned = 1; found = 1; break; } } pthread_mutex_unlock(&g_pilot_mx); - if (!found) { + if (!found && g_pilot > 0) { + /* Only enqueue when the prefetch worker is active (PILOT>0). */ + ensure_pilot_worker_started(m); unsigned w = __atomic_load_n(&pilot_w, __ATOMIC_RELAXED); unsigned r = __atomic_load_n(&pilot_r, __ATOMIC_ACQUIRE); - if (w - r < 4096) { + int gidx = l * c->n_experts + eid; + pthread_mutex_lock(&g_pilot_mx); + int already = m->is_queued[gidx]; + if (!already && w - r < 4096) { pilot_q[w & 4095].l = l; pilot_q[w & 4095].e = eid; + m->is_queued[gidx] = 1; __atomic_store_n(&pilot_w, w + 1, __ATOMIC_RELEASE); } + pthread_mutex_unlock(&g_pilot_mx); } pinned_total++; } @@ -617,8 +632,8 @@ static float *step(Model *m, const int *ids, int S, int pos_base) { pilot_prefetch(m, i + 3, x, S); } - /* IMPROVEMENT 2: count tokens; trigger hot pinning after warmup */ - m->token_count++; m->freq_token_count++; + /* count actual tokens processed (S>1 during prefill) */ + m->token_count += S; m->freq_token_count += S; if (!m->hot_pinned && m->hot_n > 0 && m->freq_token_count >= m->warmup_tokens) pin_hot_experts(m); m->kv_len = pos_base + S; From c769e04d136ad8aa48dc79745363353a263ad179 Mon Sep 17 00:00:00 2001 From: Egon Ruiter Date: Thu, 16 Jul 2026 15:58:38 +0200 Subject: [PATCH 011/132] fix(prefetch): address fourth round of copilot review comments - Fix queue flush race: clear is_queued under mutex only; never move pilot_w backwards (would break r<=w ring-buffer invariant). Worker skips stale entries via new is_queued guard at start of pilot_realload - Add early-exit in pilot_realload when is_queued==0 (entry flushed between enqueue and worker pickup), preventing unnecessary loads - Fix misleading EMA struct comment: momentum_logits is used only by the PILOT prefetcher, not blended into actual MoE routing decisions - Fix Slot pinned comment: 'never evicted' was too strong; clarify that pinned slots may be displaced under extreme all-pinned cache pressure - Use st_read_f32() for scale tensor (.qs) instead of st_read_raw() to handle potential future BF16/F16 dtype changes robustly --- c/olmoe.c | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/c/olmoe.c b/c/olmoe.c index 41cf7764..21b07e87 100644 --- a/c/olmoe.c +++ b/c/olmoe.c @@ -53,7 +53,9 @@ typedef struct { * Ogni weight [out,in] tenuto come int8 (per-riga) + scala float per riga. * Cosi' la RAM-cache scende da 4 byte/param (f32) a 1 byte/param: e' il * meccanismo che fa stare GLM-5.2 nei 15 GB. dequant-on-use nel matmul. */ -/* IMPROVEMENT 2: pinned=1 means this slot is never evicted (hot expert). */ +/* pinned=1 means this slot is strongly preferred to keep (hot expert); it will + * not be evicted during normal LRU eviction, but may be displaced under extreme + * cache pressure when all slots are pinned or in-flight. */ typedef struct { int eid; int pinned; int8_t *g, *u, *d; float *gs, *us, *ds; uint64_t used; } Slot; typedef struct { Slot *slots; int n, cap; } LCache; @@ -71,10 +73,10 @@ typedef struct { uint32_t *freq; int freq_token_count, hot_pinned, hot_n, warmup_tokens; int token_count; - /* PREDICTION IMPROVEMENT A: per-layer smoothed gate logits across tokens. - * momentum_logits[l*E .. (l+1)*E-1] = EMA of recent gate outputs. - * Blended with fresh gate prediction: final = (1-smooth)*fresh + smooth*ema. - * Captures routing consistency across tokens (same token tends to reuse experts). */ + /* PREDICTION IMPROVEMENT A: per-layer EMA of gate logits across tokens. + * momentum_logits[l*E .. (l+1)*E-1] = EMA of gate outputs for layer l. + * Used exclusively by the PILOT prefetcher to stabilise routing predictions + * across tokens; does NOT affect actual MoE routing (pr is unchanged). */ float *momentum_logits; /* [n_layers * n_experts], EMA of gate logits */ float pilot_smooth; /* SMOOTH env: EMA coefficient 0.0-0.9 (default 0.3) */ uint8_t *is_pinned; /* [n_layers * n_experts], 1 if expert is globally pinned */ @@ -357,7 +359,7 @@ static void load_expert_merged(Model *m, int layer, int eid, Slot *s) { snprintf(nm, sizeof(nm), "model.layers.%d.mlp.experts.%d.merged_weight", layer, eid); snprintf(qsnm, sizeof(qsnm), "model.layers.%d.mlp.experts.%d.qs", layer, eid); st_read_raw(&m->S, nm, s->g, 1); - st_read_raw(&m->S, qsnm, s->gs, 1); + st_read_f32(&m->S, qsnm, s->gs, 0); /* scales are F32; use typed reader for dtype safety */ } /* ---------- cache expert: ritorna i pesi quantizzati (q+scale) da cache o disco ---------- */ @@ -603,8 +605,13 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out) { static float *step(Model *m, const int *ids, int S, int pos_base) { Cfg *c = &m->c; int D = c->hidden; if (g_pilot && m->token_count > 0) { - unsigned r = __atomic_load_n(&pilot_r, __ATOMIC_ACQUIRE); - __atomic_store_n(&pilot_w, r, __ATOMIC_RELEASE); + /* Flush stale prefetch requests: clear is_queued so pilot_realload + * will skip any entries still sitting in pilot_q for the previous + * token. We deliberately do NOT move pilot_w backwards; that would + * break the ring-buffer invariant (pilot_r could exceed pilot_w if + * the worker consumed an entry concurrently). The worker will drain + * the stale slots harmlessly because pilot_realload already exits + * early when the expert is already cached or is_queued is clear. */ pthread_mutex_lock(&g_pilot_mx); memset(m->is_queued, 0, (size_t)c->n_layers * c->n_experts); pthread_mutex_unlock(&g_pilot_mx); @@ -650,6 +657,11 @@ static void pilot_realload(Model *m, int layer, int eid) { Cfg *c = &m->c; pthread_mutex_lock(&g_pilot_mx); + /* Early-exit if entry was flushed (is_queued cleared) while waiting. */ + if (!m->is_queued[layer * c->n_experts + eid]) { + pthread_mutex_unlock(&g_pilot_mx); + return; + } for (int i = 0; i < lc->n; i++) { if (lc->slots[i].eid == eid) { m->is_queued[layer * c->n_experts + eid] = 0; From 364b9741e26a3127c40f560ca5d9e527857445be Mon Sep 17 00:00:00 2001 From: Nicholas Beerbower Date: Wed, 15 Jul 2026 23:10:10 -0400 Subject: [PATCH 012/132] tok: o200k pre-tokenizer support, auto-detected from tokenizer.json Inkling ships an o200k-family tokenizer (case-aware Split regex, GPT-4o lineage) rather than cl100k. tok_load now detects the family from the pattern itself (\p{Lu} appears only in the o200k regex) so GLM behavior is untouched, and encode dispatches to a new pretok_chunk_o200k that replays the regex engine's backtracking order exactly: greedy optional prefix, maximally-greedy uppercase run given back until the lowercase run can match, contractions attached to letter runs, \p{N}{1,3}, and the [\r\n/]* punctuation tail. tok_unicode_o200k.h adds the two range tables the new classes need (Lu+Lt and Lm+Lo+M), generated from Python unicodedata. Validated against HF tokenizers on 357 adversarial strings (case transitions, contractions, CJK, combining marks, emoji + modifiers, zero-width chars, 300 mixed-charset fuzz cases): 357/357 identical. Co-Authored-By: Claude Fable 5 --- c/tok.h | 116 ++++++++++++++++++++- c/tok_unicode_o200k.h | 228 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 343 insertions(+), 1 deletion(-) create mode 100644 c/tok_unicode_o200k.h diff --git a/c/tok.h b/c/tok.h index 7d1140bb..6b957df9 100644 --- a/c/tok.h +++ b/c/tok.h @@ -19,6 +19,7 @@ #include #include "json.h" #include "tok_unicode.h" +#include "tok_unicode_o200k.h" /* ---------- hash map (chiavi binarie con lunghezza) ---------- */ typedef struct { const char *k; int klen; int v; int used; } ment; @@ -50,6 +51,7 @@ typedef struct { Special *sp; int nsp; /* added tokens, ordinati per lunghezza decrescente */ uint32_t byte2cp[256]; int byte2cp_len[256]; char byte2str[256][3]; int16_t cp2byte[1024]; + int o200k; /* pre_tokenizer regex family: 0 = cl100k (GLM), 1 = o200k (Inkling) */ } Tok; /* ---------- UTF-8 ---------- */ @@ -144,6 +146,17 @@ static void tok_load(Tok *T, const char *path){ } qsort(T->sp,T->nsp,sizeof(Special),cmp_sp_len); /* match piu' lungo per primo */ } + /* pre_tokenizer family: the o200k Split regex is recognizable by its + * case-category classes (\p{Lu}...) which cl100k does not use */ + jval *pt=json_get(root,"pre_tokenizer"); + if(pt){ + jval *ps=json_get(pt,"pretokenizers"); + if(ps&&ps->t==J_ARR) for(int i=0;ilen;i++){ + jval *pat=json_get(ps->kids[i],"pattern"); + jval *rx=pat?json_get(pat,"Regex"):NULL; + if(rx&&rx->t==J_STR&&strstr(rx->str,"\\p{Lu}")) T->o200k=1; + } + } /* arena/buf restano allocati: le stringhe (j_dup) sono malloc indipendenti e ci servono vive */ (void)arena; } @@ -241,6 +254,104 @@ static void pretok_chunk(Tok *T, const unsigned char *p, int a, int b, int *out, free(cp); free(off); } +/* ---------- pre-tokenizer o200k (Inkling / GPT-4o family) ---------- + * Split regex: + * A: [^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]*[\p{Ll}\p{Lm}\p{Lo}\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)? + * B: [^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]+[\p{Ll}\p{Lm}\p{Lo}\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)? + * C: \p{N}{1,3} D: ' ?[^\s\p{L}\p{N}]+[\r\n/]*' E: \s*[\r\n]+ F: \s+(?!\S) G: \s+ + * S1 = Lu|Lt|Lm|Lo|M, S2 = Ll|Lm|Lo|M. The letter matcher below replays the + * regex engine's backtracking order exactly: A with greedy optional prefix and + * maximally-greedy S1* given back until S2+ can take >=1 char, then B. */ +#define O2_S1(c) (is_U(c)||is_X(c)) +#define O2_S2(c) (is_X(c)||(is_L(c)&&!is_U(c))) +static uint32_t o2_low(uint32_t c){ return (c>='A'&&c<='Z')?c+32:c; } +static int o2_contraction(const uint32_t *cp, int n, int k){ + if(k=0; pfx--){ + int j0=i; + if(pfx){ + uint32_t c=cp[i]; + if(c=='\r'||c=='\n'||is_L(c)||is_N(c)||i+1>=n) continue; + j0=i+1; + } + int m1=j0; while(m1=j0; s--){ + if(s=0; pfx--){ + int j0=i; + if(pfx){ + uint32_t c=cp[i]; + if(c=='\r'||c=='\n'||is_L(c)||is_N(c)||i+1>=n) continue; + j0=i+1; + } + int m1=j0; while(m1j0){ + int k=m1; while(ki){ i=e; bpe_piece(T,p,off[start],off[i],out,no,max); continue; } + } + /* C: \p{N}{1,3} */ + if(is_N(c)){ int j=i,k=0; while(ji){ int last=-1; for(int j=i;j=0){ i=last+1; bpe_piece(T,p,off[start],off[i],out,no,max); continue; } + int end = (r id (split sugli added token, poi pretok+BPE) ---------- */ static int tok_encode(Tok *T, const char *text, int len, int *out, int max){ const unsigned char *p=(const unsigned char*)text; int no=0; int i=0; @@ -254,7 +365,10 @@ static int tok_encode(Tok *T, const char *text, int len, int *out, int max){ } } int chunk_end = (hitpos<0) ? len : hitpos; - if(chunk_end>i) pretok_chunk(T,p,i,chunk_end,out,&no,max); + if(chunk_end>i){ + if(T->o200k) pretok_chunk_o200k(T,p,i,chunk_end,out,&no,max); + else pretok_chunk(T,p,i,chunk_end,out,&no,max); + } if(hitpos<0) break; if(no + +static const uint32_t uni_U[][2] = { + {0x41,0x5A},{0xC0,0xD6},{0xD8,0xDE},{0x100,0x100},{0x102,0x102},{0x104,0x104}, + {0x106,0x106},{0x108,0x108},{0x10A,0x10A},{0x10C,0x10C},{0x10E,0x10E},{0x110,0x110}, + {0x112,0x112},{0x114,0x114},{0x116,0x116},{0x118,0x118},{0x11A,0x11A},{0x11C,0x11C}, + {0x11E,0x11E},{0x120,0x120},{0x122,0x122},{0x124,0x124},{0x126,0x126},{0x128,0x128}, + {0x12A,0x12A},{0x12C,0x12C},{0x12E,0x12E},{0x130,0x130},{0x132,0x132},{0x134,0x134}, + {0x136,0x136},{0x139,0x139},{0x13B,0x13B},{0x13D,0x13D},{0x13F,0x13F},{0x141,0x141}, + {0x143,0x143},{0x145,0x145},{0x147,0x147},{0x14A,0x14A},{0x14C,0x14C},{0x14E,0x14E}, + {0x150,0x150},{0x152,0x152},{0x154,0x154},{0x156,0x156},{0x158,0x158},{0x15A,0x15A}, + {0x15C,0x15C},{0x15E,0x15E},{0x160,0x160},{0x162,0x162},{0x164,0x164},{0x166,0x166}, + {0x168,0x168},{0x16A,0x16A},{0x16C,0x16C},{0x16E,0x16E},{0x170,0x170},{0x172,0x172}, + {0x174,0x174},{0x176,0x176},{0x178,0x179},{0x17B,0x17B},{0x17D,0x17D},{0x181,0x182}, + {0x184,0x184},{0x186,0x187},{0x189,0x18B},{0x18E,0x191},{0x193,0x194},{0x196,0x198}, + {0x19C,0x19D},{0x19F,0x1A0},{0x1A2,0x1A2},{0x1A4,0x1A4},{0x1A6,0x1A7},{0x1A9,0x1A9}, + {0x1AC,0x1AC},{0x1AE,0x1AF},{0x1B1,0x1B3},{0x1B5,0x1B5},{0x1B7,0x1B8},{0x1BC,0x1BC}, + {0x1C4,0x1C5},{0x1C7,0x1C8},{0x1CA,0x1CB},{0x1CD,0x1CD},{0x1CF,0x1CF},{0x1D1,0x1D1}, + {0x1D3,0x1D3},{0x1D5,0x1D5},{0x1D7,0x1D7},{0x1D9,0x1D9},{0x1DB,0x1DB},{0x1DE,0x1DE}, + {0x1E0,0x1E0},{0x1E2,0x1E2},{0x1E4,0x1E4},{0x1E6,0x1E6},{0x1E8,0x1E8},{0x1EA,0x1EA}, + {0x1EC,0x1EC},{0x1EE,0x1EE},{0x1F1,0x1F2},{0x1F4,0x1F4},{0x1F6,0x1F8},{0x1FA,0x1FA}, + {0x1FC,0x1FC},{0x1FE,0x1FE},{0x200,0x200},{0x202,0x202},{0x204,0x204},{0x206,0x206}, + {0x208,0x208},{0x20A,0x20A},{0x20C,0x20C},{0x20E,0x20E},{0x210,0x210},{0x212,0x212}, + {0x214,0x214},{0x216,0x216},{0x218,0x218},{0x21A,0x21A},{0x21C,0x21C},{0x21E,0x21E}, + {0x220,0x220},{0x222,0x222},{0x224,0x224},{0x226,0x226},{0x228,0x228},{0x22A,0x22A}, + {0x22C,0x22C},{0x22E,0x22E},{0x230,0x230},{0x232,0x232},{0x23A,0x23B},{0x23D,0x23E}, + {0x241,0x241},{0x243,0x246},{0x248,0x248},{0x24A,0x24A},{0x24C,0x24C},{0x24E,0x24E}, + {0x370,0x370},{0x372,0x372},{0x376,0x376},{0x37F,0x37F},{0x386,0x386},{0x388,0x38A}, + {0x38C,0x38C},{0x38E,0x38F},{0x391,0x3A1},{0x3A3,0x3AB},{0x3CF,0x3CF},{0x3D2,0x3D4}, + {0x3D8,0x3D8},{0x3DA,0x3DA},{0x3DC,0x3DC},{0x3DE,0x3DE},{0x3E0,0x3E0},{0x3E2,0x3E2}, + {0x3E4,0x3E4},{0x3E6,0x3E6},{0x3E8,0x3E8},{0x3EA,0x3EA},{0x3EC,0x3EC},{0x3EE,0x3EE}, + {0x3F4,0x3F4},{0x3F7,0x3F7},{0x3F9,0x3FA},{0x3FD,0x42F},{0x460,0x460},{0x462,0x462}, + {0x464,0x464},{0x466,0x466},{0x468,0x468},{0x46A,0x46A},{0x46C,0x46C},{0x46E,0x46E}, + {0x470,0x470},{0x472,0x472},{0x474,0x474},{0x476,0x476},{0x478,0x478},{0x47A,0x47A}, + {0x47C,0x47C},{0x47E,0x47E},{0x480,0x480},{0x48A,0x48A},{0x48C,0x48C},{0x48E,0x48E}, + {0x490,0x490},{0x492,0x492},{0x494,0x494},{0x496,0x496},{0x498,0x498},{0x49A,0x49A}, + {0x49C,0x49C},{0x49E,0x49E},{0x4A0,0x4A0},{0x4A2,0x4A2},{0x4A4,0x4A4},{0x4A6,0x4A6}, + {0x4A8,0x4A8},{0x4AA,0x4AA},{0x4AC,0x4AC},{0x4AE,0x4AE},{0x4B0,0x4B0},{0x4B2,0x4B2}, + {0x4B4,0x4B4},{0x4B6,0x4B6},{0x4B8,0x4B8},{0x4BA,0x4BA},{0x4BC,0x4BC},{0x4BE,0x4BE}, + {0x4C0,0x4C1},{0x4C3,0x4C3},{0x4C5,0x4C5},{0x4C7,0x4C7},{0x4C9,0x4C9},{0x4CB,0x4CB}, + {0x4CD,0x4CD},{0x4D0,0x4D0},{0x4D2,0x4D2},{0x4D4,0x4D4},{0x4D6,0x4D6},{0x4D8,0x4D8}, + {0x4DA,0x4DA},{0x4DC,0x4DC},{0x4DE,0x4DE},{0x4E0,0x4E0},{0x4E2,0x4E2},{0x4E4,0x4E4}, + {0x4E6,0x4E6},{0x4E8,0x4E8},{0x4EA,0x4EA},{0x4EC,0x4EC},{0x4EE,0x4EE},{0x4F0,0x4F0}, + {0x4F2,0x4F2},{0x4F4,0x4F4},{0x4F6,0x4F6},{0x4F8,0x4F8},{0x4FA,0x4FA},{0x4FC,0x4FC}, + {0x4FE,0x4FE},{0x500,0x500},{0x502,0x502},{0x504,0x504},{0x506,0x506},{0x508,0x508}, + {0x50A,0x50A},{0x50C,0x50C},{0x50E,0x50E},{0x510,0x510},{0x512,0x512},{0x514,0x514}, + {0x516,0x516},{0x518,0x518},{0x51A,0x51A},{0x51C,0x51C},{0x51E,0x51E},{0x520,0x520}, + {0x522,0x522},{0x524,0x524},{0x526,0x526},{0x528,0x528},{0x52A,0x52A},{0x52C,0x52C}, + {0x52E,0x52E},{0x531,0x556},{0x10A0,0x10C5},{0x10C7,0x10C7},{0x10CD,0x10CD},{0x13A0,0x13F5}, + {0x1C90,0x1CBA},{0x1CBD,0x1CBF},{0x1E00,0x1E00},{0x1E02,0x1E02},{0x1E04,0x1E04},{0x1E06,0x1E06}, + {0x1E08,0x1E08},{0x1E0A,0x1E0A},{0x1E0C,0x1E0C},{0x1E0E,0x1E0E},{0x1E10,0x1E10},{0x1E12,0x1E12}, + {0x1E14,0x1E14},{0x1E16,0x1E16},{0x1E18,0x1E18},{0x1E1A,0x1E1A},{0x1E1C,0x1E1C},{0x1E1E,0x1E1E}, + {0x1E20,0x1E20},{0x1E22,0x1E22},{0x1E24,0x1E24},{0x1E26,0x1E26},{0x1E28,0x1E28},{0x1E2A,0x1E2A}, + {0x1E2C,0x1E2C},{0x1E2E,0x1E2E},{0x1E30,0x1E30},{0x1E32,0x1E32},{0x1E34,0x1E34},{0x1E36,0x1E36}, + {0x1E38,0x1E38},{0x1E3A,0x1E3A},{0x1E3C,0x1E3C},{0x1E3E,0x1E3E},{0x1E40,0x1E40},{0x1E42,0x1E42}, + {0x1E44,0x1E44},{0x1E46,0x1E46},{0x1E48,0x1E48},{0x1E4A,0x1E4A},{0x1E4C,0x1E4C},{0x1E4E,0x1E4E}, + {0x1E50,0x1E50},{0x1E52,0x1E52},{0x1E54,0x1E54},{0x1E56,0x1E56},{0x1E58,0x1E58},{0x1E5A,0x1E5A}, + {0x1E5C,0x1E5C},{0x1E5E,0x1E5E},{0x1E60,0x1E60},{0x1E62,0x1E62},{0x1E64,0x1E64},{0x1E66,0x1E66}, + {0x1E68,0x1E68},{0x1E6A,0x1E6A},{0x1E6C,0x1E6C},{0x1E6E,0x1E6E},{0x1E70,0x1E70},{0x1E72,0x1E72}, + {0x1E74,0x1E74},{0x1E76,0x1E76},{0x1E78,0x1E78},{0x1E7A,0x1E7A},{0x1E7C,0x1E7C},{0x1E7E,0x1E7E}, + {0x1E80,0x1E80},{0x1E82,0x1E82},{0x1E84,0x1E84},{0x1E86,0x1E86},{0x1E88,0x1E88},{0x1E8A,0x1E8A}, + {0x1E8C,0x1E8C},{0x1E8E,0x1E8E},{0x1E90,0x1E90},{0x1E92,0x1E92},{0x1E94,0x1E94},{0x1E9E,0x1E9E}, + {0x1EA0,0x1EA0},{0x1EA2,0x1EA2},{0x1EA4,0x1EA4},{0x1EA6,0x1EA6},{0x1EA8,0x1EA8},{0x1EAA,0x1EAA}, + {0x1EAC,0x1EAC},{0x1EAE,0x1EAE},{0x1EB0,0x1EB0},{0x1EB2,0x1EB2},{0x1EB4,0x1EB4},{0x1EB6,0x1EB6}, + {0x1EB8,0x1EB8},{0x1EBA,0x1EBA},{0x1EBC,0x1EBC},{0x1EBE,0x1EBE},{0x1EC0,0x1EC0},{0x1EC2,0x1EC2}, + {0x1EC4,0x1EC4},{0x1EC6,0x1EC6},{0x1EC8,0x1EC8},{0x1ECA,0x1ECA},{0x1ECC,0x1ECC},{0x1ECE,0x1ECE}, + {0x1ED0,0x1ED0},{0x1ED2,0x1ED2},{0x1ED4,0x1ED4},{0x1ED6,0x1ED6},{0x1ED8,0x1ED8},{0x1EDA,0x1EDA}, + {0x1EDC,0x1EDC},{0x1EDE,0x1EDE},{0x1EE0,0x1EE0},{0x1EE2,0x1EE2},{0x1EE4,0x1EE4},{0x1EE6,0x1EE6}, + {0x1EE8,0x1EE8},{0x1EEA,0x1EEA},{0x1EEC,0x1EEC},{0x1EEE,0x1EEE},{0x1EF0,0x1EF0},{0x1EF2,0x1EF2}, + {0x1EF4,0x1EF4},{0x1EF6,0x1EF6},{0x1EF8,0x1EF8},{0x1EFA,0x1EFA},{0x1EFC,0x1EFC},{0x1EFE,0x1EFE}, + {0x1F08,0x1F0F},{0x1F18,0x1F1D},{0x1F28,0x1F2F},{0x1F38,0x1F3F},{0x1F48,0x1F4D},{0x1F59,0x1F59}, + {0x1F5B,0x1F5B},{0x1F5D,0x1F5D},{0x1F5F,0x1F5F},{0x1F68,0x1F6F},{0x1F88,0x1F8F},{0x1F98,0x1F9F}, + {0x1FA8,0x1FAF},{0x1FB8,0x1FBC},{0x1FC8,0x1FCC},{0x1FD8,0x1FDB},{0x1FE8,0x1FEC},{0x1FF8,0x1FFC}, + {0x2102,0x2102},{0x2107,0x2107},{0x210B,0x210D},{0x2110,0x2112},{0x2115,0x2115},{0x2119,0x211D}, + {0x2124,0x2124},{0x2126,0x2126},{0x2128,0x2128},{0x212A,0x212D},{0x2130,0x2133},{0x213E,0x213F}, + {0x2145,0x2145},{0x2183,0x2183},{0x2C00,0x2C2E},{0x2C60,0x2C60},{0x2C62,0x2C64},{0x2C67,0x2C67}, + {0x2C69,0x2C69},{0x2C6B,0x2C6B},{0x2C6D,0x2C70},{0x2C72,0x2C72},{0x2C75,0x2C75},{0x2C7E,0x2C80}, + {0x2C82,0x2C82},{0x2C84,0x2C84},{0x2C86,0x2C86},{0x2C88,0x2C88},{0x2C8A,0x2C8A},{0x2C8C,0x2C8C}, + {0x2C8E,0x2C8E},{0x2C90,0x2C90},{0x2C92,0x2C92},{0x2C94,0x2C94},{0x2C96,0x2C96},{0x2C98,0x2C98}, + {0x2C9A,0x2C9A},{0x2C9C,0x2C9C},{0x2C9E,0x2C9E},{0x2CA0,0x2CA0},{0x2CA2,0x2CA2},{0x2CA4,0x2CA4}, + {0x2CA6,0x2CA6},{0x2CA8,0x2CA8},{0x2CAA,0x2CAA},{0x2CAC,0x2CAC},{0x2CAE,0x2CAE},{0x2CB0,0x2CB0}, + {0x2CB2,0x2CB2},{0x2CB4,0x2CB4},{0x2CB6,0x2CB6},{0x2CB8,0x2CB8},{0x2CBA,0x2CBA},{0x2CBC,0x2CBC}, + {0x2CBE,0x2CBE},{0x2CC0,0x2CC0},{0x2CC2,0x2CC2},{0x2CC4,0x2CC4},{0x2CC6,0x2CC6},{0x2CC8,0x2CC8}, + {0x2CCA,0x2CCA},{0x2CCC,0x2CCC},{0x2CCE,0x2CCE},{0x2CD0,0x2CD0},{0x2CD2,0x2CD2},{0x2CD4,0x2CD4}, + {0x2CD6,0x2CD6},{0x2CD8,0x2CD8},{0x2CDA,0x2CDA},{0x2CDC,0x2CDC},{0x2CDE,0x2CDE},{0x2CE0,0x2CE0}, + {0x2CE2,0x2CE2},{0x2CEB,0x2CEB},{0x2CED,0x2CED},{0x2CF2,0x2CF2},{0xA640,0xA640},{0xA642,0xA642}, + {0xA644,0xA644},{0xA646,0xA646},{0xA648,0xA648},{0xA64A,0xA64A},{0xA64C,0xA64C},{0xA64E,0xA64E}, + {0xA650,0xA650},{0xA652,0xA652},{0xA654,0xA654},{0xA656,0xA656},{0xA658,0xA658},{0xA65A,0xA65A}, + {0xA65C,0xA65C},{0xA65E,0xA65E},{0xA660,0xA660},{0xA662,0xA662},{0xA664,0xA664},{0xA666,0xA666}, + {0xA668,0xA668},{0xA66A,0xA66A},{0xA66C,0xA66C},{0xA680,0xA680},{0xA682,0xA682},{0xA684,0xA684}, + {0xA686,0xA686},{0xA688,0xA688},{0xA68A,0xA68A},{0xA68C,0xA68C},{0xA68E,0xA68E},{0xA690,0xA690}, + {0xA692,0xA692},{0xA694,0xA694},{0xA696,0xA696},{0xA698,0xA698},{0xA69A,0xA69A},{0xA722,0xA722}, + {0xA724,0xA724},{0xA726,0xA726},{0xA728,0xA728},{0xA72A,0xA72A},{0xA72C,0xA72C},{0xA72E,0xA72E}, + {0xA732,0xA732},{0xA734,0xA734},{0xA736,0xA736},{0xA738,0xA738},{0xA73A,0xA73A},{0xA73C,0xA73C}, + {0xA73E,0xA73E},{0xA740,0xA740},{0xA742,0xA742},{0xA744,0xA744},{0xA746,0xA746},{0xA748,0xA748}, + {0xA74A,0xA74A},{0xA74C,0xA74C},{0xA74E,0xA74E},{0xA750,0xA750},{0xA752,0xA752},{0xA754,0xA754}, + {0xA756,0xA756},{0xA758,0xA758},{0xA75A,0xA75A},{0xA75C,0xA75C},{0xA75E,0xA75E},{0xA760,0xA760}, + {0xA762,0xA762},{0xA764,0xA764},{0xA766,0xA766},{0xA768,0xA768},{0xA76A,0xA76A},{0xA76C,0xA76C}, + {0xA76E,0xA76E},{0xA779,0xA779},{0xA77B,0xA77B},{0xA77D,0xA77E},{0xA780,0xA780},{0xA782,0xA782}, + {0xA784,0xA784},{0xA786,0xA786},{0xA78B,0xA78B},{0xA78D,0xA78D},{0xA790,0xA790},{0xA792,0xA792}, + {0xA796,0xA796},{0xA798,0xA798},{0xA79A,0xA79A},{0xA79C,0xA79C},{0xA79E,0xA79E},{0xA7A0,0xA7A0}, + {0xA7A2,0xA7A2},{0xA7A4,0xA7A4},{0xA7A6,0xA7A6},{0xA7A8,0xA7A8},{0xA7AA,0xA7AE},{0xA7B0,0xA7B4}, + {0xA7B6,0xA7B6},{0xA7B8,0xA7B8},{0xA7BA,0xA7BA},{0xA7BC,0xA7BC},{0xA7BE,0xA7BE},{0xA7C2,0xA7C2}, + {0xA7C4,0xA7C7},{0xA7C9,0xA7C9},{0xA7F5,0xA7F5},{0xFF21,0xFF3A},{0x10400,0x10427},{0x104B0,0x104D3}, + {0x10C80,0x10CB2},{0x118A0,0x118BF},{0x16E40,0x16E5F},{0x1D400,0x1D419},{0x1D434,0x1D44D},{0x1D468,0x1D481}, + {0x1D49C,0x1D49C},{0x1D49E,0x1D49F},{0x1D4A2,0x1D4A2},{0x1D4A5,0x1D4A6},{0x1D4A9,0x1D4AC},{0x1D4AE,0x1D4B5}, + {0x1D4D0,0x1D4E9},{0x1D504,0x1D505},{0x1D507,0x1D50A},{0x1D50D,0x1D514},{0x1D516,0x1D51C},{0x1D538,0x1D539}, + {0x1D53B,0x1D53E},{0x1D540,0x1D544},{0x1D546,0x1D546},{0x1D54A,0x1D550},{0x1D56C,0x1D585},{0x1D5A0,0x1D5B9}, + {0x1D5D4,0x1D5ED},{0x1D608,0x1D621},{0x1D63C,0x1D655},{0x1D670,0x1D689},{0x1D6A8,0x1D6C0},{0x1D6E2,0x1D6FA}, + {0x1D71C,0x1D734},{0x1D756,0x1D76E},{0x1D790,0x1D7A8},{0x1D7CA,0x1D7CA},{0x1E900,0x1E921}, +}; +static const int uni_U_n = 641; + +static const uint32_t uni_X[][2] = { + {0xAA,0xAA},{0xBA,0xBA},{0x1BB,0x1BB},{0x1C0,0x1C3},{0x294,0x294},{0x2B0,0x2C1}, + {0x2C6,0x2D1},{0x2E0,0x2E4},{0x2EC,0x2EC},{0x2EE,0x2EE},{0x300,0x36F},{0x374,0x374}, + {0x37A,0x37A},{0x483,0x489},{0x559,0x559},{0x591,0x5BD},{0x5BF,0x5BF},{0x5C1,0x5C2}, + {0x5C4,0x5C5},{0x5C7,0x5C7},{0x5D0,0x5EA},{0x5EF,0x5F2},{0x610,0x61A},{0x620,0x65F}, + {0x66E,0x6D3},{0x6D5,0x6DC},{0x6DF,0x6E8},{0x6EA,0x6EF},{0x6FA,0x6FC},{0x6FF,0x6FF}, + {0x710,0x74A},{0x74D,0x7B1},{0x7CA,0x7F5},{0x7FA,0x7FA},{0x7FD,0x7FD},{0x800,0x82D}, + {0x840,0x85B},{0x860,0x86A},{0x8A0,0x8B4},{0x8B6,0x8C7},{0x8D3,0x8E1},{0x8E3,0x963}, + {0x971,0x983},{0x985,0x98C},{0x98F,0x990},{0x993,0x9A8},{0x9AA,0x9B0},{0x9B2,0x9B2}, + {0x9B6,0x9B9},{0x9BC,0x9C4},{0x9C7,0x9C8},{0x9CB,0x9CE},{0x9D7,0x9D7},{0x9DC,0x9DD}, + {0x9DF,0x9E3},{0x9F0,0x9F1},{0x9FC,0x9FC},{0x9FE,0x9FE},{0xA01,0xA03},{0xA05,0xA0A}, + {0xA0F,0xA10},{0xA13,0xA28},{0xA2A,0xA30},{0xA32,0xA33},{0xA35,0xA36},{0xA38,0xA39}, + {0xA3C,0xA3C},{0xA3E,0xA42},{0xA47,0xA48},{0xA4B,0xA4D},{0xA51,0xA51},{0xA59,0xA5C}, + {0xA5E,0xA5E},{0xA70,0xA75},{0xA81,0xA83},{0xA85,0xA8D},{0xA8F,0xA91},{0xA93,0xAA8}, + {0xAAA,0xAB0},{0xAB2,0xAB3},{0xAB5,0xAB9},{0xABC,0xAC5},{0xAC7,0xAC9},{0xACB,0xACD}, + {0xAD0,0xAD0},{0xAE0,0xAE3},{0xAF9,0xAFF},{0xB01,0xB03},{0xB05,0xB0C},{0xB0F,0xB10}, + {0xB13,0xB28},{0xB2A,0xB30},{0xB32,0xB33},{0xB35,0xB39},{0xB3C,0xB44},{0xB47,0xB48}, + {0xB4B,0xB4D},{0xB55,0xB57},{0xB5C,0xB5D},{0xB5F,0xB63},{0xB71,0xB71},{0xB82,0xB83}, + {0xB85,0xB8A},{0xB8E,0xB90},{0xB92,0xB95},{0xB99,0xB9A},{0xB9C,0xB9C},{0xB9E,0xB9F}, + {0xBA3,0xBA4},{0xBA8,0xBAA},{0xBAE,0xBB9},{0xBBE,0xBC2},{0xBC6,0xBC8},{0xBCA,0xBCD}, + {0xBD0,0xBD0},{0xBD7,0xBD7},{0xC00,0xC0C},{0xC0E,0xC10},{0xC12,0xC28},{0xC2A,0xC39}, + {0xC3D,0xC44},{0xC46,0xC48},{0xC4A,0xC4D},{0xC55,0xC56},{0xC58,0xC5A},{0xC60,0xC63}, + {0xC80,0xC83},{0xC85,0xC8C},{0xC8E,0xC90},{0xC92,0xCA8},{0xCAA,0xCB3},{0xCB5,0xCB9}, + {0xCBC,0xCC4},{0xCC6,0xCC8},{0xCCA,0xCCD},{0xCD5,0xCD6},{0xCDE,0xCDE},{0xCE0,0xCE3}, + {0xCF1,0xCF2},{0xD00,0xD0C},{0xD0E,0xD10},{0xD12,0xD44},{0xD46,0xD48},{0xD4A,0xD4E}, + {0xD54,0xD57},{0xD5F,0xD63},{0xD7A,0xD7F},{0xD81,0xD83},{0xD85,0xD96},{0xD9A,0xDB1}, + {0xDB3,0xDBB},{0xDBD,0xDBD},{0xDC0,0xDC6},{0xDCA,0xDCA},{0xDCF,0xDD4},{0xDD6,0xDD6}, + {0xDD8,0xDDF},{0xDF2,0xDF3},{0xE01,0xE3A},{0xE40,0xE4E},{0xE81,0xE82},{0xE84,0xE84}, + {0xE86,0xE8A},{0xE8C,0xEA3},{0xEA5,0xEA5},{0xEA7,0xEBD},{0xEC0,0xEC4},{0xEC6,0xEC6}, + {0xEC8,0xECD},{0xEDC,0xEDF},{0xF00,0xF00},{0xF18,0xF19},{0xF35,0xF35},{0xF37,0xF37}, + {0xF39,0xF39},{0xF3E,0xF47},{0xF49,0xF6C},{0xF71,0xF84},{0xF86,0xF97},{0xF99,0xFBC}, + {0xFC6,0xFC6},{0x1000,0x103F},{0x1050,0x108F},{0x109A,0x109D},{0x10FC,0x10FC},{0x1100,0x1248}, + {0x124A,0x124D},{0x1250,0x1256},{0x1258,0x1258},{0x125A,0x125D},{0x1260,0x1288},{0x128A,0x128D}, + {0x1290,0x12B0},{0x12B2,0x12B5},{0x12B8,0x12BE},{0x12C0,0x12C0},{0x12C2,0x12C5},{0x12C8,0x12D6}, + {0x12D8,0x1310},{0x1312,0x1315},{0x1318,0x135A},{0x135D,0x135F},{0x1380,0x138F},{0x1401,0x166C}, + {0x166F,0x167F},{0x1681,0x169A},{0x16A0,0x16EA},{0x16F1,0x16F8},{0x1700,0x170C},{0x170E,0x1714}, + {0x1720,0x1734},{0x1740,0x1753},{0x1760,0x176C},{0x176E,0x1770},{0x1772,0x1773},{0x1780,0x17D3}, + {0x17D7,0x17D7},{0x17DC,0x17DD},{0x180B,0x180D},{0x1820,0x1878},{0x1880,0x18AA},{0x18B0,0x18F5}, + {0x1900,0x191E},{0x1920,0x192B},{0x1930,0x193B},{0x1950,0x196D},{0x1970,0x1974},{0x1980,0x19AB}, + {0x19B0,0x19C9},{0x1A00,0x1A1B},{0x1A20,0x1A5E},{0x1A60,0x1A7C},{0x1A7F,0x1A7F},{0x1AA7,0x1AA7}, + {0x1AB0,0x1AC0},{0x1B00,0x1B4B},{0x1B6B,0x1B73},{0x1B80,0x1BAF},{0x1BBA,0x1BF3},{0x1C00,0x1C37}, + {0x1C4D,0x1C4F},{0x1C5A,0x1C7D},{0x1CD0,0x1CD2},{0x1CD4,0x1CFA},{0x1D2C,0x1D6A},{0x1D78,0x1D78}, + {0x1D9B,0x1DF9},{0x1DFB,0x1DFF},{0x2071,0x2071},{0x207F,0x207F},{0x2090,0x209C},{0x20D0,0x20F0}, + {0x2135,0x2138},{0x2C7C,0x2C7D},{0x2CEF,0x2CF1},{0x2D30,0x2D67},{0x2D6F,0x2D6F},{0x2D7F,0x2D96}, + {0x2DA0,0x2DA6},{0x2DA8,0x2DAE},{0x2DB0,0x2DB6},{0x2DB8,0x2DBE},{0x2DC0,0x2DC6},{0x2DC8,0x2DCE}, + {0x2DD0,0x2DD6},{0x2DD8,0x2DDE},{0x2DE0,0x2DFF},{0x2E2F,0x2E2F},{0x3005,0x3006},{0x302A,0x302F}, + {0x3031,0x3035},{0x303B,0x303C},{0x3041,0x3096},{0x3099,0x309A},{0x309D,0x309F},{0x30A1,0x30FA}, + {0x30FC,0x30FF},{0x3105,0x312F},{0x3131,0x318E},{0x31A0,0x31BF},{0x31F0,0x31FF},{0x3400,0x4DBF}, + {0x4E00,0x9FFC},{0xA000,0xA48C},{0xA4D0,0xA4FD},{0xA500,0xA60C},{0xA610,0xA61F},{0xA62A,0xA62B}, + {0xA66E,0xA672},{0xA674,0xA67D},{0xA67F,0xA67F},{0xA69C,0xA6E5},{0xA6F0,0xA6F1},{0xA717,0xA71F}, + {0xA770,0xA770},{0xA788,0xA788},{0xA78F,0xA78F},{0xA7F7,0xA7F9},{0xA7FB,0xA827},{0xA82C,0xA82C}, + {0xA840,0xA873},{0xA880,0xA8C5},{0xA8E0,0xA8F7},{0xA8FB,0xA8FB},{0xA8FD,0xA8FF},{0xA90A,0xA92D}, + {0xA930,0xA953},{0xA960,0xA97C},{0xA980,0xA9C0},{0xA9CF,0xA9CF},{0xA9E0,0xA9EF},{0xA9FA,0xA9FE}, + {0xAA00,0xAA36},{0xAA40,0xAA4D},{0xAA60,0xAA76},{0xAA7A,0xAAC2},{0xAADB,0xAADD},{0xAAE0,0xAAEF}, + {0xAAF2,0xAAF6},{0xAB01,0xAB06},{0xAB09,0xAB0E},{0xAB11,0xAB16},{0xAB20,0xAB26},{0xAB28,0xAB2E}, + {0xAB5C,0xAB5F},{0xAB69,0xAB69},{0xABC0,0xABEA},{0xABEC,0xABED},{0xAC00,0xD7A3},{0xD7B0,0xD7C6}, + {0xD7CB,0xD7FB},{0xF900,0xFA6D},{0xFA70,0xFAD9},{0xFB1D,0xFB28},{0xFB2A,0xFB36},{0xFB38,0xFB3C}, + {0xFB3E,0xFB3E},{0xFB40,0xFB41},{0xFB43,0xFB44},{0xFB46,0xFBB1},{0xFBD3,0xFD3D},{0xFD50,0xFD8F}, + {0xFD92,0xFDC7},{0xFDF0,0xFDFB},{0xFE00,0xFE0F},{0xFE20,0xFE2F},{0xFE70,0xFE74},{0xFE76,0xFEFC}, + {0xFF66,0xFFBE},{0xFFC2,0xFFC7},{0xFFCA,0xFFCF},{0xFFD2,0xFFD7},{0xFFDA,0xFFDC},{0x10000,0x1000B}, + {0x1000D,0x10026},{0x10028,0x1003A},{0x1003C,0x1003D},{0x1003F,0x1004D},{0x10050,0x1005D},{0x10080,0x100FA}, + {0x101FD,0x101FD},{0x10280,0x1029C},{0x102A0,0x102D0},{0x102E0,0x102E0},{0x10300,0x1031F},{0x1032D,0x10340}, + {0x10342,0x10349},{0x10350,0x1037A},{0x10380,0x1039D},{0x103A0,0x103C3},{0x103C8,0x103CF},{0x10450,0x1049D}, + {0x10500,0x10527},{0x10530,0x10563},{0x10600,0x10736},{0x10740,0x10755},{0x10760,0x10767},{0x10800,0x10805}, + {0x10808,0x10808},{0x1080A,0x10835},{0x10837,0x10838},{0x1083C,0x1083C},{0x1083F,0x10855},{0x10860,0x10876}, + {0x10880,0x1089E},{0x108E0,0x108F2},{0x108F4,0x108F5},{0x10900,0x10915},{0x10920,0x10939},{0x10980,0x109B7}, + {0x109BE,0x109BF},{0x10A00,0x10A03},{0x10A05,0x10A06},{0x10A0C,0x10A13},{0x10A15,0x10A17},{0x10A19,0x10A35}, + {0x10A38,0x10A3A},{0x10A3F,0x10A3F},{0x10A60,0x10A7C},{0x10A80,0x10A9C},{0x10AC0,0x10AC7},{0x10AC9,0x10AE6}, + {0x10B00,0x10B35},{0x10B40,0x10B55},{0x10B60,0x10B72},{0x10B80,0x10B91},{0x10C00,0x10C48},{0x10D00,0x10D27}, + {0x10E80,0x10EA9},{0x10EAB,0x10EAC},{0x10EB0,0x10EB1},{0x10F00,0x10F1C},{0x10F27,0x10F27},{0x10F30,0x10F50}, + {0x10FB0,0x10FC4},{0x10FE0,0x10FF6},{0x11000,0x11046},{0x1107F,0x110BA},{0x110D0,0x110E8},{0x11100,0x11134}, + {0x11144,0x11147},{0x11150,0x11173},{0x11176,0x11176},{0x11180,0x111C4},{0x111C9,0x111CC},{0x111CE,0x111CF}, + {0x111DA,0x111DA},{0x111DC,0x111DC},{0x11200,0x11211},{0x11213,0x11237},{0x1123E,0x1123E},{0x11280,0x11286}, + {0x11288,0x11288},{0x1128A,0x1128D},{0x1128F,0x1129D},{0x1129F,0x112A8},{0x112B0,0x112EA},{0x11300,0x11303}, + {0x11305,0x1130C},{0x1130F,0x11310},{0x11313,0x11328},{0x1132A,0x11330},{0x11332,0x11333},{0x11335,0x11339}, + {0x1133B,0x11344},{0x11347,0x11348},{0x1134B,0x1134D},{0x11350,0x11350},{0x11357,0x11357},{0x1135D,0x11363}, + {0x11366,0x1136C},{0x11370,0x11374},{0x11400,0x1144A},{0x1145E,0x11461},{0x11480,0x114C5},{0x114C7,0x114C7}, + {0x11580,0x115B5},{0x115B8,0x115C0},{0x115D8,0x115DD},{0x11600,0x11640},{0x11644,0x11644},{0x11680,0x116B8}, + {0x11700,0x1171A},{0x1171D,0x1172B},{0x11800,0x1183A},{0x118FF,0x11906},{0x11909,0x11909},{0x1190C,0x11913}, + {0x11915,0x11916},{0x11918,0x11935},{0x11937,0x11938},{0x1193B,0x11943},{0x119A0,0x119A7},{0x119AA,0x119D7}, + {0x119DA,0x119E1},{0x119E3,0x119E4},{0x11A00,0x11A3E},{0x11A47,0x11A47},{0x11A50,0x11A99},{0x11A9D,0x11A9D}, + {0x11AC0,0x11AF8},{0x11C00,0x11C08},{0x11C0A,0x11C36},{0x11C38,0x11C40},{0x11C72,0x11C8F},{0x11C92,0x11CA7}, + {0x11CA9,0x11CB6},{0x11D00,0x11D06},{0x11D08,0x11D09},{0x11D0B,0x11D36},{0x11D3A,0x11D3A},{0x11D3C,0x11D3D}, + {0x11D3F,0x11D47},{0x11D60,0x11D65},{0x11D67,0x11D68},{0x11D6A,0x11D8E},{0x11D90,0x11D91},{0x11D93,0x11D98}, + {0x11EE0,0x11EF6},{0x11FB0,0x11FB0},{0x12000,0x12399},{0x12480,0x12543},{0x13000,0x1342E},{0x14400,0x14646}, + {0x16800,0x16A38},{0x16A40,0x16A5E},{0x16AD0,0x16AED},{0x16AF0,0x16AF4},{0x16B00,0x16B36},{0x16B40,0x16B43}, + {0x16B63,0x16B77},{0x16B7D,0x16B8F},{0x16F00,0x16F4A},{0x16F4F,0x16F87},{0x16F8F,0x16F9F},{0x16FE0,0x16FE1}, + {0x16FE3,0x16FE4},{0x16FF0,0x16FF1},{0x17000,0x187F7},{0x18800,0x18CD5},{0x18D00,0x18D08},{0x1B000,0x1B11E}, + {0x1B150,0x1B152},{0x1B164,0x1B167},{0x1B170,0x1B2FB},{0x1BC00,0x1BC6A},{0x1BC70,0x1BC7C},{0x1BC80,0x1BC88}, + {0x1BC90,0x1BC99},{0x1BC9D,0x1BC9E},{0x1D165,0x1D169},{0x1D16D,0x1D172},{0x1D17B,0x1D182},{0x1D185,0x1D18B}, + {0x1D1AA,0x1D1AD},{0x1D242,0x1D244},{0x1DA00,0x1DA36},{0x1DA3B,0x1DA6C},{0x1DA75,0x1DA75},{0x1DA84,0x1DA84}, + {0x1DA9B,0x1DA9F},{0x1DAA1,0x1DAAF},{0x1E000,0x1E006},{0x1E008,0x1E018},{0x1E01B,0x1E021},{0x1E023,0x1E024}, + {0x1E026,0x1E02A},{0x1E100,0x1E12C},{0x1E130,0x1E13D},{0x1E14E,0x1E14E},{0x1E2C0,0x1E2EF},{0x1E800,0x1E8C4}, + {0x1E8D0,0x1E8D6},{0x1E944,0x1E94B},{0x1EE00,0x1EE03},{0x1EE05,0x1EE1F},{0x1EE21,0x1EE22},{0x1EE24,0x1EE24}, + {0x1EE27,0x1EE27},{0x1EE29,0x1EE32},{0x1EE34,0x1EE37},{0x1EE39,0x1EE39},{0x1EE3B,0x1EE3B},{0x1EE42,0x1EE42}, + {0x1EE47,0x1EE47},{0x1EE49,0x1EE49},{0x1EE4B,0x1EE4B},{0x1EE4D,0x1EE4F},{0x1EE51,0x1EE52},{0x1EE54,0x1EE54}, + {0x1EE57,0x1EE57},{0x1EE59,0x1EE59},{0x1EE5B,0x1EE5B},{0x1EE5D,0x1EE5D},{0x1EE5F,0x1EE5F},{0x1EE61,0x1EE62}, + {0x1EE64,0x1EE64},{0x1EE67,0x1EE6A},{0x1EE6C,0x1EE72},{0x1EE74,0x1EE77},{0x1EE79,0x1EE7C},{0x1EE7E,0x1EE7E}, + {0x1EE80,0x1EE89},{0x1EE8B,0x1EE9B},{0x1EEA1,0x1EEA3},{0x1EEA5,0x1EEA9},{0x1EEAB,0x1EEBB},{0x20000,0x2A6DD}, + {0x2A700,0x2B734},{0x2B740,0x2B81D},{0x2B820,0x2CEA1},{0x2CEB0,0x2EBE0},{0x2F800,0x2FA1D},{0x30000,0x3134A}, + {0xE0100,0xE01EF}, +}; +static const int uni_X_n = 595; + +static inline int is_U(uint32_t c){ return uni_in(uni_U,uni_U_n,c); } +static inline int is_X(uint32_t c){ return uni_in(uni_X,uni_X_n,c); } +#endif From 9e2d41567c5e0902da1ef5a3daaff9a957d83997 Mon Sep 17 00:00:00 2001 From: Nicholas Beerbower Date: Thu, 16 Jul 2026 15:17:21 -0400 Subject: [PATCH 013/132] tests: o200k tokenizer coverage, no model download MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit tests/tok_o200k_tiny.json is a synthetic byte-level BPE (274 vocab, a few KB) whose Split regex is the o200k pattern; expected ids in tok_o200k_cases.txt were generated by HF tokenizers on that same file. test_tok_o200k (in TEST_BINS) scores 40/40 encode + 40/40 round-trip: case-transition splits, contractions, digit groups, the [\r\n/]* tail, whitespace branches, CJK/Greek/Cyrillic, added-token atomicity. The cl100k path is untouched by construction — dispatch requires \p{Lu} in the tokenizer's own Split pattern, which cl100k lacks — and stays covered by the GLM oracle (verified on this branch: 32/32). Co-Authored-By: Claude Fable 5 --- c/Makefile | 5 ++- c/tests/test_tok_o200k.c | 61 +++++++++++++++++++++++++++++++++++++ c/tests/tok_o200k_cases.txt | 40 ++++++++++++++++++++++++ c/tests/tok_o200k_tiny.json | 1 + 4 files changed, 106 insertions(+), 1 deletion(-) create mode 100644 c/tests/test_tok_o200k.c create mode 100644 c/tests/tok_o200k_cases.txt create mode 100644 c/tests/tok_o200k_tiny.json diff --git a/c/Makefile b/c/Makefile index 27902cb2..9887c74f 100644 --- a/c/Makefile +++ b/c/Makefile @@ -166,7 +166,7 @@ else PYTHON ?= python3 endif CUDA_OBJ = -TEST_BINS = tests/test_json$(EXE) tests/test_st$(EXE) tests/test_tier$(EXE) tests/test_grammar$(EXE) tests/test_schema_gbnf$(EXE) tests/test_decode_batch$(EXE) tests/test_idot$(EXE) tests/test_i4_grouped$(EXE) tests/test_stops$(EXE) tests/test_kv_alloc$(EXE) tests/test_i4_acc512$(EXE) tests/test_compat_direct$(EXE) +TEST_BINS = tests/test_json$(EXE) tests/test_st$(EXE) tests/test_tier$(EXE) tests/test_grammar$(EXE) tests/test_schema_gbnf$(EXE) tests/test_decode_batch$(EXE) tests/test_idot$(EXE) tests/test_i4_grouped$(EXE) tests/test_stops$(EXE) tests/test_tok_o200k$(EXE) tests/test_kv_alloc$(EXE) tests/test_i4_acc512$(EXE) tests/test_compat_direct$(EXE) ifneq (,$(LINUX)) TEST_BINS += tests/test_uring$(EXE) endif @@ -293,6 +293,9 @@ iobench$(EXE): iobench.c compat.h tests/test_json$(EXE): tests/test_json.c json.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) +tests/test_tok_o200k$(EXE): tests/test_tok_o200k.c tok.h tok_unicode.h tok_unicode_o200k.h json.h + $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) + tests/test_st$(EXE): tests/test_st.c st.h json.h compat.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) diff --git a/c/tests/test_tok_o200k.c b/c/tests/test_tok_o200k.c new file mode 100644 index 00000000..b86bda90 --- /dev/null +++ b/c/tests/test_tok_o200k.c @@ -0,0 +1,61 @@ +/* o200k pre-tokenizer validation against HF-tokenizers-generated expectations. + * Self-contained for the test-c harness: loads tests/tok_o200k_tiny.json (a + * synthetic byte-level BPE whose Split regex is the o200k pattern — a few KB, + * no model download) and scores tests/tok_o200k_cases.txt, whose expected ids + * were produced by HF `tokenizers` on the same file. Guards the case-aware + * letter matcher, contractions, digit groups, the [\r\n/]* punctuation tail, + * whitespace branches, and added-token atomicity; round-trips every case. + * The cl100k path is untouched by construction (dispatch requires \p{Lu} in + * the tokenizer's own Split pattern) and stays covered by the GLM oracle. */ +#define _GNU_SOURCE +#include "../tok.h" + +int main(void) { + Tok T; + tok_load(&T, "tests/tok_o200k_tiny.json"); + if (!T.o200k) { fprintf(stderr, "test_tok_o200k: o200k pattern not detected\n"); return 1; } + FILE *f = fopen("tests/tok_o200k_cases.txt", "rb"); + if (!f) { perror("tests/tok_o200k_cases.txt"); return 1; } + char *line = NULL; size_t cap = 0; ssize_t nr; + int pass = 0, tot = 0, dpass = 0; + while ((nr = getline(&line, &cap, f)) >= 0) { + if (nr > 0 && line[nr-1] == '\n') line[--nr] = 0; + if (nr == 0) continue; + char *tab = strchr(line, '\t'); if (!tab) continue; + *tab = 0; + const char *text = line, *idstr = tab + 1; + char tbuf[4096]; int tn = 0; + for (const char *q = text; *q && tn < 4095; q++) { + if (q[0]=='\\' && q[1]=='n') { tbuf[tn++]='\n'; q++; } + else if (q[0]=='\\' && q[1]=='t') { tbuf[tn++]='\t'; q++; } + else if (q[0]=='\\' && q[1]=='r') { tbuf[tn++]='\r'; q++; } + else if (q[0]=='\\' && q[1]=='\\') { tbuf[tn++]='\\'; q++; } + else tbuf[tn++] = *q; + } + tbuf[tn] = 0; + int exp[512], ne = 0; + for (const char *q = idstr; *q; ) { + while (*q == ',' || *q == ' ') q++; + if (!*q) break; + exp[ne++] = atoi(q); + while (*q && *q != ',') q++; + } + int got[512]; int ng = tok_encode(&T, tbuf, tn, got, 512); + int ok = (ng == ne); + for (int i = 0; i < ng && ok; i++) ok = (got[i] == exp[i]); + tot++; if (ok) pass++; + char dec[8192]; int dn = tok_decode(&T, got, ng, dec, 8191); + int drt = (dn == tn) && !memcmp(dec, tbuf, tn); + if (drt) dpass++; + if (!ok || !drt) { + fprintf(stderr, "MISMATCH text=%s\n exp(%d):", text, ne); + for (int i = 0; i < ne; i++) fprintf(stderr, " %d", exp[i]); + fprintf(stderr, "\n got(%d):", ng); + for (int i = 0; i < ng; i++) fprintf(stderr, " %d", got[i]); + fprintf(stderr, "\n decode_ok=%d\n", drt); + } + } + fclose(f); + printf("test_tok_o200k: ENCODE %d/%d DECODE %d/%d\n", pass, tot, dpass, tot); + return (pass == tot && dpass == tot) ? 0 : 2; +} diff --git a/c/tests/tok_o200k_cases.txt b/c/tests/tok_o200k_cases.txt new file mode 100644 index 00000000..38e37fa5 --- /dev/null +++ b/c/tests/tok_o200k_cases.txt @@ -0,0 +1,40 @@ +hello world 259,32,119,111,114,263 +HelloWorld 72,101,257,111,262 +XMLHttpRequest 88,77,76,72,116,116,112,82,101,113,117,101,115,116 +helloWORLDhello 259,87,79,82,76,68,259 +dog's 100,111,103,270 +DON'T 68,79,78,39,84 +don't 100,111,110,39,116 +I'll've 73,39,257,39,118,101 +O'Brien 79,39,66,114,105,101,110 +the theatre 265,32,116,256,97,116,114,101 +12345 268,52,53 +a1b22c333d4444 97,49,98,50,50,99,51,51,51,100,52,52,52,52 +3.14 51,46,49,52 +http://x.com/a/b 104,116,116,112,58,47,47,120,46,99,111,109,47,97,47,98 +path/to/file 112,97,264,47,116,111,47,102,105,108,101 +a//b///c 97,47,47,98,47,47,47,99 +!!\r\n//x 33,33,13,10,47,47,120 +one\ntwo\r\nthree 111,110,101,10,116,119,111,13,10,264,114,101,101 + \n x 32,32,10,32,32,120 + 32,32,32 +a b c 97,32,32,98,32,32,32,99 +tab\there 116,269,9,256,114,101 +Café 67,97,102,101,204,129 +naiveBayes 110,97,105,118,101,66,97,121,101,115 +Éclair 195,137,99,108,97,105,114 +北京大学 229,140,151,228,186,172,229,164,167,229,173,166 +ΑΒαβ 206,145,206,146,206,177,206,178 +Иван 208,152,208,178,208,176,208,189 +ẞßscharf 225,186,158,195,159,115,99,104,97,114,102 +i̇stanbul 105,204,135,115,116,97,110,98,117,108 +hello<|endoftext|>world 259,274,119,111,114,263 +<|message_user|>hi 275,104,105 +mixedCASEand123 109,105,120,101,100,67,65,83,69,97,110,100,268 +'s 270 + 's 32,39,115 +A 65 +aB 97,66 +Ab 65,98 +AB 65,66 +ab 269 diff --git a/c/tests/tok_o200k_tiny.json b/c/tests/tok_o200k_tiny.json new file mode 100644 index 00000000..829301ec --- /dev/null +++ b/c/tests/tok_o200k_tiny.json @@ -0,0 +1 @@ +{"version": "1.0", "truncation": null, "padding": null, "added_tokens": [{"id": 274, "content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 275, "content": "<|message_user|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}], "normalizer": null, "pre_tokenizer": {"type": "Sequence", "pretokenizers": [{"type": "Split", "pattern": {"Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"}, "behavior": "Isolated", "invert": false}, {"type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false}]}, "post_processor": null, "decoder": {"type": "ByteLevel", "add_prefix_space": true, "trim_offsets": true, "use_regex": true}, "model": {"type": "BPE", "dropout": null, "unk_token": null, "continuing_subword_prefix": null, "end_of_word_suffix": null, "fuse_unk": false, "byte_fallback": false, "ignore_merges": true, "vocab": {"Ā": 0, "ā": 1, "Ă": 2, "ă": 3, "Ą": 4, "ą": 5, "Ć": 6, "ć": 7, "Ĉ": 8, "ĉ": 9, "Ċ": 10, "ċ": 11, "Č": 12, "č": 13, "Ď": 14, "ď": 15, "Đ": 16, "đ": 17, "Ē": 18, "ē": 19, "Ĕ": 20, "ĕ": 21, "Ė": 22, "ė": 23, "Ę": 24, "ę": 25, "Ě": 26, "ě": 27, "Ĝ": 28, "ĝ": 29, "Ğ": 30, "ğ": 31, "Ġ": 32, "!": 33, "\"": 34, "#": 35, "$": 36, "%": 37, "&": 38, "'": 39, "(": 40, ")": 41, "*": 42, "+": 43, ",": 44, "-": 45, ".": 46, "/": 47, "0": 48, "1": 49, "2": 50, "3": 51, "4": 52, "5": 53, "6": 54, "7": 55, "8": 56, "9": 57, ":": 58, ";": 59, "<": 60, "=": 61, ">": 62, "?": 63, "@": 64, "A": 65, "B": 66, "C": 67, "D": 68, "E": 69, "F": 70, "G": 71, "H": 72, "I": 73, "J": 74, "K": 75, "L": 76, "M": 77, "N": 78, "O": 79, "P": 80, "Q": 81, "R": 82, "S": 83, "T": 84, "U": 85, "V": 86, "W": 87, "X": 88, "Y": 89, "Z": 90, "[": 91, "\\": 92, "]": 93, "^": 94, "_": 95, "`": 96, "a": 97, "b": 98, "c": 99, "d": 100, "e": 101, "f": 102, "g": 103, "h": 104, "i": 105, "j": 106, "k": 107, "l": 108, "m": 109, "n": 110, "o": 111, "p": 112, "q": 113, "r": 114, "s": 115, "t": 116, "u": 117, "v": 118, "w": 119, "x": 120, "y": 121, "z": 122, "{": 123, "|": 124, "}": 125, "~": 126, "ġ": 127, "Ģ": 128, "ģ": 129, "Ĥ": 130, "ĥ": 131, "Ħ": 132, "ħ": 133, "Ĩ": 134, "ĩ": 135, "Ī": 136, "ī": 137, "Ĭ": 138, "ĭ": 139, "Į": 140, "į": 141, "İ": 142, "ı": 143, "IJ": 144, "ij": 145, "Ĵ": 146, "ĵ": 147, "Ķ": 148, "ķ": 149, "ĸ": 150, "Ĺ": 151, "ĺ": 152, "Ļ": 153, "ļ": 154, "Ľ": 155, "ľ": 156, "Ŀ": 157, "ŀ": 158, "Ł": 159, "ł": 160, "¡": 161, "¢": 162, "£": 163, "¤": 164, "¥": 165, "¦": 166, "§": 167, "¨": 168, "©": 169, "ª": 170, "«": 171, "¬": 172, "Ń": 173, "®": 174, "¯": 175, "°": 176, "±": 177, "²": 178, "³": 179, "´": 180, "µ": 181, "¶": 182, "·": 183, "¸": 184, "¹": 185, "º": 186, "»": 187, "¼": 188, "½": 189, "¾": 190, "¿": 191, "À": 192, "Á": 193, "Â": 194, "Ã": 195, "Ä": 196, "Å": 197, "Æ": 198, "Ç": 199, "È": 200, "É": 201, "Ê": 202, "Ë": 203, "Ì": 204, "Í": 205, "Î": 206, "Ï": 207, "Ð": 208, "Ñ": 209, "Ò": 210, "Ó": 211, "Ô": 212, "Õ": 213, "Ö": 214, "×": 215, "Ø": 216, "Ù": 217, "Ú": 218, "Û": 219, "Ü": 220, "Ý": 221, "Þ": 222, "ß": 223, "à": 224, "á": 225, "â": 226, "ã": 227, "ä": 228, "å": 229, "æ": 230, "ç": 231, "è": 232, "é": 233, "ê": 234, "ë": 235, "ì": 236, "í": 237, "î": 238, "ï": 239, "ð": 240, "ñ": 241, "ò": 242, "ó": 243, "ô": 244, "õ": 245, "ö": 246, "÷": 247, "ø": 248, "ù": 249, "ú": 250, "û": 251, "ü": 252, "ý": 253, "þ": 254, "ÿ": 255, "he": 256, "ll": 257, "hell": 258, "hello": 259, "Wo": 260, "Wor": 261, "World": 262, "ld": 263, "th": 264, "the": 265, "Ġthe": 266, "12": 267, "123": 268, "ab": 269, "'s": 270, "Ġa": 271, "./": 272, "ĊĊ": 273}, "merges": [["h", "e"], ["l", "l"], ["he", "ll"], ["hell", "o"], ["W", "o"], ["Wo", "r"], ["Wor", "ld"], ["l", "d"], ["t", "h"], ["th", "e"], ["Ġ", "the"], ["1", "2"], ["12", "3"], ["a", "b"], ["'", "s"], ["Ġ", "a"], [".", "/"], ["Ċ", "Ċ"]]}} \ No newline at end of file From 5ad4d540ab7427e6ca134ce88fe93e3d9f869456 Mon Sep 17 00:00:00 2001 From: Nicholas Beerbower Date: Thu, 16 Jul 2026 16:33:22 -0400 Subject: [PATCH 014/132] test_tok_o200k: fgets instead of getline for the windows job MinGW's UCRT has no getline; fixed-buffer fgets with CRLF trimming reads the same case file everywhere. Co-Authored-By: Claude Fable 5 --- c/tests/test_tok_o200k.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/c/tests/test_tok_o200k.c b/c/tests/test_tok_o200k.c index b86bda90..fe628908 100644 --- a/c/tests/test_tok_o200k.c +++ b/c/tests/test_tok_o200k.c @@ -16,10 +16,13 @@ int main(void) { if (!T.o200k) { fprintf(stderr, "test_tok_o200k: o200k pattern not detected\n"); return 1; } FILE *f = fopen("tests/tok_o200k_cases.txt", "rb"); if (!f) { perror("tests/tok_o200k_cases.txt"); return 1; } - char *line = NULL; size_t cap = 0; ssize_t nr; + /* fgets, not getline: MinGW's UCRT lacks getline and this must run on + * the windows job. Case lines are short; 8 KB is generous. */ + char line[8192]; int pass = 0, tot = 0, dpass = 0; - while ((nr = getline(&line, &cap, f)) >= 0) { - if (nr > 0 && line[nr-1] == '\n') line[--nr] = 0; + while (fgets(line, sizeof(line), f)) { + size_t nr = strlen(line); + while (nr > 0 && (line[nr-1] == '\n' || line[nr-1] == '\r')) line[--nr] = 0; if (nr == 0) continue; char *tab = strchr(line, '\t'); if (!tab) continue; *tab = 0; From d52e76144781c78cd0a225af39ebe651e4aab139 Mon Sep 17 00:00:00 2001 From: noobdev-ph Date: Fri, 17 Jul 2026 02:35:34 +0800 Subject: [PATCH 015/132] =?UTF-8?q?feat:=20AMD=20GPU=20support=20via=20HIP?= =?UTF-8?q?/ROCm=20=E2=80=94=20single-source=20backend=20through=20backend?= =?UTF-8?q?=5Fgpu=5Fcompat.h?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit backend_cuda.cu compiles unchanged for both vendors: under nvcc the new header passes through to cuda_runtime.h + mma.h (preprocessed source is byte-identical to before this commit); under hipcc (make HIP=1) it maps the CUDA runtime surface onto HIP 1:1. Same one-shim-header pattern compat.h uses for the Windows port. The WMMA tensor-core dispatch is compile-gated via COLI_GPU_HAS_WMMA: gfx GPUs report compute_major >= 7 (gfx1201 reports 12), so the runtime check alone would select the __CUDA_ARCH__-guarded kernels' EMPTY bodies under HIP and return garbage silently. HIP uses the portable kernels; rocWMMA matrix-core support is a natural follow-up. Build: make HIP=1 [HIP_ARCH=native|gfxXXXX], make hip-test, make gpu-compile (CI compile-only target). HIP joins the .build-config stamp so HIP<->CPU flag flips relink correctly. CI: engine-hip-syntax job mirrors engine-cuda-syntax (rocm/dev container, pinned tag). Verified on RX 9070 XT (gfx1201, ROCm 7.2.4): hip-test passes the full upstream suite incl. tensor_update; GLM-5.2 end-to-end measures 0.32 tok/s at 61% expert hit with CUDA_RELEASE_HOST=1 (CPU: 0.22-0.27). Runtime details and hardware matrix: GPU_BACKENDS.md. --- .github/workflows/ci.yml | 21 ++++++++++ GPU_BACKENDS.md | 85 ++++++++++++++++++++++++++++++++++++++++ c/Makefile | 58 +++++++++++++++++++++------ c/backend_cuda.cu | 7 ++-- c/backend_gpu_compat.h | 62 +++++++++++++++++++++++++++++ 5 files changed, 218 insertions(+), 15 deletions(-) create mode 100644 GPU_BACKENDS.md create mode 100644 c/backend_gpu_compat.h diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e18f1b3e..500f6649 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,6 +41,27 @@ jobs: -Xcompiler=-Wall,-Wextra echo "CUDA syntax check passed" + engine-hip-syntax: + name: HIP syntax check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Free runner disk + # rocm/dev images are large; hosted runners need the headroom freed + # BEFORE the pull (which is why this is not a container: job). + run: sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL + - name: Compile backend_cuda.cu + test binary with hipcc (no GPU) + run: | + # Tag pinned: `latest` outgrew the runner disk once already. The + # gcc-side engine link is skipped: containerized gcc/ld cannot link + # ROCm runtime libs (both 22.04 and 24.04 bases); the hipcc-driver + # link of the test binary below covers the single-source claim, and + # the engine link is exercised on real ROCm hardware (GPU_BACKENDS.md). + docker run --rm -v "$PWD":/src -w /src rocm/dev-ubuntu-24.04:6.2 bash -c ' + apt-get update && apt-get install -y --no-install-recommends make gcc g++ && + make -C c gpu-compile HIP=1 HIP_ARCH=gfx1100' + echo "HIP syntax check passed" + web: name: Web UI runs-on: ubuntu-latest diff --git a/GPU_BACKENDS.md b/GPU_BACKENDS.md new file mode 100644 index 00000000..be421bfb --- /dev/null +++ b/GPU_BACKENDS.md @@ -0,0 +1,85 @@ +# GPU backends: CUDA and HIP/ROCm + +colibrì's GPU expert backend is **one source file** (`c/backend_cuda.cu`) compiled +for either vendor through `c/backend_gpu_compat.h` — the same one-shim-header +pattern `compat.h` uses for the Windows port. Compiled by nvcc the shim is a +pass-through to `cuda_runtime.h` (the NVIDIA path is byte-identical to the +pre-HIP tree); compiled by hipcc it maps the 14-symbol CUDA runtime surface the +backend uses onto HIP 1:1. The kernels use only shared syntax +(`__global__`, `__shared__`, `__syncthreads__`, `<<<>>>`), no vendor intrinsics. + +**Rule for contributors:** vendor differences go in `backend_gpu_compat.h` +only — never `#ifdef __HIP__` (or CUDA-specific code) in `backend_cuda.cu`. + +## Supported environments + +| backend | platform | toolchain | build | +|---|---|---|---| +| CUDA (`CUDA=1`) | Linux x86-64 | CUDA toolkit (nvcc), `CUDA_HOME=/usr/local/cuda` default | `make -C c glm CUDA=1 [CUDA_ARCH=native\|sm_XX]` | +| HIP (`HIP=1`) | Linux x86-64 | ROCm (hipcc), `ROCM_HOME=/opt/rocm` default; tested on ROCm 7.2 | `make -C c glm HIP=1 [HIP_ARCH=native\|gfxXXXX]` | + +`CUDA=1` and `HIP=1` are mutually exclusive and both opt-in: the default build +remains pure, dependency-free CPU. Both are refused on non-Linux with an early +`$(error)`. `*_ARCH=native` targets the local GPU; pass an explicit arch when +distributing or on machines with an unsupported iGPU visible to the runtime +(and mask iGPUs at runtime with `HIP_VISIBLE_DEVICES=` on ROCm). + +## Runtime configuration (identical for both vendors) + +- `COLI_CUDA=1` + `COLI_GPU=N` (or `COLI_GPUS=0,1,...`) — enable, select devices +- `CUDA_EXPERT_GB=G` — VRAM budget for the expert tier (clamped to free VRAM + minus projected dense set and 2 GB headroom per device) +- `CUDA_RELEASE_HOST=1` — GPU-tier experts drop their host backing after + upload (default on multi-GPU); combined with `PIN=auto`/`PIN_FILL`, VRAM + becomes additional pinned capacity at zero RAM cost. The engine + rematerializes an expert from disk (`expert_host_ensure`) whenever the CPU + path needs one whose host copy was released — validated under total GPU + failure below. +- `CUDA_DENSE=1` — experimental resident-dense path (unchanged) +- `COLI_CUDA_TC_W4A16=1` — opt-in W4A16 tensor-core path. **NVIDIA-only**: + the WMMA kernels are compile-gated (`COLI_GPU_HAS_WMMA` in the compat + header) because gfx GPUs report `compute_major >= 7` and a runtime check + alone would select empty kernel bodies under HIP. On AMD, all compute uses + the portable kernels; rocWMMA matrix-core support is a possible follow-up. + +## Validation + +### Unit tests (run on GPU hardware) + +```sh +make -C c cuda-test [CUDA_ARCH=...] # NVIDIA +make -C c hip-test [HIP_ARCH=...] # AMD (same test source) +``` + +Covers q8/q4/q2/f32 matmul correctness, multi-device placement/stats, and +`tensor_update` — the standard upstream suite, unchanged, compiled by hipcc. +(A companion PR adds failure-path tests for the backend; they are +vendor-neutral and run under `hip-test` identically.) + +### CI (no GPU required) + +The `engine-hip-syntax` job in `.github/workflows/ci.yml` compiles the +backend and its test binary with hipcc (`rocm/dev` container pinned to +`6.2`, `gfx1100`) on every PR, mirroring `engine-cuda-syntax`. Kernel +*execution* is not possible on hosted runners; that is what `hip-test` +on real hardware is for (matrix below). + +### Hardware test matrix (documented results) + +| environment | result | +|---|---| +| AMD RX 9070 XT (gfx1201), ROCm 7.2.4, Linux 7.0 | `hip-test` **pass** (all cases above); GLM-5.2 end-to-end runs (0.32 tok/s @ 61% expert hit with CUDA_RELEASE_HOST=1); benchmark series in PR #112 | +| NVIDIA | compile-verified in CI (`sm_80`); nvcc path is a pass-through include — **runtime run of `make cuda-test` on NVIDIA hardware welcomed**, the test source is vendor-neutral | + +## Known behavior notes + +- GPU float matmuls round differently than the CPU int8-dot (IDOT) kernels: + greedy output is **not token-identical** across backends (consistent with + the shape-dependence documented in #100), and MTP draft acceptance measures + lower on GPU-heavy configs (~40% → ~31% on the PR #112 machine). A + numerics-matched integer GPU kernel is the planned follow-up. +- An earlier revision of this branch carried `CUDA_EXTEND=1` (VRAM tier + holding experts beyond the RAM pin). It was superseded by upstream's + `PIN=auto` + `PIN_FILL` + `CUDA_RELEASE_HOST`, which achieve the same + capacity extension with deeper engine integration; this branch's safety + and validation work now targets that mechanism. diff --git a/c/Makefile b/c/Makefile index 27902cb2..4264d906 100644 --- a/c/Makefile +++ b/c/Makefile @@ -154,6 +154,18 @@ NVCCFLAGS ?= -O3 -std=c++17 -arch=$(CUDA_ARCH) -Xcompiler=-W3 else NVCCFLAGS ?= -O3 -std=c++17 -arch=$(CUDA_ARCH) -Xcompiler=-Wall,-Wextra endif +# HIP=1 builds the SAME backend for AMD GPUs via ROCm: backend_cuda.cu is +# compiled unchanged through backend_gpu_compat.h (one source, two vendors, +# like compat.h does for Windows). HIP_ARCH=native targets the GPU in this +# machine; set an explicit arch (e.g. HIP_ARCH=gfx1201) when distributing. +HIP ?= 0 +ROCM_HOME ?= /opt/rocm +HIPCC ?= $(ROCM_HOME)/bin/hipcc +HIP_ARCH ?= native +HIPCCFLAGS ?= -O3 -std=c++17 -x hip --offload-arch=$(HIP_ARCH) -Wall -Wextra +GPUCC = $(NVCC) +GPUFLAGS = $(NVCCFLAGS) +GPUCC_NAME = nvcc (set CUDA_HOME or NVCC) # PYTHON is a HOST tool (clean/test-c/test-python run it on the build machine), # so key it off the host, NOT $(IS_WIN) — which is derived from the *target* # triple ($(CC) -dumpmachine). Otherwise a cross build (make CC=x86_64-w64- @@ -188,11 +200,26 @@ ifneq ($(IS_WIN),) # On Windows use CUDA_DLL=1 (runtime DLL), not CUDA=1 (direct link). $(error On Windows use: make CUDA_DLL=1 cuda-dll (see backend_loader.c)) endif +ifeq ($(HIP),1) +$(error choose CUDA=1 or HIP=1, not both) +endif CFLAGS += -DCOLI_CUDA LDFLAGS += -L$(CUDA_HOME)/lib64 -Wl,-rpath,$(CUDA_HOME)/lib64 -lcudart -lstdc++ CUDA_OBJ = backend_cuda.o endif +ifeq ($(HIP),1) +ifeq (,$(LINUX)) +$(error HIP=1 is supported only on Linux) +endif +CFLAGS += -DCOLI_CUDA +LDFLAGS += -L$(ROCM_HOME)/lib -Wl,-rpath,$(ROCM_HOME)/lib -lamdhip64 -lstdc++ +CUDA_OBJ = backend_cuda.o +GPUCC = $(HIPCC) +GPUFLAGS = $(HIPCCFLAGS) +GPUCC_NAME = hipcc (set ROCM_HOME or HIPCC) +endif + # METAL=1 adds an opt-in Apple-GPU backend (macOS only). The shader is compiled at # runtime, so no Xcode / offline metal compiler is required. Default build unchanged. METAL ?= 0 @@ -219,7 +246,7 @@ glm: glm$(EXE) # it ONLY when they change (evaluated here at parse time, so the file's timestamp # moves exactly when the config moves). glm.exe and the CUDA/loader objects depend # on it, so they relink on a config change and stay put otherwise. (#306) -BUILD_CONFIG := $(CC)|$(CFLAGS)|$(LDFLAGS)|CUDA=$(CUDA)|CUDA_DLL=$(CUDA_DLL)|ARCH=$(ARCH)|CUDA_ARCH=$(CUDA_ARCH)|METAL=$(METAL) +BUILD_CONFIG := $(CC)|$(CFLAGS)|$(LDFLAGS)|CUDA=$(CUDA)|CUDA_DLL=$(CUDA_DLL)|ARCH=$(ARCH)|CUDA_ARCH=$(CUDA_ARCH)|METAL=$(METAL)|HIP=$(HIP)|HIP_ARCH=$(HIP_ARCH) BUILD_CONFIG_OLD := $(shell cat .build-config 2>/dev/null) ifneq "$(BUILD_CONFIG)" "$(BUILD_CONFIG_OLD)" $(shell printf '%s' '$(BUILD_CONFIG)' > .build-config) @@ -245,9 +272,9 @@ cuda-dll: backend_cuda.cu backend_cuda.h -L"$(CUDA_HOME)/lib/x64" -lcudart \ backend_cuda.cu -o coli_cuda.dll -backend_cuda.o: backend_cuda.cu backend_cuda.h .build-config - @command -v "$(NVCC)" >/dev/null 2>&1 || { echo "nvcc not found: set CUDA_HOME or NVCC" >&2; exit 1; } - "$(NVCC)" $(NVCCFLAGS) -c backend_cuda.cu -o $@ +backend_cuda.o: backend_cuda.cu backend_cuda.h backend_gpu_compat.h .build-config + @command -v "$(GPUCC)" >/dev/null 2>&1 || { echo "$(GPUCC_NAME) not found" >&2; exit 1; } + "$(GPUCC)" $(GPUFLAGS) -c backend_cuda.cu -o $@ backend_metal.o: backend_metal.mm backend_metal.h $(METALXX) -c backend_metal.mm -o $@ @@ -256,14 +283,23 @@ metal-test: tests/test_backend_metal.mm backend_metal.mm backend_metal.h $(METALXX) tests/test_backend_metal.mm backend_metal.mm -framework Metal -framework Foundation -o backend_metal_test ./backend_metal_test -cuda-test: backend_cuda.cu backend_cuda.h tests/test_backend_cuda.cu - @command -v "$(NVCC)" >/dev/null 2>&1 || { echo "nvcc not found: set CUDA_HOME or NVCC" >&2; exit 1; } - "$(NVCC)" $(NVCCFLAGS) backend_cuda.cu tests/test_backend_cuda.cu -o backend_cuda_test$(EXE) +cuda-test: backend_cuda.cu backend_cuda.h backend_gpu_compat.h tests/test_backend_cuda.cu + @command -v "$(GPUCC)" >/dev/null 2>&1 || { echo "$(GPUCC_NAME) not found" >&2; exit 1; } + "$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_backend_cuda.cu -o backend_cuda_test$(EXE) ./backend_cuda_test$(EXE) -cuda-bench: backend_cuda.cu backend_cuda.h tests/bench_tensor_core.cu - @command -v "$(NVCC)" >/dev/null 2>&1 || { echo "nvcc not found: set CUDA_HOME or NVCC" >&2; exit 1; } - "$(NVCC)" $(NVCCFLAGS) backend_cuda.cu tests/bench_tensor_core.cu -o backend_cuda_bench$(EXE) +# convenience alias: kernel correctness on AMD (same test, hipcc toolchain) +hip-test: + $(MAKE) cuda-test HIP=1 + +# CI: compile the backend and its test binary WITHOUT executing them (for +# runners with the toolchain but no GPU). Pass CUDA_ARCH/HIP=1+HIP_ARCH. +gpu-compile: backend_cuda.o + "$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_backend_cuda.cu -o backend_cuda_test$(EXE) + +cuda-bench: backend_cuda.cu backend_cuda.h backend_gpu_compat.h tests/bench_tensor_core.cu + @command -v "$(GPUCC)" >/dev/null 2>&1 || { echo "$(GPUCC_NAME) not found" >&2; exit 1; } + "$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/bench_tensor_core.cu -o backend_cuda_bench$(EXE) ./backend_cuda_bench$(EXE) olmoe$(EXE): olmoe.c st.h json.h compat.h @@ -362,4 +398,4 @@ clean: bench: iobench$(EXE) @if [ -n "$(ARGS)" ]; then ./iobench$(EXE) $(ARGS); else echo "built iobench$(EXE) — run: ./iobench$(EXE) "; fi -.PHONY: all glm cuda-test cuda-bench cuda-dll portable test-c test-python test check clean install uninstall bench +.PHONY: all glm cuda-test hip-test gpu-compile cuda-bench cuda-dll portable test-c test-python test check clean install uninstall bench diff --git a/c/backend_cuda.cu b/c/backend_cuda.cu index 9ce142fb..1717c63b 100644 --- a/c/backend_cuda.cu +++ b/c/backend_cuda.cu @@ -1,7 +1,6 @@ #include "backend_cuda.h" -#include -#include +#include "backend_gpu_compat.h" #include #include @@ -555,7 +554,7 @@ extern "C" int coli_cuda_shared_mlp_w4a16(ColiCudaTensor *gate,ColiCudaTensor *u if(!gate||!up||!down||!x||!y||S<1||gate->fmt!=2||up->fmt!=2||down->fmt!=2|| gate->device!=up->device||gate->device!=down->device||gate->I!=up->I|| gate->O!=up->O||down->I!=gate->O||down->O!=gate->I)return 0; - DeviceContext *ctx=find_ctx(gate->device);if(!select_ctx(ctx)||ctx->compute_major<7)return 0; + DeviceContext *ctx=find_ctx(gate->device);if(!select_ctx(ctx)||!COLI_GPU_HAS_WMMA||ctx->compute_major<7)return 0; int D=gate->I,I=gate->O;size_t xb=(size_t)S*D*sizeof(float),ib=(size_t)S*I*sizeof(float); if(!reserve(&ctx->x,&ctx->x_cap,xb)||!reserve(&ctx->gate,&ctx->gate_cap,ib)|| !reserve(&ctx->up,&ctx->up_cap,ib)||!reserve(&ctx->y,&ctx->y_cap,xb)|| @@ -635,7 +634,7 @@ extern "C" int coli_cuda_expert_group(ColiCudaTensor *const *gates, silu_mul<<<(unsigned)(((size_t)total*I+255)/256),256,0,ctx->stream>>>(ctx->gate,ctx->up,(size_t)total*I); quantize_s4_rows<<stream>>>(ctx->qx,ctx->qscale,ctx->gate,total,I); grouped_s4_wmma<<stream>>>(ctx->y,ctx->qx,ctx->qscale,dev,I,D,2); - }else if(all_s4&&ctx->compute_major>=7&&getenv("COLI_CUDA_TC_W4A16")&& + }else if(all_s4&&COLI_GPU_HAS_WMMA&&ctx->compute_major>=7&&getenv("COLI_CUDA_TC_W4A16")&& atoi(getenv("COLI_CUDA_TC_W4A16"))){ /* W4A16 Tensor Core per gruppo: attivazioni fp16 per tile (lossless al * contrario del path W4A4), un lancio per expert dentro lo stream — diff --git a/c/backend_gpu_compat.h b/c/backend_gpu_compat.h new file mode 100644 index 00000000..98ec98e0 --- /dev/null +++ b/c/backend_gpu_compat.h @@ -0,0 +1,62 @@ +/* backend_gpu_compat.h — one GPU backend source, two vendors. + * Same pattern as compat.h for Windows: every platform difference lives in + * this header and backend_cuda.cu stays untouched. Compiled by nvcc this is + * a pass-through to the CUDA runtime (and mma.h for the tensor-core paths); + * compiled by hipcc (ROCm, HIP=1) it maps the CUDA runtime surface + * backend_cuda.cu uses onto HIP 1:1. The kernel language (__global__, + * __shared__, <<<>>>) is shared syntax. + * + * COLI_GPU_HAS_WMMA: the WMMA tensor-core kernels are guarded by + * __CUDA_ARCH__ >= 700 (device side) and by this flag at the host dispatch + * sites. Under HIP the flag is 0: gfx GPUs report compute_major >= 7, so a + * runtime-only check would select empty kernel bodies. Matrix-core support + * via rocWMMA is a possible follow-up; until then HIP always uses the + * portable kernels. */ +#ifndef COLIBRI_BACKEND_GPU_COMPAT_H +#define COLIBRI_BACKEND_GPU_COMPAT_H + +#if defined(__HIP_PLATFORM_AMD__) || defined(__HIP__) +#include +#include +#define COLI_GPU_HAS_WMMA 0 +#define cudaError_t hipError_t +#define cudaSuccess hipSuccess +#define cudaGetErrorString hipGetErrorString +#define cudaGetLastError hipGetLastError +#define cudaSetDevice hipSetDevice +#define cudaGetDeviceCount hipGetDeviceCount +#define cudaDeviceProp hipDeviceProp_t +#define cudaGetDeviceProperties hipGetDeviceProperties +#define cudaMalloc hipMalloc +#define cudaFree hipFree +#define cudaMemcpy hipMemcpy +#define cudaMemcpy2D hipMemcpy2D +#define cudaMemcpyAsync hipMemcpyAsync +#define cudaMemcpyHostToDevice hipMemcpyHostToDevice +#define cudaMemcpyDeviceToHost hipMemcpyDeviceToHost +#define cudaMemGetInfo hipMemGetInfo +#define cudaStream_t hipStream_t +#define cudaStreamCreate hipStreamCreate +#define cudaStreamCreateWithFlags hipStreamCreateWithFlags +#define cudaStreamNonBlocking hipStreamNonBlocking +#define cudaStreamDestroy hipStreamDestroy +#define cudaStreamSynchronize hipStreamSynchronize +#define cudaDeviceSynchronize hipDeviceSynchronize +#define cudaEvent_t hipEvent_t +#define cudaEventCreate hipEventCreate +#define cudaEventDestroy hipEventDestroy +#define cudaEventRecord hipEventRecord +#define cudaEventSynchronize hipEventSynchronize +#define cudaEventElapsedTime hipEventElapsedTime +#define cudaMallocHost hipHostMalloc +#define cudaFreeHost hipHostFree +#define cudaMemcpyDeviceToDevice hipMemcpyDeviceToDevice +#define cudaMemcpyPeer hipMemcpyPeer +#define cudaMemsetAsync hipMemsetAsync +#else +#include +#include +#define COLI_GPU_HAS_WMMA 1 +#endif + +#endif From 288edd71903a222b79472b327640cfae844e1010 Mon Sep 17 00:00:00 2001 From: noobdev-ph Date: Fri, 17 Jul 2026 12:39:59 +0800 Subject: [PATCH 016/132] fix: GPU backend failure-path hardening + tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three vendor-neutral fixes to backend_cuda.cu, each with test coverage: 1. Upload check-order: a cached device tensor is now usable when the caller's host pointers are stale or NULL. CUDA_RELEASE_HOST slots null their host pointers after upload; the current engine reaches those tensors through direct handles (coli_cuda_expert_mlp etc.), but any caller going through coli_cuda_matmul/tensor_upload with a cached tensor — as matmul_qt does for QT tensors — hits the !weights check before the cached-tensor branch and fails spuriously. This hardens the API contract rather than fixing a measured regression; the contract is pinned by a 64x sustained-reuse test. 2. Sticky runtime error (real bug, test-caught): a failed allocation left the last-error state set, so the NEXT healthy launch's cudaGetLastError() check reported 'out of memory' and disabled a perfectly good tensor. cuda_ok() now consumes the error on the failure path; regression-covered. 3. COLI_GPU_FAIL_AFTER=N test hook: every GPU compute entry point (19 total: matmul, expert mlp/group, shared mlp, attention ops, pipe ops) reports failure after N successful calls, so the engine's CPU fallbacks and expert_host_ensure rematerialization can be exercised end-to-end without real hardware faults. Unset = zero effect; uploads/queries are never gated. Validated on GLM-5.2: total failure (N=0) completes coherently with every released expert rematerialized. Tests (run via make cuda-test on any CUDA GPU; vendor-neutral source): 64x sustained matmul reuse after host pointers are freed; upload from a scribbled-and-freed temporary; five graceful upload-failure cases with stats-integrity assertions; healthy-launch-after-failed-alloc (the sticky-error regression); fault-hook on/off restore. Verified on AMD RX 9070 XT via the companion HIP PR's compat header (same test source); a make cuda-test run on NVIDIA hardware would complete the matrix. --- c/backend_cuda.cu | 45 +++++++++++++++++++++++++++++++---- c/tests/test_backend_cuda.cu | 46 ++++++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+), 4 deletions(-) diff --git a/c/backend_cuda.cu b/c/backend_cuda.cu index 9ce142fb..69a224ff 100644 --- a/c/backend_cuda.cu +++ b/c/backend_cuda.cu @@ -45,6 +45,8 @@ static std::mutex g_group_stats_mu; static int cuda_ok(cudaError_t err, const char *what) { if (err == cudaSuccess) return 1; std::fprintf(stderr, "[CUDA] %s: %s\n", what, cudaGetErrorString(err)); + (void)cudaGetLastError(); /* consume the sticky error: a failed call must + not poison the next launch's error check */ return 0; } @@ -453,14 +455,20 @@ extern "C" void coli_cuda_group_stats(uint64_t *calls, uint64_t *experts, uint64 extern "C" int coli_cuda_tensor_upload(ColiCudaTensor **tensor, const void *weights, const float *scales, int fmt, int I, int O, int device) { - DeviceContext *ctx = find_ctx(device); - if (!tensor || !weights || I < 1 || O < 1 || !select_ctx(ctx)) return 0; - size_t rb = row_bytes(fmt, I); - if (!rb || (fmt && !scales)) return 0; + if (!tensor) return 0; if (*tensor) { + /* Cached device copy: usable even when the caller's host pointers are + * gone. CUDA_RELEASE_HOST slots null their host pointers after upload, + * and with the old order (!weights checked first) every later matmul + * on such a slot failed here — the GPU tier silently never computed + * for host-released slab experts. */ ColiCudaTensor *t = *tensor; return t->fmt == fmt && t->I == I && t->O == O && t->device == device; } + DeviceContext *ctx = find_ctx(device); + if (!weights || I < 1 || O < 1 || !select_ctx(ctx)) return 0; + size_t rb = row_bytes(fmt, I); + if (!rb || (fmt && !scales)) return 0; ColiCudaTensor *t = static_cast(std::calloc(1, sizeof(*t))); if (!t) return 0; t->fmt = fmt; t->I = I; t->O = O; t->device = device; t->weight_bytes = rb * (size_t)O; @@ -502,10 +510,21 @@ extern "C" int coli_cuda_tensor_update(ColiCudaTensor *tensor, (size_t)tensor->O*sizeof(float),cudaMemcpyHostToDevice),"scale refresh"); } +/* Test hook: COLI_GPU_FAIL_AFTER=N makes every GPU COMPUTE entry point report + * failure after N successful calls (N=0: every call fails), exercising the + * engine's CPU fallbacks and host-rematerialization end-to-end without real + * hardware faults. Uploads/queries are not gated. Unset: no effect. */ +static long g_gpu_calls; +static int fault_injected(void) { + const char *fa = std::getenv("COLI_GPU_FAIL_AFTER"); + return fa && g_gpu_calls++ >= std::atol(fa); +} + extern "C" int coli_cuda_matmul(ColiCudaTensor **tensor, float *y, const float *x, const void *weights, const float *scales, int fmt, int S, int I, int O, int device) { + if (fault_injected()) return 0; if (S < 1 || !coli_cuda_tensor_upload(tensor, weights, scales, fmt, I, O, device)) return 0; ColiCudaTensor *t = *tensor; DeviceContext *ctx = find_ctx(t->device); @@ -524,6 +543,7 @@ extern "C" int coli_cuda_matmul(ColiCudaTensor **tensor, extern "C" int coli_cuda_expert_mlp(ColiCudaTensor *gate, ColiCudaTensor *up, ColiCudaTensor *down, float *y, const float *x, int S) { + if (fault_injected()) return 0; if (!gate || !up || !down || !x || !y || S < 1 || gate->device != up->device || gate->device != down->device || gate->I != up->I || gate->O != up->O || @@ -552,6 +572,7 @@ extern "C" int coli_cuda_expert_mlp(ColiCudaTensor *gate, ColiCudaTensor *up, extern "C" int coli_cuda_shared_mlp_w4a16(ColiCudaTensor *gate,ColiCudaTensor *up, ColiCudaTensor *down,float *y,const float *x,int S){ + if (fault_injected()) return 0; if(!gate||!up||!down||!x||!y||S<1||gate->fmt!=2||up->fmt!=2||down->fmt!=2|| gate->device!=up->device||gate->device!=down->device||gate->I!=up->I|| gate->O!=up->O||down->I!=gate->O||down->O!=gate->I)return 0; @@ -583,6 +604,7 @@ extern "C" int coli_cuda_expert_group(ColiCudaTensor *const *gates, ColiCudaTensor *const *downs, const int *rows, int count, float *y, const float *x) { + if (fault_injected()) return 0; if (!gates || !ups || !downs || !rows || !x || !y || count < 1) return 0; ColiCudaTensor *first=gates[0]; if (!first) return 0; @@ -708,6 +730,7 @@ extern "C" int coli_cuda_expert_group(ColiCudaTensor *const *gates, extern "C" int coli_cuda_attention_absorb(ColiCudaTensor *w,float *ctx,const float *q, const float *latent,const float *rope,int H,int Q, int R,int V,int K,int T,float scale){ + if (fault_injected()) return 0; if(!w||!ctx||!q||!latent||!rope||H<1||Q<1||R<1||V<1||K<1||K>512||T<1||T>4096|| w->I!=K||w->O!=H*(Q+V))return 0; DeviceContext *dc=find_ctx(w->device);if(!select_ctx(dc))return 0; @@ -761,12 +784,14 @@ static int attention_absorb_batch_run(ColiCudaTensor *w,ColiCudaTensor *proj,flo extern "C" int coli_cuda_attention_absorb_batch(ColiCudaTensor *w,float *ctx,const float *q, const float *latent,const float *rope,int S,int H,int Q,int R,int V,int K,int T, float scale){ + if (fault_injected()) return 0; return attention_absorb_batch_run(w,nullptr,ctx,q,latent,rope,S,H,Q,R,V,K,T,scale); } extern "C" int coli_cuda_attention_project_batch(ColiCudaTensor *w,ColiCudaTensor *proj, float *out,const float *q,const float *latent,const float *rope,int S,int H,int Q, int R,int V,int K,int T,float scale){ + if (fault_injected()) return 0; return attention_absorb_batch_run(w,proj,out,q,latent,rope,S,H,Q,R,V,K,T,scale); } @@ -870,6 +895,7 @@ extern "C" int coli_cuda_pipe_download(int device,const void *src,void *dst,size } extern "C" int coli_cuda_pipe_rmsnorm(int device,float *y_dev,const float *x_dev, const float *w_dev,int S,int D,float eps){ + if (fault_injected()) return 0; DeviceContext *ctx=find_ctx(device); if(S<1||D<1||!select_ctx(ctx)) return 0; pipe_rmsnorm_rows<<>>(y_dev,x_dev,w_dev,D,eps,D,D); @@ -878,6 +904,7 @@ extern "C" int coli_cuda_pipe_rmsnorm(int device,float *y_dev,const float *x_dev extern "C" int coli_cuda_pipe_rmsnorm_s(int device,float *y_dev,const float *x_dev, const float *w_dev,int S,int D,float eps, int xstride,int ystride){ + if (fault_injected()) return 0; DeviceContext *ctx=find_ctx(device); if(S<1||D<1||xstride>>(y_dev,x_dev,w_dev,D,eps,xstride,ystride); @@ -886,6 +913,7 @@ extern "C" int coli_cuda_pipe_rmsnorm_s(int device,float *y_dev,const float *x_d extern "C" int coli_cuda_pipe_rope(int device,float *v_dev,const int *pos_dev, int rows,int stride,int offset,int R,int heads, float theta){ + if (fault_injected()) return 0; DeviceContext *ctx=find_ctx(device); if(rows<1||R<2||R>256||heads<1||!select_ctx(ctx)) return 0; pipe_rope_rows<<>>(v_dev,pos_dev,0,stride,offset,R,heads,theta); @@ -893,6 +921,7 @@ extern "C" int coli_cuda_pipe_rope(int device,float *v_dev,const int *pos_dev, } extern "C" int coli_cuda_pipe_rope_base(int device,float *v_dev,int pos_base,int rows, int stride,int offset,int R,int heads,float theta){ + if (fault_injected()) return 0; DeviceContext *ctx=find_ctx(device); if(rows<1||R<2||R>256||heads<1||!select_ctx(ctx)) return 0; pipe_rope_rows<<>>(v_dev,NULL,pos_base,stride,offset,R,heads,theta); @@ -910,6 +939,7 @@ extern "C" int coli_cuda_pipe_copy2d(int device,float *dst,int dpitch,const floa extern "C" int coli_cuda_attention_project_batch_dev(ColiCudaTensor *w,ColiCudaTensor *proj, float *out,const float *q_dev,const float *latent_dev,const float *rope_dev, int S,int H,int Q,int R,int V,int K,int T,float scale){ + if (fault_injected()) return 0; if(!w||!proj||!out||!q_dev||!latent_dev||!rope_dev||S<1||H<1||Q<1||R<1||V<1|| K<1||K>512||T8192||w->I!=K||w->O!=H*(Q+V)|| proj->device!=w->device||proj->I!=H*V)return 0; @@ -931,17 +961,20 @@ extern "C" int coli_cuda_attention_project_batch_dev(ColiCudaTensor *w,ColiCudaT } extern "C" int coli_cuda_pipe_silu_mul(int device,float *gate_dev,const float *up_dev, size_t n){ + if (fault_injected()) return 0; DeviceContext *ctx=find_ctx(device); if(!n||!select_ctx(ctx)) return 0; silu_mul<<<(unsigned)((n+255)/256),256>>>(gate_dev,up_dev,n); return cuda_ok(cudaGetLastError(),"pipe silu mul"); } extern "C" int coli_cuda_pipe_add(int device,float *x_dev,const float *t_dev,size_t n){ + if (fault_injected()) return 0; DeviceContext *ctx=find_ctx(device); if(!n||!select_ctx(ctx)) return 0; pipe_add_n<<<(unsigned)((n+255)/256),256>>>(x_dev,t_dev,n); return cuda_ok(cudaGetLastError(),"pipe add"); } extern "C" int coli_cuda_pipe_rows_add(int device,float *x_dev,const float *partial_dev, const int *rows_dev,int nrows,int D){ + if (fault_injected()) return 0; DeviceContext *ctx=find_ctx(device); if(nrows<1||D<1||!select_ctx(ctx)) return 0; pipe_rows_add<<>>(x_dev,partial_dev,rows_dev,D); return cuda_ok(cudaGetLastError(),"pipe rows add"); @@ -950,6 +983,7 @@ extern "C" int coli_cuda_pipe_rows_add(int device,float *x_dev,const float *part * coli_cuda_matmul, zero host transfers. */ extern "C" int coli_cuda_pipe_gemm(ColiCudaTensor *t,float *y_dev,const float *x_dev, int S){ + if (fault_injected()) return 0; if(!t||S<1) return 0; DeviceContext *ctx=find_ctx(t->device); if(!select_ctx(ctx)) return 0; dim3 grid((unsigned)t->O,(unsigned)S); @@ -969,6 +1003,7 @@ extern "C" int coli_cuda_pipe_peer_copy(int dst_dev,float *dst,int src_dev, extern "C" int coli_cuda_attention_project_batch_dev_out(ColiCudaTensor *w,ColiCudaTensor *proj, float *out_dev,const float *q_dev,const float *latent_dev,const float *rope_dev, int S,int H,int Q,int R,int V,int K,int T,float scale){ + if (fault_injected()) return 0; if(!w||!proj||!out_dev||!q_dev||!latent_dev||!rope_dev||S<1||H<1||Q<1||R<1||V<1|| K<1||K>512||T8192||w->I!=K||w->O!=H*(Q+V)|| proj->device!=w->device||proj->I!=H*V)return 0; @@ -990,6 +1025,7 @@ extern "C" int coli_cuda_attention_project_batch_dev_out(ColiCudaTensor *w,ColiC extern "C" int coli_cuda_attention_absorb_batch_dev(ColiCudaTensor *w,float *ctx_dev, const float *q_dev,const float *latent_dev,const float *rope_dev, int S,int H,int Q,int R,int V,int K,int T,float scale){ + if (fault_injected()) return 0; if(!w||!ctx_dev||!q_dev||!latent_dev||!rope_dev||S<1||H<1||Q<1||R<1||V<1|| K<1||K>512||T8192||w->I!=K||w->O!=H*(Q+V))return 0; DeviceContext *dc=find_ctx(w->device);if(!select_ctx(dc))return 0; @@ -1004,6 +1040,7 @@ extern "C" int coli_cuda_attention_absorb_batch_dev(ColiCudaTensor *w,float *ctx extern "C" int coli_cuda_attention_absorb_kvdev(ColiCudaTensor *w,float *ctx,const float *q, const float *latent_dev,const float *rope_dev,int H,int Q,int R,int V,int K,int T, float scale){ + if (fault_injected()) return 0; if(!w||!ctx||!q||!latent_dev||!rope_dev||H<1||Q<1||R<1||V<1||K<1||K>512||T<1||T>8192|| w->I!=K||w->O!=H*(Q+V))return 0; DeviceContext *dc=find_ctx(w->device);if(!select_ctx(dc))return 0; diff --git a/c/tests/test_backend_cuda.cu b/c/tests/test_backend_cuda.cu index 5550bdef..259fe6b5 100644 --- a/c/tests/test_backend_cuda.cu +++ b/c/tests/test_backend_cuda.cu @@ -50,6 +50,52 @@ int main(int argc, char **argv) { if (coli_cuda_tensor_upload(&t8, q8, s8, 1, 5, 2, d0)) return 1; if (ndev > 1 && coli_cuda_tensor_upload(&t8, q8, s8, 1, 4, 2, d1)) return 1; if (!coli_cuda_matmul(&t8, got, x, q8, s8, 1, 2, 4, 2, d0) || !close_enough(got, want8, 4)) return 1; + /* Cached tensor must stay callable without live host pointers + * (CUDA_RELEASE_HOST slots null theirs after upload) — including + * SUSTAINED reuse, not just the first call. */ + for (int rep = 0; rep < 64; rep++) + if (!coli_cuda_matmul(&t8, got, x, nullptr, nullptr, 1, 2, 4, 2, d0) || + !close_enough(got, want8, 4)) return 1; + /* A tensor uploaded from a TEMPORARY host buffer must survive the buffer + * being scribbled and freed (the release-host lifecycle). */ + { + int8_t *tmpw = static_cast(std::malloc(8)); + float *tmps = static_cast(std::malloc(2 * sizeof(float))); + if (!tmpw || !tmps) return 2; + for (int i = 0; i < 8; i++) tmpw[i] = q8[i]; + tmps[0] = s8[0]; tmps[1] = s8[1]; + ColiCudaTensor *tt = nullptr; + if (!coli_cuda_tensor_upload(&tt, tmpw, tmps, 1, 4, 2, d0)) return 1; + for (int i = 0; i < 8; i++) tmpw[i] = 99; + std::free(tmpw); std::free(tmps); + if (!coli_cuda_matmul(&tt, got, x, nullptr, nullptr, 1, 2, 4, 2, d0) || + !close_enough(got, want8, 4)) return 1; + coli_cuda_tensor_free(tt); + } + /* Upload failures must be graceful and must not corrupt accounting — + * and must not poison LATER healthy launches (sticky-error regression). */ + { + size_t c0 = 0, b0 = 0, c1 = 0, b1 = 0; + coli_cuda_stats(-1, &c0, &b0); + ColiCudaTensor *bad = nullptr; + if (coli_cuda_tensor_upload(&bad, q8, s8, 1, 4, 2, 9999)) return 1; + if (coli_cuda_tensor_upload(&bad, q8, s8, 7, 4, 2, d0)) return 1; + if (coli_cuda_tensor_upload(&bad, q8, nullptr, 1, 4, 2, d0)) return 1; + if (coli_cuda_tensor_upload(&bad, nullptr, s8, 1, 4, 2, d0)) return 1; + if (coli_cuda_tensor_upload(&bad, q8, s8, 1, 1 << 20, 1 << 24, d0)) return 1; /* ~16 TB */ + if (bad) return 1; + coli_cuda_stats(-1, &c1, &b1); + if (c0 != c1 || b0 != b1) return 1; + /* healthy launch immediately after the failed allocation */ + if (!coli_cuda_matmul(&t8, got, x, nullptr, nullptr, 1, 2, 4, 2, d0) || + !close_enough(got, want8, 4)) return 1; + } + /* Fault injection hook: on/off, restores cleanly. */ + if (setenv("COLI_GPU_FAIL_AFTER", "0", 1)) return 2; + if (coli_cuda_matmul(&t8, got, x, nullptr, nullptr, 1, 2, 4, 2, d0)) return 1; + if (unsetenv("COLI_GPU_FAIL_AFTER")) return 2; + if (!coli_cuda_matmul(&t8, got, x, nullptr, nullptr, 1, 2, 4, 2, d0) || + !close_enough(got, want8, 4)) return 1; const int8_t q8b[8]={-1,-2,-3,-4, 1,-2,3,-4}; const float s8b[2]={1.f,.5f},want8b[4]={10.f,15.f,-3.f,-2.5f}; if(!coli_cuda_tensor_update(t8,q8b,s8b)|| From dc196633f1b168a3e6801f35daabacac2f4955ba Mon Sep 17 00:00:00 2001 From: ebootheee Date: Thu, 16 Jul 2026 23:20:16 -0600 Subject: [PATCH 017/132] pipe: blocking pipe_wait (COLI_PIPE_BLOCK=1) + PIPE_WORKERS implies PIPE=1 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pipe_wait's sched_yield spin storms the scheduler for the full 0.5-3ms of each in-flight expert read; behind COLI_PIPE_BLOCK=1 it parks on a condvar instead (~5us wake, no lost-wakeup: workers store ready with release BEFORE taking the mutex to broadcast, and the waiter re-checks under the lock). Default OFF = byte-identical spin. Pthread pool only: the URING backend has no waiter spin to replace. Setting PIPE_WORKERS>0 in the env without PIPE now implies PIPE=1 with a stderr note: sizing the pool declares the intent to use it (a full benchmark campaign ran with PIPE_WORKERS=16 and the pipe silently off). The implication fires only when the platform default left the pipe off (no-op on _WIN32 where PIPE defaults to 1), only on a positive value (PIPE_WORKERS=0/empty does not enable a clamped 1-worker pipe), and an explicit PIPE=0 always wins. The rule lives in pipe_workers_imply_pipe() so the table is unit-testable. tests/test_pipe_block.c (in TEST_BINS, all platforms): pins the implication table, and drives the pool through 200 generations under each waiter against an on-disk expert fixture — spin as control, condvar arm alternating parked and fast-path waits — verifying identical slot bytes. Measured on the spin side (2x5090 + Gen5 NVMe, GLM-5.2 744B int4, CPU decode): 1.98 -> 2.16 tok/s at 192 tokens (expert-disk service 44.6s -> 33.5s). On Metal/GPU decode an M5 Pro A/B (PR thread) measured no change, consistent with the mechanism: the win is freeing the core the spinner was stealing from the CPU matmul team. Co-Authored-By: Claude Fable 5 --- c/Makefile | 5 +- c/glm.c | 39 +++++++++ c/tests/test_pipe_block.c | 169 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 212 insertions(+), 1 deletion(-) create mode 100644 c/tests/test_pipe_block.c diff --git a/c/Makefile b/c/Makefile index 27902cb2..db94a9a3 100644 --- a/c/Makefile +++ b/c/Makefile @@ -166,7 +166,7 @@ else PYTHON ?= python3 endif CUDA_OBJ = -TEST_BINS = tests/test_json$(EXE) tests/test_st$(EXE) tests/test_tier$(EXE) tests/test_grammar$(EXE) tests/test_schema_gbnf$(EXE) tests/test_decode_batch$(EXE) tests/test_idot$(EXE) tests/test_i4_grouped$(EXE) tests/test_stops$(EXE) tests/test_kv_alloc$(EXE) tests/test_i4_acc512$(EXE) tests/test_compat_direct$(EXE) +TEST_BINS = tests/test_json$(EXE) tests/test_st$(EXE) tests/test_tier$(EXE) tests/test_grammar$(EXE) tests/test_schema_gbnf$(EXE) tests/test_decode_batch$(EXE) tests/test_idot$(EXE) tests/test_i4_grouped$(EXE) tests/test_stops$(EXE) tests/test_kv_alloc$(EXE) tests/test_i4_acc512$(EXE) tests/test_compat_direct$(EXE) tests/test_pipe_block$(EXE) ifneq (,$(LINUX)) TEST_BINS += tests/test_uring$(EXE) endif @@ -329,6 +329,9 @@ tests/test_compat_direct$(EXE): tests/test_compat_direct.c compat.h tests/test_uring$(EXE): tests/test_uring.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) +tests/test_pipe_block$(EXE): tests/test_pipe_block.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h + $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) + test-c: $(TEST_BINS) $(PYTHON) tools/run_tests.py $(TEST_BINS) diff --git a/c/glm.c b/c/glm.c index 645be12b..6bd66f40 100644 --- a/c/glm.c +++ b/c/glm.c @@ -2146,6 +2146,22 @@ static int g_pipe=0; /* PIPE=1: async expert-load pipeline. Default ON for * the matmul. PIPE=0 opts back into the blocking serial path. */ static int g_pipe_nw=8; /* PIPE_WORKERS=n: I/O worker threads (disk-parallel reads) */ static int g_uring=0; /* URING=1: Linux io_uring load/completion backend; implies PIPE */ +static int g_pipe_block=0;/* COLI_PIPE_BLOCK=1: pipe_wait blocca su una condvar invece dello + * spin sched_yield (default OFF = spin byte-identico). EN: a yield + * storm on the main thread fights the OpenMP team for cycles during + * multi-ms loads; the condvar wake costs ~5us against reads that + * cost 0.5-3ms (#159). Pthread pool only: the URING backend has no + * waiter spin to replace. */ +/* PIPE_WORKERS>0 esplicito nell'env implica PIPE=1: dimensionare il pool + * dichiara l'intento di usarlo (una campagna intera l'ha impostato con la + * pipe spenta senza accorgersene). EN: fires ONLY when PIPE is unset in the + * env AND the platform default left the pipe off (on _WIN32 it already + * defaults to 1) AND PIPE_WORKERS parses positive — the internal default of + * 8 does not count, PIPE_WORKERS=0/empty does not, and an explicit PIPE=0 + * always wins. */ +static int pipe_workers_imply_pipe(const char *pipe_env, const char *pw_env, int pipe_now){ + return !pipe_now && !pipe_env && pw_env && atoi(pw_env)>0; +} typedef struct { _Atomic uint64_t cur; /* (gen<<8)|index; gen main-only, index 0..njobs (≤64) */ _Atomic int njobs; /* current batch job count */ @@ -2153,6 +2169,7 @@ typedef struct { _Atomic int layer; /* current batch layer */ _Atomic int ready[64]; /* per-slot load-done flag */ pthread_mutex_t mx; pthread_cond_t cv; /* ONLY for parking/waking idle workers */ + pthread_cond_t cv_done; /* COLI_PIPE_BLOCK: signals ready[] transitions */ Model *m; pthread_t th[16]; int nw; int started; } PipePool; @@ -2177,6 +2194,11 @@ static void *pipe_worker(void *arg){ int eid=atomic_load_explicit(&p->eids[i],memory_order_relaxed); /* AFTER winning CAS */ expert_load(p->m,L,eid,&p->m->ws[i],1); /* needed-now load: fatal on I/O error (matches serial path) */ atomic_store_explicit(&p->ready[i],1,memory_order_release); + if(g_pipe_block){ /* wake a main thread parked in pipe_wait */ + pthread_mutex_lock(&p->mx); + pthread_cond_broadcast(&p->cv_done); + pthread_mutex_unlock(&p->mx); + } } /* CAS failed → another worker advanced index (or gen advanced): re-loop */ } @@ -2194,6 +2216,7 @@ static void pipe_init(Model *m){ g_pp.m=m; g_pp.nw=g_pipe_nw; if(g_pp.nw>16) g_pp.nw=16; if(g_pp.nw<1) g_pp.nw=1; atomic_store(&g_pp.cur,0); atomic_store(&g_pp.njobs,0); pthread_mutex_init(&g_pp.mx,NULL); pthread_cond_init(&g_pp.cv,NULL); + pthread_cond_init(&g_pp.cv_done,NULL); for(int i=0;i PIPE implication table: fires ONLY when + * PIPE is unset in the env AND the platform default left the pipe off AND + * PIPE_WORKERS parses positive (PIPE_WORKERS=0/empty/negative must NOT + * silently enable a clamped 1-worker pipe). */ +#include +#include +#include +#include +#include +#include +#define main coli_glm_main_unused +#include "../glm.c" +#undef main + +static int fail(const char *s){ fprintf(stderr,"FAIL: %s\n",s); return 1; } + +enum { NE=8, LAYER=1 }; /* experts 0..NE-1 on one MoE layer */ +/* per-expert file image: [gate 12][up 12][down 12][gate.qs 12][up.qs 12][down.qs 16] */ +enum { WB=12, QS_G=12, QS_U=12, QS_D=16, ESZ=3*WB+QS_G+QS_U+QS_D }; + +static unsigned char wbyte(int e,int j){ return (unsigned char)(e*31+j+1); } +static float scale(int e,int i){ return (float)(e*8+i)+0.5f; } + +#define TMPF "test_pipe_block.tmp" + +static int write_fixture(void){ + FILE *w=fopen(TMPF,"wb"); if(!w) return fail("create temp"); + for(int e=0;ec.hidden=4; m->c.moe_inter=3; m->ebits=8; + m->S.n=NE*6; m->S.cap=NE*6; m->S.t=calloc(NE*6,sizeof(st_tensor)); + if(!m->S.t) return fail("tensor metadata allocation"); + const char *proj[3]={"gate_proj","up_proj","down_proj"}; + int sbytes[3]={QS_G,QS_U,QS_D}; + for(int e=0;eS.t[e*6+k]=(st_tensor){strdup(name),fd,wo,WB,3,WB}; wo+=WB; + size_t n=strlen(name); memcpy(name+n,".qs",4); + m->S.t[e*6+3+k]=(st_tensor){strdup(name),fd,so,sbytes[k],2,sbytes[k]/4}; so+=sbytes[k]; + } + } + return 0; +} + +static int check_slot(ESlot *s,int e){ + if(s->eid!=e || s->g.fmt!=1 || s->u.fmt!=1 || s->d.fmt!=1){ + fprintf(stderr," slot: eid=%d (want %d) fmt g/u/d=%d/%d/%d (want 1/1/1)\n", + s->eid,e,s->g.fmt,s->u.fmt,s->d.fmt); + return 1; + } + const unsigned char *g=(const unsigned char*)s->g.q8, + *u=(const unsigned char*)s->u.q8, + *d=(const unsigned char*)s->d.q8; /* q8 is int8_t; compare raw bytes */ + for(int j=0;jg.s[i]!=scale(e,i) || s->u.s[i]!=scale(e,3+i)){ + fprintf(stderr," slot e=%d scale %d: g=%g/%g u=%g/%g (got/want)\n",e,i, + (double)s->g.s[i],(double)scale(e,i),(double)s->u.s[i],(double)scale(e,3+i)); + return 1; + } + for(int i=0;i<4;i++) + if(s->d.s[i]!=scale(e,6+i)){ + fprintf(stderr," slot e=%d scale %d: d=%g/%g (got/want)\n",e,i, + (double)s->d.s[i],(double)scale(e,6+i)); + return 1; + } + return 0; +} + +static int run_generations(Model *m,int block,int gens){ + g_pipe_block=block; + for(int gen=0;genws[q],eids[q])) return fail(block?"slot contents (block)":"slot contents (spin)"); + } + } + return 0; +} + +static int test_implication_table(void){ + struct { const char *pipe_env,*pw_env; int pipe_now,want; } T[]={ + {NULL,"4",0,1}, /* pool sized, pipe off, PIPE unset → imply */ + {NULL,"16",0,1}, + {NULL,"0",0,0}, /* PIPE_WORKERS=0 must NOT enable a clamped pipe */ + {NULL,"",0,0}, + {NULL,"-3",0,0}, + {"0","4",0,0}, /* explicit PIPE=0 always wins */ + {"1","4",1,0}, /* explicit PIPE=1: nothing to imply */ + {NULL,"4",1,0}, /* platform default already ON (win32) */ + {NULL,NULL,0,0}, + }; + for(size_t i=0;i",T[i].pw_env?T[i].pw_env:"",T[i].pipe_now); + return 1; + } + return 0; +} + +int main(void){ + if(test_implication_table()) return 1; + + /* Relative to the CWD, like test_compat_direct's TMPF — NOT "/tmp/...": + * the windows job builds native .exe files and "/tmp" is not a Windows + * path. fwrite then reopen read-only: Windows compat has pread, not pwrite. */ + if(write_fixture()) return 1; + int fd=open(TMPF,COMPAT_O_RDONLY); + if(fd<0) return fail("open temp"); + + static Model m; /* zeroed: buffered pread path, no mmap/cuda */ + if(build_fixture(&m,fd)){ close(fd); remove(TMPF); return 1; } + + g_pipe=1; g_pipe_nw=4; + pipe_init(&m); + + /* spin waiter first (control), then the condvar waiter under the same + * dispatch pattern; 200 generations each cycles the gen-tagged cursor + * and alternates parked/fast-path waits. */ + if(run_generations(&m,0,200) || run_generations(&m,1,200)){ close(fd); remove(TMPF); return 1; } + + for(int q=0;q Date: Thu, 16 Jul 2026 08:55:53 -0500 Subject: [PATCH 018/132] security: harden model-file parsing, server, and build per audit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Verified & applied the ColibriFixes.md audit against the current tree (adversarially confirmed; no regressions vs. legitimate GLM-5.2 loading or normal server flows). Model-file trust boundary (untrusted-mirror threat model): - glm.c: qt_check_fmt() rejects (not infers) any quantized tensor whose weight/scale byte counts disagree with the config shape, at all three format sites (qt_from_disk + both expert_load arms) — closes the matmul OOB read and the per-row scale OOB write (SEC-1); embed_row token-id bound zero-fills out-of-range ids (SEC-5). - st.h: cross-check numel==nbytes/esz for float dtypes + shape-product overflow guard + st_read_f32_cap at config-sized sinks (SEC-2/B1); st_read_slice_f32 NULL + bounds guard (B6). - tok.h: negative-id and json_get NULL/type guards (SEC-3); tk_read_file fseek/ftell/malloc/fread error handling + size cap (B3/B4). - json.h: bound \u reads against the NUL, strncmp full true/false/null keywords (SEC-4); growable string buffer replacing the silent 64KB truncation (B5). Server (openai_server.py): - fail-closed on non-loopback bind without a key (SEC-6), Host-header allowlist incl. OPTIONS preflight (SEC-7), auth-gate /health & /experts telemetry (SEC-8), scheduler head-of-line fix (B2), [DONE] under ka_lock (B9). Build / supply chain / UI: - Makefile CUDA_ARCH=portable (sm_80..120 + compute_120 PTX fallback), native stays default (D1); env-pinnable model download revisions + c/tools/requirements.txt (D3); CUDA event-leak fix (B8); flake.lock reproducibility note (D2). - web: theme-aware ErrorBoundary, API-key autocomplete, /experts error surface, tooltip viewport clamp, 401 message, responsive badge row, type=button, Probing label. Verified: test_json/test_st pass; glm.o/olmoe.o build; backend_cuda.cu compiles under nvcc 13.3; portable gencode parses; web tsc --noEmit clean; all py_compile OK. Co-Authored-By: Claude Opus 4.8 --- c/Makefile | 20 ++++++- c/backend_cuda.cu | 3 +- c/download_fp8.py | 4 +- c/glm.c | 29 +++++++++- c/json.h | 24 ++++---- c/openai_server.py | 119 +++++++++++++++++++++++++++++--------- c/st.h | 37 +++++++++++- c/tok.h | 30 ++++++++-- c/tools/download_glm52.py | 6 +- c/tools/requirements.txt | 15 +++++ flake.nix | 3 + web/src/App.tsx | 9 +-- web/src/Brain.tsx | 8 ++- web/src/ErrorBoundary.tsx | 10 ++-- web/src/index.css | 2 +- 15 files changed, 253 insertions(+), 66 deletions(-) create mode 100644 c/tools/requirements.txt diff --git a/c/Makefile b/c/Makefile index 7210cbbb..a9315012 100644 --- a/c/Makefile +++ b/c/Makefile @@ -143,16 +143,32 @@ else CUDA_HOME ?= /usr/local/cuda NVCC ?= $(CUDA_HOME)/bin/nvcc endif +# CUDA target selection. Default `native` builds SASS for the build machine's GPU +# only (fast local dev). A headless CI/Docker build has no GPU, so `-arch=native` +# there detects nothing and the shipped coli_cuda.dll fails at first kernel dispatch. +# For a portable/release DLL that runs on any Ampere..Blackwell card AND JITs on +# newer archs, build with CUDA_ARCH=portable: it emits SASS for sm_80/86/89/90/120 +# plus a compute_120 PTX fallback (sm_120 / RTX 50-series needs CUDA >= 12.8). CUDA_ARCH ?= native +ifeq ($(CUDA_ARCH),portable) +CUDA_GENCODE = -gencode arch=compute_80,code=sm_80 \ + -gencode arch=compute_86,code=sm_86 \ + -gencode arch=compute_89,code=sm_89 \ + -gencode arch=compute_90,code=sm_90 \ + -gencode arch=compute_120,code=sm_120 \ + -gencode arch=compute_120,code=compute_120 +else +CUDA_GENCODE = -arch=$(CUDA_ARCH) +endif ifneq ($(IS_WIN),) # nvcc's host compiler on Windows is MSVC cl.exe: the GCC-style # -Xcompiler=-Wall,-Wextra is rejected ("D8021 invalid numeric argument # '/Wextra'"), which made `make cuda-dll` unbuildable as shipped. -W3 is # the MSVC warning level (dash form, NOT /W3: MSYS make would mangle the # slash-form into a filesystem path). -NVCCFLAGS ?= -O3 -std=c++17 -arch=$(CUDA_ARCH) -Xcompiler=-W3 +NVCCFLAGS ?= -O3 -std=c++17 $(CUDA_GENCODE) -Xcompiler=-W3 else -NVCCFLAGS ?= -O3 -std=c++17 -arch=$(CUDA_ARCH) -Xcompiler=-Wall,-Wextra +NVCCFLAGS ?= -O3 -std=c++17 $(CUDA_GENCODE) -Xcompiler=-Wall,-Wextra endif # PYTHON is a HOST tool (clean/test-c/test-python run it on the build machine), # so key it off the host, NOT $(IS_WIN) — which is derived from the *target* diff --git a/c/backend_cuda.cu b/c/backend_cuda.cu index 188330d9..ec1b3214 100644 --- a/c/backend_cuda.cu +++ b/c/backend_cuda.cu @@ -646,7 +646,8 @@ extern "C" int coli_cuda_expert_group(ColiCudaTensor *const *gates, if(!cuda_ok(copy_desc,"expert group descriptors"))return 0; int profile=getenv("COLI_CUDA_PROFILE")&&atoi(getenv("COLI_CUDA_PROFILE")); cudaEvent_t ev[4]={}; - if(profile) for(int i=0;i<4;i++) if(!cuda_ok(cudaEventCreate(&ev[i]),"profile event")) profile=0; + if(profile) for(int i=0;i<4;i++) if(!cuda_ok(cudaEventCreate(&ev[i]),"profile event")){ + for(int j=0;jstream); if(async)std::memcpy(ctx->host_x,x,xb); cudaError_t copy_x=async?cudaMemcpyAsync(ctx->x,ctx->host_x,xb,cudaMemcpyHostToDevice,ctx->stream) diff --git a/c/download_fp8.py b/c/download_fp8.py index fc8e2b23..dd4e96db 100644 --- a/c/download_fp8.py +++ b/c/download_fp8.py @@ -53,14 +53,14 @@ def download_file_ms(fn): model_id=REPO_MS, file_path=fn, local_dir=DEST, - revision="master", + revision=os.environ.get("GLM_MS_REVISION", "master"), # pin to a commit for supply-chain integrity ) def download_file_hf(fn): """Download a single file from HuggingFace with hf_transfer.""" os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1" from huggingface_hub import hf_hub_download - hf_hub_download(REPO_HF, fn, local_dir=DEST) + hf_hub_download(REPO_HF, fn, local_dir=DEST, revision=os.environ.get("GLM_HF_REVISION", "main")) def download_file_curl(fn, base_url, expected_size): """Fallback: download with curl to a .part file with resume.""" diff --git a/c/glm.c b/c/glm.c index 70da3201..4043d8f1 100644 --- a/c/glm.c +++ b/c/glm.c @@ -1445,6 +1445,25 @@ static int detect_group_size(int O, int I, int64_t ns){ return 0; } +/* Reject a quantized tensor whose on-disk weight/scale byte counts don't EXACTLY + * match the config-derived [O,I] shape for the detected fmt. The fmt is inferred + * from the weight byte count and falls to int2 (fmt=3) on any mismatch; without + * this guard a crafted `.qs` (untrusted mirror, the documented threat model) picks + * a wrong fmt and the SIMD kernels index the weight slab / per-row scale buffer + * past its real length (OOB read), and qt_from_disk's O-sized scale buffer is + * overrun on read (OOB write). ns is the scale-tensor byte count (F32 scales). */ +static void qt_check_fmt(const char *name, int fmt, int gs, int O, int I, int64_t nb, int64_t ns){ + int64_t ew, es; /* expected weight bytes, expected scale float count */ + if(fmt==1){ ew=(int64_t)O*I; es=(int64_t)O; } + else if(fmt==2){ ew=(int64_t)O*((I+1)/2); es=(int64_t)O; } + else if(fmt==3){ ew=(int64_t)O*((I+3)/4); es=(int64_t)O; } + else if(fmt==4){ int ng=(I+gs-1)/gs; ew=(int64_t)O*((I+1)/2); es=(int64_t)O*ng; } + else { fprintf(stderr,"%s: unknown quant fmt %d\n",name,fmt); exit(1); } + if(nb!=ew || ns!=es*4){ + fprintf(stderr,"%s: quantized size mismatch fmt=%d [O=%d,I=%d,gs=%d]: weight %lld (want %lld), scale %lld (want %lld)\n", + name,fmt,O,I,gs,(long long)nb,(long long)ew,(long long)ns,(long long)(es*4)); exit(1); } +} + /* costruisce un QT [O,I] dal disco in `t` (buffer riusabili tra chiamate). * - se esiste `name.qs`: pesi GIA' quantizzati nel container (U8 qdata + F32 scala) -> letti diretti * - altrimenti: tensore pieno (f32/bf16) -> quantizzato a runtime a `bits` (oracolo tiny / pesi pieni) @@ -1460,16 +1479,17 @@ static void qt_from_disk(Model *m, const char *name, int O, int I, int bits, int int gs=0; if(fmt==2) gs=detect_group_size(O,I,ns); if(gs>0) fmt=4; + qt_check_fmt(name,fmt,gs,O,I,nb,ns); /* reject size/format mismatch before alloc+read */ if(fmt==1){ if(t->fmt!=1||!t->q8){ t->fmt=1; t->O=O; t->I=I; t->gs=0; t->q8=qalloc(nb); t->s=qsalloc(O); } st_read_raw(&m->S,name,t->q8,drop); } else if(fmt==4){ int ng=(I+gs-1)/gs; if(t->fmt!=4||!t->q4){ t->fmt=4; t->O=O; t->I=I; t->gs=gs; t->q4=qalloc(nb); t->s=falloc((int64_t)O*ng); } st_read_raw(&m->S,name,t->q4,drop); } else { if(t->fmt!=fmt||!t->q4){ t->fmt=fmt; t->O=O; t->I=I; t->gs=0; t->q4=qalloc(nb); t->s=qsalloc(O); } st_read_raw(&m->S,name,t->q4,drop); } - st_read_f32(&m->S,sn,t->s,drop); + st_read_f32_cap(&m->S,sn,t->s, fmt==4?(int64_t)O*((I+gs-1)/gs):(int64_t)O, drop); } else { if(!t->qf && !t->q8 && !t->q4) qt_alloc(t,O,I,bits); - if(t->fmt==0) st_read_f32(&m->S,name,t->qf,drop); - else { float *tmp=falloc((int64_t)O*I); st_read_f32(&m->S,name,tmp,drop); qt_fill(t,tmp,bits); free(tmp); } + if(t->fmt==0) st_read_f32_cap(&m->S,name,t->qf,(int64_t)O*I,drop); + else { float *tmp=falloc((int64_t)O*I); st_read_f32_cap(&m->S,name,tmp,(int64_t)O*I,drop); qt_fill(t,tmp,bits); free(tmp); } } } static QT qt_load(Model *m, const char *name, int O, int I, int bits){ @@ -1678,6 +1698,7 @@ static void model_init(Model *m, const char *snap, int cap, int ebits, int dbits /* embed: dequantizza la riga del token (scala per-riga) in x[hidden] */ static void embed_row(Model *m, int tok, float *x){ int D=m->c.hidden; QT *e=&m->embed; + if(tok<0 || tok>=e->O){ memset(x,0,(size_t)D*sizeof(float)); return; } /* #SEC-5: out-of-range token id -> zero row, never OOB */ if(e->fmt==0){ memcpy(x, e->qf+(int64_t)tok*D, D*sizeof(float)); return; } if(e->fmt==1){ const int8_t *q=e->q8+(int64_t)tok*D; float s=e->s[tok]; for(int i=0;inbytes); if(gs>0) fmt=4; + qt_check_fmt(nm[k],fmt,gs,OO[k],II[k],nb,tq[k]->nbytes); /* reject crafted size/format mismatch */ qt[k]->fmt=fmt; qt[k]->O=OO[k]; qt[k]->I=II[k]; qt[k]->gs=gs; qt[k]->qf=NULL; qt[k]->q8=(int8_t*)((char*)bw[k]+tw[k]->off); qt[k]->q4=(uint8_t*)((char*)bw[k]+tw[k]->off); qt[k]->s=(float*)((char*)bq[k]+tq[k]->off); @@ -1935,6 +1957,7 @@ static int expert_load_impl(Model *m, int layer, int eid, ESlot *s, int fatal){ int gs=0; if(fmt==2) gs=detect_group_size(OO[k],II[k],tq[k]->nbytes); if(gs>0) fmt=4; + qt_check_fmt(nm[k],fmt,gs,OO[k],II[k],nb,tq[k]->nbytes); /* reject crafted size/format mismatch */ qt[k]->fmt=fmt; qt[k]->O=OO[k]; qt[k]->I=II[k]; qt[k]->gs=gs; qt[k]->qf=NULL; qt[k]->q8=(int8_t*)(s->slab+pos[k]); qt[k]->q4=s->slab+pos[k]; qt[k]->s=fp[k]; } diff --git a/c/json.h b/c/json.h index 8a35e244..ef590536 100644 --- a/c/json.h +++ b/c/json.h @@ -55,10 +55,12 @@ static jval *j_parse_val(jparser *p); static char *j_parse_str_raw(jparser *p) { /* assume *p->s == '"' */ p->s++; - const char *start = p->s; - /* trova la fine gestendo gli escape, poi copia decodificando i casi base */ - char tmp[1 << 16]; int n = 0; - #define J_PUT(ch) do{ if (n < (int)sizeof(tmp)-1) tmp[n++] = (char)(ch); }while(0) + /* buffer su heap che CRESCE: niente troncamento silenzioso a 64KB (le stringhe + * lunghe di tokenizer.json/config venivano tagliate) e niente 64KB di stack. */ + size_t cap = 64, n = 0; char *tmp = (char *)malloc(cap); + if (!tmp) { fprintf(stderr, "OOM parsing JSON string\n"); exit(1); } + #define J_PUT(ch) do{ if (n + 1 >= cap) { cap *= 2; tmp = (char *)realloc(tmp, cap); \ + if (!tmp) { fprintf(stderr, "OOM parsing JSON string\n"); exit(1); } } tmp[n++] = (char)(ch); }while(0) while (*p->s && *p->s != '"') { char c = *p->s++; if (c == '\\' && *p->s) { @@ -69,9 +71,11 @@ static char *j_parse_str_raw(jparser *p) { case 'f': c = '\f'; break; case '/': c = '/'; break; case '\\': c = '\\'; break; case '"': c = '"'; break; case 'u': { /* \uXXXX -> codepoint UTF-8 (con coppie surrogate) */ + if (!p->s[0]||!p->s[1]||!p->s[2]||!p->s[3]) { c='?'; break; } /* \u troncato: non leggere oltre il NUL */ unsigned cp = (unsigned)strtoul((char[]){p->s[0],p->s[1],p->s[2],p->s[3],0}, NULL, 16); p->s += 4; - if (cp >= 0xD800 && cp <= 0xDBFF && p->s[0]=='\\' && p->s[1]=='u') { + if (cp >= 0xD800 && cp <= 0xDBFF && p->s[0]=='\\' && p->s[1]=='u' + && p->s[2] && p->s[3] && p->s[4] && p->s[5]) { unsigned lo = (unsigned)strtoul((char[]){p->s[2],p->s[3],p->s[4],p->s[5],0}, NULL, 16); if (lo >= 0xDC00 && lo <= 0xDFFF) { cp = 0x10000 + ((cp-0xD800)<<10) + (lo-0xDC00); p->s += 6; } } @@ -88,8 +92,8 @@ static char *j_parse_str_raw(jparser *p) { } #undef J_PUT if (*p->s == '"') p->s++; - (void)start; - return j_dup(p, tmp, n); + char *out = j_dup(p, tmp, (int)n); free(tmp); + return out; } static jval *j_parse_val(jparser *p) { @@ -135,9 +139,9 @@ static jval *j_parse_val(jparser *p) { p->depth--; return v; } - if (c == 't') { p->s += 4; jval *v = j_new(J_BOOL); v->boolean = 1; return v; } - if (c == 'f') { p->s += 5; jval *v = j_new(J_BOOL); v->boolean = 0; return v; } - if (c == 'n') { p->s += 4; return j_new(J_NULL); } + if (c == 't' && !strncmp(p->s, "true", 4)) { p->s += 4; jval *v = j_new(J_BOOL); v->boolean = 1; return v; } + if (c == 'f' && !strncmp(p->s, "false", 5)) { p->s += 5; jval *v = j_new(J_BOOL); v->boolean = 0; return v; } + if (c == 'n' && !strncmp(p->s, "null", 4)) { p->s += 4; return j_new(J_NULL); } /* numero */ { char *end; double d = strtod(p->s, &end); p->s = end; jval *v = j_new(J_NUM); v->num = d; return v; } } diff --git a/c/openai_server.py b/c/openai_server.py index d55fbc84..94095d63 100644 --- a/c/openai_server.py +++ b/c/openai_server.py @@ -86,6 +86,7 @@ def __init__(self, max_queue=8, queue_timeout=300, capacity=1): @contextlib.contextmanager def admit(self, cancelled=None, slot=None): ticket = object() + entry = (ticket, slot) # (#B2) remember each waiter's target slot for fair, per-slot admission queued_at = time.monotonic() with self.condition: if self.closed: @@ -95,31 +96,46 @@ def admit(self, cancelled=None, slot=None): self.rejected += 1 raise APIError(429, "The inference queue is full.", None, "queue_full", "rate_limit_error", {"Retry-After": "1"}) - self.queue.append(ticket) + self.queue.append(entry) deadline = queued_at + self.queue_timeout while True: if self.closed: - self.queue.remove(ticket) + self.queue.remove(entry) self.condition.notify_all() raise APIError(503, "The inference scheduler is shutting down.", None, "scheduler_closed", "server_error") available = min(self.free_slots) if slot is None and self.free_slots else slot - if self.queue[0] is ticket and available in self.free_slots: + # (#B2) Admit as soon as our target slot is free AND no strictly-earlier + # waiter also wants it (an earlier waiter "wants" it if it is any-slot or + # pinned to the same slot). This replaces the old strict FIFO-head rule, + # which let a head pinned to a busy slot block every request behind it — + # even ones targeting a currently-free slot (head-of-line blocking). + # ponytail: O(queue) scan per wakeup — negligible at the default max_queue; + # switch to per-slot wait sets if max_queue is ever raised to thousands. + can_admit = available in self.free_slots + if can_admit: + for t2, s2 in self.queue: + if t2 is ticket: + break + if s2 is None or s2 == available: + can_admit = False + break + if can_admit: break if cancelled and cancelled(): - self.queue.remove(ticket) + self.queue.remove(entry) self.cancelled += 1 self.condition.notify_all() raise ClientCancelled() remaining = deadline - time.monotonic() if remaining <= 0: - self.queue.remove(ticket) + self.queue.remove(entry) self.timed_out += 1 self.condition.notify_all() raise APIError(429, "Timed out waiting for the inference engine.", None, "queue_timeout", "rate_limit_error", {"Retry-After": "1"}) self.condition.wait(min(remaining, 0.25)) - self.queue.popleft() + self.queue.remove(entry) self.free_slots.remove(available) self.active += 1 self.admitted += 1 @@ -715,14 +731,40 @@ def send_cors_headers(self): if "*" not in self.server.cors_origins: self.send_header("Vary", "Origin") + LOOPBACK_HOSTS = {"127.0.0.1", "localhost", "::1", ""} + + def _is_authed(self): + """True if no key is configured, or a correct Bearer key was presented.""" + if not self.server.api_key: + return True + import hmac + provided = self.headers.get("Authorization", "") + return hmac.compare_digest(provided, f"Bearer {self.server.api_key}") + def require_auth(self): - if self.server.api_key: - import hmac - provided = self.headers.get("Authorization", "") - expected = f"Bearer {self.server.api_key}" - if not hmac.compare_digest(provided, expected): - raise APIError(401, "Invalid or missing API key.", None, "invalid_api_key", - "authentication_error") + if not self._is_authed(): + raise APIError(401, "Invalid or missing API key.", None, "invalid_api_key", + "authentication_error") + + def _check_host(self): + """DNS-rebinding guard: a web page can resolve a hostname to 127.0.0.1 and + drive this local server unless we pin the Host header to loopback / the bind + address. Rejects requests whose Host is anything else. (#SEC-7)""" + host = self.headers.get("Host", "") + if host.startswith("["): + name = host[1:].split("]", 1)[0] # [ipv6]:port + elif host.count(":") == 1: + name = host.rsplit(":", 1)[0] # host:port / ipv4:port + else: + name = host # bare host / bracketless ipv6 + name = name.strip().lower() + allowed = set(self.LOOPBACK_HOSTS) + try: + allowed.add(str(self.server.server_address[0]).strip("[]").lower()) + except Exception: + pass + if name not in allowed: + raise APIError(403, "Host header not allowed.", None, "forbidden") def read_json(self): try: @@ -780,20 +822,26 @@ def serve_static(self, path): def do_GET(self): request_id = "req_" + uuid.uuid4().hex try: + self._check_host() path = urlsplit(self.path).path if path == "/health": - payload = {"status": "ok", "scheduler": self.server.scheduler.snapshot(), - "kv_slots": self.server.kv_slots} - tiers = getattr(self.server.engine, "tiers", None) if self.server.engine else None - if tiers: payload["tiers"] = tiers - hwinfo = getattr(self.server.engine, "hwinfo", None) if self.server.engine else None - if hwinfo: payload["hwinfo"] = hwinfo + # Liveness is always public; hardware/scheduler internals only when a + # request is authed (or no key set), so a configured key isn't leaked + # past a bare 200 to an unauthenticated probe. (#SEC-8) + payload = {"status": "ok"} + if self._is_authed(): + payload["scheduler"] = self.server.scheduler.snapshot() + payload["kv_slots"] = self.server.kv_slots + tiers = getattr(self.server.engine, "tiers", None) if self.server.engine else None + if tiers: payload["tiers"] = tiers + hwinfo = getattr(self.server.engine, "hwinfo", None) if self.server.engine else None + if hwinfo: payload["hwinfo"] = hwinfo self.send_json(200, payload, request_id) return if path == "/experts": - eng = self.server.engine payload = {"rows": 0, "cols": 0, "map": "", "hits": "", "seq": 0} - if eng and getattr(eng, "emap", None): + eng = self.server.engine + if self._is_authed() and eng and getattr(eng, "emap", None): # (#SEC-8) hide routing telemetry unless authed payload.update(eng.emap) payload["hits"] = eng.hits or "" payload["seq"] = eng.hits_seq @@ -819,6 +867,13 @@ def do_GET(self): self.send_json(error.status, error_object(error), request_id, error.headers) def do_OPTIONS(self): + try: # (#SEC-7) apply the Host guard uniformly, incl. CORS preflight + self._check_host() + except APIError: + self.send_response(403) + self.send_header("Content-Length", "0") + self.end_headers() + return self.send_response(204) self.send_header("Content-Length", "0") self.send_cors_headers() @@ -827,6 +882,7 @@ def do_OPTIONS(self): def do_POST(self): request_id = "req_" + uuid.uuid4().hex try: + self._check_host() self.require_auth() body = self.read_json() self.check_model(body) @@ -1027,11 +1083,12 @@ def emit_plain(chunk): if include_usage: event([], self.usage(stats)) if connected: - try: - self.wfile.write(b"data: [DONE]\n\n") - self.wfile.flush() - except OSError: - pass + with ka_lock: # (#B9) share the pump's lock so [DONE] can't interleave a keepalive write + try: + self.wfile.write(b"data: [DONE]\n\n") + self.wfile.flush() + except OSError: + pass self.close_connection = True def client_disconnected(self): @@ -1094,7 +1151,15 @@ def serve(model, host="127.0.0.1", port=8000, model_id="glm-5.2-colibri", api_ke if not 1 <= kv_slots <= 16: raise ValueError("kv_slots must be between 1 and 16") if host not in ("127.0.0.1", "localhost", "::1") and not api_key: - print("WARNING: API is listening beyond localhost without COLI_API_KEY", file=sys.stderr) + # (#SEC-6) Fail closed: an unauthenticated engine on a non-loopback bind exposes + # a compute-heavy API to the network. Refuse unless explicitly overridden. + if os.environ.get("COLI_ALLOW_INSECURE_BIND") == "1": + print("WARNING: binding %s beyond localhost with NO auth (COLI_ALLOW_INSECURE_BIND=1)" % host, + file=sys.stderr) + else: + print("refusing to bind %s beyond localhost without COLI_API_KEY set " + "(set COLI_ALLOW_INSECURE_BIND=1 to override)" % host, file=sys.stderr) + sys.exit(1) origins = DEFAULT_CORS_ORIGINS if cors_origins is None else tuple(cors_origins) # Bind before starting the 744B engine. A stale/occupied port must fail in # milliseconds rather than loading hundreds of GB and leaking a child. diff --git a/c/st.h b/c/st.h index efa6e5a0..67b48aa8 100644 --- a/c/st.h +++ b/c/st.h @@ -200,11 +200,26 @@ static void st_init(shards *S, const char *snap_dir) { if (a0 < 0 || b0 < a0 || data_start + b0 > fsz) { fprintf(stderr, "%s: tensor '%s' data_offsets [%lld,%lld] out of file bounds (%lld)\n", files[fi], name, (long long)a0, (long long)b0, (long long)fsz); exit(1); } - int64_t numel = 1; for (int k = 0; k < shp->len; k++) numel *= (int64_t)shp->kids[k]->num; + int64_t numel = 1; + for (int k = 0; k < shp->len; k++) { + int64_t dim = (int64_t)shp->kids[k]->num; + if (dim < 0 || (dim && numel > INT64_MAX / dim)) { /* shape-product overflow from a crafted header */ + fprintf(stderr, "%s: tensor '%s' shape overflows int64\n", files[fi], name); exit(1); } + numel *= dim; + } if (S->n == S->cap) { S->cap *= 2; S->t = realloc(S->t, S->cap*sizeof(st_tensor)); } st_tensor *t = &S->t[S->n++]; t->name = strdup(name); t->fd = fd; t->off = data_start + a0; t->nbytes = b0 - a0; t->dtype = st_dtype_code(dt->str); t->numel = numel; + /* cross-check the declared element count against the byte span for FLOAT + * dtypes: st_read_f32 writes `numel` floats (BF16/F16 loop or F32 memcpy) + * into a caller-sized buffer, so a header with numel != nbytes/esz is an + * OOB write primitive. U8/I8 (raw quant bytes) are read by byte count, so + * their numel is unused by the read path and legitimately may differ. */ + { int esz = t->dtype==2 ? 4 : (t->dtype==3 ? 1 : 2); + if (t->dtype != 3 && t->nbytes != numel * (int64_t)esz) { + fprintf(stderr, "%s: tensor '%s' numel %lld disagrees with byte span %lld (esz %d)\n", + files[fi], name, (long long)numel, (long long)t->nbytes, esz); exit(1); } } } free(arena); /* i jval restano leakati: ok, una tantum all'avvio */ free(hdr); @@ -264,6 +279,20 @@ static int64_t st_read_f32(shards *S, const char *name, float *out, int drop) { return t->numel; } +/* like st_read_f32 but refuses to write more than `cap` floats into `out`. + * Callers that size `out` from CONFIG dims (qt_alloc's O*I, per-row scales) must + * use this: a crafted header whose tensor holds more elements than the config + * shape would otherwise overrun `out`. Callers that size `out` from st_numel are + * self-consistent and may keep using st_read_f32. */ +static int64_t st_read_f32_cap(shards *S, const char *name, float *out, int64_t cap, int drop) { + st_tensor *t = st_find(S, name); + if (!t) { fprintf(stderr, "missing tensor: %s\n", name); exit(1); } + if (t->numel > cap) { + fprintf(stderr, "tensor %s: numel %lld exceeds destination capacity %lld\n", + name, (long long)t->numel, (long long)cap); exit(1); } + return st_read_f32(S, name, out, drop); +} + static int64_t st_numel(shards *S, const char *name) { st_tensor *t = st_find(S, name); return t ? t->numel : -1; } @@ -287,9 +316,13 @@ static void st_read_slice_f32(shards *S, const char *name, int64_t elem_off, int st_tensor *t = st_find(S, name); if (!t) { fprintf(stderr, "missing tensor: %s\n", name); exit(1); } int esz = (t->dtype == 2) ? 4 : 2; + if (elem_off < 0 || n_elems < 0 || elem_off + n_elems > t->numel) { /* keep the slice inside the tensor */ + fprintf(stderr, "slice %s [%lld,+%lld) out of tensor bounds (numel %lld)\n", + name, (long long)elem_off, (long long)n_elems, (long long)t->numel); exit(1); } int64_t boff = t->off + elem_off * esz, nb = n_elems * esz; void *raw = malloc(nb); - st_pread_full(t->fd, raw, nb, boff, "pread slice"); + if (!raw) { fprintf(stderr, "malloc %lld bytes for slice %s failed\n", (long long)nb, name); exit(1); } + st_pread_full(t->fd, raw, nb, boff, "pread slice"); /* dev #331: chunked + EINTR + honest short-read */ if (t->dtype == 2) memcpy(out, raw, nb); else if (t->dtype == 0) { uint16_t *p = raw; for (int64_t i = 0; i < n_elems; i++) out[i] = bf16_to_f32(p[i]); } else { uint16_t *p = raw; for (int64_t i = 0; i < n_elems; i++) out[i] = f16_to_f32(p[i]); } diff --git a/c/tok.h b/c/tok.h index 7d1140bb..df8e59c2 100644 --- a/c/tok.h +++ b/c/tok.h @@ -88,8 +88,14 @@ static void tk_build_bytemap(Tok *T){ /* ---------- caricamento tokenizer.json ---------- */ static char *tk_read_file(const char *path, long *out_n){ FILE *f=fopen(path,"rb"); if(!f){ perror(path); exit(1); } - fseek(f,0,SEEK_END); long n=ftell(f); fseek(f,0,SEEK_SET); - char *b=malloc(n+1); if(fread(b,1,n,f)!=(size_t)n){} b[n]=0; fclose(f); if(out_n)*out_n=n; return b; + if(fseek(f,0,SEEK_END)!=0){ perror(path); exit(1); } + long n=ftell(f); + if(n<0){ perror(path); exit(1); } + if(n>(1L<<30)){ fprintf(stderr,"%s: file too large (%ld bytes)\n",path,n); exit(1); } /* sanity cap vs a hostile size */ + if(fseek(f,0,SEEK_SET)!=0){ perror(path); exit(1); } + char *b=malloc((size_t)n+1); if(!b){ fprintf(stderr,"OOM reading %s (%ld bytes)\n",path,n); exit(1); } + if(fread(b,1,(size_t)n,f)!=(size_t)n){ fprintf(stderr,"%s: short read\n",path); exit(1); } + b[n]=0; fclose(f); if(out_n)*out_n=n; return b; } static int cmp_sp_len(const void *a, const void *b){ return ((const Special*)b)->len - ((const Special*)a)->len; } @@ -104,10 +110,19 @@ static void tok_load(Tok *T, const char *path){ jval *added=json_get(root,"added_tokens"); if(!vocab||!merges){ fprintf(stderr,"tokenizer.json: missing model.vocab/merges\n"); exit(1); } - /* id massimo per dimensionare id2str */ + /* id massimo per dimensionare id2str. Gli id vengono da un tokenizer.json di + * mirror non fidato: un id NEGATIVO indicizzerebbe id2str[id] SOTTO l'allocazione + * (OOB write) e un added_token privo di "id"/"content" darebbe NULL-deref. */ int maxid=0; - for(int i=0;ilen;i++){ int id=(int)vocab->kids[i]->num; if(id>maxid)maxid=id; } - if(added) for(int i=0;ilen;i++){ int id=(int)json_get(added->kids[i],"id")->num; if(id>maxid)maxid=id; } + for(int i=0;ilen;i++){ int id=(int)vocab->kids[i]->num; + if(id<0){ fprintf(stderr,"tokenizer.json: negative vocab id %d\n",id); exit(1); } + if(id>maxid)maxid=id; } + if(added) for(int i=0;ilen;i++){ + jval *ji=json_get(added->kids[i],"id"); + if(!ji||ji->t!=J_NUM){ fprintf(stderr,"tokenizer.json: added_token missing numeric id\n"); exit(1); } + int id=(int)ji->num; + if(id<0){ fprintf(stderr,"tokenizer.json: negative added id %d\n",id); exit(1); } + if(id>maxid)maxid=id; } T->n_ids=maxid+1; T->id2str=calloc(T->n_ids,sizeof(char*)); T->id_added=calloc(T->n_ids,sizeof(int)); @@ -136,7 +151,10 @@ static void tok_load(Tok *T, const char *path){ T->nsp=added->len; T->sp=calloc(T->nsp,sizeof(Special)); for(int i=0;ilen;i++){ jval *a=added->kids[i]; - char *content=json_get(a,"content")->str; int id=(int)json_get(a,"id")->num; + jval *jc=json_get(a,"content"), *ji=json_get(a,"id"); + if(!jc||jc->t!=J_STR||!jc->str||!ji||ji->t!=J_NUM){ fprintf(stderr,"tokenizer.json: malformed added_token\n"); exit(1); } + char *content=jc->str; int id=(int)ji->num; + if(id<0||id>maxid){ fprintf(stderr,"tokenizer.json: added id %d out of range\n",id); exit(1); } T->sp[i].str=content; T->sp[i].len=(int)strlen(content); T->sp[i].id=id; T->id2str[id]=content; T->id_added[id]=1; jval *sf=json_get(a,"special"); /* "special": true/false */ diff --git a/c/tools/download_glm52.py b/c/tools/download_glm52.py index aa0c791a..ebd3b2fe 100644 --- a/c/tools/download_glm52.py +++ b/c/tools/download_glm52.py @@ -18,12 +18,15 @@ from huggingface_hub import snapshot_download, HfApi REPO = "zai-org/GLM-5.2-FP8" +# Pin the model revision for supply-chain integrity: set GLM_REVISION to a commit SHA +# so a mutated/compromised upstream can't silently swap the weights you fetch. +REVISION = os.environ.get("GLM_REVISION", "main") DEST = os.environ.get("GLM_DIR", "/home/vincenzo/glm52") # su ext4 (/dev/sdd), MAI su /mnt/c def human(n): return f"{n/1e9:.0f} GB" def check(): - info = HfApi().repo_info(REPO, files_metadata=True) + info = HfApi().repo_info(REPO, revision=REVISION, files_metadata=True) tot = sum((s.size or 0) for s in info.siblings) sts = [s for s in info.siblings if s.rfilename.endswith(".safetensors")] free = shutil.disk_usage(os.path.dirname(DEST) or "/").free @@ -40,6 +43,7 @@ def download(): # resume_download e' implicito; in caso di interruzione, rilancia e riprende. snapshot_download( repo_id=REPO, + revision=REVISION, local_dir=DEST, allow_patterns=["*.safetensors", "*.json", "*.txt", "*.model"], max_workers=8, diff --git a/c/tools/requirements.txt b/c/tools/requirements.txt new file mode 100644 index 00000000..3d0087c6 --- /dev/null +++ b/c/tools/requirements.txt @@ -0,0 +1,15 @@ +# Python dependencies for the colibri offline converter / oracle / download tools +# (c/tools/*.py and c/*.py). Minimum-version pins; for a fully reproducible env, +# freeze exact versions into your own lockfile with `pip freeze > locked.txt`. +torch>=2.4 +safetensors>=0.4 +huggingface-hub>=0.24 +numpy>=1.26 +tokenizers>=0.20 +transformers>=5.11 # <5.11 applies split-half Llama RoPE and drifts vs the engine's interleaved MLA (issue #281) +datasets>=2.20 +requests>=2.31 + +# Optional download-fallback backends (only needed for download_fp8.py's alt paths): +# modelscope>=1.9 +# hf_transfer>=0.1 diff --git a/flake.nix b/flake.nix index 584a0886..c1b90792 100644 --- a/flake.nix +++ b/flake.nix @@ -1,6 +1,9 @@ { description = "colibrì — run GLM-5.2 (744B MoE) on a consumer machine with ~25 GB RAM"; + # Reproducibility: these inputs track a branch, so torch/numpy/etc. float across + # rebuilds. For deterministic builds run `nix flake lock` once and COMMIT the + # generated flake.lock (it pins each input to a commit SHA). (#D2) inputs = { nixpkgs.url = "github:NixOS/nixpkgs/nixos-26.05"; flake-utils.url = "github:numtide/flake-utils"; diff --git a/web/src/App.tsx b/web/src/App.tsx index a33cc880..3f86f74f 100644 --- a/web/src/App.tsx +++ b/web/src/App.tsx @@ -161,7 +161,8 @@ export default function App() { } catch (cause) { if (controller.signal.aborted) return setConnected(false) - setError(cause instanceof Error ? cause.message : "Could not reach the server.") + const msg = cause instanceof Error ? cause.message : "Could not reach the server." + setError(/401|unauthor/i.test(msg) ? "Authentication failed — check the API key." : msg) } finally { if (probeRef.current === controller) { probeRef.current = null; setConnecting(false) } } @@ -247,12 +248,12 @@ export default function App() {
Connection
- + -
{connected ? "Engine reachable" : "Not connected"}
+
{connected ? "Engine reachable" : connecting ? "Probing…" : "Not connected"}
@@ -339,7 +340,7 @@ export default function App() {

Ask the giant.
Keep the machine yours.

Connect to a local colibrì server and stream responses directly from your hardware. Nothing leaves the endpoint you choose.

- {["Explain how expert routing works", "Write a small C benchmark", "Compare RAM and VRAM caching"].map((item) => )} + {["Explain how expert routing works", "Write a small C benchmark", "Compare RAM and VRAM caching"].map((item) => )}
) : ( diff --git a/web/src/Brain.tsx b/web/src/Brain.tsx index 6811238f..b5635e74 100644 --- a/web/src/Brain.tsx +++ b/web/src/Brain.tsx @@ -27,6 +27,7 @@ export function Brain({ baseUrl, apiKey, connected }: { baseUrl: string; apiKey: const wrapRef = useRef(null) const [wrapSize, setWrapSize] = useState({ w: 1200, h: 700 }) const [data, setData] = useState(null) + const [probeErr, setProbeErr] = useState(false) // (#U3) surface /experts failures instead of an endless spinner const [atlas, setAtlas] = useState | null>(null) const [tip, setTip] = useState<{ x: number; y: number; row: number; col: number; tier: number; heat: number } | null>(null) const pulseRef = useRef(null) // per-expert pulse intensity 0..1 @@ -59,9 +60,11 @@ export function Brain({ baseUrl, apiKey, connected }: { baseUrl: string; apiKey: const poll = async () => { try { const res = await fetch(endpoint(base, "/experts"), { headers: apiKey ? { Authorization: `Bearer ${apiKey}` } : {} }) + if (!res.ok) throw new Error(`/experts ${res.status}`) const next = (await res.json()) as ExpertMap if (disposed || !next.rows) return setData(next) + setProbeErr(false) if (next.seq !== lastSeq.current && next.hits) { lastSeq.current = next.seq const n = next.rows * next.cols @@ -72,7 +75,7 @@ export function Brain({ baseUrl, apiKey, connected }: { baseUrl: string; apiKey: if (byte & (1 << (i & 7))) p[i] = 1 } } - } catch { /* engine busy or restarting — keep the last frame */ } + } catch { if (!disposed) setProbeErr(true) /* surface repeated failures; keep the last frame */ } } void poll() const t = window.setInterval(() => void poll(), 1500) @@ -154,13 +157,14 @@ export function Brain({ baseUrl, apiKey, connected }: { baseUrl: string; apiKey:
setTip(null)} /> {!connected &&

Connect to the engine to see the cortex.

} + {connected && probeErr && !data &&

Expert map unavailable — this engine build may not expose /experts.

}
{tip && data && (() => { const isMtp = tip.row === data.rows - 1 const realLayer = isMtp ? 78 : tip.row + 3 const entry = atlas?.[`${realLayer}:${tip.col}`] return ( -
+
Layer {realLayer}{isMtp ? " (MTP)" : ""} · Expert {tip.col}
Tier: {TIER_NAME[tip.tier]}
Heat: {tip.heat === 0 ? "never routed" : `~2^${tip.heat} selections`}
diff --git a/web/src/ErrorBoundary.tsx b/web/src/ErrorBoundary.tsx index f88b8a98..40df0647 100644 --- a/web/src/ErrorBoundary.tsx +++ b/web/src/ErrorBoundary.tsx @@ -9,11 +9,11 @@ export class ErrorBoundary extends Component<{ children: ReactNode }, State> { } render() { if (!this.state.error) return this.props.children - return
-

colibrì UI hit an error

-

The engine is unaffected. Try refreshing.

- 
{String(this.state.error)}
- + return
+

colibrì UI hit an error

+

The engine is unaffected. Try refreshing.

+ 
{String(this.state.error)}
+
} } diff --git a/web/src/index.css b/web/src/index.css index 8d777f41..ee315817 100644 --- a/web/src/index.css +++ b/web/src/index.css @@ -69,7 +69,7 @@ button:focus-visible, input:focus-visible, textarea:focus-visible, select:focus- .chat-panel { min-width: 0; height: 100vh; display: grid; grid-template-rows: 72px minmax(0, 1fr) auto; } .topbar { display: flex; align-items: center; justify-content: space-between; padding: 0 32px; border-bottom: 1px solid var(--border); } .topbar > div:first-child { min-width: 0; display: grid; gap: 3px; }.topbar strong { overflow: hidden; font: 500 14px ui-monospace, SFMono-Regular, monospace; text-overflow: ellipsis; white-space: nowrap; }.eyebrow { color: #718086; font-size: 9px; font-weight: 800; letter-spacing: .16em; } -.top-actions { display: flex; align-items: center; gap: 8px; }.top-actions [data-slot=badge] { gap: 5px; } +.top-actions { display: flex; flex-wrap: wrap; align-items: center; gap: 8px; }.top-actions [data-slot=badge] { gap: 5px; } .conversation { overflow-y: auto; } .empty-state { width: min(680px, calc(100% - 40px)); min-height: 100%; margin: 0 auto; display: flex; flex-direction: column; align-items: center; justify-content: center; padding: 56px 0; text-align: center; } .orb { position: relative; display: grid; place-items: center; width: 66px; height: 66px; margin-bottom: 24px; border: 1px solid rgba(78,214,165,.25); border-radius: 50%; color: var(--primary); background: radial-gradient(circle, rgba(78,214,165,.17), rgba(78,214,165,.02) 68%); box-shadow: 0 0 55px rgba(78,214,165,.1); }.orb svg { width: 25px; }.orb::after { content: ""; position: absolute; inset: -7px; border: 1px dashed rgba(78,214,165,.14); border-radius: inherit; } From df644dc1f758c1e7f077fd3090dd7830d751484c Mon Sep 17 00:00:00 2001 From: KingIcyCreamProjects Date: Fri, 17 Jul 2026 14:03:39 -0500 Subject: [PATCH 019/132] =?UTF-8?q?security:=20SEC-9=20=E2=80=94=20bound?= =?UTF-8?q?=20untrusted=20config/oracle=20JSON=20slurps?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit config.json arrives from unverified mirrors (same threat model as the qt_check_fmt / SEC-1 loader hardening). cfg_root() and the ref_glm.json oracle path both did an unbounded ftell -> malloc(n+1) -> fread with no size cap and no NULL check, so a hostile or oversized file caused a load-time OOM or, on malloc failure, a NULL-deref via b[got]=0. Add cfg_slurp(): 256 MB cap, NULL-checked alloc, full-read check (mirrors tok.h tk_read_file). Route both sites through it. config.json failure stays fatal (exit 1); the oracle path returns 1 as before. Found by an adversarial re-audit of the loader trust boundary. Co-Authored-By: Claude Opus 4.8 --- c/glm.c | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/c/glm.c b/c/glm.c index 4043d8f1..f1308260 100644 --- a/c/glm.c +++ b/c/glm.c @@ -1332,12 +1332,25 @@ static void rope_interleave(float *v, int pos, const Cfg *c){ } /* ---------- config ---------- */ +/* SEC-9: bounded slurp for untrusted config/oracle JSON. config.json arrives from + * unverified mirrors (see qt_check_fmt threat model); an unbounded ftell->malloc + * gave a hostile file a load-time OOM or, on malloc failure, a NULL deref via + * b[got]=0. Cap the size, NULL-check the alloc, require a full read. Returns a + * malloc'd NUL-terminated buffer, or NULL on any failure. Mirrors tok.h tk_read_file. */ +#define CFG_MAX_BYTES (256ll<<20) /* config/oracle JSON is KB-MB in practice */ +static char* cfg_slurp(const char *path){ + FILE *f=fopen(path,"rb"); if(!f) return NULL; + fseek(f,0,SEEK_END); long n=ftell(f); fseek(f,0,SEEK_SET); + if(n<0 || (long long)n>CFG_MAX_BYTES){ fclose(f); return NULL; } + char *b=malloc((size_t)n+1); if(!b){ fclose(f); return NULL; } + size_t got=fread(b,1,(size_t)n,f); fclose(f); + if((long)got!=n){ free(b); return NULL; } + b[got]=0; return b; +} static jval* cfg_root(const char *snap, char **arena){ char p[2048]; snprintf(p,sizeof(p),"%s/config.json",snap); - FILE *f=fopen(p,"rb"); if(!f){perror(p);exit(1);} - fseek(f,0,SEEK_END); long n=ftell(f); fseek(f,0,SEEK_SET); - char *b=malloc(n+1); size_t got=fread(b,1,n,f); b[got]=0; fclose(f); - if((long)got!=n) fprintf(stderr,"warning: short read on %s (%ld of %ld)\n",p,(long)got,n); + char *b=cfg_slurp(p); + if(!b){ fprintf(stderr,"%s: cannot read config.json (missing, unreadable, short, or > %lld bytes)\n",p,(long long)CFG_MAX_BYTES); exit(1); } return json_parse(b,arena); } static int gi(jval*r,const char*k){ jval*v=json_get(r,k); return v?(int)v->num:0; } @@ -6528,10 +6541,8 @@ int main(int argc, char **argv){ /* altrimenti: validazione contro l'oracolo (ref_glm.json) */ const char *refpath=getenv("REF")?getenv("REF"):"ref_glm.json"; - FILE *f=fopen(refpath,"rb"); if(!f){perror(refpath);return 1;} - fseek(f,0,SEEK_END); long n=ftell(f); fseek(f,0,SEEK_SET); - char *b=malloc(n+1); size_t got=fread(b,1,n,f); b[got]=0; fclose(f); - if((long)got!=n) fprintf(stderr,"warning: short read on %s (%ld of %ld)\n",refpath,(long)got,n); + char *b=cfg_slurp(refpath); + if(!b){ fprintf(stderr,"%s: cannot read oracle file (missing, unreadable, short, or > %lld bytes)\n",refpath,(long long)CFG_MAX_BYTES); return 1; } char *ar=NULL; jval *ref=json_parse(b,&ar); int np=0,nfull=0; int *prompt=read_arr(ref,"prompt_ids",&np); int *full=read_arr(ref,"full_ids",&nfull); if(!prompt||!full||np<1||nfull Date: Fri, 17 Jul 2026 20:21:18 -0500 Subject: [PATCH 020/132] =?UTF-8?q?security:=20extend=20the=20audit=20?= =?UTF-8?q?=E2=80=94=20olmoe=20loader,=20tokenizer,=20server=20slowloris?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up hardening found by a research+audit pass (corroborated against the llama.cpp GGUF overflow CVEs and safetensors "two-authorities" TOCTOU class). The original audit hardened glm.c thoroughly but left sibling surfaces open. - olmoe.c load_expert_w (CRITICAL): the dtype-3 (U8/I8) container path did st_read_raw(name,q) + st_read_f32(qs,scale) writing header-controlled byte counts into config-sized buffers (q=malloc(O*I), scale=falloc(O)). st_init skips its numel==nbytes/esz cross-check for dtype-3, so a crafted .safetensors with an over-declared U8 weight or oversized .qs was a controlled heap OOB write — the same SEC-1 primitive glm.c's qt_check_fmt already rejects. Add the matching size check (weight bytes == O*I, scale elems == O) in the shared loader; reject, never repair. - olmoe.c load_cfg: unbounded ftell->malloc slurp of config.json (SEC-9 class, already fixed in glm.c) and json_get(...)->num deref with no NULL check (a missing key = NULL-deref). Add a 256 MB cap + NULL/short-read guards, a req_num() that exits on missing/non-numeric keys, and CKR-style range checks so bad dims can't drive a later malloc(inter*hidden) or a div-by-zero on n_heads. - tok.h (hostile tokenizer.json): type-check vocab ids and merge-pair entries as J_NUM / J_ARR-of-J_STR before dereferencing (was a NULL-deref / silent id=0), and cap maxid so n_ids=maxid+1 can't signed-overflow into a multi-GB calloc. Verified the real 151K-vocab GLM tokenizer still loads + encodes unchanged. - openai_server.py: set APIHandler.timeout=30 — a slowloris client that dribbles its request line/body could pin a worker thread (and a KV slot) forever, before the scheduler's queue bound (B2) applies. Deferred (not shipped here): the Linux io_uring finalize path (uring_finalize_load) has the same missing qt_check_fmt + no fmt=4 handling — a 5th format site, but it is #ifdef __linux__ and cannot be compiled/tested on this Windows box, so it needs its own Linux-gated PR. /profile stays public by design (a test asserts it) — maintainer's call. Co-Authored-By: Claude Opus 4.8 --- c/olmoe.c | 47 +++++++++++++++++++++++++++++++++++++--------- c/openai_server.py | 2 ++ c/tok.h | 10 +++++++++- 3 files changed, 49 insertions(+), 10 deletions(-) diff --git a/c/olmoe.c b/c/olmoe.c index cda5913d..84d584e8 100644 --- a/c/olmoe.c +++ b/c/olmoe.c @@ -157,20 +157,35 @@ static void softmax_row(float *x, int n) { } /* ---------- caricamento ---------- */ +/* config.json arrives from an untrusted mirror: a missing key was a NULL-deref + * (json_get(...)->num), so require each dimension present + numeric. */ +static double req_num(jval *r, const char *k){ + jval *v=json_get(r,k); + if(!v||v->t!=J_NUM){ fprintf(stderr,"config.json: missing or non-numeric \"%s\"\n",k); exit(1); } + return v->num; +} static void load_cfg(Cfg *c, const char *snap) { char path[2048]; snprintf(path, sizeof(path), "%s/config.json", snap); FILE *f = fopen(path, "rb"); if(!f){perror(path);exit(1);} fseek(f,0,SEEK_END); long n=ftell(f); fseek(f,0,SEEK_SET); - char *buf = malloc(n+1); if(fread(buf,1,n,f)!=(size_t)n){} buf[n]=0; fclose(f); + if(n<0 || n>(256L<<20)){ fprintf(stderr,"%s: config.json missing or larger than 256 MB\n",path); exit(1); } /* SEC-9 */ + char *buf = malloc((size_t)n+1); if(!buf){ fprintf(stderr,"OOM reading %s\n",path); exit(1); } + if(fread(buf,1,(size_t)n,f)!=(size_t)n){ fprintf(stderr,"%s: short read\n",path); exit(1); } buf[n]=0; fclose(f); char *arena=NULL; jval *r = json_parse(buf, &arena); - c->hidden = (int)json_get(r,"hidden_size")->num; - c->n_layers = (int)json_get(r,"num_hidden_layers")->num; - c->n_heads = (int)json_get(r,"num_attention_heads")->num; - c->n_kv_heads= (int)json_get(r,"num_key_value_heads")->num; - c->n_experts = (int)json_get(r,"num_experts")->num; - c->topk = (int)json_get(r,"num_experts_per_tok")->num; - c->inter = (int)json_get(r,"intermediate_size")->num; - c->vocab = (int)json_get(r,"vocab_size")->num; + c->hidden = (int)req_num(r,"hidden_size"); + c->n_layers = (int)req_num(r,"num_hidden_layers"); + c->n_heads = (int)req_num(r,"num_attention_heads"); + c->n_kv_heads= (int)req_num(r,"num_key_value_heads"); + c->n_experts = (int)req_num(r,"num_experts"); + c->topk = (int)req_num(r,"num_experts_per_tok"); + c->inter = (int)req_num(r,"intermediate_size"); + c->vocab = (int)req_num(r,"vocab_size"); + /* range-check so bad dims can't drive a later malloc(inter*hidden) / div-by-zero */ + if(c->hidden<1||c->hidden>(1<<20) || c->n_heads<1||c->n_heads>(1<<16) || + c->inter<1||c->inter>(1<<24) || c->vocab<1||c->vocab>(1<<24) || + c->n_layers<1||c->n_layers>4096 || c->n_experts<1||c->n_experts>(1<<20) || + c->n_kv_heads<1 || c->topk<1||c->topk>c->n_experts){ + fprintf(stderr,"config.json: dimension out of range\n"); exit(1); } c->head_dim = c->hidden / c->n_heads; jval *th = json_get(r,"rope_theta"); c->theta = th ? (float)th->num : 10000.f; jval *ep = json_get(r,"rms_norm_eps"); c->eps = ep ? (float)ep->num : 1e-5f; @@ -222,6 +237,20 @@ static void load_expert_w(Model *m, const char *name, int8_t *q, float *scale, i st_tensor *t = st_find(&m->S, name); if (t && t->dtype == 3) { /* I8/U8: container colibri */ char qs[300]; snprintf(qs, sizeof(qs), "%s.qs", name); + /* SEC-1 (olmoe): st_init skips its numel==nbytes/esz cross-check for dtype-3 + * (U8/I8) tensors, so a crafted header could declare nbytes >> the config-sized + * destination and st_read_raw would write past q[O*I]; likewise an oversized .qs + * would overflow scale[O]. q is malloc(O*I) (int8 = 1 byte/elem) and scale is + * falloc(O), so require the raw span == O*I bytes and the scale == O elems. + * Reject, never repair — mirrors glm.c qt_check_fmt. */ + st_tensor *ts = st_find(&m->S, qs); + int64_t want_w = (int64_t)O * I; + if (t->nbytes != want_w) { + fprintf(stderr, "%s: I8 weight bytes %lld != expected %lld [O=%d,I=%d]\n", + name, (long long)t->nbytes, (long long)want_w, O, I); exit(1); } + if (!ts || ts->numel != (int64_t)O) { + fprintf(stderr, "%s: scale elems %lld != expected %d\n", + qs, (long long)(ts ? ts->numel : -1), O); exit(1); } st_read_raw(&m->S, name, q, 1); st_read_f32(&m->S, qs, scale, 1); return; diff --git a/c/openai_server.py b/c/openai_server.py index 94095d63..b49db9b0 100644 --- a/c/openai_server.py +++ b/c/openai_server.py @@ -700,6 +700,8 @@ def __init__(self, address, engine, model_id, api_key=None, max_tokens=1024, class APIHandler(BaseHTTPRequestHandler): protocol_version = "HTTP/1.1" + timeout = 30 # per-request socket timeout: a slowloris client that dribbles its + # request line/body can't pin a worker thread (and a slot) forever server_version = "colibri" def log_message(self, fmt, *args): diff --git a/c/tok.h b/c/tok.h index df8e59c2..e823e375 100644 --- a/c/tok.h +++ b/c/tok.h @@ -114,7 +114,9 @@ static void tok_load(Tok *T, const char *path){ * mirror non fidato: un id NEGATIVO indicizzerebbe id2str[id] SOTTO l'allocazione * (OOB write) e un added_token privo di "id"/"content" darebbe NULL-deref. */ int maxid=0; - for(int i=0;ilen;i++){ int id=(int)vocab->kids[i]->num; + for(int i=0;ilen;i++){ + if(vocab->kids[i]->t!=J_NUM){ fprintf(stderr,"tokenizer.json: non-numeric vocab id at %d\n",i); exit(1); } + int id=(int)vocab->kids[i]->num; if(id<0){ fprintf(stderr,"tokenizer.json: negative vocab id %d\n",id); exit(1); } if(id>maxid)maxid=id; } if(added) for(int i=0;ilen;i++){ @@ -123,10 +125,13 @@ static void tok_load(Tok *T, const char *path){ int id=(int)ji->num; if(id<0){ fprintf(stderr,"tokenizer.json: negative added id %d\n",id); exit(1); } if(id>maxid)maxid=id; } + /* an id near INT_MAX would overflow n_ids=maxid+1 (UB) and calloc multi-GB */ + if(maxid > (1<<21)){ fprintf(stderr,"tokenizer.json: implausible max vocab id %d\n",maxid); exit(1); } T->n_ids=maxid+1; T->id2str=calloc(T->n_ids,sizeof(char*)); T->id_added=calloc(T->n_ids,sizeof(int)); T->id_special=calloc(T->n_ids,sizeof(int)); + if(!T->id2str||!T->id_added||!T->id_special){ fprintf(stderr,"tokenizer.json: OOM sizing %d ids\n",T->n_ids); exit(1); } /* vocab: stringa -> id (capacita' potenza di 2, ~2-3x) */ int vc=1; while(vc < vocab->len*2) vc<<=1; @@ -141,6 +146,9 @@ static void tok_load(Tok *T, const char *path){ hm_init(&T->merges, mc); for(int i=0;ilen;i++){ jval *pr=merges->kids[i]; + if(!pr||pr->t!=J_ARR||pr->len<2||!pr->kids[0]||!pr->kids[1]|| + pr->kids[0]->t!=J_STR||pr->kids[1]->t!=J_STR){ + fprintf(stderr,"tokenizer.json: malformed merge entry %d\n",i); exit(1); } const char *l=pr->kids[0]->str, *r=pr->kids[1]->str; int ll=(int)strlen(l), rl=(int)strlen(r); char *key=malloc(ll+1+rl); memcpy(key,l,ll); key[ll]=0; memcpy(key+ll+1,r,rl); From d86a6b93ad4f70045138a13907682b5f936c3355 Mon Sep 17 00:00:00 2001 From: ZacharyZcR Date: Sun, 19 Jul 2026 06:10:14 +0800 Subject: [PATCH 021/132] packaging: pyproject.toml + editorconfig + clang-format Python packaging: - pyproject.toml: pip install colibri-engine (editable dev install works) - colibri/ package with __version__, CLI entry point delegating to c/coli - Optional dependency groups: [convert] (numpy, huggingface_hub), [oracle] (torch, transformers, safetensors), [bench] (tokenizers, datasets) Code style: - .editorconfig: consistent indent/charset across all file types - .clang-format: LLVM-based, 120 col, matches existing engine style Usage: pip install -e . # dev install (CLI + serve, no heavy deps) pip install -e .[convert] # adds converter dependencies pip install -e .[oracle] # adds torch/transformers for oracle validation --- .clang-format | 10 +++++++++ .editorconfig | 24 ++++++++++++++++++++ colibri/__init__.py | 5 +++++ colibri/_version.py | 3 +++ colibri/cli.py | 30 +++++++++++++++++++++++++ pyproject.toml | 53 +++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 125 insertions(+) create mode 100644 .clang-format create mode 100644 .editorconfig create mode 100644 colibri/__init__.py create mode 100644 colibri/_version.py create mode 100644 colibri/cli.py create mode 100644 pyproject.toml diff --git a/.clang-format b/.clang-format new file mode 100644 index 00000000..f9116525 --- /dev/null +++ b/.clang-format @@ -0,0 +1,10 @@ +BasedOnStyle: LLVM +IndentWidth: 4 +ColumnLimit: 120 +AllowShortFunctionsOnASingleLine: All +AllowShortIfStatementsOnASingleLine: AllIfsAndElse +AllowShortLoopsOnASingleLine: true +BreakBeforeBraces: Attach +PointerAlignment: Right +SpaceAfterCStyleCast: false +SortIncludes: false diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 00000000..e0acabb3 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,24 @@ +root = true + +[*] +charset = utf-8 +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true +indent_style = space +indent_size = 4 + +[*.{c,h,cu}] +indent_size = 4 + +[*.{ts,tsx,js,json,css}] +indent_size = 2 + +[Makefile] +indent_style = tab + +[*.yml] +indent_size = 2 + +[*.md] +trim_trailing_whitespace = false diff --git a/colibri/__init__.py b/colibri/__init__.py new file mode 100644 index 00000000..77f70615 --- /dev/null +++ b/colibri/__init__.py @@ -0,0 +1,5 @@ +"""colibrì — tiny engine, immense model.""" + +from colibri._version import __version__ + +__all__ = ["__version__"] diff --git a/colibri/_version.py b/colibri/_version.py new file mode 100644 index 00000000..9951ca18 --- /dev/null +++ b/colibri/_version.py @@ -0,0 +1,3 @@ +"""Single source of truth for the colibrì version number.""" + +__version__ = "1.0.0" diff --git a/colibri/cli.py b/colibri/cli.py new file mode 100644 index 00000000..3f192344 --- /dev/null +++ b/colibri/cli.py @@ -0,0 +1,30 @@ +"""Entry point for `coli` when installed via pip. + +Delegates to the original c/coli script which handles all subcommands. +This wrapper exists so `pip install colibri-engine` creates a `coli` console +script that works without the user having to add c/ to PATH manually. +""" + +import os +import sys +import runpy + + +def main(): + here = os.path.dirname(os.path.abspath(__file__)) + engine_dir = os.path.join(os.path.dirname(here), "c") + coli_script = os.path.join(engine_dir, "coli") + + if not os.path.exists(coli_script): + sys.exit( + "colibri engine directory not found.\n" + "Install from source: git clone + pip install -e ." + ) + + sys.path.insert(0, engine_dir) + sys.argv[0] = coli_script + runpy.run_path(coli_script, run_name="__main__") + + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..18f8f858 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,53 @@ +[build-system] +requires = ["setuptools>=68.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "colibri-engine" +dynamic = ["version"] +description = "Tiny engine, immense model — run GLM-5.2 (744B MoE) locally" +readme = "README.md" +license = "Apache-2.0" +requires-python = ">=3.10" +authors = [ + {name = "JustVugg"}, +] +classifiers = [ + "Development Status :: 4 - Beta", + "Environment :: Console", + "Intended Audience :: Science/Research", + "Operating System :: POSIX :: Linux", + "Operating System :: MacOS", + "Operating System :: Microsoft :: Windows", + "Programming Language :: Python :: 3", + "Topic :: Scientific/Engineering :: Artificial Intelligence", +] + +[project.optional-dependencies] +convert = [ + "numpy", + "huggingface_hub", +] +oracle = [ + "torch>=2.0", + "transformers>=4.40", + "safetensors", +] +bench = [ + "tokenizers", + "datasets", +] + +[project.scripts] +coli = "colibri.cli:main" + +[project.urls] +Homepage = "https://github.com/JustVugg/colibri" +Issues = "https://github.com/JustVugg/colibri/issues" + +[tool.setuptools.dynamic] +version = {attr = "colibri._version.__version__"} + +[tool.setuptools.packages.find] +where = ["."] +include = ["colibri*"] From ca39e5333f1addb67df62b836ee2d9e661fd91e4 Mon Sep 17 00:00:00 2001 From: JustVugg Date: Sun, 19 Jul 2026 12:38:11 +0200 Subject: [PATCH 022/132] packaging: single version source + honest editable-install semantics (on top of #396) colibri/_version.py now reads c/version.py (#394's single source of truth -- coli --version, the release workflow, and pip metadata can no longer drift), with an importlib.metadata fallback for the installed-wheel case where c/ is not on disk. README documents that pip install -e . is the supported form: the engine lives in c/ and is not packaged into a standalone wheel. Verified in a clean venv: pip install -e . -> colibri.__version__ == 1.0.0 read from c/version.py, coli entrypoint on PATH and functional. Co-Authored-By: Claude Fable 5 --- README.md | 4 ++++ colibri/_version.py | 24 ++++++++++++++++++++++-- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 797f1ca1..e5d16517 100644 --- a/README.md +++ b/README.md @@ -183,6 +183,10 @@ COLI_MODEL=/nvme/glm52_i4 ./coli doctor # read-only readiness check The engine at runtime is pure C — python is only used by the one-time converter and the optional API gateway. +Prefer a `coli` command on your PATH? From a checkout, `pip install -e .` +registers it (the engine itself still lives in `c/` — this is an editable +install from the clone, not a standalone wheel). + ### 3. Go deeper | topic | doc | diff --git a/colibri/_version.py b/colibri/_version.py index 9951ca18..174442aa 100644 --- a/colibri/_version.py +++ b/colibri/_version.py @@ -1,3 +1,23 @@ -"""Single source of truth for the colibrì version number.""" +"""Version accessor for the pip package. -__version__ = "1.0.0" +The single source of truth is c/version.py (#394): coli --version and the +GitHub Release workflow read it, so the pip metadata must read the SAME file +instead of carrying a second literal that would drift on the first bump. + +From a checkout (the supported install: `pip install -e .`) the file is read +directly. From an installed wheel c/ is not on disk, so fall back to the +package metadata that setuptools baked at build time from that same file. +""" +from pathlib import Path + +try: + _ns = {} + exec((Path(__file__).resolve().parent.parent / "c" / "version.py").read_text(), _ns) + __version__ = _ns["__version__"] +except OSError: + from importlib.metadata import PackageNotFoundError, version + + try: + __version__ = version("colibri-engine") + except PackageNotFoundError: + __version__ = "0.0.0+unknown" From 72e36772f51605fa947efe380614c85a395c556c Mon Sep 17 00:00:00 2001 From: JustVugg Date: Sun, 19 Jul 2026 13:05:56 +0200 Subject: [PATCH 023/132] security: reject malformed model tensors at the untrusted-mirror boundary Colibri loads model directories and safetensors from mirrors it does not control, so the file's declared shapes and byte spans are attacker-influenced input. Three memory-safety holes on that boundary, independently confirmed (incl. a from-scratch adversarial audit that re-derived the same two) and present in the shipped v1.0.0: - st.h st_read_f32: numel came from the shape, nbytes from the offsets, with no cross-check. A crafted tensor whose shape inflates numel past nbytes made the BF16/F16 loop read past the malloc'd raw buffer and the F32 memcpy write past the caller's config-sized destination (heap OOB read + write). Now enforce numel*esz == nbytes before any copy. - st.h header parse: the shape product could overflow int64 to a small/negative numel that would then pass the cross-check. Guard each multiply. - glm.c qt_resolve_fmt (new, replaces the three duplicated "?1:?2:3" fmt sites in qt_from_disk and both expert_load arms): the old inference SILENTLY fell to int2 for any unrecognized weight byte count, so a too-short weight became a valid int2 whose matmul read O*I nibbles past the buffer; and an oversized scale array overflowed the per-row t->s. Now the weight bytes must match a known int8/int4/int2 layout and the scale array must match the expected per-row (O) or grouped (O*ng) cardinality, else refuse. - glm.c config/generation_config slurp: unbounded ftell -> malloc(n+1) gave a hostile file a load-time OOM, and on malloc failure b[got]=0 was a NULL deref. Cap at 256 MB and NULL-check. Verified: TF token-exactness unchanged on every quant format (full-precision 32/32, int4 11/32, int2 1/32, mix 5/32 -- byte-identical to the pre-change binary); fmt=4 grouped path preserved (the scale check is by construction the same condition detect_group_size already imposed); a hand-crafted hostile safetensors is refused cleanly; ASan+UBSan clean on legit and hostile loads (only the pre-existing intentional startup leaks remain). These are the C trust-boundary items of #368, landed as a minimal standalone fix; the server-side and build items of that PR follow via its rebase. Co-Authored-By: Claude Fable 5 --- c/glm.c | 51 ++++++++++++++++++++++++++++++++++++--------------- c/st.h | 23 ++++++++++++++++++++++- 2 files changed, 58 insertions(+), 16 deletions(-) diff --git a/c/glm.c b/c/glm.c index a3e7ab74..26e7fc49 100644 --- a/c/glm.c +++ b/c/glm.c @@ -1336,7 +1336,12 @@ static jval* cfg_root(const char *snap, char **arena){ char p[2048]; snprintf(p,sizeof(p),"%s/config.json",snap); FILE *f=fopen(p,"rb"); if(!f){perror(p);exit(1);} fseek(f,0,SEEK_END); long n=ftell(f); fseek(f,0,SEEK_SET); - char *b=malloc(n+1); size_t got=fread(b,1,n,f); b[got]=0; fclose(f); + /* SEC: config.json arriva dalla dir modello non fidata. Limita la dimensione + * (un file ostile enorme = OOM al load) e controlla la malloc: senza il NULL + * check, b[got]=0 su malloc fallita era un NULL-deref. */ + if(n<0 || n>(256L<<20)){ fprintf(stderr,"%s: size %ld out of range (0..256 MB)\n",p,n); exit(1); } + char *b=malloc((size_t)n+1); if(!b){ fprintf(stderr,"OOM reading %s (%ld bytes)\n",p,n); exit(1); } + size_t got=fread(b,1,(size_t)n,f); b[got]=0; fclose(f); if((long)got!=n) fprintf(stderr,"warning: short read on %s (%ld of %ld)\n",p,(long)got,n); return json_parse(b,arena); } @@ -1375,8 +1380,9 @@ static void load_cfg(Cfg *c, const char *snap){ FILE *gf=fopen(gp,"rb"); /* assente = nessun problema: e' opzionale */ if(gf){ fseek(gf,0,SEEK_END); long gn=ftell(gf); fseek(gf,0,SEEK_SET); - if(gn>0){ - char *gb=malloc(gn+1); size_t gg=fread(gb,1,gn,gf); gb[gg]=0; + char *gb = (gn>0 && gn<=(256L<<20)) ? malloc((size_t)gn+1) : NULL; /* SEC: cap + NULL check */ + if(gb){ + size_t gg=fread(gb,1,(size_t)gn,gf); gb[gg]=0; char *ga=NULL; jval *gr=json_parse(gb,&ga); jval *ge=gr?json_get(gr,"eos_token_id"):NULL; if(ge){ @@ -1445,6 +1451,27 @@ static int detect_group_size(int O, int I, int64_t ns){ return 0; } +/* SEC: risolve e VALIDA il formato quantizzato di un tensore [O,I] letto da un + * container non fidato (mirror). L'inferenza precedente (`?1:?2:3`) cadeva su + * int2 per QUALSIASI conteggio byte non riconosciuto: un peso troppo corto + * diventava un int2 valido e il matmul leggeva oltre il buffer (O*I nibble a + * 4/byte). Qui i byte del peso devono corrispondere a un layout noto e i byte + * della scala alla cardinalita' attesa (O per-row, O*ng per-gruppo) — altrimenti + * si termina invece di sforare. Ritorna fmt (1/2/3/4) e scrive *gs. */ +static int qt_resolve_fmt(const char *name, int O, int I, int64_t nb, int64_t ns, int *gs){ + int64_t exp_i8=(int64_t)O*I, exp_i4=(int64_t)O*((I+1)/2), exp_i2=(int64_t)O*((I+3)/4); + int fmt = (nb==exp_i8)?1 : (nb==exp_i4)?2 : (nb==exp_i2)?3 : 0; + if(!fmt){ + fprintf(stderr,"%s: quantized weight is %lld bytes — no int8/int4/int2 layout for [%d,%d], refusing (untrusted container)\n", + name,(long long)nb,O,I); exit(1); } + *gs=0; + if(fmt==2){ int g=detect_group_size(O,I,ns); if(g>0){ fmt=4; *gs=g; } } + int64_t exp_scale = (fmt==4)? (int64_t)O*((I+*gs-1)/(*gs)) : (int64_t)O; /* in FLOAT */ + if(ns != exp_scale*4){ + fprintf(stderr,"%s: scale array is %lld bytes — expected %lld for [%d,%d] fmt=%d, refusing (untrusted container)\n", + name,(long long)ns,(long long)(exp_scale*4),O,I,fmt); exit(1); } + return fmt; +} /* costruisce un QT [O,I] dal disco in `t` (buffer riusabili tra chiamate). * - se esiste `name.qs`: pesi GIA' quantizzati nel container (U8 qdata + F32 scala) -> letti diretti * - altrimenti: tensore pieno (f32/bf16) -> quantizzato a runtime a `bits` (oracolo tiny / pesi pieni) @@ -1454,12 +1481,11 @@ static void qt_from_disk(Model *m, const char *name, int O, int I, int bits, int if(st_has(&m->S,sn)){ int64_t nb=st_nbytes(&m->S,name); int64_t ns=st_nbytes(&m->S,sn); /* scale bytes (F32) */ - /* Detect int4-grouped (fmt=4): packed int4 weight bytes BUT scale array is - * larger than O*4 — the group size is derived from the scale-array size. */ - int fmt = (nb==(int64_t)O*I)?1 : (nb==(int64_t)O*((I+1)/2))?2 : 3; + /* fmt=4 int4-grouped: byte int4 ma scala > O*4 — gs deriva dalla scala. + * qt_resolve_fmt valida entrambi i conteggi contro [O,I] e termina se + * non fidati (SEC). */ int gs=0; - if(fmt==2) gs=detect_group_size(O,I,ns); - if(gs>0) fmt=4; + int fmt = qt_resolve_fmt(name,O,I,nb,ns,&gs); if(fmt==1){ if(t->fmt!=1||!t->q8){ t->fmt=1; t->O=O; t->I=I; t->gs=0; t->q8=qalloc(nb); t->s=qsalloc(O); } st_read_raw(&m->S,name,t->q8,drop); } else if(fmt==4){ int ng=(I+gs-1)/gs; if(t->fmt!=4||!t->q4){ t->fmt=4; t->O=O; t->I=I; t->gs=gs; t->q4=qalloc(nb); t->s=falloc((int64_t)O*ng); } @@ -1802,11 +1828,8 @@ static int expert_load_impl(Model *m, int layer, int eid, ESlot *s, int fatal){ QT *qt[3]={&s->g,&s->u,&s->d}; int OO[3]={I,I,D}, II[3]={D,D,I}; for(int k=0;k<3;k++){ int64_t nb=tw[k]->nbytes; - int fmt=(nb==(int64_t)OO[k]*II[k])?1:(nb==(int64_t)OO[k]*((II[k]+1)/2))?2:3; - /* detect grouped int4 (fmt=4): int4 weight bytes + larger scale array */ int gs=0; - if(fmt==2) gs=detect_group_size(OO[k],II[k],tq[k]->nbytes); - if(gs>0) fmt=4; + int fmt=qt_resolve_fmt(tw[k]->name,OO[k],II[k],nb,tq[k]->nbytes,&gs); qt[k]->fmt=fmt; qt[k]->O=OO[k]; qt[k]->I=II[k]; qt[k]->gs=gs; qt[k]->qf=NULL; qt[k]->q8=(int8_t*)((char*)bw[k]+tw[k]->off); qt[k]->q4=(uint8_t*)((char*)bw[k]+tw[k]->off); qt[k]->s=(float*)((char*)bq[k]+tq[k]->off); @@ -1931,10 +1954,8 @@ static int expert_load_impl(Model *m, int layer, int eid, ESlot *s, int fatal){ QT *qt[3]={&s->g,&s->u,&s->d}; int OO[3]={I,I,D}, II[3]={D,D,I}; for(int k=0;k<3;k++){ int64_t nb=tw[k]->nbytes; - int fmt = (nb==(int64_t)OO[k]*II[k])?1 : (nb==(int64_t)OO[k]*((II[k]+1)/2))?2 : 3; int gs=0; - if(fmt==2) gs=detect_group_size(OO[k],II[k],tq[k]->nbytes); - if(gs>0) fmt=4; + int fmt=qt_resolve_fmt(tw[k]->name,OO[k],II[k],nb,tq[k]->nbytes,&gs); qt[k]->fmt=fmt; qt[k]->O=OO[k]; qt[k]->I=II[k]; qt[k]->gs=gs; qt[k]->qf=NULL; qt[k]->q8=(int8_t*)(s->slab+pos[k]); qt[k]->q4=s->slab+pos[k]; qt[k]->s=fp[k]; } diff --git a/c/st.h b/c/st.h index efa6e5a0..a2c2c3fa 100644 --- a/c/st.h +++ b/c/st.h @@ -200,7 +200,19 @@ static void st_init(shards *S, const char *snap_dir) { if (a0 < 0 || b0 < a0 || data_start + b0 > fsz) { fprintf(stderr, "%s: tensor '%s' data_offsets [%lld,%lld] out of file bounds (%lld)\n", files[fi], name, (long long)a0, (long long)b0, (long long)fsz); exit(1); } - int64_t numel = 1; for (int k = 0; k < shp->len; k++) numel *= (int64_t)shp->kids[k]->num; + /* SEC: lo shape viene da un file non fidato (mirror). Senza il guard + * di overflow, uno shape tipo [65535,65535,65535,...] fa avvolgere + * numel a un valore piccolo/negativo che poi passerebbe il cross-check + * numel*esz==nbytes in st_read_f32, riaprendo l'OOB. */ + int64_t numel = 1; int bad_shape = 0; + for (int k = 0; k < shp->len; k++) { + int64_t d = (int64_t)shp->kids[k]->num; + if (d < 0 || (d != 0 && numel > INT64_MAX / d)) { bad_shape = 1; break; } + numel *= d; + } + if (bad_shape) { + fprintf(stderr, "%s: tensor '%s' shape overflows int64 — refusing (hostile or corrupt file)\n", + files[fi], name); exit(1); } if (S->n == S->cap) { S->cap *= 2; S->t = realloc(S->t, S->cap*sizeof(st_tensor)); } st_tensor *t = &S->t[S->n++]; t->name = strdup(name); t->fd = fd; t->off = data_start + a0; @@ -249,6 +261,15 @@ static void st_prefetch(shards *S, const char *name) { static int64_t st_read_f32(shards *S, const char *name, float *out, int drop) { st_tensor *t = st_find(S, name); if (!t) { fprintf(stderr, "missing tensor: %s\n", name); exit(1); } + /* SEC: numel viene dallo shape, nbytes dagli offset — due campi indipendenti + * del file. Se non concordano, la memcpy F32 (nbytes) o i loop BF16/F16 + * (numel elementi da un raw di soli nbytes) sforano il buffer del chiamante, + * che e' dimensionato sul config, non sul file. Il chiamante che alloca su + * st_numel resta coerente; questo blocca l'ingresso ostile a monte. */ + int esz = (t->dtype == 2) ? 4 : 2; + if (t->numel < 0 || t->numel > t->nbytes / esz || t->numel * (int64_t)esz != t->nbytes) { + fprintf(stderr, "%s: tensor '%s' shape/bytes mismatch (numel %lld, %lld bytes, dtype %d) — refusing (hostile or corrupt file)\n", + name, name, (long long)t->numel, (long long)t->nbytes, t->dtype); exit(1); } void *raw = malloc(t->nbytes); if (!raw) { fprintf(stderr, "malloc %lld bytes for tensor %s failed\n", (long long)t->nbytes, name); exit(1); } st_pread_full(t->fd, raw, t->nbytes, t->off, "pread data"); From ac39be6b62172699e44bbab14bdafd66224661b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20Ol=C3=A1h?= Date: Sun, 19 Jul 2026 10:06:27 +0200 Subject: [PATCH 024/132] flake: remove unnecessary rec --- flake.nix | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index 584a0886..04d31339 100644 --- a/flake.nix +++ b/flake.nix @@ -90,8 +90,7 @@ mainProgram = "glm"; }; }; - in - rec { + in { packages = { default = colibri; inherit colibri; From 73aef4d010366418c74a5b5c2f737c0c530bf7bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20Ol=C3=A1h?= Date: Sun, 19 Jul 2026 10:14:24 +0200 Subject: [PATCH 025/132] nix: configure and apply a formatter This sets the formatter to alejandra. I was trying to stay as close as the original as I could; otherwise we could set it to `nixfmt` which would keep list spacing more similar, but would introduce additional indentation in a few places. --- flake.nix | 39 ++++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/flake.nix b/flake.nix index 04d31339..a63839a3 100644 --- a/flake.nix +++ b/flake.nix @@ -6,20 +6,27 @@ flake-utils.url = "github:numtide/flake-utils"; }; - outputs = { self, nixpkgs, flake-utils }: - flake-utils.lib.eachDefaultSystem (system: - let - pkgs = import nixpkgs { inherit system; }; + outputs = { + self, + nixpkgs, + flake-utils, + }: + flake-utils.lib.eachDefaultSystem ( + system: let + pkgs = import nixpkgs {inherit system;}; # Python with the packages needed by the offline converter tools - pythonEnv = pkgs.python3.withPackages (ps: with ps; [ - torch - safetensors - huggingface-hub - numpy - tokenizers - datasets - ]); + pythonEnv = pkgs.python3.withPackages ( + ps: + with ps; [ + torch + safetensors + huggingface-hub + numpy + tokenizers + datasets + ] + ); colibri = pkgs.stdenv.mkDerivation { pname = "colibri"; @@ -28,7 +35,7 @@ # python3 is needed by checkPhase: `make test-c` shells out to # `python3 tools/run_tests.py` (see c/Makefile, PYTHON ?= python3). - nativeBuildInputs = [ pkgs.makeWrapper pkgs.python3 ]; + nativeBuildInputs = [pkgs.makeWrapper pkgs.python3]; buildInputs = [ pkgs.gcc @@ -107,14 +114,16 @@ }; }; + formatter = (import nixpkgs {inherit system;}).alejandra; + devShells.default = pkgs.mkShell { - inputsFrom = [ colibri ]; + inputsFrom = [colibri]; packages = [ pythonEnv pkgs.gcc pkgs.gnumake - pkgs.clang-tools # clangd / clang-tidy for IDE support + pkgs.clang-tools # clangd / clang-tidy for IDE support pkgs.pkg-config ]; From a6d0a6c4a7e0d43ee139a65517af414eb98d20b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20Ol=C3=A1h?= Date: Sun, 19 Jul 2026 10:21:50 +0200 Subject: [PATCH 026/132] nix: use with pkgs in some package lists --- flake.nix | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/flake.nix b/flake.nix index a63839a3..96d0625c 100644 --- a/flake.nix +++ b/flake.nix @@ -35,11 +35,11 @@ # python3 is needed by checkPhase: `make test-c` shells out to # `python3 tools/run_tests.py` (see c/Makefile, PYTHON ?= python3). - nativeBuildInputs = [pkgs.makeWrapper pkgs.python3]; + nativeBuildInputs = with pkgs; [makeWrapper python3]; - buildInputs = [ - pkgs.gcc - pkgs.gmp + buildInputs = with pkgs; [ + gcc + gmp ]; # Use x86-64-v3 (AVX2) for a portable binary; override with ARCH=native for local builds @@ -119,12 +119,12 @@ devShells.default = pkgs.mkShell { inputsFrom = [colibri]; - packages = [ + packages = with pkgs; [ pythonEnv - pkgs.gcc - pkgs.gnumake - pkgs.clang-tools # clangd / clang-tidy for IDE support - pkgs.pkg-config + gcc + gnumake + clang-tools # clangd / clang-tidy for IDE support + pkg-config ]; shellHook = '' From e93d574e139995efeb921008355e7fece380ac82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20Ol=C3=A1h?= Date: Sun, 19 Jul 2026 10:29:46 +0200 Subject: [PATCH 027/132] nix: add the python env to check inputs The tests now require Python so the env should be added to the check inputs, otherwise the build fails. --- flake.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flake.nix b/flake.nix index 96d0625c..af249bd5 100644 --- a/flake.nix +++ b/flake.nix @@ -42,6 +42,8 @@ gmp ]; + checkInputs = [pythonEnv]; + # Use x86-64-v3 (AVX2) for a portable binary; override with ARCH=native for local builds ARCH = "x86-64-v3"; From bcb984a61f2a8305d543c38af11003bd503a445a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20Ol=C3=A1h?= Date: Sun, 19 Jul 2026 10:40:19 +0200 Subject: [PATCH 028/132] nix: use getExe instead of hardcoding the path --- flake.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index af249bd5..276dbb60 100644 --- a/flake.nix +++ b/flake.nix @@ -108,11 +108,11 @@ apps = { default = { type = "app"; - program = "${colibri}/bin/glm"; + program = pkgs.lib.getExe colibri; }; coli = { type = "app"; - program = "${colibri}/bin/coli"; + program = pkgs.lib.getExe' colibri "coli"; }; }; From 58fd0e557f23e95827d1be9a8226befbfffefca7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20Ol=C3=A1h?= Date: Sun, 19 Jul 2026 10:48:00 +0200 Subject: [PATCH 029/132] nix: commit lockfile, ignore result This makes the build actually reproducible by committing a working lockfile to the repo. --- .gitignore | 1 + flake.lock | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 981e7400..6e17a303 100644 --- a/.gitignore +++ b/.gitignore @@ -35,6 +35,7 @@ c/tests/test_schema_gbnf c/tests/test_schema_gbnf.exe c/tests/test_compat_direct c/tests/test_compat_direct.exe +result # oracoli tiny generati (make_glm_oracle.py) e dati benchmark scaricati c/glm_tiny/ diff --git a/flake.lock b/flake.lock index 9b788fa6..a89029e8 100644 --- a/flake.lock +++ b/flake.lock @@ -20,11 +20,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1784160687, - "narHash": "sha256-iYL/bixrb6FlHFu/gIuBYzq6c6lM5AAXsXNSWXtIgQc=", + "lastModified": 1784280462, + "narHash": "sha256-DtoqIqM7VkR6NxAkcLpMwmi02USwWb3JdmNGLyhthc0=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "4382ed2b7a6839d4280a9b386db49cbc5907414d", + "rev": "293d6abedf0478e681a4dfcfcb35b30fc796a32f", "type": "github" }, "original": { From da97f0dbf1834b6801f05744e55e73e0b926e481 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20Ol=C3=A1h?= Date: Sun, 19 Jul 2026 10:59:16 +0200 Subject: [PATCH 030/132] nix: build on darwin with -march=native --- flake.nix | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index 276dbb60..df83d4c0 100644 --- a/flake.nix +++ b/flake.nix @@ -45,7 +45,10 @@ checkInputs = [pythonEnv]; # Use x86-64-v3 (AVX2) for a portable binary; override with ARCH=native for local builds - ARCH = "x86-64-v3"; + ARCH = + if pkgs.stdenv.hostPlatform.isx86_64 + then "x86-64-v3" + else "native"; buildPhase = '' runHook preBuild @@ -95,7 +98,7 @@ description = "Run GLM-5.2 (744B MoE) on a consumer machine with ~25 GB RAM"; homepage = "https://github.com/JustVugg/colibri"; license = licenses.asl20; - platforms = platforms.linux; + platforms = with platforms; linux ++ darwin; mainProgram = "glm"; }; }; From 51fe03f615c42a07f5dd31f0287cf7105d22c2cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20Ol=C3=A1h?= Date: Sun, 19 Jul 2026 11:36:13 +0200 Subject: [PATCH 031/132] nix: set main program to coli This is the higher-level user interface, which should be the main entry point to the binary, not the engine itself. The engine is still available. --- flake.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.nix b/flake.nix index df83d4c0..17f8a36e 100644 --- a/flake.nix +++ b/flake.nix @@ -99,7 +99,7 @@ homepage = "https://github.com/JustVugg/colibri"; license = licenses.asl20; platforms = with platforms; linux ++ darwin; - mainProgram = "glm"; + mainProgram = "coli"; }; }; in { @@ -113,9 +113,9 @@ type = "app"; program = pkgs.lib.getExe colibri; }; - coli = { + glm = { type = "app"; - program = pkgs.lib.getExe' colibri "coli"; + program = "${colibri}/share/colibri/glm"; }; }; From 4775f28cebae2e127a9e2b883d8e7cf24b1f98bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20Ol=C3=A1h?= Date: Sun, 19 Jul 2026 13:50:45 +0200 Subject: [PATCH 032/132] nix: copy missing version.py to the environment --- flake.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 17f8a36e..3d62a5ea 100644 --- a/flake.nix +++ b/flake.nix @@ -68,7 +68,8 @@ cp c/glm $out/lib/colibri/glm cp c/coli $out/lib/colibri/coli chmod +x $out/lib/colibri/coli - cp c/openai_server.py c/resource_plan.py c/doctor.py $out/lib/colibri/ + cp c/openai_server.py c/resource_plan.py c/doctor.py c/version.py \ + $out/lib/colibri/ cp -r c/tools/* $out/lib/colibri/tools/ # $out/bin holds the user-facing entry points. From 845af6378d212f07906ba635b2ae4f7e4f2b0c64 Mon Sep 17 00:00:00 2001 From: JustVugg Date: Sun, 19 Jul 2026 14:08:19 +0200 Subject: [PATCH 033/132] docs: beginner-friendly Quick Start guide for Linux/Windows/macOS (#414) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds docs/quickstart.md — a step-by-step, no-experience-assumed walkthrough from installing the build tools to the first coli chat, with per-OS copy-paste commands (Ubuntu apt, Windows MSYS2 or prebuilt binary, macOS brew), the ready-made HF int4 container plus the self-convert path, and an honest 'what to expect' on disk-bound speed. Commands verified against setup.sh and the coli subcommands; every cross-linked doc exists. Linked from the README's Get started section. Closes #414 Co-Authored-By: Claude Fable 5 --- README.md | 4 ++ docs/quickstart.md | 164 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 168 insertions(+) create mode 100644 docs/quickstart.md diff --git a/README.md b/README.md index e5d16517..f9cf3f00 100644 --- a/README.md +++ b/README.md @@ -151,6 +151,10 @@ scale-granularity/rotation ablations live in ## Get started +> **New here?** The [Quick Start guide](docs/quickstart.md) walks through +> install → build → model → first chat step by step for Linux, Windows, and +> macOS, with copy-paste commands and no assumed background. + ### 1. Get the model A pre-converted **GLM-5.2 int4** container is on Hugging Face — **use the diff --git a/docs/quickstart.md b/docs/quickstart.md new file mode 100644 index 00000000..7d6a9bfd --- /dev/null +++ b/docs/quickstart.md @@ -0,0 +1,164 @@ +# Quick Start — from zero to a running model + +A step-by-step guide for first-time users on **Linux**, **Windows**, and **macOS**. +No prior experience with C, CUDA, or model conversion is assumed. If you get +stuck, `./coli doctor` (below) tells you exactly what's missing. + +> **What you're setting up:** colibrì runs a very large Mixture-of-Experts model +> (e.g. GLM-5.2, 744B parameters) on a normal machine by streaming the model's +> experts from disk instead of needing them all in RAM. The engine is a single +> C program; Python is only used once, to prepare the model files. + +--- + +## 0. What you need first (prerequisites) + +| | Minimum | Recommended | +|---|---|---| +| **RAM** | ~16 GB | 24 GB+ | +| **Free disk** | ~380 GB for the int4 model | a fast NVMe SSD (streaming speed = your token speed) | +| **OS** | Linux, Windows 10/11, or macOS | any | +| **Tools** | a C compiler + `make` + `git` + `python3` | — | + +You do **not** need a GPU. A GPU only helps if you have one; the engine runs +CPU-only by default. + +--- + +## 1. Install the build tools + +### Linux (Ubuntu / Debian) + +```bash +sudo apt update +sudo apt install -y build-essential git python3 +``` + +`build-essential` gives you `gcc`, `make`, and OpenMP (libgomp) — everything the +engine needs. + +### Windows + +You have two options. + +**Option A — download a prebuilt binary (no compiler needed).** +Grab the latest `colibri--windows-x86_64.zip` from the +[Releases page](https://github.com/JustVugg/colibri/releases), unzip it, and +skip to [step 3](#3-get-the-model). Python 3 (from +[python.org](https://www.python.org/downloads/)) is still needed if you want to +convert a model yourself. + +**Option B — build from source with MSYS2.** +Install [MSYS2](https://www.msys2.org/), open the **UCRT64** shell, and run: + +```bash +pacman -S --needed mingw-w64-ucrt-x86_64-gcc make git python +``` + +### macOS + +```bash +xcode-select --install # C compiler (clang) +brew install libomp git python # OpenMP for multithreading +``` + +--- + +## 2. Get the code and build the engine + +```bash +git clone https://github.com/JustVugg/colibri.git +cd colibri/c +./setup.sh +``` + +`setup.sh` checks your compiler and OpenMP, builds the engine, and runs a tiny +self-test. When it prints: + +``` +engine self-test: 32/32 (expected 32/32) +``` + +the engine is working correctly. (On Windows Option A you already have the +binary — you can skip this step.) + +--- + +## 3. Get the model + +You have two paths. + +### Easiest — download a ready-made int4 container + +A pre-converted **GLM-5.2 int4** model is on Hugging Face. **Use the version +with the int8 MTP heads** (the plain int4 heads disable speculative decoding — +see [#8](https://github.com/JustVugg/colibri/issues/8)): + +**https://huggingface.co/mateogrgic/GLM-5.2-colibri-int4-with-int8-mtp** + +Download it into a folder on a fast disk, e.g. `/nvme/glm52_i4` (Linux/macOS) or +`D:\glm52_i4` (Windows). It is about **372 GB**, so make sure you have the space. + +### Or convert it yourself from the FP8 source + +One resumable command downloads and converts the model shard by shard, so it +never needs the full ~756 GB on disk at once: + +```bash +./coli convert --model /nvme/glm52_i4 +``` + +This step uses Python and runs only once. Safe to interrupt and re-run — it +resumes where it left off. + +--- + +## 4. Run it + +Point `COLI_MODEL` at the folder from step 3 and start chatting: + +```bash +# Linux / macOS +COLI_MODEL=/nvme/glm52_i4 ./coli chat + +# Windows (UCRT64 shell) +COLI_MODEL=/d/glm52_i4 ./coli chat +``` + +Useful first commands: + +```bash +COLI_MODEL=/nvme/glm52_i4 ./coli doctor # read-only check: is everything ready? +COLI_MODEL=/nvme/glm52_i4 ./coli plan # shows where the model will live (RAM/disk/GPU) +COLI_MODEL=/nvme/glm52_i4 ./coli chat --topp 0.85 # faster: reads less from disk, same quality +``` + +> **Tip:** `--topp 0.85` is worth adding on a disk-bound machine — it reads +> fewer expert bytes per token with no quality loss, which directly means more +> tokens per second. + +--- + +## 5. What to expect + +- **First launch loads the resident weights** (~10 GB) — this takes a moment. +- **Speed depends on your disk.** The experts stream from storage, so a fast + NVMe SSD is the single biggest factor in tokens/second. On a slow or shared + disk, generation can be well under 1 token/second — that's expected, and it's + the honest cost of running a 744B model on a small machine. +- **It's still the full model.** Placement only changes speed, never the model's + answers or precision. + +If something doesn't work, run `./coli doctor` — it reports exactly what's +missing (compiler, model files, permissions) and how to fix it. + +--- + +## Where to go next + +| Topic | Doc | +|---|---| +| Windows native build (and CUDA DLL) | [docs/windows.md](windows.md) | +| Tuning: cache, prefetch, speculation | [docs/tuning.md](tuning.md) | +| OpenAI-compatible API + web dashboard | [docs/api.md](api.md) | +| Every environment variable | [docs/ENVIRONMENT.md](ENVIRONMENT.md) | From 40a3596354d025e686e8e1203ea16862c7887afd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20Ol=C3=A1h?= Date: Sun, 19 Jul 2026 14:09:45 +0200 Subject: [PATCH 034/132] nix: python3 is only needed for checks, not for the actual build --- flake.nix | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/flake.nix b/flake.nix index 3d62a5ea..55f95eea 100644 --- a/flake.nix +++ b/flake.nix @@ -33,16 +33,16 @@ version = "1.0"; src = ./.; - # python3 is needed by checkPhase: `make test-c` shells out to - # `python3 tools/run_tests.py` (see c/Makefile, PYTHON ?= python3). - nativeBuildInputs = with pkgs; [makeWrapper python3]; + nativeBuildInputs = with pkgs; [makeWrapper]; buildInputs = with pkgs; [ gcc gmp ]; - checkInputs = [pythonEnv]; + # python3 is needed by checkPhase: `make test-c` shells out to + # `python3 tools/run_tests.py` (see c/Makefile, PYTHON ?= python3). + nativeCheckInputs = with pkgs; [python3]; # Use x86-64-v3 (AVX2) for a portable binary; override with ARCH=native for local builds ARCH = From cfcc742591a03a79754f68a7f1a6f0a7ad1738a5 Mon Sep 17 00:00:00 2001 From: JustVugg Date: Sun, 19 Jul 2026 14:58:32 +0200 Subject: [PATCH 035/132] metal: advance the LFRU recency clock on the GPU-prerouted decode path (#417) On Metal, when routing is precomputed on the GPU (g_pre_idx), the moe fast path bumps eusage/ehit/eheat for the selected experts but skips the one thing the full CPU router does at the equivalent site: elast[layer][e] = ++eaccess_clock. So the session-local recency clock advances during prefill (full router) but freezes the moment GPU-prerouted decode starts, and REPIN's tier_pick_lfru() tie-breaker then runs on stale recency for the rest of the run. Mirror the exact update the non-Metal path already does. Inside #ifdef COLI_METAL, so CPU/CUDA are untouched; elast only feeds the LFRU eviction heuristic, so this cannot affect output, only which experts REPIN keeps warm. Found and reported by @monotophic with a source-level trace repro (ELAST_TRACE). Fix is inspection-verified against line ~3055; needs a Metal build to exercise end-to-end. Closes #417 Co-Authored-By: Claude Fable 5 --- c/glm.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/c/glm.c b/c/glm.c index 26e7fc49..0f97ad21 100644 --- a/c/glm.c +++ b/c/glm.c @@ -2931,6 +2931,11 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int m->eusage[layer][idxs[(int64_t)s*K+kk]]++; ehit_mark(m,layer,idxs[(int64_t)s*K+kk]); if(m->eheat[layer][idxs[(int64_t)s*K+kk]]eheat[layer][idxs[(int64_t)s*K+kk]]++; + /* #417: la scorciatoia GPU-prerouted deve far avanzare l'orologio di recency + * come il percorso router completo (riga ~3055), altrimenti elast/eaccess_clock + * si congelano a fine prefill e il tie-breaker LFRU di REPIN gira su punteggi + * stantii durante il decode su Metal. */ + m->elast[layer][idxs[(int64_t)s*K+kk]]=++m->eaccess_clock; } for(int d=0;d Date: Sun, 19 Jul 2026 03:56:56 +0800 Subject: [PATCH 036/132] refactor: split glm.c into colibri.c + 4 header modules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rename glm.c → colibri.c and extract four self-contained modules into header-only files (same pattern as st.h/tier.h/grammar.h): quant.h (672 lines) — SIMD matmul kernels, quantization sample.h (143 lines) — RNG, top-p sampling, stop-set kv_persist.h (121 lines) — .coli_kv disk persistence telemetry.h (189 lines) — dashboard protocol, stats, usage Main engine file shrinks from 6588 to 5396 lines (−18%). Build system: primary target is now colibri$(EXE); `make glm` remains as a phony alias for backward compat. CI, setup.sh, coli CLI, and all 10 test files that include the engine are updated. make check passes (C + Python, 73 tests, zero warnings). --- .github/workflows/ci.yml | 8 +- .gitignore | 2 + c/Makefile | 53 +- c/coli | 16 +- c/{glm.c => colibri.c} | 1239 +---------------------------- c/kv_persist.h | 121 +++ c/quant.h | 672 ++++++++++++++++ c/sample.h | 143 ++++ c/setup.sh | 4 +- c/telemetry.h | 189 +++++ c/tests/bench_dsa_select.c | 2 +- c/tests/bench_topp.c | 2 +- c/tests/test_dsa_select.c | 2 +- c/tests/test_i4_grouped.c | 2 +- c/tests/test_idot.c | 2 +- c/tests/test_kv_alloc.c | 2 +- c/tests/test_makefile_platform.py | 4 +- c/tests/test_sample_nan.c | 2 +- c/tests/test_stops.c | 2 +- c/tests/test_topp.c | 2 +- c/tests/test_uring.c | 2 +- 21 files changed, 1206 insertions(+), 1265 deletions(-) rename c/{glm.c => colibri.c} (82%) create mode 100644 c/kv_persist.h create mode 100644 c/quant.h create mode 100644 c/sample.h create mode 100644 c/telemetry.h diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 72b7149e..eb7e872c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,8 +12,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Build glm - run: cd c && make glm + - name: Build colibri + run: cd c && make colibri - name: C test suite run: cd c && make test-c @@ -105,11 +105,11 @@ jobs: make cuda-dll CUDA_ARCH=sm_80 test -f coli_cuda.dll || { echo "cuda-dll reported success but produced no DLL" >&2; exit 1; } echo "coli_cuda.dll built (MSVC host)" - - name: make glm CUDA_DLL=1 (host links backend_loader, not cudart) + - name: make colibri CUDA_DLL=1 (host links backend_loader, not cudart) shell: msys2 {0} run: | cd c - make glm CUDA_DLL=1 + make colibri CUDA_DLL=1 test -f glm.exe || { echo "glm CUDA_DLL=1 reported success but produced no exe" >&2; exit 1; } echo "glm.exe built against the DLL loader" diff --git a/.gitignore b/.gitignore index a65fd3df..b3e11938 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,8 @@ desktop/src-tauri/target/ desktop/src-tauri/gen/ # binari compilati (si rigenerano con make / coli build) +c/colibri +c/colibri.exe c/glm c/glm.exe c/olmoe diff --git a/c/Makefile b/c/Makefile index 1f8583b7..e33ad372 100644 --- a/c/Makefile +++ b/c/Makefile @@ -56,7 +56,7 @@ OMPL = endif CFLAGS = -O3 $(OMPC) -Wall -Wextra -Wno-unused-parameter -Wno-misleading-indentation -Wno-unused-function # Opt-in: ARCH=native appends -mcpu=native (arm64 clang uses -mcpu, not -march), -# which unlocks the i8mm SMMLA int8/int4 dot kernels in glm.c. ARCH unset -> +# which unlocks the i8mm SMMLA int8/int4 dot kernels in colibri.c. ARCH unset -> # no -mcpu, default build byte-identical. Apple clang knows apple-m4 / native. ifneq ($(ARCH),) CFLAGS += -mcpu=$(ARCH) @@ -70,7 +70,7 @@ else ifneq ($(IS_WIN),) # ARCH default = x86-64-v3 (portable binary with AVX2). For max speed on THIS # machine use ARCH=native: on AVX-VNNI CPUs (Intel Alder Lake+, Meteor Lake+) # it also unlocks the 128-bit VPDPBUSD int8/int4 dot kernel (dot_i8i8/dot_i4i8), -# which the x86-64-v3 baseline does not define. The #ifdef guards in glm.c mean +# which the x86-64-v3 baseline does not define. The #ifdef guards in colibri.c mean # a v3 build simply compiles out the VNNI path - safe on any x86-64. CC = gcc ARCH ?= x86-64-v3 @@ -207,17 +207,18 @@ LDFLAGS += -framework Metal -framework Foundation -lc++ METAL_OBJ = backend_metal.o endif -all: glm$(EXE) +all: colibri$(EXE) -# phony 'glm' → 'glm.exe' on Windows (so 'make glm' and 'coli build' work on every platform) -glm: glm$(EXE) +# phony targets — 'glm' kept for backward compatibility +colibri: colibri$(EXE) +glm: colibri$(EXE) # Config stamp: make only tracks file timestamps, not flag changes. Without this, -# `make glm.exe CUDA_DLL=1` after a prior CPU-only build reports "up to date" and -# silently keeps the CPU-only binary (no CUDA loader) — a build that looks like it -# worked but isn't. We record the build-affecting flags in .build-config and rewrite -# it ONLY when they change (evaluated here at parse time, so the file's timestamp -# moves exactly when the config moves). glm.exe and the CUDA/loader objects depend +# `make colibri.exe CUDA_DLL=1` after a prior CPU-only build reports "up to date" +# and silently keeps the CPU-only binary (no CUDA loader) — a build that looks like +# it worked but isn't. We record the build-affecting flags in .build-config and +# rewrite it ONLY when they change (evaluated here at parse time, so the file's +# timestamp moves exactly when the config moves). The binary and CUDA/loader objects depend # on it, so they relink on a config change and stay put otherwise. (#306) BUILD_CONFIG := $(CC)|$(CFLAGS)|$(LDFLAGS)|CUDA=$(CUDA)|CUDA_DLL=$(CUDA_DLL)|ARCH=$(ARCH)|CUDA_ARCH=$(CUDA_ARCH)|METAL=$(METAL) BUILD_CONFIG_OLD := $(shell cat .build-config 2>/dev/null) @@ -226,8 +227,8 @@ $(shell printf '%s' '$(BUILD_CONFIG)' > .build-config) endif .build-config: ; -glm$(EXE): glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h $(CUDA_OBJ) $(METAL_OBJ) .build-config - $(CC) $(CFLAGS) glm.c $(CUDA_OBJ) $(METAL_OBJ) -o glm$(EXE) $(LDFLAGS) +colibri$(EXE): colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h quant.h sample.h kv_persist.h telemetry.h $(CUDA_OBJ) $(METAL_OBJ) .build-config + $(CC) $(CFLAGS) colibri.c $(CUDA_OBJ) $(METAL_OBJ) -o colibri$(EXE) $(LDFLAGS) # Windows runtime loader object: resolves coli_cuda_* from coli_cuda.dll. backend_loader.o: backend_loader.c backend_cuda.h compat.h .build-config @@ -285,7 +286,7 @@ PORTABLE_ARCH = native endif portable: - $(MAKE) glm$(EXE) ARCH=$(PORTABLE_ARCH) + $(MAKE) colibri$(EXE) ARCH=$(PORTABLE_ARCH) iobench$(EXE): iobench.c compat.h $(CC) $(CFLAGS) iobench.c -o iobench$(EXE) $(LDFLAGS) @@ -311,27 +312,27 @@ tests/test_schema_gbnf$(EXE): tests/test_schema_gbnf.c schema_gbnf.h grammar.h j tests/test_decode_batch$(EXE): tests/test_decode_batch.c decode_batch.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) -tests/test_idot$(EXE): tests/test_idot.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h +tests/test_idot$(EXE): tests/test_idot.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) -tests/test_i4_grouped$(EXE): tests/test_i4_grouped.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h +tests/test_i4_grouped$(EXE): tests/test_i4_grouped.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) -tests/test_stops$(EXE): tests/test_stops.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h +tests/test_stops$(EXE): tests/test_stops.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) -tests/test_topp$(EXE): tests/test_topp.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h +tests/test_topp$(EXE): tests/test_topp.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) # bench_topp is a microbenchmark (old qsort vs new heap partial-select, #335), NOT a test # gate -- intentionally absent from TEST_BINS. Build on demand: make tests/bench_topp -tests/bench_topp$(EXE): tests/bench_topp.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h +tests/bench_topp$(EXE): tests/bench_topp.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) -tests/test_sample_nan$(EXE): tests/test_sample_nan.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h +tests/test_sample_nan$(EXE): tests/test_sample_nan.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) -tests/test_kv_alloc$(EXE): tests/test_kv_alloc.c glm.c st.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h +tests/test_kv_alloc$(EXE): tests/test_kv_alloc.c colibri.c st.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) tests/test_logit_nan$(EXE): tests/test_logit_nan.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h @@ -343,15 +344,15 @@ tests/test_i4_acc512$(EXE): tests/test_i4_acc512.c tests/test_compat_direct$(EXE): tests/test_compat_direct.c compat.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) -tests/test_dsa_select$(EXE): tests/test_dsa_select.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h +tests/test_dsa_select$(EXE): tests/test_dsa_select.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) # bench_dsa_select is a microbenchmark (old qsort vs new quickselect partial-select, #356), # NOT a test gate -- intentionally absent from TEST_BINS. Build on demand: make tests/bench_dsa_select -tests/bench_dsa_select$(EXE): tests/bench_dsa_select.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h +tests/bench_dsa_select$(EXE): tests/bench_dsa_select.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) -tests/test_uring$(EXE): tests/test_uring.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h +tests/test_uring$(EXE): tests/test_uring.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) test-c: $(TEST_BINS) @@ -368,12 +369,12 @@ check: $(MAKE) portable $(MAKE) test -install: glm$(EXE) olmoe$(EXE) +install: colibri$(EXE) olmoe$(EXE) $(INSTALL) -d $(DESTDIR)$(BINDIR) $(INSTALL) -d $(DESTDIR)$(LIBEXECDIR) $(INSTALL) -d $(DESTDIR)$(LIBEXECDIR)/tools $(INSTALL) -m 755 coli $(DESTDIR)$(BINDIR)/coli - $(INSTALL) -m 755 glm$(EXE) $(DESTDIR)$(LIBEXECDIR)/glm$(EXE) + $(INSTALL) -m 755 colibri$(EXE) $(DESTDIR)$(LIBEXECDIR)/colibri$(EXE) $(INSTALL) -m 755 olmoe$(EXE) $(DESTDIR)$(LIBEXECDIR)/olmoe$(EXE) $(INSTALL) -m 644 resource_plan.py doctor.py openai_server.py $(DESTDIR)$(LIBEXECDIR)/ $(INSTALL) -m 644 tools/*.py $(DESTDIR)$(LIBEXECDIR)/tools/ @@ -387,4 +388,4 @@ clean: bench: iobench$(EXE) @if [ -n "$(ARGS)" ]; then ./iobench$(EXE) $(ARGS); else echo "built iobench$(EXE) — run: ./iobench$(EXE) "; fi -.PHONY: all glm cuda-test cuda-bench cuda-dll portable test-c test-python test check clean install uninstall bench +.PHONY: all colibri glm cuda-test cuda-bench cuda-dll portable test-c test-python test check clean install uninstall bench diff --git a/c/coli b/c/coli index 803c22b5..d6728fe2 100755 --- a/c/coli +++ b/c/coli @@ -54,16 +54,20 @@ from version import __version__ as _version # guess is right (e.g. a custom packaging layout). _EXE = ".exe" if sys.platform == "win32" else "" _LIBEXEC = os.path.join(os.path.dirname(HERE), "libexec", "colibri") +_here_colibri = os.path.join(HERE, "colibri" + _EXE) _here_glm = os.path.join(HERE, "glm" + _EXE) if os.environ.get("COLI_ENGINE"): GLM = os.environ["COLI_ENGINE"] TOOLS = os.path.join(os.path.dirname(GLM), "tools") +elif os.path.exists(_here_colibri): + GLM = _here_colibri + TOOLS = os.path.join(HERE, "tools") elif os.path.exists(_here_glm): GLM = _here_glm TOOLS = os.path.join(HERE, "tools") else: - GLM = os.path.join(_LIBEXEC, "glm" + _EXE) + GLM = os.path.join(_LIBEXEC, "colibri" + _EXE) TOOLS = os.path.join(_LIBEXEC, "tools") sys.path.insert(0, _LIBEXEC) # so `import resource_plan`, `doctor`, `openai_server` still resolve @@ -241,13 +245,13 @@ def env_for(a): e["COLI_CUDA"]="0"; e.pop("CUDA_EXPERT_GB",None); e.pop("CUDA_DENSE",None) else: if not cuda_binary(): - sys.exit(f"{C.yel}--gpu needs the CUDA build:{C.r} make glm CUDA=1 (this binary is CPU-only)") + sys.exit(f"{C.yel}--gpu needs the CUDA build:{C.r} make colibri CUDA=1 (this binary is CPU-only)") e["COLI_CUDA"]="1" if a.gpu!="auto": e["COLI_GPUS"]=a.gpu e.setdefault("CUDA_DENSE","1") if a.vram and a.gpu!="none": if not cuda_binary(): - sys.exit(f"{C.yel}--vram needs the CUDA build:{C.r} make glm CUDA=1 (this binary is CPU-only)") + sys.exit(f"{C.yel}--vram needs the CUDA build:{C.r} make colibri CUDA=1 (this binary is CPU-only)") e["COLI_CUDA"]="1"; e["CUDA_EXPERT_GB"]=str(a.vram) return e @@ -421,8 +425,8 @@ def cmd_build(a): banner("build") if not os.path.exists(os.path.join(HERE, "Makefile")): sys.exit(f"{C.yel}coli build{C.r} only works from a source checkout (this is an installed copy).\n" - f" Clone https://github.com/JustVugg/colibri and run ./setup.sh, or make -C c glm.") - sys.exit(subprocess.call(["make","-C",HERE,"glm"])) + f" Clone https://github.com/JustVugg/colibri and run ./setup.sh, or make -C c colibri.") + sys.exit(subprocess.call(["make","-C",HERE,"colibri"])) def cmd_info(a): banner("info") @@ -802,7 +806,7 @@ def cmd_stop(a): if "coli" in cmd and " serve" in cmd and pid!=os.getpid(): if not any(p==pid for p,_ in targets): targets.append((pid,"coli serve (cmdline)")) comm=open(f"/proc/{pd}/comm").read().strip() - if comm in ("glm","exe","olmoe"): + if comm in ("colibri","glm","exe","olmoe"): env=open(f"/proc/{pd}/environ","rb").read().replace(b"\0",b"\n").decode("utf-8","replace") if "SERVE=1" in env: targets.append((pid,f"engine `{comm}` (SERVE=1)")) except (OSError,PermissionError): continue diff --git a/c/glm.c b/c/colibri.c similarity index 82% rename from c/glm.c rename to c/colibri.c index 0f97ad21..554a7fae 100644 --- a/c/glm.c +++ b/c/colibri.c @@ -73,21 +73,6 @@ static int g_metal_gemm_min=16; /* COLI_METAL_GEMM_MIN: min rows to send a mat static const int *g_pre_idx; static const float *g_pre_w; static const int *g_pre_keff; static const float *g_pre_sh; /* output dello shared expert gia' calcolato su GPU */ #endif -#ifdef __AVX2__ -#include -static inline float hsum256(__m256 v){ /* somma orizzontale di 8 float */ - __m128 lo=_mm256_castps256_ps128(v), hi=_mm256_extractf128_ps(v,1); - lo=_mm_add_ps(lo,hi); __m128 sh=_mm_movehl_ps(lo,lo); lo=_mm_add_ps(lo,sh); - sh=_mm_shuffle_ps(lo,lo,1); lo=_mm_add_ss(lo,sh); return _mm_cvtss_f32(lo); -} -#elif defined(__ARM_NEON) -#include /* Apple Silicon / aarch64: kernel NEON */ -#elif defined(__VSX__) -#include /* POWER8+ (ppc64le): kernel VSX */ -#undef vector /* igiene: si usano __vector/__bool espliciti */ -#undef pixel -#undef bool -#endif #ifdef __APPLE__ #include /* host_statistics64: MemAvailable di macOS */ #endif @@ -218,13 +203,21 @@ typedef struct { uint64_t bytes_mtp, bytes_main; /* byte letti da disco per tipo layer */ } Model; -static void usage_save(Model *m); /* cache che impara: definita accanto a stats_dump */ -static void tiers_emit(Model *m); -static void ehit_mark(Model *m, int layer, int eid); -static void emap_emit(Model *m); -static void hits_emit(Model *m); -static void hwinfo_emit(Model *m); -static int64_t expert_bytes_probe(Model *m, int ebits); /* PROF: tier sizes in the report */ +#include "quant.h" +static int g_no_fused_pair=0; +static int g_spec_pin=1; +static int g_spec_live=0; +static inline int spec_pinned(void){ return g_spec_pin && g_spec_live; } + +static void matmul_qt_ex(float *y, const float *x, QT *w, int S, int allow_idot); +static void matmul_qt(float *y, const float *x, QT *w, int S){ matmul_qt_ex(y,x,w,S,1); } + +static void expert_gate_up(float *g,float *u,const float *x,QT *wg,QT *wu,int S){ + if(!g_no_fused_pair&&!spec_pinned()&&S==1&&wg->fmt==2&&wu->fmt==2&&wg->I==wu->I&&wg->O==wu->O) + matmul_i4_pair(g,u,x,wg->q4,wg->s,wu->q4,wu->s,wg->I,wg->O); + else { matmul_qt(g,x,wg,S); matmul_qt(u,x,wu,S); } +} + static int g_repin; static uint64_t g_last_repin; #ifdef COLI_CUDA @@ -332,713 +325,15 @@ static float *falloc(int64_t n){ if(n<0 || (uint64_t)n > SIZE_MAX/sizeof(float)){ fprintf(stderr,"falloc: n=%lld is out of range\n",(long long)n); exit(1); } float *p=malloc((size_t)n*sizeof(float)); if(!p){fprintf(stderr,"OOM\n");exit(1);} return p; } -/* ---- Accumulatore int4->float a 512 bit / 512-bit int4->float accumulator ---- - * Stessa matematica lossless di matmul_i4 (nibble->f32, FMA), ma 32 pesi/iter su - * due catene FMA indipendenti. NON bit-identico al vecchio ordine: la riduzione - * ad albero accumula MENO errore della somma sequenziale (misurato 2-4x più - * vicino all'oracolo double sulle forme reali; perplexity invariata, +4-7% sul - * decode con routing CPU-heavy — vedi docs/experiments/glm52-6x5090-2026-07-12.md). - * EN: same lossless math as matmul_i4, 32 weights/iter on two independent FMA - * chains. Not bit-identical to the old order: tree reduction accumulates LESS - * rounding than sequential summation. I4_ACC512=0 restores the old order (A/B). */ -#if defined(__AVX512F__) && defined(__AVX512BW__) -static int g_i4_acc512=1; -static inline float dot_i4f_avx512(const uint8_t *w,const float *x,int I){ - const __m128i m4=_mm_set1_epi8(0x0F); const __m512i b8=_mm512_set1_epi32(8); - __m512 acc0=_mm512_setzero_ps(),acc1=_mm512_setzero_ps(); int i=0; - for(;i+32<=I;i+=32){ __m128i by=_mm_loadu_si128((const __m128i*)(w+(i>>1))); - __m128i lo=_mm_and_si128(by,m4),hi=_mm_and_si128(_mm_srli_epi16(by,4),m4); - __m128i n0=_mm_unpacklo_epi8(lo,hi),n1=_mm_unpackhi_epi8(lo,hi); - __m512 w0=_mm512_cvtepi32_ps(_mm512_sub_epi32(_mm512_cvtepu8_epi32(n0),b8)); - __m512 w1=_mm512_cvtepi32_ps(_mm512_sub_epi32(_mm512_cvtepu8_epi32(n1),b8)); - acc0=_mm512_fmadd_ps(_mm512_loadu_ps(x+i),w0,acc0); - acc1=_mm512_fmadd_ps(_mm512_loadu_ps(x+i+16),w1,acc1); - } - return _mm512_reduce_add_ps(_mm512_add_ps(acc0,acc1)); -} -/* selftest contro il riferimento scalare (I4_ACC512_TEST=1): copre l'ordine dei - * nibble e ogni multiplo di 32. / selftest vs the scalar reference. */ -static int i4_acc512_selftest(void){ - enum { N=224 }; uint8_t w[(N+1)/2]; float x[N]; - for(int i=0;i>1]=(uint8_t)(q+8); - else w[i>>1]|=(uint8_t)((q+8)<<4); - x[i]=(float)(((i*29+7)%101)-50)/37.f; - } - for(int n=32;n<=N;n+=32){ - float ref=0; for(int i=0;i>1]>>((i&1)*4))&15)-8); - float got=dot_i4f_avx512(w,x,n),tol=2e-5f*(1.f+fabsf(ref)); - if(fabsf(got-ref)>tol){ fprintf(stderr,"AVX512 i4 selftest n=%d: %.9g != %.9g\n",n,got,ref); return 0; } - } - return 1; -} -#endif - -/* y[S,O] = x[S,I] @ W^T, W[O,I] f32 */ -static void matmul(float *y, const float *x, const float *W, int S, int I, int O){ - #pragma omp parallel for schedule(static) - for (int o=0;o>1))); /* 8 byte=16 nibble */ - __m128i lo=_mm_and_si128(by,m4), hi=_mm_and_si128(_mm_srli_epi16(by,4),m4); - __m128i nib=_mm_unpacklo_epi8(lo,hi); /* nibble in ordine */ - __m256 w0=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(nib),b8)); - __m256 w1=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(nib,8)),b8)); - acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i), w0, acc); - acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i+8), w1, acc); } - a=hsum256(acc); -#elif defined(__ARM_NEON) - const uint8x8_t m4=vdup_n_u8(0x0F); const int8x8_t b8=vdup_n_s8(8); - float32x4_t ac0=vdupq_n_f32(0), ac1=vdupq_n_f32(0); - for(;i+16<=I;i+=16){ uint8x8_t by=vld1_u8(w+(i>>1)); /* 8 byte=16 nibble */ - uint8x8x2_t z=vzip_u8(vand_u8(by,m4), vshr_n_u8(by,4)); /* nibble in ordine */ - int16x8_t w0=vmovl_s8(vsub_s8(vreinterpret_s8_u8(z.val[0]),b8)); - int16x8_t w1=vmovl_s8(vsub_s8(vreinterpret_s8_u8(z.val[1]),b8)); - ac0=vfmaq_f32(ac0, vld1q_f32(xs+i), vcvtq_f32_s32(vmovl_s16(vget_low_s16(w0)))); - ac1=vfmaq_f32(ac1, vld1q_f32(xs+i+4), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w0)))); - ac0=vfmaq_f32(ac0, vld1q_f32(xs+i+8), vcvtq_f32_s32(vmovl_s16(vget_low_s16(w1)))); - ac1=vfmaq_f32(ac1, vld1q_f32(xs+i+12), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w1)))); } - a=vaddvq_f32(vaddq_f32(ac0,ac1)); -#endif -#if defined(__AVX512F__) && defined(__AVX512BW__) - } -#endif - for(;i+1>1]; int lo=(int)(byte&0xF)-8, hi=(int)(byte>>4)-8; - a += xs[i]*(float)lo + xs[i+1]*(float)hi; } - if(i>1]; int lo=(int)(byte&0xF)-8; a += xs[i]*(float)lo; } - y[(int64_t)s*O+o]=a*sc; } } -} -/* y[S,O] = x[S,I] @ W^T with W int4 packed (2/byte) + per-GROUP scales (fmt=4). - * Same nibble math as matmul_i4, but the scale changes every `gs` elements along I. - * The accumulator resets at each group boundary: dot(x[grp], w[grp]) * scale[grp]. - * gs MUST be a multiple of 16 (the AVX2 vector width). */ -static void matmul_i4_grouped(float *y, const float *x, const uint8_t *q4, const float *scale, - int S, int I, int O, int gs){ - int rb=(I+1)/2; int ng=(I+gs-1)/gs; - #pragma omp parallel for schedule(static) - for(int o=0;oI) glen=I-base; - float sc=scl[g]; - int i=base; -#ifdef __AVX2__ - const __m128i m4=_mm_set1_epi8(0x0F); const __m256i b8=_mm256_set1_epi32(8); - __m256 acc=_mm256_setzero_ps(); - for(; i+16<=base+glen; i+=16){ __m128i by=_mm_loadl_epi64((const __m128i*)(w+(i>>1))); - __m128i lo=_mm_and_si128(by,m4),hi=_mm_and_si128(_mm_srli_epi16(by,4),m4); - __m128i nib=_mm_unpacklo_epi8(lo,hi); - __m256 w0=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(nib),b8)); - __m256 w1=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(nib,8)),b8)); - acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i), w0, acc); - acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i+8), w1, acc); } - a+=hsum256(acc)*sc; -#endif - /* scalar tail for the group remainder */ - for(; i>1]; - a+=(xs[i]*(float)((int)(byte&0xF)-8)+xs[i+1]*(float)((int)(byte>>4)-8))*sc; } - else { uint8_t byte=w[i>>1]; a+=xs[i]*(float)((int)(byte&0xF)-8)*sc; } - } - } - y[(int64_t)s*O+o]=a; - } - } -} -/* Decode hot path for gate+up: same exact q4 dot products as matmul_i4, but one - * OpenMP dispatch covers both matrices. KTransformers uses persistent pools; - * this keeps colibri dependency-free while removing one team launch/expert. */ -static void matmul_i4_pair(float *yg, float *yu, const float *x, - const uint8_t *qg, const float *sg, - const uint8_t *qu, const float *su, int I, int O){ - int rb=(I+1)/2; - #pragma omp parallel for schedule(static) - for(int z=0;z<2*O;z++){ - int o=z>1))); - __m128i lo=_mm_and_si128(by,m4),hi=_mm_and_si128(_mm_srli_epi16(by,4),m4); - __m128i nib=_mm_unpacklo_epi8(lo,hi); - __m256 w0=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(nib),b8)); - __m256 w1=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(nib,8)),b8)); - acc=_mm256_fmadd_ps(_mm256_loadu_ps(x+i),w0,acc); - acc=_mm256_fmadd_ps(_mm256_loadu_ps(x+i+8),w1,acc); } - a=hsum256(acc); -#elif defined(__ARM_NEON) - const uint8x8_t m4=vdup_n_u8(0x0F); const int8x8_t b8=vdup_n_s8(8); - float32x4_t ac0=vdupq_n_f32(0),ac1=vdupq_n_f32(0); - for(;i+16<=I;i+=16){ uint8x8_t by=vld1_u8(w+(i>>1)); - uint8x8x2_t n=vzip_u8(vand_u8(by,m4),vshr_n_u8(by,4)); - int16x8_t w0=vmovl_s8(vsub_s8(vreinterpret_s8_u8(n.val[0]),b8)); - int16x8_t w1=vmovl_s8(vsub_s8(vreinterpret_s8_u8(n.val[1]),b8)); - ac0=vfmaq_f32(ac0,vld1q_f32(x+i),vcvtq_f32_s32(vmovl_s16(vget_low_s16(w0)))); - ac1=vfmaq_f32(ac1,vld1q_f32(x+i+4),vcvtq_f32_s32(vmovl_s16(vget_high_s16(w0)))); - ac0=vfmaq_f32(ac0,vld1q_f32(x+i+8),vcvtq_f32_s32(vmovl_s16(vget_low_s16(w1)))); - ac1=vfmaq_f32(ac1,vld1q_f32(x+i+12),vcvtq_f32_s32(vmovl_s16(vget_high_s16(w1)))); } - a=vaddvq_f32(vaddq_f32(ac0,ac1)); -#endif -#if defined(__AVX512F__) && defined(__AVX512BW__) - } -#endif - for(;i+1>1]; a+=x[i]*(float)((b&15)-8)+x[i+1]*(float)((b>>4)-8); } - if(i>1]&15)-8); - (z=2) non calcolano la STESSA funzione. Tre interruttori dipendono da S: il gate - * int4-IDOT (S>=g_i4s — asimmetrico proprio dove g_i4s>1), la fusione gate+up solo-S==1, - * e la soglia righe del GEMM Metal. Con SPEC_PIN=1 (default) ogni forward emesso mentre - * i draft del modello sono attivi resta sulla famiglia di kernel di S=1: draft e verifica - * coincidono per costruzione. Prefill e decode non speculativo sono intoccati. - * EN: MTP acceptance collapses when the draft (S=1) and verify (S>=2) forwards do not - * compute the SAME function. Three switches are S-dependent: the int4 IDOT gate - * (S>=g_i4s — asymmetric exactly on ISAs where g_i4s>1), the S==1-only gate+up fusion, - * and the Metal GEMM row threshold. SPEC_PIN=1 (default) pins every forward issued - * while model drafts are live to the platform's S=1 kernel family, so draft and verify - * agree by construction; prefill and non-speculative decode are untouched. - * SPEC_PIN=0 restores the S-dependent gates (A/B). */ -static int g_spec_pin=1; -static int g_spec_live=0; /* set by spec_decode while drafts are live */ -static inline int spec_pinned(void){ return g_spec_pin && g_spec_live; } -static void expert_gate_up(float *g,float *u,const float *x,QT *wg,QT *wu,int S){ - if(!g_no_fused_pair&&!spec_pinned()&&S==1&&wg->fmt==2&&wu->fmt==2&&wg->I==wu->I&&wg->O==wu->O) - matmul_i4_pair(g,u,x,wg->q4,wg->s,wu->q4,wu->s,wg->I,wg->O); - else { matmul_qt(g,x,wg,S); matmul_qt(u,x,wu,S); } -} -/* y[S,O] = x[S,I] @ W^T con W int2 impacchettato (4 valori/byte) + scala[O]. nibble 2-bit -> [-2,1]. */ -static void matmul_i2(float *y, const float *x, const uint8_t *q2, const float *scale, int S, int I, int O){ - int rb=(I+3)/4; - #pragma omp parallel for schedule(static) - for (int o=0;o>2))); /* 4 byte=16 valori */ - __m128i p0=_mm_and_si128(by,m2), p1=_mm_and_si128(_mm_srli_epi16(by,2),m2); - __m128i p2=_mm_and_si128(_mm_srli_epi16(by,4),m2), p3=_mm_and_si128(_mm_srli_epi16(by,6),m2); - __m128i lo=_mm_unpacklo_epi8(p0,p1), hi=_mm_unpacklo_epi8(p2,p3); - __m128i nib=_mm_unpacklo_epi16(lo,hi); /* 16 valori in ordine */ - __m256 w0=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(nib),b2)); - __m256 w1=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(nib,8)),b2)); - acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i), w0, acc); - acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i+8), w1, acc); } - a=hsum256(acc); -#elif defined(__ARM_NEON) - const uint8x8_t m2v=vdup_n_u8(3); const int8x8_t b2v=vdup_n_s8(2); - float32x4_t ac0=vdupq_n_f32(0), ac1=vdupq_n_f32(0); - for(;i+16<=I;i+=16){ uint32_t wd; memcpy(&wd, w+(i>>2), 4); /* 4 byte=16 valori */ - uint8x8_t by=vreinterpret_u8_u32(vdup_n_u32(wd)); - uint8x8x2_t z01=vzip_u8(vand_u8(by,m2v), vand_u8(vshr_n_u8(by,2),m2v)); - uint8x8x2_t z23=vzip_u8(vand_u8(vshr_n_u8(by,4),m2v), vshr_n_u8(by,6)); - uint16x4x2_t zz=vzip_u16(vreinterpret_u16_u8(z01.val[0]), vreinterpret_u16_u8(z23.val[0])); - int16x8_t w0=vmovl_s8(vsub_s8(vreinterpret_s8_u16(zz.val[0]),b2v)); /* 16 valori in ordine */ - int16x8_t w1=vmovl_s8(vsub_s8(vreinterpret_s8_u16(zz.val[1]),b2v)); - ac0=vfmaq_f32(ac0, vld1q_f32(xs+i), vcvtq_f32_s32(vmovl_s16(vget_low_s16(w0)))); - ac1=vfmaq_f32(ac1, vld1q_f32(xs+i+4), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w0)))); - ac0=vfmaq_f32(ac0, vld1q_f32(xs+i+8), vcvtq_f32_s32(vmovl_s16(vget_low_s16(w1)))); - ac1=vfmaq_f32(ac1, vld1q_f32(xs+i+12), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w1)))); } - a=vaddvq_f32(vaddq_f32(ac0,ac1)); -#endif - for(;i>2]; int sh=(i&3)*2; a += xs[i]*(float)((int)((byte>>sh)&3)-2); } - y[(int64_t)s*O+o]=a*sc; } } -} -/* ---- KERNEL INTERI (IDOT): attivazioni quantizzate a int8 per riga (absmax/127, - * stile Q8_0), prodotto scalare INTERO via maddubs/madd AVX2 — niente conversione - * f32 dei pesi nel ciclo caldo. ~2-3x sui matmul quantizzati; errore aggiunto ~0.3% - * RMS per matmul (attivazione int8), IDOT=0 torna al percorso f32 esatto. */ -#if defined(__AVX512VNNI__) && defined(__AVX512BW__) -#define IDOT_KERNEL "avx512-vnni" -#elif defined(__AVXVNNI__) && defined(__AVX2__) -#define IDOT_KERNEL "avx-vnni" -#elif defined(__AVX2__) -#define IDOT_KERNEL "avx2" -#elif defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8) -#define IDOT_KERNEL "neon-i8mm" -#elif defined(__ARM_NEON) -#define IDOT_KERNEL "neon" -#elif defined(__VSX__) -#define IDOT_KERNEL "vsx" -#else -#define IDOT_KERNEL "scalar" -#endif -static int g_idot=1; -#if defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD) -static int g_i4s=1; /* SDOT presente: int4 IDOT conviene anche a S=1 (decode). Misurato - * su Apple M-series: +14%%, expert-matmul -16%%. EN: with SDOT, int4 - * IDOT pays even at S=1 (decode); measured on Apple M-series. */ -#elif defined(__VSX__) -static int g_i4s=1; /* POWER8 vec_msum: qui il fallback f32 e' SCALARE, quindi l'IDOT - * int4 conviene anche a S=1. Misurato su POWER8 S824 (vedi PR). - * EN: on VSX the f32 fallback is plain scalar C, so int4 IDOT - * pays even at S=1. Measured on a POWER8 S824 (see PR). */ -#else -static int g_i4s=2; /* senza SDOT / altrove: soglia originale (misura AVX2 dell'autore). - * EN: without SDOT / elsewhere: original threshold (author's AVX2). */ -#endif -static inline float qrow_i8(const float *x, int8_t *q, int I){ - float amax=0; for(int i=0;iamax)amax=a; } - float s=amax/127.f; if(s<1e-12f) s=1e-12f; float inv=1.f/s; - for(int i=0;i s32 directly, 64 bytes/iter, no 16-bit intermediate. - * AVX-512 has no vpsignb: |w| via abs, sign folded into x with a mask-negate - * (w==0 -> product 0 either way). |x|<=127 (qrow_i8), |w|<=128 as u8: each - * s32 lane adds <= 4*128*127, safe up to I=16384 like the AVX2 bound. */ - __m512i acc=_mm512_setzero_si512(); - for(;i+64<=I;i+=64){ - __m512i wv=_mm512_loadu_si512((const void*)(w+i)); - __m512i xv=_mm512_loadu_si512((const void*)(x+i)); - __mmask64 neg=_mm512_movepi8_mask(wv); - __m512i xs=_mm512_mask_sub_epi8(xv,neg,_mm512_setzero_si512(),xv); - acc=_mm512_dpbusd_epi32(acc,_mm512_abs_epi8(wv),xs); - } - sum=_mm512_reduce_add_epi32(acc); -#elif defined(__AVXVNNI__) && defined(__AVX2__) - /* AVX-VNNI 128-bit: vpdpbusd u8*s8 -> s32, 16 byte/iter. Stesso trucco del - * segno della variante 512-bit: |w| via abs, segno piegato in x con maschera - * (w==0 -> product 0). __AVX2__ serve per _mm_sign_epi8 / abs. */ - __m128i acc=_mm_setzero_si128(); - for(;i+16<=I;i+=16){ - __m128i wv=_mm_loadu_si128((const __m128i*)(w+i)); - __m128i xv=_mm_loadu_si128((const __m128i*)(x+i)); - __m128i xs=_mm_sign_epi8(xv,wv); /* x * sign(w); _mm_sign zona __AVX2__ */ - acc=_mm_dpbusd_epi32(acc,_mm_abs_epi8(wv),xs); - } - sum=hsum128_i32(acc); -#elif defined(__AVX2__) - __m256i acc=_mm256_setzero_si256(); const __m256i ones=_mm256_set1_epi16(1); - for(;i+32<=I;i+=32){ - __m256i wv=_mm256_loadu_si256((const __m256i*)(w+i)); - __m256i xv=_mm256_loadu_si256((const __m256i*)(x+i)); - __m256i p=_mm256_maddubs_epi16(_mm256_sign_epi8(wv,wv),_mm256_sign_epi8(xv,wv)); - acc=_mm256_add_epi32(acc,_mm256_madd_epi16(p,ones)); - } - sum=hsum256_i32(acc); -#elif defined(__ARM_NEON) - /* ARM: SDOT nativo se disponibile (Apple Silicon: sempre); altrimenti vmull/vpadal. - * Stesso bound anti-overflow del trucco AVX2: coppie <= 128*127*2 = 32512 < 32767. */ -#if defined(__ARM_FEATURE_DOTPROD) - /* 4 accumulatori indipendenti: SDOT ha latenza ~3-4 cicli, con un solo acc la - * catena seriale strozza il core a ~26 GB/s di pesi; con 4 lane indipendenti il - * dot diventa memory-bound (misurato su M4: 26 -> 63 GB/s per core, 2.4x). */ - int32x4_t a0=vdupq_n_s32(0),a1=vdupq_n_s32(0),a2=vdupq_n_s32(0),a3=vdupq_n_s32(0); - for(;i+64<=I;i+=64){ - a0=vdotq_s32(a0,vld1q_s8(w+i), vld1q_s8(x+i)); - a1=vdotq_s32(a1,vld1q_s8(w+i+16),vld1q_s8(x+i+16)); - a2=vdotq_s32(a2,vld1q_s8(w+i+32),vld1q_s8(x+i+32)); - a3=vdotq_s32(a3,vld1q_s8(w+i+48),vld1q_s8(x+i+48)); - } - int32x4_t acc=vaddq_s32(vaddq_s32(a0,a1),vaddq_s32(a2,a3)); - for(;i+16<=I;i+=16) acc=vdotq_s32(acc,vld1q_s8(w+i),vld1q_s8(x+i)); - sum=vaddvq_s32(acc); -#else - int32x4_t acc=vdupq_n_s32(0); - for(;i+16<=I;i+=16){ - int8x16_t wv=vld1q_s8(w+i), xv=vld1q_s8(x+i); - int16x8_t p=vmull_s8(vget_low_s8(wv),vget_low_s8(xv)); - p=vmlal_s8(p,vget_high_s8(wv),vget_high_s8(xv)); - acc=vpadalq_s16(acc,p); - } - sum=vaddvq_s32(acc); -#endif -#elif defined(__VSX__) - /* POWER8: vec_msum (s8 x u8 -> s32) somma i prodotti byte DIRETTAMENTE in lane - * s32, 16 byte/iter: il bound anti-saturazione a 16 bit di maddubs qui non serve. - * Stesso trucco del segno (|w| u8 per x*sign(w) s8), ma |w| via select+sub MODULO - * e non vec_abs: -128 deve diventare 128 u8, non saturare a 127. - * EN: vec_msum accumulates byte products straight into s32 lanes; |w| is built - * with a modulo subtract select instead of vec_abs so w=-128 wraps to 128 (u8) - * rather than saturating to 127. |x|<=127 from qrow_i8, so x negation is safe. */ - __vector signed int acc=vec_splats(0); - const __vector signed char vz=vec_splats((signed char)0); - for(;i+16<=I;i+=16){ - __vector signed char wv=vec_xl(0,(const signed char*)(w+i)); - __vector signed char xv=vec_xl(0,(const signed char*)(x+i)); - __vector __bool char neg=vec_cmplt(wv,vz); - __vector signed char xs=vec_sel(xv,vec_sub(vz,xv),neg); - __vector unsigned char wa=(__vector unsigned char)vec_sel(wv,vec_sub(vz,wv),neg); - acc=vec_msum(xs,wa,acc); - } - sum=vec_extract(acc,0)+vec_extract(acc,1)+vec_extract(acc,2)+vec_extract(acc,3); -#endif - for(;i int8 [-8,7] al volo, poi stesso trucco */ -static inline int32_t dot_i4i8(const uint8_t *w4, const int8_t *x, int I){ - int32_t sum=0; int i=0; -#if defined(__AVX512VNNI__) && defined(__AVX512BW__) - /* 32 bytes = 64 nibbles -> int8 in [-8,7], one vpdpbusd per 64 values. - * 256-bit unpack leaves values in per-128-lane order [0-15][32-47]/[16-31][48-63]; - * dot pairing is order-invariant, so permute x's 128-bit blocks to match - * instead of re-ordering w (one vpermq per iter, off the critical unpack path). */ - const __m256i m4v=_mm256_set1_epi8(0x0F); - const __m512i b8v=_mm512_set1_epi8(8); - const __m512i xidx=_mm512_setr_epi64(0,1,4,5,2,3,6,7); - __m512i acc=_mm512_setzero_si512(); - for(;i+64<=I;i+=64){ - __m256i by=_mm256_loadu_si256((const __m256i*)(w4+(i>>1))); - __m256i lo=_mm256_and_si256(by,m4v), hi=_mm256_and_si256(_mm256_srli_epi16(by,4),m4v); - __m256i z0=_mm256_unpacklo_epi8(lo,hi), z1=_mm256_unpackhi_epi8(lo,hi); - __m512i wv=_mm512_sub_epi8(_mm512_inserti64x4(_mm512_castsi256_si512(z0),z1,1),b8v); - __m512i xv=_mm512_permutexvar_epi64(xidx,_mm512_loadu_si512((const void*)(x+i))); - __mmask64 neg=_mm512_movepi8_mask(wv); - __m512i xs=_mm512_mask_sub_epi8(xv,neg,_mm512_setzero_si512(),xv); - acc=_mm512_dpbusd_epi32(acc,_mm512_abs_epi8(wv),xs); - } - sum=_mm512_reduce_add_epi32(acc); -#elif defined(__AVXVNNI__) && defined(__AVX2__) - /* AVX-VNNI 128-bit, int4: 16 byte = 32 nibble -> int8 [-8,7] in due half - * (n0/n1), ciascuno alimentato a un vpdpbusd da 16 byte. Stesso unpack - * 128-bit del ramo AVX2 sotto; 32 elementi/iter come li. */ - const __m128i m4=_mm_set1_epi8(0x0F); const __m128i b8=_mm_set1_epi8(8); - __m128i acc=_mm_setzero_si128(); - for(;i+32<=I;i+=32){ - __m128i by=_mm_loadu_si128((const __m128i*)(w4+(i>>1))); /* 16 byte = 32 nibble */ - __m128i lo=_mm_and_si128(by,m4), hi=_mm_and_si128(_mm_srli_epi16(by,4),m4); - __m128i n0=_mm_unpacklo_epi8(lo,hi), n1=_mm_unpackhi_epi8(lo,hi); /* nibble in ordine */ - __m128i w0=_mm_sub_epi8(n0,b8), w1=_mm_sub_epi8(n1,b8); - __m128i x0=_mm_loadu_si128((const __m128i*)(x+i)); - __m128i x1=_mm_loadu_si128((const __m128i*)(x+i+16)); - acc=_mm_dpbusd_epi32(acc,_mm_abs_epi8(w0),_mm_sign_epi8(x0,w0)); - acc=_mm_dpbusd_epi32(acc,_mm_abs_epi8(w1),_mm_sign_epi8(x1,w1)); - } - sum=hsum128_i32(acc); -#elif defined(__AVX2__) - const __m128i m4=_mm_set1_epi8(0x0F); const __m256i b8=_mm256_set1_epi8(8); - const __m256i ones=_mm256_set1_epi16(1); - __m256i acc=_mm256_setzero_si256(); - for(;i+32<=I;i+=32){ - __m128i by=_mm_loadu_si128((const __m128i*)(w4+(i>>1))); /* 16 byte = 32 nibble */ - __m128i lo=_mm_and_si128(by,m4), hi=_mm_and_si128(_mm_srli_epi16(by,4),m4); - __m128i n0=_mm_unpacklo_epi8(lo,hi), n1=_mm_unpackhi_epi8(lo,hi); /* in ordine */ - __m256i wv=_mm256_sub_epi8(_mm256_set_m128i(n1,n0),b8); - __m256i xv=_mm256_loadu_si256((const __m256i*)(x+i)); - __m256i p=_mm256_maddubs_epi16(_mm256_sign_epi8(wv,wv),_mm256_sign_epi8(xv,wv)); - acc=_mm256_add_epi32(acc,_mm256_madd_epi16(p,ones)); - } - sum=hsum256_i32(acc); -#elif defined(__ARM_NEON) - const uint8x16_t m4q=vdupq_n_u8(0x0F); const int8x16_t b8q=vdupq_n_s8(8); -#if defined(__ARM_FEATURE_DOTPROD) - /* 4 accumulatori indipendenti (vedi dot_i8i8): spezza la catena seriale su acc. - * Misurato su M4: 12.4 -> 29.9 GB/s di pesi per core (2.4x). */ - int32x4_t a0=vdupq_n_s32(0),a1=vdupq_n_s32(0),a2=vdupq_n_s32(0),a3=vdupq_n_s32(0); - for(;i+64<=I;i+=64){ - uint8x16_t byA=vld1q_u8(w4+(i>>1)), byB=vld1q_u8(w4+(i>>1)+16); - uint8x16x2_t zA=vzipq_u8(vandq_u8(byA,m4q), vshrq_n_u8(byA,4)); /* nibble in ordine */ - uint8x16x2_t zB=vzipq_u8(vandq_u8(byB,m4q), vshrq_n_u8(byB,4)); - a0=vdotq_s32(a0,vsubq_s8(vreinterpretq_s8_u8(zA.val[0]),b8q),vld1q_s8(x+i)); - a1=vdotq_s32(a1,vsubq_s8(vreinterpretq_s8_u8(zA.val[1]),b8q),vld1q_s8(x+i+16)); - a2=vdotq_s32(a2,vsubq_s8(vreinterpretq_s8_u8(zB.val[0]),b8q),vld1q_s8(x+i+32)); - a3=vdotq_s32(a3,vsubq_s8(vreinterpretq_s8_u8(zB.val[1]),b8q),vld1q_s8(x+i+48)); - } - int32x4_t acc=vaddq_s32(vaddq_s32(a0,a1),vaddq_s32(a2,a3)); - for(;i+32<=I;i+=32){ - uint8x16_t by=vld1q_u8(w4+(i>>1)); /* 16 byte = 32 nibble */ - uint8x16x2_t z=vzipq_u8(vandq_u8(by,m4q), vshrq_n_u8(by,4)); /* nibble in ordine */ - acc=vdotq_s32(acc,vsubq_s8(vreinterpretq_s8_u8(z.val[0]),b8q),vld1q_s8(x+i)); - acc=vdotq_s32(acc,vsubq_s8(vreinterpretq_s8_u8(z.val[1]),b8q),vld1q_s8(x+i+16)); - } - sum=vaddvq_s32(acc); -#else - int32x4_t acc=vdupq_n_s32(0); - for(;i+32<=I;i+=32){ - uint8x16_t by=vld1q_u8(w4+(i>>1)); /* 16 byte = 32 nibble */ - uint8x16x2_t z=vzipq_u8(vandq_u8(by,m4q), vshrq_n_u8(by,4)); /* nibble in ordine */ - int8x16_t w0=vsubq_s8(vreinterpretq_s8_u8(z.val[0]),b8q); - int8x16_t w1=vsubq_s8(vreinterpretq_s8_u8(z.val[1]),b8q); - int8x16_t x0=vld1q_s8(x+i), x1=vld1q_s8(x+i+16); - int16x8_t p=vmull_s8(vget_low_s8(w0),vget_low_s8(x0)); /* |w|<=8: nessun overflow */ - p=vmlal_s8(p,vget_high_s8(w0),vget_high_s8(x0)); - acc=vpadalq_s16(acc,p); - p=vmull_s8(vget_low_s8(w1),vget_low_s8(x1)); - p=vmlal_s8(p,vget_high_s8(w1),vget_high_s8(x1)); - acc=vpadalq_s16(acc,p); - } - sum=vaddvq_s32(acc); -#endif -#elif defined(__VSX__) - /* 16 byte = 32 nibble. vec_mergeh/vec_mergel su ppc64le (GCC) interallacciano come - * unpacklo/unpackhi x86 (verificato empiricamente su POWER8): i nibble escono in - * ordine di memoria. |w|<=8 dopo il -8, quindi stesso trucco del segno di dot_i8i8. - * EN: vec_mergeh/l on ppc64le interleave like x86 unpacklo/hi (verified on POWER8), - * so nibbles come out in memory order; then the same sign trick as dot_i8i8. */ - const __vector unsigned char m4v=vec_splats((unsigned char)0x0F); - const __vector unsigned char sh4=vec_splats((unsigned char)4); - const __vector signed char b8v=vec_splats((signed char)8); - const __vector signed char vz=vec_splats((signed char)0); - __vector signed int acc=vec_splats(0); - for(;i+32<=I;i+=32){ - __vector unsigned char by=vec_xl(0,w4+(i>>1)); /* 16 byte = 32 nibble */ - __vector unsigned char lo=vec_and(by,m4v), hi=vec_sr(by,sh4); - __vector signed char w0=vec_sub((__vector signed char)vec_mergeh(lo,hi),b8v); - __vector signed char w1=vec_sub((__vector signed char)vec_mergel(lo,hi),b8v); - __vector signed char x0=vec_xl(0,(const signed char*)(x+i)); - __vector signed char x1=vec_xl(0,(const signed char*)(x+i+16)); - __vector __bool char n0=vec_cmplt(w0,vz), n1=vec_cmplt(w1,vz); - acc=vec_msum(vec_sel(x0,vec_sub(vz,x0),n0), - (__vector unsigned char)vec_sel(w0,vec_sub(vz,w0),n0),acc); - acc=vec_msum(vec_sel(x1,vec_sub(vz,x1),n1), - (__vector unsigned char)vec_sel(w1,vec_sub(vz,w1),n1),acc); - } - sum=vec_extract(acc,0)+vec_extract(acc,1)+vec_extract(acc,2)+vec_extract(acc,3); -#endif - for(;i+1>1]; sum+=((int)(b&0xF)-8)*x[i]+((int)(b>>4)-8)*x[i+1]; } - if(i>1]; sum+=((int)(b&0xF)-8)*x[i]; } - return sum; -} -#if defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8) -/* SMMLA (i8mm): vmmlaq_s32 vede ogni int8x16_t come matrice 2x8 row-major (byte 0-7 = - * riga 0, byte 8-15 = riga 1) e accumula C += A*B^T nel 2x2 int32: lane0=a0.b0, - * lane1=a0.b1, lane2=a1.b0, lane3=a1.b1. vcombine di due mezze-righe costruisce la - * matrice: A = due righe di peso (o,o+1), B = due righe di attivazione (s,s+1), quindi - * meta' traffico pesi e doppio lavoro per istruzione a S>=2. EN: vmmlaq_s32 treats each - * int8x16_t as a 2x8 row-major matrix and does C += A*B^T on a 2x2 int32 tile; vcombine - * of vget_low/high halves builds the 2-row register from two weight/activation rows. */ -static inline int32x4_t mm_tile16(int32x4_t acc, int8x16_t wo, int8x16_t wo1, - int8x16_t xs, int8x16_t xs1){ - acc=vmmlaq_s32(acc, vcombine_s8(vget_low_s8(wo), vget_low_s8(wo1)), - vcombine_s8(vget_low_s8(xs), vget_low_s8(xs1))); - return vmmlaq_s32(acc, vcombine_s8(vget_high_s8(wo), vget_high_s8(wo1)), - vcombine_s8(vget_high_s8(xs), vget_high_s8(xs1))); -} -static void matmul_q_idot_mm(float *y, const int8_t *xq, const float *sx, const int8_t *q, - const float *scale, int S, int I, int O){ - #pragma omp parallel for schedule(static) - for(int o=0;o<(O&~1);o+=2){ - const int8_t *wo=q+(int64_t)o*I, *wo1=q+(int64_t)(o+1)*I; - float sc0=scale[o], sc1=scale[o+1]; - for(int s=0;s<(S&~1);s+=2){ - const int8_t *xs=xq+(int64_t)s*I, *xs1=xq+(int64_t)(s+1)*I; - /* 4 accumulatori indipendenti: una sola catena vmmla e' latency-bound. - * EN: 4 independent accumulators; a single vmmla chain is latency-bound. */ - int32x4_t a0=vdupq_n_s32(0),a1=vdupq_n_s32(0),a2=vdupq_n_s32(0),a3=vdupq_n_s32(0); int i=0; - for(;i+64<=I;i+=64){ - a0=mm_tile16(a0,vld1q_s8(wo+i), vld1q_s8(wo1+i), vld1q_s8(xs+i), vld1q_s8(xs1+i)); - a1=mm_tile16(a1,vld1q_s8(wo+i+16),vld1q_s8(wo1+i+16),vld1q_s8(xs+i+16),vld1q_s8(xs1+i+16)); - a2=mm_tile16(a2,vld1q_s8(wo+i+32),vld1q_s8(wo1+i+32),vld1q_s8(xs+i+32),vld1q_s8(xs1+i+32)); - a3=mm_tile16(a3,vld1q_s8(wo+i+48),vld1q_s8(wo1+i+48),vld1q_s8(xs+i+48),vld1q_s8(xs1+i+48)); - } - for(;i+16<=I;i+=16) - a0=mm_tile16(a0,vld1q_s8(wo+i),vld1q_s8(wo1+i),vld1q_s8(xs+i),vld1q_s8(xs1+i)); - int32x4_t acc=vaddq_s32(vaddq_s32(a0,a1),vaddq_s32(a2,a3)); - int32_t d00=vgetq_lane_s32(acc,0), d01=vgetq_lane_s32(acc,1); - int32_t d10=vgetq_lane_s32(acc,2), d11=vgetq_lane_s32(acc,3); - for(;i>1)), byo1=vld1q_u8(wo1+(i>>1)); - uint8x16_t cyo=vld1q_u8(wo+(i>>1)+16), cyo1=vld1q_u8(wo1+(i>>1)+16); - uint8x16x2_t zo =vzipq_u8(vandq_u8(byo, m4q), vshrq_n_u8(byo, 4)); - uint8x16x2_t zo1=vzipq_u8(vandq_u8(byo1,m4q), vshrq_n_u8(byo1,4)); - uint8x16x2_t ko =vzipq_u8(vandq_u8(cyo, m4q), vshrq_n_u8(cyo, 4)); - uint8x16x2_t ko1=vzipq_u8(vandq_u8(cyo1,m4q), vshrq_n_u8(cyo1,4)); - a0=mm_tile16(a0, vsubq_s8(vreinterpretq_s8_u8(zo.val[0]),b8q), - vsubq_s8(vreinterpretq_s8_u8(zo1.val[0]),b8q), - vld1q_s8(xs+i), vld1q_s8(xs1+i)); - a1=mm_tile16(a1, vsubq_s8(vreinterpretq_s8_u8(zo.val[1]),b8q), - vsubq_s8(vreinterpretq_s8_u8(zo1.val[1]),b8q), - vld1q_s8(xs+i+16), vld1q_s8(xs1+i+16)); - a2=mm_tile16(a2, vsubq_s8(vreinterpretq_s8_u8(ko.val[0]),b8q), - vsubq_s8(vreinterpretq_s8_u8(ko1.val[0]),b8q), - vld1q_s8(xs+i+32), vld1q_s8(xs1+i+32)); - a3=mm_tile16(a3, vsubq_s8(vreinterpretq_s8_u8(ko.val[1]),b8q), - vsubq_s8(vreinterpretq_s8_u8(ko1.val[1]),b8q), - vld1q_s8(xs+i+48), vld1q_s8(xs1+i+48)); - } - for(;i+32<=I;i+=32){ - uint8x16_t byo=vld1q_u8(wo+(i>>1)), byo1=vld1q_u8(wo1+(i>>1)); - uint8x16x2_t zo =vzipq_u8(vandq_u8(byo, m4q), vshrq_n_u8(byo, 4)); - uint8x16x2_t zo1=vzipq_u8(vandq_u8(byo1,m4q), vshrq_n_u8(byo1,4)); - a0=mm_tile16(a0, vsubq_s8(vreinterpretq_s8_u8(zo.val[0]),b8q), - vsubq_s8(vreinterpretq_s8_u8(zo1.val[0]),b8q), - vld1q_s8(xs+i), vld1q_s8(xs1+i)); - a1=mm_tile16(a1, vsubq_s8(vreinterpretq_s8_u8(zo.val[1]),b8q), - vsubq_s8(vreinterpretq_s8_u8(zo1.val[1]),b8q), - vld1q_s8(xs+i+16), vld1q_s8(xs1+i+16)); - } - int32x4_t acc=vaddq_s32(vaddq_s32(a0,a1),vaddq_s32(a2,a3)); - int32_t d00=vgetq_lane_s32(acc,0), d01=vgetq_lane_s32(acc,1); - int32_t d10=vgetq_lane_s32(acc,2), d11=vgetq_lane_s32(acc,3); - for(;i+1>1], bo1=wo1[i>>1]; - int a0=(int)(bo&0xF)-8, a1=(int)(bo>>4)-8, b0=(int)(bo1&0xF)-8, b1=(int)(bo1>>4)-8; - int u0=xs[i],u1=xs[i+1],v0=xs1[i],v1=xs1[i+1]; - d00+=a0*u0+a1*u1; d01+=a0*v0+a1*v1; d10+=b0*u0+b1*u1; d11+=b0*v0+b1*v1; } - if(i>1], bo1=wo1[i>>1]; - int a0=(int)(bo&0xF)-8, b0=(int)(bo1&0xF)-8; - d00+=a0*xs[i]; d01+=a0*xs1[i]; d10+=b0*xs[i]; d11+=b0*xs1[i]; } - y[(int64_t)s*O+o] =(float)d00*sc0*sx[s]; - y[(int64_t)s*O+(o+1)] =(float)d10*sc1*sx[s]; - y[(int64_t)(s+1)*O+o] =(float)d01*sc0*sx[s+1]; - y[(int64_t)(s+1)*O+(o+1)]=(float)d11*sc1*sx[s+1]; - } - if(S&1){ int s=S-1; const int8_t *xs=xq+(int64_t)s*I; - y[(int64_t)s*O+o] =(float)dot_i4i8(wo, xs,I)*sc0*sx[s]; - y[(int64_t)s*O+(o+1)]=(float)dot_i4i8(wo1,xs,I)*sc1*sx[s]; } - } - if(O&1){ int o=O-1; const uint8_t *w=q4+(int64_t)o*rb; float sc=scale[o]; - #pragma omp parallel for schedule(static) - for(int s=0;s=2){ matmul_q_idot_mm(y,xq,sx,q,scale,S,I,O); return; } -#endif - #pragma omp parallel for schedule(static) - for(int o=0;o=2){ matmul_i4_idot_mm(y,xq,sx,q4,scale,S,I,O); return; } -#endif - #pragma omp parallel for schedule(static) - for(int o=0;og_qscratch.xq_cap){ - int8_t *p=realloc(g_qscratch.xq,xn); - if(!p){ fprintf(stderr,"OOM quant scratch\n"); exit(1); } - g_qscratch.xq=p; g_qscratch.xq_cap=xn; - } - if(sn>g_qscratch.sx_cap){ - float *p=realloc(g_qscratch.sx,sn*sizeof(float)); - if(!p){ fprintf(stderr,"OOM quant scales\n"); exit(1); } - g_qscratch.sx=p; g_qscratch.sx_cap=sn; - } - *xq=g_qscratch.xq; *sx=g_qscratch.sx; -} - -/* allow_idot=0: forza il kernel int4/int8 ESATTO (attivazioni f32). Serve alle proiezioni di - * attenzione: sono sensibili alla quantizzazione int8 delle attivazioni dell'IDOT. Misurato su - * GLM-5.2 int4, 1023 token, log-lik -5040.33 (esatto) -> -5160.47 (IDOT) = +0.117 nat/token, - * ~+12% perplexity. Gli altri matmul del prefill (o_proj, kv_b, expert) tengono l'IDOT. - * EN: allow_idot=0 forces the EXACT int4/int8 kernel (f32 activations). The attention - * projections need it: IDOT's int8 activation quantization costs +0.117 nats/token there - * (~+12% perplexity), measured. Every other prefill matmul keeps IDOT as before. */ -static void matmul_qt_ex(float *y, const float *x, QT *w, int S, int allow_idot); -static void matmul_qt(float *y, const float *x, QT *w, int S){ matmul_qt_ex(y,x,w,S,1); } static void matmul_qt_ex(float *y, const float *x, QT *w, int S, int allow_idot){ #ifdef COLI_METAL - /* Large row-batches (prefill: kv_b reconstruction, o_proj, dense MLP, step_all logits) - * amortize Metal's ~5ms submit latency; small-S decode matmuls stay on CPU (NEON wins). - * Weights must be registered (all dense QT allocs are, via qalloc). */ if(g_metal_enabled && S>=g_metal_gemm_min && !spec_pinned() && (w->fmt==1||w->fmt==2) && !omp_in_parallel()){ const void *wp = w->fmt==1 ? (const void*)w->q8 : (const void*)w->q4; if(coli_metal_gemm(y,x,wp,w->s,w->fmt,S,w->I,w->O)) return; } #endif #ifdef COLI_CUDA - /* The CUDA backend owns persistent copies only for model-resident tensors. - * Streaming expert slots are reused for different IDs and must never enter - * this cache. Nested OpenMP calls stay on CPU because each device context - * intentionally owns one synchronous scratch stream in this stage. */ if(g_cuda_enabled && w->cuda_eligible && !w->cuda_failed && !omp_in_parallel()){ const void *weights = w->fmt==0 ? (const void*)w->qf : w->fmt==1 ? (const void*)w->q8 : (const void*)w->q4; @@ -1049,21 +344,7 @@ static void matmul_qt_ex(float *y, const float *x, QT *w, int S, int allow_idot) } #endif if(w->fmt==0){ matmul(y,x,w->qf,S,w->I,w->O); return; } - /* fmt=4: grouped int4 — always use the exact grouped kernel (no IDOT approximation, - * since the whole point of grouped scales is better quality). */ if(w->fmt==4){ matmul_i4_grouped(y,x,w->q4,w->s,S,w->I,w->O,w->gs); return; } - /* int8 IDOT vince sempre (1.4-2.5x). int4 IDOT: l'autore su AVX2 trovo' che a S=1 - * non ripaga (soglia S>=2); ma su ARM/SDOT il singolo token CONVIENE (vedi g_i4s / - * PR #9 per il gemello VNNI). Soglia configurabile con I4S. - * EN: int8 IDOT always wins (1.4-2.5x). int4 IDOT: on AVX2 the author found S=1 didn't - * pay (S>=2 gate); on ARM/SDOT single-token DOES pay (see g_i4s / PR #9 for the VNNI - * twin). Threshold configurable via I4S. */ - /* #163: sotto SPEC_PIN il gate int4-IDOT usa la decisione di S=1 per OGNI S, cosi' - * draft e verifica restano sulla stessa famiglia. (CUDA non e' toccato: la sua - * condizione non dipende da S, quindi e' gia' coerente tra draft e verifica.) - * EN: under SPEC_PIN the int4 IDOT gate uses the S=1 decision for EVERY S, so draft - * and verify stay in one family. (CUDA untouched: its condition is S-independent, - * hence already draft/verify-consistent.) */ if(allow_idot && g_idot && (w->fmt==1 || (w->fmt==2 && (spec_pinned() ? g_i4s<=1 : S>=g_i4s)))){ int I=w->I; int8_t *xq; float *sx; if(S<0 || I<0 || (size_t)S>SIZE_MAX/(size_t)(I?I:1)){ fprintf(stderr,"matmul_qt: shape overflow\n"); exit(1); } @@ -1078,49 +359,6 @@ static void matmul_qt_ex(float *y, const float *x, QT *w, int S, int allow_idot) else matmul_i4(y,x,w->q4,w->s,S,w->I,w->O); } -/* quantizza w[O,I] f32 -> int8 q[O,I] + scala[O] simmetrica per riga */ -static void quantize_rows(const float *w, int8_t *q, float *scale, int O, int I, int bits){ - int qmax=(1<<(bits-1))-1; - #pragma omp parallel for schedule(static) - for(int o=0;oamax)amax=a; } - float s=amax/qmax; if(s<1e-8f)s=1e-8f; scale[o]=s; - int8_t *qr=q+(int64_t)o*I; - for(int i=0;iqmax)v=qmax; if(v<-qmax-1)v=-qmax-1; qr[i]=(int8_t)v; } - } -} -/* quantizza w[O,I] f32 -> int4 impacchettato (2/byte) + scala[O]. - * bits<=4: valori in [-qmax-1,qmax] stanno in un nibble [-8,7]; memorizzati come v+8 (0..15). */ -static void pack_int4(const float *w, uint8_t *q4, float *scale, int O, int I, int bits){ - int qmax=(1<<(bits-1))-1, rb=(I+1)/2; - #pragma omp parallel for schedule(static) - for(int o=0;oamax)amax=a; } - float s=amax/qmax; if(s<1e-8f)s=1e-8f; scale[o]=s; - uint8_t *qr=q4+(int64_t)o*rb; - for(int i=0;iqmax)v0=qmax; if(v0<-8)v0=-8; - int v1=0; if(i+1qmax)v1=qmax; if(v1<-8)v1=-8; } - qr[i>>1] = (uint8_t)((v0+8) | ((v1+8)<<4)); - } - } -} - -/* quantizza w[O,I] f32 -> int2 impacchettato (4/byte) + scala[O]. valori nibble 2-bit in [-2,1]. */ -static void pack_int2(const float *w, uint8_t *q2, float *scale, int O, int I, int bits){ - int qmax=(1<<(bits-1))-1, rb=(I+3)/4; - #pragma omp parallel for schedule(static) - for(int o=0;oamax)amax=a; } - float s=amax/qmax; if(s<1e-8f)s=1e-8f; scale[o]=s; - uint8_t *qr=q2+(int64_t)o*rb; - for(int i=0;iqmax)v=qmax; if(v<-2)v=-2; byte|=(uint8_t)((v+2)<<(k*2)); } - qr[i>>2]=byte; - } - } -} - static int g_nopack=0; /* NOPACK=1 -> tiene i valori <=4bit in contenitore int8 (per validare il packing) */ static int g_drop=0; /* DROP=1 -> scarta le pagine expart dopo l'uso. Default 0: le lascia in * page-cache (buff/cache, NON RSS) come L2 gratuito -> sfrutta lo @@ -1206,6 +444,11 @@ static int g_disk_split=0; /* DISK_SPLIT=1: contatori che spezzano i DISK LOAD ( * (int4), con i byte letti. Default OFF: a flag spento gli atomic * non vengono MAI toccati (zero overhead), le righe extra di stats * non vengono stampate. Solo misura: nessun effetto sull'output. */ + +#include "sample.h" +#include "kv_persist.h" +#include "telemetry.h" + /* Aligned allocator for dense QT weights/scales: under METAL, page-align + register so the * GPU reads them zero-copy (no upload duplicate). Plain malloc otherwise. */ /* ---- COLI_NUMA=1 (#82): interleave the expert slabs across NUMA nodes ---- @@ -4239,13 +3482,14 @@ static void mtp_absorb(Model *m, const int *next_ids, const float *x, int S, int } static inline int argmax_v(const float *lo, int V){ - /* skip NaN (x==x is false for NaN) so a poisoned logit can't pin the argmax - * to index 0 — pick the max finite/+Inf entry instead. */ int b=-1; float bv=-INFINITY; for(int i=0;ibv){ bv=x; b=i; } } return b<0?0:b; } +static void repin_pass_limit(Model *m,int limit); +static void repin_pass(Model *m){ repin_pass_limit(m,16); } + /* ---- METODO F: draft grammaticale (#48) ---- * gr_feed consuma i byte di ogni token EMESSO e tiene il walker in sync con l'output; * grammar_draft propone lo span FORZATO successivo (un solo byte legale per posizione) @@ -4326,113 +3570,6 @@ static int grammar_draft(int *draft, int cap){ * Rejection sampling di Leviathan: accetta il draft x_d con prob p(x_d); al rifiuto * ricampiona da p con x_d azzerato e rinormalizzato. La distribuzione risultante e' * ESATTAMENTE p: la speculazione resta invisibile all'output anche col sampling. */ -static uint64_t g_rng=0x9E3779B97F4A7C15ULL; -static inline double rndu(void){ g_rng^=g_rng<<13; g_rng^=g_rng>>7; g_rng^=g_rng<<17; - return (double)(g_rng>>11)*(1.0/9007199254740992.0); } -static float *g_pbuf=NULL; static int *g_pidx=NULL; /* buffer riusati (decode single-thread) */ -/* sift-down su max-heap in h[0..n), chiave = g_pbuf[h[i]] (#335: partial top-p select). - * Versione "a buco": porta il valore di radice e lo deposita solo alla fine, cosi' - * heapify e' O(V) e ogni pop e' O(log n) senza qsort sull'intero vocabolario. */ -static void topp_siftdown(int *h, int n, int i){ - int iv=h[i]; float kv=g_pbuf[iv]; - for(;;){ int l=2*i+1; - if(l>=n) break; /* foglia */ - int b=l; if(l+1g_pbuf[h[l]]) b=l+1; /* figlio maggiore */ - if(g_pbuf[h[b]]<=kv) break; /* nessun figlio supera la radice -> ferma */ - h[i]=h[b]; i=b; } - h[i]=iv; -} -/* costruisce in g_pbuf la distribuzione target: softmax(lo/temp) troncata a top-p g_nuc. - * Invariante per dist_sample: g_pbuf resta INDICIZZATO per token-id (mai riordinato); - * la coda troncata va AZZERATA in g_pbuf (dist_sample la legge direttamente per id). */ -static void dist_build(const float *lo, int V){ - if(!g_pbuf){ g_pbuf=falloc(V); g_pidx=malloc(V*sizeof(int)); } - /* Un solo logit NaN/+Inf avvelenava tutto (#369): +Inf diventava mx, NaN/Inf-mx - * -> expf NaN -> s NaN -> ogni prob NaN -> dist_sample cade sul fallback - * `g_pbuf[i]>0` (NaN>0 e' falso ovunque) e ritorna 0 PER SEMPRE, in silenzio. - * Difesa: mx solo sui finiti; un logit non finito contribuisce prob 0; - * se la distribuzione degenera (tutti non finiti / somma non valida) si - * ripiega sull'argmax dei finiti e si avvisa UNA volta, mai in silenzio. */ - int mxi=-1; float mx=0; - for(int i=0;imx)){ mx=lo[i]; mxi=i; } - double s=0; float invt=1.f/(g_temp>1e-4f?g_temp:1e-4f); - if(mxi>=0){ - for(int i=0;i=0)?mxi:0; /* mxi = argmax dei logit FINITI (robusto - * anche se lo[0] e' NaN, dove argmax_v fallirebbe) */ - for(int i=0;i0 && g_nuc<1.f){ - for(int i=0;i=0;i--) topp_siftdown(g_pidx,V,i); /* heapify O(V) */ - /* pop verso la coda: i vincitori (testa top-p) cadono in g_pidx[out..V-1] in ordine - * DECRESCENTE, come il vecchio qsort, quindi s2 accumula nello stesso ordine -> - * head bit-identical sui casi senza pareggi (i pareggi erano gia' non specificati - * sotto il qsort instabile e restano tali). Il prefisso g_pidx[0..out-1) e' la coda. */ - double s2=0, cum=0; int out=V; - do{ int root=g_pidx[0]; /* massimo corrente */ - g_pidx[0]=g_pidx[--out]; g_pidx[out]=root; /* sposta il max in coda */ - s2+=g_pbuf[root]; cum+=g_pbuf[root]; - if(out>0) topp_siftdown(g_pidx,out,0); - } while(cum0); - for(int i=0;i=0 -> quel token e' escluso (rinormalizzando al volo) */ -static int dist_sample(int V, int ban){ - double z = 1.0 - (ban>=0 ? g_pbuf[ban] : 0.0); if(z<=1e-12) z=1e-12; - double u = rndu()*z, cum=0; - for(int i=0;i=u) return i; } - for(int i=V-1;i>=0;i--) if(i!=ban && g_pbuf[i]>0) return i; - return 0; -} -/* prossimo token dai logits: greedy se g_temp<=0, altrimenti sampling. - * ban = token escluso perche' rifiutato dalla verifica speculativa precedente. */ -static int pick_tok(const float *lo, int V, int ban){ - if(g_temp<=0) return argmax_v(lo,V); - dist_build(lo,V); - return dist_sample(V,ban); -} - -/* stop-set attivo (popolato da run_text/run_serve dal config; vuoto in validazione, - * dove si genera un numero fisso di token da confrontare con l'oracolo) */ -static int g_stop[64], g_nstop=0; /* config eos + ogni added-token "special" del tokenizer */ -static void repin_pass_limit(Model *m,int limit); -static void repin_pass(Model *m){ repin_pass_limit(m,16); } -static inline int is_stop(int t){ for(int i=0;i solo gli stop del config (validazione/oracolo, dove il tokenizer non serve). */ -static void stops_arm_tok(const Cfg *c, int tok_eos, Tok *T){ - g_nstop=0; - for(int i=0;in_stop && g_nstop<64;i++) g_stop[g_nstop++]=c->stop_ids[i]; - if(tok_eos>=0 && !is_stop(tok_eos) && g_nstop<64) g_stop[g_nstop++]=tok_eos; - int nsp=0; - /* DIFESA IN PROFONDITA' (woolcoxm, #298): il tokenizer marca "special":true i token di - * CONTROLLO -- <|user|>, <|assistant|>, <|observation|>, , [gMASK], i marker - * image/video/audio. Nessuno di questi e' contenuto legittimo di una risposta: se il - * modello ne emette uno, il turno e' finito (infatti GLM ne elenca tre fra gli eos - * ufficiali). Senza questo, uno di quei token non elencato nel config veniva - * DETOKENIZZATO E STAMPATO IN CHAT come testo, e la generazione proseguiva oltre la - * fine reale -- l'"added stuff on the end" riportato su un checkpoint convertito. - * Fidarsi del config di pesi convertiti da terzi e' precisamente cio' che non - * possiamo controllare; il flag del tokenizer lo possiamo leggere. - * NB: // hanno "special":false e restano contenuto vero. */ - if(T) for(int id=0; idn_ids && g_nstop<64; id++) - if(T->id_special[id] && !is_stop(id)){ g_stop[g_nstop++]=id; nsp++; } - fprintf(stderr,"[stop] %d stop tokens:",g_nstop); - for(int i=0;imx){mx=lo[i];best=i;} } - double se=0; for(int i=0;i .. " (T=ctxlen+contlen) * output: riga " " per richiesta. @@ -5101,117 +4226,6 @@ static void repin_pass_limit(Model *m,int limit){ * si appendono SOLO le posizioni nuove e si riscrive nrec per ultimo: un crash a meta' * append lascia nrec vecchio = file coerente. La riga KV del layer MTP non si salva: * al resume kv_start=-1 e la finestra di draft riparte da sola. */ -static int g_kvsave=1; -#define KV_MAGIC "COLIKV1\0" -static void kv_hdr(Model *m, int32_t *h, int nrec){ - Cfg *c=&m->c; int nic=0; - for(int i=0;in_layers;i++) if(m->Ic && m->Ic[i]) nic++; - h[0]=c->n_layers; h[1]=c->kv_lora; h[2]=c->qk_rope; - h[3]=m->has_dsa?c->index_hd:0; h[4]=nic; h[5]=c->vocab; h[6]=nrec; h[7]=0; -} -/* Bytes of one on-disk record: [tok i32][Lc+Rc per layer][Ic per DSA layer]. - * Layout matches what kv_disk_append writes and kv_disk_load reads. */ -static int64_t kv_rec_bytes(Model *m){ - Cfg *c=&m->c; - int64_t rec = 4 + (int64_t)c->n_layers*(c->kv_lora+c->qk_rope)*4; - if(m->has_dsa) for(int i=0;in_layers;i++) if(m->Ic[i]) rec+=(int64_t)c->index_hd*4; - return rec; -} -/* Open the persistent handle lazily; write the header if the file is new. After - * this returns successfully, k->disk_fp is valid for the engine's lifetime and - * positioned at end-of-header (nrec==0 case) or wherever the caller seeks. */ -static int kv_disk_open(Model *m){ - KVState *k=m->kv; - if(k->disk_fp) return 1; - k->disk_fp=fopen(k->disk_path,"r+b"); - if(!k->disk_fp){ /* not there yet -> create + header */ - k->disk_fp=fopen(k->disk_path,"wb"); - if(!k->disk_fp) return 0; - int32_t h[8]; kv_hdr(m,h,0); - fwrite(KV_MAGIC,1,8,k->disk_fp); fwrite(h,4,8,k->disk_fp); - fflush(k->disk_fp); - fclose(k->disk_fp); - k->disk_fp=fopen(k->disk_path,"r+b"); /* reopen r+b for append */ - if(!k->disk_fp) return 0; - } - return 1; -} -static void kv_disk_truncate(Model *m, int nrec){ - if(!g_kvsave) return; - KVState *k=m->kv; - if(k->disk_fp){ fclose(k->disk_fp); k->disk_fp=NULL; } /* drop to shrink on disc */ - FILE *f=fopen(k->disk_path,"r+b"); - if(!f){ k->disk_nrec=0; return; } - k->disk_nrec=nrec; - int32_t nr=nrec; fseek(f,8+6*4,SEEK_SET); fwrite(&nr,4,1,f); - fflush(f); fclose(f); -} -static void kv_disk_reset(Model *m){ kv_disk_truncate(m,0); } -static void kv_disk_append(Model *m, const int *hist, int len){ - KVState *k=m->kv; - if(!g_kvsave || len<=k->disk_nrec) return; - Cfg *c=&m->c; - if(!kv_disk_open(m)) return; - FILE *f=k->disk_fp; - int64_t rec = kv_rec_bytes(m); - /* grow the contiguous staging buffer if the record is larger (#1 batching) */ - if(rec > k->disk_buf_cap){ - uint8_t *nb=realloc(k->disk_buf, rec); - if(!nb) return; /* OOM: skip this turn, retry next */ - k->disk_buf=nb; k->disk_buf_cap=rec; - } - fseek(f, 8+8*4 + (int64_t)k->disk_nrec*rec, SEEK_SET); - for(int p=k->disk_nrec;pdisk_buf; /* pack token + every layer into one record */ - *(int32_t*)b = hist[p]; b+=4; - for(int i=0;in_layers;i++){ - memcpy(b, m->Lc[i]+(int64_t)p*c->kv_lora, (size_t)c->kv_lora*4); b+=c->kv_lora*4; - memcpy(b, m->Rc[i]+(int64_t)p*c->qk_rope,(size_t)c->qk_rope*4); b+=c->qk_rope*4; - } - if(m->has_dsa) for(int i=0;in_layers;i++) if(m->Ic[i]){ - memcpy(b, m->Ic[i]+(int64_t)p*c->index_hd, (size_t)c->index_hd*4); b+=c->index_hd*4; - } - fwrite(k->disk_buf, 1, (size_t)rec, f); /* one fwrite per position (was ~157) */ - } - fflush(f); /* dati prima, contatore poi */ - int32_t nr=len; fseek(f,8+6*4,SEEK_SET); fwrite(&nr,4,1,f); - fflush(f); /* persist the counter too */ - k->disk_nrec=len; -} -static int kv_disk_load(Model *m, int *hist, int maxctx){ - if(!g_kvsave) return 0; - KVState *k=m->kv; - Cfg *c=&m->c; - FILE *f=fopen(k->disk_path,"rb"); if(!f) return 0; - char mg[8]; int32_t h[8], w[8]; kv_hdr(m,w,0); - if(fread(mg,1,8,f)!=8 || memcmp(mg,KV_MAGIC,8) || fread(h,4,8,f)!=8 || - h[0]!=w[0]||h[1]!=w[1]||h[2]!=w[2]||h[3]!=w[3]||h[4]!=w[4]||h[5]!=w[5]){ - fprintf(stderr,"[KV] ignoring .coli_kv from a different model or version\n"); fclose(f); return 0; } - int nrec=h[6]; - if(nrec<1){ fclose(f); return 0; } - if(nrec>=maxctx-8-g_draft){ - fprintf(stderr,"[KV] saved conversation (%d tokens) exceeds the context: starting over\n",nrec); - fclose(f); return 0; } - double t0=now_s(); - for(int p=0;pn_layers;i++){ - if(fread(m->Lc[i]+(int64_t)p*c->kv_lora, 4, c->kv_lora, f)!=(size_t)c->kv_lora || - fread(m->Rc[i]+(int64_t)p*c->qk_rope, 4, c->qk_rope, f)!=(size_t)c->qk_rope){ nrec=p; goto out; } - } - if(m->has_dsa) for(int i=0;in_layers;i++) if(m->Ic[i]) - if(fread(m->Ic[i]+(int64_t)p*c->index_hd, 4, c->index_hd, f)!=(size_t)c->index_hd){ nrec=p; goto out; } - } -out: - fclose(f); - if(nrec>0){ - if(m->has_mtp) m->kv_start[c->n_layers]=-1; /* la finestra MTP riparte da sola */ - fprintf(stderr,"[KV] resumed conversation from disk: %d tokens in %.1fs (no re-prefill)\n", - nrec, now_s()-t0); - } - k->disk_nrec=nrec; - return nrec; -} typedef struct { KVState kv; int *hist, len, first; } ServeCtx; static double kv_pool_bytes(Model *m, int max_ctx); @@ -5622,212 +4636,7 @@ static int *read_arr(jval*o,const char*k,int*n){ if(!r){ fprintf(stderr,"OOM read_arr\n"); exit(1); } for(int i=0;ilen;i++) r[i]=(int)a->kids[i]->num; *n=a->len; return r; } -/* byte residenti di un tensore [O,I] al numero di bit dato (specchio di qt_bytes) */ -static int64_t tbytes(int O,int I,int bits){ - if(bits>=16) return (int64_t)O*I*4; - if(bits>=5) return (int64_t)O*I + (int64_t)O*4; - return (int64_t)O*((I+1)/2) + (int64_t)O*4; -} -/* byte VERI di un expert: dal container se pre-quantizzato, altrimenti stima da ebits */ -static int64_t expert_bytes_probe(Model *m, int ebits){ - Cfg *c=&m->c; int64_t eb=0; char nm[256]; - snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.0.gate_proj.weight",c->first_dense); - if(st_nbytes(&m->S,nm)>0){ - const char *suf[3]={"gate_proj","up_proj","down_proj"}; - for(int k=0;k<3;k++){ - snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.0.%s.weight",c->first_dense,suf[k]); - eb+=st_nbytes(&m->S,nm); - snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.0.%s.weight.qs",c->first_dense,suf[k]); - int64_t q=st_nbytes(&m->S,nm); if(q>0) eb+=q; - } - } - if(eb<=0) eb = tbytes(c->moe_inter,c->hidden,ebits)*2 + tbytes(c->hidden,c->moe_inter,ebits); - return eb; -} - -/* TIERS: fotografia della piramide expert per la dashboard web — - * "TIERS " sul canale di protocollo. - * ram = pinnati non-VRAM + LRU corrente; disk = tutto il resto. */ -/* BRAIN MAP (dashboard): per-turn expert hit bitmap + full residency/heat map. - * g_ehit[layer][eid]=1 quando l'expert viene instradato in questo turno; - * hits_emit lo serializza e lo azzera. emap_emit fotografa tier+heat di TUTTI. */ -static uint8_t **g_ehit; -static void ehit_mark(Model *m, int layer, int eid){ - if(!g_ehit){ Cfg *c=&m->c; - g_ehit=calloc(c->n_layers+1,sizeof(uint8_t*)); - for(int i=0;i<=c->n_layers;i++) g_ehit[i]=calloc(c->n_experts,1); - } - g_ehit[layer][eid]=1; -} - -/* HWINFO: hardware snapshot for the web dashboard — emitted once at READY. */ -/* CPU model + cores + RAM (GB); empty/zero where unavailable. - * Shared by the dashboard HWINFO line and the PROF=1 header. */ -static void hw_probe(char *cpu, size_t cn, int *cores, double *ram_total, double *ram_avail){ - cpu[0]=0; -#ifdef _WIN32 - /* niente /proc su Windows: brand string via CPUID (0x80000002..4), zero - * dipendenze extra. La dashboard mostrava "0 GB RAM / 0 cores" perche' - * tutto questo blocco era solo-Linux mentre il ramo CUDA funzionava. */ -#if defined(__x86_64__) || defined(__i386__) - { unsigned int r[12]={0}; unsigned int *w=r; - for(unsigned int f=0x80000002u; f<=0x80000004u; f++,w+=4) - __get_cpuid(f,&w[0],&w[1],&w[2],&w[3]); - char *b=(char*)r; b[47]=0; while(*b==' ')b++; - snprintf(cpu,cn,"%s",b); } -#endif -#else - FILE *ci=fopen("/proc/cpuinfo","r"); - if(ci){ char ln[256]; - while(fgets(ln,sizeof(ln),ci)) if(!strncmp(ln,"model name",10)){ - char *p=strchr(ln,':'); if(p){ p++; while(*p==' ')p++; - int n=(int)strlen(p); if(n>0&&p[n-1]=='\n')p[--n]=0; - snprintf(cpu,cn,"%s",p); } break; } - fclose(ci); } -#endif - *cores=0; -#ifdef _WIN32 - { SYSTEM_INFO si; GetSystemInfo(&si); *cores=(int)si.dwNumberOfProcessors; } -#elif defined(_SC_NPROCESSORS_ONLN) - *cores=(int)sysconf(_SC_NPROCESSORS_ONLN); -#endif - *ram_total=*ram_avail=0; -#ifdef _WIN32 - compat_meminfo(ram_total,ram_avail); /* GlobalMemoryStatusEx, gia' in compat.h */ -#else - FILE *mi=fopen("/proc/meminfo","r"); - if(mi){ char ln[256]; double mt=0,ma=0; - while(fgets(ln,sizeof(ln),mi)){ - if(sscanf(ln,"MemTotal: %lf",&mt)==1) *ram_total=mt/1e6; - if(sscanf(ln,"MemAvailable: %lf",&ma)==1) *ram_avail=ma/1e6; - } fclose(mi); } -#endif -} - -static void hwinfo_emit(Model *m){ - Cfg *c=&m->c; (void)c; /* silence -Wunused on builds without /proc (#148 report) */ - char cpu[256]; int cores; double ram_total,ram_avail; - hw_probe(cpu,sizeof(cpu),&cores,&ram_total,&ram_avail); - /* GPU */ - int ngpu=0; double vram_total=0; - char gpu_name[128]=""; -#ifdef COLI_CUDA - ngpu=g_cuda_ndev; vram_total=m->gpu_expert_bytes/1e9; - for(int i=0;i0){ - /* We don't have a device-name API; parse from the init log line stored in stderr. - * Simpler: just read the nvidia driver sysfs or use a fixed label. */ - snprintf(gpu_name,sizeof(gpu_name),"CUDA device x%d",g_cuda_ndev); - } -#endif - printf("HWINFO %d %.1f %.1f %d %.1f %s|%s\n", - cores,ram_total,ram_avail,ngpu,vram_total,cpu,gpu_name); - fflush(stdout); -} - -static void tiers_emit(Model *m){ - Cfg *c=&m->c; int nsp=0; - for(int i=0;in_layers;i++) if(m->L[i].sparse) nsp++; - int total=(nsp+(m->has_mtp?1:0))*c->n_experts; - int pinned=0,lru=0; - for(int i=0;i<=c->n_layers;i++){ pinned+=m->npin?m->npin[i]:0; lru+=m->ecn?m->ecn[i]:0; } - int vram=0; double vram_gb=0; -#ifdef COLI_CUDA - vram=m->gpu_expert_count; vram_gb=m->gpu_expert_bytes/1e9; -#endif - int ram=pinned-vram+lru; if(ram<0) ram=0; - int disk=total-vram-ram; if(disk<0) disk=0; - double eb=(double)expert_bytes_probe(m,m->ebits); - printf("TIERS %d %d %d %.2f %.2f\n",vram,ram,disk,vram_gb,ram*eb/1e9); - fflush(stdout); -} - -/* EMAP: 1 byte/expert (2bit tier: 0=disk 1=RAM 2=VRAM | 6bit heat log2-bucket), - * righe = layer sparsi in ordine (+MTP se presente), colonne = n_experts. Hex. */ -static void emap_emit(Model *m){ - Cfg *c=&m->c; - int rows=0; - for(int i=0;in_layers;i++) if(m->L[i].sparse) rows++; - int has_mtp = m->has_mtp && m->eusage[c->n_layers]; - if(has_mtp) rows++; - int cols=c->n_experts; - char *hex=malloc((size_t)rows*cols*2+1); int w=0; - for(int i=0;i<=c->n_layers;i++){ - int is_row = (in_layers && m->L[i].sparse) || (i==c->n_layers && has_mtp); - if(!is_row) continue; - for(int e=0;epin[i]; - for(int z=0;znpin[i];z++) if(P[z].eid==e){ -#ifdef COLI_CUDA - tier = P[z].g.cuda?2:1; -#else - tier = 1; -#endif - break; } - if(!tier && m->ecache && m->ecache[i]) - for(int z=0;zecn[i];z++) if(m->ecache[i][z].eid==e){ tier=1; break; } - uint32_t u = m->eusage[i]?m->eusage[i][e]:0; - int heat=0; while(u){ heat++; u>>=1; } if(heat>63) heat=63; - int b=(tier<<6)|heat; - hex[w++]="0123456789abcdef"[b>>4]; hex[w++]="0123456789abcdef"[b&15]; - } - } - hex[w]=0; - printf("EMAP %d %d %s\n",rows,cols,hex); fflush(stdout); free(hex); -} - -/* HITS: bitmap 1bit/expert (stesso ordine di EMAP), poi azzera per il turno dopo. */ -static void hits_emit(Model *m){ - Cfg *c=&m->c; if(!g_ehit) return; - int rows=0; - for(int i=0;in_layers;i++) if(m->L[i].sparse) rows++; - int has_mtp = m->has_mtp && m->eusage[c->n_layers]; - if(has_mtp) rows++; - int cols=c->n_experts, nb=(rows*cols+7)/8; - uint8_t *bm=calloc(nb,1); int bit=0; - for(int i=0;i<=c->n_layers;i++){ - int is_row = (in_layers && m->L[i].sparse) || (i==c->n_layers && has_mtp); - if(!is_row) continue; - for(int e=0;e>3]|=1<<(bit&7); g_ehit[i][e]=0; } - } - char *hex=malloc((size_t)nb*2+1); int w=0; - for(int b=0;b>4]; hex[w++]="0123456789abcdef"[bm[b]&15]; } - hex[w]=0; - printf("HITS %d %d %s\n",rows,cols,hex); fflush(stdout); free(hex); free(bm); -} - -/* scarica su file l'istogramma d'uso degli expert: righe "layer eid count" (per PIN). - * Include la riga MTP (layer n_layers). Scrittura atomica (tmp+rename): viene chiamata - * anche a ogni turno di serve e il processo puo' morire in qualsiasi momento. */ -static void stats_dump_q(Model *m, const char *path, int quiet){ - char tmp[2100]; snprintf(tmp,sizeof(tmp),"%s.tmp",path); - FILE *f=fopen(tmp,"w"); if(!f){ if(!quiet) perror(tmp); return; } - Cfg *c=&m->c; int64_t tot=0, nz=0; - for(int i=0;i<=c->n_layers;i++){ if(!m->eusage[i]) continue; - for(int e=0;en_experts;e++) if(m->eusage[i][e]){ fprintf(f,"%d %d %u\n",i,e,m->eusage[i][e]); tot+=m->eusage[i][e]; nz++; } } - fclose(f); rename(tmp,path); - if(!quiet) fprintf(stderr,"[STATS] %lld selections across %lld distinct experts -> %s\n",(long long)tot,(long long)nz,path); -} -static void stats_dump(Model *m, const char *path){ stats_dump_q(m,path,0); } - -/* CACHE CHE IMPARA: istogramma d'uso PERSISTENTE in /.coli_usage. - * Caricato all'avvio (i contatori ripartono dalla storia), salvato a ogni turno: - * piu' usi colibri', meglio l'auto-pin conosce i TUOI expert caldi. */ -static char g_usage_path[2100]=""; -static int64_t usage_load(Model *m, const char *path){ - FILE *f=fopen(path,"r"); if(!f) return 0; - Cfg *c=&m->c; int l,e; uint32_t cnt; int64_t tot=0; - while(fscanf(f,"%d %d %u",&l,&e,&cnt)==3) - if(l>=0&&l<=c->n_layers&&e>=0&&en_experts&&m->eusage[l]){ m->eusage[l][e]+=cnt; tot+=cnt; } - fclose(f); return tot; -} -static void usage_save(Model *m){ if(g_usage_path[0]) stats_dump_q(m,g_usage_path,1); } +/* telemetry, stats, usage persistence — moved to telemetry.h */ /* HOT-STORE ("il redis del colibri'"): carica in RAM, UNA VOLTA e per sempre, i top expert * per frequenza d'uso misurata (file STATS di un run precedente), entro un budget in GB. diff --git a/c/kv_persist.h b/c/kv_persist.h new file mode 100644 index 00000000..51289a89 --- /dev/null +++ b/c/kv_persist.h @@ -0,0 +1,121 @@ +/* kv_persist.h — .coli_kv on-disk KV cache persistence. + * Conversations reopen warm across engine restarts: the compressed MLA KV-cache + * is appended incrementally after every turn, crash-safe (nrec written last). + * Include after Model/KVState/Cfg are defined; requires now_s() and g_draft. */ +#ifndef KV_PERSIST_H +#define KV_PERSIST_H + +static int g_kvsave=1; +#define KV_MAGIC "COLIKV1\0" + +static void kv_hdr(Model *m, int32_t *h, int nrec){ + Cfg *c=&m->c; int nic=0; + for(int i=0;in_layers;i++) if(m->Ic && m->Ic[i]) nic++; + h[0]=c->n_layers; h[1]=c->kv_lora; h[2]=c->qk_rope; + h[3]=m->has_dsa?c->index_hd:0; h[4]=nic; h[5]=c->vocab; h[6]=nrec; h[7]=0; +} + +static int64_t kv_rec_bytes(Model *m){ + Cfg *c=&m->c; + int64_t rec = 4 + (int64_t)c->n_layers*(c->kv_lora+c->qk_rope)*4; + if(m->has_dsa) for(int i=0;in_layers;i++) if(m->Ic[i]) rec+=(int64_t)c->index_hd*4; + return rec; +} + +static int kv_disk_open(Model *m){ + KVState *k=m->kv; + if(k->disk_fp) return 1; + k->disk_fp=fopen(k->disk_path,"r+b"); + if(!k->disk_fp){ + k->disk_fp=fopen(k->disk_path,"wb"); + if(!k->disk_fp) return 0; + int32_t h[8]; kv_hdr(m,h,0); + fwrite(KV_MAGIC,1,8,k->disk_fp); fwrite(h,4,8,k->disk_fp); + fflush(k->disk_fp); + fclose(k->disk_fp); + k->disk_fp=fopen(k->disk_path,"r+b"); + if(!k->disk_fp) return 0; + } + return 1; +} + +static void kv_disk_truncate(Model *m, int nrec){ + if(!g_kvsave) return; + KVState *k=m->kv; + if(k->disk_fp){ fclose(k->disk_fp); k->disk_fp=NULL; } + FILE *f=fopen(k->disk_path,"r+b"); + if(!f){ k->disk_nrec=0; return; } + k->disk_nrec=nrec; + int32_t nr=nrec; fseek(f,8+6*4,SEEK_SET); fwrite(&nr,4,1,f); + fflush(f); fclose(f); +} + +static void kv_disk_reset(Model *m){ kv_disk_truncate(m,0); } + +static void kv_disk_append(Model *m, const int *hist, int len){ + KVState *k=m->kv; + if(!g_kvsave || len<=k->disk_nrec) return; + Cfg *c=&m->c; + if(!kv_disk_open(m)) return; + FILE *f=k->disk_fp; + int64_t rec = kv_rec_bytes(m); + if(rec > k->disk_buf_cap){ + uint8_t *nb=realloc(k->disk_buf, rec); + if(!nb) return; + k->disk_buf=nb; k->disk_buf_cap=rec; + } + fseek(f, 8+8*4 + (int64_t)k->disk_nrec*rec, SEEK_SET); + for(int p=k->disk_nrec;pdisk_buf; + *(int32_t*)b = hist[p]; b+=4; + for(int i=0;in_layers;i++){ + memcpy(b, m->Lc[i]+(int64_t)p*c->kv_lora, (size_t)c->kv_lora*4); b+=c->kv_lora*4; + memcpy(b, m->Rc[i]+(int64_t)p*c->qk_rope,(size_t)c->qk_rope*4); b+=c->qk_rope*4; + } + if(m->has_dsa) for(int i=0;in_layers;i++) if(m->Ic[i]){ + memcpy(b, m->Ic[i]+(int64_t)p*c->index_hd, (size_t)c->index_hd*4); b+=c->index_hd*4; + } + fwrite(k->disk_buf, 1, (size_t)rec, f); + } + fflush(f); + int32_t nr=len; fseek(f,8+6*4,SEEK_SET); fwrite(&nr,4,1,f); + fflush(f); + k->disk_nrec=len; +} + +static int kv_disk_load(Model *m, int *hist, int maxctx){ + if(!g_kvsave) return 0; + KVState *k=m->kv; + Cfg *c=&m->c; + FILE *f=fopen(k->disk_path,"rb"); if(!f) return 0; + char mg[8]; int32_t h[8], w[8]; kv_hdr(m,w,0); + if(fread(mg,1,8,f)!=8 || memcmp(mg,KV_MAGIC,8) || fread(h,4,8,f)!=8 || + h[0]!=w[0]||h[1]!=w[1]||h[2]!=w[2]||h[3]!=w[3]||h[4]!=w[4]||h[5]!=w[5]){ + fprintf(stderr,"[KV] ignoring .coli_kv from a different model or version\n"); fclose(f); return 0; } + int nrec=h[6]; + if(nrec<1){ fclose(f); return 0; } + if(nrec>=maxctx-8-g_draft){ + fprintf(stderr,"[KV] saved conversation (%d tokens) exceeds the context: starting over\n",nrec); + fclose(f); return 0; } + double t0=now_s(); + for(int p=0;pn_layers;i++){ + if(fread(m->Lc[i]+(int64_t)p*c->kv_lora, 4, c->kv_lora, f)!=(size_t)c->kv_lora || + fread(m->Rc[i]+(int64_t)p*c->qk_rope, 4, c->qk_rope, f)!=(size_t)c->qk_rope){ nrec=p; goto out; } + } + if(m->has_dsa) for(int i=0;in_layers;i++) if(m->Ic[i]) + if(fread(m->Ic[i]+(int64_t)p*c->index_hd, 4, c->index_hd, f)!=(size_t)c->index_hd){ nrec=p; goto out; } + } +out: + fclose(f); + if(nrec>0){ + if(m->has_mtp) m->kv_start[c->n_layers]=-1; + fprintf(stderr,"[KV] resumed conversation from disk: %d tokens in %.1fs (no re-prefill)\n", + nrec, now_s()-t0); + } + k->disk_nrec=nrec; + return nrec; +} + +#endif /* KV_PERSIST_H */ diff --git a/c/quant.h b/c/quant.h new file mode 100644 index 00000000..928de965 --- /dev/null +++ b/c/quant.h @@ -0,0 +1,672 @@ +/* quant.h — quantized matmul kernels (header-only, all functions static). + * Multi-architecture SIMD: AVX2 / AVX-512 / AVX-VNNI / ARM NEON / NEON-SDOT / + * NEON-i8mm / POWER VSX. Pure compute — no Model or QT dependency. */ +#ifndef COLI_QUANT_H +#define COLI_QUANT_H + +#include +#include +#include +#include +#include + +#ifdef _OPENMP +#include +#endif + +/* ---- SIMD includes -------------------------------------------------------- */ +#ifdef __AVX2__ +#include +static inline float hsum256(__m256 v){ + __m128 lo=_mm256_castps256_ps128(v), hi=_mm256_extractf128_ps(v,1); + lo=_mm_add_ps(lo,hi); __m128 sh=_mm_movehl_ps(lo,lo); lo=_mm_add_ps(lo,sh); + sh=_mm_shuffle_ps(lo,lo,1); lo=_mm_add_ss(lo,sh); return _mm_cvtss_f32(lo); +} +static inline int hsum256_i32(__m256i v){ + __m128i lo=_mm256_castsi256_si128(v), hi=_mm256_extracti128_si256(v,1); + lo=_mm_add_epi32(lo,hi); lo=_mm_hadd_epi32(lo,lo); lo=_mm_hadd_epi32(lo,lo); + return _mm_cvtsi128_si32(lo); +} +#endif +#if defined(__AVXVNNI__) && defined(__AVX2__) +static inline int hsum128_i32(__m128i v){ + v=_mm_hadd_epi32(v,v); v=_mm_hadd_epi32(v,v); return _mm_cvtsi128_si32(v); +} +#endif +#ifdef __ARM_NEON +#include +#endif +#ifdef __VSX__ +#include +#undef vector +#undef pixel +#undef bool +#endif + +/* ---- AVX-512 int4->float accumulator -------------------------------------- */ +#if defined(__AVX512F__) && defined(__AVX512BW__) +static int g_i4_acc512=1; +static inline float dot_i4f_avx512(const uint8_t *w,const float *x,int I){ + const __m128i m4=_mm_set1_epi8(0x0F); const __m512i b8=_mm512_set1_epi32(8); + __m512 acc0=_mm512_setzero_ps(),acc1=_mm512_setzero_ps(); int i=0; + for(;i+32<=I;i+=32){ __m128i by=_mm_loadu_si128((const __m128i*)(w+(i>>1))); + __m128i lo=_mm_and_si128(by,m4),hi=_mm_and_si128(_mm_srli_epi16(by,4),m4); + __m128i n0=_mm_unpacklo_epi8(lo,hi),n1=_mm_unpackhi_epi8(lo,hi); + __m512 w0=_mm512_cvtepi32_ps(_mm512_sub_epi32(_mm512_cvtepu8_epi32(n0),b8)); + __m512 w1=_mm512_cvtepi32_ps(_mm512_sub_epi32(_mm512_cvtepu8_epi32(n1),b8)); + acc0=_mm512_fmadd_ps(_mm512_loadu_ps(x+i),w0,acc0); + acc1=_mm512_fmadd_ps(_mm512_loadu_ps(x+i+16),w1,acc1); + } + return _mm512_reduce_add_ps(_mm512_add_ps(acc0,acc1)); +} +static int i4_acc512_selftest(void){ + enum { N=224 }; uint8_t w[(N+1)/2]; float x[N]; + for(int i=0;i>1]=(uint8_t)(q+8); + else w[i>>1]|=(uint8_t)((q+8)<<4); + x[i]=(float)(((i*29+7)%101)-50)/37.f; + } + for(int n=32;n<=N;n+=32){ + float ref=0; for(int i=0;i>1]>>((i&1)*4))&15)-8); + float got=dot_i4f_avx512(w,x,n),tol=2e-5f*(1.f+fabsf(ref)); + if(fabsf(got-ref)>tol){ fprintf(stderr,"AVX512 i4 selftest n=%d: %.9g != %.9g\n",n,got,ref); return 0; } + } + return 1; +} +#endif + +/* ---- y[S,O] = x[S,I] @ W^T, W[O,I] f32 ---------------------------------- */ +static void matmul(float *y, const float *x, const float *W, int S, int I, int O){ + #pragma omp parallel for schedule(static) + for (int o=0;o>1))); + __m128i lo=_mm_and_si128(by,m4), hi=_mm_and_si128(_mm_srli_epi16(by,4),m4); + __m128i nib=_mm_unpacklo_epi8(lo,hi); + __m256 w0=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(nib),b8)); + __m256 w1=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(nib,8)),b8)); + acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i), w0, acc); + acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i+8), w1, acc); } + a=hsum256(acc); +#elif defined(__ARM_NEON) + const uint8x8_t m4=vdup_n_u8(0x0F); const int8x8_t b8=vdup_n_s8(8); + float32x4_t ac0=vdupq_n_f32(0), ac1=vdupq_n_f32(0); + for(;i+16<=I;i+=16){ uint8x8_t by=vld1_u8(w+(i>>1)); + uint8x8x2_t z=vzip_u8(vand_u8(by,m4), vshr_n_u8(by,4)); + int16x8_t w0=vmovl_s8(vsub_s8(vreinterpret_s8_u8(z.val[0]),b8)); + int16x8_t w1=vmovl_s8(vsub_s8(vreinterpret_s8_u8(z.val[1]),b8)); + ac0=vfmaq_f32(ac0, vld1q_f32(xs+i), vcvtq_f32_s32(vmovl_s16(vget_low_s16(w0)))); + ac1=vfmaq_f32(ac1, vld1q_f32(xs+i+4), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w0)))); + ac0=vfmaq_f32(ac0, vld1q_f32(xs+i+8), vcvtq_f32_s32(vmovl_s16(vget_low_s16(w1)))); + ac1=vfmaq_f32(ac1, vld1q_f32(xs+i+12), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w1)))); } + a=vaddvq_f32(vaddq_f32(ac0,ac1)); +#endif +#if defined(__AVX512F__) && defined(__AVX512BW__) + } +#endif + for(;i+1>1]; int lo=(int)(byte&0xF)-8, hi=(int)(byte>>4)-8; + a += xs[i]*(float)lo + xs[i+1]*(float)hi; } + if(i>1]; int lo=(int)(byte&0xF)-8; a += xs[i]*(float)lo; } + y[(int64_t)s*O+o]=a*sc; } } +} + +/* ---- y[S,O] = x[S,I] @ W^T, W int4 packed + per-GROUP scales (fmt=4) ----- */ +static void matmul_i4_grouped(float *y, const float *x, const uint8_t *q4, const float *scale, + int S, int I, int O, int gs){ + int rb=(I+1)/2; int ng=(I+gs-1)/gs; + #pragma omp parallel for schedule(static) + for(int o=0;oI) glen=I-base; + float sc=scl[g]; + int i=base; +#ifdef __AVX2__ + const __m128i m4=_mm_set1_epi8(0x0F); const __m256i b8=_mm256_set1_epi32(8); + __m256 acc=_mm256_setzero_ps(); + for(; i+16<=base+glen; i+=16){ __m128i by=_mm_loadl_epi64((const __m128i*)(w+(i>>1))); + __m128i lo=_mm_and_si128(by,m4),hi=_mm_and_si128(_mm_srli_epi16(by,4),m4); + __m128i nib=_mm_unpacklo_epi8(lo,hi); + __m256 w0=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(nib),b8)); + __m256 w1=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(nib,8)),b8)); + acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i), w0, acc); + acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i+8), w1, acc); } + a+=hsum256(acc)*sc; +#endif + for(; i>1]; + a+=(xs[i]*(float)((int)(byte&0xF)-8)+xs[i+1]*(float)((int)(byte>>4)-8))*sc; } + else { uint8_t byte=w[i>>1]; a+=xs[i]*(float)((int)(byte&0xF)-8)*sc; } + } + } + y[(int64_t)s*O+o]=a; + } + } +} + +/* ---- fused gate+up: one OMP dispatch for both matrices -------------------- */ +static void matmul_i4_pair(float *yg, float *yu, const float *x, + const uint8_t *qg, const float *sg, + const uint8_t *qu, const float *su, int I, int O){ + int rb=(I+1)/2; + #pragma omp parallel for schedule(static) + for(int z=0;z<2*O;z++){ + int o=z>1))); + __m128i lo=_mm_and_si128(by,m4),hi=_mm_and_si128(_mm_srli_epi16(by,4),m4); + __m128i nib=_mm_unpacklo_epi8(lo,hi); + __m256 w0=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(nib),b8)); + __m256 w1=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(nib,8)),b8)); + acc=_mm256_fmadd_ps(_mm256_loadu_ps(x+i),w0,acc); + acc=_mm256_fmadd_ps(_mm256_loadu_ps(x+i+8),w1,acc); } + a=hsum256(acc); +#elif defined(__ARM_NEON) + const uint8x8_t m4=vdup_n_u8(0x0F); const int8x8_t b8=vdup_n_s8(8); + float32x4_t ac0=vdupq_n_f32(0),ac1=vdupq_n_f32(0); + for(;i+16<=I;i+=16){ uint8x8_t by=vld1_u8(w+(i>>1)); + uint8x8x2_t n=vzip_u8(vand_u8(by,m4),vshr_n_u8(by,4)); + int16x8_t w0=vmovl_s8(vsub_s8(vreinterpret_s8_u8(n.val[0]),b8)); + int16x8_t w1=vmovl_s8(vsub_s8(vreinterpret_s8_u8(n.val[1]),b8)); + ac0=vfmaq_f32(ac0,vld1q_f32(x+i),vcvtq_f32_s32(vmovl_s16(vget_low_s16(w0)))); + ac1=vfmaq_f32(ac1,vld1q_f32(x+i+4),vcvtq_f32_s32(vmovl_s16(vget_high_s16(w0)))); + ac0=vfmaq_f32(ac0,vld1q_f32(x+i+8),vcvtq_f32_s32(vmovl_s16(vget_low_s16(w1)))); + ac1=vfmaq_f32(ac1,vld1q_f32(x+i+12),vcvtq_f32_s32(vmovl_s16(vget_high_s16(w1)))); } + a=vaddvq_f32(vaddq_f32(ac0,ac1)); +#endif +#if defined(__AVX512F__) && defined(__AVX512BW__) + } +#endif + for(;i+1>1]; a+=x[i]*(float)((b&15)-8)+x[i+1]*(float)((b>>4)-8); } + if(i>1]&15)-8); + (z>2))); + __m128i p0=_mm_and_si128(by,m2), p1=_mm_and_si128(_mm_srli_epi16(by,2),m2); + __m128i p2=_mm_and_si128(_mm_srli_epi16(by,4),m2), p3=_mm_and_si128(_mm_srli_epi16(by,6),m2); + __m128i lo=_mm_unpacklo_epi8(p0,p1), hi=_mm_unpacklo_epi8(p2,p3); + __m128i nib=_mm_unpacklo_epi16(lo,hi); + __m256 w0=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(nib),b2)); + __m256 w1=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(nib,8)),b2)); + acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i), w0, acc); + acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i+8), w1, acc); } + a=hsum256(acc); +#elif defined(__ARM_NEON) + const uint8x8_t m2v=vdup_n_u8(3); const int8x8_t b2v=vdup_n_s8(2); + float32x4_t ac0=vdupq_n_f32(0), ac1=vdupq_n_f32(0); + for(;i+16<=I;i+=16){ uint32_t wd; memcpy(&wd, w+(i>>2), 4); + uint8x8_t by=vreinterpret_u8_u32(vdup_n_u32(wd)); + uint8x8x2_t z01=vzip_u8(vand_u8(by,m2v), vand_u8(vshr_n_u8(by,2),m2v)); + uint8x8x2_t z23=vzip_u8(vand_u8(vshr_n_u8(by,4),m2v), vshr_n_u8(by,6)); + uint16x4x2_t zz=vzip_u16(vreinterpret_u16_u8(z01.val[0]), vreinterpret_u16_u8(z23.val[0])); + int16x8_t w0=vmovl_s8(vsub_s8(vreinterpret_s8_u16(zz.val[0]),b2v)); + int16x8_t w1=vmovl_s8(vsub_s8(vreinterpret_s8_u16(zz.val[1]),b2v)); + ac0=vfmaq_f32(ac0, vld1q_f32(xs+i), vcvtq_f32_s32(vmovl_s16(vget_low_s16(w0)))); + ac1=vfmaq_f32(ac1, vld1q_f32(xs+i+4), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w0)))); + ac0=vfmaq_f32(ac0, vld1q_f32(xs+i+8), vcvtq_f32_s32(vmovl_s16(vget_low_s16(w1)))); + ac1=vfmaq_f32(ac1, vld1q_f32(xs+i+12), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w1)))); } + a=vaddvq_f32(vaddq_f32(ac0,ac1)); +#endif + for(;i>2]; int sh=(i&3)*2; a += xs[i]*(float)((int)((byte>>sh)&3)-2); } + y[(int64_t)s*O+o]=a*sc; } } +} + +/* ---- IDOT: integer dot kernels (int8-quantized activations) --------------- */ +#if defined(__AVX512VNNI__) && defined(__AVX512BW__) +#define IDOT_KERNEL "avx512-vnni" +#elif defined(__AVXVNNI__) && defined(__AVX2__) +#define IDOT_KERNEL "avx-vnni" +#elif defined(__AVX2__) +#define IDOT_KERNEL "avx2" +#elif defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8) +#define IDOT_KERNEL "neon-i8mm" +#elif defined(__ARM_NEON) +#define IDOT_KERNEL "neon" +#elif defined(__VSX__) +#define IDOT_KERNEL "vsx" +#else +#define IDOT_KERNEL "scalar" +#endif +static int g_idot=1; +#if defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD) +static int g_i4s=1; +#elif defined(__VSX__) +static int g_i4s=1; +#else +static int g_i4s=2; +#endif + +static inline float qrow_i8(const float *x, int8_t *q, int I){ + float amax=0; for(int i=0;iamax)amax=a; } + float s=amax/127.f; if(s<1e-12f) s=1e-12f; float inv=1.f/s; + for(int i=0;i>1))); + __m256i lo=_mm256_and_si256(by,m4v), hi=_mm256_and_si256(_mm256_srli_epi16(by,4),m4v); + __m256i z0=_mm256_unpacklo_epi8(lo,hi), z1=_mm256_unpackhi_epi8(lo,hi); + __m512i wv=_mm512_sub_epi8(_mm512_inserti64x4(_mm512_castsi256_si512(z0),z1,1),b8v); + __m512i xv=_mm512_permutexvar_epi64(xidx,_mm512_loadu_si512((const void*)(x+i))); + __mmask64 neg=_mm512_movepi8_mask(wv); + __m512i xs=_mm512_mask_sub_epi8(xv,neg,_mm512_setzero_si512(),xv); + acc=_mm512_dpbusd_epi32(acc,_mm512_abs_epi8(wv),xs); + } + sum=_mm512_reduce_add_epi32(acc); +#elif defined(__AVXVNNI__) && defined(__AVX2__) + const __m128i m4=_mm_set1_epi8(0x0F); const __m128i b8=_mm_set1_epi8(8); + __m128i acc=_mm_setzero_si128(); + for(;i+32<=I;i+=32){ + __m128i by=_mm_loadu_si128((const __m128i*)(w4+(i>>1))); + __m128i lo=_mm_and_si128(by,m4), hi=_mm_and_si128(_mm_srli_epi16(by,4),m4); + __m128i n0=_mm_unpacklo_epi8(lo,hi), n1=_mm_unpackhi_epi8(lo,hi); + __m128i w0=_mm_sub_epi8(n0,b8), w1=_mm_sub_epi8(n1,b8); + __m128i x0=_mm_loadu_si128((const __m128i*)(x+i)); + __m128i x1=_mm_loadu_si128((const __m128i*)(x+i+16)); + acc=_mm_dpbusd_epi32(acc,_mm_abs_epi8(w0),_mm_sign_epi8(x0,w0)); + acc=_mm_dpbusd_epi32(acc,_mm_abs_epi8(w1),_mm_sign_epi8(x1,w1)); + } + sum=hsum128_i32(acc); +#elif defined(__AVX2__) + const __m128i m4=_mm_set1_epi8(0x0F); const __m256i b8=_mm256_set1_epi8(8); + const __m256i ones=_mm256_set1_epi16(1); + __m256i acc=_mm256_setzero_si256(); + for(;i+32<=I;i+=32){ + __m128i by=_mm_loadu_si128((const __m128i*)(w4+(i>>1))); + __m128i lo=_mm_and_si128(by,m4), hi=_mm_and_si128(_mm_srli_epi16(by,4),m4); + __m128i n0=_mm_unpacklo_epi8(lo,hi), n1=_mm_unpackhi_epi8(lo,hi); + __m256i wv=_mm256_sub_epi8(_mm256_set_m128i(n1,n0),b8); + __m256i xv=_mm256_loadu_si256((const __m256i*)(x+i)); + __m256i p=_mm256_maddubs_epi16(_mm256_sign_epi8(wv,wv),_mm256_sign_epi8(xv,wv)); + acc=_mm256_add_epi32(acc,_mm256_madd_epi16(p,ones)); + } + sum=hsum256_i32(acc); +#elif defined(__ARM_NEON) + const uint8x16_t m4q=vdupq_n_u8(0x0F); const int8x16_t b8q=vdupq_n_s8(8); +#if defined(__ARM_FEATURE_DOTPROD) + int32x4_t a0=vdupq_n_s32(0),a1=vdupq_n_s32(0),a2=vdupq_n_s32(0),a3=vdupq_n_s32(0); + for(;i+64<=I;i+=64){ + uint8x16_t byA=vld1q_u8(w4+(i>>1)), byB=vld1q_u8(w4+(i>>1)+16); + uint8x16x2_t zA=vzipq_u8(vandq_u8(byA,m4q), vshrq_n_u8(byA,4)); + uint8x16x2_t zB=vzipq_u8(vandq_u8(byB,m4q), vshrq_n_u8(byB,4)); + a0=vdotq_s32(a0,vsubq_s8(vreinterpretq_s8_u8(zA.val[0]),b8q),vld1q_s8(x+i)); + a1=vdotq_s32(a1,vsubq_s8(vreinterpretq_s8_u8(zA.val[1]),b8q),vld1q_s8(x+i+16)); + a2=vdotq_s32(a2,vsubq_s8(vreinterpretq_s8_u8(zB.val[0]),b8q),vld1q_s8(x+i+32)); + a3=vdotq_s32(a3,vsubq_s8(vreinterpretq_s8_u8(zB.val[1]),b8q),vld1q_s8(x+i+48)); + } + int32x4_t acc=vaddq_s32(vaddq_s32(a0,a1),vaddq_s32(a2,a3)); + for(;i+32<=I;i+=32){ + uint8x16_t by=vld1q_u8(w4+(i>>1)); + uint8x16x2_t z=vzipq_u8(vandq_u8(by,m4q), vshrq_n_u8(by,4)); + acc=vdotq_s32(acc,vsubq_s8(vreinterpretq_s8_u8(z.val[0]),b8q),vld1q_s8(x+i)); + acc=vdotq_s32(acc,vsubq_s8(vreinterpretq_s8_u8(z.val[1]),b8q),vld1q_s8(x+i+16)); + } + sum=vaddvq_s32(acc); +#else + int32x4_t acc=vdupq_n_s32(0); + for(;i+32<=I;i+=32){ + uint8x16_t by=vld1q_u8(w4+(i>>1)); + uint8x16x2_t z=vzipq_u8(vandq_u8(by,m4q), vshrq_n_u8(by,4)); + int8x16_t w0=vsubq_s8(vreinterpretq_s8_u8(z.val[0]),b8q); + int8x16_t w1=vsubq_s8(vreinterpretq_s8_u8(z.val[1]),b8q); + int8x16_t x0=vld1q_s8(x+i), x1=vld1q_s8(x+i+16); + int16x8_t p=vmull_s8(vget_low_s8(w0),vget_low_s8(x0)); + p=vmlal_s8(p,vget_high_s8(w0),vget_high_s8(x0)); + acc=vpadalq_s16(acc,p); + p=vmull_s8(vget_low_s8(w1),vget_low_s8(x1)); + p=vmlal_s8(p,vget_high_s8(w1),vget_high_s8(x1)); + acc=vpadalq_s16(acc,p); + } + sum=vaddvq_s32(acc); +#endif +#elif defined(__VSX__) + const __vector unsigned char m4v=vec_splats((unsigned char)0x0F); + const __vector unsigned char sh4=vec_splats((unsigned char)4); + const __vector signed char b8v=vec_splats((signed char)8); + const __vector signed char vz=vec_splats((signed char)0); + __vector signed int acc=vec_splats(0); + for(;i+32<=I;i+=32){ + __vector unsigned char by=vec_xl(0,w4+(i>>1)); + __vector unsigned char lo=vec_and(by,m4v), hi=vec_sr(by,sh4); + __vector signed char w0=vec_sub((__vector signed char)vec_mergeh(lo,hi),b8v); + __vector signed char w1=vec_sub((__vector signed char)vec_mergel(lo,hi),b8v); + __vector signed char x0=vec_xl(0,(const signed char*)(x+i)); + __vector signed char x1=vec_xl(0,(const signed char*)(x+i+16)); + __vector __bool char n0=vec_cmplt(w0,vz), n1=vec_cmplt(w1,vz); + acc=vec_msum(vec_sel(x0,vec_sub(vz,x0),n0), + (__vector unsigned char)vec_sel(w0,vec_sub(vz,w0),n0),acc); + acc=vec_msum(vec_sel(x1,vec_sub(vz,x1),n1), + (__vector unsigned char)vec_sel(w1,vec_sub(vz,w1),n1),acc); + } + sum=vec_extract(acc,0)+vec_extract(acc,1)+vec_extract(acc,2)+vec_extract(acc,3); +#endif + for(;i+1>1]; sum+=((int)(b&0xF)-8)*x[i]+((int)(b>>4)-8)*x[i+1]; } + if(i>1]; sum+=((int)(b&0xF)-8)*x[i]; } + return sum; +} + +/* ---- ARM i8mm SMMLA tiled kernels ---------------------------------------- */ +#if defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8) +static inline int32x4_t mm_tile16(int32x4_t acc, int8x16_t wo, int8x16_t wo1, + int8x16_t xs, int8x16_t xs1){ + acc=vmmlaq_s32(acc, vcombine_s8(vget_low_s8(wo), vget_low_s8(wo1)), + vcombine_s8(vget_low_s8(xs), vget_low_s8(xs1))); + return vmmlaq_s32(acc, vcombine_s8(vget_high_s8(wo), vget_high_s8(wo1)), + vcombine_s8(vget_high_s8(xs), vget_high_s8(xs1))); +} +static void matmul_q_idot_mm(float *y, const int8_t *xq, const float *sx, const int8_t *q, + const float *scale, int S, int I, int O){ + #pragma omp parallel for schedule(static) + for(int o=0;o<(O&~1);o+=2){ + const int8_t *wo=q+(int64_t)o*I, *wo1=q+(int64_t)(o+1)*I; + float sc0=scale[o], sc1=scale[o+1]; + for(int s=0;s<(S&~1);s+=2){ + const int8_t *xs=xq+(int64_t)s*I, *xs1=xq+(int64_t)(s+1)*I; + int32x4_t a0=vdupq_n_s32(0),a1=vdupq_n_s32(0),a2=vdupq_n_s32(0),a3=vdupq_n_s32(0); int i=0; + for(;i+64<=I;i+=64){ + a0=mm_tile16(a0,vld1q_s8(wo+i), vld1q_s8(wo1+i), vld1q_s8(xs+i), vld1q_s8(xs1+i)); + a1=mm_tile16(a1,vld1q_s8(wo+i+16),vld1q_s8(wo1+i+16),vld1q_s8(xs+i+16),vld1q_s8(xs1+i+16)); + a2=mm_tile16(a2,vld1q_s8(wo+i+32),vld1q_s8(wo1+i+32),vld1q_s8(xs+i+32),vld1q_s8(xs1+i+32)); + a3=mm_tile16(a3,vld1q_s8(wo+i+48),vld1q_s8(wo1+i+48),vld1q_s8(xs+i+48),vld1q_s8(xs1+i+48)); + } + for(;i+16<=I;i+=16) + a0=mm_tile16(a0,vld1q_s8(wo+i),vld1q_s8(wo1+i),vld1q_s8(xs+i),vld1q_s8(xs1+i)); + int32x4_t acc=vaddq_s32(vaddq_s32(a0,a1),vaddq_s32(a2,a3)); + int32_t d00=vgetq_lane_s32(acc,0), d01=vgetq_lane_s32(acc,1); + int32_t d10=vgetq_lane_s32(acc,2), d11=vgetq_lane_s32(acc,3); + for(;i>1)), byo1=vld1q_u8(wo1+(i>>1)); + uint8x16_t cyo=vld1q_u8(wo+(i>>1)+16), cyo1=vld1q_u8(wo1+(i>>1)+16); + uint8x16x2_t zo =vzipq_u8(vandq_u8(byo, m4q), vshrq_n_u8(byo, 4)); + uint8x16x2_t zo1=vzipq_u8(vandq_u8(byo1,m4q), vshrq_n_u8(byo1,4)); + uint8x16x2_t ko =vzipq_u8(vandq_u8(cyo, m4q), vshrq_n_u8(cyo, 4)); + uint8x16x2_t ko1=vzipq_u8(vandq_u8(cyo1,m4q), vshrq_n_u8(cyo1,4)); + a0=mm_tile16(a0, vsubq_s8(vreinterpretq_s8_u8(zo.val[0]),b8q), + vsubq_s8(vreinterpretq_s8_u8(zo1.val[0]),b8q), + vld1q_s8(xs+i), vld1q_s8(xs1+i)); + a1=mm_tile16(a1, vsubq_s8(vreinterpretq_s8_u8(zo.val[1]),b8q), + vsubq_s8(vreinterpretq_s8_u8(zo1.val[1]),b8q), + vld1q_s8(xs+i+16), vld1q_s8(xs1+i+16)); + a2=mm_tile16(a2, vsubq_s8(vreinterpretq_s8_u8(ko.val[0]),b8q), + vsubq_s8(vreinterpretq_s8_u8(ko1.val[0]),b8q), + vld1q_s8(xs+i+32), vld1q_s8(xs1+i+32)); + a3=mm_tile16(a3, vsubq_s8(vreinterpretq_s8_u8(ko.val[1]),b8q), + vsubq_s8(vreinterpretq_s8_u8(ko1.val[1]),b8q), + vld1q_s8(xs+i+48), vld1q_s8(xs1+i+48)); + } + for(;i+32<=I;i+=32){ + uint8x16_t byo=vld1q_u8(wo+(i>>1)), byo1=vld1q_u8(wo1+(i>>1)); + uint8x16x2_t zo =vzipq_u8(vandq_u8(byo, m4q), vshrq_n_u8(byo, 4)); + uint8x16x2_t zo1=vzipq_u8(vandq_u8(byo1,m4q), vshrq_n_u8(byo1,4)); + a0=mm_tile16(a0, vsubq_s8(vreinterpretq_s8_u8(zo.val[0]),b8q), + vsubq_s8(vreinterpretq_s8_u8(zo1.val[0]),b8q), + vld1q_s8(xs+i), vld1q_s8(xs1+i)); + a1=mm_tile16(a1, vsubq_s8(vreinterpretq_s8_u8(zo.val[1]),b8q), + vsubq_s8(vreinterpretq_s8_u8(zo1.val[1]),b8q), + vld1q_s8(xs+i+16), vld1q_s8(xs1+i+16)); + } + int32x4_t acc=vaddq_s32(vaddq_s32(a0,a1),vaddq_s32(a2,a3)); + int32_t d00=vgetq_lane_s32(acc,0), d01=vgetq_lane_s32(acc,1); + int32_t d10=vgetq_lane_s32(acc,2), d11=vgetq_lane_s32(acc,3); + for(;i+1>1], bo1=wo1[i>>1]; + int a0=(int)(bo&0xF)-8, a1=(int)(bo>>4)-8, b0=(int)(bo1&0xF)-8, b1=(int)(bo1>>4)-8; + int u0=xs[i],u1=xs[i+1],v0=xs1[i],v1=xs1[i+1]; + d00+=a0*u0+a1*u1; d01+=a0*v0+a1*v1; d10+=b0*u0+b1*u1; d11+=b0*v0+b1*v1; } + if(i>1], bo1=wo1[i>>1]; + int a0=(int)(bo&0xF)-8, b0=(int)(bo1&0xF)-8; + d00+=a0*xs[i]; d01+=a0*xs1[i]; d10+=b0*xs[i]; d11+=b0*xs1[i]; } + y[(int64_t)s*O+o] =(float)d00*sc0*sx[s]; + y[(int64_t)s*O+(o+1)] =(float)d10*sc1*sx[s]; + y[(int64_t)(s+1)*O+o] =(float)d01*sc0*sx[s+1]; + y[(int64_t)(s+1)*O+(o+1)]=(float)d11*sc1*sx[s+1]; + } + if(S&1){ int s=S-1; const int8_t *xs=xq+(int64_t)s*I; + y[(int64_t)s*O+o] =(float)dot_i4i8(wo, xs,I)*sc0*sx[s]; + y[(int64_t)s*O+(o+1)]=(float)dot_i4i8(wo1,xs,I)*sc1*sx[s]; } + } + if(O&1){ int o=O-1; const uint8_t *w=q4+(int64_t)o*rb; float sc=scale[o]; + #pragma omp parallel for schedule(static) + for(int s=0;s=2){ matmul_q_idot_mm(y,xq,sx,q,scale,S,I,O); return; } +#endif + #pragma omp parallel for schedule(static) + for(int o=0;o=2){ matmul_i4_idot_mm(y,xq,sx,q4,scale,S,I,O); return; } +#endif + #pragma omp parallel for schedule(static) + for(int o=0;og_qscratch.xq_cap){ + int8_t *p=realloc(g_qscratch.xq,xn); + if(!p){ fprintf(stderr,"OOM quant scratch\n"); exit(1); } + g_qscratch.xq=p; g_qscratch.xq_cap=xn; + } + if(sn>g_qscratch.sx_cap){ + float *p=realloc(g_qscratch.sx,sn*sizeof(float)); + if(!p){ fprintf(stderr,"OOM quant scales\n"); exit(1); } + g_qscratch.sx=p; g_qscratch.sx_cap=sn; + } + *xq=g_qscratch.xq; *sx=g_qscratch.sx; +} + +/* ---- f32 -> quantized packing --------------------------------------------- */ +static void quantize_rows(const float *w, int8_t *q, float *scale, int O, int I, int bits){ + int qmax=(1<<(bits-1))-1; + #pragma omp parallel for schedule(static) + for(int o=0;oamax)amax=a; } + float s=amax/qmax; if(s<1e-8f)s=1e-8f; scale[o]=s; + int8_t *qr=q+(int64_t)o*I; + for(int i=0;iqmax)v=qmax; if(v<-qmax-1)v=-qmax-1; qr[i]=(int8_t)v; } + } +} +static void pack_int4(const float *w, uint8_t *q4, float *scale, int O, int I, int bits){ + int qmax=(1<<(bits-1))-1, rb=(I+1)/2; + #pragma omp parallel for schedule(static) + for(int o=0;oamax)amax=a; } + float s=amax/qmax; if(s<1e-8f)s=1e-8f; scale[o]=s; + uint8_t *qr=q4+(int64_t)o*rb; + for(int i=0;iqmax)v0=qmax; if(v0<-8)v0=-8; + int v1=0; if(i+1qmax)v1=qmax; if(v1<-8)v1=-8; } + qr[i>>1] = (uint8_t)((v0+8) | ((v1+8)<<4)); + } + } +} +static void pack_int2(const float *w, uint8_t *q2, float *scale, int O, int I, int bits){ + int qmax=(1<<(bits-1))-1, rb=(I+3)/4; + #pragma omp parallel for schedule(static) + for(int o=0;oamax)amax=a; } + float s=amax/qmax; if(s<1e-8f)s=1e-8f; scale[o]=s; + uint8_t *qr=q2+(int64_t)o*rb; + for(int i=0;iqmax)v=qmax; if(v<-2)v=-2; byte|=(uint8_t)((v+2)<<(k*2)); } + qr[i>>2]=byte; + } + } +} + +#endif /* COLI_QUANT_H */ diff --git a/c/sample.h b/c/sample.h new file mode 100644 index 00000000..abce88c4 --- /dev/null +++ b/c/sample.h @@ -0,0 +1,143 @@ +/* sample.h — sampling (temperature + nucleus) and stop-set management. + * Header-only: all functions are static — include from the main engine file. */ +#ifndef SAMPLE_H +#define SAMPLE_H + +#include +#include +#include +#include "tok.h" + +/* ---- RNG (xorshift64*) -------------------------------------------------- */ +static uint64_t g_rng = 0x9E3779B97F4A7C15ULL; +static inline double rndu(void){ + g_rng ^= g_rng << 13; g_rng ^= g_rng >> 7; g_rng ^= g_rng << 17; + return (double)(g_rng >> 11) * (1.0 / 9007199254740992.0); +} + +/* ---- argmax over a float vector ----------------------------------------- */ +static inline int argmax_v(const float *lo, int V){ + int b = 0; float bv = lo[0]; + for (int i = 1; i < V; i++) if (lo[i] > bv) { bv = lo[i]; b = i; } + return b; +} + +/* ---- distribution buffers (reused, single-threaded decode) --------------- */ +static float *g_pbuf = NULL; +static int *g_pidx = NULL; + +/* sift-down on max-heap in h[0..n), key = g_pbuf[h[i]] (#335: partial top-p). + * "hole" variant: carries the root value and deposits only at the end, so + * heapify is O(V) and each pop is O(log n) without qsort on the full vocab. */ +static void topp_siftdown(int *h, int n, int i){ + int iv = h[i]; float kv = g_pbuf[iv]; + for (;;) { + int l = 2*i + 1; + if (l >= n) break; + int b = l; if (l+1 < n && g_pbuf[h[l+1]] > g_pbuf[h[l]]) b = l+1; + if (g_pbuf[h[b]] <= kv) break; + h[i] = h[b]; i = b; + } + h[i] = iv; +} + +/* build the target distribution in g_pbuf: softmax(lo/temp) truncated to + * top-p g_nuc. Invariant: g_pbuf stays indexed by token-id (never reordered); + * the truncated tail is zeroed (dist_sample reads by id directly). + * Requires: g_temp, g_nuc, falloc() — declared in the main engine file. */ +static void dist_build(const float *lo, int V){ + if (!g_pbuf) { g_pbuf = falloc(V); g_pidx = malloc(V * sizeof(int)); } + int mxi = -1; float mx = 0; + for (int i = 0; i < V; i++) + if (isfinite(lo[i]) && (mxi < 0 || lo[i] > mx)) { mx = lo[i]; mxi = i; } + double s = 0; float invt = 1.f / (g_temp > 1e-4f ? g_temp : 1e-4f); + if (mxi >= 0) { + for (int i = 0; i < V; i++) { + g_pbuf[i] = isfinite(lo[i]) ? expf((lo[i] - mx) * invt) : 0.f; + s += g_pbuf[i]; + } + } + if (mxi < 0 || !isfinite(s) || s <= 0.0) { + static int warned = 0; + if (!warned) { warned = 1; fprintf(stderr, + "[SAMPLE] warning: non-finite logits (NaN/Inf) — falling back to argmax; " + "output may be degraded. This usually means a numerical blow-up upstream.\n"); } + int a = (mxi >= 0) ? mxi : 0; + for (int i = 0; i < V; i++) g_pbuf[i] = 0.f; + g_pbuf[a] = 1.f; + return; + } + for (int i = 0; i < V; i++) g_pbuf[i] /= (float)s; + if (g_nuc > 0 && g_nuc < 1.f) { + for (int i = 0; i < V; i++) g_pidx[i] = i; + for (int i = V/2-1; i >= 0; i--) topp_siftdown(g_pidx, V, i); + double s2 = 0, cum = 0; int out = V; + do { + int root = g_pidx[0]; + g_pidx[0] = g_pidx[--out]; g_pidx[out] = root; + s2 += g_pbuf[root]; cum += g_pbuf[root]; + if (out > 0) topp_siftdown(g_pidx, out, 0); + } while (cum < g_nuc && out > 0); + for (int i = 0; i < out; i++) g_pbuf[g_pidx[i]] = 0; + float s2f = (float)s2; + for (int i = out; i < V; i++) g_pbuf[g_pidx[i]] /= s2f; + } +} + +/* sample from g_pbuf; ban>=0 excludes that token (renormalizing on the fly) */ +static int dist_sample(int V, int ban){ + double z = 1.0 - (ban >= 0 ? g_pbuf[ban] : 0.0); + if (z <= 1e-12) z = 1e-12; + double u = rndu() * z, cum = 0; + for (int i = 0; i < V; i++) { if (i == ban) continue; cum += g_pbuf[i]; if (cum >= u) return i; } + for (int i = V-1; i >= 0; i--) if (i != ban && g_pbuf[i] > 0) return i; + return 0; +} + +/* next token from logits: greedy if g_temp<=0, sampling otherwise. + * ban = token excluded because it was rejected by speculative verification. */ +static int pick_tok(const float *lo, int V, int ban){ + if (g_temp <= 0) return argmax_v(lo, V); + dist_build(lo, V); + return dist_sample(V, ban); +} + +/* ---- stop set ----------------------------------------------------------- */ +static int g_stop[64], g_nstop = 0; +static inline int is_stop(int t){ + for (int i = 0; i < g_nstop; i++) if (t == g_stop[i]) return 1; + return 0; +} +/* T=NULL -> config stops only (validation/oracle, where the tokenizer is not needed). */ +static void stops_arm_tok(const Cfg *c, int tok_eos, Tok *T){ + g_nstop = 0; + for (int i = 0; i < c->n_stop && g_nstop < 64; i++) g_stop[g_nstop++] = c->stop_ids[i]; + if (tok_eos >= 0 && !is_stop(tok_eos) && g_nstop < 64) g_stop[g_nstop++] = tok_eos; + int nsp = 0; + if (T) for (int id = 0; id < T->n_ids && g_nstop < 64; id++) + if (T->id_special[id] && !is_stop(id)) { g_stop[g_nstop++] = id; nsp++; } + fprintf(stderr, "[stop] %d stop tokens:", g_nstop); + for (int i = 0; i < g_nstop; i++) fprintf(stderr, " %d", g_stop[i]); + if (nsp) fprintf(stderr, " (%d from the tokenizer's special set)", nsp); + fprintf(stderr, "\n"); +} +static void stops_arm(const Cfg *c, int tok_eos){ stops_arm_tok(c, tok_eos, NULL); } + +/* ---- log-prob of a target token given the logit vector ------------------- */ +static double logprob_target(const float *lo, int V, int target, int *am){ + float mx = lo[0]; int best = 0; + for (int i = 1; i < V; i++) if (lo[i] > mx) { mx = lo[i]; best = i; } + double se = 0; + for (int i = 0; i < V; i++) se += exp((double)lo[i] - mx); + if (am) *am = (best == target); + return (double)(lo[target] - mx) - log(se); +} + +/* "glm" in model_type, case-insensitive */ +static int mt_is_glm(const char *s){ + if (s) for (; *s; s++) + if ((s[0]|32) == 'g' && (s[1]|32) == 'l' && (s[2]|32) == 'm') return 1; + return 0; +} + +#endif /* SAMPLE_H */ diff --git a/c/setup.sh b/c/setup.sh index e31e60ab..dcb851d5 100755 --- a/c/setup.sh +++ b/c/setup.sh @@ -32,11 +32,11 @@ esac # 2) build: nativa (veloce, per QUESTA macchina). Per un binario da distribuire: make portable echo " building (ARCH=${ARCH:-native})…" -make -s glm ARCH="${ARCH:-native}" +make -s colibri ARCH="${ARCH:-native}" # 3) self-test sull'oracolo tiny, se presente if [ -d glm_tiny ] && [ -f ref_glm.json ]; then - r=$(SNAP=./glm_tiny TF=1 ./glm 64 16 16 2>/dev/null | grep -oE "[0-9]+/[0-9]+ positions" || true) + r=$(SNAP=./glm_tiny TF=1 ./colibri 64 16 16 2>/dev/null | grep -oE "[0-9]+/[0-9]+ positions" || true) echo " engine self-test: ${r:-?} (expected 32/32)" fi diff --git a/c/telemetry.h b/c/telemetry.h new file mode 100644 index 00000000..3c3fe855 --- /dev/null +++ b/c/telemetry.h @@ -0,0 +1,189 @@ +/* telemetry.h — dashboard protocol lines, stats/usage persistence, hardware probe. + * Include after Model/Cfg/QT/ESlot/shards and st.h are defined; requires + * qt_bytes(), now_s(), rss_gb(), edisk_s(), and the g_cuda_* globals (ifdef). */ +#ifndef TELEMETRY_H +#define TELEMETRY_H + +static int64_t tbytes(int O,int I,int bits){ + if(bits>=16) return (int64_t)O*I*4; + if(bits>=5) return (int64_t)O*I + (int64_t)O*4; + return (int64_t)O*((I+1)/2) + (int64_t)O*4; +} + +static int64_t expert_bytes_probe(Model *m, int ebits){ + Cfg *c=&m->c; int64_t eb=0; char nm[256]; + snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.0.gate_proj.weight",c->first_dense); + if(st_nbytes(&m->S,nm)>0){ + const char *suf[3]={"gate_proj","up_proj","down_proj"}; + for(int k=0;k<3;k++){ + snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.0.%s.weight",c->first_dense,suf[k]); + eb+=st_nbytes(&m->S,nm); + snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.0.%s.weight.qs",c->first_dense,suf[k]); + int64_t q=st_nbytes(&m->S,nm); if(q>0) eb+=q; + } + } + if(eb<=0) eb = tbytes(c->moe_inter,c->hidden,ebits)*2 + tbytes(c->hidden,c->moe_inter,ebits); + return eb; +} + +/* BRAIN MAP: per-turn expert hit bitmap for the dashboard. */ +static uint8_t **g_ehit; +static void ehit_mark(Model *m, int layer, int eid){ + if(!g_ehit){ Cfg *c=&m->c; + g_ehit=calloc(c->n_layers+1,sizeof(uint8_t*)); + for(int i=0;i<=c->n_layers;i++) g_ehit[i]=calloc(c->n_experts,1); + } + g_ehit[layer][eid]=1; +} + +/* CPU model + cores + RAM (GB); empty/zero where unavailable. */ +static void hw_probe(char *cpu, size_t cn, int *cores, double *ram_total, double *ram_avail){ + cpu[0]=0; +#ifdef _WIN32 +#if defined(__x86_64__) || defined(__i386__) + { unsigned int r[12]={0}; unsigned int *w=r; + for(unsigned int f=0x80000002u; f<=0x80000004u; f++,w+=4) + __get_cpuid(f,&w[0],&w[1],&w[2],&w[3]); + char *b=(char*)r; b[47]=0; while(*b==' ')b++; + snprintf(cpu,cn,"%s",b); } +#endif +#else + FILE *ci=fopen("/proc/cpuinfo","r"); + if(ci){ char ln[256]; + while(fgets(ln,sizeof(ln),ci)) if(!strncmp(ln,"model name",10)){ + char *p=strchr(ln,':'); if(p){ p++; while(*p==' ')p++; + int n=(int)strlen(p); if(n>0&&p[n-1]=='\n')p[--n]=0; + snprintf(cpu,cn,"%s",p); } break; } + fclose(ci); } +#endif + *cores=0; +#ifdef _WIN32 + { SYSTEM_INFO si; GetSystemInfo(&si); *cores=(int)si.dwNumberOfProcessors; } +#elif defined(_SC_NPROCESSORS_ONLN) + *cores=(int)sysconf(_SC_NPROCESSORS_ONLN); +#endif + *ram_total=*ram_avail=0; +#ifdef _WIN32 + compat_meminfo(ram_total,ram_avail); +#else + FILE *mi=fopen("/proc/meminfo","r"); + if(mi){ char ln[256]; double mt=0,ma=0; + while(fgets(ln,sizeof(ln),mi)){ + if(sscanf(ln,"MemTotal: %lf",&mt)==1) *ram_total=mt/1e6; + if(sscanf(ln,"MemAvailable: %lf",&ma)==1) *ram_avail=ma/1e6; + } fclose(mi); } +#endif +} + +static void hwinfo_emit(Model *m){ + Cfg *c=&m->c; (void)c; + char cpu[256]; int cores; double ram_total,ram_avail; + hw_probe(cpu,sizeof(cpu),&cores,&ram_total,&ram_avail); + int ngpu=0; double vram_total=0; + char gpu_name[128]=""; +#ifdef COLI_CUDA + ngpu=g_cuda_ndev; vram_total=m->gpu_expert_bytes/1e9; + for(int i=0;i0) + snprintf(gpu_name,sizeof(gpu_name),"CUDA device x%d",g_cuda_ndev); +#endif + printf("HWINFO %d %.1f %.1f %d %.1f %s|%s\n", + cores,ram_total,ram_avail,ngpu,vram_total,cpu,gpu_name); + fflush(stdout); +} + +static void tiers_emit(Model *m){ + Cfg *c=&m->c; int nsp=0; + for(int i=0;in_layers;i++) if(m->L[i].sparse) nsp++; + int total=(nsp+(m->has_mtp?1:0))*c->n_experts; + int pinned=0,lru=0; + for(int i=0;i<=c->n_layers;i++){ pinned+=m->npin?m->npin[i]:0; lru+=m->ecn?m->ecn[i]:0; } + int vram=0; double vram_gb=0; +#ifdef COLI_CUDA + vram=m->gpu_expert_count; vram_gb=m->gpu_expert_bytes/1e9; +#endif + int ram=pinned-vram+lru; if(ram<0) ram=0; + int disk=total-vram-ram; if(disk<0) disk=0; + double eb=(double)expert_bytes_probe(m,m->ebits); + printf("TIERS %d %d %d %.2f %.2f\n",vram,ram,disk,vram_gb,ram*eb/1e9); + fflush(stdout); +} + +static void emap_emit(Model *m){ + Cfg *c=&m->c; + int rows=0; + for(int i=0;in_layers;i++) if(m->L[i].sparse) rows++; + int has_mtp = m->has_mtp && m->eusage[c->n_layers]; + if(has_mtp) rows++; + int cols=c->n_experts; + char *hex=malloc((size_t)rows*cols*2+1); int w=0; + for(int i=0;i<=c->n_layers;i++){ + int is_row = (in_layers && m->L[i].sparse) || (i==c->n_layers && has_mtp); + if(!is_row) continue; + for(int e=0;epin[i]; + for(int z=0;znpin[i];z++) if(P[z].eid==e){ +#ifdef COLI_CUDA + tier = P[z].g.cuda?2:1; +#else + tier = 1; +#endif + break; } + if(!tier && m->ecache && m->ecache[i]) + for(int z=0;zecn[i];z++) if(m->ecache[i][z].eid==e){ tier=1; break; } + uint32_t u = m->eusage[i]?m->eusage[i][e]:0; + int heat=0; while(u){ heat++; u>>=1; } if(heat>63) heat=63; + int b=(tier<<6)|heat; + hex[w++]="0123456789abcdef"[b>>4]; hex[w++]="0123456789abcdef"[b&15]; + } + } + hex[w]=0; + printf("EMAP %d %d %s\n",rows,cols,hex); fflush(stdout); free(hex); +} + +static void hits_emit(Model *m){ + Cfg *c=&m->c; if(!g_ehit) return; + int rows=0; + for(int i=0;in_layers;i++) if(m->L[i].sparse) rows++; + int has_mtp = m->has_mtp && m->eusage[c->n_layers]; + if(has_mtp) rows++; + int cols=c->n_experts, nb=(rows*cols+7)/8; + uint8_t *bm=calloc(nb,1); int bit=0; + for(int i=0;i<=c->n_layers;i++){ + int is_row = (in_layers && m->L[i].sparse) || (i==c->n_layers && has_mtp); + if(!is_row) continue; + for(int e=0;e>3]|=1<<(bit&7); g_ehit[i][e]=0; } + } + char *hex=malloc((size_t)nb*2+1); int w=0; + for(int b=0;b>4]; hex[w++]="0123456789abcdef"[bm[b]&15]; } + hex[w]=0; + printf("HITS %d %d %s\n",rows,cols,hex); fflush(stdout); free(hex); free(bm); +} + +static void stats_dump_q(Model *m, const char *path, int quiet){ + char tmp[2100]; snprintf(tmp,sizeof(tmp),"%s.tmp",path); + FILE *f=fopen(tmp,"w"); if(!f){ if(!quiet) perror(tmp); return; } + Cfg *c=&m->c; int64_t tot=0, nz=0; + for(int i=0;i<=c->n_layers;i++){ if(!m->eusage[i]) continue; + for(int e=0;en_experts;e++) if(m->eusage[i][e]){ fprintf(f,"%d %d %u\n",i,e,m->eusage[i][e]); tot+=m->eusage[i][e]; nz++; } } + fclose(f); rename(tmp,path); + if(!quiet) fprintf(stderr,"[STATS] %lld selections across %lld distinct experts -> %s\n",(long long)tot,(long long)nz,path); +} +static void stats_dump(Model *m, const char *path){ stats_dump_q(m,path,0); } + +static char g_usage_path[2100]=""; +static int64_t usage_load(Model *m, const char *path){ + FILE *f=fopen(path,"r"); if(!f) return 0; + Cfg *c=&m->c; int l,e; uint32_t cnt; int64_t tot=0; + while(fscanf(f,"%d %d %u",&l,&e,&cnt)==3) + if(l>=0&&l<=c->n_layers&&e>=0&&en_experts&&m->eusage[l]){ m->eusage[l][e]+=cnt; tot+=cnt; } + fclose(f); return tot; +} +static void usage_save(Model *m){ if(g_usage_path[0]) stats_dump_q(m,g_usage_path,1); } + +#endif /* TELEMETRY_H */ diff --git a/c/tests/bench_dsa_select.c b/c/tests/bench_dsa_select.c index a1f20149..676e4c27 100644 --- a/c/tests/bench_dsa_select.c +++ b/c/tests/bench_dsa_select.c @@ -24,7 +24,7 @@ * Run: make tests/bench_dsa_select && ./tests/bench_dsa_select (not in TEST_BINS) */ #define main coli_glm_main_unused -#include "../glm.c" +#include "../colibri.c" #undef main #include diff --git a/c/tests/bench_topp.c b/c/tests/bench_topp.c index 9c0682f7..b2843089 100644 --- a/c/tests/bench_topp.c +++ b/c/tests/bench_topp.c @@ -22,7 +22,7 @@ * Run: make tests/bench_topp && ./tests/bench_topp (not in TEST_BINS -- not a gate) */ #define main coli_glm_main_unused -#include "../glm.c" +#include "../colibri.c" #undef main #include diff --git a/c/tests/test_dsa_select.c b/c/tests/test_dsa_select.c index 99fecb07..ef3fd7bd 100644 --- a/c/tests/test_dsa_select.c +++ b/c/tests/test_dsa_select.c @@ -31,7 +31,7 @@ * In-memory only (no scratch files), so it builds clean on the Windows MinGW CI * job without the unmerged compat shim. */ #define main coli_glm_main_unused -#include "../glm.c" +#include "../colibri.c" #undef main #include diff --git a/c/tests/test_i4_grouped.c b/c/tests/test_i4_grouped.c index 2a9521ea..04e8d887 100644 --- a/c/tests/test_i4_grouped.c +++ b/c/tests/test_i4_grouped.c @@ -18,7 +18,7 @@ * index, a wrong group boundary or a swapped nibble cannot hide under it — * those are O(1) relative errors, not O(1e-6). */ #define main coli_glm_main_unused -#include "../glm.c" +#include "../colibri.c" #undef main static uint32_t rng_state=0xC0FFEEu; diff --git a/c/tests/test_idot.c b/c/tests/test_idot.c index b7109d40..343535c8 100644 --- a/c/tests/test_idot.c +++ b/c/tests/test_idot.c @@ -7,7 +7,7 @@ * (sign-trick kernels must treat |−128| as 128 unsigned, not saturate to 127), * and random data at qrow_i8's contract (|x| <= 127, w full int8 range). */ #define main coli_glm_main_unused -#include "../glm.c" +#include "../colibri.c" #undef main static uint32_t rng_state=0x12345678u; diff --git a/c/tests/test_kv_alloc.c b/c/tests/test_kv_alloc.c index 40ec00dd..d080c08c 100644 --- a/c/tests/test_kv_alloc.c +++ b/c/tests/test_kv_alloc.c @@ -4,7 +4,7 @@ * arrays twice on the second call -> allocator abort. No model file needed: * the CPU path of kv_alloc only reads c->n_layers/kv_lora/qk_rope. */ #define main coli_glm_main_unused -#include "../glm.c" +#include "../colibri.c" #undef main int main(void){ diff --git a/c/tests/test_makefile_platform.py b/c/tests/test_makefile_platform.py index 3c728802..50b4c7af 100644 --- a/c/tests/test_makefile_platform.py +++ b/c/tests/test_makefile_platform.py @@ -35,7 +35,7 @@ def test_windows_nt_without_uname_selects_mingw_build(self): env["PATH"] = "" result = subprocess.run( - [MAKE, "--no-print-directory", "-B", "-n", "glm"], + [MAKE, "--no-print-directory", "-B", "-n", "colibri"], cwd=C_DIR, env=env, text=True, @@ -43,7 +43,7 @@ def test_windows_nt_without_uname_selects_mingw_build(self): check=True, ) - self.assertIn("-o glm.exe", result.stdout) + self.assertIn("-o colibri.exe", result.stdout) self.assertIn("-fopenmp", result.stdout) self.assertIn("-static", result.stdout) diff --git a/c/tests/test_sample_nan.c b/c/tests/test_sample_nan.c index fc332683..4f31def9 100644 --- a/c/tests/test_sample_nan.c +++ b/c/tests/test_sample_nan.c @@ -6,7 +6,7 @@ * iniettato il token scelto e' l'argmax dei FINITI (mai 0 per default), su ogni posizione * del NaN inclusa lo[0]; (c) nessun NaN sopravvive in g_pbuf. */ #define main coli_glm_main_unused -#include "../glm.c" +#include "../colibri.c" #undef main #include diff --git a/c/tests/test_stops.c b/c/tests/test_stops.c index 00f922bf..5e7e9d2a 100644 --- a/c/tests/test_stops.c +++ b/c/tests/test_stops.c @@ -20,7 +20,7 @@ * Defense 2 is what makes this robust against checkpoints we don't control: * even with BOTH configs mutilated, a control token cannot leak into a reply. */ #define main coli_glm_main_unused -#include "../glm.c" +#include "../colibri.c" #undef main static const char *TOKJSON = diff --git a/c/tests/test_topp.c b/c/tests/test_topp.c index 9276507d..49f45d61 100644 --- a/c/tests/test_topp.c +++ b/c/tests/test_topp.c @@ -24,7 +24,7 @@ * No scratch files: the test runs entirely in memory (no mkdtemp), so it builds clean on * the Windows MinGW CI job without the unmerged compat shim (#352). */ #define main coli_glm_main_unused -#include "../glm.c" +#include "../colibri.c" #undef main #include diff --git a/c/tests/test_uring.c b/c/tests/test_uring.c index e64b9f2f..625c79bb 100644 --- a/c/tests/test_uring.c +++ b/c/tests/test_uring.c @@ -6,7 +6,7 @@ #include #include #define main coli_glm_main_unused -#include "../glm.c" +#include "../colibri.c" #undef main static int fail(const char *s){ fprintf(stderr,"FAIL: %s\n",s); return 1; } From 420a0720c31808753d03784de56a5fcd1a51a61d Mon Sep 17 00:00:00 2001 From: ZacharyZcR Date: Sun, 19 Jul 2026 04:02:00 +0800 Subject: [PATCH 037/132] docs: add simplified Chinese and Italian README, update language nav MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New files: README.zh-CN.md — simplified Chinese (大陆用词) README.it.md — Italian (the project's "mother tongue") All four READMEs now link to each other in a consistent nav bar. Updated zh-TW to reflect glm.c → colibri.c rename and new headers. --- README.it.md | 260 ++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 2 +- README.zh-CN.md | 237 +++++++++++++++++++++++++++++++++++++++++++ README.zh-TW.md | 12 ++- 4 files changed, 506 insertions(+), 5 deletions(-) create mode 100644 README.it.md create mode 100644 README.zh-CN.md diff --git a/README.it.md b/README.it.md new file mode 100644 index 00000000..76e3c21c --- /dev/null +++ b/README.it.md @@ -0,0 +1,260 @@ +

+ colibrì — motore piccolo, modello immenso +

+ +

+ English · 简体中文 · 繁體中文 · Italiano +

+ +**Motore piccolo, modello immenso.** Esegui **GLM-5.2 (744 miliardi di parametri, MoE)** su un computer consumer con ~25 GB di RAM — in C puro, zero dipendenze, caricando gli expert dal disco in streaming. + +Colibrì è un runtime MoE leggero e che preserva la qualità: tratta VRAM, RAM e +disco come un'unica gerarchia di memoria gestita. Se la memoria veloce non basta +il modello rallenta, ma la policy predefinita **non cambia mai silenziosamente la +precisione del modello né la semantica del router**. + +``` +$ ./coli chat + 🐦 colibrì v1.0 — GLM-5.2 · 744B MoE · int4 · streaming CPU + ✓ ready in 32s · resident 9.9 GB + › ciao! + ◆ Ciao! 😊 Come posso aiutarti oggi? +``` + +## Guardalo in azione + +

+ dashboard web di colibrì — metriche live, pannello hardware, livelli degli expert +

+

La dashboard web (./coli web): un modello da 744B a 4 tok/s, TTFT 1.6 s, disco 0 — +residenza completa degli expert su 6× RTX 5090, con metriche token in tempo reale, breakdown dei tempi per turno, +la barra dei livelli VRAM/RAM/disco e il mini-cervello live nell'angolo.

+ +

+ la pagina Brain — 19.456 expert come una corteccia vivente +

+

La pagina Brain: tutti i 19.456 expert come una corteccia vivente — il colore indica +il livello di archiviazione, la luminosità il calore di routing, e ogni expert instradato in un turno +lampeggia bianco. Passando il cursore si vede l'affinità +tematica misurata dell'expert.

+ +

+ la pagina Atlas — l'atlante misurato degli expert come una galassia 3D +

+

La pagina Atlas: l'atlante +misurato degli expert come una galassia 3D — 13.260 expert caratterizzati, 1.041 specialisti +replicabili che si raggruppano per argomento (poesia, legge, cinese, SQL…). La posizione deriva +dall'affinità di routing misurata, non da un embedding appreso. Trascinare per ruotare.

+ +## L'idea + +Un modello Mixture-of-Experts da 744B attiva solo ~40B parametri per token — e +solo ~11 GB di quelli cambiano da un token all'altro (gli expert instradati): + +

+ solo ~5.4% dei parametri è attivo per token +

+ +Il modello non ha bisogno di *stare* in memoria veloce — ha bisogno di essere +**piazzato**: + +- la **parte densa** (attenzione, expert condivisi, embedding — ~17B parametri) + resta **residente in RAM a int4** (~9.9 GB); +- i **19.456 expert instradati** (75 layer MoE × 256 + la testa MTP, ~19 MB + ciascuno a int4) stanno **su disco** (~370 GB) e vengono **caricati on demand + in streaming**, con una cache LRU per layer, un hot-store pinnato che impara, + e un livello VRAM opzionale. + +Il motore è un singolo file C (`c/colibri.c`) più header piccoli. Niente BLAS, +niente Python a runtime, niente GPU obbligatoria. + +## Come funziona + +### Il percorso di ogni token + +

+ instrada → unione → piazza → sovrapponi → impara +

+ +Ogni layer di ogni token percorre gli stessi cinque passi. L'obiettivo +progettuale è che **il piazzamento decide solo la velocità** — le decisioni +del router e la precisione dei pesi sono identiche sia che un expert risponda +dalla VRAM sia dal disco. + +### Una gerarchia di memoria, non un requisito di memoria + +

+ residenza expert a tre livelli: VRAM / RAM / NVMe +

+ +Lo stesso motore copre l'intero spettro: su un portatile da 25 GB tutto viene +caricato dal disco in streaming (lento, ma corretto); su un host grande l'intero +set di expert diventa residente (`CUDA_EXPERT_GB=auto PIN_GB=all`) e il disco +esce completamente dal percorso di decode. Tra i livelli c'è una **cache che +impara**: il motore registra quali expert il *tuo* carico di lavoro instrada +(`.coli_usage`, aggiornato a ogni turno) e fissa automaticamente i più caldi — +colibrì diventa letteralmente più veloce man mano che lo usi. Sugli host +multi-socket, `COLI_NUMA=1` interlaccia i pesi residenti tra i controller di +memoria ([#82](https://github.com/JustVugg/colibri/issues/82)). + +### Mai aspettare il disco due volte + +I miss nella cache costano caro, quindi il motore investe la maggior parte +della sua astuzia per evitarli e sovrapporli: le tre matrici di ogni expert sono +memorizzate contigue e lette con un unico `pread`; un pool I/O asincrono +limitato (`PIPE=1`, attivo per default) carica gli expert mancanti mentre quelli +residenti calcolano; le posizioni in batch leggono ogni expert unico una sola +volta (**batch-union**); un thread di lookahead del router (`PILOT=1`) fa il +prefetch degli expert del layer successivo — il routing è misurabilmente +**prevedibile al 71.6% un layer in anticipo**. Sulle GPU, la pipeline residente +(`COLI_CUDA_PIPE=2`) mantiene il flusso residuo on-device tra i layer, così il +loop CPU degli expert procede senza interruzioni; su Apple Silicon un backend +[Metal](docs/metal.md) sperimentale esegue la matmul batch degli expert sulla +GPU a memoria unificata. + +### Modello fedele, stato compresso + +Il forward pass è validato **token-esatto contro un oracle `transformers`** +(teacher-forcing 32/32). L'attenzione MLA memorizza uno stato KV compresso — 576 +float/token invece di 32.768 (**57× più piccolo**) — e lo persiste tra i +riavvii (`.coli_kv`): le conversazioni riaprono "calde", senza alcun re-prefill, +byte-identiche a una sessione ininterrotta. L'attenzione sparsa DSA (il +lightning indexer di GLM-5.2) è implementata fedelmente e validata forzando la +selezione di tutte le chiavi per riprodurre esattamente l'attenzione densa. + +### Decodifica speculativa, onestamente + +La testa MTP nativa di GLM-5.2 propone token che il modello principale verifica +in un unico forward batch — 2.2–2.8 token/forward quando conviene. Due regole +conquistate a caro prezzo sono i default: la testa MTP deve essere **int8** (le +teste int4 crollano al 0–4% di accettazione, +[#8](https://github.com/JustVugg/colibri/issues/8)), e draft e verifica devono +calcolare **la stessa funzione** — `SPEC_PIN=1` fissa entrambi sulla stessa +famiglia di kernel ([#163](https://github.com/JustVugg/colibri/issues/163) +contiene l'intera indagine forense). I draft forzati da grammatica +([`GRAMMAR=file.gbnf`](docs/grammar-draft.md)) aggiungono accettazione quasi +gratuita sull'output JSON vincolato. Se la speculazione conviene dipende dalla +temperatura della cache — misura, e usa `DRAFT=0` quando non paga. + +## Cosa ottiene + +

+ velocità di decode misurata per classe hardware +

+ +Stesso motore, stesso container int4 — cambia solo dove risiedono gli expert. +Punti salienti dalle [tabelle benchmark complete](docs/benchmarks.md): + +- **6× RTX 5090, residenza completa:** 5.8–6.8 tok/s in decode, TTFT ~13 s + ([log dell'esperimento](docs/experiments/glm52-6x5090-2026-07-12.md)); +- **desktop solo-CPU da 128 GB:** ~1.8 tok/s a cache calda + ([#200](https://github.com/JustVugg/colibri/issues/200)); +- **singola RTX 5070 Ti, classe laptop:** 1.07 tok/s tramite la pipeline + GPU-residente ([#273](https://github.com/JustVugg/colibri/issues/273)); +- **macchina di sviluppo da 25 GB:** 0.05–0.1 tok/s a freddo — il punto di + partenza dimostrato da cui è nato il progetto, e ancora oggi la baseline onesta. + +La qualità è misurata, non presunta: il costo di quantizzazione del container +int4 e le ablazioni su granularità delle scale e rotazione sono in +[docs/benchmarks.md](docs/benchmarks.md#quality-benchmark) e +[#108](https://github.com/JustVugg/colibri/issues/108)/[#81](https://github.com/JustVugg/colibri/issues/81). + +## Per iniziare + +### 1. Scarica il modello + +Un container **GLM-5.2 int4** pre-convertito è su Hugging Face — **usa la +versione con le teste MTP int8**: + +**https://huggingface.co/mateogrgic/GLM-5.2-colibri-int4-with-int8-mtp** + +> ⚠️ Il mirror originale contiene teste MTP int4 → accettazione dei draft allo 0% +> ([#8](https://github.com/JustVugg/colibri/issues/8)). Verifica la tua versione: +> `ls -l /out-mtp-*` — int8 (corretto) è `3527131672 / 5366238584 / 1065950496`. + +Oppure converti tu stesso dalla sorgente FP8 — un unico comando riprendibile che +non richiede mai i 756 GB completi su disco contemporaneamente: + +```bash +cd c && ./setup.sh # verifica gcc/OpenMP, compila, autotest +./coli convert --model /nvme/glm52_i4 # scarica e converti shard per shard (python, una tantum) +``` + +### 2. Esegui + +```bash +COLI_MODEL=/nvme/glm52_i4 ./coli chat # budget RAM, cache e MTP rilevati automaticamente +COLI_MODEL=/nvme/glm52_i4 ./coli plan # mostra il piazzamento pianificato VRAM/RAM/disco +COLI_MODEL=/nvme/glm52_i4 ./coli doctor # controllo di idoneità (sola lettura) +./coli web --model /nvme/glm52_i4 # API + dashboard web sulla stessa porta +./coli serve --model /nvme/glm52_i4 # solo API compatibile OpenAI +``` + +Il motore a runtime è puro C — python si usa solo per il convertitore (una tantum) +e per il gateway API opzionale. + +### 3. Approfondisci + +| argomento | documento | +|---|---| +| Benchmark, dati dalla comunità, misurazioni di qualità | [docs/benchmarks.md](docs/benchmarks.md) | +| Parametri di tuning, policy, cache che impara, prefetch | [docs/tuning.md](docs/tuning.md) | +| Build nativa su Windows 11 (con CUDA DLL) | [docs/windows.md](docs/windows.md) | +| Backend CUDA, livello expert in VRAM, residenza completa | [docs/cuda.md](docs/cuda.md) | +| Backend Metal per Apple Silicon | [docs/metal.md](docs/metal.md) | +| API compatibile OpenAI, KV slot, dashboard web | [docs/api.md](docs/api.md) | +| Draft forzati da grammatica (output strutturato) | [docs/grammar-draft.md](docs/grammar-draft.md) | +| Inventario delle variabili d'ambiente | [docs/ENVIRONMENT.md](docs/ENVIRONMENT.md) | + +## Sostenere il progetto + +colibrì è nato come progetto di una sola persona su un portatile con 12 core +e 25 GB di RAM; oggi i suoi numeri arrivano da una comunità di macchine reali. +Se ti è utile: + +- ⭐ metti una stella al repository e condividilo; +- 🐛 apri issue con i numeri di benchmark del tuo hardware — i datapoint + fanno avanzare questo progetto più di qualsiasi altra cosa; +- 💬 contattaci via GitHub issues per sponsorizzare lo sviluppo o donare hardware. + +## Struttura del repository + +``` +Makefile punto d'ingresso root per build/check +c/ +├── colibri.c motore principale +├── quant.h kernel matmul quantizzati (SIMD multi-architettura) +├── sample.h campionamento, RNG, set di stop +├── kv_persist.h persistenza KV su disco (.coli_kv) +├── telemetry.h protocollo dashboard, statistiche, usage +├── st.h, tok.h, json.h header di runtime +├── backend_cuda.* livello CUDA opzionale +├── Makefile build e check locali +├── coli CLI utente +├── openai_server.py gateway HTTP compatibile OpenAI +├── setup.sh setup locale in un solo comando +├── tools/ conversione offline, fixture e benchmark +├── scripts/ helper per conversioni lunghe +└── tests/ test C e Python senza dipendenze +web/ UI browser (puro client API OpenAI) +desktop/ shell desktop Tauri v2 che racchiude la web UI +docs/ documentazione di riferimento, esperimenti, media +``` + +Il percorso a runtime resta intenzionalmente piatto e leggibile: `colibri.c` +più i suoi header. Dalla radice del repository, `make`, `make check` e +`make clean` delegano al Makefile del motore. + +## Perché "colibrì" + +Il colibrì pesa pochi grammi, sta sospeso nel vuoto e visita un migliaio di +fiori al giorno. Questo motore tiene in vita un gigante da 744 miliardi di +parametri con le razioni di un colibrì: 25 GB di RAM, dodici core CPU e +tanta pazienza col disco. + +Il nome è rimasto in italiano perché questa è la lingua in cui è stato scritto +il primo prototipo — i commenti nel codice lo testimoniano ancora. + +## Licenza + +Apache 2.0. I pesi di GLM-5.2 sono rilasciati da Z.ai sotto licenza MIT. diff --git a/README.md b/README.md index f9cf3f00..d6f9bcb7 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@

- English · 繁體中文 + English · 简体中文 · 繁體中文 · Italiano

**Tiny engine, immense model.** Run **GLM-5.2 (744B-parameter MoE)** on a consumer machine with ~25 GB of RAM — in pure C, with zero dependencies, by streaming experts from disk. diff --git a/README.zh-CN.md b/README.zh-CN.md new file mode 100644 index 00000000..352de33c --- /dev/null +++ b/README.zh-CN.md @@ -0,0 +1,237 @@ +

+ colibrì——小巧引擎，庞大模型 +

+ +

+ English · 简体中文 · 繁體中文 · Italiano +

+ +**小巧引擎，庞大模型。**只需约 25 GB 内存，就能在消费级电脑上运行 **GLM-5.2（744B 参数的 MoE）**——以零依赖的纯 C 实现，从磁盘流式加载专家。 + +Colibrì 是一套轻量、保持模型质量的 MoE 运行时，将 VRAM、RAM +与存储设备视为统一管理的内存层级。高速内存不足可能降低速度， +但默认策略**绝不会在未告知的情况下改变模型精度或路由语义**。 + +``` +$ ./coli chat + 🐦 colibrì v1.0 — GLM-5.2 · 744B MoE · int4 · streaming CPU + ✓ ready in 32s · resident 9.9 GB + › ciao! + ◆ Ciao! 😊 Come posso aiutarti oggi? +``` + +## 实际运行效果 + +

+ colibrì 网页仪表盘——实时指标、硬件面板与专家存储层级 +

+

网页仪表盘（./coli web）：744B 模型达到 4 tok/s、TTFT 1.6 秒、磁盘读取 0—— +在 6× RTX 5090 上让所有专家常驻，并实时显示 token 指标、每轮耗时明细、 +VRAM／RAM／磁盘层级条，以及角落的实时迷你大脑。

+ +

+ 大脑页面——以实时皮层呈现 19,456 个专家 +

+

大脑（Brain）页面：将全部 19,456 个专家呈现为活的皮层——颜色代表存储层级， +亮度代表路由热度，每轮被路由到的专家都会闪白。将光标停在专家上，即可查看其 +实测主题亲和度。

+ +

+ 图谱页面——以 3D 星系呈现实测专家图谱 +

+

图谱（Atlas）页面：将实测专家图谱 +呈现为 3D 星系——共 13,260 个已分析专家，其中 1,041 个可复现的专门专家会按主题聚集 +（诗歌、法律、中文、SQL……）。位置取自实测路由亲和度，而非学习出的嵌入向量。拖拽即可旋转。

+ +## 核心概念 + +744B 的专家混合（Mixture-of-Experts）模型，每个 token 只会激活约 40B 参数—— +其中每个 token 之间会变动的只有约 11 GB（被路由到的专家）： + +

+ 每个 token 只会激活约 5.4% 的参数 +

+ +所以模型不必完整**装进**高速内存，而是需要正确**放置**： + +- **稠密部分**（注意力、共享专家、嵌入——约 17B 参数）以 int4 + **常驻 RAM**（约 9.9 GB）； +- **19,456 个路由专家**（75 个 MoE 层 × 256，加上 MTP head；每个在 int4 下约 19 MB） + **存放在磁盘**（约 370 GB），并**按需流式加载**，配合逐层 LRU 缓存、 + 会学习的热门专家固定存储区，以及可选的 VRAM 层级。 + +引擎是一个 C 主文件（`c/colibri.c`）加上若干头文件。不需要 BLAS， +运行时不需要 Python，也不需要 GPU。 + +## 工作原理 + +### 每个 token 的处理路径 + +

+ 路由 → 并集 → 放置 → 重叠执行 → 学习 +

+ +每个 token 的每一层都会经过相同的五个步骤。设计目标是让 +**放置只决定速度**——无论专家是从 VRAM 还是磁盘响应，路由器的决策与权重精度都完全相同。 + +### 统一内存层级，取代单一内存门槛 + +

+ VRAM／RAM／NVMe 三层专家常驻架构 +

+ +同一套引擎覆盖完整硬件范围：在 25 GB 笔记本上，一切都从磁盘流式加载 +（慢，但结果正确）；在大内存主机上，则可让整组专家常驻 +（`CUDA_EXPERT_GB=auto PIN_GB=all`），让磁盘完全退出解码路径。 +两端之间有一层**学习型缓存**：引擎会记录*你的*工作负载路由到哪些专家 +（`.coli_usage`，每轮更新），并自动固定最热门的专家——colibrì 确实会越用越快。 +在多路主机上，`COLI_NUMA=1` 会将常驻权重交错分配到各内存控制器 +（[#82](https://github.com/JustVugg/colibri/issues/82)）。 + +### 绝不为同一次磁盘读取等待两遍 + +缓存未命中的代价很高，因此引擎大部分的巧思都用来避免或重叠这些读取： +每个专家的三个矩阵相邻存储，并以一次 `pread` 读取；有界异步 I/O 池 +（`PIPE=1`，默认启用）会在常驻专家计算时加载缺失的专家；批量位置只读取每个 +不重复专家一次（**批量并集**）；路由前瞻线程（`PILOT=1`）则预取下一层专家—— +实测显示，路由结果提前一层时有 **71.6% 的可预测性**。 +在 GPU 上，常驻管线（`COLI_CUDA_PIPE=2`）让残差流跨层保留在设备端， +使 CPU 专家循环不中断；在 Apple Silicon 上，实验性的 +[Metal 后端](docs/metal.md)会用统一内存 GPU 执行批量专家运算。 + +### 忠实模型，压缩状态 + +前向传播已通过 `transformers` oracle 验证为**逐 token 完全一致** +（teacher-forcing 32/32）。MLA 注意力存储压缩后的 KV 状态——每个 token 为 576 个 +浮点数，而非 32,768 个（**缩小 57×**）——并跨重启持久保存 +（`.coli_kv`）：对话可暖启恢复，不需重新 prefill，结果与不中断的会话 +逐字节相同。DSA 稀疏注意力（GLM-5.2 的 lightning indexer）已忠实实现， +并通过强制选取所有 key，验证可精确复现稠密注意力。 + +### 诚实的推测解码 + +GLM-5.2 原生 MTP head 会起草 token，再由主模型以一次批量前向传播验证—— +条件合适时每次 forward 可产生 2.2–2.8 个 token。两条来之不易的规则已成为默认值： +MTP head 必须是 **int8**（int4 head 的接受率会崩塌到 0–4%，见 +[#8](https://github.com/JustVugg/colibri/issues/8)），且草稿与验证必须计算 +**相同函数**——`SPEC_PIN=1` 会把两者固定在同一 kernel family +（完整取证过程见 [#163](https://github.com/JustVugg/colibri/issues/163)）。 +语法强制草稿（[`GRAMMAR=file.gbnf`](docs/grammar-draft.md)）可在受限 JSON 输出中， +以近乎免费的代价提高接受率。推测解码是否带来净收益取决于缓存热度——请实测， +若不划算就使用 `DRAFT=0`。 + +## 实际成果 + +

+ 各硬件级别的实测解码速度 +

+ +同一套引擎、同一个 int4 容器——硬件只会改变专家的存放位置。 +[完整 benchmark 表格](docs/benchmarks.md)中的重点如下： + +- **6× RTX 5090，全部常驻：**解码 5.8–6.8 tok/s，TTFT 约 13 秒 + （[实验记录](docs/experiments/glm52-6x5090-2026-07-12.md)）； +- **128 GB、仅使用 CPU 的台式机：**热缓存后约 1.8 tok/s + （[#200](https://github.com/JustVugg/colibri/issues/200)）； +- **单张 RTX 5070 Ti 的笔记本级主机：**通过 GPU 常驻管线达到 1.07 tok/s + （[#273](https://github.com/JustVugg/colibri/issues/273)）； +- **25 GB 开发机：**冷启动 0.05–0.1 tok/s——这是项目起步时已证实的下限， + 也仍是诚实的基准。 + +质量来自测量，而非假设：int4 容器的量化损失，以及 scale granularity／rotation +消融实验，收录于 [docs/benchmarks.md](docs/benchmarks.md#quality-benchmark)、 +[#108](https://github.com/JustVugg/colibri/issues/108) 与 +[#81](https://github.com/JustVugg/colibri/issues/81)。 + +## 开始使用 + +### 1. 获取模型 + +Hugging Face 上已有预转换的 **GLM-5.2 int4** 容器——请务必使用 +**含 int8 MTP head 的版本**： + +**https://huggingface.co/mateogrgic/GLM-5.2-colibri-int4-with-int8-mtp** + +> ⚠️ 原始镜像使用 int4 MTP head → 草稿接受率为 0% +>（[#8](https://github.com/JustVugg/colibri/issues/8)）。请检查你的版本： +> `ls -l /out-mtp-*`——正确的 int8 大小为 `3527131672 / 5366238584 / 1065950496`。 + +你也可以自行从 FP8 源转换——只需一条可断点续传的命令，且任何时候都不需要 +在磁盘上同时存放完整的 756 GB： + +```bash +cd c && ./setup.sh # 检查 gcc/OpenMP、构建并运行自测 +./coli convert --model /nvme/glm52_i4 # 逐 shard 下载并转换（仅此一次需要 python） +``` + +### 2. 运行 + +```bash +COLI_MODEL=/nvme/glm52_i4 ./coli chat # 自动检测 RAM 预算、缓存与 MTP +COLI_MODEL=/nvme/glm52_i4 ./coli plan # 查看规划的 VRAM／RAM／磁盘配置 +COLI_MODEL=/nvme/glm52_i4 ./coli doctor # 只读就绪检查 +./coli web --model /nvme/glm52_i4 # 在同一端口提供 API 与网页仪表盘 +./coli serve --model /nvme/glm52_i4 # 仅提供 OpenAI 兼容 API +``` + +引擎运行时是纯 C——python 只供一次性转换工具与可选的 API gateway 使用。 + +### 3. 深入了解 + +| 主题 | 文档 | +|---|---| +| Benchmark、社区实测数据、质量测量 | [docs/benchmarks.md](docs/benchmarks.md) | +| 调优选项、策略、学习型缓存、预取 | [docs/tuning.md](docs/tuning.md) | +| Windows 11 原生构建（含 CUDA DLL） | [docs/windows.md](docs/windows.md) | +| CUDA 后端、VRAM 专家层级、全部常驻 | [docs/cuda.md](docs/cuda.md) | +| Apple Silicon Metal 后端 | [docs/metal.md](docs/metal.md) | +| OpenAI 兼容 API、KV slots、网页仪表盘 | [docs/api.md](docs/api.md) | +| 语法强制草稿（结构化输出） | [docs/grammar-draft.md](docs/grammar-draft.md) | +| 环境变量完整清单 | [docs/ENVIRONMENT.md](docs/ENVIRONMENT.md) | + +## 支持项目 + +colibrì 最初由一人使用 12 核心、25 GB RAM 的笔记本开发； +如今它的数据来自社区中各种真实机器。如果这个项目对你有用： + +- ⭐ 为仓库加星并分享； +- 🐛 以 issue 提交你的硬件 benchmark 数据——实测数据比任何其他事都更能推动项目； +- 💬 若想赞助开发或捐赠硬件，请通过 GitHub issues 联系。 + +## 仓库结构 + +``` +Makefile 根目录构建／检查入口 +c/ +├── colibri.c 引擎主文件 +├── quant.h 量化 matmul 内核（SIMD 多架构） +├── sample.h 采样与 stop-set 管理 +├── kv_persist.h .coli_kv 磁盘持久化 +├── telemetry.h 仪表盘协议、统计与用量持久化 +├── st.h, tok.h, json.h 运行时头文件 +├── backend_cuda.* 可选的 CUDA 层级 +├── Makefile 构建与本地检查 +├── coli 用户界面 CLI +├── openai_server.py OpenAI 兼容 HTTP gateway +├── setup.sh 一条命令完成本地设置 +├── tools/ 离线转换、fixtures 与 benchmarks +├── scripts/ 长时间转换辅助工具 +└── tests/ 零依赖的 C 与 Python 测试 +web/ 浏览器 UI（纯 OpenAI API client） +desktop/ 封装网页 UI 的 Tauri v2 桌面 shell +docs/ 参考文档、实验与媒体文件 +``` + +运行时路径刻意保持扁平、易读：`colibri.c` 加上若干头文件。 +在仓库根目录执行 `make`、`make check` 与 `make clean`， +都会转发给引擎的 Makefile。 + +## 为什么叫"colibrì" + +蜂鸟只有几克重，能在原地悬停，并在一天内造访上千朵花。 +这套引擎只用蜂鸟般的配给，就能让 744B 参数的巨人运转： +25 GB RAM、十二个 CPU 核心，以及对磁盘的大量耐心。 + +## 许可证 + +Apache 2.0。GLM-5.2 权重由 Z.ai 以 MIT 许可发布。 diff --git a/README.zh-TW.md b/README.zh-TW.md index 1662617c..2576eba1 100644 --- a/README.zh-TW.md +++ b/README.zh-TW.md @@ -3,7 +3,7 @@

- English · 繁體中文 + English · 简体中文 · 繁體中文 · Italiano

**小巧引擎，龐大模型。**只要約 25 GB 記憶體，就能在消費級電腦上執行 **GLM-5.2（744B 參數的 MoE）**——以零相依套件的純 C 實作，從硬碟串流載入專家。 @@ -60,7 +60,7 @@ VRAM／RAM／硬碟層級長條，以及角落的即時迷你大腦。

**存放在硬碟**（約 370 GB），並**隨需串流載入**，搭配逐層 LRU 快取、 會學習的熱門專家固定儲存區，以及選用的 VRAM 層級。 -引擎由單一 C 檔（`c/glm.c`）與少量標頭檔組成。不需要 BLAS， +引擎由主 C 檔（`c/colibri.c`）與多個標頭檔模組組成。不需要 BLAS， 執行階段不需要 Python，也不需要 GPU。 ## 運作方式 @@ -203,7 +203,11 @@ colibrì 最初是由一人使用 12 核心、25 GB RAM 的筆電開發； ``` Makefile 根目錄建置／檢查入口 c/ -├── glm.c 單檔 GLM 引擎 +├── colibri.c GLM 引擎主檔 +├── quant.h 量化 matmul kernel +├── sample.h 取樣與 stop-set +├── kv_persist.h .coli_kv 磁碟持久化 +├── telemetry.h 儀表板協定、統計 ├── st.h, tok.h, json.h 執行階段標頭檔 ├── backend_cuda.* 選用的 CUDA 層級 ├── Makefile 建置與本機檢查 @@ -218,7 +222,7 @@ desktop/ 包裝網頁 UI 的 Tauri v2 桌面 shell docs/ 參考文件、實驗與媒體檔 ``` -執行階段路徑刻意維持扁平、易讀：`glm.c` 加上少量標頭檔。 +執行階段路徑刻意維持扁平、易讀：`colibri.c` 加上模組化標頭檔。 在儲存庫根目錄執行 `make`、`make check` 與 `make clean`， 都會轉交給引擎的 Makefile。 From e486574442724ff4ba79808a1ede254368b0638a Mon Sep 17 00:00:00 2001 From: ZacharyZcR Date: Sun, 19 Jul 2026 04:22:46 +0800 Subject: [PATCH 038/132] =?UTF-8?q?web:=20i18n=20support=20=E2=80=94=20Eng?= =?UTF-8?q?lish,=20=E7=AE=80=E4=BD=93=E4=B8=AD=E6=96=87,=20=E7=B9=81?= =?UTF-8?q?=E9=AB=94=E4=B8=AD=E6=96=87,=20Italiano?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lightweight i18n without react-i18next: a LocaleProvider context + useLocale() hook with {{var}} interpolation, auto-detect from navigator.language, persisted to localStorage. 98 translation keys across 4 locale files (en/zh-CN/zh-TW/it). All user-visible strings in App/Brain/Profiling/ErrorBoundary are now t() calls. Language switcher added to the sidebar footer. Build clean (tsc + vite), 18 tests pass. --- web/src/App.tsx | 121 +++++++++++++++++++----------------- web/src/Brain.tsx | 43 +++++++------ web/src/ErrorBoundary.tsx | 21 +++++-- web/src/Profiling.tsx | 58 ++++++++---------- web/src/i18n/en.ts | 125 ++++++++++++++++++++++++++++++++++++++ web/src/i18n/index.ts | 78 ++++++++++++++++++++++++ web/src/i18n/it.ts | 113 ++++++++++++++++++++++++++++++++++ web/src/i18n/zh-CN.ts | 113 ++++++++++++++++++++++++++++++++++ web/src/i18n/zh-TW.ts | 113 ++++++++++++++++++++++++++++++++++ web/src/index.css | 4 +- web/src/main.tsx | 5 +- 11 files changed, 675 insertions(+), 119 deletions(-) create mode 100644 web/src/i18n/en.ts create mode 100644 web/src/i18n/index.ts create mode 100644 web/src/i18n/it.ts create mode 100644 web/src/i18n/zh-CN.ts create mode 100644 web/src/i18n/zh-TW.ts diff --git a/web/src/App.tsx b/web/src/App.tsx index a33cc880..39c9f3b0 100644 --- a/web/src/App.tsx +++ b/web/src/App.tsx @@ -9,6 +9,7 @@ import { Database, Feather, Gauge, + Globe, HardDrive, KeyRound, Layers, @@ -34,6 +35,7 @@ import { Brain } from "./Brain" import { Profiling } from "./Profiling" import { persistPublicSettings, stored } from "@/lib/storage" import { cn } from "@/lib/utils" +import { useLocale } from "./i18n" const message = (role: ChatMessage["role"], content: string): ChatMessage => { let id: string @@ -42,15 +44,12 @@ const message = (role: ChatMessage["role"], content: string): ChatMessage => { } export default function App() { - // When the page is served by the engine itself (coli web), same-origin is the - // right default: no CORS, no manual endpoint editing. The Vite dev server - // (port 5173) keeps the classic default. + const { t, locale, setLocale, locales } = useLocale() + const servedByEngine = typeof window !== "undefined" && window.location.port !== "5173" && window.location.protocol.startsWith("http") const defaultBase = servedByEngine ? `${window.location.origin}/v1` : "http://127.0.0.1:8000/v1" const [baseUrl, setBaseUrl] = useState(() => { const saved = stored(localStorage, "colibri.baseUrl", defaultBase) - // migrate: a stored FACTORY default pointing at another origin would trip CORS - // when the page is engine-served — upgrade it to same-origin once. if (servedByEngine && saved === "http://127.0.0.1:8000/v1" && defaultBase !== saved) return defaultBase return saved }) @@ -120,7 +119,7 @@ export default function App() { const result = await getHealth(baseUrl, apiKey) if (!disposed) { setHealth(result); setHealthError("") } } catch (cause) { - if (!disposed) setHealthError(cause instanceof Error ? cause.message : "Runtime metrics unavailable") + if (!disposed) setHealthError(cause instanceof Error ? cause.message : "status.runtimeUnavailable") } } const timer = window.setInterval(() => void poll(), 5000) @@ -155,13 +154,13 @@ export default function App() { } catch (cause) { if (!controller.signal.aborted) { setHealth(null) - setHealthError(cause instanceof Error ? cause.message : "Runtime metrics unavailable") + setHealthError(cause instanceof Error ? cause.message : "status.runtimeUnavailable") } } } catch (cause) { if (controller.signal.aborted) return setConnected(false) - setError(cause instanceof Error ? cause.message : "Could not reach the server.") + setError(cause instanceof Error ? cause.message : "status.serverError") } finally { if (probeRef.current === controller) { probeRef.current = null; setConnecting(false) } } @@ -227,7 +226,7 @@ export default function App() { if (controller.signal.aborted) { updateMessages((current) => current.filter((item) => item.id !== assistant.id || item.content)) } else { - setError(cause instanceof Error ? cause.message : "Generation failed.") + setError(cause instanceof Error ? cause.message : "status.generationFailed") updateMessages((current) => current.filter((item) => item.id !== assistant.id || item.content)) } } finally { @@ -241,22 +240,22 @@ export default function App() {
-
ACTIVE MODEL{model}
+
{t("topbar.activeModel")}{model}
- - - + + +
- {loading && tokenCount > 0 ? {tokenCount} tokens : null} - {!loading && tokPerSec != null ? {tokPerSec.toFixed(1)} tok/s : null} + {loading && tokenCount > 0 ? {t("topbar.tokens", { n: tokenCount })} : null} + {!loading && tokPerSec != null ? {t("topbar.tokPerSec", { n: tokPerSec.toFixed(1) })} : null} {!loading && ttft != null ? TTFT {(ttft/1000).toFixed(1)}s : null} {!loading && lastRun?.usage ? {lastRun.usage.prompt_tokens}→{lastRun.usage.completion_tokens} : null} {lastRun?.queueWaitMs != null ? queue {Math.round(lastRun.queueWaitMs)}ms : null} - slot {cacheSlot + 1} - + {t("topbar.slot", { n: cacheSlot + 1 })} +
@@ -335,11 +342,11 @@ export default function App() { {!messages.length ? (
- COLIBRÌ ENGINE -

Ask the giant.
Keep the machine yours.

-

Connect to a local colibrì server and stream responses directly from your hardware. Nothing leaves the endpoint you choose.

+ {t("hero.title")} +

{t("hero.subtitle")}
{t("hero.tagline")}

+

{t("hero.description")}

- {["Explain how expert routing works", "Write a small C benchmark", "Compare RAM and VRAM caching"].map((item) => )} + {[t("prompts.routing"), t("prompts.benchmark"), t("prompts.caching")].map((item) => )}
) : ( @@ -347,7 +354,7 @@ export default function App() { {messages.map((item) => (
{item.role === "user" ? "Y" : }
-
{item.role === "user" ? "You" : "colibrì"}
{item.content || }
+
{item.role === "user" ? t("chat.you") : t("chat.colibri")}
{item.content || }
))}
@@ -356,10 +363,10 @@ export default function App() {
- {error &&
{error}
} + {error &&
{t(error)}
}
-