diff --git a/.agents/issues/MODEL-TEXT-kolibri-1-kolibri1-for-causal-lm/ISSUE-LOCAL-01M4R8DVBMQA4WWCF9X6VG00E1.md b/.agents/issues/MODEL-TEXT-kolibri-1-kolibri1-for-causal-lm/ISSUE-LOCAL-01M4R8DVBMQA4WWCF9X6VG00E1.md new file mode 100644 index 000000000..7d5a98687 --- /dev/null +++ b/.agents/issues/MODEL-TEXT-kolibri-1-kolibri1-for-causal-lm/ISSUE-LOCAL-01M4R8DVBMQA4WWCF9X6VG00E1.md @@ -0,0 +1,19 @@ +ID: ISSUE-LOCAL-01M4R8DVBMQA4WWCF9X6VG00E1 +Title: Kolibri1 CPU decode GEMV stage: attribute math vs dispatch/sync overhead at t32 and price the batching restructures +Row: MODEL-TEXT-kolibri-1-kolibri1-for-causal-lm +State: OPEN +Kind: gap +GitHub: - +Mirror: PENDING +Availability: FULL +Created: 2026-10-12 +Updated: 2026-10-12 +Closed: - + +## Problem + +Production kolibri decode at t32+32GiB cache+NEON runs 175 ms/step while the linear GEMV stage aggregates only ~3.1 Gelem/s across 32 threads vs 10.5 Gelem/s single-thread (vbfdotq micro). The step looks serialization-dominated (pool kicks, per-call sync), not arithmetic- or bandwidth-bound. No per-GEMV-site attribution of math vs dispatch exists, and the candidate restructures (batch 6-expert GEMVs, batch q/k/v, eliminate per-call sync) are unpriced. + +## Resolution + +- diff --git a/scripts/aleph-alpha-inference-smoke.py b/scripts/aleph-alpha-inference-smoke.py new file mode 100644 index 000000000..8711dc594 --- /dev/null +++ b/scripts/aleph-alpha-inference-smoke.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +"""Smoke test: can vLLM's CPU backend serve Kolibri-1 via the author's own plugin? + +The question is whether the primary oracle is reachable without a GPU lease. It registers +and imports; this checks that it executes. One short prompt, one token, eager mode, minimal +context - the smallest thing that would still exercise the MoE forward. +""" +import os +import sys +import time + +MODEL = "/mnt/models/Aleph-Alpha/Kolibri-1-bf16-clean" + +os.environ.setdefault("VLLM_LOGGING_LEVEL", "INFO") +os.environ.setdefault("VLLM_PLUGINS", "aleph_alpha_inference") +os.environ.setdefault("TOKENIZERS_PARALLELISM", "false") + +from vllm import LLM, SamplingParams # noqa: E402 + + +def main(): + t0 = time.perf_counter() + llm = LLM( + model=MODEL, + tokenizer=MODEL, + + + max_model_len=64, + max_num_seqs=1, + gpu_memory_utilization=0.74, + tensor_parallel_size=4, + distributed_executor_backend="mp", + enforce_eager=True, + disable_log_stats=True, + trust_remote_code=True, + ) + print(f"[smoke] model loaded in {time.perf_counter()-t0:.1f}s", flush=True) + + prompts = ["The capital of Australia is"] + t1 = time.perf_counter() + out = llm.generate(prompts, SamplingParams(temperature=0.0, max_tokens=8)) + dt = time.perf_counter() - t1 + for o in out: + print(f"[smoke] prompt {o.prompt!r} -> {o.outputs[0].text!r}") + print(f"[smoke] token ids: {list(o.outputs[0].token_ids)}") + print(f"[smoke] 8 tokens in {dt:.1f}s ({8/dt:.3f} tok/s)", flush=True) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/compare-kolibri1-oracles.py b/scripts/compare-kolibri1-oracles.py new file mode 100644 index 000000000..3b041ddb7 --- /dev/null +++ b/scripts/compare-kolibri1-oracles.py @@ -0,0 +1,118 @@ +"""Compare the Kolibri-1 oracles position by position, with the lag auto-detected. + +The primary's rows were written one position ahead (rows[k+1] holds the distribution for +step k). That offset is detected here rather than assumed, so this works on the old file +and on a repaired one without changes. + +Usage: + python3 compare-kolibri1-oracles.py [primary_rows.json] [llamacpp_rows.json] + +Both files are expected in the primary's topk-rows format: prompts[] with +{input_ids, generated_ids, rows:[{gen_index, token, top:[[id,logprob]], logprob_of_token, +top1_top2_margin, top6_top7_margin}]}. +""" + +import json +import sys + +HERE = "tests/vllm/models/" +PRIMARY = sys.argv[1] if len(sys.argv) > 1 else HERE + "kolibri1_primary_topk_rows.json" +LLAMA = sys.argv[2] if len(sys.argv) > 2 else HERE + "kolibri1_llamacpp_topk_rows.json" + + +def load(p): + d = json.load(open(p)) + pr = d["prompts"] + pr = list(pr.values()) if isinstance(pr, dict) else pr + out = [] + for e in pr: + rows = e["rows"] + if isinstance(rows, dict): + rows = {int(k): v for k, v in rows.items()} + else: + rows = {r["gen_index"]: r for r in rows} + out.append((e, rows)) + return out + + +def detect_shift(primary, llama): + """Which offset makes the primary's chain its own rows' argmax?""" + best_rate, best_shift = -1.0, 0 + for shift in (0, 1, -1, 2, -2): + hits = tot = 0 + for (pe, prows), (le, lrows) in zip(primary, llama): + for k, tok in enumerate(pe["generated_ids"]): + rk = k + shift + if rk not in prows or rk not in lrows: + continue + top = prows[rk]["top"] + if not top: + continue + tot += 1 + if top[0][0] == tok: + hits += 1 + rate = hits / tot if tot else 0.0 + print(f" shift {shift:+d}: {hits}/{tot} = {rate*100:.1f}%") + if rate > best_rate: + best_rate, best_shift = rate, shift + return best_shift, best_rate + + +def main(): + primary, llama = load(PRIMARY), load(LLAMA) + shift, rate = detect_shift(primary, llama) + print(f"primary: {PRIMARY}") + print(f"llama : {LLAMA}") + print(f"detected primary row offset: {shift:+d} (chain-matches-own-argmax {rate*100:.1f}%)") + print() + + n = len(primary) + ft = {"p vs l": 0, "p vs t": 0} + seq = {"p vs l": 0, "p vs t": 0} + d_all = [] + margins_dis = [] + for i, ((pe, prows), (le, lrows)) in enumerate(zip(primary, llama)): + pg, lg = pe["generated_ids"], le["generated_ids"] + if pg and lg: + if pg[0] == lg[0]: + ft["p vs l"] += 1 + if pg == lg: + seq["p vs l"] += 1 + cp = 0 + for a, b in zip(pg, lg): + if a != b: + break + cp += 1 + for k in range(cp): + rk, lk = k + shift, k + if rk not in prows or lk not in lrows: + continue + ptop = dict((int(t), float(v)) for t, v in prows[rk]["top"]) + ltop = dict((int(t), float(v)) for t, v in lrows[lk]["top"]) + if not ptop or not ltop: + continue + shared = set(ptop) & set(ltop) + if shared: + d_all.append(max(abs(ptop[s] - ltop[s]) for s in shared)) + if list(ptop)[0] != list(ltop)[0]: + margins_dis.append(prows[rk].get("top1_top2_margin")) + d_all.sort() + print(f"first token agrees: {ft['p vs l']}/{n} identical full sequence: {seq['p vs l']}/{n}") + if d_all: + print( + f"max |dlogprob| on shared tokens, common prefix only (n={len(d_all)}): " + f"median={d_all[len(d_all)//2]:.4f} p90={d_all[int(0.9*len(d_all))]:.4f} max={d_all[-1]:.4f} nats" + ) + if margins_dis: + margins_dis.sort() + print( + f"argmax disagreements on the common prefix: {len(margins_dis)}, " + f"primary top1/top2 margin at those positions: " + f"median={margins_dis[len(margins_dis)//2]:.4f} min={margins_dis[0]:.4f}" + ) + else: + print("argmax disagreements on the common prefix: 0") + + +if __name__ == "__main__": + main() diff --git a/scripts/gen-kolibri1-primary-goldens.py b/scripts/gen-kolibri1-primary-goldens.py new file mode 100644 index 000000000..1a3e323af --- /dev/null +++ b/scripts/gen-kolibri1-primary-goldens.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +"""Generate Kolibri-1 goldens from the PRIMARY oracle. + +The existing goldens come from a pure-torch transcription of the plugin's math. This runs the +plugin itself, through vLLM, at the pinned commit - so the fingerprints are the model author's +own implementation rather than a proxy for it. + +Same shape as tests/vllm/models/kolibri1_goldens.json: greedy, temperature 0, 32 tokens, the +8-prompt set, so the two files can be diffed directly. + +Slow by design: 0.6 tok/s. It runs once, offline. Speed is irrelevant to a fingerprint. +""" +import json +import os +import sys + +MODEL = "/mnt/models/Aleph-Alpha/Kolibri-1-bf16-clean" +OUT = sys.argv[1] if len(sys.argv) > 1 else "/tmp/opencode/kolibri1_goldens_primary.json" + +MAX_NEW_TOKENS = 32 + +PROMPTS = [ + "alpha beta gamma delta epsilon zeta", + "der Mond scheint hell über den Bergen", + "The capital of Australia is", + "Wissen ist Macht, aber Zeit ist Geld", + "x1 = 3, x2 = 7, x3 = 11, x4 =", + "Translation to German: the weather is beautiful today", + "Ein gutes Buch liest man am besten", + "counting: one two three four five six seven", +] + +os.environ.setdefault("VLLM_PLUGINS", "aleph_alpha_inference") +os.environ.setdefault("TOKENIZERS_PARALLELISM", "false") + +from transformers import AutoTokenizer # noqa: E402 +from vllm import LLM, SamplingParams # noqa: E402 + + +def main() -> int: + tok = AutoTokenizer.from_pretrained(MODEL) + llm = LLM( + model=MODEL, + tokenizer=MODEL, + dtype="bfloat16", + max_model_len=512, + max_num_seqs=1, + gpu_memory_utilization=0.74, + tensor_parallel_size=4, + distributed_executor_backend="mp", + enforce_eager=True, + disable_log_stats=True, + trust_remote_code=True, + ) + params = SamplingParams(temperature=0.0, max_tokens=MAX_NEW_TOKENS, logprobs=8) + + entries = [] + for i, prompt in enumerate(PROMPTS): + ids = tok.encode(prompt, add_special_tokens=False) + out = llm.generate([prompt], params, use_tqdm=False)[0] + gen = list(out.outputs[0].token_ids) + # Final-position top-8, as logprobs: vLLM does not surface raw logits. + final = {} + if out.outputs[0].logprobs: + last = out.outputs[0].logprobs[-1] + final = { + "top8_token_ids": [t for t, _ in sorted(last.items(), key=lambda kv: -kv[1].logprob)], + "top8_logprobs": [ + round(float(v.logprob), 4) + for _, v in sorted(last.items(), key=lambda kv: -kv[1].logprob) + ], + } + entries.append({ + "prompt": prompt, + "input_ids": ids, + "generated_ids": gen, + "final_topk": final, + }) + print(f"[{i+1}/{len(PROMPTS)}] {prompt!r}: argmax-first {gen[0] if gen else None}, " + f"{len(gen)} tokens", flush=True) + + old = json.load(open( + "/home/lu_zero/Sources/vllm.cpp/tests/vllm/models/kolibri1_goldens.json")) + agree = sum( + 1 for a, b in zip(entries, old["prompts"]) if a["generated_ids"] == b["generated_ids"] + ) + first = sum(1 for a, b in zip(entries, old["prompts"]) if a["generated_ids"][:1] == b["generated_ids"][:1]) + + out = { + "model": MODEL, + "max_new_tokens": MAX_NEW_TOKENS, + "decode": "greedy, temperature 0", + "provenance": "PRIMARY oracle: aleph-alpha-inference 1.0.0 (049a6a7) via vLLM " + f"{__import__('vllm').__version__} CPU, tensor_parallel_size=4", + "vs_transcription_goldens": { + "identical_full_sequences": f"{agree}/{len(PROMPTS)}", + "identical_first_token": f"{first}/{len(PROMPTS)}", + }, + "prompts": entries, + } + with open(OUT, "w") as f: + json.dump(out, f, indent=1) + f.write("\n") + print(f"\nidentical full sequences vs the transcription goldens: " + f"{agree}/{len(PROMPTS)}") + print(f"identical first token: {first}/{len(PROMPTS)}") + print(f"wrote {OUT}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/vllm/models/kolibri1_goldens_llamacpp.json b/tests/vllm/models/kolibri1_goldens_llamacpp.json new file mode 100644 index 000000000..06dfa61f2 --- /dev/null +++ b/tests/vllm/models/kolibri1_goldens_llamacpp.json @@ -0,0 +1,762 @@ +{ + "model": "/tmp/kolibri1-llamacpp-auto.gguf", + "max_new_tokens": 32, + "decode": "greedy, temperature 0", + "provenance": "llama.cpp commit 62318cd57 (Kolibri-1 support), CPU aarch64, bf16 GGUF", + "prompts": [ + { + "prompt": "alpha beta gamma delta epsilon zeta", + "input_ids": [ + 5249, + 25079, + 74493, + 15774, + 121598, + 116026 + ], + "generated_ids": [ + 121598, + 121598, + 121598, + 121598, + 121598, + 121598, + 121598, + 121598, + 121598, + 121598, + 121598, + 121598, + 121598, + 121598, + 121598, + 121598, + 121598, + 121598, + 121598, + 121598, + 121598, + 121598, + 121598, + 121598, + 121598, + 121598, + 121598, + 121598, + 121598, + 121598, + 121598, + 121598 + ], + "final_topk": { + "top20_token_ids": [ + 121598, + 15930, + 33343, + 95373, + 116026, + 44, + 32, + 115060, + 67223, + 34, + 946, + 26530, + 36209, + 42046, + 68158, + 19193, + 2051, + 120410, + 125787, + 735 + ], + "top20_logprobs": [ + -0.43481317162513733, + -4.132809638977051, + -4.190994739532471, + -4.339730262756348, + -4.584716796875, + -5.4446120262146, + -5.455791473388672, + -5.506629943847656, + -6.03035306930542, + -6.092266082763672, + -6.112296104431152, + -6.240921497344971, + -6.464365482330322, + -6.4721269607543945, + -6.529815196990967, + -6.584495544433594, + -6.593481540679932, + -6.599058628082275, + -6.8685994148254395, + -7.018748760223389 + ] + } + }, + { + "prompt": "der Mond scheint hell \u00fcber den Bergen", + "input_ids": [ + 339, + 39046, + 19373, + 19573, + 1489, + 1178, + 35423 + ], + "generated_ids": [ + 948, + 46252, + 948, + 46252, + 948, + 46252, + 948, + 46252, + 948, + 46252, + 948, + 46252, + 948, + 46252, + 948, + 46252, + 948, + 46252, + 948, + 46252, + 948, + 46252, + 948, + 46252, + 948, + 46252, + 948, + 46252, + 948, + 46252, + 948, + 46252 + ], + "final_topk": { + "top20_token_ids": [ + 46252, + 22211, + 53753, + 43492, + 48716, + 35423, + 948, + 11397, + 420, + 32, + 46644, + 34748, + 42616, + 73620, + 5274, + 34991, + 9979, + 127906, + 7616, + 116371 + ], + "top20_logprobs": [ + -0.011426815763115883, + -7.311507225036621, + -8.104774475097656, + -8.499479293823242, + -8.749040603637695, + -8.85064697265625, + -9.022180557250977, + -9.073286056518555, + -9.147211074829102, + -9.214555740356445, + -9.250295639038086, + -9.30141830444336, + -9.42602825164795, + -9.461069107055664, + -9.483360290527344, + -9.487672805786133, + -9.49565315246582, + -9.532516479492188, + -9.554466247558594, + -9.721601486206055 + ] + } + }, + { + "prompt": "The capital of Australia is", + "input_ids": [ + 325, + 9255, + 279, + 5240, + 735 + ], + "generated_ids": [ + 75361, + 46, + 75361, + 735, + 262, + 9255, + 279, + 5240, + 46, + 75361, + 735, + 7789, + 622, + 262, + 7252, + 13919, + 37112, + 46, + 75361, + 735, + 262, + 6980, + 6547, + 622, + 262, + 7252, + 13919, + 37112, + 46, + 75361, + 735, + 262 + ], + "final_topk": { + "top20_token_ids": [ + 262, + 5407, + 941, + 1410, + 1075, + 502, + 15132, + 7789, + 923, + 8255, + 2602, + 10301, + 622, + 2769, + 23132, + 2403, + 13438, + 5240, + 3145, + 6846 + ], + "top20_logprobs": [ + -0.6133660078048706, + -1.952383041381836, + -1.9628639221191406, + -3.168060302734375, + -3.461732864379883, + -3.7442941665649414, + -3.7671871185302734, + -4.047459602355957, + -4.674698829650879, + -4.7004899978637695, + -6.04345703125, + -6.075115203857422, + -6.186918258666992, + -6.660128593444824, + -6.885526657104492, + -7.091333389282227, + -7.154691696166992, + -7.560610294342041, + -7.5826520919799805, + -8.211400985717773 + ] + } + }, + { + "prompt": "Wissen ist Macht, aber Zeit ist Geld", + "input_ids": [ + 81623, + 2459, + 18680, + 44, + 3504, + 2042, + 2459, + 4771 + ], + "generated_ids": [ + 44, + 936, + 936, + 936, + 936, + 936, + 936, + 936, + 936, + 936, + 936, + 936, + 936, + 936, + 936, + 936, + 936, + 936, + 936, + 936, + 936, + 936, + 936, + 936, + 936, + 936, + 936, + 936, + 936, + 936, + 936, + 936 + ], + "final_topk": { + "top20_token_ids": [ + 936, + 46, + 120022, + 7510, + 275, + 120007, + 59, + 263, + 262, + 286, + 946, + 502, + 120001, + 1064, + 766, + 120042, + 32, + 10, + 314, + 1333 + ], + "top20_logprobs": [ + -0.02787255123257637, + -5.694863319396973, + -6.215535640716553, + -6.4596171379089355, + -6.629917144775391, + -6.718471050262451, + -6.8159613609313965, + -6.976103782653809, + -7.113834857940674, + -7.251016139984131, + -7.339061260223389, + -7.539065837860107, + -7.708276748657227, + -7.783758640289307, + -7.878162860870361, + -7.878484725952148, + -7.9210286140441895, + -8.11556625366211, + -8.155978202819824, + -8.175960540771484 + ] + } + }, + { + "prompt": "x1 = 3, x2 = 7, x3 = 11, x4 =", + "input_ids": [ + 120, + 49, + 576, + 32, + 51, + 44, + 14169, + 50, + 576, + 32, + 55, + 44, + 14169, + 51, + 576, + 32, + 49, + 49, + 44, + 14169, + 52, + 576 + ], + "generated_ids": [ + 32, + 49, + 51, + 44, + 14169, + 53, + 576, + 32, + 49, + 53, + 44, + 14169, + 54, + 576, + 32, + 49, + 55, + 44, + 14169, + 55, + 576, + 32, + 49, + 57, + 44, + 14169, + 56, + 576, + 32, + 50, + 49, + 44 + ], + "final_topk": { + "top20_token_ids": [ + 44, + 263, + 10, + 120007, + 46, + 275, + 120016, + 127906, + 766, + 121119, + 1829, + 14188, + 286, + 58, + 120028, + 988, + 59, + 124969, + 63, + 120468 + ], + "top20_logprobs": [ + -0.003360128728672862, + -6.499362468719482, + -8.14769172668457, + -8.731720924377441, + -8.773417472839355, + -9.256829261779785, + -9.423718452453613, + -9.461344718933105, + -9.593433380126953, + -10.023233413696289, + -10.078298568725586, + -10.141622543334961, + -10.200352668762207, + -10.241419792175293, + -10.29602336883545, + -10.4802885055542, + -10.615851402282715, + -10.75730037689209, + -10.865551948547363, + -11.073074340820312 + ] + } + }, + { + "prompt": "Translation to German: the weather is beautiful today", + "input_ids": [ + 33607, + 318, + 4867, + 58, + 262, + 5995, + 735, + 5445, + 4484 + ], + "generated_ids": [ + 46, + 543, + 5995, + 735, + 33341, + 4484, + 46, + 543, + 5995, + 735, + 5445, + 4484, + 46, + 543, + 5995, + 735, + 33341, + 4484, + 46, + 543, + 5995, + 735, + 5445, + 4484, + 46, + 543, + 5995, + 735, + 33341, + 4484, + 46, + 543 + ], + "final_topk": { + "top20_token_ids": [ + 543, + 1047, + 24775, + 1438, + 120014, + 2297, + 127906, + 262, + 1689, + 1194, + 32, + 1231, + 611, + 14161, + 6693, + 669, + 1914, + 5995, + 325, + 9093 + ], + "top20_logprobs": [ + -0.0018977040890604258, + -7.235360145568848, + -9.038370132446289, + -9.241278648376465, + -9.527000427246094, + -9.661979675292969, + -9.931957244873047, + -9.978264808654785, + -10.347177505493164, + -10.459095001220703, + -10.478646278381348, + -10.537544250488281, + -10.97610092163086, + -11.057907104492188, + -11.071137428283691, + -11.116363525390625, + -11.122068405151367, + -11.232154846191406, + -11.309112548828125, + -11.421590805053711 + ] + } + }, + { + "prompt": "Ein gutes Buch liest man am besten", + "input_ids": [ + 1113, + 29597, + 4363, + 51092, + 836, + 10304, + 9798 + ], + "generated_ids": [ + 30135, + 420, + 30134, + 46, + 1496, + 29597, + 4363, + 51092, + 836, + 10304, + 9798, + 30135, + 420, + 30134, + 46, + 1496, + 29597, + 4363, + 51092, + 836, + 10304, + 9798, + 30135, + 420, + 30134, + 46, + 1496, + 29597, + 4363, + 51092, + 836, + 10304 + ], + "final_topk": { + "top20_token_ids": [ + 10304, + 30135, + 7314, + 120007, + 1410, + 939, + 845, + 1788, + 120022, + 120016, + 1195, + 44, + 913, + 14581, + 46, + 20758, + 1829, + 622, + 1911, + 745 + ], + "top20_logprobs": [ + -0.0034363833256065845, + -6.30530309677124, + -8.102805137634277, + -8.127137184143066, + -9.296419143676758, + -9.469003677368164, + -9.58586597442627, + -9.64742660522461, + -9.791071891784668, + -10.357354164123535, + -10.495367050170898, + -10.53777027130127, + -11.16541576385498, + -11.179350852966309, + -11.448713302612305, + -11.505209922790527, + -11.506875038146973, + -11.515031814575195, + -11.533336639404297, + -11.557818412780762 + ] + } + }, + { + "prompt": "counting: one two three four five six seven", + "input_ids": [ + 586, + 259, + 58, + 2403, + 1311, + 2467, + 2555, + 4241, + 8445, + 12843 + ], + "generated_ids": [ + 12843, + 12843, + 12843, + 12843, + 12843, + 12843, + 12843, + 12843, + 12843, + 12843, + 12843, + 12843, + 12843, + 12843, + 12843, + 12843, + 12843, + 12843, + 12843, + 12843, + 12843, + 12843, + 12843, + 12843, + 12843, + 12843, + 12843, + 12843, + 12843, + 12843, + 12843, + 12843 + ], + "final_topk": { + "top20_token_ids": [ + 12843, + 14031, + 2403, + 55333, + 19827, + 120022, + 120007, + 59, + 94021, + 31663, + 63, + 946, + 55332, + 26209, + 286, + 1311, + 46, + 14395, + 44, + 55 + ], + "top20_logprobs": [ + -0.31337517499923706, + -2.971905469894409, + -3.0842082500457764, + -3.328660726547241, + -4.9849042892456055, + -5.212173938751221, + -5.628828525543213, + -5.80322265625, + -5.806195259094238, + -5.870853900909424, + -6.010465621948242, + -6.0260772705078125, + -6.035622596740723, + -6.151800155639648, + -6.242814540863037, + -6.33974552154541, + -6.428411483764648, + -6.511769771575928, + -6.544284820556641, + -6.716195583343506 + ] + } + } + ] +} \ No newline at end of file diff --git a/tests/vllm/models/kolibri1_goldens_primary.json b/tests/vllm/models/kolibri1_goldens_primary.json new file mode 100644 index 000000000..9dbb5060f --- /dev/null +++ b/tests/vllm/models/kolibri1_goldens_primary.json @@ -0,0 +1,574 @@ +{ + "model": "/mnt/models/Aleph-Alpha/Kolibri-1-bf16-clean", + "max_new_tokens": 32, + "decode": "greedy, temperature 0", + "provenance": "PRIMARY oracle: aleph-alpha-inference 1.0.0 (049a6a7) via vLLM 0.31.0 CPU, tensor_parallel_size=4", + "vs_transcription_goldens": { + "identical_full_sequences": "3/8", + "identical_first_token": "6/8" + }, + "prompts": [ + { + "prompt": "alpha beta gamma delta epsilon zeta", + "input_ids": [ + 5249, + 25079, + 74493, + 15774, + 121598, + 116026 + ], + "generated_ids": [ + 116026, + 116026, + 116026, + 116026, + 116026, + 116026, + 116026, + 116026, + 116026, + 116026, + 116026, + 116026, + 116026, + 116026, + 116026, + 116026, + 116026, + 116026, + 116026, + 116026, + 116026, + 116026, + 116026, + 116026, + 116026, + 116026, + 116026, + 116026, + 116026, + 116026, + 116026, + 116026 + ], + "final_topk": { + "top8_token_ids": [ + 116026, + 34774, + 14153, + 125787, + 127906, + 83150, + 14620, + 34 + ], + "top8_logprobs": [ + -0.1829, + -3.8625, + -5.1458, + -5.8678, + -5.8858, + -5.9515, + -6.1266, + -6.2196 + ] + } + }, + { + "prompt": "der Mond scheint hell \u00fcber den Bergen", + "input_ids": [ + 339, + 39046, + 19373, + 19573, + 1489, + 1178, + 35423 + ], + "generated_ids": [ + 948, + 35423, + 948, + 35423, + 948, + 35423, + 948, + 35423, + 948, + 35423, + 948, + 35423, + 948, + 35423, + 948, + 35423, + 948, + 35423, + 948, + 35423, + 948, + 35423, + 948, + 35423, + 948, + 35423, + 948, + 35423, + 948, + 35423, + 948, + 35423 + ], + "final_topk": { + "top8_token_ids": [ + 35423, + 46252, + 36270, + 66863, + 7616, + 11397, + 48716, + 103140 + ], + "top8_logprobs": [ + -0.0182, + -5.2407, + -5.9327, + -6.8819, + -7.8608, + -8.3039, + -8.8993, + -9.0784 + ] + } + }, + { + "prompt": "The capital of Australia is", + "input_ids": [ + 325, + 9255, + 279, + 5240, + 735 + ], + "generated_ids": [ + 75361, + 46, + 75361, + 735, + 262, + 9255, + 279, + 5240, + 46, + 75361, + 735, + 7789, + 622, + 262, + 7252, + 13919, + 37112, + 46, + 75361, + 735, + 262, + 6980, + 6547, + 622, + 262, + 7252, + 13919, + 37112, + 46, + 75361, + 735, + 262 + ], + "final_topk": { + "top8_token_ids": [ + 262, + 941, + 5407, + 1410, + 1075, + 502, + 15132, + 7789 + ], + "top8_logprobs": [ + -0.4615, + -2.1401, + -2.2167, + -3.0757, + -3.7257, + -3.9741, + -4.1896, + -4.2381 + ] + } + }, + { + "prompt": "Wissen ist Macht, aber Zeit ist Geld", + "input_ids": [ + 81623, + 2459, + 18680, + 44, + 3504, + 2042, + 2459, + 4771 + ], + "generated_ids": [ + 46, + 4246, + 44, + 2042, + 735, + 2973, + 44, + 1409, + 44, + 1584, + 735, + 2973, + 44, + 1409, + 44, + 1584, + 735, + 2973, + 44, + 1409, + 44, + 1584, + 735, + 2973, + 44, + 1409, + 44, + 1584, + 735, + 2973, + 44, + 1409 + ], + "final_topk": { + "top8_token_ids": [ + 1409, + 7510, + 120022, + 1584, + 286, + 936, + 1914, + 120103 + ], + "top8_logprobs": [ + -0.0019, + -7.5842, + -8.7834, + -9.047, + -9.1154, + -9.9716, + -10.0424, + -10.2991 + ] + } + }, + { + "prompt": "x1 = 3, x2 = 7, x3 = 11, x4 =", + "input_ids": [ + 120, + 49, + 576, + 32, + 51, + 44, + 14169, + 50, + 576, + 32, + 55, + 44, + 14169, + 51, + 576, + 32, + 49, + 49, + 44, + 14169, + 52, + 576 + ], + "generated_ids": [ + 32, + 49, + 51, + 44, + 14169, + 53, + 576, + 32, + 49, + 53, + 44, + 14169, + 54, + 576, + 32, + 49, + 55, + 44, + 14169, + 55, + 576, + 32, + 49, + 57, + 44, + 14169, + 56, + 576, + 32, + 50, + 49, + 44 + ], + "final_topk": { + "top8_token_ids": [ + 44, + 263, + 10, + 120016, + 120007, + 46, + 275, + 127906 + ], + "top8_logprobs": [ + -0.0073, + -5.8687, + -7.5191, + -7.8801, + -7.9044, + -8.0513, + -8.1791, + -9.0839 + ] + } + }, + { + "prompt": "Translation to German: the weather is beautiful today", + "input_ids": [ + 33607, + 318, + 4867, + 58, + 262, + 5995, + 735, + 5445, + 4484 + ], + "generated_ids": [ + 46, + 543, + 5995, + 735, + 33341, + 286, + 262, + 21499, + 735, + 5657, + 46, + 543, + 5995, + 735, + 5445, + 286, + 262, + 21499, + 735, + 5657, + 46, + 543, + 5995, + 735, + 5445, + 286, + 262, + 21499, + 735, + 5657, + 46, + 543 + ], + "final_topk": { + "top8_token_ids": [ + 543, + 1047, + 669, + 1689, + 766, + 127906, + 6693, + 1580 + ], + "top8_logprobs": [ + -0.0002, + -10.7284, + -10.9057, + -11.058, + -11.1852, + -11.3034, + -11.3904, + -11.5047 + ] + } + }, + { + "prompt": "Ein gutes Buch liest man am besten", + "input_ids": [ + 1113, + 29597, + 4363, + 51092, + 836, + 10304, + 9798 + ], + "generated_ids": [ + 30135, + 420, + 30134, + 46, + 1496, + 29597, + 4363, + 2459, + 888, + 922, + 20263, + 11794, + 58, + 5084, + 46696, + 7640, + 13111, + 14225, + 1687, + 45455, + 2042, + 46, + 1496, + 29597, + 4363, + 2459, + 888, + 922, + 20263, + 11794, + 58, + 5084 + ], + "final_topk": { + "top8_token_ids": [ + 5084, + 2896, + 1415, + 120022, + 2297, + 3280, + 10694, + 1047 + ], + "top8_logprobs": [ + -0.014, + -5.2181, + -5.5617, + -7.1953, + -7.4167, + -7.7528, + -8.6536, + -8.8337 + ] + } + }, + { + "prompt": "counting: one two three four five six seven", + "input_ids": [ + 586, + 259, + 58, + 2403, + 1311, + 2467, + 2555, + 4241, + 8445, + 12843 + ], + "generated_ids": [ + 12843, + 12843, + 12843, + 12843, + 12843, + 12843, + 12843, + 12843, + 12843, + 12843, + 12843, + 12843, + 12843, + 12843, + 12843, + 12843, + 12843, + 12843, + 12843, + 12843, + 12843, + 12843, + 12843, + 12843, + 12843, + 12843, + 12843, + 12843, + 12843, + 12843, + 12843, + 12843 + ], + "final_topk": { + "top8_token_ids": [ + 12843, + 14031, + 2403, + 55333, + 19827, + 59, + 31663, + 120022 + ], + "top8_logprobs": [ + -0.261, + -3.0047, + -3.4328, + -4.0816, + -4.6841, + -5.7908, + -5.9367, + -5.943 + ] + } + } + ] +}