diff --git a/notes/imatrix-calibration-corpus.json b/notes/imatrix-calibration-corpus.json new file mode 100644 index 0000000..43d0abb --- /dev/null +++ b/notes/imatrix-calibration-corpus.json @@ -0,0 +1,56 @@ +{ + "purpose": "Reproduce the incomplete 299-context importance calibration, not a quality benchmark", + "row_selection": "Zero-based indices in pinned Parquet files, in listed order", + "sample_order": "Interleave code, reasoning, prose by sample index; skip exhausted domains", + "join": "Join samples with two newlines, append one final newline; UTF-8", + "format": { + "prose": "row.text.strip()", + "reasoning": "Question: {row.question.strip()}\nAnswer: {row.answer.strip()}", + "code": "{row.prompt.strip()}\n{row.code.strip()}" + }, + "sources": [ + { + "domain": "prose", + "repo": "Salesforce/wikitext", + "revision": "b08601e04326c79dfdd32d625aee71d232d685c3", + "parquet": "wikitext-2-raw-v1/train-00000-of-00001.parquet", + "license": "cc-by-sa-3.0", + "parquet_sha256": "e83889baabc497075506f91975be5fac0d45c5290b6b20582c8cd1e853d0c9f7", + "ordered_row_indices": [11770,18796,6213,18941,7109,35126,25988,8423,25624,5552,24162,18790,2472,5478,6212,36243,32902,27424,15870,26191,23798,20873,2476,3228,13889,12517,31794,3243,11241,30927,7247,26424,5238,3603,28874,2089,7940,21775,31860,31500,15819,15755,17512,16737,29660,26939,17247,26010,11008,1264,28706,24313,31562,7013,35485,15269,30144,30823,17700,4312,15477,28283,16454,17154,10770,34588,3250,11550,30065,18583,31522,29082,32908,36012,3154,19861,8455,28511,8746,24875,685,30425,23707,24728,2560,1011,18191,21985,28463,21870,31899,7572,17529,34717,34653,6081,28661,14863,34148,4024,29065,5583,3132,16347,35,156,30352,14389,30547,7281,33653,13444,10313,30131,16612,14790,15580,54,29230,36313,31436,32600,25969,6442,20338,21073,21933,8020,4441,10583,23382,24544,24883,12440,3642,38,7444,7368,23542,11665,16195,2648,27931,33700,8535,31490,13311,7635,33642,15280,16941,31339,34700,25287,18521,14586,4171,20382,2278,32191,29926,13757,18856,7815,3220,15381,17010,11232,17862,13193,16647,36394,8457,31915,15785,15728,31809,3230,21936,8533,22432,30401,27115,2786,25445,7867,7661,24752,27472,17606,26559,4584,5472,30622,578,768,26891,21,6457,33566,26758,17733,11631,4632,15894,7603,4252,19271,28829,12265,33730,20356,11619,36397,20971,31773,3146,29081,3955,24253,4678,7069,16742,9457,9331,32496,16607,3932,34034,8781,8462,9496,15841,13918,1832,9742,27771,20013,12358,1860,27176,5260,8108,28016,10617,30031,4217,33666,34023,33292,35664,5500,25117,18309,31884,19857,26885,1233,30117,13585,6741,10386,25565,14338,25878,6185,20519,12952,5619,9915,28707,6270,1890,10965,10177,18798,35798,34296,13922,15327,15303,13263,19875,9562,26086,7821,13447,10848,13754,11177,19599,7932,22284,11999,11457,11239,15910,15895,1336,28915,19141,22618,34330,34968,17598,858,27043,496,16574,28234,30699,25657,16304,1730,35250,22349,22853,36361,1317,22643,4956,35017,26143,7911,4674,17481,27425,15743,28436,30761,21233,22288,13390,4180,18261,27743,34474,36215,32041,6453,21063,34783,17283,20042,34904,9677,29806,614,4085,35068,21596,23649,4977,30271,12180,25566,34994,17517,16769,28329,13895,665,1445,32871,27299,14862,35244,29201,18871,17933,7965,3102,33693,7334,16365,35871,24223,745,33985,25951,30493,35696,26157,17728,19874,23673,31955,7048,32192,8406,31166,16345,30275,5453,16270,17522,10477,27189,22146,32664] + }, + { + "domain": "reasoning", + "repo": "openai/gsm8k", + "revision": "740312add88f781978c0658806c59bc2815b9866", + "parquet": "main/train-00000-of-00001.parquet", + "license": "mit", + "parquet_sha256": "ea82612ea9582142387730c793eb67d3b12849002bc0b7fa6f8efafa7351419d", + "ordered_row_indices": [4530,6017,4120,3208,2585,3335,6924,3843,4126,5579,1425,1257,3569,4974,3835,1218,5809,1875,5419,3250,6466,24,5008,4687,6689,5224,4508,5858,2527,1333,5824,405,5791,3690,689,1293,5913,5531,1204,4661,6130,5949,6389,702,5957,2793,415,1491,7284,35,607,2355,778,5567,7277,905,2520,5081,7097,5632,3830,5145,247,1269,6219,3484,3921,4045,7250,2993,5605,6889,2909,6171,5021,5552,2162,2735,3558,3257,5696,3529,3730,2759,5034,2367,6392,3290,2010,6088,3448,229,3310,4109,7084,6314,1581,4814,3382,6522,4416,2498,145,2543,369,899,456,133,3241,2728,5259,4162,1133,39,224,5892,334,2273,4541,3210,4349,4606,2151,1765,558,943,1863,5391,4031,1794,608,3728,4770,3410,3105,2215,6551,7238,6881,6795,4445,3203,3018,5686,5147,2972,7221,4528,2495,5471,7389,4708,470,5438,1994,6487,6756,4205,2858,5225,1872,1933,400,3371,4689,3512,4586,3933,6727,6535,5739,7056,1317,100,7122,5210,6132,4461,3670,6080,3345,4280,5597,1336,6669,6014,2981,6213,3837,6485,2639,144,1681,1282,1211,4927,3766,1726,5979,5360,6113,4271,3137,3886,4568,2266,2240,3631,2302,158,4003,3299,6418,5963,5856,7289,4978,740,4184,6190,3415,6872,3116,7365,3107,1717,6041,2719,4283,2539,783,1871,1836,5406,62,7388,6726,6731,1886,4488,5363,2439,4361,3708,1036,1283,5609,877,1498,4732,6753,3900,6748,289,4501,4281,5782,1266,6495,6670,4107,2797,2830,6577,194,1925,2219,6527,977,5888,6203,4943,4951,2510,3074,3301,4807,2419,2465,7382,1839,2159,5941,4867,5045,1804,2315,45,3036,3906,5013,3199,7178,1730,4581,6521,7367,1295,1362,4194,3767,4211,6684,3089,5837,4017,1044,303,7298,2007,4212,3902,1760,5027,6961,92,7316,5048,6613,6784,5051,5234,6212,4710,846,5089,7127,6703,2630,1848,4078,2524,7426,5212,4580,7433,4926,5922,6508,1877,5171,6948,3305,4991,2089,6810,2059,1343,2472,5702,1624,6347,2682,2167,4800,6100,210,2753,5649,1332,6744,4183,426,299,4902,5123,5458,4354,2226,1086,930,2594,4962,1502,6545,3408,1735,672,3882,3466,2083,964,7291,1795,6444,5822,528,3170,584,6786,3300,2896,3622,4499,6048,1046,513,2122,6396,5410] + }, + { + "domain": "code", + "repo": "google-research-datasets/mbpp", + "revision": "4bb6404fdc6cacfda99d4ac4205087b89d32030c", + "parquet": "sanitized/train-00000-of-00001.parquet", + "license": "cc-by-4.0", + "parquet_sha256": "d95f8ad6d2fff08fe4826122d6e3e31f75716825d0c5c340d297aca5e9e0de0e", + "ordered_row_indices": [29,116,6,14,119,46,118,58,86,110,47,114,81,83,1,48,21,7,20,95,22,112,115,50,82,54,94,43,34,67,44,2,59,19,16,4,104,73,76,99,56,75,88,26,109,70,13,11,57,69,102,30,28,8,41,93,31,63,78,0,85,92,23,3,15,52,80,111,84,25,37,24,72,35,91,103,66,10,107,98] + }, + { + "domain": "code", + "repo": "google-research-datasets/mbpp", + "revision": "4bb6404fdc6cacfda99d4ac4205087b89d32030c", + "parquet": "sanitized/validation-00000-of-00001.parquet", + "license": "cc-by-4.0", + "parquet_sha256": "27e065fcab3c863959933328a7fdbf404e1bcb5464b1be6fe0dcd9530e420204", + "ordered_row_indices": [36,19,25,31,27,42,8,38,13,18,37,17,1,22,5,7,33,35,11,6,30,23,40,20,28,26,3,15,12,14,24,41,39,34,21,0,16,32,9,29,2,4,10] + } + ], + "code_selection_order": [["validation",36],["train",29],["validation",19],["train",116],["validation",25],["train",6],["train",14],["train",119],["train",46],["train",118],["validation",31],["train",58],["train",86],["train",110],["validation",27],["train",47],["validation",42],["train",114],["train",81],["validation",8],["validation",38],["train",83],["train",1],["train",48],["train",21],["train",7],["train",20],["train",95],["train",22],["train",112],["train",115],["train",50],["validation",13],["validation",18],["train",82],["train",54],["train",94],["validation",37],["validation",17],["train",43],["train",34],["train",67],["train",44],["validation",1],["train",2],["train",59],["train",19],["train",16],["validation",22],["train",4],["validation",5],["validation",7],["validation",33],["train",104],["train",73],["train",76],["train",99],["train",56],["train",75],["train",88],["validation",35],["train",26],["validation",11],["validation",6],["validation",30],["train",109],["validation",23],["train",70],["train",13],["validation",40],["train",11],["train",57],["train",69],["validation",20],["train",102],["train",30],["train",28],["validation",28],["validation",26],["train",8],["validation",3],["train",41],["train",93],["train",31],["validation",15],["validation",12],["train",63],["train",78],["train",0],["validation",14],["train",85],["validation",24],["validation",41],["train",92],["train",23],["train",3],["train",15],["validation",39],["validation",34],["train",52],["train",80],["train",111],["validation",21],["validation",0],["train",84],["train",25],["train",37],["validation",16],["train",24],["validation",32],["train",72],["train",35],["validation",9],["validation",29],["train",91],["validation",2],["train",103],["validation",4],["validation",10],["train",66],["train",10],["train",107],["train",98]], + "samples_by_domain": {"code":123,"reasoning":400,"prose":400}, + "normalized_whole_sample_overlap_with_allocation_or_heldout": 0, + "mbpp_task_id_overlap_with_allocation_or_heldout": 0, + "near_duplicates_and_pretraining_overlap": "not assessed", + "output_bytes": 558041, + "output_sha256": "bd99ffc583bc6dc97f7557a9c608696760fedb58d86a9b5f7f7e8895e1380a70" +} diff --git a/notes/imatrix-probe-corpus.json b/notes/imatrix-probe-corpus.json new file mode 100644 index 0000000..1ae9354 --- /dev/null +++ b/notes/imatrix-probe-corpus.json @@ -0,0 +1,49 @@ +{ + "seed": 0, + "purpose": "Allocation-only inputs for the small HF/BF16 GGUF reference comparison", + "row_selection": "Zero-based indices in the pinned Parquet files, in listed order", + "join": "Strip each sample, join with two newlines, append one final newline; UTF-8", + "format": { + "prose": "row.text.strip()", + "reasoning": "Question: {row.question.strip()}\nAnswer: {row.answer.strip()}", + "code": "{row.prompt.strip()}\n{row.code.strip()}" + }, + "sources": [ + { + "domain": "prose", + "repo": "Salesforce/wikitext", + "revision": "b08601e04326c79dfdd32d625aee71d232d685c3", + "parquet": "wikitext-2-raw-v1/train-00000-of-00001.parquet", + "license": "cc-by-sa-3.0", + "parquet_sha256": "e83889baabc497075506f91975be5fac0d45c5290b6b20582c8cd1e853d0c9f7", + "ordered_row_indices": [33120,14172,21886,5409,13253,35844,8724,18006,16762,26694,25491,14928,6274,1051,20286,2709,20814,28338,13128,5961,19914,33245,35306,18172,30500,28201,17499,34192,26084,13791,18469,25957,33278,30439,22081,33838,31681,16527,16087,17549], + "output": "allocation-prose.txt", + "output_sha256": "b26afc33d74d5e0270343431d0c0e15a9f46a7c757beb36baac6e2bc37cefef4", + "output_bytes": 27552 + }, + { + "domain": "reasoning", + "repo": "openai/gsm8k", + "revision": "740312add88f781978c0658806c59bc2815b9866", + "parquet": "main/train-00000-of-00001.parquet", + "license": "mit", + "parquet_sha256": "ea82612ea9582142387730c793eb67d3b12849002bc0b7fa6f8efafa7351419d", + "ordered_row_indices": [1197,5330,5583,3722,5974,7399,6472,7000,494,4131,1441,3845,4651,3587,2118,2029,2923,5440,2447,6179,2831,6705,2006,6220,4656,2711,6355,5378,6845,1194,7360,263,1936,4510,5703,6373,7421,485,6005,841], + "output": "allocation-reasoning.txt", + "output_sha256": "25312cfe7310e069c5c1c57dde559c8f72cd2066aa9e76e051ac25171af4b9a4", + "output_bytes": 19364 + }, + { + "domain": "code", + "repo": "google-research-datasets/mbpp", + "revision": "4bb6404fdc6cacfda99d4ac4205087b89d32030c", + "parquet": "sanitized/train-00000-of-00001.parquet", + "license": "cc-by-4.0", + "parquet_sha256": "d95f8ad6d2fff08fe4826122d6e3e31f75716825d0c5c340d297aca5e9e0de0e", + "ordered_row_indices": [40,55,105,89,71,60,42,87,9,117,39,18,77,90,68,32,79,12,96,101,36,17,64,27,74,45,61,38,106,100,51,62,65,33,5,53,113,97,49,108], + "output": "allocation-code.txt", + "output_sha256": "eb645899d0a05c1517a655e5cd5daf091a09b101888c6d6f260b77d1f1dc235e", + "output_bytes": 10792 + } + ] +} diff --git a/notes/imatrix-probe-gb10.md b/notes/imatrix-probe-gb10.md new file mode 100644 index 0000000..31f4f44 --- /dev/null +++ b/notes/imatrix-probe-gb10.md @@ -0,0 +1,110 @@ +# BF16 and merged MoE probe checks on GB10 + +The imatrix probe needs decoded weights, not the bytes used to store BF16. +Merged expert tensors also need separate importance values for each expert. +The tests in `tests/test_probe_gguf_weights.py` cover both with real GGUF files +and costs calculated directly from small matrices. + +```sh +python -m pytest -q tests/test_probe_gguf_weights.py +``` + +All 22 cases passed on macOS ARM64 and in a Linux ARM64 environment using +Torch 2.13.0+cu130 and the pinned quantizer's GGUF reader. GPU access is not +needed for these tests. + +## Original-model check + +Source: [Qwen/Qwen3-30B-A3B](https://huggingface.co/Qwen/Qwen3-30B-A3B), revision +`ad44e777bcd18fa416d9da3bd8f70d33ebb85d39` (Apache-2.0). The original BF16 +safetensors passed content verification against the pinned Hub file hashes. + +Converter and native runtime: +[ik_llama.cpp](https://github.com/ikawrakow/ik_llama.cpp), commit +`5f89bfc81268b4d56d2af63ccbed59de17c64c09`. Conversion used `--outtype bf16`. +The resulting GGUF was 61,095,802,816 bytes, SHA256 +`1b2114d05447fa0c99ac9a4cce02c6e88b38466746d7b8356b746f0e2729baa9`. + +The probe's decoder reconstructed 26 sampled matrices exactly against the +original safetensors: all four attention projections in layers 0 and 47, plus +gate/up/down projections for experts 0, 17 and 127 in those layers. +That is 66,060,288 values. This checks those matrices, not every model weight. + +A separate prediction check on one GB10 compared the original HF BF16 model +with the native BF16 GGUF. HF used Transformers 5.17.0, eager attention and +Torch 2.13.0+cu130. Native used FlashAttention and F16 KV. The same token IDs +were checked before comparing predictions: two 256-token contexts per domain, +scoring 254 next-token predictions per domain after excluding the first half +of each context and its final token. +The exact input rows, public dataset revisions, formatting and file hashes +are listed in [imatrix-probe-corpus.json](imatrix-probe-corpus.json). No raw +dataset text or model outputs are included in that manifest. + +| Domain | Top-1 agreement | Approximate HF-to-GGUF KL, nats | Signed mean NLL difference, nats | +| --- | ---: | ---: | ---: | +| Code | 98.43% | 0.004875 | -0.004136 | +| Reasoning | 98.82% | 0.004340 | 0.019706 | +| Prose | 96.85% | 0.003958 | -0.010874 | + +The acceptance limits were fixed before the comparison: at least 90% top-1 +agreement, approximate KL at most 0.03 nats and absolute signed mean NLL +difference at most 0.1 nats, separately for each domain. All passed. Native +saved probabilities use 16-bit storage and a 24-logit floor; they were +renormalized for approximate KL. No scored target probability was clipped. +This is a small reference check, not bit-exact inference equivalence or a +held-out model-quality benchmark. + +## Router accounting and incomplete calibration + +The native collector produced entries for `ffn_gate_inp.weight` in all 48 +layers. These are MoE routers, not expert gate projections. The native +quantizer keeps them at their source type unless its separate router-type +option is used. The probe records them under `excluded_tensors` and does not +score them as expert FFNs. If a recipe quantizes routers separately, it needs +a separate measurement. Unknown tensor families still stop the probe. + +Importance collection used allocation-disjoint public calibration text. The +first 64 x 512-token run skipped eight entries and filled in missing expert +statistics in other entries. Processing the whole file (129 complete contexts) +removed the skipped entries, but still produced partial-expert warnings. +Expanding to 923 unique samples (123 code, 400 reasoning, 400 prose) processed +299 complete contexts, or 153,088 tokens. All 337 collected entries had valid +dimensions and finite nonnegative values, but 28 up/down expert tensors still +had partial-data warnings, down from 36 in the 129-context run. The expanded +run took 584 seconds. Valid stored values do not prove observed coverage: +the native collector substituted statistics for unseen experts. + +The exact expanded input is reconstructible from +[imatrix-calibration-corpus.json](imatrix-calibration-corpus.json). It uses +public training data and MBPP validation examples, excluding our frozen +allocation-selection and held-out samples. Normalized whole-sample overlap +and MBPP task-ID overlap were zero; near-duplicates and pretraining overlap +were not assessed. Held-out evaluation was not run or used to tune this corpus. + +None of these matrices establishes complete observed expert coverage. Missing gate +entries from a fused gate/up operation need explicit accounting too. + +Inspect collector warnings, not just the presence of a matrix file. This +pinned collector can substitute values of one for a few unseen experts. Its +periodic-save path also modifies those accumulated statistics. These runs +used `--output-frequency 1000000 --save-frequency 0` to defer saving until +collection finished. The expanded matrix is diagnostic evidence, not a +completed allocation-versus-uniform comparison. Its SHA256 is +`01f0526299538bd11992576f67b6023fd7f8a653982f643b2b5111ef26cc976e`. + +The score remains a Hessian-weighted RTN proxy. Expert-specific activation +averages in the saved matrix do not themselves record expert-selection +frequency. These checks do not demonstrate measured quantizer KL, a quality +gain over uniform quantization, throughput improvement or multi-node scaling. + +## Native build + +The unmodified source was built with CMake 3.31.6, GCC 13.3.0 and CUDA 13.0.88, +targeting `121-real`. GCC's native ARM detection omitted supported DOTPROD and +FP16 features on this machine. The successful build used `GGML_NATIVE=OFF` +and `GGML_ARCH_FLAGS=-march=armv8.2-a+dotprod+fp16`. All five native tools +passed driver-enabled library and help checks. This validates this build, +not the quantizer's generic ARM fallback. + +Raw corpus text, predictions and machine-specific logs are not included here. +Prepared with AI assistance and tested locally. diff --git a/tests/test_probe_gguf_weights.py b/tests/test_probe_gguf_weights.py new file mode 100644 index 0000000..6fd3aad --- /dev/null +++ b/tests/test_probe_gguf_weights.py @@ -0,0 +1,169 @@ +"""Exercise stored BF16 bytes and expert-specific importance using real GGUF files.""" +import math +from pathlib import Path +import struct +import sys + +import numpy as np +import pytest + +torch = pytest.importorskip("torch") +gguf = pytest.importorskip("gguf") +sys.path.insert(0, str(Path(__file__).parents[1] / "tools")) +import pollard_probe as P + + +def write_tensor(path, name, weights, qtype): + from gguf.quants import quantize + writer = gguf.GGUFWriter(path, "qwen3moe") + writer.add_tensor(name, quantize(weights, qtype), raw_dtype=qtype) + writer.write_header_to_file() + writer.write_kv_data_to_file() + writer.write_tensors_to_file() + writer.close() + return gguf.GGUFReader(path).tensors[0] + + +def write_imatrix(path, name, importance): + name_bytes = name.encode() + values = np.asarray(importance, dtype=np.float32).ravel() * 2 + path.write_bytes(struct.pack(" 0 + + +def test_dense_probe_uses_decoded_bf16_weights(tmp_path): + name = "blk.0.attn_q.weight" + weights = np.array([[1, .5, .25, -1], [.5, -.25, 1, 2]], dtype=np.float32) + importance = np.array([1, 2, 3, 4], dtype=np.float32) + model, matrix = tmp_path / "model.gguf", tmp_path / "imatrix.dat" + write_tensor(model, name, weights, gguf.GGMLQuantizationType.BF16) + write_imatrix(matrix, name, importance) + costs, _, _ = P._imatrix_sensitivity(str(model), str(matrix), ["attn"], 2, []) + expected = float(((torch.tensor(weights) - P._rtn(torch.tensor(weights), 2)) ** 2 * + torch.tensor(importance)).sum()) + assert costs["attn"]["0"] == pytest.approx(expected) + + +@pytest.mark.parametrize("importance", [[1, 2], [float("nan")] * 8, [-1] * 8]) +def test_bad_expert_importance_refuses_partial_profile(tmp_path, importance): + name = "blk.0.ffn_up_exps.weight" + model, matrix = tmp_path / "model.gguf", tmp_path / "imatrix.dat" + write_tensor(model, name, np.ones((2, 2, 4), dtype=np.float32), gguf.GGMLQuantizationType.F32) + write_imatrix(matrix, name, importance) + with pytest.raises(SystemExit, match="partial"): + P._imatrix_sensitivity(str(model), str(matrix), ["ffn"], 2, []) + + +@pytest.mark.parametrize("shape,importance,merged", [ + ((2, 2, 4), [1] * 4, True), ((2, 2, 4), [1] * 8, False), + ((1, 2, 2, 4), [1] * 4, True), ((2, 0), [], False), + ((2, 4), [[1] * 4], False), +]) +def test_rejects_ambiguous_or_unsupported_importance_layout(shape, importance, merged): + with pytest.raises(ValueError): + P._imatrix_weights(torch.ones(shape), importance, merged) + + +def test_quantized_expert_tensor_retains_logical_dimensions(tmp_path): + from gguf.quants import dequantize, quantize + qtype = gguf.GGMLQuantizationType.Q8_0 + weights = np.linspace(-2, 2, 128, dtype=np.float32).reshape(2, 2, 32) + tensor = write_tensor(tmp_path / "model.gguf", "blk.0.ffn_up_exps.weight", weights, qtype) + decoded = P._dequant(tensor) + assert decoded.shape == weights.shape + np.testing.assert_array_equal(decoded, dequantize(quantize(weights, qtype), qtype)) + + +def test_nonfinite_weights_are_rejected(): + with pytest.raises(ValueError, match="finite"): + P._imatrix_weights(torch.tensor([[float("inf")]]), [1]) + + +def test_cost_overflow_does_not_emit_infinite_profile(tmp_path): + name = "blk.0.attn_q.weight" + model, matrix = tmp_path / "model.gguf", tmp_path / "imatrix.dat" + weights = np.array([[1e20, 0.4e20, -1e20, 0.1e20]], dtype=np.float32) + write_tensor(model, name, weights, gguf.GGMLQuantizationType.F32) + write_imatrix(matrix, name, [1] * 4) + with pytest.raises(SystemExit, match="non-finite probe cost"): + P._imatrix_sensitivity(str(model), str(matrix), ["attn"], 2, []) diff --git a/tools/pollard_probe.py b/tools/pollard_probe.py index 671e1ca..b20870a 100644 --- a/tools/pollard_probe.py +++ b/tools/pollard_probe.py @@ -30,7 +30,7 @@ much of the protect set (the most sensitive quarter) survives from one slice to the next. STABLE means the ranking is the model's; UNSTABLE means it is the corpus's, and the card must say so. """ -import argparse, glob, json, os, re, sys +import argparse, glob, json, math, os, re, sys # `--device cpu` has to MEAN cpu, and this has to happen BEFORE torch is imported. # device_map="auto" enumerates every visible device, so accelerate places layers on the GPU even @@ -319,8 +319,15 @@ def _imatrix_sensitivity(gguf_path, imatrix_path, groups, probe_bits, ladder_bit nscored = {} # (group, layer) -> tensors scored pinned = {} # layer -> uncovered tensor names seen_covered = set() # imatrix tensors we actually scored + excluded = {} # known tensors not allocated by this profile for t in rd.tensors: + if re.fullmatch(r"blk\.\d+\.ffn_gate_inp\.weight", t.name): + # llama-quantize keeps MoE routers unchanged unless its separate + # router-type option is supplied. They are not expert FFN weights. + excluded[t.name] = {"source_type": t.tensor_type.name, + "reason": "MoE router; retain source precision"} + continue slot = _gguf_slot(t.name, groups) if slot is None: continue @@ -336,16 +343,16 @@ def _imatrix_sensitivity(gguf_path, imatrix_path, groups, probe_bits, ladder_bit continue try: W = torch.from_numpy(np.asarray(_dequant(t))).float() + h = _imatrix_weights(W, hj, t.name.endswith("_exps.weight")) except Exception as e: skipped.append(f"{t.name} ({e})") continue - if W.ndim != 2 or W.shape[1] != len(hj): - skipped.append(f"{t.name} (shape {tuple(W.shape)} vs imatrix {len(hj)})") - continue - h = torch.tensor(hj, dtype=torch.float32).unsqueeze(0) + rows = W.reshape(-1, W.shape[-1]) for b in bitset: # ONE read, every bit-width off it - dW = W - _rtn(W, b).float() + dW = W - _rtn(rows, b).reshape(W.shape).float() c = float((dW * dW * h).sum()) + if not math.isfinite(c): + raise SystemExit(f"non-finite probe cost for {t.name}; refusing to emit a profile") if b == probe_bits: cost[g][str(i)] = cost[g].get(str(i), 0.0) + c for ty, tb in ladder_bits: @@ -356,12 +363,12 @@ def _imatrix_sensitivity(gguf_path, imatrix_path, groups, probe_bits, ladder_bit seen_covered.add(t.name) del W - if not scored: - raise SystemExit("no scored tensors -- the imatrix and the GGUF do not share tensor names.") if skipped: # A partial profile that LOOKS measured is the failure mode this whole tool guards against. raise SystemExit(f"\n {len(skipped)} tensors could not be scored, so the profile would be " f"partial.\n REFUSING to emit it. first: " + ", ".join(skipped[:3])) + if not scored: + raise SystemExit("no scored tensors -- the imatrix and the GGUF do not share tensor names.") # THE INVARIANT: the imatrix is the ground truth for what actually gets quantized -- it holds an # entry for every matmul llama-quantize will touch. So a covered tensor we did not score means @@ -370,7 +377,8 @@ def _imatrix_sensitivity(gguf_path, imatrix_path, groups, probe_bits, ladder_bit # tell the allocator. Qwen3.8-27B is how this was found: 48 of 65 blocks mix with an SSM rather # than attention, 240 of 496 covered matmuls went unscored, and the profile looked fine. missed = sorted({re.sub(r"^blk\.\d+\.", "", n) for n in H - if re.match(r"^blk\.\d+\..+\.weight$", n) and n not in seen_covered}) + if re.match(r"^blk\.\d+\..+\.weight$", n) + and n not in seen_covered and n not in excluded}) if missed: # An unfamiliar architecture is an ONBOARDING case, not a dead end. Pollard's convention is # to forward it so the next person's model of that family one-shots -- every onboarding @@ -394,6 +402,10 @@ def _imatrix_sensitivity(gguf_path, imatrix_path, groups, probe_bits, ladder_bit raise SystemExit("every layer scored zero tensors -- nothing to allocate on.") print(f" scored {scored} tensors across {layers} layers", flush=True) + if excluded: + cost["excluded_tensors"] = excluded + print(f" {len(excluded)} MoE routers excluded from the score; retain their source " + "precision. Quantizing routers separately requires a separate measurement.", flush=True) if pinned: ex = sorted(pinned)[:4] print(f" {sum(len(v) for v in pinned.values())} tensors in {len(pinned)} layer(s) are NOT " @@ -402,11 +414,27 @@ def _imatrix_sensitivity(gguf_path, imatrix_path, groups, probe_bits, ladder_bit return cost, noise, layers +def _imatrix_weights(weights, importance, merged_experts=False): + """Align dense or expert-specific input-channel importance with weight rows.""" + h = torch.tensor(importance, dtype=torch.float32) + if not torch.isfinite(weights).all() or not torch.isfinite(h).all() or (h < 0).any(): + raise ValueError("weights and nonnegative importance must be finite") + if h.ndim != 1 or any(size == 0 for size in weights.shape): + raise ValueError("empty weights or non-vector importance") + if weights.ndim == 2 and weights.shape[-1] == h.numel(): + return h.unsqueeze(0) + if weights.ndim == 3 and merged_experts: + experts, _, channels = weights.shape + if experts * channels == h.numel(): + return h.reshape(experts, 1, channels) + raise ValueError(f"weight shape {tuple(weights.shape)} vs imatrix {h.numel()}") + + def _dequant(t): - """Tensor data as a 2-D float array, whatever the GGUF stored it as.""" + """Decode stored GGUF values, preserving logical tensor dimensions.""" import numpy as np from gguf import GGUFReader # noqa: F401 (import-time type table) - if t.tensor_type.name in ("F32", "F16", "BF16"): + if t.tensor_type.name in ("F32", "F16"): return t.data.astype(np.float32) from gguf.quants import dequantize # lets a Q6_K host serve as the source return dequantize(t.data, t.tensor_type).astype(np.float32)