Skip to content

Commit 68eeae7

Browse files
fluffy314cursoragent
authored and
fluffy314
committed
Mac M4 K3 in-domain spec decode evidence
In-domain cross-runtime eval passes with lossless output: acceptance_rate=0.078, acceptance_length=2.10, total_blocks=92. This preserves the Mac M4 evidence for the DFlash drafter plus MLX verifier path. Co-authored-by: Cursor <cursoragent@cursor.com>
1 parent 0c7e7f0 commit 68eeae7

2 files changed

Lines changed: 195 additions & 0 deletions

File tree

‎results/research/k3_dflash_specdecode_mac_1781024646.json‎

Lines changed: 183 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,183 @@
1+
{
2+
"schema_version": 1,
3+
"kind": "k3_dflash_specdecode_acceptance_mac",
4+
"config": {
5+
"verifier_path": "models/gemma-4-26B-A4B-it-mlx-4bit",
6+
"drafter_id": "models/dflash-kakeya-baseline",
7+
"drafter_device": "mps",
8+
"drafter_dtype": "torch.bfloat16",
9+
"block_size": 16,
10+
"num_steps": 1,
11+
"max_new_tokens": 48,
12+
"n_prompts": 4,
13+
"aux_layer_ids": [
14+
2,
15+
7,
16+
12,
17+
18,
18+
23,
19+
28
20+
],
21+
"held_out": false
22+
},
23+
"aggregate": {
24+
"acceptance_rate": 0.07805255023183925,
25+
"acceptance_length": 2.097826086956522,
26+
"total_accepted": 101,
27+
"total_drafted": 1294,
28+
"total_blocks": 92,
29+
"lossless_vs_ar": true,
30+
"reference_humaneval": {
31+
"acceptance_length": 7.7,
32+
"acceptance_rate": 0.447
33+
},
34+
"reference_cuda_held_out": {
35+
"acceptance_length": 2.45,
36+
"acceptance_rate": 0.107,
37+
"source": "results/research/k3_dflash_specdecode_corpus_heldout.json",
38+
"note": "PR #93 CUDA held-out baseline; same drafter weights, same verifier weights \u2014 Mac result should be in the same neighbourhood (modulo MLX 4-bit vs CUDA bf16 numerical differences)."
39+
}
40+
},
41+
"per_prompt": [
42+
{
43+
"prompt": "Write a Python function that returns the n-th Fibonacci number.",
44+
"blocks": 21,
45+
"block_accepts": [
46+
0,
47+
0,
48+
0,
49+
0,
50+
1,
51+
2,
52+
1,
53+
0,
54+
1,
55+
3,
56+
1,
57+
1,
58+
2,
59+
0,
60+
1,
61+
0,
62+
0,
63+
4,
64+
4,
65+
3,
66+
4
67+
],
68+
"mean_accepted_per_block": 1.3333333333333333,
69+
"tokens_generated": 48,
70+
"verifier_aux_forwards": 42,
71+
"lossless_vs_ar": true,
72+
"spec_seconds": 41.10908537497744,
73+
"decoded": "thought\nThe objective is to write a Python function that calculates the $n$-th Fibonacci number.\n\n * The Fibonacci sequence is: 0, 1, 1, 2, 3, 5,"
74+
},
75+
{
76+
"prompt": "Explain in two sentences why the sky is blue.",
77+
"blocks": 27,
78+
"block_accepts": [
79+
0,
80+
0,
81+
0,
82+
1,
83+
2,
84+
3,
85+
1,
86+
0,
87+
0,
88+
2,
89+
1,
90+
1,
91+
1,
92+
1,
93+
0,
94+
0,
95+
0,
96+
1,
97+
0,
98+
0,
99+
1,
100+
2,
101+
2,
102+
2,
103+
0,
104+
0,
105+
0
106+
],
107+
"mean_accepted_per_block": 0.7777777777777778,
108+
"tokens_generated": 48,
109+
"verifier_aux_forwards": 54,
110+
"lossless_vs_ar": true,
111+
"spec_seconds": 36.01055458281189,
112+
"decoded": "thought\n* Topic: Why the sky is blue.\n * Constraint: Exactly two sentences.\n\n * Sunlight is composed of many colors (wavelengths).\n * Earth's atmosphere contains gases"
113+
},
114+
{
115+
"prompt": "List three prime numbers greater than 100.",
116+
"blocks": 20,
117+
"block_accepts": [
118+
0,
119+
0,
120+
0,
121+
1,
122+
2,
123+
3,
124+
0,
125+
2,
126+
2,
127+
4,
128+
0,
129+
2,
130+
1,
131+
0,
132+
1,
133+
1,
134+
0,
135+
0,
136+
4,
137+
5
138+
],
139+
"mean_accepted_per_block": 1.4,
140+
"tokens_generated": 48,
141+
"verifier_aux_forwards": 40,
142+
"lossless_vs_ar": true,
143+
"spec_seconds": 27.45034712506458,
144+
"decoded": "thought\n* Target: Three prime numbers.\n * Condition: Greater than 100.\n\n * 101: Is it prime? $\\sqrt{101} \\approx 10"
145+
},
146+
{
147+
"prompt": "Summarize the plot of Romeo and Juliet in one sentence.",
148+
"blocks": 24,
149+
"block_accepts": [
150+
0,
151+
0,
152+
0,
153+
1,
154+
2,
155+
2,
156+
1,
157+
0,
158+
1,
159+
0,
160+
1,
161+
2,
162+
1,
163+
3,
164+
1,
165+
1,
166+
2,
167+
1,
168+
2,
169+
2,
170+
0,
171+
0,
172+
0,
173+
1
174+
],
175+
"mean_accepted_per_block": 1.0,
176+
"tokens_generated": 48,
177+
"verifier_aux_forwards": 48,
178+
"lossless_vs_ar": true,
179+
"spec_seconds": 32.955193375004455,
180+
"decoded": "thought\n* Subject: *Romeo and Juliet* (Shakespeare's play).\n * Task: Summarize the plot.\n * Constraint: One sentence.\n\n * Two lovers from feuding families"
181+
}
182+
]
183+
}

‎results/research/logs/k3_dflash_specdecode_mac_1781024646.log‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
[k3-sd-mac] verifier_path: models/gemma-4-26B-A4B-it-mlx-4bit
2+
[k3-sd-mac] drafter_id: models/dflash-kakeya-baseline
3+
[k3-sd-mac] drafter_device: mps
4+
[k3-sd-mac] loading MLX verifier ...
5+
[k3-sd-mac] loaded in 6.0s
6+
[k3-sd-mac] loading DFlash drafter (torch.bfloat16) ...
7+
[k3-sd-mac] loaded in 2.6s
8+
[k3-sd-mac] prompt 0: blocks=21 mean_accept=1.33 accepts=[0, 0, 0, 0, 1, 2, 1, 0, 1, 3, 1, 1, 2, 0, 1, 0, 0, 4, 4, 3, 4] lossless=True spec_time=41.1s
9+
[k3-sd-mac] prompt 1: blocks=27 mean_accept=0.78 accepts=[0, 0, 0, 1, 2, 3, 1, 0, 0, 2, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 1, 2, 2, 2, 0, 0, 0] lossless=True spec_time=36.0s
10+
[k3-sd-mac] prompt 2: blocks=20 mean_accept=1.40 accepts=[0, 0, 0, 1, 2, 3, 0, 2, 2, 4, 0, 2, 1, 0, 1, 1, 0, 0, 4, 5] lossless=True spec_time=27.5s
11+
[k3-sd-mac] prompt 3: blocks=24 mean_accept=1.00 accepts=[0, 0, 0, 1, 2, 2, 1, 0, 1, 0, 1, 2, 1, 3, 1, 1, 2, 1, 2, 2, 0, 0, 0, 1] lossless=True spec_time=33.0s
12+
[k3-sd-mac] AGGREGATE acceptance_rate=0.078 acceptance_length=2.10 lossless=True (CUDA held-out ref: 0.107 / 2.45) -> results/research/k3_dflash_specdecode_mac_1781024646.json

0 commit comments

Comments
 (0)