Repository navigation
Expand file tree
/
Copy pathwolfbench-chart.py
More file actions
5393 lines (4991 loc) · 229 KB
/
Copy pathwolfbench-chart.py
File metadata and controls
5393 lines (4991 loc) · 229 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/usr/bin/env python3
"""WolfBench Chart — HTML/CSS Chart Generator.
Generates a WolfBench chart as a standalone HTML file with metallic-gradient
bars per model/agent, using the Five-Metric Framework.
Usage:
python wolfbench-chart.py # Generate wolfbench.html
python wolfbench-chart.py -i results.json # Custom input
python wolfbench-chart.py --min-runs 3 # Require ≥3 runs
"""
from __future__ import annotations
import argparse
import base64
import json
import re
from collections import Counter, defaultdict
from datetime import date, datetime, timezone
from html import escape as _html_escape
from io import BytesIO
from pathlib import Path
try:
from PIL import Image
except ImportError: # pragma: no cover - only used when Pillow is unavailable.
Image = None
TOTAL_TASKS = 89
AGENT_LOGO_EMBED_SIZE_PX = 32
DEFAULT_RUNS = 5 # Baseline run count — hidden on bars; only deviations shown
DEFAULT_TIMEOUT_H = 1 # Baseline timeout in hours — hidden on bars; only deviations shown
DEFAULT_VERSIONS = { # Baseline agent versions — hidden on bars; only deviations shown
"terminus-2": "2.0.0",
}
AGENT_LOGOS = {
"terminus-2": """<svg viewBox="0 0 24 24" aria-hidden="true" focusable="false"><path fill-rule="evenodd" d="M4 4.5a2 2 0 0 0-2 2v11a2 2 0 0 0 2 2h16a2 2 0 0 0 2-2v-11a2 2 0 0 0-2-2H4Zm0 3h16v10H4v-10Zm3.2 2.1-1 1.1 2 1.8-2 1.8 1 1.1 3.3-2.9-3.3-2.9Zm4.4 4.5v1.5h5.5v-1.5h-5.5Z"/></svg>""",
"claude-code": """<svg viewBox="0 0 24 24" aria-hidden="true" focusable="false"><path d="m4.7144 15.9555 4.7174-2.6471.079-.2307-.079-.1275h-.2307l-.7893-.0486-2.6956-.0729-2.3375-.0971-2.2646-.1214-.5707-.1215-.5343-.7042.0546-.3522.4797-.3218.686.0608 1.5179.1032 2.2767.1578 1.6514.0972 2.4468.255h.3886l.0546-.1579-.1336-.0971-.1032-.0972L6.973 9.8356l-2.55-1.6879-1.3356-.9714-.7225-.4918-.3643-.4614-.1578-1.0078.6557-.7225.8803.0607.2246.0607.8925.686 1.9064 1.4754 2.4893 1.8336.3643.3035.1457-.1032.0182-.0728-.164-.2733-1.3539-2.4467-1.445-2.4893-.6435-1.032-.17-.6194c-.0607-.255-.1032-.4674-.1032-.7285L6.287.1335 6.6997 0l.9957.1336.419.3642.6192 1.4147 1.0018 2.2282 1.5543 3.0296.4553.8985.2429.8318.091.255h.1579v-.1457l.1275-1.706.2368-2.0947.2307-2.6957.0789-.7589.3764-.9107.7468-.4918.5828.2793.4797.686-.0668.4433-.2853 1.8517-.5586 2.9021-.3643 1.9429h.2125l.2429-.2429.9835-1.3053 1.6514-2.0643.7286-.8196.85-.9046.5464-.4311h1.0321l.759 1.1293-.34 1.1657-1.0625 1.3478-.8804 1.1414-1.2628 1.7-.7893 1.36.0729.1093.1882-.0183 2.8535-.607 1.5421-.2794 1.8396-.3157.8318.3886.091.3946-.3278.8075-1.967.4857-2.3072.4614-3.4364.8136-.0425.0304.0486.0607 1.5482.1457.6618.0364h1.621l3.0175.2247.7892.522.4736.6376-.079.4857-1.2142.6193-1.6393-.3886-3.825-.9107-1.3113-.3279h-.1822v.1093l1.0929 1.0686 2.0035 1.8092 2.5075 2.3314.1275.5768-.3218.4554-.34-.0486-2.2039-1.6575-.85-.7468-1.9246-1.621h-.1275v.17l.4432.6496 2.3436 3.5214.1214 1.0807-.17.3521-.6071.2125-.6679-.1214-1.3721-1.9246L14.38 17.959l-1.1414-1.9428-.1397.079-.674 7.2552-.3156.3703-.7286.2793-.6071-.4614-.3218-.7468.3218-1.4753.3886-1.9246.3157-1.53.2853-1.9004.17-.6314-.0121-.0425-.1397.0182-1.4328 1.9672-2.1796 2.9446-1.7243 1.8456-.4128.164-.7164-.3704.0667-.6618.4008-.5889 2.386-3.0357 1.4389-1.882.929-1.0868-.0062-.1579h-.0546l-6.3385 4.1164-1.1293.1457-.4857-.4554.0608-.7467.2307-.2429 1.9064-1.3114Z"/></svg>""",
"hermes": """<svg viewBox="0 0 24 24" aria-hidden="true" focusable="false"><path d="M13.1 1.7 5.9 13h4.5l-1.5 9.3 9.3-12.4h-5.1l2.1-8.2h-2.1Z"/><path d="M2 7.1c3.9-.4 6.4.4 8.1 2.4l-1 1.6C7.4 9.9 5.4 9.3 2 9.6V7.1Zm20 0c-3.2-.3-5.6.2-7.3 1.5l-.6 2.4c1.7-1.2 3.9-1.7 7.9-1.4V7.1Z"/></svg>""",
"openclaw": """<svg viewBox="0 0 24 24" aria-hidden="true" focusable="false"><path d="M6.1 6.4 1.8 12l4.3 5.6 2-1.5L5 12l3.1-4.1-2-1.5Zm11.8 0-2 1.5L19 12l-3.1 4.1 2 1.5 4.3-5.6-4.3-5.6ZM11 20.2 14.5 3.8h-2.4L8.6 20.2H11Z"/></svg>""",
"cline-cli": """<svg viewBox="0 0 24 24" aria-hidden="true" focusable="false"><path d="m23.365 13.556-1.442-2.895V8.994c0-2.764-2.218-5.002-4.954-5.002h-2.464c.178-.367.276-.779.276-1.213A2.77 2.77 0 0 0 12.018 0a2.77 2.77 0 0 0-2.763 2.779c0 .434.098.846.276 1.213H7.067c-2.736 0-4.954 2.238-4.954 5.002v1.667L.64 13.549c-.149.29-.149.636 0 .927l1.472 2.855v1.667C2.113 21.762 4.33 24 7.067 24h9.902c2.736 0 4.954-2.238 4.954-5.002V17.33l1.44-2.865c.143-.286.143-.622.002-.91m-12.854 2.36a2.27 2.27 0 0 1-2.261 2.273 2.27 2.27 0 0 1-2.261-2.273v-4.042A2.27 2.27 0 0 1 8.249 9.6a2.267 2.267 0 0 1 2.262 2.274zm7.285 0a2.27 2.27 0 0 1-2.26 2.273 2.27 2.27 0 0 1-2.262-2.273v-4.042A2.267 2.267 0 0 1 15.535 9.6a2.267 2.267 0 0 1 2.261 2.274z"/></svg>""",
"cursor-cli": """<svg viewBox="0 0 24 24" aria-hidden="true" focusable="false"><path d="M11.503.131 1.891 5.678a.84.84 0 0 0-.42.726v11.188c0 .3.162.575.42.724l9.609 5.55a1 1 0 0 0 .998 0l9.61-5.55a.84.84 0 0 0 .42-.724V6.404a.84.84 0 0 0-.42-.726L12.497.131a1.01 1.01 0 0 0-.996 0M2.657 6.338h18.55c.263 0 .43.287.297.515L12.23 22.918c-.062.107-.229.064-.229-.06V12.335a.59.59 0 0 0-.295-.51l-9.11-5.257c-.109-.063-.064-.23.061-.23"/></svg>""",
"codex": """<svg viewBox="0 0 24 24" aria-hidden="true" focusable="false"><path d="M22.2819 9.8211a5.9847 5.9847 0 0 0-.5157-4.9108 6.0462 6.0462 0 0 0-6.5098-2.9A6.0651 6.0651 0 0 0 4.9807 4.1818a5.9847 5.9847 0 0 0-3.9977 2.9 6.0462 6.0462 0 0 0 .7427 7.0966 5.98 5.98 0 0 0 .511 4.9107 6.051 6.051 0 0 0 6.5146 2.9001A5.9847 5.9847 0 0 0 13.2599 24a6.0557 6.0557 0 0 0 5.7718-4.2058 5.9894 5.9894 0 0 0 3.9977-2.9001 6.0557 6.0557 0 0 0-.7475-7.0729Zm-9.022 12.6081a4.4755 4.4755 0 0 1-2.8764-1.0408l.1419-.0804 4.7783-2.7582a.7948.7948 0 0 0 .3927-.6813v-6.7369l2.02 1.1686a.071.071 0 0 1 .038.052v5.5826a4.504 4.504 0 0 1-4.4945 4.4944Zm-9.6607-4.1254a4.4708 4.4708 0 0 1-.5346-3.0137l.142.0852 4.783 2.7582a.7712.7712 0 0 0 .7806 0l5.8428-3.3685v2.3324a.0804.0804 0 0 1-.0332.0615L9.74 19.9502a4.4992 4.4992 0 0 1-6.1408-1.6464ZM2.3408 7.8956a4.485 4.485 0 0 1 2.3655-1.9728V11.6a.7664.7664 0 0 0 .3879.6765l5.8144 3.3543-2.0201 1.1685a.0757.0757 0 0 1-.071 0l-4.8303-2.7865A4.504 4.504 0 0 1 2.3408 7.872Zm16.5963 3.8558L13.1038 8.364 15.1192 7.2a.0757.0757 0 0 1 .071 0l4.8303 2.7913a4.4944 4.4944 0 0 1-.6765 8.1042v-5.6772a.79.79 0 0 0-.407-.667Zm2.0107-3.0231-.142-.0852-4.7735-2.7818a.7759.7759 0 0 0-.7854 0L9.409 9.2297V6.8974a.0662.0662 0 0 1 .0284-.0615l4.8303-2.7866a4.4992 4.4992 0 0 1 6.6802 4.66ZM8.3065 12.863l-2.02-1.1638a.0804.0804 0 0 1-.038-.0567V6.0742a4.4992 4.4992 0 0 1 7.3757-3.4537l-.142.0805L8.704 5.459a.7948.7948 0 0 0-.3927.6813Zm1.0976-2.3654 2.602-1.4998 2.6069 1.4998v2.9994l-2.5974 1.4997-2.6067-1.4997Z"/></svg>""",
}
AGENT_LOGO_ASSETS = {
"terminus-2": ("agent-icons/terminus-2.png", "image/png"),
"claude-code": ("agent-icons/claude-code.svg", "image/svg+xml"),
"hermes": ("agent-icons/hermes.svg", "image/svg+xml"),
"openclaw": ("agent-icons/openclaw.svg", "image/svg+xml"),
"cline-cli": ("agent-icons/cline-cli.svg", "image/svg+xml"),
"cursor-cli": ("agent-icons/cursor-cli.svg", "image/svg+xml"),
"codex": ("agent-icons/codex.svg", "image/svg+xml"),
}
AGENT_LOGO_BRANDED: set[str] = {"openclaw"}
_AGENT_LOGO_ASSET_CACHE: dict[tuple[str, str], str] = {}
AGENT_CONFIG = {
"terminus-2": {
"label": "T2",
"name": "Terminus-2",
"gradient": {
"solid": ("linear-gradient(135deg, #0a3d1a 0%, #145a32 40%, #1e8449 70%, #27ae60 100%)",
"0 0 12px rgba(39,174,96,0.3)"),
"average": ("linear-gradient(135deg, #1a7a3a 0%, #27ae60 40%, #2ecc71 70%, #58d68d 100%)",
"0 0 12px rgba(46,204,113,0.3)"),
"best": ("linear-gradient(135deg, #27ae60 0%, #58d68d 40%, #82e0aa 70%, #a9dfbf 100%)",
"0 0 12px rgba(88,214,141,0.3)"),
"ceiling": ("linear-gradient(135deg, #58d68d 0%, #abebc6 40%, #d5f5e3 70%, #eafaf1 100%)",
"0 0 8px rgba(171,235,198,0.2)"),
},
},
"claude-code": {
"label": "CC",
"name": "Claude Code",
"gradient": {
"solid": ("linear-gradient(135deg, #5a2d00 0%, #7d3c00 40%, #b35900 70%, #e67e22 100%)",
"0 0 12px rgba(230,126,34,0.3)"),
"average": ("linear-gradient(135deg, #7d3c00 0%, #e67e22 40%, #f0b27a 70%, #f5cba7 100%)",
"0 0 12px rgba(240,178,122,0.3)"),
"best": ("linear-gradient(135deg, #e67e22 0%, #f0b27a 40%, #f5cba7 70%, #fdebd0 100%)",
"0 0 12px rgba(245,203,167,0.3)"),
"ceiling": ("linear-gradient(135deg, #f0b27a 0%, #fdebd0 40%, #fef5e7 70%, #fffaf2 100%)",
"0 0 8px rgba(253,235,208,0.2)"),
},
},
"hermes": {
"label": "HA",
"name": "Hermes Agent",
"gradient": {
"solid": ("linear-gradient(135deg, #5a4d00 0%, #7d6b00 40%, #b39700 70%, #f1c40f 100%)",
"0 0 12px rgba(241,196,15,0.3)"),
"average": ("linear-gradient(135deg, #7d6b00 0%, #f1c40f 40%, #f4d03f 70%, #f7dc6f 100%)",
"0 0 12px rgba(244,208,63,0.3)"),
"best": ("linear-gradient(135deg, #f1c40f 0%, #f7dc6f 40%, #f9e79f 70%, #fcf3cf 100%)",
"0 0 12px rgba(247,220,111,0.3)"),
"ceiling": ("linear-gradient(135deg, #f7dc6f 0%, #fcf3cf 40%, #fef9e7 70%, #fffdf2 100%)",
"0 0 8px rgba(252,243,207,0.2)"),
},
},
"openclaw": {
"label": "OC",
"name": "OpenClaw",
"gradient": {
"solid": ("linear-gradient(135deg, #641e16 0%, #922b21 40%, #c0392b 70%, #e74c3c 100%)",
"0 0 12px rgba(231,76,60,0.3)"),
"average": ("linear-gradient(135deg, #922b21 0%, #e74c3c 40%, #ec7063 70%, #f1948a 100%)",
"0 0 12px rgba(241,148,138,0.3)"),
"best": ("linear-gradient(135deg, #e74c3c 0%, #f1948a 40%, #f5b7b1 70%, #fadbd8 100%)",
"0 0 12px rgba(245,183,177,0.3)"),
"ceiling": ("linear-gradient(135deg, #f1948a 0%, #fadbd8 40%, #fdedec 70%, #fef9f8 100%)",
"0 0 8px rgba(250,219,216,0.2)"),
},
},
"cline-cli": {
"label": "Cl",
"name": "Cline CLI",
"gradient": {
"solid": ("linear-gradient(135deg, #2d1054 0%, #4a1a8a 40%, #6c3483 70%, #8e44ad 100%)",
"0 0 12px rgba(142,68,173,0.3)"),
"average": ("linear-gradient(135deg, #4a1a8a 0%, #8e44ad 40%, #a569bd 70%, #bb8fce 100%)",
"0 0 12px rgba(165,105,189,0.3)"),
"best": ("linear-gradient(135deg, #8e44ad 0%, #bb8fce 40%, #d2b4de 70%, #e8daef 100%)",
"0 0 12px rgba(210,180,222,0.3)"),
"ceiling": ("linear-gradient(135deg, #bb8fce 0%, #e8daef 40%, #f4ecf7 70%, #faf5fc 100%)",
"0 0 8px rgba(232,218,239,0.2)"),
},
},
"cursor-cli": {
"label": "CA",
"name": "Cursor",
"gradient": {
"solid": ("linear-gradient(135deg, #0e3855 0%, #1a5276 40%, #2874a6 70%, #3498db 100%)",
"0 0 12px rgba(52,152,219,0.3)"),
"average": ("linear-gradient(135deg, #1a5276 0%, #3498db 40%, #5dade2 70%, #85c1e9 100%)",
"0 0 12px rgba(93,173,226,0.3)"),
"best": ("linear-gradient(135deg, #3498db 0%, #85c1e9 40%, #aed6f1 70%, #d6eaf8 100%)",
"0 0 12px rgba(133,193,233,0.3)"),
"ceiling": ("linear-gradient(135deg, #85c1e9 0%, #d6eaf8 40%, #eaf5fb 70%, #f4fafd 100%)",
"0 0 8px rgba(214,234,248,0.2)"),
},
},
"codex": {
"label": "CX",
"name": "Codex",
"gradient": {
"solid": ("linear-gradient(135deg, #1e1b4b 0%, #3730a3 40%, #4f46e5 70%, #6366f1 100%)",
"0 0 12px rgba(99,102,241,0.3)"),
"average": ("linear-gradient(135deg, #3730a3 0%, #6366f1 40%, #818cf8 70%, #a5b4fc 100%)",
"0 0 12px rgba(129,140,248,0.3)"),
"best": ("linear-gradient(135deg, #6366f1 0%, #a5b4fc 40%, #c7d2fe 70%, #e0e7ff 100%)",
"0 0 12px rgba(165,180,252,0.3)"),
"ceiling": ("linear-gradient(135deg, #a5b4fc 0%, #e0e7ff 40%, #eef2ff 70%, #f5f7ff 100%)",
"0 0 8px rgba(224,231,255,0.2)"),
},
},
}
def _raster_logo_data_uri(logo_path: Path, mime: str) -> str:
if Image is None:
raise RuntimeError("Pillow is required to normalize raster agent logos.")
with Image.open(logo_path) as source:
image = source.convert("RGBA")
alpha_bbox = image.getchannel("A").getbbox()
if alpha_bbox:
image = image.crop(alpha_bbox)
resampling = getattr(getattr(Image, "Resampling", Image), "LANCZOS")
image.thumbnail((AGENT_LOGO_EMBED_SIZE_PX, AGENT_LOGO_EMBED_SIZE_PX), resampling)
canvas = Image.new("RGBA", (AGENT_LOGO_EMBED_SIZE_PX, AGENT_LOGO_EMBED_SIZE_PX), (0, 0, 0, 0))
offset = (
(AGENT_LOGO_EMBED_SIZE_PX - image.width) // 2,
(AGENT_LOGO_EMBED_SIZE_PX - image.height) // 2,
)
canvas.alpha_composite(image, offset)
buffer = BytesIO()
canvas.save(buffer, format="PNG", optimize=True)
encoded = base64.b64encode(buffer.getvalue()).decode("ascii")
return f"data:image/png;base64,{encoded}"
def _strip_svg_tooltip_nodes(svg: str) -> str:
return re.sub(r"<(?:title|desc)\b[^>]*>.*?</(?:title|desc)>", "", svg, flags=re.IGNORECASE | re.DOTALL)
def _agent_logo_html(agent: str, title: str) -> str:
safe_title = _html_escape(title)
if agent in AGENT_LOGO_ASSETS:
cache_key = (agent, title)
cached_logo = _AGENT_LOGO_ASSET_CACHE.get(cache_key)
if cached_logo is not None:
return cached_logo
rel_path, mime = AGENT_LOGO_ASSETS[agent]
logo_path = Path(__file__).parent / rel_path
if logo_path.exists():
safe_agent = _html_escape(agent)
logo_classes = f"agent-logo agent-logo-{safe_agent}"
if agent in AGENT_LOGO_BRANDED:
logo_classes += " agent-logo-branded"
if mime == "image/svg+xml":
svg = _strip_svg_tooltip_nodes(logo_path.read_text(encoding="utf-8").strip())
logo = f'<span class="{logo_classes}" title="{safe_title}" aria-hidden="true">{svg}</span>'
else:
data_uri = _raster_logo_data_uri(logo_path, mime)
logo = (
f'<span class="{logo_classes}" title="{safe_title}" aria-hidden="true">'
f'<img src="{data_uri}" alt="" decoding="async">'
f'</span>'
)
_AGENT_LOGO_ASSET_CACHE[cache_key] = logo
return logo
logo = AGENT_LOGOS.get(agent)
if not logo:
return f'<span class="agent-logo agent-logo-missing" title="{safe_title}" aria-hidden="true"></span>'
svg = _strip_svg_tooltip_nodes(logo)
return f'<span class="agent-logo agent-logo-{_html_escape(agent)}" title="{safe_title}" aria-hidden="true">{svg}</span>'
def _agent_badge_html(agent: str, name: str) -> str:
safe_name = _html_escape(name)
return (
f'<span class="agent-badge" title="{safe_name}" aria-label="{safe_name}">'
f'{_agent_logo_html(agent, name)}</span>'
)
_HEX_RE = re.compile(r"#[0-9a-fA-F]{6}")
def _blend_hex(a: str, b: str, amount: float) -> str:
"""Blend two #rrggbb colors."""
a = a.lstrip("#")
b = b.lstrip("#")
ar, ag, ab = (int(a[i:i + 2], 16) for i in (0, 2, 4))
br, bg, bb = (int(b[i:i + 2], 16) for i in (0, 2, 4))
mix = lambda x, y: round(x * (1 - amount) + y * amount)
return f"#{mix(ar, br):02x}{mix(ag, bg):02x}{mix(ab, bb):02x}"
def _worst_gradient(grads: dict[str, tuple[str, str]]) -> tuple[str, str]:
"""Create a visible Worst-of segment style between Solid and Average."""
solid_grad, _ = grads["solid"]
avg_grad, avg_shadow = grads["average"]
solid_colors = _HEX_RE.findall(solid_grad)
avg_colors = _HEX_RE.findall(avg_grad)
if not solid_colors or len(solid_colors) != len(avg_colors):
return avg_grad, avg_shadow
stops = [0, 40, 70, 100]
blended = [
f"{_blend_hex(s, a, 0.55)} {stops[i] if i < len(stops) else round(i / max(1, len(solid_colors) - 1) * 100)}%"
for i, (s, a) in enumerate(zip(solid_colors, avg_colors))
]
return "linear-gradient(135deg, " + ", ".join(blended) + ")", avg_shadow
MODEL_DISPLAY = {
"claude-opus-4-6": "Claude Opus 4.6",
"claude-sonnet-4-6": "Claude Sonnet 4.6",
"Kimi-K2.5": "Kimi K2.5",
"MiniMax-M2.5": "MiniMax M2.5",
"GLM-5-FP8": "GLM-5 FP8",
}
def _resolve_display_name(r: dict) -> str:
"""Return display name: model_display override if set, else auto-derive from model path."""
md = r.get("model_display") or ""
if md:
return md
model_full = r.get("model", "unknown")
parts = model_full.split("/")
if len(parts) >= 3:
return "/".join(parts[2:])
elif len(parts) == 2:
return parts[-1]
return model_full
def _normalize_thinking(t) -> str:
"""Normalize thinking/reasoning_effort to a display string."""
if t is None:
return "-"
if t is True or t == "enabled":
return "on"
if t is False or t == "disabled":
return "off"
return str(t)
def _resolve_thinking(r: dict) -> str:
"""Return thinking display: thinking_display override if set, else auto-normalize."""
td = r.get("thinking_display") or ""
if td:
return td
return _normalize_thinking(r.get("thinking"))
def _resolve_version(r: dict) -> str:
"""Return agent version: version_display override if set, else agent_version."""
vd = r.get("version_display") or ""
if vd:
return vd
return r.get("agent_version") or "-"
def _resolve_provider_vendor(r: dict) -> tuple[str, str]:
"""Return (provider, vendor): overrides if set, else split from model path."""
model_full = r.get("model", "unknown")
parts = model_full.split("/")
if len(parts) >= 3:
provider = parts[0]
vendor = parts[1]
elif len(parts) == 2:
provider = parts[0]
vendor = parts[0]
else:
provider = "-"
vendor = "-"
pd = r.get("provider_display") or ""
vd = r.get("vendor_display") or ""
return (pd or provider, vd or vendor)
def _positive_cost_usd(value) -> float | None:
try:
cost = float(value)
except (TypeError, ValueError):
return None
return cost if cost > 0 else None
def _fmt_cost_usd(value) -> str:
cost = _positive_cost_usd(value)
return f"${cost:,.2f}" if cost is not None else "-"
METRIC_LABELS = {
"ceiling": ("★", "Ceiling", "ever solved"),
"best": ("▲", "Best-of", "peak run"),
"average": ("∅", "Average", "mean score"),
"worst": ("▼", "Worst-of", "lowest run"),
"solid": ("■", "Solid", "always solved"),
}
def compute_metrics(runs: list[dict]) -> dict | None:
n_runs = len(runs)
scores = [r["score"] for r in runs if r["score"] is not None]
if not scores:
return None
task_pass_counts = Counter()
for r in runs:
for t in r["passed_tasks"]:
task_pass_counts[t] += 1
solid = sum(1 for c in task_pass_counts.values() if c == n_runs)
ceiling = len(task_pass_counts)
# Extract timeout (most common value; typically uniform per group)
timeouts = [r.get("timeout_sec") for r in runs if r.get("timeout_sec") is not None]
timeout_sec = max(set(timeouts), key=timeouts.count) if timeouts else None
token_input_total = 0
token_output_total = 0
cost_total = 0.0
cost_runs = 0
for r in runs:
token_input_total += (r.get("tokens_in") or 0) + (r.get("tokens_cache_write") or 0)
token_output_total += r.get("tokens_out") or 0
cost = _positive_cost_usd(r.get("cost_usd"))
if cost is not None:
cost_total += cost
cost_runs += 1
avg_score = sum(scores) / len(scores)
return {
"n_runs": n_runs,
"min": round(min(scores) * 100),
"solid": round(solid / TOTAL_TASKS * 100),
"average": round(avg_score * 100),
"best": round(max(scores) * 100),
"ceiling": round(ceiling / TOTAL_TASKS * 100),
"min_abs": round(min(scores) * TOTAL_TASKS),
"solid_abs": solid,
"avg_abs": round(avg_score * TOTAL_TASKS),
"best_abs": round(max(scores) * TOTAL_TASKS),
"ceiling_abs": ceiling,
# Raw unrounded values (0-TOTAL_TASKS range) for tiebreak comparisons —
# avoid rounding artifacts like 63 vs 64 both displaying as 71%.
"min_raw": min(scores) * TOTAL_TASKS,
"solid_raw": float(solid),
"avg_raw": avg_score * TOTAL_TASKS,
"best_raw": max(scores) * TOTAL_TASKS,
"ceiling_raw": float(ceiling),
"timeout_sec": timeout_sec,
"tokens_in_total": token_input_total,
"tokens_out_total": token_output_total,
"tokens_total": token_input_total + token_output_total,
"tokens_runs": n_runs,
"cost_total_usd": cost_total,
"cost_runs": cost_runs,
}
def _fmt_timeout_h(sec: float | int | None) -> str:
"""Convert timeout seconds to compact hours string: 7200→'2h', 5400→'1.5h'."""
if sec is None:
return ""
h = sec / 3600
return f"{h:.0f}h" if h == int(h) else f"{h:.1f}h"
def _parse_run_datetime(value: str | None) -> datetime | None:
"""Parse ISO datetimes or Harbor run-dir timestamps."""
if not value:
return None
raw = str(value).strip()
if not raw:
return None
normalized = raw.replace("Z", "+00:00")
try:
parsed = datetime.fromisoformat(normalized)
if parsed.tzinfo is not None:
return parsed.astimezone(timezone.utc).replace(tzinfo=None)
return parsed
except ValueError:
pass
match = re.match(r"^(\d{4}-\d{2}-\d{2})__(\d{2})-(\d{2})(?:-(\d{2}))?", raw)
if not match:
return None
date_part, hh, mm, ss = match.groups()
try:
return datetime.fromisoformat(f"{date_part}T{hh}:{mm}:{ss or '00'}")
except ValueError:
return None
def _run_datetime_label_and_sort(run: dict) -> tuple[str, str]:
dt = _parse_run_datetime(run.get("started_at")) or _parse_run_datetime(run.get("timestamp"))
if dt is None:
ts = str(run.get("timestamp") or "").strip()
return (ts[:10] if ts else "-", "")
return dt.strftime("%Y-%m-%d %H:%M"), str(int(dt.timestamp()))
# Scale: pixels per percentage point
PX_PER_PCT = 6
CHART_HEIGHT = 100 * PX_PER_PCT # 600px for 100%
def _bar_segments_html(metrics: dict, agent_cfg: dict) -> str:
"""Generate the stacked bar segments for one agent/model combo."""
grads = agent_cfg["gradient"]
_abs_px = lambda v: v / TOTAL_TASKS * 100 * PX_PER_PCT
seg = [
("solid", metrics["solid"], metrics["solid_abs"]),
("worst", metrics["min"] - metrics["solid"], metrics["min_abs"] - metrics["solid_abs"]),
("average", metrics["average"] - metrics["min"], metrics["avg_abs"] - metrics["min_abs"]),
("best", metrics["best"] - metrics["average"], metrics["best_abs"] - metrics["avg_abs"]),
("ceiling", metrics["ceiling"] - metrics["best"], metrics["ceiling_abs"] - metrics["best_abs"]),
]
# Segment divs (no labels inside — labels are positioned separately)
parts = []
for key, height_pct, height_abs in seg:
if height_pct <= 0 and height_abs <= 0:
continue
gradient, shadow = _worst_gradient(grads) if key == "worst" else grads[key]
px_h = max(height_pct * PX_PER_PCT, 2)
px_h_abs = max(_abs_px(height_abs), 2)
parts.append(f'''
<div class="segment segment-{key}" data-metric="{key}"
data-h-pct="{px_h:.1f}" data-h-abs="{px_h_abs:.1f}" style="
height: {px_h}px;
background: {gradient};
box-shadow: {shadow}, inset 0 1px 0 rgba(255,255,255,0.15), inset 0 -1px 0 rgba(0,0,0,0.2);
">
<div class="segment-shine"></div>
</div>''')
# Reverse so ceiling is on top (we build bottom-up)
parts.reverse()
# Collect labels, then nudge to avoid overlap
total_h = metrics["ceiling"] * PX_PER_PCT
total_h_abs = _abs_px(metrics["ceiling_abs"])
if metrics["n_runs"] == 1:
# Single run: all five metrics are identical — just show the value, no symbols
val = metrics["ceiling"]
abs_val = metrics["ceiling_abs"]
bottom_px = val * PX_PER_PCT
abs_bottom_px = _abs_px(abs_val)
label_parts = [
f'<span class="seg-label seg-label-single" data-metric="solid"'
f' data-true-bottom="{bottom_px:.1f}"'
f' data-bottom-pct="{bottom_px:.1f}" data-bottom-abs="{abs_bottom_px:.1f}"'
f' style="bottom: {bottom_px:.1f}px;">'
f'<span class="seg-pct" data-pct="{val}%" data-abs="{abs_val}">{val}%</span></span>'
]
else:
_val_key = {"worst": "min", "solid": "solid", "average": "average",
"best": "best", "ceiling": "ceiling"}
_abs_key = {"worst": "min_abs", "solid": "solid_abs", "average": "avg_abs",
"best": "best_abs", "ceiling": "ceiling_abs"}
labels = []
for key in ("worst", "solid", "average", "best", "ceiling"):
val = metrics[_val_key[key]]
abs_val = metrics[_abs_key[key]]
sym, name, _ = METRIC_LABELS[key]
labels.append((val, abs_val, sym, f"{val}%", str(abs_val), key))
labels.sort(key=lambda t: t[0])
# Collision avoidance helper
def _snap_positions(raw_px: list[float], max_bottom: float) -> list[float]:
min_gap_px = 15 # ~label height at 0.7rem
positions = list(raw_px)
for _ in range(40):
changed = False
for i in range(len(positions) - 1):
gap = positions[i + 1] - positions[i]
if gap < min_gap_px:
push = (min_gap_px - gap) / 2
positions[i] -= push
positions[i + 1] += push
changed = True
if positions and positions[-1] > max_bottom:
overshoot = positions[-1] - max_bottom
positions = [p - overshoot for p in positions]
changed = True
if positions and positions[0] < 0:
undershoot = -positions[0]
positions = [p + undershoot for p in positions]
changed = True
if not changed:
break
return positions
# Compute positions for both percentage and absolute modes
pct_raw = [val * PX_PER_PCT for val, _, _, _, _, _ in labels]
abs_raw = [abs_v / TOTAL_TASKS * 100 * PX_PER_PCT for _, abs_v, _, _, _, _ in labels]
pct_positions = _snap_positions(pct_raw, total_h)
abs_positions = _snap_positions(abs_raw, total_h)
label_parts = []
for (val, abs_v, sym, pct, abs_val, metric_key), bottom_px, abs_bottom_px in zip(
labels, pct_positions, abs_positions):
true_px = val * PX_PER_PCT
label_parts.append(
f'<span class="seg-label" data-metric="{metric_key}"'
f' data-true-bottom="{true_px:.1f}"'
f' data-bottom-pct="{bottom_px:.1f}" data-bottom-abs="{abs_bottom_px:.1f}"'
f' style="bottom: {bottom_px:.1f}px;">'
f'<span class="seg-sym">{sym}</span>'
f'<span class="seg-pct" data-pct="{pct}" data-abs="{abs_val}">{pct}</span></span>'
)
segments_html = "\n".join(parts)
labels_html = "\n".join(label_parts)
worst_px = metrics["min"] * PX_PER_PCT
return f'''
<div class="bar-inner" style="height: {total_h}px;"
data-h-pct="{total_h:.1f}" data-h-abs="{total_h_abs:.1f}"
data-h-worst="{worst_px:.1f}"
data-h-solid="{metrics['solid'] * PX_PER_PCT:.1f}"
data-h-average="{metrics['average'] * PX_PER_PCT:.1f}"
data-h-best="{metrics['best'] * PX_PER_PCT:.1f}"
data-h-ceiling="{total_h:.1f}">
<div class="bar-segments">{segments_html}</div>
<div class="bar-labels">{labels_html}</div>
</div>'''
def _build_runs_table_html(
runs: list[dict],
weave_run_urls: dict[tuple, str] | None = None,
) -> str:
"""Build a collapsed HTML <details> table showing individual run data."""
if not runs:
return ""
def _fmt_tok(n):
if not n:
return "-"
if n >= 1_000_000_000:
return f"{n / 1_000_000_000:.2f}".rstrip("0").rstrip(".") + "B"
if n >= 1_000_000:
return f"{n / 1_000_000:.1f}M"
if n >= 1_000:
return f"{n / 1_000:.0f}K"
return str(n)
def _safe_float(value) -> float:
try:
f = float(value or 0)
except (TypeError, ValueError):
return 0.0
return f if f > 0 else 0.0
def _fmt_duration_total(seconds: float) -> str:
total = int(round(seconds))
if total <= 0:
return "n/a"
days, rem = divmod(total, 86_400)
hours, rem = divmod(rem, 3_600)
minutes, _ = divmod(rem, 60)
parts = []
if days:
parts.append(f"{days}d")
if hours or days:
parts.append(f"{hours}h")
parts.append(f"{minutes}m")
return " ".join(parts)
def _build_resource_stats_text(visible_runs: list[dict]) -> str:
n_visible = len(visible_runs)
duration_total = 0.0
duration_runs = 0
token_in_total = 0
token_out_total = 0
token_runs = 0
cost_total = 0.0
cost_runs = 0
for _r in visible_runs:
_duration = _safe_float(_r.get("duration_sec"))
if _duration:
duration_total += _duration
duration_runs += 1
_tok_in = (_r.get("tokens_in") or 0) + (_r.get("tokens_cache_write") or 0)
_tok_out = _r.get("tokens_out") or 0
_tok_total = _tok_in + _tok_out
if _tok_total:
token_in_total += _tok_in
token_out_total += _tok_out
token_runs += 1
_cost = _positive_cost_usd(_r.get("cost_usd"))
if _cost is not None:
cost_total += _cost
cost_runs += 1
duration_label = _fmt_duration_total(duration_total)
if duration_runs and duration_runs != n_visible:
duration_label += f" (timing data for {duration_runs}/{n_visible} runs)"
token_total = token_in_total + token_out_total
if token_total:
token_label = f"{_fmt_tok(token_total)} ({_fmt_tok(token_in_total)} in, {_fmt_tok(token_out_total)} out)"
if token_runs != n_visible:
token_label += f" (token data for {token_runs}/{n_visible} runs)"
else:
token_label = "n/a"
if cost_runs:
cost_label = _fmt_cost_usd(cost_total)
if cost_runs != n_visible:
cost_label += f" (cost data for {cost_runs}/{n_visible} runs)"
else:
cost_label = "n/a"
return (
f"Total runtime: {duration_label} · "
f"Total tokens: {token_label} · "
f"Total cost: {cost_label}."
)
# Sort: date descending (latest first)
sorted_runs = sorted(
runs,
key=lambda r: _parse_run_datetime(r.get("started_at")) or _parse_run_datetime(r.get("timestamp")) or datetime.min,
reverse=True,
)
rows: list[str] = []
for r in sorted_runs:
ts = r.get("timestamp", "")
date_time, date_sort_value = _run_datetime_label_and_sort(r)
# Agent (full name) + version in brackets: "OpenClaw (v2026.2.17)"
agent = r.get("agent", "?")
agent_name = AGENT_CONFIG.get(agent, {}).get("name", agent)
ver = _resolve_version(r)
agent_ver = f"{agent_name} ({ver})" if ver != "-" else agent_name
# Provider / Vendor (override if set, else split from full model path)
provider, vendor = _resolve_provider_vendor(r)
model = _resolve_display_name(r)
think = _resolve_thinking(r)
run_url = (weave_run_urls or {}).get(
(agent, ver if ver != "-" else "unknown", model, r.get("timeout_sec"), think, ts)
)
if run_url:
date_cell = (
f'<a class="run-link" href="{_html_escape(run_url)}" target="_blank" '
f'rel="noopener noreferrer" title="View this run trace on W&B Weave">'
f'{_html_escape(date_time)}</a>'
)
else:
date_cell = _html_escape(date_time)
score = f'{r["score"]:.1%}' if r.get("score") is not None else "?"
n_p = r.get("n_passed", 0)
n_f = r.get("n_failed", 0)
n_t = r.get("n_trials", 0)
to_sec = r.get("timeout_sec")
to_str = f"{int(to_sec)}s" if to_sec is not None else "-"
eb = r.get("error_breakdown", {})
ato = eb.get("AgentTimeoutError", 0)
lost = max(0, n_t - n_p - n_f)
duration_sec = _safe_float(r.get("duration_sec"))
dur = "-"
if duration_sec:
h, rem = divmod(int(duration_sec), 3600)
m, _ = divmod(rem, 60)
dur = f"{h}h{m:02d}m"
# tokens_cache is a discounted subset of tokens_in, not extra input.
tok_in_raw = r.get("tokens_in")
tok_cache_write_raw = r.get("tokens_cache_write")
tok_cache_raw = r.get("tokens_cache")
tok_total_in = None
if tok_in_raw is not None or tok_cache_write_raw is not None:
tok_total_in = (tok_in_raw or 0) + (tok_cache_write_raw or 0)
tok_in = _fmt_tok(tok_total_in)
tok_out_raw = r.get("tokens_out")
tok_out = _fmt_tok(tok_out_raw)
tok_total_raw = (
(tok_total_in or 0) + (tok_out_raw or 0)
if tok_total_in is not None or tok_out_raw is not None else None
)
tok_total = _fmt_tok(tok_total_raw)
cost_value = _positive_cost_usd(r.get("cost_usd"))
cost_str = _fmt_cost_usd(cost_value)
cost_sort_value = f"{cost_value:.6f}" if cost_value is not None else ""
cost_attr = f"{cost_value:.6f}" if cost_value is not None else "0"
_passed_json = json.dumps(r.get("passed_tasks", []), separators=(",", ":"))
rows.append(
f"<tr data-agent=\"{_html_escape(agent)}\" data-model=\"{_html_escape(model)}\" data-passed='{_passed_json}' "
f'data-duration-sec="{duration_sec:.3f}" data-token-input="{int(tok_total_in or 0)}" '
f'data-token-output="{int(tok_out_raw or 0)}" data-token-total="{int(tok_total_raw or 0)}" '
f'data-cost-usd="{cost_attr}">'
f"<td data-sort-value=\"{date_sort_value}\">{date_cell}</td>"
f"<td>{_html_escape(agent_ver)}</td>"
f"<td>{_html_escape(provider)}</td>"
f"<td>{_html_escape(vendor)}</td>"
f"<td>{_html_escape(model)}</td>"
f"<td>{think}</td>"
f"<td>{score}</td>"
f"<td>{n_p}</td>"
f"<td>{n_f}</td>"
f"<td>{to_str}</td>"
f"<td>{ato}</td>"
f"<td>{lost}</td>"
f"<td>{dur}</td>"
f"<td>{tok_in}</td>"
f"<td>{tok_out}</td>"
f"<td>{tok_total}</td>"
f"<td class='n' data-sort-value=\"{cost_sort_value}\">{cost_str}</td>"
f"</tr>"
)
# Compute aggregate task stats across all runs
_task_counts = Counter()
for _r in runs:
for _t in _r.get("passed_tasks", []):
_task_counts[_t] += 1
_n_total_runs = len(runs)
_solved_once = len(_task_counts)
_solved_always = sum(1 for _c in _task_counts.values() if _c == _n_total_runs) if _n_total_runs > 0 else 0
_never_solved = TOTAL_TASKS - _solved_once
_pct = lambda x: round(x / TOTAL_TASKS * 100)
_task_stats_html = (
f'<p class="task-stats" id="taskStats" data-total-tasks="{TOTAL_TASKS}">Across these {_n_total_runs} runs, '
f'{_solved_once} ({_pct(_solved_once)}%) of the {TOTAL_TASKS} tasks were solved at least once, '
f'{_solved_always} ({_pct(_solved_always)}%) were solved every time, '
f'and {_never_solved} ({_pct(_never_solved)}%) were never solved.</p>'
)
_resource_stats_html = (
f'<p class="task-stats run-resource-stats" id="runResourceStats">'
f'{_html_escape(_build_resource_stats_text(runs))}</p>'
)
return (
f'<details class="runs-details">\n'
f'<summary>Run Details ({len(runs)} runs)</summary>\n'
f'{_task_stats_html}\n'
f'{_resource_stats_html}\n'
f'<div class="runs-table-wrap">\n'
f'<table class="runs-table">\n'
f'<thead><tr>'
f'<th data-sort="desc">Date</th><th>Agent</th>'
f'<th>Provider</th><th>Vendor</th><th>Model</th><th>Think</th>'
f'<th>Score</th><th>Pass</th><th>Fail</th>'
f'<th>Timeout</th><th>Timeouts</th><th>Err</th>'
f'<th>Duration</th>'
f'<th>In</th><th>Out</th><th>Total</th>'
f'<th>Cost</th>'
f'</tr></thead>\n'
f'<tbody>\n{"".join(rows)}\n</tbody>\n'
f'</table>\n'
f'</div>\n'
f'</details>'
)
def generate_html(
groups: dict[tuple[str, str, str, float | None, str], dict],
output_path: Path,
min_runs: int = 1,
agent_versions: dict[str, set[str]] | None = None,
chart_date: str | None = None,
runs: list[dict] | None = None,
weave_urls: dict[tuple, str] | None = None,
weave_run_urls: dict[tuple, str] | None = None,
) -> Path:
# Embed logos as base64 so the generated HTML stays standalone.
_asset_dir = Path(__file__).parent
_wolfbench_logo_path = _asset_dir / "WolfBench-Logo-256.png"
_wandb_cw_logo_path = _asset_dir / "Endorsed_secondary_goldwhite.png"
wolfbench_logo_b64 = base64.b64encode(_wolfbench_logo_path.read_bytes()).decode("ascii")
wandb_cw_logo_b64 = base64.b64encode(_wandb_cw_logo_path.read_bytes()).decode("ascii")
three_bars_js = (Path(__file__).parent / "wolfbench-threejs-bars.js").read_text()
# Filter
groups = {k: v for k, v in groups.items() if v["n_runs"] >= min_runs}
if not groups:
print("No groups meet the minimum run threshold.")
return None
# Organize by model — keyed by (agent, version, timeout, thinking) tuple
models_with_data: dict[str, dict[tuple[str, str, float | None, str], dict]] = defaultdict(dict)
for (agent, ver, model, timeout, thinking), metrics in groups.items():
models_with_data[model][(agent, ver, timeout, thinking)] = metrics
# Tiebreaker cascade within each agent: primary metric (avg) first,
# then others in legend-display order. Missing agent ranks last.
# Uses RAW (unrounded) values to avoid rounding artifacts in ties.
_TIEBREAK_METRICS = ("avg_raw", "ceiling_raw", "best_raw", "min_raw", "solid_raw")
def _model_sort_key(m: str) -> tuple[float, ...]:
key: list[float] = []
for a in AGENT_CONFIG:
agent_data = [v for (ag, *_), v in models_with_data[m].items() if ag == a]
for data_key in _TIEBREAK_METRICS:
if agent_data:
key.append(-max(v[data_key] for v in agent_data))
else:
key.append(1) # missing agent ranks after any real score (negated: -N..0)
return tuple(key)
model_order = sorted(models_with_data.keys(), key=_model_sort_key)
# Ordered (agent, version, timeout, thinking) quads: AGENT_CONFIG order, then version/timeout/thinking asc
_seen_quads: set[tuple[str, str, float | None, str]] = set()
for m_data in models_with_data.values():
_seen_quads.update(m_data.keys())
agent_ver_order: list[tuple[str, str, float | None, str]] = []
for a in AGENT_CONFIG:
matches = sorted(
((ver, to, th) for (ag, ver, to, th) in _seen_quads if ag == a),
key=lambda x: (x[0], x[1] or 0, x[2]),
)
for v, t, th in matches:
agent_ver_order.append((a, v, t, th))
# Flat agent list (unique, preserving AGENT_CONFIG order) for legend/footer
agent_order = [a for a in AGENT_CONFIG
if any(ag == a for ag, _, _, _ in agent_ver_order)]
# Build model groups HTML
model_groups_html = []
model_group_widths = []
for _model_idx, model in enumerate(model_order):
bars_html = []
group_bar_widths = []
# Per-agent per-metric scores for JS re-sorting on agent/metric filter
_scores: dict[str, dict[str, float]] = {}
_raw_key = {"solid": "solid_raw", "average": "avg_raw", "best": "best_raw",
"ceiling": "ceiling_raw", "worst": "min_raw"}
for _a in agent_order:
_am: dict[str, float] = {}
for (ag, ver_, to_, th_), v in models_with_data[model].items():
if ag == _a:
for _mk in ("solid", "average", "best", "ceiling"):
_am[_mk] = max(_am.get(_mk, 0.0), v[_mk])
_rk = _raw_key[_mk]
_am[_mk + "_raw"] = max(_am.get(_mk + "_raw", 0.0), v[_rk])
_am["worst"] = max(_am.get("worst", 0.0), v["min"])
_am["worst_raw"] = max(_am.get("worst_raw", 0.0), v["min_raw"])
if _am:
_scores[_a] = {k: (round(v, 4) if k.endswith("_raw") else round(v, 1))
for k, v in _am.items()}
_scores_json = json.dumps(_scores, separators=(",", ":"))
# Within each agent, sort variants by score cascade (avg → ceiling → best → worst → solid).
# Uses raw values to avoid rounding ties. Matches JS within-agent tiebreaker.
_ordered_quads: list[tuple[str, str, float | None, str]] = []
for _a in AGENT_CONFIG:
_variants = [
(ag, ver, to, th)
for (ag, ver, to, th) in models_with_data[model]
if ag == _a
]
_variants.sort(key=lambda q: (
-models_with_data[model][q]["avg_raw"],
-models_with_data[model][q]["ceiling_raw"],
-models_with_data[model][q]["best_raw"],
-models_with_data[model][q]["min_raw"],
-models_with_data[model][q]["solid_raw"],
))
_ordered_quads.extend(_variants)
for agent, ver, timeout, thinking in _ordered_quads:
m = models_with_data[model][(agent, ver, timeout, thinking)]
cfg = AGENT_CONFIG[agent]
segments = _bar_segments_html(m, cfg)
n_runs = m["n_runs"]
bar_w = 56 + 8 * max(0, min(n_runs, 10) - 5)
group_bar_widths.append(bar_w)
_ver_is_default = ver == "unknown" or ver == DEFAULT_VERSIONS.get(agent)
top_label_parts = []
if not _ver_is_default:
top_label_parts.append(f'<span class="version-label">{ver}</span>')
if thinking != "-":
top_label_parts.append(f'<span class="thinking-label">\U0001f9e0 {thinking}</span>')
top_label_html = (
f'<div class="bar-top-label">{"<br>".join(top_label_parts)}</div>'
if top_label_parts else ""
)
_wurl = (weave_urls or {}).get((agent, ver, model, m.get("timeout_sec"), thinking))
_wopen = f'<a href="{_html_escape(_wurl)}" target="_blank" class="bar-link" title="View on W&B Weave">' if _wurl else ""
_wclose = "</a>" if _wurl else ""
_bar_scores = {k: round(m[k], 1) for k in ("solid", "average", "best", "ceiling")}
_bar_scores["worst"] = round(m["min"], 1)
_bar_scores["solid_raw"] = round(m["solid_raw"], 4)
_bar_scores["average_raw"] = round(m["avg_raw"], 4)
_bar_scores["best_raw"] = round(m["best_raw"], 4)
_bar_scores["ceiling_raw"] = round(m["ceiling_raw"], 4)
_bar_scores["worst_raw"] = round(m["min_raw"], 4)
_bar_scores_json = json.dumps(_bar_scores, separators=(",", ":"))
_tok_in = int(round(m.get("tokens_in_total") or 0))
_tok_out = int(round(m.get("tokens_out_total") or 0))
_tok_total = int(round(m.get("tokens_total") or (_tok_in + _tok_out)))
_tok_runs = int(m.get("tokens_runs") or n_runs)
_cost_total = float(m.get("cost_total_usd") or 0)
_cost_runs = int(m.get("cost_runs") or 0)
_agent_version = "" if ver == "unknown" else ver
_agent_badge = _agent_badge_html(agent, cfg["name"])
bars_html.append(f'''
<div class="bar-wrapper" data-agent="{agent}" data-runs="{n_runs}" data-bar-scores='{_bar_scores_json}'
data-agent-name="{_html_escape(cfg["name"])}" data-agent-version="{_html_escape(_agent_version)}"
data-token-input="{_tok_in}" data-token-output="{_tok_out}" data-token-total="{_tok_total}" data-token-runs="{_tok_runs}"
data-cost-total="{_cost_total:.6f}" data-cost-runs="{_cost_runs}" draggable="true">
{top_label_html}
{_wopen}<div class="bar" data-agent="{agent}" style="width: {bar_w}px;">
{segments}
<div class="bar-bottom-label">{_agent_badge}</div>
</div>{_wclose}
</div>''')
group_w = (sum(group_bar_widths) + max(0, len(group_bar_widths) - 1) * 8) if group_bar_widths else 0
if not group_bar_widths:
continue # Skip models with no bars (agent not in AGENT_CONFIG)
model_group_widths.append(group_w)
display = MODEL_DISPLAY.get(model, model)
model_groups_html.append(f'''
<div class="model-group" data-model="{_html_escape(model)}" data-width="{group_w}" data-orig-order="{_model_idx}" data-scores='{_scores_json}'>
<div class="model-highlight-box" aria-hidden="true"></div>
<div class="model-label">{display}</div>
<div class="bars-row">
{"".join(bars_html)}
</div>
</div>''')
# Minimum chart width based on actual bar content
n_groups = len(model_group_widths)
chart_min_w = (
sum(model_group_widths) + max(0, n_groups - 1) * 64 + 2 * 48
) if n_groups else 400