abi/scale_validation_results.json at master · Yoder23/abi · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
[
  {
    "tier": 1,
    "name": "S1_domain_on_pretrained_gpt2",
    "passed": true,
    "metrics": {
      "backbone": "pretrained GPT-2-medium",
      "backbone_params_M": 354.8,
      "d_model": 1024,
      "d_abi": 256,
      "domain_corpus": "Python source code (workspace .py files)",
      "general_corpus": "WikiText-2 (public benchmark)",
      "ppl_no_domain": 20.05,
      "ppl_with_domain": 7.15,
      "improvement_pct": 64.36,
      "threshold_pct": 3.0
    },
    "evidence": "Pretrained GPT-2-medium (354.8M backbone, d_model=1024, d_abi=256). Python domain module (trained 500 steps, backbone frozen): PPL 20.1 \u2192 7.1 (+64.4%). Threshold >3%: True",
    "note": "ABI interface learned on top of frozen GPT-2-medium's pretrained representations. The domain module specialises signals that GPT-2 already understands as Python, operating purely in the fixed d_abi=256 space."
  },
  {
    "tier": 2,
    "name": "S2_zero_shot_after_backbone_update",
    "passed": true,
    "metrics": {
      "update_corpus": "WikiText-2 (public benchmark, external)",
      "update_steps": 1000,
      "alpha_stability": 1.0,
      "proj_out_frozen": true,
      "abi_cos_sim_post_update": 0.6388,
      "abi_ratio_vs_random": 342.7,
      "ppl_updated_base": 30.12,
      "ppl_updated_with": 17.55,
      "improvement_pct": 41.74,
      "source_imp_pct": 64.36,
      "threshold_pct": 3.0
    },
    "evidence": "Backbone fine-tuned on WikiText-2 (1000 steps, 5\u00d7 production test). ABI stability: alpha=1.0, proj_out frozen. Post-update ABI cos_sim=0.6388 (343x random). Zero-shot: 30.1 \u2192 17.5 (+41.7%). Threshold >3%: True",
    "note": "CORE PRODUCTION TEST AT SCALE. Backbone updated 1000 steps on WikiText-2 (external public benchmark, distinct distribution from Python domain). Python domain module reused zero-shot \u2014 no domain fine-tuning. This is the peer reviewer's explicit requested experiment."
  },
  {
    "tier": 3,
    "name": "S3_transfer_efficacy_vs_native",
    "passed": true,
    "metrics": {
      "imp_zero_shot_pct": 41.74,
      "imp_native_pct": 63.96,
      "transfer_efficacy_pct": 65.3,
      "threshold_efficacy_pct": 50.0,
      "ppl_updated_base": 30.12,
      "ppl_updated_with": 17.55,
      "ppl_native_base": 21.68,
      "ppl_native_with": 7.81
    },
    "evidence": "Zero-shot efficacy: 65% (+41.7% zero-shot vs +64.0% native cold-start). Threshold \u226550%: True",
    "note": "Transfer efficacy at 354M scale with WikiText-2 backbone update. Peer reviewer bar: \u226550\u201370% = 'people will pay attention', \u226590% = 'this changes assumptions'. "
  },
  {
    "tier": 4,
    "name": "S4_rollout_stability",
    "passed": true,
    "metrics": {
      "window_ppls": [
        1.3,
        1.02,
        1.01,
        1.01
      ],
      "last_first_ratio": 0.78,
      "threshold": 3.0
    },
    "evidence": "Window PPLs: [1.3, 1.0, 1.0, 1.0]. Last/First = 0.78 (< 3.0 = stable). Domain active.",
    "note": "Autoregressive generation stability at 354M scale with domain active."
  },
  {
    "tier": 5,
    "name": "S5_scale_claim_assessment",
    "passed": true,
    "metrics": {
      "model": "gpt2-medium",
      "backbone_params_M": 354.8,
      "general_corpus": "WikiText-2 (public benchmark)",
      "domain_corpus": "Python source code",
      "update_steps": 1000,
      "transfer_efficacy_pct": 65.3,
      "claim_level": "STRONG \u2014 validated (above peer minimum bar)",
      "all_4_tests_pass": true
    },
    "evidence": "GPT-2-medium (354.8M) | WikiText-2 | 1000-step update | S1=True S2=True S3=True S4=True | Transfer efficacy: 65.3%",
    "note": "STRONG \u2014 validated (above peer minimum bar)"
  }
]