Close native formula coverage gaps and measure the frozen corpus

.github/workflows/corpus.yml

@@ -0,0 +1,108 @@
+name: Frozen formula corpus
+
+on:
+  workflow_dispatch:
+    inputs:
+      baseline:
+        description: Baseline commit or branch to compare on the same runner
+        required: true
+        default: main
+  pull_request:
+    paths:
+      - .github/workflows/corpus.yml
+
+permissions:
+  contents: read
+
+concurrency:
+  group: corpus-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  compare:
+    name: Frozen Enron before and after
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    env:
+      CARGO_INCREMENTAL: "0"
+    steps:
+      - name: Set runner-local evidence directories
+        run: |
+          echo "CORPUS_TARGETS=$RUNNER_TEMP/omasheets-corpus-targets" >> "$GITHUB_ENV"
+          echo "CORPUS_EVIDENCE=$RUNNER_TEMP/omasheets-corpus-evidence" >> "$GITHUB_ENV"
+          echo "CORPUS_DATA=$RUNNER_TEMP/omasheets-corpus-data" >> "$GITHUB_ENV"
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: ${{ github.event.pull_request.head.sha || github.sha }}
+          path: candidate
+          persist-credentials: false
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: ${{ github.event.pull_request.base.sha || inputs.baseline }}
+          path: baseline
+          persist-credentials: false
+      - name: Install the minimum toolchain and archive extractor
+        run: |
+          sudo apt-get update
+          sudo apt-get install --yes p7zip-full
+          rustup toolchain install 1.88.0 --profile minimal --no-self-update
+          rustup default 1.88.0
+      - name: Fetch the registered archive and preserve the frozen denominator
+        working-directory: candidate
+        run: |
+          cmp corpus/sources/enron-figshare.jsonl ../baseline/corpus/sources/enron-figshare.jsonl
+          cmp corpus/sources/enron-figshare.json ../baseline/corpus/sources/enron-figshare.json
+          python scripts/fetch_corpus.py corpus/sources/enron-figshare.json "$CORPUS_DATA"
+          mkdir -p "$CORPUS_EVIDENCE/before" "$CORPUS_EVIDENCE/after"
+      - name: Build the baseline scorer
+        working-directory: baseline
+        run: cargo build --locked --release --no-default-features -p omasheets-corpus --target-dir "$CORPUS_TARGETS/baseline"
+      - name: Verify and score the baseline alone
+        working-directory: candidate
+        run: |
+          "$CORPUS_TARGETS/baseline/release/omasheets-corpus" verify corpus/sources/enron-figshare.jsonl "$CORPUS_DATA/enron-figshare" > "$CORPUS_EVIDENCE/verify.json"
+          python scripts/performance.py run --name frozen-corpus-before --timeout 900 --output "$CORPUS_EVIDENCE/before/performance.json" -- "$CORPUS_TARGETS/baseline/release/omasheets-corpus" score corpus/sources/enron-figshare.jsonl "$CORPUS_DATA/enron-figshare" "$CORPUS_EVIDENCE/before/score.json" --timeout-seconds 30
+          python scripts/update_corpus_summary.py --score "$CORPUS_EVIDENCE/before/score.json" --performance "$CORPUS_EVIDENCE/before/performance.json" --manifest corpus/sources/enron-figshare.jsonl --baseline-summary corpus/sources/enron-figshare.score-summary.json --summary "$CORPUS_EVIDENCE/before/summary.json" --delta "$CORPUS_EVIDENCE/before/delta.json" --engine-commit "$(git -C ../baseline rev-parse HEAD)" --runner "GitHub ubuntu-latest x86_64; native-only build; sequential baseline then candidate; hosted runner, not Omarchy hardware" --note "Formualizer candidate lane disabled for this native compatibility iteration; only the owned lane is measured."
+      - name: Build the candidate scorer
+        working-directory: candidate
+        run: cargo build --locked --release --no-default-features -p omasheets-corpus --target-dir "$CORPUS_TARGETS/candidate"
+      - name: Score the candidate alone and report aggregate changes
+        working-directory: candidate
+        run: |
+          python scripts/performance.py run --name frozen-corpus-after --timeout 900 --output "$CORPUS_EVIDENCE/after/performance.json" -- "$CORPUS_TARGETS/candidate/release/omasheets-corpus" score corpus/sources/enron-figshare.jsonl "$CORPUS_DATA/enron-figshare" "$CORPUS_EVIDENCE/after/score.json" --timeout-seconds 30
+          python scripts/update_corpus_summary.py --score "$CORPUS_EVIDENCE/after/score.json" --performance "$CORPUS_EVIDENCE/after/performance.json" --manifest corpus/sources/enron-figshare.jsonl --baseline-summary "$CORPUS_EVIDENCE/before/summary.json" --summary "$CORPUS_EVIDENCE/after/summary.json" --delta "$CORPUS_EVIDENCE/after/delta.json" --runner "GitHub ubuntu-latest x86_64; native-only build; sequential baseline then candidate; hosted runner, not Omarchy hardware" --note "Formualizer candidate lane disabled for this native compatibility iteration; only the owned lane is measured."
+          python - <<'PY'
+          import json
+          import os
+          from pathlib import Path
+
+          root = Path(os.environ["CORPUS_EVIDENCE"])
+          before_report = json.loads((root / "before/summary.json").read_text())
+          before = before_report["owned_summary"]
+          after_report = json.loads((root / "after/summary.json").read_text())
+          after = after_report["owned_summary"]
+          # Retain aggregate evidence in logs if artifact storage is unavailable.
+          print("FROZEN_CORPUS_BEFORE " + json.dumps(before_report, sort_keys=True))
+          print("FROZEN_CORPUS_AFTER " + json.dumps(after_report, sort_keys=True))
+          print("FROZEN_CORPUS_DELTA " + (root / "after/delta.json").read_text().replace("\n", " "))
+          assert "syntax_failure_tokens" in after, "Candidate scorer lacks the new diagnostic schema"
+          keys = ["opened", "formula_cells_observed", "formula_cells_loaded", "formula_cells_compared", "stored_values_matched", "stored_values_mismatched", "formula_parse_rate", "stored_value_match_rate"]
+          lines = ["| Metric | Before | After |", "|---|---:|---:|"]
+          lines.extend(f"| {key} | {before[key]} | {after[key]} |" for key in keys)
+          with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as output:
+              output.write("\n".join(lines) + "\n")
+          assert after["opened"] == before["opened"], "Workbook-open denominator changed"
+          assert after["formula_cells_observed"] == before["formula_cells_observed"], "Formula denominator changed"
+          assert after["formula_cells_loaded"] >= before["formula_cells_loaded"], "Formula compilation regressed"
+          assert after["stored_values_matched"] >= before["stored_values_matched"], "Stored-value matches regressed"
+          PY
+      - name: Upload aggregate evidence only
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: frozen-formula-corpus
+          if-no-files-found: warn
+          path: |
+            ${{ runner.temp }}/omasheets-corpus-evidence/before/summary.json
+            ${{ runner.temp }}/omasheets-corpus-evidence/after/summary.json
+            ${{ runner.temp }}/omasheets-corpus-evidence/after/delta.json

corpus/README.md

@@ -124,6 +124,15 @@ access terms. See `corpus/sources/README.md` for what is registered.
 
 ## Corpus policy
 
+The **Frozen formula corpus** Actions workflow scores a baseline and candidate
+sequentially on the same hosted runner, in separate build directories, using
+the registered archive and the
+unchanged 1,000-workbook manifest. Run it manually with a baseline revision;
+changes to the workflow itself also exercise it in a pull request. Only
+aggregate before/after summaries are uploaded. Syntax failures include a fixed
+token-class histogram to guide parser work without exposing formula contents.
+This is corpus compatibility evidence, not target-desktop performance evidence.
+
 - Record the upstream corpus name, retrieval date, license or access terms, and
   sampling method beside every frozen manifest.
 - Do not commit source workbooks, extracted cell contents, local paths, or model

corpus/sources/README.md

@@ -28,8 +28,44 @@ the frozen sample for the current engine (engine commit, wall time,
 process-tree peak memory, both lane summaries, the owned lane's
 unsupported-function distribution and the failure kinds), and
 `enron-figshare.score-delta.json` records the owned lane against the
-baseline engine that first scored the sample. Both are aggregate only. Scored
-on 2026-09-01 on a 4-vCPU Linux container:
+baseline engine named in that delta. Both are aggregate only; the JSON records
+the exact engine revisions and runners.
+
+The latest comparison was scored on 2026-09-08, sequentially on one GitHub
+`ubuntu-latest` runner with separate build directories. Both revisions used
+the same frozen 1,000-workbook manifest and exposed 924,235 formula cells.
+[Workflow evidence](https://github.com/tcballard/OmaSheets/actions/runs/34254502652).
+
+| Owned engine lane | Baseline (`ecf0036`) | Formula coverage (`12d1c8c`) |
+|---|---:|---:|
+| Workbooks opened | 996 / 1,000 | 996 / 1,000 |
+| Formula cells observed | 924,235 | 924,235 |
+| Loaded and compared | 825,016 (89.26%) | 829,529 (89.75%) |
+| Stored values matched | 823,932 | 828,437 |
+| Match rate of compared | 99.87% | 99.87% |
+| Stored values mismatched | 1,084 | 1,092 |
+| Not compiled | 99,219 | 94,706 |
+
+This adds 4,513 compiled formulas and 4,505 stored-value matches without
+changing the sample, comparison tolerance or denominator. The eight additional
+mismatches remain visible. The aggregate cannot attribute them to individual
+formulas or establish that every previously matched cell is unchanged.
+
+Parser work also reveals later failures: the 10,791 syntax classifications
+are gone, while invalid references rise from 4,260 to 10,551 and unsupported
+function classifications rise from 26,617 to 28,967. `OFFSET` now accounts for
+3,619 first failures; previously some of these stopped at a syntax or name
+error. A lower count in an early failure class does not mean every affected
+formula now compiles.
+
+The remaining first-failure groups are 47,007 external workbook references,
+28,967 unsupported functions, 10,551 invalid references, 5,767 unsupported
+name definitions, 2,292 cycles and 122 unknown names. Within unsupported
+functions, 21,737 cells call proprietary add-ins. Ordinary gaps include
+`OFFSET`, locale-sensitive `DATEVALUE`, volatile functions and `INDIRECT`.
+External/add-in execution and hidden clock or random state remain refused.
+These are compatibility measurements, not target-desktop performance claims.
+
 
 Generate both files from a measured schema-2 scorer report with
 `scripts/update_corpus_summary.py`. The command validates the manifest digest,
@@ -49,6 +85,12 @@ python scripts/update_corpus_summary.py \
   --runner "linux x86_64, eight-core baseline, cold run"
 ```
 
+### Historical comparison
+
+The following 2026-09-01 run predates importer fixes that exposed additional
+formula cells. Its smaller denominator makes the raw rate unsuitable for a
+direct comparison with the current table.
+
 | Owned M0 engine lane | Baseline (`08f38d1`) | After formula gaps (`0eccaff`) |
 |---|---|---|
 | Workbooks opened | 971 / 1000 | 971 / 1000 |

corpus/sources/enron-figshare.score-delta.json

@@ -1,86 +1,108 @@
 {
-  "schema": 1,
-  "source": "enron-figshare",
+  "baseline_engine_commit": "ecf00367004ccbeb56a736fd0aef30dd4e45c1ba",
+  "engine_commit": "12d1c8c69522ee7ee81ce3d7090a1bba60cd2ac5",
   "manifest": "enron-figshare.jsonl",
   "manifest_sha256": "2f5ea94c02ae3779206d6ca1b1da9e77002450bc070fa968ca566cfd526ff80d",
-  "baseline_engine_commit": "3d32520f91d51b63d327202cca82920125e7d36d",
-  "engine_commit": "71f21c5e285c7ec98543b498088c17ee648ef858",
-  "scored": "2026-09-02",
+  "notes": [
+    "Frozen 1,000-workbook manifest and 924,235-formula denominator unchanged.",
+    "Workflow evidence: https://github.com/tcballard/OmaSheets/actions/runs/34254502652",
+    "First-failure categories move as parsing advances: syntax 10,791 -> 0, invalid references 4,260 -> 10,551, unsupported functions 26,617 -> 28,967. This is not a claim that every former syntax failure now evaluates.",
+    "4,513 more formulas compiled; 4,505 more stored-value matches; total mismatches 1,084 -> 1,092. Aggregate evidence cannot assign the eight additional mismatches to individual formulas.",
+    "No performance comparison to Omarchy desktop hardware is implied."
+  ],
   "owned_lane": {
-    "opened": {
-      "before": 996,
-      "after": 996
+    "comparison_coverage": {
+      "after": 0.8975303899982148,
+      "before": 0.8926474327416728
     },
     "failed": {
-      "before": 4,
-      "after": 4
-    },
-    "timed_out": {
-      "before": 0,
-      "after": 0
+      "after": 4,
+      "before": 4
     },
-    "formula_cells_observed": {
-      "before": 924235,
-      "after": 924235
+    "formula_cells_compared": {
+      "after": 829529,
+      "before": 825016
     },
     "formula_cells_loaded": {
-      "before": 817225,
-      "after": 825016
+      "after": 829529,
+      "before": 825016
     },
-    "formula_cells_compared": {
-      "before": 817225,
-      "after": 825016
+    "formula_cells_observed": {
+      "after": 924235,
+      "before": 924235
     },
-    "stored_values_matched": {
-      "before": 816141,
-      "after": 823932
+    "formula_parse_rate": {
+      "after": 0.8975303899982148,
+      "before": 0.8926474327416728
     },
-    "stored_values_mismatched": {
-      "before": 1084,
-      "after": 1084
+    "opened": {
+      "after": 996,
+      "before": 996
     },
-    "unsupported_formulas": {
-      "before": 107010,
-      "after": 99219
+    "peak_rss_bytes_max": {
+      "after": 142831616,
+      "before": 142925824
     },
-    "formula_parse_rate": {
-      "before": 0.8842177584705189,
-      "after": 0.8926474327416728
+    "stored_value_match_rate": {
+      "after": 0.9986835903265588,
+      "before": 0.9986860860880273
     },
-    "comparison_coverage": {
-      "before": 0.8842177584705189,
-      "after": 0.8926474327416728
+    "stored_values_matched": {
+      "after": 828437,
+      "before": 823932
     },
-    "stored_value_match_rate": {
-      "before": 0.9986735599131206,
-      "after": 0.9986860860880273
+    "stored_values_mismatched": {
+      "after": 1092,
+      "before": 1084
     },
-    "peak_rss_bytes_max": {
-      "before": 116269056,
-      "after": 116408320
+    "timed_out": {
+      "after": 0,
+      "before": 0
+    },
+    "unsupported_formulas": {
+      "after": 94706,
+      "before": 99219
     }
   },
   "owned_lane_added": {
-    "workbooks_with_skipped_sheets": {
-      "before": 23,
-      "after": 23
-    },
     "skipped_sheets": {
-      "before": 48,
-      "after": 48
+      "after": 48,
+      "before": 48
+    },
+    "workbooks_with_skipped_sheets": {
+      "after": 23,
+      "before": 23
     }
   },
+  "process_tree_peak_rss_bytes": {
+    "after": 1493118976,
+    "before": 1502441472
+  },
+  "resolved_defects": [
+    "Array constants in aggregate, elementwise and lookup inputs",
+    "Sheet-qualified defined names and known modern function prefixes",
+    "Qualified and deleted reference range endpoints"
+  ],
+  "resolved_functions": [
+    "COVAR",
+    "IRR",
+    "NORMSDIST",
+    "PV"
+  ],
+  "resolved_open_failures": {},
+  "schema": 1,
+  "scored": "2026-09-08",
+  "source": "enron-figshare",
   "unsupported_reasons": {
-    "before": {
-      "cycle": 2286,
+    "after": {
+      "cycle": 2292,
       "external_reference": 47007,
-      "invalid_reference": 12051,
-      "syntax": 10791,
+      "invalid_reference": 10551,
       "unknown_name": 122,
-      "unsupported_function": 26617,
-      "unsupported_name": 8136
+      "unsupported_function": 28967,
+      "unsupported_name": 5767
     },
-    "after": {
+    "before": {
       "cycle": 2286,
       "external_reference": 47007,
       "invalid_reference": 4260,
@@ -90,20 +112,8 @@
       "unsupported_name": 8136
     }
   },
-  "resolved_functions": [],
-  "resolved_defects": [
-    "a sheet-qualified #REF! (a deleted cell on another sheet) was refused as an invalid reference instead of compiling to the #REF! error"
-  ],
-  "resolved_open_failures": {},
   "wall_seconds": {
-    "before": 139.0,
-    "after": 208.0
-  },
-  "process_tree_peak_rss_bytes": {
-    "before": 1539121152,
-    "after": 1544372224
-  },
-  "notes": [
-    "Timing evidence in this entry comes from a restarted container that is about 35% slower than the one the previous entries ran on: the candidate lane, which this engine change does not touch, went from 44 s to 60 s of summed import time. Per-cell results are identical across two runs on the new container; compare cell counts across entries, not wall seconds."
-  ]
+    "after": 117.606367,
+    "before": 118.763427
+  }
 }