From 18b23229c4c2a1724f90d608d0c4fbb0c8a2eb13 Mon Sep 17 00:00:00 2001
From: Oscar Villavicencio <9220505+odvcencio@users.noreply.github.com>
Date: Sun, 23 Aug 2026 14:30:28 -0700
Subject: [PATCH 1/2] add: Add parity probes, retry profiles, and diagnostic
token forks
- Add locked C oracle parity checks for Doxygen, Templ, Wolfram, WGSL, and Objective-C inputs.
- Introduce benchmark cases for incremental recovery deletion and early newline edits.
- Provide diagnostic token source forking to capture parser state and range digests during retries.
- Guard against cyclic node references in trees produced by repeated concurrent parses.
- Record Doxygen normalization blocker receipts and expected dispatch rewrite counts.
Buckley-Change-Hash: sha256:9f86426c9a11d6035e0f27923664d3c8dea2ae32d0e89f688c26220058a61e19
Buckley-Change-Stats: files=20 insertions=4265 deletions=0 binaries=0
---
...benchmark_go_canonical_incremental_test.go | 113 +++
cgo_harness/doxygen_probe_parity_test.go | 103 +++
cgo_harness/p14_profile_test.go | 64 ++
cgo_harness/p25b_issue454_137k_parity_test.go | 85 +++
cgo_harness/p25v_retry_paired_test.go | 452 ++++++++++++
cgo_harness/p25w_exact_language_route_test.go | 150 ++++
cgo_harness/templ_next_live_probe_test.go | 388 ++++++++++
cgo_harness/wgsl_next_live_probe_test.go | 440 ++++++++++++
cgo_harness/wolfram_retirement_parity_test.go | 146 ++++
diagnostic_token_source_fork.go | 88 +++
diagnostic_token_source_fork_test.go | 165 +++++
...-normalization-blocker-receipt-proposal.md | 129 ++++
doxygen_probe_bridge.go | 21 +
doxygen_probe_test.go | 254 +++++++
grammars/c_token_source_diagnostic_fork.go | 101 +++
.../c_token_source_diagnostic_fork_test.go | 308 ++++++++
p17_attribution_test.go | 671 ++++++++++++++++++
p25b_issue454_profile_test.go | 184 +++++
p25c_objc_probe_test.go | 324 +++++++++
...o_materialization_cycle_regression_test.go | 79 +++
20 files changed, 4265 insertions(+)
create mode 100644 cgo_harness/doxygen_probe_parity_test.go
create mode 100644 cgo_harness/p14_profile_test.go
create mode 100644 cgo_harness/p25b_issue454_137k_parity_test.go
create mode 100644 cgo_harness/p25v_retry_paired_test.go
create mode 100644 cgo_harness/p25w_exact_language_route_test.go
create mode 100644 cgo_harness/templ_next_live_probe_test.go
create mode 100644 cgo_harness/wgsl_next_live_probe_test.go
create mode 100644 cgo_harness/wolfram_retirement_parity_test.go
create mode 100644 diagnostic_token_source_fork.go
create mode 100644 diagnostic_token_source_fork_test.go
create mode 100644 docs/doxygen-normalization-blocker-receipt-proposal.md
create mode 100644 doxygen_probe_bridge.go
create mode 100644 doxygen_probe_test.go
create mode 100644 grammars/c_token_source_diagnostic_fork.go
create mode 100644 grammars/c_token_source_diagnostic_fork_test.go
create mode 100644 p17_attribution_test.go
create mode 100644 p25b_issue454_profile_test.go
create mode 100644 p25c_objc_probe_test.go
create mode 100644 parser_go_materialization_cycle_regression_test.go
diff --git a/cgo_harness/benchmark_go_canonical_incremental_test.go b/cgo_harness/benchmark_go_canonical_incremental_test.go
index ef042714d..f6a53585e 100644
--- a/cgo_harness/benchmark_go_canonical_incremental_test.go
+++ b/cgo_harness/benchmark_go_canonical_incremental_test.go
@@ -4,6 +4,7 @@ package cgoharness
import (
"bytes"
+ "context"
"crypto/sha256"
_ "embed"
"encoding/hex"
@@ -12,6 +13,7 @@ import (
"io"
"os"
"path/filepath"
+ "runtime/pprof"
"strings"
"testing"
"time"
@@ -298,6 +300,117 @@ func BenchmarkParityGoCanonicalIncremental(b *testing.B) {
}
}
+// BenchmarkP25tRecoveryDeletionForwardOnly repeats the forward recovery edit.
+// Keep this temporary benchmark outside the canonical tree.
+func BenchmarkP25tRecoveryDeletionForwardOnly(b *testing.B) {
+ cases := loadCanonicalGoIncrementalCases(b)
+ var tc *canonicalGoIncrementalCase
+ for i := range cases {
+ if cases[i].spec.Name == "recovery_deletion" {
+ tc = &cases[i]
+ break
+ }
+ }
+ if tc == nil {
+ b.Fatal("canonical recovery_deletion case is missing")
+ }
+ goLang := canonicalIncrementalGoLanguage(b, tc.spec.Language)
+ cLang := canonicalIncrementalCLanguage(b, tc.spec.Language)
+ goParser := gotreesitter.NewParser(goLang)
+ cParser := sitter.NewParser()
+ defer cParser.Close()
+ if err := cParser.SetLanguage(cLang); err != nil {
+ b.Fatalf("recovery_deletion set pinned Go C reference: %v", err)
+ }
+ direction := tc.directions()[0]
+ admitCanonicalGoIncrementalDirection(b, tc.spec.Name, direction, goParser, cParser, goLang)
+
+ parser := gotreesitter.NewParser(goLang)
+ tree, err := parser.Parse(tc.source)
+ requireCanonicalGoIncrementalTree(b, tree, tc.source, tc.spec.Name+" initial Go", err)
+ b.ReportAllocs()
+ b.SetBytes(int64((len(tc.source) + len(tc.edited)) / 2))
+ b.ResetTimer()
+ b.StopTimer()
+ var totals realCorpusIncrementalProfileTotals
+ iterationLogPath := os.Getenv("P25T_ITERATION_LOG")
+ var iterationLog *os.File
+ if iterationLogPath != "" {
+ iterationLog, err = os.Create(iterationLogPath)
+ if err != nil {
+ b.Fatalf("create P25t iteration log: %v", err)
+ }
+ defer iterationLog.Close()
+ }
+ labels := pprof.Labels("phase", "incremental")
+ for i := 0; i < b.N; i++ {
+ if i > 0 {
+ releaseCanonicalGoTree(tree)
+ tree, err = parser.Parse(tc.source)
+ requireCanonicalGoIncrementalTree(b, tree, tc.source, tc.spec.Name+" reset Go", err)
+ }
+ var editElapsed, parseElapsed time.Duration
+ var profile gotreesitter.IncrementalParseProfile
+ var newTree *gotreesitter.Tree
+ var oldTree *gotreesitter.Tree
+ pprof.Do(context.Background(), labels, func(context.Context) {
+ b.StartTimer()
+ editStart := time.Now()
+ tree.Edit(tc.forward)
+ editElapsed = time.Since(editStart)
+ oldTree = tree
+ parseStart := time.Now()
+ newTree, profile, err = parser.ParseIncrementalProfiled(tc.edited, oldTree)
+ parseElapsed = time.Since(parseStart)
+ requireCanonicalGoIncrementalTree(b, newTree, tc.edited, tc.spec.Name+" timed Go", err)
+ b.StopTimer()
+ })
+ totals.addEdit(editElapsed)
+ totals.addParseWall(parseElapsed)
+ totals.add(profile)
+ if newTree != oldTree {
+ releaseCanonicalGoTree(oldTree)
+ }
+ tree = newTree
+ if iterationLog != nil {
+ _, _ = fmt.Fprintf(iterationLog,
+ "iteration=%d edit_ns=%d parse_wall_ns=%d reuse_ns=%d reparse_ns=%d reused_subtrees=%d reused_bytes=%d new_nodes=%d tokens=%d max_stacks=%d single_iters=%d multi_iters=%d parser_loop_ns=%d token_next_ns=%d result_tree_build_ns=%d result_finalize_root_ns=%d result_compatibility_ns=%d gss_alloc=%d gss_retained=%d gss_dropped=%d parent_alloc=%d parent_retained=%d parent_dropped=%d leaf_alloc=%d leaf_retained=%d leaf_dropped=%d retry_attempts=%d retry_adopted=%t old_tree_reuse=%t\n",
+ i+1,
+ editElapsed.Nanoseconds(),
+ parseElapsed.Nanoseconds(),
+ profile.ReuseCursorNanos,
+ profile.ReparseNanos,
+ profile.ReusedSubtrees,
+ profile.ReusedBytes,
+ profile.NewNodesAllocated,
+ profile.TokensConsumed,
+ profile.MaxStacksSeen,
+ profile.SingleStackIterations,
+ profile.MultiStackIterations,
+ profile.ParserLoopNanos,
+ profile.TokenNextNanos,
+ profile.ResultTreeBuildNanos,
+ profile.ResultFinalizeRootNanos,
+ profile.ResultCompatibilityNanos,
+ profile.GSSNodesAllocated,
+ profile.GSSNodesRetained,
+ profile.GSSNodesDroppedSameToken,
+ profile.ParentNodesAllocated,
+ profile.ParentNodesRetained,
+ profile.ParentNodesDroppedSameToken,
+ profile.LeafNodesAllocated,
+ profile.LeafNodesRetained,
+ profile.LeafNodesDroppedSameToken,
+ profile.AcceptedErrorRetryAttempts,
+ profile.AcceptedErrorRetryAdopted,
+ profile.OldTreeReuseRoute,
+ )
+ }
+ }
+ totals.report(b, b.N)
+ releaseCanonicalGoTree(tree)
+}
+
func loadCanonicalGoIncrementalCases(tb testing.TB) []canonicalGoIncrementalCase {
tb.Helper()
decoder := json.NewDecoder(bytes.NewReader(canonicalGoIncrementalManifestJSON))
diff --git a/cgo_harness/doxygen_probe_parity_test.go b/cgo_harness/doxygen_probe_parity_test.go
new file mode 100644
index 000000000..ec39be91d
--- /dev/null
+++ b/cgo_harness/doxygen_probe_parity_test.go
@@ -0,0 +1,103 @@
+//go:build cgo && treesitter_c_parity
+
+package cgoharness
+
+import (
+ "crypto/sha256"
+ "os"
+ "path/filepath"
+ "testing"
+
+ gotreesitter "github.com/odvcencio/gotreesitter"
+ "github.com/odvcencio/gotreesitter/grammars"
+ "github.com/odvcencio/gotreesitter/internal/benchfixtures"
+ sitter "github.com/tree-sitter/go-tree-sitter"
+)
+
+func TestDoxygenRegisteredWitnessesLockedCDeepParity(t *testing.T) {
+ t.Setenv("GTS_DISPATCHER_CENSUS", "1")
+ entry, ok := parityEntriesByName["doxygen"]
+ if !ok {
+ t.Fatal("missing Doxygen grammar entry")
+ }
+ goLang := entry.Language()
+ cLang, err := COracleLanguage("doxygen")
+ if err != nil {
+ t.Fatal(err)
+ }
+ witnesses := []struct {
+ name, path, source string
+ }{
+ {name: "a0_CMakeLists", path: "../testdata/dispatcher_census_a0/doxygen/medium__CMakeLists.txt"},
+ {name: "a0_metrics", path: "../testdata/dispatcher_census_a0/doxygen/medium__metrics.py"},
+ {name: "a0_example_cfg", path: "../testdata/dispatcher_census_a0/doxygen/small__example.cfg"},
+ {name: "registered_smoke", source: grammars.ParseSmokeSample("doxygen")},
+ {name: "historical_childless_error", source: "/** Adds all words in \\a s to document \\a doc with weight \\a wfd */"},
+ {name: "historical_recovered_document", source: "/**\n * @param {int} value\n * @brief Example\n */"},
+ }
+ for _, witness := range witnesses {
+ witness := witness
+ t.Run(witness.name, func(t *testing.T) {
+ source := []byte(witness.source)
+ if witness.path != "" {
+ var err error
+ source, err = os.ReadFile(filepath.Join("..", "testdata", "dispatcher_census_a0", "doxygen", filepath.Base(witness.path)))
+ if err != nil {
+ t.Fatal(err)
+ }
+ }
+ t.Logf("source_sha256=%x bytes=%d", sha256.Sum256(source), len(source))
+
+ rawParser := gotreesitter.NewParser(goLang)
+ rawParser.SetAdmissionCandidateRoute(false)
+ raw, err := rawParser.ParseNoResultCompatibilityBenchmarkOnly(source)
+ if err != nil {
+ t.Fatalf("raw parse: %v", err)
+ }
+ t.Cleanup(raw.Release)
+ productionParser := gotreesitter.NewParser(goLang)
+ productionParser.SetAdmissionCandidateRoute(false)
+ production, err := productionParser.Parse(source)
+ if err != nil {
+ t.Fatalf("production parse: %v", err)
+ }
+ t.Cleanup(production.Release)
+
+ cParser := sitter.NewParser()
+ t.Cleanup(cParser.Close)
+ if err := cParser.SetLanguage(cLang); err != nil {
+ t.Fatal(err)
+ }
+ cTree := cParser.Parse(source, nil)
+ if cTree == nil || cTree.RootNode() == nil {
+ t.Fatal("C oracle returned a nil tree")
+ }
+ t.Cleanup(cTree.Close)
+ cDigest, err := COracleDeepDigest(cTree)
+ if err != nil {
+ t.Fatalf("C digest: %v", err)
+ }
+ rawInspection, err := benchfixtures.InspectGoTree(raw.RootNode(), goLang)
+ if err != nil {
+ t.Fatalf("raw digest: %v", err)
+ }
+ productionInspection, err := benchfixtures.InspectGoTree(production.RootNode(), goLang)
+ if err != nil {
+ t.Fatalf("production digest: %v", err)
+ }
+ t.Logf("raw_digest=%s production_digest=%s c_digest=%s raw_rewrites=%d production_rewrites=%d raw_root=%s raw_error=%t production_root=%s production_error=%t", rawInspection.SHA256, productionInspection.SHA256, cDigest, raw.ParseRuntime().NormalizationNodesRewritten, production.ParseRuntime().NormalizationNodesRewritten, raw.RootNode().Type(goLang), raw.RootNode().HasError(), production.RootNode().Type(goLang), production.RootNode().HasError())
+ if diff := FirstDivergenceDumpV1(production.RootNode(), goLang, cTree.RootNode()); diff != nil {
+ t.Fatalf("production diverges from locked C: %+v", diff)
+ }
+ if diff := firstLockedCTreeFlagDivergence(production.RootNode(), goLang, cTree.RootNode(), "/"+production.RootNode().Type(goLang)); diff != nil {
+ t.Fatalf("production flags diverge from locked C: %v", diff)
+ }
+ if productionInspection.SHA256 != cDigest {
+ t.Fatalf("production deep digest=%s C=%s", productionInspection.SHA256, cDigest)
+ }
+ if rawInspection.SHA256 != productionInspection.SHA256 {
+ t.Errorf("raw and production deep digests differ: raw=%s production=%s", rawInspection.SHA256, productionInspection.SHA256)
+ }
+ })
+ }
+}
diff --git a/cgo_harness/p14_profile_test.go b/cgo_harness/p14_profile_test.go
new file mode 100644
index 000000000..d1e533ea2
--- /dev/null
+++ b/cgo_harness/p14_profile_test.go
@@ -0,0 +1,64 @@
+//go:build cgo && treesitter_c_parity
+
+package cgoharness
+
+import (
+ "testing"
+
+ gotreesitter "github.com/odvcencio/gotreesitter"
+)
+
+func benchmarkP14EarlyNewlineDirection(b *testing.B, reverse bool) {
+ b.Helper()
+ cases := loadCanonicalGoIncrementalCases(b)
+ var tc canonicalGoIncrementalCase
+ for _, candidate := range cases {
+ if candidate.spec.Name == "early_newline" {
+ tc = candidate
+ break
+ }
+ }
+ if tc.spec.Name == "" {
+ b.Fatal("canonical early_newline case is missing")
+ }
+ var from, target []byte
+ var edit gotreesitter.InputEdit
+ if reverse {
+ from, target, edit = tc.edited, tc.source, tc.reverse
+ } else {
+ from, target, edit = tc.source, tc.edited, tc.forward
+ }
+ parser := gotreesitter.NewParser(canonicalIncrementalGoLanguage(b, "go"))
+ tree, err := parser.Parse(from)
+ requireCanonicalGoIncrementalTree(b, tree, from, "P14 profile initial", err)
+ defer releaseCanonicalGoTree(tree)
+
+ b.ReportAllocs()
+ b.SetBytes(int64((len(tc.source) + len(tc.edited)) / 2))
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ if i > 0 {
+ b.StopTimer()
+ releaseCanonicalGoTree(tree)
+ tree, err = parser.Parse(from)
+ requireCanonicalGoIncrementalTree(b, tree, from, "P14 profile reset", err)
+ b.StartTimer()
+ }
+ tree.Edit(edit)
+ newTree, _, err := parser.ParseIncrementalProfiled(target, tree)
+ requireCanonicalGoIncrementalTree(b, newTree, target, "P14 profile direction", err)
+ if newTree != tree {
+ tree.Release()
+ }
+ tree = newTree
+ }
+ b.StopTimer()
+}
+
+func BenchmarkP14EarlyNewlineForward(b *testing.B) {
+ benchmarkP14EarlyNewlineDirection(b, false)
+}
+
+func BenchmarkP14EarlyNewlineReverse(b *testing.B) {
+ benchmarkP14EarlyNewlineDirection(b, true)
+}
diff --git a/cgo_harness/p25b_issue454_137k_parity_test.go b/cgo_harness/p25b_issue454_137k_parity_test.go
new file mode 100644
index 000000000..1d198656b
--- /dev/null
+++ b/cgo_harness/p25b_issue454_137k_parity_test.go
@@ -0,0 +1,85 @@
+//go:build cgo && treesitter_c_parity
+
+package cgoharness
+
+import (
+ "bytes"
+ "testing"
+
+ gotreesitter "github.com/odvcencio/gotreesitter"
+ "github.com/odvcencio/gotreesitter/internal/benchfixtures"
+ sitter "github.com/tree-sitter/go-tree-sitter"
+)
+
+func TestP25BIssue454C137KiBParityBaseline(t *testing.T) {
+ source := benchfixtures.Issue454CSource()
+ site := bytes.Index(source, []byte("x0"))
+ if site < 0 {
+ t.Fatal("C edit marker is absent")
+ }
+ edited := append(append([]byte(nil), source[:site]...), source[site+1:]...)
+
+ goBase, goLang, err := parseWithGo(parityCase{name: "c", source: string(source)}, source, nil)
+ if err != nil {
+ t.Fatalf("Go base parse: %v", err)
+ }
+ defer releaseGoTree(goBase)
+ goFresh, _, err := parseWithGo(parityCase{name: "c", source: string(edited)}, edited, nil)
+ if err != nil {
+ t.Fatalf("Go fresh edited parse: %v", err)
+ }
+ defer releaseGoTree(goFresh)
+ goIncremental, _, err := parseWithGo(parityCase{name: "c", source: string(edited)}, edited, goBase)
+ if err != nil {
+ t.Fatalf("Go incremental edited parse: %v", err)
+ }
+ defer releaseGoTree(goIncremental)
+
+ cLang, err := ParityCLanguage("c")
+ if err != nil {
+ t.Fatalf("load C oracle: %v", err)
+ }
+ cParser := sitter.NewParser()
+ defer cParser.Close()
+ if err := cParser.SetLanguage(cLang); err != nil {
+ t.Fatalf("set C oracle language: %v", err)
+ }
+ cBase := cParser.Parse(source, nil)
+ if cBase == nil || cBase.RootNode() == nil {
+ t.Fatal("C base parse returned no tree")
+ }
+ defer cBase.Close()
+ cFresh := cParser.Parse(edited, nil)
+ if cFresh == nil || cFresh.RootNode() == nil {
+ t.Fatal("C fresh edited parse returned no tree")
+ }
+ defer cFresh.Close()
+
+ goBaseDigest := p25bGoDigest(t, goBase, goLang)
+ goFreshDigest := p25bGoDigest(t, goFresh, goLang)
+ goIncrementalDigest := p25bGoDigest(t, goIncremental, goLang)
+ cBaseDigest, err := COracleDeepDigest(cBase)
+ if err != nil {
+ t.Fatalf("C base digest: %v", err)
+ }
+ cFreshDigest, err := COracleDeepDigest(cFresh)
+ if err != nil {
+ t.Fatalf("C edited digest: %v", err)
+ }
+ baseDiff := FirstDivergenceDumpV1(goBase.RootNode(), goLang, cBase.RootNode())
+ freshDiff := FirstDivergenceDumpV1(goFresh.RootNode(), goLang, cFresh.RootNode())
+ incrementalDiff := FirstDivergenceDumpV1(goIncremental.RootNode(), goLang, cFresh.RootNode())
+ t.Logf("P25B_C137K bytes=%d site=%d go_base_digest=%s c_base_digest=%s go_fresh_digest=%s c_fresh_digest=%s go_incremental_digest=%s base_diff=%+v fresh_c_diff=%+v incremental_c_diff=%+v incremental_fresh_go_equal=%v go_base_error=%v go_fresh_error=%v go_incremental_error=%v c_base_error=%v c_fresh_error=%v", len(source), site, goBaseDigest, cBaseDigest, goFreshDigest, cFreshDigest, goIncrementalDigest, baseDiff, freshDiff, incrementalDiff, goIncrementalDigest == goFreshDigest, goBase.RootNode().HasError(), goFresh.RootNode().HasError(), goIncremental.RootNode().HasError(), cBase.RootNode().HasError(), cFresh.RootNode().HasError())
+ if goIncrementalDigest != goFreshDigest {
+ t.Fatalf("incremental Go digest %s differs from fresh Go digest %s", goIncrementalDigest, goFreshDigest)
+ }
+}
+
+func p25bGoDigest(t *testing.T, tree *gotreesitter.Tree, lang *gotreesitter.Language) string {
+ t.Helper()
+ inspection, err := benchfixtures.InspectGoTree(tree.RootNode(), lang)
+ if err != nil {
+ t.Fatalf("inspect Go tree: %v", err)
+ }
+ return inspection.SHA256
+}
diff --git a/cgo_harness/p25v_retry_paired_test.go b/cgo_harness/p25v_retry_paired_test.go
new file mode 100644
index 000000000..a80d26555
--- /dev/null
+++ b/cgo_harness/p25v_retry_paired_test.go
@@ -0,0 +1,452 @@
+//go:build cgo && treesitter_c_parity
+
+package cgoharness
+
+import (
+ "encoding/json"
+ "os"
+ "testing"
+ "time"
+
+ gotreesitter "github.com/odvcencio/gotreesitter"
+ sitter "github.com/tree-sitter/go-tree-sitter"
+)
+
+const p25vRetryBypassEnv = "GOT_DIAGNOSTIC_SKIP_ACCEPTED_ERROR_BASE_MERGE_RETRY"
+
+type p25vSpan struct {
+ Start uint32 `json:"start_byte"`
+ End uint32 `json:"end_byte"`
+ Kind string `json:"kind"`
+}
+
+type p25vRoot struct {
+ StartByte uint32 `json:"start_byte"`
+ EndByte uint32 `json:"end_byte"`
+ HasError bool `json:"has_error"`
+ ChildCount int `json:"child_count"`
+ FirstError *p25vSpan `json:"first_error,omitempty"`
+}
+
+type p25vRuntime struct {
+ StopReason string `json:"stop_reason"`
+ Truncated bool `json:"truncated"`
+ TokenSourceEOFEarly bool `json:"token_source_eof_early"`
+ RootEndByte uint32 `json:"root_end_byte"`
+ RetryAttempts uint8 `json:"retry_attempts"`
+ RetryAdopted bool `json:"retry_adopted"`
+ RetryMergePerKey int `json:"retry_merge_per_key"`
+ RetryCause string `json:"retry_cause"`
+ OldTreeReuseRoute bool `json:"old_tree_reuse_route"`
+ CRecoveryEnteredErrorState bool `json:"c_recovery_entered_error_state"`
+ CRecoveryDroppedErrorForClean bool `json:"c_recovery_dropped_error_for_clean"`
+ CRecoverySwallowedFallbackAttempted bool `json:"c_recovery_swallowed_fallback_attempted"`
+ TokensConsumed uint64 `json:"tokens_consumed"`
+ Iterations int `json:"iterations"`
+ NodesAllocated int `json:"nodes_allocated"`
+ MaxStacksSeen int `json:"max_stacks_seen"`
+ SingleStackIterations int `json:"single_stack_iterations"`
+ MultiStackIterations int `json:"multi_stack_iterations"`
+ GSSNodesAllocated uint64 `json:"gss_nodes_allocated"`
+ GSSNodesRetained uint64 `json:"gss_nodes_retained"`
+ GSSNodesDropped uint64 `json:"gss_nodes_dropped_same_token"`
+ ParentNodesAllocated uint64 `json:"parent_nodes_allocated"`
+ ParentNodesRetained uint64 `json:"parent_nodes_retained"`
+ ParentNodesDropped uint64 `json:"parent_nodes_dropped_same_token"`
+ LeafNodesAllocated uint64 `json:"leaf_nodes_allocated"`
+ LeafNodesRetained uint64 `json:"leaf_nodes_retained"`
+ LeafNodesDropped uint64 `json:"leaf_nodes_dropped_same_token"`
+ ArenaBytesAllocated int64 `json:"arena_bytes_allocated"`
+ ScratchBytesAllocated int64 `json:"scratch_bytes_allocated"`
+ GSSBytesAllocated int64 `json:"gss_bytes_allocated"`
+ MergeStacksIn uint64 `json:"merge_stacks_in"`
+ MergeStacksOut uint64 `json:"merge_stacks_out"`
+ MergeSlotsUsed uint64 `json:"merge_slots_used"`
+}
+
+type p25vProfile struct {
+ ReuseCursorNanos int64 `json:"reuse_cursor_nanos"`
+ ReparseNanos int64 `json:"reparse_nanos"`
+ ReusedSubtrees uint64 `json:"reused_subtrees"`
+ ReusedBytes uint64 `json:"reused_bytes"`
+ NewNodesAllocated uint64 `json:"new_nodes_allocated"`
+ RetryAttempts uint8 `json:"retry_attempts"`
+ RetryAdopted bool `json:"retry_adopted"`
+ RetryMergePerKey int `json:"retry_merge_per_key"`
+ RetryCause string `json:"retry_cause"`
+ OldTreeReuseRoute bool `json:"old_tree_reuse_route"`
+ ReuseRejectDirty uint64 `json:"reuse_reject_dirty"`
+ ReuseRejectHasError uint64 `json:"reuse_reject_has_error"`
+ ReuseRejectFragileNonLeaf uint64 `json:"reuse_reject_fragile_non_leaf"`
+ RecoverSearches uint64 `json:"recover_searches"`
+ RecoverStateChecks uint64 `json:"recover_state_checks"`
+ RecoverHits uint64 `json:"recover_hits"`
+ TokensConsumed uint64 `json:"tokens_consumed"`
+ MaxStacksSeen int `json:"max_stacks_seen"`
+ SingleStackIterations int `json:"single_stack_iterations"`
+ MultiStackIterations int `json:"multi_stack_iterations"`
+ GSSNodesAllocated uint64 `json:"gss_nodes_allocated"`
+ GSSNodesRetained uint64 `json:"gss_nodes_retained"`
+ GSSNodesDropped uint64 `json:"gss_nodes_dropped_same_token"`
+ ParentNodesAllocated uint64 `json:"parent_nodes_allocated"`
+ ParentNodesRetained uint64 `json:"parent_nodes_retained"`
+ ParentNodesDropped uint64 `json:"parent_nodes_dropped_same_token"`
+ LeafNodesAllocated uint64 `json:"leaf_nodes_allocated"`
+ LeafNodesRetained uint64 `json:"leaf_nodes_retained"`
+ LeafNodesDropped uint64 `json:"leaf_nodes_dropped_same_token"`
+ ArenaBytesAllocated int64 `json:"arena_bytes_allocated"`
+ ScratchBytesAllocated int64 `json:"scratch_bytes_allocated"`
+ GSSBytesAllocated int64 `json:"gss_bytes_allocated"`
+ MergeStacksIn uint64 `json:"merge_stacks_in"`
+ MergeStacksOut uint64 `json:"merge_stacks_out"`
+ MergeSlotsUsed uint64 `json:"merge_slots_used"`
+ GlobalCullStacksIn uint64 `json:"global_cull_stacks_in"`
+ GlobalCullStacksOut uint64 `json:"global_cull_stacks_out"`
+}
+
+type p25vRoute struct {
+ DigestEqualFreshGoC bool `json:"digest_equal_fresh_go_c"`
+ DigestEqualIncrementalGoC bool `json:"digest_equal_incremental_go_c"`
+ DigestEqualFreshIncremental bool `json:"digest_equal_fresh_incremental"`
+ AllDeepDigestsEqual bool `json:"all_deep_digests_equal"`
+ GoFreshDigest string `json:"go_fresh_digest"`
+ CFreshDigest string `json:"c_fresh_digest"`
+ GoIncrementalDigest string `json:"go_incremental_digest"`
+ CIncrementalDigest string `json:"c_incremental_digest"`
+ FreshFirstDifference string `json:"fresh_first_difference,omitempty"`
+ IncrementalFirstDifference string `json:"incremental_first_difference,omitempty"`
+}
+
+type p25vCaseResult struct {
+ Name string `json:"name"`
+ Role string `json:"role"`
+ Mode string `json:"mode"`
+ SourceBytes int `json:"source_bytes"`
+ EditedBytes int `json:"edited_bytes"`
+ SourceSHA256 string `json:"source_sha256"`
+ EditedSHA256 string `json:"edited_sha256"`
+ GoFreshWallNanos int64 `json:"go_fresh_wall_nanos"`
+ CFreshWallNanos int64 `json:"c_fresh_wall_nanos"`
+ GoIncrementalWallNanos int64 `json:"go_incremental_wall_nanos"`
+ CIncrementalWallNanos int64 `json:"c_incremental_wall_nanos"`
+ GoFreshRoot p25vRoot `json:"go_fresh_root"`
+ CFreshRoot p25vRoot `json:"c_fresh_root"`
+ GoIncrementalRoot p25vRoot `json:"go_incremental_root"`
+ CIncrementalRoot p25vRoot `json:"c_incremental_root"`
+ GoFreshRuntime p25vRuntime `json:"go_fresh_runtime"`
+ GoIncrementalRuntime p25vRuntime `json:"go_incremental_runtime"`
+ GoIncrementalProfile p25vProfile `json:"go_incremental_profile"`
+ Route p25vRoute `json:"route"`
+}
+
+type p25vReport struct {
+ Schema string `json:"schema"`
+ Mode string `json:"mode"`
+ BypassEnv string `json:"bypass_env"`
+ Cases []p25vCaseResult `json:"cases"`
+}
+
+func TestP25vRetryPaired(t *testing.T) {
+ bypass := os.Getenv(p25vRetryBypassEnv) == "1"
+ mode := "retry-enabled"
+ if bypass {
+ mode = "retry-bypassed"
+ }
+ caseNames := []struct {
+ name string
+ role string
+ }{
+ {name: "recovery_deletion", role: "authenticated-forward-malformed-recovery"},
+ {name: "same_length_leaf_validation", role: "clean-incremental-control"},
+ {name: "same_line_length_change", role: "accepted-error-recovery-control"},
+ }
+ cases := loadCanonicalGoIncrementalCases(t)
+ report := p25vReport{Schema: "p25v-retry-paired-v1", Mode: mode, BypassEnv: os.Getenv(p25vRetryBypassEnv)}
+ allEqual := true
+ for _, want := range caseNames {
+ var tc *canonicalGoIncrementalCase
+ for i := range cases {
+ if cases[i].spec.Name == want.name {
+ candidate := cases[i]
+ tc = &candidate
+ break
+ }
+ }
+ if tc == nil {
+ t.Fatalf("canonical case %q is missing", want.name)
+ }
+ result := runP25vCase(t, tc, want.role, mode)
+ report.Cases = append(report.Cases, result)
+ if !result.Route.AllDeepDigestsEqual {
+ allEqual = false
+ }
+ }
+ encoded, err := json.Marshal(report)
+ if err != nil {
+ t.Fatalf("marshal P25v report: %v", err)
+ }
+ t.Logf("P25V_RESULT %s", encoded)
+ if path := os.Getenv("P25V_RESULT_OUT"); path != "" {
+ if err := os.WriteFile(path, encoded, 0o644); err != nil {
+ t.Fatalf("write P25v report %q: %v", path, err)
+ }
+ }
+ if !allEqual {
+ t.Fatalf("P25v %s mode changed a fresh, incremental, or locked-C deep digest", mode)
+ }
+}
+
+func runP25vCase(t *testing.T, tc *canonicalGoIncrementalCase, role, mode string) p25vCaseResult {
+ t.Helper()
+ direction := tc.directions()[0]
+ goLang := canonicalIncrementalGoLanguage(t, tc.spec.Language)
+ cLang := canonicalIncrementalCLanguage(t, tc.spec.Language)
+ goParser := gotreesitter.NewParser(goLang)
+ cParser := sitter.NewParser()
+ if err := cParser.SetLanguage(cLang); err != nil {
+ t.Fatalf("%s set C language: %v", tc.spec.Name, err)
+ }
+
+ goFreshStart := time.Now()
+ goFresh, err := goParser.Parse(direction.to)
+ goFreshWall := time.Since(goFreshStart)
+ if err != nil {
+ t.Fatalf("%s fresh Go parse: %v", tc.spec.Name, err)
+ }
+ requireCanonicalGoIncrementalTree(t, goFresh, direction.to, tc.spec.Name+" P25v fresh Go", nil)
+ goFreshDigest := canonicalGoTreeDigest(t, goFresh, goLang, tc.spec.Name+" P25v fresh Go")
+
+ cFreshStart := time.Now()
+ cFresh := cParser.Parse(direction.to, nil)
+ cFreshWall := time.Since(cFreshStart)
+ requireCanonicalCIncrementalTree(t, cFresh, direction.to, tc.spec.Name+" P25v fresh C")
+ cFreshDigest := canonicalCTreeDigest(t, cFresh, tc.spec.Name+" P25v fresh C")
+
+ goOld, err := goParser.Parse(direction.from)
+ if err != nil {
+ t.Fatalf("%s old Go parse: %v", tc.spec.Name, err)
+ }
+ requireCanonicalGoIncrementalTree(t, goOld, direction.from, tc.spec.Name+" P25v old Go", nil)
+ goOld.Edit(direction.goEdit)
+ goIncrementalStart := time.Now()
+ goIncremental, profile, err := goParser.ParseIncrementalProfiled(direction.to, goOld)
+ goIncrementalWall := time.Since(goIncrementalStart)
+ if err != nil {
+ t.Fatalf("%s incremental Go parse: %v", tc.spec.Name, err)
+ }
+ if goIncremental != goOld {
+ releaseCanonicalGoTree(goOld)
+ }
+ requireCanonicalGoIncrementalTree(t, goIncremental, direction.to, tc.spec.Name+" P25v incremental Go", nil)
+ goIncrementalDigest := canonicalGoTreeDigest(t, goIncremental, goLang, tc.spec.Name+" P25v incremental Go")
+
+ cOld := cParser.Parse(direction.from, nil)
+ requireCanonicalCIncrementalTree(t, cOld, direction.from, tc.spec.Name+" P25v old C")
+ cOld.Edit(&direction.cEdit)
+ cIncrementalStart := time.Now()
+ cIncremental := cParser.Parse(direction.to, cOld)
+ cIncrementalWall := time.Since(cIncrementalStart)
+ if cIncremental != cOld {
+ closeCanonicalCTree(cOld)
+ }
+ requireCanonicalCIncrementalTree(t, cIncremental, direction.to, tc.spec.Name+" P25v incremental C")
+ cIncrementalDigest := canonicalCTreeDigest(t, cIncremental, tc.spec.Name+" P25v incremental C")
+
+ goFreshRoot := p25vGoRoot(goFresh.RootNode())
+ cFreshRoot := p25vCRoot(cFresh.RootNode())
+ goIncrementalRoot := p25vGoRoot(goIncremental.RootNode())
+ cIncrementalRoot := p25vCRoot(cIncremental.RootNode())
+ goFreshRuntime := p25vRuntimeFrom(goFresh.ParseRuntime())
+ goIncrementalRuntime := p25vRuntimeFrom(goIncremental.ParseRuntime())
+ freshFirstDifference := ""
+ incrementalFirstDifference := ""
+ if diff := FirstDivergenceDumpV1(goFresh.RootNode(), goLang, cFresh.RootNode()); diff != nil {
+ freshFirstDifference = formatRealCorpusDivergence(diff)
+ }
+ if diff := FirstDivergenceDumpV1(goIncremental.RootNode(), goLang, cIncremental.RootNode()); diff != nil {
+ incrementalFirstDifference = formatRealCorpusDivergence(diff)
+ }
+ route := p25vRoute{
+ DigestEqualFreshGoC: goFreshDigest == cFreshDigest,
+ DigestEqualIncrementalGoC: goIncrementalDigest == cIncrementalDigest,
+ DigestEqualFreshIncremental: goFreshDigest == goIncrementalDigest,
+ AllDeepDigestsEqual: goFreshDigest == cFreshDigest && goFreshDigest == goIncrementalDigest && goFreshDigest == cIncrementalDigest,
+ GoFreshDigest: goFreshDigest,
+ CFreshDigest: cFreshDigest,
+ GoIncrementalDigest: goIncrementalDigest,
+ CIncrementalDigest: cIncrementalDigest,
+ FreshFirstDifference: freshFirstDifference,
+ IncrementalFirstDifference: incrementalFirstDifference,
+ }
+
+ result := p25vCaseResult{
+ Name: tc.spec.Name,
+ Role: role,
+ Mode: mode,
+ SourceBytes: len(direction.from),
+ EditedBytes: len(direction.to),
+ SourceSHA256: direction.fromSHA256(),
+ EditedSHA256: direction.toSHA256(),
+ GoFreshWallNanos: goFreshWall.Nanoseconds(),
+ CFreshWallNanos: cFreshWall.Nanoseconds(),
+ GoIncrementalWallNanos: goIncrementalWall.Nanoseconds(),
+ CIncrementalWallNanos: cIncrementalWall.Nanoseconds(),
+ GoFreshRoot: goFreshRoot,
+ CFreshRoot: cFreshRoot,
+ GoIncrementalRoot: goIncrementalRoot,
+ CIncrementalRoot: cIncrementalRoot,
+ GoFreshRuntime: goFreshRuntime,
+ GoIncrementalRuntime: goIncrementalRuntime,
+ GoIncrementalProfile: p25vProfileFrom(profile),
+ Route: route,
+ }
+
+ t.Logf("P25V_CASE mode=%s case=%s role=%s all_deep_equal=%t fresh_go=%s fresh_c=%s incremental_go=%s incremental_c=%s go_inc_wall_ns=%d c_inc_wall_ns=%d retry_attempts=%d retry_adopted=%t retry_cap=%d stop=%s root_error=%t", mode, result.Name, result.Role, route.AllDeepDigestsEqual, goFreshDigest, cFreshDigest, goIncrementalDigest, cIncrementalDigest, result.GoIncrementalWallNanos, result.CIncrementalWallNanos, profile.AcceptedErrorRetryAttempts, profile.AcceptedErrorRetryAdopted, profile.AcceptedErrorRetryMergePerKey, goIncrementalRuntime.StopReason, goIncrementalRoot.HasError)
+
+ releaseCanonicalGoTree(goFresh)
+ releaseCanonicalGoTree(goIncremental)
+ closeCanonicalCTree(cFresh)
+ closeCanonicalCTree(cIncremental)
+ cParser.Close()
+ return result
+}
+
+func (d canonicalGoIncrementalDirection) fromSHA256() string {
+ return sha256Hex(d.from)
+}
+
+func (d canonicalGoIncrementalDirection) toSHA256() string {
+ return sha256Hex(d.to)
+}
+
+func p25vGoRoot(root *gotreesitter.Node) p25vRoot {
+ if root == nil {
+ return p25vRoot{}
+ }
+ return p25vRoot{StartByte: root.StartByte(), EndByte: root.EndByte(), HasError: root.HasError(), ChildCount: root.ChildCount(), FirstError: p25vFirstGoError(root)}
+}
+
+func p25vCRoot(root *sitter.Node) p25vRoot {
+ if root == nil {
+ return p25vRoot{}
+ }
+ return p25vRoot{StartByte: uint32(root.StartByte()), EndByte: uint32(root.EndByte()), HasError: root.HasError(), ChildCount: int(root.ChildCount()), FirstError: p25vFirstCError(root)}
+}
+
+func p25vFirstGoError(node *gotreesitter.Node) *p25vSpan {
+ if node == nil {
+ return nil
+ }
+ if node.IsError() || node.IsMissing() {
+ kind := "ERROR"
+ if node.IsMissing() {
+ kind = "MISSING"
+ }
+ return &p25vSpan{Start: node.StartByte(), End: node.EndByte(), Kind: kind}
+ }
+ for i := 0; i < node.ChildCount(); i++ {
+ if span := p25vFirstGoError(node.Child(i)); span != nil {
+ return span
+ }
+ }
+ return nil
+}
+
+func p25vFirstCError(node *sitter.Node) *p25vSpan {
+ if node == nil {
+ return nil
+ }
+ if node.IsError() || node.IsMissing() {
+ kind := "ERROR"
+ if node.IsMissing() {
+ kind = "MISSING"
+ }
+ return &p25vSpan{Start: uint32(node.StartByte()), End: uint32(node.EndByte()), Kind: kind}
+ }
+ for i := uint(0); i < node.ChildCount(); i++ {
+ if span := p25vFirstCError(node.Child(i)); span != nil {
+ return span
+ }
+ }
+ return nil
+}
+
+func p25vRuntimeFrom(runtime gotreesitter.ParseRuntime) p25vRuntime {
+ return p25vRuntime{
+ StopReason: string(runtime.StopReason),
+ Truncated: runtime.Truncated,
+ TokenSourceEOFEarly: runtime.TokenSourceEOFEarly,
+ RootEndByte: runtime.RootEndByte,
+ RetryAttempts: runtime.IncrementalAcceptedErrorRetryAttempts,
+ RetryAdopted: runtime.IncrementalAcceptedErrorRetryAdopted,
+ RetryMergePerKey: runtime.IncrementalAcceptedErrorRetryMergePerKey,
+ RetryCause: canonicalIncrementalRetryCause(runtime.IncrementalAcceptedErrorRetryCause),
+ OldTreeReuseRoute: runtime.IncrementalOldTreeReuseRoute,
+ CRecoveryEnteredErrorState: runtime.CRecoveryEnteredErrorState,
+ CRecoveryDroppedErrorForClean: runtime.CRecoveryDroppedErrorForClean,
+ CRecoverySwallowedFallbackAttempted: runtime.CRecoverySwallowedErrorFallbackAttempted,
+ TokensConsumed: runtime.TokensConsumed,
+ Iterations: runtime.Iterations,
+ NodesAllocated: runtime.NodesAllocated,
+ MaxStacksSeen: runtime.MaxStacksSeen,
+ SingleStackIterations: runtime.SingleStackIterations,
+ MultiStackIterations: runtime.MultiStackIterations,
+ GSSNodesAllocated: runtime.GSSNodesAllocated,
+ GSSNodesRetained: runtime.GSSNodesRetained,
+ GSSNodesDropped: runtime.GSSNodesDroppedSameToken,
+ ParentNodesAllocated: runtime.ParentNodesAllocated,
+ ParentNodesRetained: runtime.ParentNodesRetained,
+ ParentNodesDropped: runtime.ParentNodesDroppedSameToken,
+ LeafNodesAllocated: runtime.LeafNodesAllocated,
+ LeafNodesRetained: runtime.LeafNodesRetained,
+ LeafNodesDropped: runtime.LeafNodesDroppedSameToken,
+ ArenaBytesAllocated: runtime.ArenaBytesAllocated,
+ ScratchBytesAllocated: runtime.ScratchBytesAllocated,
+ GSSBytesAllocated: runtime.GSSBytesAllocated,
+ MergeStacksIn: runtime.MergeStacksIn,
+ MergeStacksOut: runtime.MergeStacksOut,
+ MergeSlotsUsed: runtime.MergeSlotsUsed,
+ }
+}
+
+func p25vProfileFrom(profile gotreesitter.IncrementalParseProfile) p25vProfile {
+ return p25vProfile{
+ ReuseCursorNanos: profile.ReuseCursorNanos,
+ ReparseNanos: profile.ReparseNanos,
+ ReusedSubtrees: profile.ReusedSubtrees,
+ ReusedBytes: profile.ReusedBytes,
+ NewNodesAllocated: profile.NewNodesAllocated,
+ RetryAttempts: profile.AcceptedErrorRetryAttempts,
+ RetryAdopted: profile.AcceptedErrorRetryAdopted,
+ RetryMergePerKey: profile.AcceptedErrorRetryMergePerKey,
+ RetryCause: canonicalIncrementalRetryCause(profile.AcceptedErrorRetryCause),
+ OldTreeReuseRoute: profile.OldTreeReuseRoute,
+ ReuseRejectDirty: profile.ReuseRejectDirty,
+ ReuseRejectHasError: profile.ReuseRejectHasError,
+ ReuseRejectFragileNonLeaf: profile.ReuseRejectFragileNonLeaf,
+ RecoverSearches: profile.RecoverSearches,
+ RecoverStateChecks: profile.RecoverStateChecks,
+ RecoverHits: profile.RecoverHits,
+ TokensConsumed: profile.TokensConsumed,
+ MaxStacksSeen: profile.MaxStacksSeen,
+ SingleStackIterations: profile.SingleStackIterations,
+ MultiStackIterations: profile.MultiStackIterations,
+ GSSNodesAllocated: profile.GSSNodesAllocated,
+ GSSNodesRetained: profile.GSSNodesRetained,
+ GSSNodesDropped: profile.GSSNodesDroppedSameToken,
+ ParentNodesAllocated: profile.ParentNodesAllocated,
+ ParentNodesRetained: profile.ParentNodesRetained,
+ ParentNodesDropped: profile.ParentNodesDroppedSameToken,
+ LeafNodesAllocated: profile.LeafNodesAllocated,
+ LeafNodesRetained: profile.LeafNodesRetained,
+ LeafNodesDropped: profile.LeafNodesDroppedSameToken,
+ ArenaBytesAllocated: profile.ArenaBytesAllocated,
+ ScratchBytesAllocated: profile.ScratchBytesAllocated,
+ GSSBytesAllocated: profile.GSSBytesAllocated,
+ MergeStacksIn: profile.MergeStacksIn,
+ MergeStacksOut: profile.MergeStacksOut,
+ MergeSlotsUsed: profile.MergeSlotsUsed,
+ GlobalCullStacksIn: profile.GlobalCullStacksIn,
+ GlobalCullStacksOut: profile.GlobalCullStacksOut,
+ }
+}
diff --git a/cgo_harness/p25w_exact_language_route_test.go b/cgo_harness/p25w_exact_language_route_test.go
new file mode 100644
index 000000000..344a75f80
--- /dev/null
+++ b/cgo_harness/p25w_exact_language_route_test.go
@@ -0,0 +1,150 @@
+//go:build cgo && treesitter_c_parity
+
+package cgoharness
+
+import (
+ "encoding/json"
+ "os"
+ "testing"
+ "time"
+
+ gotreesitter "github.com/odvcencio/gotreesitter"
+ sitter "github.com/tree-sitter/go-tree-sitter"
+)
+
+type p25wExactLanguageResult struct {
+ Mode string `json:"mode"`
+ Case string `json:"case"`
+ SourceSHA256 string `json:"source_sha256"`
+ EditedSHA256 string `json:"edited_sha256"`
+ GoFreshDigest string `json:"go_fresh_digest"`
+ GoIncrementalDigest string `json:"go_incremental_digest"`
+ CFreshDigest string `json:"c_fresh_digest"`
+ CIncrementalDigest string `json:"c_incremental_digest"`
+ GoFreshRoot p25vRoot `json:"go_fresh_root"`
+ GoIncrementalRoot p25vRoot `json:"go_incremental_root"`
+ CFreshRoot p25vRoot `json:"c_fresh_root"`
+ CIncrementalRoot p25vRoot `json:"c_incremental_root"`
+ GoIncrementalRuntime p25vRuntime `json:"go_incremental_runtime"`
+ GoIncrementalProfile p25vProfile `json:"go_incremental_profile"`
+ FreshDifference string `json:"fresh_difference,omitempty"`
+ IncrementalDifference string `json:"incremental_difference,omitempty"`
+ GoFreshWallNanos int64 `json:"go_fresh_wall_nanos"`
+ GoIncrementalWallNanos int64 `json:"go_incremental_wall_nanos"`
+}
+
+func TestP25wExactLanguageRoute(t *testing.T) {
+ mode := "retry-enabled"
+ if os.Getenv(p25vRetryBypassEnv) == "1" {
+ mode = "retry-bypassed"
+ }
+ cases := loadCanonicalGoIncrementalCases(t)
+ var tc *canonicalGoIncrementalCase
+ for i := range cases {
+ if cases[i].spec.Name == "same_line_length_change" {
+ candidate := cases[i]
+ tc = &candidate
+ break
+ }
+ }
+ if tc == nil {
+ t.Fatal("same_line_length_change case is missing")
+ }
+ direction := tc.directions()[0]
+ goLang := canonicalIncrementalGoLanguage(t, tc.spec.Language)
+ cLang := canonicalIncrementalCLanguage(t, tc.spec.Language)
+
+ goFreshParser := gotreesitter.NewParser(goLang)
+ goFreshParser.SetAdmissionCandidateRoute(false)
+ goFreshStart := time.Now()
+ goFresh, err := goFreshParser.Parse(direction.to)
+ goFreshWall := time.Since(goFreshStart)
+ if err != nil {
+ t.Fatalf("fresh Go parse: %v", err)
+ }
+ requireCanonicalGoIncrementalTree(t, goFresh, direction.to, "P25w fresh Go", nil)
+ goFreshDigest := canonicalGoTreeDigest(t, goFresh, goLang, "P25w fresh Go")
+
+ goOldParser := gotreesitter.NewParser(goLang)
+ goOldParser.SetAdmissionCandidateRoute(false)
+ goOld, err := goOldParser.Parse(direction.from)
+ if err != nil {
+ t.Fatalf("old Go parse: %v", err)
+ }
+ requireCanonicalGoIncrementalTree(t, goOld, direction.from, "P25w old Go", nil)
+ goOld.Edit(direction.goEdit)
+ goIncrementalParser := gotreesitter.NewParser(goLang)
+ goIncrementalParser.SetAdmissionCandidateRoute(false)
+ goIncrementalStart := time.Now()
+ goIncremental, profile, err := goIncrementalParser.ParseIncrementalProfiled(direction.to, goOld)
+ goIncrementalWall := time.Since(goIncrementalStart)
+ if err != nil {
+ t.Fatalf("incremental Go parse: %v", err)
+ }
+ if goIncremental != goOld {
+ releaseCanonicalGoTree(goOld)
+ }
+ requireCanonicalGoIncrementalTree(t, goIncremental, direction.to, "P25w incremental Go", nil)
+ goIncrementalDigest := canonicalGoTreeDigest(t, goIncremental, goLang, "P25w incremental Go")
+
+ cParser := sitter.NewParser()
+ if err := cParser.SetLanguage(cLang); err != nil {
+ t.Fatalf("set C language: %v", err)
+ }
+ cFresh := cParser.Parse(direction.to, nil)
+ requireCanonicalCIncrementalTree(t, cFresh, direction.to, "P25w fresh C")
+ cFreshDigest := canonicalCTreeDigest(t, cFresh, "P25w fresh C")
+ cOld := cParser.Parse(direction.from, nil)
+ requireCanonicalCIncrementalTree(t, cOld, direction.from, "P25w old C")
+ cOld.Edit(&direction.cEdit)
+ cIncremental := cParser.Parse(direction.to, cOld)
+ if cIncremental != cOld {
+ closeCanonicalCTree(cOld)
+ }
+ requireCanonicalCIncrementalTree(t, cIncremental, direction.to, "P25w incremental C")
+ cIncrementalDigest := canonicalCTreeDigest(t, cIncremental, "P25w incremental C")
+
+ result := p25wExactLanguageResult{
+ Mode: mode,
+ Case: tc.spec.Name,
+ SourceSHA256: direction.fromSHA256(),
+ EditedSHA256: direction.toSHA256(),
+ GoFreshDigest: goFreshDigest,
+ GoIncrementalDigest: goIncrementalDigest,
+ CFreshDigest: cFreshDigest,
+ CIncrementalDigest: cIncrementalDigest,
+ GoFreshRoot: p25vGoRoot(goFresh.RootNode()),
+ GoIncrementalRoot: p25vGoRoot(goIncremental.RootNode()),
+ CFreshRoot: p25vCRoot(cFresh.RootNode()),
+ CIncrementalRoot: p25vCRoot(cIncremental.RootNode()),
+ GoIncrementalRuntime: p25vRuntimeFrom(goIncremental.ParseRuntime()),
+ GoIncrementalProfile: p25vProfileFrom(profile),
+ GoFreshWallNanos: goFreshWall.Nanoseconds(),
+ GoIncrementalWallNanos: goIncrementalWall.Nanoseconds(),
+ }
+ if diff := FirstDivergenceDumpV1(goFresh.RootNode(), goLang, cFresh.RootNode()); diff != nil {
+ result.FreshDifference = formatRealCorpusDivergence(diff)
+ }
+ if diff := FirstDivergenceDumpV1(goIncremental.RootNode(), goLang, cIncremental.RootNode()); diff != nil {
+ result.IncrementalDifference = formatRealCorpusDivergence(diff)
+ }
+ encoded, err := json.Marshal(result)
+ if err != nil {
+ t.Fatalf("marshal P25w result: %v", err)
+ }
+ t.Logf("P25W_RESULT %s", encoded)
+ if path := os.Getenv("P25W_RESULT_OUT"); path != "" {
+ if err := os.WriteFile(path, encoded, 0o644); err != nil {
+ t.Fatalf("write P25w result: %v", err)
+ }
+ }
+ if goIncrementalDigest != goFreshDigest || goIncrementalDigest != cFreshDigest || goIncrementalDigest != cIncrementalDigest {
+ t.Errorf("P25w exact route deep digest mismatch: Go fresh=%s Go incremental=%s C fresh=%s C incremental=%s", goFreshDigest, goIncrementalDigest, cFreshDigest, cIncrementalDigest)
+ }
+
+ releaseCanonicalGoTree(goFresh)
+ releaseCanonicalGoTree(goIncremental)
+ closeCanonicalCTree(cFresh)
+ closeCanonicalCTree(cIncremental)
+ cParser.Close()
+}
diff --git a/cgo_harness/templ_next_live_probe_test.go b/cgo_harness/templ_next_live_probe_test.go
new file mode 100644
index 000000000..41d55eeb4
--- /dev/null
+++ b/cgo_harness/templ_next_live_probe_test.go
@@ -0,0 +1,388 @@
+//go:build cgo && treesitter_c_parity
+
+package cgoharness
+
+import (
+ "bytes"
+ "crypto/sha256"
+ "fmt"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ gotreesitter "github.com/odvcencio/gotreesitter"
+ "github.com/odvcencio/gotreesitter/grammars"
+ "github.com/odvcencio/gotreesitter/internal/benchfixtures"
+ sitter "github.com/tree-sitter/go-tree-sitter"
+)
+
+type templNextWitness struct {
+ name string
+ source []byte
+}
+
+// TestTemplNextLiveArmLockedCRoutes records Templ on raw, production, compact,
+// forest, and incremental routes. It keeps rewrite and recovery evidence visible.
+func TestTemplNextLiveArmLockedCRoutes(t *testing.T) {
+ t.Setenv("GTS_DISPATCHER_CENSUS", "1")
+ t.Setenv("GOT_PARSE_PHASE_TIMING", "1")
+ goLanguage := grammars.TemplLanguage()
+ cLanguage, err := COracleLanguage("templ")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ witnesses := []templNextWitness{
+ {name: "a0-medium-main", source: templNextReadA0(t, "medium__main.templ")},
+ {name: "a0-medium-template", source: templNextReadA0(t, "medium__template.templ")},
+ {name: "a0-small-template", source: templNextReadA0(t, "small__template.templ")},
+ {name: "positive-no-op-control", source: []byte("templ T() { ok
}\n")},
+ {name: "qualified-component-import", source: []byte("package main\n\ntempl T() {\n\t@templ.JSONScript(\"scriptData\", scriptData)\n}\n")},
+ {name: "malformed-dangling-attribute-quote", source: []byte("templ Broken() {\n\t\n}\n")},
+ {name: "malformed-component-import", source: []byte("package main\n\ntempl Broken() {\n\t@counts(global,\n}\n")},
+ }
+
+ for _, witness := range witnesses {
+ witness := witness
+ t.Run(witness.name, func(t *testing.T) {
+ cParser := sitter.NewParser()
+ t.Cleanup(cParser.Close)
+ if err := cParser.SetLanguage(cLanguage); err != nil {
+ t.Fatal(err)
+ }
+ cTree := cParser.Parse(witness.source, nil)
+ if cTree == nil || cTree.RootNode() == nil {
+ t.Fatal("locked C parser returned no tree")
+ }
+ t.Cleanup(cTree.Close)
+ cDigest, err := COracleDeepDigest(cTree)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if want := templNextExpectedCDigest(witness.name); want != "" && cDigest != want {
+ t.Fatalf("%s locked-C digest=%s, want %s", witness.name, cDigest, want)
+ }
+
+ raw := templNextParseRoute(t, goLanguage, witness.source, "raw", func(p *gotreesitter.Parser, source []byte) (*gotreesitter.Tree, error) {
+ p.SetAdmissionCandidateRoute(false)
+ return p.ParseNoResultCompatibilityBenchmarkOnly(source)
+ })
+ production := templNextParseRoute(t, goLanguage, witness.source, "production", func(p *gotreesitter.Parser, source []byte) (*gotreesitter.Tree, error) {
+ p.SetAdmissionCandidateRoute(false)
+ return p.Parse(source)
+ })
+
+ routedBefore, fallbackBefore := gotreesitter.AdmissionCandidateCounters()
+ compactParser := gotreesitter.NewParser(goLanguage)
+ compactParser.SetAdmissionCandidateRoute(true)
+ compact, err := compactParser.Parse(witness.source)
+ if err != nil {
+ t.Fatalf("compact parse: %v", err)
+ }
+ t.Cleanup(compact.Release)
+ routedAfter, fallbackAfter := gotreesitter.AdmissionCandidateCounters()
+ compactRoute := "accepted"
+ if fallbackAfter > fallbackBefore {
+ compactRoute = "fallback:" + gotreesitter.AdmissionCandidateLastFallbackReason()
+ }
+
+ forestParser := gotreesitter.NewParser(goLanguage)
+ forest, forestOK := forestParser.ParseForestExperimental(witness.source)
+ forestRoute := "declined"
+ forestDetail := ""
+ if forestOK && forest != nil {
+ forestRoute = "accepted"
+ t.Cleanup(forest.Release)
+ } else {
+ offset, symbol, reason, _ := forestParser.ForestDeclineInfo()
+ forestDetail = fmt.Sprintf("offset=%d symbol=%d reason=%q", offset, symbol, reason)
+ }
+
+ incremental, profile := templNextIncrementalRoute(t, goLanguage, witness.source)
+
+ templNextLogRoute(t, "raw", raw, goLanguage, cTree, cDigest, "")
+ templNextAssertKnownRoute(t, witness.name, "raw", raw, goLanguage, cTree)
+ templNextLogRoute(t, "production", production, goLanguage, cTree, cDigest, "")
+ templNextAssertKnownRoute(t, witness.name, "production", production, goLanguage, cTree)
+ templNextLogRoute(t, "compact", compact, goLanguage, cTree, cDigest, compactRoute)
+ templNextAssertKnownRoute(t, witness.name, "compact", compact, goLanguage, cTree)
+ if forest != nil {
+ templNextLogRoute(t, "forest", forest, goLanguage, cTree, cDigest, forestRoute)
+ templNextAssertKnownRoute(t, witness.name, "forest", forest, goLanguage, cTree)
+ } else {
+ t.Logf("route=forest witness=%s result=declined %s", witness.name, forestDetail)
+ }
+ templNextLogRoute(t, "incremental", incremental, goLanguage, cTree, cDigest, fmt.Sprintf("reuse=%t unsupported=%t reason=%q reused_subtrees=%d reused_bytes=%d", profile.OldTreeReuseRoute, profile.ReuseUnsupported, profile.ReuseUnsupportedReason, profile.ReusedSubtrees, profile.ReusedBytes))
+ templNextAssertKnownRoute(t, witness.name, "incremental", incremental, goLanguage, cTree)
+ if profile.ReuseUnsupportedReason != "external_scanner_unsupported" || profile.OldTreeReuseRoute || !profile.ReuseUnsupported {
+ t.Fatalf("incremental route profile = reuse=%t unsupported=%t reason=%q", profile.OldTreeReuseRoute, profile.ReuseUnsupported, profile.ReuseUnsupportedReason)
+ }
+ templNextAssertForestRoute(t, witness.name, forestParser, forestOK)
+ t.Logf("witness=%s bytes=%d source_sha256=%x c_digest=%s compact=%s counters=%d/%d->%d/%d forest=%s %s incremental_reuse=%t incremental_unsupported=%t incremental_reason=%q", witness.name, len(witness.source), sha256.Sum256(witness.source), cDigest, compactRoute, routedBefore, fallbackBefore, routedAfter, fallbackAfter, forestRoute, forestDetail, profile.OldTreeReuseRoute, profile.ReuseUnsupported, profile.ReuseUnsupportedReason)
+ templNextAssertCompactRoute(t, witness.name, compactRoute, routedAfter-routedBefore, fallbackAfter-fallbackBefore)
+ })
+ }
+}
+
+func templNextReadA0(t *testing.T, name string) []byte {
+ t.Helper()
+ source, err := os.ReadFile(filepath.Join("..", "testdata", "dispatcher_census_a0", "templ", name))
+ if err != nil {
+ t.Fatal(err)
+ }
+ return source
+}
+
+func templNextParseRoute(t *testing.T, language *gotreesitter.Language, source []byte, route string, parse func(*gotreesitter.Parser, []byte) (*gotreesitter.Tree, error)) *gotreesitter.Tree {
+ t.Helper()
+ parser := gotreesitter.NewParser(language)
+ tree, err := parse(parser, source)
+ if err != nil {
+ t.Fatalf("%s parse: %v", route, err)
+ }
+ if tree == nil || tree.RootNode() == nil {
+ t.Fatalf("%s returned no tree", route)
+ }
+ t.Cleanup(tree.Release)
+ return tree
+}
+
+func templNextIncrementalRoute(t *testing.T, language *gotreesitter.Language, source []byte) (*gotreesitter.Tree, gotreesitter.IncrementalParseProfile) {
+ t.Helper()
+ if !bytes.HasSuffix(source, []byte{'\n'}) {
+ t.Fatalf("incremental witness does not end with newline")
+ }
+ base := bytes.TrimSuffix(source, []byte{'\n'})
+ parser := gotreesitter.NewParser(language)
+ oldTree, err := parser.Parse(base)
+ if err != nil {
+ t.Fatalf("incremental base parse: %v", err)
+ }
+ t.Cleanup(oldTree.Release)
+ point := templNextPointAtByte(base)
+ oldTree.Edit(gotreesitter.InputEdit{
+ StartByte: uint32(len(base)),
+ OldEndByte: uint32(len(base)),
+ NewEndByte: uint32(len(source)),
+ StartPoint: point,
+ OldEndPoint: point,
+ NewEndPoint: templNextPointAtByte(source),
+ })
+ tree, profile, err := parser.ParseIncrementalProfiled(source, oldTree)
+ if err != nil {
+ t.Fatalf("incremental parse: %v", err)
+ }
+ if tree == nil || tree.RootNode() == nil {
+ t.Fatal("incremental returned no tree")
+ }
+ t.Cleanup(tree.Release)
+ return tree, profile
+}
+
+func templNextPointAtByte(source []byte) gotreesitter.Point {
+ var point gotreesitter.Point
+ for _, b := range source {
+ if b == '\n' {
+ point.Row++
+ point.Column = 0
+ continue
+ }
+ point.Column++
+ }
+ return point
+}
+
+func templNextLogRoute(t *testing.T, route string, tree *gotreesitter.Tree, language *gotreesitter.Language, cTree *sitter.Tree, cDigest, detail string) {
+ t.Helper()
+ root := tree.RootNode()
+ inspection, err := benchfixtures.InspectGoTree(root, language)
+ if err != nil {
+ t.Fatalf("%s inspect Go tree: %v", route, err)
+ }
+ diff := FirstDivergenceDumpV1(root, language, cTree.RootNode())
+ checked, run, visited, rewritten := templNextDispatchStats(tree)
+ if detail != "" {
+ detail = " " + detail
+ }
+ t.Logf("route=%s error=%t digest=%s c_digest=%s divergence=%+v dispatch_checked=%d dispatch_run=%d dispatch_visited=%d dispatch_rewritten=%d%s", route, root.HasError(), inspection.SHA256, cDigest, diff, checked, run, visited, rewritten, detail)
+}
+
+func templNextDispatchStats(tree *gotreesitter.Tree) (checked, run, visited, rewritten uint64) {
+ runtime := tree.ParseRuntime()
+ if runtime.NormalizationPasses == nil {
+ return 0, 0, 0, 0
+ }
+ for _, pass := range *runtime.NormalizationPasses {
+ if pass.Name != "dispatch.templ" {
+ continue
+ }
+ checked += pass.Checked
+ run += pass.Run
+ visited += pass.NodesVisited
+ rewritten += pass.NodesRewritten
+ }
+ return checked, run, visited, rewritten
+}
+
+type templNextExpectedRoute struct {
+ digest string
+ error bool
+ rewrite uint64
+ diff *DumpV1Divergence
+}
+
+func templNextExpectedDiff(path, category, goValue, cValue string) *DumpV1Divergence {
+ return &DumpV1Divergence{Path: path, Category: category, GoValue: goValue, CValue: cValue}
+}
+
+func templNextExpectedCDigest(witness string) string {
+ switch witness {
+ case "a0-medium-main":
+ return "efab90f3a4a75a4deba8c94d67c741dd842a7c8c6708bed3f59e37e0a994a11f"
+ case "a0-medium-template":
+ return "7de9788750436a485bee98ec6200da09d5062700368333fe380562d71f171891"
+ case "a0-small-template":
+ return "cb81fe10587416eae568216d16d2f7258bda32d00136030d8a4fcd2198e12594"
+ case "positive-no-op-control":
+ return "ac11fb7f49572a2132e31c3a46328e17a103c4ae73fd75745427a188c5987e11"
+ case "qualified-component-import":
+ return "631955c93c466e736f59aa22039f2558b7685d3b4641b71b874cac52b5e70a23"
+ case "malformed-dangling-attribute-quote":
+ return "3e6eb2d96ca843d122d8f1e952fcd465feee7f67d289c8de118c08436f1b4491"
+ case "malformed-component-import":
+ return "922cf68ffcfd8e890f27d0cb8fb995b5a5fdfb10c3853bc661bd8c722bbd34cb"
+ default:
+ return ""
+ }
+}
+
+func templNextExpectedRouteFor(witness, route string) (templNextExpectedRoute, bool) {
+ shapeRaw := func(digest, path, goValue, cValue string) templNextExpectedRoute {
+ return templNextExpectedRoute{digest: digest, diff: templNextExpectedDiff(path, "shape", goValue, cValue)}
+ }
+ shapeNormalized := func(digest string) templNextExpectedRoute {
+ return shapeRaw(digest, "/source_file/component_declaration[26]/component_block[3]", "children=12", "children=11")
+ }
+ errorRoot := templNextExpectedDiff("/source_file", "error", "true", "false")
+ malformedRoot := templNextExpectedDiff("/source_file", "error", "false", "true")
+ switch witness {
+ case "a0-medium-main":
+ return templNextExpectedRoute{digest: "895657a1c4978896653cf968b2dedddf7badd40f464d558e97dd95a9d9675595", error: true, diff: errorRoot}, true
+ case "a0-medium-template":
+ if route == "raw" {
+ return shapeRaw("33a54940b5da62255e5a03056b2ed7935994773b53a746b9a7e706b60a1a8dcb", "/source_file/component_declaration[26]/component_block[3]", "children=20", "children=11"), true
+ }
+ want := shapeNormalized("2499953c81a152ca9db474f121b1a8a9de0c888c6f00a25125301c157bcb0b0e")
+ want.rewrite = 53
+ return want, true
+ case "a0-small-template":
+ if route == "raw" {
+ return shapeRaw("80e67baee0a78d252f4621c42b4eab3e1334bc919bdcba000d17034d04f954f3", "/source_file/component_declaration[3]/component_block[3]/element[2]/element[1]", "children=4", "children=3"), true
+ }
+ return templNextExpectedRoute{digest: "cb81fe10587416eae568216d16d2f7258bda32d00136030d8a4fcd2198e12594", rewrite: 23}, true
+ case "positive-no-op-control":
+ return templNextExpectedRoute{digest: "ac11fb7f49572a2132e31c3a46328e17a103c4ae73fd75745427a188c5987e11"}, true
+ case "qualified-component-import":
+ if route == "raw" {
+ return shapeRaw("ed7d7f4155a6bdc6153038f7637dd2a14878abfcb26d27bbfc4a30529a4e0da7", "/source_file/component_declaration[1]/component_block[3]", "children=4", "children=3"), true
+ }
+ want := templNextExpectedRoute{digest: "631955c93c466e736f59aa22039f2558b7685d3b4641b71b874cac52b5e70a23"}
+ if route == "production" || route == "forest" || route == "incremental" {
+ want.rewrite = 15
+ }
+ return want, true
+ case "malformed-dangling-attribute-quote":
+ return templNextExpectedRoute{digest: "3e6eb2d96ca843d122d8f1e952fcd465feee7f67d289c8de118c08436f1b4491", error: true}, true
+ case "malformed-component-import":
+ return templNextExpectedRoute{digest: "8954432deb8e607319a63b17b665508434a8d87fac8c6fe9923ad1bbe062760f", diff: malformedRoot}, true
+ default:
+ return templNextExpectedRoute{}, false
+ }
+}
+
+func templNextAssertKnownRoute(t *testing.T, witness, route string, tree *gotreesitter.Tree, language *gotreesitter.Language, cTree *sitter.Tree) {
+ t.Helper()
+ want, ok := templNextExpectedRouteFor(witness, route)
+ if !ok {
+ return
+ }
+ root := tree.RootNode()
+ if root == nil {
+ t.Fatalf("%s/%s returned no root", witness, route)
+ }
+ if root.HasError() != want.error {
+ t.Fatalf("%s/%s error_root=%t, want %t", witness, route, root.HasError(), want.error)
+ }
+ inspection, err := benchfixtures.InspectGoTree(root, language)
+ if err != nil {
+ t.Fatalf("%s/%s inspect: %v", witness, route, err)
+ }
+ if inspection.SHA256 != want.digest {
+ t.Fatalf("%s/%s digest=%s, want %s", witness, route, inspection.SHA256, want.digest)
+ }
+ diff := FirstDivergenceDumpV1(root, language, cTree.RootNode())
+ if want.diff == nil {
+ if diff != nil {
+ t.Fatalf("%s/%s divergence=%+v, want exact locked-C parity", witness, route, diff)
+ }
+ } else if diff == nil || *diff != *want.diff {
+ t.Fatalf("%s/%s divergence=%+v, want %+v", witness, route, diff, want.diff)
+ }
+ _, _, _, rewritten := templNextDispatchStats(tree)
+ if rewritten != want.rewrite {
+ t.Fatalf("%s/%s dispatch.templ rewrites=%d, want %d", witness, route, rewritten, want.rewrite)
+ }
+}
+
+func templNextAssertCompactRoute(t *testing.T, witness, route string, routedDelta, fallbackDelta uint64) {
+ t.Helper()
+ if routedDelta+fallbackDelta != 1 {
+ t.Fatalf("%s compact counters routed_delta=%d fallback_delta=%d", witness, routedDelta, fallbackDelta)
+ }
+ wantFallback := witness == "a0-medium-main" || witness == "a0-medium-template" || witness == "a0-small-template" || witness == "malformed-dangling-attribute-quote"
+ if wantFallback && (route == "accepted" || routedDelta != 0 || fallbackDelta != 1) {
+ t.Fatalf("%s compact route=%q counters=%d/%d, want fallback", witness, route, routedDelta, fallbackDelta)
+ }
+ if !wantFallback && (route != "accepted" || routedDelta != 1 || fallbackDelta != 0) {
+ t.Fatalf("%s compact route=%q counters=%d/%d, want accepted", witness, route, routedDelta, fallbackDelta)
+ }
+}
+
+func templNextAssertForestRoute(t *testing.T, witness string, parser *gotreesitter.Parser, accepted bool) {
+ t.Helper()
+ wantDeclined := witness == "a0-medium-main" || witness == "malformed-dangling-attribute-quote"
+ if wantDeclined != !accepted {
+ t.Fatalf("%s forest accepted=%t, want %t", witness, accepted, !wantDeclined)
+ }
+ if wantDeclined {
+ offset, symbol, reason, _ := parser.ForestDeclineInfo()
+ if witness == "a0-medium-main" && (offset != 1025 || symbol != 74 || reason != "dead_end") {
+ t.Fatalf("%s forest decline=%d/%d/%q, want 1025/74/dead_end", witness, offset, symbol, reason)
+ }
+ if witness == "malformed-dangling-attribute-quote" && (offset != 47 || symbol != 24 || reason != "dead_end") {
+ t.Fatalf("%s forest decline=%d/%d/%q, want 47/24/dead_end", witness, offset, symbol, reason)
+ }
+ }
+}
+
+// TestTemplNextLiveArmReceiptDocument guards the final blocker receipt markers.
+func TestTemplNextLiveArmReceiptDocument(t *testing.T) {
+ raw, err := os.ReadFile("../docs/root-normalization-retirement.md")
+ if err != nil {
+ t.Fatal(err)
+ }
+ document := strings.ReplaceAll(strings.Join(strings.Fields(string(raw)), " "), "`", "")
+ for _, marker := range []string{
+ "Status: NO-GO. KEEP LIVE: dispatch.templ.",
+ "The authenticated Templ A0 receipt records three files, three checks, three runs, 1138 visited nodes, 76 rewritten nodes, one error root, and zero parse errors.",
+ "The medium template witness rewrites 53 nodes on production, compact, forest, and incremental routes.",
+ "The full real-corpus census is unavailable because cgo_harness/corpus_real is absent.",
+ "Do not change the registry or production state.",
+ } {
+ marker = strings.Join(strings.Fields(marker), " ")
+ if !strings.Contains(document, marker) {
+ t.Fatalf("Templ blocker receipt lacks marker %q", marker)
+ }
+ }
+}
diff --git a/cgo_harness/wgsl_next_live_probe_test.go b/cgo_harness/wgsl_next_live_probe_test.go
new file mode 100644
index 000000000..80095d6fc
--- /dev/null
+++ b/cgo_harness/wgsl_next_live_probe_test.go
@@ -0,0 +1,440 @@
+//go:build cgo && treesitter_c_parity
+
+package cgoharness
+
+import (
+ "bytes"
+ "crypto/sha256"
+ "fmt"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ gotreesitter "github.com/odvcencio/gotreesitter"
+ "github.com/odvcencio/gotreesitter/grammars"
+ "github.com/odvcencio/gotreesitter/internal/benchfixtures"
+ sitter "github.com/tree-sitter/go-tree-sitter"
+)
+
+type wgslNextWitness struct {
+ name string
+ source []byte
+ class string
+}
+
+type wgslNextRouteExpectation struct {
+ digest string
+ receipt string
+ diff *DumpV1Divergence
+ rootError bool
+ rootErrorSet bool
+}
+
+type wgslNextExpectation struct {
+ cDigest string
+ wantError bool
+ compactRoute string
+ forestRoute string
+ reuseSubtrees uint64
+ reuseBytes uint64
+ routes map[string]wgslNextRouteExpectation
+ forest *wgslNextRouteExpectation
+}
+
+// TestWGSLNextLiveArmLockedCRoutes records every WGSL route for A0 and controls.
+func TestWGSLNextLiveArmLockedCRoutes(t *testing.T) {
+ t.Setenv("GTS_DISPATCHER_CENSUS", "1")
+ t.Setenv("GOT_PARSE_PHASE_TIMING", "1")
+ goLanguage := grammars.WgslLanguage()
+ cLanguage, err := COracleLanguage("wgsl")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ witnesses := []wgslNextWitness{
+ {name: "a0-small-fragmentTextureQuad", source: wgslReadA0(t, "small__fragmentTextureQuad.wgsl"), class: "a0"},
+ {name: "a0-medium-normalMap", source: wgslReadA0(t, "medium__normalMap.wgsl"), class: "a0"},
+ {name: "a0-medium-radiosity", source: wgslReadA0(t, "medium__radiosity.wgsl"), class: "a0"},
+ {name: "recovery-empty-return", source: []byte("fn malformed() { return; }\n"), class: "recovery"},
+ {name: "malformed-missing-expression", source: []byte("fn malformed() { let value: f32 = ; }\n"), class: "malformed"},
+ {name: "malformed-argument-list", source: []byte("fn malformed() { textureLoad(texture, coord,); }\n"), class: "malformed"},
+ {name: "positive-control", source: []byte("fn identity(value: f32) -> f32 { return value; }\n"), class: "control"},
+ }
+
+ for _, witness := range witnesses {
+ witness := witness
+ t.Run(witness.name, func(t *testing.T) {
+ cParser := sitter.NewParser()
+ t.Cleanup(cParser.Close)
+ if err := cParser.SetLanguage(cLanguage); err != nil {
+ t.Fatal(err)
+ }
+ cTree := cParser.Parse(witness.source, nil)
+ if cTree == nil || cTree.RootNode() == nil {
+ t.Fatal("locked C parser returned no tree")
+ }
+ t.Cleanup(cTree.Close)
+ cDigest, err := COracleDeepDigest(cTree)
+ if err != nil {
+ t.Fatal(err)
+ }
+ want := wgslNextWant(witness.name)
+ if cDigest != want.cDigest {
+ t.Fatalf("locked C digest=%s, want %s", cDigest, want.cDigest)
+ }
+
+ raw := wgslParseRoute(t, goLanguage, witness.source, "raw", func(p *gotreesitter.Parser, source []byte) (*gotreesitter.Tree, error) {
+ return p.ParseNoResultCompatibilityBenchmarkOnly(source)
+ })
+ production := wgslParseRoute(t, goLanguage, witness.source, "production", func(p *gotreesitter.Parser, source []byte) (*gotreesitter.Tree, error) {
+ return p.Parse(source)
+ })
+ compactParser := gotreesitter.NewParser(goLanguage)
+ compactParser.SetAdmissionCandidateRoute(true)
+ routedBefore, fallbackBefore := gotreesitter.AdmissionCandidateCounters()
+ compact := wgslParseRoute(t, goLanguage, witness.source, "compact", func(_ *gotreesitter.Parser, source []byte) (*gotreesitter.Tree, error) {
+ return compactParser.Parse(source)
+ })
+ routedAfter, fallbackAfter := gotreesitter.AdmissionCandidateCounters()
+ compactRoute := "accepted"
+ if fallbackAfter > fallbackBefore {
+ compactRoute = "fallback:" + gotreesitter.AdmissionCandidateLastFallbackReason()
+ }
+
+ forestParser := gotreesitter.NewParser(goLanguage)
+ forest, forestOK := forestParser.ParseForestExperimental(witness.source)
+ forestRoute := "declined"
+ if forestOK && forest != nil {
+ forestRoute = "accepted"
+ t.Cleanup(forest.Release)
+ } else {
+ offset, symbol, reason, _ := forestParser.ForestDeclineInfo()
+ forestRoute = fmt.Sprintf("declined:%s@%d/%d", reason, offset, symbol)
+ }
+
+ base := bytes.TrimSuffix(witness.source, []byte{'\n'})
+ incrementalParser := gotreesitter.NewParser(goLanguage)
+ oldTree, err := incrementalParser.Parse(base)
+ if err != nil {
+ t.Fatalf("incremental base parse: %v", err)
+ }
+ t.Cleanup(oldTree.Release)
+ oldTree.Edit(gotreesitter.InputEdit{
+ StartByte: uint32(len(base)),
+ OldEndByte: uint32(len(base)),
+ NewEndByte: uint32(len(witness.source)),
+ StartPoint: wgslPointAtByte(base),
+ OldEndPoint: wgslPointAtByte(base),
+ NewEndPoint: wgslPointAtByte(witness.source),
+ })
+ incremental, profile, err := incrementalParser.ParseIncrementalProfiled(witness.source, oldTree)
+ if err != nil {
+ t.Fatalf("incremental parse: %v", err)
+ }
+ t.Cleanup(incremental.Release)
+
+ for _, route := range []struct {
+ name string
+ tree *gotreesitter.Tree
+ }{
+ {name: "raw", tree: raw},
+ {name: "production", tree: production},
+ {name: "compact", tree: compact},
+ {name: "incremental", tree: incremental},
+ } {
+ if route.tree == nil || route.tree.RootNode() == nil {
+ t.Fatalf("%s route returned no root", route.name)
+ }
+ inspection, err := benchfixtures.InspectGoTree(route.tree.RootNode(), goLanguage)
+ if err != nil {
+ t.Fatalf("%s inspect Go tree: %v", route.name, err)
+ }
+ diff := FirstDivergenceDumpV1(route.tree.RootNode(), goLanguage, cTree.RootNode())
+ expectation, ok := want.routes[route.name]
+ if !ok {
+ t.Fatalf("missing %s expectation", route.name)
+ }
+ wantError := want.wantError
+ if expectation.rootErrorSet {
+ wantError = expectation.rootError
+ }
+ if route.tree.RootNode().HasError() != wantError {
+ t.Fatalf("%s root error=%t, want %t", route.name, route.tree.RootNode().HasError(), wantError)
+ }
+ if inspection.SHA256 != expectation.digest {
+ t.Fatalf("%s digest=%s, want %s", route.name, inspection.SHA256, expectation.digest)
+ }
+ if got := wgslDispatchReceipt(route.tree); got != expectation.receipt {
+ t.Fatalf("%s dispatch receipt=%s, want %s", route.name, got, expectation.receipt)
+ }
+ wgslRequireDivergence(t, route.name, diff, expectation.diff)
+ t.Logf("witness=%s class=%s route=%s bytes=%d source_sha256=%x root_error=%t digest=%s c_digest=%s divergence=%+v dispatch=%s", witness.name, witness.class, route.name, len(witness.source), sha256.Sum256(witness.source), route.tree.RootNode().HasError(), inspection.SHA256, cDigest, diff, wgslDispatchReceipt(route.tree))
+ }
+ if forest != nil {
+ inspection, err := benchfixtures.InspectGoTree(forest.RootNode(), goLanguage)
+ if err != nil {
+ t.Fatalf("forest inspect Go tree: %v", err)
+ }
+ diff := FirstDivergenceDumpV1(forest.RootNode(), goLanguage, cTree.RootNode())
+ if want.forest == nil {
+ t.Fatalf("forest returned a tree, want decline %s", want.forestRoute)
+ }
+ if forest.RootNode().HasError() != want.wantError {
+ t.Fatalf("forest root error=%t, want %t", forest.RootNode().HasError(), want.wantError)
+ }
+ if inspection.SHA256 != want.forest.digest {
+ t.Fatalf("forest digest=%s, want %s", inspection.SHA256, want.forest.digest)
+ }
+ if got := wgslDispatchReceipt(forest); got != want.forest.receipt {
+ t.Fatalf("forest dispatch receipt=%s, want %s", got, want.forest.receipt)
+ }
+ wgslRequireDivergence(t, "forest", diff, want.forest.diff)
+ t.Logf("witness=%s class=%s route=forest bytes=%d source_sha256=%x root_error=%t digest=%s c_digest=%s divergence=%+v dispatch=%s", witness.name, witness.class, len(witness.source), sha256.Sum256(witness.source), forest.RootNode().HasError(), inspection.SHA256, cDigest, diff, wgslDispatchReceipt(forest))
+ } else if want.forest != nil {
+ t.Fatalf("forest declined, want digest %s", want.forest.digest)
+ }
+ if compactRoute != want.compactRoute {
+ t.Fatalf("compact route=%q, want %q", compactRoute, want.compactRoute)
+ }
+ if forestRoute != want.forestRoute {
+ t.Fatalf("forest route=%q, want %q", forestRoute, want.forestRoute)
+ }
+ if profile.OldTreeReuseRoute != true || profile.ReuseUnsupported {
+ t.Fatalf("incremental reuse route=%t unsupported=%t reason=%q, want reuse without fallback", profile.OldTreeReuseRoute, profile.ReuseUnsupported, profile.ReuseUnsupportedReason)
+ }
+ if profile.ReusedSubtrees != want.reuseSubtrees || profile.ReusedBytes != want.reuseBytes {
+ t.Fatalf("incremental reuse=%d subtrees/%d bytes, want %d/%d", profile.ReusedSubtrees, profile.ReusedBytes, want.reuseSubtrees, want.reuseBytes)
+ }
+ t.Logf("witness=%s compact=%s counters=%d/%d->%d/%d forest=%s incremental_reuse=%t reuse_unsupported=%t reuse_reason=%q reused_subtrees=%d reused_bytes=%d", witness.name, compactRoute, routedBefore, fallbackBefore, routedAfter, fallbackAfter, forestRoute, profile.OldTreeReuseRoute, profile.ReuseUnsupported, profile.ReuseUnsupportedReason, profile.ReusedSubtrees, profile.ReusedBytes)
+ })
+ }
+}
+
+func wgslRequireDivergence(t *testing.T, route string, got, want *DumpV1Divergence) {
+ t.Helper()
+ if got == nil || want == nil {
+ if got != want {
+ t.Fatalf("%s divergence=%+v, want %+v", route, got, want)
+ }
+ return
+ }
+ if *got != *want {
+ t.Fatalf("%s divergence=%+v, want %+v", route, got, want)
+ }
+}
+
+func wgslDiff(path, category, goValue, cValue string) *DumpV1Divergence {
+ return &DumpV1Divergence{Path: path, Category: category, GoValue: goValue, CValue: cValue}
+}
+
+func wgslNextWant(name string) wgslNextExpectation {
+ switch name {
+ case "a0-small-fragmentTextureQuad":
+ const digest = "d3e58954c750ed560edd3177a165bbf701c159467a1b4677996bec620c377804"
+ return wgslNextExpectation{
+ cDigest: digest,
+ compactRoute: "accepted",
+ forestRoute: "accepted",
+ reuseSubtrees: 49,
+ reuseBytes: 192,
+ routes: map[string]wgslNextRouteExpectation{
+ "raw": {digest: digest, receipt: "none"},
+ "production": {digest: digest, receipt: "none"},
+ "compact": {digest: digest, receipt: "none"},
+ "incremental": {digest: digest, receipt: "1/1/112/51"},
+ },
+ forest: &wgslNextRouteExpectation{digest: digest, receipt: "1/1/111/0"},
+ }
+ case "a0-medium-normalMap":
+ return wgslNextExpectation{
+ cDigest: "231e10ca2215945a5fb51670620c9f5ba2ea1ca7d445cb2c9443fb51b8e0e18a",
+ wantError: true,
+ compactRoute: "fallback:compact route declined at recovery [mechanism=recovery-entered]: did not accept EOF: generic scheduler has no table action for the elected token",
+ forestRoute: "declined:dead_end@0/1",
+ reuseSubtrees: 452,
+ reuseBytes: 2318,
+ routes: map[string]wgslNextRouteExpectation{
+ "raw": {digest: "77fdfd002d6937e6f5784fc19e21a6f63ab8f2280ca8ba0dcfc1ee5b1d3d42cc", receipt: "none", diff: wgslDiff("/source_file", "shape", "children=70", "children=79")},
+ "production": {digest: "9d802a0e9af71176c0520496ae99425406aa76dc8560f8c7e939e366f9fbbd44", receipt: "1/1/1289/120", diff: wgslDiff("/source_file", "shape", "children=70", "children=79")},
+ "compact": {digest: "9d802a0e9af71176c0520496ae99425406aa76dc8560f8c7e939e366f9fbbd44", receipt: "1/1/1289/120", diff: wgslDiff("/source_file", "shape", "children=70", "children=79")},
+ "incremental": {digest: "fe0cb9f758eaace140619c81a9ef3347d89633580b18492e46dcfae6fb8f57c7", receipt: "1/1/1287/684", diff: wgslDiff("/source_file", "shape", "children=69", "children=79")},
+ },
+ }
+ case "a0-medium-radiosity":
+ return wgslNextExpectation{
+ cDigest: "22b9d004c33c6a8229b56876282125e04efddf59deef6224eddd61f38c9952b2",
+ wantError: true,
+ compactRoute: "fallback:compact route declined at recovery [mechanism=recovery-entered]: did not accept EOF: generic scheduler has no table action for the elected token",
+ forestRoute: "declined:dead_end@247/43",
+ reuseSubtrees: 185,
+ reuseBytes: 1197,
+ routes: map[string]wgslNextRouteExpectation{
+ "raw": {digest: "c591b9329ad2fc946b6b8b7c4bc80adb7305f41934dd51d4505e4f606787b127", receipt: "none", diff: wgslDiff("/source_file/global_variable_declaration[3]/variable_declaration[2]/variable_identifier_declaration[2]/type_declaration[2]/ERROR[1]", "type", "ERROR", "<")},
+ "production": {digest: "592abfad21a9a3170c11fd2e18888a9c5ecac7f681af5cf81e2d1c352873df63", receipt: "1/1/1230/51", diff: wgslDiff("/source_file/function_declaration[29]", "error", "false", "true")},
+ "compact": {digest: "592abfad21a9a3170c11fd2e18888a9c5ecac7f681af5cf81e2d1c352873df63", receipt: "1/1/1230/51", diff: wgslDiff("/source_file/function_declaration[29]", "error", "false", "true")},
+ "incremental": {digest: "9e0113134b748e66ba22390bdf09e6e083a7f09dc0fc3b1dec67a407fed979cd", receipt: "1/1/1230/56", diff: wgslDiff("/source_file", "shape", "children=46", "children=48")},
+ },
+ }
+ case "recovery-empty-return":
+ const digest = "909c5e3b5efd2201372daf83848ea91cc459c1069537f89fcebf1f7c49cd58f4"
+ return wgslNextExpectation{
+ cDigest: digest,
+ compactRoute: "accepted",
+ forestRoute: "accepted",
+ reuseSubtrees: 6,
+ reuseBytes: 20,
+ routes: map[string]wgslNextRouteExpectation{
+ "raw": {digest: digest, receipt: "none"},
+ "production": {digest: digest, receipt: "none"},
+ "compact": {digest: digest, receipt: "none"},
+ "incremental": {digest: digest, receipt: "1/1/12/0"},
+ },
+ forest: &wgslNextRouteExpectation{digest: digest, receipt: "1/1/12/0"},
+ }
+ case "malformed-missing-expression":
+ const digest = "dcdd782dbf759ef627faf475817ba14b68ae5b0783e431fc06a1bee5a879e73b"
+ const normalizedDigest = "88373bc4d0bc650c42c159273d5cc2a6915b2c69334adc3d5e8923a421fce02b"
+ return wgslNextExpectation{
+ cDigest: digest,
+ wantError: true,
+ compactRoute: "fallback:compact route declined at recovery [mechanism=recovery-entered]: did not accept EOF: generic scheduler has no table action for the elected token",
+ forestRoute: "declined:dead_end@34/3",
+ reuseSubtrees: 9,
+ reuseBytes: 26,
+ routes: map[string]wgslNextRouteExpectation{
+ "raw": {digest: digest, receipt: "none"},
+ "production": {digest: normalizedDigest, receipt: "1/1/19/5", diff: wgslDiff("/source_file", "error", "false", "true"), rootErrorSet: true, rootError: false},
+ "compact": {digest: normalizedDigest, receipt: "1/1/19/5", diff: wgslDiff("/source_file", "error", "false", "true"), rootErrorSet: true, rootError: false},
+ "incremental": {digest: normalizedDigest, receipt: "1/1/19/5", diff: wgslDiff("/source_file", "error", "false", "true"), rootErrorSet: true, rootError: false},
+ },
+ }
+ case "malformed-argument-list":
+ return wgslNextExpectation{
+ cDigest: "4a43e477628bba014be5e5863dc87ab460c14c7232c96aa737414f299a407e81",
+ wantError: true,
+ compactRoute: "fallback:compact route declined at recovery [mechanism=recovery-entered]: did not accept EOF: generic scheduler has no table action for the elected token",
+ forestRoute: "declined:dead_end@28/7",
+ reuseSubtrees: 6,
+ reuseBytes: 25,
+ routes: map[string]wgslNextRouteExpectation{
+ "raw": {digest: "40a24a35095668102b40bd1bd92f5576211eaf92b3aac01f24b5e81a6ab755a9", receipt: "none", diff: wgslDiff("/source_file/function_declaration[0]/compound_statement[4]/assignment_statement[1]/parenthesized_expression[2]/ERROR[2]/,[0]", "error", "true", "false")},
+ "production": {digest: "2555f0b34dfb5e295bc72d8d5532ceadfd6807686edc2e417a5e8ae911aa9df7", receipt: "1/1/23/5", diff: wgslDiff("/source_file/function_declaration[0]/compound_statement[4]/assignment_statement[1]/compound_assignment_operator[1]", "error", "false", "true")},
+ "compact": {digest: "2555f0b34dfb5e295bc72d8d5532ceadfd6807686edc2e417a5e8ae911aa9df7", receipt: "1/1/23/5", diff: wgslDiff("/source_file/function_declaration[0]/compound_statement[4]/assignment_statement[1]/compound_assignment_operator[1]", "error", "false", "true")},
+ "incremental": {digest: "2555f0b34dfb5e295bc72d8d5532ceadfd6807686edc2e417a5e8ae911aa9df7", receipt: "1/1/23/5", diff: wgslDiff("/source_file/function_declaration[0]/compound_statement[4]/assignment_statement[1]/compound_assignment_operator[1]", "error", "false", "true")},
+ },
+ }
+ case "positive-control":
+ const digest = "7868a6b43efc8d14b73746e2472a596c9eeb3cd215f734021528aaa3668f05ea"
+ return wgslNextExpectation{
+ cDigest: digest,
+ compactRoute: "accepted",
+ forestRoute: "accepted",
+ reuseSubtrees: 11,
+ reuseBytes: 37,
+ routes: map[string]wgslNextRouteExpectation{
+ "raw": {digest: digest, receipt: "none"},
+ "production": {digest: digest, receipt: "none"},
+ "compact": {digest: digest, receipt: "none"},
+ "incremental": {digest: digest, receipt: "1/1/24/0"},
+ },
+ forest: &wgslNextRouteExpectation{digest: digest, receipt: "1/1/24/0"},
+ }
+ default:
+ panic("missing WGSL expectation for " + name)
+ }
+}
+
+func wgslReadA0(t *testing.T, name string) []byte {
+ t.Helper()
+ source, err := os.ReadFile(filepath.Join("..", "testdata", "dispatcher_census_a0", "wgsl", name))
+ if err != nil {
+ t.Fatal(err)
+ }
+ return source
+}
+
+func wgslParseRoute(t *testing.T, language *gotreesitter.Language, source []byte, route string, parse func(*gotreesitter.Parser, []byte) (*gotreesitter.Tree, error)) *gotreesitter.Tree {
+ t.Helper()
+ parser := gotreesitter.NewParser(language)
+ tree, err := parse(parser, source)
+ if err != nil {
+ t.Fatalf("%s parse: %v", route, err)
+ }
+ if tree == nil || tree.RootNode() == nil {
+ t.Fatalf("%s parse returned no tree", route)
+ }
+ t.Cleanup(tree.Release)
+ return tree
+}
+
+func wgslDispatchReceipt(tree *gotreesitter.Tree) string {
+ if tree == nil || tree.ParseRuntime().NormalizationPasses == nil {
+ return "none"
+ }
+ for _, pass := range *tree.ParseRuntime().NormalizationPasses {
+ if pass.Name == "dispatch.wgsl" {
+ return fmt.Sprintf("%d/%d/%d/%d", pass.Checked, pass.Run, pass.NodesVisited, pass.NodesRewritten)
+ }
+ }
+ return "none"
+}
+
+func wgslPointAtByte(source []byte) gotreesitter.Point {
+ var point gotreesitter.Point
+ for _, value := range source {
+ if value == '\n' {
+ point.Row++
+ point.Column = 0
+ } else {
+ point.Column++
+ }
+ }
+ return point
+}
+
+// TestWGSLNextLiveArmReceiptDocument guards the blocker markers.
+func TestWGSLNextLiveArmReceiptDocument(t *testing.T) {
+ raw, err := os.ReadFile("../docs/root-normalization-retirement.md")
+ if err != nil {
+ t.Fatal(err)
+ }
+ document := strings.Join(strings.Fields(string(raw)), " ")
+ for _, marker := range []string{
+ "Status: NO-GO. KEEP LIVE: `dispatch.wgsl`.",
+ "The WGSL A0 receipt contains three files, three checks, three runs, 2,630 visited nodes, 171 rewrites, two error roots, and zero parse errors.",
+ "The tracked census has seven fixtures across six languages. It excludes WGSL.",
+ "The focused receipt uses seven witnesses.",
+ "The `normalMap` A0 witness exposes the first live blocker.",
+ "The malformed missing-expression control matches locked C on its raw route.",
+ "The malformed argument-list control keeps a live rewrite.",
+ "The two malformed controls each add five rewrites.",
+ "The probe changes only the receipt test, changelog, and retirement ledger.",
+ "Keep `dispatch.wgsl` live until the producer emits the locked-C trees for all registered witnesses and all listed routes.",
+ } {
+ marker = strings.Join(strings.Fields(marker), " ")
+ if !strings.Contains(document, marker) {
+ t.Fatalf("WGSL blocker receipt lacks marker %q", marker)
+ }
+ }
+}
+
+// TestWGSLNextLiveChangelogReceipt guards the changelog marker.
+func TestWGSLNextLiveChangelogReceipt(t *testing.T) {
+ raw, err := os.ReadFile("../CHANGELOG.md")
+ if err != nil {
+ t.Fatal(err)
+ }
+ document := strings.Join(strings.Fields(string(raw)), " ")
+ for _, marker := range []string{
+ "Record the `dispatch.wgsl` blocker receipt at base `7498a678c52029a82f312e9637ecb66b15defa0b`.",
+ "The A0 manifest records three WGSL files, three checks, three runs, and 171 rewrites.",
+ "Both malformed controls rewrite five nodes on normalized routes. The production, compact, and incremental routes for both controls diverge from locked C.",
+ "No registry or production code changes are included.",
+ } {
+ marker = strings.Join(strings.Fields(marker), " ")
+ if !strings.Contains(document, marker) {
+ t.Fatalf("WGSL changelog lacks marker %q", marker)
+ }
+ }
+}
diff --git a/cgo_harness/wolfram_retirement_parity_test.go b/cgo_harness/wolfram_retirement_parity_test.go
new file mode 100644
index 000000000..0df495a46
--- /dev/null
+++ b/cgo_harness/wolfram_retirement_parity_test.go
@@ -0,0 +1,146 @@
+//go:build cgo && treesitter_c_parity
+
+package cgoharness
+
+import (
+ "crypto/sha256"
+ "fmt"
+ "os"
+ "path/filepath"
+ "testing"
+
+ gotreesitter "github.com/odvcencio/gotreesitter"
+ "github.com/odvcencio/gotreesitter/internal/benchfixtures"
+ sitter "github.com/tree-sitter/go-tree-sitter"
+)
+
+// TestWolframDispatchRetirementLockedCParity compares all three parser-produced
+// Wolfram A0 witnesses with the locked C parser on raw and production routes.
+func TestWolframDispatchRetirementLockedCParity(t *testing.T) {
+ entry, ok := parityEntriesByName["wolfram"]
+ if !ok {
+ t.Fatal("missing Wolfram grammar entry")
+ }
+ language := entry.Language()
+ cLanguage, err := COracleLanguage("wolfram")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ tests := []struct {
+ name string
+ file string
+ sha256 string
+ }{
+ {
+ name: "evaluation-utilities",
+ file: "large__EvaluationUtilities.wl",
+ sha256: "e03c8588214ce3a0a5ba48d1f1335276c1826356052c33df1f3184a6d6303a53",
+ },
+ {
+ name: "output-handling-utilities",
+ file: "medium__OutputHandlingUtilities.wl",
+ sha256: "45a6287c3c8ad5f4f37298d4915d1bfb29e6e91ee0eccde1c842efb7c90e3dec",
+ },
+ {
+ name: "paclet-info",
+ file: "small__PacletInfo.m",
+ sha256: "55be9b6143e5dd68ddb433bb9c95c0388a505b65c452fb6036e064d537e3f602",
+ },
+ }
+
+ for _, test := range tests {
+ test := test
+ t.Run(test.name, func(t *testing.T) {
+ source, err := os.ReadFile(filepath.Join(
+ "..", "testdata", "dispatcher_census_a0", "wolfram", test.file,
+ ))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got := fmt.Sprintf("%x", sha256.Sum256(source)); got != test.sha256 {
+ t.Fatalf("source SHA-256 = %s, want %s", got, test.sha256)
+ }
+
+ rawParser := gotreesitter.NewParser(language)
+ rawParser.SetAdmissionCandidateRoute(false)
+ rawTree, err := rawParser.ParseNoResultCompatibilityBenchmarkOnly(source)
+ if err != nil {
+ t.Fatalf("raw parse: %v", err)
+ }
+ t.Cleanup(rawTree.Release)
+ rawRuntime := rawTree.ParseRuntime()
+ if rawRuntime.NormalizationNodesRewritten != 0 {
+ t.Fatalf("raw normalization rewrote %d nodes", rawRuntime.NormalizationNodesRewritten)
+ }
+
+ productionParser := gotreesitter.NewParser(language)
+ productionParser.SetAdmissionCandidateRoute(false)
+ productionTree, err := productionParser.Parse(source)
+ if err != nil {
+ t.Fatalf("production parse: %v", err)
+ }
+ t.Cleanup(productionTree.Release)
+ productionRuntime := productionTree.ParseRuntime()
+ if productionRuntime.NormalizationNodesRewritten != 0 {
+ t.Fatalf("production normalization rewrote %d nodes", productionRuntime.NormalizationNodesRewritten)
+ }
+
+ cParser := sitter.NewParser()
+ t.Cleanup(cParser.Close)
+ if err := cParser.SetLanguage(cLanguage); err != nil {
+ t.Fatal(err)
+ }
+ cTree := cParser.Parse(source, nil)
+ if cTree == nil || cTree.RootNode() == nil {
+ t.Fatal("C oracle returned a nil tree")
+ }
+ t.Cleanup(cTree.Close)
+
+ cDigest, err := COracleDeepDigest(cTree)
+ if err != nil {
+ t.Fatalf("inspect C deep tree: %v", err)
+ }
+ rawDigest := assertWolframLockedCTreeExact(t, "raw", rawTree, language, cTree, cDigest)
+ productionDigest := assertWolframLockedCTreeExact(t, "production", productionTree, language, cTree, cDigest)
+ t.Logf(
+ "witness=%s bytes=%d source_sha256=%s raw_digest=%s production_digest=%s raw_rewrites=%d production_rewrites=%d",
+ test.file,
+ len(source),
+ test.sha256,
+ rawDigest,
+ productionDigest,
+ rawRuntime.NormalizationNodesRewritten,
+ productionRuntime.NormalizationNodesRewritten,
+ )
+ })
+ }
+}
+
+func assertWolframLockedCTreeExact(
+ t *testing.T,
+ label string,
+ goTree *gotreesitter.Tree,
+ goLang *gotreesitter.Language,
+ cTree *sitter.Tree,
+ wantDigest string,
+) string {
+ t.Helper()
+ goRoot := goTree.RootNode()
+ cRoot := cTree.RootNode()
+ if diff := FirstDivergenceDumpV1(goRoot, goLang, cRoot); diff != nil {
+ t.Fatalf("%s tree diverges from the locked C oracle: %+v", label, diff)
+ }
+ if diff := firstLockedCTreeFlagDivergence(goRoot, goLang, cRoot, "/"+goRoot.Type(goLang)); diff != nil {
+ t.Fatalf("%s tree has a missing or error flag divergence: %v", label, diff)
+ }
+ inspection, err := benchfixtures.InspectGoTree(goRoot, goLang)
+ if err != nil {
+ t.Fatalf("inspect %s Go deep tree: %v", label, err)
+ }
+ if inspection.SHA256 != wantDigest {
+ t.Fatalf("%s deep digest Go=%s C=%s", label, inspection.SHA256, wantDigest)
+ }
+ t.Logf("%s route matches locked C exactly: symbols, fields, spans, points, extras, missing/error flags, deep digest=%s", label, inspection.SHA256)
+ return inspection.SHA256
+}
diff --git a/diagnostic_token_source_fork.go b/diagnostic_token_source_fork.go
new file mode 100644
index 000000000..bd1da46cf
--- /dev/null
+++ b/diagnostic_token_source_fork.go
@@ -0,0 +1,88 @@
+//go:build gts_workcount
+
+package gotreesitter
+
+import (
+ "crypto/sha256"
+ "encoding/binary"
+ "fmt"
+)
+
+// DiagnosticTokenSourceForkReceipt records the state at a diagnostic fork.
+// Production builds do not include this type.
+type DiagnosticTokenSourceForkReceipt struct {
+ TokenSourceKind string
+ CursorByte uint32
+ CursorPoint Point
+ ParserState StateID
+ ActiveGLRStates []StateID
+ PendingTokenCount int
+ PendingTokenDigest [sha256.Size]byte
+ ZeroWidthOffset int64
+ IncludedRangeIndex int
+ IncludedRangeCount int
+ IncludedRangeDigest [sha256.Size]byte
+ ExternalScannerCheckpointPresent bool
+ ExternalScannerCheckpointDigest [sha256.Size]byte
+}
+
+type diagnosticTokenSourceForker interface {
+ ForkTokenSourceForDiagnostic() (TokenSource, DiagnosticTokenSourceForkReceipt, error)
+}
+
+func forkTokenSourceForDiagnostic(source TokenSource) (TokenSource, DiagnosticTokenSourceForkReceipt, error) {
+ if source == nil {
+ return nil, DiagnosticTokenSourceForkReceipt{}, fmt.Errorf("diagnostic token fork: source is nil")
+ }
+ forker, ok := source.(diagnosticTokenSourceForker)
+ if !ok {
+ return nil, DiagnosticTokenSourceForkReceipt{}, fmt.Errorf("diagnostic token fork: source %T cannot fork", source)
+ }
+ return forker.ForkTokenSourceForDiagnostic()
+}
+
+func (s *includedRangeTokenSource) ForkTokenSourceForDiagnostic() (TokenSource, DiagnosticTokenSourceForkReceipt, error) {
+ if s == nil || s.base == nil {
+ return nil, DiagnosticTokenSourceForkReceipt{}, fmt.Errorf("diagnostic token fork: included-range source is nil")
+ }
+ if s.idx < 0 || s.idx > len(s.ranges) {
+ return nil, DiagnosticTokenSourceForkReceipt{}, fmt.Errorf("diagnostic token fork: included-range index %d is outside 0..%d", s.idx, len(s.ranges))
+ }
+
+ base, receipt, err := forkTokenSourceForDiagnostic(s.base)
+ if err != nil {
+ return nil, DiagnosticTokenSourceForkReceipt{}, err
+ }
+ ranges := append([]Range(nil), s.ranges...)
+ fork := &includedRangeTokenSource{
+ base: base,
+ ranges: ranges,
+ idx: s.idx,
+ }
+ receipt.TokenSourceKind = "included-range/" + receipt.TokenSourceKind
+ receipt.IncludedRangeIndex = s.idx
+ receipt.IncludedRangeCount = len(s.ranges)
+ receipt.IncludedRangeDigest = diagnosticIncludedRangeDigest(s.ranges)
+ return fork, receipt, nil
+}
+
+func diagnosticIncludedRangeDigest(ranges []Range) [sha256.Size]byte {
+ hash := sha256.New()
+ var value [8]byte
+ write := func(v uint64) {
+ binary.LittleEndian.PutUint64(value[:], v)
+ _, _ = hash.Write(value[:])
+ }
+ write(uint64(len(ranges)))
+ for _, item := range ranges {
+ write(uint64(item.StartByte))
+ write(uint64(item.EndByte))
+ write(uint64(item.StartPoint.Row))
+ write(uint64(item.StartPoint.Column))
+ write(uint64(item.EndPoint.Row))
+ write(uint64(item.EndPoint.Column))
+ }
+ var digest [sha256.Size]byte
+ copy(digest[:], hash.Sum(nil))
+ return digest
+}
diff --git a/diagnostic_token_source_fork_test.go b/diagnostic_token_source_fork_test.go
new file mode 100644
index 000000000..38261bb52
--- /dev/null
+++ b/diagnostic_token_source_fork_test.go
@@ -0,0 +1,165 @@
+//go:build gts_workcount
+
+package gotreesitter
+
+import (
+ "crypto/sha256"
+ "reflect"
+ "testing"
+)
+
+type diagnosticForkStub struct {
+ tokens []Token
+ index int
+ state StateID
+ glrStates []StateID
+}
+
+func (s *diagnosticForkStub) Next() Token {
+ if s.index >= len(s.tokens) {
+ return Token{}
+ }
+ token := s.tokens[s.index]
+ s.index++
+ return token
+}
+
+func (s *diagnosticForkStub) SetParserState(state StateID) {
+ s.state = state
+}
+
+func (s *diagnosticForkStub) SetGLRStates(states []StateID) {
+ s.glrStates = append(s.glrStates[:0], states...)
+}
+
+func (s *diagnosticForkStub) ForkTokenSourceForDiagnostic() (TokenSource, DiagnosticTokenSourceForkReceipt, error) {
+ fork := *s
+ fork.tokens = append([]Token(nil), s.tokens...)
+ fork.glrStates = append([]StateID(nil), s.glrStates...)
+ return &fork, DiagnosticTokenSourceForkReceipt{
+ TokenSourceKind: "stub",
+ ParserState: s.state,
+ ActiveGLRStates: append([]StateID(nil), s.glrStates...),
+ IncludedRangeIndex: -1,
+ ExternalScannerCheckpointPresent: true,
+ ExternalScannerCheckpointDigest: sha256.Sum256([]byte("stub scanner")),
+ }, nil
+}
+
+func TestDiagnosticIncludedRangeForkIsIndependent(t *testing.T) {
+ base := &diagnosticForkStub{
+ tokens: []Token{
+ {Symbol: 1, Text: "outside", StartByte: 0, EndByte: 7},
+ {Symbol: 2, Text: "first", StartByte: 12, EndByte: 17},
+ {Symbol: 3, Text: "second", StartByte: 31, EndByte: 37},
+ {Symbol: 4, Text: "third", StartByte: 38, EndByte: 39},
+ {},
+ },
+ state: 77,
+ glrStates: []StateID{7, 11},
+ }
+ ranges := []Range{
+ {StartByte: 10, EndByte: 20, StartPoint: Point{Column: 10}, EndPoint: Point{Column: 20}},
+ {StartByte: 30, EndByte: 40, StartPoint: Point{Row: 1}, EndPoint: Point{Row: 1, Column: 10}},
+ }
+ live := newIncludedRangeTokenSource(base, ranges).(*includedRangeTokenSource)
+ if token := live.Next(); token.Symbol != 2 {
+ t.Fatalf("first included token symbol = %d, want 2", token.Symbol)
+ }
+ if live.idx != 0 {
+ t.Fatalf("live range index = %d, want 0", live.idx)
+ }
+
+ liveBaseIndex := base.index
+ liveRanges := append([]Range(nil), live.ranges...)
+ forkSource, receipt, err := forkTokenSourceForDiagnostic(live)
+ if err != nil {
+ t.Fatalf("forkTokenSourceForDiagnostic failed: %v", err)
+ }
+ fork, ok := forkSource.(*includedRangeTokenSource)
+ if !ok {
+ t.Fatalf("fork type = %T, want *includedRangeTokenSource", forkSource)
+ }
+ forkBase, ok := fork.base.(*diagnosticForkStub)
+ if !ok {
+ t.Fatalf("fork base type = %T, want *diagnosticForkStub", fork.base)
+ }
+ if !reflect.DeepEqual(fork, live) {
+ t.Fatalf("fork state differs before advance:\n got: %#v\nwant: %#v", fork, live)
+ }
+ if fork == live || forkBase == base {
+ t.Fatal("fork shares a mutable source object with the live source")
+ }
+ if &fork.ranges[0] == &live.ranges[0] {
+ t.Fatal("fork shares the included-range backing array")
+ }
+ if &forkBase.tokens[0] == &base.tokens[0] {
+ t.Fatal("fork shares the base token backing array")
+ }
+ if &forkBase.glrStates[0] == &base.glrStates[0] {
+ t.Fatal("fork shares the base GLR-state backing array")
+ }
+ if got, want := receipt.TokenSourceKind, "included-range/stub"; got != want {
+ t.Fatalf("receipt kind = %q, want %q", got, want)
+ }
+ if receipt.IncludedRangeIndex != live.idx || receipt.IncludedRangeCount != len(live.ranges) {
+ t.Fatalf("receipt range state = %d/%d, want %d/%d", receipt.IncludedRangeIndex, receipt.IncludedRangeCount, live.idx, len(live.ranges))
+ }
+ if receipt.ParserState != base.state {
+ t.Fatalf("receipt parser state = %d, want %d", receipt.ParserState, base.state)
+ }
+ if !reflect.DeepEqual(receipt.ActiveGLRStates, base.glrStates) {
+ t.Fatalf("receipt GLR states = %v, want %v", receipt.ActiveGLRStates, base.glrStates)
+ }
+ if got, want := receipt.IncludedRangeDigest, diagnosticIncludedRangeDigest(live.ranges); got != want {
+ t.Fatalf("receipt range digest = %x, want %x", got, want)
+ }
+ if !receipt.ExternalScannerCheckpointPresent {
+ t.Fatal("wrapper discarded the base scanner checkpoint state")
+ }
+
+ fork.SetParserState(99)
+ fork.SetGLRStates([]StateID{13, 17})
+ if base.state != 77 || !reflect.DeepEqual(base.glrStates, []StateID{7, 11}) {
+ t.Fatalf("fork state update changed live base: state=%d glr=%v", base.state, base.glrStates)
+ }
+ forkToken := fork.Next()
+ if forkToken.Symbol != 3 || fork.idx != 1 {
+ t.Fatalf("fork next state = symbol %d at range %d, want symbol 3 at range 1", forkToken.Symbol, fork.idx)
+ }
+ if base.index != liveBaseIndex || live.idx != 0 || !reflect.DeepEqual(live.ranges, liveRanges) {
+ t.Fatalf("fork advance changed live state: base index=%d range index=%d ranges=%v", base.index, live.idx, live.ranges)
+ }
+ if liveToken := live.Next(); !reflect.DeepEqual(liveToken, forkToken) {
+ t.Fatalf("live next token = %+v, want fork token %+v", liveToken, forkToken)
+ }
+
+ liveBaseIndex = base.index
+ forkToken = fork.Next()
+ if forkToken.Symbol != 4 {
+ t.Fatalf("fork next symbol = %d, want 4", forkToken.Symbol)
+ }
+ if base.index != liveBaseIndex || live.idx != 1 {
+ t.Fatalf("second fork advance changed live state: base index=%d range index=%d", base.index, live.idx)
+ }
+ if liveToken := live.Next(); !reflect.DeepEqual(liveToken, forkToken) {
+ t.Fatalf("second live token = %+v, want fork token %+v", liveToken, forkToken)
+ }
+
+ fork.ranges[0].EndByte++
+ if reflect.DeepEqual(fork.ranges, live.ranges) {
+ t.Fatal("fork range mutation reached the live wrapper")
+ }
+}
+
+func TestDiagnosticIncludedRangeForkRejectsUnsupportedBase(t *testing.T) {
+ base := &stubTokenSource{}
+ live := newIncludedRangeTokenSource(base, []Range{{StartByte: 1, EndByte: 2}}).(*includedRangeTokenSource)
+ indexBefore := live.idx
+ if _, _, err := forkTokenSourceForDiagnostic(live); err == nil {
+ t.Fatal("fork with an unsupported base succeeded")
+ }
+ if base.nextCalls != 0 || base.skipCalls != 0 || live.idx != indexBefore {
+ t.Fatalf("rejected fork changed live source: next=%d skip=%d range index=%d, want 0/0/%d", base.nextCalls, base.skipCalls, live.idx, indexBefore)
+ }
+}
diff --git a/docs/doxygen-normalization-blocker-receipt-proposal.md b/docs/doxygen-normalization-blocker-receipt-proposal.md
new file mode 100644
index 000000000..dea3cfceb
--- /dev/null
+++ b/docs/doxygen-normalization-blocker-receipt-proposal.md
@@ -0,0 +1,129 @@
+# Doxygen normalization blocker receipt proposal
+
+Receipt date: 2026-08-22.
+
+Base commit: `0c34a681db29a3e8d27e488c5f26d6d7a5f02592`.
+
+Disposition: `NO-GO`. Keep `dispatch.doxygen` live. Do not remove the arm,
+its source file, or its registry entry.
+
+## Registry denominator
+
+The registry test passed with these values:
+
+| Measure | Value |
+| --- | ---: |
+| Dispatcher arms | 31 |
+| Dispatcher languages | 33 |
+| Dispatcher predicates | 1 |
+| Generic passes | 0 |
+| Post-finalization arms | 0 |
+| Post-finalization languages | 0 |
+| Live entries | 32 |
+| Retired entries | 56 |
+| Live language labels | 35 |
+
+The A0 manifest test passed with 14 languages and 14 receipts after JSDoc
+retirement. Doxygen retains three A0 fixtures.
+
+## A0 raw and production receipt
+
+All three registered A0 Doxygen fixtures produced equal raw and production
+trees. Each route reported zero rewrites.
+
+| Witness | Source SHA-256 | Raw and production digest | Root | Error |
+| --- | --- | --- | --- | --- |
+| `medium__CMakeLists.txt` | `66408d6539b27d7c49b1e51777605c38c91b6d924267db5109ee00e2a1cfcf41` | `01d09d1ffd9d09af0333bcd887c35e68bcb4a96d15ff0d96c29a1780971b7e04` | `document` | true |
+| `medium__metrics.py` | `31622a6c075ffa6f78a16af6e379f517213d42ff67729bbd0d10551c5fca9702` | `5adbacb1ec949237a802a56a5c95c3c7a1ce17fe9c8db5423b63f083da62d5d1` | `ERROR` | true |
+| `small__example.cfg` | `86998161914382f8152e4984db091e7bf486799c1091fc6c57db4e704eee4a3b` | `3b803e3d4b9ffcf99c771c352118f3f7026420ea5f26c8d934349ac848789b23` | `document` | true |
+
+The A0 census receipt is `files=3 checked=2 run=2 visited=4 rewritten=0
+error_roots=3 parse_errors=0`. The arm does not record a pass for the
+`ERROR` root in `medium__metrics.py`.
+
+The production, compact, forest, and incremental route receipt passed for all
+three fixtures. The compact routes fell back with `parser-core fresh-full
+runner did not accept EOF`. The forest routes fell back at `dead_end`. The
+incremental routes fell back at `external_scanner_unsupported`. Each covered
+route recorded zero Doxygen rewrites.
+
+## Locked-C blocker
+
+The one-grammar locked-C deep probe used the Doxygen lock
+`https://github.com/amaanq/tree-sitter-doxygen` at commit
+`ccd998f378c3f9345ea4eeb223f56d7b84d16687`. It used the C runtime contract
+`tree-sitter-c-v1`, binding `github.com/tree-sitter/go-tree-sitter@v0.25.0`,
+runtime `0.25.1`, and the locked C deep digest.
+
+The production route diverges from locked C for every A0 fixture:
+
+- `medium__CMakeLists.txt`: Go digest `01d09d1ffd9d09af0333bcd887c35e68bcb4a96d15ff0d96c29a1780971b7e04`; C digest `d6f623d2b87344001e98de5528b44e38b102e564491871a9ffb64c1b73d193c5`; first divergence `/document`, type `document` versus `ERROR`.
+- `medium__metrics.py`: Go digest `5adbacb1ec949237a802a56a5c95c3c7a1ce17fe9c8db5423b63f083da62d5d1`; C digest `6660931c2bf1bf1e0f909a1cac1e4cd8446853ae4466781c943e28fbcc61e860`; first divergence `/ERROR`, children `0` versus `279`.
+- `small__example.cfg`: Go digest `3b803e3d4b9ffcf99c771c352118f3f7026420ea5f26c8d934349ac848789b23`; C digest `f1938d5c7bc544856a5df6c204af75af10a5395bd1f89f560c74caef5acf191f`; first divergence `/document`, type `document` versus `ERROR`.
+
+The registered smoke witness passes locked C exactly. Its source SHA-256 is
+`e2d564b999c40b0a53450771ffa82adf7880375449e8628fefd118aae21056d7`, and
+its raw, production, and C digest is
+`1ae089a98760be594f06d0820951e01714097e99621cc2cd4428ce09ba867083`.
+
+## Historical trigger evidence
+
+The live arm has actual rewrites on the historical trigger shapes that its
+unit test protects:
+
+- Childless whole-comment `ERROR`: source SHA-256
+ `ff90d209911d0d32bf44ebff0742e6f42ff40a6f4978860a00ec3f7228b2af24`.
+ The direct arm changes `(ERROR (ERROR (_multiline_begin)))` to `(ERROR)`;
+ the transcript records a root `child_count_delta=-1` and the census records
+ `3` rewritten nodes. The production digest is
+ `0e1129b2130636e62dd05b2494c22a9a2b5b6ec044aea2eeb4dc836380e38b38`, which
+ matches locked C. Raw digest is
+ `6c16ff1b99a3b116d575f90aa0fe5456381b442a58af021dac36e6954345ce4c`.
+- Recovered document: source SHA-256
+ `f6deae068bcf0fe684f8623d671ee5dfbfab47c93d7827ec03c3b4b5330f8309`.
+ The direct arm changes the root from `ERROR` to `document`, removes two
+ root children, and records `14` rewritten nodes. The production digest is
+ `21374502deb13653ec081dd59a4e21311f501aa9adfd34ea1fe3a2f09bc5f8d5`.
+ Locked C reports digest
+ `05813d8b13788902a7f9b9322ca16127ecf5e9c3694d60726acc7a511be622fe` and
+ first diverges at `/document`, children `4` versus `3`.
+
+The route receipt confirms that these rewrites remain live on every covered
+route. The childless witness reports `3` rewrites on compact fallback at
+`compact route error`, forest fallback at `eof_no_root`, and incremental
+fallback at `external_scanner_unsupported`. The recovered-document witness
+reports `14` rewrites on compact fallback at `accepted-leaf-tiling-gap`,
+forest fallback at `dead_end`, and incremental fallback at
+`external_scanner_unsupported`.
+
+## Census and structural trace
+
+The mounted real-corpus census passed with 20 languages, 5 inert arms, 15
+active arms, 14 uncovered registered arms, and 31 languages without a
+dispatcher arm. Doxygen was uncovered because the mounted real corpus has no
+Doxygen directory. The dedicated A0 fixtures supply its current firing
+receipt.
+
+Canopy traced the call path as follows:
+
+`runLanguageResultCompatibility` -> `dispatcherArmCensus` ->
+`normalizeDoxygenCompatibility` -> `normalizeDoxygenWholeBlockCommentError`
+-> `doxygenErrorTreeHasRecoveredStructure` and
+`retypeDoxygenRecoveredErrorRoot`.
+
+## Docker artifacts
+
+- A0 registry and route receipt:
+ `harness_out/docker/20260822T201143Z-doxygen-a0-routes-final-main`.
+- Locked-C deep parity receipt:
+ `harness_out/docker/20260822T200654Z-doxygen-locked-c-deep-main`.
+- Historical rewrite route receipt:
+ `harness_out/docker/20260822T201217Z-doxygen-historical-routes-final-main`.
+- Direct-arm transcript receipt:
+ `harness_out/docker/20260822T201002Z-doxygen-direct-arm-transcript-main`.
+- Post-JSDoc full census with the corpus mounted:
+ `harness_out/docker/20260822T200749Z-doxygen-census-after-jsdoc-main-mounted`.
+- Post-JSDoc registry and A0 manifest gates:
+ `harness_out/docker/20260822T200855Z-doxygen-registry-post-jsdoc-main2`.
+
+No production source, registry entry, changelog, or normalizer arm changed.
diff --git a/doxygen_probe_bridge.go b/doxygen_probe_bridge.go
new file mode 100644
index 000000000..da6350f16
--- /dev/null
+++ b/doxygen_probe_bridge.go
@@ -0,0 +1,21 @@
+//go:build doxygen_probe
+
+package gotreesitter
+
+// DoxygenProbeNormalization applies the live result normalizer to a raw root
+// and returns the direct arm counters for the temporary probe.
+func DoxygenProbeNormalization(root *Node, source []byte, lang *Language) (before, after string, checked, run, visited, rewritten uint64) {
+ if root == nil || lang == nil {
+ return "", "", 0, 0, 0, 0
+ }
+ before = root.SExpr(lang)
+ p := &Parser{language: lang}
+ _ = normalizeResultCompatibility(root, source, p, nil)
+ after = root.SExpr(lang)
+ for _, pass := range p.normalizationStats.namedPasses {
+ if pass.name == "dispatch.doxygen" {
+ return before, after, pass.checked, pass.run, pass.nodesVisited, pass.nodesRewritten
+ }
+ }
+ return before, after, p.normalizationStats.passesChecked, p.normalizationStats.passesRun, p.normalizationStats.nodesVisited, p.normalizationStats.nodesRewritten
+}
diff --git a/doxygen_probe_test.go b/doxygen_probe_test.go
new file mode 100644
index 000000000..111b18e1a
--- /dev/null
+++ b/doxygen_probe_test.go
@@ -0,0 +1,254 @@
+//go:build doxygen_probe
+
+package gotreesitter_test
+
+import (
+ "crypto/sha256"
+ "fmt"
+ "os"
+ "path/filepath"
+ "testing"
+
+ gotreesitter "github.com/odvcencio/gotreesitter"
+ "github.com/odvcencio/gotreesitter/grammars"
+ "github.com/odvcencio/gotreesitter/internal/benchfixtures"
+)
+
+func TestDoxygenA0RawProductionProbe(t *testing.T) {
+ t.Setenv("GTS_DISPATCHER_CENSUS", "1")
+ lang := grammars.DoxygenLanguage()
+ paths := []string{
+ "testdata/dispatcher_census_a0/doxygen/medium__CMakeLists.txt",
+ "testdata/dispatcher_census_a0/doxygen/medium__metrics.py",
+ "testdata/dispatcher_census_a0/doxygen/small__example.cfg",
+ }
+ for _, rel := range paths {
+ rel := rel
+ t.Run(filepath.Base(rel), func(t *testing.T) {
+ source, err := os.ReadFile(rel)
+ if err != nil {
+ t.Fatal(err)
+ }
+ probeDoxygenRawProduction(t, lang, source, rel)
+ })
+ }
+}
+
+func TestDoxygenPositiveRewriteProbe(t *testing.T) {
+ t.Setenv("GTS_DISPATCHER_CENSUS", "1")
+ lang := grammars.DoxygenLanguage()
+ for _, tc := range []struct{ name, source string }{
+ {"registered_smoke", grammars.ParseSmokeSample("doxygen")},
+ {"childless_error", "/** Adds all words in \\a s to document \\a doc with weight \\a wfd */"},
+ {"recovered_document", "/**\\n * @param {int} value\\n * @brief Example\\n */"},
+ } {
+ t.Run(tc.name, func(t *testing.T) { probeDoxygenRawProduction(t, lang, []byte(tc.source), tc.name) })
+ }
+}
+
+func TestDoxygenDirectArmMutationProbe(t *testing.T) {
+ t.Setenv("GTS_DISPATCHER_CENSUS", "1")
+ lang := grammars.DoxygenLanguage()
+ for _, tc := range []struct {
+ name, source string
+ }{
+ {"childless_error", "/** Adds all words in \\a s to document \\a doc with weight \\a wfd */"},
+ {"recovered_document", "/**\n * @param {int} value\n * @brief Example\n */"},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ rawParser := gotreesitter.NewParser(lang)
+ rawParser.SetAdmissionCandidateRoute(false)
+ tree, err := rawParser.ParseNoResultCompatibilityBenchmarkOnly([]byte(tc.source))
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(tree.Release)
+ before, after, checked, run, visited, rewritten := gotreesitter.DoxygenProbeNormalization(tree.RootNode(), []byte(tc.source), lang)
+ t.Logf("source=%q before=%s after=%s checked=%d run=%d visited=%d rewritten=%d", tc.source, before, after, checked, run, visited, rewritten)
+ if transcriptPath := os.Getenv("GTS_DISPATCHER_TRANSCRIPT_OUT"); transcriptPath != "" {
+ transcript, readErr := os.ReadFile(transcriptPath)
+ if readErr != nil {
+ t.Fatalf("read dispatcher transcript: %v", readErr)
+ }
+ t.Logf("dispatcher_transcript=%s", transcript)
+ }
+ })
+ }
+}
+
+func probeDoxygenRawProduction(t *testing.T, lang *gotreesitter.Language, source []byte, label string) {
+ t.Helper()
+ rawParser := gotreesitter.NewParser(lang)
+ rawParser.SetAdmissionCandidateRoute(false)
+ raw, err := rawParser.ParseNoResultCompatibilityBenchmarkOnly(source)
+ if err != nil {
+ t.Fatalf("raw parse: %v", err)
+ }
+ t.Cleanup(raw.Release)
+ productionParser := gotreesitter.NewParser(lang)
+ productionParser.SetAdmissionCandidateRoute(false)
+ production, err := productionParser.Parse(source)
+ if err != nil {
+ t.Fatalf("production parse: %v", err)
+ }
+ t.Cleanup(production.Release)
+ rawInspect, err := benchfixtures.InspectGoTree(raw.RootNode(), lang)
+ if err != nil {
+ t.Fatalf("raw inspect: %v", err)
+ }
+ productionInspect, err := benchfixtures.InspectGoTree(production.RootNode(), lang)
+ if err != nil {
+ t.Fatalf("production inspect: %v", err)
+ }
+ rawRuntime, productionRuntime := raw.ParseRuntime(), production.ParseRuntime()
+ rootRaw, rootProduction := raw.RootNode(), production.RootNode()
+ t.Logf("label=%s bytes=%d source_sha256=%x raw_digest=%s production_digest=%s raw_rewrites=%d production_rewrites=%d raw_passes=%d production_passes=%d raw_root=%s[%d,%d) raw_error=%t production_root=%s[%d,%d) production_error=%t raw_sexpr=%s production_sexpr=%s", label, len(source), sha256.Sum256(source), rawInspect.SHA256, productionInspect.SHA256, rawRuntime.NormalizationNodesRewritten, productionRuntime.NormalizationNodesRewritten, rawRuntime.NormalizationPassesRun, productionRuntime.NormalizationPassesRun, rootRaw.Type(lang), rootRaw.StartByte(), rootRaw.EndByte(), rootRaw.HasError(), rootProduction.Type(lang), rootProduction.StartByte(), rootProduction.EndByte(), rootProduction.HasError(), rootRaw.SExpr(lang), rootProduction.SExpr(lang))
+}
+
+func TestDoxygenA0RouteTrace(t *testing.T) {
+ t.Setenv("GTS_DISPATCHER_CENSUS", "1")
+ lang := grammars.DoxygenLanguage()
+ paths := []string{
+ "testdata/dispatcher_census_a0/doxygen/medium__CMakeLists.txt",
+ "testdata/dispatcher_census_a0/doxygen/medium__metrics.py",
+ "testdata/dispatcher_census_a0/doxygen/small__example.cfg",
+ }
+ for _, rel := range paths {
+ rel := rel
+ t.Run(filepath.Base(rel), func(t *testing.T) {
+ source, err := os.ReadFile(rel)
+ if err != nil {
+ t.Fatal(err)
+ }
+ productionParser := gotreesitter.NewParser(lang)
+ productionParser.SetAdmissionCandidateRoute(false)
+ production, err := productionParser.Parse(source)
+ if err != nil {
+ t.Fatalf("production parse: %v", err)
+ }
+ t.Cleanup(production.Release)
+ want, err := benchfixtures.InspectGoTree(production.RootNode(), lang)
+ if err != nil {
+ t.Fatal(err)
+ }
+ traceDoxygenRoutes(t, lang, source, want.SHA256)
+ })
+ }
+}
+
+func TestDoxygenHistoricalRouteTrace(t *testing.T) {
+ t.Setenv("GTS_DISPATCHER_CENSUS", "1")
+ lang := grammars.DoxygenLanguage()
+ for _, tc := range []struct {
+ name, source string
+ }{
+ {"childless_error", "/** Adds all words in \\a s to document \\a doc with weight \\a wfd */"},
+ {"recovered_document", "/**\n * @param {int} value\n * @brief Example\n */"},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ productionParser := gotreesitter.NewParser(lang)
+ productionParser.SetAdmissionCandidateRoute(false)
+ production, err := productionParser.Parse([]byte(tc.source))
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(production.Release)
+ want, err := benchfixtures.InspectGoTree(production.RootNode(), lang)
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Logf("source_sha256=%x production_digest=%s production_sexpr=%s", sha256.Sum256([]byte(tc.source)), want.SHA256, production.RootNode().SExpr(lang))
+ traceDoxygenRoutes(t, lang, []byte(tc.source), want.SHA256)
+ })
+ }
+}
+
+func traceDoxygenRoutes(t *testing.T, lang *gotreesitter.Language, source []byte, wantDigest string) {
+ t.Helper()
+ check := func(route string, tree *gotreesitter.Tree) {
+ t.Helper()
+ inspection, err := benchfixtures.InspectGoTree(tree.RootNode(), lang)
+ if err != nil {
+ t.Fatalf("%s inspect: %v", route, err)
+ }
+ runtime := tree.ParseRuntime()
+ var named []gotreesitter.NormalizationPassRuntime
+ if runtime.NormalizationPasses != nil {
+ named = *runtime.NormalizationPasses
+ }
+ t.Logf("route=%s digest=%s root=%s[%d,%d) error=%t passes=%d checked=%d visited=%d rewritten=%d forest=%t named=%+v", route, inspection.SHA256, tree.RootNode().Type(lang), tree.RootNode().StartByte(), tree.RootNode().EndByte(), tree.RootNode().HasError(), runtime.NormalizationPassesRun, runtime.NormalizationPassesChecked, runtime.NormalizationNodesVisited, runtime.NormalizationNodesRewritten, runtime.ForestFastPath, named)
+ if inspection.SHA256 != wantDigest {
+ t.Fatalf("%s digest=%s want production=%s", route, inspection.SHA256, wantDigest)
+ }
+ t.Cleanup(tree.Release)
+ }
+ compactParser := gotreesitter.NewParser(lang)
+ compactParser.SetAdmissionCandidateRoute(true)
+ routedBefore, fallbackBefore := gotreesitter.AdmissionCandidateCounters()
+ compact, err := compactParser.Parse(source)
+ if err != nil {
+ t.Fatalf("compact parse: %v", err)
+ }
+ routedAfter, fallbackAfter := gotreesitter.AdmissionCandidateCounters()
+ compactRoute := "compact-direct"
+ if routedAfter == routedBefore && fallbackAfter == fallbackBefore+1 {
+ compactRoute = "compact-fallback:" + gotreesitter.AdmissionCandidateLastFallbackReason()
+ }
+ check(compactRoute, compact)
+ forestParser := gotreesitter.NewParser(lang)
+ forest, ok := forestParser.ParseForestExperimental(source)
+ if ok && forest != nil {
+ check("forest-direct", forest)
+ } else {
+ if forest != nil {
+ forest.Release()
+ }
+ _, _, reason, _ := forestParser.ForestDeclineInfo()
+ fallback, fallbackErr := forestParser.Parse(source)
+ if fallbackErr != nil {
+ t.Fatalf("forest fallback parse: %v", fallbackErr)
+ }
+ check("forest-fallback:"+reason, fallback)
+ }
+ if len(source) == 0 {
+ return
+ }
+ oldSource := append([]byte(nil), source...)
+ oldSource = append(oldSource, ' ')
+ incrementalParser := gotreesitter.NewParser(lang)
+ incrementalParser.SetAdmissionCandidateRoute(false)
+ oldTree, err := incrementalParser.Parse(oldSource)
+ if err != nil {
+ t.Fatalf("incremental base parse: %v", err)
+ }
+ t.Cleanup(oldTree.Release)
+ p := doxygenProbePointAtByte(source, len(source))
+ oldTree.Edit(gotreesitter.InputEdit{StartByte: uint32(len(source)), OldEndByte: uint32(len(oldSource)), NewEndByte: uint32(len(source)), StartPoint: p, OldEndPoint: doxygenProbePointAtByte(oldSource, len(oldSource)), NewEndPoint: p})
+ incremental, profile, err := incrementalParser.ParseIncrementalProfiled(source, oldTree)
+ if err != nil {
+ t.Fatalf("incremental parse: %v", err)
+ }
+ route := "incremental-fresh"
+ if profile.ReuseUnsupported {
+ route = "incremental-fallback:" + profile.ReuseUnsupportedReason
+ } else if profile.OldTreeReuseRoute && profile.ReusedSubtrees > 0 {
+ route = fmt.Sprintf("incremental-reuse:%d:%d", profile.ReusedSubtrees, profile.ReusedBytes)
+ }
+ check(route, incremental)
+}
+
+func doxygenProbePointAtByte(source []byte, offset int) gotreesitter.Point {
+ var p gotreesitter.Point
+ for i, b := range source {
+ if i >= offset {
+ break
+ }
+ if b == '\n' {
+ p.Row++
+ p.Column = 0
+ } else {
+ p.Column++
+ }
+ }
+ return p
+}
diff --git a/grammars/c_token_source_diagnostic_fork.go b/grammars/c_token_source_diagnostic_fork.go
new file mode 100644
index 000000000..a0cf39f80
--- /dev/null
+++ b/grammars/c_token_source_diagnostic_fork.go
@@ -0,0 +1,101 @@
+//go:build gts_workcount && (!grammar_subset || grammar_subset_c || grammar_subset_cpp)
+
+package grammars
+
+import (
+ "crypto/sha256"
+ "encoding/binary"
+ "fmt"
+ "hash"
+
+ "github.com/odvcencio/gotreesitter"
+)
+
+// ForkTokenSourceForDiagnostic copies the current C token-source state.
+// The fork shares only immutable source bytes, language data, and lexer tables.
+func (ts *CTokenSource) ForkTokenSourceForDiagnostic() (gotreesitter.TokenSource, gotreesitter.DiagnosticTokenSourceForkReceipt, error) {
+ if ts == nil {
+ return nil, gotreesitter.DiagnosticTokenSourceForkReceipt{}, fmt.Errorf("diagnostic C token fork: source is nil")
+ }
+ if ts.cur.offset < 0 || ts.cur.offset > len(ts.cur.src) {
+ return nil, gotreesitter.DiagnosticTokenSourceForkReceipt{}, fmt.Errorf("diagnostic C token fork: cursor %d is outside source length %d", ts.cur.offset, len(ts.cur.src))
+ }
+ if uint64(ts.cur.offset) > uint64(^uint32(0)) {
+ return nil, gotreesitter.DiagnosticTokenSourceForkReceipt{}, fmt.Errorf("diagnostic C token fork: cursor %d exceeds the token offset limit", ts.cur.offset)
+ }
+ if len(ts.src) != len(ts.cur.src) || (len(ts.src) > 0 && &ts.src[0] != &ts.cur.src[0]) {
+ return nil, gotreesitter.DiagnosticTokenSourceForkReceipt{}, fmt.Errorf("diagnostic C token fork: source cursor does not share the immutable source")
+ }
+
+ fork := *ts
+ fork.pending = diagnosticCloneCTokens(ts.pending)
+ fork.glrStates = diagnosticCloneCStates(ts.glrStates)
+
+ receipt := gotreesitter.DiagnosticTokenSourceForkReceipt{
+ TokenSourceKind: "c",
+ CursorByte: uint32(ts.cur.offset),
+ CursorPoint: ts.cur.point(),
+ ParserState: ts.parserState,
+ ActiveGLRStates: diagnosticCloneCStates(ts.glrStates),
+ PendingTokenCount: len(ts.pending),
+ PendingTokenDigest: diagnosticCTokenDigest(ts.pending),
+ ZeroWidthOffset: int64(ts.lastSyntheticOffset),
+ IncludedRangeIndex: -1,
+ }
+ return &fork, receipt, nil
+}
+
+func diagnosticCloneCTokens(tokens []gotreesitter.Token) []gotreesitter.Token {
+ if tokens == nil {
+ return nil
+ }
+ clone := make([]gotreesitter.Token, len(tokens))
+ copy(clone, tokens)
+ return clone
+}
+
+func diagnosticCloneCStates(states []gotreesitter.StateID) []gotreesitter.StateID {
+ if states == nil {
+ return nil
+ }
+ clone := make([]gotreesitter.StateID, len(states))
+ copy(clone, states)
+ return clone
+}
+
+func diagnosticCTokenDigest(tokens []gotreesitter.Token) [sha256.Size]byte {
+ digest := sha256.New()
+ diagnosticWriteUint64(digest, uint64(len(tokens)))
+ for _, token := range tokens {
+ diagnosticWriteUint64(digest, uint64(token.Symbol))
+ diagnosticWriteUint64(digest, uint64(len(token.Text)))
+ _, _ = digest.Write([]byte(token.Text))
+ diagnosticWriteUint64(digest, uint64(token.StartByte))
+ diagnosticWriteUint64(digest, uint64(token.EndByte))
+ diagnosticWriteUint64(digest, uint64(token.StartPoint.Row))
+ diagnosticWriteUint64(digest, uint64(token.StartPoint.Column))
+ diagnosticWriteUint64(digest, uint64(token.EndPoint.Row))
+ diagnosticWriteUint64(digest, uint64(token.EndPoint.Column))
+ diagnosticWriteBool(digest, token.Missing)
+ diagnosticWriteBool(digest, token.NoLookahead)
+ diagnosticWriteBool(digest, token.ExternalScannerToken)
+ diagnosticWriteUint64(digest, uint64(token.ExternalScannerStartByte))
+ }
+ var out [sha256.Size]byte
+ copy(out[:], digest.Sum(nil))
+ return out
+}
+
+func diagnosticWriteUint64(digest hash.Hash, value uint64) {
+ var encoded [8]byte
+ binary.LittleEndian.PutUint64(encoded[:], value)
+ _, _ = digest.Write(encoded[:])
+}
+
+func diagnosticWriteBool(digest hash.Hash, value bool) {
+ if value {
+ _, _ = digest.Write([]byte{1})
+ return
+ }
+ _, _ = digest.Write([]byte{0})
+}
diff --git a/grammars/c_token_source_diagnostic_fork_test.go b/grammars/c_token_source_diagnostic_fork_test.go
new file mode 100644
index 000000000..b51bbd7f8
--- /dev/null
+++ b/grammars/c_token_source_diagnostic_fork_test.go
@@ -0,0 +1,308 @@
+//go:build gts_workcount && (!grammar_subset || grammar_subset_c || grammar_subset_cpp)
+
+package grammars
+
+import (
+ "bytes"
+ "crypto/sha256"
+ "reflect"
+ "testing"
+
+ "github.com/odvcencio/gotreesitter"
+)
+
+type cDiagnosticMutableState struct {
+ cursor sourceCursor
+ done bool
+ pending []gotreesitter.Token
+ preprocState int
+ parserState gotreesitter.StateID
+ glrStates []gotreesitter.StateID
+ lastSyntheticOffset int
+ preprocDefineNameEnd int
+ preprocOpaqueArgPending bool
+ preprocOpaqueArgActive bool
+}
+
+func captureCDiagnosticMutableState(source *CTokenSource) cDiagnosticMutableState {
+ return cDiagnosticMutableState{
+ cursor: source.cur,
+ done: source.done,
+ pending: append([]gotreesitter.Token(nil), source.pending...),
+ preprocState: source.preprocState,
+ parserState: source.parserState,
+ glrStates: append([]gotreesitter.StateID(nil), source.glrStates...),
+ lastSyntheticOffset: source.lastSyntheticOffset,
+ preprocDefineNameEnd: source.preprocDefineNameEnd,
+ preprocOpaqueArgPending: source.preprocOpaqueArgPending,
+ preprocOpaqueArgActive: source.preprocOpaqueArgActive,
+ }
+}
+
+func assertCDiagnosticStateEqual(t *testing.T, got, want cDiagnosticMutableState) {
+ t.Helper()
+ if !reflect.DeepEqual(got, want) {
+ t.Fatalf("C token-source state changed:\n got: %#v\nwant: %#v", got, want)
+ }
+}
+
+func assertCDiagnosticForkStateEqual(t *testing.T, fork, live *CTokenSource) {
+ t.Helper()
+ if !reflect.DeepEqual(fork, live) {
+ t.Fatalf("C fork state differs before advance:\n got: %#v\nwant: %#v", fork, live)
+ }
+}
+
+func advanceCDiagnosticTokens(source *CTokenSource, count int) []gotreesitter.Token {
+ out := make([]gotreesitter.Token, count)
+ for index := range out {
+ out[index] = source.Next()
+ }
+ return out
+}
+
+func requireCDiagnosticTokenSequenceEqual(t *testing.T, got, want []gotreesitter.Token) {
+ t.Helper()
+ if !reflect.DeepEqual(got, want) {
+ t.Fatalf("token sequence differs:\n got: %#v\nwant: %#v", got, want)
+ }
+}
+
+func TestCTokenSourceDiagnosticForkPreservesPendingTokens(t *testing.T) {
+ lang := CLanguage()
+ sourceBytes := []byte("const char *value = \"a\\n\"; int tail;\n")
+ live, err := NewCTokenSource(sourceBytes, lang)
+ if err != nil {
+ t.Fatalf("NewCTokenSource failed: %v", err)
+ }
+ t.Cleanup(live.Close)
+ live.SetParserState(101)
+ live.SetGLRStates([]gotreesitter.StateID{4, 8, 15})
+
+ foundOpener := false
+ for index := 0; index < 12; index++ {
+ if token := live.Next(); token.Text == "\"" {
+ foundOpener = true
+ break
+ }
+ }
+ if !foundOpener {
+ t.Fatal("setup did not reach a string opener")
+ }
+ if len(live.pending) < 2 {
+ t.Fatalf("pending token count = %d, want at least 2", len(live.pending))
+ }
+ liveBefore := captureCDiagnosticMutableState(live)
+ sourceBefore := append([]byte(nil), live.src...)
+
+ forkSource, receipt, err := live.ForkTokenSourceForDiagnostic()
+ if err != nil {
+ t.Fatalf("ForkTokenSourceForDiagnostic failed: %v", err)
+ }
+ fork, ok := forkSource.(*CTokenSource)
+ if !ok {
+ t.Fatalf("fork type = %T, want *CTokenSource", forkSource)
+ }
+ t.Cleanup(fork.Close)
+ assertCDiagnosticForkStateEqual(t, fork, live)
+ if fork == live {
+ t.Fatal("fork shares the live C token-source object")
+ }
+ if &fork.pending[0] == &live.pending[0] {
+ t.Fatal("fork shares the pending-token backing array")
+ }
+ if &fork.glrStates[0] == &live.glrStates[0] {
+ t.Fatal("fork shares the GLR-state backing array")
+ }
+ if &receipt.ActiveGLRStates[0] == &live.glrStates[0] || &receipt.ActiveGLRStates[0] == &fork.glrStates[0] {
+ t.Fatal("receipt shares a mutable GLR-state backing array")
+ }
+ if got, want := receipt.TokenSourceKind, "c"; got != want {
+ t.Fatalf("receipt kind = %q, want %q", got, want)
+ }
+ if receipt.CursorByte != uint32(live.cur.offset) || receipt.CursorPoint != live.cur.point() {
+ t.Fatalf("receipt cursor = %d/%+v, want %d/%+v", receipt.CursorByte, receipt.CursorPoint, live.cur.offset, live.cur.point())
+ }
+ if receipt.ParserState != live.parserState {
+ t.Fatalf("receipt parser state = %d, want %d", receipt.ParserState, live.parserState)
+ }
+ if !reflect.DeepEqual(receipt.ActiveGLRStates, live.glrStates) {
+ t.Fatalf("receipt GLR states = %v, want %v", receipt.ActiveGLRStates, live.glrStates)
+ }
+ if receipt.PendingTokenCount != len(live.pending) || receipt.PendingTokenDigest != diagnosticCTokenDigest(live.pending) {
+ t.Fatalf("receipt pending state = %d/%x, want %d/%x", receipt.PendingTokenCount, receipt.PendingTokenDigest, len(live.pending), diagnosticCTokenDigest(live.pending))
+ }
+ if receipt.ExternalScannerCheckpointPresent || receipt.ExternalScannerCheckpointDigest != [sha256.Size]byte{} {
+ t.Fatalf("C receipt reports external scanner state: present=%v digest=%x", receipt.ExternalScannerCheckpointPresent, receipt.ExternalScannerCheckpointDigest)
+ }
+
+ forkTokens := advanceCDiagnosticTokens(fork, 7)
+ assertCDiagnosticStateEqual(t, captureCDiagnosticMutableState(live), liveBefore)
+ if !bytes.Equal(live.src, sourceBefore) {
+ t.Fatalf("fork advance changed live source bytes: got %q, want %q", live.src, sourceBefore)
+ }
+ liveTokens := advanceCDiagnosticTokens(live, 7)
+ requireCDiagnosticTokenSequenceEqual(t, forkTokens, liveTokens)
+
+ fork.glrStates[0] = 99
+ receipt.ActiveGLRStates[0] = 100
+ if live.glrStates[0] != 4 {
+ t.Fatalf("fork or receipt mutation changed live GLR state to %d", live.glrStates[0])
+ }
+}
+
+func TestCTokenSourceDiagnosticForkPreservesPreprocessorState(t *testing.T) {
+ lang := CLanguage()
+ sourceBytes := []byte("#if __has_include()\nint enabled;\n")
+ live, err := NewCTokenSource(sourceBytes, lang)
+ if err != nil {
+ t.Fatalf("NewCTokenSource failed: %v", err)
+ }
+ t.Cleanup(live.Close)
+
+ setup := advanceCDiagnosticTokens(live, 3)
+ if got, want := setup[2].Text, "("; got != want {
+ t.Fatalf("third setup token text = %q, want %q", got, want)
+ }
+ if live.preprocState != cPreprocConditionalExpr || !live.preprocOpaqueArgActive || live.preprocOpaqueArgPending {
+ t.Fatalf("setup preprocessor state = %d/%v/%v", live.preprocState, live.preprocOpaqueArgPending, live.preprocOpaqueArgActive)
+ }
+ liveBefore := captureCDiagnosticMutableState(live)
+
+ forkSource, _, err := live.ForkTokenSourceForDiagnostic()
+ if err != nil {
+ t.Fatalf("ForkTokenSourceForDiagnostic failed: %v", err)
+ }
+ fork := forkSource.(*CTokenSource)
+ t.Cleanup(fork.Close)
+ assertCDiagnosticForkStateEqual(t, fork, live)
+ if fork.preprocState != live.preprocState || fork.preprocOpaqueArgPending != live.preprocOpaqueArgPending || fork.preprocOpaqueArgActive != live.preprocOpaqueArgActive {
+ t.Fatalf("fork preprocessor state = %d/%v/%v, want %d/%v/%v", fork.preprocState, fork.preprocOpaqueArgPending, fork.preprocOpaqueArgActive, live.preprocState, live.preprocOpaqueArgPending, live.preprocOpaqueArgActive)
+ }
+
+ forkTokens := advanceCDiagnosticTokens(fork, 6)
+ assertCDiagnosticStateEqual(t, captureCDiagnosticMutableState(live), liveBefore)
+ liveTokens := advanceCDiagnosticTokens(live, 6)
+ requireCDiagnosticTokenSequenceEqual(t, forkTokens, liveTokens)
+}
+
+func TestCTokenSourceDiagnosticForkPreservesDefinitionBoundary(t *testing.T) {
+ lang := CLanguage()
+ sourceBytes := []byte("#define FLAG(x) x\nint tail;\n")
+ live, err := NewCTokenSource(sourceBytes, lang)
+ if err != nil {
+ t.Fatalf("NewCTokenSource failed: %v", err)
+ }
+ t.Cleanup(live.Close)
+
+ setup := advanceCDiagnosticTokens(live, 2)
+ if got, want := setup[1].Text, "FLAG"; got != want {
+ t.Fatalf("second setup token text = %q, want %q", got, want)
+ }
+ if live.preprocState != cPreprocAfterDefineName || live.preprocDefineNameEnd != live.cur.offset {
+ t.Fatalf("definition boundary state = %d/%d, want %d/%d", live.preprocState, live.preprocDefineNameEnd, cPreprocAfterDefineName, live.cur.offset)
+ }
+ liveBefore := captureCDiagnosticMutableState(live)
+
+ forkSource, _, err := live.ForkTokenSourceForDiagnostic()
+ if err != nil {
+ t.Fatalf("ForkTokenSourceForDiagnostic failed: %v", err)
+ }
+ fork := forkSource.(*CTokenSource)
+ t.Cleanup(fork.Close)
+ assertCDiagnosticForkStateEqual(t, fork, live)
+ if fork.preprocDefineNameEnd != live.preprocDefineNameEnd {
+ t.Fatalf("fork definition boundary = %d, want %d", fork.preprocDefineNameEnd, live.preprocDefineNameEnd)
+ }
+
+ forkTokens := advanceCDiagnosticTokens(fork, 6)
+ assertCDiagnosticStateEqual(t, captureCDiagnosticMutableState(live), liveBefore)
+ liveTokens := advanceCDiagnosticTokens(live, 6)
+ requireCDiagnosticTokenSequenceEqual(t, forkTokens, liveTokens)
+}
+
+func TestCTokenSourceDiagnosticForkPreservesZeroWidthState(t *testing.T) {
+ lang := CLanguage()
+ sourceBytes := []byte("#ifdef __cplusplus\nextern \"C\" {\n#endif\n\nint x;\n\n#ifdef __cplusplus\n}\n#endif\n")
+ live, err := NewCTokenSource(sourceBytes, lang)
+ if err != nil {
+ t.Fatalf("NewCTokenSource failed: %v", err)
+ }
+ t.Cleanup(live.Close)
+ live.cur.advanceBytes(66)
+ live.SetParserState(10)
+ liveBefore := captureCDiagnosticMutableState(live)
+
+ forkSource, receipt, err := live.ForkTokenSourceForDiagnostic()
+ if err != nil {
+ t.Fatalf("ForkTokenSourceForDiagnostic failed: %v", err)
+ }
+ fork := forkSource.(*CTokenSource)
+ t.Cleanup(fork.Close)
+ assertCDiagnosticForkStateEqual(t, fork, live)
+ if got, want := receipt.ZeroWidthOffset, int64(-1); got != want {
+ t.Fatalf("receipt zero-width offset = %d, want %d", got, want)
+ }
+
+ forkSynthetic := fork.Next()
+ if forkSynthetic.Symbol != live.endifSymbol || forkSynthetic.StartByte != 66 || forkSynthetic.EndByte != 66 || !forkSynthetic.Missing {
+ t.Fatalf("fork synthetic token = %+v, want a missing #endif at byte 66", forkSynthetic)
+ }
+ assertCDiagnosticStateEqual(t, captureCDiagnosticMutableState(live), liveBefore)
+ liveSynthetic := live.Next()
+ if !reflect.DeepEqual(liveSynthetic, forkSynthetic) {
+ t.Fatalf("live synthetic token = %+v, want %+v", liveSynthetic, forkSynthetic)
+ }
+
+ guardedForkSource, guardedReceipt, err := live.ForkTokenSourceForDiagnostic()
+ if err != nil {
+ t.Fatalf("guarded ForkTokenSourceForDiagnostic failed: %v", err)
+ }
+ guardedFork := guardedForkSource.(*CTokenSource)
+ t.Cleanup(guardedFork.Close)
+ assertCDiagnosticForkStateEqual(t, guardedFork, live)
+ if got, want := guardedReceipt.ZeroWidthOffset, int64(66); got != want {
+ t.Fatalf("guarded receipt zero-width offset = %d, want %d", got, want)
+ }
+ liveBeforeGuardedAdvance := captureCDiagnosticMutableState(live)
+ forkNext := guardedFork.Next()
+ assertCDiagnosticStateEqual(t, captureCDiagnosticMutableState(live), liveBeforeGuardedAdvance)
+ if liveNext := live.Next(); !reflect.DeepEqual(liveNext, forkNext) {
+ t.Fatalf("guarded live token = %+v, want fork token %+v", liveNext, forkNext)
+ }
+}
+
+func TestCTokenSourceDiagnosticForkPreservesDoneState(t *testing.T) {
+ lang := CLanguage()
+ live, err := NewCTokenSource([]byte("int"), lang)
+ if err != nil {
+ t.Fatalf("NewCTokenSource failed: %v", err)
+ }
+ t.Cleanup(live.Close)
+ _ = live.Next()
+ _ = live.Next()
+ if !live.done {
+ t.Fatal("setup source is not done")
+ }
+ liveBefore := captureCDiagnosticMutableState(live)
+
+ forkSource, receipt, err := live.ForkTokenSourceForDiagnostic()
+ if err != nil {
+ t.Fatalf("ForkTokenSourceForDiagnostic failed: %v", err)
+ }
+ fork := forkSource.(*CTokenSource)
+ t.Cleanup(fork.Close)
+ assertCDiagnosticForkStateEqual(t, fork, live)
+ if !fork.done {
+ t.Fatal("fork did not preserve the done state")
+ }
+ if receipt.CursorByte != uint32(len(live.src)) || receipt.CursorPoint != (gotreesitter.Point{Column: 3}) {
+ t.Fatalf("done receipt cursor = %d/%+v", receipt.CursorByte, receipt.CursorPoint)
+ }
+
+ forkToken := fork.Next()
+ assertCDiagnosticStateEqual(t, captureCDiagnosticMutableState(live), liveBefore)
+ if liveToken := live.Next(); !reflect.DeepEqual(liveToken, forkToken) {
+ t.Fatalf("done live token = %+v, want fork token %+v", liveToken, forkToken)
+ }
+}
diff --git a/p17_attribution_test.go b/p17_attribution_test.go
new file mode 100644
index 000000000..9fd99c76b
--- /dev/null
+++ b/p17_attribution_test.go
@@ -0,0 +1,671 @@
+//go:build p17_attribution
+
+package gotreesitter
+
+// This file is a temporary P17 attribution harness. It must not remain in the
+// worktree after the evidence packet is frozen.
+
+import (
+ "compress/gzip"
+ "crypto/sha256"
+ "encoding/hex"
+ "encoding/json"
+ "fmt"
+ "io"
+ "os"
+ "path/filepath"
+ "runtime"
+ "runtime/pprof"
+ "sort"
+ "strings"
+ "testing"
+ "time"
+ "unsafe"
+)
+
+type p17Manifest struct {
+ Schema string `json:"schema"`
+ Edits []p17EditSpec `json:"edits"`
+}
+
+type p17EditSpec struct {
+ Name string `json:"name"`
+ Language string `json:"language"`
+ Fixture string `json:"fixture"`
+ Role string `json:"role"`
+ StartByte int `json:"start_byte"`
+ OldEndByte int `json:"old_end_byte"`
+ OldText string `json:"old_text"`
+ NewText string `json:"new_text"`
+ SourceSHA256 string `json:"source_sha256"`
+ EditedSHA256 string `json:"edited_sha256"`
+}
+
+type p17Case struct {
+ Spec p17EditSpec
+ Source []byte
+ Edited []byte
+}
+
+type p17ArenaSample struct {
+ CaseName string `json:"case"`
+ Direction string `json:"direction"`
+ Mode string `json:"mode"`
+ Repeat int `json:"repeat"`
+ SourceLen int `json:"source_len"`
+ TargetLen int `json:"target_len"`
+ HintBefore int `json:"incremental_arena_hint_before"`
+ HintAfter int `json:"incremental_arena_hint_after"`
+ CapacityTarget int `json:"parse_incremental_arena_node_capacity_target"`
+ InitialPrimary int `json:"initial_primary_arena_capacity"`
+ ArenaClass string `json:"returned_arena_class"`
+ PrimaryLen int `json:"primary_arena_len"`
+ PrimaryCap int `json:"primary_arena_cap"`
+ ArenaUsed int `json:"arena_used"`
+ PrimaryBytes int64 `json:"primary_arena_bytes"`
+ OverflowCaps []int `json:"overflow_slab_capacities"`
+ OverflowBytes int64 `json:"overflow_node_bytes"`
+ TotalNodeBytes int64 `json:"total_node_bytes"`
+ OldTarget int `json:"old_geometric_target"`
+ OldCrossesUsed bool `json:"old_geometric_target_crosses_used"`
+ ExactCrosses bool `json:"exact_target_crosses_used"`
+ BytesPerOp uint64 `json:"bytes_per_op"`
+ AllocsPerOp uint64 `json:"allocs_per_op"`
+ NewNodes uint64 `json:"new_nodes_allocated"`
+ ReusedSubtrees uint64 `json:"reused_subtrees"`
+ ReusedBytes uint64 `json:"reused_bytes"`
+ TokensConsumed uint64 `json:"tokens_consumed"`
+ MaxStacksSeen int `json:"max_stacks_seen"`
+ EntryPeak uint64 `json:"entry_scratch_peak"`
+ StopReason string `json:"stop_reason"`
+ ArenaBytes int64 `json:"arena_bytes_allocated"`
+ ScratchBytes int64 `json:"scratch_bytes_allocated"`
+ EntryBytes int64 `json:"entry_scratch_bytes_allocated"`
+ GSSBytes int64 `json:"gss_bytes_allocated"`
+ ParserLoopNs int64 `json:"parser_loop_nanos"`
+ DispatchNs int64 `json:"action_dispatch_nanos"`
+ MergeNs int64 `json:"glr_merge_nanos"`
+ ProfilePath string `json:"profile_path,omitempty"`
+ ArenaEvents []p17ArenaEvent `json:"arena_events,omitempty"`
+}
+
+type p17Aggregate struct {
+ CaseName string `json:"case"`
+ Direction string `json:"direction"`
+ Mode string `json:"mode"`
+ SourceLen int `json:"source_len"`
+ TargetLen int `json:"target_len"`
+ HintBefore int `json:"incremental_arena_hint_before"`
+ HintAfter int `json:"incremental_arena_hint_after"`
+ CapacityTarget int `json:"parse_incremental_arena_node_capacity_target"`
+ InitialPrimary int `json:"initial_primary_arena_capacity"`
+ ArenaClass string `json:"returned_arena_class"`
+ PrimaryLen int `json:"primary_arena_len"`
+ PrimaryCap int `json:"primary_arena_cap"`
+ ArenaUsed int `json:"arena_used"`
+ PrimaryBytes int64 `json:"primary_arena_bytes"`
+ OverflowCaps []int `json:"overflow_slab_capacities"`
+ OverflowBytes int64 `json:"overflow_node_bytes"`
+ TotalNodeBytes int64 `json:"total_node_bytes"`
+ OldTarget int `json:"old_geometric_target"`
+ OldCrossesUsed bool `json:"old_geometric_target_crosses_used"`
+ ExactCrosses bool `json:"exact_target_crosses_used"`
+ ArenaBytes int64 `json:"arena_bytes_allocated"`
+ ScratchBytes int64 `json:"scratch_bytes_allocated"`
+ EntryBytes int64 `json:"entry_scratch_bytes_allocated"`
+ GSSBytes int64 `json:"gss_bytes_allocated"`
+ MedianBytes uint64 `json:"median_bytes_per_op"`
+ MedianAllocs uint64 `json:"median_allocs_per_op"`
+ MinBytes uint64 `json:"min_bytes_per_op"`
+ MaxBytes uint64 `json:"max_bytes_per_op"`
+ MinAllocs uint64 `json:"min_allocs_per_op"`
+ MaxAllocs uint64 `json:"max_allocs_per_op"`
+ Samples []p17ArenaSample `json:"samples"`
+}
+
+type p17Receipt struct {
+ Schema string `json:"schema"`
+ GeneratedUTC string `json:"generated_utc"`
+ GoVersion string `json:"go_version"`
+ NodeSize int `json:"node_size"`
+ ManifestSHA string `json:"manifest_sha256"`
+ SourceSHA string `json:"tracked_source_sha256,omitempty"`
+ Samples []p17Aggregate `json:"aggregates"`
+ Sequences []p17SequenceReceipt `json:"sequences,omitempty"`
+}
+
+type p17SequenceStep struct {
+ Iteration int `json:"iteration"`
+ Direction string `json:"direction"`
+ SourceLen int `json:"source_len"`
+ TargetLen int `json:"target_len"`
+ HintBefore int `json:"incremental_arena_hint_before"`
+ HintAfter int `json:"incremental_arena_hint_after"`
+ CapacityTarget int `json:"parse_incremental_arena_node_capacity_target"`
+ InitialPrimary int `json:"initial_primary_arena_capacity"`
+ ArenaClass string `json:"returned_arena_class"`
+ PrimaryLen int `json:"primary_arena_len"`
+ ArenaUsed int `json:"arena_used"`
+ OverflowCaps []int `json:"overflow_slab_capacities"`
+ OverflowBytes int64 `json:"overflow_node_bytes"`
+ OldTarget int `json:"old_geometric_target"`
+ OldCrossesUsed bool `json:"old_geometric_target_crosses_used"`
+ ExactCrosses bool `json:"exact_target_crosses_used"`
+ BytesPerOp uint64 `json:"bytes_per_op"`
+ AllocsPerOp uint64 `json:"allocs_per_op"`
+ NewNodes uint64 `json:"new_nodes_allocated"`
+ ArenaBytes int64 `json:"arena_bytes_allocated"`
+ StopReason string `json:"stop_reason"`
+ ArenaEvents []p17ArenaEvent `json:"arena_events,omitempty"`
+}
+
+type p17SequenceReceipt struct {
+ CaseName string `json:"case"`
+ Cycles int `json:"cycles"`
+ ProfilePath string `json:"profile_path,omitempty"`
+ Steps []p17SequenceStep `json:"steps"`
+}
+
+type p17ArenaEvent struct {
+ Kind string `json:"kind"`
+ Class string `json:"arena_class"`
+ Before int `json:"before"`
+ Requested int `json:"requested"`
+ After int `json:"after"`
+ Used int `json:"used"`
+}
+
+type p17ArenaTrace struct {
+ Events []p17ArenaEvent
+ Count int
+}
+
+var p17CurrentArenaTrace *p17ArenaTrace
+
+func p17TraceArenaEvent(kind string, class arenaClass, before, requested, after, used int) {
+ trace := p17CurrentArenaTrace
+ if trace == nil || trace.Count >= len(trace.Events) {
+ return
+ }
+ trace.Events[trace.Count] = p17ArenaEvent{
+ Kind: kind, Class: p17ArenaClassName(class), Before: before,
+ Requested: requested, After: after, Used: used,
+ }
+ trace.Count++
+}
+
+func p17ArenaTraceSnapshot(trace *p17ArenaTrace) []p17ArenaEvent {
+ if trace == nil || trace.Count == 0 {
+ return nil
+ }
+ out := make([]p17ArenaEvent, trace.Count)
+ copy(out, trace.Events[:trace.Count])
+ return out
+}
+
+func TestP17CanonicalArenaAttribution(t *testing.T) {
+ if os.Getenv("P17_OUT_DIR") == "" {
+ t.Skip("P17_OUT_DIR is required")
+ }
+ manifestBytes, err := os.ReadFile("cgo_harness/testdata/canonical_go_incremental_edits.json")
+ if err != nil {
+ t.Fatal(err)
+ }
+ var manifest p17Manifest
+ if err := json.Unmarshal(manifestBytes, &manifest); err != nil {
+ t.Fatal(err)
+ }
+ fixtureByID := make(map[string][]byte)
+ var cases []p17Case
+ for _, spec := range manifest.Edits {
+ if spec.Language != "go" || spec.Role != "representative" {
+ continue
+ }
+ source, ok := fixtureByID[spec.Fixture]
+ if !ok {
+ asset := map[string]string{
+ "query_compile": "internal/benchfixtures/testdata/query_compile.go.gz",
+ "language": "internal/benchfixtures/testdata/language.go.gz",
+ "grammargen_lr": "internal/benchfixtures/testdata/grammargen_lr.go.gz",
+ "rewrite": "internal/benchfixtures/testdata/rewrite.go.gz",
+ }[spec.Fixture]
+ if asset == "" {
+ t.Fatalf("fixture %q not found", spec.Fixture)
+ }
+ var loadErr error
+ source, loadErr = p17LoadGzip(asset)
+ if loadErr != nil {
+ t.Fatalf("load fixture %s: %v", spec.Fixture, loadErr)
+ }
+ fixtureByID[spec.Fixture] = source
+ }
+ if got := sha256HexP17(source); got != spec.SourceSHA256 {
+ t.Fatalf("%s source sha=%s want=%s", spec.Name, got, spec.SourceSHA256)
+ }
+ if spec.StartByte < 0 || spec.OldEndByte < spec.StartByte || spec.OldEndByte > len(source) {
+ t.Fatalf("%s invalid edit range", spec.Name)
+ }
+ if got := string(source[spec.StartByte:spec.OldEndByte]); got != spec.OldText {
+ t.Fatalf("%s old text=%q want=%q", spec.Name, got, spec.OldText)
+ }
+ edited := make([]byte, 0, len(source)-(spec.OldEndByte-spec.StartByte)+len(spec.NewText))
+ edited = append(edited, source[:spec.StartByte]...)
+ edited = append(edited, spec.NewText...)
+ edited = append(edited, source[spec.OldEndByte:]...)
+ if got := sha256HexP17(edited); got != spec.EditedSHA256 {
+ t.Fatalf("%s edited sha=%s want=%s", spec.Name, got, spec.EditedSHA256)
+ }
+ cases = append(cases, p17Case{Spec: spec, Source: source, Edited: edited})
+ }
+ sort.Slice(cases, func(i, j int) bool { return cases[i].Spec.Name < cases[j].Spec.Name })
+ if len(cases) != 4 {
+ t.Fatalf("representative Go cases=%d want=4", len(cases))
+ }
+
+ selectedCase := os.Getenv("P17_CASE")
+ selectedDirection := os.Getenv("P17_DIRECTION")
+ selectedMode := os.Getenv("P17_MODE")
+ profile := strings.TrimSpace(os.Getenv("P17_PROFILE")) != ""
+ if selectedMode == "alternating" {
+ sequences := make([]p17SequenceReceipt, 0, len(cases))
+ for _, tc := range cases {
+ if selectedCase != "" && selectedCase != tc.Spec.Name {
+ continue
+ }
+ sequences = append(sequences, p17RunSequence(t, tc, profile, os.Getenv("P17_OUT_DIR")))
+ }
+ if len(sequences) == 0 {
+ t.Fatal("P17 alternating selection matched no cases")
+ }
+ receipt := p17Receipt{
+ Schema: "p17-arena-attribution-v1", GeneratedUTC: time.Now().UTC().Format(time.RFC3339Nano),
+ GoVersion: runtime.Version(), NodeSize: int(unsafe.Sizeof(Node{})),
+ ManifestSHA: sha256HexP17(manifestBytes), Sequences: sequences,
+ }
+ name := "p17-sequence"
+ if len(sequences) == 1 {
+ name += "-" + sequences[0].CaseName
+ }
+ p17WriteReceipt(t, receipt, os.Getenv("P17_OUT_DIR"), name)
+ return
+ }
+ repeats := 3
+ if profile {
+ repeats = 1
+ }
+ outDir := os.Getenv("P17_OUT_DIR")
+ if err := os.MkdirAll(outDir, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ var aggregates []p17Aggregate
+ for _, tc := range cases {
+ if selectedCase != "" && selectedCase != tc.Spec.Name {
+ continue
+ }
+ for _, direction := range []string{"forward", "reverse"} {
+ if selectedDirection != "" && selectedDirection != direction {
+ continue
+ }
+ for _, mode := range []string{"first_call", "warmed_parser"} {
+ if selectedMode != "" && selectedMode != mode {
+ continue
+ }
+ from, to, edit := p17Direction(tc, direction)
+ aggregate := p17Measure(t, tc.Spec.Name, direction, mode, from, to, edit, repeats, profile, outDir)
+ aggregates = append(aggregates, aggregate)
+ }
+ }
+ }
+ if len(aggregates) == 0 {
+ t.Fatal("P17 selection matched no cases")
+ }
+ receipt := p17Receipt{
+ Schema: "p17-arena-attribution-v1",
+ GeneratedUTC: time.Now().UTC().Format(time.RFC3339Nano),
+ GoVersion: runtime.Version(),
+ NodeSize: int(unsafe.Sizeof(Node{})),
+ ManifestSHA: sha256HexP17(manifestBytes),
+ Samples: aggregates,
+ }
+ name := "p17-arena-attribution"
+ if selectedCase != "" {
+ name += "-" + selectedCase
+ }
+ if selectedDirection != "" {
+ name += "-" + selectedDirection
+ }
+ if selectedMode != "" {
+ name += "-" + selectedMode
+ }
+ p17WriteReceipt(t, receipt, outDir, name)
+}
+
+func p17WriteReceipt(t *testing.T, receipt p17Receipt, outDir, name string) {
+ t.Helper()
+ receiptBytes, err := json.MarshalIndent(receipt, "", " ")
+ if err != nil {
+ t.Fatal(err)
+ }
+ path := filepath.Join(outDir, name+".json")
+ if err := os.WriteFile(path, append(receiptBytes, '\n'), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ t.Logf("P17 receipt=%s sha256=%s", path, sha256HexP17(append(receiptBytes, '\n')))
+}
+
+func p17Direction(tc p17Case, direction string) ([]byte, []byte, InputEdit) {
+ if direction == "forward" {
+ return tc.Source, tc.Edited, p17InputEdit(tc.Source, tc.Edited, tc.Spec.StartByte, tc.Spec.OldEndByte, tc.Spec.StartByte+len(tc.Spec.NewText))
+ }
+ return tc.Edited, tc.Source, p17InputEdit(tc.Edited, tc.Source, tc.Spec.StartByte, tc.Spec.StartByte+len(tc.Spec.NewText), tc.Spec.StartByte+len(tc.Spec.OldText))
+}
+
+func p17InputEdit(oldSource, newSource []byte, start, oldEnd, newEnd int) InputEdit {
+ return InputEdit{
+ StartByte: uint32(start),
+ OldEndByte: uint32(oldEnd),
+ NewEndByte: uint32(newEnd),
+ StartPoint: p17PointAt(oldSource, start),
+ OldEndPoint: p17PointAt(oldSource, oldEnd),
+ NewEndPoint: p17PointAt(newSource, newEnd),
+ }
+}
+
+func p17PointAt(source []byte, offset int) Point {
+ if offset < 0 {
+ offset = 0
+ }
+ if offset > len(source) {
+ offset = len(source)
+ }
+ row, col := uint32(0), uint32(0)
+ for _, b := range source[:offset] {
+ if b == '\n' {
+ row++
+ col = 0
+ } else {
+ col++
+ }
+ }
+ return Point{Row: row, Column: col}
+}
+
+func p17Measure(t *testing.T, caseName, direction, mode string, from, to []byte, edit InputEdit, repeats int, profile bool, outDir string) p17Aggregate {
+ t.Helper()
+ lang, err := LoadLanguage(parserCoreCertifiedGoBlob)
+ if err != nil {
+ t.Fatalf("load Go language: %v", err)
+ }
+ samples := make([]p17ArenaSample, 0, repeats)
+ for repeat := 0; repeat < repeats; repeat++ {
+ DrainArenaPools()
+ runtime.GC()
+ parser := NewParser(lang)
+ initialPrimary := nodeCapacityForClass(arenaClassIncremental)
+ if mode == "warmed_parser" {
+ warmOld, err := parser.Parse(from)
+ if err != nil {
+ t.Fatalf("%s/%s warm old parse: %v", caseName, direction, err)
+ }
+ warmOld.Edit(edit)
+ warmNew, _, err := parser.ParseIncrementalProfiled(to, warmOld)
+ if err != nil {
+ t.Fatalf("%s/%s warm incremental: %v", caseName, direction, err)
+ }
+ if warmNew != warmOld {
+ warmOld.Release()
+ }
+ if warmNew.arena != nil {
+ initialPrimary = len(warmNew.arena.nodes)
+ }
+ warmNew.Release()
+ runtime.GC()
+ }
+ oldTree, err := parser.Parse(from)
+ if err != nil {
+ t.Fatalf("%s/%s old parse: %v", caseName, direction, err)
+ }
+ oldTree.Edit(edit)
+ hintBefore := parser.incrementalArenaHintCapacity()
+ target := parseIncrementalArenaNodeCapacity(len(to), hintBefore)
+ // The full-parse old tree owns a different arena pool. The incremental
+ // pool is cold for first_call and retains the warm parse for warmed_parser.
+ // The old tree owns a full-parse arena. Draining the pool is safe here,
+ // and the incremental parse acquires a fresh incremental arena below.
+ trace := &p17ArenaTrace{Events: make([]p17ArenaEvent, 128)}
+ var before, after runtime.MemStats
+ runtime.ReadMemStats(&before)
+ if profile {
+ runtime.MemProfileRate = 1
+ }
+ p17CurrentArenaTrace = trace
+ newTree, prof, err := parser.ParseIncrementalProfiled(to, oldTree)
+ p17CurrentArenaTrace = nil
+ if profile {
+ runtime.MemProfileRate = 0
+ }
+ if err != nil {
+ t.Fatalf("%s/%s %s incremental: %v", caseName, direction, mode, err)
+ }
+ runtime.ReadMemStats(&after)
+ if newTree == nil || newTree.arena == nil {
+ t.Fatalf("%s/%s %s returned tree without arena", caseName, direction, mode)
+ }
+ arena := newTree.arena
+ hintAfter := parser.incrementalArenaHintCapacity()
+ caps := make([]int, len(arena.nodeSlabs))
+ var overflowBytes int64
+ for i, slab := range arena.nodeSlabs {
+ caps[i] = len(slab.data)
+ overflowBytes += int64(len(slab.data)) * int64(unsafe.Sizeof(Node{}))
+ }
+ primaryBytes := int64(len(arena.nodes)) * int64(unsafe.Sizeof(Node{}))
+ used := arena.used
+ oldTarget := p17GeometricCapacity(initialPrimary, target)
+ profilePath := ""
+ if profile {
+ profilePath = filepath.Join(outDir, fmt.Sprintf("p17-alloc-%s-%s-%s-%d.pprof", caseName, direction, mode, repeat))
+ f, openErr := os.Create(profilePath)
+ if openErr != nil {
+ t.Fatalf("create allocation profile: %v", openErr)
+ }
+ if writeErr := pprof.Lookup("allocs").WriteTo(f, 0); writeErr != nil {
+ _ = f.Close()
+ t.Fatalf("write allocation profile: %v", writeErr)
+ }
+ if closeErr := f.Close(); closeErr != nil {
+ t.Fatalf("close allocation profile: %v", closeErr)
+ }
+ }
+ sample := p17ArenaSample{
+ CaseName: caseName, Direction: direction, Mode: mode, Repeat: repeat,
+ SourceLen: len(from), TargetLen: len(to), HintBefore: hintBefore, HintAfter: hintAfter,
+ CapacityTarget: target, InitialPrimary: initialPrimary,
+ ArenaClass: p17ArenaClassName(arena.class),
+ PrimaryLen: len(arena.nodes), PrimaryCap: cap(arena.nodes), ArenaUsed: used,
+ PrimaryBytes: primaryBytes, OverflowCaps: caps, OverflowBytes: overflowBytes,
+ TotalNodeBytes: primaryBytes + overflowBytes, OldTarget: oldTarget,
+ OldCrossesUsed: used > oldTarget, ExactCrosses: used > target,
+ BytesPerOp: after.TotalAlloc - before.TotalAlloc, AllocsPerOp: after.Mallocs - before.Mallocs,
+ NewNodes: prof.NewNodesAllocated, ReusedSubtrees: prof.ReusedSubtrees, ReusedBytes: prof.ReusedBytes,
+ TokensConsumed: prof.TokensConsumed, MaxStacksSeen: prof.MaxStacksSeen, EntryPeak: prof.EntryScratchPeak,
+ StopReason: string(prof.StopReason), ParserLoopNs: prof.ParserLoopNanos,
+ DispatchNs: prof.ActionDispatchNanos, MergeNs: prof.GLRMergeNanos, ProfilePath: profilePath,
+ ArenaBytes: prof.ArenaBytesAllocated, ScratchBytes: prof.ScratchBytesAllocated,
+ EntryBytes: prof.EntryScratchBytesAllocated, GSSBytes: prof.GSSBytesAllocated,
+ ArenaEvents: p17ArenaTraceSnapshot(trace),
+ }
+ samples = append(samples, sample)
+ if newTree != oldTree {
+ oldTree.Release()
+ }
+ newTree.Release()
+ DrainArenaPools()
+ runtime.GC()
+ }
+ bytes := make([]uint64, len(samples))
+ allocs := make([]uint64, len(samples))
+ for i, sample := range samples {
+ bytes[i] = sample.BytesPerOp
+ allocs[i] = sample.AllocsPerOp
+ }
+ sort.Slice(bytes, func(i, j int) bool { return bytes[i] < bytes[j] })
+ sort.Slice(allocs, func(i, j int) bool { return allocs[i] < allocs[j] })
+ median := func(values []uint64) uint64 { return values[len(values)/2] }
+ first := samples[0]
+ return p17Aggregate{
+ CaseName: caseName, Direction: direction, Mode: mode,
+ SourceLen: first.SourceLen, TargetLen: first.TargetLen,
+ HintBefore: first.HintBefore, HintAfter: first.HintAfter, CapacityTarget: first.CapacityTarget,
+ InitialPrimary: first.InitialPrimary, PrimaryLen: first.PrimaryLen, PrimaryCap: first.PrimaryCap,
+ ArenaClass: first.ArenaClass,
+ ArenaUsed: first.ArenaUsed, PrimaryBytes: first.PrimaryBytes, OverflowCaps: first.OverflowCaps,
+ OverflowBytes: first.OverflowBytes, TotalNodeBytes: first.TotalNodeBytes, OldTarget: first.OldTarget,
+ OldCrossesUsed: first.OldCrossesUsed, ExactCrosses: first.ExactCrosses,
+ ArenaBytes: first.ArenaBytes, ScratchBytes: first.ScratchBytes,
+ EntryBytes: first.EntryBytes, GSSBytes: first.GSSBytes,
+ MedianBytes: median(bytes), MedianAllocs: median(allocs), MinBytes: bytes[0], MaxBytes: bytes[len(bytes)-1],
+ MinAllocs: allocs[0], MaxAllocs: allocs[len(allocs)-1], Samples: samples,
+ }
+}
+
+func p17RunSequence(t *testing.T, tc p17Case, profile bool, outDir string) p17SequenceReceipt {
+ t.Helper()
+ lang, err := LoadLanguage(parserCoreCertifiedGoBlob)
+ if err != nil {
+ t.Fatalf("load Go language: %v", err)
+ }
+ parser := NewParser(lang)
+ tree, err := parser.Parse(tc.Source)
+ if err != nil {
+ t.Fatalf("%s initial parse: %v", tc.Spec.Name, err)
+ }
+ const cycles = 12
+ steps := make([]p17SequenceStep, 0, cycles)
+ var lastIncrementalPrimary int
+ for i := 0; i < cycles; i++ {
+ direction := "forward"
+ from, to, edit := tc.Source, tc.Edited, p17InputEdit(tc.Source, tc.Edited, tc.Spec.StartByte, tc.Spec.OldEndByte, tc.Spec.StartByte+len(tc.Spec.NewText))
+ if i%2 == 1 {
+ direction = "reverse"
+ from, to, edit = tc.Edited, tc.Source, p17InputEdit(tc.Edited, tc.Source, tc.Spec.StartByte, tc.Spec.StartByte+len(tc.Spec.NewText), tc.Spec.StartByte+len(tc.Spec.OldText))
+ }
+ if tree.Source() == nil {
+ t.Fatalf("%s sequence tree has nil source before iteration %d", tc.Spec.Name, i)
+ }
+ tree.Edit(edit)
+ hintBefore := parser.incrementalArenaHintCapacity()
+ target := parseIncrementalArenaNodeCapacity(len(to), hintBefore)
+ initialPrimary := nodeCapacityForClass(arenaClassIncremental)
+ if i > 0 && lastIncrementalPrimary > 0 {
+ initialPrimary = lastIncrementalPrimary
+ }
+ trace := &p17ArenaTrace{Events: make([]p17ArenaEvent, 128)}
+ var before, after runtime.MemStats
+ runtime.ReadMemStats(&before)
+ if profile && i == 0 {
+ runtime.MemProfileRate = 1
+ }
+ p17CurrentArenaTrace = trace
+ newTree, prof, err := parser.ParseIncrementalProfiled(to, tree)
+ p17CurrentArenaTrace = nil
+ if err != nil {
+ t.Fatalf("%s sequence iteration %d: %v", tc.Spec.Name, i, err)
+ }
+ runtime.ReadMemStats(&after)
+ if newTree == nil || newTree.arena == nil {
+ t.Fatalf("%s sequence iteration %d returned nil arena", tc.Spec.Name, i)
+ }
+ arena := newTree.arena
+ caps := make([]int, len(arena.nodeSlabs))
+ var overflowBytes int64
+ for j, slab := range arena.nodeSlabs {
+ caps[j] = len(slab.data)
+ overflowBytes += int64(len(slab.data)) * int64(unsafe.Sizeof(Node{}))
+ }
+ used := arena.used
+ oldTarget := p17GeometricCapacity(initialPrimary, target)
+ steps = append(steps, p17SequenceStep{
+ Iteration: i, Direction: direction, SourceLen: len(from), TargetLen: len(to),
+ HintBefore: hintBefore, HintAfter: parser.incrementalArenaHintCapacity(), CapacityTarget: target,
+ InitialPrimary: initialPrimary, ArenaClass: p17ArenaClassName(arena.class),
+ PrimaryLen: len(arena.nodes), ArenaUsed: used, OverflowCaps: caps, OverflowBytes: overflowBytes,
+ OldTarget: oldTarget, OldCrossesUsed: used > oldTarget, ExactCrosses: used > target,
+ BytesPerOp: after.TotalAlloc - before.TotalAlloc, AllocsPerOp: after.Mallocs - before.Mallocs,
+ NewNodes: prof.NewNodesAllocated, ArenaBytes: prof.ArenaBytesAllocated, StopReason: string(prof.StopReason),
+ ArenaEvents: p17ArenaTraceSnapshot(trace),
+ })
+ if newTree != tree {
+ if tree.arena != nil && tree.arena.class == arenaClassIncremental {
+ lastIncrementalPrimary = len(tree.arena.nodes)
+ }
+ tree.Release()
+ }
+ tree = newTree
+ }
+ if profile {
+ runtime.MemProfileRate = 0
+ }
+ profilePath := ""
+ if profile {
+ profilePath = filepath.Join(outDir, "p17-sequence-"+tc.Spec.Name+".pprof")
+ f, createErr := os.Create(profilePath)
+ if createErr != nil {
+ t.Fatal(createErr)
+ }
+ if writeErr := pprof.Lookup("allocs").WriteTo(f, 0); writeErr != nil {
+ _ = f.Close()
+ t.Fatal(writeErr)
+ }
+ if closeErr := f.Close(); closeErr != nil {
+ t.Fatal(closeErr)
+ }
+ }
+ tree.Release()
+ DrainArenaPools()
+ runtime.GC()
+ return p17SequenceReceipt{CaseName: tc.Spec.Name, Cycles: cycles, ProfilePath: profilePath, Steps: steps}
+}
+
+func p17GeometricCapacity(initial, target int) int {
+ if initial < minArenaNodeCap {
+ initial = minArenaNodeCap
+ }
+ if target <= initial {
+ return initial
+ }
+ for initial < target {
+ initial *= 2
+ }
+ return initial
+}
+
+func p17ArenaClassName(class arenaClass) string {
+ if class == arenaClassFull {
+ return "full"
+ }
+ return "incremental"
+}
+
+func sha256HexP17(data []byte) string {
+ digest := sha256.Sum256(data)
+ return hex.EncodeToString(digest[:])
+}
+
+func p17LoadGzip(path string) ([]byte, error) {
+ f, err := os.Open(path)
+ if err != nil {
+ return nil, err
+ }
+ defer f.Close()
+ zr, err := gzip.NewReader(f)
+ if err != nil {
+ return nil, err
+ }
+ data, readErr := io.ReadAll(zr)
+ closeErr := zr.Close()
+ if readErr != nil {
+ return nil, readErr
+ }
+ return data, closeErr
+}
diff --git a/p25b_issue454_profile_test.go b/p25b_issue454_profile_test.go
new file mode 100644
index 000000000..c6ed31676
--- /dev/null
+++ b/p25b_issue454_profile_test.go
@@ -0,0 +1,184 @@
+package gotreesitter_test
+
+import (
+ "bytes"
+ "fmt"
+ "os"
+ "runtime"
+ "strconv"
+ "testing"
+ "time"
+
+ gotreesitter "github.com/odvcencio/gotreesitter"
+ "github.com/odvcencio/gotreesitter/grammars"
+ "github.com/odvcencio/gotreesitter/internal/benchfixtures"
+)
+
+func TestP25BIssue454Profile(t *testing.T) {
+ mode := os.Getenv("P25B_MODE")
+ if mode == "" {
+ t.Skip("set P25B_MODE=fresh or incremental")
+ }
+ runs := 3
+ if raw := os.Getenv("P25B_RUNS"); raw != "" {
+ value, err := strconv.Atoi(raw)
+ if err != nil || value < 1 {
+ t.Fatalf("invalid P25B_RUNS=%q", raw)
+ }
+ runs = value
+ }
+
+ source := benchfixtures.Issue454CSource()
+ site := bytes.Index(source, []byte("x0"))
+ if site < 0 {
+ t.Fatal("C edit marker is absent")
+ }
+ edited := append(append([]byte(nil), source[:site]...), source[site+1:]...)
+ point := p25bPointAt(source, site)
+ edit := gotreesitter.InputEdit{
+ StartByte: uint32(site),
+ OldEndByte: uint32(site + 1),
+ NewEndByte: uint32(site),
+ StartPoint: point,
+ OldEndPoint: p25bPointAt(source, site+1),
+ NewEndPoint: point,
+ }
+
+ gotreesitter.EnableRecoveryRuntimeTelemetry(true)
+ t.Cleanup(func() { gotreesitter.EnableRecoveryRuntimeTelemetry(false) })
+
+ lang := grammars.CLanguage()
+ switch mode {
+ case "fresh":
+ p25bRunFresh(t, lang, edited, runs)
+ case "incremental":
+ p25bRunIncremental(t, lang, source, edited, edit, runs)
+ default:
+ t.Fatalf("unsupported P25B_MODE=%q", mode)
+ }
+}
+
+func p25bRunFresh(t *testing.T, lang *gotreesitter.Language, source []byte, runs int) {
+ t.Helper()
+ parser := gotreesitter.NewParser(lang)
+ for run := 0; run < runs; run++ {
+ runtime.GC()
+ before := p25bMemStats()
+ started := time.Now()
+ tree, err := parser.Parse(source)
+ wall := time.Since(started)
+ if err != nil {
+ t.Fatalf("fresh run %d: %v", run, err)
+ }
+ if tree == nil || tree.RootNode() == nil {
+ t.Fatalf("fresh run %d returned no tree", run)
+ }
+ runtimeStats := tree.ParseRuntime()
+ recoveryStats := parser.DebugRecoveryRuntimeStats()
+ digest := p25bDigest(t, tree, lang)
+ after := p25bMemStats()
+ fmt.Printf("P25B_FRESH run=%d bytes=%d wall_ns=%d mallocs=%d total_alloc=%d heap_alloc_delta=%d digest=%s root_end=%d has_error=%v stop=%s nodes=%d arena=%d scratch=%d gss=%d memory_budget=%d memory_stop_source=%s runtime_heap_growth=%d runtime_sys_growth=%d tokens=%d max_stacks=%d parser_loop_ns=%d recovery_entries=%d recovery_cost_competitions=%d recovery_cost_walk_ns=%d\n",
+ run, len(source), wall.Nanoseconds(), after.Mallocs-before.Mallocs, after.TotalAlloc-before.TotalAlloc,
+ int64(after.HeapAlloc)-int64(before.HeapAlloc), digest, tree.RootNode().EndByte(), tree.RootNode().HasError(),
+ runtimeStats.StopReason, runtimeStats.NodesAllocated, runtimeStats.ArenaBytesAllocated, runtimeStats.ScratchBytesAllocated,
+ runtimeStats.GSSBytesAllocated, runtimeStats.MemoryBudgetBytes, runtimeStats.MemoryBudgetStopSource,
+ runtimeStats.RuntimeHeapGrowthBytes, runtimeStats.RuntimeSysGrowthBytes, runtimeStats.TokensConsumed,
+ runtimeStats.MaxStacksSeen, runtimeStats.ParserLoopNanos, recoveryStats.RecoveryEntryCount,
+ recoveryStats.RecoveryCostCompetitionCount, recoveryStats.RecoveryCostWalkNanos)
+ p25bPrintAttempts(parser, "fresh", run)
+ tree.Release()
+ }
+}
+
+func p25bRunIncremental(t *testing.T, lang *gotreesitter.Language, source, edited []byte, edit gotreesitter.InputEdit, runs int) {
+ t.Helper()
+ for run := 0; run < runs; run++ {
+ parser := gotreesitter.NewParser(lang)
+ oldTree, err := parser.Parse(source)
+ if err != nil {
+ t.Fatalf("base run %d: %v", run, err)
+ }
+ baseDigest := p25bDigest(t, oldTree, lang)
+ runtime.GC()
+ beforeEdit := p25bMemStats()
+ editStarted := time.Now()
+ oldTree.Edit(edit)
+ editWall := time.Since(editStarted)
+ afterEdit := p25bMemStats()
+ parseStarted := time.Now()
+ incremental, profile, err := parser.ParseIncrementalProfiled(edited, oldTree)
+ parseWall := time.Since(parseStarted)
+ if err != nil {
+ oldTree.Release()
+ t.Fatalf("incremental run %d: %v", run, err)
+ }
+ if incremental == nil || incremental.RootNode() == nil {
+ t.Fatalf("incremental run %d returned no tree", run)
+ }
+ afterParse := p25bMemStats()
+ runtimeStats := incremental.ParseRuntime()
+ recoveryStats := parser.DebugRecoveryRuntimeStats()
+ digest := p25bDigest(t, incremental, lang)
+ fmt.Printf("P25B_INCREMENTAL run=%d bytes=%d site=%d base_digest=%s incremental_digest=%s edit_ns=%d edit_mallocs=%d edit_total_alloc=%d parse_wall_ns=%d parse_mallocs=%d parse_total_alloc=%d parse_heap_alloc_delta=%d reuse_ns=%d reparse_ns=%d reused_subtrees=%d reused_bytes=%d new_nodes=%d reuse_unsupported=%v reuse_reason=%s old_tree_reuse_route=%v reject_dirty=%d reject_ancestor_dirty=%d reject_root_nonleaf=%d reject_fragile=%d reject_scanner=%d block_splice_steps=%d recover_searches=%d recover_state_checks=%d recover_state_skips=%d recover_symbol_skips=%d recover_lookups=%d recover_hits=%d tokens=%d max_stacks=%d entry_scratch_peak=%d stop=%s nodes=%d arena=%d scratch=%d gss=%d memory_budget=%d memory_stop_source=%s runtime_heap_growth=%d runtime_sys_growth=%d c_recovery_entered=%v c_recovery_dropped_clean=%v retry_passes=%d recovery_entries=%d recovery_cost_competitions=%d recovery_cost_walk_ns=%d parser_loop_ns=%d result_selection_ns=%d result_tree_build_ns=%d normalization_ns=%d root_end=%d has_error=%v\n",
+ run, len(edited), edit.StartByte, baseDigest, digest, editWall.Nanoseconds(),
+ afterEdit.Mallocs-beforeEdit.Mallocs, afterEdit.TotalAlloc-beforeEdit.TotalAlloc,
+ parseWall.Nanoseconds(), afterParse.Mallocs-afterEdit.Mallocs, afterParse.TotalAlloc-afterEdit.TotalAlloc,
+ int64(afterParse.HeapAlloc)-int64(afterEdit.HeapAlloc), profile.ReuseCursorNanos, profile.ReparseNanos,
+ profile.ReusedSubtrees, profile.ReusedBytes, profile.NewNodesAllocated, profile.ReuseUnsupported,
+ profile.ReuseUnsupportedReason, profile.OldTreeReuseRoute, profile.ReuseRejectDirty,
+ profile.ReuseRejectAncestorDirtyBeforeEdit, profile.ReuseRejectRootNonLeafChanged,
+ profile.ReuseRejectFragileNonLeaf, profile.ReuseRejectScannerUnquiescent, profile.BlockSpliceSteps,
+ profile.RecoverSearches, profile.RecoverStateChecks, profile.RecoverStateSkips, profile.RecoverSymbolSkips,
+ profile.RecoverLookups, profile.RecoverHits, profile.TokensConsumed, profile.MaxStacksSeen,
+ profile.EntryScratchPeak, profile.StopReason, runtimeStats.NodesAllocated, runtimeStats.ArenaBytesAllocated,
+ runtimeStats.ScratchBytesAllocated, runtimeStats.GSSBytesAllocated, runtimeStats.MemoryBudgetBytes,
+ runtimeStats.MemoryBudgetStopSource, runtimeStats.RuntimeHeapGrowthBytes, runtimeStats.RuntimeSysGrowthBytes,
+ runtimeStats.CRecoveryEnteredErrorState, runtimeStats.CRecoveryDroppedErrorForClean,
+ runtimeStats.IncrementalAcceptedErrorRetryAttempts, recoveryStats.RecoveryEntryCount,
+ recoveryStats.RecoveryCostCompetitionCount, recoveryStats.RecoveryCostWalkNanos, runtimeStats.ParserLoopNanos,
+ runtimeStats.ResultSelectionNanos, runtimeStats.ResultTreeBuildNanos, profile.NormalizationNanos,
+ incremental.RootNode().EndByte(), incremental.RootNode().HasError())
+ p25bPrintAttempts(parser, "incremental", run)
+ if incremental != oldTree {
+ oldTree.Release()
+ }
+ incremental.Release()
+ }
+}
+
+func p25bDigest(t *testing.T, tree *gotreesitter.Tree, lang *gotreesitter.Language) string {
+ t.Helper()
+ inspection, err := benchfixtures.InspectGoTree(tree.RootNode(), lang)
+ if err != nil {
+ t.Fatalf("inspect tree: %v", err)
+ }
+ return inspection.SHA256
+}
+
+func p25bPointAt(source []byte, offset int) gotreesitter.Point {
+ row := bytes.Count(source[:offset], []byte{'\n'})
+ column := offset
+ if newline := bytes.LastIndexByte(source[:offset], '\n'); newline >= 0 {
+ column = offset - newline - 1
+ }
+ return gotreesitter.Point{Row: uint32(row), Column: uint32(column)}
+}
+
+func p25bMemStats() runtime.MemStats {
+ var stats runtime.MemStats
+ runtime.ReadMemStats(&stats)
+ return stats
+}
+
+func p25bPrintAttempts(parser *gotreesitter.Parser, phase string, run int) {
+ for _, attempt := range parser.DebugRecoveryRuntimeAttempts() {
+ fmt.Printf("P25B_ATTEMPT phase=%s run=%d ordinal=%d rung=%s cause=%s stop=%s truncated=%v full_span=%v has_error=%v wall_ns=%d heap_delta=%d total_alloc=%d mallocs=%d recovery_entries=%d recovery_cost_competitions=%d recovery_cost_walk_ns=%d materialization_ns=%d arena_peak=%d scratch_peak=%d entry_scratch_peak=%d gss_peak=%d nodes=%d max_stacks=%d peak_depth=%d live_versions=%d peak_live_versions=%d selected=%v replaced=%v\n",
+ phase, run, attempt.Ordinal, attempt.Rung, attempt.Cause, attempt.StopReason, attempt.Truncated,
+ attempt.AttemptFullSpan, attempt.AttemptHasError, attempt.WallNanos, attempt.HeapAllocDeltaBytes,
+ attempt.TotalAllocDeltaBytes, attempt.MallocsDelta, attempt.RecoveryEntryCount,
+ attempt.RecoveryCostCompetitionCount, attempt.RecoveryCostWalkNanos, attempt.MaterializationNanos,
+ attempt.ArenaBytesPeak, attempt.ScratchBytesPeak, attempt.EntryScratchBytesPeak, attempt.GSSBytesPeak,
+ attempt.NodesAllocated, attempt.MaxStacksSeen, attempt.PeakStackDepth, attempt.LiveVersions,
+ attempt.PeakLiveVersions, attempt.CandidateSelected, attempt.CandidateReplacedIncumbent)
+ }
+}
diff --git a/p25c_objc_probe_test.go b/p25c_objc_probe_test.go
new file mode 100644
index 000000000..ae7d976ec
--- /dev/null
+++ b/p25c_objc_probe_test.go
@@ -0,0 +1,324 @@
+package gotreesitter_test
+
+import (
+ "crypto/sha256"
+ "encoding/json"
+ "fmt"
+ "os"
+ "runtime"
+ "strconv"
+ "strings"
+ "testing"
+ "time"
+
+ gotreesitter "github.com/odvcencio/gotreesitter"
+ "github.com/odvcencio/gotreesitter/grammars"
+)
+
+type p25cOperationReceipt struct {
+ Label string `json:"label"`
+ Bytes int `json:"bytes"`
+ SourceSHA256 string `json:"source_sha256"`
+ WallNanos int64 `json:"wall_nanos"`
+ TreeEditNanos int64 `json:"tree_edit_nanos,omitempty"`
+ HeapAllocDelta int64 `json:"heap_alloc_delta"`
+ TotalAllocDelta uint64 `json:"total_alloc_delta"`
+ MallocsDelta uint64 `json:"mallocs_delta"`
+ TreeDigest string `json:"tree_digest"`
+ FreshDigest string `json:"fresh_digest,omitempty"`
+ IncrementalDigest string `json:"incremental_digest,omitempty"`
+ FreshEqual bool `json:"fresh_equal,omitempty"`
+ Profile gotreesitter.IncrementalParseProfile `json:"profile,omitempty"`
+ Runtime gotreesitter.ParseRuntime `json:"runtime"`
+ Recovery gotreesitter.RecoveryRuntimeStats `json:"recovery"`
+ Attempts gotreesitter.RecoveryRuntimeAttempts `json:"attempts,omitempty"`
+ Error string `json:"error,omitempty"`
+}
+
+func TestP25CObjcProbe(t *testing.T) {
+ target := p25cTargetBytes(t)
+ source, editOffset := p25cObjcSource(target)
+ if len(source) != target {
+ t.Fatalf("source bytes=%d want=%d", len(source), target)
+ }
+ if editOffset <= 0 || editOffset >= len(source) || source[editOffset] != 'x' {
+ t.Fatalf("edit offset=%d does not identify x", editOffset)
+ }
+ gotreesitter.EnableRecoveryRuntimeTelemetry(true)
+ defer gotreesitter.EnableRecoveryRuntimeTelemetry(false)
+
+ t.Logf("P25C_FIXTURE bytes=%d edit_offset=%d source_sha256=%x", len(source), editOffset, sha256.Sum256(source))
+ base, baseReceipt := p25cFresh("full", source, editOffset, nil)
+ if baseReceipt.Error != "" {
+ t.Fatal(baseReceipt.Error)
+ }
+ if base == nil || base.RootNode() == nil {
+ t.Fatal("base parse returned no root")
+ }
+ base.Release()
+
+ for _, kind := range []string{"insert", "delete"} {
+ t.Run(kind, func(t *testing.T) {
+ edited, edit := p25cApplyEdit(source, editOffset, kind)
+ fresh, freshReceipt := p25cFresh("fresh_"+kind, edited, editOffset, nil)
+ if freshReceipt.Error != "" {
+ t.Fatal(freshReceipt.Error)
+ }
+ if fresh == nil || fresh.RootNode() == nil {
+ t.Fatal("fresh edited parse returned no root")
+ }
+ freshDigest := p25cTreeDigest(fresh, grammars.ObjcLanguage())
+ fresh.Release()
+ old, oldReceipt := p25cFresh("old_"+kind, source, editOffset, nil)
+ if oldReceipt.Error != "" {
+ t.Fatal(oldReceipt.Error)
+ }
+ if old == nil || old.RootNode() == nil {
+ t.Fatal("old parse returned no root")
+ }
+ t.Cleanup(old.Release)
+ beforeEdit := time.Now()
+ old.Edit(edit)
+ treeEditNanos := time.Since(beforeEdit).Nanoseconds()
+ parser := gotreesitter.NewParser(grammars.ObjcLanguage())
+ parser.SetAdmissionCandidateRoute(false)
+ var before, after runtime.MemStats
+ runtime.GC()
+ runtime.ReadMemStats(&before)
+ started := time.Now()
+ incremental, profile, err := parser.ParseIncrementalProfiled(edited, old)
+ wall := time.Since(started).Nanoseconds()
+ runtime.ReadMemStats(&after)
+ receipt := p25cOperationReceipt{
+ Label: "incremental_" + kind,
+ Bytes: len(edited),
+ SourceSHA256: fmt.Sprintf("%x", sha256.Sum256(edited)),
+ WallNanos: wall,
+ TreeEditNanos: treeEditNanos,
+ HeapAllocDelta: int64(after.HeapAlloc) - int64(before.HeapAlloc),
+ TotalAllocDelta: after.TotalAlloc - before.TotalAlloc,
+ MallocsDelta: after.Mallocs - before.Mallocs,
+ Profile: profile,
+ }
+ if err != nil {
+ receipt.Error = err.Error()
+ }
+ if incremental != nil && incremental.RootNode() != nil {
+ receipt.TreeDigest = p25cTreeDigest(incremental, grammars.ObjcLanguage())
+ receipt.Runtime = incremental.ParseRuntime()
+ receipt.Recovery = parser.DebugRecoveryRuntimeStats()
+ receipt.Attempts = parser.DebugRecoveryRuntimeAttempts()
+ }
+ receipt.FreshDigest = freshDigest
+ receipt.IncrementalDigest = receipt.TreeDigest
+ receipt.FreshEqual = receipt.FreshDigest != "" && receipt.FreshDigest == receipt.IncrementalDigest
+ p25cLogReceipt(t, receipt)
+ if incremental != nil && incremental != old {
+ t.Cleanup(incremental.Release)
+ }
+ })
+ }
+}
+
+func TestP25CObjcCorrectness(t *testing.T) {
+ target := p25cTargetBytes(t)
+ source, editOffset := p25cObjcSource(target)
+ if len(source) != target {
+ t.Fatalf("source bytes=%d want=%d", len(source), target)
+ }
+ for _, kind := range []string{"insert", "delete"} {
+ t.Run(kind, func(t *testing.T) {
+ edited, edit := p25cApplyEdit(source, editOffset, kind)
+ fresh, err := gotreesitter.NewParser(grammars.ObjcLanguage()).Parse(edited)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if fresh == nil || fresh.RootNode() == nil {
+ t.Fatal("fresh parse returned no root")
+ }
+ t.Cleanup(fresh.Release)
+ old, err := gotreesitter.NewParser(grammars.ObjcLanguage()).Parse(source)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if old == nil || old.RootNode() == nil {
+ t.Fatal("old parse returned no root")
+ }
+ t.Cleanup(old.Release)
+ old.Edit(edit)
+ parser := gotreesitter.NewParser(grammars.ObjcLanguage())
+ parser.SetAdmissionCandidateRoute(false)
+ incremental, _, err := parser.ParseIncrementalProfiled(edited, old)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if incremental == nil || incremental.RootNode() == nil {
+ t.Fatal("incremental parse returned no root")
+ }
+ t.Cleanup(func() {
+ if incremental != old {
+ incremental.Release()
+ }
+ })
+ freshRoot := fresh.RootNode()
+ incrementalRoot := incremental.RootNode()
+ if got, want := freshRoot.EndByte(), uint32(len(edited)); got != want {
+ t.Fatalf("fresh root end=%d want=%d", got, want)
+ }
+ if got, want := incrementalRoot.EndByte(), uint32(len(edited)); got != want {
+ t.Fatalf("incremental root end=%d want=%d", got, want)
+ }
+ if fresh.ParseRuntime().StopReason != gotreesitter.ParseStopAccepted ||
+ incremental.ParseRuntime().StopReason != gotreesitter.ParseStopAccepted {
+ t.Fatalf("stop reasons fresh=%s incremental=%s", fresh.ParseRuntime().StopReason, incremental.ParseRuntime().StopReason)
+ }
+ if fresh.ParseRuntime().Truncated || incremental.ParseRuntime().Truncated {
+ t.Fatal("correctness parse truncated")
+ }
+ if got, want := freshRoot.HasError(), kind == "delete"; got != want {
+ t.Fatalf("fresh has_error=%v want=%v", got, want)
+ }
+ freshDigest := p25cTreeDigest(fresh, grammars.ObjcLanguage())
+ incrementalDigest := p25cTreeDigest(incremental, grammars.ObjcLanguage())
+ if freshDigest != incrementalDigest {
+ t.Fatalf("fresh/incremental digest mismatch fresh=%s incremental=%s", freshDigest, incrementalDigest)
+ }
+ t.Logf("P25C_CORRECTNESS kind=%s bytes=%d fresh=%s incremental=%s has_error=%v", kind, len(edited), freshDigest, incrementalDigest, freshRoot.HasError())
+ })
+ }
+}
+
+func p25cFresh(label string, source []byte, editOffset int, _ *gotreesitter.Parser) (*gotreesitter.Tree, p25cOperationReceipt) {
+ parser := gotreesitter.NewParser(grammars.ObjcLanguage())
+ parser.SetAdmissionCandidateRoute(false)
+ runtime.GC()
+ var before, after runtime.MemStats
+ runtime.ReadMemStats(&before)
+ started := time.Now()
+ tree, err := parser.Parse(source)
+ wall := time.Since(started).Nanoseconds()
+ runtime.ReadMemStats(&after)
+ receipt := p25cOperationReceipt{
+ Label: label,
+ Bytes: len(source),
+ SourceSHA256: fmt.Sprintf("%x", sha256.Sum256(source)),
+ WallNanos: wall,
+ HeapAllocDelta: int64(after.HeapAlloc) - int64(before.HeapAlloc),
+ TotalAllocDelta: after.TotalAlloc - before.TotalAlloc,
+ MallocsDelta: after.Mallocs - before.Mallocs,
+ }
+ if err != nil {
+ receipt.Error = err.Error()
+ }
+ if tree != nil && tree.RootNode() != nil {
+ receipt.TreeDigest = p25cTreeDigest(tree, grammars.ObjcLanguage())
+ receipt.Runtime = tree.ParseRuntime()
+ receipt.Recovery = parser.DebugRecoveryRuntimeStats()
+ receipt.Attempts = parser.DebugRecoveryRuntimeAttempts()
+ }
+ if os.Getenv("P25C_LOG_FRESH") != "0" {
+ fmt.Printf("P25C_RECEIPT %s\n", p25cJSON(receipt))
+ }
+ _ = editOffset
+ return tree, receipt
+}
+
+func p25cApplyEdit(source []byte, offset int, kind string) ([]byte, gotreesitter.InputEdit) {
+ replacement := []byte("x")
+ oldEnd := offset
+ switch kind {
+ case "insert":
+ replacement = []byte("x")
+ case "delete":
+ oldEnd = offset + 1
+ replacement = nil
+ default:
+ panic("unknown edit kind " + kind)
+ }
+ edited := make([]byte, 0, len(source)-oldEnd+offset+len(replacement))
+ edited = append(edited, source[:offset]...)
+ edited = append(edited, replacement...)
+ edited = append(edited, source[oldEnd:]...)
+ line := uint32(bytesLine(source, offset))
+ point := gotreesitter.Point{Row: line, Column: uint32(bytesColumn(source, offset))}
+ return edited, gotreesitter.InputEdit{
+ StartByte: uint32(offset),
+ OldEndByte: uint32(oldEnd),
+ NewEndByte: uint32(offset + len(replacement)),
+ StartPoint: point,
+ OldEndPoint: point,
+ NewEndPoint: point,
+ }
+}
+
+func p25cObjcSource(target int) ([]byte, int) {
+ const prefix = "@interface P25CBox : NSObject\n@end\n@implementation P25CBox\n"
+ const suffix = "@end\n"
+ var b strings.Builder
+ b.Grow(target)
+ b.WriteString(prefix)
+ editOffset := -1
+ for i := 0; ; i++ {
+ line := fmt.Sprintf("- (int)m%d:(int)arg { int x%d = %d; return x%d; }\n", i, i, i, i)
+ if b.Len()+len(line)+len(suffix) > target {
+ break
+ }
+ if editOffset < 0 {
+ editOffset = b.Len() + strings.Index(line, "x0")
+ }
+ b.WriteString(line)
+ }
+ b.WriteString(suffix)
+ if b.Len() < target {
+ b.WriteString(strings.Repeat(" ", target-b.Len()))
+ }
+ return []byte(b.String()), editOffset
+}
+
+func p25cTargetBytes(t *testing.T) int {
+ t.Helper()
+ value := os.Getenv("P25C_BYTES")
+ if value == "" {
+ return 4 * 1024
+ }
+ n, err := strconv.Atoi(value)
+ if err != nil || n < 256 {
+ t.Fatalf("invalid P25C_BYTES=%q", value)
+ }
+ return n
+}
+
+func p25cTreeDigest(tree *gotreesitter.Tree, lang *gotreesitter.Language) string {
+ if tree == nil || tree.RootNode() == nil {
+ return ""
+ }
+ sum := sha256.Sum256([]byte(tree.RootNode().SExpr(lang)))
+ return fmt.Sprintf("%x", sum)
+}
+
+func p25cLogReceipt(t *testing.T, receipt p25cOperationReceipt) {
+ t.Helper()
+ line := "P25C_RECEIPT " + p25cJSON(receipt)
+ t.Log(line)
+ fmt.Println(line)
+}
+
+func p25cJSON(value any) string {
+ encoded, err := json.Marshal(value)
+ if err != nil {
+ return fmt.Sprintf("json_error:%v", err)
+ }
+ return string(encoded)
+}
+
+func bytesLine(source []byte, offset int) int {
+ return strings.Count(string(source[:offset]), "\n")
+}
+
+func bytesColumn(source []byte, offset int) int {
+ last := strings.LastIndexByte(string(source[:offset]), '\n')
+ if last < 0 {
+ return offset
+ }
+ return offset - last - 1
+}
diff --git a/parser_go_materialization_cycle_regression_test.go b/parser_go_materialization_cycle_regression_test.go
new file mode 100644
index 000000000..d05802926
--- /dev/null
+++ b/parser_go_materialization_cycle_regression_test.go
@@ -0,0 +1,79 @@
+package gotreesitter_test
+
+import (
+ "os"
+ "testing"
+ "time"
+
+ "github.com/odvcencio/gotreesitter"
+ "github.com/odvcencio/gotreesitter/grammars"
+)
+
+func TestGoRepeatedNestedSubtestsReturnedTreeAcyclic(t *testing.T) {
+ fixture, err := os.ReadFile("internal/parsercorephase0/core_test.go")
+ if err != nil {
+ t.Fatal(err)
+ }
+ const start, end = 60255, 68583
+ if len(fixture) < end {
+ t.Fatalf("fixture size = %d, want at least %d", len(fixture), end)
+ }
+ source := append([]byte("package p\n"), fixture[start:end]...)
+ function := append([]byte(nil), source[len("package p\n"):]...)
+ for range 7 {
+ source = append(source, function...)
+ }
+
+ type parseResult struct {
+ tree *gotreesitter.Tree
+ err error
+ }
+ done := make(chan parseResult, 1)
+ go func() {
+ tree, err := gotreesitter.NewParser(grammars.GoLanguage()).Parse(source)
+ done <- parseResult{tree: tree, err: err}
+ }()
+
+ var tree *gotreesitter.Tree
+ select {
+ case result := <-done:
+ if result.err != nil {
+ t.Fatal(result.err)
+ }
+ tree = result.tree
+ case <-time.After(30 * time.Second):
+ t.Fatal("parse did not return")
+ }
+ defer tree.Release()
+
+ root := tree.RootNode()
+ type frame struct {
+ node *gotreesitter.Node
+ child int
+ }
+ active := map[*gotreesitter.Node]struct{}{root: {}}
+ stack := []frame{{node: root}}
+ for len(stack) > 0 {
+ top := &stack[len(stack)-1]
+ if top.child >= top.node.ChildCount() {
+ delete(active, top.node)
+ stack = stack[:len(stack)-1]
+ continue
+ }
+ child := top.node.Child(top.child)
+ top.child++
+ if child == nil {
+ continue
+ }
+ if _, ok := active[child]; ok {
+ t.Fatalf(
+ "returned tree contains a back-edge to %s [%d,%d)",
+ child.Type(grammars.GoLanguage()),
+ child.StartByte(),
+ child.EndByte(),
+ )
+ }
+ active[child] = struct{}{}
+ stack = append(stack, frame{node: child})
+ }
+}
From 3416e9a1175433961ed63b1fb898fccb8bb0bb42 Mon Sep 17 00:00:00 2001
From: Oscar Villavicencio <9220505+odvcencio@users.noreply.github.com>
Date: Sun, 23 Aug 2026 14:47:09 -0700
Subject: [PATCH 2/2] add: Add external scanner checkpoint lifecycle
- Add external scanner checkpoint lifecycle to manage payload ownership across parse epochs and GLR branches.
- Verify checkpoint deserialization before accepting scan results. Roll back state on failed restores.
- Track version forks and merge exact duplicates. Discard mismatched branches after failed scans.
- Add unit tests for lifecycle opt-in, failed restore cleanup, exact versus mismatched fork handling, and payload isolation.
- Extend CGO parity harness with locked-C verification for BitBake and Corn normalization fixtures.
- Add Swift probe tests for associated type conformance witnesses and masking shift operators.
- Validate Rust dispatch routes against the locked C deep digest.
- Expand Issue 454 diagnostics across five file sizes and four target languages.
Buckley-Change-Hash: sha256:31565164b2d60bd430f5c14f7d419d021adbe92d98b486f18ce23a9d673e5bb9
Buckley-Change-Stats: files=12 insertions=2026 deletions=0 binaries=0
---
cgo_harness/bitbake_retirement_parity_test.go | 205 ++++++++
cgo_harness/c26b_swift_assoc_minimal.swift | 3 +
.../c26b_swift_associatedtype_probe_test.go | 180 +++++++
cgo_harness/c26c_swift_mask_minimal.swift | 1 +
.../c26c_swift_masking_shift_probe_test.go | 116 +++++
cgo_harness/corn_retirement_parity_test.go | 148 ++++++
.../issue454_variant_cguard_fivesize_test.go | 60 +++
.../issue454_variant_locked_c_diag_test.go | 94 ++++
.../rust_dispatch_locked_c_probe_test.go | 216 +++++++++
external_scanner_checkpoint_lifecycle.go | 447 ++++++++++++++++++
external_scanner_checkpoint_lifecycle_test.go | 273 +++++++++++
rust_dispatch_probe_test.go | 283 +++++++++++
12 files changed, 2026 insertions(+)
create mode 100644 cgo_harness/bitbake_retirement_parity_test.go
create mode 100644 cgo_harness/c26b_swift_assoc_minimal.swift
create mode 100644 cgo_harness/c26b_swift_associatedtype_probe_test.go
create mode 100644 cgo_harness/c26c_swift_mask_minimal.swift
create mode 100644 cgo_harness/c26c_swift_masking_shift_probe_test.go
create mode 100644 cgo_harness/corn_retirement_parity_test.go
create mode 100644 cgo_harness/issue454_variant_cguard_fivesize_test.go
create mode 100644 cgo_harness/issue454_variant_locked_c_diag_test.go
create mode 100644 cgo_harness/rust_dispatch_locked_c_probe_test.go
create mode 100644 external_scanner_checkpoint_lifecycle.go
create mode 100644 external_scanner_checkpoint_lifecycle_test.go
create mode 100644 rust_dispatch_probe_test.go
diff --git a/cgo_harness/bitbake_retirement_parity_test.go b/cgo_harness/bitbake_retirement_parity_test.go
new file mode 100644
index 000000000..f408e24e3
--- /dev/null
+++ b/cgo_harness/bitbake_retirement_parity_test.go
@@ -0,0 +1,205 @@
+//go:build cgo && treesitter_c_parity
+
+package cgoharness
+
+import (
+ "crypto/sha256"
+ "fmt"
+ "os"
+ "path/filepath"
+ "testing"
+
+ gotreesitter "github.com/odvcencio/gotreesitter"
+ "github.com/odvcencio/gotreesitter/internal/benchfixtures"
+ sitter "github.com/tree-sitter/go-tree-sitter"
+)
+
+const bitbakeAddtaskTriggerSource = `SUMMARY = "Test recipe for fetching git submodules"
+HOMEPAGE = "http://git.yoctoproject.org/cgit/cgit.cgi/git-submodule-test/"
+LICENSE = "MIT"
+LIC_FILES_CHKSUM = "file://${COMMON_LICENSE_DIR}/MIT;md5=0835ade698e0bcf8506ecda2f7b4f302"
+
+INHIBIT_DEFAULT_DEPS = "1"
+
+# Note: this is intentionally not the latest version in the original .bb
+SRCREV = "f280847494763cdcf71197557a81ba7d8a6bce42"
+PV = "0.1+git"
+PR = "r2"
+
+SRC_URI = "gitsm://git.yoctoproject.org/git-submodule-test;branch=master;protocol=https"
+UPSTREAM_CHECK_COMMITS = "1"
+RECIPE_NO_UPDATE_REASON = "This recipe is used to test devtool upgrade feature"
+
+EXCLUDE_FROM_WORLD = "1"
+
+do_test_git_as_user() {
+ cd ${S}
+ git status
+ git submodule status
+}
+addtask test_git_as_user after do_unpack
+
+fakeroot do_test_git_as_root() {
+ cd ${S}
+ git status
+ git submodule status
+}
+do_test_git_as_root[depends] += "virtual/fakeroot-native:do_populate_sysroot"
+addtask test_git_as_root after do_unpack`
+
+const bitbakeFunctionFlagTriggerSource = `do_run_tests () {
+ meson test -C "${B}" --no-rebuild
+}
+do_run_tests[doc] = "Run meson test using qemu-user"
+addtask do_run_tests after do_compile`
+
+const bitbakeAdjacentOverrideTriggerSource = `do_install:append() {
+ ln -sf am335x-bonegreen-ext.dtb "${D}/boot/devicetree/am335x-bonegreen-ext-alias.dtb"
+}
+
+do_deploy:append() {
+ ln -sf am335x-bonegreen-ext.dtb "${DEPLOYDIR}/devicetree/am335x-bonegreen-ext-alias.dtb"
+}`
+
+// TestBitbakeNormalizationCensusLockedCExact compares all A0 BitBake
+// fixtures and the existing parser trigger sources against the pinned C grammar.
+func TestBitbakeNormalizationCensusLockedCExact(t *testing.T) {
+ entry, ok := parityEntriesByName["bitbake"]
+ if !ok {
+ t.Fatal("missing BitBake grammar entry")
+ }
+ language := entry.Language()
+ cLanguage, err := COracleLanguage("bitbake")
+ if err != nil {
+ t.Fatal(err)
+ }
+ identity, err := COracleIdentity("bitbake")
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Logf("C oracle contract=%s binding=%s runtime=%s grammar_commit=%s artifact_sha256=%s", identity.Contract, identity.BindingVersion, identity.RuntimeVersion, identity.GrammarCommit, identity.GrammarArtifactSHA256)
+
+ tests := []struct {
+ name string
+ file string
+ sha256 string
+ source []byte
+ }{
+ {
+ name: "a0-small-error",
+ file: "small__error.bb",
+ sha256: "fbdb85e443edd378e944e5a1416c0c4a1e485f0cd38f70a5ac75748073a15d12",
+ },
+ {
+ name: "a0-medium-clang-git",
+ file: "medium__clang_git.bb",
+ sha256: "7deb41efd839d8b5b8b2c98589614377d12fd81fa6033824330084e07c5eaf9f",
+ },
+ {
+ name: "a0-large-linux-firmware",
+ file: "large__linux-firmware_20260519.bb",
+ sha256: "eaa9e3f2354345d558717c4791a67144d8b27767674bf5468e157a0e0a332ff6",
+ },
+ {
+ name: "trigger-addtask-error-wrapper",
+ sha256: "35ccf9d007ef76548258088b18adb39d7c0509452b4fb62bd7dfdc94cdcdf780",
+ source: []byte(bitbakeAddtaskTriggerSource),
+ },
+ {
+ name: "trigger-function-flag-assignment",
+ sha256: "8655832f5acd3b4ada197881f41b0110657572aa0c791ecf4524fbc10603a1eb",
+ source: []byte(bitbakeFunctionFlagTriggerSource),
+ },
+ {
+ name: "trigger-adjacent-override-functions",
+ sha256: "068037e13b101cf01d0a36da586f798972fa39b99ba0c2a86dc17996bc34d185",
+ source: []byte(bitbakeAdjacentOverrideTriggerSource),
+ },
+ }
+
+ for _, test := range tests {
+ test := test
+ if !t.Run(test.name, func(t *testing.T) {
+ source := test.source
+ if source == nil {
+ var err error
+ source, err = os.ReadFile(filepath.Join(
+ "..", "testdata", "dispatcher_census_a0", "bitbake", test.file,
+ ))
+ if err != nil {
+ t.Fatal(err)
+ }
+ }
+ if got := fmt.Sprintf("%x", sha256.Sum256(source)); got != test.sha256 {
+ t.Fatalf("source SHA-256 = %s, want %s", got, test.sha256)
+ }
+
+ cParser := sitter.NewParser()
+ t.Cleanup(cParser.Close)
+ if err := cParser.SetLanguage(cLanguage); err != nil {
+ t.Fatal(err)
+ }
+ cTree := cParser.Parse(source, nil)
+ if cTree == nil || cTree.RootNode() == nil {
+ t.Fatal("locked C parser returned a nil tree")
+ }
+ t.Cleanup(cTree.Close)
+ cDigest, err := COracleDeepDigest(cTree)
+ if err != nil {
+ t.Fatalf("inspect locked C deep tree: %v", err)
+ }
+
+ rawParser := gotreesitter.NewParser(language)
+ rawParser.SetAdmissionCandidateRoute(false)
+ rawTree, err := rawParser.ParseNoResultCompatibilityBenchmarkOnly(source)
+ if err != nil {
+ t.Fatalf("raw parse: %v", err)
+ }
+ t.Cleanup(rawTree.Release)
+ rawRuntime := rawTree.ParseRuntime()
+ rawDigest := assertBitbakeLockedCTreeExact(t, "raw", rawTree, language, cTree, cDigest)
+
+ productionParser := gotreesitter.NewParser(language)
+ productionParser.SetAdmissionCandidateRoute(false)
+ productionTree, err := productionParser.Parse(source)
+ if err != nil {
+ t.Fatalf("production parse: %v", err)
+ }
+ t.Cleanup(productionTree.Release)
+ productionRuntime := productionTree.ParseRuntime()
+ productionDigest := assertBitbakeLockedCTreeExact(t, "production", productionTree, language, cTree, cDigest)
+
+ t.Logf("witness=%s bytes=%d source_sha256=%s c_digest=%s raw_digest=%s production_digest=%s raw_rewrites=%d production_rewrites=%d", test.name, len(source), test.sha256, cDigest, rawDigest, productionDigest, rawRuntime.NormalizationNodesRewritten, productionRuntime.NormalizationNodesRewritten)
+ }) {
+ return
+ }
+ }
+}
+
+func assertBitbakeLockedCTreeExact(
+ t *testing.T,
+ label string,
+ goTree *gotreesitter.Tree,
+ goLang *gotreesitter.Language,
+ cTree *sitter.Tree,
+ wantDigest string,
+) string {
+ t.Helper()
+ goRoot := goTree.RootNode()
+ cRoot := cTree.RootNode()
+ if diff := FirstDivergenceDumpV1(goRoot, goLang, cRoot); diff != nil {
+ t.Fatalf("%s tree diverges from the locked C oracle: %+v", label, diff)
+ }
+ if diff := firstLockedCTreeFlagDivergence(goRoot, goLang, cRoot, "/"+goRoot.Type(goLang)); diff != nil {
+ t.Fatalf("%s tree has a missing or error flag divergence: %v", label, diff)
+ }
+ inspection, err := benchfixtures.InspectGoTree(goRoot, goLang)
+ if err != nil {
+ t.Fatalf("inspect %s Go deep tree: %v", label, err)
+ }
+ if inspection.SHA256 != wantDigest {
+ t.Fatalf("%s deep digest Go=%s C=%s", label, inspection.SHA256, wantDigest)
+ }
+ t.Logf("%s route matches locked C exactly: symbols, fields, spans, points, child order, named/extra/missing/error flags, deep digest=%s", label, inspection.SHA256)
+ return inspection.SHA256
+}
diff --git a/cgo_harness/c26b_swift_assoc_minimal.swift b/cgo_harness/c26b_swift_assoc_minimal.swift
new file mode 100644
index 000000000..66f415e49
--- /dev/null
+++ b/cgo_harness/c26b_swift_assoc_minimal.swift
@@ -0,0 +1,3 @@
+protocol P {
+ associatedtype Stride: SignedNumeric, Comparable
+}
diff --git a/cgo_harness/c26b_swift_associatedtype_probe_test.go b/cgo_harness/c26b_swift_associatedtype_probe_test.go
new file mode 100644
index 000000000..84643cb90
--- /dev/null
+++ b/cgo_harness/c26b_swift_associatedtype_probe_test.go
@@ -0,0 +1,180 @@
+//go:build cgo && treesitter_c_parity
+
+package cgoharness
+
+import (
+ "crypto/sha256"
+ "fmt"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ gotreesitter "github.com/odvcencio/gotreesitter"
+ "github.com/odvcencio/gotreesitter/grammars"
+ "github.com/odvcencio/gotreesitter/internal/benchfixtures"
+ sitter "github.com/tree-sitter/go-tree-sitter"
+)
+
+func TestC26BAssociatedtypeConformanceWitness(t *testing.T) {
+ cases := []struct {
+ name string
+ src []byte
+ }{
+ {
+ name: "minimal",
+ src: []byte("protocol P {\n associatedtype Stride: SignedNumeric, Comparable\n}\n"),
+ },
+ {
+ name: "corpus",
+ src: func() []byte {
+ src, err := os.ReadFile(filepath.Join("..", "grammars", "testdata", "swift_corpus", "stdlib_Stride.swift"))
+ if err != nil {
+ t.Fatalf("read Swift corpus witness: %v", err)
+ }
+ return src
+ }(),
+ },
+ }
+
+ goLang := grammars.SwiftLanguage()
+ cLang, err := ParityCLanguage("swift")
+ if err != nil {
+ t.Fatalf("load locked Swift C parser: %v", err)
+ }
+ for _, test := range cases {
+ t.Run(test.name, func(t *testing.T) {
+ goParser := gotreesitter.NewParser(goLang)
+ goTree, err := goParser.Parse(test.src)
+ if err != nil {
+ t.Fatalf("parse Swift with Go: %v", err)
+ }
+ defer goTree.Release()
+
+ cParser := sitter.NewParser()
+ defer cParser.Close()
+ if err := cParser.SetLanguage(cLang); err != nil {
+ t.Fatalf("set locked Swift language: %v", err)
+ }
+ cTree := cParser.Parse(test.src, nil)
+ if cTree == nil || cTree.RootNode() == nil {
+ t.Fatal("locked C parser returned no tree")
+ }
+ defer cTree.Close()
+
+ goInspection, err := benchfixtures.InspectGoTree(goTree.RootNode(), goLang)
+ if err != nil {
+ t.Fatalf("inspect Go tree: %v", err)
+ }
+ cDigest, err := COracleDeepDigest(cTree)
+ if err != nil {
+ t.Fatalf("inspect locked C tree: %v", err)
+ }
+ fmt.Printf("C26B_ASSOC name=%s bytes=%d source_sha256=%x go_sha256=%s c_sha256=%s go_root=%d:%d go_error=%t c_root=%d:%d c_error=%t go_error_node=%s c_error_node=%s first_diff=%+v\n",
+ test.name,
+ len(test.src),
+ sha256.Sum256(test.src),
+ goInspection.SHA256,
+ cDigest,
+ goTree.RootNode().StartByte(),
+ goTree.RootNode().EndByte(),
+ goTree.RootNode().HasError(),
+ cTree.RootNode().StartByte(),
+ cTree.RootNode().EndByte(),
+ cTree.RootNode().HasError(),
+ c26bFirstGoError(goTree.RootNode(), goLang, "/source_file"),
+ c26bFirstCError(cTree.RootNode(), "/source_file"),
+ FirstDivergenceDumpV1(goTree.RootNode(), goLang, cTree.RootNode()),
+ )
+ if target := strings.Index(string(test.src), "associatedtype Stride: SignedNumeric, Comparable"); target >= 0 {
+ fmt.Printf("C26B_TARGET name=%s target=%d go_context=%s c_context=%s\n",
+ test.name,
+ target,
+ strings.Join(c26bGoErrorContext(goTree.RootNode(), goLang, uint32(target), "/source_file"), " -> "),
+ strings.Join(c26bCErrorContext(cTree.RootNode(), uint32(target), "/source_file"), " -> "),
+ )
+ }
+ if FirstDivergenceDumpV1(goTree.RootNode(), goLang, cTree.RootNode()) == nil {
+ t.Fatal("associatedtype witness unexpectedly matches locked C")
+ }
+ })
+ }
+}
+
+func c26bGoErrorContext(node *gotreesitter.Node, lang *gotreesitter.Language, target uint32, path string) []string {
+ if node == nil || target < node.StartByte() || target > node.EndByte() {
+ return nil
+ }
+ current := []string(nil)
+ if node.Type(lang) == "ERROR" {
+ current = append(current, fmt.Sprintf("%s[%d:%d] children=%d", path, node.StartByte(), node.EndByte(), node.ChildCount()))
+ }
+ for i := 0; i < node.ChildCount(); i++ {
+ child := node.Child(i)
+ if child == nil {
+ continue
+ }
+ if nested := c26bGoErrorContext(child, lang, target, fmt.Sprintf("%s/%s[%d]", path, child.Type(lang), i)); nested != nil {
+ current = append(current, nested...)
+ }
+ }
+ return current
+}
+
+func c26bCErrorContext(node *sitter.Node, target uint32, path string) []string {
+ if node == nil || target < uint32(node.StartByte()) || target > uint32(node.EndByte()) {
+ return nil
+ }
+ current := []string(nil)
+ if node.Kind() == "ERROR" {
+ current = append(current, fmt.Sprintf("%s[%d:%d] children=%d", path, node.StartByte(), node.EndByte(), node.ChildCount()))
+ }
+ for i := 0; i < int(node.ChildCount()); i++ {
+ child := node.Child(uint(i))
+ if child == nil {
+ continue
+ }
+ if nested := c26bCErrorContext(child, target, fmt.Sprintf("%s/%s[%d]", path, child.Kind(), i)); nested != nil {
+ current = append(current, nested...)
+ }
+ }
+ return current
+}
+
+func c26bFirstGoError(node *gotreesitter.Node, lang *gotreesitter.Language, path string) string {
+ if node == nil {
+ return ""
+ }
+ if node.Type(lang) == "ERROR" {
+ return fmt.Sprintf("%s[%d:%d] children=%d", path, node.StartByte(), node.EndByte(), node.ChildCount())
+ }
+ for i := 0; i < node.ChildCount(); i++ {
+ child := node.Child(i)
+ if child == nil {
+ continue
+ }
+ if got := c26bFirstGoError(child, lang, fmt.Sprintf("%s/%s[%d]", path, child.Type(lang), i)); got != "" {
+ return got
+ }
+ }
+ return ""
+}
+
+func c26bFirstCError(node *sitter.Node, path string) string {
+ if node == nil {
+ return ""
+ }
+ if node.Kind() == "ERROR" {
+ return fmt.Sprintf("%s[%d:%d] children=%d", path, node.StartByte(), node.EndByte(), node.ChildCount())
+ }
+ for i := 0; i < int(node.ChildCount()); i++ {
+ child := node.Child(uint(i))
+ if child == nil {
+ continue
+ }
+ if got := c26bFirstCError(child, fmt.Sprintf("%s/%s[%d]", path, child.Kind(), i)); got != "" {
+ return got
+ }
+ }
+ return ""
+}
diff --git a/cgo_harness/c26c_swift_mask_minimal.swift b/cgo_harness/c26c_swift_mask_minimal.swift
new file mode 100644
index 000000000..82de7eb86
--- /dev/null
+++ b/cgo_harness/c26c_swift_mask_minimal.swift
@@ -0,0 +1 @@
+let x = 1&<<7
diff --git a/cgo_harness/c26c_swift_masking_shift_probe_test.go b/cgo_harness/c26c_swift_masking_shift_probe_test.go
new file mode 100644
index 000000000..d4843223a
--- /dev/null
+++ b/cgo_harness/c26c_swift_masking_shift_probe_test.go
@@ -0,0 +1,116 @@
+//go:build cgo && treesitter_c_parity
+
+package cgoharness
+
+import (
+ "crypto/sha256"
+ "fmt"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ gotreesitter "github.com/odvcencio/gotreesitter"
+ "github.com/odvcencio/gotreesitter/internal/benchfixtures"
+ sitter "github.com/tree-sitter/go-tree-sitter"
+)
+
+func TestC26CMaskingShiftWitness(t *testing.T) {
+ tests := []struct {
+ name string
+ src []byte
+ }{
+ {
+ name: "minimal",
+ src: []byte("let x = 1&<<7\n"),
+ },
+ {
+ name: "corpus",
+ src: func() []byte {
+ path := filepath.Join("..", "grammars", "testdata", "swift_corpus", "stdlib_ASCII.swift")
+ src, err := os.ReadFile(path)
+ if err != nil {
+ panic(err)
+ }
+ return src
+ }(),
+ },
+ }
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ source := test.src
+ sourceDigest := sha256.Sum256(source)
+ goTree, goLang, err := parseWithGo(parityCase{name: "swift", source: string(source)}, source, nil)
+ if err != nil {
+ t.Fatalf("parse Swift with Go: %v", err)
+ }
+ defer releaseGoTree(goTree)
+
+ cLang, err := ParityCLanguage("swift")
+ if err != nil {
+ t.Fatalf("load locked Swift C parser: %v", err)
+ }
+ cParser := sitter.NewParser()
+ defer cParser.Close()
+ if err := cParser.SetLanguage(cLang); err != nil {
+ t.Fatalf("set locked Swift C parser language: %v", err)
+ }
+ cTree := cParser.Parse(source, nil)
+ if cTree == nil || cTree.RootNode() == nil {
+ t.Fatal("locked C parser returned no tree")
+ }
+ defer cTree.Close()
+
+ goRoot := goTree.RootNode()
+ cRoot := cTree.RootNode()
+ goInspection, err := benchfixtures.InspectGoTree(goRoot, goLang)
+ if err != nil {
+ t.Fatalf("inspect Go tree: %v", err)
+ }
+ cDigest, err := COracleDeepDigest(cTree)
+ if err != nil {
+ t.Fatalf("inspect C tree: %v", err)
+ }
+ diff := FirstDivergenceDumpV1(goRoot, goLang, cRoot)
+ t.Logf("C26C_MASK name=%s bytes=%d source_sha256=%x go_sha256=%s c_sha256=%s go_root=%d:%d go_children=%d go_error=%v c_root=%d:%d c_children=%d c_error=%v first_diff=%+v target=%d", test.name, len(source), sourceDigest, goInspection.SHA256, cDigest, goRoot.StartByte(), goRoot.EndByte(), goRoot.ChildCount(), goRoot.HasError(), cRoot.StartByte(), cRoot.EndByte(), int(cRoot.ChildCount()), cRoot.HasError(), diff, strings.Index(string(source), "1&<<7"))
+ c26cLogGoRootChildren(t, "go", goRoot, goLang, source)
+ c26cLogCRootChildren(t, "c", cRoot, source)
+ })
+ }
+}
+
+func c26cLogGoRootChildren(t *testing.T, label string, root *gotreesitter.Node, lang *gotreesitter.Language, source []byte) {
+ t.Helper()
+ limit := root.ChildCount()
+ if limit > 12 {
+ limit = 12
+ }
+ for i := 0; i < limit; i++ {
+ child := root.Child(i)
+ t.Logf("%s.root.child[%d] type=%q span=%d:%d named=%v extra=%v children=%d text=%s", label, i, child.Type(lang), child.StartByte(), child.EndByte(), child.IsNamed(), child.IsExtra(), child.ChildCount(), c26cText(source, child.StartByte(), child.EndByte()))
+ }
+}
+
+func c26cLogCRootChildren(t *testing.T, label string, root *sitter.Node, source []byte) {
+ t.Helper()
+ limit := int(root.ChildCount())
+ if limit > 12 {
+ limit = 12
+ }
+ for i := 0; i < limit; i++ {
+ child := root.Child(uint(i))
+ t.Logf("%s.root.child[%d] kind=%q span=%d:%d named=%v extra=%v children=%d text=%q", label, i, child.Kind(), child.StartByte(), child.EndByte(), child.IsNamed(), child.IsExtra(), int(child.ChildCount()), c26cText(source, uint32(child.StartByte()), uint32(child.EndByte())))
+ }
+}
+
+func c26cText(source []byte, start, end uint32) string {
+ if start > end || end > uint32(len(source)) {
+ return ""
+ }
+ text := string(source[start:end])
+ if len(text) > 100 {
+ text = text[:100]
+ }
+ return fmt.Sprintf("%q", text)
+}
diff --git a/cgo_harness/corn_retirement_parity_test.go b/cgo_harness/corn_retirement_parity_test.go
new file mode 100644
index 000000000..37748a349
--- /dev/null
+++ b/cgo_harness/corn_retirement_parity_test.go
@@ -0,0 +1,148 @@
+//go:build cgo && treesitter_c_parity
+
+package cgoharness
+
+import (
+ "crypto/sha256"
+ "fmt"
+ "os"
+ "path/filepath"
+ "testing"
+
+ gotreesitter "github.com/odvcencio/gotreesitter"
+ "github.com/odvcencio/gotreesitter/internal/benchfixtures"
+ sitter "github.com/tree-sitter/go-tree-sitter"
+)
+
+const cornQuotedKeysSource = "{\n 'foo.bar' = 42\n 'green.eggs'.and.ham = \"hello world\"\n 'with spaces' = true\n 'escaped\\'quote' = false\n 'escaped=equals' = -3\n}"
+
+// TestCornNormalizationCensusLockedCExact compares all A0 Corn witnesses and
+// the existing quoted-keys trigger against the pinned C grammar.
+func TestCornNormalizationCensusLockedCExact(t *testing.T) {
+ entry, ok := parityEntriesByName["corn"]
+ if !ok {
+ t.Fatal("missing Corn grammar entry")
+ }
+ language := entry.Language()
+ cLanguage, err := COracleLanguage("corn")
+ if err != nil {
+ t.Fatal(err)
+ }
+ identity, err := COracleIdentity("corn")
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Logf("C oracle contract=%s binding=%s runtime=%s grammar_commit=%s artifact_sha256=%s", identity.Contract, identity.BindingVersion, identity.RuntimeVersion, identity.GrammarCommit, identity.GrammarArtifactSHA256)
+
+ tests := []struct {
+ name string
+ file string
+ sha256 string
+ source []byte
+ }{
+ {
+ name: "a0-compact",
+ file: "small__compact.corn",
+ sha256: "e9793277f21b19593024cf7f670934333deefb54eae122caaa1f66cf41c7606a",
+ },
+ {
+ name: "a0-complex",
+ file: "small__complex.corn",
+ sha256: "98aaba0d478418a7855fa538cb9cd52ab81b2d2e581402d212bc3951a6a5db02",
+ },
+ {
+ name: "a0-readme-example",
+ file: "small__readme_example.corn",
+ sha256: "7d412d6e3c5e396818885601df14ad092b8cbe4aac690a45d7a4c29e0410da94",
+ },
+ {
+ name: "quoted-keys-trigger",
+ sha256: "1e541539a857d075420cd73d64381b4f452d98a44884356a09b3e3ca7faf0b68",
+ source: []byte(cornQuotedKeysSource),
+ },
+ }
+
+ for _, test := range tests {
+ test := test
+ t.Run(test.name, func(t *testing.T) {
+ source := test.source
+ if source == nil {
+ var err error
+ source, err = os.ReadFile(filepath.Join(
+ "..", "testdata", "dispatcher_census_a0", "corn", test.file,
+ ))
+ if err != nil {
+ t.Fatal(err)
+ }
+ }
+ if got := fmt.Sprintf("%x", sha256.Sum256(source)); got != test.sha256 {
+ t.Fatalf("source SHA-256 = %s, want %s", got, test.sha256)
+ }
+
+ cParser := sitter.NewParser()
+ t.Cleanup(cParser.Close)
+ if err := cParser.SetLanguage(cLanguage); err != nil {
+ t.Fatal(err)
+ }
+ cTree := cParser.Parse(source, nil)
+ if cTree == nil || cTree.RootNode() == nil {
+ t.Fatal("locked C parser returned a nil tree")
+ }
+ t.Cleanup(cTree.Close)
+ cDigest, err := COracleDeepDigest(cTree)
+ if err != nil {
+ t.Fatalf("inspect locked C deep tree: %v", err)
+ }
+
+ rawParser := gotreesitter.NewParser(language)
+ rawParser.SetAdmissionCandidateRoute(false)
+ rawTree, err := rawParser.ParseNoResultCompatibilityBenchmarkOnly(source)
+ if err != nil {
+ t.Fatalf("raw parse: %v", err)
+ }
+ t.Cleanup(rawTree.Release)
+ rawRuntime := rawTree.ParseRuntime()
+ rawDigest := assertCornLockedCTreeExact(t, "raw", rawTree, language, cTree, cDigest)
+
+ productionParser := gotreesitter.NewParser(language)
+ productionParser.SetAdmissionCandidateRoute(false)
+ productionTree, err := productionParser.Parse(source)
+ if err != nil {
+ t.Fatalf("production parse: %v", err)
+ }
+ t.Cleanup(productionTree.Release)
+ productionRuntime := productionTree.ParseRuntime()
+ productionDigest := assertCornLockedCTreeExact(t, "production", productionTree, language, cTree, cDigest)
+
+ t.Logf("witness=%s bytes=%d source_sha256=%s c_digest=%s raw_digest=%s production_digest=%s raw_rewrites=%d production_rewrites=%d", test.name, len(source), test.sha256, cDigest, rawDigest, productionDigest, rawRuntime.NormalizationNodesRewritten, productionRuntime.NormalizationNodesRewritten)
+ })
+ }
+}
+
+func assertCornLockedCTreeExact(
+ t *testing.T,
+ label string,
+ goTree *gotreesitter.Tree,
+ goLang *gotreesitter.Language,
+ cTree *sitter.Tree,
+ wantDigest string,
+) string {
+ t.Helper()
+ goRoot := goTree.RootNode()
+ cRoot := cTree.RootNode()
+ if diff := FirstDivergenceDumpV1(goRoot, goLang, cRoot); diff != nil {
+ t.Fatalf("%s tree diverges from the locked C oracle: %+v", label, diff)
+ }
+ if diff := firstLockedCTreeFlagDivergence(goRoot, goLang, cRoot, "/"+goRoot.Type(goLang)); diff != nil {
+ t.Fatalf("%s tree has a missing or error flag divergence: %v", label, diff)
+ }
+ inspection, err := benchfixtures.InspectGoTree(goRoot, goLang)
+ if err != nil {
+ t.Fatalf("inspect %s Go deep tree: %v", label, err)
+ }
+ if inspection.SHA256 != wantDigest {
+ t.Fatalf("%s deep digest Go=%s C=%s", label, inspection.SHA256, wantDigest)
+ }
+ t.Logf("%s route matches locked C exactly: symbols, fields, spans, points, extras, missing/error flags, deep digest=%s", label, inspection.SHA256)
+ return inspection.SHA256
+}
diff --git a/cgo_harness/issue454_variant_cguard_fivesize_test.go b/cgo_harness/issue454_variant_cguard_fivesize_test.go
new file mode 100644
index 000000000..2e9a0c246
--- /dev/null
+++ b/cgo_harness/issue454_variant_cguard_fivesize_test.go
@@ -0,0 +1,60 @@
+//go:build cgo && treesitter_c_parity
+
+package cgoharness
+
+import (
+ "bytes"
+ "strconv"
+ "testing"
+
+ "github.com/odvcencio/gotreesitter/internal/benchfixtures"
+ sitter "github.com/tree-sitter/go-tree-sitter"
+)
+
+func TestIssue454VariantCGuardFiveSizes(t *testing.T) {
+ sizes := []int{1024, 4096, 16384, 65536, benchfixtures.Issue454CFixtureBytes}
+ for _, size := range sizes {
+ size := size
+ t.Run(sizeLabel(size), func(t *testing.T) {
+ source := append([]byte(nil), benchfixtures.Issue454CSource()[:size]...)
+ site := bytes.Index(source, []byte("x0"))
+ if site < 0 {
+ t.Fatal("C edit marker is absent")
+ }
+ edited := append(append([]byte(nil), source[:site]...), source[site+1:]...)
+
+ goTree, goLang, err := parseWithGo(parityCase{name: "c"}, edited, nil)
+ if err != nil {
+ t.Fatalf("parse edited C witness with Go: %v", err)
+ }
+ t.Cleanup(goTree.Release)
+
+ cLang, err := ParityCLanguage("c")
+ if err != nil {
+ t.Fatalf("load locked C grammar: %v", err)
+ }
+ cParser := sitter.NewParser()
+ t.Cleanup(cParser.Close)
+ if err := cParser.SetLanguage(cLang); err != nil {
+ t.Fatalf("set locked C grammar: %v", err)
+ }
+ cTree := cParser.Parse(edited, nil)
+ if cTree == nil || cTree.RootNode() == nil {
+ t.Fatal("locked C parser returned no tree")
+ }
+ t.Cleanup(cTree.Close)
+
+ if diff := FirstDivergenceDumpV1(goTree.RootNode(), goLang, cTree.RootNode()); diff != nil {
+ t.Fatalf("size=%d locked-C parity diverged: %+v", size, *diff)
+ }
+ t.Logf("size=%d locked-C parity passed", size)
+ })
+ }
+}
+
+func sizeLabel(size int) string {
+ if size == benchfixtures.Issue454CFixtureBytes {
+ return "137KiB"
+ }
+ return strconv.Itoa(size) + "B"
+}
diff --git a/cgo_harness/issue454_variant_locked_c_diag_test.go b/cgo_harness/issue454_variant_locked_c_diag_test.go
new file mode 100644
index 000000000..abfd6e60e
--- /dev/null
+++ b/cgo_harness/issue454_variant_locked_c_diag_test.go
@@ -0,0 +1,94 @@
+//go:build cgo && treesitter_c_parity
+
+package cgoharness
+
+import (
+ "crypto/sha256"
+ "fmt"
+ "os"
+ "testing"
+
+ sitter "github.com/tree-sitter/go-tree-sitter"
+
+ gotreesitter "github.com/odvcencio/gotreesitter"
+ "github.com/odvcencio/gotreesitter/grammars"
+ "github.com/odvcencio/gotreesitter/internal/benchfixtures"
+)
+
+func TestIssue454VariantLockedCDiag(t *testing.T) {
+ cases := []struct {
+ name string
+ lang string
+ load func() *gotreesitter.Language
+ path string
+ source string
+ }{
+ {name: "cobol_a0_small", lang: "cobol", load: grammars.CobolLanguage, path: "../testdata/dispatcher_census_a0/cobol/small__CVERSNP1.cpy"},
+ {name: "cobol_a0_medium", lang: "cobol", load: grammars.CobolLanguage, path: "../testdata/dispatcher_census_a0/cobol/medium__DBANK02P.cbl"},
+ {name: "cobol_a0_large", lang: "cobol", load: grammars.CobolLanguage, path: "../testdata/dispatcher_census_a0/cobol/large__MBANK30.cpy"},
+ {name: "wgsl_a0_small", lang: "wgsl", load: grammars.WgslLanguage, path: "../testdata/dispatcher_census_a0/wgsl/small__fragmentTextureQuad.wgsl"},
+ {name: "wgsl_a0_medium_normal", lang: "wgsl", load: grammars.WgslLanguage, path: "../testdata/dispatcher_census_a0/wgsl/medium__normalMap.wgsl"},
+ {name: "wgsl_a0_medium_radiosity", lang: "wgsl", load: grammars.WgslLanguage, path: "../testdata/dispatcher_census_a0/wgsl/medium__radiosity.wgsl"},
+ {name: "cooklang_punctuation", lang: "cooklang", load: grammars.CooklangLanguage, source: "Add @salt{1%tsp}.\n"},
+ {name: "cooklang_recovered", lang: "cooklang", load: grammars.CooklangLanguage, source: "---\nservings: 4\nemoji: 🥟\ntags: warm, fried, starter\n---\n\nServe hot.\n"},
+ {name: "cooklang_no_newline", lang: "cooklang", load: grammars.CooklangLanguage, source: "Add @salt{1%tsp}."},
+ }
+ for _, tc := range cases {
+ tc := tc
+ t.Run(tc.name, func(t *testing.T) {
+ var source []byte
+ if tc.path != "" {
+ var err error
+ source, err = os.ReadFile(tc.path)
+ if err != nil {
+ t.Fatalf("read source: %v", err)
+ }
+ } else {
+ source = []byte(tc.source)
+ }
+ cLang, err := ParityCLanguage(tc.lang)
+ if err != nil {
+ t.Fatalf("load locked C language: %v", err)
+ }
+ cParser := sitter.NewParser()
+ defer cParser.Close()
+ if err := cParser.SetLanguage(cLang); err != nil {
+ t.Fatalf("set C language: %v", err)
+ }
+ goLang := tc.load()
+ goParser := gotreesitter.NewParser(goLang)
+ goParser.SetAdmissionCandidateRoute(false)
+ goTree, err := goParser.ParseNoResultCompatibilityBenchmarkOnly(source)
+ if err != nil {
+ t.Fatalf("parse Go tree: %v", err)
+ }
+ defer goTree.Release()
+ cTree := cParser.Parse(source, nil)
+ if cTree == nil || cTree.RootNode() == nil {
+ t.Fatal("C parse returned no tree")
+ }
+ defer cTree.Close()
+ goRoot := goTree.RootNode()
+ cRoot := cTree.RootNode()
+ var diffs []compactT3StructuralDivergence
+ compactT3WalkStructuralDivergences(goRoot, goLang, cRoot, "root", &diffs)
+ goDigest := sha256.Sum256([]byte(goRoot.SExpr(goLang)))
+ cDigest := sha256.Sum256([]byte(dumpCTree(cRoot, 0)))
+ goInspection, err := benchfixtures.InspectGoTree(goRoot, goLang)
+ if err != nil {
+ t.Fatalf("inspect Go tree: %v", err)
+ }
+ cDeepDigest, err := COracleDeepDigest(cTree)
+ if err != nil {
+ t.Fatalf("inspect C tree: %v", err)
+ }
+ t.Logf("raw_error=%v c_error=%v raw_sexpr_sha256=%x c_tree_sha256=%x go_deep_digest=%s c_deep_digest=%s structural_diffs=%d", goRoot.HasError(), cRoot.HasError(), goDigest, cDigest, goInspection.SHA256, cDeepDigest, len(diffs))
+ for i, diff := range diffs {
+ if i == 8 {
+ break
+ }
+ t.Logf("diff[%d]=%s", i, fmt.Sprint(diff))
+ }
+ })
+ }
+}
diff --git a/cgo_harness/rust_dispatch_locked_c_probe_test.go b/cgo_harness/rust_dispatch_locked_c_probe_test.go
new file mode 100644
index 000000000..a310c5807
--- /dev/null
+++ b/cgo_harness/rust_dispatch_locked_c_probe_test.go
@@ -0,0 +1,216 @@
+//go:build cgo && treesitter_c_parity
+
+package cgoharness
+
+import (
+ "crypto/sha256"
+ "encoding/hex"
+ "fmt"
+ "os"
+ "testing"
+
+ gotreesitter "github.com/odvcencio/gotreesitter"
+ "github.com/odvcencio/gotreesitter/grammars"
+ "github.com/odvcencio/gotreesitter/internal/benchfixtures"
+ sitter "github.com/tree-sitter/go-tree-sitter"
+)
+
+type rustDispatchLockedCWitness struct {
+ name string
+ src []byte
+}
+
+// TestRustDispatchRegisteredWitnessLockedCDeepParity checks every Rust
+// witness named by the ownership registry against the locked C oracle.
+// It also checks the production, compact, forest, and incremental routes.
+func TestRustDispatchRegisteredWitnessLockedCDeepParity(t *testing.T) {
+ t.Setenv("GTS_DISPATCHER_CENSUS", "1")
+ fixtures := []rustDispatchLockedCWitness{
+ {name: "ownership-registry-smoke", src: []byte(grammars.ParseSmokeSample("rust"))},
+ {name: "tracked-census-rust_ast.rs", src: rustDispatchReadFixture(t, "../testdata/incremental_gate/rust_ast.rs")},
+ }
+ goLanguage := grammars.RustLanguage()
+ cLanguage, err := ParityCLanguage("rust")
+ if err != nil {
+ t.Fatalf("load locked Rust C oracle: %v", err)
+ }
+
+ for _, fixture := range fixtures {
+ fixture := fixture
+ t.Run(fixture.name, func(t *testing.T) {
+ sourceHash := sha256.Sum256(fixture.src)
+ t.Logf("witness=%s bytes=%d source_sha256=%s", fixture.name, len(fixture.src), hex.EncodeToString(sourceHash[:]))
+
+ raw := rustDispatchParseRaw(t, goLanguage, fixture.src)
+ defer raw.Release()
+ production := rustDispatchParseProduction(t, goLanguage, fixture.src)
+ defer production.Release()
+ rawDigest := rustDispatchGoDigest(t, raw, goLanguage)
+ productionDigest := rustDispatchGoDigest(t, production, goLanguage)
+ if rawDigest != productionDigest {
+ t.Fatalf("raw/production deep digest mismatch: raw=%s production=%s", rawDigest, productionDigest)
+ }
+ cDigest := rustDispatchCDigest(t, cLanguage, fixture.src)
+ if rawDigest != cDigest {
+ t.Fatalf("raw/locked-C deep digest mismatch: go=%s c=%s", rawDigest, cDigest)
+ }
+ t.Logf("source route=raw production raw_digest=%s c_digest=%s raw_rewrites=%d production_rewrites=%d", rawDigest, cDigest, raw.ParseRuntime().NormalizationNodesRewritten, production.ParseRuntime().NormalizationNodesRewritten)
+ rLogRustDispatchLockedC(t, "production", production.ParseRuntime())
+
+ routeSource := append(append([]byte(nil), fixture.src...), '\n')
+ routeCDigest := rustDispatchCDigest(t, cLanguage, routeSource)
+ productionRoute := rustDispatchParseProduction(t, goLanguage, routeSource)
+ defer productionRoute.Release()
+ rustDispatchAssertGoDigest(t, "production-route", productionRoute, goLanguage, routeCDigest)
+ rLogRustDispatchLockedC(t, "production-route", productionRoute.ParseRuntime())
+
+ beforeCandidate, beforeFallback := gotreesitter.AdmissionCandidateCounters()
+ compactParser := gotreesitter.NewParser(goLanguage)
+ compactParser.SetAdmissionCandidateRoute(true)
+ compact, err := compactParser.Parse(routeSource)
+ if err != nil {
+ t.Fatalf("compact route: %v", err)
+ }
+ defer compact.Release()
+ rustDispatchAssertGoDigest(t, "compact", compact, goLanguage, routeCDigest)
+ afterCandidate, afterFallback := gotreesitter.AdmissionCandidateCounters()
+ t.Logf("route=compact candidate_routed_delta=%d candidate_fallback_delta=%d digest=%s", afterCandidate-beforeCandidate, afterFallback-beforeFallback, rustDispatchGoDigest(t, compact, goLanguage))
+ if afterCandidate <= beforeCandidate {
+ t.Fatalf("compact route did not enter the admission candidate")
+ }
+ if afterFallback != beforeFallback {
+ t.Fatalf("compact route fell back: before=%d after=%d reason=%q", beforeFallback, afterFallback, gotreesitter.AdmissionCandidateLastFallbackReason())
+ }
+
+ forestParser := gotreesitter.NewParser(goLanguage)
+ forest, ok := forestParser.ParseForestExperimental(routeSource)
+ if !ok || forest == nil {
+ offset, symbol, reason, states := forestParser.ForestDeclineInfo()
+ t.Fatalf("forest route declined: offset=%d symbol=%d reason=%q states=%v", offset, symbol, reason, states)
+ }
+ defer forest.Release()
+ rustDispatchAssertGoDigest(t, "forest", forest, goLanguage, routeCDigest)
+ t.Logf("route=forest outcome=accepted digest=%s forest_fast_path=%t", rustDispatchGoDigest(t, forest, goLanguage), forest.ParseRuntime().ForestFastPath)
+ rLogRustDispatchLockedC(t, "forest", forest.ParseRuntime())
+
+ oldTree := rustDispatchParseProduction(t, goLanguage, fixture.src)
+ defer oldTree.Release()
+ oldEnd := rustDispatchPointAtByte(fixture.src, len(fixture.src))
+ oldTree.Edit(gotreesitter.InputEdit{
+ StartByte: uint32(len(fixture.src)),
+ OldEndByte: uint32(len(fixture.src)),
+ NewEndByte: uint32(len(routeSource)),
+ StartPoint: oldEnd,
+ OldEndPoint: oldEnd,
+ NewEndPoint: rustDispatchPointAtByte(routeSource, len(routeSource)),
+ })
+ incrementalParser := gotreesitter.NewParser(goLanguage)
+ incremental, profile, err := incrementalParser.ParseIncrementalProfiled(routeSource, oldTree)
+ if err != nil {
+ t.Fatalf("incremental route: %v", err)
+ }
+ defer incremental.Release()
+ rustDispatchAssertGoDigest(t, "incremental", incremental, goLanguage, routeCDigest)
+ t.Logf("route=incremental digest=%s reused_subtrees=%d reused_bytes=%d reuse_unsupported=%t reason=%q", rustDispatchGoDigest(t, incremental, goLanguage), profile.ReusedSubtrees, profile.ReusedBytes, profile.ReuseUnsupported, profile.ReuseUnsupportedReason)
+ rLogRustDispatchLockedC(t, "incremental", incremental.ParseRuntime())
+ })
+ }
+}
+
+func rustDispatchReadFixture(t *testing.T, path string) []byte {
+ t.Helper()
+ source, err := os.ReadFile(path)
+ if err != nil {
+ t.Fatalf("read Rust fixture %s: %v", path, err)
+ }
+ return source
+}
+
+func rustDispatchParseRaw(t *testing.T, language *gotreesitter.Language, source []byte) *gotreesitter.Tree {
+ t.Helper()
+ parser := gotreesitter.NewParser(language)
+ tree, err := parser.ParseNoResultCompatibilityBenchmarkOnly(source)
+ if err != nil {
+ t.Fatalf("raw parse: %v", err)
+ }
+ return tree
+}
+
+func rustDispatchParseProduction(t *testing.T, language *gotreesitter.Language, source []byte) *gotreesitter.Tree {
+ t.Helper()
+ parser := gotreesitter.NewParser(language)
+ parser.SetAdmissionCandidateRoute(false)
+ tree, err := parser.Parse(source)
+ if err != nil {
+ t.Fatalf("production parse: %v", err)
+ }
+ return tree
+}
+
+func rustDispatchGoDigest(t *testing.T, tree *gotreesitter.Tree, language *gotreesitter.Language) string {
+ t.Helper()
+ inspection, err := benchfixtures.InspectGoTree(tree.RootNode(), language)
+ if err != nil {
+ t.Fatalf("inspect Go deep tree: %v", err)
+ }
+ return inspection.SHA256
+}
+
+func rustDispatchCDigest(t *testing.T, language *sitter.Language, source []byte) string {
+ t.Helper()
+ parser := sitter.NewParser()
+ defer parser.Close()
+ if err := parser.SetLanguage(language); err != nil {
+ t.Fatalf("set C oracle language: %v", err)
+ }
+ tree := parser.Parse(source, nil)
+ if tree == nil || tree.RootNode() == nil {
+ t.Fatal("C oracle returned a nil tree")
+ }
+ defer tree.Close()
+ digest, err := COracleDeepDigest(tree)
+ if err != nil {
+ t.Fatalf("inspect C deep tree: %v", err)
+ }
+ return digest
+}
+
+func rustDispatchAssertGoDigest(t *testing.T, route string, tree *gotreesitter.Tree, language *gotreesitter.Language, want string) {
+ t.Helper()
+ got := rustDispatchGoDigest(t, tree, language)
+ if got != want {
+ t.Fatalf("route=%s deep digest Go=%s, want locked C %s", route, got, want)
+ }
+ if tree.RootNode().HasError() {
+ t.Fatalf("route=%s produced an error root", route)
+ }
+}
+
+func rLogRustDispatchLockedC(t *testing.T, route string, runtime gotreesitter.ParseRuntime) {
+ t.Helper()
+ if runtime.NormalizationPasses == nil {
+ t.Logf("route=%s dispatch.rust=absent", route)
+ return
+ }
+ for _, pass := range *runtime.NormalizationPasses {
+ if pass.Name == "dispatch.rust" {
+ t.Logf("route=%s pass=%s checked=%d run=%d visited=%d rewritten=%d", route, pass.Name, pass.Checked, pass.Run, pass.NodesVisited, pass.NodesRewritten)
+ }
+ }
+}
+
+func rustDispatchPointAtByte(source []byte, offset int) gotreesitter.Point {
+ if offset < 0 || offset > len(source) {
+ panic(fmt.Sprintf("point offset %d outside source length %d", offset, len(source)))
+ }
+ var point gotreesitter.Point
+ for _, value := range source[:offset] {
+ if value == '\n' {
+ point.Row++
+ point.Column = 0
+ } else {
+ point.Column++
+ }
+ }
+ return point
+}
diff --git a/external_scanner_checkpoint_lifecycle.go b/external_scanner_checkpoint_lifecycle.go
new file mode 100644
index 000000000..0a7d06656
--- /dev/null
+++ b/external_scanner_checkpoint_lifecycle.go
@@ -0,0 +1,447 @@
+package gotreesitter
+
+// externalScannerCheckpointVersionKey identifies one parser version inside a
+// single checkpoint lifecycle. It is not a parser stack field.
+type externalScannerCheckpointVersionKey struct {
+ parseEpoch uint64
+ versionID uint64
+}
+
+// externalScannerCheckpointVersion keeps scanner ownership beside, rather
+// than inside, a parser stack. The lifecycle is opt-in and has no production
+// parser caller yet.
+type externalScannerCheckpointVersion struct {
+ key externalScannerCheckpointVersionKey
+ payload any
+ checkpoint externalScannerCheckpointRecord
+ alive bool
+}
+
+// externalScannerCheckpointLifecycle is a bounded ownership ledger for the
+// synthetic parser lifecycle proof. It does not wire scanner state into GLR.
+type externalScannerCheckpointLifecycle struct {
+ scanner ExternalScanner
+ epoch uint64
+ nextID uint64
+ versions map[externalScannerCheckpointVersionKey]*externalScannerCheckpointVersion
+}
+
+// newExternalScannerCheckpointLifecycle requires an explicit opt-in. A
+// missing or disabled capability returns no lifecycle and no ledger.
+func newExternalScannerCheckpointLifecycle(scanner ExternalScanner, enabled bool) (*externalScannerCheckpointLifecycle, bool) {
+ if !enabled || scanner == nil {
+ return nil, false
+ }
+ provider, ok := scanner.(ExternalScannerCheckpointIdentityProvider)
+ if !ok || !provider.UsesExternalScannerCheckpoints() {
+ return nil, false
+ }
+ return &externalScannerCheckpointLifecycle{
+ scanner: scanner,
+ epoch: 1,
+ nextID: 1,
+ versions: make(map[externalScannerCheckpointVersionKey]*externalScannerCheckpointVersion),
+ }, true
+}
+
+func (l *externalScannerCheckpointLifecycle) nextVersionKey() (externalScannerCheckpointVersionKey, bool) {
+ if l == nil || l.epoch == 0 || l.nextID == 0 {
+ return externalScannerCheckpointVersionKey{}, false
+ }
+ key := externalScannerCheckpointVersionKey{
+ parseEpoch: l.epoch,
+ versionID: l.nextID,
+ }
+ if _, exists := l.versions[key]; exists {
+ return externalScannerCheckpointVersionKey{}, false
+ }
+ if l.nextID == ^uint64(0) {
+ l.nextID = 0
+ } else {
+ l.nextID++
+ }
+ return key, true
+}
+
+func (l *externalScannerCheckpointLifecycle) close() {
+ if l == nil {
+ return
+ }
+ for _, version := range l.versions {
+ if version != nil && version.payload != nil {
+ l.scanner.Destroy(version.payload)
+ version.payload = nil
+ }
+ if version != nil {
+ version.alive = false
+ }
+ }
+ clear(l.versions)
+}
+
+func (l *externalScannerCheckpointLifecycle) version(key externalScannerCheckpointVersionKey) (*externalScannerCheckpointVersion, bool) {
+ if l == nil || l.versions == nil {
+ return nil, false
+ }
+ version, ok := l.versions[key]
+ if !ok || version == nil || !version.alive || !version.checkpoint.complete() {
+ return nil, false
+ }
+ return version, true
+}
+
+func (l *externalScannerCheckpointLifecycle) addRoot(
+ payload any,
+ sourceByte uint32,
+ sourcePoint Point,
+ externalLexState uint16,
+ tokenStartByte uint32,
+ tokenEndByte uint32,
+) (externalScannerCheckpointVersionKey, bool) {
+ if l == nil {
+ return externalScannerCheckpointVersionKey{}, false
+ }
+ record, ok := captureExternalScannerCheckpointRecord(
+ l.scanner,
+ payload,
+ sourceByte,
+ sourcePoint,
+ externalLexState,
+ tokenStartByte,
+ tokenEndByte,
+ )
+ if !ok {
+ return externalScannerCheckpointVersionKey{}, false
+ }
+ key, ok := l.nextVersionKey()
+ if !ok {
+ if payload != nil {
+ l.scanner.Destroy(payload)
+ }
+ return externalScannerCheckpointVersionKey{}, false
+ }
+ l.versions[key] = &externalScannerCheckpointVersion{
+ key: key,
+ payload: payload,
+ checkpoint: record,
+ alive: true,
+ }
+ return key, true
+}
+
+// restoreAndVerify restores the owned bytes, then captures them again. The
+// second capture proves that Deserialize accepted the complete checkpoint.
+func (l *externalScannerCheckpointLifecycle) restoreAndVerify(version *externalScannerCheckpointVersion) bool {
+ if l == nil || version == nil || !version.alive || !version.checkpoint.complete() {
+ return false
+ }
+ if !version.checkpoint.restore(l.scanner, version.payload) {
+ return false
+ }
+ restored, ok := captureExternalScannerCheckpointRecord(
+ l.scanner,
+ version.payload,
+ version.checkpoint.sourceByte,
+ version.checkpoint.sourcePoint,
+ version.checkpoint.externalLexState,
+ version.checkpoint.tokenStartByte,
+ version.checkpoint.tokenEndByte,
+ )
+ return ok && restored.equal(version.checkpoint)
+}
+
+// discardVersion destroys a payload after failed restore verification. It
+// also removes an owned version from the lifecycle ledger.
+func (l *externalScannerCheckpointLifecycle) discardVersion(version *externalScannerCheckpointVersion) {
+ if version == nil {
+ return
+ }
+ if l != nil && l.versions != nil {
+ if current, ok := l.versions[version.key]; ok && current == version {
+ version.alive = false
+ _ = l.deleteDead(version.key)
+ return
+ }
+ }
+ if version.payload != nil {
+ if l != nil && l.scanner != nil {
+ l.scanner.Destroy(version.payload)
+ }
+ version.payload = nil
+ }
+ version.alive = false
+}
+
+// elect runs one synthetic token election. A failed scan restores the prior
+// checkpoint and keeps the version only after verified restoration.
+func (l *externalScannerCheckpointLifecycle) elect(
+ key externalScannerCheckpointVersionKey,
+ sourceByte uint32,
+ sourcePoint Point,
+ externalLexState uint16,
+ scan func(any) (Token, bool),
+) (Token, bool) {
+ version, ok := l.version(key)
+ if !ok || scan == nil {
+ return Token{}, false
+ }
+ before, ok := captureExternalScannerCheckpointRecord(
+ l.scanner,
+ version.payload,
+ sourceByte,
+ sourcePoint,
+ externalLexState,
+ sourceByte,
+ sourceByte,
+ )
+ if !ok {
+ return Token{}, false
+ }
+ tok, ok := scan(version.payload)
+ if !ok {
+ if !before.restore(l.scanner, version.payload) {
+ l.discardVersion(version)
+ return Token{}, false
+ }
+ if restored, restoreOK := captureExternalScannerCheckpointRecord(
+ l.scanner,
+ version.payload,
+ before.sourceByte,
+ before.sourcePoint,
+ before.externalLexState,
+ before.tokenStartByte,
+ before.tokenEndByte,
+ ); !restoreOK || !restored.equal(before) {
+ l.discardVersion(version)
+ return Token{}, false
+ }
+ return Token{}, false
+ }
+ after, ok := captureExternalScannerCheckpointRecord(
+ l.scanner,
+ version.payload,
+ sourceByte,
+ sourcePoint,
+ externalLexState,
+ tok.StartByte,
+ tok.EndByte,
+ )
+ if !ok {
+ if !before.restore(l.scanner, version.payload) {
+ l.discardVersion(version)
+ return Token{}, false
+ }
+ restored, restoreOK := captureExternalScannerCheckpointRecord(
+ l.scanner,
+ version.payload,
+ before.sourceByte,
+ before.sourcePoint,
+ before.externalLexState,
+ before.tokenStartByte,
+ before.tokenEndByte,
+ )
+ if !restoreOK || !restored.equal(before) {
+ l.discardVersion(version)
+ }
+ return Token{}, false
+ }
+ version.checkpoint = after
+ return tok, true
+}
+
+func (l *externalScannerCheckpointLifecycle) fork(parentKey externalScannerCheckpointVersionKey) (externalScannerCheckpointVersionKey, bool) {
+ parent, ok := l.version(parentKey)
+ if !ok {
+ return externalScannerCheckpointVersionKey{}, false
+ }
+ childPayload := l.scanner.Create()
+ child := &externalScannerCheckpointVersion{
+ payload: childPayload,
+ checkpoint: parent.checkpoint.clone(),
+ alive: true,
+ }
+ if !l.restoreAndVerify(child) {
+ l.discardVersion(child)
+ return externalScannerCheckpointVersionKey{}, false
+ }
+ key, ok := l.nextVersionKey()
+ if !ok {
+ l.discardVersion(child)
+ return externalScannerCheckpointVersionKey{}, false
+ }
+ child.key = key
+ l.versions[key] = child
+ return key, true
+}
+
+func (l *externalScannerCheckpointLifecycle) canShare(aKey, bKey externalScannerCheckpointVersionKey) bool {
+ a, aOK := l.version(aKey)
+ b, bOK := l.version(bKey)
+ if !aOK || !bOK {
+ return false
+ }
+ if !l.currentStateMatches(a) {
+ l.discardVersion(a)
+ return false
+ }
+ if !l.currentStateMatches(b) {
+ l.discardVersion(b)
+ return false
+ }
+ return canShareExternalScannerCheckpoint(a.checkpoint, b.checkpoint)
+}
+
+func (l *externalScannerCheckpointLifecycle) currentStateMatches(version *externalScannerCheckpointVersion) bool {
+ if l == nil || version == nil || !version.alive || !version.checkpoint.complete() {
+ return false
+ }
+ record, ok := captureExternalScannerCheckpointRecord(
+ l.scanner,
+ version.payload,
+ version.checkpoint.sourceByte,
+ version.checkpoint.sourcePoint,
+ version.checkpoint.externalLexState,
+ version.checkpoint.tokenStartByte,
+ version.checkpoint.tokenEndByte,
+ )
+ return ok && record.equal(version.checkpoint)
+}
+
+// merge removes the candidate only when its complete record equals the keep
+// record. A mismatch leaves both versions alive for a safe fallback.
+func (l *externalScannerCheckpointLifecycle) merge(keepKey, candidateKey externalScannerCheckpointVersionKey) bool {
+ if keepKey == candidateKey || !l.canShare(keepKey, candidateKey) {
+ return false
+ }
+ return l.markDeadAndDelete(candidateKey)
+}
+
+// condense selects one version and deletes only dead or exact-record siblings.
+// It rejects a live sibling with a different scanner state.
+func (l *externalScannerCheckpointLifecycle) condense(selectedKey externalScannerCheckpointVersionKey, siblingKeys []externalScannerCheckpointVersionKey) (any, bool) {
+ selected, ok := l.version(selectedKey)
+ if !ok || !l.restoreAndVerify(selected) {
+ if ok {
+ l.discardVersion(selected)
+ }
+ return nil, false
+ }
+ for _, siblingKey := range siblingKeys {
+ if siblingKey == selectedKey {
+ continue
+ }
+ sibling, siblingOK := l.version(siblingKey)
+ if !siblingOK {
+ continue
+ }
+ if sibling.alive && !l.canShare(selectedKey, siblingKey) {
+ return nil, false
+ }
+ }
+ for _, siblingKey := range siblingKeys {
+ if siblingKey == selectedKey {
+ continue
+ }
+ if sibling, siblingOK := l.version(siblingKey); siblingOK {
+ sibling.alive = false
+ _ = l.deleteDead(siblingKey)
+ }
+ }
+ return selected.payload, true
+}
+
+// resume restores the selected version before the recovery callback. A failed
+// callback keeps the version only after verified restoration.
+func (l *externalScannerCheckpointLifecycle) resume(
+ key externalScannerCheckpointVersionKey,
+ sourceByte uint32,
+ sourcePoint Point,
+ externalLexState uint16,
+ recover func(any) (Token, bool),
+) (Token, bool) {
+ version, ok := l.version(key)
+ if !ok || recover == nil {
+ return Token{}, false
+ }
+ if !l.restoreAndVerify(version) {
+ l.discardVersion(version)
+ return Token{}, false
+ }
+ before := version.checkpoint.clone()
+ tok, ok := recover(version.payload)
+ if !ok {
+ if !before.restore(l.scanner, version.payload) {
+ l.discardVersion(version)
+ return Token{}, false
+ }
+ if !l.restoreAndVerify(version) {
+ l.discardVersion(version)
+ }
+ return Token{}, false
+ }
+ after, ok := captureExternalScannerCheckpointRecord(
+ l.scanner,
+ version.payload,
+ sourceByte,
+ sourcePoint,
+ externalLexState,
+ tok.StartByte,
+ tok.EndByte,
+ )
+ if !ok {
+ if !before.restore(l.scanner, version.payload) {
+ l.discardVersion(version)
+ return Token{}, false
+ }
+ restored, restoreOK := captureExternalScannerCheckpointRecord(
+ l.scanner,
+ version.payload,
+ before.sourceByte,
+ before.sourcePoint,
+ before.externalLexState,
+ before.tokenStartByte,
+ before.tokenEndByte,
+ )
+ if !restoreOK || !restored.equal(before) {
+ l.discardVersion(version)
+ }
+ return Token{}, false
+ }
+ version.checkpoint = after
+ return tok, true
+}
+
+func (l *externalScannerCheckpointLifecycle) markDead(key externalScannerCheckpointVersionKey) bool {
+ version, ok := l.version(key)
+ if !ok {
+ return false
+ }
+ version.alive = false
+ return true
+}
+
+func (l *externalScannerCheckpointLifecycle) deleteDead(key externalScannerCheckpointVersionKey) bool {
+ if l == nil {
+ return false
+ }
+ version, ok := l.versions[key]
+ if !ok || version == nil || version.alive {
+ return false
+ }
+ if version.payload != nil {
+ payload := version.payload
+ version.payload = nil
+ l.scanner.Destroy(payload)
+ }
+ delete(l.versions, key)
+ return true
+}
+
+func (l *externalScannerCheckpointLifecycle) markDeadAndDelete(key externalScannerCheckpointVersionKey) bool {
+ version, ok := l.versions[key]
+ if !ok || version == nil || !version.alive {
+ return false
+ }
+ version.alive = false
+ return l.deleteDead(key)
+}
diff --git a/external_scanner_checkpoint_lifecycle_test.go b/external_scanner_checkpoint_lifecycle_test.go
new file mode 100644
index 000000000..b884427cc
--- /dev/null
+++ b/external_scanner_checkpoint_lifecycle_test.go
@@ -0,0 +1,273 @@
+package gotreesitter
+
+import (
+ "bytes"
+ "testing"
+)
+
+type c26nCheckpointPayload struct {
+ state []byte
+}
+
+type c26nCheckpointScanner struct {
+ scannerID []byte
+ grammarID []byte
+ destroyed int
+ deserializeMismatch bool
+}
+
+func (s *c26nCheckpointScanner) Create() any {
+ return &c26nCheckpointPayload{state: []byte{1, 2, 3}}
+}
+
+func (s *c26nCheckpointScanner) Destroy(any) { s.destroyed++ }
+
+func (s *c26nCheckpointScanner) Serialize(payload any, buf []byte) int {
+ p, ok := payload.(*c26nCheckpointPayload)
+ if !ok || len(p.state) == 0 || len(p.state) > len(buf) {
+ return 0
+ }
+ return copy(buf, p.state)
+}
+
+func (s *c26nCheckpointScanner) Deserialize(payload any, buf []byte) {
+ p, ok := payload.(*c26nCheckpointPayload)
+ if !ok {
+ return
+ }
+ if s.deserializeMismatch {
+ p.state = []byte{9, 9, 9}
+ return
+ }
+ p.state = append(p.state[:0], buf...)
+}
+
+func (*c26nCheckpointScanner) Scan(any, *ExternalLexer, []bool) bool {
+ return false
+}
+
+func (*c26nCheckpointScanner) UsesExternalScannerCheckpoints() bool {
+ return true
+}
+
+func (s *c26nCheckpointScanner) CheckpointIdentity() (ExternalScannerCheckpointIdentity, bool) {
+ return ExternalScannerCheckpointIdentity{
+ Scanner: s.scannerID,
+ Grammar: s.grammarID,
+ }, true
+}
+
+func newC26nCheckpointScanner() *c26nCheckpointScanner {
+ return &c26nCheckpointScanner{
+ scannerID: []byte("scanner-c26n"),
+ grammarID: []byte("grammar-c26n"),
+ }
+}
+
+func c26nPayloadState(t *testing.T, payload any) []byte {
+ t.Helper()
+ p, ok := payload.(*c26nCheckpointPayload)
+ if !ok {
+ t.Fatalf("payload type = %T, want *c26nCheckpointPayload", payload)
+ }
+ return append([]byte(nil), p.state...)
+}
+
+func TestC26nCheckpointLifecycleRequiresExplicitOptIn(t *testing.T) {
+ scanner := newC26nCheckpointScanner()
+ if lifecycle, ok := newExternalScannerCheckpointLifecycle(scanner, false); ok || lifecycle != nil {
+ t.Fatal("disabled lifecycle allocated an ownership ledger")
+ }
+ if lifecycle, ok := newExternalScannerCheckpointLifecycle(nil, true); ok || lifecycle != nil {
+ t.Fatal("nil scanner enabled a lifecycle")
+ }
+}
+
+func TestC26nCheckpointLifecycleDiscardsFailedRestore(t *testing.T) {
+ scanner := newC26nCheckpointScanner()
+ lifecycle, ok := newExternalScannerCheckpointLifecycle(scanner, true)
+ if !ok {
+ t.Fatal("explicit lifecycle opt-in was rejected")
+ }
+
+ rootPayload := scanner.Create()
+ rootKey, ok := lifecycle.addRoot(rootPayload, 10, Point{Row: 1}, 7, 10, 10)
+ if !ok {
+ t.Fatal("root checkpoint was rejected")
+ }
+ scanner.deserializeMismatch = true
+ if _, ok := lifecycle.elect(rootKey, 10, Point{Row: 1}, 7, func(payload any) (Token, bool) {
+ payload.(*c26nCheckpointPayload).state = []byte{9, 9, 9}
+ return Token{}, false
+ }); ok {
+ t.Fatal("failed election unexpectedly succeeded")
+ }
+ if _, exists := lifecycle.versions[rootKey]; exists {
+ t.Fatal("failed election retained an unverified payload")
+ }
+ if scanner.destroyed != 1 {
+ t.Fatalf("destroy count after failed election = %d, want 1", scanner.destroyed)
+ }
+
+ scanner.deserializeMismatch = false
+ resumePayload := scanner.Create()
+ resumeKey, ok := lifecycle.addRoot(resumePayload, 11, Point{Row: 1}, 7, 11, 11)
+ if !ok {
+ t.Fatal("resume checkpoint was rejected")
+ }
+ scanner.deserializeMismatch = true
+ if _, ok := lifecycle.resume(resumeKey, 11, Point{Row: 1}, 7, func(any) (Token, bool) {
+ return Token{}, false
+ }); ok {
+ t.Fatal("failed resume unexpectedly succeeded")
+ }
+ if _, exists := lifecycle.versions[resumeKey]; exists {
+ t.Fatal("failed resume retained an unverified payload")
+ }
+ if scanner.destroyed != 2 {
+ t.Fatalf("destroy count after failed resume = %d, want 2", scanner.destroyed)
+ }
+ lifecycle.close()
+ if scanner.destroyed != 2 {
+ t.Fatalf("close destroyed discarded payloads again: count = %d", scanner.destroyed)
+ }
+}
+
+func TestC26nCheckpointLifecycle(t *testing.T) {
+ scanner := newC26nCheckpointScanner()
+ lifecycle, ok := newExternalScannerCheckpointLifecycle(scanner, true)
+ if !ok {
+ t.Fatal("explicit lifecycle opt-in was rejected")
+ }
+ defer lifecycle.close()
+
+ rootPayload := scanner.Create()
+ rootKey, ok := lifecycle.addRoot(rootPayload, 10, Point{Row: 1, Column: 2}, 7, 10, 10)
+ if !ok {
+ t.Fatal("root checkpoint was rejected")
+ }
+
+ selectedToken, ok := lifecycle.elect(rootKey, 10, Point{Row: 1, Column: 2}, 7, func(payload any) (Token, bool) {
+ p := payload.(*c26nCheckpointPayload)
+ p.state = []byte{4, 5, 6}
+ return Token{Symbol: 11, StartByte: 10, EndByte: 12}, true
+ })
+ if !ok || selectedToken.Symbol != 11 || selectedToken.EndByte != 12 {
+ t.Fatalf("election = (%+v, %t), want symbol 11 and success", selectedToken, ok)
+ }
+ if got := c26nPayloadState(t, rootPayload); !bytes.Equal(got, []byte{4, 5, 6}) {
+ t.Fatalf("elected payload = %v, want [4 5 6]", got)
+ }
+ rootRecord := lifecycle.versions[rootKey].checkpoint.clone()
+
+ failedToken, ok := lifecycle.elect(rootKey, 12, Point{Row: 1, Column: 4}, 7, func(payload any) (Token, bool) {
+ p := payload.(*c26nCheckpointPayload)
+ p.state = []byte{9, 9, 9}
+ return Token{}, false
+ })
+ if ok || failedToken != (Token{}) {
+ t.Fatalf("failed election = (%+v, %t), want zero token and failure", failedToken, ok)
+ }
+ if got := c26nPayloadState(t, rootPayload); !bytes.Equal(got, []byte{4, 5, 6}) {
+ t.Fatalf("failed-election payload = %v, want [4 5 6]", got)
+ }
+ if !lifecycle.versions[rootKey].checkpoint.equal(rootRecord) {
+ t.Fatal("failed election changed the owned checkpoint")
+ }
+
+ exactFork, ok := lifecycle.fork(rootKey)
+ if !ok {
+ t.Fatal("exact fork was rejected")
+ }
+ if !lifecycle.canShare(rootKey, exactFork) {
+ t.Fatal("exact fork did not share its copied checkpoint")
+ }
+ childPayload := lifecycle.versions[exactFork].payload
+ if got := c26nPayloadState(t, childPayload); !bytes.Equal(got, []byte{4, 5, 6}) {
+ t.Fatalf("fork payload = %v, want [4 5 6]", got)
+ }
+ childPayload.(*c26nCheckpointPayload).state[0] = 99
+ if got := c26nPayloadState(t, rootPayload); !bytes.Equal(got, []byte{4, 5, 6}) {
+ t.Fatalf("fork mutation changed root payload = %v, want [4 5 6]", got)
+ }
+ if !lifecycle.restoreAndVerify(lifecycle.versions[exactFork]) {
+ t.Fatal("exact fork did not restore after an independent payload mutation")
+ }
+ staleFork, ok := lifecycle.fork(rootKey)
+ if !ok {
+ t.Fatal("stale fork was rejected")
+ }
+ lifecycle.versions[staleFork].payload.(*c26nCheckpointPayload).state[0] = 77
+ if lifecycle.merge(rootKey, staleFork) {
+ t.Fatal("stale checkpoint merge was accepted")
+ }
+ if _, exists := lifecycle.versions[staleFork]; exists {
+ t.Fatal("stale checkpoint remained shareable")
+ }
+ if !lifecycle.merge(rootKey, exactFork) {
+ t.Fatal("exact checkpoint merge was rejected")
+ }
+ if _, exists := lifecycle.versions[exactFork]; exists {
+ t.Fatal("merged fork was not deleted")
+ }
+ if got := c26nPayloadState(t, rootPayload); !bytes.Equal(got, []byte{4, 5, 6}) {
+ t.Fatalf("root payload after merge = %v, want [4 5 6]", got)
+ }
+
+ mismatchedFork, ok := lifecycle.fork(rootKey)
+ if !ok {
+ t.Fatal("mismatched fork was rejected")
+ }
+ if _, ok := lifecycle.elect(mismatchedFork, 12, Point{Row: 1, Column: 4}, 7, func(payload any) (Token, bool) {
+ p := payload.(*c26nCheckpointPayload)
+ p.state = []byte{8, 8, 8}
+ return Token{Symbol: 12, StartByte: 12, EndByte: 13}, true
+ }); !ok {
+ t.Fatal("mismatched election was rejected")
+ }
+ if lifecycle.merge(rootKey, mismatchedFork) {
+ t.Fatal("mismatched checkpoint merge was accepted")
+ }
+ if _, exists := lifecycle.versions[mismatchedFork]; !exists {
+ t.Fatal("mismatched version was dropped after merge rejection")
+ }
+
+ if _, ok := lifecycle.condense(rootKey, []externalScannerCheckpointVersionKey{mismatchedFork}); ok {
+ t.Fatal("condense dropped a live mismatched version")
+ }
+ if !lifecycle.markDead(mismatchedFork) {
+ t.Fatal("mismatched version was not marked dead")
+ }
+ if !lifecycle.deleteDead(mismatchedFork) {
+ t.Fatal("dead version was not deleted")
+ }
+ if _, exists := lifecycle.versions[mismatchedFork]; exists {
+ t.Fatal("dead version remains in the ledger")
+ }
+
+ condenseFork, ok := lifecycle.fork(rootKey)
+ if !ok {
+ t.Fatal("condense fork was rejected")
+ }
+ if _, ok := lifecycle.condense(rootKey, []externalScannerCheckpointVersionKey{condenseFork}); !ok {
+ t.Fatal("exact condense selection was rejected")
+ }
+ if _, exists := lifecycle.versions[condenseFork]; exists {
+ t.Fatal("exact condense sibling was not deleted")
+ }
+
+ resumedToken, ok := lifecycle.resume(rootKey, 13, Point{Row: 1, Column: 5}, 7, func(payload any) (Token, bool) {
+ p := payload.(*c26nCheckpointPayload)
+ if !bytes.Equal(p.state, []byte{4, 5, 6}) {
+ return Token{}, false
+ }
+ p.state = []byte{7, 7, 7}
+ return Token{Symbol: 13, StartByte: 13, EndByte: 14}, true
+ })
+ if !ok || resumedToken.Symbol != 13 {
+ t.Fatalf("recovery resume = (%+v, %t), want symbol 13 and success", resumedToken, ok)
+ }
+ if got := c26nPayloadState(t, rootPayload); !bytes.Equal(got, []byte{7, 7, 7}) {
+ t.Fatalf("resumed payload = %v, want [7 7 7]", got)
+ }
+}
diff --git a/rust_dispatch_probe_test.go b/rust_dispatch_probe_test.go
new file mode 100644
index 000000000..17660121c
--- /dev/null
+++ b/rust_dispatch_probe_test.go
@@ -0,0 +1,283 @@
+package gotreesitter_test
+
+import (
+ "crypto/sha256"
+ "encoding/hex"
+ "fmt"
+ "os"
+ "testing"
+
+ gotreesitter "github.com/odvcencio/gotreesitter"
+ "github.com/odvcencio/gotreesitter/grammars"
+ "github.com/odvcencio/gotreesitter/internal/benchfixtures"
+)
+
+type rustDispatchProbeWitness struct {
+ name string
+ src []byte
+}
+
+func TestRustDispatchRawProductionAndRouteProbe(t *testing.T) {
+ t.Setenv("GTS_DISPATCHER_CENSUS", "1")
+ fixtures := []rustDispatchProbeWitness{
+ {name: "ownership-registry-smoke", src: []byte(grammars.ParseSmokeSample("rust"))},
+ {name: "tracked-census-rust_ast.rs", src: mustRustProbeFile(t, "testdata/incremental_gate/rust_ast.rs")},
+ {name: "admission-depth", src: []byte("fn main() {\n let values = [1, 2, 3];\n println!(\"{}\", values.len());\n}\n")},
+ {name: "token-tree", src: []byte("fn main() {\n let v = vec![1 + 2, 3 * 4, 5 - 6, a & b, c | d];\n println!(\"{} {}\", x && y, p || q);\n assert_eq!(lhs << 2, rhs >> 1);\n my_macro!(a : b, c => d, e $ f, g ? h);\n nested!(inner!(deep!(x % y ^ z, !flag)));\n paths!(std::collections::HashMap, core::mem);\n mixed! { key: value; arr[idx] = func(arg) + 1 }\n}\n")},
+ {name: "recovered-impl-item", src: []byte("pub type ExplicitSelf = Spanned;\n\nimpl Arg {\n pub fn to_self(&self) -> Option {\n if let PatKind::Ident(BindingMode::ByValue(mutbl), ident, _) = self.pat.node {\n if ident.node.name == keywords::SelfValue.name() {\n return match self.ty.node {\n TyKind::ImplicitSelf => Some(respan(self.pat.span, SelfKind::Value(mutbl))),\n _ => None,\n };\n }\n }\n None\n }\n}\n")},
+ {name: "doc-comment", src: []byte("//! crate docs\n/// item docs\nfn f() {}\n")},
+ {name: "token-binding", src: []byte("macro_rules! m { ($e:expr) => {} }\n")},
+ {name: "pattern-statement", src: []byte("if let A(x) | B(x) = expr {\n do_stuff_with(x);\n}\n")},
+ {name: "struct-expression", src: []byte("let a = SomeStruct { field1, field2: expression, field3, };\n")},
+ }
+
+ lang := grammars.RustLanguage()
+ for _, witness := range fixtures {
+ witness := witness
+ t.Run(witness.name, func(t *testing.T) {
+ sourceSHA := sha256.Sum256(witness.src)
+ rawParser := gotreesitter.NewParser(lang)
+ raw, err := rawParser.ParseNoResultCompatibilityBenchmarkOnly(witness.src)
+ if err != nil {
+ t.Fatalf("raw parse: %v", err)
+ }
+ defer raw.Release()
+ productionParser := gotreesitter.NewParser(lang)
+ productionParser.SetAdmissionCandidateRoute(false)
+ production, err := productionParser.Parse(witness.src)
+ if err != nil {
+ t.Fatalf("production parse: %v", err)
+ }
+ defer production.Release()
+ rawInspection, err := benchfixtures.InspectGoTree(raw.RootNode(), lang)
+ if err != nil {
+ t.Fatal(err)
+ }
+ productionInspection, err := benchfixtures.InspectGoTree(production.RootNode(), lang)
+ if err != nil {
+ t.Fatal(err)
+ }
+ rawRuntime := raw.ParseRuntime()
+ productionRuntime := production.ParseRuntime()
+ t.Logf("witness=%s bytes=%d source_sha256=%s raw_digest=%s production_digest=%s raw_rewrites=%d production_rewrites=%d raw_errors=%t production_errors=%t production_runtime=%s", witness.name, len(witness.src), hex.EncodeToString(sourceSHA[:]), rawInspection.SHA256, productionInspection.SHA256, rawRuntime.NormalizationNodesRewritten, productionRuntime.NormalizationNodesRewritten, raw.RootNode().HasError(), production.RootNode().HasError(), productionRuntime.Summary())
+ if rawInspection.SHA256 != productionInspection.SHA256 {
+ t.Errorf("raw and production deep digests differ: raw=%s production=%s", rawInspection.SHA256, productionInspection.SHA256)
+ }
+ logRustDispatchPass(t, "production", productionRuntime)
+
+ routeSource := append(append([]byte(nil), witness.src...), '\n')
+ routeProduction := parseRustProbeProduction(t, lang, routeSource)
+ defer routeProduction.Release()
+ routeProductionInspection, err := benchfixtures.InspectGoTree(routeProduction.RootNode(), lang)
+ if err != nil {
+ t.Fatal(err)
+ }
+ logRustRoute(t, "production-route", routeProduction, routeProductionInspection.SHA256)
+
+ compactParser := gotreesitter.NewParser(lang)
+ compactParser.SetAdmissionCandidateRoute(true)
+ compact, err := compactParser.Parse(routeSource)
+ if err != nil {
+ t.Fatalf("compact route: %v", err)
+ }
+ defer compact.Release()
+ compactInspection, err := benchfixtures.InspectGoTree(compact.RootNode(), lang)
+ if err != nil {
+ t.Fatal(err)
+ }
+ routed, fallback := gotreesitter.AdmissionCandidateCounters()
+ t.Logf("route=compact candidate_routed=%d candidate_fallback=%d fallback_reason=%q digest=%s errors=%t runtime=%s", routed, fallback, gotreesitter.AdmissionCandidateLastFallbackReason(), compactInspection.SHA256, compact.RootNode().HasError(), compact.ParseRuntime().Summary())
+ logRustDispatchPass(t, "compact", compact.ParseRuntime())
+ if compactInspection.SHA256 != routeProductionInspection.SHA256 {
+ t.Errorf("compact digest=%s, want production-route %s", compactInspection.SHA256, routeProductionInspection.SHA256)
+ }
+
+ forestParser := gotreesitter.NewParser(lang)
+ forest, forestOK := forestParser.ParseForestExperimental(routeSource)
+ if !forestOK || forest == nil {
+ offset, symbol, reason, states := forestParser.ForestDeclineInfo()
+ t.Logf("route=forest outcome=declined offset=%d symbol=%d reason=%q states=%v", offset, symbol, reason, states)
+ } else {
+ defer forest.Release()
+ forestInspection, inspectErr := benchfixtures.InspectGoTree(forest.RootNode(), lang)
+ if inspectErr != nil {
+ t.Fatal(inspectErr)
+ }
+ t.Logf("route=forest outcome=accepted digest=%s errors=%t runtime=%s", forestInspection.SHA256, forest.RootNode().HasError(), forest.ParseRuntime().Summary())
+ logRustDispatchPass(t, "forest", forest.ParseRuntime())
+ if forestInspection.SHA256 != routeProductionInspection.SHA256 {
+ t.Errorf("forest digest=%s, want production-route %s", forestInspection.SHA256, routeProductionInspection.SHA256)
+ }
+ }
+
+ oldTree := parseRustProbeProduction(t, lang, witness.src)
+ defer oldTree.Release()
+ oldEnd := rustProbePointAtByte(witness.src, len(witness.src))
+ oldTree.Edit(gotreesitter.InputEdit{StartByte: uint32(len(witness.src)), OldEndByte: uint32(len(witness.src)), NewEndByte: uint32(len(routeSource)), StartPoint: oldEnd, OldEndPoint: oldEnd, NewEndPoint: rustProbePointAtByte(routeSource, len(routeSource))})
+ incrementalParser := gotreesitter.NewParser(lang)
+ incremental, profile, err := incrementalParser.ParseIncrementalProfiled(routeSource, oldTree)
+ if err != nil {
+ t.Fatalf("incremental route: %v", err)
+ }
+ defer incremental.Release()
+ incrementalInspection, err := benchfixtures.InspectGoTree(incremental.RootNode(), lang)
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Logf("route=incremental digest=%s errors=%t profile=%+v runtime=%s", incrementalInspection.SHA256, incremental.RootNode().HasError(), profile, incremental.ParseRuntime().Summary())
+ logRustDispatchPass(t, "incremental", incremental.ParseRuntime())
+ if incrementalInspection.SHA256 != routeProductionInspection.SHA256 {
+ t.Errorf("incremental digest=%s, want production-route %s", incrementalInspection.SHA256, routeProductionInspection.SHA256)
+ }
+ })
+ }
+}
+
+// TestRustDispatchRewriteTraceExpandedCorpus expands the receipt beyond the
+// registered smoke and tracked witnesses. It records every Rust arm rewrite.
+func TestRustDispatchRewriteTraceExpandedCorpus(t *testing.T) {
+ t.Setenv("GTS_DISPATCHER_CENSUS", "1")
+ fixtures := []rustDispatchProbeWitness{
+ {name: "admission-direct-external-payload", src: mustRustProbeFile(t, "testdata/admission_direct/external_payload/rust.rs")},
+ {name: "outline-rust-lib", src: mustRustProbeFile(t, "testdata/outline/rust/lib.rs")},
+ {name: "parity-lifetime-and-abstract-types", src: []byte("fn main() {}\n\nfn add(x: i32, y: i32) -> i32 { return x + y; }\n\nfn foo(x: impl FnOnce() -> result::Result) {}\n\nfn foo(bar: impl for<'a> Baz>) {}\n")},
+ {name: "parity-pattern-statements", src: []byte("if let A(x) | B(x) = expr { do_stuff_with(x); }\nwhile let A(x) | B(x) = expr { do_stuff_with(x); }\nlet Ok(index) | Err(index) = slice.binary_search(&x);\nfor A | B | C in c {}\n")},
+ {name: "parity-macro-invocations", src: []byte("a!(* a *);\na!(& a &);\na!(- a -);\na!(b + c + +);\na!('a'..='z');\na!($);\na!($());\na!($ a $);\na!(${$([ a ])});\na!($a $a:ident $($a);*);\n")},
+ {name: "parity-weird-expressions", src: []byte("fn angrydome() {\n loop { if break { } }\n let mut i = 0;\n loop { i += 1; if i == 1 { match (continue) { 1 => { }, _ => panic!(\"wat\") } }\n break; }\n}\n\nfn special_characters() {\n let val = !((|(..):(_,_),(|__@_|__)|__)((&*\"\\\\\",'🤔')/**/,{})=={&[..=..][..];})//\n ;\n assert!(!val);\n}\n\nfn function() {\n struct foo;\n impl Deref for foo {\n type Target = fn() -> Self;\n fn deref(&self) -> &Self::Target {\n &((|| foo) as _)\n }\n }\n let foo = foo () ()() ()()() ()()()() ()()()()();\n}\n\nfn closure_matching() {\n let x = |_| Some(1);\n let (|x| x) = match x(..) {\n |_| Some(2) => |_| Some(3),\n |_| _ => unreachable!(),\n };\n assert!(matches!(x(..), |_| Some(4)));\n}\n")},
+ {name: "parity-weird-top-level", src: []byte("// Just a grab bag of stuff that you would not want to write.\n\nfn strange() -> bool { let _x: bool = return true; }\n\nfn what() {\n fn the(x: &Cell) {\n return while !x.get() { x.set(true); };\n }\n let i = &Cell::new(false);\n let dont = {||the(i)};\n dont();\n assert!((i.get()));\n}\n\nfn punch_card() -> impl std::fmt::Debug {\n ..=..=.. .. .. .. .. .. .. .. .. .. .. ..=.. ..\n ..=.. ..=.. .. .. .. .. .. .. .. .. ..=..=..=..\n ..=.. ..=.. ..=.. ..=.. .. ..=..=.. .. ..=.. ..\n ..=..=.. .. ..=.. ..=.. ..=.. .. .. .. ..=.. ..\n ..=.. ..=.. ..=.. ..=.. .. ..=.. .. .. ..=.. ..\n ..=.. ..=.. ..=.. ..=.. .. .. ..=.. .. ..=.. ..\n ..=.. ..=.. .. ..=..=.. ..=..=.. .. .. ..=.. ..\n}\n")},
+ }
+ lang := grammars.RustLanguage()
+ for _, fixture := range fixtures {
+ fixture := fixture
+ t.Run(fixture.name, func(t *testing.T) {
+ rawParser := gotreesitter.NewParser(lang)
+ raw, err := rawParser.ParseNoResultCompatibilityBenchmarkOnly(fixture.src)
+ if err != nil {
+ t.Fatalf("raw parse: %v", err)
+ }
+ defer raw.Release()
+ productionParser := gotreesitter.NewParser(lang)
+ productionParser.SetAdmissionCandidateRoute(false)
+ production, err := productionParser.Parse(fixture.src)
+ if err != nil {
+ t.Fatalf("production parse: %v", err)
+ }
+ defer production.Release()
+ rawInspection, err := benchfixtures.InspectGoTree(raw.RootNode(), lang)
+ if err != nil {
+ t.Fatal(err)
+ }
+ productionInspection, err := benchfixtures.InspectGoTree(production.RootNode(), lang)
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Logf("expanded=%s bytes=%d raw_digest=%s production_digest=%s raw_rewrites=%d production_rewrites=%d raw_errors=%t production_errors=%t", fixture.name, len(fixture.src), rawInspection.SHA256, productionInspection.SHA256, raw.ParseRuntime().NormalizationNodesRewritten, production.ParseRuntime().NormalizationNodesRewritten, raw.RootNode().HasError(), production.RootNode().HasError())
+ logRustDispatchPass(t, "expanded-production", production.ParseRuntime())
+ if rawInspection.SHA256 != productionInspection.SHA256 {
+ t.Logf("expanded raw/production deep mismatch: raw=%s production=%s", rawInspection.SHA256, productionInspection.SHA256)
+ }
+ })
+ }
+}
+
+// TestRustDispatchRewriteTraceMalformedCorpus probes the recovery-only paths
+// that the clean corpus does not reach. Keep this diagnostic receipt separate.
+func TestRustDispatchRewriteTraceMalformedCorpus(t *testing.T) {
+ t.Setenv("GTS_DISPATCHER_CENSUS", "1")
+ fixtures := []rustDispatchProbeWitness{
+ {name: "malformed-function-impl-type", src: []byte("fn foo(bar: impl for<'a> Baz>) {\n")},
+ {name: "malformed-top-level-impl", src: []byte("impl Arg {\n pub fn f(&self) -> bool { self.value }\n")},
+ {name: "malformed-let-closure", src: []byte("fn f() { let x = (|a| a + 1; }\n")},
+ {name: "malformed-token-tree", src: []byte("macro_rules! m { ($e:expr => {\n")},
+ {name: "malformed-pattern-statement", src: []byte("if let A(x) | B(x) = expr {\n")},
+ {name: "malformed-struct-expression", src: []byte("let a = SomeStruct { field1, field2: expression\n")},
+ {name: "malformed-doc-comment", src: []byte("/// docs\nfn f(\n")},
+ }
+ lang := grammars.RustLanguage()
+ for _, fixture := range fixtures {
+ fixture := fixture
+ t.Run(fixture.name, func(t *testing.T) {
+ rawParser := gotreesitter.NewParser(lang)
+ raw, err := rawParser.ParseNoResultCompatibilityBenchmarkOnly(fixture.src)
+ if err != nil {
+ t.Fatalf("raw parse: %v", err)
+ }
+ defer raw.Release()
+ productionParser := gotreesitter.NewParser(lang)
+ productionParser.SetAdmissionCandidateRoute(false)
+ production, err := productionParser.Parse(fixture.src)
+ if err != nil {
+ t.Fatalf("production parse: %v", err)
+ }
+ defer production.Release()
+ rawInspection, err := benchfixtures.InspectGoTree(raw.RootNode(), lang)
+ if err != nil {
+ t.Fatal(err)
+ }
+ productionInspection, err := benchfixtures.InspectGoTree(production.RootNode(), lang)
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Logf("malformed=%s bytes=%d raw_digest=%s production_digest=%s raw_rewrites=%d production_rewrites=%d raw_errors=%t production_errors=%t", fixture.name, len(fixture.src), rawInspection.SHA256, productionInspection.SHA256, raw.ParseRuntime().NormalizationNodesRewritten, production.ParseRuntime().NormalizationNodesRewritten, raw.RootNode().HasError(), production.RootNode().HasError())
+ logRustDispatchPass(t, "malformed-production", production.ParseRuntime())
+ if rawInspection.SHA256 != productionInspection.SHA256 {
+ t.Logf("malformed raw/production deep mismatch: raw=%s production=%s", rawInspection.SHA256, productionInspection.SHA256)
+ }
+ })
+ }
+}
+
+func mustRustProbeFile(t *testing.T, path string) []byte {
+ t.Helper()
+ source, err := os.ReadFile(path)
+ if err != nil {
+ t.Fatalf("read %s: %v", path, err)
+ }
+ return source
+}
+
+func parseRustProbeProduction(t *testing.T, lang *gotreesitter.Language, source []byte) *gotreesitter.Tree {
+ t.Helper()
+ parser := gotreesitter.NewParser(lang)
+ parser.SetAdmissionCandidateRoute(false)
+ tree, err := parser.Parse(source)
+ if err != nil {
+ t.Fatalf("production parse: %v", err)
+ }
+ return tree
+}
+
+func logRustDispatchPass(t *testing.T, route string, runtime gotreesitter.ParseRuntime) {
+ t.Helper()
+ if runtime.NormalizationPasses == nil {
+ t.Logf("route=%s dispatch.rust=absent", route)
+ return
+ }
+ for _, pass := range *runtime.NormalizationPasses {
+ if pass.Name == "dispatch.rust" || pass.Name == "rust_source_file_root_pre" || pass.Name == "rust_recovered_pattern_statements_root" || pass.Name == "rust_recovered_function_items" || pass.Name == "rust_recovered_struct_expression_root" || pass.Name == "rust_token_binding_patterns" || pass.Name == "rust_source_file_root_post" || pass.Name == "rust_doc_comment_ranges" {
+ t.Logf("route=%s pass=%s checked=%d run=%d visited=%d rewritten=%d", route, pass.Name, pass.Checked, pass.Run, pass.NodesVisited, pass.NodesRewritten)
+ }
+ }
+}
+
+func logRustRoute(t *testing.T, route string, tree *gotreesitter.Tree, digest string) {
+ t.Helper()
+ t.Logf("route=%s digest=%s errors=%t runtime=%s", route, digest, tree.RootNode().HasError(), tree.ParseRuntime().Summary())
+ logRustDispatchPass(t, route, tree.ParseRuntime())
+}
+
+func rustProbePointAtByte(source []byte, offset int) gotreesitter.Point {
+ if offset < 0 || offset > len(source) {
+ panic(fmt.Sprintf("point offset %d outside source length %d", offset, len(source)))
+ }
+ var point gotreesitter.Point
+ for _, value := range source[:offset] {
+ if value == '\n' {
+ point.Row++
+ point.Column = 0
+ } else {
+ point.Column++
+ }
+ }
+ return point
+}