[archive] Preserve Issue 454 combined recovery variant

cgo_harness/issue454_variant_locked_c_diag_test.go

@@ -0,0 +1,94 @@
+//go:build cgo && treesitter_c_parity
+
+package cgoharness
+
+import (
+	"crypto/sha256"
+	"fmt"
+	"os"
+	"testing"
+
+	sitter "github.com/tree-sitter/go-tree-sitter"
+
+	gotreesitter "github.com/odvcencio/gotreesitter"
+	"github.com/odvcencio/gotreesitter/grammars"
+	"github.com/odvcencio/gotreesitter/internal/benchfixtures"
+)
+
+func TestIssue454VariantLockedCDiag(t *testing.T) {
+	cases := []struct {
+		name   string
+		lang   string
+		load   func() *gotreesitter.Language
+		path   string
+		source string
+	}{
+		{name: "cobol_a0_small", lang: "cobol", load: grammars.CobolLanguage, path: "../testdata/dispatcher_census_a0/cobol/small__CVERSNP1.cpy"},
+		{name: "cobol_a0_medium", lang: "cobol", load: grammars.CobolLanguage, path: "../testdata/dispatcher_census_a0/cobol/medium__DBANK02P.cbl"},
+		{name: "cobol_a0_large", lang: "cobol", load: grammars.CobolLanguage, path: "../testdata/dispatcher_census_a0/cobol/large__MBANK30.cpy"},
+		{name: "wgsl_a0_small", lang: "wgsl", load: grammars.WgslLanguage, path: "../testdata/dispatcher_census_a0/wgsl/small__fragmentTextureQuad.wgsl"},
+		{name: "wgsl_a0_medium_normal", lang: "wgsl", load: grammars.WgslLanguage, path: "../testdata/dispatcher_census_a0/wgsl/medium__normalMap.wgsl"},
+		{name: "wgsl_a0_medium_radiosity", lang: "wgsl", load: grammars.WgslLanguage, path: "../testdata/dispatcher_census_a0/wgsl/medium__radiosity.wgsl"},
+		{name: "cooklang_punctuation", lang: "cooklang", load: grammars.CooklangLanguage, source: "Add @salt{1%tsp}.\n"},
+		{name: "cooklang_recovered", lang: "cooklang", load: grammars.CooklangLanguage, source: "---\nservings: 4\nemoji: 🥟\ntags: warm, fried, starter\n---\n\nServe hot.\n"},
+		{name: "cooklang_no_newline", lang: "cooklang", load: grammars.CooklangLanguage, source: "Add @salt{1%tsp}."},
+	}
+	for _, tc := range cases {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			var source []byte
+			if tc.path != "" {
+				var err error
+				source, err = os.ReadFile(tc.path)
+				if err != nil {
+					t.Fatalf("read source: %v", err)
+				}
+			} else {
+				source = []byte(tc.source)
+			}
+			cLang, err := ParityCLanguage(tc.lang)
+			if err != nil {
+				t.Fatalf("load locked C language: %v", err)
+			}
+			cParser := sitter.NewParser()
+			defer cParser.Close()
+			if err := cParser.SetLanguage(cLang); err != nil {
+				t.Fatalf("set C language: %v", err)
+			}
+			goLang := tc.load()
+			goParser := gotreesitter.NewParser(goLang)
+			goParser.SetAdmissionCandidateRoute(false)
+			goTree, err := goParser.ParseNoResultCompatibilityBenchmarkOnly(source)
+			if err != nil {
+				t.Fatalf("parse Go tree: %v", err)
+			}
+			defer goTree.Release()
+			cTree := cParser.Parse(source, nil)
+			if cTree == nil || cTree.RootNode() == nil {
+				t.Fatal("C parse returned no tree")
+			}
+			defer cTree.Close()
+			goRoot := goTree.RootNode()
+			cRoot := cTree.RootNode()
+			var diffs []compactT3StructuralDivergence
+			compactT3WalkStructuralDivergences(goRoot, goLang, cRoot, "root", &diffs)
+			goDigest := sha256.Sum256([]byte(goRoot.SExpr(goLang)))
+			cDigest := sha256.Sum256([]byte(dumpCTree(cRoot, 0)))
+			goInspection, err := benchfixtures.InspectGoTree(goRoot, goLang)
+			if err != nil {
+				t.Fatalf("inspect Go tree: %v", err)
+			}
+			cDeepDigest, err := COracleDeepDigest(cTree)
+			if err != nil {
+				t.Fatalf("inspect C tree: %v", err)
+			}
+			t.Logf("raw_error=%v c_error=%v raw_sexpr_sha256=%x c_tree_sha256=%x go_deep_digest=%s c_deep_digest=%s structural_diffs=%d", goRoot.HasError(), cRoot.HasError(), goDigest, cDigest, goInspection.SHA256, cDeepDigest, len(diffs))
+			for i, diff := range diffs {
+				if i == 8 {
+					break
+				}
+				t.Logf("diff[%d]=%s", i, fmt.Sprint(diff))
+			}
+		})
+	}
+}

parser_recover_c.go

@@ -4074,6 +4074,21 @@ func (p *Parser) cAppendVisibleSplice(dst []*Node, n *Node) []*Node {
 	return dst
 }
 
+func (p *Parser) cAppendVisibleSpliceWithFields(scratch *reduceBuildScratch, n *Node) {
+	if p == nil || scratch == nil || n == nil {
+		return
+	}
+	if n.symbol == errorSymbol || n.isMissing() || p.cSymbolVisible(n.symbol) {
+		scratch.appendNode(n)
+		return
+	}
+	if hiddenTreeHasFieldIDs(n) {
+		appendFlattenedHiddenChildrenWithFieldScratch(scratch, n, p.language.SymbolMetadata, nil)
+		return
+	}
+	appendFlattenedHiddenChildrenToScratch(scratch, n, p.language.SymbolMetadata, nil)
+}
+
 func (p *Parser) cAppendVisibleSpliceUntil(dst []*Node, n *Node, limit int) ([]*Node, bool) {
 	if n == nil {
 		return dst, true
@@ -4162,7 +4177,8 @@ func (p *Parser) cRecoverToState(v *glrStack, depth int, goal StateID, arena *no
 	// open ERROR node's children) and splice invisible nodes the way the
 	// engine's reduce does. The raw popped extent pins the ERROR span (C
 	// error regions cover invisible subtrees too).
-	children := make([]*Node, 0, len(wrapped)+2)
+	splice := reduceBuildScratch{}
+	splice.nodes = make([]*Node, 0, len(wrapped)+2)
 	openErr := (*cRecoverState)(nil)
 	if v.cRec != nil {
 		openErr = v.cRec
@@ -4175,14 +4191,17 @@ func (p *Parser) cRecoverToState(v *glrStack, depth int, goal StateID, arena *no
 		rawLast = n
 		if openErr != nil && n == openErr.openErr {
 			// Open-region children were visible-spliced at absorb time.
-			children = append(children, n.children...)
+			for _, child := range n.children {
+				splice.appendNode(child)
+			}
 			continue
 		}
 		// C parity: popped closed subtrees (ERROR carriers included) keep
 		// their identity inside the new ERROR; only invisible subtrees
 		// flatten.
-		children = p.cAppendVisibleSplice(children, n)
+		p.cAppendVisibleSpliceWithFields(&splice, n)
 	}
+	children, fieldIDs, fieldSources := materializeReduceChildrenFromScratch(&splice, arena)
 
 	fork := v.cloneWithScratch(gssScratch)
 	fork.cRec = nil
@@ -4215,6 +4234,7 @@ func (p *Parser) cRecoverToState(v *glrStack, depth int, goal StateID, arena *no
 
 	if rawFirst != nil {
 		errNode := p.newRecoveryParentNodeInArena(arena, errorSymbol, true, children, 0)
+		errNode.setFieldMetadata(fieldIDs, fieldSources)
 		cSetNodeSpan(errNode, rawFirst.startByte, rawLast.endByte, rawFirst.startPoint, rawLast.endPoint)
 		errNode.setHasError(true)
 		errNode.setExtra(true)
@@ -4277,7 +4297,6 @@ func (p *Parser) cAbsorbTokenIntoError(v *glrStack, tok Token, nodeCount *int, a
 	if leafVisible {
 		leaf = newLeafNodeInArena(arena, tok.Symbol, tok.Symbol == errorSymbol || p.isNamedSymbol(tok.Symbol),
 			tok.StartByte, tok.EndByte, tok.StartPoint, tok.EndPoint)
-		leaf.setHasError(true)
 		// C: if the token shifts as extra in state 1, mark it extra so it is
 		// not counted in error cost calculations.
 		if idx := p.lookupActionIndex(1, tok.Symbol); idx != 0 && int(idx) < len(p.language.ParseActions) {