diff --git a/cgo_harness/issue454_variant_locked_c_diag_test.go b/cgo_harness/issue454_variant_locked_c_diag_test.go new file mode 100644 index 000000000..abfd6e60e --- /dev/null +++ b/cgo_harness/issue454_variant_locked_c_diag_test.go @@ -0,0 +1,94 @@ +//go:build cgo && treesitter_c_parity + +package cgoharness + +import ( + "crypto/sha256" + "fmt" + "os" + "testing" + + sitter "github.com/tree-sitter/go-tree-sitter" + + gotreesitter "github.com/odvcencio/gotreesitter" + "github.com/odvcencio/gotreesitter/grammars" + "github.com/odvcencio/gotreesitter/internal/benchfixtures" +) + +func TestIssue454VariantLockedCDiag(t *testing.T) { + cases := []struct { + name string + lang string + load func() *gotreesitter.Language + path string + source string + }{ + {name: "cobol_a0_small", lang: "cobol", load: grammars.CobolLanguage, path: "../testdata/dispatcher_census_a0/cobol/small__CVERSNP1.cpy"}, + {name: "cobol_a0_medium", lang: "cobol", load: grammars.CobolLanguage, path: "../testdata/dispatcher_census_a0/cobol/medium__DBANK02P.cbl"}, + {name: "cobol_a0_large", lang: "cobol", load: grammars.CobolLanguage, path: "../testdata/dispatcher_census_a0/cobol/large__MBANK30.cpy"}, + {name: "wgsl_a0_small", lang: "wgsl", load: grammars.WgslLanguage, path: "../testdata/dispatcher_census_a0/wgsl/small__fragmentTextureQuad.wgsl"}, + {name: "wgsl_a0_medium_normal", lang: "wgsl", load: grammars.WgslLanguage, path: "../testdata/dispatcher_census_a0/wgsl/medium__normalMap.wgsl"}, + {name: "wgsl_a0_medium_radiosity", lang: "wgsl", load: grammars.WgslLanguage, path: "../testdata/dispatcher_census_a0/wgsl/medium__radiosity.wgsl"}, + {name: "cooklang_punctuation", lang: "cooklang", load: grammars.CooklangLanguage, source: "Add @salt{1%tsp}.\n"}, + {name: "cooklang_recovered", lang: "cooklang", load: grammars.CooklangLanguage, source: "---\nservings: 4\nemoji: 🥟\ntags: warm, fried, starter\n---\n\nServe hot.\n"}, + {name: "cooklang_no_newline", lang: "cooklang", load: grammars.CooklangLanguage, source: "Add @salt{1%tsp}."}, + } + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + var source []byte + if tc.path != "" { + var err error + source, err = os.ReadFile(tc.path) + if err != nil { + t.Fatalf("read source: %v", err) + } + } else { + source = []byte(tc.source) + } + cLang, err := ParityCLanguage(tc.lang) + if err != nil { + t.Fatalf("load locked C language: %v", err) + } + cParser := sitter.NewParser() + defer cParser.Close() + if err := cParser.SetLanguage(cLang); err != nil { + t.Fatalf("set C language: %v", err) + } + goLang := tc.load() + goParser := gotreesitter.NewParser(goLang) + goParser.SetAdmissionCandidateRoute(false) + goTree, err := goParser.ParseNoResultCompatibilityBenchmarkOnly(source) + if err != nil { + t.Fatalf("parse Go tree: %v", err) + } + defer goTree.Release() + cTree := cParser.Parse(source, nil) + if cTree == nil || cTree.RootNode() == nil { + t.Fatal("C parse returned no tree") + } + defer cTree.Close() + goRoot := goTree.RootNode() + cRoot := cTree.RootNode() + var diffs []compactT3StructuralDivergence + compactT3WalkStructuralDivergences(goRoot, goLang, cRoot, "root", &diffs) + goDigest := sha256.Sum256([]byte(goRoot.SExpr(goLang))) + cDigest := sha256.Sum256([]byte(dumpCTree(cRoot, 0))) + goInspection, err := benchfixtures.InspectGoTree(goRoot, goLang) + if err != nil { + t.Fatalf("inspect Go tree: %v", err) + } + cDeepDigest, err := COracleDeepDigest(cTree) + if err != nil { + t.Fatalf("inspect C tree: %v", err) + } + t.Logf("raw_error=%v c_error=%v raw_sexpr_sha256=%x c_tree_sha256=%x go_deep_digest=%s c_deep_digest=%s structural_diffs=%d", goRoot.HasError(), cRoot.HasError(), goDigest, cDigest, goInspection.SHA256, cDeepDigest, len(diffs)) + for i, diff := range diffs { + if i == 8 { + break + } + t.Logf("diff[%d]=%s", i, fmt.Sprint(diff)) + } + }) + } +} diff --git a/parser_recover_c.go b/parser_recover_c.go index a5079c823..692e798a3 100644 --- a/parser_recover_c.go +++ b/parser_recover_c.go @@ -4074,6 +4074,21 @@ func (p *Parser) cAppendVisibleSplice(dst []*Node, n *Node) []*Node { return dst } +func (p *Parser) cAppendVisibleSpliceWithFields(scratch *reduceBuildScratch, n *Node) { + if p == nil || scratch == nil || n == nil { + return + } + if n.symbol == errorSymbol || n.isMissing() || p.cSymbolVisible(n.symbol) { + scratch.appendNode(n) + return + } + if hiddenTreeHasFieldIDs(n) { + appendFlattenedHiddenChildrenWithFieldScratch(scratch, n, p.language.SymbolMetadata, nil) + return + } + appendFlattenedHiddenChildrenToScratch(scratch, n, p.language.SymbolMetadata, nil) +} + func (p *Parser) cAppendVisibleSpliceUntil(dst []*Node, n *Node, limit int) ([]*Node, bool) { if n == nil { return dst, true @@ -4162,7 +4177,8 @@ func (p *Parser) cRecoverToState(v *glrStack, depth int, goal StateID, arena *no // open ERROR node's children) and splice invisible nodes the way the // engine's reduce does. The raw popped extent pins the ERROR span (C // error regions cover invisible subtrees too). - children := make([]*Node, 0, len(wrapped)+2) + splice := reduceBuildScratch{} + splice.nodes = make([]*Node, 0, len(wrapped)+2) openErr := (*cRecoverState)(nil) if v.cRec != nil { openErr = v.cRec @@ -4175,14 +4191,17 @@ func (p *Parser) cRecoverToState(v *glrStack, depth int, goal StateID, arena *no rawLast = n if openErr != nil && n == openErr.openErr { // Open-region children were visible-spliced at absorb time. - children = append(children, n.children...) + for _, child := range n.children { + splice.appendNode(child) + } continue } // C parity: popped closed subtrees (ERROR carriers included) keep // their identity inside the new ERROR; only invisible subtrees // flatten. - children = p.cAppendVisibleSplice(children, n) + p.cAppendVisibleSpliceWithFields(&splice, n) } + children, fieldIDs, fieldSources := materializeReduceChildrenFromScratch(&splice, arena) fork := v.cloneWithScratch(gssScratch) fork.cRec = nil @@ -4215,6 +4234,7 @@ func (p *Parser) cRecoverToState(v *glrStack, depth int, goal StateID, arena *no if rawFirst != nil { errNode := p.newRecoveryParentNodeInArena(arena, errorSymbol, true, children, 0) + errNode.setFieldMetadata(fieldIDs, fieldSources) cSetNodeSpan(errNode, rawFirst.startByte, rawLast.endByte, rawFirst.startPoint, rawLast.endPoint) errNode.setHasError(true) errNode.setExtra(true) @@ -4277,7 +4297,6 @@ func (p *Parser) cAbsorbTokenIntoError(v *glrStack, tok Token, nodeCount *int, a if leafVisible { leaf = newLeafNodeInArena(arena, tok.Symbol, tok.Symbol == errorSymbol || p.isNamedSymbol(tok.Symbol), tok.StartByte, tok.EndByte, tok.StartPoint, tok.EndPoint) - leaf.setHasError(true) // C: if the token shifts as extra in state 1, mark it extra so it is // not counted in error cost calculations. if idx := p.lookupActionIndex(1, tok.Symbol); idx != 0 && int(idx) < len(p.language.ParseActions) {