From 784aa4e81afe2f20b6b48dd85a88649db68e337e Mon Sep 17 00:00:00 2001 From: Oscar Villavicencio <9220505+odvcencio@users.noreply.github.com> Date: Mon, 24 Aug 2026 16:26:00 -0700 Subject: [PATCH] fix(included-ranges): start included-range parse at first selected byte - Re-seek or preserve overlapping tokens at range boundaries. - Align initial parser stack offset with the first included byte. - Route empty-stack end points through a new accessor method. - Add parity checks for root start bytes and start points. - Update docs to record the root-start fix as partial parity. Buckley-Change-Hash: sha256:57418d5725d1b68369e0708b92c5ab1222839c12865933a4812ea5f978a0b0da Buckley-Change-Stats: files=8 insertions=311 deletions=37 binaries=0 --- CHANGELOG.md | 9 ++ cgo_harness/go_next_live_probe_test.go | 2 +- ...included_ranges_go_root_parity_cgo_test.go | 30 +++-- docs/root-normalization-retirement.md | 53 ++++++--- included_ranges.go | 26 ++++- included_ranges_test.go | 87 ++++++++++++++ parser.go | 35 +++++- parser_included_ranges_go_root_test.go | 106 ++++++++++++++++++ 8 files changed, 311 insertions(+), 37 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ed1fd49d..85a2eedde 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -91,6 +91,15 @@ for tags and release notes while still in `0.x`. ### Correctness +- Included-range parsing now starts byte-seek sources at the first selected + byte. The parser seeds its initial stack there and preserves the configured + start point for recovery gaps. A non-seek source preserves a complete + overlapping token because it cannot reproduce a trimmed boundary token. + The locked Go route now matches C at root byte 26. Child counts remain 10 + for Go and seven for C. Compact and forest included-range routes remain + uncertified. Keep `dispatch.go.source-file-root` live until exact shape and + all required route proofs pass. See `docs/root-normalization-retirement.md`. + - Recorded a generic Swift issue #576 recovery candidate at base `da6f71471aaaa835503accaa1bc2083ced90b4e6`. The active deterministic finite automaton (DFA) source now replays recovery from the exact skipped-prefix diff --git a/cgo_harness/go_next_live_probe_test.go b/cgo_harness/go_next_live_probe_test.go index c99bf415a..a4cd2afb5 100644 --- a/cgo_harness/go_next_live_probe_test.go +++ b/cgo_harness/go_next_live_probe_test.go @@ -311,7 +311,7 @@ func goNextRunIncludedRanges(t *testing.T, language *gotreesitter.Language, cLan if cDigest != "9c8e5bb506bb345a577beb351f7b9230cca5e2e02cc4fd619e21f607657f290f" { t.Fatalf("included-ranges locked-C digest=%s", cDigest) } - goNextCheckRoute(t, "included-ranges", goTree, language, cTree, cDigest, "8231fb8782e5699708e183619b68e1d030ab09e60de99ac52e3c092ffc56d59c", goNextDivergence("/source_file", "range", "0:0-22:0 @0..276", "0:26-22:0 @26..276"), goNextPassSet{source: goNextPass(0), compat: goNextPass(25), newMake: goNextPass(0)}) + goNextCheckRoute(t, "included-ranges", goTree, language, cTree, cDigest, "00c4d0aa190209d83625248618330de3cc1173c0e5e1dbc705b265fe9b79d09d", goNextDivergence("/source_file", "shape", "children=10", "children=7"), goNextPassSet{source: goNextPass(0), compat: goNextPass(25), newMake: goNextPass(0)}) if got := goTree.RootNode().Type(language); got != "source_file" { t.Fatalf("included-ranges Go root=%q", got) } diff --git a/cgo_harness/included_ranges_go_root_parity_cgo_test.go b/cgo_harness/included_ranges_go_root_parity_cgo_test.go index 19794131f..612fb1ec5 100644 --- a/cgo_harness/included_ranges_go_root_parity_cgo_test.go +++ b/cgo_harness/included_ranges_go_root_parity_cgo_test.go @@ -18,16 +18,15 @@ package cgoharness // geometries pinned below now agree with the C oracle on the root symbol // (before the fix, one of the four produced an ERROR root here where C // produced source_file), and the Go arm's root member no longer fires on any -// of them. The root span and child count still diverge from C on every -// geometry. +// of them. The producer anchors each tested root at its first included byte. +// Child counts and some trailing spans still diverge from C. // // READ THIS BEFORE CITING THIS TEST AS EVIDENCE. gotreesitter and the C // oracle do NOT agree on this route in general. They agree on the root symbol -// for all four geometries pinned here, and on the root span for exactly one: -// the geometry whose first range starts at byte 0 and whose last range ends -// at end of file. No geometry reaches child-count parity. This test pins -// each observation per geometry. It is a change detector for the route, not -// a parity certificate for it. +// for the four geometries pinned here. Their root start spans also match. +// The root end and child count still diverge on selected geometries. +// This test pins each observation per geometry. It is a change detector for +// the route, not a parity certificate for the complete tree. // // Run: GTS_PARITY_ALLOW_HOST=1 go test ./cgo_harness -tags treesitter_c_parity \ // -run TestIncludedRangesGo -v @@ -86,24 +85,21 @@ type includedRangesGeometry struct { // excluded bytes between two ranges as if they had to be whitespace, so a // non-whitespace gap between ranges no longer kills every GLR stack and // forces recovery. That is what padding_kills_stacks used to pin. Every -// geometry below now reaches a `source_file` root, and the Go arm's root -// member no longer fires on any of them, because none of them hands it an -// ERROR root to repair. The root span and child count still diverge from C -// on every geometry — a separate, pre-existing set of divergences this fix -// does not address. +// tested geometry reaches a `source_file` root. The Go arm's root member does +// not fire on these four cases because none supplies an ERROR root. Their root +// starts match C. Child counts and some trailing spans still differ. var includedRangesGoGeometries = []includedRangesGeometry{ { name: "interior_anchors_arm_live", spans: [2][2]int{{26, 150}, {203, 276}}, armRewrites: 0, c: includedRangesRootObservation{"source_file", 26, 276, 7, true}, - gts: includedRangesRootObservation{"source_file", 0, 276, 10, true}, + gts: includedRangesRootObservation{"source_file", 26, 276, 10, true}, note: "The realistic injection shape: both ranges start at a positive " + "offset. The root symbol matches C; the Go arm member does not " + - "fire, because the root already carries the right symbol before " + - "the member inspects it. The root span still diverges: " + - "gotreesitter starts at byte 0 while C starts at the first " + - "included range. The child count now diverges too (10 vs. 7): the " + + "fire, because the root already carries the right symbol and span " + + "before the member inspects it. The child count still diverges (10 " + + "vs. 7): the " + "parser folds both ranges into one pass instead of the recovery " + "reparse the old, unclipped scan used to force.", }, diff --git a/docs/root-normalization-retirement.md b/docs/root-normalization-retirement.md index bce8b77c0..55d68d68d 100644 --- a/docs/root-normalization-retirement.md +++ b/docs/root-normalization-retirement.md @@ -2965,38 +2965,59 @@ The included-ranges source has 276 bytes and SHA-256 The C digest is `9c8e5bb506bb345a577beb351f7b9230cca5e2e02cc4fd619e21f607657f290f`. The Go digest is -`8231fb8782e5699708e183619b68e1d030ab09e60de99ac52e3c092ffc56d59c`. +`00c4d0aa190209d83625248618330de3cc1173c0e5e1dbc705b265fe9b79d09d`. Both roots are `source_file` with an error flag. -Go uses range `0..276` with 10 children. -C uses range `26..276` with seven children. -The first difference is the root range. +Both roots use range `26..276`. +Go has 10 children, while C has seven children. +The first difference is the root child shape. The Go arm records zero root rewrites, 25 compatibility rewrites, and zero `new`/`make` rewrites on this route. +A byte-seek token source now reproduces a token that overlaps the first range +boundary. The parser starts its initial stack at that boundary. It also uses +the configured range point for an initial recovery gap. A source without byte +seeking preserves the complete overlapping token. Such a token can start +before the selected boundary. This limitation does not provide strict trim +parity for non-seek sources. + +The included-range production route now proves the root-start correction. +Focused unit coverage also checks the supported incremental API path. +Compact and forest included-range routes remain uncertified. Do not use this +receipt as compact, forest, or complete tree-parity evidence. + Every Go tree reports `NativeRecoveredStructureAuthoritative=false`. Forest returns nil when it declines and a non-nil tree when it accepts. The guard releases an unexpected tree before it fails. Incremental routes require positive old-tree reuse and reject unsupported reuse. The exact reuse values remain receipt evidence, not a brittle counter contract. -The route receipt is under -`/tmp/gts-n31j-go-current-artifacts/20260823T124728Z-current-go-route`. -Its `container.log` SHA-256 is -`da58f728f6ad9d42059abb3314c1aad2ccb65372d2a2d4fadcce971b054b7bf4`. -The document guard receipt is under -`/tmp/gts-n31j-go-current-artifacts/20260823T125100Z-current-go-document-first`. -Its `container.log` SHA-256 is -`f01faa740a06015d0ff19a831219e368699eb537ceb02f59bb4b058657b16af1`. -The run used one Go grammar, one CPU, 4 GiB, 512 process IDs, +- The focused Go receipt is under + `/tmp/gts-included-range-root-start-artifacts/20260824T232146Z-included-range-root-start-go`. + Its `container.log` SHA-256 is + `265dd56598ca4cda600b1ef627ef57dff1d0b57c3fcc8726bcadf7673e20c292`. +- The locked-C geometry and arm-guard receipt is under + `/tmp/gts-included-range-root-start-artifacts/20260824T232228Z-included-range-root-start-locked-c`. + Its `container.log` SHA-256 is + `5606b899c0abc3bb16639beb317090d20d1b207e60623d70fd4aafe8ad02f262`. +- The locked-C live-arm receipt is under + `/tmp/gts-included-range-root-start-artifacts/20260824T232309Z-included-range-root-start-go-next`. + Its `container.log` SHA-256 is + `72672bb50bf9e4564a01a35afaeaf59eb02dba00ade1c8d2c3a34943621d9e44`. + +Each run used one Go grammar, one CPU, 4 GiB, 512 process IDs, `GOMEMLIMIT=3GiB`, `GOMAXPROCS=1`, `GOFLAGS=-p=1`, `-parallel=1`, and a -20-minute timeout. It exited zero without timeout or out-of-memory failure. +20-minute timeout. All three runs exited zero without timeout or +out-of-memory failure. -No safe shared producer invariant was identified. +The first included byte is now a shared producer invariant for byte-seek +sources. This invariant fixes the root start only. Keep `dispatch.go` live until a producer emits exact C output for every authenticated witness and route. Require zero rewrites in all three subpasses. Require an authenticated Go corpus and source lock. Require included-ranges root, semicolon, sibling-boundary, and `new`/`make` -parity before retirement. Do not change production or registry state. +parity before retirement. Reopen `dispatch.go.source-file-root` retirement +only after exact child shape and production, compact, forest, and incremental +proofs pass. Do not change production or registry state. ## 2026-08-23 Doxygen dispatcher blocker receipt diff --git a/included_ranges.go b/included_ranges.go index fef40c1f3..a480ff788 100644 --- a/included_ranges.go +++ b/included_ranges.go @@ -12,9 +12,13 @@ func newIncludedRangeTokenSource(base TokenSource, ranges []Range) TokenSource { if base == nil || len(ranges) == 0 { return base } + ranges = normalizeIncludedRanges(ranges) + if len(ranges) == 0 { + return base + } return &includedRangeTokenSource{ base: base, - ranges: normalizeIncludedRanges(ranges), + ranges: ranges, } } @@ -190,6 +194,26 @@ func (s *includedRangeTokenSource) filterToken(tok Token, hasToken bool) Token { } r := s.ranges[s.idx] + if tok.StartByte < r.StartByte && tok.EndByte <= r.StartByte { + if skipper, ok := s.base.(ByteSkippableTokenSource); ok { + tok = skipper.SkipToByte(r.StartByte) + hasToken = true + } + continue + } + if tok.StartByte < r.StartByte { + // Re-seek a token that overlaps the selected boundary. A source + // with byte seeking can reproduce the token from that boundary. + if skipper, ok := s.base.(ByteSkippableTokenSource); ok { + tok = skipper.SkipToByte(r.StartByte) + hasToken = true + continue + } + // A source without byte seeking cannot reproduce a trimmed token. + // Preserve the complete overlap, which is the conservative existing + // behavior. Its start can remain before the selected boundary. + return tok + } if tok.EndByte <= r.StartByte { if skipper, ok := s.base.(ByteSkippableTokenSource); ok { tok = skipper.SkipToByte(r.StartByte) diff --git a/included_ranges_test.go b/included_ranges_test.go index dcc7e77e8..e0592be05 100644 --- a/included_ranges_test.go +++ b/included_ranges_test.go @@ -15,6 +15,20 @@ type stubTokenSource struct { relexCalls int } +type nextOnlyTokenSource struct { + tokens []Token + i int +} + +func (s *nextOnlyTokenSource) Next() Token { + if s.i >= len(s.tokens) { + return Token{} + } + tok := s.tokens[s.i] + s.i++ + return tok +} + func (s *stubTokenSource) Next() Token { s.nextCalls++ if s.i >= len(s.tokens) { @@ -80,6 +94,17 @@ func TestNormalizeIncludedRanges(t *testing.T) { } } +func TestIncludedRangeTokenSourceDropsAllEmptyRanges(t *testing.T) { + base := &nextOnlyTokenSource{tokens: []Token{{Symbol: 1, StartByte: 0, EndByte: 1}}} + ts := newIncludedRangeTokenSource(base, []Range{ + {StartByte: 3, EndByte: 3}, + {StartByte: 8, EndByte: 4}, + }) + if ts != base { + t.Fatalf("empty ranges returned %T, want the original token source", ts) + } +} + func TestIncludedRangeTokenSourceFiltersTokens(t *testing.T) { base := &stubTokenSource{ tokens: []Token{ @@ -101,6 +126,68 @@ func TestIncludedRangeTokenSourceFiltersTokens(t *testing.T) { } } +func TestIncludedRangeTokenSourceReseeksOverlappingTokenAtRangeStart(t *testing.T) { + base := &stubTokenSource{ + tokens: []Token{ + {Symbol: 1, StartByte: 0, EndByte: 5}, + {Symbol: 2, StartByte: 3, EndByte: 5}, + {Symbol: 3, StartByte: 12, EndByte: 15}, + {}, + }, + } + ts := newIncludedRangeTokenSource(base, []Range{{StartByte: 3, EndByte: 20}}).(*includedRangeTokenSource) + + tok := ts.Next() + if tok.Symbol != 2 { + t.Fatalf("overlapping token: got %d, want 2", tok.Symbol) + } +} + +func TestIncludedRangeTokenSourcePreservesOverlapWithoutReseek(t *testing.T) { + base := &nextOnlyTokenSource{tokens: []Token{ + {Symbol: 1, StartByte: 0, EndByte: 5}, + {Symbol: 2, StartByte: 8, EndByte: 10}, + {}, + }} + ts := newIncludedRangeTokenSource(base, []Range{{StartByte: 3, EndByte: 12}}).(*includedRangeTokenSource) + + tok := ts.Next() + if tok.Symbol != 1 || tok.StartByte != 0 || tok.EndByte != 5 { + t.Fatalf("overlapping token without seek: got %+v, want the original token", tok) + } +} + +func TestIncludedRangeTokenSourceSkipsExcludedTokenWithoutReseek(t *testing.T) { + base := &nextOnlyTokenSource{tokens: []Token{ + {Symbol: 1, StartByte: 0, EndByte: 2}, + {Symbol: 2, StartByte: 1, EndByte: 5}, + {Symbol: 3, StartByte: 8, EndByte: 10}, + {}, + }} + ts := newIncludedRangeTokenSource(base, []Range{{StartByte: 3, EndByte: 12}}).(*includedRangeTokenSource) + + tok := ts.Next() + if tok.Symbol != 2 || tok.StartByte != 1 || tok.EndByte != 5 { + t.Fatalf("first selected token = %+v, want the preserved overlap", tok) + } + if base.i != 2 { + t.Fatalf("base token index = %d, want 2 after skipping one excluded token", base.i) + } +} + +func TestInitialParseStackClampsIncludedRangeStartPastSource(t *testing.T) { + var scratch parserScratch + parser := &Parser{ + language: &Language{InitialState: 1}, + included: []Range{{StartByte: 100, EndByte: 200}}, + } + + stacks, _ := parser.newInitialParseStacks(&scratch, nil, nil, 12) + if got, want := stacks[0].byteOffset, uint32(12); got != want { + t.Fatalf("initial byte offset = %d, want clamped source end %d", got, want) + } +} + func TestIncludedRangeTokenSourceDelegatesParserState(t *testing.T) { base := &stubTokenSource{ tokens: []Token{{}}, diff --git a/parser.go b/parser.go index 333dc6f20..5d342c125 100644 --- a/parser.go +++ b/parser.go @@ -4086,6 +4086,27 @@ func (p *Parser) lineContinuationEscapeByte() byte { return languageLineContinuationEscapeByte(p.language) } +// parserStackEndPoint returns the parser-owned point at a stack's current +// byte offset. An included-range parse starts with an empty stack at the +// first selected byte, so its point comes from the range rather than from a +// stack entry that does not exist yet. +func (p *Parser) parserStackEndPoint(s *glrStack) Point { + if s == nil { + return Point{} + } + top := s.top() + if stackEntryHasNode(top) { + return stackEntryNodeEndPoint(top) + } + if p != nil && len(p.included) > 0 { + first := p.included[0] + if s.byteOffset == first.StartByte { + return first.StartPoint + } + } + return stackEntryNodeEndPoint(top) +} + // materializeSkippedGapAsExtraError covers a lexer-skipped mid-production gap // (a stray run of bytes immediately after an anonymous separator, where the // real lookahead continues the production via a single deterministic shift, @@ -4130,7 +4151,7 @@ func (p *Parser) materializeSkippedGapAsExtraError(s *glrStack, state StateID, t // this branch only after p.skippedRealGapContinuesSeparatedList already // returned true, and that function itself returns false for a nil p. p.markCRecoveryCostCompetitionRelevant() - startPoint := stackEntryNodeEndPoint(s.top()) + startPoint := p.parserStackEndPoint(s) leaf := newLeafNodeInArena(arena, errorSymbol, true, s.byteOffset, tok.StartByte, startPoint, tok.StartPoint) leaf.setHasError(true) leaf.setExtra(true) @@ -4174,7 +4195,7 @@ func (p *Parser) tryMaterializeSkippedRealGap(source []byte, s *glrStack, state return s.byteOffset == tok.StartByte } top := stackEntryNode(s.top()) - startPoint := stackEntryNodeEndPoint(s.top()) + startPoint := p.parserStackEndPoint(s) if top != nil && top.symbol == errorSymbol { if top.isMissing() || len(top.children) != 0 || @@ -7211,6 +7232,16 @@ func (p *Parser) newInitialParseStacks(scratch *parserScratch, reuse *reuseCurso initialStackCap = parseFullEntryScratchReservation(sourceLen) } stacks[0] = newGLRStackWithScratchCap(p.language.InitialState, &scratch.entries, initialStackCap) + // Included-range parsing starts at the first selected byte. Keep the + // initial stack offset aligned with the token source so a skipped prefix + // cannot become a parser-owned ERROR span before the first token. + if p != nil && len(p.included) > 0 { + start := p.included[0].StartByte + if uint64(start) > uint64(sourceLen) { + start = uint32(sourceLen) + } + stacks[0].byteOffset = start + } stacks[0].recoverabilityKnown = true stacks[0].mayRecover = p.stateCanRecover(p.language.InitialState) if timing != nil && timing.maxStacksSeen < len(stacks) { diff --git a/parser_included_ranges_go_root_test.go b/parser_included_ranges_go_root_test.go index 1e9349e54..f378d14a8 100644 --- a/parser_included_ranges_go_root_test.go +++ b/parser_included_ranges_go_root_test.go @@ -1,6 +1,7 @@ package gotreesitter_test import ( + "bytes" "os" "testing" @@ -105,6 +106,111 @@ func TestIncludedRangesGoRootStaysSourceFile(t *testing.T) { if root.IsError() { t.Fatal("included-ranges root is an ERROR node") } + if got, want := root.StartByte(), ranges[0].StartByte; got != want { + t.Fatalf("included-ranges root start = %d, want first included byte %d", got, want) + } + if got, want := root.StartPoint(), ranges[0].StartPoint; got != want { + t.Fatalf("included-ranges root start point = %+v, want %+v", got, want) + } +} + +func TestIncludedRangesGoRecoveryKeepsSelectedStart(t *testing.T) { + lang := grammars.GoLanguage() + if lang == nil { + t.Skip("go grammar unavailable") + } + src, ranges := loadIncludedRangesGoFixture(t) + parser := gotreesitter.NewParser(lang) + parser.SetIncludedRanges(ranges) + + tree, err := parser.Parse(src) + if err != nil { + t.Fatalf("included-ranges recovery parse: %v", err) + } + if tree == nil || tree.RootNode() == nil { + t.Fatal("included-ranges recovery parse returned no tree") + } + defer tree.Release() + + root := tree.RootNode() + if !root.HasError() { + t.Fatal("included-ranges recovery root has no error") + } + if got, want := root.StartByte(), ranges[0].StartByte; got != want { + t.Fatalf("recovery root start = %d, want %d", got, want) + } + if got, want := root.StartPoint(), ranges[0].StartPoint; got != want { + t.Fatalf("recovery root start point = %+v, want %+v", got, want) + } +} + +func TestIncludedRangesGoIncrementalKeepsSelectedStart(t *testing.T) { + lang := grammars.GoLanguage() + if lang == nil { + t.Skip("go grammar unavailable") + } + src := []byte("// host prefix\npackage p\n\nvar value = 1\n") + start := bytes.Index(src, []byte("package")) + if start < 0 { + t.Fatal("included-range start token is absent") + } + ranges := []gotreesitter.Range{{ + StartByte: uint32(start), + EndByte: uint32(len(src)), + StartPoint: includedRangesGoPointAt(src, start), + EndPoint: includedRangesGoPointAt(src, len(src)), + }} + editAt := bytes.Index(src, []byte("1\n")) + if editAt < 0 { + t.Fatal("incremental edit token is absent") + } + edited := append([]byte(nil), src...) + edited[editAt] = '2' + + parser := gotreesitter.NewParser(lang) + parser.SetIncludedRanges(ranges) + oldTree, err := parser.Parse(src) + if err != nil { + t.Fatalf("included-ranges base parse: %v", err) + } + if oldTree == nil { + t.Fatal("included-ranges base parse returned no tree") + } + defer oldTree.Release() + if oldTree.RootNode().HasError() { + t.Fatal("included-ranges base parse has an error") + } + + startPoint := includedRangesGoPointAt(src, editAt) + endPoint := includedRangesGoPointAt(src, editAt+1) + oldTree.Edit(gotreesitter.InputEdit{ + StartByte: uint32(editAt), + OldEndByte: uint32(editAt + 1), + NewEndByte: uint32(editAt + 1), + StartPoint: startPoint, + OldEndPoint: endPoint, + NewEndPoint: endPoint, + }) + + tree, profile, err := parser.ParseIncrementalProfiled(edited, oldTree) + if err != nil { + t.Fatalf("included-ranges incremental parse: %v", err) + } + if tree == nil || tree.RootNode() == nil { + t.Fatal("included-ranges incremental parse returned no tree") + } + defer tree.Release() + if profile.ReuseUnsupported { + t.Fatalf("included-ranges incremental reuse unsupported: %s", profile.ReuseUnsupportedReason) + } + + root := tree.RootNode() + if got, want := root.StartByte(), ranges[0].StartByte; got != want { + t.Fatalf("incremental root start = %d, want %d", got, want) + } + if got, want := root.StartPoint(), ranges[0].StartPoint; got != want { + t.Fatalf("incremental root start point = %+v, want %+v", got, want) + } } // TestIncludedRangesGoArmMemberNowInert records the current state of the