Start included-range roots at the selected byte

CHANGELOG.md

@@ -91,6 +91,15 @@ for tags and release notes while still in `0.x`.
 
 ### Correctness
 
+- Included-range parsing now starts byte-seek sources at the first selected
+  byte. The parser seeds its initial stack there and preserves the configured
+  start point for recovery gaps. A non-seek source preserves a complete
+  overlapping token because it cannot reproduce a trimmed boundary token.
+  The locked Go route now matches C at root byte 26. Child counts remain 10
+  for Go and seven for C. Compact and forest included-range routes remain
+  uncertified. Keep `dispatch.go.source-file-root` live until exact shape and
+  all required route proofs pass. See `docs/root-normalization-retirement.md`.
+
 - Recorded a generic Swift issue #576 recovery candidate at base
   `da6f71471aaaa835503accaa1bc2083ced90b4e6`. The active deterministic finite
   automaton (DFA) source now replays recovery from the exact skipped-prefix

cgo_harness/go_next_live_probe_test.go

@@ -311,7 +311,7 @@ func goNextRunIncludedRanges(t *testing.T, language *gotreesitter.Language, cLan
 	if cDigest != "9c8e5bb506bb345a577beb351f7b9230cca5e2e02cc4fd619e21f607657f290f" {
 		t.Fatalf("included-ranges locked-C digest=%s", cDigest)
 	}
-	goNextCheckRoute(t, "included-ranges", goTree, language, cTree, cDigest, "8231fb8782e5699708e183619b68e1d030ab09e60de99ac52e3c092ffc56d59c", goNextDivergence("/source_file", "range", "0:0-22:0 @0..276", "0:26-22:0 @26..276"), goNextPassSet{source: goNextPass(0), compat: goNextPass(25), newMake: goNextPass(0)})
+	goNextCheckRoute(t, "included-ranges", goTree, language, cTree, cDigest, "00c4d0aa190209d83625248618330de3cc1173c0e5e1dbc705b265fe9b79d09d", goNextDivergence("/source_file", "shape", "children=10", "children=7"), goNextPassSet{source: goNextPass(0), compat: goNextPass(25), newMake: goNextPass(0)})
 	if got := goTree.RootNode().Type(language); got != "source_file" {
 		t.Fatalf("included-ranges Go root=%q", got)
 	}

cgo_harness/included_ranges_go_root_parity_cgo_test.go

@@ -18,16 +18,15 @@ package cgoharness
 // geometries pinned below now agree with the C oracle on the root symbol
 // (before the fix, one of the four produced an ERROR root here where C
 // produced source_file), and the Go arm's root member no longer fires on any
-// of them. The root span and child count still diverge from C on every
-// geometry.
+// of them. The producer anchors each tested root at its first included byte.
+// Child counts and some trailing spans still diverge from C.
 //
 // READ THIS BEFORE CITING THIS TEST AS EVIDENCE. gotreesitter and the C
 // oracle do NOT agree on this route in general. They agree on the root symbol
-// for all four geometries pinned here, and on the root span for exactly one:
-// the geometry whose first range starts at byte 0 and whose last range ends
-// at end of file. No geometry reaches child-count parity. This test pins
-// each observation per geometry. It is a change detector for the route, not
-// a parity certificate for it.
+// for the four geometries pinned here. Their root start spans also match.
+// The root end and child count still diverge on selected geometries.
+// This test pins each observation per geometry. It is a change detector for
+// the route, not a parity certificate for the complete tree.
 //
 // Run: GTS_PARITY_ALLOW_HOST=1 go test ./cgo_harness -tags treesitter_c_parity \
 //        -run TestIncludedRangesGo -v
@@ -86,24 +85,21 @@ type includedRangesGeometry struct {
 // excluded bytes between two ranges as if they had to be whitespace, so a
 // non-whitespace gap between ranges no longer kills every GLR stack and
 // forces recovery. That is what padding_kills_stacks used to pin. Every
-// geometry below now reaches a `source_file` root, and the Go arm's root
-// member no longer fires on any of them, because none of them hands it an
-// ERROR root to repair. The root span and child count still diverge from C
-// on every geometry — a separate, pre-existing set of divergences this fix
-// does not address.
+// tested geometry reaches a `source_file` root. The Go arm's root member does
+// not fire on these four cases because none supplies an ERROR root. Their root
+// starts match C. Child counts and some trailing spans still differ.
 var includedRangesGoGeometries = []includedRangesGeometry{
 	{
 		name:        "interior_anchors_arm_live",
 		spans:       [2][2]int{{26, 150}, {203, 276}},
 		armRewrites: 0,
 		c:           includedRangesRootObservation{"source_file", 26, 276, 7, true},
-		gts:         includedRangesRootObservation{"source_file", 0, 276, 10, true},
+		gts:         includedRangesRootObservation{"source_file", 26, 276, 10, true},
 		note: "The realistic injection shape: both ranges start at a positive " +
 			"offset. The root symbol matches C; the Go arm member does not " +
-			"fire, because the root already carries the right symbol before " +
-			"the member inspects it. The root span still diverges: " +
-			"gotreesitter starts at byte 0 while C starts at the first " +
-			"included range. The child count now diverges too (10 vs. 7): the " +
+			"fire, because the root already carries the right symbol and span " +
+			"before the member inspects it. The child count still diverges (10 " +
+			"vs. 7): the " +
 			"parser folds both ranges into one pass instead of the recovery " +
 			"reparse the old, unclipped scan used to force.",
 	},

docs/root-normalization-retirement.md

@@ -2965,38 +2965,59 @@ The included-ranges source has 276 bytes and SHA-256
 The C digest is
 `9c8e5bb506bb345a577beb351f7b9230cca5e2e02cc4fd619e21f607657f290f`.
 The Go digest is
-`8231fb8782e5699708e183619b68e1d030ab09e60de99ac52e3c092ffc56d59c`.
+`00c4d0aa190209d83625248618330de3cc1173c0e5e1dbc705b265fe9b79d09d`.
 Both roots are `source_file` with an error flag.
-Go uses range `0..276` with 10 children.
-C uses range `26..276` with seven children.
-The first difference is the root range.
+Both roots use range `26..276`.
+Go has 10 children, while C has seven children.
+The first difference is the root child shape.
 The Go arm records zero root rewrites, 25 compatibility rewrites, and zero
 `new`/`make` rewrites on this route.
 
+A byte-seek token source now reproduces a token that overlaps the first range
+boundary. The parser starts its initial stack at that boundary. It also uses
+the configured range point for an initial recovery gap. A source without byte
+seeking preserves the complete overlapping token. Such a token can start
+before the selected boundary. This limitation does not provide strict trim
+parity for non-seek sources.
+
+The included-range production route now proves the root-start correction.
+Focused unit coverage also checks the supported incremental API path.
+Compact and forest included-range routes remain uncertified. Do not use this
+receipt as compact, forest, or complete tree-parity evidence.
+
 Every Go tree reports `NativeRecoveredStructureAuthoritative=false`.
 Forest returns nil when it declines and a non-nil tree when it accepts.
 The guard releases an unexpected tree before it fails.
 Incremental routes require positive old-tree reuse and reject unsupported reuse.
 The exact reuse values remain receipt evidence, not a brittle counter contract.
 
-The route receipt is under
-`/tmp/gts-n31j-go-current-artifacts/20260823T124728Z-current-go-route`.
-Its `container.log` SHA-256 is
-`da58f728f6ad9d42059abb3314c1aad2ccb65372d2a2d4fadcce971b054b7bf4`.
-The document guard receipt is under
-`/tmp/gts-n31j-go-current-artifacts/20260823T125100Z-current-go-document-first`.
-Its `container.log` SHA-256 is
-`f01faa740a06015d0ff19a831219e368699eb537ceb02f59bb4b058657b16af1`.
-The run used one Go grammar, one CPU, 4 GiB, 512 process IDs,
+- The focused Go receipt is under
+  `/tmp/gts-included-range-root-start-artifacts/20260824T232146Z-included-range-root-start-go`.
+  Its `container.log` SHA-256 is
+  `265dd56598ca4cda600b1ef627ef57dff1d0b57c3fcc8726bcadf7673e20c292`.
+- The locked-C geometry and arm-guard receipt is under
+  `/tmp/gts-included-range-root-start-artifacts/20260824T232228Z-included-range-root-start-locked-c`.
+  Its `container.log` SHA-256 is
+  `5606b899c0abc3bb16639beb317090d20d1b207e60623d70fd4aafe8ad02f262`.
+- The locked-C live-arm receipt is under
+  `/tmp/gts-included-range-root-start-artifacts/20260824T232309Z-included-range-root-start-go-next`.
+  Its `container.log` SHA-256 is
+  `72672bb50bf9e4564a01a35afaeaf59eb02dba00ade1c8d2c3a34943621d9e44`.
+
+Each run used one Go grammar, one CPU, 4 GiB, 512 process IDs,
 `GOMEMLIMIT=3GiB`, `GOMAXPROCS=1`, `GOFLAGS=-p=1`, `-parallel=1`, and a
-20-minute timeout. It exited zero without timeout or out-of-memory failure.
+20-minute timeout. All three runs exited zero without timeout or
+out-of-memory failure.
 
-No safe shared producer invariant was identified.
+The first included byte is now a shared producer invariant for byte-seek
+sources. This invariant fixes the root start only.
 Keep `dispatch.go` live until a producer emits exact C output for every
 authenticated witness and route. Require zero rewrites in all three subpasses.
 Require an authenticated Go corpus and source lock.
 Require included-ranges root, semicolon, sibling-boundary, and `new`/`make`
-parity before retirement. Do not change production or registry state.
+parity before retirement. Reopen `dispatch.go.source-file-root` retirement
+only after exact child shape and production, compact, forest, and incremental
+proofs pass. Do not change production or registry state.
 
 ## 2026-08-23 Doxygen dispatcher blocker receipt
 

included_ranges.go

@@ -12,9 +12,13 @@ func newIncludedRangeTokenSource(base TokenSource, ranges []Range) TokenSource {
 	if base == nil || len(ranges) == 0 {
 		return base
 	}
+	ranges = normalizeIncludedRanges(ranges)
+	if len(ranges) == 0 {
+		return base
+	}
 	return &includedRangeTokenSource{
 		base:   base,
-		ranges: normalizeIncludedRanges(ranges),
+		ranges: ranges,
 	}
 }
 
@@ -190,6 +194,26 @@ func (s *includedRangeTokenSource) filterToken(tok Token, hasToken bool) Token {
 		}
 
 		r := s.ranges[s.idx]
+		if tok.StartByte < r.StartByte && tok.EndByte <= r.StartByte {
+			if skipper, ok := s.base.(ByteSkippableTokenSource); ok {
+				tok = skipper.SkipToByte(r.StartByte)
+				hasToken = true
+			}
+			continue
+		}
+		if tok.StartByte < r.StartByte {
+			// Re-seek a token that overlaps the selected boundary. A source
+			// with byte seeking can reproduce the token from that boundary.
+			if skipper, ok := s.base.(ByteSkippableTokenSource); ok {
+				tok = skipper.SkipToByte(r.StartByte)
+				hasToken = true
+				continue
+			}
+			// A source without byte seeking cannot reproduce a trimmed token.
+			// Preserve the complete overlap, which is the conservative existing
+			// behavior. Its start can remain before the selected boundary.
+			return tok
+		}
 		if tok.EndByte <= r.StartByte {
 			if skipper, ok := s.base.(ByteSkippableTokenSource); ok {
 				tok = skipper.SkipToByte(r.StartByte)

included_ranges_test.go

@@ -15,6 +15,20 @@ type stubTokenSource struct {
 	relexCalls int
 }
 
+type nextOnlyTokenSource struct {
+	tokens []Token
+	i      int
+}
+
+func (s *nextOnlyTokenSource) Next() Token {
+	if s.i >= len(s.tokens) {
+		return Token{}
+	}
+	tok := s.tokens[s.i]
+	s.i++
+	return tok
+}
+
 func (s *stubTokenSource) Next() Token {
 	s.nextCalls++
 	if s.i >= len(s.tokens) {
@@ -80,6 +94,17 @@ func TestNormalizeIncludedRanges(t *testing.T) {
 	}
 }
 
+func TestIncludedRangeTokenSourceDropsAllEmptyRanges(t *testing.T) {
+	base := &nextOnlyTokenSource{tokens: []Token{{Symbol: 1, StartByte: 0, EndByte: 1}}}
+	ts := newIncludedRangeTokenSource(base, []Range{
+		{StartByte: 3, EndByte: 3},
+		{StartByte: 8, EndByte: 4},
+	})
+	if ts != base {
+		t.Fatalf("empty ranges returned %T, want the original token source", ts)
+	}
+}
+
 func TestIncludedRangeTokenSourceFiltersTokens(t *testing.T) {
 	base := &stubTokenSource{
 		tokens: []Token{
@@ -101,6 +126,68 @@ func TestIncludedRangeTokenSourceFiltersTokens(t *testing.T) {
 	}
 }
 
+func TestIncludedRangeTokenSourceReseeksOverlappingTokenAtRangeStart(t *testing.T) {
+	base := &stubTokenSource{
+		tokens: []Token{
+			{Symbol: 1, StartByte: 0, EndByte: 5},
+			{Symbol: 2, StartByte: 3, EndByte: 5},
+			{Symbol: 3, StartByte: 12, EndByte: 15},
+			{},
+		},
+	}
+	ts := newIncludedRangeTokenSource(base, []Range{{StartByte: 3, EndByte: 20}}).(*includedRangeTokenSource)
+
+	tok := ts.Next()
+	if tok.Symbol != 2 {
+		t.Fatalf("overlapping token: got %d, want 2", tok.Symbol)
+	}
+}
+
+func TestIncludedRangeTokenSourcePreservesOverlapWithoutReseek(t *testing.T) {
+	base := &nextOnlyTokenSource{tokens: []Token{
+		{Symbol: 1, StartByte: 0, EndByte: 5},
+		{Symbol: 2, StartByte: 8, EndByte: 10},
+		{},
+	}}
+	ts := newIncludedRangeTokenSource(base, []Range{{StartByte: 3, EndByte: 12}}).(*includedRangeTokenSource)
+
+	tok := ts.Next()
+	if tok.Symbol != 1 || tok.StartByte != 0 || tok.EndByte != 5 {
+		t.Fatalf("overlapping token without seek: got %+v, want the original token", tok)
+	}
+}
+
+func TestIncludedRangeTokenSourceSkipsExcludedTokenWithoutReseek(t *testing.T) {
+	base := &nextOnlyTokenSource{tokens: []Token{
+		{Symbol: 1, StartByte: 0, EndByte: 2},
+		{Symbol: 2, StartByte: 1, EndByte: 5},
+		{Symbol: 3, StartByte: 8, EndByte: 10},
+		{},
+	}}
+	ts := newIncludedRangeTokenSource(base, []Range{{StartByte: 3, EndByte: 12}}).(*includedRangeTokenSource)
+
+	tok := ts.Next()
+	if tok.Symbol != 2 || tok.StartByte != 1 || tok.EndByte != 5 {
+		t.Fatalf("first selected token = %+v, want the preserved overlap", tok)
+	}
+	if base.i != 2 {
+		t.Fatalf("base token index = %d, want 2 after skipping one excluded token", base.i)
+	}
+}
+
+func TestInitialParseStackClampsIncludedRangeStartPastSource(t *testing.T) {
+	var scratch parserScratch
+	parser := &Parser{
+		language: &Language{InitialState: 1},
+		included: []Range{{StartByte: 100, EndByte: 200}},
+	}
+
+	stacks, _ := parser.newInitialParseStacks(&scratch, nil, nil, 12)
+	if got, want := stacks[0].byteOffset, uint32(12); got != want {
+		t.Fatalf("initial byte offset = %d, want clamped source end %d", got, want)
+	}
+}
+
 func TestIncludedRangeTokenSourceDelegatesParserState(t *testing.T) {
 	base := &stubTokenSource{
 		tokens: []Token{{}},

parser.go

@@ -4086,6 +4086,27 @@ func (p *Parser) lineContinuationEscapeByte() byte {
 	return languageLineContinuationEscapeByte(p.language)
 }
 
+// parserStackEndPoint returns the parser-owned point at a stack's current
+// byte offset. An included-range parse starts with an empty stack at the
+// first selected byte, so its point comes from the range rather than from a
+// stack entry that does not exist yet.
+func (p *Parser) parserStackEndPoint(s *glrStack) Point {
+	if s == nil {
+		return Point{}
+	}
+	top := s.top()
+	if stackEntryHasNode(top) {
+		return stackEntryNodeEndPoint(top)
+	}
+	if p != nil && len(p.included) > 0 {
+		first := p.included[0]
+		if s.byteOffset == first.StartByte {
+			return first.StartPoint
+		}
+	}
+	return stackEntryNodeEndPoint(top)
+}
+
 // materializeSkippedGapAsExtraError covers a lexer-skipped mid-production gap
 // (a stray run of bytes immediately after an anonymous separator, where the
 // real lookahead continues the production via a single deterministic shift,
@@ -4130,7 +4151,7 @@ func (p *Parser) materializeSkippedGapAsExtraError(s *glrStack, state StateID, t
 	// this branch only after p.skippedRealGapContinuesSeparatedList already
 	// returned true, and that function itself returns false for a nil p.
 	p.markCRecoveryCostCompetitionRelevant()
-	startPoint := stackEntryNodeEndPoint(s.top())
+	startPoint := p.parserStackEndPoint(s)
 	leaf := newLeafNodeInArena(arena, errorSymbol, true, s.byteOffset, tok.StartByte, startPoint, tok.StartPoint)
 	leaf.setHasError(true)
 	leaf.setExtra(true)
@@ -4174,7 +4195,7 @@ func (p *Parser) tryMaterializeSkippedRealGap(source []byte, s *glrStack, state
 		return s.byteOffset == tok.StartByte
 	}
 	top := stackEntryNode(s.top())
-	startPoint := stackEntryNodeEndPoint(s.top())
+	startPoint := p.parserStackEndPoint(s)
 	if top != nil && top.symbol == errorSymbol {
 		if top.isMissing() ||
 			len(top.children) != 0 ||
@@ -7211,6 +7232,16 @@ func (p *Parser) newInitialParseStacks(scratch *parserScratch, reuse *reuseCurso
 		initialStackCap = parseFullEntryScratchReservation(sourceLen)
 	}
 	stacks[0] = newGLRStackWithScratchCap(p.language.InitialState, &scratch.entries, initialStackCap)
+	// Included-range parsing starts at the first selected byte. Keep the
+	// initial stack offset aligned with the token source so a skipped prefix
+	// cannot become a parser-owned ERROR span before the first token.
+	if p != nil && len(p.included) > 0 {
+		start := p.included[0].StartByte
+		if uint64(start) > uint64(sourceLen) {
+			start = uint32(sourceLen)
+		}
+		stacks[0].byteOffset = start
+	}
 	stacks[0].recoverabilityKnown = true
 	stacks[0].mayRecover = p.stateCanRecover(p.language.InitialState)
 	if timing != nil && timing.maxStacksSeen < len(stacks) {