Archive the C8 token fork work

diagnostic_token_source_fork.go

@@ -0,0 +1,88 @@
+//go:build gts_workcount
+
+package gotreesitter
+
+import (
+	"crypto/sha256"
+	"encoding/binary"
+	"fmt"
+)
+
+// DiagnosticTokenSourceForkReceipt records the state at a diagnostic fork.
+// Production builds do not include this type.
+type DiagnosticTokenSourceForkReceipt struct {
+	TokenSourceKind                  string
+	CursorByte                       uint32
+	CursorPoint                      Point
+	ParserState                      StateID
+	ActiveGLRStates                  []StateID
+	PendingTokenCount                int
+	PendingTokenDigest               [sha256.Size]byte
+	ZeroWidthOffset                  int64
+	IncludedRangeIndex               int
+	IncludedRangeCount               int
+	IncludedRangeDigest              [sha256.Size]byte
+	ExternalScannerCheckpointPresent bool
+	ExternalScannerCheckpointDigest  [sha256.Size]byte
+}
+
+type diagnosticTokenSourceForker interface {
+	ForkTokenSourceForDiagnostic() (TokenSource, DiagnosticTokenSourceForkReceipt, error)
+}
+
+func forkTokenSourceForDiagnostic(source TokenSource) (TokenSource, DiagnosticTokenSourceForkReceipt, error) {
+	if source == nil {
+		return nil, DiagnosticTokenSourceForkReceipt{}, fmt.Errorf("diagnostic token fork: source is nil")
+	}
+	forker, ok := source.(diagnosticTokenSourceForker)
+	if !ok {
+		return nil, DiagnosticTokenSourceForkReceipt{}, fmt.Errorf("diagnostic token fork: source %T cannot fork", source)
+	}
+	return forker.ForkTokenSourceForDiagnostic()
+}
+
+func (s *includedRangeTokenSource) ForkTokenSourceForDiagnostic() (TokenSource, DiagnosticTokenSourceForkReceipt, error) {
+	if s == nil || s.base == nil {
+		return nil, DiagnosticTokenSourceForkReceipt{}, fmt.Errorf("diagnostic token fork: included-range source is nil")
+	}
+	if s.idx < 0 || s.idx > len(s.ranges) {
+		return nil, DiagnosticTokenSourceForkReceipt{}, fmt.Errorf("diagnostic token fork: included-range index %d is outside 0..%d", s.idx, len(s.ranges))
+	}
+
+	base, receipt, err := forkTokenSourceForDiagnostic(s.base)
+	if err != nil {
+		return nil, DiagnosticTokenSourceForkReceipt{}, err
+	}
+	ranges := append([]Range(nil), s.ranges...)
+	fork := &includedRangeTokenSource{
+		base:   base,
+		ranges: ranges,
+		idx:    s.idx,
+	}
+	receipt.TokenSourceKind = "included-range/" + receipt.TokenSourceKind
+	receipt.IncludedRangeIndex = s.idx
+	receipt.IncludedRangeCount = len(s.ranges)
+	receipt.IncludedRangeDigest = diagnosticIncludedRangeDigest(s.ranges)
+	return fork, receipt, nil
+}
+
+func diagnosticIncludedRangeDigest(ranges []Range) [sha256.Size]byte {
+	hash := sha256.New()
+	var value [8]byte
+	write := func(v uint64) {
+		binary.LittleEndian.PutUint64(value[:], v)
+		_, _ = hash.Write(value[:])
+	}
+	write(uint64(len(ranges)))
+	for _, item := range ranges {
+		write(uint64(item.StartByte))
+		write(uint64(item.EndByte))
+		write(uint64(item.StartPoint.Row))
+		write(uint64(item.StartPoint.Column))
+		write(uint64(item.EndPoint.Row))
+		write(uint64(item.EndPoint.Column))
+	}
+	var digest [sha256.Size]byte
+	copy(digest[:], hash.Sum(nil))
+	return digest
+}

diagnostic_token_source_fork_test.go

@@ -0,0 +1,165 @@
+//go:build gts_workcount
+
+package gotreesitter
+
+import (
+	"crypto/sha256"
+	"reflect"
+	"testing"
+)
+
+type diagnosticForkStub struct {
+	tokens    []Token
+	index     int
+	state     StateID
+	glrStates []StateID
+}
+
+func (s *diagnosticForkStub) Next() Token {
+	if s.index >= len(s.tokens) {
+		return Token{}
+	}
+	token := s.tokens[s.index]
+	s.index++
+	return token
+}
+
+func (s *diagnosticForkStub) SetParserState(state StateID) {
+	s.state = state
+}
+
+func (s *diagnosticForkStub) SetGLRStates(states []StateID) {
+	s.glrStates = append(s.glrStates[:0], states...)
+}
+
+func (s *diagnosticForkStub) ForkTokenSourceForDiagnostic() (TokenSource, DiagnosticTokenSourceForkReceipt, error) {
+	fork := *s
+	fork.tokens = append([]Token(nil), s.tokens...)
+	fork.glrStates = append([]StateID(nil), s.glrStates...)
+	return &fork, DiagnosticTokenSourceForkReceipt{
+		TokenSourceKind:                  "stub",
+		ParserState:                      s.state,
+		ActiveGLRStates:                  append([]StateID(nil), s.glrStates...),
+		IncludedRangeIndex:               -1,
+		ExternalScannerCheckpointPresent: true,
+		ExternalScannerCheckpointDigest:  sha256.Sum256([]byte("stub scanner")),
+	}, nil
+}
+
+func TestDiagnosticIncludedRangeForkIsIndependent(t *testing.T) {
+	base := &diagnosticForkStub{
+		tokens: []Token{
+			{Symbol: 1, Text: "outside", StartByte: 0, EndByte: 7},
+			{Symbol: 2, Text: "first", StartByte: 12, EndByte: 17},
+			{Symbol: 3, Text: "second", StartByte: 31, EndByte: 37},
+			{Symbol: 4, Text: "third", StartByte: 38, EndByte: 39},
+			{},
+		},
+		state:     77,
+		glrStates: []StateID{7, 11},
+	}
+	ranges := []Range{
+		{StartByte: 10, EndByte: 20, StartPoint: Point{Column: 10}, EndPoint: Point{Column: 20}},
+		{StartByte: 30, EndByte: 40, StartPoint: Point{Row: 1}, EndPoint: Point{Row: 1, Column: 10}},
+	}
+	live := newIncludedRangeTokenSource(base, ranges).(*includedRangeTokenSource)
+	if token := live.Next(); token.Symbol != 2 {
+		t.Fatalf("first included token symbol = %d, want 2", token.Symbol)
+	}
+	if live.idx != 0 {
+		t.Fatalf("live range index = %d, want 0", live.idx)
+	}
+
+	liveBaseIndex := base.index
+	liveRanges := append([]Range(nil), live.ranges...)
+	forkSource, receipt, err := forkTokenSourceForDiagnostic(live)
+	if err != nil {
+		t.Fatalf("forkTokenSourceForDiagnostic failed: %v", err)
+	}
+	fork, ok := forkSource.(*includedRangeTokenSource)
+	if !ok {
+		t.Fatalf("fork type = %T, want *includedRangeTokenSource", forkSource)
+	}
+	forkBase, ok := fork.base.(*diagnosticForkStub)
+	if !ok {
+		t.Fatalf("fork base type = %T, want *diagnosticForkStub", fork.base)
+	}
+	if !reflect.DeepEqual(fork, live) {
+		t.Fatalf("fork state differs before advance:\n got: %#v\nwant: %#v", fork, live)
+	}
+	if fork == live || forkBase == base {
+		t.Fatal("fork shares a mutable source object with the live source")
+	}
+	if &fork.ranges[0] == &live.ranges[0] {
+		t.Fatal("fork shares the included-range backing array")
+	}
+	if &forkBase.tokens[0] == &base.tokens[0] {
+		t.Fatal("fork shares the base token backing array")
+	}
+	if &forkBase.glrStates[0] == &base.glrStates[0] {
+		t.Fatal("fork shares the base GLR-state backing array")
+	}
+	if got, want := receipt.TokenSourceKind, "included-range/stub"; got != want {
+		t.Fatalf("receipt kind = %q, want %q", got, want)
+	}
+	if receipt.IncludedRangeIndex != live.idx || receipt.IncludedRangeCount != len(live.ranges) {
+		t.Fatalf("receipt range state = %d/%d, want %d/%d", receipt.IncludedRangeIndex, receipt.IncludedRangeCount, live.idx, len(live.ranges))
+	}
+	if receipt.ParserState != base.state {
+		t.Fatalf("receipt parser state = %d, want %d", receipt.ParserState, base.state)
+	}
+	if !reflect.DeepEqual(receipt.ActiveGLRStates, base.glrStates) {
+		t.Fatalf("receipt GLR states = %v, want %v", receipt.ActiveGLRStates, base.glrStates)
+	}
+	if got, want := receipt.IncludedRangeDigest, diagnosticIncludedRangeDigest(live.ranges); got != want {
+		t.Fatalf("receipt range digest = %x, want %x", got, want)
+	}
+	if !receipt.ExternalScannerCheckpointPresent {
+		t.Fatal("wrapper discarded the base scanner checkpoint state")
+	}
+
+	fork.SetParserState(99)
+	fork.SetGLRStates([]StateID{13, 17})
+	if base.state != 77 || !reflect.DeepEqual(base.glrStates, []StateID{7, 11}) {
+		t.Fatalf("fork state update changed live base: state=%d glr=%v", base.state, base.glrStates)
+	}
+	forkToken := fork.Next()
+	if forkToken.Symbol != 3 || fork.idx != 1 {
+		t.Fatalf("fork next state = symbol %d at range %d, want symbol 3 at range 1", forkToken.Symbol, fork.idx)
+	}
+	if base.index != liveBaseIndex || live.idx != 0 || !reflect.DeepEqual(live.ranges, liveRanges) {
+		t.Fatalf("fork advance changed live state: base index=%d range index=%d ranges=%v", base.index, live.idx, live.ranges)
+	}
+	if liveToken := live.Next(); !reflect.DeepEqual(liveToken, forkToken) {
+		t.Fatalf("live next token = %+v, want fork token %+v", liveToken, forkToken)
+	}
+
+	liveBaseIndex = base.index
+	forkToken = fork.Next()
+	if forkToken.Symbol != 4 {
+		t.Fatalf("fork next symbol = %d, want 4", forkToken.Symbol)
+	}
+	if base.index != liveBaseIndex || live.idx != 1 {
+		t.Fatalf("second fork advance changed live state: base index=%d range index=%d", base.index, live.idx)
+	}
+	if liveToken := live.Next(); !reflect.DeepEqual(liveToken, forkToken) {
+		t.Fatalf("second live token = %+v, want fork token %+v", liveToken, forkToken)
+	}
+
+	fork.ranges[0].EndByte++
+	if reflect.DeepEqual(fork.ranges, live.ranges) {
+		t.Fatal("fork range mutation reached the live wrapper")
+	}
+}
+
+func TestDiagnosticIncludedRangeForkRejectsUnsupportedBase(t *testing.T) {
+	base := &stubTokenSource{}
+	live := newIncludedRangeTokenSource(base, []Range{{StartByte: 1, EndByte: 2}}).(*includedRangeTokenSource)
+	indexBefore := live.idx
+	if _, _, err := forkTokenSourceForDiagnostic(live); err == nil {
+		t.Fatal("fork with an unsupported base succeeded")
+	}
+	if base.nextCalls != 0 || base.skipCalls != 0 || live.idx != indexBefore {
+		t.Fatalf("rejected fork changed live source: next=%d skip=%d range index=%d, want 0/0/%d", base.nextCalls, base.skipCalls, live.idx, indexBefore)
+	}
+}

grammars/c_token_source_diagnostic_fork.go

@@ -0,0 +1,101 @@
+//go:build gts_workcount && (!grammar_subset || grammar_subset_c || grammar_subset_cpp)
+
+package grammars
+
+import (
+	"crypto/sha256"
+	"encoding/binary"
+	"fmt"
+	"hash"
+
+	"github.com/odvcencio/gotreesitter"
+)
+
+// ForkTokenSourceForDiagnostic copies the current C token-source state.
+// The fork shares only immutable source bytes, language data, and lexer tables.
+func (ts *CTokenSource) ForkTokenSourceForDiagnostic() (gotreesitter.TokenSource, gotreesitter.DiagnosticTokenSourceForkReceipt, error) {
+	if ts == nil {
+		return nil, gotreesitter.DiagnosticTokenSourceForkReceipt{}, fmt.Errorf("diagnostic C token fork: source is nil")
+	}
+	if ts.cur.offset < 0 || ts.cur.offset > len(ts.cur.src) {
+		return nil, gotreesitter.DiagnosticTokenSourceForkReceipt{}, fmt.Errorf("diagnostic C token fork: cursor %d is outside source length %d", ts.cur.offset, len(ts.cur.src))
+	}
+	if uint64(ts.cur.offset) > uint64(^uint32(0)) {
+		return nil, gotreesitter.DiagnosticTokenSourceForkReceipt{}, fmt.Errorf("diagnostic C token fork: cursor %d exceeds the token offset limit", ts.cur.offset)
+	}
+	if len(ts.src) != len(ts.cur.src) || (len(ts.src) > 0 && &ts.src[0] != &ts.cur.src[0]) {
+		return nil, gotreesitter.DiagnosticTokenSourceForkReceipt{}, fmt.Errorf("diagnostic C token fork: source cursor does not share the immutable source")
+	}
+
+	fork := *ts
+	fork.pending = diagnosticCloneCTokens(ts.pending)
+	fork.glrStates = diagnosticCloneCStates(ts.glrStates)
+
+	receipt := gotreesitter.DiagnosticTokenSourceForkReceipt{
+		TokenSourceKind:    "c",
+		CursorByte:         uint32(ts.cur.offset),
+		CursorPoint:        ts.cur.point(),
+		ParserState:        ts.parserState,
+		ActiveGLRStates:    diagnosticCloneCStates(ts.glrStates),
+		PendingTokenCount:  len(ts.pending),
+		PendingTokenDigest: diagnosticCTokenDigest(ts.pending),
+		ZeroWidthOffset:    int64(ts.lastSyntheticOffset),
+		IncludedRangeIndex: -1,
+	}
+	return &fork, receipt, nil
+}
+
+func diagnosticCloneCTokens(tokens []gotreesitter.Token) []gotreesitter.Token {
+	if tokens == nil {
+		return nil
+	}
+	clone := make([]gotreesitter.Token, len(tokens))
+	copy(clone, tokens)
+	return clone
+}
+
+func diagnosticCloneCStates(states []gotreesitter.StateID) []gotreesitter.StateID {
+	if states == nil {
+		return nil
+	}
+	clone := make([]gotreesitter.StateID, len(states))
+	copy(clone, states)
+	return clone
+}
+
+func diagnosticCTokenDigest(tokens []gotreesitter.Token) [sha256.Size]byte {
+	digest := sha256.New()
+	diagnosticWriteUint64(digest, uint64(len(tokens)))
+	for _, token := range tokens {
+		diagnosticWriteUint64(digest, uint64(token.Symbol))
+		diagnosticWriteUint64(digest, uint64(len(token.Text)))
+		_, _ = digest.Write([]byte(token.Text))
+		diagnosticWriteUint64(digest, uint64(token.StartByte))
+		diagnosticWriteUint64(digest, uint64(token.EndByte))
+		diagnosticWriteUint64(digest, uint64(token.StartPoint.Row))
+		diagnosticWriteUint64(digest, uint64(token.StartPoint.Column))
+		diagnosticWriteUint64(digest, uint64(token.EndPoint.Row))
+		diagnosticWriteUint64(digest, uint64(token.EndPoint.Column))
+		diagnosticWriteBool(digest, token.Missing)
+		diagnosticWriteBool(digest, token.NoLookahead)
+		diagnosticWriteBool(digest, token.ExternalScannerToken)
+		diagnosticWriteUint64(digest, uint64(token.ExternalScannerStartByte))
+	}
+	var out [sha256.Size]byte
+	copy(out[:], digest.Sum(nil))
+	return out
+}
+
+func diagnosticWriteUint64(digest hash.Hash, value uint64) {
+	var encoded [8]byte
+	binary.LittleEndian.PutUint64(encoded[:], value)
+	_, _ = digest.Write(encoded[:])
+}
+
+func diagnosticWriteBool(digest hash.Hash, value bool) {
+	if value {
+		_, _ = digest.Write([]byte{1})
+		return
+	}
+	_, _ = digest.Write([]byte{0})
+}