diff --git a/diagnostic_token_source_fork.go b/diagnostic_token_source_fork.go new file mode 100644 index 000000000..bd1da46cf --- /dev/null +++ b/diagnostic_token_source_fork.go @@ -0,0 +1,88 @@ +//go:build gts_workcount + +package gotreesitter + +import ( + "crypto/sha256" + "encoding/binary" + "fmt" +) + +// DiagnosticTokenSourceForkReceipt records the state at a diagnostic fork. +// Production builds do not include this type. +type DiagnosticTokenSourceForkReceipt struct { + TokenSourceKind string + CursorByte uint32 + CursorPoint Point + ParserState StateID + ActiveGLRStates []StateID + PendingTokenCount int + PendingTokenDigest [sha256.Size]byte + ZeroWidthOffset int64 + IncludedRangeIndex int + IncludedRangeCount int + IncludedRangeDigest [sha256.Size]byte + ExternalScannerCheckpointPresent bool + ExternalScannerCheckpointDigest [sha256.Size]byte +} + +type diagnosticTokenSourceForker interface { + ForkTokenSourceForDiagnostic() (TokenSource, DiagnosticTokenSourceForkReceipt, error) +} + +func forkTokenSourceForDiagnostic(source TokenSource) (TokenSource, DiagnosticTokenSourceForkReceipt, error) { + if source == nil { + return nil, DiagnosticTokenSourceForkReceipt{}, fmt.Errorf("diagnostic token fork: source is nil") + } + forker, ok := source.(diagnosticTokenSourceForker) + if !ok { + return nil, DiagnosticTokenSourceForkReceipt{}, fmt.Errorf("diagnostic token fork: source %T cannot fork", source) + } + return forker.ForkTokenSourceForDiagnostic() +} + +func (s *includedRangeTokenSource) ForkTokenSourceForDiagnostic() (TokenSource, DiagnosticTokenSourceForkReceipt, error) { + if s == nil || s.base == nil { + return nil, DiagnosticTokenSourceForkReceipt{}, fmt.Errorf("diagnostic token fork: included-range source is nil") + } + if s.idx < 0 || s.idx > len(s.ranges) { + return nil, DiagnosticTokenSourceForkReceipt{}, fmt.Errorf("diagnostic token fork: included-range index %d is outside 0..%d", s.idx, len(s.ranges)) + } + + base, receipt, err := forkTokenSourceForDiagnostic(s.base) + if err != nil { + return nil, DiagnosticTokenSourceForkReceipt{}, err + } + ranges := append([]Range(nil), s.ranges...) + fork := &includedRangeTokenSource{ + base: base, + ranges: ranges, + idx: s.idx, + } + receipt.TokenSourceKind = "included-range/" + receipt.TokenSourceKind + receipt.IncludedRangeIndex = s.idx + receipt.IncludedRangeCount = len(s.ranges) + receipt.IncludedRangeDigest = diagnosticIncludedRangeDigest(s.ranges) + return fork, receipt, nil +} + +func diagnosticIncludedRangeDigest(ranges []Range) [sha256.Size]byte { + hash := sha256.New() + var value [8]byte + write := func(v uint64) { + binary.LittleEndian.PutUint64(value[:], v) + _, _ = hash.Write(value[:]) + } + write(uint64(len(ranges))) + for _, item := range ranges { + write(uint64(item.StartByte)) + write(uint64(item.EndByte)) + write(uint64(item.StartPoint.Row)) + write(uint64(item.StartPoint.Column)) + write(uint64(item.EndPoint.Row)) + write(uint64(item.EndPoint.Column)) + } + var digest [sha256.Size]byte + copy(digest[:], hash.Sum(nil)) + return digest +} diff --git a/diagnostic_token_source_fork_test.go b/diagnostic_token_source_fork_test.go new file mode 100644 index 000000000..38261bb52 --- /dev/null +++ b/diagnostic_token_source_fork_test.go @@ -0,0 +1,165 @@ +//go:build gts_workcount + +package gotreesitter + +import ( + "crypto/sha256" + "reflect" + "testing" +) + +type diagnosticForkStub struct { + tokens []Token + index int + state StateID + glrStates []StateID +} + +func (s *diagnosticForkStub) Next() Token { + if s.index >= len(s.tokens) { + return Token{} + } + token := s.tokens[s.index] + s.index++ + return token +} + +func (s *diagnosticForkStub) SetParserState(state StateID) { + s.state = state +} + +func (s *diagnosticForkStub) SetGLRStates(states []StateID) { + s.glrStates = append(s.glrStates[:0], states...) +} + +func (s *diagnosticForkStub) ForkTokenSourceForDiagnostic() (TokenSource, DiagnosticTokenSourceForkReceipt, error) { + fork := *s + fork.tokens = append([]Token(nil), s.tokens...) + fork.glrStates = append([]StateID(nil), s.glrStates...) + return &fork, DiagnosticTokenSourceForkReceipt{ + TokenSourceKind: "stub", + ParserState: s.state, + ActiveGLRStates: append([]StateID(nil), s.glrStates...), + IncludedRangeIndex: -1, + ExternalScannerCheckpointPresent: true, + ExternalScannerCheckpointDigest: sha256.Sum256([]byte("stub scanner")), + }, nil +} + +func TestDiagnosticIncludedRangeForkIsIndependent(t *testing.T) { + base := &diagnosticForkStub{ + tokens: []Token{ + {Symbol: 1, Text: "outside", StartByte: 0, EndByte: 7}, + {Symbol: 2, Text: "first", StartByte: 12, EndByte: 17}, + {Symbol: 3, Text: "second", StartByte: 31, EndByte: 37}, + {Symbol: 4, Text: "third", StartByte: 38, EndByte: 39}, + {}, + }, + state: 77, + glrStates: []StateID{7, 11}, + } + ranges := []Range{ + {StartByte: 10, EndByte: 20, StartPoint: Point{Column: 10}, EndPoint: Point{Column: 20}}, + {StartByte: 30, EndByte: 40, StartPoint: Point{Row: 1}, EndPoint: Point{Row: 1, Column: 10}}, + } + live := newIncludedRangeTokenSource(base, ranges).(*includedRangeTokenSource) + if token := live.Next(); token.Symbol != 2 { + t.Fatalf("first included token symbol = %d, want 2", token.Symbol) + } + if live.idx != 0 { + t.Fatalf("live range index = %d, want 0", live.idx) + } + + liveBaseIndex := base.index + liveRanges := append([]Range(nil), live.ranges...) + forkSource, receipt, err := forkTokenSourceForDiagnostic(live) + if err != nil { + t.Fatalf("forkTokenSourceForDiagnostic failed: %v", err) + } + fork, ok := forkSource.(*includedRangeTokenSource) + if !ok { + t.Fatalf("fork type = %T, want *includedRangeTokenSource", forkSource) + } + forkBase, ok := fork.base.(*diagnosticForkStub) + if !ok { + t.Fatalf("fork base type = %T, want *diagnosticForkStub", fork.base) + } + if !reflect.DeepEqual(fork, live) { + t.Fatalf("fork state differs before advance:\n got: %#v\nwant: %#v", fork, live) + } + if fork == live || forkBase == base { + t.Fatal("fork shares a mutable source object with the live source") + } + if &fork.ranges[0] == &live.ranges[0] { + t.Fatal("fork shares the included-range backing array") + } + if &forkBase.tokens[0] == &base.tokens[0] { + t.Fatal("fork shares the base token backing array") + } + if &forkBase.glrStates[0] == &base.glrStates[0] { + t.Fatal("fork shares the base GLR-state backing array") + } + if got, want := receipt.TokenSourceKind, "included-range/stub"; got != want { + t.Fatalf("receipt kind = %q, want %q", got, want) + } + if receipt.IncludedRangeIndex != live.idx || receipt.IncludedRangeCount != len(live.ranges) { + t.Fatalf("receipt range state = %d/%d, want %d/%d", receipt.IncludedRangeIndex, receipt.IncludedRangeCount, live.idx, len(live.ranges)) + } + if receipt.ParserState != base.state { + t.Fatalf("receipt parser state = %d, want %d", receipt.ParserState, base.state) + } + if !reflect.DeepEqual(receipt.ActiveGLRStates, base.glrStates) { + t.Fatalf("receipt GLR states = %v, want %v", receipt.ActiveGLRStates, base.glrStates) + } + if got, want := receipt.IncludedRangeDigest, diagnosticIncludedRangeDigest(live.ranges); got != want { + t.Fatalf("receipt range digest = %x, want %x", got, want) + } + if !receipt.ExternalScannerCheckpointPresent { + t.Fatal("wrapper discarded the base scanner checkpoint state") + } + + fork.SetParserState(99) + fork.SetGLRStates([]StateID{13, 17}) + if base.state != 77 || !reflect.DeepEqual(base.glrStates, []StateID{7, 11}) { + t.Fatalf("fork state update changed live base: state=%d glr=%v", base.state, base.glrStates) + } + forkToken := fork.Next() + if forkToken.Symbol != 3 || fork.idx != 1 { + t.Fatalf("fork next state = symbol %d at range %d, want symbol 3 at range 1", forkToken.Symbol, fork.idx) + } + if base.index != liveBaseIndex || live.idx != 0 || !reflect.DeepEqual(live.ranges, liveRanges) { + t.Fatalf("fork advance changed live state: base index=%d range index=%d ranges=%v", base.index, live.idx, live.ranges) + } + if liveToken := live.Next(); !reflect.DeepEqual(liveToken, forkToken) { + t.Fatalf("live next token = %+v, want fork token %+v", liveToken, forkToken) + } + + liveBaseIndex = base.index + forkToken = fork.Next() + if forkToken.Symbol != 4 { + t.Fatalf("fork next symbol = %d, want 4", forkToken.Symbol) + } + if base.index != liveBaseIndex || live.idx != 1 { + t.Fatalf("second fork advance changed live state: base index=%d range index=%d", base.index, live.idx) + } + if liveToken := live.Next(); !reflect.DeepEqual(liveToken, forkToken) { + t.Fatalf("second live token = %+v, want fork token %+v", liveToken, forkToken) + } + + fork.ranges[0].EndByte++ + if reflect.DeepEqual(fork.ranges, live.ranges) { + t.Fatal("fork range mutation reached the live wrapper") + } +} + +func TestDiagnosticIncludedRangeForkRejectsUnsupportedBase(t *testing.T) { + base := &stubTokenSource{} + live := newIncludedRangeTokenSource(base, []Range{{StartByte: 1, EndByte: 2}}).(*includedRangeTokenSource) + indexBefore := live.idx + if _, _, err := forkTokenSourceForDiagnostic(live); err == nil { + t.Fatal("fork with an unsupported base succeeded") + } + if base.nextCalls != 0 || base.skipCalls != 0 || live.idx != indexBefore { + t.Fatalf("rejected fork changed live source: next=%d skip=%d range index=%d, want 0/0/%d", base.nextCalls, base.skipCalls, live.idx, indexBefore) + } +} diff --git a/grammars/c_token_source_diagnostic_fork.go b/grammars/c_token_source_diagnostic_fork.go new file mode 100644 index 000000000..a0cf39f80 --- /dev/null +++ b/grammars/c_token_source_diagnostic_fork.go @@ -0,0 +1,101 @@ +//go:build gts_workcount && (!grammar_subset || grammar_subset_c || grammar_subset_cpp) + +package grammars + +import ( + "crypto/sha256" + "encoding/binary" + "fmt" + "hash" + + "github.com/odvcencio/gotreesitter" +) + +// ForkTokenSourceForDiagnostic copies the current C token-source state. +// The fork shares only immutable source bytes, language data, and lexer tables. +func (ts *CTokenSource) ForkTokenSourceForDiagnostic() (gotreesitter.TokenSource, gotreesitter.DiagnosticTokenSourceForkReceipt, error) { + if ts == nil { + return nil, gotreesitter.DiagnosticTokenSourceForkReceipt{}, fmt.Errorf("diagnostic C token fork: source is nil") + } + if ts.cur.offset < 0 || ts.cur.offset > len(ts.cur.src) { + return nil, gotreesitter.DiagnosticTokenSourceForkReceipt{}, fmt.Errorf("diagnostic C token fork: cursor %d is outside source length %d", ts.cur.offset, len(ts.cur.src)) + } + if uint64(ts.cur.offset) > uint64(^uint32(0)) { + return nil, gotreesitter.DiagnosticTokenSourceForkReceipt{}, fmt.Errorf("diagnostic C token fork: cursor %d exceeds the token offset limit", ts.cur.offset) + } + if len(ts.src) != len(ts.cur.src) || (len(ts.src) > 0 && &ts.src[0] != &ts.cur.src[0]) { + return nil, gotreesitter.DiagnosticTokenSourceForkReceipt{}, fmt.Errorf("diagnostic C token fork: source cursor does not share the immutable source") + } + + fork := *ts + fork.pending = diagnosticCloneCTokens(ts.pending) + fork.glrStates = diagnosticCloneCStates(ts.glrStates) + + receipt := gotreesitter.DiagnosticTokenSourceForkReceipt{ + TokenSourceKind: "c", + CursorByte: uint32(ts.cur.offset), + CursorPoint: ts.cur.point(), + ParserState: ts.parserState, + ActiveGLRStates: diagnosticCloneCStates(ts.glrStates), + PendingTokenCount: len(ts.pending), + PendingTokenDigest: diagnosticCTokenDigest(ts.pending), + ZeroWidthOffset: int64(ts.lastSyntheticOffset), + IncludedRangeIndex: -1, + } + return &fork, receipt, nil +} + +func diagnosticCloneCTokens(tokens []gotreesitter.Token) []gotreesitter.Token { + if tokens == nil { + return nil + } + clone := make([]gotreesitter.Token, len(tokens)) + copy(clone, tokens) + return clone +} + +func diagnosticCloneCStates(states []gotreesitter.StateID) []gotreesitter.StateID { + if states == nil { + return nil + } + clone := make([]gotreesitter.StateID, len(states)) + copy(clone, states) + return clone +} + +func diagnosticCTokenDigest(tokens []gotreesitter.Token) [sha256.Size]byte { + digest := sha256.New() + diagnosticWriteUint64(digest, uint64(len(tokens))) + for _, token := range tokens { + diagnosticWriteUint64(digest, uint64(token.Symbol)) + diagnosticWriteUint64(digest, uint64(len(token.Text))) + _, _ = digest.Write([]byte(token.Text)) + diagnosticWriteUint64(digest, uint64(token.StartByte)) + diagnosticWriteUint64(digest, uint64(token.EndByte)) + diagnosticWriteUint64(digest, uint64(token.StartPoint.Row)) + diagnosticWriteUint64(digest, uint64(token.StartPoint.Column)) + diagnosticWriteUint64(digest, uint64(token.EndPoint.Row)) + diagnosticWriteUint64(digest, uint64(token.EndPoint.Column)) + diagnosticWriteBool(digest, token.Missing) + diagnosticWriteBool(digest, token.NoLookahead) + diagnosticWriteBool(digest, token.ExternalScannerToken) + diagnosticWriteUint64(digest, uint64(token.ExternalScannerStartByte)) + } + var out [sha256.Size]byte + copy(out[:], digest.Sum(nil)) + return out +} + +func diagnosticWriteUint64(digest hash.Hash, value uint64) { + var encoded [8]byte + binary.LittleEndian.PutUint64(encoded[:], value) + _, _ = digest.Write(encoded[:]) +} + +func diagnosticWriteBool(digest hash.Hash, value bool) { + if value { + _, _ = digest.Write([]byte{1}) + return + } + _, _ = digest.Write([]byte{0}) +} diff --git a/grammars/c_token_source_diagnostic_fork_test.go b/grammars/c_token_source_diagnostic_fork_test.go new file mode 100644 index 000000000..b51bbd7f8 --- /dev/null +++ b/grammars/c_token_source_diagnostic_fork_test.go @@ -0,0 +1,308 @@ +//go:build gts_workcount && (!grammar_subset || grammar_subset_c || grammar_subset_cpp) + +package grammars + +import ( + "bytes" + "crypto/sha256" + "reflect" + "testing" + + "github.com/odvcencio/gotreesitter" +) + +type cDiagnosticMutableState struct { + cursor sourceCursor + done bool + pending []gotreesitter.Token + preprocState int + parserState gotreesitter.StateID + glrStates []gotreesitter.StateID + lastSyntheticOffset int + preprocDefineNameEnd int + preprocOpaqueArgPending bool + preprocOpaqueArgActive bool +} + +func captureCDiagnosticMutableState(source *CTokenSource) cDiagnosticMutableState { + return cDiagnosticMutableState{ + cursor: source.cur, + done: source.done, + pending: append([]gotreesitter.Token(nil), source.pending...), + preprocState: source.preprocState, + parserState: source.parserState, + glrStates: append([]gotreesitter.StateID(nil), source.glrStates...), + lastSyntheticOffset: source.lastSyntheticOffset, + preprocDefineNameEnd: source.preprocDefineNameEnd, + preprocOpaqueArgPending: source.preprocOpaqueArgPending, + preprocOpaqueArgActive: source.preprocOpaqueArgActive, + } +} + +func assertCDiagnosticStateEqual(t *testing.T, got, want cDiagnosticMutableState) { + t.Helper() + if !reflect.DeepEqual(got, want) { + t.Fatalf("C token-source state changed:\n got: %#v\nwant: %#v", got, want) + } +} + +func assertCDiagnosticForkStateEqual(t *testing.T, fork, live *CTokenSource) { + t.Helper() + if !reflect.DeepEqual(fork, live) { + t.Fatalf("C fork state differs before advance:\n got: %#v\nwant: %#v", fork, live) + } +} + +func advanceCDiagnosticTokens(source *CTokenSource, count int) []gotreesitter.Token { + out := make([]gotreesitter.Token, count) + for index := range out { + out[index] = source.Next() + } + return out +} + +func requireCDiagnosticTokenSequenceEqual(t *testing.T, got, want []gotreesitter.Token) { + t.Helper() + if !reflect.DeepEqual(got, want) { + t.Fatalf("token sequence differs:\n got: %#v\nwant: %#v", got, want) + } +} + +func TestCTokenSourceDiagnosticForkPreservesPendingTokens(t *testing.T) { + lang := CLanguage() + sourceBytes := []byte("const char *value = \"a\\n\"; int tail;\n") + live, err := NewCTokenSource(sourceBytes, lang) + if err != nil { + t.Fatalf("NewCTokenSource failed: %v", err) + } + t.Cleanup(live.Close) + live.SetParserState(101) + live.SetGLRStates([]gotreesitter.StateID{4, 8, 15}) + + foundOpener := false + for index := 0; index < 12; index++ { + if token := live.Next(); token.Text == "\"" { + foundOpener = true + break + } + } + if !foundOpener { + t.Fatal("setup did not reach a string opener") + } + if len(live.pending) < 2 { + t.Fatalf("pending token count = %d, want at least 2", len(live.pending)) + } + liveBefore := captureCDiagnosticMutableState(live) + sourceBefore := append([]byte(nil), live.src...) + + forkSource, receipt, err := live.ForkTokenSourceForDiagnostic() + if err != nil { + t.Fatalf("ForkTokenSourceForDiagnostic failed: %v", err) + } + fork, ok := forkSource.(*CTokenSource) + if !ok { + t.Fatalf("fork type = %T, want *CTokenSource", forkSource) + } + t.Cleanup(fork.Close) + assertCDiagnosticForkStateEqual(t, fork, live) + if fork == live { + t.Fatal("fork shares the live C token-source object") + } + if &fork.pending[0] == &live.pending[0] { + t.Fatal("fork shares the pending-token backing array") + } + if &fork.glrStates[0] == &live.glrStates[0] { + t.Fatal("fork shares the GLR-state backing array") + } + if &receipt.ActiveGLRStates[0] == &live.glrStates[0] || &receipt.ActiveGLRStates[0] == &fork.glrStates[0] { + t.Fatal("receipt shares a mutable GLR-state backing array") + } + if got, want := receipt.TokenSourceKind, "c"; got != want { + t.Fatalf("receipt kind = %q, want %q", got, want) + } + if receipt.CursorByte != uint32(live.cur.offset) || receipt.CursorPoint != live.cur.point() { + t.Fatalf("receipt cursor = %d/%+v, want %d/%+v", receipt.CursorByte, receipt.CursorPoint, live.cur.offset, live.cur.point()) + } + if receipt.ParserState != live.parserState { + t.Fatalf("receipt parser state = %d, want %d", receipt.ParserState, live.parserState) + } + if !reflect.DeepEqual(receipt.ActiveGLRStates, live.glrStates) { + t.Fatalf("receipt GLR states = %v, want %v", receipt.ActiveGLRStates, live.glrStates) + } + if receipt.PendingTokenCount != len(live.pending) || receipt.PendingTokenDigest != diagnosticCTokenDigest(live.pending) { + t.Fatalf("receipt pending state = %d/%x, want %d/%x", receipt.PendingTokenCount, receipt.PendingTokenDigest, len(live.pending), diagnosticCTokenDigest(live.pending)) + } + if receipt.ExternalScannerCheckpointPresent || receipt.ExternalScannerCheckpointDigest != [sha256.Size]byte{} { + t.Fatalf("C receipt reports external scanner state: present=%v digest=%x", receipt.ExternalScannerCheckpointPresent, receipt.ExternalScannerCheckpointDigest) + } + + forkTokens := advanceCDiagnosticTokens(fork, 7) + assertCDiagnosticStateEqual(t, captureCDiagnosticMutableState(live), liveBefore) + if !bytes.Equal(live.src, sourceBefore) { + t.Fatalf("fork advance changed live source bytes: got %q, want %q", live.src, sourceBefore) + } + liveTokens := advanceCDiagnosticTokens(live, 7) + requireCDiagnosticTokenSequenceEqual(t, forkTokens, liveTokens) + + fork.glrStates[0] = 99 + receipt.ActiveGLRStates[0] = 100 + if live.glrStates[0] != 4 { + t.Fatalf("fork or receipt mutation changed live GLR state to %d", live.glrStates[0]) + } +} + +func TestCTokenSourceDiagnosticForkPreservesPreprocessorState(t *testing.T) { + lang := CLanguage() + sourceBytes := []byte("#if __has_include()\nint enabled;\n") + live, err := NewCTokenSource(sourceBytes, lang) + if err != nil { + t.Fatalf("NewCTokenSource failed: %v", err) + } + t.Cleanup(live.Close) + + setup := advanceCDiagnosticTokens(live, 3) + if got, want := setup[2].Text, "("; got != want { + t.Fatalf("third setup token text = %q, want %q", got, want) + } + if live.preprocState != cPreprocConditionalExpr || !live.preprocOpaqueArgActive || live.preprocOpaqueArgPending { + t.Fatalf("setup preprocessor state = %d/%v/%v", live.preprocState, live.preprocOpaqueArgPending, live.preprocOpaqueArgActive) + } + liveBefore := captureCDiagnosticMutableState(live) + + forkSource, _, err := live.ForkTokenSourceForDiagnostic() + if err != nil { + t.Fatalf("ForkTokenSourceForDiagnostic failed: %v", err) + } + fork := forkSource.(*CTokenSource) + t.Cleanup(fork.Close) + assertCDiagnosticForkStateEqual(t, fork, live) + if fork.preprocState != live.preprocState || fork.preprocOpaqueArgPending != live.preprocOpaqueArgPending || fork.preprocOpaqueArgActive != live.preprocOpaqueArgActive { + t.Fatalf("fork preprocessor state = %d/%v/%v, want %d/%v/%v", fork.preprocState, fork.preprocOpaqueArgPending, fork.preprocOpaqueArgActive, live.preprocState, live.preprocOpaqueArgPending, live.preprocOpaqueArgActive) + } + + forkTokens := advanceCDiagnosticTokens(fork, 6) + assertCDiagnosticStateEqual(t, captureCDiagnosticMutableState(live), liveBefore) + liveTokens := advanceCDiagnosticTokens(live, 6) + requireCDiagnosticTokenSequenceEqual(t, forkTokens, liveTokens) +} + +func TestCTokenSourceDiagnosticForkPreservesDefinitionBoundary(t *testing.T) { + lang := CLanguage() + sourceBytes := []byte("#define FLAG(x) x\nint tail;\n") + live, err := NewCTokenSource(sourceBytes, lang) + if err != nil { + t.Fatalf("NewCTokenSource failed: %v", err) + } + t.Cleanup(live.Close) + + setup := advanceCDiagnosticTokens(live, 2) + if got, want := setup[1].Text, "FLAG"; got != want { + t.Fatalf("second setup token text = %q, want %q", got, want) + } + if live.preprocState != cPreprocAfterDefineName || live.preprocDefineNameEnd != live.cur.offset { + t.Fatalf("definition boundary state = %d/%d, want %d/%d", live.preprocState, live.preprocDefineNameEnd, cPreprocAfterDefineName, live.cur.offset) + } + liveBefore := captureCDiagnosticMutableState(live) + + forkSource, _, err := live.ForkTokenSourceForDiagnostic() + if err != nil { + t.Fatalf("ForkTokenSourceForDiagnostic failed: %v", err) + } + fork := forkSource.(*CTokenSource) + t.Cleanup(fork.Close) + assertCDiagnosticForkStateEqual(t, fork, live) + if fork.preprocDefineNameEnd != live.preprocDefineNameEnd { + t.Fatalf("fork definition boundary = %d, want %d", fork.preprocDefineNameEnd, live.preprocDefineNameEnd) + } + + forkTokens := advanceCDiagnosticTokens(fork, 6) + assertCDiagnosticStateEqual(t, captureCDiagnosticMutableState(live), liveBefore) + liveTokens := advanceCDiagnosticTokens(live, 6) + requireCDiagnosticTokenSequenceEqual(t, forkTokens, liveTokens) +} + +func TestCTokenSourceDiagnosticForkPreservesZeroWidthState(t *testing.T) { + lang := CLanguage() + sourceBytes := []byte("#ifdef __cplusplus\nextern \"C\" {\n#endif\n\nint x;\n\n#ifdef __cplusplus\n}\n#endif\n") + live, err := NewCTokenSource(sourceBytes, lang) + if err != nil { + t.Fatalf("NewCTokenSource failed: %v", err) + } + t.Cleanup(live.Close) + live.cur.advanceBytes(66) + live.SetParserState(10) + liveBefore := captureCDiagnosticMutableState(live) + + forkSource, receipt, err := live.ForkTokenSourceForDiagnostic() + if err != nil { + t.Fatalf("ForkTokenSourceForDiagnostic failed: %v", err) + } + fork := forkSource.(*CTokenSource) + t.Cleanup(fork.Close) + assertCDiagnosticForkStateEqual(t, fork, live) + if got, want := receipt.ZeroWidthOffset, int64(-1); got != want { + t.Fatalf("receipt zero-width offset = %d, want %d", got, want) + } + + forkSynthetic := fork.Next() + if forkSynthetic.Symbol != live.endifSymbol || forkSynthetic.StartByte != 66 || forkSynthetic.EndByte != 66 || !forkSynthetic.Missing { + t.Fatalf("fork synthetic token = %+v, want a missing #endif at byte 66", forkSynthetic) + } + assertCDiagnosticStateEqual(t, captureCDiagnosticMutableState(live), liveBefore) + liveSynthetic := live.Next() + if !reflect.DeepEqual(liveSynthetic, forkSynthetic) { + t.Fatalf("live synthetic token = %+v, want %+v", liveSynthetic, forkSynthetic) + } + + guardedForkSource, guardedReceipt, err := live.ForkTokenSourceForDiagnostic() + if err != nil { + t.Fatalf("guarded ForkTokenSourceForDiagnostic failed: %v", err) + } + guardedFork := guardedForkSource.(*CTokenSource) + t.Cleanup(guardedFork.Close) + assertCDiagnosticForkStateEqual(t, guardedFork, live) + if got, want := guardedReceipt.ZeroWidthOffset, int64(66); got != want { + t.Fatalf("guarded receipt zero-width offset = %d, want %d", got, want) + } + liveBeforeGuardedAdvance := captureCDiagnosticMutableState(live) + forkNext := guardedFork.Next() + assertCDiagnosticStateEqual(t, captureCDiagnosticMutableState(live), liveBeforeGuardedAdvance) + if liveNext := live.Next(); !reflect.DeepEqual(liveNext, forkNext) { + t.Fatalf("guarded live token = %+v, want fork token %+v", liveNext, forkNext) + } +} + +func TestCTokenSourceDiagnosticForkPreservesDoneState(t *testing.T) { + lang := CLanguage() + live, err := NewCTokenSource([]byte("int"), lang) + if err != nil { + t.Fatalf("NewCTokenSource failed: %v", err) + } + t.Cleanup(live.Close) + _ = live.Next() + _ = live.Next() + if !live.done { + t.Fatal("setup source is not done") + } + liveBefore := captureCDiagnosticMutableState(live) + + forkSource, receipt, err := live.ForkTokenSourceForDiagnostic() + if err != nil { + t.Fatalf("ForkTokenSourceForDiagnostic failed: %v", err) + } + fork := forkSource.(*CTokenSource) + t.Cleanup(fork.Close) + assertCDiagnosticForkStateEqual(t, fork, live) + if !fork.done { + t.Fatal("fork did not preserve the done state") + } + if receipt.CursorByte != uint32(len(live.src)) || receipt.CursorPoint != (gotreesitter.Point{Column: 3}) { + t.Fatalf("done receipt cursor = %d/%+v", receipt.CursorByte, receipt.CursorPoint) + } + + forkToken := fork.Next() + assertCDiagnosticStateEqual(t, captureCDiagnosticMutableState(live), liveBefore) + if liveToken := live.Next(); !reflect.DeepEqual(liveToken, forkToken) { + t.Fatalf("done live token = %+v, want fork token %+v", liveToken, forkToken) + } +}