diff --git a/.gitignore b/.gitignore index 75f0aa7..56a6874 100644 --- a/.gitignore +++ b/.gitignore @@ -372,3 +372,7 @@ src/.vscode/launch.json .idea/.idea.TextMateSharp.dir/.idea/indexLayout.xml .idea/.idea.TextMateSharp.dir/.idea/vcs.xml .idea/.idea.TextMateSharp/.idea/riderMarkupCache.xml +.idea/.idea.TextMateSharp/.idea/copilot.data.migration.edit.xml +.idea/.idea.TextMateSharp/.idea/copilot.data.migration.ask2agent.xml +.idea/.idea.TextMateSharp/.idea/copilot.data.migration.ask.xml +.idea/.idea.TextMateSharp/.idea/copilot.data.migration.agent.xml diff --git a/build/Directory.Build.props b/build/Directory.Build.props index d2bc446..7d9da74 100644 --- a/build/Directory.Build.props +++ b/build/Directory.Build.props @@ -3,6 +3,6 @@ latest true 8.0.5 - 1.0.8 + 1.0.9 diff --git a/src/TextMateSharp.Grammars.Tests/TextMateSharp.Grammars.Tests.csproj b/src/TextMateSharp.Grammars.Tests/TextMateSharp.Grammars.Tests.csproj index 449f714..3db4421 100644 --- a/src/TextMateSharp.Grammars.Tests/TextMateSharp.Grammars.Tests.csproj +++ b/src/TextMateSharp.Grammars.Tests/TextMateSharp.Grammars.Tests.csproj @@ -1,7 +1,7 @@ - net6.0 + net8.0 false True ..\TextMateSharp.snk diff --git a/src/TextMateSharp.Tests/Model/TMModelTests.cs b/src/TextMateSharp.Tests/Model/TMModelTests.cs index e02e1c8..5e6f901 100644 --- a/src/TextMateSharp.Tests/Model/TMModelTests.cs +++ b/src/TextMateSharp.Tests/Model/TMModelTests.cs @@ -125,7 +125,7 @@ public override int GetLineLength(int lineIndex) { return _lines[lineIndex].Length; } - public override string GetLineText(int lineIndex) + public override LineText GetLineText(int lineIndex) { return _lines[lineIndex]; } diff --git a/src/TextMateSharp.Tests/TextMateSharp.Tests.csproj b/src/TextMateSharp.Tests/TextMateSharp.Tests.csproj index ad51fbf..61499ce 100644 --- a/src/TextMateSharp.Tests/TextMateSharp.Tests.csproj +++ b/src/TextMateSharp.Tests/TextMateSharp.Tests.csproj @@ -1,7 +1,7 @@  - net6.0 + net8.0 False True ..\TextMateSharp.snk diff --git a/src/TextMateSharp/Grammar/IGrammar.cs b/src/TextMateSharp/Grammar/IGrammar.cs index e82df51..05e0d03 100644 --- a/src/TextMateSharp/Grammar/IGrammar.cs +++ b/src/TextMateSharp/Grammar/IGrammar.cs @@ -9,9 +9,9 @@ public interface IGrammar string GetName(); string GetScopeName(); ICollection GetFileTypes(); - ITokenizeLineResult TokenizeLine(string lineText); - ITokenizeLineResult TokenizeLine(string lineText, IStateStack prevState, TimeSpan timeLimit); - ITokenizeLineResult2 TokenizeLine2(string lineText); - ITokenizeLineResult2 TokenizeLine2(string lineText, IStateStack prevState, TimeSpan timeLimit); + ITokenizeLineResult TokenizeLine(LineText lineText); + ITokenizeLineResult TokenizeLine(LineText lineText, IStateStack prevState, TimeSpan timeLimit); + ITokenizeLineResult2 TokenizeLine2(LineText lineText); + ITokenizeLineResult2 TokenizeLine2(LineText lineText, IStateStack prevState, TimeSpan timeLimit); } } \ No newline at end of file diff --git a/src/TextMateSharp/Grammar/LineText.cs b/src/TextMateSharp/Grammar/LineText.cs new file mode 100644 index 0000000..e78c1c0 --- /dev/null +++ b/src/TextMateSharp/Grammar/LineText.cs @@ -0,0 +1,33 @@ +using System; + +namespace TextMateSharp.Grammars +{ + public readonly struct LineText + { + private readonly ReadOnlyMemory _memory; + + public LineText(ReadOnlyMemory memory) + { + _memory = memory; + } + + public LineText(string text) + { + _memory = text?.AsMemory() ?? ReadOnlyMemory.Empty; + } + + public ReadOnlyMemory Memory => _memory; + + public int Length => _memory.Length; + + public bool IsEmpty => _memory.IsEmpty; + + public static implicit operator LineText(string text) => new LineText(text); + + public static implicit operator LineText(ReadOnlyMemory memory) => new LineText(memory); + + public static implicit operator ReadOnlyMemory(LineText lineText) => lineText._memory; + + public override string ToString() => _memory.Span.ToString(); + } +} diff --git a/src/TextMateSharp/Internal/Grammars/Grammar.cs b/src/TextMateSharp/Internal/Grammars/Grammar.cs index 8700f84..4651538 100644 --- a/src/TextMateSharp/Internal/Grammars/Grammar.cs +++ b/src/TextMateSharp/Internal/Grammars/Grammar.cs @@ -200,27 +200,27 @@ private IRawGrammar Clone(IRawGrammar grammar) return (IRawGrammar)((Raw)grammar).Clone(); } - public ITokenizeLineResult TokenizeLine(string lineText) + public ITokenizeLineResult TokenizeLine(LineText lineText) { return TokenizeLine(lineText, null, TimeSpan.MaxValue); } - public ITokenizeLineResult TokenizeLine(string lineText, IStateStack prevState, TimeSpan timeLimit) + public ITokenizeLineResult TokenizeLine(LineText lineText, IStateStack prevState, TimeSpan timeLimit) { - return (ITokenizeLineResult)Tokenize(lineText, (StateStack)prevState, false, timeLimit); + return (ITokenizeLineResult)Tokenize(lineText.Memory, (StateStack)prevState, false, timeLimit); } - public ITokenizeLineResult2 TokenizeLine2(string lineText) + public ITokenizeLineResult2 TokenizeLine2(LineText lineText) { return TokenizeLine2(lineText, null, TimeSpan.MaxValue); } - public ITokenizeLineResult2 TokenizeLine2(string lineText, IStateStack prevState, TimeSpan timeLimit) + public ITokenizeLineResult2 TokenizeLine2(LineText lineText, IStateStack prevState, TimeSpan timeLimit) { - return (ITokenizeLineResult2)Tokenize(lineText, (StateStack)prevState, true, timeLimit); + return (ITokenizeLineResult2)Tokenize(lineText.Memory, (StateStack)prevState, true, timeLimit); } - private object Tokenize(string lineText, StateStack prevState, bool emitBinaryTokens, TimeSpan timeLimit) + private object Tokenize(ReadOnlyMemory lineText, StateStack prevState, bool emitBinaryTokens, TimeSpan timeLimit) { if (this._rootId == null) { @@ -237,7 +237,7 @@ private object Tokenize(string lineText, StateStack prevState, bool emitBinaryTo rawDefaultMetadata.TokenType, null, defaultTheme.fontStyle, defaultTheme.foreground, defaultTheme.background); - string rootScopeName = this.GetRule(this._rootId)?.GetName(null, null); + string rootScopeName = this.GetRule(this._rootId)?.GetName(ReadOnlyMemory.Empty, null); if (rootScopeName == null) return null; BasicScopeAttributes rawRootMetadata = this._basicScopeAttributesProvider.GetBasicScopeAttributes(rootScopeName); @@ -253,14 +253,25 @@ private object Tokenize(string lineText, StateStack prevState, bool emitBinaryTo prevState.Reset(); } - if (string.IsNullOrEmpty(lineText) || lineText[lineText.Length - 1] != '\n') + // Check if we need to append newline + ReadOnlyMemory effectiveLineText; + if (lineText.Length == 0 || lineText.Span[lineText.Length - 1] != '\n') { // Only add \n if the passed lineText didn't have it. - lineText += '\n'; + // We need to allocate a new buffer with the newline + char[] buffer = new char[lineText.Length + 1]; + lineText.Span.CopyTo(buffer); + buffer[lineText.Length] = '\n'; + effectiveLineText = buffer.AsMemory(); } - int lineLength = lineText.Length; - LineTokens lineTokens = new LineTokens(emitBinaryTokens, lineText, _tokenTypeMatchers, _balancedBracketSelectors); - TokenizeStringResult tokenizeResult = LineTokenizer.TokenizeString(this, lineText, isFirstLine, 0, prevState, + else + { + effectiveLineText = lineText; + } + + int lineLength = effectiveLineText.Length; + LineTokens lineTokens = new LineTokens(emitBinaryTokens, effectiveLineText, _tokenTypeMatchers, _balancedBracketSelectors); + TokenizeStringResult tokenizeResult = LineTokenizer.TokenizeString(this, effectiveLineText, isFirstLine, 0, prevState, lineTokens, true, timeLimit); if (emitBinaryTokens) diff --git a/src/TextMateSharp/Internal/Grammars/LineTokenizer.cs b/src/TextMateSharp/Internal/Grammars/LineTokenizer.cs index 05095e0..33ed2bf 100644 --- a/src/TextMateSharp/Internal/Grammars/LineTokenizer.cs +++ b/src/TextMateSharp/Internal/Grammars/LineTokenizer.cs @@ -13,7 +13,7 @@ namespace TextMateSharp.Internal.Grammars class LineTokenizer { private Grammar _grammar; - private string _lineText; + private ReadOnlyMemory _lineText; private bool _isFirstLine; private int _linePos; private StateStack _stack; @@ -22,7 +22,7 @@ class LineTokenizer private bool _stop; private int _lineLength; - public LineTokenizer(Grammar grammar, string lineText, bool isFirstLine, int linePos, StateStack stack, + public LineTokenizer(Grammar grammar, ReadOnlyMemory lineText, bool isFirstLine, int linePos, StateStack stack, LineTokens lineTokens) { this._grammar = grammar; @@ -253,7 +253,7 @@ private void ScanNext() } } - private MatchResult MatchRule(Grammar grammar, string lineText, in bool isFirstLine, in int linePos, + private MatchResult MatchRule(Grammar grammar, ReadOnlyMemory lineText, in bool isFirstLine, in int linePos, StateStack stack, in int anchorPosition) { Rule rule = stack.GetRule(grammar); @@ -277,7 +277,7 @@ private MatchResult MatchRule(Grammar grammar, string lineText, in bool isFirstL return null; } - private MatchResult MatchRuleOrInjections(Grammar grammar, string lineText, bool isFirstLine, + private MatchResult MatchRuleOrInjections(Grammar grammar, ReadOnlyMemory lineText, bool isFirstLine, in int linePos, StateStack stack, in int anchorPosition) { // Look for normal grammar rule @@ -319,7 +319,7 @@ private MatchResult MatchRuleOrInjections(Grammar grammar, string lineText, bool return matchResult; } - private MatchInjectionsResult MatchInjections(List injections, Grammar grammar, string lineText, + private MatchInjectionsResult MatchInjections(List injections, Grammar grammar, ReadOnlyMemory lineText, bool isFirstLine, in int linePos, StateStack stack, in int anchorPosition) { // The lower the better @@ -383,7 +383,7 @@ private MatchInjectionsResult MatchInjections(List injections, Gramma return null; } - private void HandleCaptures(Grammar grammar, string lineText, bool isFirstLine, StateStack stack, + private void HandleCaptures(Grammar grammar, ReadOnlyMemory lineText, bool isFirstLine, StateStack stack, LineTokens lineTokens, List captures, IOnigCaptureIndex[] captureIndices) { if (captures.Count == 0) @@ -457,7 +457,7 @@ private void HandleCaptures(Grammar grammar, string lineText, bool isFirstLine, contentNameScopesList); TokenizeString(grammar, - lineText.SubstringAtIndexes(0, captureIndex.End), + lineText.SliceAtIndexes(0, captureIndex.End), (isFirstLine && captureIndex.Start == 0), captureIndex.Start, stackClone, lineTokens, false, TimeSpan.MaxValue); continue; } @@ -488,7 +488,7 @@ private void HandleCaptures(Grammar grammar, string lineText, bool isFirstLine, * order. If any fails, cut off the entire stack above the failed while * condition. While conditions may also advance the linePosition. */ - private WhileCheckResult CheckWhileConditions(Grammar grammar, string lineText, bool isFirstLine, + private WhileCheckResult CheckWhileConditions(Grammar grammar, ReadOnlyMemory lineText, bool isFirstLine, int linePos, StateStack stack, LineTokens lineTokens) { int anchorPosition = stack.BeginRuleCapturedEOL ? 0 : -1; @@ -541,7 +541,7 @@ private WhileCheckResult CheckWhileConditions(Grammar grammar, string lineText, return new WhileCheckResult(stack, linePos, anchorPosition, isFirstLine); } - public static TokenizeStringResult TokenizeString(Grammar grammar, string lineText, bool isFirstLine, int linePos, + public static TokenizeStringResult TokenizeString(Grammar grammar, ReadOnlyMemory lineText, bool isFirstLine, int linePos, StateStack stack, LineTokens lineTokens, bool checkWhileConditions, TimeSpan timeLimit) { return new LineTokenizer(grammar, lineText, isFirstLine, linePos, stack, lineTokens).Scan(checkWhileConditions, timeLimit); diff --git a/src/TextMateSharp/Internal/Grammars/LineTokens.cs b/src/TextMateSharp/Internal/Grammars/LineTokens.cs index d535d46..dcf9f34 100644 --- a/src/TextMateSharp/Internal/Grammars/LineTokens.cs +++ b/src/TextMateSharp/Internal/Grammars/LineTokens.cs @@ -1,3 +1,4 @@ +using System; using System.Collections.Generic; using TextMateSharp.Grammars; using TextMateSharp.Themes; @@ -6,7 +7,7 @@ namespace TextMateSharp.Internal.Grammars { internal class LineTokens { - private string _lineText; + private ReadOnlyMemory _lineText; // used only if `_emitBinaryTokens` is false. private List _tokens; @@ -23,7 +24,7 @@ internal class LineTokens internal LineTokens( bool emitBinaryTokens, - string lineText, + ReadOnlyMemory lineText, List tokenTypeOverrides, BalancedBracketSelectors balancedBracketSelectors) { diff --git a/src/TextMateSharp/Internal/Rules/BeginEndRule.cs b/src/TextMateSharp/Internal/Rules/BeginEndRule.cs index a925a71..38d0259 100644 --- a/src/TextMateSharp/Internal/Rules/BeginEndRule.cs +++ b/src/TextMateSharp/Internal/Rules/BeginEndRule.cs @@ -1,3 +1,4 @@ +using System; using System.Collections.Generic; using Onigwrap; @@ -33,7 +34,7 @@ public BeginEndRule(RuleId id, string name, string contentName, string begin, Li _cachedCompiledPatterns = null; } - public string GetEndWithResolvedBackReferences(string lineText, IOnigCaptureIndex[] captureIndices) + public string GetEndWithResolvedBackReferences(ReadOnlyMemory lineText, IOnigCaptureIndex[] captureIndices) { return this._end.ResolveBackReferences(lineText, captureIndices); } diff --git a/src/TextMateSharp/Internal/Rules/BeginWhileRule.cs b/src/TextMateSharp/Internal/Rules/BeginWhileRule.cs index f3d52c5..99290e2 100644 --- a/src/TextMateSharp/Internal/Rules/BeginWhileRule.cs +++ b/src/TextMateSharp/Internal/Rules/BeginWhileRule.cs @@ -1,3 +1,4 @@ +using System; using System.Collections.Generic; using Onigwrap; @@ -33,7 +34,7 @@ public BeginWhileRule(RuleId id, string name, string contentName, string begin, _cachedCompiledWhilePatterns = null; } - public string getWhileWithResolvedBackReferences(string lineText, IOnigCaptureIndex[] captureIndices) + public string getWhileWithResolvedBackReferences(ReadOnlyMemory lineText, IOnigCaptureIndex[] captureIndices) { return this._while.ResolveBackReferences(lineText, captureIndices); } diff --git a/src/TextMateSharp/Internal/Rules/RegExpSource.cs b/src/TextMateSharp/Internal/Rules/RegExpSource.cs index faef2ca..a3dee4a 100644 --- a/src/TextMateSharp/Internal/Rules/RegExpSource.cs +++ b/src/TextMateSharp/Internal/Rules/RegExpSource.cs @@ -119,7 +119,7 @@ private void HandleAnchors(string regExpSource) } } - public string ResolveBackReferences(string lineText, IOnigCaptureIndex[] captureIndices) + public string ResolveBackReferences(ReadOnlyMemory lineText, IOnigCaptureIndex[] captureIndices) { List capturedValues = new List(); @@ -151,7 +151,7 @@ public string ResolveBackReferences(string lineText, IOnigCaptureIndex[] capture System.Diagnostics.Debug.WriteLine(ex.Message); } - return lineText; + return lineText.Span.ToString(); } private string EscapeRegExpCharacters(string value) diff --git a/src/TextMateSharp/Internal/Rules/Rule.cs b/src/TextMateSharp/Internal/Rules/Rule.cs index 0641e23..a238361 100644 --- a/src/TextMateSharp/Internal/Rules/Rule.cs +++ b/src/TextMateSharp/Internal/Rules/Rule.cs @@ -1,3 +1,4 @@ +using System; using Onigwrap; using TextMateSharp.Internal.Utils; @@ -24,7 +25,7 @@ public Rule(RuleId id, string name, string contentName) _contentNameIsCapturing = RegexSource.HasCaptures(this._contentName); } - public string GetName(string lineText, IOnigCaptureIndex[] captureIndices) + public string GetName(ReadOnlyMemory lineText, IOnigCaptureIndex[] captureIndices) { if (!this._nameIsCapturing) { @@ -34,7 +35,7 @@ public string GetName(string lineText, IOnigCaptureIndex[] captureIndices) return RegexSource.ReplaceCaptures(this._name, lineText, captureIndices); } - public string GetContentName(string lineText, IOnigCaptureIndex[] captureIndices) + public string GetContentName(ReadOnlyMemory lineText, IOnigCaptureIndex[] captureIndices) { if (!this._contentNameIsCapturing) { diff --git a/src/TextMateSharp/Internal/Utils/RegexSource.cs b/src/TextMateSharp/Internal/Utils/RegexSource.cs index b8c26c3..762948b 100644 --- a/src/TextMateSharp/Internal/Utils/RegexSource.cs +++ b/src/TextMateSharp/Internal/Utils/RegexSource.cs @@ -62,13 +62,13 @@ public static bool HasCaptures(string regexSource) return CAPTURING_REGEX_SOURCE.Match(regexSource).Success; } - public static string ReplaceCaptures(string regexSource, string captureSource, IOnigCaptureIndex[] captureIndices) + public static string ReplaceCaptures(string regexSource, ReadOnlyMemory captureSource, IOnigCaptureIndex[] captureIndices) { return CAPTURING_REGEX_SOURCE.Replace( regexSource, m => GetReplacement(m.Value, captureSource, captureIndices)); } - private static string GetReplacement(string match, string captureSource, IOnigCaptureIndex[] captureIndices) + private static string GetReplacement(string match, ReadOnlyMemory captureSource, IOnigCaptureIndex[] captureIndices) { int index = -1; string command = null; @@ -82,7 +82,7 @@ private static string GetReplacement(string match, string captureSource, IOnigCa { index = int.Parse(match.SubstringAtIndexes(1, match.Length)); } - IOnigCaptureIndex capture = captureIndices.Length > index ? captureIndices[index] : null; + IOnigCaptureIndex capture = captureIndices != null && captureIndices.Length > index ? captureIndices[index] : null; if (capture != null) { string result = captureSource.SubstringAtIndexes(capture.Start, capture.End); diff --git a/src/TextMateSharp/Internal/Utils/StringUtils.cs b/src/TextMateSharp/Internal/Utils/StringUtils.cs index 991d221..0047324 100644 --- a/src/TextMateSharp/Internal/Utils/StringUtils.cs +++ b/src/TextMateSharp/Internal/Utils/StringUtils.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using System.Text.RegularExpressions; namespace TextMateSharp.Internal.Utils @@ -15,6 +16,21 @@ internal static string SubstringAtIndexes(this string str, int startIndex, int e return str.Substring(startIndex, endIndex - startIndex); } + internal static ReadOnlyMemory SliceAtIndexes(this ReadOnlyMemory memory, int startIndex, int endIndex) + { + return memory.Slice(startIndex, endIndex - startIndex); + } + + internal static ReadOnlySpan SliceAtIndexes(this ReadOnlySpan span, int startIndex, int endIndex) + { + return span.Slice(startIndex, endIndex - startIndex); + } + + internal static string SubstringAtIndexes(this ReadOnlyMemory memory, int startIndex, int endIndex) + { + return memory.Slice(startIndex, endIndex - startIndex).Span.ToString(); + } + internal static bool IsValidHexColor(string hex) { if (hex == null || hex.Length < 1) diff --git a/src/TextMateSharp/Model/AbstractLineList.cs b/src/TextMateSharp/Model/AbstractLineList.cs index 56d2c57..fb255da 100644 --- a/src/TextMateSharp/Model/AbstractLineList.cs +++ b/src/TextMateSharp/Model/AbstractLineList.cs @@ -1,6 +1,8 @@ using System; using System.Collections.Generic; +using TextMateSharp.Grammars; + namespace TextMateSharp.Model { public abstract class AbstractLineList : IModelLines @@ -94,7 +96,7 @@ public int GetSize() public abstract int GetNumberOfLines(); - public abstract string GetLineText(int lineIndex); + public abstract LineText GetLineText(int lineIndex); public abstract int GetLineLength(int lineIndex); diff --git a/src/TextMateSharp/Model/IModelLines.cs b/src/TextMateSharp/Model/IModelLines.cs index 593592e..80f7568 100644 --- a/src/TextMateSharp/Model/IModelLines.cs +++ b/src/TextMateSharp/Model/IModelLines.cs @@ -1,5 +1,7 @@ using System; +using TextMateSharp.Grammars; + namespace TextMateSharp.Model { public interface IModelLines @@ -11,7 +13,7 @@ public interface IModelLines ModelLine Get(int lineIndex); void ForEach(Action action); int GetNumberOfLines(); - string GetLineText(int lineIndex); + LineText GetLineText(int lineIndex); int GetLineLength(int lineIndex); void Dispose(); } diff --git a/src/TextMateSharp/Model/ITokenizationSupport.cs b/src/TextMateSharp/Model/ITokenizationSupport.cs index 81cb75a..813bb0f 100644 --- a/src/TextMateSharp/Model/ITokenizationSupport.cs +++ b/src/TextMateSharp/Model/ITokenizationSupport.cs @@ -1,14 +1,13 @@ using System; +using TextMateSharp.Grammars; + namespace TextMateSharp.Model { public interface ITokenizationSupport { TMState GetInitialState(); - - LineTokens Tokenize(string line, TMState state, TimeSpan timeLimit); - - LineTokens Tokenize(string line, TMState state, int offsetDelta, int maxLen, TimeSpan timeLimit); - + LineTokens Tokenize(LineText line, TMState state, TimeSpan timeLimit); + LineTokens Tokenize(LineText line, TMState state, int offsetDelta, int maxLen, TimeSpan timeLimit); } } \ No newline at end of file diff --git a/src/TextMateSharp/Model/TMModel.cs b/src/TextMateSharp/Model/TMModel.cs index 5550583..977cb35 100644 --- a/src/TextMateSharp/Model/TMModel.cs +++ b/src/TextMateSharp/Model/TMModel.cs @@ -190,13 +190,11 @@ public int UpdateTokensInRange(ModelTokensChangedEventBuilder eventBuilder, int int endStateIndex = lineIndex + 1; LineTokens r = null; - string text = null; + LineText text = default; ModelLine modeLine = model._lines.Get(lineIndex); try { text = model._lines.GetLineText(lineIndex); - if (text == null) - continue; // Tokenize only the first X characters r = model._tokenizer.Tokenize(text, modeLine.State, 0, MAX_LEN_TO_TOKENIZE, stopLineTokenizationAfter); } diff --git a/src/TextMateSharp/Model/Tokenizer.cs b/src/TextMateSharp/Model/Tokenizer.cs index 2859b92..d0caeda 100644 --- a/src/TextMateSharp/Model/Tokenizer.cs +++ b/src/TextMateSharp/Model/Tokenizer.cs @@ -21,22 +21,23 @@ public TMState GetInitialState() return new TMState(null, null); } - public LineTokens Tokenize(string line, TMState state, TimeSpan timeLimit) + public LineTokens Tokenize(LineText line, TMState state, TimeSpan timeLimit) { return Tokenize(line, state, 0, 0, timeLimit); } - public LineTokens Tokenize(string line, TMState state, int offsetDelta, int maxLen, TimeSpan timeLimit) + public LineTokens Tokenize(LineText line, TMState state, int offsetDelta, int maxLen, TimeSpan timeLimit) { if (_grammar == null) return null; TMState freshState = state != null ? state.Clone() : GetInitialState(); - if (line.Length > 0 && line.Length > maxLen) - line = line.Substring(0, maxLen); + ReadOnlyMemory effectiveLine = line.Memory; + if (maxLen > 0 && effectiveLine.Length > maxLen) + effectiveLine = effectiveLine.Slice(0, maxLen); - ITokenizeLineResult textMateResult = _grammar.TokenizeLine(line, freshState.GetRuleStack(), timeLimit); + ITokenizeLineResult textMateResult = _grammar.TokenizeLine(effectiveLine, freshState.GetRuleStack(), timeLimit); freshState.SetRuleStack(textMateResult.RuleStack); // Create the result early and fill in the tokens later @@ -57,7 +58,7 @@ public LineTokens Tokenize(string line, TMState state, int offsetDelta, int maxL lastTokenType = tokenType; } } - return new LineTokens(tokens, offsetDelta + line.Length, freshState); + return new LineTokens(tokens, offsetDelta + effectiveLine.Length, freshState); } private string DecodeTextMateToken(DecodeMap decodeMap, List scopes)