diff --git a/.gitignore b/.gitignore
index 75f0aa7..56a6874 100644
--- a/.gitignore
+++ b/.gitignore
@@ -372,3 +372,7 @@ src/.vscode/launch.json
.idea/.idea.TextMateSharp.dir/.idea/indexLayout.xml
.idea/.idea.TextMateSharp.dir/.idea/vcs.xml
.idea/.idea.TextMateSharp/.idea/riderMarkupCache.xml
+.idea/.idea.TextMateSharp/.idea/copilot.data.migration.edit.xml
+.idea/.idea.TextMateSharp/.idea/copilot.data.migration.ask2agent.xml
+.idea/.idea.TextMateSharp/.idea/copilot.data.migration.ask.xml
+.idea/.idea.TextMateSharp/.idea/copilot.data.migration.agent.xml
diff --git a/build/Directory.Build.props b/build/Directory.Build.props
index d2bc446..7d9da74 100644
--- a/build/Directory.Build.props
+++ b/build/Directory.Build.props
@@ -3,6 +3,6 @@
latest
true
8.0.5
- 1.0.8
+ 1.0.9
diff --git a/src/TextMateSharp.Grammars.Tests/TextMateSharp.Grammars.Tests.csproj b/src/TextMateSharp.Grammars.Tests/TextMateSharp.Grammars.Tests.csproj
index 449f714..3db4421 100644
--- a/src/TextMateSharp.Grammars.Tests/TextMateSharp.Grammars.Tests.csproj
+++ b/src/TextMateSharp.Grammars.Tests/TextMateSharp.Grammars.Tests.csproj
@@ -1,7 +1,7 @@
- net6.0
+ net8.0
false
True
..\TextMateSharp.snk
diff --git a/src/TextMateSharp.Tests/Model/TMModelTests.cs b/src/TextMateSharp.Tests/Model/TMModelTests.cs
index e02e1c8..5e6f901 100644
--- a/src/TextMateSharp.Tests/Model/TMModelTests.cs
+++ b/src/TextMateSharp.Tests/Model/TMModelTests.cs
@@ -125,7 +125,7 @@ public override int GetLineLength(int lineIndex)
{
return _lines[lineIndex].Length;
}
- public override string GetLineText(int lineIndex)
+ public override LineText GetLineText(int lineIndex)
{
return _lines[lineIndex];
}
diff --git a/src/TextMateSharp.Tests/TextMateSharp.Tests.csproj b/src/TextMateSharp.Tests/TextMateSharp.Tests.csproj
index ad51fbf..61499ce 100644
--- a/src/TextMateSharp.Tests/TextMateSharp.Tests.csproj
+++ b/src/TextMateSharp.Tests/TextMateSharp.Tests.csproj
@@ -1,7 +1,7 @@
- net6.0
+ net8.0
False
True
..\TextMateSharp.snk
diff --git a/src/TextMateSharp/Grammar/IGrammar.cs b/src/TextMateSharp/Grammar/IGrammar.cs
index e82df51..05e0d03 100644
--- a/src/TextMateSharp/Grammar/IGrammar.cs
+++ b/src/TextMateSharp/Grammar/IGrammar.cs
@@ -9,9 +9,9 @@ public interface IGrammar
string GetName();
string GetScopeName();
ICollection GetFileTypes();
- ITokenizeLineResult TokenizeLine(string lineText);
- ITokenizeLineResult TokenizeLine(string lineText, IStateStack prevState, TimeSpan timeLimit);
- ITokenizeLineResult2 TokenizeLine2(string lineText);
- ITokenizeLineResult2 TokenizeLine2(string lineText, IStateStack prevState, TimeSpan timeLimit);
+ ITokenizeLineResult TokenizeLine(LineText lineText);
+ ITokenizeLineResult TokenizeLine(LineText lineText, IStateStack prevState, TimeSpan timeLimit);
+ ITokenizeLineResult2 TokenizeLine2(LineText lineText);
+ ITokenizeLineResult2 TokenizeLine2(LineText lineText, IStateStack prevState, TimeSpan timeLimit);
}
}
\ No newline at end of file
diff --git a/src/TextMateSharp/Grammar/LineText.cs b/src/TextMateSharp/Grammar/LineText.cs
new file mode 100644
index 0000000..e78c1c0
--- /dev/null
+++ b/src/TextMateSharp/Grammar/LineText.cs
@@ -0,0 +1,33 @@
+using System;
+
+namespace TextMateSharp.Grammars
+{
+ public readonly struct LineText
+ {
+ private readonly ReadOnlyMemory _memory;
+
+ public LineText(ReadOnlyMemory memory)
+ {
+ _memory = memory;
+ }
+
+ public LineText(string text)
+ {
+ _memory = text?.AsMemory() ?? ReadOnlyMemory.Empty;
+ }
+
+ public ReadOnlyMemory Memory => _memory;
+
+ public int Length => _memory.Length;
+
+ public bool IsEmpty => _memory.IsEmpty;
+
+ public static implicit operator LineText(string text) => new LineText(text);
+
+ public static implicit operator LineText(ReadOnlyMemory memory) => new LineText(memory);
+
+ public static implicit operator ReadOnlyMemory(LineText lineText) => lineText._memory;
+
+ public override string ToString() => _memory.Span.ToString();
+ }
+}
diff --git a/src/TextMateSharp/Internal/Grammars/Grammar.cs b/src/TextMateSharp/Internal/Grammars/Grammar.cs
index 8700f84..4651538 100644
--- a/src/TextMateSharp/Internal/Grammars/Grammar.cs
+++ b/src/TextMateSharp/Internal/Grammars/Grammar.cs
@@ -200,27 +200,27 @@ private IRawGrammar Clone(IRawGrammar grammar)
return (IRawGrammar)((Raw)grammar).Clone();
}
- public ITokenizeLineResult TokenizeLine(string lineText)
+ public ITokenizeLineResult TokenizeLine(LineText lineText)
{
return TokenizeLine(lineText, null, TimeSpan.MaxValue);
}
- public ITokenizeLineResult TokenizeLine(string lineText, IStateStack prevState, TimeSpan timeLimit)
+ public ITokenizeLineResult TokenizeLine(LineText lineText, IStateStack prevState, TimeSpan timeLimit)
{
- return (ITokenizeLineResult)Tokenize(lineText, (StateStack)prevState, false, timeLimit);
+ return (ITokenizeLineResult)Tokenize(lineText.Memory, (StateStack)prevState, false, timeLimit);
}
- public ITokenizeLineResult2 TokenizeLine2(string lineText)
+ public ITokenizeLineResult2 TokenizeLine2(LineText lineText)
{
return TokenizeLine2(lineText, null, TimeSpan.MaxValue);
}
- public ITokenizeLineResult2 TokenizeLine2(string lineText, IStateStack prevState, TimeSpan timeLimit)
+ public ITokenizeLineResult2 TokenizeLine2(LineText lineText, IStateStack prevState, TimeSpan timeLimit)
{
- return (ITokenizeLineResult2)Tokenize(lineText, (StateStack)prevState, true, timeLimit);
+ return (ITokenizeLineResult2)Tokenize(lineText.Memory, (StateStack)prevState, true, timeLimit);
}
- private object Tokenize(string lineText, StateStack prevState, bool emitBinaryTokens, TimeSpan timeLimit)
+ private object Tokenize(ReadOnlyMemory lineText, StateStack prevState, bool emitBinaryTokens, TimeSpan timeLimit)
{
if (this._rootId == null)
{
@@ -237,7 +237,7 @@ private object Tokenize(string lineText, StateStack prevState, bool emitBinaryTo
rawDefaultMetadata.TokenType, null, defaultTheme.fontStyle, defaultTheme.foreground,
defaultTheme.background);
- string rootScopeName = this.GetRule(this._rootId)?.GetName(null, null);
+ string rootScopeName = this.GetRule(this._rootId)?.GetName(ReadOnlyMemory.Empty, null);
if (rootScopeName == null)
return null;
BasicScopeAttributes rawRootMetadata = this._basicScopeAttributesProvider.GetBasicScopeAttributes(rootScopeName);
@@ -253,14 +253,25 @@ private object Tokenize(string lineText, StateStack prevState, bool emitBinaryTo
prevState.Reset();
}
- if (string.IsNullOrEmpty(lineText) || lineText[lineText.Length - 1] != '\n')
+ // Check if we need to append newline
+ ReadOnlyMemory effectiveLineText;
+ if (lineText.Length == 0 || lineText.Span[lineText.Length - 1] != '\n')
{
// Only add \n if the passed lineText didn't have it.
- lineText += '\n';
+ // We need to allocate a new buffer with the newline
+ char[] buffer = new char[lineText.Length + 1];
+ lineText.Span.CopyTo(buffer);
+ buffer[lineText.Length] = '\n';
+ effectiveLineText = buffer.AsMemory();
}
- int lineLength = lineText.Length;
- LineTokens lineTokens = new LineTokens(emitBinaryTokens, lineText, _tokenTypeMatchers, _balancedBracketSelectors);
- TokenizeStringResult tokenizeResult = LineTokenizer.TokenizeString(this, lineText, isFirstLine, 0, prevState,
+ else
+ {
+ effectiveLineText = lineText;
+ }
+
+ int lineLength = effectiveLineText.Length;
+ LineTokens lineTokens = new LineTokens(emitBinaryTokens, effectiveLineText, _tokenTypeMatchers, _balancedBracketSelectors);
+ TokenizeStringResult tokenizeResult = LineTokenizer.TokenizeString(this, effectiveLineText, isFirstLine, 0, prevState,
lineTokens, true, timeLimit);
if (emitBinaryTokens)
diff --git a/src/TextMateSharp/Internal/Grammars/LineTokenizer.cs b/src/TextMateSharp/Internal/Grammars/LineTokenizer.cs
index 05095e0..33ed2bf 100644
--- a/src/TextMateSharp/Internal/Grammars/LineTokenizer.cs
+++ b/src/TextMateSharp/Internal/Grammars/LineTokenizer.cs
@@ -13,7 +13,7 @@ namespace TextMateSharp.Internal.Grammars
class LineTokenizer
{
private Grammar _grammar;
- private string _lineText;
+ private ReadOnlyMemory _lineText;
private bool _isFirstLine;
private int _linePos;
private StateStack _stack;
@@ -22,7 +22,7 @@ class LineTokenizer
private bool _stop;
private int _lineLength;
- public LineTokenizer(Grammar grammar, string lineText, bool isFirstLine, int linePos, StateStack stack,
+ public LineTokenizer(Grammar grammar, ReadOnlyMemory lineText, bool isFirstLine, int linePos, StateStack stack,
LineTokens lineTokens)
{
this._grammar = grammar;
@@ -253,7 +253,7 @@ private void ScanNext()
}
}
- private MatchResult MatchRule(Grammar grammar, string lineText, in bool isFirstLine, in int linePos,
+ private MatchResult MatchRule(Grammar grammar, ReadOnlyMemory lineText, in bool isFirstLine, in int linePos,
StateStack stack, in int anchorPosition)
{
Rule rule = stack.GetRule(grammar);
@@ -277,7 +277,7 @@ private MatchResult MatchRule(Grammar grammar, string lineText, in bool isFirstL
return null;
}
- private MatchResult MatchRuleOrInjections(Grammar grammar, string lineText, bool isFirstLine,
+ private MatchResult MatchRuleOrInjections(Grammar grammar, ReadOnlyMemory lineText, bool isFirstLine,
in int linePos, StateStack stack, in int anchorPosition)
{
// Look for normal grammar rule
@@ -319,7 +319,7 @@ private MatchResult MatchRuleOrInjections(Grammar grammar, string lineText, bool
return matchResult;
}
- private MatchInjectionsResult MatchInjections(List injections, Grammar grammar, string lineText,
+ private MatchInjectionsResult MatchInjections(List injections, Grammar grammar, ReadOnlyMemory lineText,
bool isFirstLine, in int linePos, StateStack stack, in int anchorPosition)
{
// The lower the better
@@ -383,7 +383,7 @@ private MatchInjectionsResult MatchInjections(List injections, Gramma
return null;
}
- private void HandleCaptures(Grammar grammar, string lineText, bool isFirstLine, StateStack stack,
+ private void HandleCaptures(Grammar grammar, ReadOnlyMemory lineText, bool isFirstLine, StateStack stack,
LineTokens lineTokens, List captures, IOnigCaptureIndex[] captureIndices)
{
if (captures.Count == 0)
@@ -457,7 +457,7 @@ private void HandleCaptures(Grammar grammar, string lineText, bool isFirstLine,
contentNameScopesList);
TokenizeString(grammar,
- lineText.SubstringAtIndexes(0, captureIndex.End),
+ lineText.SliceAtIndexes(0, captureIndex.End),
(isFirstLine && captureIndex.Start == 0), captureIndex.Start, stackClone, lineTokens, false, TimeSpan.MaxValue);
continue;
}
@@ -488,7 +488,7 @@ private void HandleCaptures(Grammar grammar, string lineText, bool isFirstLine,
* order. If any fails, cut off the entire stack above the failed while
* condition. While conditions may also advance the linePosition.
*/
- private WhileCheckResult CheckWhileConditions(Grammar grammar, string lineText, bool isFirstLine,
+ private WhileCheckResult CheckWhileConditions(Grammar grammar, ReadOnlyMemory lineText, bool isFirstLine,
int linePos, StateStack stack, LineTokens lineTokens)
{
int anchorPosition = stack.BeginRuleCapturedEOL ? 0 : -1;
@@ -541,7 +541,7 @@ private WhileCheckResult CheckWhileConditions(Grammar grammar, string lineText,
return new WhileCheckResult(stack, linePos, anchorPosition, isFirstLine);
}
- public static TokenizeStringResult TokenizeString(Grammar grammar, string lineText, bool isFirstLine, int linePos,
+ public static TokenizeStringResult TokenizeString(Grammar grammar, ReadOnlyMemory lineText, bool isFirstLine, int linePos,
StateStack stack, LineTokens lineTokens, bool checkWhileConditions, TimeSpan timeLimit)
{
return new LineTokenizer(grammar, lineText, isFirstLine, linePos, stack, lineTokens).Scan(checkWhileConditions, timeLimit);
diff --git a/src/TextMateSharp/Internal/Grammars/LineTokens.cs b/src/TextMateSharp/Internal/Grammars/LineTokens.cs
index d535d46..dcf9f34 100644
--- a/src/TextMateSharp/Internal/Grammars/LineTokens.cs
+++ b/src/TextMateSharp/Internal/Grammars/LineTokens.cs
@@ -1,3 +1,4 @@
+using System;
using System.Collections.Generic;
using TextMateSharp.Grammars;
using TextMateSharp.Themes;
@@ -6,7 +7,7 @@ namespace TextMateSharp.Internal.Grammars
{
internal class LineTokens
{
- private string _lineText;
+ private ReadOnlyMemory _lineText;
// used only if `_emitBinaryTokens` is false.
private List _tokens;
@@ -23,7 +24,7 @@ internal class LineTokens
internal LineTokens(
bool emitBinaryTokens,
- string lineText,
+ ReadOnlyMemory lineText,
List tokenTypeOverrides,
BalancedBracketSelectors balancedBracketSelectors)
{
diff --git a/src/TextMateSharp/Internal/Rules/BeginEndRule.cs b/src/TextMateSharp/Internal/Rules/BeginEndRule.cs
index a925a71..38d0259 100644
--- a/src/TextMateSharp/Internal/Rules/BeginEndRule.cs
+++ b/src/TextMateSharp/Internal/Rules/BeginEndRule.cs
@@ -1,3 +1,4 @@
+using System;
using System.Collections.Generic;
using Onigwrap;
@@ -33,7 +34,7 @@ public BeginEndRule(RuleId id, string name, string contentName, string begin, Li
_cachedCompiledPatterns = null;
}
- public string GetEndWithResolvedBackReferences(string lineText, IOnigCaptureIndex[] captureIndices)
+ public string GetEndWithResolvedBackReferences(ReadOnlyMemory lineText, IOnigCaptureIndex[] captureIndices)
{
return this._end.ResolveBackReferences(lineText, captureIndices);
}
diff --git a/src/TextMateSharp/Internal/Rules/BeginWhileRule.cs b/src/TextMateSharp/Internal/Rules/BeginWhileRule.cs
index f3d52c5..99290e2 100644
--- a/src/TextMateSharp/Internal/Rules/BeginWhileRule.cs
+++ b/src/TextMateSharp/Internal/Rules/BeginWhileRule.cs
@@ -1,3 +1,4 @@
+using System;
using System.Collections.Generic;
using Onigwrap;
@@ -33,7 +34,7 @@ public BeginWhileRule(RuleId id, string name, string contentName, string begin,
_cachedCompiledWhilePatterns = null;
}
- public string getWhileWithResolvedBackReferences(string lineText, IOnigCaptureIndex[] captureIndices)
+ public string getWhileWithResolvedBackReferences(ReadOnlyMemory lineText, IOnigCaptureIndex[] captureIndices)
{
return this._while.ResolveBackReferences(lineText, captureIndices);
}
diff --git a/src/TextMateSharp/Internal/Rules/RegExpSource.cs b/src/TextMateSharp/Internal/Rules/RegExpSource.cs
index faef2ca..a3dee4a 100644
--- a/src/TextMateSharp/Internal/Rules/RegExpSource.cs
+++ b/src/TextMateSharp/Internal/Rules/RegExpSource.cs
@@ -119,7 +119,7 @@ private void HandleAnchors(string regExpSource)
}
}
- public string ResolveBackReferences(string lineText, IOnigCaptureIndex[] captureIndices)
+ public string ResolveBackReferences(ReadOnlyMemory lineText, IOnigCaptureIndex[] captureIndices)
{
List capturedValues = new List();
@@ -151,7 +151,7 @@ public string ResolveBackReferences(string lineText, IOnigCaptureIndex[] capture
System.Diagnostics.Debug.WriteLine(ex.Message);
}
- return lineText;
+ return lineText.Span.ToString();
}
private string EscapeRegExpCharacters(string value)
diff --git a/src/TextMateSharp/Internal/Rules/Rule.cs b/src/TextMateSharp/Internal/Rules/Rule.cs
index 0641e23..a238361 100644
--- a/src/TextMateSharp/Internal/Rules/Rule.cs
+++ b/src/TextMateSharp/Internal/Rules/Rule.cs
@@ -1,3 +1,4 @@
+using System;
using Onigwrap;
using TextMateSharp.Internal.Utils;
@@ -24,7 +25,7 @@ public Rule(RuleId id, string name, string contentName)
_contentNameIsCapturing = RegexSource.HasCaptures(this._contentName);
}
- public string GetName(string lineText, IOnigCaptureIndex[] captureIndices)
+ public string GetName(ReadOnlyMemory lineText, IOnigCaptureIndex[] captureIndices)
{
if (!this._nameIsCapturing)
{
@@ -34,7 +35,7 @@ public string GetName(string lineText, IOnigCaptureIndex[] captureIndices)
return RegexSource.ReplaceCaptures(this._name, lineText, captureIndices);
}
- public string GetContentName(string lineText, IOnigCaptureIndex[] captureIndices)
+ public string GetContentName(ReadOnlyMemory lineText, IOnigCaptureIndex[] captureIndices)
{
if (!this._contentNameIsCapturing)
{
diff --git a/src/TextMateSharp/Internal/Utils/RegexSource.cs b/src/TextMateSharp/Internal/Utils/RegexSource.cs
index b8c26c3..762948b 100644
--- a/src/TextMateSharp/Internal/Utils/RegexSource.cs
+++ b/src/TextMateSharp/Internal/Utils/RegexSource.cs
@@ -62,13 +62,13 @@ public static bool HasCaptures(string regexSource)
return CAPTURING_REGEX_SOURCE.Match(regexSource).Success;
}
- public static string ReplaceCaptures(string regexSource, string captureSource, IOnigCaptureIndex[] captureIndices)
+ public static string ReplaceCaptures(string regexSource, ReadOnlyMemory captureSource, IOnigCaptureIndex[] captureIndices)
{
return CAPTURING_REGEX_SOURCE.Replace(
regexSource, m => GetReplacement(m.Value, captureSource, captureIndices));
}
- private static string GetReplacement(string match, string captureSource, IOnigCaptureIndex[] captureIndices)
+ private static string GetReplacement(string match, ReadOnlyMemory captureSource, IOnigCaptureIndex[] captureIndices)
{
int index = -1;
string command = null;
@@ -82,7 +82,7 @@ private static string GetReplacement(string match, string captureSource, IOnigCa
{
index = int.Parse(match.SubstringAtIndexes(1, match.Length));
}
- IOnigCaptureIndex capture = captureIndices.Length > index ? captureIndices[index] : null;
+ IOnigCaptureIndex capture = captureIndices != null && captureIndices.Length > index ? captureIndices[index] : null;
if (capture != null)
{
string result = captureSource.SubstringAtIndexes(capture.Start, capture.End);
diff --git a/src/TextMateSharp/Internal/Utils/StringUtils.cs b/src/TextMateSharp/Internal/Utils/StringUtils.cs
index 991d221..0047324 100644
--- a/src/TextMateSharp/Internal/Utils/StringUtils.cs
+++ b/src/TextMateSharp/Internal/Utils/StringUtils.cs
@@ -1,4 +1,5 @@
-using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
using System.Text.RegularExpressions;
namespace TextMateSharp.Internal.Utils
@@ -15,6 +16,21 @@ internal static string SubstringAtIndexes(this string str, int startIndex, int e
return str.Substring(startIndex, endIndex - startIndex);
}
+ internal static ReadOnlyMemory SliceAtIndexes(this ReadOnlyMemory memory, int startIndex, int endIndex)
+ {
+ return memory.Slice(startIndex, endIndex - startIndex);
+ }
+
+ internal static ReadOnlySpan SliceAtIndexes(this ReadOnlySpan span, int startIndex, int endIndex)
+ {
+ return span.Slice(startIndex, endIndex - startIndex);
+ }
+
+ internal static string SubstringAtIndexes(this ReadOnlyMemory memory, int startIndex, int endIndex)
+ {
+ return memory.Slice(startIndex, endIndex - startIndex).Span.ToString();
+ }
+
internal static bool IsValidHexColor(string hex)
{
if (hex == null || hex.Length < 1)
diff --git a/src/TextMateSharp/Model/AbstractLineList.cs b/src/TextMateSharp/Model/AbstractLineList.cs
index 56d2c57..fb255da 100644
--- a/src/TextMateSharp/Model/AbstractLineList.cs
+++ b/src/TextMateSharp/Model/AbstractLineList.cs
@@ -1,6 +1,8 @@
using System;
using System.Collections.Generic;
+using TextMateSharp.Grammars;
+
namespace TextMateSharp.Model
{
public abstract class AbstractLineList : IModelLines
@@ -94,7 +96,7 @@ public int GetSize()
public abstract int GetNumberOfLines();
- public abstract string GetLineText(int lineIndex);
+ public abstract LineText GetLineText(int lineIndex);
public abstract int GetLineLength(int lineIndex);
diff --git a/src/TextMateSharp/Model/IModelLines.cs b/src/TextMateSharp/Model/IModelLines.cs
index 593592e..80f7568 100644
--- a/src/TextMateSharp/Model/IModelLines.cs
+++ b/src/TextMateSharp/Model/IModelLines.cs
@@ -1,5 +1,7 @@
using System;
+using TextMateSharp.Grammars;
+
namespace TextMateSharp.Model
{
public interface IModelLines
@@ -11,7 +13,7 @@ public interface IModelLines
ModelLine Get(int lineIndex);
void ForEach(Action action);
int GetNumberOfLines();
- string GetLineText(int lineIndex);
+ LineText GetLineText(int lineIndex);
int GetLineLength(int lineIndex);
void Dispose();
}
diff --git a/src/TextMateSharp/Model/ITokenizationSupport.cs b/src/TextMateSharp/Model/ITokenizationSupport.cs
index 81cb75a..813bb0f 100644
--- a/src/TextMateSharp/Model/ITokenizationSupport.cs
+++ b/src/TextMateSharp/Model/ITokenizationSupport.cs
@@ -1,14 +1,13 @@
using System;
+using TextMateSharp.Grammars;
+
namespace TextMateSharp.Model
{
public interface ITokenizationSupport
{
TMState GetInitialState();
-
- LineTokens Tokenize(string line, TMState state, TimeSpan timeLimit);
-
- LineTokens Tokenize(string line, TMState state, int offsetDelta, int maxLen, TimeSpan timeLimit);
-
+ LineTokens Tokenize(LineText line, TMState state, TimeSpan timeLimit);
+ LineTokens Tokenize(LineText line, TMState state, int offsetDelta, int maxLen, TimeSpan timeLimit);
}
}
\ No newline at end of file
diff --git a/src/TextMateSharp/Model/TMModel.cs b/src/TextMateSharp/Model/TMModel.cs
index 5550583..977cb35 100644
--- a/src/TextMateSharp/Model/TMModel.cs
+++ b/src/TextMateSharp/Model/TMModel.cs
@@ -190,13 +190,11 @@ public int UpdateTokensInRange(ModelTokensChangedEventBuilder eventBuilder, int
int endStateIndex = lineIndex + 1;
LineTokens r = null;
- string text = null;
+ LineText text = default;
ModelLine modeLine = model._lines.Get(lineIndex);
try
{
text = model._lines.GetLineText(lineIndex);
- if (text == null)
- continue;
// Tokenize only the first X characters
r = model._tokenizer.Tokenize(text, modeLine.State, 0, MAX_LEN_TO_TOKENIZE, stopLineTokenizationAfter);
}
diff --git a/src/TextMateSharp/Model/Tokenizer.cs b/src/TextMateSharp/Model/Tokenizer.cs
index 2859b92..d0caeda 100644
--- a/src/TextMateSharp/Model/Tokenizer.cs
+++ b/src/TextMateSharp/Model/Tokenizer.cs
@@ -21,22 +21,23 @@ public TMState GetInitialState()
return new TMState(null, null);
}
- public LineTokens Tokenize(string line, TMState state, TimeSpan timeLimit)
+ public LineTokens Tokenize(LineText line, TMState state, TimeSpan timeLimit)
{
return Tokenize(line, state, 0, 0, timeLimit);
}
- public LineTokens Tokenize(string line, TMState state, int offsetDelta, int maxLen, TimeSpan timeLimit)
+ public LineTokens Tokenize(LineText line, TMState state, int offsetDelta, int maxLen, TimeSpan timeLimit)
{
if (_grammar == null)
return null;
TMState freshState = state != null ? state.Clone() : GetInitialState();
- if (line.Length > 0 && line.Length > maxLen)
- line = line.Substring(0, maxLen);
+ ReadOnlyMemory effectiveLine = line.Memory;
+ if (maxLen > 0 && effectiveLine.Length > maxLen)
+ effectiveLine = effectiveLine.Slice(0, maxLen);
- ITokenizeLineResult textMateResult = _grammar.TokenizeLine(line, freshState.GetRuleStack(), timeLimit);
+ ITokenizeLineResult textMateResult = _grammar.TokenizeLine(effectiveLine, freshState.GetRuleStack(), timeLimit);
freshState.SetRuleStack(textMateResult.RuleStack);
// Create the result early and fill in the tokens later
@@ -57,7 +58,7 @@ public LineTokens Tokenize(string line, TMState state, int offsetDelta, int maxL
lastTokenType = tokenType;
}
}
- return new LineTokens(tokens, offsetDelta + line.Length, freshState);
+ return new LineTokens(tokens, offsetDelta + effectiveLine.Length, freshState);
}
private string DecodeTextMateToken(DecodeMap decodeMap, List scopes)