From 1965039be1d68e916ef4a54c6663c878756d078f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Pen=CC=83alba?= Date: Thu, 11 Dec 2025 19:31:27 +0100 Subject: [PATCH 1/8] Empty change just to pass the tests on the CI --- src/TextMateSharp.Demo/Program.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/TextMateSharp.Demo/Program.cs b/src/TextMateSharp.Demo/Program.cs index eed263a..c1b6453 100644 --- a/src/TextMateSharp.Demo/Program.cs +++ b/src/TextMateSharp.Demo/Program.cs @@ -13,6 +13,8 @@ class Program { static void Main(string[] args) { + // empty change just to pass the tests + try { if (args.Length < 1) From 16a458bd63be899a54d198c92f893ff92a9d7c84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Pen=CC=83alba?= Date: Thu, 11 Dec 2025 19:32:26 +0100 Subject: [PATCH 2/8] Revert the previous change --- src/TextMateSharp.Demo/Program.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/TextMateSharp.Demo/Program.cs b/src/TextMateSharp.Demo/Program.cs index c1b6453..eed263a 100644 --- a/src/TextMateSharp.Demo/Program.cs +++ b/src/TextMateSharp.Demo/Program.cs @@ -13,8 +13,6 @@ class Program { static void Main(string[] args) { - // empty change just to pass the tests - try { if (args.Length < 1) From 6c5f75a9f79dfd9a4cf2f9c9693e7953b40ce427 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Pen=CC=83alba?= Date: Thu, 11 Dec 2025 19:32:48 +0100 Subject: [PATCH 3/8] Update onigwrap to 1.0.9 --- build/Directory.Build.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/Directory.Build.props b/build/Directory.Build.props index d2bc446..7d9da74 100644 --- a/build/Directory.Build.props +++ b/build/Directory.Build.props @@ -3,6 +3,6 @@ latest true 8.0.5 - 1.0.8 + 1.0.9 From 40a2c216637f9ca4edb84d95f134a32dbeafcb6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Pen=CC=83alba?= Date: Thu, 11 Dec 2025 19:37:53 +0100 Subject: [PATCH 4/8] Use onigwrap with ReadonlyMemory instead of string --- src/TextMateSharp/Internal/Grammars/LineTokenizer.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/TextMateSharp/Internal/Grammars/LineTokenizer.cs b/src/TextMateSharp/Internal/Grammars/LineTokenizer.cs index 05095e0..f080417 100644 --- a/src/TextMateSharp/Internal/Grammars/LineTokenizer.cs +++ b/src/TextMateSharp/Internal/Grammars/LineTokenizer.cs @@ -266,7 +266,7 @@ private MatchResult MatchRule(Grammar grammar, string lineText, in bool isFirstL if (ruleScanner == null) return null; - IOnigNextMatchResult r = ruleScanner.Scanner.FindNextMatchSync(lineText, linePos); + IOnigNextMatchResult r = ruleScanner.Scanner.FindNextMatchSync(lineText.AsMemory(), linePos); if (r != null) { @@ -340,7 +340,7 @@ private MatchInjectionsResult MatchInjections(List injections, Gramma CompiledRule ruleScanner = grammar.GetRule(injection.RuleId).Compile(grammar, null, isFirstLine, linePos == anchorPosition); - IOnigNextMatchResult matchResult = ruleScanner.Scanner.FindNextMatchSync(lineText, linePos); + IOnigNextMatchResult matchResult = ruleScanner.Scanner.FindNextMatchSync(lineText.AsMemory(), linePos); if (matchResult == null) { @@ -506,7 +506,7 @@ private WhileCheckResult CheckWhileConditions(Grammar grammar, string lineText, WhileStack whileRule = whileRules[i]; CompiledRule ruleScanner = whileRule.Rule.CompileWhile(whileRule.Stack.EndRule, isFirstLine, anchorPosition == linePos); - IOnigNextMatchResult r = ruleScanner.Scanner.FindNextMatchSync(lineText, linePos); + IOnigNextMatchResult r = ruleScanner.Scanner.FindNextMatchSync(lineText.AsMemory(), linePos); if (r != null) { From e71abf1d93ec6edde029cd6d55dcbbf56a19f84f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Pen=CC=83alba?= Date: Thu, 11 Dec 2025 14:11:12 +0100 Subject: [PATCH 5/8] Change public APIs to get LineText instead string Line text can be implicitly converted from string and from ReadonlyMemory --- src/TextMateSharp.Tests/Model/TMModelTests.cs | 2 +- src/TextMateSharp/Grammar/IGrammar.cs | 8 ++-- src/TextMateSharp/Grammar/LineText.cs | 33 +++++++++++++++++ .../Internal/Grammars/Grammar.cs | 37 ++++++++++++------- .../Internal/Grammars/LineTokenizer.cs | 18 ++++----- .../Internal/Grammars/LineTokens.cs | 5 ++- .../Internal/Rules/BeginEndRule.cs | 3 +- .../Internal/Rules/BeginWhileRule.cs | 3 +- .../Internal/Rules/RegExpSource.cs | 4 +- src/TextMateSharp/Internal/Rules/Rule.cs | 5 ++- .../Internal/Utils/RegexSource.cs | 6 +-- .../Internal/Utils/StringUtils.cs | 18 ++++++++- src/TextMateSharp/Model/AbstractLineList.cs | 4 +- src/TextMateSharp/Model/IModelLines.cs | 4 +- .../Model/ITokenizationSupport.cs | 9 ++--- src/TextMateSharp/Model/TMModel.cs | 4 +- src/TextMateSharp/Model/Tokenizer.cs | 13 ++++--- 17 files changed, 122 insertions(+), 54 deletions(-) create mode 100644 src/TextMateSharp/Grammar/LineText.cs diff --git a/src/TextMateSharp.Tests/Model/TMModelTests.cs b/src/TextMateSharp.Tests/Model/TMModelTests.cs index e02e1c8..5e6f901 100644 --- a/src/TextMateSharp.Tests/Model/TMModelTests.cs +++ b/src/TextMateSharp.Tests/Model/TMModelTests.cs @@ -125,7 +125,7 @@ public override int GetLineLength(int lineIndex) { return _lines[lineIndex].Length; } - public override string GetLineText(int lineIndex) + public override LineText GetLineText(int lineIndex) { return _lines[lineIndex]; } diff --git a/src/TextMateSharp/Grammar/IGrammar.cs b/src/TextMateSharp/Grammar/IGrammar.cs index e82df51..05e0d03 100644 --- a/src/TextMateSharp/Grammar/IGrammar.cs +++ b/src/TextMateSharp/Grammar/IGrammar.cs @@ -9,9 +9,9 @@ public interface IGrammar string GetName(); string GetScopeName(); ICollection GetFileTypes(); - ITokenizeLineResult TokenizeLine(string lineText); - ITokenizeLineResult TokenizeLine(string lineText, IStateStack prevState, TimeSpan timeLimit); - ITokenizeLineResult2 TokenizeLine2(string lineText); - ITokenizeLineResult2 TokenizeLine2(string lineText, IStateStack prevState, TimeSpan timeLimit); + ITokenizeLineResult TokenizeLine(LineText lineText); + ITokenizeLineResult TokenizeLine(LineText lineText, IStateStack prevState, TimeSpan timeLimit); + ITokenizeLineResult2 TokenizeLine2(LineText lineText); + ITokenizeLineResult2 TokenizeLine2(LineText lineText, IStateStack prevState, TimeSpan timeLimit); } } \ No newline at end of file diff --git a/src/TextMateSharp/Grammar/LineText.cs b/src/TextMateSharp/Grammar/LineText.cs new file mode 100644 index 0000000..e78c1c0 --- /dev/null +++ b/src/TextMateSharp/Grammar/LineText.cs @@ -0,0 +1,33 @@ +using System; + +namespace TextMateSharp.Grammars +{ + public readonly struct LineText + { + private readonly ReadOnlyMemory _memory; + + public LineText(ReadOnlyMemory memory) + { + _memory = memory; + } + + public LineText(string text) + { + _memory = text?.AsMemory() ?? ReadOnlyMemory.Empty; + } + + public ReadOnlyMemory Memory => _memory; + + public int Length => _memory.Length; + + public bool IsEmpty => _memory.IsEmpty; + + public static implicit operator LineText(string text) => new LineText(text); + + public static implicit operator LineText(ReadOnlyMemory memory) => new LineText(memory); + + public static implicit operator ReadOnlyMemory(LineText lineText) => lineText._memory; + + public override string ToString() => _memory.Span.ToString(); + } +} diff --git a/src/TextMateSharp/Internal/Grammars/Grammar.cs b/src/TextMateSharp/Internal/Grammars/Grammar.cs index 8700f84..4651538 100644 --- a/src/TextMateSharp/Internal/Grammars/Grammar.cs +++ b/src/TextMateSharp/Internal/Grammars/Grammar.cs @@ -200,27 +200,27 @@ private IRawGrammar Clone(IRawGrammar grammar) return (IRawGrammar)((Raw)grammar).Clone(); } - public ITokenizeLineResult TokenizeLine(string lineText) + public ITokenizeLineResult TokenizeLine(LineText lineText) { return TokenizeLine(lineText, null, TimeSpan.MaxValue); } - public ITokenizeLineResult TokenizeLine(string lineText, IStateStack prevState, TimeSpan timeLimit) + public ITokenizeLineResult TokenizeLine(LineText lineText, IStateStack prevState, TimeSpan timeLimit) { - return (ITokenizeLineResult)Tokenize(lineText, (StateStack)prevState, false, timeLimit); + return (ITokenizeLineResult)Tokenize(lineText.Memory, (StateStack)prevState, false, timeLimit); } - public ITokenizeLineResult2 TokenizeLine2(string lineText) + public ITokenizeLineResult2 TokenizeLine2(LineText lineText) { return TokenizeLine2(lineText, null, TimeSpan.MaxValue); } - public ITokenizeLineResult2 TokenizeLine2(string lineText, IStateStack prevState, TimeSpan timeLimit) + public ITokenizeLineResult2 TokenizeLine2(LineText lineText, IStateStack prevState, TimeSpan timeLimit) { - return (ITokenizeLineResult2)Tokenize(lineText, (StateStack)prevState, true, timeLimit); + return (ITokenizeLineResult2)Tokenize(lineText.Memory, (StateStack)prevState, true, timeLimit); } - private object Tokenize(string lineText, StateStack prevState, bool emitBinaryTokens, TimeSpan timeLimit) + private object Tokenize(ReadOnlyMemory lineText, StateStack prevState, bool emitBinaryTokens, TimeSpan timeLimit) { if (this._rootId == null) { @@ -237,7 +237,7 @@ private object Tokenize(string lineText, StateStack prevState, bool emitBinaryTo rawDefaultMetadata.TokenType, null, defaultTheme.fontStyle, defaultTheme.foreground, defaultTheme.background); - string rootScopeName = this.GetRule(this._rootId)?.GetName(null, null); + string rootScopeName = this.GetRule(this._rootId)?.GetName(ReadOnlyMemory.Empty, null); if (rootScopeName == null) return null; BasicScopeAttributes rawRootMetadata = this._basicScopeAttributesProvider.GetBasicScopeAttributes(rootScopeName); @@ -253,14 +253,25 @@ private object Tokenize(string lineText, StateStack prevState, bool emitBinaryTo prevState.Reset(); } - if (string.IsNullOrEmpty(lineText) || lineText[lineText.Length - 1] != '\n') + // Check if we need to append newline + ReadOnlyMemory effectiveLineText; + if (lineText.Length == 0 || lineText.Span[lineText.Length - 1] != '\n') { // Only add \n if the passed lineText didn't have it. - lineText += '\n'; + // We need to allocate a new buffer with the newline + char[] buffer = new char[lineText.Length + 1]; + lineText.Span.CopyTo(buffer); + buffer[lineText.Length] = '\n'; + effectiveLineText = buffer.AsMemory(); } - int lineLength = lineText.Length; - LineTokens lineTokens = new LineTokens(emitBinaryTokens, lineText, _tokenTypeMatchers, _balancedBracketSelectors); - TokenizeStringResult tokenizeResult = LineTokenizer.TokenizeString(this, lineText, isFirstLine, 0, prevState, + else + { + effectiveLineText = lineText; + } + + int lineLength = effectiveLineText.Length; + LineTokens lineTokens = new LineTokens(emitBinaryTokens, effectiveLineText, _tokenTypeMatchers, _balancedBracketSelectors); + TokenizeStringResult tokenizeResult = LineTokenizer.TokenizeString(this, effectiveLineText, isFirstLine, 0, prevState, lineTokens, true, timeLimit); if (emitBinaryTokens) diff --git a/src/TextMateSharp/Internal/Grammars/LineTokenizer.cs b/src/TextMateSharp/Internal/Grammars/LineTokenizer.cs index f080417..440bab7 100644 --- a/src/TextMateSharp/Internal/Grammars/LineTokenizer.cs +++ b/src/TextMateSharp/Internal/Grammars/LineTokenizer.cs @@ -13,7 +13,7 @@ namespace TextMateSharp.Internal.Grammars class LineTokenizer { private Grammar _grammar; - private string _lineText; + private ReadOnlyMemory _lineText; private bool _isFirstLine; private int _linePos; private StateStack _stack; @@ -22,7 +22,7 @@ class LineTokenizer private bool _stop; private int _lineLength; - public LineTokenizer(Grammar grammar, string lineText, bool isFirstLine, int linePos, StateStack stack, + public LineTokenizer(Grammar grammar, ReadOnlyMemory lineText, bool isFirstLine, int linePos, StateStack stack, LineTokens lineTokens) { this._grammar = grammar; @@ -253,7 +253,7 @@ private void ScanNext() } } - private MatchResult MatchRule(Grammar grammar, string lineText, in bool isFirstLine, in int linePos, + private MatchResult MatchRule(Grammar grammar, ReadOnlyMemory lineText, in bool isFirstLine, in int linePos, StateStack stack, in int anchorPosition) { Rule rule = stack.GetRule(grammar); @@ -277,7 +277,7 @@ private MatchResult MatchRule(Grammar grammar, string lineText, in bool isFirstL return null; } - private MatchResult MatchRuleOrInjections(Grammar grammar, string lineText, bool isFirstLine, + private MatchResult MatchRuleOrInjections(Grammar grammar, ReadOnlyMemory lineText, bool isFirstLine, in int linePos, StateStack stack, in int anchorPosition) { // Look for normal grammar rule @@ -319,7 +319,7 @@ private MatchResult MatchRuleOrInjections(Grammar grammar, string lineText, bool return matchResult; } - private MatchInjectionsResult MatchInjections(List injections, Grammar grammar, string lineText, + private MatchInjectionsResult MatchInjections(List injections, Grammar grammar, ReadOnlyMemory lineText, bool isFirstLine, in int linePos, StateStack stack, in int anchorPosition) { // The lower the better @@ -383,7 +383,7 @@ private MatchInjectionsResult MatchInjections(List injections, Gramma return null; } - private void HandleCaptures(Grammar grammar, string lineText, bool isFirstLine, StateStack stack, + private void HandleCaptures(Grammar grammar, ReadOnlyMemory lineText, bool isFirstLine, StateStack stack, LineTokens lineTokens, List captures, IOnigCaptureIndex[] captureIndices) { if (captures.Count == 0) @@ -457,7 +457,7 @@ private void HandleCaptures(Grammar grammar, string lineText, bool isFirstLine, contentNameScopesList); TokenizeString(grammar, - lineText.SubstringAtIndexes(0, captureIndex.End), + lineText.SliceAtIndexes(0, captureIndex.End), (isFirstLine && captureIndex.Start == 0), captureIndex.Start, stackClone, lineTokens, false, TimeSpan.MaxValue); continue; } @@ -488,7 +488,7 @@ private void HandleCaptures(Grammar grammar, string lineText, bool isFirstLine, * order. If any fails, cut off the entire stack above the failed while * condition. While conditions may also advance the linePosition. */ - private WhileCheckResult CheckWhileConditions(Grammar grammar, string lineText, bool isFirstLine, + private WhileCheckResult CheckWhileConditions(Grammar grammar, ReadOnlyMemory lineText, bool isFirstLine, int linePos, StateStack stack, LineTokens lineTokens) { int anchorPosition = stack.BeginRuleCapturedEOL ? 0 : -1; @@ -541,7 +541,7 @@ private WhileCheckResult CheckWhileConditions(Grammar grammar, string lineText, return new WhileCheckResult(stack, linePos, anchorPosition, isFirstLine); } - public static TokenizeStringResult TokenizeString(Grammar grammar, string lineText, bool isFirstLine, int linePos, + public static TokenizeStringResult TokenizeString(Grammar grammar, ReadOnlyMemory lineText, bool isFirstLine, int linePos, StateStack stack, LineTokens lineTokens, bool checkWhileConditions, TimeSpan timeLimit) { return new LineTokenizer(grammar, lineText, isFirstLine, linePos, stack, lineTokens).Scan(checkWhileConditions, timeLimit); diff --git a/src/TextMateSharp/Internal/Grammars/LineTokens.cs b/src/TextMateSharp/Internal/Grammars/LineTokens.cs index d535d46..dcf9f34 100644 --- a/src/TextMateSharp/Internal/Grammars/LineTokens.cs +++ b/src/TextMateSharp/Internal/Grammars/LineTokens.cs @@ -1,3 +1,4 @@ +using System; using System.Collections.Generic; using TextMateSharp.Grammars; using TextMateSharp.Themes; @@ -6,7 +7,7 @@ namespace TextMateSharp.Internal.Grammars { internal class LineTokens { - private string _lineText; + private ReadOnlyMemory _lineText; // used only if `_emitBinaryTokens` is false. private List _tokens; @@ -23,7 +24,7 @@ internal class LineTokens internal LineTokens( bool emitBinaryTokens, - string lineText, + ReadOnlyMemory lineText, List tokenTypeOverrides, BalancedBracketSelectors balancedBracketSelectors) { diff --git a/src/TextMateSharp/Internal/Rules/BeginEndRule.cs b/src/TextMateSharp/Internal/Rules/BeginEndRule.cs index a925a71..38d0259 100644 --- a/src/TextMateSharp/Internal/Rules/BeginEndRule.cs +++ b/src/TextMateSharp/Internal/Rules/BeginEndRule.cs @@ -1,3 +1,4 @@ +using System; using System.Collections.Generic; using Onigwrap; @@ -33,7 +34,7 @@ public BeginEndRule(RuleId id, string name, string contentName, string begin, Li _cachedCompiledPatterns = null; } - public string GetEndWithResolvedBackReferences(string lineText, IOnigCaptureIndex[] captureIndices) + public string GetEndWithResolvedBackReferences(ReadOnlyMemory lineText, IOnigCaptureIndex[] captureIndices) { return this._end.ResolveBackReferences(lineText, captureIndices); } diff --git a/src/TextMateSharp/Internal/Rules/BeginWhileRule.cs b/src/TextMateSharp/Internal/Rules/BeginWhileRule.cs index f3d52c5..99290e2 100644 --- a/src/TextMateSharp/Internal/Rules/BeginWhileRule.cs +++ b/src/TextMateSharp/Internal/Rules/BeginWhileRule.cs @@ -1,3 +1,4 @@ +using System; using System.Collections.Generic; using Onigwrap; @@ -33,7 +34,7 @@ public BeginWhileRule(RuleId id, string name, string contentName, string begin, _cachedCompiledWhilePatterns = null; } - public string getWhileWithResolvedBackReferences(string lineText, IOnigCaptureIndex[] captureIndices) + public string getWhileWithResolvedBackReferences(ReadOnlyMemory lineText, IOnigCaptureIndex[] captureIndices) { return this._while.ResolveBackReferences(lineText, captureIndices); } diff --git a/src/TextMateSharp/Internal/Rules/RegExpSource.cs b/src/TextMateSharp/Internal/Rules/RegExpSource.cs index faef2ca..a3dee4a 100644 --- a/src/TextMateSharp/Internal/Rules/RegExpSource.cs +++ b/src/TextMateSharp/Internal/Rules/RegExpSource.cs @@ -119,7 +119,7 @@ private void HandleAnchors(string regExpSource) } } - public string ResolveBackReferences(string lineText, IOnigCaptureIndex[] captureIndices) + public string ResolveBackReferences(ReadOnlyMemory lineText, IOnigCaptureIndex[] captureIndices) { List capturedValues = new List(); @@ -151,7 +151,7 @@ public string ResolveBackReferences(string lineText, IOnigCaptureIndex[] capture System.Diagnostics.Debug.WriteLine(ex.Message); } - return lineText; + return lineText.Span.ToString(); } private string EscapeRegExpCharacters(string value) diff --git a/src/TextMateSharp/Internal/Rules/Rule.cs b/src/TextMateSharp/Internal/Rules/Rule.cs index 0641e23..a238361 100644 --- a/src/TextMateSharp/Internal/Rules/Rule.cs +++ b/src/TextMateSharp/Internal/Rules/Rule.cs @@ -1,3 +1,4 @@ +using System; using Onigwrap; using TextMateSharp.Internal.Utils; @@ -24,7 +25,7 @@ public Rule(RuleId id, string name, string contentName) _contentNameIsCapturing = RegexSource.HasCaptures(this._contentName); } - public string GetName(string lineText, IOnigCaptureIndex[] captureIndices) + public string GetName(ReadOnlyMemory lineText, IOnigCaptureIndex[] captureIndices) { if (!this._nameIsCapturing) { @@ -34,7 +35,7 @@ public string GetName(string lineText, IOnigCaptureIndex[] captureIndices) return RegexSource.ReplaceCaptures(this._name, lineText, captureIndices); } - public string GetContentName(string lineText, IOnigCaptureIndex[] captureIndices) + public string GetContentName(ReadOnlyMemory lineText, IOnigCaptureIndex[] captureIndices) { if (!this._contentNameIsCapturing) { diff --git a/src/TextMateSharp/Internal/Utils/RegexSource.cs b/src/TextMateSharp/Internal/Utils/RegexSource.cs index b8c26c3..762948b 100644 --- a/src/TextMateSharp/Internal/Utils/RegexSource.cs +++ b/src/TextMateSharp/Internal/Utils/RegexSource.cs @@ -62,13 +62,13 @@ public static bool HasCaptures(string regexSource) return CAPTURING_REGEX_SOURCE.Match(regexSource).Success; } - public static string ReplaceCaptures(string regexSource, string captureSource, IOnigCaptureIndex[] captureIndices) + public static string ReplaceCaptures(string regexSource, ReadOnlyMemory captureSource, IOnigCaptureIndex[] captureIndices) { return CAPTURING_REGEX_SOURCE.Replace( regexSource, m => GetReplacement(m.Value, captureSource, captureIndices)); } - private static string GetReplacement(string match, string captureSource, IOnigCaptureIndex[] captureIndices) + private static string GetReplacement(string match, ReadOnlyMemory captureSource, IOnigCaptureIndex[] captureIndices) { int index = -1; string command = null; @@ -82,7 +82,7 @@ private static string GetReplacement(string match, string captureSource, IOnigCa { index = int.Parse(match.SubstringAtIndexes(1, match.Length)); } - IOnigCaptureIndex capture = captureIndices.Length > index ? captureIndices[index] : null; + IOnigCaptureIndex capture = captureIndices != null && captureIndices.Length > index ? captureIndices[index] : null; if (capture != null) { string result = captureSource.SubstringAtIndexes(capture.Start, capture.End); diff --git a/src/TextMateSharp/Internal/Utils/StringUtils.cs b/src/TextMateSharp/Internal/Utils/StringUtils.cs index 991d221..0047324 100644 --- a/src/TextMateSharp/Internal/Utils/StringUtils.cs +++ b/src/TextMateSharp/Internal/Utils/StringUtils.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using System.Text.RegularExpressions; namespace TextMateSharp.Internal.Utils @@ -15,6 +16,21 @@ internal static string SubstringAtIndexes(this string str, int startIndex, int e return str.Substring(startIndex, endIndex - startIndex); } + internal static ReadOnlyMemory SliceAtIndexes(this ReadOnlyMemory memory, int startIndex, int endIndex) + { + return memory.Slice(startIndex, endIndex - startIndex); + } + + internal static ReadOnlySpan SliceAtIndexes(this ReadOnlySpan span, int startIndex, int endIndex) + { + return span.Slice(startIndex, endIndex - startIndex); + } + + internal static string SubstringAtIndexes(this ReadOnlyMemory memory, int startIndex, int endIndex) + { + return memory.Slice(startIndex, endIndex - startIndex).Span.ToString(); + } + internal static bool IsValidHexColor(string hex) { if (hex == null || hex.Length < 1) diff --git a/src/TextMateSharp/Model/AbstractLineList.cs b/src/TextMateSharp/Model/AbstractLineList.cs index 56d2c57..fb255da 100644 --- a/src/TextMateSharp/Model/AbstractLineList.cs +++ b/src/TextMateSharp/Model/AbstractLineList.cs @@ -1,6 +1,8 @@ using System; using System.Collections.Generic; +using TextMateSharp.Grammars; + namespace TextMateSharp.Model { public abstract class AbstractLineList : IModelLines @@ -94,7 +96,7 @@ public int GetSize() public abstract int GetNumberOfLines(); - public abstract string GetLineText(int lineIndex); + public abstract LineText GetLineText(int lineIndex); public abstract int GetLineLength(int lineIndex); diff --git a/src/TextMateSharp/Model/IModelLines.cs b/src/TextMateSharp/Model/IModelLines.cs index 593592e..80f7568 100644 --- a/src/TextMateSharp/Model/IModelLines.cs +++ b/src/TextMateSharp/Model/IModelLines.cs @@ -1,5 +1,7 @@ using System; +using TextMateSharp.Grammars; + namespace TextMateSharp.Model { public interface IModelLines @@ -11,7 +13,7 @@ public interface IModelLines ModelLine Get(int lineIndex); void ForEach(Action action); int GetNumberOfLines(); - string GetLineText(int lineIndex); + LineText GetLineText(int lineIndex); int GetLineLength(int lineIndex); void Dispose(); } diff --git a/src/TextMateSharp/Model/ITokenizationSupport.cs b/src/TextMateSharp/Model/ITokenizationSupport.cs index 81cb75a..813bb0f 100644 --- a/src/TextMateSharp/Model/ITokenizationSupport.cs +++ b/src/TextMateSharp/Model/ITokenizationSupport.cs @@ -1,14 +1,13 @@ using System; +using TextMateSharp.Grammars; + namespace TextMateSharp.Model { public interface ITokenizationSupport { TMState GetInitialState(); - - LineTokens Tokenize(string line, TMState state, TimeSpan timeLimit); - - LineTokens Tokenize(string line, TMState state, int offsetDelta, int maxLen, TimeSpan timeLimit); - + LineTokens Tokenize(LineText line, TMState state, TimeSpan timeLimit); + LineTokens Tokenize(LineText line, TMState state, int offsetDelta, int maxLen, TimeSpan timeLimit); } } \ No newline at end of file diff --git a/src/TextMateSharp/Model/TMModel.cs b/src/TextMateSharp/Model/TMModel.cs index 5550583..b76199b 100644 --- a/src/TextMateSharp/Model/TMModel.cs +++ b/src/TextMateSharp/Model/TMModel.cs @@ -190,12 +190,12 @@ public int UpdateTokensInRange(ModelTokensChangedEventBuilder eventBuilder, int int endStateIndex = lineIndex + 1; LineTokens r = null; - string text = null; + LineText text = default; ModelLine modeLine = model._lines.Get(lineIndex); try { text = model._lines.GetLineText(lineIndex); - if (text == null) + if (text.IsEmpty) continue; // Tokenize only the first X characters r = model._tokenizer.Tokenize(text, modeLine.State, 0, MAX_LEN_TO_TOKENIZE, stopLineTokenizationAfter); diff --git a/src/TextMateSharp/Model/Tokenizer.cs b/src/TextMateSharp/Model/Tokenizer.cs index 2859b92..d0caeda 100644 --- a/src/TextMateSharp/Model/Tokenizer.cs +++ b/src/TextMateSharp/Model/Tokenizer.cs @@ -21,22 +21,23 @@ public TMState GetInitialState() return new TMState(null, null); } - public LineTokens Tokenize(string line, TMState state, TimeSpan timeLimit) + public LineTokens Tokenize(LineText line, TMState state, TimeSpan timeLimit) { return Tokenize(line, state, 0, 0, timeLimit); } - public LineTokens Tokenize(string line, TMState state, int offsetDelta, int maxLen, TimeSpan timeLimit) + public LineTokens Tokenize(LineText line, TMState state, int offsetDelta, int maxLen, TimeSpan timeLimit) { if (_grammar == null) return null; TMState freshState = state != null ? state.Clone() : GetInitialState(); - if (line.Length > 0 && line.Length > maxLen) - line = line.Substring(0, maxLen); + ReadOnlyMemory effectiveLine = line.Memory; + if (maxLen > 0 && effectiveLine.Length > maxLen) + effectiveLine = effectiveLine.Slice(0, maxLen); - ITokenizeLineResult textMateResult = _grammar.TokenizeLine(line, freshState.GetRuleStack(), timeLimit); + ITokenizeLineResult textMateResult = _grammar.TokenizeLine(effectiveLine, freshState.GetRuleStack(), timeLimit); freshState.SetRuleStack(textMateResult.RuleStack); // Create the result early and fill in the tokens later @@ -57,7 +58,7 @@ public LineTokens Tokenize(string line, TMState state, int offsetDelta, int maxL lastTokenType = tokenType; } } - return new LineTokens(tokens, offsetDelta + line.Length, freshState); + return new LineTokens(tokens, offsetDelta + effectiveLine.Length, freshState); } private string DecodeTextMateToken(DecodeMap decodeMap, List scopes) From e968d4cafdfdc0f7ce9ae0166b126156c1c5af6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Pen=CC=83alba?= Date: Thu, 11 Dec 2025 19:56:40 +0100 Subject: [PATCH 6/8] Fixed build and tests --- src/TextMateSharp/Internal/Grammars/LineTokenizer.cs | 6 +++--- src/TextMateSharp/Model/TMModel.cs | 2 -- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/TextMateSharp/Internal/Grammars/LineTokenizer.cs b/src/TextMateSharp/Internal/Grammars/LineTokenizer.cs index 440bab7..33ed2bf 100644 --- a/src/TextMateSharp/Internal/Grammars/LineTokenizer.cs +++ b/src/TextMateSharp/Internal/Grammars/LineTokenizer.cs @@ -266,7 +266,7 @@ private MatchResult MatchRule(Grammar grammar, ReadOnlyMemory lineText, in if (ruleScanner == null) return null; - IOnigNextMatchResult r = ruleScanner.Scanner.FindNextMatchSync(lineText.AsMemory(), linePos); + IOnigNextMatchResult r = ruleScanner.Scanner.FindNextMatchSync(lineText, linePos); if (r != null) { @@ -340,7 +340,7 @@ private MatchInjectionsResult MatchInjections(List injections, Gramma CompiledRule ruleScanner = grammar.GetRule(injection.RuleId).Compile(grammar, null, isFirstLine, linePos == anchorPosition); - IOnigNextMatchResult matchResult = ruleScanner.Scanner.FindNextMatchSync(lineText.AsMemory(), linePos); + IOnigNextMatchResult matchResult = ruleScanner.Scanner.FindNextMatchSync(lineText, linePos); if (matchResult == null) { @@ -506,7 +506,7 @@ private WhileCheckResult CheckWhileConditions(Grammar grammar, ReadOnlyMemory Date: Thu, 11 Dec 2025 20:19:16 +0100 Subject: [PATCH 7/8] Update .gitignore --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 75f0aa7..56a6874 100644 --- a/.gitignore +++ b/.gitignore @@ -372,3 +372,7 @@ src/.vscode/launch.json .idea/.idea.TextMateSharp.dir/.idea/indexLayout.xml .idea/.idea.TextMateSharp.dir/.idea/vcs.xml .idea/.idea.TextMateSharp/.idea/riderMarkupCache.xml +.idea/.idea.TextMateSharp/.idea/copilot.data.migration.edit.xml +.idea/.idea.TextMateSharp/.idea/copilot.data.migration.ask2agent.xml +.idea/.idea.TextMateSharp/.idea/copilot.data.migration.ask.xml +.idea/.idea.TextMateSharp/.idea/copilot.data.migration.agent.xml From 824e5968d5c9d1aad853e390b7b7a3b3f35d62ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Pen=CC=83alba?= Date: Thu, 11 Dec 2025 20:19:26 +0100 Subject: [PATCH 8/8] Update test projects to .net 8 --- .../TextMateSharp.Grammars.Tests.csproj | 2 +- src/TextMateSharp.Tests/TextMateSharp.Tests.csproj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/TextMateSharp.Grammars.Tests/TextMateSharp.Grammars.Tests.csproj b/src/TextMateSharp.Grammars.Tests/TextMateSharp.Grammars.Tests.csproj index 449f714..3db4421 100644 --- a/src/TextMateSharp.Grammars.Tests/TextMateSharp.Grammars.Tests.csproj +++ b/src/TextMateSharp.Grammars.Tests/TextMateSharp.Grammars.Tests.csproj @@ -1,7 +1,7 @@ - net6.0 + net8.0 false True ..\TextMateSharp.snk diff --git a/src/TextMateSharp.Tests/TextMateSharp.Tests.csproj b/src/TextMateSharp.Tests/TextMateSharp.Tests.csproj index ad51fbf..61499ce 100644 --- a/src/TextMateSharp.Tests/TextMateSharp.Tests.csproj +++ b/src/TextMateSharp.Tests/TextMateSharp.Tests.csproj @@ -1,7 +1,7 @@  - net6.0 + net8.0 False True ..\TextMateSharp.snk