Discarded - Update onigwrap to 1.0.9

.gitignore

@@ -372,3 +372,7 @@ src/.vscode/launch.json
 .idea/.idea.TextMateSharp.dir/.idea/indexLayout.xml
 .idea/.idea.TextMateSharp.dir/.idea/vcs.xml
 .idea/.idea.TextMateSharp/.idea/riderMarkupCache.xml
+.idea/.idea.TextMateSharp/.idea/copilot.data.migration.edit.xml
+.idea/.idea.TextMateSharp/.idea/copilot.data.migration.ask2agent.xml
+.idea/.idea.TextMateSharp/.idea/copilot.data.migration.ask.xml
+.idea/.idea.TextMateSharp/.idea/copilot.data.migration.agent.xml

build/Directory.Build.props

@@ -3,6 +3,6 @@
     <LangVersion>latest</LangVersion>
     <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
     <SystemTextJsonVersion>8.0.5</SystemTextJsonVersion>
-    <OnigwrapVersion>1.0.8</OnigwrapVersion>
+    <OnigwrapVersion>1.0.9</OnigwrapVersion>
   </PropertyGroup>
 </Project>

src/TextMateSharp.Grammars.Tests/TextMateSharp.Grammars.Tests.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <IsPackable>false</IsPackable>
     <SignAssembly>True</SignAssembly>
     <AssemblyOriginatorKeyFile>..\TextMateSharp.snk</AssemblyOriginatorKeyFile>

src/TextMateSharp.Tests/Model/TMModelTests.cs

@@ -125,7 +125,7 @@ public override int GetLineLength(int lineIndex)
             {
                 return _lines[lineIndex].Length;
             }
-            public override string GetLineText(int lineIndex)
+            public override LineText GetLineText(int lineIndex)
             {
                 return _lines[lineIndex];
             }

src/TextMateSharp.Tests/TextMateSharp.Tests.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <IsPackable>False</IsPackable>
     <SignAssembly>True</SignAssembly>
     <AssemblyOriginatorKeyFile>..\TextMateSharp.snk</AssemblyOriginatorKeyFile>

src/TextMateSharp/Grammar/IGrammar.cs

@@ -9,9 +9,9 @@ public interface IGrammar
         string GetName();
         string GetScopeName();
         ICollection<string> GetFileTypes();
-        ITokenizeLineResult TokenizeLine(string lineText);
-        ITokenizeLineResult TokenizeLine(string lineText, IStateStack prevState, TimeSpan timeLimit);
-        ITokenizeLineResult2 TokenizeLine2(string lineText);
-        ITokenizeLineResult2 TokenizeLine2(string lineText, IStateStack prevState, TimeSpan timeLimit);
+        ITokenizeLineResult TokenizeLine(LineText lineText);
+        ITokenizeLineResult TokenizeLine(LineText lineText, IStateStack prevState, TimeSpan timeLimit);
+        ITokenizeLineResult2 TokenizeLine2(LineText lineText);
+        ITokenizeLineResult2 TokenizeLine2(LineText lineText, IStateStack prevState, TimeSpan timeLimit);
     }
 }

src/TextMateSharp/Grammar/LineText.cs

@@ -0,0 +1,33 @@
+using System;
+
+namespace TextMateSharp.Grammars
+{
+    public readonly struct LineText
+    {
+        private readonly ReadOnlyMemory<char> _memory;
+
+        public LineText(ReadOnlyMemory<char> memory)
+        {
+            _memory = memory;
+        }
+
+        public LineText(string text)
+        {
+            _memory = text?.AsMemory() ?? ReadOnlyMemory<char>.Empty;
+        }
+
+        public ReadOnlyMemory<char> Memory => _memory;
+
+        public int Length => _memory.Length;
+
+        public bool IsEmpty => _memory.IsEmpty;
+
+        public static implicit operator LineText(string text) => new LineText(text);
+
+        public static implicit operator LineText(ReadOnlyMemory<char> memory) => new LineText(memory);
+
+        public static implicit operator ReadOnlyMemory<char>(LineText lineText) => lineText._memory;
+
+        public override string ToString() => _memory.Span.ToString();
+    }
+}

src/TextMateSharp/Internal/Grammars/Grammar.cs

@@ -200,27 +200,27 @@ private IRawGrammar Clone(IRawGrammar grammar)
             return (IRawGrammar)((Raw)grammar).Clone();
         }
 
-        public ITokenizeLineResult TokenizeLine(string lineText)
+        public ITokenizeLineResult TokenizeLine(LineText lineText)
         {
             return TokenizeLine(lineText, null, TimeSpan.MaxValue);
         }
 
-        public ITokenizeLineResult TokenizeLine(string lineText, IStateStack prevState, TimeSpan timeLimit)
+        public ITokenizeLineResult TokenizeLine(LineText lineText, IStateStack prevState, TimeSpan timeLimit)
         {
-            return (ITokenizeLineResult)Tokenize(lineText, (StateStack)prevState, false, timeLimit);
+            return (ITokenizeLineResult)Tokenize(lineText.Memory, (StateStack)prevState, false, timeLimit);
         }
 
-        public ITokenizeLineResult2 TokenizeLine2(string lineText)
+        public ITokenizeLineResult2 TokenizeLine2(LineText lineText)
         {
             return TokenizeLine2(lineText, null, TimeSpan.MaxValue);
         }
 
-        public ITokenizeLineResult2 TokenizeLine2(string lineText, IStateStack prevState, TimeSpan timeLimit)
+        public ITokenizeLineResult2 TokenizeLine2(LineText lineText, IStateStack prevState, TimeSpan timeLimit)
         {
-            return (ITokenizeLineResult2)Tokenize(lineText, (StateStack)prevState, true, timeLimit);
+            return (ITokenizeLineResult2)Tokenize(lineText.Memory, (StateStack)prevState, true, timeLimit);
         }
 
-        private object Tokenize(string lineText, StateStack prevState, bool emitBinaryTokens, TimeSpan timeLimit)
+        private object Tokenize(ReadOnlyMemory<char> lineText, StateStack prevState, bool emitBinaryTokens, TimeSpan timeLimit)
         {
             if (this._rootId == null)
             {
@@ -237,7 +237,7 @@ private object Tokenize(string lineText, StateStack prevState, bool emitBinaryTo
                         rawDefaultMetadata.TokenType, null, defaultTheme.fontStyle, defaultTheme.foreground,
                         defaultTheme.background);
 
-                string rootScopeName = this.GetRule(this._rootId)?.GetName(null, null);
+                string rootScopeName = this.GetRule(this._rootId)?.GetName(ReadOnlyMemory<char>.Empty, null);
                 if (rootScopeName == null)
                     return null;
                 BasicScopeAttributes rawRootMetadata = this._basicScopeAttributesProvider.GetBasicScopeAttributes(rootScopeName);
@@ -253,14 +253,25 @@ private object Tokenize(string lineText, StateStack prevState, bool emitBinaryTo
                 prevState.Reset();
             }
 
-            if (string.IsNullOrEmpty(lineText) || lineText[lineText.Length - 1] != '\n')
+            // Check if we need to append newline
+            ReadOnlyMemory<char> effectiveLineText;
+            if (lineText.Length == 0 || lineText.Span[lineText.Length - 1] != '\n')
             {
                 // Only add \n if the passed lineText didn't have it.
-                lineText += '\n';
+                // We need to allocate a new buffer with the newline
+                char[] buffer = new char[lineText.Length + 1];
+                lineText.Span.CopyTo(buffer);
+                buffer[lineText.Length] = '\n';
+                effectiveLineText = buffer.AsMemory();
             }
-            int lineLength = lineText.Length;
-            LineTokens lineTokens = new LineTokens(emitBinaryTokens, lineText, _tokenTypeMatchers, _balancedBracketSelectors);
-            TokenizeStringResult tokenizeResult = LineTokenizer.TokenizeString(this, lineText, isFirstLine, 0, prevState,
+            else
+            {
+                effectiveLineText = lineText;
+            }
+
+            int lineLength = effectiveLineText.Length;
+            LineTokens lineTokens = new LineTokens(emitBinaryTokens, effectiveLineText, _tokenTypeMatchers, _balancedBracketSelectors);
+            TokenizeStringResult tokenizeResult = LineTokenizer.TokenizeString(this, effectiveLineText, isFirstLine, 0, prevState,
                 lineTokens, true, timeLimit);
 
             if (emitBinaryTokens)

src/TextMateSharp/Internal/Grammars/LineTokenizer.cs

@@ -13,7 +13,7 @@ namespace TextMateSharp.Internal.Grammars
     class LineTokenizer
     {
         private Grammar _grammar;
-        private string _lineText;
+        private ReadOnlyMemory<char> _lineText;
         private bool _isFirstLine;
         private int _linePos;
         private StateStack _stack;
@@ -22,7 +22,7 @@ class LineTokenizer
         private bool _stop;
         private int _lineLength;
 
-        public LineTokenizer(Grammar grammar, string lineText, bool isFirstLine, int linePos, StateStack stack,
+        public LineTokenizer(Grammar grammar, ReadOnlyMemory<char> lineText, bool isFirstLine, int linePos, StateStack stack,
                 LineTokens lineTokens)
         {
             this._grammar = grammar;
@@ -253,7 +253,7 @@ private void ScanNext()
             }
         }
 
-        private MatchResult MatchRule(Grammar grammar, string lineText, in bool isFirstLine, in int linePos,
+        private MatchResult MatchRule(Grammar grammar, ReadOnlyMemory<char> lineText, in bool isFirstLine, in int linePos,
                 StateStack stack, in int anchorPosition)
         {
             Rule rule = stack.GetRule(grammar);
@@ -277,7 +277,7 @@ private MatchResult MatchRule(Grammar grammar, string lineText, in bool isFirstL
             return null;
         }
 
-        private MatchResult MatchRuleOrInjections(Grammar grammar, string lineText, bool isFirstLine,
+        private MatchResult MatchRuleOrInjections(Grammar grammar, ReadOnlyMemory<char> lineText, bool isFirstLine,
             in int linePos, StateStack stack, in int anchorPosition)
         {
             // Look for normal grammar rule
@@ -319,7 +319,7 @@ private MatchResult MatchRuleOrInjections(Grammar grammar, string lineText, bool
             return matchResult;
         }
 
-        private MatchInjectionsResult MatchInjections(List<Injection> injections, Grammar grammar, string lineText,
+        private MatchInjectionsResult MatchInjections(List<Injection> injections, Grammar grammar, ReadOnlyMemory<char> lineText,
                 bool isFirstLine, in int linePos, StateStack stack, in int anchorPosition)
         {
             // The lower the better
@@ -383,7 +383,7 @@ private MatchInjectionsResult MatchInjections(List<Injection> injections, Gramma
             return null;
         }
 
-        private void HandleCaptures(Grammar grammar, string lineText, bool isFirstLine, StateStack stack,
+        private void HandleCaptures(Grammar grammar, ReadOnlyMemory<char> lineText, bool isFirstLine, StateStack stack,
                 LineTokens lineTokens, List<CaptureRule> captures, IOnigCaptureIndex[] captureIndices)
         {
             if (captures.Count == 0)
@@ -457,7 +457,7 @@ private void HandleCaptures(Grammar grammar, string lineText, bool isFirstLine,
                         contentNameScopesList);
 
                     TokenizeString(grammar,
-                            lineText.SubstringAtIndexes(0, captureIndex.End),
+                            lineText.SliceAtIndexes(0, captureIndex.End),
                             (isFirstLine && captureIndex.Start == 0), captureIndex.Start, stackClone, lineTokens, false, TimeSpan.MaxValue);
                     continue;
                 }
@@ -488,7 +488,7 @@ private void HandleCaptures(Grammar grammar, string lineText, bool isFirstLine,
          * order. If any fails, cut off the entire stack above the failed while
          * condition. While conditions may also advance the linePosition.
          */
-        private WhileCheckResult CheckWhileConditions(Grammar grammar, string lineText, bool isFirstLine,
+        private WhileCheckResult CheckWhileConditions(Grammar grammar, ReadOnlyMemory<char> lineText, bool isFirstLine,
                 int linePos, StateStack stack, LineTokens lineTokens)
         {
             int anchorPosition = stack.BeginRuleCapturedEOL ? 0 : -1;
@@ -541,7 +541,7 @@ private WhileCheckResult CheckWhileConditions(Grammar grammar, string lineText,
             return new WhileCheckResult(stack, linePos, anchorPosition, isFirstLine);
         }
 
-        public static TokenizeStringResult TokenizeString(Grammar grammar, string lineText, bool isFirstLine, int linePos,
+        public static TokenizeStringResult TokenizeString(Grammar grammar, ReadOnlyMemory<char> lineText, bool isFirstLine, int linePos,
                 StateStack stack, LineTokens lineTokens, bool checkWhileConditions, TimeSpan timeLimit)
         {
             return new LineTokenizer(grammar, lineText, isFirstLine, linePos, stack, lineTokens).Scan(checkWhileConditions, timeLimit);

src/TextMateSharp/Internal/Grammars/LineTokens.cs

@@ -1,3 +1,4 @@
+using System;
 using System.Collections.Generic;
 using TextMateSharp.Grammars;
 using TextMateSharp.Themes;
@@ -6,7 +7,7 @@ namespace TextMateSharp.Internal.Grammars
 {
     internal class LineTokens
     {
-        private string _lineText;
+        private ReadOnlyMemory<char> _lineText;
 
         // used only if `_emitBinaryTokens` is false.
         private List<IToken> _tokens;
@@ -23,7 +24,7 @@ internal class LineTokens
 
         internal LineTokens(
             bool emitBinaryTokens,
-            string lineText,
+            ReadOnlyMemory<char> lineText,
             List<TokenTypeMatcher> tokenTypeOverrides,
             BalancedBracketSelectors balancedBracketSelectors)
         {

src/TextMateSharp/Internal/Rules/BeginEndRule.cs

@@ -1,3 +1,4 @@
+using System;
 using System.Collections.Generic;
 using Onigwrap;
 
@@ -33,7 +34,7 @@ public BeginEndRule(RuleId id, string name, string contentName, string begin, Li
             _cachedCompiledPatterns = null;
         }
 
-        public string GetEndWithResolvedBackReferences(string lineText, IOnigCaptureIndex[] captureIndices)
+        public string GetEndWithResolvedBackReferences(ReadOnlyMemory<char> lineText, IOnigCaptureIndex[] captureIndices)
         {
             return this._end.ResolveBackReferences(lineText, captureIndices);
         }

src/TextMateSharp/Internal/Rules/BeginWhileRule.cs

@@ -1,3 +1,4 @@
+using System;
 using System.Collections.Generic;
 using Onigwrap;
 
@@ -33,7 +34,7 @@ public BeginWhileRule(RuleId id, string name, string contentName, string begin,
             _cachedCompiledWhilePatterns = null;
         }
 
-        public string getWhileWithResolvedBackReferences(string lineText, IOnigCaptureIndex[] captureIndices)
+        public string getWhileWithResolvedBackReferences(ReadOnlyMemory<char> lineText, IOnigCaptureIndex[] captureIndices)
         {
             return this._while.ResolveBackReferences(lineText, captureIndices);
         }

src/TextMateSharp/Internal/Rules/RegExpSource.cs

@@ -119,7 +119,7 @@ private void HandleAnchors(string regExpSource)
             }
         }
 
-        public string ResolveBackReferences(string lineText, IOnigCaptureIndex[] captureIndices)
+        public string ResolveBackReferences(ReadOnlyMemory<char> lineText, IOnigCaptureIndex[] captureIndices)
         {
             List<string> capturedValues = new List<string>();
 
@@ -151,7 +151,7 @@ public string ResolveBackReferences(string lineText, IOnigCaptureIndex[] capture
                 System.Diagnostics.Debug.WriteLine(ex.Message);
             }
 
-            return lineText;
+            return lineText.Span.ToString();
         }
 
         private string EscapeRegExpCharacters(string value)

src/TextMateSharp/Internal/Rules/Rule.cs

@@ -1,3 +1,4 @@
+using System;
 using Onigwrap;
 
 using TextMateSharp.Internal.Utils;
@@ -24,7 +25,7 @@ public Rule(RuleId id, string name, string contentName)
             _contentNameIsCapturing = RegexSource.HasCaptures(this._contentName);
         }
 
-        public string GetName(string lineText, IOnigCaptureIndex[] captureIndices)
+        public string GetName(ReadOnlyMemory<char> lineText, IOnigCaptureIndex[] captureIndices)
         {
             if (!this._nameIsCapturing)
             {
@@ -34,7 +35,7 @@ public string GetName(string lineText, IOnigCaptureIndex[] captureIndices)
             return RegexSource.ReplaceCaptures(this._name, lineText, captureIndices);
         }
 
-        public string GetContentName(string lineText, IOnigCaptureIndex[] captureIndices)
+        public string GetContentName(ReadOnlyMemory<char> lineText, IOnigCaptureIndex[] captureIndices)
         {
             if (!this._contentNameIsCapturing)
             {

src/TextMateSharp/Internal/Utils/RegexSource.cs

@@ -62,13 +62,13 @@ public static bool HasCaptures(string regexSource)
             return CAPTURING_REGEX_SOURCE.Match(regexSource).Success;
         }
 
-        public static string ReplaceCaptures(string regexSource, string captureSource, IOnigCaptureIndex[] captureIndices)
+        public static string ReplaceCaptures(string regexSource, ReadOnlyMemory<char> captureSource, IOnigCaptureIndex[] captureIndices)
         {
             return CAPTURING_REGEX_SOURCE.Replace(
                 regexSource, m => GetReplacement(m.Value, captureSource, captureIndices));
         }
 
-        private static string GetReplacement(string match, string captureSource, IOnigCaptureIndex[] captureIndices)
+        private static string GetReplacement(string match, ReadOnlyMemory<char> captureSource, IOnigCaptureIndex[] captureIndices)
         {
             int index = -1;
             string command = null;
@@ -82,7 +82,7 @@ private static string GetReplacement(string match, string captureSource, IOnigCa
             {
                 index = int.Parse(match.SubstringAtIndexes(1, match.Length));
             }
-            IOnigCaptureIndex capture = captureIndices.Length > index ? captureIndices[index] : null;
+            IOnigCaptureIndex capture = captureIndices != null && captureIndices.Length > index ? captureIndices[index] : null;
             if (capture != null)
             {
                 string result = captureSource.SubstringAtIndexes(capture.Start, capture.End);

src/TextMateSharp/Internal/Utils/StringUtils.cs

@@ -1,4 +1,5 @@
-using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
 using System.Text.RegularExpressions;
 
 namespace TextMateSharp.Internal.Utils
@@ -15,6 +16,21 @@ internal static string SubstringAtIndexes(this string str, int startIndex, int e
             return str.Substring(startIndex, endIndex - startIndex);
         }
 
+        internal static ReadOnlyMemory<char> SliceAtIndexes(this ReadOnlyMemory<char> memory, int startIndex, int endIndex)
+        {
+            return memory.Slice(startIndex, endIndex - startIndex);
+        }
+
+        internal static ReadOnlySpan<char> SliceAtIndexes(this ReadOnlySpan<char> span, int startIndex, int endIndex)
+        {
+            return span.Slice(startIndex, endIndex - startIndex);
+        }
+
+        internal static string SubstringAtIndexes(this ReadOnlyMemory<char> memory, int startIndex, int endIndex)
+        {
+            return memory.Slice(startIndex, endIndex - startIndex).Span.ToString();
+        }
+
         internal static bool IsValidHexColor(string hex)
         {
             if (hex == null || hex.Length < 1)