Optimize Query.Newline + reduce query/region allocations

TinyTokenizer.Benchmarks/NewlineQueryBenchmarks.cs

@@ -0,0 +1,72 @@
+using System.Text;
+using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Configs;
+using TinyTokenizer.Ast;
+using Q = TinyTokenizer.Ast.Query;
+
+namespace TinyTokenizer.Benchmarks;
+
+/// <summary>
+/// Benchmarks for token-centric newline detection via Query.Newline.
+/// Measures the cost of scanning a newline-heavy tree and matching nodes
+/// that follow a newline (current leading newline OR previous sibling trailing newline).
+/// </summary>
+[MemoryDiagnoser]
+[GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByCategory)]
+[CategoriesColumn]
+public class NewlineQueryBenchmarks
+{
+    [Params(1_000, 10_000)]
+    public int Lines { get; set; }
+
+    private SyntaxTree _tree = null!;
+
+    [GlobalSetup]
+    public void Setup()
+    {
+        var source = GenerateNewlineHeavyInput(Lines);
+        _tree = SyntaxTree.Parse(source);
+    }
+
+    [Benchmark(Description = "Select(Query.Newline) - Count")]
+    [BenchmarkCategory("Query", "Newline")]
+    public int SelectNewline_Count()
+    {
+        int count = 0;
+        foreach (var _ in _tree.Select(Q.Newline))
+            count++;
+        return count;
+    }
+
+    [Benchmark(Description = "Select(Query.Newline.First()) - First match")]
+    [BenchmarkCategory("Query", "Newline")]
+    public int SelectNewline_First()
+    {
+        foreach (var node in _tree.Select(Q.Newline.First()))
+            return node.Position;
+        return -1;
+    }
+
+    private static string GenerateNewlineHeavyInput(int lines)
+    {
+        // Intentionally mixes: trailing-newline ownership (end-of-line) and
+        // leading-newline ownership (own-line comments become leading trivia).
+        var builder = new StringBuilder(capacity: lines * 32);
+
+        for (int i = 0; i < lines; i++)
+        {
+            builder.Append("x");
+            builder.Append(i);
+            builder.Append(" = ");
+            builder.Append(i);
+            builder.Append(";\n");
+
+            if ((i & 7) == 0)
+            {
+                builder.Append("// comment\n");
+            }
+        }
+
+        return builder.ToString();
+    }
+}

TinyTokenizer.Tests/GreenNodeFlagsTests.cs

@@ -139,16 +139,16 @@ public void GreenBlock_Flags_AggregateContains_AndUseOpenerLeadingPlusCloserTrai
     }
 
     [Fact]
-    public void GreenList_Flags_UseFirstLeadingAndLastTrailingAsBoundary()
+    public void GreenList_Flags_DoNotPropagateBoundaryFlags_FromChildren()
     {
         var first = new GreenLeaf(NodeKind.Ident, "a", leadingTrivia: ImmutableArray.Create(GreenTrivia.Whitespace("  ")));
         var middle = new GreenLeaf(NodeKind.Ident, "b", trailingTrivia: ImmutableArray.Create(GreenTrivia.SingleLineComment("// c")));
         var last = new GreenLeaf(NodeKind.Ident, "c", trailingTrivia: ImmutableArray.Create(GreenTrivia.Newline("\n")));
 
         var list = new GreenList(ImmutableArray.Create<GreenNode>(first, middle, last));
 
-        AssertHas(list.Flags, GreenNodeFlags.HasLeadingWhitespaceTrivia);
-        AssertHas(list.Flags, GreenNodeFlags.HasTrailingNewlineTrivia);
+        // Token-centric boundary semantics: lists do not own boundary trivia.
+        AssertNotHas(list.Flags, GreenNodeFlagMasks.Boundary);
 
         AssertHas(list.Flags, GreenNodeFlags.ContainsWhitespaceTrivia);
         AssertHas(list.Flags, GreenNodeFlags.ContainsCommentTrivia);
@@ -165,8 +165,8 @@ public void GreenSyntaxNode_Flags_AggregateLikeList()
 
         var node = new GreenSyntaxNode(kind, first, last);
 
-        AssertHas(node.Flags, GreenNodeFlags.HasLeadingWhitespaceTrivia);
-        AssertHas(node.Flags, GreenNodeFlags.HasTrailingNewlineTrivia);
+        // Token-centric boundary semantics: syntax containers do not own boundary trivia.
+        AssertNotHas(node.Flags, GreenNodeFlagMasks.Boundary);
         AssertHas(node.Flags, GreenNodeFlags.ContainsWhitespaceTrivia);
         AssertHas(node.Flags, GreenNodeFlags.ContainsNewlineTrivia);
     }

TinyTokenizer.Tests/NewlineQueryTests.cs

@@ -0,0 +1,121 @@
+using System.Linq;
+using TinyTokenizer.Ast;
+using Xunit;
+
+namespace TinyTokenizer.Tests;
+
+[Trait("Category", "Query")]
+public sealed class NewlineQueryTests
+{
+    private static Schema CreateSyntaxBindingSchema()
+    {
+        return Schema.Create()
+            .DefineSyntax(Syntax.Define<FunctionCallSyntax>("FunctionCall")
+                .Match(Query.AnyIdent, Query.ParenBlock)
+                .Build())
+            .Build();
+    }
+
+    [Fact]
+    public void Newline_MatchesNodeWithLeadingTriviaNewline_TopLevelFirstSibling()
+    {
+        var tree = SyntaxTree.Parse("\nfoo");
+        var foo = tree.Root.Children.OfType<SyntaxToken>().First(n => n.Kind == NodeKind.Ident);
+
+        Assert.True(Query.Newline.Matches(foo));
+        Assert.False(Query.NotNewline.Matches(foo));
+    }
+
+    [Fact]
+    public void Newline_MatchesNodeAfterNewlineViaPreviousSiblingTrailingTrivia_TopLevel()
+    {
+        var tree = SyntaxTree.Parse("x\ny");
+        var idents = tree.Root.Children.OfType<SyntaxToken>().Where(n => n.Kind == NodeKind.Ident).ToList();
+
+        Assert.Equal(2, idents.Count);
+        Assert.False(Query.Newline.Matches(idents[0]));
+        Assert.True(Query.Newline.Matches(idents[1]));
+    }
+
+    [Fact]
+    public void Newline_MatchesFirstInnerNodeAfterOpenerViaPreviousSiblingTrailingTrivia_InBlock()
+    {
+        var tree = SyntaxTree.Parse("{\na}");
+        var block = tree.Root.Children.OfType<SyntaxBlock>().Single();
+        var a = block.InnerChildren.OfType<SyntaxToken>().Single(n => n.Kind == NodeKind.Ident);
+
+        Assert.True(Query.Newline.Matches(a));
+        Assert.False(Query.NotNewline.Matches(a));
+    }
+
+    [Fact]
+    public void Newline_MatchesInnerNodeAfterNewlineBetweenSiblings_InBlock()
+    {
+        var tree = SyntaxTree.Parse("{a\nb}");
+        var block = tree.Root.Children.OfType<SyntaxBlock>().Single();
+        var idents = block.InnerChildren.OfType<SyntaxToken>().Where(n => n.Kind == NodeKind.Ident).ToList();
+
+        Assert.Equal(2, idents.Count);
+        Assert.False(Query.Newline.Matches(idents[0]));
+        Assert.True(Query.Newline.Matches(idents[1]));
+    }
+
+    [Fact]
+    public void NotNewline_IsExactNegationOfNewline_ForIdentifiersInSameTree()
+    {
+        var tree = SyntaxTree.Parse("a b\nc\n\nd");
+
+        var allIdents = tree.Select(Query.AnyIdent).ToList();
+        var newlineIdents = tree.Select(Query.AnyIdent & Query.Newline).ToList();
+        var notNewlineIdents = tree.Select(Query.AnyIdent & Query.NotNewline).ToList();
+
+        Assert.All(allIdents, n => Assert.True(newlineIdents.Contains(n) ^ notNewlineIdents.Contains(n)));
+        Assert.Empty(newlineIdents.Intersect(notNewlineIdents));
+        Assert.Equal(allIdents.Count, newlineIdents.Count + notNewlineIdents.Count);
+    }
+
+    [Fact]
+    public void Newline_DoesNotThrow_OnEmptyBlockOrEmptyTree()
+    {
+        var emptyTree = SyntaxTree.Parse(string.Empty);
+        Assert.Empty(emptyTree.Select(Query.Newline));
+
+        var emptyBlockTree = SyntaxTree.Parse("{}");
+        Assert.Empty(emptyBlockTree.Select(Query.Newline));
+    }
+
+    [Fact]
+    public void Newline_MatchesCloserAfterNewlineInEmptyBlock()
+    {
+        var tree = SyntaxTree.Parse("{\n}");
+        var block = tree.Root.Children.OfType<SyntaxBlock>().Single();
+
+        Assert.True(Query.Newline.Matches(block.CloserNode));
+        Assert.False(Query.Newline.Matches(block.OpenerNode));
+    }
+
+    [Fact]
+    public void Newline_DoesNotMatchRootContainerNode()
+    {
+        var tree = SyntaxTree.Parse("\nfoo");
+
+        Assert.False(Query.Newline.Matches(tree.Root));
+        Assert.DoesNotContain(tree.Root, tree.Select(Query.Newline));
+    }
+
+    [Fact]
+    public void Newline_DoesNotMatchBoundSyntaxContainerNode()
+    {
+        var schema = CreateSyntaxBindingSchema();
+        var tree = SyntaxTree.Parse("\nfoo()", schema);
+
+        var funcCall = tree.Root.Children.OfType<FunctionCallSyntax>().First();
+
+        // Token-centric newline: matches tokens, not the bound syntax container.
+        Assert.False(Query.Newline.Matches(funcCall));
+        Assert.DoesNotContain(funcCall, tree.Select(Query.Newline));
+
+        // Still matches the first token after newline.
+        Assert.True(Query.Newline.Matches(funcCall.NameNode));
+    }
+}