Green-node flags for fast trivia/content queries

TinyTokenizer.Tests/GreenNodeEqualityTests.cs

@@ -0,0 +1,99 @@
+using System.Collections.Immutable;
+using TinyTokenizer.Ast;
+
+namespace TinyTokenizer.Tests;
+
+[Trait("Category", "Equality")]
+public class GreenNodeEqualityTests
+{
+    [Fact]
+    public void GreenLeaf_EquivalentConstructions_AreEqual_AndHashMatches()
+    {
+        var leading = ImmutableArray.Create(GreenTrivia.Whitespace(" "));
+        var trailing = ImmutableArray.Create(GreenTrivia.Newline("\n"));
+
+        var a = new GreenLeaf(NodeKind.Ident, "x", leading, trailing);
+
+        var b = new GreenLeaf(NodeKind.Ident, "x")
+            .WithLeadingTrivia(leading)
+            .WithTrailingTrivia(trailing);
+
+        Assert.True(a.Equals(b));
+        Assert.Equal(a.GetHashCode(), b.GetHashCode());
+    }
+
+    [Fact]
+    public void GreenLeaf_CacheAndNonCachePaths_BehaveConsistently()
+    {
+        // No trivia -> cached
+        var cached1 = GreenNodeCache.GetOrCreate(NodeKind.Symbol, "{");
+        var cached2 = GreenNodeCache.CreateDelimiter('{');
+
+        Assert.Same(cached1, cached2);
+        Assert.True(cached1.Equals(cached2));
+        Assert.Equal(cached1.GetHashCode(), cached2.GetHashCode());
+
+        // Trailing space -> cached "with space"
+        var spaceTrivia = ImmutableArray.Create(GreenTrivia.Whitespace(" "));
+        var cachedWithSpace = GreenNodeCache.GetOrCreateWithTrailingSpace(NodeKind.Symbol, "{");
+        var viaFactory = GreenNodeCache.Create(NodeKind.Symbol, "{", ImmutableArray<GreenTrivia>.Empty, spaceTrivia);
+
+        Assert.Same(cachedWithSpace, viaFactory);
+        Assert.Equal(cachedWithSpace.GetHashCode(), viaFactory.GetHashCode());
+        AssertHasWhitespaceButNoNewlineOrComment(cachedWithSpace);
+    }
+
+    [Fact]
+    public void StructuralSharing_WithMethods_DoNotMutateOriginals()
+    {
+        var leaf = new GreenLeaf(NodeKind.Ident, "x");
+        var changed = leaf.WithText("y");
+
+        Assert.Equal("x", leaf.Text);
+        Assert.Equal("y", changed.Text);
+        Assert.False(leaf.Equals(changed));
+
+        var child1 = new GreenLeaf(NodeKind.Ident, "a");
+        var child2 = new GreenLeaf(NodeKind.Ident, "b");
+        var block = GreenBlock.Create('{', ImmutableArray.Create<GreenNode>(child1, child2));
+
+        var inserted = block.WithInsert(2, ImmutableArray.Create<GreenNode>(new GreenLeaf(NodeKind.Ident, "c")));
+
+        Assert.Equal(4, block.SlotCount); // opener + 2 children + closer
+        Assert.Equal(5, inserted.SlotCount); // opener + 3 children + closer
+
+        // Original is unchanged and still shares children
+        Assert.Same(child1, block.GetSlot(1));
+        Assert.Same(child2, block.GetSlot(2));
+
+        // Inserted block shares originals too
+        Assert.Same(child1, inserted.GetSlot(1));
+        Assert.Same(child2, inserted.GetSlot(3));
+
+        Assert.False(block.Equals(inserted));
+    }
+
+    [Fact]
+    public void OnlyContentChanges_CauseInequality()
+    {
+        // Same kind/text/trivia should be equal (baseline)
+        var a = new GreenLeaf(NodeKind.Ident, "x");
+        var b = new GreenLeaf(NodeKind.Ident, "x");
+        Assert.True(a.Equals(b));
+
+        // Different text -> not equal
+        var c = new GreenLeaf(NodeKind.Ident, "y");
+        Assert.False(a.Equals(c));
+
+        // Different kind -> not equal
+        var d = new GreenLeaf(NodeKind.Numeric, "x");
+        Assert.False(a.Equals(d));
+    }
+
+    private static void AssertHasWhitespaceButNoNewlineOrComment(GreenLeaf leaf)
+    {
+        Assert.True((leaf.Flags & GreenNodeFlags.ContainsWhitespaceTrivia) != 0);
+        Assert.True((leaf.Flags & GreenNodeFlags.ContainsNewlineTrivia) == 0);
+        Assert.True((leaf.Flags & GreenNodeFlags.ContainsCommentTrivia) == 0);
+    }
+}

TinyTokenizer.Tests/GreenNodeFlagsTests.cs

@@ -0,0 +1,222 @@
+using System.Collections.Immutable;
+using TinyTokenizer.Ast;
+
+namespace TinyTokenizer.Tests;
+
+[Trait("Category", "Flags")]
+public class GreenNodeFlagsTests
+{
+    #region Helpers
+
+    private static ImmutableArray<GreenLeaf> ParseLeaves(string source, TokenizerOptions? options = null)
+    {
+        var lexer = new GreenLexer(options ?? TokenizerOptions.Default);
+        var nodes = lexer.ParseToGreenNodes(source);
+        return CollectLeaves(nodes);
+    }
+
+    private static ImmutableArray<GreenLeaf> CollectLeaves(ImmutableArray<GreenNode> nodes)
+    {
+        var builder = ImmutableArray.CreateBuilder<GreenLeaf>();
+        foreach (var node in nodes)
+        {
+            CollectLeavesRecursive(node, builder);
+        }
+        return builder.ToImmutable();
+    }
+
+    private static void CollectLeavesRecursive(GreenNode node, ImmutableArray<GreenLeaf>.Builder builder)
+    {
+        if (node is GreenLeaf leaf)
+        {
+            builder.Add(leaf);
+            return;
+        }
+
+        for (int i = 0; i < node.SlotCount; i++)
+        {
+            var child = node.GetSlot(i);
+            if (child != null)
+                CollectLeavesRecursive(child, builder);
+        }
+    }
+
+    private static GreenLeaf FindLeaf(ImmutableArray<GreenLeaf> leaves, string text) =>
+        Assert.Single(leaves.Where(l => l.Text == text));
+
+    private static void AssertHas(GreenNodeFlags flags, GreenNodeFlags expected)
+    {
+        Assert.True((flags & expected) == expected, $"Expected flags to include {expected} but was {flags}");
+    }
+
+    private static void AssertNotHas(GreenNodeFlags flags, GreenNodeFlags unexpected)
+    {
+        Assert.True((flags & unexpected) == 0, $"Expected flags to NOT include {unexpected} but was {flags}");
+    }
+
+    #endregion
+
+    [Fact]
+    public void GreenLeaf_Flags_ReflectLeadingAndTrailingTriviaKinds()
+    {
+        var leading = ImmutableArray.Create(
+            GreenTrivia.Whitespace("  "),
+            GreenTrivia.SingleLineComment("// c"),
+            GreenTrivia.Newline("\n"));
+
+        var trailing = ImmutableArray.Create(
+            GreenTrivia.Whitespace("\t"),
+            GreenTrivia.MultiLineComment("/* m */"));
+
+        var leaf = new GreenLeaf(NodeKind.Ident, "x", leading, trailing);
+
+        AssertHas(leaf.Flags, GreenNodeFlags.HasLeadingWhitespaceTrivia);
+        AssertHas(leaf.Flags, GreenNodeFlags.HasLeadingCommentTrivia);
+        AssertHas(leaf.Flags, GreenNodeFlags.HasLeadingNewlineTrivia);
+
+        AssertHas(leaf.Flags, GreenNodeFlags.HasTrailingWhitespaceTrivia);
+        AssertHas(leaf.Flags, GreenNodeFlags.HasTrailingCommentTrivia);
+        AssertNotHas(leaf.Flags, GreenNodeFlags.HasTrailingNewlineTrivia);
+
+        AssertHas(leaf.Flags, GreenNodeFlags.ContainsWhitespaceTrivia);
+        AssertHas(leaf.Flags, GreenNodeFlags.ContainsCommentTrivia);
+        AssertHas(leaf.Flags, GreenNodeFlags.ContainsNewlineTrivia);
+    }
+
+    [Fact]
+    public void GreenLeaf_Flags_IncludeKindDerivedSubtreeBits()
+    {
+        var keywordKind = NodeKindExtensions.KeywordKind(0);
+
+        var keywordLeaf = new GreenLeaf(keywordKind, "if");
+        AssertHas(keywordLeaf.Flags, GreenNodeFlags.ContainsKeyword);
+        AssertNotHas(keywordLeaf.Flags, GreenNodeFlagMasks.Boundary | GreenNodeFlags.ContainsWhitespaceTrivia | GreenNodeFlags.ContainsCommentTrivia | GreenNodeFlags.ContainsNewlineTrivia);
+
+        var taggedLeaf = new GreenLeaf(NodeKind.TaggedIdent, "#define");
+        AssertHas(taggedLeaf.Flags, GreenNodeFlags.ContainsTaggedIdent);
+
+        var errorLeaf = new GreenLeaf(NodeKind.Error, "<error>");
+        AssertHas(errorLeaf.Flags, GreenNodeFlags.ContainsErrorNode);
+
+        var normalLeaf = new GreenLeaf(NodeKind.Ident, "x");
+        Assert.Equal(GreenNodeFlags.None, normalLeaf.Flags);
+    }
+
+    [Fact]
+    public void GreenBlock_Flags_AggregateContains_AndUseOpenerLeadingPlusCloserTrailingAsBoundary()
+    {
+        var opener = new GreenLeaf(
+            NodeKind.Symbol,
+            "{",
+            leadingTrivia: ImmutableArray.Create(GreenTrivia.Newline("\n")),
+            trailingTrivia: ImmutableArray.Create(GreenTrivia.Whitespace(" ")));
+
+        var closer = new GreenLeaf(
+            NodeKind.Symbol,
+            "}",
+            leadingTrivia: ImmutableArray.Create(GreenTrivia.Newline("\n")),
+            trailingTrivia: ImmutableArray.Create(GreenTrivia.Whitespace(" ")));
+
+        var inner = new GreenLeaf(
+            NodeKind.Ident,
+            "x",
+            trailingTrivia: ImmutableArray.Create(GreenTrivia.SingleLineComment("// c")));
+
+        var block = new GreenBlock(opener, closer, ImmutableArray.Create<GreenNode>(inner));
+
+        // Boundary behavior
+        AssertHas(block.Flags, GreenNodeFlags.HasLeadingNewlineTrivia);
+        AssertHas(block.Flags, GreenNodeFlags.HasTrailingWhitespaceTrivia);
+
+        // Not from opener trailing / closer leading
+        AssertNotHas(block.Flags, GreenNodeFlags.HasLeadingWhitespaceTrivia);
+        AssertNotHas(block.Flags, GreenNodeFlags.HasTrailingNewlineTrivia);
+
+        // Subtree contains aggregation
+        AssertHas(block.Flags, GreenNodeFlags.ContainsNewlineTrivia);
+        AssertHas(block.Flags, GreenNodeFlags.ContainsWhitespaceTrivia);
+        AssertHas(block.Flags, GreenNodeFlags.ContainsCommentTrivia);
+    }
+
+    [Fact]
+    public void GreenList_Flags_UseFirstLeadingAndLastTrailingAsBoundary()
+    {
+        var first = new GreenLeaf(NodeKind.Ident, "a", leadingTrivia: ImmutableArray.Create(GreenTrivia.Whitespace("  ")));
+        var middle = new GreenLeaf(NodeKind.Ident, "b", trailingTrivia: ImmutableArray.Create(GreenTrivia.SingleLineComment("// c")));
+        var last = new GreenLeaf(NodeKind.Ident, "c", trailingTrivia: ImmutableArray.Create(GreenTrivia.Newline("\n")));
+
+        var list = new GreenList(ImmutableArray.Create<GreenNode>(first, middle, last));
+
+        AssertHas(list.Flags, GreenNodeFlags.HasLeadingWhitespaceTrivia);
+        AssertHas(list.Flags, GreenNodeFlags.HasTrailingNewlineTrivia);
+
+        AssertHas(list.Flags, GreenNodeFlags.ContainsWhitespaceTrivia);
+        AssertHas(list.Flags, GreenNodeFlags.ContainsCommentTrivia);
+        AssertHas(list.Flags, GreenNodeFlags.ContainsNewlineTrivia);
+    }
+
+    [Fact]
+    public void GreenSyntaxNode_Flags_AggregateLikeList()
+    {
+        var kind = NodeKindExtensions.SemanticKind(0);
+
+        var first = new GreenLeaf(NodeKind.Ident, "a", leadingTrivia: ImmutableArray.Create(GreenTrivia.Whitespace(" ")));
+        var last = new GreenLeaf(NodeKind.Ident, "b", trailingTrivia: ImmutableArray.Create(GreenTrivia.Newline("\n")));
+
+        var node = new GreenSyntaxNode(kind, first, last);
+
+        AssertHas(node.Flags, GreenNodeFlags.HasLeadingWhitespaceTrivia);
+        AssertHas(node.Flags, GreenNodeFlags.HasTrailingNewlineTrivia);
+        AssertHas(node.Flags, GreenNodeFlags.ContainsWhitespaceTrivia);
+        AssertHas(node.Flags, GreenNodeFlags.ContainsNewlineTrivia);
+    }
+
+    [Fact]
+    public void ParsedNewlineOwnership_ReflectedInFlags_CommonCase()
+    {
+        var leaves = ParseLeaves("a\nb");
+
+        var a = FindLeaf(leaves, "a");
+        var b = FindLeaf(leaves, "b");
+
+        AssertHas(a.Flags, GreenNodeFlags.HasTrailingNewlineTrivia);
+        AssertHas(a.Flags, GreenNodeFlags.ContainsNewlineTrivia);
+
+        AssertNotHas(b.Flags, GreenNodeFlags.HasLeadingNewlineTrivia);
+    }
+
+    [Fact]
+    public void ParsedSameLineCommentOwnership_ReflectedInFlags()
+    {
+        var options = TokenizerOptions.Default.WithCommentStyles(CommentStyle.CStyleSingleLine);
+        var leaves = ParseLeaves("a // c\nb", options);
+
+        var a = FindLeaf(leaves, "a");
+        var b = FindLeaf(leaves, "b");
+
+        AssertHas(a.Flags, GreenNodeFlags.HasTrailingCommentTrivia);
+        AssertHas(a.Flags, GreenNodeFlags.HasTrailingNewlineTrivia);
+        AssertHas(a.Flags, GreenNodeFlags.ContainsCommentTrivia);
+        AssertHas(a.Flags, GreenNodeFlags.ContainsNewlineTrivia);
+
+        Assert.Empty(b.LeadingTrivia);
+        AssertNotHas(b.Flags, GreenNodeFlags.HasLeadingCommentTrivia | GreenNodeFlags.HasLeadingNewlineTrivia);
+    }
+
+    [Fact]
+    public void ParsedOwnLineCommentOwnership_ReflectedInFlags()
+    {
+        var options = TokenizerOptions.Default.WithCommentStyles(CommentStyle.CStyleSingleLine);
+        var leaves = ParseLeaves("a\n// c\nb", options);
+
+        var a = FindLeaf(leaves, "a");
+        var b = FindLeaf(leaves, "b");
+
+        AssertHas(a.Flags, GreenNodeFlags.HasTrailingNewlineTrivia);
+
+        AssertHas(b.Flags, GreenNodeFlags.HasLeadingCommentTrivia);
+        AssertHas(b.Flags, GreenNodeFlags.HasLeadingNewlineTrivia);
+        AssertHas(b.Flags, GreenNodeFlags.ContainsCommentTrivia);
+        AssertHas(b.Flags, GreenNodeFlags.ContainsNewlineTrivia);
+    }
+}

TinyTokenizer/Ast/GreenBlock.cs

@@ -24,10 +24,14 @@ internal sealed record GreenBlock : GreenContainer
 
     private readonly ImmutableArray<GreenNode> _children;
     private readonly int _width;
+    private readonly GreenNodeFlags _flags;
     private readonly int[]? _childOffsets; // Pre-computed for ≥10 children (O(1) lookup)
 
     /// <inheritdoc/>
     public override NodeKind Kind { get; }
+
+    /// <inheritdoc/>
+    internal override GreenNodeFlags Flags => _flags;
 
     /// <summary>The opening delimiter node (e.g., '{', '[', '(') with its trivia.</summary>
     public GreenLeaf OpenerNode { get; }
@@ -75,6 +79,19 @@ public GreenBlock(
             childrenWidth += child.Width;
 
         _width = OpenerNode.Width + childrenWidth + CloserNode.Width;
+
+        // Flags
+        // - Boundary comes only from the first/last leaf boundaries (opener leading, closer trailing).
+        // - Subtree contains flags are ORed across opener/inner/closer (excluding boundary bits).
+        var boundary =
+            (OpenerNode.Flags & GreenNodeFlagMasks.LeadingBoundary) |
+            (CloserNode.Flags & GreenNodeFlagMasks.TrailingBoundary);
+
+        var contains = (OpenerNode.Flags | CloserNode.Flags) & GreenNodeFlagMasks.Contains;
+        foreach (var child in _children)
+            contains |= child.Flags & GreenNodeFlagMasks.Contains;
+
+        _flags = boundary | contains;
 
         // Pre-compute child offsets for large blocks
         if (_children.Length >= 10)