diff --git a/TinyTokenizer.Benchmarks/NewlineQueryBenchmarks.cs b/TinyTokenizer.Benchmarks/NewlineQueryBenchmarks.cs new file mode 100644 index 0000000..b071863 --- /dev/null +++ b/TinyTokenizer.Benchmarks/NewlineQueryBenchmarks.cs @@ -0,0 +1,72 @@ +using System.Text; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using TinyTokenizer.Ast; +using Q = TinyTokenizer.Ast.Query; + +namespace TinyTokenizer.Benchmarks; + +/// +/// Benchmarks for token-centric newline detection via Query.Newline. +/// Measures the cost of scanning a newline-heavy tree and matching nodes +/// that follow a newline (current leading newline OR previous sibling trailing newline). +/// +[MemoryDiagnoser] +[GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByCategory)] +[CategoriesColumn] +public class NewlineQueryBenchmarks +{ + [Params(1_000, 10_000)] + public int Lines { get; set; } + + private SyntaxTree _tree = null!; + + [GlobalSetup] + public void Setup() + { + var source = GenerateNewlineHeavyInput(Lines); + _tree = SyntaxTree.Parse(source); + } + + [Benchmark(Description = "Select(Query.Newline) - Count")] + [BenchmarkCategory("Query", "Newline")] + public int SelectNewline_Count() + { + int count = 0; + foreach (var _ in _tree.Select(Q.Newline)) + count++; + return count; + } + + [Benchmark(Description = "Select(Query.Newline.First()) - First match")] + [BenchmarkCategory("Query", "Newline")] + public int SelectNewline_First() + { + foreach (var node in _tree.Select(Q.Newline.First())) + return node.Position; + return -1; + } + + private static string GenerateNewlineHeavyInput(int lines) + { + // Intentionally mixes: trailing-newline ownership (end-of-line) and + // leading-newline ownership (own-line comments become leading trivia). + var builder = new StringBuilder(capacity: lines * 32); + + for (int i = 0; i < lines; i++) + { + builder.Append("x"); + builder.Append(i); + builder.Append(" = "); + builder.Append(i); + builder.Append(";\n"); + + if ((i & 7) == 0) + { + builder.Append("// comment\n"); + } + } + + return builder.ToString(); + } +} diff --git a/TinyTokenizer.Tests/GreenNodeFlagsTests.cs b/TinyTokenizer.Tests/GreenNodeFlagsTests.cs index e0a8973..cf50f54 100644 --- a/TinyTokenizer.Tests/GreenNodeFlagsTests.cs +++ b/TinyTokenizer.Tests/GreenNodeFlagsTests.cs @@ -139,7 +139,7 @@ public void GreenBlock_Flags_AggregateContains_AndUseOpenerLeadingPlusCloserTrai } [Fact] - public void GreenList_Flags_UseFirstLeadingAndLastTrailingAsBoundary() + public void GreenList_Flags_DoNotPropagateBoundaryFlags_FromChildren() { var first = new GreenLeaf(NodeKind.Ident, "a", leadingTrivia: ImmutableArray.Create(GreenTrivia.Whitespace(" "))); var middle = new GreenLeaf(NodeKind.Ident, "b", trailingTrivia: ImmutableArray.Create(GreenTrivia.SingleLineComment("// c"))); @@ -147,8 +147,8 @@ public void GreenList_Flags_UseFirstLeadingAndLastTrailingAsBoundary() var list = new GreenList(ImmutableArray.Create(first, middle, last)); - AssertHas(list.Flags, GreenNodeFlags.HasLeadingWhitespaceTrivia); - AssertHas(list.Flags, GreenNodeFlags.HasTrailingNewlineTrivia); + // Token-centric boundary semantics: lists do not own boundary trivia. + AssertNotHas(list.Flags, GreenNodeFlagMasks.Boundary); AssertHas(list.Flags, GreenNodeFlags.ContainsWhitespaceTrivia); AssertHas(list.Flags, GreenNodeFlags.ContainsCommentTrivia); @@ -165,8 +165,8 @@ public void GreenSyntaxNode_Flags_AggregateLikeList() var node = new GreenSyntaxNode(kind, first, last); - AssertHas(node.Flags, GreenNodeFlags.HasLeadingWhitespaceTrivia); - AssertHas(node.Flags, GreenNodeFlags.HasTrailingNewlineTrivia); + // Token-centric boundary semantics: syntax containers do not own boundary trivia. + AssertNotHas(node.Flags, GreenNodeFlagMasks.Boundary); AssertHas(node.Flags, GreenNodeFlags.ContainsWhitespaceTrivia); AssertHas(node.Flags, GreenNodeFlags.ContainsNewlineTrivia); } diff --git a/TinyTokenizer.Tests/NewlineQueryTests.cs b/TinyTokenizer.Tests/NewlineQueryTests.cs new file mode 100644 index 0000000..90b0505 --- /dev/null +++ b/TinyTokenizer.Tests/NewlineQueryTests.cs @@ -0,0 +1,121 @@ +using System.Linq; +using TinyTokenizer.Ast; +using Xunit; + +namespace TinyTokenizer.Tests; + +[Trait("Category", "Query")] +public sealed class NewlineQueryTests +{ + private static Schema CreateSyntaxBindingSchema() + { + return Schema.Create() + .DefineSyntax(Syntax.Define("FunctionCall") + .Match(Query.AnyIdent, Query.ParenBlock) + .Build()) + .Build(); + } + + [Fact] + public void Newline_MatchesNodeWithLeadingTriviaNewline_TopLevelFirstSibling() + { + var tree = SyntaxTree.Parse("\nfoo"); + var foo = tree.Root.Children.OfType().First(n => n.Kind == NodeKind.Ident); + + Assert.True(Query.Newline.Matches(foo)); + Assert.False(Query.NotNewline.Matches(foo)); + } + + [Fact] + public void Newline_MatchesNodeAfterNewlineViaPreviousSiblingTrailingTrivia_TopLevel() + { + var tree = SyntaxTree.Parse("x\ny"); + var idents = tree.Root.Children.OfType().Where(n => n.Kind == NodeKind.Ident).ToList(); + + Assert.Equal(2, idents.Count); + Assert.False(Query.Newline.Matches(idents[0])); + Assert.True(Query.Newline.Matches(idents[1])); + } + + [Fact] + public void Newline_MatchesFirstInnerNodeAfterOpenerViaPreviousSiblingTrailingTrivia_InBlock() + { + var tree = SyntaxTree.Parse("{\na}"); + var block = tree.Root.Children.OfType().Single(); + var a = block.InnerChildren.OfType().Single(n => n.Kind == NodeKind.Ident); + + Assert.True(Query.Newline.Matches(a)); + Assert.False(Query.NotNewline.Matches(a)); + } + + [Fact] + public void Newline_MatchesInnerNodeAfterNewlineBetweenSiblings_InBlock() + { + var tree = SyntaxTree.Parse("{a\nb}"); + var block = tree.Root.Children.OfType().Single(); + var idents = block.InnerChildren.OfType().Where(n => n.Kind == NodeKind.Ident).ToList(); + + Assert.Equal(2, idents.Count); + Assert.False(Query.Newline.Matches(idents[0])); + Assert.True(Query.Newline.Matches(idents[1])); + } + + [Fact] + public void NotNewline_IsExactNegationOfNewline_ForIdentifiersInSameTree() + { + var tree = SyntaxTree.Parse("a b\nc\n\nd"); + + var allIdents = tree.Select(Query.AnyIdent).ToList(); + var newlineIdents = tree.Select(Query.AnyIdent & Query.Newline).ToList(); + var notNewlineIdents = tree.Select(Query.AnyIdent & Query.NotNewline).ToList(); + + Assert.All(allIdents, n => Assert.True(newlineIdents.Contains(n) ^ notNewlineIdents.Contains(n))); + Assert.Empty(newlineIdents.Intersect(notNewlineIdents)); + Assert.Equal(allIdents.Count, newlineIdents.Count + notNewlineIdents.Count); + } + + [Fact] + public void Newline_DoesNotThrow_OnEmptyBlockOrEmptyTree() + { + var emptyTree = SyntaxTree.Parse(string.Empty); + Assert.Empty(emptyTree.Select(Query.Newline)); + + var emptyBlockTree = SyntaxTree.Parse("{}"); + Assert.Empty(emptyBlockTree.Select(Query.Newline)); + } + + [Fact] + public void Newline_MatchesCloserAfterNewlineInEmptyBlock() + { + var tree = SyntaxTree.Parse("{\n}"); + var block = tree.Root.Children.OfType().Single(); + + Assert.True(Query.Newline.Matches(block.CloserNode)); + Assert.False(Query.Newline.Matches(block.OpenerNode)); + } + + [Fact] + public void Newline_DoesNotMatchRootContainerNode() + { + var tree = SyntaxTree.Parse("\nfoo"); + + Assert.False(Query.Newline.Matches(tree.Root)); + Assert.DoesNotContain(tree.Root, tree.Select(Query.Newline)); + } + + [Fact] + public void Newline_DoesNotMatchBoundSyntaxContainerNode() + { + var schema = CreateSyntaxBindingSchema(); + var tree = SyntaxTree.Parse("\nfoo()", schema); + + var funcCall = tree.Root.Children.OfType().First(); + + // Token-centric newline: matches tokens, not the bound syntax container. + Assert.False(Query.Newline.Matches(funcCall)); + Assert.DoesNotContain(funcCall, tree.Select(Query.Newline)); + + // Still matches the first token after newline. + Assert.True(Query.Newline.Matches(funcCall.NameNode)); + } +} diff --git a/TinyTokenizer.Tests/SyntaxEditorTests.cs b/TinyTokenizer.Tests/SyntaxEditorTests.cs index d800955..eaa9a68 100644 --- a/TinyTokenizer.Tests/SyntaxEditorTests.cs +++ b/TinyTokenizer.Tests/SyntaxEditorTests.cs @@ -1614,6 +1614,636 @@ public void InsertAfter_InsertedTextWithTrailingNewline_PreservesFollowingTrivia Assert.Equal(2, bAfter.LeadingTriviaWidth); // b retains its " " leading trivia } + #region Green Flag Mutation Tests + + private static void AssertHasFlags(GreenNodeFlags actual, GreenNodeFlags expected) + { + Assert.True((actual & expected) == expected, $"Expected flags to include {expected} but was {actual}"); + } + + private static void AssertNotHasFlags(GreenNodeFlags actual, GreenNodeFlags unexpected) + { + Assert.True((actual & unexpected) == 0, $"Expected flags to NOT include {unexpected} but was {actual}"); + } + + private static SyntaxToken FindToken(SyntaxTree tree, NodeKind kind, string text, int occurrence = 0) + { + ArgumentNullException.ThrowIfNull(tree); + ArgumentNullException.ThrowIfNull(text); + + var matches = tree + .Select(Query.Kind(kind).WithText(text)) + .OfType() + .ToList(); + + Assert.True(matches.Count > 0, $"Expected to find at least 1 token of kind '{kind}' with text '{text}', but found none."); + Assert.True( + occurrence >= 0 && occurrence < matches.Count, + $"Expected occurrence {occurrence} for token kind '{kind}' text '{text}', but only found {matches.Count} match(es)." + ); + + return matches[occurrence]; + } + + private static void AssertReparseOracleFlagsMatch(SyntaxTree tree, SyntaxTree oracle) + { + ArgumentNullException.ThrowIfNull(tree); + ArgumentNullException.ThrowIfNull(oracle); + + Assert.Equal(oracle.GreenRoot.Flags, tree.GreenRoot.Flags); + + var treeLeaves = tree.Leaves.ToList(); + var oracleLeaves = oracle.Leaves.ToList(); + Assert.Equal(oracleLeaves.Count, treeLeaves.Count); + + for (int i = 0; i < treeLeaves.Count; i++) + { + Assert.Equal(oracleLeaves[i].Kind, treeLeaves[i].Kind); + Assert.Equal(oracleLeaves[i].Text, treeLeaves[i].Text); + + var expected = oracleLeaves[i].Green.Flags; + var actual = treeLeaves[i].Green.Flags; + Assert.True( + expected == actual, + $"Leaf flags mismatch at index {i}: {treeLeaves[i].Kind} '{treeLeaves[i].Text}'. Expected={expected} Actual={actual}" + ); + } + } + + [Fact] + public void Replace_OwnLineCommentLeadingTrivia_PreservesGreenBoundaryFlags_OnReplacement() + { + var options = TokenizerOptions.Default.WithCommentStyles(CommentStyle.CStyleSingleLine); + var tree = SyntaxTree.Parse("a\n// c\nb", options); + + var bBefore = Assert.Single(tree.Select(Q.Ident("b")).OfType()); + AssertHasFlags(bBefore.Green.Flags, GreenNodeFlags.HasLeadingNewlineTrivia | GreenNodeFlags.HasLeadingCommentTrivia); + + tree.CreateEditor() + .Replace(Q.Ident("b"), "X") + .Commit(); + + var xAfter = Assert.Single(tree.Select(Q.Ident("X")).OfType()); + AssertHasFlags(xAfter.Green.Flags, GreenNodeFlags.HasLeadingNewlineTrivia | GreenNodeFlags.HasLeadingCommentTrivia); + AssertHasFlags(xAfter.Green.Flags, GreenNodeFlags.ContainsNewlineTrivia | GreenNodeFlags.ContainsCommentTrivia); + } + + [Fact] + public void Replace_TokenOwningTrailingNewline_PreservesGreenBoundaryFlags_OnReplacement() + { + var tree = SyntaxTree.Parse("a\nb"); + + var aBefore = FindToken(tree, NodeKind.Ident, "a"); + AssertHasFlags(aBefore.Green.Flags, GreenNodeFlags.HasTrailingNewlineTrivia); + + tree.CreateEditor() + .Replace(Q.Ident("a"), "X") + .Commit(); + + var xAfter = FindToken(tree, NodeKind.Ident, "X"); + var bAfter = FindToken(tree, NodeKind.Ident, "b"); + + AssertHasFlags(xAfter.Green.Flags, GreenNodeFlags.HasTrailingNewlineTrivia); + AssertNotHasFlags(bAfter.Green.Flags, GreenNodeFlags.HasLeadingNewlineTrivia); + AssertHasFlags(tree.GreenRoot.Flags, GreenNodeFlags.ContainsNewlineTrivia); + } + + [Fact] + public void Replace_SameLineCommentTrailingTrivia_PreservesGreenBoundaryFlags_OnReplacement() + { + var options = TokenizerOptions.Default.WithCommentStyles(CommentStyle.CStyleSingleLine); + var tree = SyntaxTree.Parse("a // c\nb", options); + + var aBefore = FindToken(tree, NodeKind.Ident, "a"); + AssertHasFlags(aBefore.Green.Flags, GreenNodeFlags.HasTrailingCommentTrivia | GreenNodeFlags.HasTrailingNewlineTrivia); + + tree.CreateEditor() + .Replace(Q.Ident("a"), "X") + .Commit(); + + var xAfter = FindToken(tree, NodeKind.Ident, "X"); + var bAfter = FindToken(tree, NodeKind.Ident, "b"); + + AssertHasFlags(xAfter.Green.Flags, GreenNodeFlags.HasTrailingCommentTrivia | GreenNodeFlags.HasTrailingNewlineTrivia); + AssertNotHasFlags(bAfter.Green.Flags, GreenNodeFlags.HasLeadingCommentTrivia | GreenNodeFlags.HasLeadingNewlineTrivia); + AssertHasFlags(tree.GreenRoot.Flags, GreenNodeFlags.ContainsCommentTrivia | GreenNodeFlags.ContainsNewlineTrivia); + } + + [Fact] + public void Replace_TokenOwningTrailingWhitespace_PreservesGreenBoundaryFlags_OnReplacement() + { + var tree = SyntaxTree.Parse("a b"); + + var aBefore = FindToken(tree, NodeKind.Ident, "a"); + AssertHasFlags(aBefore.Green.Flags, GreenNodeFlags.HasTrailingWhitespaceTrivia); + + tree.CreateEditor() + .Replace(Q.Ident("a"), "X") + .Commit(); + + var xAfter = FindToken(tree, NodeKind.Ident, "X"); + var bAfter = FindToken(tree, NodeKind.Ident, "b"); + + AssertHasFlags(xAfter.Green.Flags, GreenNodeFlags.HasTrailingWhitespaceTrivia); + AssertNotHasFlags(bAfter.Green.Flags, GreenNodeFlags.HasLeadingWhitespaceTrivia); + AssertHasFlags(tree.GreenRoot.Flags, GreenNodeFlags.ContainsWhitespaceTrivia); + } + + [Fact] + public void InsertAfter_InsertedTextWithTrailingNewline_SetsGreenFlags_OnInsertedAndFollowingTokens() + { + var tree = SyntaxTree.Parse("a b"); + var aNode = Assert.Single(tree.Select(Q.Ident("a")).OfType()); + + tree.CreateEditor() + .InsertAfter(aNode, " X\n") + .Commit(); + + var xAfter = Assert.Single(tree.Select(Q.Ident("X")).OfType()); + var bAfter = Assert.Single(tree.Select(Q.Ident("b")).OfType()); + + // Inserted token owns the trailing newline; following token should NOT gain leading newline ownership. + AssertHasFlags(xAfter.Green.Flags, GreenNodeFlags.HasTrailingNewlineTrivia | GreenNodeFlags.ContainsNewlineTrivia); + AssertNotHasFlags(bAfter.Green.Flags, GreenNodeFlags.HasLeadingNewlineTrivia); + + // Root list should reflect subtree contains. + AssertHasFlags(tree.GreenRoot.Flags, GreenNodeFlags.ContainsNewlineTrivia); + } + + [Fact] + public void InsertBefore_DoesNotStealLeadingWhitespace_FromFollowingToken_GreenFlags() + { + // In the token-centric trivia model, indentation after a newline is leading trivia on the following token. + var tree = SyntaxTree.Parse("a\n b"); + + var bBefore = FindToken(tree, NodeKind.Ident, "b"); + AssertHasFlags(bBefore.Green.Flags, GreenNodeFlags.HasLeadingWhitespaceTrivia); + + tree.CreateEditor() + .InsertBefore(bBefore, "X") + .Commit(); + + var xAfter = FindToken(tree, NodeKind.Ident, "X"); + var bAfter = FindToken(tree, NodeKind.Ident, "b"); + + // Insertion must not transfer whitespace ownership from b to X. + AssertHasFlags(bAfter.Green.Flags, GreenNodeFlags.HasLeadingWhitespaceTrivia); + AssertNotHasFlags(xAfter.Green.Flags, GreenNodeFlags.HasLeadingWhitespaceTrivia); + + // Root should reflect subtree contains. + AssertHasFlags(tree.GreenRoot.Flags, GreenNodeFlags.ContainsWhitespaceTrivia); + } + + [Fact] + public void InsertAfter_InsertedTextWithTrailingCRLF_SetsGreenFlags_OnInsertedAndFollowingTokens() + { + var tree = SyntaxTree.Parse("a b"); + var aNode = FindToken(tree, NodeKind.Ident, "a"); + + tree.CreateEditor() + .InsertAfter(aNode, " X\r\n") + .Commit(); + + var xAfter = FindToken(tree, NodeKind.Ident, "X"); + var bAfter = FindToken(tree, NodeKind.Ident, "b"); + + // Inserted token owns the trailing newline; following token should NOT gain leading newline ownership. + AssertHasFlags(xAfter.Green.Flags, GreenNodeFlags.HasTrailingNewlineTrivia | GreenNodeFlags.ContainsNewlineTrivia); + AssertNotHasFlags(bAfter.Green.Flags, GreenNodeFlags.HasLeadingNewlineTrivia); + + AssertHasFlags(tree.GreenRoot.Flags, GreenNodeFlags.ContainsNewlineTrivia); + } + + [Fact] + public void Remove_TokenOwningTrailingNewline_RemovesNewlineBoundaryAndContainsFlags() + { + var tree = SyntaxTree.Parse("a\nb"); + var aNode = Assert.Single(tree.Select(Q.Ident("a")).OfType()); + + // Sanity: 'a' owns the trailing newline in the token-centric trivia model. + AssertHasFlags(aNode.Green.Flags, GreenNodeFlags.HasTrailingNewlineTrivia); + AssertHasFlags(tree.GreenRoot.Flags, GreenNodeFlags.ContainsNewlineTrivia); + + tree.CreateEditor() + .Remove(Q.Ident("a")) + .Commit(); + + Assert.Equal("b", tree.ToText()); + var bAfter = Assert.Single(tree.Select(Q.Ident("b")).OfType()); + AssertNotHasFlags(bAfter.Green.Flags, GreenNodeFlags.HasLeadingNewlineTrivia | GreenNodeFlags.HasTrailingNewlineTrivia); + AssertNotHasFlags(tree.GreenRoot.Flags, GreenNodeFlags.ContainsNewlineTrivia); + } + + [Fact] + public void Remove_OneNewlineOwner_DoesNotClearContainsNewlineTrivia_WhenOthersRemain() + { + var tree = SyntaxTree.Parse("a\nb\nc"); + AssertHasFlags(tree.GreenRoot.Flags, GreenNodeFlags.ContainsNewlineTrivia); + + tree.CreateEditor() + .Remove(Q.Ident("a")) + .Commit(); + + // The newline between b and c should still exist. + AssertHasFlags(tree.GreenRoot.Flags, GreenNodeFlags.ContainsNewlineTrivia); + } + + [Fact] + public void Remove_OneCommentOwner_DoesNotClearContainsCommentTrivia_WhenOthersRemain() + { + var options = TokenizerOptions.Default.WithCommentStyles(CommentStyle.CStyleSingleLine); + var tree = SyntaxTree.Parse("a // c1\nb // c2\nc", options); + AssertHasFlags(tree.GreenRoot.Flags, GreenNodeFlags.ContainsCommentTrivia); + + tree.CreateEditor() + .Remove(Q.Ident("a")) + .Commit(); + + // The comment after b should still exist. + AssertHasFlags(tree.GreenRoot.Flags, GreenNodeFlags.ContainsCommentTrivia); + } + + [Fact] + public void Remove_OneWhitespaceOwner_DoesNotClearContainsWhitespaceTrivia_WhenOthersRemain() + { + // Two separate whitespace regions: after 'a' and after 'b'. + var tree = SyntaxTree.Parse("a b c"); + AssertHasFlags(tree.GreenRoot.Flags, GreenNodeFlags.ContainsWhitespaceTrivia); + + tree.CreateEditor() + .Remove(Q.Ident("a")) + .Commit(); + + // The remaining space between b and c should still exist. + AssertHasFlags(tree.GreenRoot.Flags, GreenNodeFlags.ContainsWhitespaceTrivia); + } + + [Fact] + public void Replace_MultipleNodes_TransfersLeadingToFirst_AndTrailingToLast_GreenBoundaryFlags() + { + // b owns leading whitespace (indentation) and trailing newline. + var tree = SyntaxTree.Parse("a\n b\nc"); + + var bBefore = FindToken(tree, NodeKind.Ident, "b"); + AssertHasFlags(bBefore.Green.Flags, GreenNodeFlags.HasLeadingWhitespaceTrivia | GreenNodeFlags.HasTrailingNewlineTrivia); + + tree.CreateEditor() + .Replace(Q.Ident("b"), "X Y") + .Commit(); + + var xAfter = FindToken(tree, NodeKind.Ident, "X"); + var yAfter = FindToken(tree, NodeKind.Ident, "Y"); + var cAfter = FindToken(tree, NodeKind.Ident, "c"); + + // Leading boundary transfers to first replacement node. + AssertHasFlags(xAfter.Green.Flags, GreenNodeFlags.HasLeadingWhitespaceTrivia); + AssertNotHasFlags(xAfter.Green.Flags, GreenNodeFlags.HasTrailingNewlineTrivia); + + // Trailing boundary transfers to last replacement node. + AssertHasFlags(yAfter.Green.Flags, GreenNodeFlags.HasTrailingNewlineTrivia); + AssertNotHasFlags(yAfter.Green.Flags, GreenNodeFlags.HasLeadingWhitespaceTrivia); + + // Following token should not incorrectly gain leading newline ownership. + AssertNotHasFlags(cAfter.Green.Flags, GreenNodeFlags.HasLeadingNewlineTrivia); + + AssertHasFlags(tree.GreenRoot.Flags, GreenNodeFlags.ContainsWhitespaceTrivia | GreenNodeFlags.ContainsNewlineTrivia); + } + + [Fact] + public void Replace_WithBlockAtEdges_TransfersTriviaToBlockBoundaries_GreenFlags() + { + // Replacement begins/ends with a container (block). Trivia should attach to opener/closer boundaries. + var tree = SyntaxTree.Parse("a\n b\nc"); + + var bBefore = FindToken(tree, NodeKind.Ident, "b"); + AssertHasFlags(bBefore.Green.Flags, GreenNodeFlags.HasLeadingWhitespaceTrivia | GreenNodeFlags.HasTrailingNewlineTrivia); + + tree.CreateEditor() + .Replace(Q.Ident("b"), "{x}") + .Commit(); + + var block = Assert.Single(tree.Select(Q.BraceBlock).OfType()); + + // Boundary trivia should be preserved on the block boundaries. + AssertHasFlags(block.Green.Flags, GreenNodeFlags.HasLeadingWhitespaceTrivia | GreenNodeFlags.HasTrailingNewlineTrivia); + + // Containers must not accidentally carry child boundary flags beyond their own semantics. + // (Block boundary flags are only opener-leading and closer-trailing.) + AssertHasFlags(tree.GreenRoot.Flags, GreenNodeFlags.ContainsWhitespaceTrivia | GreenNodeFlags.ContainsNewlineTrivia); + } + + [Fact] + public void UndoRedo_RestoresLeafBoundaryFlags_NotJustRootFlags() + { + var tree = SyntaxTree.Parse("a b"); + var aBefore = FindToken(tree, NodeKind.Ident, "a"); + var bBefore = FindToken(tree, NodeKind.Ident, "b"); + + var aFlagsBefore = aBefore.Green.Flags; + var bFlagsBefore = bBefore.Green.Flags; + var rootFlagsBefore = tree.GreenRoot.Flags; + + tree.CreateEditor() + .InsertAfter(aBefore, " X\n") + .Commit(); + + var aAfter = FindToken(tree, NodeKind.Ident, "a"); + var xAfter = FindToken(tree, NodeKind.Ident, "X"); + var bAfter = FindToken(tree, NodeKind.Ident, "b"); + + var aFlagsAfter = aAfter.Green.Flags; + var xFlagsAfter = xAfter.Green.Flags; + var bFlagsAfter = bAfter.Green.Flags; + var rootFlagsAfter = tree.GreenRoot.Flags; + + // Sanity: mutation should introduce a newline owner. + AssertHasFlags(xFlagsAfter, GreenNodeFlags.HasTrailingNewlineTrivia); + AssertHasFlags(rootFlagsAfter, GreenNodeFlags.ContainsNewlineTrivia); + + Assert.True(tree.Undo()); + + var aUndo = FindToken(tree, NodeKind.Ident, "a"); + var bUndo = FindToken(tree, NodeKind.Ident, "b"); + Assert.Equal(aFlagsBefore, aUndo.Green.Flags); + Assert.Equal(bFlagsBefore, bUndo.Green.Flags); + Assert.Equal(rootFlagsBefore, tree.GreenRoot.Flags); + + Assert.True(tree.Redo()); + + var aRedo = FindToken(tree, NodeKind.Ident, "a"); + var xRedo = FindToken(tree, NodeKind.Ident, "X"); + var bRedo = FindToken(tree, NodeKind.Ident, "b"); + Assert.Equal(aFlagsAfter, aRedo.Green.Flags); + Assert.Equal(xFlagsAfter, xRedo.Green.Flags); + Assert.Equal(bFlagsAfter, bRedo.Green.Flags); + Assert.Equal(rootFlagsAfter, tree.GreenRoot.Flags); + } + + [Fact] + public void SchemaRebind_ReplaceInsideSyntaxNode_PreservesLeafBoundaryFlags_AndKeepsSyntaxContainersBoundaryFree() + { + const GreenNodeFlags boundaryMask = + GreenNodeFlags.HasLeadingNewlineTrivia | + GreenNodeFlags.HasTrailingNewlineTrivia | + GreenNodeFlags.HasLeadingWhitespaceTrivia | + GreenNodeFlags.HasTrailingWhitespaceTrivia | + GreenNodeFlags.HasLeadingCommentTrivia | + GreenNodeFlags.HasTrailingCommentTrivia; + + var schema = Schema.Create() + .WithCommentStyles(CommentStyle.CStyleSingleLine) + .WithTagPrefixes('@') + .DefineSyntax(Syntax.Define("testTagged") + .Match(Q.AnyTaggedIdent, Q.AnyString) + .Build()) + .Build(); + + var tree = SyntaxTree.Parse("before\n @tag \"value\" // c\nnext", schema); + + var syntaxBefore = Assert.Single(tree.Select(Q.Syntax()).OfType()); + AssertNotHasFlags(syntaxBefore.Green.Flags, boundaryMask); + + var tagBefore = FindToken(tree, NodeKind.TaggedIdent, "@tag"); + var valueBefore = FindToken(tree, NodeKind.String, "\"value\""); + var nextBefore = FindToken(tree, NodeKind.Ident, "next"); + + // Indentation is leading whitespace on the first token inside the syntax node. + AssertHasFlags(tagBefore.Green.Flags, GreenNodeFlags.HasLeadingWhitespaceTrivia); + + // The string token owns same-line comment + newline. + AssertHasFlags(valueBefore.Green.Flags, GreenNodeFlags.HasTrailingCommentTrivia | GreenNodeFlags.HasTrailingNewlineTrivia); + AssertNotHasFlags(nextBefore.Green.Flags, GreenNodeFlags.HasLeadingCommentTrivia | GreenNodeFlags.HasLeadingNewlineTrivia); + + tree.CreateEditor() + .Replace(Q.String("\"value\""), "\"X\"") + .Commit(); + + var syntaxAfter = Assert.Single(tree.Select(Q.Syntax()).OfType()); + AssertNotHasFlags(syntaxAfter.Green.Flags, boundaryMask); + + var tagAfter = FindToken(tree, NodeKind.TaggedIdent, "@tag"); + var xAfter = FindToken(tree, NodeKind.String, "\"X\""); + var nextAfter = FindToken(tree, NodeKind.Ident, "next"); + + AssertHasFlags(tagAfter.Green.Flags, GreenNodeFlags.HasLeadingWhitespaceTrivia); + AssertHasFlags(xAfter.Green.Flags, GreenNodeFlags.HasTrailingCommentTrivia | GreenNodeFlags.HasTrailingNewlineTrivia); + AssertNotHasFlags(nextAfter.Green.Flags, GreenNodeFlags.HasLeadingCommentTrivia | GreenNodeFlags.HasLeadingNewlineTrivia); + + AssertHasFlags(tree.GreenRoot.Flags, GreenNodeFlags.ContainsWhitespaceTrivia | GreenNodeFlags.ContainsCommentTrivia | GreenNodeFlags.ContainsNewlineTrivia); + } + + [Fact] + public void SchemaRebind_InsertBeforeSyntaxNode_DoesNotStealLeadingWhitespace_AndKeepsSyntaxContainersBoundaryFree() + { + const GreenNodeFlags boundaryMask = + GreenNodeFlags.HasLeadingNewlineTrivia | + GreenNodeFlags.HasTrailingNewlineTrivia | + GreenNodeFlags.HasLeadingWhitespaceTrivia | + GreenNodeFlags.HasTrailingWhitespaceTrivia | + GreenNodeFlags.HasLeadingCommentTrivia | + GreenNodeFlags.HasTrailingCommentTrivia; + + var schema = Schema.Create() + .WithTagPrefixes('@') + .DefineSyntax(Syntax.Define("testTagged") + .Match(Q.AnyTaggedIdent, Q.AnyString) + .Build()) + .Build(); + + var tree = SyntaxTree.Parse("before\n @tag \"value\"\nafter", schema); + + var syntaxBefore = Assert.Single(tree.Select(Q.Syntax()).OfType()); + AssertNotHasFlags(syntaxBefore.Green.Flags, boundaryMask); + + var tagBefore = FindToken(tree, NodeKind.TaggedIdent, "@tag"); + AssertHasFlags(tagBefore.Green.Flags, GreenNodeFlags.HasLeadingWhitespaceTrivia); + + tree.CreateEditor() + .InsertBefore(Q.Syntax(), "X\n") + .Commit(); + + var syntaxAfter = Assert.Single(tree.Select(Q.Syntax()).OfType()); + AssertNotHasFlags(syntaxAfter.Green.Flags, boundaryMask); + + var xAfter = FindToken(tree, NodeKind.Ident, "X"); + var tagAfter = FindToken(tree, NodeKind.TaggedIdent, "@tag"); + var afterAfter = FindToken(tree, NodeKind.Ident, "after"); + + // Inserted token owns its trailing newline; syntax node's first token keeps its indentation. + AssertHasFlags(xAfter.Green.Flags, GreenNodeFlags.HasTrailingNewlineTrivia); + AssertHasFlags(tagAfter.Green.Flags, GreenNodeFlags.HasLeadingWhitespaceTrivia); + AssertNotHasFlags(afterAfter.Green.Flags, GreenNodeFlags.HasLeadingNewlineTrivia); + + AssertHasFlags(tree.GreenRoot.Flags, GreenNodeFlags.ContainsWhitespaceTrivia | GreenNodeFlags.ContainsNewlineTrivia); + } + + [Fact] + public void ReparseOracle_AfterComplexEdit_MatchesRootAndLeafFlags_WithOptions() + { + var options = TokenizerOptions.Default + .WithCommentStyles(CommentStyle.CStyleSingleLine, CommentStyle.CStyleMultiLine); + + const string source = "a{\n b // c1\n d(e) /* c2 */\n}\n"; + var tree = SyntaxTree.Parse(source, options); + + tree.CreateEditor(options) + .Replace(Q.Ident("b"), "{x}") + .InsertAfter(Q.Ident("d"), ".Y\n") + .Commit(); + + var editedText = tree.ToText(); + var oracle = SyntaxTree.Parse(editedText, options); + + AssertReparseOracleFlagsMatch(tree, oracle); + } + + [Fact] + public void ReparseOracle_AfterComplexEdit_MatchesRootAndLeafFlags_WithSchemaAndBinding() + { + var schema = Schema.Create() + .WithCommentStyles(CommentStyle.CStyleSingleLine, CommentStyle.CStyleMultiLine) + .WithTagPrefixes('@') + .DefineSyntax(Syntax.Define("testTagged") + .Match(Q.AnyTaggedIdent, Q.AnyString) + .Build()) + .Build(); + + var tree = SyntaxTree.Parse("before\n @tag \"value\" // c\nafter", schema); + + tree.CreateEditor() + .InsertBefore(Q.Syntax(), "X\n") + .Replace(Q.String("\"value\""), "\"Z\"") + .Commit(); + + var editedText = tree.ToText(); + var oracle = SyntaxTree.Parse(editedText, schema); + + AssertReparseOracleFlagsMatch(tree, oracle); + } + + [Fact] + public void ReparseOracle_AfterRemoveReplaceAndCRLFInsert_MatchesRootAndLeafFlags_WithOptions() + { + var options = TokenizerOptions.Default + .WithCommentStyles(CommentStyle.CStyleSingleLine, CommentStyle.CStyleMultiLine); + + const string source = "a // c1\r\n b /* c2 */\r\nc d"; + var tree = SyntaxTree.Parse(source, options); + + tree.CreateEditor(options) + // Remove token that owns trailing comment+newline + .Remove(Q.Ident("a")) + // Replace a token that owns leading indentation + .Replace(Q.Ident("b"), "{x}") + // Insert a token that owns CRLF; avoid leading trivia by starting with a symbol + .InsertAfter(Q.Ident("c"), ".Y\r\n") + .Commit(); + + var editedText = tree.ToText(); + var oracle = SyntaxTree.Parse(editedText, options); + + AssertReparseOracleFlagsMatch(tree, oracle); + } + + [Fact] + public void ReparseOracle_AfterEditsOnMultipleSyntaxNodes_MatchesRootAndLeafFlags_WithSchemaAndBinding() + { + var schema = Schema.Create() + .WithCommentStyles(CommentStyle.CStyleSingleLine) + .WithTagPrefixes('@') + .DefineSyntax(Syntax.Define("testTagged") + .Match(Q.AnyTaggedIdent, Q.AnyString) + .Build()) + .DefineSyntax(Syntax.Define("funcCall") + .Match(Q.AnyIdent, Q.ParenBlock) + .Build()) + .Build(); + + var tree = SyntaxTree.Parse("@tag \"value\"\nfoo(a, b)\n", schema); + + tree.CreateEditor() + .Replace(Q.String("\"value\""), "\"Z\"") + // Insert inside the function call argument list. Start with a symbol to avoid leading trivia. + .InsertAfter(Q.Ident("a"), ",c") + .Commit(); + + var editedText = tree.ToText(); + var oracle = SyntaxTree.Parse(editedText, schema); + + AssertReparseOracleFlagsMatch(tree, oracle); + } + + [Fact] + public void ReparseOracle_AfterDeeplyNestedBlockEdits_MatchesRootAndLeafFlags_WithOptions() + { + var options = TokenizerOptions.Default + .WithCommentStyles(CommentStyle.CStyleSingleLine, CommentStyle.CStyleMultiLine); + + const string source = + "root {\n" + + " a(b[c{d(e)}]) // c1\n" + + " { x /* c2 */ }\n" + + "}\n"; + + var tree = SyntaxTree.Parse(source, options); + + // Perform multiple edits inside deeply nested structures. + tree.CreateEditor(options) + // Replace an inner identifier. + .Replace(Q.Ident("d"), "D") + // Insert inside the deepest paren. Start with a symbol to avoid leading-trivia ambiguity. + .InsertAfter(Q.Ident("e"), ".Y\n") + // Replace an identifier with a block to exercise block-edge trivia semantics. + .Replace(Q.Ident("x"), "{z}") + .Commit(); + + var editedText = tree.ToText(); + var oracle = SyntaxTree.Parse(editedText, options); + + AssertReparseOracleFlagsMatch(tree, oracle); + } + + [Fact] + public void InsertAfter_BlockContainsNewlineFlag_UpdatesAfterMutation() + { + var tree = SyntaxTree.Parse("{a}"); + var blockBefore = (SyntaxBlock)Assert.Single(tree.Select(Q.BraceBlock)); + + AssertNotHasFlags(blockBefore.Green.Flags, GreenNodeFlags.ContainsNewlineTrivia); + + var aNode = Assert.Single(tree.Select(Q.Ident("a")).OfType()); + tree.CreateEditor() + .InsertAfter(aNode, "X\n") + .Commit(); + + var blockAfter = (SyntaxBlock)Assert.Single(tree.Select(Q.BraceBlock)); + AssertHasFlags(blockAfter.Green.Flags, GreenNodeFlags.ContainsNewlineTrivia); + } + + [Fact] + public void GreenFlags_UndoRedo_RestoreFlagState_AfterMutation() + { + var tree = SyntaxTree.Parse("a b"); + var before = tree.GreenRoot.Flags; + AssertNotHasFlags(before, GreenNodeFlags.ContainsNewlineTrivia); + + var aNode = Assert.Single(tree.Select(Q.Ident("a")).OfType()); + tree.CreateEditor() + .InsertAfter(aNode, "X\n") + .Commit(); + + var after = tree.GreenRoot.Flags; + AssertHasFlags(after, GreenNodeFlags.ContainsNewlineTrivia); + + Assert.True(tree.Undo()); + Assert.Equal(before, tree.GreenRoot.Flags); + + Assert.True(tree.Redo()); + Assert.Equal(after, tree.GreenRoot.Flags); + } + + #endregion + /// /// Tests that block opener trivia follows the correct trivia model: /// - Trailing trivia: up to AND INCLUDING the newline diff --git a/TinyTokenizer/Ast/GreenBlock.cs b/TinyTokenizer/Ast/GreenBlock.cs index 115f71d..6a09b47 100644 --- a/TinyTokenizer/Ast/GreenBlock.cs +++ b/TinyTokenizer/Ast/GreenBlock.cs @@ -24,14 +24,10 @@ internal sealed record GreenBlock : GreenContainer private readonly ImmutableArray _children; private readonly int _width; - private readonly GreenNodeFlags _flags; private readonly int[]? _childOffsets; // Pre-computed for ≥10 children (O(1) lookup) /// public override NodeKind Kind { get; } - - /// - internal override GreenNodeFlags Flags => _flags; /// The opening delimiter node (e.g., '{', '[', '(') with its trivia. public GreenLeaf OpenerNode { get; } @@ -67,44 +63,66 @@ public GreenBlock( GreenLeaf openerNode, GreenLeaf closerNode, ImmutableArray children) + : this( + openerNode, + closerNode, + children.IsDefault ? ImmutableArray.Empty : children, + Compute(openerNode, closerNode, children.IsDefault ? ImmutableArray.Empty : children)) + { + } + + private GreenBlock( + GreenLeaf openerNode, + GreenLeaf closerNode, + ImmutableArray children, + BlockComputed computed) + : base(computed.Flags) { OpenerNode = openerNode; CloserNode = closerNode; Kind = GetBlockKind(Opener); - _children = children.IsDefault ? ImmutableArray.Empty : children; - - // Compute width: opener (with trivia) + children + closer (with trivia) + _children = children; + _width = computed.Width; + _childOffsets = computed.ChildOffsets; + } + + private static BlockComputed Compute(GreenLeaf openerNode, GreenLeaf closerNode, ImmutableArray children) + { int childrenWidth = 0; - foreach (var child in _children) + + int[]? offsets = null; + if (children.Length >= 10) + offsets = new int[children.Length]; + + int offset = openerNode.Width; + for (int i = 0; i < children.Length; i++) + { + if (offsets != null) + offsets[i] = offset; + + var child = children[i]; childrenWidth += child.Width; - - _width = OpenerNode.Width + childrenWidth + CloserNode.Width; + offset += child.Width; + } + + int width = openerNode.Width + childrenWidth + closerNode.Width; // Flags // - Boundary comes only from the first/last leaf boundaries (opener leading, closer trailing). // - Subtree contains flags are ORed across opener/inner/closer (excluding boundary bits). var boundary = - (OpenerNode.Flags & GreenNodeFlagMasks.LeadingBoundary) | - (CloserNode.Flags & GreenNodeFlagMasks.TrailingBoundary); + (openerNode.Flags & GreenNodeFlagMasks.LeadingBoundary) | + (closerNode.Flags & GreenNodeFlagMasks.TrailingBoundary); - var contains = (OpenerNode.Flags | CloserNode.Flags) & GreenNodeFlagMasks.Contains; - foreach (var child in _children) + var contains = (openerNode.Flags | closerNode.Flags) & GreenNodeFlagMasks.Contains; + foreach (var child in children) contains |= child.Flags & GreenNodeFlagMasks.Contains; - _flags = boundary | contains; - - // Pre-compute child offsets for large blocks - if (_children.Length >= 10) - { - _childOffsets = new int[_children.Length]; - int offset = OpenerNode.Width; // After opener (including its trivia) - for (int i = 0; i < _children.Length; i++) - { - _childOffsets[i] = offset; - offset += _children[i].Width; - } - } + var flags = boundary | contains; + return new BlockComputed(width, flags, offsets); } + + private readonly record struct BlockComputed(int Width, GreenNodeFlags Flags, int[]? ChildOffsets); /// /// Creates a new block node with automatic opener/closer creation. diff --git a/TinyTokenizer/Ast/GreenContainer.cs b/TinyTokenizer/Ast/GreenContainer.cs index 2df38dc..40859a4 100644 --- a/TinyTokenizer/Ast/GreenContainer.cs +++ b/TinyTokenizer/Ast/GreenContainer.cs @@ -18,6 +18,11 @@ namespace TinyTokenizer.Ast; [DebuggerDisplay("{DebuggerDisplay,nq}")] internal abstract record GreenContainer : GreenNode { + protected GreenContainer(GreenNodeFlags flags) + : base(flags) + { + } + /// protected override string DebuggerDisplay => $"{Kind}[{Width}] ({SlotCount} children)"; diff --git a/TinyTokenizer/Ast/GreenLeaf.cs b/TinyTokenizer/Ast/GreenLeaf.cs index bc1f0a6..503193f 100644 --- a/TinyTokenizer/Ast/GreenLeaf.cs +++ b/TinyTokenizer/Ast/GreenLeaf.cs @@ -17,13 +17,9 @@ internal sealed record GreenLeaf : GreenNode $"{Kind}[{Width}] \"{Truncate(Text, 20)}\""; private readonly int _width; - private readonly GreenNodeFlags _flags; /// public override NodeKind Kind { get; } - - /// - internal override GreenNodeFlags Flags => _flags; /// The text content of this token (excluding trivia). public string Text { get; } @@ -60,36 +56,34 @@ public GreenLeaf( string text, ImmutableArray leadingTrivia = default, ImmutableArray trailingTrivia = default) + : this( + kind, + text, + leadingTrivia.IsDefault ? ImmutableArray.Empty : leadingTrivia, + trailingTrivia.IsDefault ? ImmutableArray.Empty : trailingTrivia, + Compute(kind, + text, + leadingTrivia.IsDefault ? ImmutableArray.Empty : leadingTrivia, + trailingTrivia.IsDefault ? ImmutableArray.Empty : trailingTrivia)) + { + } + + private GreenLeaf( + NodeKind kind, + string text, + ImmutableArray leadingTrivia, + ImmutableArray trailingTrivia, + LeafComputed computed) + : base(computed.Flags) { Kind = kind; Text = text; - LeadingTrivia = leadingTrivia.IsDefault ? ImmutableArray.Empty : leadingTrivia; - TrailingTrivia = trailingTrivia.IsDefault ? ImmutableArray.Empty : trailingTrivia; - - LeadingTriviaWidth = ComputeTriviaWidthAndFlags( - LeadingTrivia, - isLeading: true, - out var leadingBoundaryFlags, - out var leadingContainsFlags); - - TrailingTriviaWidth = ComputeTriviaWidthAndFlags( - TrailingTrivia, - isLeading: false, - out var trailingBoundaryFlags, - out var trailingContainsFlags); - - _width = LeadingTriviaWidth + Text.Length + TrailingTriviaWidth; - - // Subtree flags based on node kind - var kindFlags = GreenNodeFlags.None; - if (kind == NodeKind.Error) - kindFlags |= GreenNodeFlags.ContainsErrorNode; - if (kind == NodeKind.TaggedIdent) - kindFlags |= GreenNodeFlags.ContainsTaggedIdent; - if (kind.IsKeyword()) - kindFlags |= GreenNodeFlags.ContainsKeyword; + LeadingTrivia = leadingTrivia; + TrailingTrivia = trailingTrivia; - _flags = leadingBoundaryFlags | trailingBoundaryFlags | leadingContainsFlags | trailingContainsFlags | kindFlags; + LeadingTriviaWidth = computed.LeadingTriviaWidth; + TrailingTriviaWidth = computed.TrailingTriviaWidth; + _width = computed.Width; } /// @@ -138,6 +132,45 @@ public GreenLeaf WithTrailingTrivia(ImmutableArray trivia) /// public GreenLeaf WithText(string text) => new(Kind, text, LeadingTrivia, TrailingTrivia); + + private static LeafComputed Compute( + NodeKind kind, + string text, + ImmutableArray leadingTrivia, + ImmutableArray trailingTrivia) + { + int leadingWidth = ComputeTriviaWidthAndFlags( + leadingTrivia, + isLeading: true, + out var leadingBoundaryFlags, + out var leadingContainsFlags); + + int trailingWidth = ComputeTriviaWidthAndFlags( + trailingTrivia, + isLeading: false, + out var trailingBoundaryFlags, + out var trailingContainsFlags); + + int width = leadingWidth + text.Length + trailingWidth; + + // Subtree flags based on node kind + var kindFlags = GreenNodeFlags.None; + if (kind == NodeKind.Error) + kindFlags |= GreenNodeFlags.ContainsErrorNode; + if (kind == NodeKind.TaggedIdent) + kindFlags |= GreenNodeFlags.ContainsTaggedIdent; + if (kind.IsKeyword()) + kindFlags |= GreenNodeFlags.ContainsKeyword; + + var flags = leadingBoundaryFlags | trailingBoundaryFlags | leadingContainsFlags | trailingContainsFlags | kindFlags; + return new LeafComputed(leadingWidth, trailingWidth, width, flags); + } + + private readonly record struct LeafComputed( + int LeadingTriviaWidth, + int TrailingTriviaWidth, + int Width, + GreenNodeFlags Flags); private static int ComputeTriviaWidthAndFlags( ImmutableArray trivia, diff --git a/TinyTokenizer/Ast/GreenList.cs b/TinyTokenizer/Ast/GreenList.cs index c6a3ab5..89dac24 100644 --- a/TinyTokenizer/Ast/GreenList.cs +++ b/TinyTokenizer/Ast/GreenList.cs @@ -18,7 +18,6 @@ internal sealed record GreenList : GreenContainer private readonly ImmutableArray _children; private readonly int _width; - private readonly GreenNodeFlags _flags; private readonly int[]? _childOffsets; /// @@ -26,9 +25,6 @@ internal sealed record GreenList : GreenContainer /// public override int Width => _width; - - /// - internal override GreenNodeFlags Flags => _flags; /// public override ImmutableArray Children => _children; @@ -37,48 +33,50 @@ internal sealed record GreenList : GreenContainer /// Creates a new token list. /// public GreenList(ImmutableArray children) + : this( + children.IsDefault ? ImmutableArray.Empty : children, + Compute(children.IsDefault ? ImmutableArray.Empty : children)) { - _children = children.IsDefault ? ImmutableArray.Empty : children; - - // Compute width - int width = 0; - foreach (var child in _children) - width += child.Width; - _width = width; + } - // Flags - if (_children.Length == 0) - { - _flags = GreenNodeFlags.None; - } - else - { - var first = _children[0]; - var last = _children[^1]; + private GreenList(ImmutableArray children, ListComputed computed) + : base(computed.Flags) + { + _children = children; + _width = computed.Width; + _childOffsets = computed.ChildOffsets; + } + + private static ListComputed Compute(ImmutableArray children) + { + if (children.Length == 0) + return new ListComputed(Width: 0, Flags: GreenNodeFlags.None, ChildOffsets: null); - var boundary = - (first.Flags & GreenNodeFlagMasks.LeadingBoundary) | - (last.Flags & GreenNodeFlagMasks.TrailingBoundary); + int width = 0; + var contains = GreenNodeFlags.None; - var contains = GreenNodeFlags.None; - foreach (var child in _children) - contains |= child.Flags & GreenNodeFlagMasks.Contains; + int[]? offsets = null; + if (children.Length >= 10) + offsets = new int[children.Length]; - _flags = boundary | contains; - } - - // Pre-compute offsets for large lists - if (_children.Length >= 10) + int offset = 0; + for (int i = 0; i < children.Length; i++) { - _childOffsets = new int[_children.Length]; - int offset = 0; - for (int i = 0; i < _children.Length; i++) - { - _childOffsets[i] = offset; - offset += _children[i].Width; - } + if (offsets != null) + offsets[i] = offset; + + var child = children[i]; + width += child.Width; + offset += child.Width; + contains |= child.Flags & GreenNodeFlagMasks.Contains; } + + // Token-centric boundary semantics: lists do not own boundary trivia. + var flags = contains; + return new ListComputed(width, flags, offsets); } + + private readonly record struct ListComputed(int Width, GreenNodeFlags Flags, int[]? ChildOffsets); /// public override GreenNode? GetSlot(int index) diff --git a/TinyTokenizer/Ast/GreenNode.cs b/TinyTokenizer/Ast/GreenNode.cs index a9e65e5..6809d31 100644 --- a/TinyTokenizer/Ast/GreenNode.cs +++ b/TinyTokenizer/Ast/GreenNode.cs @@ -14,6 +14,11 @@ namespace TinyTokenizer.Ast; [DebuggerDisplay("{DebuggerDisplay,nq}")] internal abstract record GreenNode : IFormattable, ITextSerializable { + protected GreenNode(GreenNodeFlags flags) + { + Flags = flags; + } + /// /// Gets the debugger display string for this node. /// Override in derived classes for specialized display. @@ -39,9 +44,9 @@ protected static string Truncate(string text, int maxLength) /// /// Cached flags describing trivia/content properties for fast queries. - /// Concrete green node types override this once flags are computed. + /// Stored directly on the green node for O(1) access. /// - internal virtual GreenNodeFlags Flags => GreenNodeFlags.None; + internal readonly GreenNodeFlags Flags; /// /// Total character width of this node, including any trivia. diff --git a/TinyTokenizer/Ast/GreenNodeFlags.cs b/TinyTokenizer/Ast/GreenNodeFlags.cs index 89124a7..1c6a0b0 100644 --- a/TinyTokenizer/Ast/GreenNodeFlags.cs +++ b/TinyTokenizer/Ast/GreenNodeFlags.cs @@ -6,12 +6,41 @@ namespace TinyTokenizer.Ast; /// Bitflags describing trivia and content properties of a green node. /// Intended for O(1) query checks and subtree pruning. /// +/// +/// +/// These flags intentionally distinguish between boundary trivia and subtree contains. +/// +/// +/// +/// +/// Boundary flags (, +/// , etc.) mean the node itself +/// owns trivia on its boundary. For token-centric newline queries, this is the only +/// correct interpretation. +/// +/// +/// +/// +/// Contains flags (, etc.) mean the +/// trivia exists somewhere within the node's subtree (including children). +/// +/// +/// +/// +/// Important: boundary flags MUST NOT be used to represent "first child has boundary trivia". +/// Container nodes should not automatically inherit boundary flags from their first/last child. +/// +/// [Flags] internal enum GreenNodeFlags : uint { None = 0, - // Boundary trivia (left/right edge of the node's text span) + // Boundary trivia (owned by this node's boundary) + // + // For leaves: these correspond to the leaf's own leading/trailing trivia. + // For blocks: these correspond to opener leading and closer trailing trivia. + // For containers/lists: these should remain None unless the container explicitly owns trivia. HasLeadingNewlineTrivia = 1u << 0, HasTrailingNewlineTrivia = 1u << 1, @@ -21,7 +50,7 @@ internal enum GreenNodeFlags : uint HasLeadingCommentTrivia = 1u << 4, HasTrailingCommentTrivia = 1u << 5, - // Subtree flags (anywhere within the node's subtree) + // Subtree flags (anywhere within the node's subtree, including children) ContainsNewlineTrivia = 1u << 8, ContainsWhitespaceTrivia = 1u << 9, ContainsCommentTrivia = 1u << 10, diff --git a/TinyTokenizer/Ast/GreenSyntaxNode.cs b/TinyTokenizer/Ast/GreenSyntaxNode.cs index ed21515..36d8e58 100644 --- a/TinyTokenizer/Ast/GreenSyntaxNode.cs +++ b/TinyTokenizer/Ast/GreenSyntaxNode.cs @@ -24,7 +24,6 @@ internal sealed record GreenSyntaxNode : GreenContainer private readonly ImmutableArray _children; private readonly NodeKind _kind; private readonly int _width; - private readonly GreenNodeFlags _flags; /// /// Creates a green syntax node wrapping the specified children. @@ -32,39 +31,40 @@ internal sealed record GreenSyntaxNode : GreenContainer /// The semantic NodeKind for this syntax construct. /// The child green nodes that make up this syntax construct. public GreenSyntaxNode(NodeKind kind, ImmutableArray children) + : this( + kind, + children.IsDefault ? ImmutableArray.Empty : children, + Compute(children.IsDefault ? ImmutableArray.Empty : children)) + { + } + + private GreenSyntaxNode(NodeKind kind, ImmutableArray children, SyntaxNodeComputed computed) + : base(computed.Flags) { _kind = kind; - _children = children.IsDefault ? ImmutableArray.Empty : children; - - // Calculate total width + _children = children; + _width = computed.Width; + } + + private static SyntaxNodeComputed Compute(ImmutableArray children) + { + if (children.Length == 0) + return new SyntaxNodeComputed(Width: 0, Flags: GreenNodeFlags.None); + int width = 0; - foreach (var child in _children) + var contains = GreenNodeFlags.None; + foreach (var child in children) { width += child.Width; + contains |= child.Flags & GreenNodeFlagMasks.Contains; } - _width = width; - // Flags - if (_children.Length == 0) - { - _flags = GreenNodeFlags.None; - } - else - { - var first = _children[0]; - var last = _children[^1]; - - var boundary = - (first.Flags & GreenNodeFlagMasks.LeadingBoundary) | - (last.Flags & GreenNodeFlagMasks.TrailingBoundary); - - var contains = GreenNodeFlags.None; - foreach (var child in _children) - contains |= child.Flags & GreenNodeFlagMasks.Contains; - - _flags = boundary | contains; - } + // Token-centric boundary semantics: syntax containers do not own boundary trivia. + var flags = contains; + return new SyntaxNodeComputed(width, flags); } + + private readonly record struct SyntaxNodeComputed(int Width, GreenNodeFlags Flags); /// /// Creates a green syntax node from params array of children. @@ -79,9 +79,6 @@ public GreenSyntaxNode(NodeKind kind, params GreenNode[] children) /// public override int Width => _width; - - /// - internal override GreenNodeFlags Flags => _flags; /// public override ImmutableArray Children => _children; diff --git a/TinyTokenizer/Ast/NodeQuery.cs b/TinyTokenizer/Ast/NodeQuery.cs index ac62315..97d2f17 100644 --- a/TinyTokenizer/Ast/NodeQuery.cs +++ b/TinyTokenizer/Ast/NodeQuery.cs @@ -124,9 +124,9 @@ IEnumerable IRegionQuery.SelectRegions(SyntaxNode root) => SelectRegionsCore(root); /// - /// Default region resolution: traverses tree with PathTrackingWalker, calls TryMatch once per node, + /// Default region resolution: traverses tree with a stack-based walker, calls TryMatch once per node, /// then applies selection filtering (First/Last/Nth via ApplyRegionFilter). - /// Uses incremental path tracking for O(1) per node instead of O(depth). + /// Snapshots a NodePath only when a match is found. /// internal virtual IEnumerable SelectRegionsCore(SyntaxNode root) { @@ -135,29 +135,11 @@ internal virtual IEnumerable SelectRegionsCore(SyntaxNode root) /// /// Traverses tree and yields a region for each matching node. - /// Uses PathTrackingWalker for O(1) path computation per node. + /// Uses RegionTraversal to avoid per-visited-node allocations. /// private IEnumerable SelectAllRegions(SyntaxNode root) { - var walker = new PathTrackingWalker(root); - foreach (var (node, parentPath) in walker.DescendantsAndSelfWithPath()) - { - if (TryMatch(node, out var consumedCount)) - { - var parent = node.Parent; - if (parent != null) - { - yield return new QueryRegion( - parentPath: parentPath, - parent: parent, - startSlot: node.SiblingIndex, - endSlot: node.SiblingIndex + consumedCount, - firstNode: node, - position: node.Position - ); - } - } - } + return RegionTraversal.SelectRegions(root, TryMatch); } /// diff --git a/TinyTokenizer/Ast/NodeQueryTypes.cs b/TinyTokenizer/Ast/NodeQueryTypes.cs index 0648bf3..de15079 100644 --- a/TinyTokenizer/Ast/NodeQueryTypes.cs +++ b/TinyTokenizer/Ast/NodeQueryTypes.cs @@ -7,16 +7,95 @@ namespace TinyTokenizer.Ast; /// internal static class SelectionModeHelper { - public static IEnumerable Apply(IEnumerable regions, SelectionMode mode, int modeArg) => - mode switch + public static IEnumerable Apply(IEnumerable source, SelectionMode mode, int modeArg) + { + return mode switch { - SelectionMode.First => regions.Take(1), - SelectionMode.Last => regions.TakeLast(1), - SelectionMode.Nth => regions.Skip(modeArg).Take(1), - SelectionMode.Skip => regions.Skip(modeArg), - SelectionMode.Take => regions.Take(modeArg), - _ => regions + SelectionMode.First => TakeFirst(source), + SelectionMode.Last => TakeLast(source), + SelectionMode.Nth => TakeNth(source, modeArg), + SelectionMode.Skip => Skip(source, modeArg), + SelectionMode.Take => Take(source, modeArg), + _ => source }; + } + + private static IEnumerable TakeFirst(IEnumerable source) + { + foreach (var item in source) + { + yield return item; + yield break; + } + } + + private static IEnumerable TakeLast(IEnumerable source) + { + T? last = default; + var found = false; + + foreach (var item in source) + { + last = item; + found = true; + } + + if (found) + yield return last!; + } + + private static IEnumerable TakeNth(IEnumerable source, int n) + { + if (n < 0) + yield break; + + var index = 0; + foreach (var item in source) + { + if (index == n) + { + yield return item; + yield break; + } + index++; + } + } + + private static IEnumerable Skip(IEnumerable source, int count) + { + if (count <= 0) + { + foreach (var item in source) + yield return item; + yield break; + } + + var skipped = 0; + foreach (var item in source) + { + if (skipped < count) + { + skipped++; + continue; + } + yield return item; + } + } + + private static IEnumerable Take(IEnumerable source, int count) + { + if (count <= 0) + yield break; + + var taken = 0; + foreach (var item in source) + { + yield return item; + taken++; + if (taken >= count) + yield break; + } + } } /// @@ -66,19 +145,18 @@ private KindNodeQuery(NodeKind kind, Func? predicate, Selectio /// public override IEnumerable Select(SyntaxNode root) + { + return SelectionModeHelper.Apply(EnumerateMatches(root), _mode, _modeArg); + } + + private IEnumerable EnumerateMatches(SyntaxNode root) { var walker = new TreeWalker(root); - var matches = walker.DescendantsAndSelf().Where(Matches); - - return _mode switch + foreach (var node in walker.DescendantsAndSelf()) { - SelectionMode.First => matches.Take(1), - SelectionMode.Last => matches.TakeLast(1), - SelectionMode.Nth => matches.Skip(_modeArg).Take(1), - SelectionMode.Skip => matches.Skip(_modeArg), - SelectionMode.Take => matches.Take(_modeArg), - _ => matches - }; + if (node.Kind == Kind && (_predicate == null || _predicate(node))) + yield return node; + } } /// @@ -119,35 +197,23 @@ protected override KindNodeQuery CreateFiltered(Func predicate /// /// Optimized region resolution: single-pass traversal that checks Kind directly /// and applies selection mode inline for efficient First()/Take() short-circuit. - /// Uses PathTrackingWalker for O(1) path computation per node. + /// Uses RegionTraversal to avoid per-visited-node allocations. /// internal override IEnumerable SelectRegionsCore(SyntaxNode root) { - var walker = new PathTrackingWalker(root); - var regions = SelectRegionsFromWalker(walker); + var regions = RegionTraversal.SelectRegions(root, TryGetRegion); return ApplyRegionFilter(regions); - } - - private IEnumerable SelectRegionsFromWalker(PathTrackingWalker walker) - { - foreach (var (node, parentPath) in walker.DescendantsAndSelfWithPath()) + + bool TryGetRegion(SyntaxNode node, out int consumedCount) { - // Inline match check - avoids virtual TryMatch call if (node.Kind == Kind && (_predicate == null || _predicate(node))) { - var parent = node.Parent; - if (parent != null) - { - yield return new QueryRegion( - parentPath: parentPath, - parent: parent, - startSlot: node.SiblingIndex, - endSlot: node.SiblingIndex + 1, // KindNodeQuery always consumes 1 - firstNode: node, - position: node.Position - ); - } + consumedCount = 1; + return true; } + + consumedCount = 0; + return false; } } @@ -188,19 +254,26 @@ private protected BlockNodeQuery(char? opener, Func? predicate /// public override IEnumerable Select(SyntaxNode root) + { + return SelectionModeHelper.Apply(EnumerateMatches(root), _mode, _modeArg); + } + + private IEnumerable EnumerateMatches(SyntaxNode root) { var walker = new TreeWalker(root); - var matches = walker.DescendantsAndSelf().Where(Matches); - - return _mode switch + foreach (var node in walker.DescendantsAndSelf()) { - SelectionMode.First => matches.Take(1), - SelectionMode.Last => matches.TakeLast(1), - SelectionMode.Nth => matches.Skip(_modeArg).Take(1), - SelectionMode.Skip => matches.Skip(_modeArg), - SelectionMode.Take => matches.Take(_modeArg), - _ => matches - }; + if (node is not SyntaxBlock block) + continue; + + if (_opener != null && block.Opener != _opener.Value) + continue; + + if (_predicate != null && !_predicate(node)) + continue; + + yield return node; + } } /// @@ -233,37 +306,25 @@ protected override BlockNodeQuery CreateFiltered(Func predicat /// /// Optimized region resolution: single-pass traversal that checks block type directly /// and applies selection mode inline for efficient First()/Take() short-circuit. - /// Uses PathTrackingWalker for O(1) path computation per node. + /// Uses RegionTraversal to avoid per-visited-node allocations. /// internal override IEnumerable SelectRegionsCore(SyntaxNode root) { - var walker = new PathTrackingWalker(root); - var regions = SelectRegionsFromWalker(walker); + var regions = RegionTraversal.SelectRegions(root, TryGetRegion); return ApplyRegionFilter(regions); - } - - private IEnumerable SelectRegionsFromWalker(PathTrackingWalker walker) - { - foreach (var (node, parentPath) in walker.DescendantsAndSelfWithPath()) + + bool TryGetRegion(SyntaxNode node, out int consumedCount) { - // Inline match check - avoids virtual TryMatch call if (node is SyntaxBlock block && (_opener == null || block.Opener == _opener.Value) && (_predicate == null || _predicate(node))) { - var parent = node.Parent; - if (parent != null) - { - yield return new QueryRegion( - parentPath: parentPath, - parent: parent, - startSlot: node.SiblingIndex, - endSlot: node.SiblingIndex + 1, // BlockNodeQuery always consumes 1 - firstNode: node, - position: node.Position - ); - } + consumedCount = 1; + return true; } + + consumedCount = 0; + return false; } } @@ -303,9 +364,6 @@ private IEnumerable SelectRegionsFromWalker(PathTrackingWalker walk /// /// // Replace content between braces /// editor.Replace(Query.BraceBlock.Inner(), "new content") - /// - /// // Works with empty blocks too - /// editor.Replace(Query.BraceBlock.Inner(), "inserted into empty") /// /// public InnerContentQuery Inner() => new InnerContentQuery(this); @@ -327,12 +385,9 @@ public enum BoundarySide } /// -/// A query that selects the boundary (start or end) of container nodes. -/// For blocks, this returns the opener or closer token. -/// For lists/containers without delimiters, this returns first/last child (or empty for empty containers). +/// A query that selects the boundary node (start or end) of containers matched by an inner query. /// /// -/// This query carries metadata about which container and boundary is being targeted. /// uses this metadata to compute insertion positions, /// even for empty containers where returns no results. /// @@ -532,7 +587,12 @@ private IEnumerable SelectRegionsCore(SyntaxNode root) if (container is SyntaxBlock block) { var innerCount = block.ChildCount; - var firstInner = block.InnerChildren.FirstOrDefault(); + SyntaxNode? firstInner = null; + foreach (var child in block.InnerChildren) + { + firstInner = child; + break; + } yield return new QueryRegion( parent: block, @@ -574,19 +634,20 @@ private AnyNodeQuery(Func? predicate, SelectionMode mode, int /// public override IEnumerable Select(SyntaxNode root) { - var matches = new TreeWalker(root).DescendantsAndSelf(); - if (_predicate != null) - matches = matches.Where(_predicate); - - return _mode switch + if (_predicate == null) + return SelectionModeHelper.Apply(new TreeWalker(root).DescendantsAndSelf(), _mode, _modeArg); + + return SelectionModeHelper.Apply(EnumerateMatches(root, _predicate), _mode, _modeArg); + } + + private static IEnumerable EnumerateMatches(SyntaxNode root, Func predicate) + { + var walker = new TreeWalker(root); + foreach (var node in walker.DescendantsAndSelf()) { - SelectionMode.First => matches.Take(1), - SelectionMode.Last => matches.TakeLast(1), - SelectionMode.Nth => matches.Skip(_modeArg).Take(1), - SelectionMode.Skip => matches.Skip(_modeArg), - SelectionMode.Take => matches.Take(_modeArg), - _ => matches - }; + if (predicate(node)) + yield return node; + } } /// @@ -637,19 +698,18 @@ private LeafNodeQuery(Func? predicate, SelectionMode mode, int /// public override IEnumerable Select(SyntaxNode root) + { + return SelectionModeHelper.Apply(EnumerateMatches(root), _mode, _modeArg); + } + + private IEnumerable EnumerateMatches(SyntaxNode root) { var walker = new TreeWalker(root, NodeFilter.Leaves); - var matches = walker.DescendantsAndSelf().Where(Matches); - - return _mode switch + foreach (var node in walker.DescendantsAndSelf()) { - SelectionMode.First => matches.Take(1), - SelectionMode.Last => matches.TakeLast(1), - SelectionMode.Nth => matches.Skip(_modeArg).Take(1), - SelectionMode.Skip => matches.Skip(_modeArg), - SelectionMode.Take => matches.Take(_modeArg), - _ => matches - }; + if (node is SyntaxToken && (_predicate == null || _predicate(node))) + yield return node; + } } /// @@ -679,13 +739,15 @@ protected override LeafNodeQuery CreateFiltered(Func predicate #region Newline Query /// -/// Matches nodes that represent or are preceded by a newline. -/// Checks: -/// 1. The node itself is a whitespace token containing newline characters. -/// 2. The node's leading trivia contains a newline. -/// 3. The previous sibling's trailing trivia contains a newline. +/// Matches nodes that occur after a newline. +/// A node matches when either: +/// 1) The node owns leading newline trivia, OR +/// 2) The previous sibling owns trailing newline trivia. /// /// +/// Newline detection is token-centric: the newline boundary is owned by a token's +/// leading/trailing trivia, not by container nodes. +/// /// This query is particularly useful for line-based pattern matching, such as /// matching directive lines that should consume tokens until a newline. /// @@ -716,19 +778,18 @@ private NewlineNodeQuery(Func? predicate, SelectionMode mode, /// public override IEnumerable Select(SyntaxNode root) + { + return SelectionModeHelper.Apply(EnumerateMatches(root), _mode, _modeArg); + } + + private IEnumerable EnumerateMatches(SyntaxNode root) { var walker = new TreeWalker(root); - var matches = walker.DescendantsAndSelf().Where(Matches); - - return _mode switch + foreach (var node in walker.DescendantsAndSelf()) { - SelectionMode.First => matches.Take(1), - SelectionMode.Last => matches.TakeLast(1), - SelectionMode.Nth => matches.Skip(_modeArg).Take(1), - SelectionMode.Skip => matches.Skip(_modeArg), - SelectionMode.Take => matches.Take(_modeArg), - _ => matches - }; + if (Matches(node)) + yield return node; + } } /// @@ -744,61 +805,56 @@ internal override bool MatchesGreen(GreenNode node) bool hasNewline = HasGreenNewline(node); return _negated ? !hasNewline : hasNewline; } - - private static bool HasGreenNewline(GreenNode node) + + internal override bool TryMatchGreen(IReadOnlyList siblings, int startIndex, out int consumedCount) { - // Check leading trivia for newline - var leadingTrivia = node switch + if ((uint)startIndex >= (uint)siblings.Count) { - GreenLeaf gl => gl.LeadingTrivia, - GreenBlock gb => gb.LeadingTrivia, - _ => System.Collections.Immutable.ImmutableArray.Empty - }; - - foreach (var t in leadingTrivia) + consumedCount = 0; + return false; + } + + var node = siblings[startIndex]; + + // Token-centric newline semantics: + // - current node matches if it owns leading newline trivia + // - OR if previous sibling owns trailing newline trivia + bool hasNewline = HasGreenNewline(node); + if (!hasNewline && startIndex > 0) { - if (t.Kind == TriviaKind.Newline) - return true; + hasNewline = HasGreenTrailingNewline(siblings[startIndex - 1]); } - + + bool matched = _negated ? !hasNewline : hasNewline; + if (matched) + { + consumedCount = 1; + return true; + } + + consumedCount = 0; return false; } + private static bool HasGreenNewline(GreenNode node) + { + return (node.Flags & GreenNodeFlags.HasLeadingNewlineTrivia) != 0; + } + + private static bool HasGreenTrailingNewline(GreenNode node) + { + return (node.Flags & GreenNodeFlags.HasTrailingNewlineTrivia) != 0; + } + private static bool HasNewline(SyntaxNode node) { - // Check 1: Does leading trivia contain newline? - var leadingTrivia = node.Green switch - { - GreenLeaf gl => gl.LeadingTrivia, - GreenBlock gb => gb.LeadingTrivia, - _ => System.Collections.Immutable.ImmutableArray.Empty - }; - - foreach (var t in leadingTrivia) - { - if (t.Kind == TriviaKind.Newline) - return true; - } - - // Check 2: Does previous sibling's trailing trivia contain newline? + // Check 1: does this node own leading newline trivia? + if ((node.Green.Flags & GreenNodeFlags.HasLeadingNewlineTrivia) != 0) + return true; + + // Check 2: does previous sibling own trailing newline trivia? var prev = node.PreviousSibling(); - if (prev != null) - { - var trailingTrivia = prev.Green switch - { - GreenLeaf gl => gl.TrailingTrivia, - GreenBlock gb => gb.TrailingTrivia, - _ => System.Collections.Immutable.ImmutableArray.Empty - }; - - foreach (var t in trailingTrivia) - { - if (t.Kind == TriviaKind.Newline) - return true; - } - } - - return false; + return prev != null && (prev.Green.Flags & GreenNodeFlags.HasTrailingNewlineTrivia) != 0; } protected override NewlineNodeQuery CreateFiltered(Func predicate) => @@ -955,10 +1011,26 @@ public sealed record AnyOfQuery : INodeQuery, IGreenNodeQuery, ISchemaResolvable public AnyOfQuery(params INodeQuery[] queries) => _queries = queries; /// Creates a query that matches any of the specified queries. - public AnyOfQuery(IEnumerable queries) => _queries = queries.ToArray(); + public AnyOfQuery(IEnumerable queries) + { + ArgumentNullException.ThrowIfNull(queries); + _queries = MaterializeQueries(queries); + } /// - public bool IsResolved => _queries.All(q => q is not ISchemaResolvableQuery r || r.IsResolved); + public bool IsResolved + { + get + { + foreach (var query in _queries) + { + if (query is ISchemaResolvableQuery r && !r.IsResolved) + return false; + } + + return true; + } + } /// public void ResolveWithSchema(Schema schema) @@ -1035,6 +1107,27 @@ bool IGreenNodeQuery.TryMatchGreen(IReadOnlyList siblings, int startI consumedCount = 0; return false; } + + private static INodeQuery[] MaterializeQueries(IEnumerable queries) + { + if (queries is INodeQuery[] arr) + return arr; + + if (queries is ICollection col) + { + if (col.Count == 0) + return []; + + var buffer = new INodeQuery[col.Count]; + col.CopyTo(buffer, 0); + return buffer; + } + + var list = new List(); + foreach (var q in queries) + list.Add(q); + return list.ToArray(); + } } #endregion @@ -1053,10 +1146,26 @@ public sealed record NoneOfQuery : INodeQuery, IGreenNodeQuery, ISchemaResolvabl public NoneOfQuery(params INodeQuery[] queries) => _queries = queries; /// Creates a query that matches when none of the specified queries match. - public NoneOfQuery(IEnumerable queries) => _queries = queries.ToArray(); + public NoneOfQuery(IEnumerable queries) + { + ArgumentNullException.ThrowIfNull(queries); + _queries = MaterializeQueries(queries); + } /// - public bool IsResolved => _queries.All(q => q is not ISchemaResolvableQuery r || r.IsResolved); + public bool IsResolved + { + get + { + foreach (var query in _queries) + { + if (query is ISchemaResolvableQuery r && !r.IsResolved) + return false; + } + + return true; + } + } /// public void ResolveWithSchema(Schema schema) @@ -1143,6 +1252,27 @@ bool IGreenNodeQuery.TryMatchGreen(IReadOnlyList siblings, int startI consumedCount = 1; return true; } + + private static INodeQuery[] MaterializeQueries(IEnumerable queries) + { + if (queries is INodeQuery[] arr) + return arr; + + if (queries is ICollection col) + { + if (col.Count == 0) + return []; + + var buffer = new INodeQuery[col.Count]; + col.CopyTo(buffer, 0); + return buffer; + } + + var list = new List(); + foreach (var q in queries) + list.Add(q); + return list.ToArray(); + } } #endregion @@ -1162,9 +1292,11 @@ public sealed record BeginningOfFileQuery : INodeQuery, IGreenNodeQuery public IEnumerable Select(SyntaxNode root) { // BOF only matches the first node in the file - var firstChild = root.Children.FirstOrDefault(); - if (firstChild != null) - yield return firstChild; + foreach (var child in root.Children) + { + yield return child; + yield break; + } } /// @@ -1220,9 +1352,14 @@ public sealed record EndOfFileQuery : INodeQuery, IGreenNodeQuery public IEnumerable Select(SyntaxNode root) { // EOF only matches the last node in the file - var lastChild = root.Children.LastOrDefault(); - if (lastChild != null) - yield return lastChild; + SyntaxNode? last = null; + foreach (var child in root.Children) + { + last = child; + } + + if (last != null) + yield return last; } /// @@ -1565,19 +1702,18 @@ private AnyKeywordQuery(Func? predicate, SelectionMode mode, i /// public override IEnumerable Select(SyntaxNode root) + { + return SelectionModeHelper.Apply(EnumerateMatches(root), _mode, _modeArg); + } + + private IEnumerable EnumerateMatches(SyntaxNode root) { var walker = new TreeWalker(root); - var matches = walker.DescendantsAndSelf().Where(Matches); - - return _mode switch + foreach (var node in walker.DescendantsAndSelf()) { - SelectionMode.First => matches.Take(1), - SelectionMode.Last => matches.TakeLast(1), - SelectionMode.Nth => matches.Skip(_modeArg).Take(1), - SelectionMode.Skip => matches.Skip(_modeArg), - SelectionMode.Take => matches.Take(_modeArg), - _ => matches - }; + if (node.Kind.IsKeyword() && (_predicate == null || _predicate(node))) + yield return node; + } } /// @@ -1681,20 +1817,18 @@ public override IEnumerable Select(SyntaxNode root) if (!_isResolved || _resolvedKind == null) return []; - var targetKind = _resolvedKind.Value; + return SelectionModeHelper.Apply(EnumerateMatches(root), _mode, _modeArg); + } + + private IEnumerable EnumerateMatches(SyntaxNode root) + { + var targetKind = _resolvedKind!.Value; var walker = new TreeWalker(root); - var matches = walker.DescendantsAndSelf() - .Where(n => n.Kind == targetKind && (_predicate == null || _predicate(n))); - - return _mode switch + foreach (var node in walker.DescendantsAndSelf()) { - SelectionMode.First => matches.Take(1), - SelectionMode.Last => matches.TakeLast(1), - SelectionMode.Nth => matches.Skip(_modeArg).Take(1), - SelectionMode.Skip => matches.Skip(_modeArg), - SelectionMode.Take => matches.Take(_modeArg), - _ => matches - }; + if (node.Kind == targetKind && (_predicate == null || _predicate(node))) + yield return node; + } } /// @@ -1840,38 +1974,34 @@ public override IEnumerable Select(SyntaxTree tree) return []; var kindSet = categoryKinds.ToHashSet(); - var walker = new TreeWalker(tree.Root); - var matches = walker.DescendantsAndSelf() - .Where(n => kindSet.Contains(n.Kind) && (_predicate == null || _predicate(n))); - - return _mode switch + return SelectionModeHelper.Apply(EnumerateMatchesByKindSet(tree.Root, kindSet), _mode, _modeArg); + } + + private IEnumerable EnumerateMatchesByKindSet(SyntaxNode root, HashSet kindSet) + { + var walker = new TreeWalker(root); + foreach (var node in walker.DescendantsAndSelf()) { - SelectionMode.First => matches.Take(1), - SelectionMode.Last => matches.TakeLast(1), - SelectionMode.Nth => matches.Skip(_modeArg).Take(1), - SelectionMode.Skip => matches.Skip(_modeArg), - SelectionMode.Take => matches.Take(_modeArg), - _ => matches - }; + if (kindSet.Contains(node.Kind) && (_predicate == null || _predicate(node))) + yield return node; + } } /// public override IEnumerable Select(SyntaxNode root) { // Without tree context, we can't resolve category - match any keyword + return SelectionModeHelper.Apply(EnumerateMatchesAnyKeyword(root), _mode, _modeArg); + } + + private IEnumerable EnumerateMatchesAnyKeyword(SyntaxNode root) + { var walker = new TreeWalker(root); - var matches = walker.DescendantsAndSelf() - .Where(n => n.Kind.IsKeyword() && (_predicate == null || _predicate(n))); - - return _mode switch + foreach (var node in walker.DescendantsAndSelf()) { - SelectionMode.First => matches.Take(1), - SelectionMode.Last => matches.TakeLast(1), - SelectionMode.Nth => matches.Skip(_modeArg).Take(1), - SelectionMode.Skip => matches.Skip(_modeArg), - SelectionMode.Take => matches.Take(_modeArg), - _ => matches - }; + if (node.Kind.IsKeyword() && (_predicate == null || _predicate(node))) + yield return node; + } } /// diff --git a/TinyTokenizer/Ast/Query.cs b/TinyTokenizer/Ast/Query.cs index 8b430f1..56e88e3 100644 --- a/TinyTokenizer/Ast/Query.cs +++ b/TinyTokenizer/Ast/Query.cs @@ -104,13 +104,17 @@ public static class Query public static LeafNodeQuery Leaf => new LeafNodeQuery(); /// - /// Matches nodes that are preceded by a newline (in trivia or as whitespace token). + /// Matches nodes that occur after a newline. + /// A node matches when either: + /// - The node owns leading newline trivia, OR + /// - The previous sibling owns trailing newline trivia. /// Useful for line-based pattern matching. /// public static NewlineNodeQuery Newline => new NewlineNodeQuery(); /// - /// Matches nodes that are NOT preceded by a newline. + /// Matches nodes that do NOT occur after a newline. + /// This is the exact negation of under the same context. /// Useful for matching tokens on the same line. /// public static NewlineNodeQuery NotNewline => new NewlineNodeQuery(negated: true); diff --git a/TinyTokenizer/Ast/QueryCombinators.cs b/TinyTokenizer/Ast/QueryCombinators.cs index 6d2373d..6744a35 100644 --- a/TinyTokenizer/Ast/QueryCombinators.cs +++ b/TinyTokenizer/Ast/QueryCombinators.cs @@ -215,25 +215,7 @@ IEnumerable IRegionQuery.SelectRegions(SyntaxTree tree) /// IEnumerable IRegionQuery.SelectRegions(SyntaxNode root) { - var walker = new PathTrackingWalker(root); - foreach (var (node, parentPath) in walker.DescendantsAndSelfWithPath()) - { - if (TryMatch(node, out var consumedCount)) - { - var parent = node.Parent; - if (parent != null) - { - yield return new QueryRegion( - parentPath: parentPath, - parent: parent, - startSlot: node.SiblingIndex, - endSlot: node.SiblingIndex + consumedCount, - firstNode: node, - position: node.Position - ); - } - } - } + return RegionTraversal.SelectRegions(root, TryMatch); } } @@ -270,7 +252,19 @@ public SequenceQuery(params INodeQuery[] parts) } /// - public bool IsResolved => _parts.All(p => p is not ISchemaResolvableQuery r || r.IsResolved); + public bool IsResolved + { + get + { + foreach (var part in _parts) + { + if (part is ISchemaResolvableQuery r && !r.IsResolved) + return false; + } + + return true; + } + } /// public void ResolveWithSchema(Schema schema) @@ -381,25 +375,7 @@ IEnumerable IRegionQuery.SelectRegions(SyntaxTree tree) /// IEnumerable IRegionQuery.SelectRegions(SyntaxNode root) { - var walker = new PathTrackingWalker(root); - foreach (var (node, parentPath) in walker.DescendantsAndSelfWithPath()) - { - if (TryMatch(node, out var consumedCount)) - { - var parent = node.Parent; - if (parent != null) - { - yield return new QueryRegion( - parentPath: parentPath, - parent: parent, - startSlot: node.SiblingIndex, - endSlot: node.SiblingIndex + consumedCount, - firstNode: node, - position: node.Position - ); - } - } - } + return RegionTraversal.SelectRegions(root, TryMatch); } } @@ -466,25 +442,16 @@ IEnumerable IRegionQuery.SelectRegions(SyntaxTree tree) /// IEnumerable IRegionQuery.SelectRegions(SyntaxNode root) { - // Optional delegates to inner query - traverse and match inner directly - var walker = new PathTrackingWalker(root); - foreach (var (node, parentPath) in walker.DescendantsAndSelfWithPath()) + // Optional delegates to inner query - only yield when inner consumes > 0 + return RegionTraversal.SelectRegions(root, TryGetInnerRegion); + + bool TryGetInnerRegion(SyntaxNode node, out int consumedCount) { - if (_inner.TryMatch(node, out var consumedCount) && consumedCount > 0) - { - var parent = node.Parent; - if (parent != null) - { - yield return new QueryRegion( - parentPath: parentPath, - parent: parent, - startSlot: node.SiblingIndex, - endSlot: node.SiblingIndex + consumedCount, - firstNode: node, - position: node.Position - ); - } - } + if (_inner.TryMatch(node, out consumedCount) && consumedCount > 0) + return true; + + consumedCount = 0; + return false; } } } @@ -613,24 +580,15 @@ IEnumerable IRegionQuery.SelectRegions(SyntaxTree tree) /// IEnumerable IRegionQuery.SelectRegions(SyntaxNode root) { - var walker = new PathTrackingWalker(root); - foreach (var (node, parentPath) in walker.DescendantsAndSelfWithPath()) + return RegionTraversal.SelectRegions(root, TryGetNonEmptyRegion); + + bool TryGetNonEmptyRegion(SyntaxNode node, out int consumedCount) { - if (TryMatch(node, out var consumedCount) && consumedCount > 0) - { - var parent = node.Parent; - if (parent != null) - { - yield return new QueryRegion( - parentPath: parentPath, - parent: parent, - startSlot: node.SiblingIndex, - endSlot: node.SiblingIndex + consumedCount, - firstNode: node, - position: node.Position - ); - } - } + if (TryMatch(node, out consumedCount) && consumedCount > 0) + return true; + + consumedCount = 0; + return false; } } } @@ -834,25 +792,7 @@ IEnumerable IRegionQuery.SelectRegions(SyntaxTree tree) /// IEnumerable IRegionQuery.SelectRegions(SyntaxNode root) { - var walker = new PathTrackingWalker(root); - foreach (var (node, parentPath) in walker.DescendantsAndSelfWithPath()) - { - if (TryMatch(node, out var consumedCount)) - { - var parent = node.Parent; - if (parent != null) - { - yield return new QueryRegion( - parentPath: parentPath, - parent: parent, - startSlot: node.SiblingIndex, - endSlot: node.SiblingIndex + consumedCount, - firstNode: node, - position: node.Position - ); - } - } - } + return RegionTraversal.SelectRegions(root, TryMatch); } } @@ -990,25 +930,7 @@ IEnumerable IRegionQuery.SelectRegions(SyntaxTree tree) /// IEnumerable IRegionQuery.SelectRegions(SyntaxNode root) { - var walker = new PathTrackingWalker(root); - foreach (var (node, parentPath) in walker.DescendantsAndSelfWithPath()) - { - if (TryMatch(node, out var consumedCount)) - { - var parent = node.Parent; - if (parent != null) - { - yield return new QueryRegion( - parentPath: parentPath, - parent: parent, - startSlot: node.SiblingIndex, - endSlot: node.SiblingIndex + consumedCount, - firstNode: node, - position: node.Position - ); - } - } - } + return RegionTraversal.SelectRegions(root, TryMatch); } } diff --git a/TinyTokenizer/Ast/QueryRegion.cs b/TinyTokenizer/Ast/QueryRegion.cs index 63c7ad1..616b2d2 100644 --- a/TinyTokenizer/Ast/QueryRegion.cs +++ b/TinyTokenizer/Ast/QueryRegion.cs @@ -133,113 +133,103 @@ internal interface IRegionQuery } /// -/// A tree walker that incrementally tracks the path during traversal. -/// O(1) per traversal step instead of O(depth) for NodePath.FromNode(). +/// Low-allocation traversal helper for region resolution. +/// Maintains an incremental slot-index stack while walking the tree and only snapshots +/// a when a match is found. /// -internal sealed class PathTrackingWalker +internal static class RegionTraversal { - private readonly SyntaxNode _root; - private readonly List _pathStack; - private SyntaxNode _current; - - public PathTrackingWalker(SyntaxNode root) - { - _root = root; - _current = root; - _pathStack = new List(8); // Pre-allocate for typical tree depth - } - - /// The current node. - public SyntaxNode Current => _current; - - /// - /// Gets the current path as a NodePath. - /// The path leads to the CURRENT node (not its parent). - /// - public NodePath CurrentPath => new NodePath(ImmutableArray.CreateRange(_pathStack)); - - /// - /// Gets the path to the parent of the current node. - /// Returns Root path if current is the root. - /// - public NodePath ParentPath + internal delegate bool TryGetRegionDelegate(SyntaxNode node, out int consumedCount); + + internal static IEnumerable SelectRegions(SyntaxNode root, TryGetRegionDelegate tryGetRegion) { - get + ArgumentNullException.ThrowIfNull(tryGetRegion); + + var pathStack = new List(8); + var current = root; + + while (true) { - if (_pathStack.Count == 0) - return NodePath.Root; - - // Return path without the last index (current node's sibling index) - return new NodePath(ImmutableArray.CreateRange(_pathStack.Take(_pathStack.Count - 1))); + if (current.Parent != null && tryGetRegion(current, out var consumedCount)) + { + var parent = current.Parent; + var startSlot = current.SiblingIndex; + yield return new QueryRegion( + parentPath: CreateParentPath(pathStack), + parent: parent, + startSlot: startSlot, + endSlot: startSlot + consumedCount, + firstNode: current, + position: current.Position + ); + } + + if (TryMoveToFirstChild(ref current, pathStack)) + continue; + + if (!TryMoveToNextSiblingOrAncestor(ref current, pathStack)) + break; } } - - /// - /// Enumerates all descendants of the root in document order, - /// yielding each node along with its parent path. - /// - public IEnumerable<(SyntaxNode Node, NodePath ParentPath)> DescendantsAndSelfWithPath() + + private static NodePath CreateParentPath(List pathStack) { - // Yield root first - yield return (_root, NodePath.Root); - - // Reset state for traversal - _current = _root; - _pathStack.Clear(); - - while (MoveNext()) - { - yield return (_current, ParentPath); - } + // pathStack is the path to the CURRENT node; parent path is pathStack without the last element. + var parentDepth = pathStack.Count - 1; + if (parentDepth <= 0) + return NodePath.Root; + + var builder = ImmutableArray.CreateBuilder(parentDepth); + for (int i = 0; i < parentDepth; i++) + builder.Add(pathStack[i]); + + return new NodePath(builder.ToImmutable()); } - - /// - /// Moves to the next node in document order (depth-first pre-order). - /// Returns true if moved, false if at end. - /// - private bool MoveNext() + + private static bool TryMoveToFirstChild(ref SyntaxNode current, List pathStack) { - // Try first child - if (_current.SlotCount > 0) + if (current.SlotCount == 0) + return false; + + for (int i = 0; i < current.SlotCount; i++) { - for (int i = 0; i < _current.SlotCount; i++) + var child = current.GetChild(i); + if (child != null) { - var child = _current.GetChild(i); - if (child != null) - { - _pathStack.Add(i); - _current = child; - return true; - } + pathStack.Add(i); + current = child; + return true; } } - - // Try next sibling or ancestor's next sibling - while (_pathStack.Count > 0) + + return false; + } + + private static bool TryMoveToNextSiblingOrAncestor(ref SyntaxNode current, List pathStack) + { + while (pathStack.Count > 0) { - var parent = _current.Parent; + var parent = current.Parent; if (parent == null) - break; - - var currentIndex = _pathStack[_pathStack.Count - 1]; - _pathStack.RemoveAt(_pathStack.Count - 1); - - // Try next sibling + return false; + + var currentIndex = pathStack[^1]; + pathStack.RemoveAt(pathStack.Count - 1); + for (int i = currentIndex + 1; i < parent.SlotCount; i++) { var sibling = parent.GetChild(i); if (sibling != null) { - _pathStack.Add(i); - _current = sibling; + pathStack.Add(i); + current = sibling; return true; } } - - // Move up to try parent's siblings - _current = parent; + + current = parent; } - + return false; } } diff --git a/TinyTokenizer/Ast/SemanticMatchExtensions.cs b/TinyTokenizer/Ast/SemanticMatchExtensions.cs index d28b1c4..21e3b0b 100644 --- a/TinyTokenizer/Ast/SemanticMatchExtensions.cs +++ b/TinyTokenizer/Ast/SemanticMatchExtensions.cs @@ -134,19 +134,18 @@ private SyntaxNodeQuery(NodeKind kind, Func? predicate, Select public override IEnumerable Select(SyntaxTree tree) => Select(tree.Root); public override IEnumerable Select(SyntaxNode root) + { + return SelectionModeHelper.Apply(EnumerateMatches(root), _mode, _modeArg); + } + + private IEnumerable EnumerateMatches(SyntaxNode root) { var walker = new TreeWalker(root); - var matches = walker.DescendantsAndSelf().Where(Matches); - - return _mode switch + foreach (var node in walker.DescendantsAndSelf()) { - SelectionMode.First => matches.Take(1), - SelectionMode.Last => matches.TakeLast(1), - SelectionMode.Nth => matches.Skip(_modeArg).Take(1), - SelectionMode.Skip => matches.Skip(_modeArg), - SelectionMode.Take => matches.Take(_modeArg), - _ => matches - }; + if (node.Kind == _kind && (_predicate == null || _predicate(node))) + yield return node; + } } public override bool Matches(SyntaxNode node) => @@ -206,45 +205,72 @@ public override IEnumerable Select(SyntaxTree tree) } private IEnumerable SelectWithKind(SyntaxNode root, NodeKind kind) + { + return ApplyMode(EnumerateMatchesByKind(root, kind)); + } + + private IEnumerable EnumerateMatchesByKind(SyntaxNode root, NodeKind kind) { var walker = new TreeWalker(root); - var matches = walker.DescendantsAndSelf() - .Where(n => n.Kind == kind && (_predicate == null || _predicate((T)n))); - - return ApplyMode(matches); + foreach (var node in walker.DescendantsAndSelf()) + { + if (node.Kind != kind) + continue; + + if (node is not T typed) + continue; + + if (_predicate != null && !_predicate(typed)) + continue; + + yield return node; + } } public override IEnumerable Select(SyntaxNode root) { // Match by C# type (no schema available) + return ApplyMode(EnumerateMatchesByType(root)); + } + + private IEnumerable EnumerateMatchesByType(SyntaxNode root) + { var walker = new TreeWalker(root); - var matches = walker.DescendantsAndSelf().Where(Matches); - - return ApplyMode(matches); + foreach (var node in walker.DescendantsAndSelf()) + { + if (node is T typed && (_predicate == null || _predicate(typed))) + yield return node; + } } private IEnumerable ApplyMode(IEnumerable matches) { - return _mode switch - { - SelectionMode.First => matches.Take(1), - SelectionMode.Last => matches.TakeLast(1), - SelectionMode.Nth => matches.Skip(_modeArg).Take(1), - SelectionMode.Skip => matches.Skip(_modeArg), - SelectionMode.Take => matches.Take(_modeArg), - _ => matches - }; + return SelectionModeHelper.Apply(matches, _mode, _modeArg); } /// /// Selects and casts to the strongly-typed syntax node. /// - public IEnumerable SelectTyped(SyntaxTree tree) => Select(tree).Cast(); + public IEnumerable SelectTyped(SyntaxTree tree) + { + foreach (var node in Select(tree)) + { + if (node is T typed) + yield return typed; + } + } /// /// Selects and casts to the strongly-typed syntax node. /// - public IEnumerable SelectTyped(SyntaxNode root) => Select(root).Cast(); + public IEnumerable SelectTyped(SyntaxNode root) + { + foreach (var node in Select(root)) + { + if (node is T typed) + yield return typed; + } + } public override bool Matches(SyntaxNode node) => node is T typed && (_predicate == null || _predicate(typed)); diff --git a/TinyTokenizer/Ast/SyntaxEditor.cs b/TinyTokenizer/Ast/SyntaxEditor.cs index 47efd84..77e3148 100644 --- a/TinyTokenizer/Ast/SyntaxEditor.cs +++ b/TinyTokenizer/Ast/SyntaxEditor.cs @@ -1066,21 +1066,43 @@ private static ImmutableArray TransferTrivia( var result = nodes.ToBuilder(); // Add leading trivia to first node - if (!leading.IsEmpty && result[0] is GreenLeaf firstLeaf) + if (!leading.IsEmpty) { - var newLeading = firstLeaf.LeadingTrivia.IsEmpty - ? leading - : leading.AddRange(firstLeaf.LeadingTrivia); - result[0] = firstLeaf.WithLeadingTrivia(newLeading); + if (result[0] is GreenLeaf firstLeaf) + { + var newLeading = firstLeaf.LeadingTrivia.IsEmpty + ? leading + : leading.AddRange(firstLeaf.LeadingTrivia); + result[0] = firstLeaf.WithLeadingTrivia(newLeading); + } + else if (result[0] is GreenBlock firstBlock) + { + var existingLeading = firstBlock.OpenerNode.LeadingTrivia; + var newLeading = existingLeading.IsEmpty + ? leading + : leading.AddRange(existingLeading); + result[0] = firstBlock.WithLeadingTrivia(newLeading); + } } // Add trailing trivia to last node - if (!trailing.IsEmpty && result[^1] is GreenLeaf lastLeaf) + if (!trailing.IsEmpty) { - var newTrailing = lastLeaf.TrailingTrivia.IsEmpty - ? trailing - : lastLeaf.TrailingTrivia.AddRange(trailing); - result[^1] = lastLeaf.WithTrailingTrivia(newTrailing); + if (result[^1] is GreenLeaf lastLeaf) + { + var newTrailing = lastLeaf.TrailingTrivia.IsEmpty + ? trailing + : lastLeaf.TrailingTrivia.AddRange(trailing); + result[^1] = lastLeaf.WithTrailingTrivia(newTrailing); + } + else if (result[^1] is GreenBlock lastBlock) + { + var existingTrailing = lastBlock.CloserNode.TrailingTrivia; + var newTrailing = existingTrailing.IsEmpty + ? trailing + : existingTrailing.AddRange(trailing); + result[^1] = lastBlock.WithTrailingTrivia(newTrailing); + } } return result.ToImmutable(); @@ -1143,21 +1165,43 @@ private static ImmutableArray TransferTrivia( var result = nodes.ToBuilder(); // Add leading trivia to first node - if (!leading.IsEmpty && result[0] is GreenLeaf firstLeaf) + if (!leading.IsEmpty) { - var newLeading = firstLeaf.LeadingTrivia.IsEmpty - ? leading - : leading.AddRange(firstLeaf.LeadingTrivia); - result[0] = firstLeaf.WithLeadingTrivia(newLeading); + if (result[0] is GreenLeaf firstLeaf) + { + var newLeading = firstLeaf.LeadingTrivia.IsEmpty + ? leading + : leading.AddRange(firstLeaf.LeadingTrivia); + result[0] = firstLeaf.WithLeadingTrivia(newLeading); + } + else if (result[0] is GreenBlock firstBlock) + { + var existingLeading = firstBlock.OpenerNode.LeadingTrivia; + var newLeading = existingLeading.IsEmpty + ? leading + : leading.AddRange(existingLeading); + result[0] = firstBlock.WithLeadingTrivia(newLeading); + } } // Add trailing trivia to last node - if (!trailing.IsEmpty && result[^1] is GreenLeaf lastLeaf) + if (!trailing.IsEmpty) { - var newTrailing = lastLeaf.TrailingTrivia.IsEmpty - ? trailing - : lastLeaf.TrailingTrivia.AddRange(trailing); - result[^1] = lastLeaf.WithTrailingTrivia(newTrailing); + if (result[^1] is GreenLeaf lastLeaf) + { + var newTrailing = lastLeaf.TrailingTrivia.IsEmpty + ? trailing + : lastLeaf.TrailingTrivia.AddRange(trailing); + result[^1] = lastLeaf.WithTrailingTrivia(newTrailing); + } + else if (result[^1] is GreenBlock lastBlock) + { + var existingTrailing = lastBlock.CloserNode.TrailingTrivia; + var newTrailing = existingTrailing.IsEmpty + ? trailing + : existingTrailing.AddRange(trailing); + result[^1] = lastBlock.WithTrailingTrivia(newTrailing); + } } return result.ToImmutable(); @@ -1241,20 +1285,42 @@ private static ImmutableArray TransferTrivia( var result = nodes.ToBuilder(); - if (!leading.IsEmpty && result[0] is GreenLeaf firstLeaf) + if (!leading.IsEmpty) { - var newLeading = firstLeaf.LeadingTrivia.IsEmpty - ? leading - : leading.AddRange(firstLeaf.LeadingTrivia); - result[0] = firstLeaf.WithLeadingTrivia(newLeading); + if (result[0] is GreenLeaf firstLeaf) + { + var newLeading = firstLeaf.LeadingTrivia.IsEmpty + ? leading + : leading.AddRange(firstLeaf.LeadingTrivia); + result[0] = firstLeaf.WithLeadingTrivia(newLeading); + } + else if (result[0] is GreenBlock firstBlock) + { + var existingLeading = firstBlock.OpenerNode.LeadingTrivia; + var newLeading = existingLeading.IsEmpty + ? leading + : leading.AddRange(existingLeading); + result[0] = firstBlock.WithLeadingTrivia(newLeading); + } } - if (!trailing.IsEmpty && result[^1] is GreenLeaf lastLeaf) + if (!trailing.IsEmpty) { - var newTrailing = lastLeaf.TrailingTrivia.IsEmpty - ? trailing - : lastLeaf.TrailingTrivia.AddRange(trailing); - result[^1] = lastLeaf.WithTrailingTrivia(newTrailing); + if (result[^1] is GreenLeaf lastLeaf) + { + var newTrailing = lastLeaf.TrailingTrivia.IsEmpty + ? trailing + : lastLeaf.TrailingTrivia.AddRange(trailing); + result[^1] = lastLeaf.WithTrailingTrivia(newTrailing); + } + else if (result[^1] is GreenBlock lastBlock) + { + var existingTrailing = lastBlock.CloserNode.TrailingTrivia; + var newTrailing = existingTrailing.IsEmpty + ? trailing + : existingTrailing.AddRange(trailing); + result[^1] = lastBlock.WithTrailingTrivia(newTrailing); + } } return result.ToImmutable(); @@ -1309,20 +1375,42 @@ private static ImmutableArray TransferTrivia( var result = nodes.ToBuilder(); - if (!leading.IsEmpty && result[0] is GreenLeaf firstLeaf) + if (!leading.IsEmpty) { - var newLeading = firstLeaf.LeadingTrivia.IsEmpty - ? leading - : leading.AddRange(firstLeaf.LeadingTrivia); - result[0] = firstLeaf.WithLeadingTrivia(newLeading); + if (result[0] is GreenLeaf firstLeaf) + { + var newLeading = firstLeaf.LeadingTrivia.IsEmpty + ? leading + : leading.AddRange(firstLeaf.LeadingTrivia); + result[0] = firstLeaf.WithLeadingTrivia(newLeading); + } + else if (result[0] is GreenBlock firstBlock) + { + var existingLeading = firstBlock.OpenerNode.LeadingTrivia; + var newLeading = existingLeading.IsEmpty + ? leading + : leading.AddRange(existingLeading); + result[0] = firstBlock.WithLeadingTrivia(newLeading); + } } - if (!trailing.IsEmpty && result[^1] is GreenLeaf lastLeaf) + if (!trailing.IsEmpty) { - var newTrailing = lastLeaf.TrailingTrivia.IsEmpty - ? trailing - : lastLeaf.TrailingTrivia.AddRange(trailing); - result[^1] = lastLeaf.WithTrailingTrivia(newTrailing); + if (result[^1] is GreenLeaf lastLeaf) + { + var newTrailing = lastLeaf.TrailingTrivia.IsEmpty + ? trailing + : lastLeaf.TrailingTrivia.AddRange(trailing); + result[^1] = lastLeaf.WithTrailingTrivia(newTrailing); + } + else if (result[^1] is GreenBlock lastBlock) + { + var existingTrailing = lastBlock.CloserNode.TrailingTrivia; + var newTrailing = existingTrailing.IsEmpty + ? trailing + : existingTrailing.AddRange(trailing); + result[^1] = lastBlock.WithTrailingTrivia(newTrailing); + } } return result.ToImmutable(); diff --git a/syntaxeditor-flag-mutation-test-gaps.todo b/syntaxeditor-flag-mutation-test-gaps.todo new file mode 100644 index 0000000..572680d --- /dev/null +++ b/syntaxeditor-flag-mutation-test-gaps.todo @@ -0,0 +1,159 @@ +```todo +# SyntaxEditor Green-Flag Mutation Test Gaps + +Goal: expand `SyntaxEditor` test coverage to ensure `GreenNodeFlags` stay correct after mutations, +with emphasis on boundary-vs-contains semantics and undo/redo correctness. + +Context: +- Boundary flags are token-owned (`HasLeading*` / `HasTrailing*`), and containers must not inherit them. +- Contains flags (`Contains*`) must accurately reflect subtree presence. +- We already have a small set of green-flag mutation tests in `TinyTokenizer.Tests/SyntaxEditorTests.cs`. + +------------------------------------------------------------------------------ + +## Phase 0 — Inventory + Harness + +- [x] Decide location for new tests + - [x] Keep under `#region Green Flag Mutation Tests` in `TinyTokenizer.Tests/SyntaxEditorTests.cs`, OR + - [ ] Create a dedicated test class file (e.g., `TinyTokenizer.Tests/SyntaxEditorGreenFlagsTests.cs`) for clarity + +- [x] Add a small helper to find leaves consistently after edits + - [x] Helper: `FindToken(tree, kind, text)` (or `Query.Ident("...")` + `.OfType()`) + - [x] Avoid relying on indexes when trivia can shift; prefer `Query` + text predicates + +Acceptance criteria: +- Tests are easy to read and robust to trivia shape. + +------------------------------------------------------------------------------ + +## Phase 1 — Boundary Flags After Replace + +- [x] Replace preserves *trailing newline* boundary ownership + - [x] Source: `"a\nb"` + - [x] Replace `a` -> `X` + - [x] Assert `X.Green.Flags` includes `HasTrailingNewlineTrivia` + - [x] Assert `b.Green.Flags` does NOT include `HasLeadingNewlineTrivia` + - [x] Assert `tree.GreenRoot.Flags` includes `ContainsNewlineTrivia` + +- [x] Replace preserves *same-line comment* trailing boundary flags + - [x] Use `TokenizerOptions.Default.WithCommentStyles(CommentStyle.CStyleSingleLine)` + - [x] Source: `"a // c\nb"` + - [x] Replace `a` -> `X` + - [x] Assert `X.Green.Flags` includes `HasTrailingCommentTrivia` AND `HasTrailingNewlineTrivia` + - [x] Assert `b.Green.Flags` does NOT include `HasLeadingCommentTrivia` or `HasLeadingNewlineTrivia` + +- [x] Replace preserves/tranfers *trailing whitespace* boundary (non-newline) + - [x] Source that produces trailing whitespace trivia on the replaced token (e.g., `"a \nb"` or similar) + - [x] Replace target token + - [x] Assert `HasTrailingWhitespaceTrivia` is preserved on replacement token + +Acceptance criteria: +- Boundary flags remain token-centric and are preserved on replacement where trivia is preserved. + +------------------------------------------------------------------------------ + +## Phase 2 — Boundary Flags After InsertBefore / InsertAfter + +- [x] InsertBefore does NOT steal leading whitespace from the following node + - [x] Source: `"a\n b"` (indentation is leading whitespace on `b`) + - [x] InsertBefore(`b`, `"X"`) + - [x] Assert `b.Green.Flags` includes `HasLeadingWhitespaceTrivia` + - [x] Assert `X.Green.Flags` does NOT unexpectedly include `HasLeadingWhitespaceTrivia` + - [x] Assert `tree.GreenRoot.Flags` includes `ContainsWhitespaceTrivia` + +- [x] InsertAfter does not accidentally manufacture leading newline on following token + - [x] Already partially covered for inserted `X\n`; add coverage for variations: + - [x] InsertAfter with `"X\r\n"` (CRLF) + - [x] InsertAfter with `" X\n"` (inserted node has leading whitespace + trailing newline) + - [x] Assert following token does NOT gain `HasLeadingNewlineTrivia` + +Acceptance criteria: +- Insertion doesn’t transfer boundary ownership incorrectly and doesn’t corrupt following-token boundary flags. + +------------------------------------------------------------------------------ + +## Phase 3 — Contains Flags Correctness Under Partial Removal + +- [x] Removing one newline owner does not clear `ContainsNewlineTrivia` when others remain + - [x] Source: `"a\nb\nc"` + - [x] Remove `a` + - [x] Assert `tree.GreenRoot.Flags` still includes `ContainsNewlineTrivia` + +- [x] Removing one comment owner does not clear `ContainsCommentTrivia` when others remain + - [x] Use `CommentStyle.CStyleSingleLine` + - [x] Source with multiple comments + - [x] Remove one token owning a comment + - [x] Assert `ContainsCommentTrivia` remains set + +- [x] Removing one whitespace owner does not clear `ContainsWhitespaceTrivia` when others remain + - [x] Source with multiple leading/trailing whitespace trivia across tokens + - [x] Remove one token + - [x] Assert `ContainsWhitespaceTrivia` remains set + +Acceptance criteria: +- Contains flags behave like true subtree aggregations (don’t clear prematurely). + +------------------------------------------------------------------------------ + +## Phase 4 — Multi-node Replacement Trivia Transfer (First/Last) + +- [x] Replace a token with *multiple nodes* transfers leading boundary to first and trailing boundary to last + - [x] Source where target has leading boundary trivia and trailing boundary trivia + - [x] Replace target with `"X Y"` + - [x] Assert `X` receives the leading boundary flags expected + - [x] Assert `Y` receives the trailing boundary flags expected + - [x] Assert middle nodes (if any) do not get boundary flags unless their own trivia demands it + +- [x] Replace with nodes that begin/end with containers (edge-case semantics) + - [x] Replace a token with something like `"{x}"` or `"(x)"` + - [x] Decide expected behavior: preserve boundary trivia on block opener/closer when replacement is a block + - [x] Add assertions to lock in intended behavior + +Acceptance criteria: +- Trivia transfer semantics are explicitly tested and stable. + +------------------------------------------------------------------------------ + +## Phase 5 — Undo/Redo Leaf-Level Flag Restoration + +- [x] Undo/Redo restores *leaf* boundary flags, not just root flags + - [x] Start with a tree where a target leaf’s flags are known + - [x] Commit an edit that changes boundary/contains flags (e.g., insert `X\n`) + - [x] Capture the leaf flags before/after + - [x] Undo and assert leaf flags are restored + - [x] Redo and assert leaf flags match the mutated state + +Acceptance criteria: +- Undo/Redo correctness is verified at the leaf level. + +------------------------------------------------------------------------------ + +## Phase 6 — Schema/Rebind Interaction (Syntax Binding) + +- [x] Ensure edits with schema + syntax definitions do not violate token-centric boundary semantics + - [x] Parse with a schema that includes syntax definitions (so `Commit()` triggers `RebindAt`) + - [x] Perform insert/replace/remove inside or around bound syntax nodes + - [x] Assert: + - [x] leaf boundary flags are correct + - [x] syntax containers do NOT have `GreenNodeFlagMasks.Boundary` + - [x] contains flags reflect subtree + +Acceptance criteria: +- Rebinding does not break boundary/contains invariants. + +------------------------------------------------------------------------------ + +## Phase 7 — Oracle-Style Regression Test (Optional “Big Hammer”) + +- [x] Add a “reparse oracle” test for flags + - [x] After an edit, compute `editedText = tree.ToText()` + - [x] Parse a fresh `oracle = SyntaxTree.Parse(editedText, sameOptionsOrSchema)` + - [x] Compare: + - [x] `tree.GreenRoot.Flags == oracle.GreenRoot.Flags` + - [x] Per-leaf flags match for corresponding leaves (by `(Kind, Text, occurrence index)` or by positions) + - [x] Use this for 1–2 representative complex edits (nested blocks + comments) + +Acceptance criteria: +- A broad regression net exists that catches stale/miscomputed flags without enumerating every case. + +```