diff --git a/TinyTokenizer.Benchmarks/SyntaxEditorBenchmarks.cs b/TinyTokenizer.Benchmarks/SyntaxEditorBenchmarks.cs new file mode 100644 index 0000000..9f73d1c --- /dev/null +++ b/TinyTokenizer.Benchmarks/SyntaxEditorBenchmarks.cs @@ -0,0 +1,375 @@ +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using TinyTokenizer.Ast; +using Q = TinyTokenizer.Ast.Query; + +namespace TinyTokenizer.Benchmarks; + +/// +/// Benchmarks for SyntaxEditor operations including replacements, insertions, +/// and region resolution on various tree sizes and depths. +/// +[MemoryDiagnoser] +[GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByCategory)] +[CategoriesColumn] +public class SyntaxEditorBenchmarks +{ + #region Test Data + + /// + /// Simple expression for single-token replacements. + /// + private const string SimpleExpression = "foo + bar * baz"; + + /// + /// Empty block for inner replacement benchmarks. + /// + private const string EmptyBlockInput = "function test() { }"; + + /// + /// Block with content for inner replacement benchmarks. + /// + private const string BlockWithContent = """ + function test() { + var x = 1; + var y = 2; + return x + y; + } + """; + + /// + /// Multiple functions for multi-position insertions. + /// + private static readonly string MultiFunctionInput = GenerateMultiFunctionInput(); + + /// + /// Deeply nested blocks for deep tree traversal. + /// + private static readonly string DeeplyNestedInput = GenerateDeeplyNestedInput(20); + + /// + /// Very deeply nested blocks for stress testing. + /// + private static readonly string VeryDeeplyNestedInput = GenerateDeeplyNestedInput(50); + + private static readonly Schema DefaultSchema = Schema.Create() + .AddCommentStyles(CommentStyle.CStyleSingleLine, CommentStyle.CStyleMultiLine) + .WithOperators(CommonOperators.CFamily) + .Build(); + + // Pre-parsed trees + private static readonly SyntaxTree SimpleTree = SyntaxTree.Parse(SimpleExpression, DefaultSchema); + private static readonly SyntaxTree EmptyBlockTree = SyntaxTree.Parse(EmptyBlockInput, DefaultSchema); + private static readonly SyntaxTree BlockWithContentTree = SyntaxTree.Parse(BlockWithContent, DefaultSchema); + private static readonly SyntaxTree MultiFunctionTree = SyntaxTree.Parse(MultiFunctionInput, DefaultSchema); + private static readonly SyntaxTree DeeplyNestedTree = SyntaxTree.Parse(DeeplyNestedInput, DefaultSchema); + private static readonly SyntaxTree VeryDeeplyNestedTree = SyntaxTree.Parse(VeryDeeplyNestedInput, DefaultSchema); + + private static string GenerateMultiFunctionInput() + { + var template = """ + function func{0}(a, b) {{ + return a + b; + }} + + """; + + return string.Concat(Enumerable.Range(0, 50).Select(i => + string.Format(template, i))); + } + + private static string GenerateDeeplyNestedInput(int depth) + { + var open = string.Concat(Enumerable.Repeat("{ ", depth)); + var close = string.Concat(Enumerable.Repeat(" }", depth)); + return $"function deep() {open}x{close}"; + } + + #endregion + + #region Replace - Single Token + + /// + /// Replace a single identifier token with another identifier. + /// Baseline for minimal edit operation. + /// + [Benchmark(Baseline = true)] + [BenchmarkCategory("Replace")] + public SyntaxTree Replace_SingleToken() + { + var tree = SyntaxTree.Parse(SimpleExpression, DefaultSchema); + tree.CreateEditor() + .Replace(Q.Ident("foo"), "replaced") + .Commit(); + return tree; + } + + /// + /// Replace multiple tokens in a single edit batch. + /// + [Benchmark] + [BenchmarkCategory("Replace")] + public SyntaxTree Replace_MultipleTokens() + { + var tree = SyntaxTree.Parse(SimpleExpression, DefaultSchema); + tree.CreateEditor() + .Replace(Q.Ident("foo"), "a") + .Replace(Q.Ident("bar"), "b") + .Replace(Q.Ident("baz"), "c") + .Commit(); + return tree; + } + + /// + /// Replace all identifiers using a single query. + /// + [Benchmark] + [BenchmarkCategory("Replace")] + public SyntaxTree Replace_AllIdents() + { + var tree = SyntaxTree.Parse(SimpleExpression, DefaultSchema); + tree.CreateEditor() + .Replace(Q.AnyIdent, "x") + .Commit(); + return tree; + } + + #endregion + + #region Replace - Block Inner + + /// + /// Replace the inner content of an empty block. + /// Tests zero-width region handling. + /// + [Benchmark] + [BenchmarkCategory("BlockInner")] + public SyntaxTree Replace_BlockInner_Empty() + { + var tree = SyntaxTree.Parse(EmptyBlockInput, DefaultSchema); + tree.CreateEditor() + .Replace(Q.BraceBlock.First().Inner(), "return 42;") + .Commit(); + return tree; + } + + /// + /// Replace the inner content of a block with existing content. + /// Tests multi-slot region replacement. + /// + [Benchmark] + [BenchmarkCategory("BlockInner")] + public SyntaxTree Replace_BlockInner_WithContent() + { + var tree = SyntaxTree.Parse(BlockWithContent, DefaultSchema); + tree.CreateEditor() + .Replace(Q.BraceBlock.First().Inner(), "return 0;") + .Commit(); + return tree; + } + + /// + /// Replace inner content with larger replacement text. + /// Tests tree rebuilding with size change. + /// + [Benchmark] + [BenchmarkCategory("BlockInner")] + public SyntaxTree Replace_BlockInner_Larger() + { + var tree = SyntaxTree.Parse(EmptyBlockInput, DefaultSchema); + var largeContent = string.Join("\n", Enumerable.Range(0, 20).Select(i => $"var x{i} = {i};")); + tree.CreateEditor() + .Replace(Q.BraceBlock.First().Inner(), largeContent) + .Commit(); + return tree; + } + + #endregion + + #region Insert - Multiple Positions + + /// + /// Insert after a single position. + /// + [Benchmark] + [BenchmarkCategory("Insert")] + public SyntaxTree InsertAfter_SinglePosition() + { + var tree = SyntaxTree.Parse(MultiFunctionInput, DefaultSchema); + tree.CreateEditor() + .InsertAfter(Q.BraceBlock.First().Start(), "// inserted\n") + .Commit(); + return tree; + } + + /// + /// Insert after all function body starts (50 positions). + /// Tests edit batching with many positions. + /// + [Benchmark] + [BenchmarkCategory("Insert")] + public SyntaxTree InsertAfter_ManyPositions() + { + var tree = SyntaxTree.Parse(MultiFunctionInput, DefaultSchema); + tree.CreateEditor() + .InsertAfter(Q.BraceBlock.Start(), "// inserted\n") + .Commit(); + return tree; + } + + /// + /// Insert before all closing braces. + /// + [Benchmark] + [BenchmarkCategory("Insert")] + public SyntaxTree InsertBefore_ManyPositions() + { + var tree = SyntaxTree.Parse(MultiFunctionInput, DefaultSchema); + tree.CreateEditor() + .InsertBefore(Q.BraceBlock.End(), "\n// end") + .Commit(); + return tree; + } + + #endregion + + #region SelectRegions - Deep Trees + + /// + /// Select regions in a moderately deep tree (20 levels). + /// + [Benchmark] + [BenchmarkCategory("SelectRegions")] + public int SelectRegions_DeepTree() + { + var count = 0; + foreach (var _ in ((IRegionQuery)Q.AnyIdent).SelectRegions(DeeplyNestedTree)) + { + count++; + } + return count; + } + + /// + /// Select regions in a very deep tree (50 levels). + /// + [Benchmark] + [BenchmarkCategory("SelectRegions")] + public int SelectRegions_VeryDeepTree() + { + var count = 0; + foreach (var _ in ((IRegionQuery)Q.AnyIdent).SelectRegions(VeryDeeplyNestedTree)) + { + count++; + } + return count; + } + + /// + /// Select block regions in a deep tree. + /// Tests BlockNodeQuery optimization. + /// + [Benchmark] + [BenchmarkCategory("SelectRegions")] + public int SelectRegions_DeepTree_Blocks() + { + var count = 0; + foreach (var _ in ((IRegionQuery)Q.BraceBlock).SelectRegions(DeeplyNestedTree)) + { + count++; + } + return count; + } + + /// + /// Select first match only - tests short-circuit. + /// + [Benchmark] + [BenchmarkCategory("SelectRegions")] + public int SelectRegions_DeepTree_First() + { + var count = 0; + foreach (var _ in ((IRegionQuery)Q.AnyIdent.First()).SelectRegions(VeryDeeplyNestedTree)) + { + count++; + } + return count; + } + + #endregion + + #region Edit with Transformer + + /// + /// Edit using a transformer function (uppercase identifiers). + /// + [Benchmark] + [BenchmarkCategory("Transform")] + public SyntaxTree Edit_Transform_SingleToken() + { + var tree = SyntaxTree.Parse(SimpleExpression, DefaultSchema); + tree.CreateEditor() + .Edit(Q.AnyIdent, text => text.ToUpperInvariant()) + .Commit(); + return tree; + } + + /// + /// Edit using Replace with RedNode transformer. + /// + [Benchmark] + [BenchmarkCategory("Transform")] + public SyntaxTree Replace_WithNodeTransformer() + { + var tree = SyntaxTree.Parse(SimpleExpression, DefaultSchema); + tree.CreateEditor() + .Replace(Q.AnyIdent, node => node.ToText().ToUpperInvariant()) + .Commit(); + return tree; + } + + #endregion + + #region Undo/Redo + + /// + /// Perform edit then undo. + /// + [Benchmark] + [BenchmarkCategory("UndoRedo")] + public SyntaxTree Edit_ThenUndo() + { + var tree = SyntaxTree.Parse(SimpleExpression, DefaultSchema); + tree.CreateEditor() + .Replace(Q.Ident("foo"), "replaced") + .Commit(); + tree.Undo(); + return tree; + } + + /// + /// Perform multiple edits then undo all. + /// + [Benchmark] + [BenchmarkCategory("UndoRedo")] + public SyntaxTree MultipleEdits_ThenUndoAll() + { + var tree = SyntaxTree.Parse(SimpleExpression, DefaultSchema); + + for (int i = 0; i < 5; i++) + { + tree.CreateEditor() + .Replace(Q.AnyIdent.First(), $"v{i}") + .Commit(); + } + + while (tree.CanUndo) + { + tree.Undo(); + } + + return tree; + } + + #endregion +} diff --git a/TinyTokenizer.Tests/GreenNodeTests.cs b/TinyTokenizer.Tests/GreenNodeTests.cs index d4ca5bd..20512b2 100644 --- a/TinyTokenizer.Tests/GreenNodeTests.cs +++ b/TinyTokenizer.Tests/GreenNodeTests.cs @@ -46,12 +46,16 @@ public void GreenNode_GetSlotOffset_ComputesCorrectly() var child3 = new GreenLeaf(NodeKind.Ident, "c"); // width 1 var block = GreenBlock.Create('{', ImmutableArray.Create(child1, child2, child3)); - // Offset 0: after opener '{' (1 char) - Assert.Equal(1, block.GetSlotOffset(0)); - // Offset 1: after child1 (1 + 3 = 4) - Assert.Equal(4, block.GetSlotOffset(1)); - // Offset 2: after child2 (1 + 3 + 2 = 6) - Assert.Equal(6, block.GetSlotOffset(2)); + // Slot 0: opener '{' at offset 0 + Assert.Equal(0, block.GetSlotOffset(0)); + // Slot 1: first inner child after opener (1) + Assert.Equal(1, block.GetSlotOffset(1)); + // Slot 2: after child1 (1 + 3 = 4) + Assert.Equal(4, block.GetSlotOffset(2)); + // Slot 3: after child2 (1 + 3 + 2 = 6) + Assert.Equal(6, block.GetSlotOffset(3)); + // Slot 4: closer after all children (1 + 3 + 2 + 1 = 7) + Assert.Equal(7, block.GetSlotOffset(4)); } #endregion @@ -327,7 +331,10 @@ public void GreenBlock_SlotCount_ReturnsChildCount() new GreenLeaf(NodeKind.Ident, "c")); var block = GreenBlock.Create('{', children); - Assert.Equal(3, block.SlotCount); + // SlotCount = opener + 3 inner children + closer = 5 + Assert.Equal(5, block.SlotCount); + // InnerChildren should still be 3 + Assert.Equal(3, block.InnerChildren.Length); } [Fact] @@ -336,7 +343,10 @@ public void GreenBlock_GetSlot_ReturnsChild() var child = new GreenLeaf(NodeKind.Ident, "test"); var block = GreenBlock.Create('{', ImmutableArray.Create(child)); - Assert.Same(child, block.GetSlot(0)); + // Slot 0 is opener, slot 1 is first inner child, slot 2 is closer + Assert.Equal(NodeKind.Symbol, block.GetSlot(0)!.Kind); // opener + Assert.Same(child, block.GetSlot(1)); // inner child + Assert.Equal(NodeKind.Symbol, block.GetSlot(2)!.Kind); // closer } [Fact] @@ -344,7 +354,10 @@ public void GreenBlock_GetSlot_OutOfRange_ReturnsNull() { var block = GreenBlock.Create('{', ImmutableArray.Empty); - Assert.Null(block.GetSlot(0)); + // Empty block has 2 slots: opener (0) and closer (1) + Assert.NotNull(block.GetSlot(0)); // opener + Assert.NotNull(block.GetSlot(1)); // closer + Assert.Null(block.GetSlot(2)); // out of range Assert.Null(block.GetSlot(-1)); Assert.Null(block.GetSlot(100)); } @@ -371,10 +384,14 @@ public void GreenBlock_GetSlotOffset_SmallBlock() var child2 = new GreenLeaf(NodeKind.Ident, "bbb"); // width 3 var block = GreenBlock.Create('{', ImmutableArray.Create(child1, child2)); - // slot 0: after opener = 1 - Assert.Equal(1, block.GetSlotOffset(0)); - // slot 1: after opener + child1 = 1 + 2 = 3 - Assert.Equal(3, block.GetSlotOffset(1)); + // slot 0: opener at offset 0 + Assert.Equal(0, block.GetSlotOffset(0)); + // slot 1: first child after opener = 1 + Assert.Equal(1, block.GetSlotOffset(1)); + // slot 2: after opener + child1 = 1 + 2 = 3 + Assert.Equal(3, block.GetSlotOffset(2)); + // slot 3: closer after all = 1 + 2 + 3 = 6 + Assert.Equal(6, block.GetSlotOffset(3)); } [Fact] @@ -384,8 +401,10 @@ public void GreenBlock_GetSlotOffset_WithLeadingTrivia() var child = new GreenLeaf(NodeKind.Ident, "x"); var block = GreenBlock.Create('{', ImmutableArray.Create(child), leading); - // slot 0: leading trivia(2) + opener(1) = 3 - Assert.Equal(3, block.GetSlotOffset(0)); + // slot 0: opener at offset 0 (trivia is part of opener's width) + Assert.Equal(0, block.GetSlotOffset(0)); + // slot 1: first child after opener = leading trivia(2) + opener(1) = 3 + Assert.Equal(3, block.GetSlotOffset(1)); } [Fact] @@ -399,13 +418,19 @@ public void GreenBlock_GetSlotOffset_LargeBlock_UsesPrecomputed() } var block = GreenBlock.Create('{', children.ToImmutableArray()); - // Verify offsets are computed correctly + // Slot 0 is opener at offset 0 + Assert.Equal(0, block.GetSlotOffset(0)); + + // Verify inner child offsets are computed correctly (slots 1..15) int expectedOffset = 1; // After opener for (int i = 0; i < 15; i++) { - Assert.Equal(expectedOffset, block.GetSlotOffset(i)); + Assert.Equal(expectedOffset, block.GetSlotOffset(i + 1)); // +1 because slot 0 is opener expectedOffset += children[i].Width; } + + // Slot 16 is closer + Assert.Equal(expectedOffset, block.GetSlotOffset(16)); } #endregion @@ -419,11 +444,12 @@ public void GreenBlock_WithSlot_ReplacesChild() var replacement = new GreenLeaf(NodeKind.Ident, "new"); var block = GreenBlock.Create('{', ImmutableArray.Create(original)); - var modified = block.WithSlot(0, replacement); + // Replace slot 1 (first inner child) + var modified = block.WithSlot(1, replacement); Assert.NotSame(block, modified); - Assert.Same(replacement, modified.GetSlot(0)); - Assert.Same(original, block.GetSlot(0)); // Original unchanged + Assert.Same(replacement, modified.GetSlot(1)); + Assert.Same(original, block.GetSlot(1)); // Original unchanged } [Fact] @@ -435,12 +461,13 @@ public void GreenBlock_WithSlot_SharesUnchangedChildren() var replacement = new GreenLeaf(NodeKind.Ident, "X"); var block = GreenBlock.Create('{', ImmutableArray.Create(child1, child2, child3)); - var modified = block.WithSlot(1, replacement); + // Replace slot 2 (second inner child) + var modified = block.WithSlot(2, replacement); - // child1 and child3 should be shared - Assert.Same(child1, modified.GetSlot(0)); - Assert.Same(replacement, modified.GetSlot(1)); - Assert.Same(child3, modified.GetSlot(2)); + // child1 and child3 should be shared (at slots 1 and 3) + Assert.Same(child1, modified.GetSlot(1)); + Assert.Same(replacement, modified.GetSlot(2)); + Assert.Same(child3, modified.GetSlot(3)); } [Fact] @@ -449,7 +476,10 @@ public void GreenBlock_WithSlot_InvalidIndex_Throws() var block = GreenBlock.Create('{', ImmutableArray.Empty); var child = new GreenLeaf(NodeKind.Ident, "x"); - Assert.Throws(() => block.WithSlot(0, child)); + // Empty block has 2 slots: opener (0) and closer (1) + // Slot 0 can only be replaced with a valid opener leaf + // Slot 2+ is out of range + Assert.Throws(() => block.WithSlot(2, child)); Assert.Throws(() => block.WithSlot(-1, child)); } @@ -461,12 +491,14 @@ public void GreenBlock_WithInsert_InsertsAtIndex() var inserted = new GreenLeaf(NodeKind.Ident, "X"); var block = GreenBlock.Create('{', ImmutableArray.Create(child1, child2)); - var modified = block.WithInsert(1, ImmutableArray.Create(inserted)); + // Insert at slot 2 (between a and b) + var modified = block.WithInsert(2, ImmutableArray.Create(inserted)); - Assert.Equal(3, modified.SlotCount); - Assert.Same(child1, modified.GetSlot(0)); - Assert.Same(inserted, modified.GetSlot(1)); - Assert.Same(child2, modified.GetSlot(2)); + // 5 slots: opener + 3 inner + closer + Assert.Equal(5, modified.SlotCount); + Assert.Same(child1, modified.GetSlot(1)); + Assert.Same(inserted, modified.GetSlot(2)); + Assert.Same(child2, modified.GetSlot(3)); } [Fact] @@ -476,10 +508,11 @@ public void GreenBlock_WithInsert_AtStart() var inserted = new GreenLeaf(NodeKind.Ident, "X"); var block = GreenBlock.Create('{', ImmutableArray.Create(child)); - var modified = block.WithInsert(0, ImmutableArray.Create(inserted)); + // Insert at slot 1 (after opener, before first inner child) + var modified = block.WithInsert(1, ImmutableArray.Create(inserted)); - Assert.Same(inserted, modified.GetSlot(0)); - Assert.Same(child, modified.GetSlot(1)); + Assert.Same(inserted, modified.GetSlot(1)); // inserted at slot 1 + Assert.Same(child, modified.GetSlot(2)); // original moved to slot 2 } [Fact] @@ -489,10 +522,11 @@ public void GreenBlock_WithInsert_AtEnd() var inserted = new GreenLeaf(NodeKind.Ident, "X"); var block = GreenBlock.Create('{', ImmutableArray.Create(child)); - var modified = block.WithInsert(1, ImmutableArray.Create(inserted)); + // Insert at slot 2 (after last inner child, before closer) + var modified = block.WithInsert(2, ImmutableArray.Create(inserted)); - Assert.Same(child, modified.GetSlot(0)); - Assert.Same(inserted, modified.GetSlot(1)); + Assert.Same(child, modified.GetSlot(1)); // original at slot 1 + Assert.Same(inserted, modified.GetSlot(2)); // inserted at slot 2 } [Fact] @@ -503,12 +537,14 @@ public void GreenBlock_WithInsert_MultipleNodes() var insert2 = new GreenLeaf(NodeKind.Ident, "Y"); var block = GreenBlock.Create('{', ImmutableArray.Create(original)); - var modified = block.WithInsert(0, ImmutableArray.Create(insert1, insert2)); + // Insert at slot 1 (start of inner content) + var modified = block.WithInsert(1, ImmutableArray.Create(insert1, insert2)); - Assert.Equal(3, modified.SlotCount); - Assert.Same(insert1, modified.GetSlot(0)); - Assert.Same(insert2, modified.GetSlot(1)); - Assert.Same(original, modified.GetSlot(2)); + // 5 slots: opener + 3 inner + closer + Assert.Equal(5, modified.SlotCount); + Assert.Same(insert1, modified.GetSlot(1)); + Assert.Same(insert2, modified.GetSlot(2)); + Assert.Same(original, modified.GetSlot(3)); } [Fact] @@ -517,10 +553,17 @@ public void GreenBlock_WithInsert_InvalidIndex_Throws() var block = GreenBlock.Create('{', ImmutableArray.Empty); var child = new GreenLeaf(NodeKind.Ident, "x"); + // Cannot insert at slot 0 (opener) or past slot 1 (closer) + Assert.Throws(() => + block.WithInsert(0, ImmutableArray.Create(child))); Assert.Throws(() => block.WithInsert(-1, ImmutableArray.Create(child))); Assert.Throws(() => block.WithInsert(10, ImmutableArray.Create(child))); + + // Valid: insert at slot 1 (the only valid position for empty block) + var modified = block.WithInsert(1, ImmutableArray.Create(child)); + Assert.Equal(3, modified.SlotCount); // opener + 1 inner + closer } [Fact] @@ -531,11 +574,13 @@ public void GreenBlock_WithRemove_RemovesChildren() var child3 = new GreenLeaf(NodeKind.Ident, "c"); var block = GreenBlock.Create('{', ImmutableArray.Create(child1, child2, child3)); - var modified = block.WithRemove(1, 1); + // Remove slot 2 (second inner child 'b') + var modified = block.WithRemove(2, 1); - Assert.Equal(2, modified.SlotCount); - Assert.Same(child1, modified.GetSlot(0)); - Assert.Same(child3, modified.GetSlot(1)); + // 4 slots remaining: opener + 2 inner + closer + Assert.Equal(4, modified.SlotCount); + Assert.Same(child1, modified.GetSlot(1)); // 'a' at slot 1 + Assert.Same(child3, modified.GetSlot(2)); // 'c' moved to slot 2 } [Fact] @@ -547,11 +592,13 @@ public void GreenBlock_WithRemove_MultipleChildren() var child4 = new GreenLeaf(NodeKind.Ident, "d"); var block = GreenBlock.Create('{', ImmutableArray.Create(child1, child2, child3, child4)); - var modified = block.WithRemove(1, 2); // Remove b and c + // Remove slots 2-3 (b and c, inner indices 1-2) + var modified = block.WithRemove(2, 2); - Assert.Equal(2, modified.SlotCount); - Assert.Same(child1, modified.GetSlot(0)); - Assert.Same(child4, modified.GetSlot(1)); + // 4 slots remaining: opener + 2 inner + closer + Assert.Equal(4, modified.SlotCount); + Assert.Same(child1, modified.GetSlot(1)); // 'a' at slot 1 + Assert.Same(child4, modified.GetSlot(2)); // 'd' moved to slot 2 } [Fact] @@ -560,8 +607,12 @@ public void GreenBlock_WithRemove_InvalidRange_Throws() var child = new GreenLeaf(NodeKind.Ident, "a"); var block = GreenBlock.Create('{', ImmutableArray.Create(child)); + // Cannot remove opener (slot 0) + Assert.Throws(() => block.WithRemove(0, 1)); + // Cannot remove with negative index Assert.Throws(() => block.WithRemove(-1, 1)); - Assert.Throws(() => block.WithRemove(0, 5)); + // Cannot remove past inner children (would affect closer) + Assert.Throws(() => block.WithRemove(1, 5)); } [Fact] @@ -573,12 +624,14 @@ public void GreenBlock_WithReplace_ReplacesRange() var replacement = new GreenLeaf(NodeKind.Ident, "X"); var block = GreenBlock.Create('{', ImmutableArray.Create(child1, child2, child3)); - var modified = block.WithReplace(1, 1, ImmutableArray.Create(replacement)); + // Replace slot 2 (second inner child 'b') + var modified = block.WithReplace(2, 1, ImmutableArray.Create(replacement)); - Assert.Equal(3, modified.SlotCount); - Assert.Same(child1, modified.GetSlot(0)); - Assert.Same(replacement, modified.GetSlot(1)); - Assert.Same(child3, modified.GetSlot(2)); + // 5 slots: opener + 3 inner + closer + Assert.Equal(5, modified.SlotCount); + Assert.Same(child1, modified.GetSlot(1)); + Assert.Same(replacement, modified.GetSlot(2)); + Assert.Same(child3, modified.GetSlot(3)); } [Fact] @@ -591,14 +644,15 @@ public void GreenBlock_WithReplace_ExpandsRange() var repl3 = new GreenLeaf(NodeKind.Ident, "Z"); var block = GreenBlock.Create('{', ImmutableArray.Create(child1, child2)); - // Replace 1 child with 3 - var modified = block.WithReplace(1, 1, ImmutableArray.Create(repl1, repl2, repl3)); + // Replace slot 2 (second inner child 'b') with 3 nodes + var modified = block.WithReplace(2, 1, ImmutableArray.Create(repl1, repl2, repl3)); - Assert.Equal(4, modified.SlotCount); - Assert.Same(child1, modified.GetSlot(0)); - Assert.Same(repl1, modified.GetSlot(1)); - Assert.Same(repl2, modified.GetSlot(2)); - Assert.Same(repl3, modified.GetSlot(3)); + // 6 slots: opener + 4 inner + closer + Assert.Equal(6, modified.SlotCount); + Assert.Same(child1, modified.GetSlot(1)); + Assert.Same(repl1, modified.GetSlot(2)); + Assert.Same(repl2, modified.GetSlot(3)); + Assert.Same(repl3, modified.GetSlot(4)); } [Fact] @@ -610,12 +664,13 @@ public void GreenBlock_WithReplace_ContractsRange() var replacement = new GreenLeaf(NodeKind.Ident, "X"); var block = GreenBlock.Create('{', ImmutableArray.Create(child1, child2, child3)); - // Replace 2 children with 1 - var modified = block.WithReplace(0, 2, ImmutableArray.Create(replacement)); + // Replace slots 1-2 (a and b) with 1 node + var modified = block.WithReplace(1, 2, ImmutableArray.Create(replacement)); - Assert.Equal(2, modified.SlotCount); - Assert.Same(replacement, modified.GetSlot(0)); - Assert.Same(child3, modified.GetSlot(1)); + // 4 slots: opener + 2 inner + closer + Assert.Equal(4, modified.SlotCount); + Assert.Same(replacement, modified.GetSlot(1)); + Assert.Same(child3, modified.GetSlot(2)); } [Fact] @@ -624,10 +679,15 @@ public void GreenBlock_WithReplace_InvalidRange_Throws() var block = GreenBlock.Create('{', ImmutableArray.Empty); var child = new GreenLeaf(NodeKind.Ident, "x"); + // Cannot replace with negative index Assert.Throws(() => block.WithReplace(-1, 1, ImmutableArray.Create(child))); + // Cannot replace range that exceeds total slots Assert.Throws(() => - block.WithReplace(0, 5, ImmutableArray.Create(child))); + block.WithReplace(1, 5, ImmutableArray.Create(child))); + // Cannot partially replace starting at opener without replacing entire block + Assert.Throws(() => + block.WithReplace(0, 1, ImmutableArray.Create(child))); } [Fact] @@ -711,7 +771,8 @@ public void GreenBlock_NestedBlocks() var outer = GreenBlock.Create('{', ImmutableArray.Create(inner)); Assert.Equal(4, outer.Width); // { + ( + ) + } - Assert.Same(inner, outer.GetSlot(0)); + // Inner block is at slot 1 (slot 0 is opener) + Assert.Same(inner, outer.GetSlot(1)); } [Fact] diff --git a/TinyTokenizer.Tests/QueryCombinatorTests.cs b/TinyTokenizer.Tests/QueryCombinatorTests.cs index dbac949..02f2992 100644 --- a/TinyTokenizer.Tests/QueryCombinatorTests.cs +++ b/TinyTokenizer.Tests/QueryCombinatorTests.cs @@ -1085,7 +1085,8 @@ public void Ancestor_MatchesDirectParent() var tree = Parse("{ foo }"); var root = tree.Root; var block = root.Children.First() as SyntaxBlock; - var innerIdent = block!.Children.First(); + // Use InnerChildren to skip opener/closer + var innerIdent = block!.InnerChildren.First(); var query = Query.Ancestor(Query.BraceBlock); Assert.True(query.Matches(innerIdent)); @@ -1097,8 +1098,9 @@ public void Ancestor_MatchesGrandparent() var tree = Parse("{ [ foo ] }"); var root = tree.Root; var braceBlock = root.Children.First() as SyntaxBlock; - var bracketBlock = braceBlock!.Children.First() as SyntaxBlock; - var innerIdent = bracketBlock!.Children.First(); + // Use InnerChildren to skip opener/closer + var bracketBlock = braceBlock!.InnerChildren.First() as SyntaxBlock; + var innerIdent = bracketBlock!.InnerChildren.First(); // innerIdent's grandparent is brace block var query = Query.Ancestor(Query.BraceBlock); @@ -1111,7 +1113,8 @@ public void Ancestor_FailsWhenNoAncestorMatches() var tree = Parse("{ foo }"); var root = tree.Root; var block = root.Children.First() as SyntaxBlock; - var innerIdent = block!.Children.First(); + // Use InnerChildren to skip opener/closer + var innerIdent = block!.InnerChildren.First(); var query = Query.Ancestor(Query.ParenBlock); // No paren ancestor Assert.False(query.Matches(innerIdent)); @@ -1165,7 +1168,8 @@ public void AsParent_Extension_CreatesParentQuery() var tree = Parse("{ foo }"); var root = tree.Root; var block = root.Children.First() as SyntaxBlock; - var innerIdent = block!.Children.First(); + // Use InnerChildren to skip opener/closer + var innerIdent = block!.InnerChildren.First(); var query = Query.BraceBlock.AsParent(); Assert.True(query.Matches(innerIdent)); @@ -1178,8 +1182,9 @@ public void AsAncestor_Extension_CreatesAncestorQuery() var tree = Parse("{ [ foo ] }"); var root = tree.Root; var braceBlock = root.Children.First() as SyntaxBlock; - var bracketBlock = braceBlock!.Children.First() as SyntaxBlock; - var innerIdent = bracketBlock!.Children.First(); + // Use InnerChildren to skip opener/closer + var bracketBlock = braceBlock!.InnerChildren.First() as SyntaxBlock; + var innerIdent = bracketBlock!.InnerChildren.First(); var query = Query.BraceBlock.AsAncestor(); Assert.True(query.Matches(innerIdent)); @@ -1638,8 +1643,9 @@ public void Ancestor_Combined_With_Parent() var tree = Parse("{ [ foo ] }"); var root = tree.Root; var braceBlock = root.Children.First() as SyntaxBlock; - var bracketBlock = braceBlock!.Children.First() as SyntaxBlock; - var innerIdent = bracketBlock!.Children.First(); + // Use InnerChildren to skip opener/closer + var bracketBlock = braceBlock!.InnerChildren.First() as SyntaxBlock; + var innerIdent = bracketBlock!.InnerChildren.First(); // Direct parent is bracket, ancestor is brace Assert.True(Query.Parent(Query.BracketBlock).Matches(innerIdent)); diff --git a/TinyTokenizer.Tests/SyntaxEditorTests.cs b/TinyTokenizer.Tests/SyntaxEditorTests.cs index ab9591e..abbeafe 100644 --- a/TinyTokenizer.Tests/SyntaxEditorTests.cs +++ b/TinyTokenizer.Tests/SyntaxEditorTests.cs @@ -656,6 +656,230 @@ public void Insert_ParenBlock_InnerEnd() #endregion + #region Inner() Query Tests + + [Fact] + public void Inner_Select_ReturnsInnerChildren() + { + var tree = SyntaxTree.Parse("{a b c}"); + + var inner = tree.Select(Q.BraceBlock.First().Inner()).ToList(); + + // "a", "b", "c" (whitespace is trivia attached to tokens) + Assert.Equal(3, inner.Count); + } + + [Fact] + public void Inner_Replace_ReplacesContentPreservingDelimiters() + { + var tree = SyntaxTree.Parse("{old content}"); + + tree.CreateEditor() + .Replace(Q.BraceBlock.First().Inner(), "new") + .Commit(); + + Assert.Equal("{new}", tree.ToText()); + } + + [Fact] + public void Inner_Replace_EmptyBlock_InsertsContent() + { + var tree = SyntaxTree.Parse("{}"); + + tree.CreateEditor() + .Replace(Q.BraceBlock.First().Inner(), "inserted") + .Commit(); + + Assert.Equal("{inserted}", tree.ToText()); + } + + [Fact] + public void Inner_Replace_WithWhitespace() + { + var tree = SyntaxTree.Parse("{ old }"); + + tree.CreateEditor() + .Replace(Q.BraceBlock.First().Inner(), " new ") + .Commit(); + + // Original trivia (" ") + new content (" new ") + original trailing trivia (" ") + // But trivia from "old" is transferred: leading space + content + trailing space + Assert.Equal("{ new }", tree.ToText()); + } + + [Fact] + public void Inner_Replace_MultipleBlocks() + { + var tree = SyntaxTree.Parse("{a} {b}"); + + tree.CreateEditor() + .Replace(Q.BraceBlock.Inner(), "X") + .Commit(); + + Assert.Equal("{X} {X}", tree.ToText()); + } + + [Fact] + public void Inner_Remove_ClearsBlockContent() + { + var tree = SyntaxTree.Parse("{content}"); + + tree.CreateEditor() + .Remove(Q.BraceBlock.First().Inner()) + .Commit(); + + Assert.Equal("{}", tree.ToText()); + } + + [Fact] + public void Inner_Remove_EmptyBlock_NoChange() + { + var tree = SyntaxTree.Parse("{}"); + + tree.CreateEditor() + .Remove(Q.BraceBlock.First().Inner()) + .Commit(); + + Assert.Equal("{}", tree.ToText()); + } + + [Fact] + public void Inner_InsertBefore_InsertsAtStart() + { + var tree = SyntaxTree.Parse("{existing}"); + + tree.CreateEditor() + .InsertBefore(Q.BraceBlock.First().Inner(), "prefix ") + .Commit(); + + Assert.Equal("{prefix existing}", tree.ToText()); + } + + [Fact] + public void Inner_InsertAfter_InsertsAtEnd() + { + var tree = SyntaxTree.Parse("{existing}"); + + tree.CreateEditor() + .InsertAfter(Q.BraceBlock.First().Inner(), " suffix") + .Commit(); + + Assert.Equal("{existing suffix}", tree.ToText()); + } + + [Fact] + public void Inner_InsertBefore_EmptyBlock_InsertsContent() + { + var tree = SyntaxTree.Parse("{}"); + + tree.CreateEditor() + .InsertBefore(Q.BraceBlock.First().Inner(), "new") + .Commit(); + + Assert.Equal("{new}", tree.ToText()); + } + + [Fact] + public void Inner_InsertAfter_EmptyBlock_InsertsContent() + { + var tree = SyntaxTree.Parse("{}"); + + tree.CreateEditor() + .InsertAfter(Q.BraceBlock.First().Inner(), "new") + .Commit(); + + Assert.Equal("{new}", tree.ToText()); + } + + [Fact] + public void Inner_BracketBlock_Works() + { + var tree = SyntaxTree.Parse("[old]"); + + tree.CreateEditor() + .Replace(Q.BracketBlock.First().Inner(), "new") + .Commit(); + + Assert.Equal("[new]", tree.ToText()); + } + + [Fact] + public void Inner_ParenBlock_Works() + { + var tree = SyntaxTree.Parse("(old)"); + + tree.CreateEditor() + .Replace(Q.ParenBlock.First().Inner(), "new") + .Commit(); + + Assert.Equal("(new)", tree.ToText()); + } + + [Fact] + public void Inner_NestedBlocks_WorksOnOuter() + { + var tree = SyntaxTree.Parse("{outer {inner}}"); + + tree.CreateEditor() + .Replace(Q.BraceBlock.First().Inner(), "replaced") + .Commit(); + + Assert.Equal("{replaced}", tree.ToText()); + } + + [Fact] + public void Inner_NestedBlocks_WorksOnInner() + { + var tree = SyntaxTree.Parse("{outer {inner}}"); + + // Get the inner brace block (second one) + var innerBlock = tree.Select(Q.BraceBlock).Skip(1).First(); + + tree.CreateEditor() + .Replace(innerBlock, "{replaced}") + .Commit(); + + Assert.Equal("{outer {replaced}}", tree.ToText()); + } + + [Fact] + public void Inner_WithFirst_SelectsFirstBlockInner() + { + var tree = SyntaxTree.Parse("{a} {b}"); + + tree.CreateEditor() + .Replace(Q.BraceBlock.First().Inner(), "X") + .Commit(); + + Assert.Equal("{X} {b}", tree.ToText()); + } + + [Fact] + public void Inner_WithLast_SelectsLastBlockInner() + { + var tree = SyntaxTree.Parse("{a} {b}"); + + tree.CreateEditor() + .Replace(Q.BraceBlock.Last().Inner(), "X") + .Commit(); + + Assert.Equal("{a} {X}", tree.ToText()); + } + + [Fact] + public void Inner_WithPredicate_SelectsMatchingBlockInner() + { + var tree = SyntaxTree.Parse("{short} {longer content}"); + + tree.CreateEditor() + .Replace(Q.BraceBlock.Where(b => b.Width > 10).First().Inner(), "X") + .Commit(); + + Assert.Equal("{short} {X}", tree.ToText()); + } + + #endregion + #region Function-Like Block Insertion Scenarios [Fact] @@ -1813,4 +2037,142 @@ public void TriviaPreservation_NestedBlocksWithSchema_IndentationBeforeElseLost( } #endregion + + #region Query.Between with SyntaxEditor + + /// + /// Tests that Query.Between correctly replaces a range of nodes. + /// + [Fact] + public void Replace_QueryBetween_ReplacesEntireRange() + { + // Arrange: a, b, c - we want to replace from 'a' to 'c' inclusive + var tree = SyntaxTree.Parse("x a b c y"); + + // Act: replace everything from 'a' to 'c' (inclusive) + tree.CreateEditor() + .Replace(Q.Between(Q.Ident("a"), Q.Ident("c")), "REPLACED") + .Commit(); + + // Assert: the range [a, b, c] should be replaced with REPLACED + Assert.Equal("x REPLACED y", tree.ToText()); + } + + /// + /// Tests that Remove with Query.Between removes the entire matched range. + /// + [Fact] + public void Remove_QueryBetween_RemovesEntireRange() + { + var tree = SyntaxTree.Parse("start a b c end"); + + tree.CreateEditor() + .Remove(Q.Between(Q.Ident("a"), Q.Ident("c"))) + .Commit(); + + // Trivia handling: leading trivia of 'a' (space) is removed with 'a', + // but 'end' keeps its leading trivia (space) + Assert.Equal("start end", tree.ToText()); + } + + /// + /// Tests that InsertBefore with Query.Between inserts before the start of the range. + /// + [Fact] + public void InsertBefore_QueryBetween_InsertsBeforeRange() + { + var tree = SyntaxTree.Parse("x a b c y"); + + tree.CreateEditor() + .InsertBefore(Q.Between(Q.Ident("a"), Q.Ident("c")), "BEFORE ") + .Commit(); + + Assert.Equal("x BEFORE a b c y", tree.ToText()); + } + + /// + /// Tests that InsertAfter with Query.Between inserts after the end of the range. + /// Note: With trailing trivia model, the preceding token keeps its trailing whitespace, + /// so inserted content should use trailing space (not leading) for proper separation. + /// + [Fact] + public void InsertAfter_QueryBetween_InsertsAfterRange() + { + // Test with whitespace-separated tokens where spaces are trailing trivia + var tree = SyntaxTree.Parse("x a b c y"); + + // Insert "AFTER " after the range a..c + // - c has trailing trivia (space) -> "c " + // - Insert "AFTER " (with trailing space for separation from y) + // - y has no trivia -> "y" + tree.CreateEditor() + .InsertAfter(Q.Between(Q.Ident("a"), Q.Ident("c")), "AFTER ") + .Commit(); + + Assert.Equal("x a b c AFTER y", tree.ToText()); + } + + /// + /// Tests that InsertAfter preserves the target token's trailing trivia and + /// uses trailing space on inserted content for proper separation. + /// + /// + /// In TinyAst's trivia model, same-line whitespace is trailing trivia on the preceding token. + /// When inserting after a token, the token keeps its trailing trivia, so inserted content + /// should use trailing whitespace (not leading) for proper spacing with following tokens. + /// + [Fact] + public void InsertAfter_RedNode_InMiddle_PreservesFollowingTrivia() + { + var tree = SyntaxTree.Parse("x a b c y"); + + // Verify trivia model: each token (except last) has trailing whitespace + var beforeTrivia = string.Join(", ", tree.Leaves.Select(l => + $"{l.Text}(T:{l.TrailingTriviaWidth})")); + Assert.Equal("x(T:1), a(T:1), b(T:1), c(T:1), y(T:0)", beforeTrivia); + + var cNode = tree.Root.Children.First(n => n is SyntaxToken leaf && leaf.Text == "c"); + + // Insert "AFTER " with trailing space for proper separation from y + tree.CreateEditor() + .InsertAfter(cNode, "AFTER ") + .Commit(); + + Assert.Equal("x a b c AFTER y", tree.ToText()); + } + + /// + /// Tests that Edit with Query.Between transforms the concatenated content of the range. + /// + [Fact] + public void Edit_QueryBetween_TransformsRangeContent() + { + var tree = SyntaxTree.Parse("x abc def ghi y"); + + tree.CreateEditor() + .Edit(Q.Between(Q.Ident("abc"), Q.Ident("ghi")), content => content.ToUpper()) + .Commit(); + + // The content between abc and ghi (inclusive) should be uppercased + Assert.Equal("x ABC DEF GHI y", tree.ToText()); + } + + /// + /// Tests Query.Sequence with SyntaxEditor Replace. + /// + [Fact] + public void Replace_QuerySequence_ReplacesAllMatchedNodes() + { + var tree = SyntaxTree.Parse("a = 1 ; b = 2"); + + // Replace sequence "a =" with "x =" + // Note: '=' is parsed as an operator by default, not a symbol + tree.CreateEditor() + .Replace(Q.Sequence(Q.Ident("a"), Q.Operator("=")), "x =") + .Commit(); + + Assert.Equal("x = 1 ; b = 2", tree.ToText()); + } + + #endregion } diff --git a/TinyTokenizer.Tests/SyntaxTreeTests.cs b/TinyTokenizer.Tests/SyntaxTreeTests.cs index 906291e..c8f48df 100644 --- a/TinyTokenizer.Tests/SyntaxTreeTests.cs +++ b/TinyTokenizer.Tests/SyntaxTreeTests.cs @@ -128,12 +128,13 @@ public void GreenBlock_WithSlot_SharesSiblings() var block = GreenBlock.Create('{', ImmutableArray.Create(child1, child2, child3)); var newChild = new GreenLeaf(NodeKind.Ident, "X"); - var newBlock = block.WithSlot(1, newChild); + // Replace slot 2 (second inner child) + var newBlock = block.WithSlot(2, newChild); - // Siblings should be shared - Assert.Same(child1, newBlock.GetSlot(0)); - Assert.Same(child3, newBlock.GetSlot(2)); - Assert.NotSame(child2, newBlock.GetSlot(1)); + // Siblings should be shared (slot 0 is opener, slots 1,3 are first/third children) + Assert.Same(child1, newBlock.GetSlot(1)); + Assert.Same(child3, newBlock.GetSlot(3)); + Assert.NotSame(child2, newBlock.GetSlot(2)); } [Fact] @@ -144,12 +145,14 @@ public void GreenBlock_WithInsert_PreservesExisting() var block = GreenBlock.Create('{', ImmutableArray.Create(child1, child2)); var newChild = new GreenLeaf(NodeKind.Ident, "X"); - var newBlock = block.WithInsert(1, ImmutableArray.Create(newChild)); + // Insert at slot 2 (between a and b) + var newBlock = block.WithInsert(2, ImmutableArray.Create(newChild)); - Assert.Equal(3, newBlock.SlotCount); - Assert.Same(child1, newBlock.GetSlot(0)); - Assert.Same(newChild, newBlock.GetSlot(1)); - Assert.Same(child2, newBlock.GetSlot(2)); + // 5 slots: opener + 3 inner + closer + Assert.Equal(5, newBlock.SlotCount); + Assert.Same(child1, newBlock.GetSlot(1)); + Assert.Same(newChild, newBlock.GetSlot(2)); + Assert.Same(child2, newBlock.GetSlot(3)); } #endregion @@ -1466,7 +1469,8 @@ public void GreenTreeBuilder_InsertAt_InsertsNodes() var newNodes = lexer.ParseToGreenNodes("b"); var builder = new GreenTreeBuilder(tree.GreenRoot); - var newRoot = builder.InsertAt(new[] { 0 }, 1, newNodes); + // Insert at slot 2 (after first inner child 'a', before closer) + var newRoot = builder.InsertAt(new[] { 0 }, 2, newNodes); Assert.NotNull(newRoot); } @@ -1477,7 +1481,8 @@ public void GreenTreeBuilder_RemoveAt_RemovesNodes() var tree = SyntaxTree.Parse("{a b}"); var builder = new GreenTreeBuilder(tree.GreenRoot); - var newRoot = builder.RemoveAt(new[] { 0 }, 0, 1); + // Remove slot 1 (first inner child 'a') + var newRoot = builder.RemoveAt(new[] { 0 }, 1, 1); Assert.NotNull(newRoot); } @@ -1490,7 +1495,8 @@ public void GreenTreeBuilder_ReplaceAt_ReplacesNodes() var newNodes = lexer.ParseToGreenNodes("new"); var builder = new GreenTreeBuilder(tree.GreenRoot); - var newRoot = builder.ReplaceAt(new[] { 0 }, 0, 1, newNodes); + // Replace slot 1 (first inner child 'old') + var newRoot = builder.ReplaceAt(new[] { 0 }, 1, 1, newNodes); Assert.NotNull(newRoot); } @@ -1505,7 +1511,8 @@ public void GreenTreeBuilder_ReplaceChild_ReplacesChild() if (newNode != null) { var builder = new GreenTreeBuilder(tree.GreenRoot); - var newRoot = builder.ReplaceChild(new[] { 0 }, 0, newNode); + // Replace slot 1 (first inner child 'a') + var newRoot = builder.ReplaceChild(new[] { 0 }, 1, newNode); Assert.NotNull(newRoot); } @@ -1518,9 +1525,10 @@ public void GreenTreeBuilder_DeepPath_WorksCorrectly() var lexer = new GreenLexer(); var newNodes = lexer.ParseToGreenNodes("x"); - // Path: root -> first child (outer block) -> first child (inner block) + // Path: root -> first child (outer block at 0) -> inner block (at slot 1, since slot 0 is opener) + // Insert at slot 1 (start of inner block content) var builder = new GreenTreeBuilder(tree.GreenRoot); - var newRoot = builder.InsertAt(new[] { 0, 0 }, 0, newNodes); + var newRoot = builder.InsertAt(new[] { 0, 1 }, 1, newNodes); Assert.NotNull(newRoot); } diff --git a/TinyTokenizer.Tests/TreeWalkerTests.cs b/TinyTokenizer.Tests/TreeWalkerTests.cs index 93c5389..e15b245 100644 --- a/TinyTokenizer.Tests/TreeWalkerTests.cs +++ b/TinyTokenizer.Tests/TreeWalkerTests.cs @@ -651,9 +651,16 @@ public void NextNode_WithSkipOnFirstChild_RecursesCorrectly() var first = walker.NextNode(); - // Should skip the block and return the ident inside + // Should skip the block and recurse into it + // With Roslyn-style slots, first child of block is opener '{' Assert.NotNull(first); - Assert.Equal(NodeKind.Ident, first.Kind); + Assert.Equal(NodeKind.Symbol, first.Kind); + Assert.Equal("{", first.ToText()); + + // Continue to get the actual content + var second = walker.NextNode(); + Assert.NotNull(second); + Assert.Equal(NodeKind.Ident, second.Kind); } [Fact] @@ -757,9 +764,11 @@ public void TraverseChildren_WithSkip_RecursesIntoSkippedNode() var first = walker.FirstChild(); - // Should skip outer block, skip inner block, and return "deep" + // Should skip outer block, skip inner block + // With Roslyn-style slots, first child encountered is the outer opener '{' Assert.NotNull(first); - Assert.Equal(NodeKind.Ident, first.Kind); + Assert.Equal(NodeKind.Symbol, first.Kind); + Assert.Equal("{", first.ToText()); } [Fact] diff --git a/TinyTokenizer.code-workspace b/TinyTokenizer.code-workspace index c4e866c..6dd2aa4 100644 --- a/TinyTokenizer.code-workspace +++ b/TinyTokenizer.code-workspace @@ -7,5 +7,7 @@ "path": "../TinyTokenizer.wiki" } ], - "settings": {} + "settings": { + "powershell.cwd": "TinyTokenizer" + } } \ No newline at end of file diff --git a/TinyTokenizer/Ast/GreenBlock.cs b/TinyTokenizer/Ast/GreenBlock.cs index 6604168..968dc1e 100644 --- a/TinyTokenizer/Ast/GreenBlock.cs +++ b/TinyTokenizer/Ast/GreenBlock.cs @@ -106,22 +106,74 @@ public static GreenBlock Create( return new GreenBlock(openerNode, closerNode, children); } - /// Gets the children of this block. - public override ImmutableArray Children => _children; + /// Gets the inner children of this block (excluding delimiters). + public ImmutableArray InnerChildren => _children; + + /// + /// Gets all children including delimiters (opener at slot 0, closer at slot N+1). + /// This is the Roslyn-style slot model where delimiters are traversable children. + /// + public override ImmutableArray Children + { + get + { + var builder = ImmutableArray.CreateBuilder(_children.Length + 2); + builder.Add(OpenerNode); + builder.AddRange(_children); + builder.Add(CloserNode); + return builder.MoveToImmutable(); + } + } + + /// + /// Number of slots: opener + inner children + closer. + /// + public override int SlotCount => _children.Length + 2; /// + /// + /// Slot 0 = opener, slots 1..N = inner children, slot N+1 = closer. + /// public override GreenNode? GetSlot(int index) - => index >= 0 && index < _children.Length ? _children[index] : null; + { + if (index < 0 || index > _children.Length + 1) + return null; + if (index == 0) + return OpenerNode; + if (index == _children.Length + 1) + return CloserNode; + return _children[index - 1]; // Adjust for opener at slot 0 + } /// + /// + /// Returns offset from block start: + /// - Slot 0 (opener): 0 + /// - Slot 1..N (inner children): opener width + sum of preceding inner children + /// - Slot N+1 (closer): opener width + all inner children widths + /// public override int GetSlotOffset(int index) { + if (index == 0) + return 0; // Opener starts at block start + + if (index == _children.Length + 1) + { + // Closer is after opener and all children + int closerOffset = OpenerNode.Width; + foreach (var child in _children) + closerOffset += child.Width; + return closerOffset; + } + + // Inner child: use precomputed offsets if available + int innerIndex = index - 1; // Convert to inner children index if (_childOffsets != null) - return _childOffsets[index]; // O(1) + return _childOffsets[innerIndex]; // O(1) // O(index) for small blocks int offset = OpenerNode.Width; // After opener (including its trivia) - for (int i = 0; i < index; i++) + for (int i = 0; i < innerIndex; i++) offset += _children[i].Width; return offset; } @@ -147,50 +199,157 @@ public override void WriteTo(IBufferWriter writer) #region Structural Sharing Mutations /// + /// + /// Creates a new block with one slot replaced. + /// Slot 0 = opener, slots 1..N = inner children, slot N+1 = closer. + /// public override GreenBlock WithSlot(int index, GreenNode newChild) { - if (index < 0 || index >= _children.Length) + if (index < 0 || index > _children.Length + 1) throw new ArgumentOutOfRangeException(nameof(index)); - var newChildren = _children.SetItem(index, newChild); + // Slot 0 = opener + if (index == 0) + { + if (newChild is not GreenLeaf newOpener) + throw new ArgumentException("Opener slot must be a GreenLeaf", nameof(newChild)); + return new GreenBlock(newOpener, CloserNode, _children); + } + + // Slot N+1 = closer + if (index == _children.Length + 1) + { + if (newChild is not GreenLeaf newCloser) + throw new ArgumentException("Closer slot must be a GreenLeaf", nameof(newChild)); + return new GreenBlock(OpenerNode, newCloser, _children); + } + + // Inner child slot (1..N) + var innerIndex = index - 1; + var newChildren = _children.SetItem(innerIndex, newChild); return new GreenBlock(OpenerNode, CloserNode, newChildren); } - /// + /// + /// Creates a new block with all children replaced (including delimiters). + /// First element must be opener, last must be closer. + /// public override GreenBlock WithChildren(ImmutableArray newChildren) - => new(OpenerNode, CloserNode, newChildren); + { + if (newChildren.Length < 2) + throw new ArgumentException("Children must include at least opener and closer", nameof(newChildren)); + + if (newChildren[0] is not GreenLeaf opener) + throw new ArgumentException("First child must be opener (GreenLeaf)", nameof(newChildren)); + if (newChildren[^1] is not GreenLeaf closer) + throw new ArgumentException("Last child must be closer (GreenLeaf)", nameof(newChildren)); + + // Extract inner children (everything between opener and closer) + var innerChildren = newChildren.RemoveAt(newChildren.Length - 1).RemoveAt(0); + return new GreenBlock(opener, closer, innerChildren); + } - /// + /// + /// Creates a new block with inner children replaced (preserves delimiters). + /// + public GreenBlock WithInnerChildren(ImmutableArray newInnerChildren) + => new(OpenerNode, CloserNode, newInnerChildren); + + /// + /// Inserts nodes at the specified slot index. + /// Slot 0 = opener position, slots 1..N = inner content, slot N+1 = after last inner/before closer. + /// Cannot insert before slot 0 or after slot N+2 (would be outside block bounds). + /// + /// + /// Inserting at slot 1 places content after the opener. + /// Inserting at slot N+1 (where N+1 = _children.Length + 1) places content before the closer. + /// public override GreenBlock WithInsert(int index, ImmutableArray nodes) { - if (index < 0 || index > _children.Length) - throw new ArgumentOutOfRangeException(nameof(index)); + // Valid insertion range: 1 through _children.Length + 1 (after opener through before closer) + // Slot 0 is the opener, slot _children.Length + 1 is the closer + if (index < 1 || index > _children.Length + 1) + throw new ArgumentOutOfRangeException(nameof(index), + $"Insert index must be between 1 and {_children.Length + 1} (inner content range)"); + var innerIndex = index - 1; // Convert to inner children index var builder = _children.ToBuilder(); - builder.InsertRange(index, nodes); + builder.InsertRange(innerIndex, nodes); return new GreenBlock(OpenerNode, CloserNode, builder.ToImmutable()); } - /// + /// + /// Removes nodes starting at the specified slot index. + /// Can only remove inner children (slots 1..N). Cannot remove opener (slot 0) or closer (slot N+1). + /// public override GreenBlock WithRemove(int index, int count) { - if (index < 0 || count < 0 || index + count > _children.Length) - throw new ArgumentOutOfRangeException(nameof(index)); + // Valid removal range: slots 1..N (inner children only) + // Slot 0 is opener, slot _children.Length + 1 is closer - neither can be removed + if (index < 1 || count < 0) + throw new ArgumentOutOfRangeException(nameof(index), + "Cannot remove opener (slot 0). Use WithSlot to replace delimiters."); + + var innerIndex = index - 1; + if (innerIndex + count > _children.Length) + throw new ArgumentOutOfRangeException(nameof(count), + "Cannot remove closer. Removal range extends past inner children."); var builder = _children.ToBuilder(); - builder.RemoveRange(index, count); + builder.RemoveRange(innerIndex, count); return new GreenBlock(OpenerNode, CloserNode, builder.ToImmutable()); } - /// + /// + /// Replaces nodes starting at the specified slot index. + /// Can replace inner children (slots 1..N) or entire block content including delimiters. + /// + /// + /// If replacing only inner content (index >= 1, not touching closer), preserves delimiters. + /// If replacing from slot 0 through the closer, the replacement becomes the new children + /// (first replacement node becomes opener if it's a leaf, etc.). + /// public override GreenBlock WithReplace(int index, int count, ImmutableArray replacement) { - if (index < 0 || count < 0 || index + count > _children.Length) - throw new ArgumentOutOfRangeException(nameof(index)); + var totalSlots = _children.Length + 2; // opener + children + closer + + if (index < 0 || count < 0 || index + count > totalSlots) + throw new ArgumentOutOfRangeException(nameof(index), + $"Replace range [{index}..{index + count}) is out of bounds for slot count {totalSlots}"); + + // Special case: replacing entire block content (slot 0 through last slot) + // This replaces opener, all inner children, and closer + if (index == 0 && count == totalSlots) + { + // The replacement becomes the new full children array + return WithChildren(replacement); + } + + // Special case: replacing from opener through some inner content + // This is invalid - can't partially replace including opener without replacing all + if (index == 0) + { + throw new ArgumentException( + "Cannot replace range starting at opener (slot 0) without replacing entire block. " + + "Use WithSlot to replace just the opener, or WithChildren to replace all.", + nameof(index)); + } + + // Special case: replacing through the closer + // This is invalid - can't partially replace including closer without replacing all + if (index + count == totalSlots && index != 0) + { + throw new ArgumentException( + "Cannot replace range including closer without replacing entire block. " + + "Use WithSlot to replace just the closer, or WithChildren to replace all.", + nameof(count)); + } + // Normal case: replacing inner content only (slots 1 through N) + var innerIndex = index - 1; var builder = _children.ToBuilder(); - builder.RemoveRange(index, count); - builder.InsertRange(index, replacement); + builder.RemoveRange(innerIndex, count); + builder.InsertRange(innerIndex, replacement); return new GreenBlock(OpenerNode, CloserNode, builder.ToImmutable()); } diff --git a/TinyTokenizer/Ast/GreenNode.cs b/TinyTokenizer/Ast/GreenNode.cs index 9516151..b563cae 100644 --- a/TinyTokenizer/Ast/GreenNode.cs +++ b/TinyTokenizer/Ast/GreenNode.cs @@ -171,6 +171,94 @@ public virtual int GetSlotOffset(int index) /// public bool IsLeaf => !IsContainer; + #region Trivia Access + + /// + /// Gets the leading trivia of this node by finding the first leaf descendant. + /// For leaves, returns the leaf's leading trivia directly. + /// + public ImmutableArray GetLeadingTrivia() + { + var firstLeaf = GetFirstLeaf(); + return firstLeaf?.LeadingTrivia ?? ImmutableArray.Empty; + } + + /// + /// Gets the trailing trivia of this node by finding the last leaf descendant. + /// For leaves, returns the leaf's trailing trivia directly. + /// + public ImmutableArray GetTrailingTrivia() + { + var lastLeaf = GetLastLeaf(); + return lastLeaf?.TrailingTrivia ?? ImmutableArray.Empty; + } + + /// + /// Gets the width of leading trivia (precomputed for leaves). + /// + public int GetLeadingTriviaWidth() + { + var firstLeaf = GetFirstLeaf(); + return firstLeaf?.LeadingTriviaWidth ?? 0; + } + + /// + /// Gets the width of trailing trivia (precomputed for leaves). + /// + public int GetTrailingTriviaWidth() + { + var lastLeaf = GetLastLeaf(); + return lastLeaf?.TrailingTriviaWidth ?? 0; + } + + /// + /// Gets the first leaf descendant of this node. + /// For leaves, returns self. For containers, descends to find leftmost leaf. + /// + internal GreenLeaf? GetFirstLeaf() + { + if (this is GreenLeaf leaf) + return leaf; + + for (int i = 0; i < SlotCount; i++) + { + var child = GetSlot(i); + if (child != null) + { + var firstLeaf = child.GetFirstLeaf(); + if (firstLeaf != null) + return firstLeaf; + } + } + + return null; + } + + /// + /// Gets the last leaf descendant of this node. + /// For leaves, returns self. For containers, descends to find rightmost leaf. + /// + internal GreenLeaf? GetLastLeaf() + { + if (this is GreenLeaf leaf) + return leaf; + + for (int i = SlotCount - 1; i >= 0; i--) + { + var child = GetSlot(i); + if (child != null) + { + var lastLeaf = child.GetLastLeaf(); + if (lastLeaf != null) + return lastLeaf; + } + } + + return null; + } + + #endregion + #region IFormattable /// diff --git a/TinyTokenizer/Ast/NodeQuery.cs b/TinyTokenizer/Ast/NodeQuery.cs index 7315f0a..acfe1d6 100644 --- a/TinyTokenizer/Ast/NodeQuery.cs +++ b/TinyTokenizer/Ast/NodeQuery.cs @@ -27,11 +27,13 @@ public interface INodeQuery { /// /// Selects all nodes matching this query from the tree. + /// For range queries (e.g., BetweenQuery), returns only the start node of each match. /// IEnumerable Select(SyntaxTree tree); /// /// Selects all nodes matching this query from a subtree. + /// For range queries, returns only the start node of each match. /// IEnumerable Select(SyntaxNode root); @@ -56,7 +58,7 @@ public interface INodeQuery /// preserving type-specific methods through the fluent chain. /// /// The derived query type. -public abstract record NodeQuery : INodeQuery, IGreenNodeQuery where TSelf : NodeQuery +public abstract record NodeQuery : INodeQuery, IGreenNodeQuery, IRegionQuery where TSelf : NodeQuery { /// /// Selects all nodes matching this query from the tree. @@ -68,6 +70,76 @@ public abstract record NodeQuery : INodeQuery, IGreenNodeQuery where TSel /// public abstract IEnumerable Select(SyntaxNode root); + #region IRegionQuery Implementation + + /// + /// Resolves this query to regions in the tree. + /// + IEnumerable IRegionQuery.SelectRegions(SyntaxTree tree) + => SelectRegionsCore(tree.Root); + + /// + /// Resolves this query to regions in a subtree. + /// + IEnumerable IRegionQuery.SelectRegions(SyntaxNode root) + => SelectRegionsCore(root); + + /// + /// Default region resolution: traverses tree with PathTrackingWalker, calls TryMatch once per node, + /// then applies selection filtering (First/Last/Nth via ApplyRegionFilter). + /// Uses incremental path tracking for O(1) per node instead of O(depth). + /// + internal virtual IEnumerable SelectRegionsCore(SyntaxNode root) + { + return ApplyRegionFilter(SelectAllRegions(root)); + } + + /// + /// Traverses tree and yields a region for each matching node. + /// Uses PathTrackingWalker for O(1) path computation per node. + /// + private IEnumerable SelectAllRegions(SyntaxNode root) + { + var walker = new PathTrackingWalker(root); + foreach (var (node, parentPath) in walker.DescendantsAndSelfWithPath()) + { + if (TryMatch(node, out var consumedCount)) + { + var parent = node.Parent; + if (parent != null) + { + yield return new QueryRegion( + parentPath: parentPath, + parent: parent, + startSlot: node.SiblingIndex, + endSlot: node.SiblingIndex + consumedCount, + firstNode: node, + position: node.Position + ); + } + } + } + } + + /// + /// Gets the selection mode for this query. Override in derived classes. + /// + internal virtual SelectionMode Mode => SelectionMode.All; + + /// + /// Gets the selection mode argument (e.g., N for Nth, count for Skip/Take). + /// + internal virtual int ModeArg => 0; + + /// + /// Applies selection mode filtering (First/Last/Nth/Skip/Take) to regions. + /// Uses and properties. + /// + internal IEnumerable ApplyRegionFilter(IEnumerable regions) => + SelectionModeHelper.Apply(regions, Mode, ModeArg); + + #endregion + /// /// Tests whether a single node matches this query's criteria. /// diff --git a/TinyTokenizer/Ast/NodeQueryTypes.cs b/TinyTokenizer/Ast/NodeQueryTypes.cs index dcdd1d2..74bdbfe 100644 --- a/TinyTokenizer/Ast/NodeQueryTypes.cs +++ b/TinyTokenizer/Ast/NodeQueryTypes.cs @@ -2,6 +2,23 @@ namespace TinyTokenizer.Ast; +/// +/// Helper for applying selection mode filtering to regions. +/// +internal static class SelectionModeHelper +{ + public static IEnumerable Apply(IEnumerable regions, SelectionMode mode, int modeArg) => + mode switch + { + SelectionMode.First => regions.Take(1), + SelectionMode.Last => regions.TakeLast(1), + SelectionMode.Nth => regions.Skip(modeArg).Take(1), + SelectionMode.Skip => regions.Skip(modeArg), + SelectionMode.Take => regions.Take(modeArg), + _ => regions + }; +} + /// /// Compares red nodes for equality using SyntaxNode's equality semantics. /// Two red nodes are equal if they wrap the same green node and have the same position. @@ -96,6 +113,44 @@ protected override KindNodeQuery CreateFiltered(Func predicate protected override KindNodeQuery CreateSkip(int count) => new(Kind, _predicate, SelectionMode.Skip, count, _textConstraint); protected override KindNodeQuery CreateTake(int count) => new(Kind, _predicate, SelectionMode.Take, count, _textConstraint); + internal override SelectionMode Mode => _mode; + internal override int ModeArg => _modeArg; + + /// + /// Optimized region resolution: single-pass traversal that checks Kind directly + /// and applies selection mode inline for efficient First()/Take() short-circuit. + /// Uses PathTrackingWalker for O(1) path computation per node. + /// + internal override IEnumerable SelectRegionsCore(SyntaxNode root) + { + var walker = new PathTrackingWalker(root); + var regions = SelectRegionsFromWalker(walker); + return ApplyRegionFilter(regions); + } + + private IEnumerable SelectRegionsFromWalker(PathTrackingWalker walker) + { + foreach (var (node, parentPath) in walker.DescendantsAndSelfWithPath()) + { + // Inline match check - avoids virtual TryMatch call + if (node.Kind == Kind && (_predicate == null || _predicate(node))) + { + var parent = node.Parent; + if (parent != null) + { + yield return new QueryRegion( + parentPath: parentPath, + parent: parent, + startSlot: node.SiblingIndex, + endSlot: node.SiblingIndex + 1, // KindNodeQuery always consumes 1 + firstNode: node, + position: node.Position + ); + } + } + } + } + private static Func? CombinePredicates(Func? a, Func b) => a == null ? b : n => a(n) && b(n); } @@ -172,9 +227,49 @@ protected override BlockNodeQuery CreateFiltered(Func predicat protected override BlockNodeQuery CreateSkip(int count) => new(_opener, _predicate, SelectionMode.Skip, count); protected override BlockNodeQuery CreateTake(int count) => new(_opener, _predicate, SelectionMode.Take, count); + internal override SelectionMode Mode => _mode; + internal override int ModeArg => _modeArg; + + /// + /// Optimized region resolution: single-pass traversal that checks block type directly + /// and applies selection mode inline for efficient First()/Take() short-circuit. + /// Uses PathTrackingWalker for O(1) path computation per node. + /// + internal override IEnumerable SelectRegionsCore(SyntaxNode root) + { + var walker = new PathTrackingWalker(root); + var regions = SelectRegionsFromWalker(walker); + return ApplyRegionFilter(regions); + } + + private IEnumerable SelectRegionsFromWalker(PathTrackingWalker walker) + { + foreach (var (node, parentPath) in walker.DescendantsAndSelfWithPath()) + { + // Inline match check - avoids virtual TryMatch call + if (node is SyntaxBlock block && + (_opener == null || block.Opener == _opener.Value) && + (_predicate == null || _predicate(node))) + { + var parent = node.Parent; + if (parent != null) + { + yield return new QueryRegion( + parentPath: parentPath, + parent: parent, + startSlot: node.SiblingIndex, + endSlot: node.SiblingIndex + 1, // BlockNodeQuery always consumes 1 + firstNode: node, + position: node.Position + ); + } + } + } + } + private static Func? CombinePredicates(Func? a, Func b) => a == null ? b : n => a(n) && b(n); - + /// /// Returns a query that selects the opening delimiter (start) of matched blocks. /// Use with InsertAfter to insert at the beginning of block content. @@ -198,6 +293,22 @@ protected override BlockNodeQuery CreateFiltered(Func predicat /// /// public BoundaryQuery End() => new BoundaryQuery(this, BoundarySide.End); + + /// + /// Returns a query that selects all inner children of matched blocks as a range. + /// Use with Replace/Edit to modify block content while preserving delimiters. + /// Empty blocks yield an empty region, enabling insertion via Replace. + /// + /// + /// + /// // Replace content between braces + /// editor.Replace(Query.BraceBlock.Inner(), "new content") + /// + /// // Works with empty blocks too + /// editor.Replace(Query.BraceBlock.Inner(), "inserted into empty") + /// + /// + public InnerContentQuery Inner() => new InnerContentQuery(this); } #endregion @@ -225,7 +336,7 @@ public enum BoundarySide /// uses this metadata to compute insertion positions, /// even for empty containers where returns no results. /// -public sealed record BoundaryQuery : INodeQuery +public sealed record BoundaryQuery : INodeQuery, IRegionQuery { /// Gets the underlying container query. public INodeQuery ContainerQuery { get; } @@ -282,33 +393,156 @@ public bool TryMatch(SyntaxNode startNode, out int consumedCount) return false; } + #region IRegionQuery Implementation + /// - /// Gets the boundary node for a container. - /// For blocks: returns OpenerNode or CloserNode. - /// For other containers: returns first or last child. + /// Resolves this query to regions in the tree. /// - private SyntaxNode? GetBoundaryNode(SyntaxNode container) + IEnumerable IRegionQuery.SelectRegions(SyntaxTree tree) + => SelectRegionsCore(tree.Root); + + /// + /// Resolves this query to regions in a subtree. + /// + IEnumerable IRegionQuery.SelectRegions(SyntaxNode root) + => SelectRegionsCore(root); + + private IEnumerable SelectRegionsCore(SyntaxNode root) { - if (container is SyntaxBlock block) + foreach (var container in ContainerQuery.Select(root)) { - return Side == BoundarySide.Start ? block.OpenerNode : block.CloserNode; + if (container.SlotCount == 0) + continue; + + int slot = Side == BoundarySide.Start ? 0 : container.SlotCount - 1; + var boundaryNode = container.GetChild(slot); + + if (boundaryNode != null) + { + yield return new QueryRegion( + parent: container, + startSlot: slot, + endSlot: slot + 1, + firstNode: boundaryNode, + position: boundaryNode.Position + ); + } } - - // For non-block containers (lists, syntax nodes), return first/last child - var children = container.Children.ToList(); - if (children.Count == 0) + } + + #endregion + + /// + /// Gets the boundary node for a container. + /// Uses slot-based access which works uniformly for all container types: + /// - Blocks: slot 0 = opener, slot N = closer (Roslyn-style) + /// - Lists/other: slot 0 = first child, slot N = last child + /// + private SyntaxNode? GetBoundaryNode(SyntaxNode container) + { + if (container.SlotCount == 0) return null; // Empty container - no boundary node to return - return Side == BoundarySide.Start ? children[0] : children[^1]; + return Side == BoundarySide.Start + ? container.GetChild(0) + : container.GetChild(container.SlotCount - 1); + } +} + +#endregion + +#region Inner Content Query + +/// +/// A query that selects all inner children of a block as a single range. +/// Returns a region spanning slots 1 through SlotCount-2 (excluding opener/closer). +/// Empty blocks yield an empty region at slot 1, enabling insertion. +/// +/// +/// Use Query.BraceBlock.Inner() to create this query. +/// Works with to replace block content while preserving delimiters. +/// +public sealed record InnerContentQuery : INodeQuery, IRegionQuery +{ + /// The container query that selects blocks. + public BlockNodeQuery ContainerQuery { get; } + + internal InnerContentQuery(BlockNodeQuery containerQuery) + { + ContainerQuery = containerQuery; + } + + /// + public IEnumerable Select(SyntaxTree tree) => Select(tree.Root); + + /// + public IEnumerable Select(SyntaxNode root) + { + foreach (var region in SelectRegionsCore(root)) + { + foreach (var node in region.Nodes) + yield return node; + } + } + + /// + public bool Matches(SyntaxNode node) + { + // Check if node is an inner child of a matching block + var parent = node.Parent; + if (parent is SyntaxBlock block && ContainerQuery.Matches(block)) + { + var index = node.SiblingIndex; + return index >= 1 && index < block.SlotCount - 1; + } + return false; + } + + /// + public bool TryMatch(SyntaxNode startNode, out int consumedCount) + { + var parent = startNode.Parent; + if (parent is SyntaxBlock block && + ContainerQuery.Matches(block) && + startNode.SiblingIndex == 1) + { + consumedCount = block.ChildCount; + return true; + } + consumedCount = 0; + return false; } /// - /// Resolves the containers matched by this boundary query. - /// Used by to compute insertion positions. + /// Resolves this query to regions in the tree. /// - internal IEnumerable ResolveContainers(SyntaxTree tree) + IEnumerable IRegionQuery.SelectRegions(SyntaxTree tree) + => SelectRegionsCore(tree.Root); + + /// + /// Resolves this query to regions in a subtree. + /// + IEnumerable IRegionQuery.SelectRegions(SyntaxNode root) + => SelectRegionsCore(root); + + private IEnumerable SelectRegionsCore(SyntaxNode root) { - return ContainerQuery.Select(tree); + foreach (var container in ContainerQuery.Select(root)) + { + if (container is SyntaxBlock block) + { + var innerCount = block.ChildCount; + var firstInner = block.InnerChildren.FirstOrDefault(); + + yield return new QueryRegion( + parent: block, + startSlot: 1, + endSlot: 1 + innerCount, + firstNode: firstInner, + position: block.InnerStartPosition + ); + } + } } } @@ -369,6 +603,9 @@ protected override AnyNodeQuery CreateFiltered(Func predicate) protected override AnyNodeQuery CreateSkip(int count) => new(_predicate, SelectionMode.Skip, count); protected override AnyNodeQuery CreateTake(int count) => new(_predicate, SelectionMode.Take, count); + internal override SelectionMode Mode => _mode; + internal override int ModeArg => _modeArg; + private static Func? CombinePredicates(Func? a, Func b) => a == null ? b : n => a(n) && b(n); } @@ -430,6 +667,9 @@ protected override LeafNodeQuery CreateFiltered(Func predicate protected override LeafNodeQuery CreateSkip(int count) => new(_predicate, SelectionMode.Skip, count); protected override LeafNodeQuery CreateTake(int count) => new(_predicate, SelectionMode.Take, count); + internal override SelectionMode Mode => _mode; + internal override int ModeArg => _modeArg; + private static Func? CombinePredicates(Func? a, Func b) => a == null ? b : n => a(n) && b(n); } @@ -570,6 +810,9 @@ protected override NewlineNodeQuery CreateFiltered(Func predic protected override NewlineNodeQuery CreateSkip(int count) => new(_predicate, SelectionMode.Skip, count, _negated); protected override NewlineNodeQuery CreateTake(int count) => new(_predicate, SelectionMode.Take, count, _negated); + internal override SelectionMode Mode => _mode; + internal override int ModeArg => _modeArg; + private static Func? CombinePredicates(Func? a, Func b) => a == null ? b : n => a(n) && b(n); } @@ -1246,6 +1489,9 @@ protected override ExactNodeQuery CreateFiltered(Func predicat protected override ExactNodeQuery CreateSkip(int count) => new(_target, _predicate, SelectionMode.Skip, count); protected override ExactNodeQuery CreateTake(int count) => new(_target, _predicate, SelectionMode.Take, count); + internal override SelectionMode Mode => _mode; + internal override int ModeArg => _modeArg; + private static Func? CombinePredicates(Func? a, Func b) => a == null ? b : n => a(n) && b(n); @@ -1320,6 +1566,9 @@ protected override AnyKeywordQuery CreateFiltered(Func predica protected override AnyKeywordQuery CreateSkip(int count) => new(_predicate, SelectionMode.Skip, count); protected override AnyKeywordQuery CreateTake(int count) => new(_predicate, SelectionMode.Take, count); + internal override SelectionMode Mode => _mode; + internal override int ModeArg => _modeArg; + private static Func? CombinePredicates(Func? a, Func b) => a == null ? b : n => a(n) && b(n); } @@ -1411,6 +1660,9 @@ protected override KeywordCategoryQuery CreateFiltered(Func pr protected override KeywordCategoryQuery CreateSkip(int count) => new(_categoryName, _predicate, SelectionMode.Skip, count); protected override KeywordCategoryQuery CreateTake(int count) => new(_categoryName, _predicate, SelectionMode.Take, count); + internal override SelectionMode Mode => _mode; + internal override int ModeArg => _modeArg; + private static Func? CombinePredicates(Func? a, Func b) => a == null ? b : n => a(n) && b(n); } diff --git a/TinyTokenizer/Ast/QueryCombinators.cs b/TinyTokenizer/Ast/QueryCombinators.cs index 4952a31..0c8dd84 100644 --- a/TinyTokenizer/Ast/QueryCombinators.cs +++ b/TinyTokenizer/Ast/QueryCombinators.cs @@ -76,7 +76,7 @@ bool IGreenNodeQuery.TryMatchGreen(IReadOnlyList siblings, int startI /// Useful for extracting content between matching patterns: /// Query.Between(Query.Symbol("("), Query.Symbol(")")) /// -public sealed record BetweenQuery : INodeQuery, IGreenNodeQuery +public sealed record BetweenQuery : INodeQuery, IGreenNodeQuery, IRegionQuery { private readonly INodeQuery _start; private readonly INodeQuery _end; @@ -183,6 +183,34 @@ bool IGreenNodeQuery.TryMatchGreen(IReadOnlyList siblings, int startI // End not found return false; } + + /// + IEnumerable IRegionQuery.SelectRegions(SyntaxTree tree) + => ((IRegionQuery)this).SelectRegions(tree.Root); + + /// + IEnumerable IRegionQuery.SelectRegions(SyntaxNode root) + { + var walker = new PathTrackingWalker(root); + foreach (var (node, parentPath) in walker.DescendantsAndSelfWithPath()) + { + if (TryMatch(node, out var consumedCount)) + { + var parent = node.Parent; + if (parent != null) + { + yield return new QueryRegion( + parentPath: parentPath, + parent: parent, + startSlot: node.SiblingIndex, + endSlot: node.SiblingIndex + consumedCount, + firstNode: node, + position: node.Position + ); + } + } + } + } } #endregion @@ -192,7 +220,7 @@ bool IGreenNodeQuery.TryMatchGreen(IReadOnlyList siblings, int startI /// /// Matches a sequence of queries in order, consuming multiple sibling nodes. /// -public sealed record SequenceQuery : INodeQuery, IGreenNodeQuery +public sealed record SequenceQuery : INodeQuery, IGreenNodeQuery, IRegionQuery { private readonly ImmutableArray _parts; @@ -308,6 +336,34 @@ bool IGreenNodeQuery.TryMatchGreen(IReadOnlyList siblings, int startI consumedCount = totalConsumed; return true; } + + /// + IEnumerable IRegionQuery.SelectRegions(SyntaxTree tree) + => ((IRegionQuery)this).SelectRegions(tree.Root); + + /// + IEnumerable IRegionQuery.SelectRegions(SyntaxNode root) + { + var walker = new PathTrackingWalker(root); + foreach (var (node, parentPath) in walker.DescendantsAndSelfWithPath()) + { + if (TryMatch(node, out var consumedCount)) + { + var parent = node.Parent; + if (parent != null) + { + yield return new QueryRegion( + parentPath: parentPath, + parent: parent, + startSlot: node.SiblingIndex, + endSlot: node.SiblingIndex + consumedCount, + firstNode: node, + position: node.Position + ); + } + } + } + } } #endregion @@ -318,7 +374,7 @@ bool IGreenNodeQuery.TryMatchGreen(IReadOnlyList siblings, int startI /// Matches zero or one occurrence of the inner query. /// Always succeeds - returns empty match if inner doesn't match. /// -public sealed record OptionalQuery : INodeQuery, IGreenNodeQuery +public sealed record OptionalQuery : INodeQuery, IGreenNodeQuery, IRegionQuery { private readonly INodeQuery _inner; @@ -355,6 +411,35 @@ bool IGreenNodeQuery.TryMatchGreen(IReadOnlyList siblings, int startI consumedCount = 0; return true; // Optional always succeeds with 0 consumed } + + /// + IEnumerable IRegionQuery.SelectRegions(SyntaxTree tree) + => ((IRegionQuery)this).SelectRegions(tree.Root); + + /// + IEnumerable IRegionQuery.SelectRegions(SyntaxNode root) + { + // Optional delegates to inner query - traverse and match inner directly + var walker = new PathTrackingWalker(root); + foreach (var (node, parentPath) in walker.DescendantsAndSelfWithPath()) + { + if (_inner.TryMatch(node, out var consumedCount) && consumedCount > 0) + { + var parent = node.Parent; + if (parent != null) + { + yield return new QueryRegion( + parentPath: parentPath, + parent: parent, + startSlot: node.SiblingIndex, + endSlot: node.SiblingIndex + consumedCount, + firstNode: node, + position: node.Position + ); + } + } + } + } } #endregion @@ -364,7 +449,7 @@ bool IGreenNodeQuery.TryMatchGreen(IReadOnlyList siblings, int startI /// /// Matches the inner query multiple times (min to max occurrences). /// -public sealed record RepeatQuery : INodeQuery, IGreenNodeQuery +public sealed record RepeatQuery : INodeQuery, IGreenNodeQuery, IRegionQuery { private readonly INodeQuery _inner; private readonly int _min; @@ -463,6 +548,34 @@ bool IGreenNodeQuery.TryMatchGreen(IReadOnlyList siblings, int startI consumedCount = totalConsumed; return true; } + + /// + IEnumerable IRegionQuery.SelectRegions(SyntaxTree tree) + => ((IRegionQuery)this).SelectRegions(tree.Root); + + /// + IEnumerable IRegionQuery.SelectRegions(SyntaxNode root) + { + var walker = new PathTrackingWalker(root); + foreach (var (node, parentPath) in walker.DescendantsAndSelfWithPath()) + { + if (TryMatch(node, out var consumedCount) && consumedCount > 0) + { + var parent = node.Parent; + if (parent != null) + { + yield return new QueryRegion( + parentPath: parentPath, + parent: parent, + startSlot: node.SiblingIndex, + endSlot: node.SiblingIndex + consumedCount, + firstNode: node, + position: node.Position + ); + } + } + } + } } #endregion @@ -473,7 +586,7 @@ bool IGreenNodeQuery.TryMatchGreen(IReadOnlyList siblings, int startI /// Matches the inner query repeatedly until a terminator is encountered. /// The terminator is NOT consumed (lookahead-style matching). /// -public sealed record RepeatUntilQuery : INodeQuery, IGreenNodeQuery +public sealed record RepeatUntilQuery : INodeQuery, IGreenNodeQuery, IRegionQuery { private readonly INodeQuery _inner; private readonly INodeQuery _terminator; @@ -642,6 +755,34 @@ private bool TerminatorMatchesGreen(IReadOnlyList siblings, int index return false; } + + /// + IEnumerable IRegionQuery.SelectRegions(SyntaxTree tree) + => ((IRegionQuery)this).SelectRegions(tree.Root); + + /// + IEnumerable IRegionQuery.SelectRegions(SyntaxNode root) + { + var walker = new PathTrackingWalker(root); + foreach (var (node, parentPath) in walker.DescendantsAndSelfWithPath()) + { + if (TryMatch(node, out var consumedCount)) + { + var parent = node.Parent; + if (parent != null) + { + yield return new QueryRegion( + parentPath: parentPath, + parent: parent, + startSlot: node.SiblingIndex, + endSlot: node.SiblingIndex + consumedCount, + firstNode: node, + position: node.Position + ); + } + } + } + } } #endregion @@ -652,7 +793,7 @@ private bool TerminatorMatchesGreen(IReadOnlyList siblings, int index /// Matches the inner query only if followed by (or not followed by) the lookahead query. /// The lookahead is not consumed (zero-width assertion). /// -public sealed record LookaheadQuery : INodeQuery, IGreenNodeQuery +public sealed record LookaheadQuery : INodeQuery, IGreenNodeQuery, IRegionQuery { private readonly INodeQuery _inner; private readonly INodeQuery _lookahead; @@ -756,6 +897,34 @@ _lookahead is IGreenNodeQuery lookaheadGreen && consumedCount = innerConsumed; return true; } + + /// + IEnumerable IRegionQuery.SelectRegions(SyntaxTree tree) + => ((IRegionQuery)this).SelectRegions(tree.Root); + + /// + IEnumerable IRegionQuery.SelectRegions(SyntaxNode root) + { + var walker = new PathTrackingWalker(root); + foreach (var (node, parentPath) in walker.DescendantsAndSelfWithPath()) + { + if (TryMatch(node, out var consumedCount)) + { + var parent = node.Parent; + if (parent != null) + { + yield return new QueryRegion( + parentPath: parentPath, + parent: parent, + startSlot: node.SiblingIndex, + endSlot: node.SiblingIndex + consumedCount, + firstNode: node, + position: node.Position + ); + } + } + } + } } #endregion diff --git a/TinyTokenizer/Ast/QueryRegion.cs b/TinyTokenizer/Ast/QueryRegion.cs new file mode 100644 index 0000000..63c7ad1 --- /dev/null +++ b/TinyTokenizer/Ast/QueryRegion.cs @@ -0,0 +1,245 @@ +using System.Collections.Immutable; + +namespace TinyTokenizer.Ast; + +/// +/// Represents a resolved region in the syntax tree. +/// A region is a contiguous range of slots within a parent container. +/// May be empty (StartSlot == EndSlot) representing a zero-width position. +/// +/// +/// This is an internal type used by to translate +/// queries into edit operations. Users interact with the public Query API. +/// +/// is computed lazily to avoid tree-walking overhead +/// when regions are enumerated but not all are used for editing. +/// +/// +internal readonly struct QueryRegion +{ + private readonly NodePath? _parentPath; + private readonly SyntaxNode _parent; + + /// + /// Creates a new region with lazy path computation. + /// + public QueryRegion( + SyntaxNode parent, + int startSlot, + int endSlot, + SyntaxNode? firstNode, + int position) + { + _parentPath = null; + _parent = parent; + StartSlot = startSlot; + EndSlot = endSlot; + FirstNode = firstNode; + Position = position; + } + + /// + /// Creates a new region with pre-computed path. + /// Use when path is already available to avoid recomputation. + /// + public QueryRegion( + NodePath parentPath, + SyntaxNode parent, + int startSlot, + int endSlot, + SyntaxNode? firstNode, + int position) + { + _parentPath = parentPath; + _parent = parent; + StartSlot = startSlot; + EndSlot = endSlot; + FirstNode = firstNode; + Position = position; + } + + /// + /// Path to the parent container holding this region. + /// Computed lazily on first access. + /// + public NodePath ParentPath => _parentPath ?? NodePath.FromNode(_parent); + + /// Parent container node. + public SyntaxNode Parent => _parent; + + /// First slot index in the region (inclusive). + public int StartSlot { get; } + + /// Slot index after the last slot in the region (exclusive). + public int EndSlot { get; } + + /// Number of slots in the region. + public int SlotCount => EndSlot - StartSlot; + + /// Whether this region is empty (zero-width position). + public bool IsEmpty => StartSlot == EndSlot; + + /// The first node in the region, if any exist. Null for empty regions. + public SyntaxNode? FirstNode { get; } + + /// Document position at the start of this region. + public int Position { get; } + + /// Enumerates all nodes currently in this region. + public IEnumerable Nodes + { + get + { + for (int i = StartSlot; i < EndSlot; i++) + { + var child = Parent.GetChild(i); + if (child != null) + yield return child; + } + } + } + + /// Gets the last node in the region, or null if empty. + public SyntaxNode? LastNode + { + get + { + if (IsEmpty) return null; + for (int i = EndSlot - 1; i >= StartSlot; i--) + { + var child = Parent.GetChild(i); + if (child != null) + return child; + } + return null; + } + } + + /// Document position at the end of this region. + public int EndPosition => LastNode?.EndPosition ?? Position; +} + +/// +/// Internal interface for queries that can resolve to regions. +/// Used by SyntaxEditor to translate queries into edit operations. +/// +internal interface IRegionQuery +{ + /// Resolves this query to regions in the tree. + IEnumerable SelectRegions(SyntaxTree tree); + + /// Resolves this query to regions in a subtree. + IEnumerable SelectRegions(SyntaxNode root); +} + +/// +/// A tree walker that incrementally tracks the path during traversal. +/// O(1) per traversal step instead of O(depth) for NodePath.FromNode(). +/// +internal sealed class PathTrackingWalker +{ + private readonly SyntaxNode _root; + private readonly List _pathStack; + private SyntaxNode _current; + + public PathTrackingWalker(SyntaxNode root) + { + _root = root; + _current = root; + _pathStack = new List(8); // Pre-allocate for typical tree depth + } + + /// The current node. + public SyntaxNode Current => _current; + + /// + /// Gets the current path as a NodePath. + /// The path leads to the CURRENT node (not its parent). + /// + public NodePath CurrentPath => new NodePath(ImmutableArray.CreateRange(_pathStack)); + + /// + /// Gets the path to the parent of the current node. + /// Returns Root path if current is the root. + /// + public NodePath ParentPath + { + get + { + if (_pathStack.Count == 0) + return NodePath.Root; + + // Return path without the last index (current node's sibling index) + return new NodePath(ImmutableArray.CreateRange(_pathStack.Take(_pathStack.Count - 1))); + } + } + + /// + /// Enumerates all descendants of the root in document order, + /// yielding each node along with its parent path. + /// + public IEnumerable<(SyntaxNode Node, NodePath ParentPath)> DescendantsAndSelfWithPath() + { + // Yield root first + yield return (_root, NodePath.Root); + + // Reset state for traversal + _current = _root; + _pathStack.Clear(); + + while (MoveNext()) + { + yield return (_current, ParentPath); + } + } + + /// + /// Moves to the next node in document order (depth-first pre-order). + /// Returns true if moved, false if at end. + /// + private bool MoveNext() + { + // Try first child + if (_current.SlotCount > 0) + { + for (int i = 0; i < _current.SlotCount; i++) + { + var child = _current.GetChild(i); + if (child != null) + { + _pathStack.Add(i); + _current = child; + return true; + } + } + } + + // Try next sibling or ancestor's next sibling + while (_pathStack.Count > 0) + { + var parent = _current.Parent; + if (parent == null) + break; + + var currentIndex = _pathStack[_pathStack.Count - 1]; + _pathStack.RemoveAt(_pathStack.Count - 1); + + // Try next sibling + for (int i = currentIndex + 1; i < parent.SlotCount; i++) + { + var sibling = parent.GetChild(i); + if (sibling != null) + { + _pathStack.Add(i); + _current = sibling; + return true; + } + } + + // Move up to try parent's siblings + _current = parent; + } + + return false; + } +} diff --git a/TinyTokenizer/Ast/SemanticMatchExtensions.cs b/TinyTokenizer/Ast/SemanticMatchExtensions.cs index d5af20b..d28b1c4 100644 --- a/TinyTokenizer/Ast/SemanticMatchExtensions.cs +++ b/TinyTokenizer/Ast/SemanticMatchExtensions.cs @@ -163,6 +163,9 @@ protected override SyntaxNodeQuery CreateFiltered(Func predica protected override SyntaxNodeQuery CreateSkip(int count) => new(_kind, _predicate, SelectionMode.Skip, count); protected override SyntaxNodeQuery CreateTake(int count) => new(_kind, _predicate, SelectionMode.Take, count); + internal override SelectionMode Mode => _mode; + internal override int ModeArg => _modeArg; + private static Func? CombinePredicates(Func? a, Func b) => a == null ? b : n => a(n) && b(n); } @@ -272,4 +275,7 @@ protected override SyntaxNodeQuery CreateFiltered(Func pred protected override SyntaxNodeQuery CreateNth(int n) => new SyntaxNodeQuery(_predicate, SelectionMode.Nth, n); protected override SyntaxNodeQuery CreateSkip(int count) => new SyntaxNodeQuery(_predicate, SelectionMode.Skip, count); protected override SyntaxNodeQuery CreateTake(int count) => new SyntaxNodeQuery(_predicate, SelectionMode.Take, count); + + internal override SelectionMode Mode => _mode; + internal override int ModeArg => _modeArg; } diff --git a/TinyTokenizer/Ast/SyntaxBlock.cs b/TinyTokenizer/Ast/SyntaxBlock.cs index 6952ee9..3adc7f6 100644 --- a/TinyTokenizer/Ast/SyntaxBlock.cs +++ b/TinyTokenizer/Ast/SyntaxBlock.cs @@ -1,5 +1,6 @@ using System.Collections.Immutable; using System.Diagnostics; +using CommunityToolkit.HighPerformance.Buffers; namespace TinyTokenizer.Ast; @@ -32,20 +33,15 @@ internal SyntaxBlock(GreenBlock green, SyntaxNode? parent, int position, int sib public char Closer => Green.Closer; /// The opening delimiter node with its trivia. - public SyntaxToken OpenerNode => (SyntaxToken)Green.OpenerNode.CreateRed(this, Position, -1, Tree); + /// Opener is slot 0 in the Roslyn-style slot model. + public SyntaxToken OpenerNode => (SyntaxToken)GetChild(0)!; /// The closing delimiter node with its trivia. - public SyntaxToken CloserNode - { - get - { - var closerPosition = EndPosition - Green.CloserNode.Width; - return (SyntaxToken)Green.CloserNode.CreateRed(this, closerPosition, -1, Tree); - } - } + /// Closer is the last slot (N+1) in the Roslyn-style slot model. + public SyntaxToken CloserNode => (SyntaxToken)GetChild(SlotCount - 1)!; - /// Number of children in this block. - public int ChildCount => Green.SlotCount; + /// Number of inner children in this block (excluding opener/closer). + public int ChildCount => Green.InnerChildren.Length; /// Leading trivia before the opening delimiter (from opener node). internal ImmutableArray GreenLeadingTrivia => Green.OpenerNode.LeadingTrivia; @@ -140,13 +136,48 @@ public IEnumerable GetTrailingTrivia() public int InnerEndPosition => EndPosition - Green.CloserNode.Width; /// - /// Gets all children as an enumerable (lazy creation). + /// Gets the text of inner children only (excluding opener and closer delimiters). + /// + public string InnerText + { + get + { + using var buffer = new ArrayPoolBufferWriter(); + foreach (var child in InnerChildren) + { + child.WriteTo(buffer); + } + return buffer.WrittenSpan.ToString(); + } + } + + /// + /// Gets all children including opener and closer (Roslyn-style traversal). + /// Slot 0 = opener, slots 1..N = inner children, slot N+1 = closer. /// public new IEnumerable Children { get { - for (int i = 0; i < ChildCount; i++) + for (int i = 0; i < SlotCount; i++) + { + var child = GetChild(i); + if (child != null) + yield return child; + } + } + } + + /// + /// Gets inner children only (excluding opener and closer delimiters). + /// Use this for content-focused traversal. + /// + public IEnumerable InnerChildren + { + get + { + // Inner children are slots 1 through SlotCount-2 (excluding opener at 0 and closer at N+1) + for (int i = 1; i < SlotCount - 1; i++) { var child = GetChild(i); if (child != null) @@ -156,11 +187,11 @@ public IEnumerable GetTrailingTrivia() } /// - /// Gets children of a specific kind. + /// Gets inner children of a specific kind. /// public IEnumerable ChildrenOfKind(NodeKind kind) { - foreach (var child in Children) + foreach (var child in InnerChildren) { if (child.Kind == kind) yield return child; @@ -168,13 +199,13 @@ public IEnumerable ChildrenOfKind(NodeKind kind) } /// - /// Gets all leaf children. + /// Gets all inner leaf children (excluding opener/closer). /// public IEnumerable LeafChildren { get { - foreach (var child in Children) + foreach (var child in InnerChildren) { if (child is SyntaxToken leaf) yield return leaf; @@ -183,13 +214,13 @@ public IEnumerable LeafChildren } /// - /// Gets all block children. + /// Gets all inner block children. /// public IEnumerable BlockChildren { get { - foreach (var child in Children) + foreach (var child in InnerChildren) { if (child is SyntaxBlock block) yield return block; @@ -198,13 +229,14 @@ public IEnumerable BlockChildren } /// - /// Finds the index of a child node. + /// Finds the index of a child node within inner children. + /// Returns slot index (1-based for inner children, 0 for opener, SlotCount-1 for closer). /// Uses SyntaxNode equality which compares by green node identity and position. /// public int IndexOf(SyntaxNode child) { // First check if the child has a valid sibling index from its parent - if (child.SiblingIndex >= 0 && child.SiblingIndex < ChildCount) + if (child.SiblingIndex >= 0 && child.SiblingIndex < SlotCount) { // Verify it's actually from this block var candidate = GetChild(child.SiblingIndex); @@ -214,8 +246,8 @@ public int IndexOf(SyntaxNode child) } } - // Fall back to linear search - for (int i = 0; i < ChildCount; i++) + // Fall back to linear search across all slots + for (int i = 0; i < SlotCount; i++) { var candidate = GetChild(i); if (candidate == child) diff --git a/TinyTokenizer/Ast/SyntaxEditor.cs b/TinyTokenizer/Ast/SyntaxEditor.cs index fada12b..47efd84 100644 --- a/TinyTokenizer/Ast/SyntaxEditor.cs +++ b/TinyTokenizer/Ast/SyntaxEditor.cs @@ -1,4 +1,5 @@ using System.Collections.Immutable; +using CommunityToolkit.HighPerformance.Buffers; namespace TinyTokenizer.Ast; @@ -44,11 +45,53 @@ internal SyntaxEditor(SyntaxTree tree, TokenizerOptions? options = null) /// public bool HasPendingEdits => _edits.Count > 0; + #region Region Resolution + + /// + /// Gets regions from a query, using IRegionQuery if available, + /// otherwise falling back to match-based resolution. + /// + private IEnumerable GetRegions(INodeQuery query) + { + if (query is IRegionQuery regionQuery) + { + return regionQuery.SelectRegions(_tree); + } + + // Fallback for queries that don't implement IRegionQuery + return GetRegionsFromMatches(query); + } + + private IEnumerable GetRegionsFromMatches(INodeQuery query) + { + // Use Select + TryMatch to get matches without SelectMatches + foreach (var node in query.Select(_tree)) + { + if (query.TryMatch(node, out var consumedCount)) + { + var parent = node.Parent; + if (parent != null) + { + yield return new QueryRegion( + parent: parent, + startSlot: node.SiblingIndex, + endSlot: node.SiblingIndex + consumedCount, + firstNode: node, + position: node.Position + ); + } + } + } + } + + #endregion + #region Insert (Query-based) /// /// Queues an insertion of text before all nodes matching the query. - /// For , handles empty containers by using container metadata. + /// For range queries (e.g., Query.Between), inserts before the start of the matched range. + /// For empty regions (e.g., empty blocks via Inner()), inserts at the region position. /// /// A query specifying which nodes to insert before. /// The text to insert (will be parsed into nodes). @@ -60,29 +103,20 @@ internal SyntaxEditor(SyntaxTree tree, TokenizerOptions? options = null) /// public SyntaxEditor InsertBefore(INodeQuery query, string text) { - // Handle BoundaryQuery specially for empty container support - if (query is BoundaryQuery boundaryQuery) - { - foreach (var container in boundaryQuery.ResolveContainers(_tree)) - { - var pos = CreateBoundaryInsertionPosition(container, boundaryQuery.Side, before: true); - if (pos.HasValue) - _edits.Add(new InsertEdit(pos.Value, text) { SequenceNumber = _sequenceNumber++ }); - } - return this; - } - - // Standard query - insert before each matched node - foreach (var node in query.Select(_tree)) + foreach (var region in GetRegions(query)) { - InsertBefore(node, text); + _edits.Add(new InsertAtSlotEdit(region.ParentPath, region.StartSlot, region.Position, text) + { + SequenceNumber = _sequenceNumber++ + }); } return this; } /// /// Queues an insertion of text after all nodes matching the query. - /// For , handles empty containers by using container metadata. + /// For range queries (e.g., Query.Between), inserts after the end of the matched range. + /// For empty regions (e.g., empty blocks via Inner()), inserts at the region position. /// /// A query specifying which nodes to insert after. /// The text to insert (will be parsed into nodes). @@ -94,22 +128,12 @@ public SyntaxEditor InsertBefore(INodeQuery query, string text) /// public SyntaxEditor InsertAfter(INodeQuery query, string text) { - // Handle BoundaryQuery specially for empty container support - if (query is BoundaryQuery boundaryQuery) + foreach (var region in GetRegions(query)) { - foreach (var container in boundaryQuery.ResolveContainers(_tree)) - { - var pos = CreateBoundaryInsertionPosition(container, boundaryQuery.Side, before: false); - if (pos.HasValue) - _edits.Add(new InsertEdit(pos.Value, text) { SequenceNumber = _sequenceNumber++ }); - } - return this; - } - - // Standard query - insert after each matched node - foreach (var node in query.Select(_tree)) - { - InsertAfter(node, text); + _edits.Add(new InsertAtSlotEdit(region.ParentPath, region.EndSlot, region.EndPosition, text) + { + SequenceNumber = _sequenceNumber++ + }); } return this; } @@ -304,15 +328,20 @@ internal SyntaxEditor InsertAfter(IEnumerable targets, IEnumerable /// Queues removal of all nodes matching the query. + /// For range queries (e.g., Query.Between), removes all nodes in the matched range. /// public SyntaxEditor Remove(INodeQuery query) { - var nodes = query.Select(_tree).ToList(); - - foreach (var node in nodes) + foreach (var region in GetRegions(query)) { - var path = NodePath.FromNode(node); - _edits.Add(new RemoveEdit(path, node.Position) { SequenceNumber = _sequenceNumber++ }); + if (!region.IsEmpty) + { + var path = region.FirstNode != null + ? NodePath.FromNode(region.FirstNode) + : region.ParentPath.Child(region.StartSlot); + _edits.Add(new RemoveEdit(path, region.Position, region.SlotCount) { SequenceNumber = _sequenceNumber++ }); + } + // Empty regions: nothing to remove, silently skip } return this; @@ -362,16 +391,17 @@ public SyntaxEditor Remove(IEnumerable nodes) /// /// Queues replacement of all nodes matching the query with new text. + /// For range queries (e.g., Query.Between), replaces all nodes in the matched range. /// public SyntaxEditor Replace(INodeQuery query, string text) { - var nodes = query.Select(_tree).ToList(); - - foreach (var node in nodes) + foreach (var region in GetRegions(query)) { - var path = NodePath.FromNode(node); - var (leading, trailing) = GetTrivia(node); - _edits.Add(new ReplaceEdit(path, text, node.Position, leading, trailing) { SequenceNumber = _sequenceNumber++ }); + var (leading, trailing) = GetTriviaForRegion(region); + _edits.Add(new ReplaceRegionEdit(region, text, leading, trailing) + { + SequenceNumber = _sequenceNumber++ + }); } return this; @@ -396,14 +426,17 @@ public SyntaxEditor Replace(IEnumerable queries, string text) /// public SyntaxEditor Replace(INodeQuery query, Func replacer) { - var nodes = query.Select(_tree).ToList(); - - foreach (var node in nodes) + foreach (var region in GetRegions(query)) { + var node = region.FirstNode; + if (node == null) continue; + var text = replacer(node); - var path = NodePath.FromNode(node); - var (leading, trailing) = GetTrivia(node); - _edits.Add(new ReplaceEdit(path, text, node.Position, leading, trailing) { SequenceNumber = _sequenceNumber++ }); + var (leading, trailing) = GetTriviaForRegion(region); + _edits.Add(new ReplaceRegionEdit(region, text, leading, trailing) + { + SequenceNumber = _sequenceNumber++ + }); } return this; @@ -428,13 +461,13 @@ public SyntaxEditor Replace(IEnumerable queries, Func internal SyntaxEditor Replace(INodeQuery query, ImmutableArray nodes) { - var matchedNodes = query.Select(_tree).ToList(); - - foreach (var node in matchedNodes) + foreach (var region in GetRegions(query)) { - var path = NodePath.FromNode(node); - var (leading, trailing) = GetTrivia(node); - _edits.Add(new ReplaceNodesEdit(path, nodes, node.Position, leading, trailing) { SequenceNumber = _sequenceNumber++ }); + var (leading, trailing) = GetTriviaForRegion(region); + _edits.Add(new ReplaceNodesRegionEdit(region, nodes, leading, trailing) + { + SequenceNumber = _sequenceNumber++ + }); } return this; @@ -550,20 +583,24 @@ internal SyntaxEditor Replace(SyntaxNode node, IEnumerable replacemen /// /// Queues a transformation edit of all nodes matching the query. /// The transformer receives the node's content WITHOUT trivia, and trivia is automatically preserved. + /// For range queries, receives the concatenated content of all matched nodes. /// /// Query to select nodes to edit. /// Function that transforms the node's content (without trivia). public SyntaxEditor Edit(INodeQuery query, Func transformer) { - var nodes = query.Select(_tree).ToList(); - - foreach (var node in nodes) + foreach (var region in GetRegions(query)) { - var path = NodePath.FromNode(node); - var (leading, trailing) = GetTrivia(node); - var contentWithoutTrivia = GetContentWithoutTrivia(node); + if (region.IsEmpty) continue; + + var firstNode = region.FirstNode; + if (firstNode == null) continue; + + var path = NodePath.FromNode(firstNode); + var (leading, trailing) = GetTriviaForRegion(region); + var contentWithoutTrivia = GetContentWithoutTriviaForRegion(region); var newText = transformer(contentWithoutTrivia); - _edits.Add(new ReplaceEdit(path, newText, node.Position, leading, trailing) { SequenceNumber = _sequenceNumber++ }); + _edits.Add(new ReplaceEdit(path, newText, region.Position, leading, trailing, region.SlotCount) { SequenceNumber = _sequenceNumber++ }); } return this; @@ -623,6 +660,7 @@ public SyntaxEditor Edit(IEnumerable nodes, Func tra /// Gets the content of a node without its leading and trailing trivia. /// For leaves, returns the token text. For blocks, returns the full content minus outer trivia. /// For syntax nodes (containers), returns full content minus the trivia from first/last children. + /// Uses precomputed trivia widths from green nodes for efficiency. /// private static string GetContentWithoutTrivia(SyntaxNode node) { @@ -645,75 +683,126 @@ private static string GetContentWithoutTrivia(SyntaxNode node) if (contentLength <= 0) return string.Empty; - return fullText.Substring(leadingWidth, contentLength); + return fullText.AsSpan(leadingWidth, contentLength).ToString(); } - // For syntax nodes and other containers, check first/last children for trivia - var (leadingTrivia, trailingTrivia) = GetTrivia(node); - if (leadingTrivia.IsEmpty && trailingTrivia.IsEmpty) + // For syntax nodes and other containers, use precomputed trivia widths + var green = node.Green; + var leadingTriviaWidth = green.GetLeadingTriviaWidth(); + var trailingTriviaWidth = green.GetTrailingTriviaWidth(); + + if (leadingTriviaWidth == 0 && trailingTriviaWidth == 0) { return node.ToText(); } var text = node.ToText(); - var leadingTriviaWidth = leadingTrivia.Sum(t => t.Width); - var trailingTriviaWidth = trailingTrivia.Sum(t => t.Width); - var contentLen = text.Length - leadingTriviaWidth - trailingTriviaWidth; if (contentLen <= 0) return string.Empty; - return text.Substring(leadingTriviaWidth, contentLen); + return text.AsSpan(leadingTriviaWidth, contentLen).ToString(); } /// - /// Extracts leading and trailing trivia from a node. - /// For leaves, returns the leaf's trivia. - /// For blocks, returns the block's trivia. - /// For syntax nodes (containers), returns trivia from the first child (leading) and last child (trailing). + /// Gets the concatenated content of a range of siblings without leading/trailing trivia. + /// Leading trivia from first node and trailing trivia from last node are excluded. + /// Uses precomputed trivia widths from green nodes for efficiency. /// - private static (ImmutableArray Leading, ImmutableArray Trailing) GetTrivia(SyntaxNode node) + private static string GetContentWithoutTriviaForRange(SyntaxNode startNode, int count) { - if (node is SyntaxToken leaf) - { - var greenLeaf = (GreenLeaf)leaf.Green; - return (greenLeaf.LeadingTrivia, greenLeaf.TrailingTrivia); - } + if (count <= 0) + return string.Empty; - if (node is SyntaxBlock block) - { - return (block.GreenLeadingTrivia, block.GreenTrailingTrivia); - } + if (count == 1) + return GetContentWithoutTrivia(startNode); - // For syntax nodes and other containers, get trivia from first/last children - var leading = ImmutableArray.Empty; - var trailing = ImmutableArray.Empty; + // For multiple nodes, concatenate their content using pooled buffer + using var buffer = new ArrayPoolBufferWriter(); + var current = startNode; - // Get leading trivia from first child - var firstChild = node.Children.FirstOrDefault(); - if (firstChild != null) + for (int i = 0; i < count && current != null; i++) { - var (childLeading, _) = GetTrivia(firstChild); - leading = childLeading; + var content = current.ToText(); + + if (i == 0) + { + // First node: exclude leading trivia only (use precomputed width) + var leadingWidth = current.Green.GetLeadingTriviaWidth(); + var span = content.AsSpan(leadingWidth); + var dest = buffer.GetSpan(span.Length); + span.CopyTo(dest); + buffer.Advance(span.Length); + } + else if (i == count - 1) + { + // Last node: exclude trailing trivia only (use precomputed width) + var trailingWidth = current.Green.GetTrailingTriviaWidth(); + var span = content.AsSpan(0, content.Length - trailingWidth); + var dest = buffer.GetSpan(span.Length); + span.CopyTo(dest); + buffer.Advance(span.Length); + } + else + { + // Middle nodes: include everything + var span = content.AsSpan(); + var dest = buffer.GetSpan(span.Length); + span.CopyTo(dest); + buffer.Advance(span.Length); + } + + current = current.NextSibling(); } - // Get trailing trivia from last child - var lastChild = node.Children.LastOrDefault(); - if (lastChild != null && !ReferenceEquals(lastChild, firstChild)) - { - var (_, childTrailing) = GetTrivia(lastChild); - trailing = childTrailing; - } - else if (firstChild != null) - { - // Only one child - get trailing from that same child - var (_, childTrailing) = GetTrivia(firstChild); - trailing = childTrailing; - } + return buffer.WrittenSpan.ToString(); + } + + /// + /// Extracts leading and trailing trivia from a node. + /// Uses green node's GetFirstLeaf/GetLastLeaf for O(depth) access instead of recursive red node traversal. + /// + private static (ImmutableArray Leading, ImmutableArray Trailing) GetTrivia(SyntaxNode node) + { + // Delegate to green node which efficiently finds first/last leaf + var green = node.Green; + return (green.GetLeadingTrivia(), green.GetTrailingTrivia()); + } + + /// + /// Gets trivia for a query region. + /// For empty regions, returns empty trivia. + /// For non-empty regions, returns leading from first and trailing from last. + /// + private static (ImmutableArray Leading, ImmutableArray Trailing) GetTriviaForRegion(QueryRegion region) + { + if (region.IsEmpty) + return (ImmutableArray.Empty, ImmutableArray.Empty); + + var first = region.FirstNode; + var last = region.LastNode; + + var leading = first != null ? GetTrivia(first).Leading : ImmutableArray.Empty; + var trailing = last != null ? GetTrivia(last).Trailing : ImmutableArray.Empty; return (leading, trailing); } + /// + /// Gets the content of a region without leading/trailing trivia. + /// + private static string GetContentWithoutTriviaForRegion(QueryRegion region) + { + if (region.IsEmpty) + return string.Empty; + + var first = region.FirstNode; + if (first == null) + return string.Empty; + + return GetContentWithoutTriviaForRange(first, region.SlotCount); + } + /// /// Creates an InsertionPosition for inserting before or after a target node. /// @@ -737,94 +826,6 @@ private static InsertionPosition CreateInsertionPosition(SyntaxNode target, bool : new InsertionPosition(parentPath, childIndex + 1, target.EndPosition, targetPath, targetLeading, targetTrailing); } - /// - /// Creates an InsertionPosition for a boundary query (Start/End of a container). - /// Handles empty containers by computing position from container metadata. - /// - private static InsertionPosition? CreateBoundaryInsertionPosition(SyntaxNode container, BoundarySide side, bool before) - { - if (container is SyntaxBlock block) - { - // For blocks, the boundary nodes are the opener/closer - // Start + InsertAfter = insert at beginning of content (after opener) - // End + InsertBefore = insert at end of content (before closer) - var blockPath = NodePath.FromNode(block); - - if (side == BoundarySide.Start) - { - // Insert relative to opener - if (before) - { - // InsertBefore(Start) = insert before the opener (before the block content) - var parent = block.Parent; - if (parent == null) return null; - var parentPath = NodePath.FromNode(parent); - return new InsertionPosition(parentPath, block.SiblingIndex, block.Position, null, - ImmutableArray.Empty, ImmutableArray.Empty); - } - else - { - // InsertAfter(Start) = insert at beginning of block content (child index 0) - return new InsertionPosition(blockPath, 0, block.InnerStartPosition, null, - ImmutableArray.Empty, ImmutableArray.Empty); - } - } - else // BoundarySide.End - { - if (before) - { - // InsertBefore(End) = insert at end of block content (after last child) - return new InsertionPosition(blockPath, block.ChildCount, block.InnerEndPosition, null, - ImmutableArray.Empty, ImmutableArray.Empty); - } - else - { - // InsertAfter(End) = insert after the closer (after the block) - var parent = block.Parent; - if (parent == null) return null; - var parentPath = NodePath.FromNode(parent); - return new InsertionPosition(parentPath, block.SiblingIndex + 1, block.EndPosition, null, - ImmutableArray.Empty, ImmutableArray.Empty); - } - } - } - - // For non-block containers (lists, syntax nodes), use first/last child - var children = container.Children.ToList(); - var containerPath = NodePath.FromNode(container); - - if (side == BoundarySide.Start) - { - if (children.Count == 0) - { - // Empty container - insert at position 0 (child index 0) - return before - ? null // Can't insert before nothing - : new InsertionPosition(containerPath, 0, container.Position, null, - ImmutableArray.Empty, ImmutableArray.Empty); - } - - // Has children - delegate to first child - var first = children[0]; - return CreateInsertionPosition(first, before); - } - else // BoundarySide.End - { - if (children.Count == 0) - { - // Empty container - insert at position 0 (child index 0) - return before - ? new InsertionPosition(containerPath, 0, container.Position, null, - ImmutableArray.Empty, ImmutableArray.Empty) - : null; // Can't insert after nothing - } - - // Has children - delegate to last child - var last = children[^1]; - return CreateInsertionPosition(last, before); - } - } - /// /// Converts an enumerable of RedNodes to an ImmutableArray of their underlying GreenNodes. /// @@ -987,11 +988,13 @@ internal sealed class RemoveEdit : PendingEdit { private readonly NodePath _path; private readonly int _position; + private readonly int _count; - public RemoveEdit(NodePath path, int position) + public RemoveEdit(NodePath path, int position, int count = 1) { _path = path; _position = position; + _count = count; } public override int Position => _position; @@ -1002,7 +1005,7 @@ public override GreenNode Apply(GreenNode root, GreenTreeBuilder builder, Tokeni { var parentPath = _path.Parent(); var childIndex = _path.Depth > 0 ? _path[_path.Depth - 1] : 0; - return builder.RemoveAt(parentPath.ToArray(), childIndex, 1); + return builder.RemoveAt(parentPath.ToArray(), childIndex, _count); } } @@ -1011,16 +1014,19 @@ internal sealed class ReplaceEdit : PendingEdit private readonly NodePath _path; private readonly string _text; private readonly int _position; + private readonly int _count; private readonly ImmutableArray _leadingTrivia; private readonly ImmutableArray _trailingTrivia; public ReplaceEdit(NodePath path, string text, int position, ImmutableArray leadingTrivia = default, - ImmutableArray trailingTrivia = default) + ImmutableArray trailingTrivia = default, + int count = 1) { _path = path; _text = text; _position = position; + _count = count; _leadingTrivia = leadingTrivia.IsDefault ? ImmutableArray.Empty : leadingTrivia; _trailingTrivia = trailingTrivia.IsDefault ? ImmutableArray.Empty : trailingTrivia; } @@ -1043,7 +1049,7 @@ public override GreenNode Apply(GreenNode root, GreenTreeBuilder builder, Tokeni nodes = TransferTrivia(nodes, _leadingTrivia, _trailingTrivia); } - return builder.ReplaceAt(parentPath.ToArray(), childIndex, 1, nodes); + return builder.ReplaceAt(parentPath.ToArray(), childIndex, _count, nodes); } /// @@ -1086,16 +1092,19 @@ internal sealed class ReplaceNodesEdit : PendingEdit private readonly NodePath _path; private readonly ImmutableArray _nodes; private readonly int _position; + private readonly int _count; private readonly ImmutableArray _leadingTrivia; private readonly ImmutableArray _trailingTrivia; public ReplaceNodesEdit(NodePath path, ImmutableArray nodes, int position, ImmutableArray leadingTrivia = default, - ImmutableArray trailingTrivia = default) + ImmutableArray trailingTrivia = default, + int count = 1) { _path = path; _nodes = nodes; _position = position; + _count = count; _leadingTrivia = leadingTrivia.IsDefault ? ImmutableArray.Empty : leadingTrivia; _trailingTrivia = trailingTrivia.IsDefault ? ImmutableArray.Empty : trailingTrivia; } @@ -1117,7 +1126,7 @@ public override GreenNode Apply(GreenNode root, GreenTreeBuilder builder, Tokeni nodes = TransferTrivia(nodes, _leadingTrivia, _trailingTrivia); } - return builder.ReplaceAt(parentPath.ToArray(), childIndex, 1, nodes); + return builder.ReplaceAt(parentPath.ToArray(), childIndex, _count, nodes); } /// @@ -1155,6 +1164,171 @@ private static ImmutableArray TransferTrivia( } } +internal sealed class InsertAtSlotEdit : PendingEdit +{ + private readonly NodePath _parentPath; + private readonly int _slot; + private readonly int _position; + private readonly string _text; + + public InsertAtSlotEdit(NodePath parentPath, int slot, int position, string text) + { + _parentPath = parentPath; + _slot = slot; + _position = position; + _text = text; + } + + public override int Position => _position; + + public override NodePath AffectedPath => _parentPath; + + public override GreenNode Apply(GreenNode root, GreenTreeBuilder builder, TokenizerOptions options) + { + var lexer = new GreenLexer(options); + var newNodes = lexer.ParseToGreenNodes(_text); + + return builder.InsertAt(_parentPath.ToArray(), _slot, newNodes); + } +} + +internal sealed class ReplaceRegionEdit : PendingEdit +{ + private readonly QueryRegion _region; + private readonly string _text; + private readonly ImmutableArray _leadingTrivia; + private readonly ImmutableArray _trailingTrivia; + + public ReplaceRegionEdit(QueryRegion region, string text, + ImmutableArray leadingTrivia, + ImmutableArray trailingTrivia) + { + _region = region; + _text = text; + _leadingTrivia = leadingTrivia.IsDefault ? ImmutableArray.Empty : leadingTrivia; + _trailingTrivia = trailingTrivia.IsDefault ? ImmutableArray.Empty : trailingTrivia; + } + + public override int Position => _region.Position; + + public override NodePath AffectedPath => _region.ParentPath; + + public override GreenNode Apply(GreenNode root, GreenTreeBuilder builder, TokenizerOptions options) + { + var lexer = new GreenLexer(options); + var newNodes = lexer.ParseToGreenNodes(_text); + + if (newNodes.Length > 0) + { + newNodes = TransferTrivia(newNodes, _leadingTrivia, _trailingTrivia); + } + + return builder.ReplaceAt( + _region.ParentPath.ToArray(), + _region.StartSlot, + _region.SlotCount, + newNodes + ); + } + + private static ImmutableArray TransferTrivia( + ImmutableArray nodes, + ImmutableArray leading, + ImmutableArray trailing) + { + if (leading.IsEmpty && trailing.IsEmpty) + return nodes; + + var result = nodes.ToBuilder(); + + if (!leading.IsEmpty && result[0] is GreenLeaf firstLeaf) + { + var newLeading = firstLeaf.LeadingTrivia.IsEmpty + ? leading + : leading.AddRange(firstLeaf.LeadingTrivia); + result[0] = firstLeaf.WithLeadingTrivia(newLeading); + } + + if (!trailing.IsEmpty && result[^1] is GreenLeaf lastLeaf) + { + var newTrailing = lastLeaf.TrailingTrivia.IsEmpty + ? trailing + : lastLeaf.TrailingTrivia.AddRange(trailing); + result[^1] = lastLeaf.WithTrailingTrivia(newTrailing); + } + + return result.ToImmutable(); + } +} + +internal sealed class ReplaceNodesRegionEdit : PendingEdit +{ + private readonly QueryRegion _region; + private readonly ImmutableArray _nodes; + private readonly ImmutableArray _leadingTrivia; + private readonly ImmutableArray _trailingTrivia; + + public ReplaceNodesRegionEdit(QueryRegion region, ImmutableArray nodes, + ImmutableArray leadingTrivia, + ImmutableArray trailingTrivia) + { + _region = region; + _nodes = nodes; + _leadingTrivia = leadingTrivia.IsDefault ? ImmutableArray.Empty : leadingTrivia; + _trailingTrivia = trailingTrivia.IsDefault ? ImmutableArray.Empty : trailingTrivia; + } + + public override int Position => _region.Position; + + public override NodePath AffectedPath => _region.ParentPath; + + public override GreenNode Apply(GreenNode root, GreenTreeBuilder builder, TokenizerOptions options) + { + var nodes = _nodes; + + if (nodes.Length > 0 && (!_leadingTrivia.IsEmpty || !_trailingTrivia.IsEmpty)) + { + nodes = TransferTrivia(nodes, _leadingTrivia, _trailingTrivia); + } + + return builder.ReplaceAt( + _region.ParentPath.ToArray(), + _region.StartSlot, + _region.SlotCount, + nodes + ); + } + + private static ImmutableArray TransferTrivia( + ImmutableArray nodes, + ImmutableArray leading, + ImmutableArray trailing) + { + if (leading.IsEmpty && trailing.IsEmpty) + return nodes; + + var result = nodes.ToBuilder(); + + if (!leading.IsEmpty && result[0] is GreenLeaf firstLeaf) + { + var newLeading = firstLeaf.LeadingTrivia.IsEmpty + ? leading + : leading.AddRange(firstLeaf.LeadingTrivia); + result[0] = firstLeaf.WithLeadingTrivia(newLeading); + } + + if (!trailing.IsEmpty && result[^1] is GreenLeaf lastLeaf) + { + var newTrailing = lastLeaf.TrailingTrivia.IsEmpty + ? trailing + : lastLeaf.TrailingTrivia.AddRange(trailing); + result[^1] = lastLeaf.WithTrailingTrivia(newTrailing); + } + + return result.ToImmutable(); + } +} + /// /// Contains all information needed to perform an insertion. /// diff --git a/project.todo b/project.todo index e0b3660..e69de29 100644 --- a/project.todo +++ b/project.todo @@ -1,371 +0,0 @@ -# TinyTokenizer Improvement Plan - Project Tasks -# Based on: docs/improvement-plan-2026.md -# Created: January 2, 2026 -# Target Completion: Q2 2026 - -═══════════════════════════════════════════════════════════════════════════════ -PHASE 1: Quick Wins (v0.6.6) -Timeline: 1-2 days | Breaking Changes: None -═══════════════════════════════════════════════════════════════════════════════ - -[x] 1.1 Pre-compute Comment Style Flags (30 min) ✓ DONE - File: TokenParser.cs - [x] Add _hasCSingleLineComment private readonly field - [x] Add _hasCMultiLineComment private readonly field - [x] Initialize fields in constructor from options.CommentStyles - [x] Update TryParseComment to use cached fields instead of LINQ - [x] Verify no LINQ allocation in TryParseComment - [x] Benchmark comment-heavy input to confirm improvement - -[x] 1.2 Add IFormattable to Token (1 hr) ✓ DONE - File: Token.cs - [x] Add IFormattable interface to base Token record - [x] Implement ToString(string?, IFormatProvider?) method - [x] "G" or null → ContentSpan.ToString() - [x] "T" → Type.ToString() - [x] "P" → Position.ToString() - [x] "R" → Range format "{Position}..{Position + Content.Length}" - [x] "D" → Debug format "{Type}[{Range}]" - [x] Add XML documentation for format specifiers - [x] Add unit tests for each format specifier - -[x] 1.3 Fix AppendToBuffer Inefficiency (15 min) ✓ DONE - File: TokenParser.cs - [x] Add EnsureCapacity call before loop - [x] Replace indexed access with foreach over span - [x] Verify single capacity allocation instead of multiple resizes - [x] Run all parsing tests to confirm no regressions - -[x] 1.4 Cache SiblingIndex in RedNode (1 hr) ✓ DONE - File: Ast/RedNode.cs - [x] Add private readonly int _siblingIndex field - [x] Update internal constructor to accept siblingIndex parameter (default -1) - [x] Update GetRedChild to pass slot index when creating red child - [x] Update CreateRed method signature to accept siblingIndex - [x] Change SiblingIndex property to return _siblingIndex directly - [x] Verify O(1) access time - [x] Test NextSibling() and PreviousSibling() still work correctly - -[ ] 1.5 Phase 1 Release - [ ] Run full test suite (dotnet test) - [ ] Run benchmarks and document results - [ ] Update CHANGELOG.md with changes - [ ] Bump version to 0.6.6 in TinyTokenizer.csproj - [ ] Commit and tag as v0.6.6 - [ ] Create GitHub release - [ ] Verify NuGet package published via CI - -═══════════════════════════════════════════════════════════════════════════════ -PHASE 2: Performance (v0.7.0) -Timeline: 1-2 weeks | Breaking Changes: Minor (internal APIs only) -═══════════════════════════════════════════════════════════════════════════════ - -[x] 2.1 Replace List with ArrayPoolBufferWriter (4 hrs) ✓ DONE - Files: TokenParser.cs, TinyTokenizer.csproj - [x] Add NuGet reference: CommunityToolkit.HighPerformance 8.2.2 - [x] Add using CommunityToolkit.HighPerformance.Buffers - [x] Update ParseBlock method - [x] Replace List with ArrayPoolBufferWriter - [x] Use buffer.Write(span) for appending - [x] Add using statement for proper disposal - [x] Update ParseString method - [x] Replace List with ArrayPoolBufferWriter - [x] Ensure disposal in all code paths - [x] Update ParseNumericFromDigits method - [x] Update ParseNumericFromDot method - [x] Update ParseSingleLineComment method - [x] Update ParseMultiLineComment method - [x] Update TryParseOperator method - [x] Update TryParseTaggedIdent method - [x] Remove AppendToBuffer helper method (replaced with WriteToBuffer) - [x] Run all parsing tests to confirm no regressions - -[x] 2.2 Build Operator Trie (6 hrs) ✓ DONE - Files: TokenParser.cs, new OperatorTrie.cs - [x] Create OperatorTrie.cs file - [x] Create internal sealed class OperatorTrie - [x] Create private sealed class TrieNode - [x] Dictionary? Children - [x] string? Operator (non-null if end of operator) - [x] Implement Add(string op) method - [x] Implement TryMatch(ReadOnlySpan) method - [x] Update TokenParser constructor - [x] Add private readonly OperatorTrie _operatorTrie field - [x] Build trie from options.Operators - [x] Update TryParseOperator to use trie lookup - [x] Verify O(k) matching where k = operator length - [x] Verify greedy matching still works (longest match first) - [x] Add unit tests for OperatorTrie - [x] Benchmark with 50+ operators to confirm improvement - -[x] 2.3 Standardize Position Types (2 hrs) ⚠️ BREAKING ✓ DONE - Files: SimpleToken.cs, Token.cs, all derived types - [x] Update SimpleToken.cs - [x] Change Position parameter: long → int - [x] Update Token.cs - [x] Change Position parameter: long → int - [x] Update all derived token types (if any explicit Position usage) - [x] Update Lexer.cs position tracking - [x] Update TokenParser.cs position handling - [x] Search codebase for all long position usages and update - [x] Add XML doc noting 2GB file size practical limit - [x] Update all tests with position assertions - [ ] Document breaking change in CHANGELOG - -[x] 2.4 Unify Operator Matching with OperatorTrie (1 hr) ✓ DONE - Files: Tokenizer.cs, Ast/GreenLexer.cs - [x] Update Tokenizer.cs to use OperatorTrie instead of sorted array - [x] Replace _sortedOperators field with _operatorTrie - [x] Build trie in constructor from options.Operators - [x] Update TryParseOperator to use trie-based O(k) matching - [x] Update GreenLexer.cs to use OperatorTrie - [x] Replace _sortedOperators field with _operatorTrie - [x] Build trie in constructor - [x] Update TryParseOperator to use trie-based matching - [x] Use stackalloc for character buffer (zero allocation) - [x] Keep TokenizerOptions.Operators as ImmutableHashSet (no breaking change) - [x] Verify all tests pass - -[ ] 2.5 Phase 2 Release - [ ] Run full test suite - [ ] Run before/after allocation benchmarks - [ ] Document benchmark results - [ ] Write migration guide for position type change (long → int) - [ ] Update CHANGELOG.md with all changes and breaking changes - [ ] Bump version to 0.7.0 in TinyTokenizer.csproj - [ ] Commit and tag as v0.7.0 - [ ] Create GitHub release with migration notes - [ ] Verify NuGet package published via CI - -═══════════════════════════════════════════════════════════════════════════════ -PHASE 3: API Cleanup (v0.8.0) -Timeline: 2-3 weeks | Breaking Changes: Yes (major cleanup) -═══════════════════════════════════════════════════════════════════════════════ - -[x] 3.1 Consolidate Duplicate Parsing Logic (4 hrs) ✓ DONE - Files: TokenizerCore.cs, TokenParser.cs, Ast/GreenLexer.cs - [x] Extend TokenizerCore.cs with shared helper methods - [x] Add IsOperatorCapableToken(SimpleTokenType) method - [x] Add GetOperatorChar(SimpleTokenType) method - [x] Add GetMatchingCloser(SimpleTokenType) method - [x] Add GetBlockTokenType(SimpleTokenType) method - [x] Add IsOpeningDelimiter(SimpleTokenType) overload - [x] Add IsClosingDelimiter(SimpleTokenType) overload - [x] Refactor TokenParser.cs to use TokenizerCore - [x] Remove duplicate IsOperatorCapableToken - [x] Remove duplicate GetTokenChar (use GetOperatorChar) - [x] Remove duplicate IsOpeningDelimiter/IsClosingDelimiter - [x] Remove duplicate GetMatchingCloser/GetBlockTokenType - [x] Refactor GreenLexer.cs to use TokenizerCore - [x] Remove duplicate GetOperatorChar - [x] Remove duplicate IsOpeningDelimiter/IsClosingDelimiter - [x] Remove duplicate GetMatchingCloser - [x] Verify all 1,165 tests pass - Note: Full parsing logic unification not practical due to different - data flow (char vs SimpleToken streams, different output types). - Focused on sharing classification/mapping helpers instead. - -[x] 3.2 Address Schema Nullability (4 hrs) ✓ DONE - File: Ast/SyntaxTree.cs - [x] Add HasSchema property - [x] public bool HasSchema => Schema != null; - [x] Add private RequireSchema() helper method - [x] Throws InvalidOperationException with clear message if null - [x] Returns Schema if non-null (for use in expressions) - [x] Add WithSchema method - [x] public SyntaxTree WithSchema(Schema schema) - [x] Creates new tree with schema attached - [x] Auto-applies syntax binding if schema has definitions - [x] Throws ArgumentNullException if schema is null - [x] Update all schema-dependent methods to use RequireSchema() - [x] Match(SemanticContext?) - uses RequireSchema() - [x] MatchAll(SemanticContext?) - uses RequireSchema() - [x] Add comprehensive XML documentation - [x] Document when schema is required - [x] Document how to attach schema (WithSchema) - [x] Document error behavior - [x] Add cross-references - [x] Add unit tests (9 new tests) - [x] HasSchema_WithoutSchema_ReturnsFalse - [x] HasSchema_WithSchema_ReturnsTrue - [x] WithSchema_AttachesSchemaToExistingTree - [x] WithSchema_DoesNotModifyOriginalTree - [x] WithSchema_ThrowsOnNull - [x] Match_WithoutSchema_ThrowsInvalidOperationException - [x] MatchAll_WithoutSchema_ThrowsInvalidOperationException - [x] Match_WithExplicitSchema_DoesNotRequireAttachedSchema - [x] MatchAll_WithExplicitSchema_DoesNotRequireAttachedSchema - [x] All 1,174 tests pass - -[x] 3.3 Make Trivia Types Public (3 hrs) ✓ DONE - Files: Ast/GreenTrivia.cs, new Trivia.cs, Ast/RedLeaf.cs, Ast/RedBlock.cs - [x] Create Trivia.cs file - [x] Create public readonly struct Trivia - [x] internal Trivia(GreenTrivia green) constructor - [x] TriviaKind Kind property - [x] string Text property - [x] int Width property - [x] bool IsWhitespace, IsNewline, IsComment helper properties - [x] IEquatable implementation - [x] override ToString() - [x] TriviaKind enum already public (in GreenTrivia.cs) - [x] Update RedLeaf.cs - [x] Add IEnumerable GetLeadingTrivia() method - [x] Add IEnumerable GetTrailingTrivia() method - [x] Add bool HasLeadingTrivia property - [x] Add bool HasTrailingTrivia property - [x] Update RedBlock.cs - [x] Add IEnumerable GetLeadingTrivia() method - [x] Add IEnumerable GetInnerTrivia() method - [x] Add IEnumerable GetTrailingTrivia() method - [x] Add bool HasLeadingTrivia, HasInnerTrivia, HasTrailingTrivia properties - [x] Add XML documentation to all new types - [x] Add unit tests (21 new tests in TriviaTests.cs) - [x] All 1,195 tests pass - -[x] 3.4 Comprehensive Documentation Pass (4 hrs) ✓ VERIFIED COMPLETE - Files: All public API files - Note: Documentation review shows codebase already has comprehensive XML docs: - [x] Query.cs - Has , , tags on all methods - [x] TreeWalker.cs - Has for NodeFilter, FilterResult, all methods - [x] NodePattern.cs - Has and (marked obsolete with guidance) - [x] SyntaxBinder.cs - Has and on class and methods - [x] Schema.cs - Has for all properties, SchemaBuilder documented - [x] SemanticNode.cs - Has for base class and all built-in nodes - [x] NodeQuery.cs/NodeQueryTypes.cs - Has full documentation - [x] QueryCombinators.cs/QueryExtensions.cs - Has and - [x] SemanticContext.cs - Has for all members - [x] SemanticNodeDefinition.cs - Has full interface/class documentation - [x] RedSyntaxNode.cs - Has code and - [x] Trivia.cs - Has comprehensive , , - Template used consistently: , , , - -[x] 3.5 Remove Obsolete APIs (30 min) ⚠️ BREAKING ✓ DONE - File: Ast/SyntaxTree.cs - [x] Remove [Obsolete] attribute from ToFullString() - [x] Remove ToFullString() method entirely - [x] Search for and update any internal usages (none found) - [x] All 1,195 tests pass - [ ] Document removal in CHANGELOG (at release time) - -[ ] 3.6 Phase 3 Release - [ ] Run full test suite - [ ] Verify API documentation is complete - [ ] Write migration guide for removed/changed APIs - [ ] Update CHANGELOG.md with all changes and breaking changes - [ ] Bump version to 0.8.0 in TinyTokenizer.csproj - [ ] Commit and tag as v0.8.0 - [ ] Create GitHub release with migration notes - [ ] Verify NuGet package published via CI - -═══════════════════════════════════════════════════════════════════════════════ -PHASE 4: Testing (Ongoing) -Timeline: Continuous -═══════════════════════════════════════════════════════════════════════════════ - -[ ] 4.1 Error Recovery Test Suite (4 hrs) - File: new TinyTokenizer.Tests/ErrorRecoveryTests.cs - [x] Create ErrorRecoveryTests.cs file - [x] Unclosed block tests - [x] Test "{" produces ErrorToken - [x] Test "[" produces ErrorToken - [x] Test "(" produces ErrorToken - [x] Test "{ { }" (nested unclosed) produces ErrorToken - [x] Mismatched delimiter tests - [x] Test "{]" produces ErrorToken - [x] Test "[)" produces ErrorToken - [x] Test "(}" produces ErrorToken - [x] Unclosed string tests - [x] Test "\"hello" produces ErrorToken or Symbol - [x] Test "'world" produces ErrorToken or Symbol - [x] Test "\"test\nmore\"" handles newline in string - [x] Recovery tests - [x] Test tokenizer continues after error token - -[ ] 4.2 Unicode Test Suite (3 hrs) - File: new TinyTokenizer.Tests/UnicodeTests.cs - [x] Create UnicodeTests.cs file - [x] Unicode identifier tests - [x] Test "変数" (Japanese) recognized as identifier - [x] Test "переменная" (Cyrillic) recognized as identifier - [x] Test "משתנה" (Hebrew RTL) recognized as identifier - [x] Emoji tests - [x] Test "🚀rocket" (emoji prefix) handled correctly - [x] Test "var_🎉" (emoji suffix) handled correctly - [x] Invisible character tests - [x] Test "\u200B" (zero-width space) handled - [x] Test "\uFEFF" (BOM) handled - -[x] 4.3 Numeric Edge Cases Tests (2 hrs) ✓ DONE - File: TinyTokenizer.Tests/LexerParserTests.cs (extend existing) - [x] Add numeric edge case theory tests - [x] Test "0" → Integer - [x] Test "0.0" → FloatingPoint - [x] Test ".0" → FloatingPoint - [x] Test "0." → Integer + Dot (trailing dot separate) - [x] Test "00123" → Integer (leading zeros) - [x] Test "1.2.3" → 1.2 FloatingPoint + .3 FloatingPoint (greedy parsing) - -[x] 4.4 Thread Safety Tests (3 hrs) ✓ DONE - File: new TinyTokenizer.Tests/ConcurrencyTests.cs - [x] Create ConcurrencyTests.cs file - [x] Concurrent RedNode access test - [x] Parse a tree once - [x] Spawn 100 tasks accessing Root.Children - [x] Each task accesses SiblingIndex, NextSibling() - [x] Verify no exceptions thrown - [x] Concurrent tree parsing test - [x] Spawn 100 tasks each parsing different input - [x] Await all tasks - [x] Verify all trees have valid Root - -[x] 4.5 Performance Benchmarks (4 hrs) ✓ DONE - File: TinyTokenizer.Benchmarks/ - [x] Create AllocationBenchmarks.cs - [x] Add [MemoryDiagnoser] attribute - [x] Setup small input (~1 KB) - [x] Setup medium input (~100 KB) - [x] Setup large input (~1 MB) - [x] Add ParseSmall benchmark - [x] Add ParseMedium benchmark - [x] Add ParseLarge benchmark - [x] Create OperatorMatchingBenchmarks.cs - [x] Add [MemoryDiagnoser] attribute - [x] Add [Params(10, 50, 100)] OperatorCount - [x] Add MatchOperators benchmark - [x] Create SyntaxTreeBenchmarks.cs - [x] Tree parsing benchmarks (small/medium/large/nested) - [x] Schema integration benchmarks - [x] Red node creation and traversal benchmarks - [x] TreeWalker benchmarks - [x] Query/filter benchmarks - [x] Sibling navigation benchmarks - [x] Edit/mutation benchmarks - -═══════════════════════════════════════════════════════════════════════════════ -DEPENDENCIES -═══════════════════════════════════════════════════════════════════════════════ - -[x] Add CommunityToolkit.HighPerformance 8.2.2 (Phase 2.1) ✓ DONE - Purpose: ArrayPoolBufferWriter for zero-allocation parsing - -═══════════════════════════════════════════════════════════════════════════════ -SUCCESS METRICS -═══════════════════════════════════════════════════════════════════════════════ - -[ ] Allocation reduction: >50% fewer allocations in parsing benchmarks -[ ] API consistency: All public types implement IFormattable where applicable -[ ] Test coverage: >85% line coverage on core parsing logic -[ ] Documentation: 100% XML doc coverage on public APIs -[ ] Performance: No regression in existing benchmarks - -═══════════════════════════════════════════════════════════════════════════════ -RISK ASSESSMENT -═══════════════════════════════════════════════════════════════════════════════ - -Risk | Likelihood | Impact | Mitigation -------------------------------|------------|--------|-------------------------------- -Breaking change regression | Medium | High | Comprehensive test suite -Performance regression | Low | Medium | Benchmark before/after each phase -Memory leak from pooling | Low | High | Memory profiler, dispose patterns -Thread safety issues | Medium | High | Dedicated concurrency tests