Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions TinyTokenizer.E2ETests/GlslEditorTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,23 @@ public void DumpTokenTree()
_output.WriteLine(dump);
Assert.NotNull(dump);
}

[Fact]
public void Editor_Between_MethodBodyStartEnd_Replace_WithNonDelimitedText()
{
var schema = CreateGlslSchema();
var tree = SyntaxTree.Parse("void main() { return 0.0; }", schema);

var editor = tree.CreateEditor();

var functionName = "main";
var methodBody = Query.Syntax<GlFunctionNode>().Named(functionName).Block("body");
var bodyContents = Query.Between(methodBody.Start(), methodBody.End());

editor.Replace(bodyContents, "\nreturn 1.0;");

editor.Commit();
}

[Fact]
public void Parse_RecognizesGlslFunctions()
Expand Down
41 changes: 27 additions & 14 deletions TinyTokenizer.Tests/QueryCombinatorTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -1294,21 +1294,34 @@ public void BOF_MatchesOnlyAtRootLevel()
#region Edge Case Tests - Between

[Fact]
public void Between_Exclusive_DoesNotCountDelimiters()
public void Between_Default_IsExclusive_ForEditorRegions()
{
var tree = Parse("< a b c >");
var root = tree.Root;
var children = root.Children.ToList();
var openAngle = children.First(c => c.ToText().Trim() == "<");

var queryInclusive = Query.Between(Query.Operator("<"), Query.Operator(">"), inclusive: true);
var queryExclusive = Query.Between(Query.Operator("<"), Query.Operator(">"), inclusive: false);

Assert.True(queryInclusive.TryMatch(openAngle, out var consumedInclusive));
Assert.True(queryExclusive.TryMatch(openAngle, out var consumedExclusive));

// Exclusive should report fewer consumed (just the content, not delimiters)
Assert.True(consumedExclusive < consumedInclusive);
var tree = Parse("before < a b c > after");

// Default Between should replace ONLY the content between delimiters.
tree.CreateEditor()
.Replace(Query.Between(Query.Operator("<"), Query.Operator(">")), " CONTENT ")
.Commit();

var text = tree.ToText();
Assert.Contains("<", text);
Assert.Contains(">", text);
Assert.Matches(@"<\s*CONTENT\s*>", text);
}

[Fact]
public void Between_Inclusive_ReplacesDelimitersToo()
{
var tree = Parse("before < a b c > after");

tree.CreateEditor()
.Replace(Query.Between(Query.Operator("<"), Query.Operator(">"), inclusive: true), " CONTENT ")
.Commit();

var text = tree.ToText();
Assert.DoesNotContain("<", text);
Assert.DoesNotContain(">", text);
Assert.Contains("CONTENT", text);
}

[Fact]
Expand Down
45 changes: 39 additions & 6 deletions TinyTokenizer.Tests/SyntaxEditorTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -935,7 +935,7 @@ public void NamedBlockInner_Replace_EmptyBlock_InsertsContent()

Assert.Equal("f{inserted}", tree.ToText());
}

#endregion

#region Function-Like Block Insertion Scenarios
Expand Down Expand Up @@ -3006,7 +3006,7 @@ public void Replace_QueryBetween_ReplacesEntireRange()

// Act: replace everything from 'a' to 'c' (inclusive)
tree.CreateEditor()
.Replace(Q.Between(Q.Ident("a"), Q.Ident("c")), "REPLACED")
.Replace(Q.Between(Q.Ident("a"), Q.Ident("c"), inclusive: true), "REPLACED")
.Commit();

// Assert: the range [a, b, c] should be replaced with REPLACED
Expand All @@ -3022,7 +3022,7 @@ public void Remove_QueryBetween_RemovesEntireRange()
var tree = SyntaxTree.Parse("start a b c end");

tree.CreateEditor()
.Remove(Q.Between(Q.Ident("a"), Q.Ident("c")))
.Remove(Q.Between(Q.Ident("a"), Q.Ident("c"), inclusive: true))
.Commit();

// Trivia handling: leading trivia of 'a' (space) is removed with 'a',
Expand All @@ -3039,7 +3039,7 @@ public void InsertBefore_QueryBetween_InsertsBeforeRange()
var tree = SyntaxTree.Parse("x a b c y");

tree.CreateEditor()
.InsertBefore(Q.Between(Q.Ident("a"), Q.Ident("c")), "BEFORE ")
.InsertBefore(Q.Between(Q.Ident("a"), Q.Ident("c"), inclusive: true), "BEFORE ")
.Commit();

Assert.Equal("x BEFORE a b c y", tree.ToText());
Expand All @@ -3061,7 +3061,7 @@ public void InsertAfter_QueryBetween_InsertsAfterRange()
// - Insert "AFTER " (with trailing space for separation from y)
// - y has no trivia -> "y"
tree.CreateEditor()
.InsertAfter(Q.Between(Q.Ident("a"), Q.Ident("c")), "AFTER ")
.InsertAfter(Q.Between(Q.Ident("a"), Q.Ident("c"), inclusive: true), "AFTER ")
.Commit();

Assert.Equal("x a b c AFTER y", tree.ToText());
Expand Down Expand Up @@ -3105,7 +3105,7 @@ public void Edit_QueryBetween_TransformsRangeContent()
var tree = SyntaxTree.Parse("x abc def ghi y");

tree.CreateEditor()
.Edit(Q.Between(Q.Ident("abc"), Q.Ident("ghi")), content => content.ToUpper())
.Edit(Q.Between(Q.Ident("abc"), Q.Ident("ghi"), inclusive: true), content => content.ToUpper())
.Commit();

// The content between abc and ghi (inclusive) should be uppercased
Expand All @@ -3130,4 +3130,37 @@ public void Replace_QuerySequence_ReplacesAllMatchedNodes()
}

#endregion

#region Query.Wrap / Query.Inner (BlockNode)

[Fact]
public void Replace_QueryWrapBlock_Inner_ReplacesOnlyThatBlock()
{
var tree = SyntaxTree.Parse("{a}{b}");
var blocks = tree.Select(Query.BraceBlock).OfType<SyntaxBlock>().ToList();
Assert.Equal(2, blocks.Count);

var second = blocks[1];

tree.CreateEditor()
.Replace(Query.Wrap(second).Inner(), "c")
.Commit();

Assert.Equal("{a}{c}", tree.ToText());
}

[Fact]
public void Replace_QueryInnerBlock_ReplacesOnlyThatBlock()
{
var tree = SyntaxTree.Parse("{a}{b}");
var second = tree.Select(Query.BraceBlock).OfType<SyntaxBlock>().Skip(1).First();

tree.CreateEditor()
.Replace(Query.Inner(second), "c")
.Commit();

Assert.Equal("{a}{c}", tree.ToText());
}

#endregion
}
51 changes: 48 additions & 3 deletions TinyTokenizer/Ast/Query.cs
Original file line number Diff line number Diff line change
Expand Up @@ -189,19 +189,19 @@ public static class Query

/// <summary>
/// Creates a query that matches content between a start and end pattern.
/// Consumes all nodes from start through end (inclusive).
/// By default, the matched region excludes the start/end delimiters.
/// </summary>
/// <param name="start">The starting delimiter/pattern.</param>
/// <param name="end">The ending delimiter/pattern.</param>
/// <param name="inclusive">If true (default), includes start/end in consumed count.</param>
/// <param name="inclusive">If true, includes start/end delimiters in the matched region.</param>
/// <returns>A query matching the content between start and end.</returns>
/// <example>
/// <code>
/// // Match content between parentheses
/// Query.Between(Query.Symbol("("), Query.Symbol(")"))
/// </code>
/// </example>
public static BetweenQuery Between(INodeQuery start, INodeQuery end, bool inclusive = true) =>
public static BetweenQuery Between(INodeQuery start, INodeQuery end, bool inclusive = false) =>
new BetweenQuery(start, end, inclusive);

#endregion
Expand Down Expand Up @@ -309,6 +309,51 @@ public static BetweenQuery Between(INodeQuery start, INodeQuery end, bool inclus
/// </example>
public static ExactNodeQuery Exact(SyntaxNode node) => new ExactNodeQuery(node);

#endregion

#region Wrap / Inner (Node-based)

/// <summary>
/// Wraps an existing <see cref="SyntaxBlock"/> instance as a <see cref="BlockNodeQuery"/>.
/// This is useful when you already have a block node and want to use block-specific query helpers
/// like <see cref="BlockNodeQuery.Inner"/>, <see cref="BlockNodeQuery.Start"/>, and <see cref="BlockNodeQuery.End"/>.
/// </summary>
/// <remarks>
/// Like <see cref="Exact"/>, this query is intended for immediate use within the current tree state.
/// Red nodes are recreated on tree mutations.
/// </remarks>
public static BlockNodeQuery Wrap(SyntaxBlock block)
{
ArgumentNullException.ThrowIfNull(block);

// Match the block by red-node equality (same green node + position).
// Also constrain by opener for fast rejection.
return new BlockNodeQuery(block.Opener).Where(n => n == block);
}

/// <summary>
/// Wraps an existing node instance as a query.
/// If the node is a <see cref="SyntaxBlock"/>, returns a <see cref="BlockNodeQuery"/>;
/// otherwise returns an <see cref="ExactNodeQuery"/>.
/// </summary>
/// <remarks>
/// Like <see cref="Exact"/>, this matches the specific node instance and is intended for immediate use.
/// </remarks>
public static INodeQuery Wrap(SyntaxNode node)
{
ArgumentNullException.ThrowIfNull(node);
return node is SyntaxBlock block ? Wrap(block) : Exact(node);
}

/// <summary>
/// Creates a query selecting the inner content region of an existing block node.
/// Equivalent to <c>Query.Wrap(block).Inner()</c>.
/// </summary>
/// <remarks>
/// This query matches the current node instance and is intended for immediate use.
/// </remarks>
public static InnerContentQuery Inner(SyntaxBlock block) => Wrap(block).Inner();

#endregion

#region Keyword Queries
Expand Down
81 changes: 72 additions & 9 deletions TinyTokenizer/Ast/QueryCombinators.cs
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,8 @@ bool IGreenNodeQuery.TryMatchGreen(IReadOnlyList<GreenNode> siblings, int startI

/// <summary>
/// Matches and captures content between a start and end query/delimiter.
/// Consumes all nodes from start through end (inclusive of delimiters).
/// Matches a contiguous region between a start and end pattern.
/// By default, the matched region excludes the start/end delimiters.
/// </summary>
/// <remarks>
/// Useful for extracting content between matching patterns:
Expand All @@ -95,8 +96,8 @@ public sealed record BetweenQuery : INodeQuery, IGreenNodeQuery, IRegionQuery, I
/// <summary>Creates a query matching content between start and end.</summary>
/// <param name="start">The starting delimiter/pattern.</param>
/// <param name="end">The ending delimiter/pattern.</param>
/// <param name="inclusive">If true, includes start/end in consumed count; if false, only content between.</param>
public BetweenQuery(INodeQuery start, INodeQuery end, bool inclusive = true)
/// <param name="inclusive">If true, includes the start/end delimiters in the matched region.</param>
public BetweenQuery(INodeQuery start, INodeQuery end, bool inclusive = false)
{
_start = start;
_end = end;
Expand Down Expand Up @@ -156,7 +157,11 @@ public bool TryMatch(SyntaxNode startNode, out int consumedCount)
if (_end.TryMatch(current, out var endConsumed))
{
totalConsumed += endConsumed;
consumedCount = _inclusive ? totalConsumed : totalConsumed - startConsumed - endConsumed;
// Always report the full span consumed (start..end, inclusive) so this query
// is safe to use in sequences and other sibling-consuming contexts.
// The inclusive/exclusive behavior is applied when translating this query
// into edit regions via IRegionQuery.
consumedCount = totalConsumed;
return true;
}

Expand Down Expand Up @@ -196,7 +201,9 @@ bool IGreenNodeQuery.TryMatchGreen(IReadOnlyList<GreenNode> siblings, int startI
if (endGreen.TryMatchGreen(siblings, currentIndex, out var endConsumed))
{
totalConsumed += endConsumed;
consumedCount = _inclusive ? totalConsumed : totalConsumed - startConsumed - endConsumed;
// Always report the full span consumed (start..end, inclusive).
// Inclusive/exclusive is handled at the region-translation layer.
consumedCount = totalConsumed;
return true;
}

Expand All @@ -209,13 +216,69 @@ bool IGreenNodeQuery.TryMatchGreen(IReadOnlyList<GreenNode> siblings, int startI
}

/// <inheritdoc/>
IEnumerable<QueryRegion> IRegionQuery.SelectRegions(SyntaxTree tree)
IEnumerable<QueryRegion> IRegionQuery.SelectRegions(SyntaxTree tree)
=> ((IRegionQuery)this).SelectRegions(tree.Root);

/// <inheritdoc/>

IEnumerable<QueryRegion> IRegionQuery.SelectRegions(SyntaxNode root)
{
return RegionTraversal.SelectRegions(root, TryMatch);
return RegionTraversal.SelectRegions(root, TryGetBetweenRegion);

bool TryGetBetweenRegion(
SyntaxNode startNode,
out int startSlotOffset,
out int slotCount,
out SyntaxNode? firstNode,
out int position)
{
startSlotOffset = 0;
slotCount = 0;
firstNode = null;
position = 0;

if (!_start.TryMatch(startNode, out var startConsumed))
return false;

// Navigate to the first node AFTER the start match.
var afterStart = startNode;
for (int i = 0; i < startConsumed && afterStart != null; i++)
afterStart = afterStart.NextSibling();

if (afterStart == null)
return false;

// Scan for end starting at afterStart.
var current = afterStart;
int totalConsumed = startConsumed;

while (current != null)
{
if (_end.TryMatch(current, out var endConsumed))
{
totalConsumed += endConsumed;

if (_inclusive)
{
startSlotOffset = 0;
slotCount = totalConsumed;
firstNode = startNode;
position = startNode.Position;
return true;
}

// Exclusive: region starts after the start match and ends before the end delimiter.
startSlotOffset = startConsumed;
slotCount = totalConsumed - startConsumed - endConsumed;
position = afterStart.Position;
firstNode = slotCount > 0 ? afterStart : null;
return true;
}

totalConsumed++;
current = current.NextSibling();
}

return false;
}
}
}

Expand Down
4 changes: 2 additions & 2 deletions TinyTokenizer/Ast/QueryExtensions.cs
Original file line number Diff line number Diff line change
Expand Up @@ -119,9 +119,9 @@ public static NoneOfQuery ExceptFor(this INodeQuery query, params INodeQuery[] o

/// <summary>
/// Creates a query matching content between this query (start) and the end query.
/// Consumes all nodes from start through end (inclusive).
/// By default, the matched region excludes the start/end delimiters.
/// </summary>
public static BetweenQuery Between(this INodeQuery start, INodeQuery end, bool inclusive = true) =>
public static BetweenQuery Between(this INodeQuery start, INodeQuery end, bool inclusive = false) =>
new(start, end, inclusive);

#endregion
Expand Down
Loading
Loading