Skip to content

Commit 2a31999

Browse files
pksorensenpksorensen
committed
fix: empty whitespace solved
1 parent c10a97e commit 2a31999

6 files changed

Lines changed: 359 additions & 8 deletions

File tree

‎src/Compo.Test/ASTSerializationTest.cs‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
using System.Text.Json;
2+
using Compo.Serialization;
3+
4+
namespace Compo.Test;
5+
6+
public class ASTSerializationTest
7+
{
8+
[Fact]
9+
public void AST_ShouldPreserveSpaceSeparator()
10+
{
11+
var engine = new ExpressionParser();
12+
var expression = "@split(payload()['category'], ' ')";
13+
var ast = engine.BuildAst(expression);
14+
15+
ast.Success.Should().BeTrue();
16+
17+
// Serialize the AST
18+
var serializer = new AstSerializer();
19+
var json = serializer.Serialize(ast.Value!);
20+
21+
Console.WriteLine($"Serialized AST: {json}");
22+
23+
// Check that the space is preserved in the JSON
24+
json.Should().Contain("\"value\":\" \"", "the space separator should be preserved in the AST");
25+
26+
// Deserialize and verify
27+
var deserialized = serializer.Deserialize(json);
28+
var functionNode = deserialized as FunctionNode;
29+
functionNode.Should().NotBeNull();
30+
functionNode!.Function.Should().Be("split");
31+
functionNode.Arguments.Should().HaveCount(2);
32+
33+
var secondArg = functionNode.Arguments[1] as ValueNode<string>;
34+
secondArg.Should().NotBeNull();
35+
secondArg!.Value.Should().Be(" ", "the space separator should be preserved after deserialization");
36+
}
37+
38+
[Fact]
39+
public void AST_ComplexExpression_ShouldPreserveSpaceSeparator()
40+
{
41+
var engine = new ExpressionParser();
42+
var expression = "@lookup('product','product_key', split(payload()['product_code'],' ')[0],split(payload()['product_code'],' ')[1],split(payload()['product_code'],' ')[2])";
43+
var ast = engine.BuildAst(expression);
44+
45+
ast.Success.Should().BeTrue();
46+
47+
// Serialize the AST
48+
var serializer = new AstSerializer();
49+
var json = serializer.Serialize(ast.Value!);
50+
51+
Console.WriteLine($"Expression: {expression}");
52+
Console.WriteLine($"Serialized AST: {json}");
53+
54+
// Check that ALL space separators are preserved
55+
var spaceCount = json.Split(new[] { "\"value\":\" \"" }, StringSplitOptions.None).Length - 1;
56+
spaceCount.Should().Be(3, "there should be 3 space separators in the expression");
57+
}
58+
}

‎src/Compo.Test/Functions/SplitArrayAccessTest.cs‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
using Microsoft.Extensions.DependencyInjection;
2+
3+
namespace Compo.Test.Functions;
4+
5+
public class SplitArrayAccessTest
6+
{
7+
[Fact]
8+
public void Split_ShouldAllowIndexAccess_ToAllElements()
9+
{
10+
var services = new ServiceCollection();
11+
services.AddLogging();
12+
services.DiscoverFunctions();
13+
services.AddSingleton<IExpressionEvaluator, ExpressionEvaluator>();
14+
var serviceProvider = services.BuildServiceProvider();
15+
var evaluator = serviceProvider.GetRequiredService<IExpressionEvaluator>();
16+
var engine = new ExpressionParser();
17+
18+
// Test accessing all three elements like in the real use case
19+
var expr0 = "@split('one two tree', ' ')[0]";
20+
var expr1 = "@split('one two tree', ' ')[1]";
21+
var expr2 = "@split('one two tree', ' ')[2]";
22+
23+
var result0 = evaluator.Evaluate(engine.BuildAst(expr0).Value!);
24+
var result1 = evaluator.Evaluate(engine.BuildAst(expr1).Value!);
25+
var result2 = evaluator.Evaluate(engine.BuildAst(expr2).Value!);
26+
27+
result0.Should().Be("one");
28+
result1.Should().Be("two");
29+
result2.Should().Be("tree");
30+
}
31+
32+
[Fact]
33+
public void Split_WithPayload_ShouldAllowIndexAccess()
34+
{
35+
var services = new ServiceCollection();
36+
services.AddLogging();
37+
38+
// Register a payload function
39+
services.RegisterFunction<PayloadFunction>("payload");
40+
services.DiscoverFunctions();
41+
services.AddSingleton<IExpressionEvaluator, ExpressionEvaluator>();
42+
var serviceProvider = services.BuildServiceProvider();
43+
var evaluator = serviceProvider.GetRequiredService<IExpressionEvaluator>();
44+
var engine = new ExpressionParser();
45+
46+
// Test use case with payload field access
47+
var expr0 = "@split(payload()['product_code'], ' ')[0]";
48+
var expr1 = "@split(payload()['product_code'], ' ')[1]";
49+
var expr2 = "@split(payload()['product_code'], ' ')[2]";
50+
51+
var result0 = evaluator.Evaluate(engine.BuildAst(expr0).Value!);
52+
var result1 = evaluator.Evaluate(engine.BuildAst(expr1).Value!);
53+
var result2 = evaluator.Evaluate(engine.BuildAst(expr2).Value!);
54+
55+
result0.Should().Be("one");
56+
result1.Should().Be("two");
57+
result2.Should().Be("tree");
58+
}
59+
60+
public class PayloadFunction : IFunction<IDictionary<string, object>>
61+
{
62+
public IDictionary<string, object> Execute()
63+
{
64+
return new Dictionary<string, object>
65+
{
66+
{ "product_code", "one two tree" }
67+
};
68+
}
69+
}
70+
}

‎src/Compo.Test/Functions/SplitFunctionTest.cs‎

Lines changed: 185 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,185 @@
1+
using Microsoft.Extensions.DependencyInjection;
2+
3+
namespace Compo.Test.Functions;
4+
5+
public class SplitFunctionTest
6+
{
7+
private static IExpressionEvaluator CreateEvaluator()
8+
{
9+
var services = new ServiceCollection();
10+
services.AddLogging();
11+
services.DiscoverFunctions();
12+
services.AddSingleton<IExpressionEvaluator, ExpressionEvaluator>();
13+
var serviceProvider = services.BuildServiceProvider();
14+
return serviceProvider.GetRequiredService<IExpressionEvaluator>();
15+
}
16+
17+
/// <summary>
18+
/// Function that returns a dictionary with strings to split
19+
/// </summary>
20+
public class DataFunction : IFunction<IDictionary<string, object>>
21+
{
22+
private readonly IDictionary<string, object> _data = new Dictionary<string, object>
23+
{
24+
{ "spaceSeparated", "one two three" },
25+
{ "commaSeparated", "apple,banana,orange" },
26+
{ "pipeSeparated", "a|b|c|d" },
27+
{ "separator", " " },
28+
};
29+
30+
public IDictionary<string, object> Execute()
31+
{
32+
return _data;
33+
}
34+
}
35+
36+
[Theory]
37+
[InlineData("@split('apple,banana,orange', ',')", new[] { "apple", "banana", "orange" })]
38+
[InlineData("@split('a|b|c|d', '|')", new[] { "a", "b", "c", "d" })]
39+
[InlineData("@split('single', ',')", new[] { "single" })]
40+
[InlineData("@split('one two tree', ' ')", new[] { "one", "two", "tree" })]
41+
public void Split_ShouldSplitStringBySeparator(string expression, string[] expected)
42+
{
43+
var services = new ServiceCollection();
44+
services.AddLogging();
45+
services.DiscoverFunctions();
46+
services.AddSingleton<IExpressionEvaluator, ExpressionEvaluator>();
47+
var serviceProvider = services.BuildServiceProvider();
48+
var evaluator = serviceProvider.GetRequiredService<IExpressionEvaluator>();
49+
var engine = new ExpressionParser();
50+
51+
var result = evaluator.Evaluate(engine.BuildAst(expression).Value!);
52+
53+
result.Should().BeAssignableTo<IEnumerable<object>>();
54+
var actual = ((IEnumerable<object>)result!).Cast<string>().ToArray();
55+
actual.Should().BeEquivalentTo(expected);
56+
}
57+
58+
[Theory]
59+
[InlineData("@split(data()['commaSeparated'], ',')[0]", "apple")]
60+
[InlineData("@split(data()['commaSeparated'], ',')[1]", "banana")]
61+
[InlineData("@split(data()['commaSeparated'], ',')[2]", "orange")]
62+
public void Split_ShouldAccessArrayElementsWithComma(string expression, string expected)
63+
{
64+
var services = new ServiceCollection();
65+
services.AddLogging();
66+
services.RegisterFunction<DataFunction>("data");
67+
services.DiscoverFunctions();
68+
services.AddSingleton<IExpressionEvaluator, ExpressionEvaluator>();
69+
var serviceProvider = services.BuildServiceProvider();
70+
var evaluator = serviceProvider.GetRequiredService<IExpressionEvaluator>();
71+
var engine = new ExpressionParser();
72+
73+
var result = evaluator.Evaluate(engine.BuildAst(expression).Value!);
74+
75+
result.Should().Be(expected);
76+
}
77+
78+
[Theory]
79+
[InlineData("@split(data()['spaceSeparated'], data()['separator'])[0]", "one")]
80+
[InlineData("@split(data()['spaceSeparated'], data()['separator'])[1]", "two")]
81+
[InlineData("@split(data()['spaceSeparated'], data()['separator'])[2]", "three")]
82+
public void Split_WithSpaceSeparatorFromData_ShouldWork(string expression, string expected)
83+
{
84+
var services = new ServiceCollection();
85+
services.AddLogging();
86+
services.RegisterFunction<DataFunction>("data");
87+
services.DiscoverFunctions();
88+
services.AddSingleton<IExpressionEvaluator, ExpressionEvaluator>();
89+
var serviceProvider = services.BuildServiceProvider();
90+
var evaluator = serviceProvider.GetRequiredService<IExpressionEvaluator>();
91+
var engine = new ExpressionParser();
92+
93+
var result = evaluator.Evaluate(engine.BuildAst(expression).Value!);
94+
95+
result.Should().Be(expected);
96+
}
97+
98+
[Fact]
99+
public void Split_WithEmptyString_ShouldReturnEmptyArray()
100+
{
101+
var evaluator = CreateEvaluator();
102+
var engine = new ExpressionParser();
103+
104+
var expression = "@split('', ',')";
105+
var result = evaluator.Evaluate(engine.BuildAst(expression).Value!);
106+
107+
result.Should().BeAssignableTo<IEnumerable<object>>();
108+
var actual = ((IEnumerable<object>)result!).ToArray();
109+
actual.Should().BeEmpty();
110+
}
111+
112+
[Theory]
113+
[InlineData("@split('a,,b,,c', ',', false)", new[] { "a", "", "b", "", "c" })]
114+
[InlineData("@split('a,,b,,c', ',', true)", new[] { "a", "b", "c" })]
115+
public void Split_WithRemoveEmptyEntries_ShouldHandleEmptyValues(string expression, string[] expected)
116+
{
117+
var evaluator = CreateEvaluator();
118+
var engine = new ExpressionParser();
119+
120+
var result = evaluator.Evaluate(engine.BuildAst(expression).Value!);
121+
122+
result.Should().BeAssignableTo<IEnumerable<object>>();
123+
var actual = ((IEnumerable<object>)result!).Cast<string>().ToArray();
124+
actual.Should().BeEquivalentTo(expected);
125+
}
126+
127+
[Fact]
128+
public void Split_WithMultiCharacterSeparator_ShouldWork()
129+
{
130+
var evaluator = CreateEvaluator();
131+
var engine = new ExpressionParser();
132+
133+
var expression = "@split('apple::banana::orange', '::')";
134+
var result = evaluator.Evaluate(engine.BuildAst(expression).Value!);
135+
136+
result.Should().BeAssignableTo<IEnumerable<object>>();
137+
var actual = ((IEnumerable<object>)result!).Cast<string>().ToArray();
138+
actual.Should().BeEquivalentTo(new[] { "apple", "banana", "orange" });
139+
}
140+
141+
[Fact]
142+
public void Split_WithEmptySeparator_ShouldReturnSingleElement()
143+
{
144+
var evaluator = CreateEvaluator();
145+
var engine = new ExpressionParser();
146+
147+
// Empty separator returns the whole string as single element (standard .NET behavior)
148+
var expression = "@split('test', '')";
149+
var result = evaluator.Evaluate(engine.BuildAst(expression).Value!);
150+
151+
result.Should().BeAssignableTo<IEnumerable<object>>();
152+
var actual = ((IEnumerable<object>)result!).Cast<string>().ToArray();
153+
actual.Should().BeEquivalentTo(new[] { "test" });
154+
}
155+
156+
/// <summary>
157+
/// Test the use case from the original requirement - splitting and accessing multiple parts
158+
/// This simulates: split(payload()['field'],' ')[0], split(...)[1], split(...)[2]
159+
/// </summary>
160+
[Fact]
161+
public void Split_RealWorldExample_ShouldAccessMultipleParts()
162+
{
163+
var services = new ServiceCollection();
164+
services.AddLogging();
165+
services.RegisterFunction<DataFunction>("data");
166+
services.DiscoverFunctions();
167+
services.AddSingleton<IExpressionEvaluator, ExpressionEvaluator>();
168+
var serviceProvider = services.BuildServiceProvider();
169+
var evaluator = serviceProvider.GetRequiredService<IExpressionEvaluator>();
170+
var engine = new ExpressionParser();
171+
172+
// Using comma separator (space separators from literals are trimmed by parser)
173+
var expr0 = "@split('part1,part2,part3', ',')[0]";
174+
var expr1 = "@split('part1,part2,part3', ',')[1]";
175+
var expr2 = "@split('part1,part2,part3', ',')[2]";
176+
177+
var result0 = evaluator.Evaluate(engine.BuildAst(expr0).Value!);
178+
var result1 = evaluator.Evaluate(engine.BuildAst(expr1).Value!);
179+
var result2 = evaluator.Evaluate(engine.BuildAst(expr2).Value!);
180+
181+
result0.Should().Be("part1");
182+
result1.Should().Be("part2");
183+
result2.Should().Be("part3");
184+
}
185+
}

‎src/Compo.Test/PayloadNullHandlingTest.cs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -151,8 +151,8 @@ public void DateTimeConversion_WithValidFormat_ShouldWork(string expression)
151151
[Fact]
152152
public void DateTimeConversion_WithInvalidFormat_ShouldThrow()
153153
{
154-
// Replicates the production error:
155-
// failedFields: { bd_dateofconsent: {
154+
// Demonstrates error when datetime conversion fails:
155+
// failedFields: { consent_date: {
156156
// message: 'Exception has been thrown by the target of an invocation.',
157157
// innerMessage: 'Cannot convert '2025-03-14-13.20.24.208783' to DateTime'
158158
// }}
@@ -247,7 +247,7 @@ public void DictionaryAccess_WithValidKey_ShouldReturnMappedValue(string express
247247
public void DictionaryAccess_WithDynamicKey_ShouldReturnMappedValue(string expression, int expected)
248248
{
249249
// Test dictionary access with dynamic keys from payload
250-
// This simulates: @picklist(payload()['bd_type'], {'Work email': 121140000, 'Private email': 121140001})
250+
// This simulates: @picklist(payload()['email_type'], {'Work email': 121140000, 'Private email': 121140001})
251251
var services = new ServiceCollection();
252252
services.AddLogging();
253253
services.RegisterFunction<PayloadFunction>("payload");

‎src/Compo/Core/ExpressionParser.cs‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ static Parser<char, T> Tok<T>(Parser<char, T> p)
1818
private static readonly Parser<char, char> OpenBracket = Tok('[');
1919
private static readonly Parser<char, char> CloseBracket = Tok(']');
2020
private static readonly Parser<char, char> Dot = Tok('.');
21-
private static readonly Parser<char, char> Quote = Tok('\'');
21+
private static readonly Parser<char, char> Quote = Char('\''); // Don't skip whitespace after quotes in string literals
2222

2323
#region PidingPaste
2424

@@ -99,10 +99,11 @@ public ExpressionParser()
9999
access.Aggregate((Node)new FunctionNode(functionName, args.Where(x => x != null!).ToList()),
100100
(node, node1) => new AccessNode(node, node1.b, node1.a.HasValue)),
101101
AnyCharExcept('(', ']', '[').ManyString().Before(OpenParen),
102-
Try(Terminal)
103-
.Or(Try(Rec(() => _function)))
104-
.Or(SkipWhitespaces.Select<Node>(_ => null!))
105-
.Between(Whitespaces.IgnoreResult())
102+
SkipWhitespaces.Then(
103+
Try(Terminal)
104+
.Or(Try(Rec(() => _function)))
105+
.Or(SkipWhitespaces.Select<Node>(_ => null!))
106+
).Before(SkipWhitespaces)
106107
.Separated(Comma),
107108
CloseParen.Then(
108109
Map(

‎src/Compo/Functions/String/SplitFunction.cs‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
namespace Compo.Functions.String;
2+
3+
/// <summary>
4+
/// Splits a string into an array of substrings based on a separator.
5+
/// split(string, separator) - splits on separator with default options
6+
/// split(string, separator, removeEmptyEntries) - optionally removes empty entries
7+
///
8+
/// Examples:
9+
/// - split('one two three', ' ') returns ["one", "two", "three"]
10+
/// - split('a,b,c', ',') returns ["a", "b", "c"]
11+
/// - split('a,,b,,c', ',', true) returns ["a", "b", "c"] (empty entries removed)
12+
/// </summary>
13+
[FunctionRegistration("split")]
14+
public class SplitFunction :
15+
IFunction<string, string, IEnumerable<string>>,
16+
IFunction<string, string, bool, IEnumerable<string>>
17+
{
18+
public IEnumerable<string> Execute(string text, string separator)
19+
{
20+
return Execute(text, separator, false);
21+
}
22+
23+
public IEnumerable<string> Execute(string text, string separator, bool removeEmptyEntries)
24+
{
25+
if (string.IsNullOrEmpty(text))
26+
return Array.Empty<string>();
27+
28+
if (separator == null)
29+
throw new ArgumentNullException(nameof(separator), "Separator cannot be null");
30+
31+
var options = removeEmptyEntries
32+
? StringSplitOptions.RemoveEmptyEntries
33+
: StringSplitOptions.None;
34+
35+
return text.Split(new[] { separator }, options);
36+
}
37+
}

0 commit comments

Comments
 (0)