From 1e93323fe77fc751590acc367e48d9294485f03b Mon Sep 17 00:00:00 2001 From: Callum Sykes Date: Sun, 17 May 2026 16:19:07 -0700 Subject: [PATCH 1/3] Fix tool invocation, add NUnit constraints, wire attributes - AgentRunner now wraps the IChatClient in FunctionInvokingChatClient so the tools the model calls are actually executed. Previously the runner recorded FunctionCallContent but nothing fed results back, so against a real LLM the loop would spin until max turns with no progress. - Add Ngentic.NUnit Constraint-based API: Did.CallTool, Did.NotCallTool, Did.CallToolsInOrder, Did.MakeToolCalls. Plays with NUnit's Assert.That, Assert.Multiple, and the rich failure formatter. WithArg/WithArgs lets tests assert on tool arguments, not just the name. - AgenticTestBase now reads [MaxTurns] and [Model] from the current test method in [SetUp] and threads them into AgentBuilder so the attributes do something instead of being silent. - Thread modelId through AgentRunner into ChatOptions.ModelId. - Add ToolCall.CallId so calls and results are paired by id, not by index/best-effort. - New Ngentic.Tests project covering constraints, the legacy Expect DSL, and the runner's tool-invocation + model-id + system-prompt plumbing. --- Ngentic.slnx | 3 + samples/Ngentic.Samples/SampleTest.cs | 9 +- samples/Ngentic.Samples/UnitTest1.cs | 4 + src/Ngentic.Core/AgentRunner.cs | 113 ++++++----- src/Ngentic.Core/ToolCall.cs | 1 + src/Ngentic.NUnit/AgenticTestBase.cs | 49 ++++- src/Ngentic.NUnit/Constraints.cs | 244 +++++++++++++++++++++++ tests/Ngentic.Tests/AgentRunnerTests.cs | 82 ++++++++ tests/Ngentic.Tests/ConstraintTests.cs | 108 ++++++++++ tests/Ngentic.Tests/ExpectTests.cs | 59 ++++++ tests/Ngentic.Tests/FakeChatClient.cs | 56 ++++++ tests/Ngentic.Tests/GlobalUsings.cs | 1 + tests/Ngentic.Tests/Ngentic.Tests.csproj | 25 +++ tests/Ngentic.Tests/TestHelpers.cs | 42 ++++ tests/Ngentic.Tests/UnitTest1.cs | 4 + 15 files changed, 738 insertions(+), 62 deletions(-) create mode 100644 samples/Ngentic.Samples/UnitTest1.cs create mode 100644 src/Ngentic.NUnit/Constraints.cs create mode 100644 tests/Ngentic.Tests/AgentRunnerTests.cs create mode 100644 tests/Ngentic.Tests/ConstraintTests.cs create mode 100644 tests/Ngentic.Tests/ExpectTests.cs create mode 100644 tests/Ngentic.Tests/FakeChatClient.cs create mode 100644 tests/Ngentic.Tests/GlobalUsings.cs create mode 100644 tests/Ngentic.Tests/Ngentic.Tests.csproj create mode 100644 tests/Ngentic.Tests/TestHelpers.cs create mode 100644 tests/Ngentic.Tests/UnitTest1.cs diff --git a/Ngentic.slnx b/Ngentic.slnx index d2c5f7a..fd8f695 100644 --- a/Ngentic.slnx +++ b/Ngentic.slnx @@ -6,4 +6,7 @@ + + + diff --git a/samples/Ngentic.Samples/SampleTest.cs b/samples/Ngentic.Samples/SampleTest.cs index c41e806..4c82b43 100644 --- a/samples/Ngentic.Samples/SampleTest.cs +++ b/samples/Ngentic.Samples/SampleTest.cs @@ -34,16 +34,17 @@ protected override void ConfigureHarness() } [Test] + [MaxTurns(4)] public async Task Agent_uses_calculator_to_add() { AgentRun run = await Agent .WithSystemPrompt("You are a math assistant.") .WithAllowedTools("mcp__calculator__*") - .WithMaxTurns(4) .RunAsync("What is 2 + 3?"); - Expect.That(run).CalledTool("mcp__calculator__add").AtLeastOnce(); - Expect.That(run).HasCount(1); - Expect.That(run.FinalOutput).Contains("5"); + Assert.That(run, Did.CallTool("mcp__calculator__add").WithArg("a", 2).WithArg("b", 3)); + Assert.That(run, Did.NotCallTool("mcp__calculator__divide")); + Assert.That(run, Did.MakeToolCalls.Exactly(1)); + Assert.That(run.FinalOutput, Does.Contain("5")); } } diff --git a/samples/Ngentic.Samples/UnitTest1.cs b/samples/Ngentic.Samples/UnitTest1.cs new file mode 100644 index 0000000..3495d55 --- /dev/null +++ b/samples/Ngentic.Samples/UnitTest1.cs @@ -0,0 +1,4 @@ +// Sample lives in SampleTest.cs. +namespace Ngentic.Samples.Internal; + +internal static class AssemblyMarker { } diff --git a/src/Ngentic.Core/AgentRunner.cs b/src/Ngentic.Core/AgentRunner.cs index 8cba00b..a4844f0 100644 --- a/src/Ngentic.Core/AgentRunner.cs +++ b/src/Ngentic.Core/AgentRunner.cs @@ -7,26 +7,39 @@ namespace Ngentic; public sealed class AgentRunner { - private readonly IChatClient _client; + private readonly IChatClient _innerClient; private readonly IList _tools; private readonly string? _systemPrompt; private readonly int _maxTurns; + private readonly string? _modelId; public AgentRunner( IChatClient client, IList tools, string? systemPrompt = null, - int maxTurns = 16) + int maxTurns = 16, + string? modelId = null) { - _client = client; + _innerClient = client; _tools = tools; _systemPrompt = systemPrompt; _maxTurns = maxTurns; + _modelId = modelId; } public async Task RunAsync(string prompt, CancellationToken ct = default) { Stopwatch stopwatch = Stopwatch.StartNew(); + + // Wrap the caller's IChatClient so MEAI invokes tools on the model's behalf. + // Without this, FunctionCallContent comes back from the model but nothing + // executes the AITool — the loop would spin until max turns with no progress. + FunctionInvokingChatClient functionClient = new FunctionInvokingChatClient(_innerClient) + { + MaximumIterationsPerRequest = _maxTurns, + AllowConcurrentInvocation = false, + }; + List messages = new(); if (!string.IsNullOrEmpty(_systemPrompt)) { @@ -34,81 +47,73 @@ public async Task RunAsync(string prompt, CancellationToken ct = defau } messages.Add(new ChatMessage(ChatRole.User, prompt)); - List toolCalls = new(); ChatOptions options = new() { Tools = _tools, ToolMode = ChatToolMode.Auto, + ModelId = _modelId, }; + ChatResponse response = await functionClient.GetResponseAsync(messages, options, ct); + + List toolCalls = new(); StringBuilder finalText = new(); - int turn = 0; - bool hitLimit = false; - while (turn < _maxTurns) + foreach (ChatMessage msg in response.Messages) { - turn++; - ChatResponse response = await _client.GetResponseAsync(messages, options, ct); - foreach (ChatMessage msg in response.Messages) + foreach (AIContent content in msg.Contents) { - messages.Add(msg); - foreach (AIContent content in msg.Contents) + switch (content) { - switch (content) - { - case FunctionCallContent call: - // Tool invocation is handled by FunctionInvokingChatClient downstream; - // we record the call shape here. Result is filled below. - toolCalls.Add(new ToolCall - { - Name = call.Name, - Arguments = SerializeArgs(call.Arguments), - Result = "", - IsError = false, - }); - break; - case FunctionResultContent result: - int idx = toolCalls.FindLastIndex(t => t.Result == "" && t.Name != ""); - if (idx >= 0) + case FunctionCallContent call: + toolCalls.Add(new ToolCall + { + CallId = call.CallId, + Name = call.Name, + Arguments = SerializeArgs(call.Arguments), + Result = "", + IsError = false, + }); + break; + case FunctionResultContent result: + int idx = toolCalls.FindIndex(t => t.CallId == result.CallId); + if (idx >= 0) + { + ToolCall existing = toolCalls[idx]; + toolCalls[idx] = new ToolCall { - toolCalls[idx] = new ToolCall - { - Name = toolCalls[idx].Name, - Arguments = toolCalls[idx].Arguments, - Result = result.Result?.ToString() ?? "", - IsError = result.Exception != null, - }; - } - break; - case TextContent text: + CallId = existing.CallId, + Name = existing.Name, + Arguments = existing.Arguments, + Result = result.Result?.ToString() ?? "", + IsError = result.Exception != null, + }; + } + break; + case TextContent text: + if (msg.Role == ChatRole.Assistant) + { finalText.Append(text.Text); - break; - } + } + break; } } - - bool moreWork = response.Messages - .SelectMany(m => m.Contents) - .Any(c => c is FunctionCallContent); - - if (!moreWork) - { - break; - } - if (turn >= _maxTurns) - { - hitLimit = true; - } } stopwatch.Stop(); + + int turns = Math.Max(1, response.Messages.Count(m => m.Role == ChatRole.Assistant)); + bool hitLimit = turns >= _maxTurns + && response.Messages.Any(m => m.Contents.OfType().Any() + && !response.Messages.Any(r => r.Contents.OfType().Any())); + return new AgentRun { Prompt = prompt, ToolCalls = toolCalls, FinalOutput = finalText.ToString(), Duration = stopwatch.Elapsed, - Turns = turn, + Turns = turns, HitTurnLimit = hitLimit, }; } diff --git a/src/Ngentic.Core/ToolCall.cs b/src/Ngentic.Core/ToolCall.cs index 878c3d1..51caf6d 100644 --- a/src/Ngentic.Core/ToolCall.cs +++ b/src/Ngentic.Core/ToolCall.cs @@ -4,6 +4,7 @@ namespace Ngentic; public sealed class ToolCall { + public string CallId { get; init; } = ""; public string Name { get; init; } = ""; public JsonElement Arguments { get; init; } public string Result { get; init; } = ""; diff --git a/src/Ngentic.NUnit/AgenticTestBase.cs b/src/Ngentic.NUnit/AgenticTestBase.cs index cdd281e..23d1dd4 100644 --- a/src/Ngentic.NUnit/AgenticTestBase.cs +++ b/src/Ngentic.NUnit/AgenticTestBase.cs @@ -1,6 +1,7 @@ using System.Reflection; using Microsoft.Extensions.AI; using NUnit.Framework; +using NUnit.Framework.Interfaces; namespace Ngentic.NUnit; @@ -8,8 +9,15 @@ public abstract class AgenticTestBase { private IChatClient? _chatClient; private IMcpRegistry? _registry; + private int? _currentMaxTurns; + private string? _currentModel; - protected AgentBuilder Agent => new AgentBuilder(RequireClient(), RequireRegistry(), CollectMcpNames()); + protected AgentBuilder Agent => new AgentBuilder( + RequireClient(), + RequireRegistry(), + CollectMcpNames(), + _currentMaxTurns, + _currentModel); protected void UseClient(IChatClient client) => _chatClient = client; protected void UseRegistry(IMcpRegistry registry) => _registry = registry; @@ -21,6 +29,25 @@ public void __NgenticOneTimeSetUp() VerifyMcpDependencies(); } + [SetUp] + public void __NgenticSetUp() + { + // Pick up [MaxTurns] / [Model] declared on the current test method so the + // attributes I documented are actually load-bearing. + string? methodName = TestContext.CurrentContext.Test.MethodName; + if (methodName == null) + { + _currentMaxTurns = null; + _currentModel = null; + return; + } + + MethodInfo? method = GetType().GetMethod(methodName, + BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance); + _currentMaxTurns = method?.GetCustomAttribute()?.Value; + _currentModel = method?.GetCustomAttribute()?.Id; + } + protected abstract void ConfigureHarness(); private void VerifyMcpDependencies() @@ -73,14 +100,22 @@ public sealed class AgentBuilder private readonly IMcpRegistry _registry; private readonly IReadOnlyList _mcpNames; private string? _systemPrompt; - private int _maxTurns = 16; + private int _maxTurns; + private string? _model; private readonly List _allowedPatterns = new(); - internal AgentBuilder(IChatClient client, IMcpRegistry registry, IReadOnlyList mcpNames) + internal AgentBuilder( + IChatClient client, + IMcpRegistry registry, + IReadOnlyList mcpNames, + int? maxTurnsFromAttribute, + string? modelFromAttribute) { _client = client; _registry = registry; _mcpNames = mcpNames; + _maxTurns = maxTurnsFromAttribute ?? 16; + _model = modelFromAttribute; } public AgentBuilder WithSystemPrompt(string prompt) @@ -95,6 +130,12 @@ public AgentBuilder WithMaxTurns(int turns) return this; } + public AgentBuilder WithModel(string modelId) + { + _model = modelId; + return this; + } + public AgentBuilder WithAllowedTools(params string[] patterns) { _allowedPatterns.AddRange(patterns); @@ -116,7 +157,7 @@ public async Task RunAsync(string prompt, CancellationToken ct = defau } } - AgentRunner runner = new AgentRunner(_client, tools, _systemPrompt, _maxTurns); + AgentRunner runner = new AgentRunner(_client, tools, _systemPrompt, _maxTurns, _model); return await runner.RunAsync(prompt, ct); } diff --git a/src/Ngentic.NUnit/Constraints.cs b/src/Ngentic.NUnit/Constraints.cs new file mode 100644 index 0000000..83a0527 --- /dev/null +++ b/src/Ngentic.NUnit/Constraints.cs @@ -0,0 +1,244 @@ +using System.Text.Json; +using NUnit.Framework.Constraints; + +namespace Ngentic.NUnit; + +/// +/// Entry point for trajectory constraints. Use with NUnit's Assert.That: +/// Assert.That(run, Did.CallTool("mcp__rhino__spawn_slot")); +/// +public static class Did +{ + public static CalledToolConstraint CallTool(string name) => new CalledToolConstraint(name); + public static NotCalledToolConstraint NotCallTool(string name) => new NotCalledToolConstraint(name); + public static CalledInOrderConstraint CallToolsInOrder(params string[] names) => new CalledInOrderConstraint(names); + public static ToolCallCountConstraint MakeToolCalls => new ToolCallCountConstraint(); +} + +public abstract class AgentRunConstraint : Constraint +{ + private string _description = ""; + public override string Description => _description; + protected void SetDescription(string text) => _description = text; + + protected static bool NameMatches(string actual, string pattern) + { + if (pattern.EndsWith('*')) + { + return actual.StartsWith(pattern[..^1], StringComparison.Ordinal); + } + return actual == pattern; + } +} + +public sealed class CalledToolConstraint : AgentRunConstraint +{ + private readonly string _pattern; + private int _expectedCount = -1; + private readonly List> _argChecks = new(); + private readonly List _argDescriptions = new(); + + public CalledToolConstraint(string pattern) + { + _pattern = pattern; + SetDescription($"agent called tool '{pattern}'"); + } + + public CalledToolConstraint AtLeastOnce() + { + _expectedCount = -1; + SetDescription($"agent called tool '{_pattern}' at least once"); + return this; + } + + public CalledToolConstraint Times(int count) + { + _expectedCount = count; + SetDescription($"agent called tool '{_pattern}' exactly {count} time(s)"); + return this; + } + + public CalledToolConstraint WithArg(string name, object expected) + { + _argChecks.Add(args => + { + if (!args.TryGetProperty(name, out JsonElement value)) + { + return false; + } + return JsonEquals(value, expected); + }); + _argDescriptions.Add($"{name}={expected}"); + SetDescription($"agent called tool '{_pattern}' with {string.Join(", ", _argDescriptions)}"); + return this; + } + + public CalledToolConstraint WithArgs(Func predicate, string? describe = null) + { + _argChecks.Add(predicate); + _argDescriptions.Add(describe ?? ""); + SetDescription($"agent called tool '{_pattern}' with {string.Join(", ", _argDescriptions)}"); + return this; + } + + public override ConstraintResult ApplyTo(TActual actual) + { + if (actual is not AgentRun run) + { + return new ConstraintResult(this, actual, ConstraintStatus.Error); + } + + List matchingFully = run.ToolCalls + .Where(c => NameMatches(c.Name, _pattern)) + .Where(c => _argChecks.All(check => check(c.Arguments))) + .ToList(); + + bool success = _expectedCount < 0 + ? matchingFully.Count > 0 + : matchingFully.Count == _expectedCount; + + return new AgentRunConstraintResult(this, run, success); + } + + private static bool JsonEquals(JsonElement value, object expected) + { + return expected switch + { + int i => value.ValueKind == JsonValueKind.Number && value.TryGetInt32(out int v) && v == i, + long l => value.ValueKind == JsonValueKind.Number && value.TryGetInt64(out long v) && v == l, + double d => value.ValueKind == JsonValueKind.Number && Math.Abs(value.GetDouble() - d) < 1e-9, + bool b => value.ValueKind == (b ? JsonValueKind.True : JsonValueKind.False), + string s => value.ValueKind == JsonValueKind.String && value.GetString() == s, + null => value.ValueKind == JsonValueKind.Null, + _ => value.ToString() == expected.ToString(), + }; + } +} + +public sealed class NotCalledToolConstraint : AgentRunConstraint +{ + private readonly string _pattern; + + public NotCalledToolConstraint(string pattern) + { + _pattern = pattern; + SetDescription($"agent did NOT call tool '{pattern}'"); + } + + public override ConstraintResult ApplyTo(TActual actual) + { + if (actual is not AgentRun run) + { + return new ConstraintResult(this, actual, ConstraintStatus.Error); + } + bool success = !run.ToolCalls.Any(c => NameMatches(c.Name, _pattern)); + return new AgentRunConstraintResult(this, run, success); + } +} + +public sealed class CalledInOrderConstraint : AgentRunConstraint +{ + private readonly string[] _names; + + public CalledInOrderConstraint(string[] names) + { + _names = names; + SetDescription($"agent called tools in order: [{string.Join(", ", names)}]"); + } + + public override ConstraintResult ApplyTo(TActual actual) + { + if (actual is not AgentRun run) + { + return new ConstraintResult(this, actual, ConstraintStatus.Error); + } + int idx = 0; + foreach (ToolCall call in run.ToolCalls) + { + if (idx < _names.Length && NameMatches(call.Name, _names[idx])) + { + idx++; + } + } + return new AgentRunConstraintResult(this, run, idx >= _names.Length); + } +} + +public sealed class ToolCallCountConstraint : AgentRunConstraint +{ + private int _min = -1; + private int _max = -1; + private int _exact = -1; + + public ToolCallCountConstraint() + { + SetDescription("agent made tool calls"); + } + + public ToolCallCountConstraint Exactly(int count) + { + _exact = count; + SetDescription($"agent made exactly {count} tool call(s)"); + return this; + } + + public ToolCallCountConstraint AtLeast(int count) + { + _min = count; + SetDescription($"agent made at least {count} tool call(s)"); + return this; + } + + public ToolCallCountConstraint AtMost(int count) + { + _max = count; + SetDescription($"agent made at most {count} tool call(s)"); + return this; + } + + public override ConstraintResult ApplyTo(TActual actual) + { + if (actual is not AgentRun run) + { + return new ConstraintResult(this, actual, ConstraintStatus.Error); + } + int n = run.ToolCalls.Count; + bool ok = true; + if (_exact >= 0) ok &= n == _exact; + if (_min >= 0) ok &= n >= _min; + if (_max >= 0) ok &= n <= _max; + return new AgentRunConstraintResult(this, run, ok); + } +} + +internal sealed class AgentRunConstraintResult : ConstraintResult +{ + private readonly AgentRun _run; + + public AgentRunConstraintResult(IConstraint constraint, AgentRun run, bool success) + : base(constraint, FormatActual(run), success) + { + _run = run; + } + + private static string FormatActual(AgentRun run) + { + if (run.ToolCalls.Count == 0) + { + return "(no tool calls)"; + } + return $"trajectory of {run.ToolCalls.Count} call(s): " + + string.Join(", ", run.ToolCalls.Select(c => c.Name)); + } + + public override void WriteMessageTo(MessageWriter writer) + { + writer.WriteLine($" Expected: {Description}"); + writer.WriteLine($" But was: trajectory of {_run.ToolCalls.Count} call(s)"); + for (int i = 0; i < _run.ToolCalls.Count; i++) + { + ToolCall c = _run.ToolCalls[i]; + writer.WriteLine($" [{i}] {c.Name}({c.Arguments.GetRawText()})"); + } + } +} diff --git a/tests/Ngentic.Tests/AgentRunnerTests.cs b/tests/Ngentic.Tests/AgentRunnerTests.cs new file mode 100644 index 0000000..cb8f10b --- /dev/null +++ b/tests/Ngentic.Tests/AgentRunnerTests.cs @@ -0,0 +1,82 @@ +using Microsoft.Extensions.AI; +using NUnit.Framework; + +namespace Ngentic.Tests; + +[TestFixture] +public sealed class AgentRunnerTests +{ + [Test] + public async Task Runner_invokes_tool_and_captures_call() + { + int invocationCount = 0; + AITool addTool = AIFunctionFactory.Create( + (int a, int b) => { invocationCount++; return a + b; }, + name: "add", + description: "Add two integers."); + + FakeChatClient client = new FakeChatClient(scriptedReplies: new AIContent[] + { + new FunctionCallContent("call-1", "add", + new Dictionary { ["a"] = 2, ["b"] = 3 }), + }, finalText: "answer: 5"); + + AgentRunner runner = new AgentRunner(client, new List { addTool }); + AgentRun run = await runner.RunAsync("what is 2 + 3?"); + + Assert.That(invocationCount, Is.EqualTo(1), "FunctionInvokingChatClient should have invoked the tool"); + Assert.That(run.ToolCalls, Has.Count.EqualTo(1)); + Assert.That(run.ToolCalls[0].Name, Is.EqualTo("add")); + Assert.That(run.ToolCalls[0].Result, Is.EqualTo("5")); + Assert.That(run.FinalOutput, Does.Contain("answer: 5")); + } + + [Test] + public async Task Runner_links_call_and_result_by_callId() + { + int counter = 0; + AITool tool = AIFunctionFactory.Create( + () => $"result-{++counter}", + name: "next", + description: "Returns sequential results."); + + FakeChatClient client = new FakeChatClient(scriptedReplies: new AIContent[] + { + new FunctionCallContent("call-A", "next", new Dictionary()), + new FunctionCallContent("call-B", "next", new Dictionary()), + }, finalText: "done"); + + AgentRunner runner = new AgentRunner(client, new List { tool }); + AgentRun run = await runner.RunAsync("call twice"); + + Assert.That(run.ToolCalls, Has.Count.EqualTo(2)); + Assert.That(run.ToolCalls[0].CallId, Is.EqualTo("call-A")); + Assert.That(run.ToolCalls[0].Result, Is.EqualTo("result-1")); + Assert.That(run.ToolCalls[1].CallId, Is.EqualTo("call-B")); + Assert.That(run.ToolCalls[1].Result, Is.EqualTo("result-2")); + } + + [Test] + public async Task Runner_propagates_model_id_to_options() + { + FakeChatClient client = new FakeChatClient(Array.Empty(), finalText: "hi"); + AgentRunner runner = new AgentRunner(client, new List(), modelId: "claude-sonnet-4-6"); + await runner.RunAsync("hello"); + + Assert.That(client.LastOptions?.ModelId, Is.EqualTo("claude-sonnet-4-6")); + } + + [Test] + public async Task Runner_includes_system_prompt_in_messages() + { + FakeChatClient client = new FakeChatClient(Array.Empty(), finalText: "hi"); + AgentRunner runner = new AgentRunner(client, new List(), systemPrompt: "BE BRIEF"); + await runner.RunAsync("hello"); + + IList firstCallMessages = client.MessageHistoryPerCall[0]; + Assert.That(firstCallMessages, Has.Count.EqualTo(2)); + Assert.That(firstCallMessages[0].Role, Is.EqualTo(ChatRole.System)); + Assert.That(firstCallMessages[0].Text, Is.EqualTo("BE BRIEF")); + Assert.That(firstCallMessages[1].Role, Is.EqualTo(ChatRole.User)); + } +} diff --git a/tests/Ngentic.Tests/ConstraintTests.cs b/tests/Ngentic.Tests/ConstraintTests.cs new file mode 100644 index 0000000..78fcd19 --- /dev/null +++ b/tests/Ngentic.Tests/ConstraintTests.cs @@ -0,0 +1,108 @@ +using Ngentic.NUnit; +using NUnit.Framework; + +namespace Ngentic.Tests; + +[TestFixture] +public sealed class ConstraintTests +{ + private static void AssertFails(Action test) + { + Assert.Throws(test); + } + + [Test] + public void CallTool_passes_when_tool_was_called() + { + AgentRun run = TestHelpers.MakeRun(("foo", null)); + Assert.That(run, Did.CallTool("foo")); + } + + [Test] + public void CallTool_fails_when_tool_was_not_called() + { + AgentRun run = TestHelpers.MakeRun(("bar", null)); + AssertFails(() => Assert.That(run, Did.CallTool("foo"))); + } + + [Test] + public void CallTool_supports_wildcard_suffix() + { + AgentRun run = TestHelpers.MakeRun(("mcp__rhino__spawn_slot", null)); + Assert.That(run, Did.CallTool("mcp__rhino__*")); + } + + [Test] + public void CallTool_WithArg_matches_int_argument() + { + AgentRun run = TestHelpers.MakeRun(("add", new { a = 2, b = 3 })); + Assert.That(run, Did.CallTool("add").WithArg("a", 2).WithArg("b", 3)); + } + + [Test] + public void CallTool_WithArg_fails_on_wrong_value() + { + AgentRun run = TestHelpers.MakeRun(("add", new { a = 2, b = 3 })); + AssertFails(() => Assert.That(run, Did.CallTool("add").WithArg("a", 99))); + } + + [Test] + public void CallTool_Times_counts_invocations() + { + AgentRun run = TestHelpers.MakeRun(("foo", null), ("foo", null), ("bar", null)); + Assert.That(run, Did.CallTool("foo").Times(2)); + } + + [Test] + public void NotCallTool_passes_when_tool_absent() + { + AgentRun run = TestHelpers.MakeRun(("foo", null)); + Assert.That(run, Did.NotCallTool("bar")); + } + + [Test] + public void NotCallTool_fails_when_tool_present() + { + AgentRun run = TestHelpers.MakeRun(("foo", null)); + AssertFails(() => Assert.That(run, Did.NotCallTool("foo"))); + } + + [Test] + public void CallToolsInOrder_matches_subsequence() + { + AgentRun run = TestHelpers.MakeRun(("a", null), ("b", null), ("c", null), ("d", null)); + Assert.That(run, Did.CallToolsInOrder("a", "c")); + } + + [Test] + public void CallToolsInOrder_fails_when_out_of_order() + { + AgentRun run = TestHelpers.MakeRun(("c", null), ("a", null)); + AssertFails(() => Assert.That(run, Did.CallToolsInOrder("a", "c"))); + } + + [Test] + public void MakeToolCalls_Exactly_counts_total() + { + AgentRun run = TestHelpers.MakeRun(("a", null), ("b", null)); + Assert.That(run, Did.MakeToolCalls.Exactly(2)); + } + + [Test] + public void MakeToolCalls_AtMost_caps_count() + { + AgentRun run = TestHelpers.MakeRun(("a", null), ("b", null)); + Assert.That(run, Did.MakeToolCalls.AtMost(5)); + AssertFails(() => Assert.That(run, Did.MakeToolCalls.AtMost(1))); + } + + [Test] + public void Constraint_failure_shows_actual_trajectory_in_message() + { + AgentRun run = TestHelpers.MakeRun(("actual_tool", null)); + AssertionException ex = Assert.Throws( + (Action)(() => Assert.That(run, Did.CallTool("missing_tool"))))!; + Assert.That(ex.Message, Does.Contain("actual_tool")); + Assert.That(ex.Message, Does.Contain("missing_tool")); + } +} diff --git a/tests/Ngentic.Tests/ExpectTests.cs b/tests/Ngentic.Tests/ExpectTests.cs new file mode 100644 index 0000000..324f842 --- /dev/null +++ b/tests/Ngentic.Tests/ExpectTests.cs @@ -0,0 +1,59 @@ +using NUnit.Framework; + +namespace Ngentic.Tests; + +/// +/// Covers the framework-agnostic Expect DSL. The constraint-based API in +/// Ngentic.NUnit is preferred for NUnit consumers, but Expect remains for +/// non-NUnit callers. +/// +[TestFixture] +public sealed class ExpectTests +{ + private static void AssertPasses(Action act) => Assert.DoesNotThrow(act); + private static void AssertFails(Action act) => Assert.Throws(act); + + [Test] + public void CalledTool_AtLeastOnce_passes_when_present() + { + AgentRun run = TestHelpers.MakeRun(("foo", null)); + AssertPasses(() => Expect.That(run).CalledTool("foo").AtLeastOnce()); + } + + [Test] + public void CalledTool_throws_when_absent() + { + AgentRun run = TestHelpers.MakeRun(("bar", null)); + AssertFails(() => Expect.That(run).CalledTool("foo").AtLeastOnce()); + } + + [Test] + public void DidNotCall_throws_when_present() + { + AgentRun run = TestHelpers.MakeRun(("foo", null)); + AssertFails(() => Expect.That(run).DidNotCall("foo")); + } + + [Test] + public void HasCount_matches_exact() + { + AgentRun run = TestHelpers.MakeRun(("a", null), ("b", null)); + AssertPasses(() => Expect.That(run).HasCount(2)); + AssertFails(() => Expect.That(run).HasCount(3)); + } + + [Test] + public void HasCountLessThan_enforces_upper_bound() + { + AgentRun run = TestHelpers.MakeRun(("a", null), ("b", null)); + AssertPasses(() => Expect.That(run).HasCountLessThan(5)); + AssertFails(() => Expect.That(run).HasCountLessThan(2)); + } + + [Test] + public void OutputAssertion_Contains_is_case_insensitive() + { + AssertPasses(() => Expect.That("Hello World").Contains("hello")); + AssertFails(() => Expect.That("Hello World").Contains("missing")); + } +} diff --git a/tests/Ngentic.Tests/FakeChatClient.cs b/tests/Ngentic.Tests/FakeChatClient.cs new file mode 100644 index 0000000..7f08010 --- /dev/null +++ b/tests/Ngentic.Tests/FakeChatClient.cs @@ -0,0 +1,56 @@ +using Microsoft.Extensions.AI; + +namespace Ngentic.Tests; + +/// +/// Test double: returns a scripted sequence of replies. Each call to GetResponseAsync +/// advances one step. Each reply is either a tool call or a final text message. +/// +internal sealed class FakeChatClient : IChatClient +{ + private readonly IReadOnlyList _scriptedReplies; + private readonly string _finalText; + private int _turn; + + public int CallsObserved { get; private set; } + public List> MessageHistoryPerCall { get; } = new(); + public ChatOptions? LastOptions { get; private set; } + + public FakeChatClient(IEnumerable scriptedReplies, string finalText = "") + { + _scriptedReplies = scriptedReplies.ToList(); + _finalText = finalText; + } + + public Task GetResponseAsync( + IEnumerable messages, + ChatOptions? options = null, + CancellationToken cancellationToken = default) + { + CallsObserved++; + MessageHistoryPerCall.Add(messages.ToList()); + LastOptions = options; + + ChatMessage reply; + if (_turn < _scriptedReplies.Count) + { + reply = new ChatMessage(ChatRole.Assistant, new List { _scriptedReplies[_turn] }); + } + else + { + reply = new ChatMessage(ChatRole.Assistant, _finalText); + } + _turn++; + return Task.FromResult(new ChatResponse(reply)); + } + + public IAsyncEnumerable GetStreamingResponseAsync( + IEnumerable messages, + ChatOptions? options = null, + CancellationToken cancellationToken = default) + => throw new NotImplementedException(); + + public object? GetService(Type serviceType, object? serviceKey = null) => null; + + public void Dispose() { } +} diff --git a/tests/Ngentic.Tests/GlobalUsings.cs b/tests/Ngentic.Tests/GlobalUsings.cs new file mode 100644 index 0000000..cefced4 --- /dev/null +++ b/tests/Ngentic.Tests/GlobalUsings.cs @@ -0,0 +1 @@ +global using NUnit.Framework; \ No newline at end of file diff --git a/tests/Ngentic.Tests/Ngentic.Tests.csproj b/tests/Ngentic.Tests/Ngentic.Tests.csproj new file mode 100644 index 0000000..2ef6655 --- /dev/null +++ b/tests/Ngentic.Tests/Ngentic.Tests.csproj @@ -0,0 +1,25 @@ + + + + net8.0 + 12 + enable + enable + false + true + + + + + + + + + + + + + + + + diff --git a/tests/Ngentic.Tests/TestHelpers.cs b/tests/Ngentic.Tests/TestHelpers.cs new file mode 100644 index 0000000..e6f98a3 --- /dev/null +++ b/tests/Ngentic.Tests/TestHelpers.cs @@ -0,0 +1,42 @@ +using System.Text.Json; + +namespace Ngentic.Tests; + +internal static class TestHelpers +{ + public static AgentRun MakeRun(params (string name, object? args)[] calls) + { + List toolCalls = new(); + int i = 0; + foreach ((string name, object? args) in calls) + { + toolCalls.Add(new ToolCall + { + CallId = $"call-{i++}", + Name = name, + Arguments = ToJson(args), + Result = "", + IsError = false, + }); + } + return new AgentRun + { + Prompt = "test", + ToolCalls = toolCalls, + FinalOutput = "", + Duration = TimeSpan.Zero, + Turns = 1, + HitTurnLimit = false, + }; + } + + private static JsonElement ToJson(object? args) + { + if (args == null) + { + return JsonDocument.Parse("{}").RootElement; + } + string json = JsonSerializer.Serialize(args); + return JsonDocument.Parse(json).RootElement; + } +} diff --git a/tests/Ngentic.Tests/UnitTest1.cs b/tests/Ngentic.Tests/UnitTest1.cs new file mode 100644 index 0000000..0210ba8 --- /dev/null +++ b/tests/Ngentic.Tests/UnitTest1.cs @@ -0,0 +1,4 @@ +// Placeholder — real tests live in sibling files. +namespace Ngentic.Tests.Internal; + +internal static class AssemblyMarker { } From 14d3eb359a39558ff64af124807913fd0d68c45d Mon Sep 17 00:00:00 2001 From: Callum Sykes Date: Mon, 18 May 2026 08:37:30 -0700 Subject: [PATCH 2/3] Rewrite --- Ngentic.slnx | 3 - samples/Ngentic.Samples/FakeChatClient.cs | 46 ---- samples/Ngentic.Samples/GlobalUsings.cs | 1 - .../Ngentic.Samples/Ngentic.Samples.csproj | 24 -- samples/Ngentic.Samples/SampleTest.cs | 50 ---- samples/Ngentic.Samples/UnitTest1.cs | 4 - src/Ngentic.Core/AgentRun.cs | 14 ++ src/Ngentic.Core/AgentRunner.cs | 130 ---------- src/Ngentic.Core/Class1.cs | 3 - src/Ngentic.Core/Expect.cs | 159 ------------ src/Ngentic.Core/IStateScorer.cs | 6 - src/Ngentic.Core/McpRegistry.cs | 36 --- src/Ngentic.Core/Ngentic.Core.csproj | 6 - src/Ngentic.NUnit/AgenticTestBase.cs | 179 +++++++------- src/Ngentic.NUnit/Class1.cs | 3 - src/Ngentic.NUnit/ClaudeAgent.cs | 138 +++++++++++ src/Ngentic.NUnit/McpServerSpec.cs | 12 + src/Ngentic.NUnit/TrajectoryParser.cs | 229 ++++++++++++++++++ tests/Ngentic.Tests/AgentRunnerTests.cs | 82 ------- tests/Ngentic.Tests/ExpectTests.cs | 59 ----- tests/Ngentic.Tests/FakeChatClient.cs | 56 ----- tests/Ngentic.Tests/TrajectoryParserTests.cs | 138 +++++++++++ 22 files changed, 613 insertions(+), 765 deletions(-) delete mode 100644 samples/Ngentic.Samples/FakeChatClient.cs delete mode 100644 samples/Ngentic.Samples/GlobalUsings.cs delete mode 100644 samples/Ngentic.Samples/Ngentic.Samples.csproj delete mode 100644 samples/Ngentic.Samples/SampleTest.cs delete mode 100644 samples/Ngentic.Samples/UnitTest1.cs delete mode 100644 src/Ngentic.Core/AgentRunner.cs delete mode 100644 src/Ngentic.Core/Class1.cs delete mode 100644 src/Ngentic.Core/Expect.cs delete mode 100644 src/Ngentic.Core/IStateScorer.cs delete mode 100644 src/Ngentic.Core/McpRegistry.cs delete mode 100644 src/Ngentic.NUnit/Class1.cs create mode 100644 src/Ngentic.NUnit/ClaudeAgent.cs create mode 100644 src/Ngentic.NUnit/McpServerSpec.cs create mode 100644 src/Ngentic.NUnit/TrajectoryParser.cs delete mode 100644 tests/Ngentic.Tests/AgentRunnerTests.cs delete mode 100644 tests/Ngentic.Tests/ExpectTests.cs delete mode 100644 tests/Ngentic.Tests/FakeChatClient.cs create mode 100644 tests/Ngentic.Tests/TrajectoryParserTests.cs diff --git a/Ngentic.slnx b/Ngentic.slnx index fd8f695..9dbf830 100644 --- a/Ngentic.slnx +++ b/Ngentic.slnx @@ -1,7 +1,4 @@ - - - diff --git a/samples/Ngentic.Samples/FakeChatClient.cs b/samples/Ngentic.Samples/FakeChatClient.cs deleted file mode 100644 index cc6de78..0000000 --- a/samples/Ngentic.Samples/FakeChatClient.cs +++ /dev/null @@ -1,46 +0,0 @@ -using Microsoft.Extensions.AI; - -namespace Ngentic.Samples; - -internal sealed class FakeChatClient : IChatClient -{ - private readonly IReadOnlyList _scriptedCalls; - private readonly string _finalText; - private int _turn; - - public FakeChatClient(IEnumerable scriptedCalls, string finalText) - { - _scriptedCalls = scriptedCalls.ToList(); - _finalText = finalText; - } - - public Task GetResponseAsync( - IEnumerable messages, - ChatOptions? options = null, - CancellationToken cancellationToken = default) - { - ChatMessage reply; - if (_turn < _scriptedCalls.Count) - { - FunctionCallContent call = _scriptedCalls[_turn]; - reply = new ChatMessage(ChatRole.Assistant, new List { call }); - } - else - { - reply = new ChatMessage(ChatRole.Assistant, _finalText); - } - _turn++; - ChatResponse response = new ChatResponse(reply); - return Task.FromResult(response); - } - - public IAsyncEnumerable GetStreamingResponseAsync( - IEnumerable messages, - ChatOptions? options = null, - CancellationToken cancellationToken = default) - => throw new NotImplementedException(); - - public object? GetService(Type serviceType, object? serviceKey = null) => null; - - public void Dispose() { } -} diff --git a/samples/Ngentic.Samples/GlobalUsings.cs b/samples/Ngentic.Samples/GlobalUsings.cs deleted file mode 100644 index cefced4..0000000 --- a/samples/Ngentic.Samples/GlobalUsings.cs +++ /dev/null @@ -1 +0,0 @@ -global using NUnit.Framework; \ No newline at end of file diff --git a/samples/Ngentic.Samples/Ngentic.Samples.csproj b/samples/Ngentic.Samples/Ngentic.Samples.csproj deleted file mode 100644 index bb444f9..0000000 --- a/samples/Ngentic.Samples/Ngentic.Samples.csproj +++ /dev/null @@ -1,24 +0,0 @@ - - - - net8.0 - enable - enable - - false - true - - - - - - - - - - - - - - - diff --git a/samples/Ngentic.Samples/SampleTest.cs b/samples/Ngentic.Samples/SampleTest.cs deleted file mode 100644 index 4c82b43..0000000 --- a/samples/Ngentic.Samples/SampleTest.cs +++ /dev/null @@ -1,50 +0,0 @@ -using Microsoft.Extensions.AI; -using Ngentic; -using Ngentic.NUnit; -using NUnit.Framework; - -namespace Ngentic.Samples; - -[TestFixture] -[McpDependency("calculator")] -public sealed class CalculatorAgentTests : AgenticTestBase -{ - // Sample uses a fake IChatClient + a fake MCP registry so the test compiles - // and runs without network calls. Real usage swaps these for the official - // Anthropic IChatClient and the ModelContextProtocol client. - protected override void ConfigureHarness() - { - FakeChatClient fakeClient = new FakeChatClient(scriptedCalls: new[] - { - new FunctionCallContent("call-1", "mcp__calculator__add", - new Dictionary { ["a"] = 2, ["b"] = 3 }), - }, finalText: "2 + 3 = 5"); - - InMemoryMcpRegistry registry = new InMemoryMcpRegistry(); - registry.Register("calculator", _ => Task.FromResult>(new List - { - AIFunctionFactory.Create( - (int a, int b) => a + b, - name: "mcp__calculator__add", - description: "Add two integers."), - })); - - UseClient(fakeClient); - UseRegistry(registry); - } - - [Test] - [MaxTurns(4)] - public async Task Agent_uses_calculator_to_add() - { - AgentRun run = await Agent - .WithSystemPrompt("You are a math assistant.") - .WithAllowedTools("mcp__calculator__*") - .RunAsync("What is 2 + 3?"); - - Assert.That(run, Did.CallTool("mcp__calculator__add").WithArg("a", 2).WithArg("b", 3)); - Assert.That(run, Did.NotCallTool("mcp__calculator__divide")); - Assert.That(run, Did.MakeToolCalls.Exactly(1)); - Assert.That(run.FinalOutput, Does.Contain("5")); - } -} diff --git a/samples/Ngentic.Samples/UnitTest1.cs b/samples/Ngentic.Samples/UnitTest1.cs deleted file mode 100644 index 3495d55..0000000 --- a/samples/Ngentic.Samples/UnitTest1.cs +++ /dev/null @@ -1,4 +0,0 @@ -// Sample lives in SampleTest.cs. -namespace Ngentic.Samples.Internal; - -internal static class AssemblyMarker { } diff --git a/src/Ngentic.Core/AgentRun.cs b/src/Ngentic.Core/AgentRun.cs index 62ca273..11c0af3 100644 --- a/src/Ngentic.Core/AgentRun.cs +++ b/src/Ngentic.Core/AgentRun.cs @@ -8,4 +8,18 @@ public sealed class AgentRun public required TimeSpan Duration { get; init; } public required int Turns { get; init; } public required bool HitTurnLimit { get; init; } + + /// + /// Total cost reported by the agent driver, in USD. Populated by the + /// Claude CLI driver from the final stream-json `result` event; null + /// when the driver does not surface cost. + /// + public double? CostUsd { get; init; } + + /// + /// Free-form payload for driver-specific information that does not earn + /// a top-level field. Populated by the Claude CLI driver with stderr, + /// exit code, and the raw stream-json transcript for debugging. + /// + public IReadOnlyDictionary? Metadata { get; init; } } diff --git a/src/Ngentic.Core/AgentRunner.cs b/src/Ngentic.Core/AgentRunner.cs deleted file mode 100644 index a4844f0..0000000 --- a/src/Ngentic.Core/AgentRunner.cs +++ /dev/null @@ -1,130 +0,0 @@ -using System.Diagnostics; -using System.Text; -using System.Text.Json; -using Microsoft.Extensions.AI; - -namespace Ngentic; - -public sealed class AgentRunner -{ - private readonly IChatClient _innerClient; - private readonly IList _tools; - private readonly string? _systemPrompt; - private readonly int _maxTurns; - private readonly string? _modelId; - - public AgentRunner( - IChatClient client, - IList tools, - string? systemPrompt = null, - int maxTurns = 16, - string? modelId = null) - { - _innerClient = client; - _tools = tools; - _systemPrompt = systemPrompt; - _maxTurns = maxTurns; - _modelId = modelId; - } - - public async Task RunAsync(string prompt, CancellationToken ct = default) - { - Stopwatch stopwatch = Stopwatch.StartNew(); - - // Wrap the caller's IChatClient so MEAI invokes tools on the model's behalf. - // Without this, FunctionCallContent comes back from the model but nothing - // executes the AITool — the loop would spin until max turns with no progress. - FunctionInvokingChatClient functionClient = new FunctionInvokingChatClient(_innerClient) - { - MaximumIterationsPerRequest = _maxTurns, - AllowConcurrentInvocation = false, - }; - - List messages = new(); - if (!string.IsNullOrEmpty(_systemPrompt)) - { - messages.Add(new ChatMessage(ChatRole.System, _systemPrompt)); - } - messages.Add(new ChatMessage(ChatRole.User, prompt)); - - ChatOptions options = new() - { - Tools = _tools, - ToolMode = ChatToolMode.Auto, - ModelId = _modelId, - }; - - ChatResponse response = await functionClient.GetResponseAsync(messages, options, ct); - - List toolCalls = new(); - StringBuilder finalText = new(); - - foreach (ChatMessage msg in response.Messages) - { - foreach (AIContent content in msg.Contents) - { - switch (content) - { - case FunctionCallContent call: - toolCalls.Add(new ToolCall - { - CallId = call.CallId, - Name = call.Name, - Arguments = SerializeArgs(call.Arguments), - Result = "", - IsError = false, - }); - break; - case FunctionResultContent result: - int idx = toolCalls.FindIndex(t => t.CallId == result.CallId); - if (idx >= 0) - { - ToolCall existing = toolCalls[idx]; - toolCalls[idx] = new ToolCall - { - CallId = existing.CallId, - Name = existing.Name, - Arguments = existing.Arguments, - Result = result.Result?.ToString() ?? "", - IsError = result.Exception != null, - }; - } - break; - case TextContent text: - if (msg.Role == ChatRole.Assistant) - { - finalText.Append(text.Text); - } - break; - } - } - } - - stopwatch.Stop(); - - int turns = Math.Max(1, response.Messages.Count(m => m.Role == ChatRole.Assistant)); - bool hitLimit = turns >= _maxTurns - && response.Messages.Any(m => m.Contents.OfType().Any() - && !response.Messages.Any(r => r.Contents.OfType().Any())); - - return new AgentRun - { - Prompt = prompt, - ToolCalls = toolCalls, - FinalOutput = finalText.ToString(), - Duration = stopwatch.Elapsed, - Turns = turns, - HitTurnLimit = hitLimit, - }; - } - - private static JsonElement SerializeArgs(IDictionary? args) - { - if (args == null || args.Count == 0) - { - return JsonDocument.Parse("{}").RootElement; - } - string json = JsonSerializer.Serialize(args); - return JsonDocument.Parse(json).RootElement; - } -} diff --git a/src/Ngentic.Core/Class1.cs b/src/Ngentic.Core/Class1.cs deleted file mode 100644 index f5a9b57..0000000 --- a/src/Ngentic.Core/Class1.cs +++ /dev/null @@ -1,3 +0,0 @@ -namespace Ngentic.Internal; - -internal static class AssemblyMarker { } diff --git a/src/Ngentic.Core/Expect.cs b/src/Ngentic.Core/Expect.cs deleted file mode 100644 index 7f571c7..0000000 --- a/src/Ngentic.Core/Expect.cs +++ /dev/null @@ -1,159 +0,0 @@ -namespace Ngentic; - -public static class Expect -{ - public static TrajectoryAssertion That(AgentRun run) => new TrajectoryAssertion(run); - public static OutputAssertion That(string finalOutput) => new OutputAssertion(finalOutput); -} - -public sealed class TrajectoryAssertion -{ - private readonly AgentRun _run; - private readonly List> _pending = new(); - private bool _orMode; - - internal TrajectoryAssertion(AgentRun run) - { - _run = run; - } - - public TrajectoryAssertion CalledTool(string name) - { - AddClause(r => r.ToolCalls.Any(c => Matches(c.Name, name))); - return this; - } - - public TrajectoryAssertion DidNotCall(string name) - { - Flush(); - if (_run.ToolCalls.Any(c => Matches(c.Name, name))) - { - throw new AgenticAssertionException( - $"Expected agent NOT to call '{name}', but it did.\n{Trace(_run)}"); - } - return this; - } - - public TrajectoryAssertion CalledInOrder(params string[] names) - { - Flush(); - int idx = 0; - foreach (ToolCall call in _run.ToolCalls) - { - if (idx < names.Length && Matches(call.Name, names[idx])) - { - idx++; - } - } - if (idx < names.Length) - { - throw new AgenticAssertionException( - $"Expected tools called in order [{string.Join(", ", names)}], but order was not satisfied.\n{Trace(_run)}"); - } - return this; - } - - public TrajectoryAssertion Or => Pivot(true); - - public TrajectoryAssertion AtLeastOnce() - { - Flush(); - return this; - } - - public TrajectoryAssertion HasCountLessThan(int limit) - { - Flush(); - if (_run.ToolCalls.Count >= limit) - { - throw new AgenticAssertionException( - $"Expected fewer than {limit} tool calls, got {_run.ToolCalls.Count}.\n{Trace(_run)}"); - } - return this; - } - - public TrajectoryAssertion HasCount(int count) - { - Flush(); - if (_run.ToolCalls.Count != count) - { - throw new AgenticAssertionException( - $"Expected exactly {count} tool calls, got {_run.ToolCalls.Count}.\n{Trace(_run)}"); - } - return this; - } - - private TrajectoryAssertion Pivot(bool orMode) - { - _orMode = orMode; - return this; - } - - private void AddClause(Predicate clause) - { - if (_orMode && _pending.Count > 0) - { - Predicate last = _pending[^1]; - _pending[^1] = r => last(r) || clause(r); - _orMode = false; - } - else - { - _pending.Add(clause); - } - } - - private void Flush() - { - foreach (Predicate clause in _pending) - { - if (!clause(_run)) - { - throw new AgenticAssertionException( - $"Trajectory assertion failed.\n{Trace(_run)}"); - } - } - _pending.Clear(); - } - - private static bool Matches(string actual, string pattern) - { - if (pattern.EndsWith('*')) - { - string prefix = pattern[..^1]; - return actual.StartsWith(prefix, StringComparison.Ordinal); - } - return actual == pattern; - } - - private static string Trace(AgentRun run) - { - if (run.ToolCalls.Count == 0) - { - return " (no tool calls)"; - } - return " Trajectory:\n" + string.Join("\n", - run.ToolCalls.Select((c, i) => $" [{i}] {c.Name}")); - } -} - -public sealed class OutputAssertion -{ - private readonly string _output; - internal OutputAssertion(string output) { _output = output; } - - public OutputAssertion Contains(string substring) - { - if (!_output.Contains(substring, StringComparison.OrdinalIgnoreCase)) - { - throw new AgenticAssertionException( - $"Expected output to contain '{substring}'. Output was:\n{_output}"); - } - return this; - } -} - -public sealed class AgenticAssertionException : Exception -{ - public AgenticAssertionException(string message) : base(message) { } -} diff --git a/src/Ngentic.Core/IStateScorer.cs b/src/Ngentic.Core/IStateScorer.cs deleted file mode 100644 index dca71ef..0000000 --- a/src/Ngentic.Core/IStateScorer.cs +++ /dev/null @@ -1,6 +0,0 @@ -namespace Ngentic; - -public interface IStateScorer -{ - Task SnapshotAsync(CancellationToken ct = default); -} diff --git a/src/Ngentic.Core/McpRegistry.cs b/src/Ngentic.Core/McpRegistry.cs deleted file mode 100644 index be44b83..0000000 --- a/src/Ngentic.Core/McpRegistry.cs +++ /dev/null @@ -1,36 +0,0 @@ -using Microsoft.Extensions.AI; - -namespace Ngentic; - -public sealed record McpServerSpec( - string Name, - string Command, - IReadOnlyList Args, - IReadOnlyDictionary? Env = null); - -public interface IMcpRegistry -{ - bool IsConfigured(string name); - Task> GetToolsAsync(string name, CancellationToken ct = default); -} - -public sealed class InMemoryMcpRegistry : IMcpRegistry -{ - private readonly Dictionary>>> _factories = new(); - - public void Register(string name, Func>> factory) - { - _factories[name] = factory; - } - - public bool IsConfigured(string name) => _factories.ContainsKey(name); - - public Task> GetToolsAsync(string name, CancellationToken ct = default) - { - if (!_factories.TryGetValue(name, out Func>>? factory)) - { - throw new InvalidOperationException($"MCP server '{name}' is not registered."); - } - return factory(ct); - } -} diff --git a/src/Ngentic.Core/Ngentic.Core.csproj b/src/Ngentic.Core/Ngentic.Core.csproj index 7957654..fa71b7a 100644 --- a/src/Ngentic.Core/Ngentic.Core.csproj +++ b/src/Ngentic.Core/Ngentic.Core.csproj @@ -6,10 +6,4 @@ enable - - - - - - diff --git a/src/Ngentic.NUnit/AgenticTestBase.cs b/src/Ngentic.NUnit/AgenticTestBase.cs index 23d1dd4..805d997 100644 --- a/src/Ngentic.NUnit/AgenticTestBase.cs +++ b/src/Ngentic.NUnit/AgenticTestBase.cs @@ -1,26 +1,48 @@ using System.Reflection; -using Microsoft.Extensions.AI; using NUnit.Framework; -using NUnit.Framework.Interfaces; namespace Ngentic.NUnit; +/// +/// Base class for tests that drive an MCP-enabled Claude session via the +/// local claude CLI. Subclasses implement +/// to register the MCP servers their tests depend on. Per-test +/// [MaxTurns] / [Model] attributes are honoured automatically. +/// public abstract class AgenticTestBase { - private IChatClient? _chatClient; - private IMcpRegistry? _registry; + private readonly List _mcpServers = new(); + private string? _defaultModel; + private double _defaultMaxBudgetUsd = 0.50; private int? _currentMaxTurns; private string? _currentModel; protected AgentBuilder Agent => new AgentBuilder( - RequireClient(), - RequireRegistry(), - CollectMcpNames(), + _mcpServers, + _defaultModel, + _defaultMaxBudgetUsd, _currentMaxTurns, _currentModel); - protected void UseClient(IChatClient client) => _chatClient = client; - protected void UseRegistry(IMcpRegistry registry) => _registry = registry; + /// + /// Register an MCP server that the agent should be able to talk to. Called + /// from . + /// + protected void UseMcp( + string name, + string command, + IReadOnlyList? args = null, + IReadOnlyDictionary? env = null) + { + _mcpServers.Add(new McpServerSpec(name, command, args, env)); + } + + /// Set default model + budget for every agent run in this fixture. + protected void UseDefaults(string? model = null, double maxBudgetUsd = 0.50) + { + _defaultModel = model; + _defaultMaxBudgetUsd = maxBudgetUsd; + } [OneTimeSetUp] public void __NgenticOneTimeSetUp() @@ -32,8 +54,8 @@ public void __NgenticOneTimeSetUp() [SetUp] public void __NgenticSetUp() { - // Pick up [MaxTurns] / [Model] declared on the current test method so the - // attributes I documented are actually load-bearing. + // Surface [MaxTurns] / [Model] from the test method so the attributes + // are load-bearing instead of decorative. string? methodName = TestContext.CurrentContext.Test.MethodName; if (methodName == null) { @@ -41,7 +63,6 @@ public void __NgenticSetUp() _currentModel = null; return; } - MethodInfo? method = GetType().GetMethod(methodName, BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance); _currentMaxTurns = method?.GetCustomAttribute()?.Value; @@ -52,135 +73,99 @@ public void __NgenticSetUp() private void VerifyMcpDependencies() { - IMcpRegistry registry = RequireRegistry(); + HashSet registered = new(_mcpServers.Select(s => s.Name), StringComparer.Ordinal); foreach (McpDependencyAttribute dep in GetType().GetCustomAttributes()) { - if (dep.Required && !registry.IsConfigured(dep.Name)) + if (dep.Required && !registered.Contains(dep.Name)) { - Assert.Fail($"Required MCP dependency '{dep.Name}' is not configured. " + - $"Register it in {GetType().Name}.ConfigureHarness()."); + Assert.Fail($"Required MCP dependency '{dep.Name}' was not registered. " + + $"Call UseMcp(\"{dep.Name}\", ...) inside {GetType().Name}.ConfigureHarness()."); } } } - - private IReadOnlyList CollectMcpNames() - { - List names = new(); - foreach (McpDependencyAttribute dep in GetType().GetCustomAttributes()) - { - names.Add(dep.Name); - } - return names; - } - - private IChatClient RequireClient() - { - if (_chatClient == null) - { - throw new InvalidOperationException( - "No IChatClient configured. Call UseClient(...) inside ConfigureHarness()."); - } - return _chatClient; - } - - private IMcpRegistry RequireRegistry() - { - if (_registry == null) - { - throw new InvalidOperationException( - "No IMcpRegistry configured. Call UseRegistry(...) inside ConfigureHarness()."); - } - return _registry; - } } public sealed class AgentBuilder { - private readonly IChatClient _client; - private readonly IMcpRegistry _registry; - private readonly IReadOnlyList _mcpNames; - private string? _systemPrompt; - private int _maxTurns; + private readonly IReadOnlyList _mcpServers; + private readonly double _defaultMaxBudgetUsd; + private readonly string? _defaultModel; + private string? _systemPromptAppend; private string? _model; - private readonly List _allowedPatterns = new(); + private double? _maxBudgetUsd; + private readonly List _allowedTools = new(); + // _maxTurns is preserved for [MaxTurns] reporting but the CLI uses --max-budget-usd + // rather than a hard turn cap; it gets stashed into AgentRun.Metadata. + private readonly int? _maxTurns; internal AgentBuilder( - IChatClient client, - IMcpRegistry registry, - IReadOnlyList mcpNames, + IReadOnlyList mcpServers, + string? defaultModel, + double defaultMaxBudgetUsd, int? maxTurnsFromAttribute, string? modelFromAttribute) { - _client = client; - _registry = registry; - _mcpNames = mcpNames; - _maxTurns = maxTurnsFromAttribute ?? 16; + _mcpServers = mcpServers; + _defaultModel = defaultModel; + _defaultMaxBudgetUsd = defaultMaxBudgetUsd; + _maxTurns = maxTurnsFromAttribute; _model = modelFromAttribute; } public AgentBuilder WithSystemPrompt(string prompt) { - _systemPrompt = prompt; + _systemPromptAppend = prompt; return this; } - public AgentBuilder WithMaxTurns(int turns) + public AgentBuilder WithModel(string modelId) { - _maxTurns = turns; + _model = modelId; return this; } - public AgentBuilder WithModel(string modelId) + public AgentBuilder WithMaxBudgetUsd(double usd) { - _model = modelId; + _maxBudgetUsd = usd; return this; } public AgentBuilder WithAllowedTools(params string[] patterns) { - _allowedPatterns.AddRange(patterns); + _allowedTools.AddRange(patterns); return this; } public async Task RunAsync(string prompt, CancellationToken ct = default) { - List tools = new(); - foreach (string mcpName in _mcpNames) - { - IList mcpTools = await _registry.GetToolsAsync(mcpName, ct); - foreach (AITool tool in mcpTools) - { - if (IsAllowed(tool.Name)) - { - tools.Add(tool); - } - } - } + ClaudeAgentRequest request = new( + Prompt: prompt, + McpServers: _mcpServers, + AllowedTools: _allowedTools, + SystemPromptAppend: _systemPromptAppend, + Model: _model ?? _defaultModel, + MaxBudgetUsd: _maxBudgetUsd ?? _defaultMaxBudgetUsd); - AgentRunner runner = new AgentRunner(_client, tools, _systemPrompt, _maxTurns, _model); - return await runner.RunAsync(prompt, ct); - } + AgentRun run = await ClaudeAgent.RunAsync(request, ct).ConfigureAwait(false); - private bool IsAllowed(string toolName) - { - if (_allowedPatterns.Count == 0) + if (_maxTurns is not null) { - return true; - } - foreach (string pattern in _allowedPatterns) - { - if (pattern.EndsWith('*')) + Dictionary meta = new(run.Metadata ?? new Dictionary()) { - if (toolName.StartsWith(pattern[..^1], StringComparison.Ordinal)) - { - return true; - } - } - else if (toolName == pattern) + ["max_turns_attribute"] = _maxTurns.Value, + }; + run = new AgentRun { - return true; - } + Prompt = run.Prompt, + ToolCalls = run.ToolCalls, + FinalOutput = run.FinalOutput, + Duration = run.Duration, + Turns = run.Turns, + HitTurnLimit = run.Turns > _maxTurns.Value, + CostUsd = run.CostUsd, + Metadata = meta, + }; } - return false; + return run; } } diff --git a/src/Ngentic.NUnit/Class1.cs b/src/Ngentic.NUnit/Class1.cs deleted file mode 100644 index 2c06bbe..0000000 --- a/src/Ngentic.NUnit/Class1.cs +++ /dev/null @@ -1,3 +0,0 @@ -namespace Ngentic.NUnit.Internal; - -internal static class AssemblyMarker { } diff --git a/src/Ngentic.NUnit/ClaudeAgent.cs b/src/Ngentic.NUnit/ClaudeAgent.cs new file mode 100644 index 0000000..6983ab3 --- /dev/null +++ b/src/Ngentic.NUnit/ClaudeAgent.cs @@ -0,0 +1,138 @@ +using System.Diagnostics; +using System.Globalization; +using System.Runtime.InteropServices; +using System.Text; +using System.Text.Json; + +namespace Ngentic.NUnit; + +/// +/// Drives a real Claude session via the local claude CLI in headless +/// mode (-p --output-format stream-json). The CLI handles MCP wiring +/// itself — we hand it an inline --mcp-config built from the supplied +/// s. Tests that depend on this should be +/// [Explicit]: real LLM calls cost subscription quota and are +/// non-deterministic. +/// +public static class ClaudeAgent +{ + public static async Task RunAsync( + ClaudeAgentRequest request, + CancellationToken ct = default) + { + Stopwatch stopwatch = Stopwatch.StartNew(); + + string mcpConfigJson = BuildMcpConfig(request.McpServers); + ProcessStartInfo psi = new() + { + FileName = ResolveClaudeCli(), + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + WorkingDirectory = request.WorkingDirectory ?? Environment.CurrentDirectory, + }; + psi.ArgumentList.Add("-p"); + psi.ArgumentList.Add(request.Prompt); + psi.ArgumentList.Add("--output-format"); + psi.ArgumentList.Add("stream-json"); + psi.ArgumentList.Add("--verbose"); + psi.ArgumentList.Add("--strict-mcp-config"); + psi.ArgumentList.Add("--mcp-config"); + psi.ArgumentList.Add(mcpConfigJson); + psi.ArgumentList.Add("--permission-mode"); + psi.ArgumentList.Add("bypassPermissions"); + psi.ArgumentList.Add("--max-budget-usd"); + psi.ArgumentList.Add(request.MaxBudgetUsd.ToString("0.00", CultureInfo.InvariantCulture)); + psi.ArgumentList.Add("--no-session-persistence"); + if (request.AllowedTools.Count > 0) + { + psi.ArgumentList.Add("--allowedTools"); + psi.ArgumentList.Add(string.Join(" ", request.AllowedTools)); + } + if (request.SystemPromptAppend is not null) + { + psi.ArgumentList.Add("--append-system-prompt"); + psi.ArgumentList.Add(request.SystemPromptAppend); + } + if (request.Model is not null) + { + psi.ArgumentList.Add("--model"); + psi.ArgumentList.Add(request.Model); + } + + using Process proc = Process.Start(psi) + ?? throw new InvalidOperationException("Failed to start `claude` CLI."); + + // Pipes can deadlock if either stream fills its buffer while we wait + // for the process — drain both concurrently. + Task> stdoutTask = ReadLinesAsync(proc.StandardOutput, ct); + Task stderrTask = proc.StandardError.ReadToEndAsync(ct); + + try + { + await proc.WaitForExitAsync(ct).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + try { proc.Kill(entireProcessTree: true); } catch { /* best effort */ } + throw; + } + + List stdoutLines = await stdoutTask.ConfigureAwait(false); + string stderr = await stderrTask.ConfigureAwait(false); + + stopwatch.Stop(); + + return TrajectoryParser.Parse( + request.Prompt, + stdoutLines, + stderr, + proc.ExitCode, + stopwatch.Elapsed); + } + + private static string BuildMcpConfig(IReadOnlyList servers) + { + Dictionary map = new(); + foreach (McpServerSpec spec in servers) + { + map[spec.Name] = new Dictionary + { + ["command"] = spec.Command, + ["args"] = spec.Args ?? (IReadOnlyList)Array.Empty(), + ["env"] = spec.Env, + }; + } + return JsonSerializer.Serialize(new { mcpServers = map }); + } + + private static string ResolveClaudeCli() + { + string? overrideExe = Environment.GetEnvironmentVariable("CLAUDE_CLI"); + if (!string.IsNullOrEmpty(overrideExe)) + { + return overrideExe; + } + return RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "claude.exe" : "claude"; + } + + private static async Task> ReadLinesAsync(StreamReader reader, CancellationToken ct) + { + List lines = new(); + string? line; + while ((line = await reader.ReadLineAsync(ct).ConfigureAwait(false)) is not null) + { + lines.Add(line); + } + return lines; + } +} + +public sealed record ClaudeAgentRequest( + string Prompt, + IReadOnlyList McpServers, + IReadOnlyList AllowedTools, + string? SystemPromptAppend = null, + string? Model = null, + double MaxBudgetUsd = 0.50, + string? WorkingDirectory = null); diff --git a/src/Ngentic.NUnit/McpServerSpec.cs b/src/Ngentic.NUnit/McpServerSpec.cs new file mode 100644 index 0000000..030bac7 --- /dev/null +++ b/src/Ngentic.NUnit/McpServerSpec.cs @@ -0,0 +1,12 @@ +namespace Ngentic.NUnit; + +/// +/// Declarative description of an MCP server to hand to the local Claude CLI +/// via --mcp-config. Consumer code (e.g. a rhino-specific test) builds +/// these in their fixture setup and passes them to . +/// +public sealed record McpServerSpec( + string Name, + string Command, + IReadOnlyList? Args = null, + IReadOnlyDictionary? Env = null); diff --git a/src/Ngentic.NUnit/TrajectoryParser.cs b/src/Ngentic.NUnit/TrajectoryParser.cs new file mode 100644 index 0000000..6ac4df7 --- /dev/null +++ b/src/Ngentic.NUnit/TrajectoryParser.cs @@ -0,0 +1,229 @@ +using System.Text; +using System.Text.Json; + +namespace Ngentic.NUnit; + +/// +/// Parses claude -p --output-format stream-json output into a +/// . Exposed publicly so consumers can replay captured +/// transcripts in unit tests without spawning the CLI. +/// +public static class TrajectoryParser +{ + public static AgentRun Parse( + string prompt, + IReadOnlyList stdoutLines, + string stderr, + int exitCode, + TimeSpan duration) + { + List calls = new(); + Dictionary indexByCallId = new(); + StringBuilder finalText = new(); + int assistantMessages = 0; + bool isError = false; + string? errorPayload = null; + double? costUsd = null; + + foreach (string line in stdoutLines) + { + if (string.IsNullOrWhiteSpace(line)) + { + continue; + } + JsonDocument doc; + try + { + doc = JsonDocument.Parse(line); + } + catch (JsonException) + { + continue; + } + + using (doc) + { + JsonElement root = doc.RootElement; + if (!root.TryGetProperty("type", out JsonElement typeEl)) + { + continue; + } + switch (typeEl.GetString()) + { + case "assistant": + assistantMessages++; + ExtractAssistantContent(root, calls, indexByCallId, finalText); + break; + case "user": + ExtractToolResults(root, calls, indexByCallId); + break; + case "result": + if (root.TryGetProperty("is_error", out JsonElement errEl) + && errEl.ValueKind == JsonValueKind.True) + { + isError = true; + errorPayload = root.TryGetProperty("result", out JsonElement r) + ? r.GetString() + : null; + } + if (root.TryGetProperty("total_cost_usd", out JsonElement costEl) + && costEl.TryGetDouble(out double c)) + { + costUsd = c; + } + break; + } + } + } + + Dictionary metadata = new() + { + ["exit_code"] = exitCode, + ["stderr"] = stderr, + ["raw_stdout_lines"] = stdoutLines, + ["is_error"] = isError, + ["error_payload"] = errorPayload, + }; + + return new AgentRun + { + Prompt = prompt, + ToolCalls = calls, + FinalOutput = finalText.ToString(), + Duration = duration, + Turns = Math.Max(1, assistantMessages), + HitTurnLimit = false, + CostUsd = costUsd, + Metadata = metadata, + }; + } + + private static void ExtractAssistantContent( + JsonElement root, + List calls, + Dictionary indexByCallId, + StringBuilder finalText) + { + if (!root.TryGetProperty("message", out JsonElement msg) + || !msg.TryGetProperty("content", out JsonElement content) + || content.ValueKind != JsonValueKind.Array) + { + return; + } + + foreach (JsonElement block in content.EnumerateArray()) + { + if (!block.TryGetProperty("type", out JsonElement blockTypeEl)) + { + continue; + } + switch (blockTypeEl.GetString()) + { + case "text": + if (block.TryGetProperty("text", out JsonElement textEl)) + { + finalText.Append(textEl.GetString()); + } + break; + case "tool_use": + string? id = block.TryGetProperty("id", out JsonElement idEl) + ? idEl.GetString() + : null; + string? name = block.TryGetProperty("name", out JsonElement nameEl) + ? nameEl.GetString() + : null; + JsonElement input = block.TryGetProperty("input", out JsonElement inEl) + ? inEl.Clone() + : JsonDocument.Parse("{}").RootElement; + if (name is null) + { + break; + } + string callId = id ?? Guid.NewGuid().ToString("N"); + indexByCallId[callId] = calls.Count; + calls.Add(new ToolCall + { + CallId = callId, + Name = name, + Arguments = input, + Result = "", + IsError = false, + }); + break; + } + } + } + + private static void ExtractToolResults( + JsonElement root, + List calls, + Dictionary indexByCallId) + { + if (!root.TryGetProperty("message", out JsonElement msg) + || !msg.TryGetProperty("content", out JsonElement content) + || content.ValueKind != JsonValueKind.Array) + { + return; + } + + foreach (JsonElement block in content.EnumerateArray()) + { + if (!block.TryGetProperty("type", out JsonElement blockTypeEl) + || blockTypeEl.GetString() != "tool_result") + { + continue; + } + + string? id = block.TryGetProperty("tool_use_id", out JsonElement idEl) + ? idEl.GetString() + : null; + if (id is null || !indexByCallId.TryGetValue(id, out int idx)) + { + continue; + } + + string resultText = ExtractToolResultText(block); + bool isErr = block.TryGetProperty("is_error", out JsonElement errEl) + && errEl.ValueKind == JsonValueKind.True; + + ToolCall prev = calls[idx]; + calls[idx] = new ToolCall + { + CallId = prev.CallId, + Name = prev.Name, + Arguments = prev.Arguments, + Result = resultText, + IsError = isErr, + }; + } + } + + private static string ExtractToolResultText(JsonElement block) + { + if (!block.TryGetProperty("content", out JsonElement contentEl)) + { + return ""; + } + if (contentEl.ValueKind == JsonValueKind.String) + { + return contentEl.GetString() ?? ""; + } + if (contentEl.ValueKind != JsonValueKind.Array) + { + return contentEl.GetRawText(); + } + StringBuilder sb = new(); + foreach (JsonElement c in contentEl.EnumerateArray()) + { + if (c.ValueKind == JsonValueKind.String) + { + sb.Append(c.GetString()); + } + else if (c.TryGetProperty("text", out JsonElement t)) + { + sb.Append(t.GetString()); + } + } + return sb.ToString(); + } +} diff --git a/tests/Ngentic.Tests/AgentRunnerTests.cs b/tests/Ngentic.Tests/AgentRunnerTests.cs deleted file mode 100644 index cb8f10b..0000000 --- a/tests/Ngentic.Tests/AgentRunnerTests.cs +++ /dev/null @@ -1,82 +0,0 @@ -using Microsoft.Extensions.AI; -using NUnit.Framework; - -namespace Ngentic.Tests; - -[TestFixture] -public sealed class AgentRunnerTests -{ - [Test] - public async Task Runner_invokes_tool_and_captures_call() - { - int invocationCount = 0; - AITool addTool = AIFunctionFactory.Create( - (int a, int b) => { invocationCount++; return a + b; }, - name: "add", - description: "Add two integers."); - - FakeChatClient client = new FakeChatClient(scriptedReplies: new AIContent[] - { - new FunctionCallContent("call-1", "add", - new Dictionary { ["a"] = 2, ["b"] = 3 }), - }, finalText: "answer: 5"); - - AgentRunner runner = new AgentRunner(client, new List { addTool }); - AgentRun run = await runner.RunAsync("what is 2 + 3?"); - - Assert.That(invocationCount, Is.EqualTo(1), "FunctionInvokingChatClient should have invoked the tool"); - Assert.That(run.ToolCalls, Has.Count.EqualTo(1)); - Assert.That(run.ToolCalls[0].Name, Is.EqualTo("add")); - Assert.That(run.ToolCalls[0].Result, Is.EqualTo("5")); - Assert.That(run.FinalOutput, Does.Contain("answer: 5")); - } - - [Test] - public async Task Runner_links_call_and_result_by_callId() - { - int counter = 0; - AITool tool = AIFunctionFactory.Create( - () => $"result-{++counter}", - name: "next", - description: "Returns sequential results."); - - FakeChatClient client = new FakeChatClient(scriptedReplies: new AIContent[] - { - new FunctionCallContent("call-A", "next", new Dictionary()), - new FunctionCallContent("call-B", "next", new Dictionary()), - }, finalText: "done"); - - AgentRunner runner = new AgentRunner(client, new List { tool }); - AgentRun run = await runner.RunAsync("call twice"); - - Assert.That(run.ToolCalls, Has.Count.EqualTo(2)); - Assert.That(run.ToolCalls[0].CallId, Is.EqualTo("call-A")); - Assert.That(run.ToolCalls[0].Result, Is.EqualTo("result-1")); - Assert.That(run.ToolCalls[1].CallId, Is.EqualTo("call-B")); - Assert.That(run.ToolCalls[1].Result, Is.EqualTo("result-2")); - } - - [Test] - public async Task Runner_propagates_model_id_to_options() - { - FakeChatClient client = new FakeChatClient(Array.Empty(), finalText: "hi"); - AgentRunner runner = new AgentRunner(client, new List(), modelId: "claude-sonnet-4-6"); - await runner.RunAsync("hello"); - - Assert.That(client.LastOptions?.ModelId, Is.EqualTo("claude-sonnet-4-6")); - } - - [Test] - public async Task Runner_includes_system_prompt_in_messages() - { - FakeChatClient client = new FakeChatClient(Array.Empty(), finalText: "hi"); - AgentRunner runner = new AgentRunner(client, new List(), systemPrompt: "BE BRIEF"); - await runner.RunAsync("hello"); - - IList firstCallMessages = client.MessageHistoryPerCall[0]; - Assert.That(firstCallMessages, Has.Count.EqualTo(2)); - Assert.That(firstCallMessages[0].Role, Is.EqualTo(ChatRole.System)); - Assert.That(firstCallMessages[0].Text, Is.EqualTo("BE BRIEF")); - Assert.That(firstCallMessages[1].Role, Is.EqualTo(ChatRole.User)); - } -} diff --git a/tests/Ngentic.Tests/ExpectTests.cs b/tests/Ngentic.Tests/ExpectTests.cs deleted file mode 100644 index 324f842..0000000 --- a/tests/Ngentic.Tests/ExpectTests.cs +++ /dev/null @@ -1,59 +0,0 @@ -using NUnit.Framework; - -namespace Ngentic.Tests; - -/// -/// Covers the framework-agnostic Expect DSL. The constraint-based API in -/// Ngentic.NUnit is preferred for NUnit consumers, but Expect remains for -/// non-NUnit callers. -/// -[TestFixture] -public sealed class ExpectTests -{ - private static void AssertPasses(Action act) => Assert.DoesNotThrow(act); - private static void AssertFails(Action act) => Assert.Throws(act); - - [Test] - public void CalledTool_AtLeastOnce_passes_when_present() - { - AgentRun run = TestHelpers.MakeRun(("foo", null)); - AssertPasses(() => Expect.That(run).CalledTool("foo").AtLeastOnce()); - } - - [Test] - public void CalledTool_throws_when_absent() - { - AgentRun run = TestHelpers.MakeRun(("bar", null)); - AssertFails(() => Expect.That(run).CalledTool("foo").AtLeastOnce()); - } - - [Test] - public void DidNotCall_throws_when_present() - { - AgentRun run = TestHelpers.MakeRun(("foo", null)); - AssertFails(() => Expect.That(run).DidNotCall("foo")); - } - - [Test] - public void HasCount_matches_exact() - { - AgentRun run = TestHelpers.MakeRun(("a", null), ("b", null)); - AssertPasses(() => Expect.That(run).HasCount(2)); - AssertFails(() => Expect.That(run).HasCount(3)); - } - - [Test] - public void HasCountLessThan_enforces_upper_bound() - { - AgentRun run = TestHelpers.MakeRun(("a", null), ("b", null)); - AssertPasses(() => Expect.That(run).HasCountLessThan(5)); - AssertFails(() => Expect.That(run).HasCountLessThan(2)); - } - - [Test] - public void OutputAssertion_Contains_is_case_insensitive() - { - AssertPasses(() => Expect.That("Hello World").Contains("hello")); - AssertFails(() => Expect.That("Hello World").Contains("missing")); - } -} diff --git a/tests/Ngentic.Tests/FakeChatClient.cs b/tests/Ngentic.Tests/FakeChatClient.cs deleted file mode 100644 index 7f08010..0000000 --- a/tests/Ngentic.Tests/FakeChatClient.cs +++ /dev/null @@ -1,56 +0,0 @@ -using Microsoft.Extensions.AI; - -namespace Ngentic.Tests; - -/// -/// Test double: returns a scripted sequence of replies. Each call to GetResponseAsync -/// advances one step. Each reply is either a tool call or a final text message. -/// -internal sealed class FakeChatClient : IChatClient -{ - private readonly IReadOnlyList _scriptedReplies; - private readonly string _finalText; - private int _turn; - - public int CallsObserved { get; private set; } - public List> MessageHistoryPerCall { get; } = new(); - public ChatOptions? LastOptions { get; private set; } - - public FakeChatClient(IEnumerable scriptedReplies, string finalText = "") - { - _scriptedReplies = scriptedReplies.ToList(); - _finalText = finalText; - } - - public Task GetResponseAsync( - IEnumerable messages, - ChatOptions? options = null, - CancellationToken cancellationToken = default) - { - CallsObserved++; - MessageHistoryPerCall.Add(messages.ToList()); - LastOptions = options; - - ChatMessage reply; - if (_turn < _scriptedReplies.Count) - { - reply = new ChatMessage(ChatRole.Assistant, new List { _scriptedReplies[_turn] }); - } - else - { - reply = new ChatMessage(ChatRole.Assistant, _finalText); - } - _turn++; - return Task.FromResult(new ChatResponse(reply)); - } - - public IAsyncEnumerable GetStreamingResponseAsync( - IEnumerable messages, - ChatOptions? options = null, - CancellationToken cancellationToken = default) - => throw new NotImplementedException(); - - public object? GetService(Type serviceType, object? serviceKey = null) => null; - - public void Dispose() { } -} diff --git a/tests/Ngentic.Tests/TrajectoryParserTests.cs b/tests/Ngentic.Tests/TrajectoryParserTests.cs new file mode 100644 index 0000000..433cac4 --- /dev/null +++ b/tests/Ngentic.Tests/TrajectoryParserTests.cs @@ -0,0 +1,138 @@ +using Ngentic.NUnit; +using NUnit.Framework; + +namespace Ngentic.Tests; + +/// +/// Exercises against captured stream-json +/// transcripts. No CLI process required — just feed in the lines and assert. +/// +[TestFixture] +public sealed class TrajectoryParserTests +{ + [Test] + public void Parses_a_single_tool_call_and_result() + { + string[] lines = + { + """{"type":"assistant","message":{"content":[{"type":"tool_use","id":"toolu_1","name":"mcp__rhino__close_slot","input":{"slot_id":"abc"}}]}}""", + """{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"toolu_1","content":"{\"closed\":false,\"error\":\"slot_not_found\"}"}]}}""", + """{"type":"assistant","message":{"content":[{"type":"text","text":"slot_not_found, as you said."}]}}""", + """{"type":"result","total_cost_usd":0.0123,"is_error":false}""", + }; + + AgentRun run = TrajectoryParser.Parse( + prompt: "Close slot abc", + stdoutLines: lines, + stderr: "", + exitCode: 0, + duration: TimeSpan.FromSeconds(2)); + + Assert.That(run.ToolCalls, Has.Count.EqualTo(1)); + Assert.That(run.ToolCalls[0].Name, Is.EqualTo("mcp__rhino__close_slot")); + Assert.That(run.ToolCalls[0].CallId, Is.EqualTo("toolu_1")); + Assert.That(run.ToolCalls[0].Result, Does.Contain("slot_not_found")); + Assert.That(run.FinalOutput, Does.Contain("slot_not_found")); + Assert.That(run.CostUsd, Is.EqualTo(0.0123).Within(1e-9)); + Assert.That(run.Turns, Is.EqualTo(2)); + } + + [Test] + public void Pairs_call_and_result_by_tool_use_id() + { + string[] lines = + { + """{"type":"assistant","message":{"content":[{"type":"tool_use","id":"a","name":"foo","input":{}}]}}""", + """{"type":"assistant","message":{"content":[{"type":"tool_use","id":"b","name":"foo","input":{}}]}}""", + """{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"b","content":"second"}]}}""", + """{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"a","content":"first"}]}}""", + """{"type":"result","is_error":false}""", + }; + + AgentRun run = TrajectoryParser.Parse("t", lines, "", 0, TimeSpan.Zero); + + Assert.That(run.ToolCalls, Has.Count.EqualTo(2)); + Assert.That(run.ToolCalls[0].CallId, Is.EqualTo("a")); + Assert.That(run.ToolCalls[0].Result, Is.EqualTo("first")); + Assert.That(run.ToolCalls[1].CallId, Is.EqualTo("b")); + Assert.That(run.ToolCalls[1].Result, Is.EqualTo("second")); + } + + [Test] + public void Handles_array_content_in_tool_results() + { + string[] lines = + { + """{"type":"assistant","message":{"content":[{"type":"tool_use","id":"x","name":"foo","input":{}}]}}""", + """{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"x","content":[{"type":"text","text":"part 1"},{"type":"text","text":" part 2"}]}]}}""", + """{"type":"result"}""", + }; + + AgentRun run = TrajectoryParser.Parse("t", lines, "", 0, TimeSpan.Zero); + + Assert.That(run.ToolCalls[0].Result, Is.EqualTo("part 1 part 2")); + } + + [Test] + public void Marks_tool_errors_via_is_error_flag() + { + string[] lines = + { + """{"type":"assistant","message":{"content":[{"type":"tool_use","id":"x","name":"foo","input":{}}]}}""", + """{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"x","content":"boom","is_error":true}]}}""", + """{"type":"result"}""", + }; + + AgentRun run = TrajectoryParser.Parse("t", lines, "", 0, TimeSpan.Zero); + + Assert.That(run.ToolCalls[0].IsError, Is.True); + } + + [Test] + public void Surfaces_stderr_and_exit_code_in_metadata() + { + AgentRun run = TrajectoryParser.Parse( + prompt: "x", + stdoutLines: Array.Empty(), + stderr: "warning: foo", + exitCode: 7, + duration: TimeSpan.Zero); + + Assert.That(run.Metadata, Is.Not.Null); + Assert.That(run.Metadata!["exit_code"], Is.EqualTo(7)); + Assert.That(run.Metadata!["stderr"], Is.EqualTo("warning: foo")); + } + + [Test] + public void Ignores_non_json_lines() + { + string[] lines = + { + "this is not json", + "", + """{"type":"assistant","message":{"content":[{"type":"text","text":"hi"}]}}""", + "garbage", + """{"type":"result","total_cost_usd":0.01}""", + }; + + AgentRun run = TrajectoryParser.Parse("t", lines, "", 0, TimeSpan.Zero); + + Assert.That(run.FinalOutput, Is.EqualTo("hi")); + Assert.That(run.CostUsd, Is.EqualTo(0.01).Within(1e-9)); + } + + [Test] + public void Captures_tool_call_arguments_as_JsonElement() + { + string[] lines = + { + """{"type":"assistant","message":{"content":[{"type":"tool_use","id":"x","name":"add","input":{"a":2,"b":3}}]}}""", + """{"type":"result"}""", + }; + + AgentRun run = TrajectoryParser.Parse("t", lines, "", 0, TimeSpan.Zero); + + Assert.That(run.ToolCalls[0].Arguments.GetProperty("a").GetInt32(), Is.EqualTo(2)); + Assert.That(run.ToolCalls[0].Arguments.GetProperty("b").GetInt32(), Is.EqualTo(3)); + } +} From e41c680b0d8720efbb8f4a656637e413fe28b35a Mon Sep 17 00:00:00 2001 From: Callum Sykes Date: Mon, 18 May 2026 13:44:33 -0700 Subject: [PATCH 3/3] More fluid assertions for JSON payloads --- src/Ngentic.Core/Args.cs | 20 ++ src/Ngentic.NUnit/JsonConstraints.cs | 244 +++++++++++++++++++++ tests/Ngentic.Tests/JsonConstraintTests.cs | 183 ++++++++++++++++ 3 files changed, 447 insertions(+) create mode 100644 src/Ngentic.Core/Args.cs create mode 100644 src/Ngentic.NUnit/JsonConstraints.cs create mode 100644 tests/Ngentic.Tests/JsonConstraintTests.cs diff --git a/src/Ngentic.Core/Args.cs b/src/Ngentic.Core/Args.cs new file mode 100644 index 0000000..d74bbc8 --- /dev/null +++ b/src/Ngentic.Core/Args.cs @@ -0,0 +1,20 @@ +namespace Ngentic; + +/// +/// Lightweight builder for the Dictionary<string, object?> argument +/// bag that MCP client APIs typically take. Lets tests write +/// Args.Of(("slot", slotA), ("script", "...")) instead of a full +/// dictionary initializer. +/// +public static class Args +{ + public static Dictionary Of(params (string name, object? value)[] pairs) + { + Dictionary result = new(pairs.Length); + foreach ((string name, object? value) in pairs) + { + result[name] = value; + } + return result; + } +} diff --git a/src/Ngentic.NUnit/JsonConstraints.cs b/src/Ngentic.NUnit/JsonConstraints.cs new file mode 100644 index 0000000..6599b5c --- /dev/null +++ b/src/Ngentic.NUnit/JsonConstraints.cs @@ -0,0 +1,244 @@ +using System.Text; +using System.Text.Json; +using NUnit.Framework.Constraints; + +namespace Ngentic.NUnit; + +/// +/// Entry point for JSON-aware constraints. Use with NUnit's Assert.That: +/// +/// Assert.That(json, Json.HasProperty("error", Is.EqualTo("slot_not_found"))); +/// Assert.That(json, Json.HasProperty("slotId", Is.Not.Empty)); +/// Assert.That(json, Json.IsArrayOfLength(0)); +/// +/// Accepts either a raw JSON string or a . If the +/// payload is an MCP content envelope ({"content":[{"type":"text","text":"..."}]}) +/// the inner text is auto-unwrapped, so callers never have to walk +/// content[0].text by hand. +/// +public static class Json +{ + public static JsonHasPropertyConstraint HasProperty(string name) => new(name, inner: null); + + public static JsonHasPropertyConstraint HasProperty(string name, IResolveConstraint inner) + => new(name, inner); + + public static JsonArrayLengthConstraint IsArrayOfLength(int length) => new(length); +} + +public sealed class JsonHasPropertyConstraint : Constraint +{ + private readonly string _name; + private readonly IResolveConstraint? _inner; + + public JsonHasPropertyConstraint(string name, IResolveConstraint? inner) + { + _name = name; + _inner = inner; + } + + public override string Description => _inner is null + ? $"JSON has property '{_name}'" + : $"JSON property '{_name}' {_inner.Resolve().Description}"; + + public override ConstraintResult ApplyTo(TActual actual) + { + if (!JsonHelpers.TryGetElement(actual, out JsonElement root)) + { + return new ConstraintResult(this, actual, ConstraintStatus.Error); + } + + JsonElement unwrapped = JsonHelpers.UnwrapMcpEnvelope(root); + + if (unwrapped.ValueKind != JsonValueKind.Object + || !unwrapped.TryGetProperty(_name, out JsonElement value)) + { + return new JsonPropertyMissingResult(this, unwrapped, _name); + } + + if (_inner is null) + { + return new JsonPropertyFoundResult(this, unwrapped, _name, value); + } + + object? clr = JsonHelpers.ToClr(value); + IConstraint resolved = _inner.Resolve(); + ConstraintResult innerResult = resolved.ApplyTo(clr); + return new JsonPropertyInnerResult(this, unwrapped, _name, value, innerResult); + } +} + +public sealed class JsonArrayLengthConstraint : Constraint +{ + private readonly int _expected; + + public JsonArrayLengthConstraint(int expected) + { + _expected = expected; + } + + public override string Description => $"JSON array of length {_expected}"; + + public override ConstraintResult ApplyTo(TActual actual) + { + if (!JsonHelpers.TryGetElement(actual, out JsonElement root)) + { + return new ConstraintResult(this, actual, ConstraintStatus.Error); + } + + JsonElement unwrapped = JsonHelpers.UnwrapMcpEnvelope(root); + + if (unwrapped.ValueKind != JsonValueKind.Array) + { + return new ConstraintResult(this, $"{unwrapped.ValueKind}: {unwrapped.GetRawText()}", false); + } + + int actualLength = unwrapped.GetArrayLength(); + return new ConstraintResult( + this, + $"array of length {actualLength}: {unwrapped.GetRawText()}", + actualLength == _expected); + } +} + +internal static class JsonHelpers +{ + public static bool TryGetElement(object? actual, out JsonElement element) + { + switch (actual) + { + case JsonElement e: + element = e; + return true; + case string s: + try + { + element = JsonDocument.Parse(s).RootElement; + return true; + } + catch (JsonException) + { + element = default; + return false; + } + default: + element = default; + return false; + } + } + + // Recognises the MCP tool-result envelope shape and digs out the inner + // payload. The envelope wraps a JSON-encoded string inside a content + // block: {"content":[{"type":"text","text":"{...}"}]}. We concatenate + // the text fields (some tools split into multiple blocks) and try to + // parse the result as JSON; if parsing fails we leave the envelope + // alone, so non-JSON text responses don't get mangled. Recurses to + // handle nested envelopes if a server ever produces them. + public static JsonElement UnwrapMcpEnvelope(JsonElement element) + { + if (element.ValueKind != JsonValueKind.Object) + { + return element; + } + if (!element.TryGetProperty("content", out JsonElement content) + || content.ValueKind != JsonValueKind.Array + || content.GetArrayLength() == 0) + { + return element; + } + + StringBuilder sb = new(); + foreach (JsonElement block in content.EnumerateArray()) + { + if (block.ValueKind != JsonValueKind.Object + || !block.TryGetProperty("text", out JsonElement text) + || text.ValueKind != JsonValueKind.String) + { + return element; + } + sb.Append(text.GetString()); + } + + try + { + JsonElement inner = JsonDocument.Parse(sb.ToString()).RootElement; + return UnwrapMcpEnvelope(inner); + } + catch (JsonException) + { + return element; + } + } + + public static object? ToClr(JsonElement value) + { + return value.ValueKind switch + { + JsonValueKind.String => value.GetString(), + JsonValueKind.True => true, + JsonValueKind.False => false, + JsonValueKind.Null => null, + JsonValueKind.Number => value.TryGetInt64(out long l) ? l : value.GetDouble(), + _ => value, + }; + } +} + +internal sealed class JsonPropertyMissingResult : ConstraintResult +{ + private readonly JsonElement _root; + private readonly string _name; + + public JsonPropertyMissingResult(IConstraint constraint, JsonElement root, string name) + : base(constraint, "property missing", isSuccess: false) + { + _root = root; + _name = name; + } + + public override void WriteMessageTo(MessageWriter writer) + { + writer.WriteLine($" Expected: {Description}"); + writer.WriteLine($" But was: property '{_name}' not found"); + writer.WriteLine($" JSON: {_root.GetRawText()}"); + } +} + +internal sealed class JsonPropertyFoundResult : ConstraintResult +{ + public JsonPropertyFoundResult(IConstraint constraint, JsonElement root, string name, JsonElement value) + : base(constraint, value.GetRawText(), isSuccess: true) + { + _ = root; + _ = name; + } +} + +internal sealed class JsonPropertyInnerResult : ConstraintResult +{ + private readonly JsonElement _root; + private readonly string _name; + private readonly JsonElement _propertyValue; + private readonly ConstraintResult _inner; + + public JsonPropertyInnerResult( + IConstraint constraint, + JsonElement root, + string name, + JsonElement propertyValue, + ConstraintResult inner) + : base(constraint, inner.ActualValue, inner.IsSuccess) + { + _root = root; + _name = name; + _propertyValue = propertyValue; + _inner = inner; + } + + public override void WriteMessageTo(MessageWriter writer) + { + writer.WriteLine($" Expected: {Description}"); + writer.WriteLine($" But was: property '{_name}' = {_propertyValue.GetRawText()}"); + writer.WriteLine($" JSON: {_root.GetRawText()}"); + } +} diff --git a/tests/Ngentic.Tests/JsonConstraintTests.cs b/tests/Ngentic.Tests/JsonConstraintTests.cs new file mode 100644 index 0000000..a52eb02 --- /dev/null +++ b/tests/Ngentic.Tests/JsonConstraintTests.cs @@ -0,0 +1,183 @@ +using System.Text.Json; +using Ngentic.NUnit; +using NUnit.Framework; + +namespace Ngentic.Tests; + +[TestFixture] +public sealed class JsonConstraintTests +{ + private static void AssertFails(Action test) + { + Assert.Throws(test); + } + + [Test] + public void HasProperty_passes_when_property_exists() + { + string json = """{"slotId":"abc-123","adopted":false}"""; + Assert.That(json, Json.HasProperty("slotId")); + } + + [Test] + public void HasProperty_fails_when_property_is_missing() + { + string json = """{"slotId":"abc-123"}"""; + AssertFails(() => Assert.That(json, Json.HasProperty("error"))); + } + + [Test] + public void HasProperty_with_inner_passes_on_string_equality() + { + string json = """{"error":"rhino_not_installed","message":"version 'garbage' was not found"}"""; + Assert.That(json, Json.HasProperty("error", Is.EqualTo("rhino_not_installed"))); + } + + [Test] + public void HasProperty_with_inner_passes_on_substring() + { + string json = """{"error":"rhino_not_installed","message":"version 'garbage' was not found"}"""; + Assert.That(json, Json.HasProperty("message", Does.Contain("garbage"))); + } + + [Test] + public void HasProperty_with_inner_passes_on_non_empty() + { + string json = """{"slotId":"abc-123"}"""; + Assert.That(json, Json.HasProperty("slotId", Is.Not.Empty)); + } + + [Test] + public void HasProperty_with_inner_fails_on_empty_string() + { + string json = """{"slotId":""}"""; + AssertFails(() => Assert.That(json, Json.HasProperty("slotId", Is.Not.Empty))); + } + + [Test] + public void HasProperty_with_inner_passes_on_bool_false() + { + string json = """{"adopted":false}"""; + Assert.That(json, Json.HasProperty("adopted", Is.False)); + } + + [Test] + public void HasProperty_with_inner_passes_on_int_equality() + { + string json = """{"port":54321}"""; + Assert.That(json, Json.HasProperty("port", Is.EqualTo(54321))); + } + + [Test] + public void HasProperty_with_inner_fails_on_wrong_value() + { + string json = """{"error":"rhino_not_installed"}"""; + AssertFails(() => Assert.That(json, Json.HasProperty("error", Is.EqualTo("something_else")))); + } + + [Test] + public void HasProperty_accepts_JsonElement_directly() + { + JsonElement root = JsonDocument.Parse("""{"adopted":true}""").RootElement; + Assert.That(root, Json.HasProperty("adopted", Is.True)); + } + + [Test] + public void HasProperty_unwraps_mcp_content_envelope_for_inner_json() + { + // The plugin tools return a content envelope wrapping a JSON-encoded + // string. The constraint should peek through it transparently. + string envelope = """{"content":[{"type":"text","text":"{\"count\":3,\"layer\":\"Default\"}"}]}"""; + Assert.That(envelope, Json.HasProperty("count", Is.EqualTo(3))); + Assert.That(envelope, Json.HasProperty("layer", Is.EqualTo("Default"))); + } + + [Test] + public void HasProperty_unwraps_envelope_with_multiple_text_blocks() + { + // Some servers split JSON across multiple text blocks. Concatenated + // text should still parse. + string envelope = """ + {"content":[ + {"type":"text","text":"{\"error\":\"slot_"}, + {"type":"text","text":"not_found\"}"} + ]} + """; + Assert.That(envelope, Json.HasProperty("error", Is.EqualTo("slot_not_found"))); + } + + [Test] + public void HasProperty_does_not_unwrap_when_inner_text_is_not_json() + { + // A "plain string" response (e.g. close_doc's "Document closed.") must + // not get mangled — falling back to the envelope means HasProperty for + // the inner thing simply won't find the property. + string envelope = """{"content":[{"type":"text","text":"Document closed."}]}"""; + Assert.That(envelope, Json.HasProperty("content")); + } + + [Test] + public void HasProperty_supports_nesting_via_inner_constraint() + { + // Nested objects: Json.HasProperty itself is an IResolveConstraint, so + // it composes. + string json = """{"outer":{"inner":"value"}}"""; + Assert.That(json, Json.HasProperty("outer", Json.HasProperty("inner", Is.EqualTo("value")))); + } + + [Test] + public void IsArrayOfLength_passes_for_empty_array() + { + string json = "[]"; + Assert.That(json, Json.IsArrayOfLength(0)); + } + + [Test] + public void IsArrayOfLength_passes_for_matching_length() + { + string json = "[1,2,3]"; + Assert.That(json, Json.IsArrayOfLength(3)); + } + + [Test] + public void IsArrayOfLength_fails_for_wrong_length() + { + string json = "[1,2]"; + AssertFails(() => Assert.That(json, Json.IsArrayOfLength(3))); + } + + [Test] + public void IsArrayOfLength_fails_when_root_is_not_an_array() + { + string json = """{"foo":"bar"}"""; + AssertFails(() => Assert.That(json, Json.IsArrayOfLength(0))); + } +} + +[TestFixture] +public sealed class ArgsTests +{ + [Test] + public void Of_builds_dictionary_from_tuple_pairs() + { + Dictionary args = Args.Of(("slot", "abc"), ("count", 3)); + Assert.That(args["slot"], Is.EqualTo("abc")); + Assert.That(args["count"], Is.EqualTo(3)); + Assert.That(args, Has.Count.EqualTo(2)); + } + + [Test] + public void Of_with_no_pairs_returns_empty_dictionary() + { + Dictionary args = Args.Of(); + Assert.That(args, Is.Empty); + } + + [Test] + public void Of_accepts_null_values() + { + Dictionary args = Args.Of(("optional", null)); + Assert.That(args.ContainsKey("optional"), Is.True); + Assert.That(args["optional"], Is.Null); + } +}