diff --git a/Ngentic.slnx b/Ngentic.slnx
index d2c5f7a..9dbf830 100644
--- a/Ngentic.slnx
+++ b/Ngentic.slnx
@@ -1,9 +1,9 @@
-
-
-
+
+
+
diff --git a/samples/Ngentic.Samples/FakeChatClient.cs b/samples/Ngentic.Samples/FakeChatClient.cs
deleted file mode 100644
index cc6de78..0000000
--- a/samples/Ngentic.Samples/FakeChatClient.cs
+++ /dev/null
@@ -1,46 +0,0 @@
-using Microsoft.Extensions.AI;
-
-namespace Ngentic.Samples;
-
-internal sealed class FakeChatClient : IChatClient
-{
- private readonly IReadOnlyList _scriptedCalls;
- private readonly string _finalText;
- private int _turn;
-
- public FakeChatClient(IEnumerable scriptedCalls, string finalText)
- {
- _scriptedCalls = scriptedCalls.ToList();
- _finalText = finalText;
- }
-
- public Task GetResponseAsync(
- IEnumerable messages,
- ChatOptions? options = null,
- CancellationToken cancellationToken = default)
- {
- ChatMessage reply;
- if (_turn < _scriptedCalls.Count)
- {
- FunctionCallContent call = _scriptedCalls[_turn];
- reply = new ChatMessage(ChatRole.Assistant, new List { call });
- }
- else
- {
- reply = new ChatMessage(ChatRole.Assistant, _finalText);
- }
- _turn++;
- ChatResponse response = new ChatResponse(reply);
- return Task.FromResult(response);
- }
-
- public IAsyncEnumerable GetStreamingResponseAsync(
- IEnumerable messages,
- ChatOptions? options = null,
- CancellationToken cancellationToken = default)
- => throw new NotImplementedException();
-
- public object? GetService(Type serviceType, object? serviceKey = null) => null;
-
- public void Dispose() { }
-}
diff --git a/samples/Ngentic.Samples/SampleTest.cs b/samples/Ngentic.Samples/SampleTest.cs
deleted file mode 100644
index c41e806..0000000
--- a/samples/Ngentic.Samples/SampleTest.cs
+++ /dev/null
@@ -1,49 +0,0 @@
-using Microsoft.Extensions.AI;
-using Ngentic;
-using Ngentic.NUnit;
-using NUnit.Framework;
-
-namespace Ngentic.Samples;
-
-[TestFixture]
-[McpDependency("calculator")]
-public sealed class CalculatorAgentTests : AgenticTestBase
-{
- // Sample uses a fake IChatClient + a fake MCP registry so the test compiles
- // and runs without network calls. Real usage swaps these for the official
- // Anthropic IChatClient and the ModelContextProtocol client.
- protected override void ConfigureHarness()
- {
- FakeChatClient fakeClient = new FakeChatClient(scriptedCalls: new[]
- {
- new FunctionCallContent("call-1", "mcp__calculator__add",
- new Dictionary { ["a"] = 2, ["b"] = 3 }),
- }, finalText: "2 + 3 = 5");
-
- InMemoryMcpRegistry registry = new InMemoryMcpRegistry();
- registry.Register("calculator", _ => Task.FromResult>(new List
- {
- AIFunctionFactory.Create(
- (int a, int b) => a + b,
- name: "mcp__calculator__add",
- description: "Add two integers."),
- }));
-
- UseClient(fakeClient);
- UseRegistry(registry);
- }
-
- [Test]
- public async Task Agent_uses_calculator_to_add()
- {
- AgentRun run = await Agent
- .WithSystemPrompt("You are a math assistant.")
- .WithAllowedTools("mcp__calculator__*")
- .WithMaxTurns(4)
- .RunAsync("What is 2 + 3?");
-
- Expect.That(run).CalledTool("mcp__calculator__add").AtLeastOnce();
- Expect.That(run).HasCount(1);
- Expect.That(run.FinalOutput).Contains("5");
- }
-}
diff --git a/src/Ngentic.Core/AgentRun.cs b/src/Ngentic.Core/AgentRun.cs
index 62ca273..11c0af3 100644
--- a/src/Ngentic.Core/AgentRun.cs
+++ b/src/Ngentic.Core/AgentRun.cs
@@ -8,4 +8,18 @@ public sealed class AgentRun
public required TimeSpan Duration { get; init; }
public required int Turns { get; init; }
public required bool HitTurnLimit { get; init; }
+
+ ///
+ /// Total cost reported by the agent driver, in USD. Populated by the
+ /// Claude CLI driver from the final stream-json `result` event; null
+ /// when the driver does not surface cost.
+ ///
+ public double? CostUsd { get; init; }
+
+ ///
+ /// Free-form payload for driver-specific information that does not earn
+ /// a top-level field. Populated by the Claude CLI driver with stderr,
+ /// exit code, and the raw stream-json transcript for debugging.
+ ///
+ public IReadOnlyDictionary? Metadata { get; init; }
}
diff --git a/src/Ngentic.Core/AgentRunner.cs b/src/Ngentic.Core/AgentRunner.cs
deleted file mode 100644
index 8cba00b..0000000
--- a/src/Ngentic.Core/AgentRunner.cs
+++ /dev/null
@@ -1,125 +0,0 @@
-using System.Diagnostics;
-using System.Text;
-using System.Text.Json;
-using Microsoft.Extensions.AI;
-
-namespace Ngentic;
-
-public sealed class AgentRunner
-{
- private readonly IChatClient _client;
- private readonly IList _tools;
- private readonly string? _systemPrompt;
- private readonly int _maxTurns;
-
- public AgentRunner(
- IChatClient client,
- IList tools,
- string? systemPrompt = null,
- int maxTurns = 16)
- {
- _client = client;
- _tools = tools;
- _systemPrompt = systemPrompt;
- _maxTurns = maxTurns;
- }
-
- public async Task RunAsync(string prompt, CancellationToken ct = default)
- {
- Stopwatch stopwatch = Stopwatch.StartNew();
- List messages = new();
- if (!string.IsNullOrEmpty(_systemPrompt))
- {
- messages.Add(new ChatMessage(ChatRole.System, _systemPrompt));
- }
- messages.Add(new ChatMessage(ChatRole.User, prompt));
-
- List toolCalls = new();
- ChatOptions options = new()
- {
- Tools = _tools,
- ToolMode = ChatToolMode.Auto,
- };
-
- StringBuilder finalText = new();
- int turn = 0;
- bool hitLimit = false;
-
- while (turn < _maxTurns)
- {
- turn++;
- ChatResponse response = await _client.GetResponseAsync(messages, options, ct);
- foreach (ChatMessage msg in response.Messages)
- {
- messages.Add(msg);
- foreach (AIContent content in msg.Contents)
- {
- switch (content)
- {
- case FunctionCallContent call:
- // Tool invocation is handled by FunctionInvokingChatClient downstream;
- // we record the call shape here. Result is filled below.
- toolCalls.Add(new ToolCall
- {
- Name = call.Name,
- Arguments = SerializeArgs(call.Arguments),
- Result = "",
- IsError = false,
- });
- break;
- case FunctionResultContent result:
- int idx = toolCalls.FindLastIndex(t => t.Result == "" && t.Name != "");
- if (idx >= 0)
- {
- toolCalls[idx] = new ToolCall
- {
- Name = toolCalls[idx].Name,
- Arguments = toolCalls[idx].Arguments,
- Result = result.Result?.ToString() ?? "",
- IsError = result.Exception != null,
- };
- }
- break;
- case TextContent text:
- finalText.Append(text.Text);
- break;
- }
- }
- }
-
- bool moreWork = response.Messages
- .SelectMany(m => m.Contents)
- .Any(c => c is FunctionCallContent);
-
- if (!moreWork)
- {
- break;
- }
- if (turn >= _maxTurns)
- {
- hitLimit = true;
- }
- }
-
- stopwatch.Stop();
- return new AgentRun
- {
- Prompt = prompt,
- ToolCalls = toolCalls,
- FinalOutput = finalText.ToString(),
- Duration = stopwatch.Elapsed,
- Turns = turn,
- HitTurnLimit = hitLimit,
- };
- }
-
- private static JsonElement SerializeArgs(IDictionary? args)
- {
- if (args == null || args.Count == 0)
- {
- return JsonDocument.Parse("{}").RootElement;
- }
- string json = JsonSerializer.Serialize(args);
- return JsonDocument.Parse(json).RootElement;
- }
-}
diff --git a/src/Ngentic.Core/Args.cs b/src/Ngentic.Core/Args.cs
new file mode 100644
index 0000000..d74bbc8
--- /dev/null
+++ b/src/Ngentic.Core/Args.cs
@@ -0,0 +1,20 @@
+namespace Ngentic;
+
+///
+/// Lightweight builder for the Dictionary<string, object?> argument
+/// bag that MCP client APIs typically take. Lets tests write
+/// Args.Of(("slot", slotA), ("script", "...")) instead of a full
+/// dictionary initializer.
+///
+public static class Args
+{
+ public static Dictionary Of(params (string name, object? value)[] pairs)
+ {
+ Dictionary result = new(pairs.Length);
+ foreach ((string name, object? value) in pairs)
+ {
+ result[name] = value;
+ }
+ return result;
+ }
+}
diff --git a/src/Ngentic.Core/Class1.cs b/src/Ngentic.Core/Class1.cs
deleted file mode 100644
index f5a9b57..0000000
--- a/src/Ngentic.Core/Class1.cs
+++ /dev/null
@@ -1,3 +0,0 @@
-namespace Ngentic.Internal;
-
-internal static class AssemblyMarker { }
diff --git a/src/Ngentic.Core/Expect.cs b/src/Ngentic.Core/Expect.cs
deleted file mode 100644
index 7f571c7..0000000
--- a/src/Ngentic.Core/Expect.cs
+++ /dev/null
@@ -1,159 +0,0 @@
-namespace Ngentic;
-
-public static class Expect
-{
- public static TrajectoryAssertion That(AgentRun run) => new TrajectoryAssertion(run);
- public static OutputAssertion That(string finalOutput) => new OutputAssertion(finalOutput);
-}
-
-public sealed class TrajectoryAssertion
-{
- private readonly AgentRun _run;
- private readonly List> _pending = new();
- private bool _orMode;
-
- internal TrajectoryAssertion(AgentRun run)
- {
- _run = run;
- }
-
- public TrajectoryAssertion CalledTool(string name)
- {
- AddClause(r => r.ToolCalls.Any(c => Matches(c.Name, name)));
- return this;
- }
-
- public TrajectoryAssertion DidNotCall(string name)
- {
- Flush();
- if (_run.ToolCalls.Any(c => Matches(c.Name, name)))
- {
- throw new AgenticAssertionException(
- $"Expected agent NOT to call '{name}', but it did.\n{Trace(_run)}");
- }
- return this;
- }
-
- public TrajectoryAssertion CalledInOrder(params string[] names)
- {
- Flush();
- int idx = 0;
- foreach (ToolCall call in _run.ToolCalls)
- {
- if (idx < names.Length && Matches(call.Name, names[idx]))
- {
- idx++;
- }
- }
- if (idx < names.Length)
- {
- throw new AgenticAssertionException(
- $"Expected tools called in order [{string.Join(", ", names)}], but order was not satisfied.\n{Trace(_run)}");
- }
- return this;
- }
-
- public TrajectoryAssertion Or => Pivot(true);
-
- public TrajectoryAssertion AtLeastOnce()
- {
- Flush();
- return this;
- }
-
- public TrajectoryAssertion HasCountLessThan(int limit)
- {
- Flush();
- if (_run.ToolCalls.Count >= limit)
- {
- throw new AgenticAssertionException(
- $"Expected fewer than {limit} tool calls, got {_run.ToolCalls.Count}.\n{Trace(_run)}");
- }
- return this;
- }
-
- public TrajectoryAssertion HasCount(int count)
- {
- Flush();
- if (_run.ToolCalls.Count != count)
- {
- throw new AgenticAssertionException(
- $"Expected exactly {count} tool calls, got {_run.ToolCalls.Count}.\n{Trace(_run)}");
- }
- return this;
- }
-
- private TrajectoryAssertion Pivot(bool orMode)
- {
- _orMode = orMode;
- return this;
- }
-
- private void AddClause(Predicate clause)
- {
- if (_orMode && _pending.Count > 0)
- {
- Predicate last = _pending[^1];
- _pending[^1] = r => last(r) || clause(r);
- _orMode = false;
- }
- else
- {
- _pending.Add(clause);
- }
- }
-
- private void Flush()
- {
- foreach (Predicate clause in _pending)
- {
- if (!clause(_run))
- {
- throw new AgenticAssertionException(
- $"Trajectory assertion failed.\n{Trace(_run)}");
- }
- }
- _pending.Clear();
- }
-
- private static bool Matches(string actual, string pattern)
- {
- if (pattern.EndsWith('*'))
- {
- string prefix = pattern[..^1];
- return actual.StartsWith(prefix, StringComparison.Ordinal);
- }
- return actual == pattern;
- }
-
- private static string Trace(AgentRun run)
- {
- if (run.ToolCalls.Count == 0)
- {
- return " (no tool calls)";
- }
- return " Trajectory:\n" + string.Join("\n",
- run.ToolCalls.Select((c, i) => $" [{i}] {c.Name}"));
- }
-}
-
-public sealed class OutputAssertion
-{
- private readonly string _output;
- internal OutputAssertion(string output) { _output = output; }
-
- public OutputAssertion Contains(string substring)
- {
- if (!_output.Contains(substring, StringComparison.OrdinalIgnoreCase))
- {
- throw new AgenticAssertionException(
- $"Expected output to contain '{substring}'. Output was:\n{_output}");
- }
- return this;
- }
-}
-
-public sealed class AgenticAssertionException : Exception
-{
- public AgenticAssertionException(string message) : base(message) { }
-}
diff --git a/src/Ngentic.Core/IStateScorer.cs b/src/Ngentic.Core/IStateScorer.cs
deleted file mode 100644
index dca71ef..0000000
--- a/src/Ngentic.Core/IStateScorer.cs
+++ /dev/null
@@ -1,6 +0,0 @@
-namespace Ngentic;
-
-public interface IStateScorer
-{
- Task SnapshotAsync(CancellationToken ct = default);
-}
diff --git a/src/Ngentic.Core/McpRegistry.cs b/src/Ngentic.Core/McpRegistry.cs
deleted file mode 100644
index be44b83..0000000
--- a/src/Ngentic.Core/McpRegistry.cs
+++ /dev/null
@@ -1,36 +0,0 @@
-using Microsoft.Extensions.AI;
-
-namespace Ngentic;
-
-public sealed record McpServerSpec(
- string Name,
- string Command,
- IReadOnlyList Args,
- IReadOnlyDictionary? Env = null);
-
-public interface IMcpRegistry
-{
- bool IsConfigured(string name);
- Task> GetToolsAsync(string name, CancellationToken ct = default);
-}
-
-public sealed class InMemoryMcpRegistry : IMcpRegistry
-{
- private readonly Dictionary>>> _factories = new();
-
- public void Register(string name, Func>> factory)
- {
- _factories[name] = factory;
- }
-
- public bool IsConfigured(string name) => _factories.ContainsKey(name);
-
- public Task> GetToolsAsync(string name, CancellationToken ct = default)
- {
- if (!_factories.TryGetValue(name, out Func>>? factory))
- {
- throw new InvalidOperationException($"MCP server '{name}' is not registered.");
- }
- return factory(ct);
- }
-}
diff --git a/src/Ngentic.Core/Ngentic.Core.csproj b/src/Ngentic.Core/Ngentic.Core.csproj
index 7957654..fa71b7a 100644
--- a/src/Ngentic.Core/Ngentic.Core.csproj
+++ b/src/Ngentic.Core/Ngentic.Core.csproj
@@ -6,10 +6,4 @@
enable
-
-
-
-
-
-
diff --git a/src/Ngentic.Core/ToolCall.cs b/src/Ngentic.Core/ToolCall.cs
index 878c3d1..51caf6d 100644
--- a/src/Ngentic.Core/ToolCall.cs
+++ b/src/Ngentic.Core/ToolCall.cs
@@ -4,6 +4,7 @@ namespace Ngentic;
public sealed class ToolCall
{
+ public string CallId { get; init; } = "";
public string Name { get; init; } = "";
public JsonElement Arguments { get; init; }
public string Result { get; init; } = "";
diff --git a/src/Ngentic.NUnit/AgenticTestBase.cs b/src/Ngentic.NUnit/AgenticTestBase.cs
index cdd281e..805d997 100644
--- a/src/Ngentic.NUnit/AgenticTestBase.cs
+++ b/src/Ngentic.NUnit/AgenticTestBase.cs
@@ -1,18 +1,48 @@
using System.Reflection;
-using Microsoft.Extensions.AI;
using NUnit.Framework;
namespace Ngentic.NUnit;
+///
+/// Base class for tests that drive an MCP-enabled Claude session via the
+/// local claude CLI. Subclasses implement
+/// to register the MCP servers their tests depend on. Per-test
+/// [MaxTurns] / [Model] attributes are honoured automatically.
+///
public abstract class AgenticTestBase
{
- private IChatClient? _chatClient;
- private IMcpRegistry? _registry;
-
- protected AgentBuilder Agent => new AgentBuilder(RequireClient(), RequireRegistry(), CollectMcpNames());
+ private readonly List _mcpServers = new();
+ private string? _defaultModel;
+ private double _defaultMaxBudgetUsd = 0.50;
+ private int? _currentMaxTurns;
+ private string? _currentModel;
+
+ protected AgentBuilder Agent => new AgentBuilder(
+ _mcpServers,
+ _defaultModel,
+ _defaultMaxBudgetUsd,
+ _currentMaxTurns,
+ _currentModel);
+
+ ///
+ /// Register an MCP server that the agent should be able to talk to. Called
+ /// from .
+ ///
+ protected void UseMcp(
+ string name,
+ string command,
+ IReadOnlyList? args = null,
+ IReadOnlyDictionary? env = null)
+ {
+ _mcpServers.Add(new McpServerSpec(name, command, args, env));
+ }
- protected void UseClient(IChatClient client) => _chatClient = client;
- protected void UseRegistry(IMcpRegistry registry) => _registry = registry;
+ /// Set default model + budget for every agent run in this fixture.
+ protected void UseDefaults(string? model = null, double maxBudgetUsd = 0.50)
+ {
+ _defaultModel = model;
+ _defaultMaxBudgetUsd = maxBudgetUsd;
+ }
[OneTimeSetUp]
public void __NgenticOneTimeSetUp()
@@ -21,125 +51,121 @@ public void __NgenticOneTimeSetUp()
VerifyMcpDependencies();
}
- protected abstract void ConfigureHarness();
-
- private void VerifyMcpDependencies()
- {
- IMcpRegistry registry = RequireRegistry();
- foreach (McpDependencyAttribute dep in GetType().GetCustomAttributes())
- {
- if (dep.Required && !registry.IsConfigured(dep.Name))
- {
- Assert.Fail($"Required MCP dependency '{dep.Name}' is not configured. " +
- $"Register it in {GetType().Name}.ConfigureHarness().");
- }
- }
- }
-
- private IReadOnlyList CollectMcpNames()
+ [SetUp]
+ public void __NgenticSetUp()
{
- List names = new();
- foreach (McpDependencyAttribute dep in GetType().GetCustomAttributes())
+ // Surface [MaxTurns] / [Model] from the test method so the attributes
+ // are load-bearing instead of decorative.
+ string? methodName = TestContext.CurrentContext.Test.MethodName;
+ if (methodName == null)
{
- names.Add(dep.Name);
+ _currentMaxTurns = null;
+ _currentModel = null;
+ return;
}
- return names;
+ MethodInfo? method = GetType().GetMethod(methodName,
+ BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance);
+ _currentMaxTurns = method?.GetCustomAttribute()?.Value;
+ _currentModel = method?.GetCustomAttribute()?.Id;
}
- private IChatClient RequireClient()
- {
- if (_chatClient == null)
- {
- throw new InvalidOperationException(
- "No IChatClient configured. Call UseClient(...) inside ConfigureHarness().");
- }
- return _chatClient;
- }
+ protected abstract void ConfigureHarness();
- private IMcpRegistry RequireRegistry()
+ private void VerifyMcpDependencies()
{
- if (_registry == null)
+ HashSet registered = new(_mcpServers.Select(s => s.Name), StringComparer.Ordinal);
+ foreach (McpDependencyAttribute dep in GetType().GetCustomAttributes())
{
- throw new InvalidOperationException(
- "No IMcpRegistry configured. Call UseRegistry(...) inside ConfigureHarness().");
+ if (dep.Required && !registered.Contains(dep.Name))
+ {
+ Assert.Fail($"Required MCP dependency '{dep.Name}' was not registered. " +
+ $"Call UseMcp(\"{dep.Name}\", ...) inside {GetType().Name}.ConfigureHarness().");
+ }
}
- return _registry;
}
}
public sealed class AgentBuilder
{
- private readonly IChatClient _client;
- private readonly IMcpRegistry _registry;
- private readonly IReadOnlyList _mcpNames;
- private string? _systemPrompt;
- private int _maxTurns = 16;
- private readonly List _allowedPatterns = new();
-
- internal AgentBuilder(IChatClient client, IMcpRegistry registry, IReadOnlyList mcpNames)
+ private readonly IReadOnlyList _mcpServers;
+ private readonly double _defaultMaxBudgetUsd;
+ private readonly string? _defaultModel;
+ private string? _systemPromptAppend;
+ private string? _model;
+ private double? _maxBudgetUsd;
+ private readonly List _allowedTools = new();
+ // _maxTurns is preserved for [MaxTurns] reporting but the CLI uses --max-budget-usd
+ // rather than a hard turn cap; it gets stashed into AgentRun.Metadata.
+ private readonly int? _maxTurns;
+
+ internal AgentBuilder(
+ IReadOnlyList mcpServers,
+ string? defaultModel,
+ double defaultMaxBudgetUsd,
+ int? maxTurnsFromAttribute,
+ string? modelFromAttribute)
{
- _client = client;
- _registry = registry;
- _mcpNames = mcpNames;
+ _mcpServers = mcpServers;
+ _defaultModel = defaultModel;
+ _defaultMaxBudgetUsd = defaultMaxBudgetUsd;
+ _maxTurns = maxTurnsFromAttribute;
+ _model = modelFromAttribute;
}
public AgentBuilder WithSystemPrompt(string prompt)
{
- _systemPrompt = prompt;
+ _systemPromptAppend = prompt;
+ return this;
+ }
+
+ public AgentBuilder WithModel(string modelId)
+ {
+ _model = modelId;
return this;
}
- public AgentBuilder WithMaxTurns(int turns)
+ public AgentBuilder WithMaxBudgetUsd(double usd)
{
- _maxTurns = turns;
+ _maxBudgetUsd = usd;
return this;
}
public AgentBuilder WithAllowedTools(params string[] patterns)
{
- _allowedPatterns.AddRange(patterns);
+ _allowedTools.AddRange(patterns);
return this;
}
public async Task RunAsync(string prompt, CancellationToken ct = default)
{
- List tools = new();
- foreach (string mcpName in _mcpNames)
- {
- IList mcpTools = await _registry.GetToolsAsync(mcpName, ct);
- foreach (AITool tool in mcpTools)
- {
- if (IsAllowed(tool.Name))
- {
- tools.Add(tool);
- }
- }
- }
+ ClaudeAgentRequest request = new(
+ Prompt: prompt,
+ McpServers: _mcpServers,
+ AllowedTools: _allowedTools,
+ SystemPromptAppend: _systemPromptAppend,
+ Model: _model ?? _defaultModel,
+ MaxBudgetUsd: _maxBudgetUsd ?? _defaultMaxBudgetUsd);
- AgentRunner runner = new AgentRunner(_client, tools, _systemPrompt, _maxTurns);
- return await runner.RunAsync(prompt, ct);
- }
+ AgentRun run = await ClaudeAgent.RunAsync(request, ct).ConfigureAwait(false);
- private bool IsAllowed(string toolName)
- {
- if (_allowedPatterns.Count == 0)
- {
- return true;
- }
- foreach (string pattern in _allowedPatterns)
+ if (_maxTurns is not null)
{
- if (pattern.EndsWith('*'))
+ Dictionary meta = new(run.Metadata ?? new Dictionary())
{
- if (toolName.StartsWith(pattern[..^1], StringComparison.Ordinal))
- {
- return true;
- }
- }
- else if (toolName == pattern)
+ ["max_turns_attribute"] = _maxTurns.Value,
+ };
+ run = new AgentRun
{
- return true;
- }
+ Prompt = run.Prompt,
+ ToolCalls = run.ToolCalls,
+ FinalOutput = run.FinalOutput,
+ Duration = run.Duration,
+ Turns = run.Turns,
+ HitTurnLimit = run.Turns > _maxTurns.Value,
+ CostUsd = run.CostUsd,
+ Metadata = meta,
+ };
}
- return false;
+ return run;
}
}
diff --git a/src/Ngentic.NUnit/Class1.cs b/src/Ngentic.NUnit/Class1.cs
deleted file mode 100644
index 2c06bbe..0000000
--- a/src/Ngentic.NUnit/Class1.cs
+++ /dev/null
@@ -1,3 +0,0 @@
-namespace Ngentic.NUnit.Internal;
-
-internal static class AssemblyMarker { }
diff --git a/src/Ngentic.NUnit/ClaudeAgent.cs b/src/Ngentic.NUnit/ClaudeAgent.cs
new file mode 100644
index 0000000..6983ab3
--- /dev/null
+++ b/src/Ngentic.NUnit/ClaudeAgent.cs
@@ -0,0 +1,138 @@
+using System.Diagnostics;
+using System.Globalization;
+using System.Runtime.InteropServices;
+using System.Text;
+using System.Text.Json;
+
+namespace Ngentic.NUnit;
+
+///
+/// Drives a real Claude session via the local claude CLI in headless
+/// mode (-p --output-format stream-json). The CLI handles MCP wiring
+/// itself — we hand it an inline --mcp-config built from the supplied
+/// s. Tests that depend on this should be
+/// [Explicit]: real LLM calls cost subscription quota and are
+/// non-deterministic.
+///
+public static class ClaudeAgent
+{
+ public static async Task RunAsync(
+ ClaudeAgentRequest request,
+ CancellationToken ct = default)
+ {
+ Stopwatch stopwatch = Stopwatch.StartNew();
+
+ string mcpConfigJson = BuildMcpConfig(request.McpServers);
+ ProcessStartInfo psi = new()
+ {
+ FileName = ResolveClaudeCli(),
+ RedirectStandardOutput = true,
+ RedirectStandardError = true,
+ UseShellExecute = false,
+ WorkingDirectory = request.WorkingDirectory ?? Environment.CurrentDirectory,
+ };
+ psi.ArgumentList.Add("-p");
+ psi.ArgumentList.Add(request.Prompt);
+ psi.ArgumentList.Add("--output-format");
+ psi.ArgumentList.Add("stream-json");
+ psi.ArgumentList.Add("--verbose");
+ psi.ArgumentList.Add("--strict-mcp-config");
+ psi.ArgumentList.Add("--mcp-config");
+ psi.ArgumentList.Add(mcpConfigJson);
+ psi.ArgumentList.Add("--permission-mode");
+ psi.ArgumentList.Add("bypassPermissions");
+ psi.ArgumentList.Add("--max-budget-usd");
+ psi.ArgumentList.Add(request.MaxBudgetUsd.ToString("0.00", CultureInfo.InvariantCulture));
+ psi.ArgumentList.Add("--no-session-persistence");
+ if (request.AllowedTools.Count > 0)
+ {
+ psi.ArgumentList.Add("--allowedTools");
+ psi.ArgumentList.Add(string.Join(" ", request.AllowedTools));
+ }
+ if (request.SystemPromptAppend is not null)
+ {
+ psi.ArgumentList.Add("--append-system-prompt");
+ psi.ArgumentList.Add(request.SystemPromptAppend);
+ }
+ if (request.Model is not null)
+ {
+ psi.ArgumentList.Add("--model");
+ psi.ArgumentList.Add(request.Model);
+ }
+
+ using Process proc = Process.Start(psi)
+ ?? throw new InvalidOperationException("Failed to start `claude` CLI.");
+
+ // Pipes can deadlock if either stream fills its buffer while we wait
+ // for the process — drain both concurrently.
+ Task> stdoutTask = ReadLinesAsync(proc.StandardOutput, ct);
+ Task stderrTask = proc.StandardError.ReadToEndAsync(ct);
+
+ try
+ {
+ await proc.WaitForExitAsync(ct).ConfigureAwait(false);
+ }
+ catch (OperationCanceledException)
+ {
+ try { proc.Kill(entireProcessTree: true); } catch { /* best effort */ }
+ throw;
+ }
+
+ List stdoutLines = await stdoutTask.ConfigureAwait(false);
+ string stderr = await stderrTask.ConfigureAwait(false);
+
+ stopwatch.Stop();
+
+ return TrajectoryParser.Parse(
+ request.Prompt,
+ stdoutLines,
+ stderr,
+ proc.ExitCode,
+ stopwatch.Elapsed);
+ }
+
+ private static string BuildMcpConfig(IReadOnlyList servers)
+ {
+ Dictionary map = new();
+ foreach (McpServerSpec spec in servers)
+ {
+ map[spec.Name] = new Dictionary
+ {
+ ["command"] = spec.Command,
+ ["args"] = spec.Args ?? (IReadOnlyList)Array.Empty(),
+ ["env"] = spec.Env,
+ };
+ }
+ return JsonSerializer.Serialize(new { mcpServers = map });
+ }
+
+ private static string ResolveClaudeCli()
+ {
+ string? overrideExe = Environment.GetEnvironmentVariable("CLAUDE_CLI");
+ if (!string.IsNullOrEmpty(overrideExe))
+ {
+ return overrideExe;
+ }
+ return RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "claude.exe" : "claude";
+ }
+
+ private static async Task> ReadLinesAsync(StreamReader reader, CancellationToken ct)
+ {
+ List lines = new();
+ string? line;
+ while ((line = await reader.ReadLineAsync(ct).ConfigureAwait(false)) is not null)
+ {
+ lines.Add(line);
+ }
+ return lines;
+ }
+}
+
+public sealed record ClaudeAgentRequest(
+ string Prompt,
+ IReadOnlyList McpServers,
+ IReadOnlyList AllowedTools,
+ string? SystemPromptAppend = null,
+ string? Model = null,
+ double MaxBudgetUsd = 0.50,
+ string? WorkingDirectory = null);
diff --git a/src/Ngentic.NUnit/Constraints.cs b/src/Ngentic.NUnit/Constraints.cs
new file mode 100644
index 0000000..83a0527
--- /dev/null
+++ b/src/Ngentic.NUnit/Constraints.cs
@@ -0,0 +1,244 @@
+using System.Text.Json;
+using NUnit.Framework.Constraints;
+
+namespace Ngentic.NUnit;
+
+///
+/// Entry point for trajectory constraints. Use with NUnit's Assert.That:
+/// Assert.That(run, Did.CallTool("mcp__rhino__spawn_slot"));
+///
+public static class Did
+{
+ public static CalledToolConstraint CallTool(string name) => new CalledToolConstraint(name);
+ public static NotCalledToolConstraint NotCallTool(string name) => new NotCalledToolConstraint(name);
+ public static CalledInOrderConstraint CallToolsInOrder(params string[] names) => new CalledInOrderConstraint(names);
+ public static ToolCallCountConstraint MakeToolCalls => new ToolCallCountConstraint();
+}
+
+public abstract class AgentRunConstraint : Constraint
+{
+ private string _description = "";
+ public override string Description => _description;
+ protected void SetDescription(string text) => _description = text;
+
+ protected static bool NameMatches(string actual, string pattern)
+ {
+ if (pattern.EndsWith('*'))
+ {
+ return actual.StartsWith(pattern[..^1], StringComparison.Ordinal);
+ }
+ return actual == pattern;
+ }
+}
+
+public sealed class CalledToolConstraint : AgentRunConstraint
+{
+ private readonly string _pattern;
+ private int _expectedCount = -1;
+ private readonly List> _argChecks = new();
+ private readonly List _argDescriptions = new();
+
+ public CalledToolConstraint(string pattern)
+ {
+ _pattern = pattern;
+ SetDescription($"agent called tool '{pattern}'");
+ }
+
+ public CalledToolConstraint AtLeastOnce()
+ {
+ _expectedCount = -1;
+ SetDescription($"agent called tool '{_pattern}' at least once");
+ return this;
+ }
+
+ public CalledToolConstraint Times(int count)
+ {
+ _expectedCount = count;
+ SetDescription($"agent called tool '{_pattern}' exactly {count} time(s)");
+ return this;
+ }
+
+ public CalledToolConstraint WithArg(string name, object expected)
+ {
+ _argChecks.Add(args =>
+ {
+ if (!args.TryGetProperty(name, out JsonElement value))
+ {
+ return false;
+ }
+ return JsonEquals(value, expected);
+ });
+ _argDescriptions.Add($"{name}={expected}");
+ SetDescription($"agent called tool '{_pattern}' with {string.Join(", ", _argDescriptions)}");
+ return this;
+ }
+
+ public CalledToolConstraint WithArgs(Func predicate, string? describe = null)
+ {
+ _argChecks.Add(predicate);
+ _argDescriptions.Add(describe ?? "");
+ SetDescription($"agent called tool '{_pattern}' with {string.Join(", ", _argDescriptions)}");
+ return this;
+ }
+
+ public override ConstraintResult ApplyTo(TActual actual)
+ {
+ if (actual is not AgentRun run)
+ {
+ return new ConstraintResult(this, actual, ConstraintStatus.Error);
+ }
+
+ List matchingFully = run.ToolCalls
+ .Where(c => NameMatches(c.Name, _pattern))
+ .Where(c => _argChecks.All(check => check(c.Arguments)))
+ .ToList();
+
+ bool success = _expectedCount < 0
+ ? matchingFully.Count > 0
+ : matchingFully.Count == _expectedCount;
+
+ return new AgentRunConstraintResult(this, run, success);
+ }
+
+ private static bool JsonEquals(JsonElement value, object expected)
+ {
+ return expected switch
+ {
+ int i => value.ValueKind == JsonValueKind.Number && value.TryGetInt32(out int v) && v == i,
+ long l => value.ValueKind == JsonValueKind.Number && value.TryGetInt64(out long v) && v == l,
+ double d => value.ValueKind == JsonValueKind.Number && Math.Abs(value.GetDouble() - d) < 1e-9,
+ bool b => value.ValueKind == (b ? JsonValueKind.True : JsonValueKind.False),
+ string s => value.ValueKind == JsonValueKind.String && value.GetString() == s,
+ null => value.ValueKind == JsonValueKind.Null,
+ _ => value.ToString() == expected.ToString(),
+ };
+ }
+}
+
+public sealed class NotCalledToolConstraint : AgentRunConstraint
+{
+ private readonly string _pattern;
+
+ public NotCalledToolConstraint(string pattern)
+ {
+ _pattern = pattern;
+ SetDescription($"agent did NOT call tool '{pattern}'");
+ }
+
+ public override ConstraintResult ApplyTo(TActual actual)
+ {
+ if (actual is not AgentRun run)
+ {
+ return new ConstraintResult(this, actual, ConstraintStatus.Error);
+ }
+ bool success = !run.ToolCalls.Any(c => NameMatches(c.Name, _pattern));
+ return new AgentRunConstraintResult(this, run, success);
+ }
+}
+
+public sealed class CalledInOrderConstraint : AgentRunConstraint
+{
+ private readonly string[] _names;
+
+ public CalledInOrderConstraint(string[] names)
+ {
+ _names = names;
+ SetDescription($"agent called tools in order: [{string.Join(", ", names)}]");
+ }
+
+ public override ConstraintResult ApplyTo(TActual actual)
+ {
+ if (actual is not AgentRun run)
+ {
+ return new ConstraintResult(this, actual, ConstraintStatus.Error);
+ }
+ int idx = 0;
+ foreach (ToolCall call in run.ToolCalls)
+ {
+ if (idx < _names.Length && NameMatches(call.Name, _names[idx]))
+ {
+ idx++;
+ }
+ }
+ return new AgentRunConstraintResult(this, run, idx >= _names.Length);
+ }
+}
+
+public sealed class ToolCallCountConstraint : AgentRunConstraint
+{
+ private int _min = -1;
+ private int _max = -1;
+ private int _exact = -1;
+
+ public ToolCallCountConstraint()
+ {
+ SetDescription("agent made tool calls");
+ }
+
+ public ToolCallCountConstraint Exactly(int count)
+ {
+ _exact = count;
+ SetDescription($"agent made exactly {count} tool call(s)");
+ return this;
+ }
+
+ public ToolCallCountConstraint AtLeast(int count)
+ {
+ _min = count;
+ SetDescription($"agent made at least {count} tool call(s)");
+ return this;
+ }
+
+ public ToolCallCountConstraint AtMost(int count)
+ {
+ _max = count;
+ SetDescription($"agent made at most {count} tool call(s)");
+ return this;
+ }
+
+ public override ConstraintResult ApplyTo(TActual actual)
+ {
+ if (actual is not AgentRun run)
+ {
+ return new ConstraintResult(this, actual, ConstraintStatus.Error);
+ }
+ int n = run.ToolCalls.Count;
+ bool ok = true;
+ if (_exact >= 0) ok &= n == _exact;
+ if (_min >= 0) ok &= n >= _min;
+ if (_max >= 0) ok &= n <= _max;
+ return new AgentRunConstraintResult(this, run, ok);
+ }
+}
+
+internal sealed class AgentRunConstraintResult : ConstraintResult
+{
+ private readonly AgentRun _run;
+
+ public AgentRunConstraintResult(IConstraint constraint, AgentRun run, bool success)
+ : base(constraint, FormatActual(run), success)
+ {
+ _run = run;
+ }
+
+ private static string FormatActual(AgentRun run)
+ {
+ if (run.ToolCalls.Count == 0)
+ {
+ return "(no tool calls)";
+ }
+ return $"trajectory of {run.ToolCalls.Count} call(s): " +
+ string.Join(", ", run.ToolCalls.Select(c => c.Name));
+ }
+
+ public override void WriteMessageTo(MessageWriter writer)
+ {
+ writer.WriteLine($" Expected: {Description}");
+ writer.WriteLine($" But was: trajectory of {_run.ToolCalls.Count} call(s)");
+ for (int i = 0; i < _run.ToolCalls.Count; i++)
+ {
+ ToolCall c = _run.ToolCalls[i];
+ writer.WriteLine($" [{i}] {c.Name}({c.Arguments.GetRawText()})");
+ }
+ }
+}
diff --git a/src/Ngentic.NUnit/JsonConstraints.cs b/src/Ngentic.NUnit/JsonConstraints.cs
new file mode 100644
index 0000000..6599b5c
--- /dev/null
+++ b/src/Ngentic.NUnit/JsonConstraints.cs
@@ -0,0 +1,244 @@
+using System.Text;
+using System.Text.Json;
+using NUnit.Framework.Constraints;
+
+namespace Ngentic.NUnit;
+
+///
+/// Entry point for JSON-aware constraints. Use with NUnit's Assert.That:
+///
+/// Assert.That(json, Json.HasProperty("error", Is.EqualTo("slot_not_found")));
+/// Assert.That(json, Json.HasProperty("slotId", Is.Not.Empty));
+/// Assert.That(json, Json.IsArrayOfLength(0));
+///
+/// Accepts either a raw JSON string or a . If the
+/// payload is an MCP content envelope ({"content":[{"type":"text","text":"..."}]})
+/// the inner text is auto-unwrapped, so callers never have to walk
+/// content[0].text by hand.
+///
+public static class Json
+{
+ public static JsonHasPropertyConstraint HasProperty(string name) => new(name, inner: null);
+
+ public static JsonHasPropertyConstraint HasProperty(string name, IResolveConstraint inner)
+ => new(name, inner);
+
+ public static JsonArrayLengthConstraint IsArrayOfLength(int length) => new(length);
+}
+
+public sealed class JsonHasPropertyConstraint : Constraint
+{
+ private readonly string _name;
+ private readonly IResolveConstraint? _inner;
+
+ public JsonHasPropertyConstraint(string name, IResolveConstraint? inner)
+ {
+ _name = name;
+ _inner = inner;
+ }
+
+ public override string Description => _inner is null
+ ? $"JSON has property '{_name}'"
+ : $"JSON property '{_name}' {_inner.Resolve().Description}";
+
+ public override ConstraintResult ApplyTo(TActual actual)
+ {
+ if (!JsonHelpers.TryGetElement(actual, out JsonElement root))
+ {
+ return new ConstraintResult(this, actual, ConstraintStatus.Error);
+ }
+
+ JsonElement unwrapped = JsonHelpers.UnwrapMcpEnvelope(root);
+
+ if (unwrapped.ValueKind != JsonValueKind.Object
+ || !unwrapped.TryGetProperty(_name, out JsonElement value))
+ {
+ return new JsonPropertyMissingResult(this, unwrapped, _name);
+ }
+
+ if (_inner is null)
+ {
+ return new JsonPropertyFoundResult(this, unwrapped, _name, value);
+ }
+
+ object? clr = JsonHelpers.ToClr(value);
+ IConstraint resolved = _inner.Resolve();
+ ConstraintResult innerResult = resolved.ApplyTo(clr);
+ return new JsonPropertyInnerResult(this, unwrapped, _name, value, innerResult);
+ }
+}
+
+public sealed class JsonArrayLengthConstraint : Constraint
+{
+ private readonly int _expected;
+
+ public JsonArrayLengthConstraint(int expected)
+ {
+ _expected = expected;
+ }
+
+ public override string Description => $"JSON array of length {_expected}";
+
+ public override ConstraintResult ApplyTo(TActual actual)
+ {
+ if (!JsonHelpers.TryGetElement(actual, out JsonElement root))
+ {
+ return new ConstraintResult(this, actual, ConstraintStatus.Error);
+ }
+
+ JsonElement unwrapped = JsonHelpers.UnwrapMcpEnvelope(root);
+
+ if (unwrapped.ValueKind != JsonValueKind.Array)
+ {
+ return new ConstraintResult(this, $"{unwrapped.ValueKind}: {unwrapped.GetRawText()}", false);
+ }
+
+ int actualLength = unwrapped.GetArrayLength();
+ return new ConstraintResult(
+ this,
+ $"array of length {actualLength}: {unwrapped.GetRawText()}",
+ actualLength == _expected);
+ }
+}
+
+internal static class JsonHelpers
+{
+ public static bool TryGetElement(object? actual, out JsonElement element)
+ {
+ switch (actual)
+ {
+ case JsonElement e:
+ element = e;
+ return true;
+ case string s:
+ try
+ {
+ element = JsonDocument.Parse(s).RootElement;
+ return true;
+ }
+ catch (JsonException)
+ {
+ element = default;
+ return false;
+ }
+ default:
+ element = default;
+ return false;
+ }
+ }
+
+ // Recognises the MCP tool-result envelope shape and digs out the inner
+ // payload. The envelope wraps a JSON-encoded string inside a content
+ // block: {"content":[{"type":"text","text":"{...}"}]}. We concatenate
+ // the text fields (some tools split into multiple blocks) and try to
+ // parse the result as JSON; if parsing fails we leave the envelope
+ // alone, so non-JSON text responses don't get mangled. Recurses to
+ // handle nested envelopes if a server ever produces them.
+ public static JsonElement UnwrapMcpEnvelope(JsonElement element)
+ {
+ if (element.ValueKind != JsonValueKind.Object)
+ {
+ return element;
+ }
+ if (!element.TryGetProperty("content", out JsonElement content)
+ || content.ValueKind != JsonValueKind.Array
+ || content.GetArrayLength() == 0)
+ {
+ return element;
+ }
+
+ StringBuilder sb = new();
+ foreach (JsonElement block in content.EnumerateArray())
+ {
+ if (block.ValueKind != JsonValueKind.Object
+ || !block.TryGetProperty("text", out JsonElement text)
+ || text.ValueKind != JsonValueKind.String)
+ {
+ return element;
+ }
+ sb.Append(text.GetString());
+ }
+
+ try
+ {
+ JsonElement inner = JsonDocument.Parse(sb.ToString()).RootElement;
+ return UnwrapMcpEnvelope(inner);
+ }
+ catch (JsonException)
+ {
+ return element;
+ }
+ }
+
+ public static object? ToClr(JsonElement value)
+ {
+ return value.ValueKind switch
+ {
+ JsonValueKind.String => value.GetString(),
+ JsonValueKind.True => true,
+ JsonValueKind.False => false,
+ JsonValueKind.Null => null,
+ JsonValueKind.Number => value.TryGetInt64(out long l) ? l : value.GetDouble(),
+ _ => value,
+ };
+ }
+}
+
+internal sealed class JsonPropertyMissingResult : ConstraintResult
+{
+ private readonly JsonElement _root;
+ private readonly string _name;
+
+ public JsonPropertyMissingResult(IConstraint constraint, JsonElement root, string name)
+ : base(constraint, "property missing", isSuccess: false)
+ {
+ _root = root;
+ _name = name;
+ }
+
+ public override void WriteMessageTo(MessageWriter writer)
+ {
+ writer.WriteLine($" Expected: {Description}");
+ writer.WriteLine($" But was: property '{_name}' not found");
+ writer.WriteLine($" JSON: {_root.GetRawText()}");
+ }
+}
+
+internal sealed class JsonPropertyFoundResult : ConstraintResult
+{
+ public JsonPropertyFoundResult(IConstraint constraint, JsonElement root, string name, JsonElement value)
+ : base(constraint, value.GetRawText(), isSuccess: true)
+ {
+ _ = root;
+ _ = name;
+ }
+}
+
+internal sealed class JsonPropertyInnerResult : ConstraintResult
+{
+ private readonly JsonElement _root;
+ private readonly string _name;
+ private readonly JsonElement _propertyValue;
+ private readonly ConstraintResult _inner;
+
+ public JsonPropertyInnerResult(
+ IConstraint constraint,
+ JsonElement root,
+ string name,
+ JsonElement propertyValue,
+ ConstraintResult inner)
+ : base(constraint, inner.ActualValue, inner.IsSuccess)
+ {
+ _root = root;
+ _name = name;
+ _propertyValue = propertyValue;
+ _inner = inner;
+ }
+
+ public override void WriteMessageTo(MessageWriter writer)
+ {
+ writer.WriteLine($" Expected: {Description}");
+ writer.WriteLine($" But was: property '{_name}' = {_propertyValue.GetRawText()}");
+ writer.WriteLine($" JSON: {_root.GetRawText()}");
+ }
+}
diff --git a/src/Ngentic.NUnit/McpServerSpec.cs b/src/Ngentic.NUnit/McpServerSpec.cs
new file mode 100644
index 0000000..030bac7
--- /dev/null
+++ b/src/Ngentic.NUnit/McpServerSpec.cs
@@ -0,0 +1,12 @@
+namespace Ngentic.NUnit;
+
+///
+/// Declarative description of an MCP server to hand to the local Claude CLI
+/// via --mcp-config. Consumer code (e.g. a rhino-specific test) builds
+/// these in their fixture setup and passes them to .
+///
+public sealed record McpServerSpec(
+ string Name,
+ string Command,
+ IReadOnlyList? Args = null,
+ IReadOnlyDictionary? Env = null);
diff --git a/src/Ngentic.NUnit/TrajectoryParser.cs b/src/Ngentic.NUnit/TrajectoryParser.cs
new file mode 100644
index 0000000..6ac4df7
--- /dev/null
+++ b/src/Ngentic.NUnit/TrajectoryParser.cs
@@ -0,0 +1,229 @@
+using System.Text;
+using System.Text.Json;
+
+namespace Ngentic.NUnit;
+
+///
+/// Parses claude -p --output-format stream-json output into a
+/// . Exposed publicly so consumers can replay captured
+/// transcripts in unit tests without spawning the CLI.
+///
+public static class TrajectoryParser
+{
+ public static AgentRun Parse(
+ string prompt,
+ IReadOnlyList stdoutLines,
+ string stderr,
+ int exitCode,
+ TimeSpan duration)
+ {
+ List calls = new();
+ Dictionary indexByCallId = new();
+ StringBuilder finalText = new();
+ int assistantMessages = 0;
+ bool isError = false;
+ string? errorPayload = null;
+ double? costUsd = null;
+
+ foreach (string line in stdoutLines)
+ {
+ if (string.IsNullOrWhiteSpace(line))
+ {
+ continue;
+ }
+ JsonDocument doc;
+ try
+ {
+ doc = JsonDocument.Parse(line);
+ }
+ catch (JsonException)
+ {
+ continue;
+ }
+
+ using (doc)
+ {
+ JsonElement root = doc.RootElement;
+ if (!root.TryGetProperty("type", out JsonElement typeEl))
+ {
+ continue;
+ }
+ switch (typeEl.GetString())
+ {
+ case "assistant":
+ assistantMessages++;
+ ExtractAssistantContent(root, calls, indexByCallId, finalText);
+ break;
+ case "user":
+ ExtractToolResults(root, calls, indexByCallId);
+ break;
+ case "result":
+ if (root.TryGetProperty("is_error", out JsonElement errEl)
+ && errEl.ValueKind == JsonValueKind.True)
+ {
+ isError = true;
+ errorPayload = root.TryGetProperty("result", out JsonElement r)
+ ? r.GetString()
+ : null;
+ }
+ if (root.TryGetProperty("total_cost_usd", out JsonElement costEl)
+ && costEl.TryGetDouble(out double c))
+ {
+ costUsd = c;
+ }
+ break;
+ }
+ }
+ }
+
+ Dictionary metadata = new()
+ {
+ ["exit_code"] = exitCode,
+ ["stderr"] = stderr,
+ ["raw_stdout_lines"] = stdoutLines,
+ ["is_error"] = isError,
+ ["error_payload"] = errorPayload,
+ };
+
+ return new AgentRun
+ {
+ Prompt = prompt,
+ ToolCalls = calls,
+ FinalOutput = finalText.ToString(),
+ Duration = duration,
+ Turns = Math.Max(1, assistantMessages),
+ HitTurnLimit = false,
+ CostUsd = costUsd,
+ Metadata = metadata,
+ };
+ }
+
+ private static void ExtractAssistantContent(
+ JsonElement root,
+ List calls,
+ Dictionary indexByCallId,
+ StringBuilder finalText)
+ {
+ if (!root.TryGetProperty("message", out JsonElement msg)
+ || !msg.TryGetProperty("content", out JsonElement content)
+ || content.ValueKind != JsonValueKind.Array)
+ {
+ return;
+ }
+
+ foreach (JsonElement block in content.EnumerateArray())
+ {
+ if (!block.TryGetProperty("type", out JsonElement blockTypeEl))
+ {
+ continue;
+ }
+ switch (blockTypeEl.GetString())
+ {
+ case "text":
+ if (block.TryGetProperty("text", out JsonElement textEl))
+ {
+ finalText.Append(textEl.GetString());
+ }
+ break;
+ case "tool_use":
+ string? id = block.TryGetProperty("id", out JsonElement idEl)
+ ? idEl.GetString()
+ : null;
+ string? name = block.TryGetProperty("name", out JsonElement nameEl)
+ ? nameEl.GetString()
+ : null;
+ JsonElement input = block.TryGetProperty("input", out JsonElement inEl)
+ ? inEl.Clone()
+ : JsonDocument.Parse("{}").RootElement;
+ if (name is null)
+ {
+ break;
+ }
+ string callId = id ?? Guid.NewGuid().ToString("N");
+ indexByCallId[callId] = calls.Count;
+ calls.Add(new ToolCall
+ {
+ CallId = callId,
+ Name = name,
+ Arguments = input,
+ Result = "",
+ IsError = false,
+ });
+ break;
+ }
+ }
+ }
+
+ private static void ExtractToolResults(
+ JsonElement root,
+ List calls,
+ Dictionary indexByCallId)
+ {
+ if (!root.TryGetProperty("message", out JsonElement msg)
+ || !msg.TryGetProperty("content", out JsonElement content)
+ || content.ValueKind != JsonValueKind.Array)
+ {
+ return;
+ }
+
+ foreach (JsonElement block in content.EnumerateArray())
+ {
+ if (!block.TryGetProperty("type", out JsonElement blockTypeEl)
+ || blockTypeEl.GetString() != "tool_result")
+ {
+ continue;
+ }
+
+ string? id = block.TryGetProperty("tool_use_id", out JsonElement idEl)
+ ? idEl.GetString()
+ : null;
+ if (id is null || !indexByCallId.TryGetValue(id, out int idx))
+ {
+ continue;
+ }
+
+ string resultText = ExtractToolResultText(block);
+ bool isErr = block.TryGetProperty("is_error", out JsonElement errEl)
+ && errEl.ValueKind == JsonValueKind.True;
+
+ ToolCall prev = calls[idx];
+ calls[idx] = new ToolCall
+ {
+ CallId = prev.CallId,
+ Name = prev.Name,
+ Arguments = prev.Arguments,
+ Result = resultText,
+ IsError = isErr,
+ };
+ }
+ }
+
+ private static string ExtractToolResultText(JsonElement block)
+ {
+ if (!block.TryGetProperty("content", out JsonElement contentEl))
+ {
+ return "";
+ }
+ if (contentEl.ValueKind == JsonValueKind.String)
+ {
+ return contentEl.GetString() ?? "";
+ }
+ if (contentEl.ValueKind != JsonValueKind.Array)
+ {
+ return contentEl.GetRawText();
+ }
+ StringBuilder sb = new();
+ foreach (JsonElement c in contentEl.EnumerateArray())
+ {
+ if (c.ValueKind == JsonValueKind.String)
+ {
+ sb.Append(c.GetString());
+ }
+ else if (c.TryGetProperty("text", out JsonElement t))
+ {
+ sb.Append(t.GetString());
+ }
+ }
+ return sb.ToString();
+ }
+}
diff --git a/tests/Ngentic.Tests/ConstraintTests.cs b/tests/Ngentic.Tests/ConstraintTests.cs
new file mode 100644
index 0000000..78fcd19
--- /dev/null
+++ b/tests/Ngentic.Tests/ConstraintTests.cs
@@ -0,0 +1,108 @@
+using Ngentic.NUnit;
+using NUnit.Framework;
+
+namespace Ngentic.Tests;
+
+[TestFixture]
+public sealed class ConstraintTests
+{
+ private static void AssertFails(Action test)
+ {
+ Assert.Throws(test);
+ }
+
+ [Test]
+ public void CallTool_passes_when_tool_was_called()
+ {
+ AgentRun run = TestHelpers.MakeRun(("foo", null));
+ Assert.That(run, Did.CallTool("foo"));
+ }
+
+ [Test]
+ public void CallTool_fails_when_tool_was_not_called()
+ {
+ AgentRun run = TestHelpers.MakeRun(("bar", null));
+ AssertFails(() => Assert.That(run, Did.CallTool("foo")));
+ }
+
+ [Test]
+ public void CallTool_supports_wildcard_suffix()
+ {
+ AgentRun run = TestHelpers.MakeRun(("mcp__rhino__spawn_slot", null));
+ Assert.That(run, Did.CallTool("mcp__rhino__*"));
+ }
+
+ [Test]
+ public void CallTool_WithArg_matches_int_argument()
+ {
+ AgentRun run = TestHelpers.MakeRun(("add", new { a = 2, b = 3 }));
+ Assert.That(run, Did.CallTool("add").WithArg("a", 2).WithArg("b", 3));
+ }
+
+ [Test]
+ public void CallTool_WithArg_fails_on_wrong_value()
+ {
+ AgentRun run = TestHelpers.MakeRun(("add", new { a = 2, b = 3 }));
+ AssertFails(() => Assert.That(run, Did.CallTool("add").WithArg("a", 99)));
+ }
+
+ [Test]
+ public void CallTool_Times_counts_invocations()
+ {
+ AgentRun run = TestHelpers.MakeRun(("foo", null), ("foo", null), ("bar", null));
+ Assert.That(run, Did.CallTool("foo").Times(2));
+ }
+
+ [Test]
+ public void NotCallTool_passes_when_tool_absent()
+ {
+ AgentRun run = TestHelpers.MakeRun(("foo", null));
+ Assert.That(run, Did.NotCallTool("bar"));
+ }
+
+ [Test]
+ public void NotCallTool_fails_when_tool_present()
+ {
+ AgentRun run = TestHelpers.MakeRun(("foo", null));
+ AssertFails(() => Assert.That(run, Did.NotCallTool("foo")));
+ }
+
+ [Test]
+ public void CallToolsInOrder_matches_subsequence()
+ {
+ AgentRun run = TestHelpers.MakeRun(("a", null), ("b", null), ("c", null), ("d", null));
+ Assert.That(run, Did.CallToolsInOrder("a", "c"));
+ }
+
+ [Test]
+ public void CallToolsInOrder_fails_when_out_of_order()
+ {
+ AgentRun run = TestHelpers.MakeRun(("c", null), ("a", null));
+ AssertFails(() => Assert.That(run, Did.CallToolsInOrder("a", "c")));
+ }
+
+ [Test]
+ public void MakeToolCalls_Exactly_counts_total()
+ {
+ AgentRun run = TestHelpers.MakeRun(("a", null), ("b", null));
+ Assert.That(run, Did.MakeToolCalls.Exactly(2));
+ }
+
+ [Test]
+ public void MakeToolCalls_AtMost_caps_count()
+ {
+ AgentRun run = TestHelpers.MakeRun(("a", null), ("b", null));
+ Assert.That(run, Did.MakeToolCalls.AtMost(5));
+ AssertFails(() => Assert.That(run, Did.MakeToolCalls.AtMost(1)));
+ }
+
+ [Test]
+ public void Constraint_failure_shows_actual_trajectory_in_message()
+ {
+ AgentRun run = TestHelpers.MakeRun(("actual_tool", null));
+ AssertionException ex = Assert.Throws(
+ (Action)(() => Assert.That(run, Did.CallTool("missing_tool"))))!;
+ Assert.That(ex.Message, Does.Contain("actual_tool"));
+ Assert.That(ex.Message, Does.Contain("missing_tool"));
+ }
+}
diff --git a/samples/Ngentic.Samples/GlobalUsings.cs b/tests/Ngentic.Tests/GlobalUsings.cs
similarity index 100%
rename from samples/Ngentic.Samples/GlobalUsings.cs
rename to tests/Ngentic.Tests/GlobalUsings.cs
diff --git a/tests/Ngentic.Tests/JsonConstraintTests.cs b/tests/Ngentic.Tests/JsonConstraintTests.cs
new file mode 100644
index 0000000..a52eb02
--- /dev/null
+++ b/tests/Ngentic.Tests/JsonConstraintTests.cs
@@ -0,0 +1,183 @@
+using System.Text.Json;
+using Ngentic.NUnit;
+using NUnit.Framework;
+
+namespace Ngentic.Tests;
+
+[TestFixture]
+public sealed class JsonConstraintTests
+{
+ private static void AssertFails(Action test)
+ {
+ Assert.Throws(test);
+ }
+
+ [Test]
+ public void HasProperty_passes_when_property_exists()
+ {
+ string json = """{"slotId":"abc-123","adopted":false}""";
+ Assert.That(json, Json.HasProperty("slotId"));
+ }
+
+ [Test]
+ public void HasProperty_fails_when_property_is_missing()
+ {
+ string json = """{"slotId":"abc-123"}""";
+ AssertFails(() => Assert.That(json, Json.HasProperty("error")));
+ }
+
+ [Test]
+ public void HasProperty_with_inner_passes_on_string_equality()
+ {
+ string json = """{"error":"rhino_not_installed","message":"version 'garbage' was not found"}""";
+ Assert.That(json, Json.HasProperty("error", Is.EqualTo("rhino_not_installed")));
+ }
+
+ [Test]
+ public void HasProperty_with_inner_passes_on_substring()
+ {
+ string json = """{"error":"rhino_not_installed","message":"version 'garbage' was not found"}""";
+ Assert.That(json, Json.HasProperty("message", Does.Contain("garbage")));
+ }
+
+ [Test]
+ public void HasProperty_with_inner_passes_on_non_empty()
+ {
+ string json = """{"slotId":"abc-123"}""";
+ Assert.That(json, Json.HasProperty("slotId", Is.Not.Empty));
+ }
+
+ [Test]
+ public void HasProperty_with_inner_fails_on_empty_string()
+ {
+ string json = """{"slotId":""}""";
+ AssertFails(() => Assert.That(json, Json.HasProperty("slotId", Is.Not.Empty)));
+ }
+
+ [Test]
+ public void HasProperty_with_inner_passes_on_bool_false()
+ {
+ string json = """{"adopted":false}""";
+ Assert.That(json, Json.HasProperty("adopted", Is.False));
+ }
+
+ [Test]
+ public void HasProperty_with_inner_passes_on_int_equality()
+ {
+ string json = """{"port":54321}""";
+ Assert.That(json, Json.HasProperty("port", Is.EqualTo(54321)));
+ }
+
+ [Test]
+ public void HasProperty_with_inner_fails_on_wrong_value()
+ {
+ string json = """{"error":"rhino_not_installed"}""";
+ AssertFails(() => Assert.That(json, Json.HasProperty("error", Is.EqualTo("something_else"))));
+ }
+
+ [Test]
+ public void HasProperty_accepts_JsonElement_directly()
+ {
+ JsonElement root = JsonDocument.Parse("""{"adopted":true}""").RootElement;
+ Assert.That(root, Json.HasProperty("adopted", Is.True));
+ }
+
+ [Test]
+ public void HasProperty_unwraps_mcp_content_envelope_for_inner_json()
+ {
+ // The plugin tools return a content envelope wrapping a JSON-encoded
+ // string. The constraint should peek through it transparently.
+ string envelope = """{"content":[{"type":"text","text":"{\"count\":3,\"layer\":\"Default\"}"}]}""";
+ Assert.That(envelope, Json.HasProperty("count", Is.EqualTo(3)));
+ Assert.That(envelope, Json.HasProperty("layer", Is.EqualTo("Default")));
+ }
+
+ [Test]
+ public void HasProperty_unwraps_envelope_with_multiple_text_blocks()
+ {
+ // Some servers split JSON across multiple text blocks. Concatenated
+ // text should still parse.
+ string envelope = """
+ {"content":[
+ {"type":"text","text":"{\"error\":\"slot_"},
+ {"type":"text","text":"not_found\"}"}
+ ]}
+ """;
+ Assert.That(envelope, Json.HasProperty("error", Is.EqualTo("slot_not_found")));
+ }
+
+ [Test]
+ public void HasProperty_does_not_unwrap_when_inner_text_is_not_json()
+ {
+ // A "plain string" response (e.g. close_doc's "Document closed.") must
+ // not get mangled — falling back to the envelope means HasProperty for
+ // the inner thing simply won't find the property.
+ string envelope = """{"content":[{"type":"text","text":"Document closed."}]}""";
+ Assert.That(envelope, Json.HasProperty("content"));
+ }
+
+ [Test]
+ public void HasProperty_supports_nesting_via_inner_constraint()
+ {
+ // Nested objects: Json.HasProperty itself is an IResolveConstraint, so
+ // it composes.
+ string json = """{"outer":{"inner":"value"}}""";
+ Assert.That(json, Json.HasProperty("outer", Json.HasProperty("inner", Is.EqualTo("value"))));
+ }
+
+ [Test]
+ public void IsArrayOfLength_passes_for_empty_array()
+ {
+ string json = "[]";
+ Assert.That(json, Json.IsArrayOfLength(0));
+ }
+
+ [Test]
+ public void IsArrayOfLength_passes_for_matching_length()
+ {
+ string json = "[1,2,3]";
+ Assert.That(json, Json.IsArrayOfLength(3));
+ }
+
+ [Test]
+ public void IsArrayOfLength_fails_for_wrong_length()
+ {
+ string json = "[1,2]";
+ AssertFails(() => Assert.That(json, Json.IsArrayOfLength(3)));
+ }
+
+ [Test]
+ public void IsArrayOfLength_fails_when_root_is_not_an_array()
+ {
+ string json = """{"foo":"bar"}""";
+ AssertFails(() => Assert.That(json, Json.IsArrayOfLength(0)));
+ }
+}
+
+[TestFixture]
+public sealed class ArgsTests
+{
+ [Test]
+ public void Of_builds_dictionary_from_tuple_pairs()
+ {
+ Dictionary args = Args.Of(("slot", "abc"), ("count", 3));
+ Assert.That(args["slot"], Is.EqualTo("abc"));
+ Assert.That(args["count"], Is.EqualTo(3));
+ Assert.That(args, Has.Count.EqualTo(2));
+ }
+
+ [Test]
+ public void Of_with_no_pairs_returns_empty_dictionary()
+ {
+ Dictionary args = Args.Of();
+ Assert.That(args, Is.Empty);
+ }
+
+ [Test]
+ public void Of_accepts_null_values()
+ {
+ Dictionary args = Args.Of(("optional", null));
+ Assert.That(args.ContainsKey("optional"), Is.True);
+ Assert.That(args["optional"], Is.Null);
+ }
+}
diff --git a/samples/Ngentic.Samples/Ngentic.Samples.csproj b/tests/Ngentic.Tests/Ngentic.Tests.csproj
similarity index 79%
rename from samples/Ngentic.Samples/Ngentic.Samples.csproj
rename to tests/Ngentic.Tests/Ngentic.Tests.csproj
index bb444f9..2ef6655 100644
--- a/samples/Ngentic.Samples/Ngentic.Samples.csproj
+++ b/tests/Ngentic.Tests/Ngentic.Tests.csproj
@@ -2,9 +2,9 @@
net8.0
- enable
+ 12
enable
-
+ enable
false
true
@@ -18,7 +18,8 @@
-
+
+
diff --git a/tests/Ngentic.Tests/TestHelpers.cs b/tests/Ngentic.Tests/TestHelpers.cs
new file mode 100644
index 0000000..e6f98a3
--- /dev/null
+++ b/tests/Ngentic.Tests/TestHelpers.cs
@@ -0,0 +1,42 @@
+using System.Text.Json;
+
+namespace Ngentic.Tests;
+
+internal static class TestHelpers
+{
+ public static AgentRun MakeRun(params (string name, object? args)[] calls)
+ {
+ List toolCalls = new();
+ int i = 0;
+ foreach ((string name, object? args) in calls)
+ {
+ toolCalls.Add(new ToolCall
+ {
+ CallId = $"call-{i++}",
+ Name = name,
+ Arguments = ToJson(args),
+ Result = "",
+ IsError = false,
+ });
+ }
+ return new AgentRun
+ {
+ Prompt = "test",
+ ToolCalls = toolCalls,
+ FinalOutput = "",
+ Duration = TimeSpan.Zero,
+ Turns = 1,
+ HitTurnLimit = false,
+ };
+ }
+
+ private static JsonElement ToJson(object? args)
+ {
+ if (args == null)
+ {
+ return JsonDocument.Parse("{}").RootElement;
+ }
+ string json = JsonSerializer.Serialize(args);
+ return JsonDocument.Parse(json).RootElement;
+ }
+}
diff --git a/tests/Ngentic.Tests/TrajectoryParserTests.cs b/tests/Ngentic.Tests/TrajectoryParserTests.cs
new file mode 100644
index 0000000..433cac4
--- /dev/null
+++ b/tests/Ngentic.Tests/TrajectoryParserTests.cs
@@ -0,0 +1,138 @@
+using Ngentic.NUnit;
+using NUnit.Framework;
+
+namespace Ngentic.Tests;
+
+///
+/// Exercises against captured stream-json
+/// transcripts. No CLI process required — just feed in the lines and assert.
+///
+[TestFixture]
+public sealed class TrajectoryParserTests
+{
+ [Test]
+ public void Parses_a_single_tool_call_and_result()
+ {
+ string[] lines =
+ {
+ """{"type":"assistant","message":{"content":[{"type":"tool_use","id":"toolu_1","name":"mcp__rhino__close_slot","input":{"slot_id":"abc"}}]}}""",
+ """{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"toolu_1","content":"{\"closed\":false,\"error\":\"slot_not_found\"}"}]}}""",
+ """{"type":"assistant","message":{"content":[{"type":"text","text":"slot_not_found, as you said."}]}}""",
+ """{"type":"result","total_cost_usd":0.0123,"is_error":false}""",
+ };
+
+ AgentRun run = TrajectoryParser.Parse(
+ prompt: "Close slot abc",
+ stdoutLines: lines,
+ stderr: "",
+ exitCode: 0,
+ duration: TimeSpan.FromSeconds(2));
+
+ Assert.That(run.ToolCalls, Has.Count.EqualTo(1));
+ Assert.That(run.ToolCalls[0].Name, Is.EqualTo("mcp__rhino__close_slot"));
+ Assert.That(run.ToolCalls[0].CallId, Is.EqualTo("toolu_1"));
+ Assert.That(run.ToolCalls[0].Result, Does.Contain("slot_not_found"));
+ Assert.That(run.FinalOutput, Does.Contain("slot_not_found"));
+ Assert.That(run.CostUsd, Is.EqualTo(0.0123).Within(1e-9));
+ Assert.That(run.Turns, Is.EqualTo(2));
+ }
+
+ [Test]
+ public void Pairs_call_and_result_by_tool_use_id()
+ {
+ string[] lines =
+ {
+ """{"type":"assistant","message":{"content":[{"type":"tool_use","id":"a","name":"foo","input":{}}]}}""",
+ """{"type":"assistant","message":{"content":[{"type":"tool_use","id":"b","name":"foo","input":{}}]}}""",
+ """{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"b","content":"second"}]}}""",
+ """{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"a","content":"first"}]}}""",
+ """{"type":"result","is_error":false}""",
+ };
+
+ AgentRun run = TrajectoryParser.Parse("t", lines, "", 0, TimeSpan.Zero);
+
+ Assert.That(run.ToolCalls, Has.Count.EqualTo(2));
+ Assert.That(run.ToolCalls[0].CallId, Is.EqualTo("a"));
+ Assert.That(run.ToolCalls[0].Result, Is.EqualTo("first"));
+ Assert.That(run.ToolCalls[1].CallId, Is.EqualTo("b"));
+ Assert.That(run.ToolCalls[1].Result, Is.EqualTo("second"));
+ }
+
+ [Test]
+ public void Handles_array_content_in_tool_results()
+ {
+ string[] lines =
+ {
+ """{"type":"assistant","message":{"content":[{"type":"tool_use","id":"x","name":"foo","input":{}}]}}""",
+ """{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"x","content":[{"type":"text","text":"part 1"},{"type":"text","text":" part 2"}]}]}}""",
+ """{"type":"result"}""",
+ };
+
+ AgentRun run = TrajectoryParser.Parse("t", lines, "", 0, TimeSpan.Zero);
+
+ Assert.That(run.ToolCalls[0].Result, Is.EqualTo("part 1 part 2"));
+ }
+
+ [Test]
+ public void Marks_tool_errors_via_is_error_flag()
+ {
+ string[] lines =
+ {
+ """{"type":"assistant","message":{"content":[{"type":"tool_use","id":"x","name":"foo","input":{}}]}}""",
+ """{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"x","content":"boom","is_error":true}]}}""",
+ """{"type":"result"}""",
+ };
+
+ AgentRun run = TrajectoryParser.Parse("t", lines, "", 0, TimeSpan.Zero);
+
+ Assert.That(run.ToolCalls[0].IsError, Is.True);
+ }
+
+ [Test]
+ public void Surfaces_stderr_and_exit_code_in_metadata()
+ {
+ AgentRun run = TrajectoryParser.Parse(
+ prompt: "x",
+ stdoutLines: Array.Empty(),
+ stderr: "warning: foo",
+ exitCode: 7,
+ duration: TimeSpan.Zero);
+
+ Assert.That(run.Metadata, Is.Not.Null);
+ Assert.That(run.Metadata!["exit_code"], Is.EqualTo(7));
+ Assert.That(run.Metadata!["stderr"], Is.EqualTo("warning: foo"));
+ }
+
+ [Test]
+ public void Ignores_non_json_lines()
+ {
+ string[] lines =
+ {
+ "this is not json",
+ "",
+ """{"type":"assistant","message":{"content":[{"type":"text","text":"hi"}]}}""",
+ "garbage",
+ """{"type":"result","total_cost_usd":0.01}""",
+ };
+
+ AgentRun run = TrajectoryParser.Parse("t", lines, "", 0, TimeSpan.Zero);
+
+ Assert.That(run.FinalOutput, Is.EqualTo("hi"));
+ Assert.That(run.CostUsd, Is.EqualTo(0.01).Within(1e-9));
+ }
+
+ [Test]
+ public void Captures_tool_call_arguments_as_JsonElement()
+ {
+ string[] lines =
+ {
+ """{"type":"assistant","message":{"content":[{"type":"tool_use","id":"x","name":"add","input":{"a":2,"b":3}}]}}""",
+ """{"type":"result"}""",
+ };
+
+ AgentRun run = TrajectoryParser.Parse("t", lines, "", 0, TimeSpan.Zero);
+
+ Assert.That(run.ToolCalls[0].Arguments.GetProperty("a").GetInt32(), Is.EqualTo(2));
+ Assert.That(run.ToolCalls[0].Arguments.GetProperty("b").GetInt32(), Is.EqualTo(3));
+ }
+}
diff --git a/tests/Ngentic.Tests/UnitTest1.cs b/tests/Ngentic.Tests/UnitTest1.cs
new file mode 100644
index 0000000..0210ba8
--- /dev/null
+++ b/tests/Ngentic.Tests/UnitTest1.cs
@@ -0,0 +1,4 @@
+// Placeholder — real tests live in sibling files.
+namespace Ngentic.Tests.Internal;
+
+internal static class AssemblyMarker { }