From 9ca7e8998572b68bcf50783951c1cff591c1503b Mon Sep 17 00:00:00 2001 From: Abbinayaa Subramanian Date: Thu, 21 May 2026 11:59:45 -0700 Subject: [PATCH 01/27] Adding validation subproject covering T0-T3 checks --- CHANGELOG.md | 2 + src/Microsoft.Agents.A365.DevTools.Cli.sln | 12 + .../Commands/ValidateCommand.cs | 611 ++++++++++++++ .../Constants/CommandNames.cs | 1 + .../Microsoft.Agents.A365.DevTools.Cli.csproj | 4 + .../Program.cs | 2 + .../HttpListenerBotCallbackReceiver.cs | 242 ++++++ .../Services/IBotCallbackReceiver.cs | 36 + .../Services/Requirements/RequirementCheck.cs | 14 +- .../Requirements/RequirementCheckResult.cs | 216 +++-- .../ConversationRequirementCheck.cs | 782 ++++++++++++++++++ .../LocalRuntimeRequirementCheck.cs | 378 +++++++++ .../PowerShellModulesRequirementCheck.cs | 1 - .../ProjectBuildRequirementCheck.cs | 203 +++++ .../ToolingManifestRequirementCheck.cs | 91 ++ ...oft.Agents.A365.DevTools.Validation.csproj | 8 + .../ValidateReport.cs | 254 ++++++ .../ValidationContracts.cs | 183 ++++ .../Commands/ValidateCommandTests.cs | 190 +++++ ...soft.Agents.A365.DevTools.Cli.Tests.csproj | 1 + .../ConversationRequirementCheckTests.cs | 703 ++++++++++++++++ .../LocalRuntimeRequirementCheckTests.cs | 281 +++++++ .../ProjectBuildRequirementCheckTests.cs | 308 +++++++ .../ToolingManifestRequirementCheckTests.cs | 224 +++++ ...ents.A365.DevTools.Validation.Tests.csproj | 18 + .../Validation/ValidationContractsTests.cs | 40 + 26 files changed, 4719 insertions(+), 86 deletions(-) create mode 100644 src/Microsoft.Agents.A365.DevTools.Cli/Commands/ValidateCommand.cs create mode 100644 src/Microsoft.Agents.A365.DevTools.Cli/Services/HttpListenerBotCallbackReceiver.cs create mode 100644 src/Microsoft.Agents.A365.DevTools.Cli/Services/IBotCallbackReceiver.cs create mode 100644 src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/ConversationRequirementCheck.cs create mode 100644 src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/LocalRuntimeRequirementCheck.cs create mode 100644 src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/ProjectBuildRequirementCheck.cs create mode 100644 src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/ToolingManifestRequirementCheck.cs create mode 100644 src/Microsoft.Agents.A365.DevTools.Validation/Microsoft.Agents.A365.DevTools.Validation.csproj create mode 100644 src/Microsoft.Agents.A365.DevTools.Validation/ValidateReport.cs create mode 100644 src/Microsoft.Agents.A365.DevTools.Validation/ValidationContracts.cs create mode 100644 src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Commands/ValidateCommandTests.cs create mode 100644 src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/ConversationRequirementCheckTests.cs create mode 100644 src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/LocalRuntimeRequirementCheckTests.cs create mode 100644 src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/ProjectBuildRequirementCheckTests.cs create mode 100644 src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/ToolingManifestRequirementCheckTests.cs create mode 100644 src/Tests/Microsoft.Agents.A365.DevTools.Validation.Tests/Microsoft.Agents.A365.DevTools.Validation.Tests.csproj create mode 100644 src/Tests/Microsoft.Agents.A365.DevTools.Validation.Tests/Validation/ValidationContractsTests.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index ee3005d5..867e7d8e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,8 @@ Agents provisioned before this release need `Agent365.Observability.OtelWrite` g **Option B — CLI** (`a365 setup admin`) has been removed in this release. Use Option A above, or copy the PowerShell instructions printed in the `a365 setup all` summary output. ### Added +- `a365 validate` — validates the local `a365.config.json` plus prerequisite checks without making changes. Reports missing or invalid config, then runs the existing setup prerequisite checks so users can catch problems before starting a setup workflow. +- New `Microsoft.Agents.A365.DevTools.Validation` subproject for reusable validation contracts and helpers. - `logs export [command] [--output ]` — exports a redacted copy of a CLI diagnostic log safe to share with Microsoft support. Redacts JWT tokens, email addresses, OS-path usernames, and tenant-specific GUIDs; replaces identical values with consistent aliases so log correlation is preserved. Preserves diagnostic IDs that aren't sensitive but are useful for debugging — `TraceId`, `CorrelationId`, Microsoft Graph `request-id` and `client-request-id` values, and well-known public Microsoft / Agent 365 resource appIds (such as the Microsoft Graph appId `00000003-0000-0000-c000-000000000000`). Omit `[command]` to export all available logs at once. - `setup blueprint --show-secret` — displays the blueprint client secret stored in `a365.generated.config.json` in plaintext without re-running any setup steps. On Windows, decryption requires the same machine and user account that ran setup (DPAPI). When no secret is found, the command prints instructions to run `a365 setup blueprint --agent-name `. - Blueprint client secret is now printed to the terminal at creation time with a "copy this value now" warning. Use `a365 setup blueprint --show-secret` to retrieve it afterwards. diff --git a/src/Microsoft.Agents.A365.DevTools.Cli.sln b/src/Microsoft.Agents.A365.DevTools.Cli.sln index 61867381..16aa439f 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli.sln +++ b/src/Microsoft.Agents.A365.DevTools.Cli.sln @@ -5,8 +5,12 @@ VisualStudioVersion = 17.0.31903.59 MinimumVisualStudioVersion = 10.0.40219.1 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Agents.A365.DevTools.Cli", "Microsoft.Agents.A365.DevTools.Cli\Microsoft.Agents.A365.DevTools.Cli.csproj", "{B46A53DE-09FD-8E1D-83E8-F1DC1DB32397}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Agents.A365.DevTools.Validation", "Microsoft.Agents.A365.DevTools.Validation\Microsoft.Agents.A365.DevTools.Validation.csproj", "{A1A5F7A7-1F3A-4D32-8B28-A2B95B2F1F11}" +EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Agents.A365.DevTools.Cli.Tests", "Tests\Microsoft.Agents.A365.DevTools.Cli.Tests\Microsoft.Agents.A365.DevTools.Cli.Tests.csproj", "{ACDAF3A6-DD04-9652-7C37-1C5EC3FA9FB5}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Agents.A365.DevTools.Validation.Tests", "Tests\Microsoft.Agents.A365.DevTools.Validation.Tests\Microsoft.Agents.A365.DevTools.Validation.Tests.csproj", "{DAD2B4B5-2E68-46EA-97FC-30A0A1E0FE91}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -17,10 +21,18 @@ Global {B46A53DE-09FD-8E1D-83E8-F1DC1DB32397}.Debug|Any CPU.Build.0 = Debug|Any CPU {B46A53DE-09FD-8E1D-83E8-F1DC1DB32397}.Release|Any CPU.ActiveCfg = Release|Any CPU {B46A53DE-09FD-8E1D-83E8-F1DC1DB32397}.Release|Any CPU.Build.0 = Release|Any CPU + {A1A5F7A7-1F3A-4D32-8B28-A2B95B2F1F11}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A1A5F7A7-1F3A-4D32-8B28-A2B95B2F1F11}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A1A5F7A7-1F3A-4D32-8B28-A2B95B2F1F11}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A1A5F7A7-1F3A-4D32-8B28-A2B95B2F1F11}.Release|Any CPU.Build.0 = Release|Any CPU {ACDAF3A6-DD04-9652-7C37-1C5EC3FA9FB5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {ACDAF3A6-DD04-9652-7C37-1C5EC3FA9FB5}.Debug|Any CPU.Build.0 = Debug|Any CPU {ACDAF3A6-DD04-9652-7C37-1C5EC3FA9FB5}.Release|Any CPU.ActiveCfg = Release|Any CPU {ACDAF3A6-DD04-9652-7C37-1C5EC3FA9FB5}.Release|Any CPU.Build.0 = Release|Any CPU + {DAD2B4B5-2E68-46EA-97FC-30A0A1E0FE91}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {DAD2B4B5-2E68-46EA-97FC-30A0A1E0FE91}.Debug|Any CPU.Build.0 = Debug|Any CPU + {DAD2B4B5-2E68-46EA-97FC-30A0A1E0FE91}.Release|Any CPU.ActiveCfg = Release|Any CPU + {DAD2B4B5-2E68-46EA-97FC-30A0A1E0FE91}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Commands/ValidateCommand.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Commands/ValidateCommand.cs new file mode 100644 index 00000000..23f27ecf --- /dev/null +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Commands/ValidateCommand.cs @@ -0,0 +1,611 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.Agents.A365.DevTools.Cli.Constants; +using Microsoft.Agents.A365.DevTools.Cli.Exceptions; +using Microsoft.Agents.A365.DevTools.Cli.Models; +using Microsoft.Agents.A365.DevTools.Cli.Services; +using Microsoft.Agents.A365.DevTools.Cli.Services.Requirements; +using Microsoft.Agents.A365.DevTools.Cli.Services.Requirements.RequirementChecks; +using Microsoft.Extensions.Logging; +using System.CommandLine; +using System.CommandLine.Invocation; +using System.Text.Json; +using Microsoft.Agents.A365.DevTools.Validation; + +namespace Microsoft.Agents.A365.DevTools.Cli.Commands; + +/// +/// Validates the local Agent 365 CLI configuration and prerequisite state. +/// Writes a structured report to a365.validate.json. +/// +public sealed class ValidateCommand +{ + internal const string ReportFileName = "a365.validate.json"; + + // Status markers — use characters supported across Windows/macOS/Linux terminals + private const string PassMark = "\u221A"; // √ (square root, same as Windows renders for checkmark) + private const string FailMark = "X"; + private const string SkipMark = "-"; + + private static readonly JsonSerializerOptions ReportSerializerOptions = new() + { + WriteIndented = true, + DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.Never + }; + + public static Command CreateCommand( + ILogger logger, + IConfigService configService, + PlatformDetector? platformDetector = null, + CommandExecutor? commandExecutor = null, + IProcessService? processService = null, + IEnumerable? requirementChecksOverride = null) + { + var command = new Command(CommandNames.Validate, + "Validate the local Agent 365 CLI configuration and prerequisite state\n" + + "Checks config validity and code health. Run 'a365 setup all' before using this command."); + + var playgroundOption = new Option( + "--playground", + "Launch AgentsPlayground after automated conversation turns for interactive testing"); + command.AddOption(playgroundOption); + + command.SetHandler(async (InvocationContext context) => + { + var ct = context.GetCancellationToken(); + var cwd = Directory.GetCurrentDirectory(); + var configPath = Path.Combine(cwd, ConfigConstants.DefaultConfigFileName); + var report = new ValidateReport(); + var launchPlayground = context.ParseResult.GetValueForOption(playgroundOption); + + try + { + // Phase 1: Config validation (structural tier) + var (config, configOk) = await ValidateConfigAsync(configService, configPath, logger, report); + + if (!configOk || config is null) + { + report.Summary = new SummaryResult { Ok = false, Blocker = "structural" }; + context.ExitCode = 1; + return; + } + + logger.LogDebug("Configuration file validated successfully"); + + // Populate agent info from config + var projectPath = ResolveProjectPath(config); + var language = platformDetector?.Detect(projectPath); + report.Agent = new AgentInfo + { + Path = projectPath, + Language = language is not null and not ProjectPlatform.Unknown + ? language.Value.ToString().ToLowerInvariant() + : null + }; + + // Phase 2: Run requirement checks and map to tiers + var checks = requirementChecksOverride?.ToList() + ?? BuildValidationChecks(platformDetector, commandExecutor, processService, includeConversation: false); + + var results = await RunChecksDetailedAsync(checks, config, logger, ct); + MapResultsToTiers(results, report); + + // Phase 2b: Run conversation check only if boot tier passed + var bootPassed = report.Tiers.Boot is { Skipped: false, Ok: true }; + if (bootPassed && requirementChecksOverride is null) + { + var conversationChecks = BuildConversationChecks(platformDetector, processService, launchPlayground); + if (conversationChecks.Count > 0) + { + var conversationResults = await RunChecksDetailedAsync(conversationChecks, config, logger, ct); + MapResultsToTiers(conversationResults, report); + results.AddRange(conversationResults); + } + } + else if (!bootPassed && report.Tiers.Boot is not { Skipped: true }) + { + report.Tiers.Conversation = new ConversationTierResult + { + Skipped = true, + Reason = "boot tier failed" + }; + } + + // For test overrides, also map conversation checks + if (requirementChecksOverride is not null) + { + // Conversation checks from override are already in results via MapResultsToTiers + } + + // Phase 3: Build summary — any failed check is a blocker + var anyFailed = results.Any(r => !r.Result.Passed); + var blocker = FindBlocker(report.Tiers); + report.Summary = new SummaryResult + { + Ok = !anyFailed && blocker is null, + Blocker = blocker + }; + + context.ExitCode = report.Summary.Ok ? 0 : 1; + + // Print formatted summary to console + PrintSummary(report, logger); + } + finally + { + await WriteReportAsync(report, cwd, logger); + } + }); + + return command; + } + + private static async Task<(Agent365Config? Config, bool Ok)> ValidateConfigAsync( + IConfigService configService, + string configPath, + ILogger logger, + ValidateReport report) + { + var structuralChecks = new List(); + + if (!await configService.ConfigExistsAsync(configPath)) + { + structuralChecks.Add(new StructuralCheck { Name = "config-exists", Ok = false, Message = "a365.config.json not found" }); + report.Tiers.Structural = new StructuralTierResult { Ok = false, Checks = structuralChecks }; + + logger.LogError("Fail: Configuration File"); + logger.LogInformation(" {Message}", "a365.config.json not found in the current directory."); + logger.LogInformation(""); + logger.LogInformation(" {Step}", "Run 'a365 setup all --agent-name ' to set up first."); + return (null, false); + } + + structuralChecks.Add(new StructuralCheck { Name = "config-exists", Ok = true }); + + Agent365Config config; + try + { + config = await configService.LoadAsync(configPath); + } + catch (ConfigurationValidationException ex) + { + structuralChecks.Add(new StructuralCheck { Name = "config-format", Ok = false, Message = ex.IssueDescription }); + report.Tiers.Structural = new StructuralTierResult { Ok = false, Checks = structuralChecks }; + logger.LogError("Fail: Configuration File"); + logger.LogInformation(" {Message}", ex.IssueDescription); + return (null, false); + } + catch (ConfigFileNotFoundException ex) + { + structuralChecks.Add(new StructuralCheck { Name = "config-format", Ok = false, Message = ex.IssueDescription }); + report.Tiers.Structural = new StructuralTierResult { Ok = false, Checks = structuralChecks }; + logger.LogError("Fail: Configuration File"); + logger.LogInformation(" {Message}", ex.IssueDescription); + return (null, false); + } + catch (JsonException) + { + structuralChecks.Add(new StructuralCheck { Name = "config-format", Ok = false, Message = ErrorMessages.InvalidConfigFormat }); + report.Tiers.Structural = new StructuralTierResult { Ok = false, Checks = structuralChecks }; + logger.LogError("Fail: Configuration File"); + logger.LogInformation(" {Message}", ErrorMessages.InvalidConfigFormat); + return (null, false); + } + + structuralChecks.Add(new StructuralCheck { Name = "config-format", Ok = true }); + + var configErrors = config.Validate(); + if (configErrors.Count > 0) + { + structuralChecks.Add(new StructuralCheck + { + Name = "config-schema", + Ok = false, + Message = string.Join("; ", configErrors) + }); + report.Tiers.Structural = new StructuralTierResult { Ok = false, Checks = structuralChecks }; + + logger.LogError("Fail: Configuration File"); + foreach (var error in configErrors) + { + logger.LogInformation(" {Message}", error); + } + logger.LogInformation(""); + logger.LogInformation(" {Step}", "Fix the configuration errors in a365.config.json and try again."); + return (null, false); + } + + structuralChecks.Add(new StructuralCheck { Name = "config-schema", Ok = true }); + + report.Tiers.Structural = new StructuralTierResult + { + Ok = true, + Checks = structuralChecks + }; + + return (config, true); + } + + private static async Task> RunChecksDetailedAsync( + List checks, + Agent365Config config, + ILogger logger, + CancellationToken ct) + { + var results = new List<(IRequirementCheck Check, RequirementCheckResult Result)>(); + + logger.LogDebug("Checking requirements..."); + + foreach (var check in checks) + { + var result = await check.CheckAsync(config, logger, ct); + results.Add((check, result)); + } + + var passed = results.Count(r => r.Result.Passed && !r.Result.IsWarning); + var warnings = results.Count(r => r.Result.IsWarning); + var failed = results.Count(r => !r.Result.Passed); + + logger.LogDebug("Requirements: {Passed} passed, {Warning} warnings, {Failed} failed", + passed, warnings, failed); + + return results; + } + + private static void MapResultsToTiers( + List<(IRequirementCheck Check, RequirementCheckResult Result)> results, + ValidateReport report) + { + foreach (var (check, result) in results) + { + switch (check) + { + case ToolingManifestRequirementCheck: + // Add to structural tier + var structural = report.Tiers.Structural; + if (structural.Skipped) + { + structural = new StructuralTierResult { Ok = true, Checks = new List() }; + report.Tiers.Structural = structural; + } + structural.Checks ??= new List(); + structural.Checks.Add(new StructuralCheck + { + Name = "tooling-manifest", + Ok = result.Passed, + Message = result.Passed ? result.Details : result.ErrorMessage + }); + if (!result.Passed) + { + structural.Ok = false; + } + break; + + case ProjectBuildRequirementCheck: + if (result.IsWarning) + { + report.Tiers.Build = new BuildTierResult + { + Skipped = true, + Reason = result.ErrorMessage ?? result.Details + }; + } + else + { + report.Tiers.Build = new BuildTierResult + { + Ok = result.Passed, + Log = result.Metadata?.Log, + ExitCode = result.Metadata?.ExitCode + }; + } + break; + + case LocalRuntimeRequirementCheck: + if (result.IsWarning) + { + report.Tiers.Boot = new BootTierResult + { + Skipped = true, + Reason = result.ErrorMessage ?? result.Details + }; + } + else + { + report.Tiers.Boot = new BootTierResult + { + Ok = result.Passed, + Port = result.Metadata?.Port, + BootMs = result.Metadata?.BootMs + }; + } + break; + + case ConversationRequirementCheck: + if (result.IsWarning) + { + report.Tiers.Conversation = new ConversationTierResult + { + Skipped = true, + Reason = result.ErrorMessage ?? result.Details + }; + } + else + { + report.Tiers.Conversation = new ConversationTierResult + { + Ok = result.Passed, + PlaygroundLaunched = result.Metadata?.PlaygroundLaunched, + Turns = result.Metadata?.Turns?.Select(t => new ConversationTurnResult + { + Input = t.Input, + StatusCode = t.StatusCode, + ResponseSnippet = t.ResponseSnippet, + LatencyMs = t.LatencyMs, + Ok = t.Ok, + Error = t.Error, + AgentResponded = t.AgentResponded, + AgentResponseText = t.AgentResponseText + }).ToList() + }; + } + break; + } + } + } + + private static string? FindBlocker(ValidationTiers tiers) + { + if (tiers.Structural is { Skipped: false, Ok: false }) return "structural"; + if (tiers.Build is { Skipped: false, Ok: false }) return "build"; + if (tiers.Boot is { Skipped: false, Ok: false }) return "boot"; + if (tiers.Conversation is { Skipped: false, Ok: false }) return "conversation"; + if (tiers.Telemetry is { Skipped: false, Ok: false }) return "telemetry"; + if (tiers.Blueprint is { Skipped: false, Ok: false }) return "blueprint"; + if (tiers.Mac is { Skipped: false, Ok: false }) return "mac"; + if (tiers.M365 is { Skipped: false, Ok: false }) return "m365"; + if (tiers.Judge is { Skipped: false, Ok: false }) return "judge"; + return null; + } + + internal static void PrintSummary(ValidateReport report, ILogger logger) + { + logger.LogInformation(""); + + // Group related tiers into user-facing rows + var rows = BuildDisplayRows(report); + + int passCount = 0; + int failCount = 0; + int localChecks = 0; + + foreach (var row in rows) + { + if (row.Skipped) + { + var reason = row.Reason ?? "not configured"; + logger.LogInformation(" {Skip} {Name,-20} skipped ({Reason})", SkipMark, row.Label, reason); + } + else if (row.Ok) + { + passCount++; + localChecks++; + logger.LogInformation(" {Pass} {Name,-20} {Description}", PassMark, row.Label, row.Description); + } + else + { + failCount++; + localChecks++; + logger.LogInformation(" {Fail} {Name,-20} {Description}", FailMark, row.Label, row.Description); + + if (row.Suggestion is not null) + { + logger.LogInformation(" -> suggestion: {Suggestion}", row.Suggestion); + } + } + } + + logger.LogInformation(""); + + if (failCount == 0 && localChecks > 0) + { + logger.LogInformation(" All {PassCount} checks passed.", passCount); + } + else if (failCount > 0) + { + logger.LogInformation( + " {FailCount} of {LocalChecks} checks failed. Run `a365 validate --fix` to attempt auto-repair.", + failCount, localChecks); + } + + logger.LogInformation(""); + } + + private static List BuildDisplayRows(ValidateReport report) + { + var rows = new List(); + var tiers = report.Tiers; + + // Row 1: Code health (structural + build + manifest) + var codeHealthTiers = new[] { tiers.Structural, tiers.Build as TierResult }; + var codeHealthActive = codeHealthTiers.Where(t => !t.Skipped).ToList(); + if (codeHealthActive.Count > 0) + { + var allOk = codeHealthActive.All(t => t.Ok == true); + rows.Add(new DisplayRow + { + Label = "Code health", + Ok = allOk, + Description = allOk ? "project structure, manifest, build" : "code health check failed", + Suggestion = allOk ? null : "fix build errors and re-run `a365 validate`" + }); + } + else + { + rows.Add(new DisplayRow { Label = "Code health", Skipped = true, Reason = "not configured" }); + } + + // Row 2: Boot (api/health) + if (!tiers.Boot.Skipped) + { + var bootOk = tiers.Boot.Ok == true; + rows.Add(new DisplayRow + { + Label = "Runs locally", + Ok = bootOk, + Description = bootOk + ? $"/api/health OK{(tiers.Boot is BootTierResult b && b.Port is not null ? $" (port {b.Port})" : "")}" + : "health check failed", + Suggestion = bootOk ? null : "ensure the agent starts locally with `dotnet run` or `npm start`" + }); + } + else + { + rows.Add(new DisplayRow + { + Label = "Runs locally", + Skipped = true, + Reason = tiers.Boot.Reason ?? "boot skipped" + }); + } + + // Row 3: Conversation + if (!tiers.Conversation.Skipped) + { + var conv = tiers.Conversation; + var convOk = conv.Ok == true; + var turnCount = conv.Turns?.Count ?? 0; + var respondedCount = conv.Turns?.Count(t => t.AgentResponded == true) ?? 0; + var failedCount = conv.Turns?.Count(t => !t.Ok) ?? 0; + + rows.Add(new DisplayRow + { + Label = "Conversation", + Ok = convOk, + Description = convOk + ? $"{turnCount}-turn conversation OK, {respondedCount} agent responses" + : $"{turnCount}-turn conversation, {failedCount} failed", + Suggestion = convOk ? null : "check agent logs or a365.validate.json for details" + }); + } + else + { + rows.Add(new DisplayRow + { + Label = "Conversation", + Skipped = true, + Reason = tiers.Conversation.Reason ?? "boot tier failed" + }); + } + + // Remaining individual tiers + rows.Add(CreateTierRow("Telemetry", tiers.Telemetry, + "tracing and observability", + "re-run \"instrument-observability\" skill")); + rows.Add(CreateTierRow("Registered", tiers.Blueprint, + "blueprint registration", + null)); + rows.Add(CreateTierRow("Visible in MAC", tiers.Mac, + "app compliance checks", + null)); + rows.Add(CreateTierRow("Visible in M365", tiers.M365, + "Teams/M365 visibility", + null)); + + return rows; + } + + private static DisplayRow CreateTierRow(string label, TierResult tier, string description, string? suggestion) + { + if (tier.Skipped) + { + return new DisplayRow { Label = label, Skipped = true, Reason = tier.Reason ?? "not yet implemented" }; + } + + return new DisplayRow + { + Label = label, + Ok = tier.Ok == true, + Description = tier.Ok == true ? description : (tier.Reason ?? tier.Warning ?? "check failed"), + Suggestion = tier.Ok == true ? null : suggestion + }; + } + + private sealed class DisplayRow + { + public string Label { get; init; } = string.Empty; + public bool Skipped { get; init; } + public string? Reason { get; init; } + public bool Ok { get; init; } + public string? Description { get; init; } + public string? Suggestion { get; init; } + } + + private static async Task WriteReportAsync(ValidateReport report, string directory, ILogger logger) + { + try + { + var reportPath = Path.Combine(directory, ReportFileName); + var json = JsonSerializer.Serialize(report, ReportSerializerOptions); + await File.WriteAllTextAsync(reportPath, json); + logger.LogInformation("Report written to {ReportPath}", reportPath); + } + catch (Exception ex) + { + logger.LogWarning(ex, "Failed to write validation report"); + } + } + + private static string ResolveProjectPath(Agent365Config config) + { + return string.IsNullOrWhiteSpace(config.DeploymentProjectPath) + ? Directory.GetCurrentDirectory() + : Path.GetFullPath(config.DeploymentProjectPath); + } + + private static List BuildValidationChecks( + PlatformDetector? platformDetector, + CommandExecutor? commandExecutor, + IProcessService? processService, + bool includeConversation = false) + { + var checks = new List + { + new ToolingManifestRequirementCheck() + }; + + if (platformDetector is not null && commandExecutor is not null) + { + checks.Add(new ProjectBuildRequirementCheck(platformDetector, commandExecutor)); + } + + if (platformDetector is not null && processService is not null) + { + checks.Add(new LocalRuntimeRequirementCheck(platformDetector, processService)); + + if (includeConversation) + { + checks.Add(new ConversationRequirementCheck(platformDetector, processService)); + } + } + + return checks; + } + + private static List BuildConversationChecks( + PlatformDetector? platformDetector, + IProcessService? processService, + bool launchPlayground = false) + { + var checks = new List(); + + if (platformDetector is not null && processService is not null) + { + checks.Add(new ConversationRequirementCheck( + platformDetector, processService, launchPlayground: launchPlayground)); + } + + return checks; + } +} \ No newline at end of file diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Constants/CommandNames.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Constants/CommandNames.cs index 8c82ee86..7ce7b4bd 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Constants/CommandNames.cs +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Constants/CommandNames.cs @@ -11,6 +11,7 @@ public static class CommandNames { // Main commands public const string Setup = "setup"; + public const string Validate = "validate"; public const string Deploy = "deploy"; public const string Config = "config"; public const string Publish = "publish"; diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Microsoft.Agents.A365.DevTools.Cli.csproj b/src/Microsoft.Agents.A365.DevTools.Cli/Microsoft.Agents.A365.DevTools.Cli.csproj index b38adb2b..e24be59f 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Microsoft.Agents.A365.DevTools.Cli.csproj +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Microsoft.Agents.A365.DevTools.Cli.csproj @@ -22,6 +22,10 @@ + + + + diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Program.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Program.cs index 37f5ec84..06d8e029 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Program.cs +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Program.cs @@ -130,6 +130,7 @@ await Task.WhenAll( var queryEntraLogger = serviceProvider.GetRequiredService>(); var cleanupLogger = serviceProvider.GetRequiredService>(); var publishLogger = serviceProvider.GetRequiredService>(); + var validateLogger = serviceProvider.GetRequiredService>(); var developLogger = serviceProvider.GetRequiredService>(); var configService = serviceProvider.GetRequiredService(); var executor = serviceProvider.GetRequiredService(); @@ -156,6 +157,7 @@ await Task.WhenAll( var confirmationProvider = serviceProvider.GetRequiredService(); rootCommand.AddCommand(SetupCommand.CreateCommand(setupLogger, configService, executor, backendConfigurator, azureAuthValidator, platformDetector, graphApiService, agentBlueprintService, blueprintLookupService, federatedCredentialService, clientAppValidator, confirmationProvider, armApiService, resolver: bootstrapResolver)); + rootCommand.AddCommand(ValidateCommand.CreateCommand(validateLogger, configService, platformDetector, executor, processService)); var manifestTemplateService = serviceProvider.GetRequiredService(); rootCommand.AddCommand(QueryEntraCommand.CreateCommand(queryEntraLogger, configService, executor, graphApiService, agentBlueprintService, resolver: bootstrapResolver)); rootCommand.AddCommand(CleanupCommand.CreateCommand(cleanupLogger, configService, backendConfigurator, executor, agentBlueprintService, confirmationProvider, federatedCredentialService, azureAuthValidator, graphApiService, resolver: bootstrapResolver)); diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Services/HttpListenerBotCallbackReceiver.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Services/HttpListenerBotCallbackReceiver.cs new file mode 100644 index 00000000..adf070f1 --- /dev/null +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Services/HttpListenerBotCallbackReceiver.cs @@ -0,0 +1,242 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Net; +using System.Net.Sockets; +using System.Text.Json; + +namespace Microsoft.Agents.A365.DevTools.Cli.Services; + +/// +/// HttpListener-based implementation of . +/// Listens on a random available localhost port for Bot Framework callback activities +/// sent by the agent to POST /v3/conversations/{id}/activities. +/// +internal sealed class HttpListenerBotCallbackReceiver : IBotCallbackReceiver +{ + private readonly HttpListener _listener; + private readonly int _port; + private readonly SemaphoreSlim _responseReceived = new(0); + private readonly object _lock = new(); + private readonly List _responses = new(); + private CancellationTokenSource? _cts; + private Task? _listenTask; + + /// + /// After the first callback arrives, continue collecting for this long to capture + /// the actual response (agents often send an acknowledgment before the real reply). + /// + internal static readonly TimeSpan GracePeriod = TimeSpan.FromSeconds(5); + + public string ServiceUrl => $"http://localhost:{_port}"; + + public HttpListenerBotCallbackReceiver() + { + _port = FindAvailablePort(); + _listener = new HttpListener(); + _listener.Prefixes.Add($"http://localhost:{_port}/"); + } + + /// + /// Initializes with a specific port (for testing). + /// + internal HttpListenerBotCallbackReceiver(int port) + { + _port = port; + _listener = new HttpListener(); + _listener.Prefixes.Add($"http://localhost:{_port}/"); + } + + public Task StartAsync(CancellationToken cancellationToken = default) + { + _cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + _listener.Start(); + _listenTask = Task.Run(() => ListenLoopAsync(_cts.Token), _cts.Token); + return Task.CompletedTask; + } + + public async Task WaitForResponseAsync( + TimeSpan timeout, + CancellationToken cancellationToken = default) + { + using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + timeoutCts.CancelAfter(timeout); + + try + { + // Wait for the first callback activity + await _responseReceived.WaitAsync(timeoutCts.Token); + + // Grace period: keep collecting to capture the actual response + // after an initial acknowledgment (e.g., "Got it - working on it...") + var graceDeadline = DateTime.UtcNow + GracePeriod; + + while (DateTime.UtcNow < graceDeadline && !timeoutCts.Token.IsCancellationRequested) + { + var remaining = graceDeadline - DateTime.UtcNow; + if (remaining <= TimeSpan.Zero) + { + break; + } + + try + { + if (!await _responseReceived.WaitAsync(remaining, timeoutCts.Token)) + { + break; // No more responses within grace period + } + // Got another response — continue collecting + } + catch (OperationCanceledException) + { + break; + } + } + } + catch (OperationCanceledException) + { + // Overall timeout or cancellation — return whatever we collected + } + + lock (_lock) + { + return SelectBestResponse(); + } + } + + public void ClearResponses() + { + lock (_lock) + { + _responses.Clear(); + } + + while (_responseReceived.Wait(0)) + { + // Drain any pending signals + } + } + + private async Task ListenLoopAsync(CancellationToken cancellationToken) + { + while (!cancellationToken.IsCancellationRequested) + { + try + { + var context = await _listener.GetContextAsync(); + _ = HandleRequestAsync(context); + } + catch (HttpListenerException) when (cancellationToken.IsCancellationRequested) + { + break; + } + catch (HttpListenerException) + { + break; + } + catch (ObjectDisposedException) + { + break; + } + } + } + + private async Task HandleRequestAsync(HttpListenerContext context) + { + try + { + // Bot Framework sends responses to POST /v3/conversations/{id}/activities[/{id}] + if (context.Request.HttpMethod == "POST" && + context.Request.Url?.AbsolutePath.Contains("/v3/conversations/", StringComparison.OrdinalIgnoreCase) == true) + { + using var reader = new StreamReader(context.Request.InputStream, context.Request.ContentEncoding); + var body = await reader.ReadToEndAsync(); + + try + { + using var doc = JsonDocument.Parse(body); + var text = doc.RootElement.TryGetProperty("text", out var textProp) ? textProp.GetString() : null; + var type = doc.RootElement.TryGetProperty("type", out var typeProp) ? typeProp.GetString() : null; + + lock (_lock) + { + _responses.Add(new BotCallbackResponse(text, type)); + } + + _responseReceived.Release(); + } + catch (JsonException) + { + lock (_lock) + { + _responses.Add(new BotCallbackResponse(null, null)); + } + + _responseReceived.Release(); + } + } + + // Return 200 OK with a ResourceResponse (required by Bot Framework SDK) + context.Response.StatusCode = 200; + context.Response.ContentType = "application/json"; + var responseJson = JsonSerializer.SerializeToUtf8Bytes(new { id = Guid.NewGuid().ToString("N") }); + await context.Response.OutputStream.WriteAsync(responseJson); + } + finally + { + context.Response.Close(); + } + } + + /// + /// Selects the best response from collected callbacks. + /// Prefers the last message-type response with substantive text, + /// falling back to the last response of any type. + /// + private BotCallbackResponse? SelectBestResponse() + { + if (_responses.Count == 0) + { + return null; + } + + // Prefer the last message with non-trivial text (skip short acknowledgments) + var bestMessage = _responses + .LastOrDefault(r => r.Type == "message" && !string.IsNullOrWhiteSpace(r.Text) && r.Text.Length > 30); + + // Fall back to last message with any text + bestMessage ??= _responses.LastOrDefault(r => r.Type == "message" && !string.IsNullOrWhiteSpace(r.Text)); + + // Fall back to last response of any kind + return bestMessage ?? _responses[^1]; + } + + public async ValueTask DisposeAsync() + { + _cts?.Cancel(); + + try { _listener.Stop(); } + catch { /* best effort */ } + + if (_listenTask is not null) + { + try { await _listenTask; } + catch { /* best effort */ } + } + + try { _listener.Close(); } + catch { /* best effort */ } + + _cts?.Dispose(); + _responseReceived.Dispose(); + } + + private static int FindAvailablePort() + { + using var listener = new TcpListener(IPAddress.Loopback, 0); + listener.Start(); + var port = ((IPEndPoint)listener.LocalEndpoint).Port; + listener.Stop(); + return port; + } +} diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Services/IBotCallbackReceiver.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Services/IBotCallbackReceiver.cs new file mode 100644 index 00000000..77bef734 --- /dev/null +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Services/IBotCallbackReceiver.cs @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.Agents.A365.DevTools.Cli.Services; + +/// +/// Receives Bot Framework callback activities sent by the agent to its serviceUrl. +/// Used during conversation validation to detect whether the agent actually responded. +/// +public interface IBotCallbackReceiver : IAsyncDisposable +{ + /// + /// The service URL that should be set in outgoing activities so the bot sends responses here. + /// + string ServiceUrl { get; } + + /// + /// Starts listening for callback activities. + /// + Task StartAsync(CancellationToken cancellationToken = default); + + /// + /// Waits for a response activity from the bot. Returns null if no response arrives within the timeout. + /// + Task WaitForResponseAsync(TimeSpan timeout, CancellationToken cancellationToken = default); + + /// + /// Clears any previously received responses. Call before each conversation turn. + /// + void ClearResponses(); +} + +/// +/// A response activity received from the bot via the serviceUrl callback. +/// +public sealed record BotCallbackResponse(string? Text, string? Type); diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementCheck.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementCheck.cs index 0ee8fac7..0d0dae90 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementCheck.cs +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementCheck.cs @@ -28,7 +28,7 @@ public abstract class RequirementCheck : IRequirementCheck /// protected virtual void LogCheckSuccess(ILogger logger, string? details = null) { - logger.LogInformation("Pass: {Name}{Details}", Name, + logger.LogDebug("Pass: {Name}{Details}", Name, string.IsNullOrWhiteSpace(details) ? "" : $" ({details})"); } @@ -37,7 +37,7 @@ protected virtual void LogCheckSuccess(ILogger logger, string? details = null) /// protected virtual void LogCheckWarning(ILogger logger, string? message = null) { - logger.LogWarning("Warn: {Name}{Details}", Name, + logger.LogDebug("Warn: {Name}{Details}", Name, string.IsNullOrWhiteSpace(message) ? "" : $" - {message}"); } @@ -46,18 +46,16 @@ protected virtual void LogCheckWarning(ILogger logger, string? message = null) /// protected virtual void LogCheckFailure(ILogger logger, string errorMessage, string resolutionGuidance) { - // Single red line — AZ CLI convention: one ERROR line per failure, not per detail - logger.LogError("Fail: {Name}", Name); + logger.LogDebug("Fail: {Name}", Name); - // Error details and resolution guidance in white — they describe and guide, not error foreach (var line in errorMessage.Split('\n', StringSplitOptions.RemoveEmptyEntries)) - logger.LogInformation(" {Line}", line.TrimEnd()); + logger.LogDebug(" {Line}", line.TrimEnd()); if (!string.IsNullOrWhiteSpace(resolutionGuidance)) { - logger.LogInformation(""); + logger.LogDebug(""); foreach (var step in resolutionGuidance.Split('\n', StringSplitOptions.RemoveEmptyEntries)) - logger.LogInformation(" {Step}", step.TrimEnd()); + logger.LogDebug(" {Step}", step.TrimEnd()); } } diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementCheckResult.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementCheckResult.cs index 25543bd0..dc2217ee 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementCheckResult.cs +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementCheckResult.cs @@ -1,77 +1,139 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -namespace Microsoft.Agents.A365.DevTools.Cli.Services.Requirements; - -/// -/// Result of a requirement check execution -/// -public class RequirementCheckResult -{ - /// - /// Whether the requirement check passed - /// - public bool Passed { get; set; } - - /// - /// Whether this is a warning (informational, doesn't block setup) - /// - public bool IsWarning { get; set; } - - /// - /// Error message if the check failed - /// - public string? ErrorMessage { get; set; } - - /// - /// Guidance on how to resolve the issue if the check failed - /// - public string? ResolutionGuidance { get; set; } - - /// - /// Additional details about the check result - /// - public string? Details { get; set; } - - /// - /// Creates a successful result - /// - public static RequirementCheckResult Success(string? details = null) - { - return new RequirementCheckResult - { - Passed = true, - IsWarning = false, - Details = details - }; - } - - /// - /// Creates a warning result (informational, doesn't block setup) - /// - public static RequirementCheckResult Warning(string message, string? details = null) - { - return new RequirementCheckResult - { - Passed = true, - IsWarning = true, - ErrorMessage = message, - Details = details - }; - } - - /// - /// Creates a failed result - /// - public static RequirementCheckResult Failure(string errorMessage, string resolutionGuidance, string? details = null) - { - return new RequirementCheckResult - { - Passed = false, - IsWarning = false, - ErrorMessage = errorMessage, - ResolutionGuidance = resolutionGuidance, - Details = details - }; - } -} \ No newline at end of file +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.Agents.A365.DevTools.Cli.Services.Requirements; + +/// +/// Result of a requirement check execution +/// +public class RequirementCheckResult +{ + /// + /// Whether the requirement check passed + /// + public bool Passed { get; set; } + + /// + /// Whether this is a warning (informational, doesn't block setup) + /// + public bool IsWarning { get; set; } + + /// + /// Error message if the check failed + /// + public string? ErrorMessage { get; set; } + + /// + /// Guidance on how to resolve the issue if the check failed + /// + public string? ResolutionGuidance { get; set; } + + /// + /// Additional details about the check result + /// + public string? Details { get; set; } + + /// + /// Optional typed metadata for structured report output. + /// + public RequirementCheckMetadata? Metadata { get; set; } + + /// + /// Creates a successful result + /// + public static RequirementCheckResult Success(string? details = null) + { + return new RequirementCheckResult + { + Passed = true, + IsWarning = false, + Details = details + }; + } + + /// + /// Creates a warning result (informational, doesn't block setup) + /// + public static RequirementCheckResult Warning(string message, string? details = null) + { + return new RequirementCheckResult + { + Passed = true, + IsWarning = true, + ErrorMessage = message, + Details = details + }; + } + + /// + /// Creates a failed result + /// + public static RequirementCheckResult Failure(string errorMessage, string resolutionGuidance, string? details = null) + { + return new RequirementCheckResult + { + Passed = false, + IsWarning = false, + ErrorMessage = errorMessage, + ResolutionGuidance = resolutionGuidance, + Details = details + }; + } +} + +/// +/// Typed metadata for structured validation report output. +/// +public sealed class RequirementCheckMetadata +{ + /// Port the app is running on (boot tier). + public int? Port { get; init; } + + /// Time in milliseconds for the app to respond (boot tier). + public long? BootMs { get; init; } + + /// Build or runtime log output (build/boot tier). + public string? Log { get; init; } + + /// Process exit code (build tier). + public int? ExitCode { get; init; } + + /// Detected platform name (build/boot tier). + public string? Platform { get; init; } + + /// Conversation turn results (conversation tier). + public List? Turns { get; init; } + + /// Whether AgentsPlayground was launched for interactive testing. + public bool? PlaygroundLaunched { get; init; } +} + +/// +/// Metadata for a single conversation turn. +/// +public sealed class ConversationTurnMetadata +{ + /// The message sent to the agent. + public string Input { get; init; } = string.Empty; + + /// HTTP status code returned by /api/messages. + public int? StatusCode { get; init; } + + /// Truncated response body snippet. + public string? ResponseSnippet { get; init; } + + /// Round-trip latency in milliseconds. + public long? LatencyMs { get; init; } + + /// Whether this turn succeeded. + public bool Ok { get; init; } + + /// Error description if the turn failed. + public string? Error { get; init; } + + /// Whether the agent sent a response via the serviceUrl callback. Null if tracking was unavailable. + public bool? AgentResponded { get; init; } + + /// The text content of the agent's callback response, if any. + public string? AgentResponseText { get; init; } +} diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/ConversationRequirementCheck.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/ConversationRequirementCheck.cs new file mode 100644 index 00000000..5efe82cb --- /dev/null +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/ConversationRequirementCheck.cs @@ -0,0 +1,782 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Diagnostics; +using System.Net; +using System.Net.Http; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using Microsoft.Agents.A365.DevTools.Cli.Models; +using Microsoft.Extensions.Logging; + +namespace Microsoft.Agents.A365.DevTools.Cli.Services.Requirements.RequirementChecks; + +/// +/// Validates that the agent can hold a multi-turn conversation by spawning the agent locally, +/// waiting for readiness via /api/health, then POSTing Bot Framework Activity messages to /api/messages. +/// +public class ConversationRequirementCheck : RequirementCheck +{ + private readonly PlatformDetector _platformDetector; + private readonly IProcessService _processService; + private readonly HttpClient _httpClient; + private readonly IBotCallbackReceiver? _callbackReceiver; + private readonly bool _launchPlayground; + + /// + /// Maximum time to wait for the app to start and respond on the health endpoint. + /// + internal static readonly TimeSpan StartupTimeout = TimeSpan.FromSeconds(30); + + /// + /// Maximum time to wait for a single conversation turn response. + /// + internal static readonly TimeSpan TurnTimeout = TimeSpan.FromSeconds(15); + + /// + /// Maximum time to wait for the agent to respond via the serviceUrl callback after a successful POST. + /// + internal static readonly TimeSpan ResponseWaitTimeout = TimeSpan.FromSeconds(10); + + /// + /// Interval between health endpoint polls during startup. + /// + internal static readonly TimeSpan PollInterval = TimeSpan.FromMilliseconds(500); + + /// + /// Maximum number of stdout/stderr lines to capture for diagnostics. + /// + internal const int MaxOutputLines = 50; + + /// + /// Default port used when no port can be inferred from configuration. + /// + internal const int DefaultPort = 5000; + + /// + /// Multi-turn conversation prompts used for validation. + /// + internal static readonly string[] ConversationPrompts = new[] + { + "Hello", + "What can you do?", + "Thanks" + }; + + public ConversationRequirementCheck( + PlatformDetector platformDetector, + IProcessService processService, + HttpClient? httpClient = null, + IBotCallbackReceiver? callbackReceiver = null, + bool launchPlayground = false) + { + _platformDetector = platformDetector ?? throw new ArgumentNullException(nameof(platformDetector)); + _processService = processService ?? throw new ArgumentNullException(nameof(processService)); + _httpClient = httpClient ?? new HttpClient(); + _callbackReceiver = callbackReceiver; + _launchPlayground = launchPlayground; + } + + /// + public override string Name => "Conversation"; + + /// + public override string Description => "Validates multi-turn conversation with the agent via /api/messages"; + + /// + public override string Category => "Code Health"; + + /// + public override async Task CheckAsync( + Agent365Config config, + ILogger logger, + CancellationToken cancellationToken = default) + { + return await ExecuteCheckWithLoggingAsync(config, logger, CheckImplementationAsync, cancellationToken); + } + + private async Task CheckImplementationAsync( + Agent365Config config, + ILogger logger, + CancellationToken cancellationToken) + { + var projectPath = ResolveProjectPath(config); + + if (!Directory.Exists(projectPath)) + { + return RequirementCheckResult.Failure( + $"Project path does not exist: {projectPath}", + "Ensure the project directory exists, or set deploymentProjectPath in a365.config.json"); + } + + var platform = _platformDetector.Detect(projectPath); + if (platform == ProjectPlatform.Unknown) + { + return RequirementCheckResult.Warning( + "Could not detect project platform, skipping conversation validation", + details: $"No .NET, Node.js, or Python project detected in {projectPath}"); + } + + var port = LocalRuntimeRequirementCheck.ResolvePort(config.MessagingEndpoint); + var healthUrl = $"http://localhost:{port}{LocalRuntimeRequirementCheck.DefaultHealthPath}"; + var messagesUrl = $"http://localhost:{port}/api/messages"; + var conversationId = $"validate-{Guid.NewGuid():N}"; + + logger.LogDebug( + "Starting conversation check: platform={Platform}, port={Port}, projectPath={ProjectPath}", + platform, port, projectPath); + + var startInfo = BuildProcessStartInfo(platform, projectPath, port); + return await SpawnAndConverse(startInfo, healthUrl, messagesUrl, conversationId, platform, port, logger, cancellationToken); + } + + private async Task SpawnAndConverse( + ProcessStartInfo startInfo, + string healthUrl, + string messagesUrl, + string conversationId, + ProjectPlatform platform, + int port, + ILogger logger, + CancellationToken cancellationToken) + { + var outputLines = new LocalRuntimeRequirementCheck.BoundedLineBuffer(MaxOutputLines); + var errorLines = new LocalRuntimeRequirementCheck.BoundedLineBuffer(MaxOutputLines); + Process? process = null; + IBotCallbackReceiver? receiver = _callbackReceiver; + bool ownedReceiver = false; + + try + { + // Start callback receiver for agent response tracking + if (receiver is null) + { + try + { + var httpReceiver = new HttpListenerBotCallbackReceiver(); + await httpReceiver.StartAsync(cancellationToken); + receiver = httpReceiver; + ownedReceiver = true; + logger.LogDebug("Callback receiver started on {ServiceUrl}", receiver.ServiceUrl); + } + catch (Exception ex) + { + logger.LogDebug(ex, "Could not start callback receiver, agent response tracking unavailable"); + } + } + + process = _processService.Start(startInfo); + if (process is null) + { + return RequirementCheckResult.Failure( + $"Failed to start {platform} process", + GetRunGuidance(platform)); + } + + process.OutputDataReceived += (_, args) => + { + if (args.Data is not null) outputLines.Add(args.Data); + }; + process.ErrorDataReceived += (_, args) => + { + if (args.Data is not null) errorLines.Add(args.Data); + }; + process.BeginOutputReadLine(); + process.BeginErrorReadLine(); + + // Phase 1: Wait for health endpoint + var bootResult = await WaitForHealthAsync(process, healthUrl, platform, logger, cancellationToken); + if (bootResult is not null) + { + return bootResult; + } + + // Phase 2: Multi-turn conversation + var turns = new List(); + var allOk = true; + + for (int i = 0; i < ConversationPrompts.Length; i++) + { + if (process.HasExited) + { + var exitOutput = GetCapturedOutput(outputLines, errorLines); + return RequirementCheckResult.Failure( + $"Agent process exited during conversation (turn {i + 1}):\n{exitOutput}", + GetRunGuidance(platform)); + } + + var turnResult = await SendTurnAsync(messagesUrl, conversationId, ConversationPrompts[i], i, port, receiver, logger, cancellationToken); + turns.Add(turnResult); + + if (!turnResult.Ok) + { + allOk = false; + // Continue to remaining turns for a complete report + } + } + + var turnSummary = $"{turns.Count(t => t.Ok)}/{turns.Count} turns succeeded"; + var respondedCount = turns.Count(t => t.AgentResponded == true); + var trackedCount = turns.Count(t => t.AgentResponded is not null); + if (trackedCount > 0) + { + turnSummary += $", {respondedCount}/{trackedCount} agent responses received"; + } + + // Phase 3: Launch AgentsPlayground for interactive testing if requested + bool playgroundLaunched = false; + if (_launchPlayground && !process.HasExited) + { + playgroundLaunched = await LaunchPlaygroundAsync(messagesUrl, logger, cancellationToken); + } + + return new RequirementCheckResult + { + Passed = allOk, + ErrorMessage = allOk ? null : $"Conversation validation failed: {turnSummary}", + ResolutionGuidance = allOk ? null : GetConversationGuidance(turns, platform), + Details = turnSummary, + Metadata = new RequirementCheckMetadata + { + Port = port, + Platform = platform.ToString(), + PlaygroundLaunched = playgroundLaunched ? true : null, + Turns = turns.Select(t => new ConversationTurnMetadata + { + Input = t.Input, + StatusCode = t.StatusCode, + ResponseSnippet = t.ResponseSnippet, + LatencyMs = t.LatencyMs, + Ok = t.Ok, + Error = t.Error, + AgentResponded = t.AgentResponded, + AgentResponseText = t.AgentResponseText + }).ToList() + } + }; + } + finally + { + if (process is not null) + { + try + { + if (!process.HasExited) + { + process.Kill(entireProcessTree: true); + } + } + catch (Exception ex) + { + logger.LogDebug(ex, "Failed to kill process during cleanup"); + } + + process.Dispose(); + } + + if (ownedReceiver && receiver is not null) + { + try + { + await receiver.DisposeAsync(); + } + catch (Exception ex) + { + logger.LogDebug(ex, "Failed to dispose callback receiver during cleanup"); + } + } + } + } + + private async Task WaitForHealthAsync( + Process process, + string healthUrl, + ProjectPlatform platform, + ILogger logger, + CancellationToken cancellationToken) + { + using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + timeoutCts.CancelAfter(StartupTimeout); + + while (!timeoutCts.Token.IsCancellationRequested) + { + if (process.HasExited) + { + return RequirementCheckResult.Failure( + $"App exited with code {process.ExitCode} before health endpoint responded", + GetRunGuidance(platform)); + } + + try + { + using var response = await _httpClient.GetAsync(healthUrl, timeoutCts.Token); + if (response.IsSuccessStatusCode) + { + logger.LogDebug("Health endpoint ready, starting conversation"); + return null; // Ready + } + } + catch (HttpRequestException) + { + // App not ready yet + } + catch (TaskCanceledException) when (timeoutCts.Token.IsCancellationRequested && !cancellationToken.IsCancellationRequested) + { + break; + } + + try + { + await Task.Delay(PollInterval, timeoutCts.Token); + } + catch (TaskCanceledException) when (!cancellationToken.IsCancellationRequested) + { + break; + } + } + + cancellationToken.ThrowIfCancellationRequested(); + + return RequirementCheckResult.Failure( + $"App did not respond on {healthUrl} within {(int)StartupTimeout.TotalSeconds} seconds", + GetRunGuidance(platform)); + } + + private async Task SendTurnAsync( + string messagesUrl, + string conversationId, + string text, + int turnIndex, + int port, + IBotCallbackReceiver? callbackReceiver, + ILogger logger, + CancellationToken cancellationToken) + { + var stopwatch = Stopwatch.StartNew(); + + // Clear previous responses before each turn + callbackReceiver?.ClearResponses(); + + var activity = new BotActivity + { + Type = "message", + Id = Guid.NewGuid().ToString("N"), + Text = text, + From = new BotChannelAccount { Id = "validate-user", Name = "Validate" }, + Recipient = new BotChannelAccount { Id = "agent", Name = "Agent" }, + Conversation = new BotConversationAccount { Id = conversationId }, + ChannelId = "emulator", + ServiceUrl = callbackReceiver?.ServiceUrl ?? $"http://localhost:{port}", + Timestamp = DateTimeOffset.UtcNow + }; + + var json = JsonSerializer.Serialize(activity, ActivitySerializerOptions); + using var content = new StringContent(json, Encoding.UTF8, "application/json"); + + try + { + using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + timeoutCts.CancelAfter(TurnTimeout); + + using var response = await _httpClient.PostAsync(messagesUrl, content, timeoutCts.Token); + stopwatch.Stop(); + var latencyMs = stopwatch.ElapsedMilliseconds; + var statusCode = (int)response.StatusCode; + + // Auth failure — distinct guidance + if (response.StatusCode is HttpStatusCode.Unauthorized or HttpStatusCode.Forbidden) + { + return new ConversationTurnData + { + Input = text, + StatusCode = statusCode, + LatencyMs = latencyMs, + Ok = false, + Error = $"Auth rejected (HTTP {statusCode}). Set channelId='emulator' bypass or requireAuth=false for local testing.", + AgentResponded = false + }; + } + + if (!response.IsSuccessStatusCode) + { + return new ConversationTurnData + { + Input = text, + StatusCode = statusCode, + LatencyMs = latencyMs, + Ok = false, + Error = $"HTTP {statusCode} from /api/messages", + AgentResponded = false + }; + } + + // Try to read the HTTP response body (some bots return inline responses) + var responseBody = await response.Content.ReadAsStringAsync(cancellationToken); + var snippet = TruncateResponse(responseBody, 200); + + // Wait for agent to respond via the serviceUrl callback + bool? agentResponded = null; + string? agentResponseText = null; + + if (callbackReceiver is not null) + { + var botResponse = await callbackReceiver.WaitForResponseAsync(ResponseWaitTimeout, cancellationToken); + agentResponded = botResponse is not null; + agentResponseText = botResponse?.Text; + + if (agentResponseText is not null) + { + agentResponseText = TruncateResponse(agentResponseText, 200); + } + } + + // In non-playground mode, require a valid agent response + bool turnOk = true; + string? turnError = null; + + if (!_launchPlayground && callbackReceiver is not null) + { + if (agentResponded != true) + { + turnOk = false; + turnError = "Agent did not respond within timeout"; + } + else if (IsErrorResponse(agentResponseText)) + { + turnOk = false; + turnError = $"Agent returned an error response: {agentResponseText}"; + } + } + + logger.LogDebug( + "Turn {Turn} ({Text}): HTTP {StatusCode}, latency {Latency}ms, agentResponded={AgentResponded}", + turnIndex + 1, text, statusCode, latencyMs, agentResponded); + + return new ConversationTurnData + { + Input = text, + StatusCode = statusCode, + ResponseSnippet = string.IsNullOrWhiteSpace(responseBody) ? null : snippet, + LatencyMs = latencyMs, + Ok = turnOk, + Error = turnError, + AgentResponded = agentResponded, + AgentResponseText = agentResponseText + }; + } + catch (TaskCanceledException) when (!cancellationToken.IsCancellationRequested) + { + stopwatch.Stop(); + return new ConversationTurnData + { + Input = text, + LatencyMs = stopwatch.ElapsedMilliseconds, + Ok = false, + Error = $"Turn timed out after {(int)TurnTimeout.TotalSeconds} seconds", + AgentResponded = false + }; + } + catch (HttpRequestException ex) + { + stopwatch.Stop(); + return new ConversationTurnData + { + Input = text, + LatencyMs = stopwatch.ElapsedMilliseconds, + Ok = false, + Error = $"Connection failed: {ex.Message}", + AgentResponded = false + }; + } + } + + private static ProcessStartInfo BuildProcessStartInfo(ProjectPlatform platform, string projectPath, int port) + { + var startInfo = new ProcessStartInfo + { + WorkingDirectory = projectPath, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + }; + + // Disable auth so the bot accepts unauthenticated local requests + startInfo.EnvironmentVariables["BYPASS_AUTH"] = "true"; + + switch (platform) + { + case ProjectPlatform.DotNet: + startInfo.FileName = "dotnet"; + startInfo.Arguments = "run --no-build"; + startInfo.EnvironmentVariables["ASPNETCORE_URLS"] = $"http://localhost:{port}"; + break; + + case ProjectPlatform.NodeJs: + startInfo.FileName = "npm"; + startInfo.Arguments = "start"; + startInfo.EnvironmentVariables["PORT"] = port.ToString(); + break; + + case ProjectPlatform.Python: + startInfo.FileName = "python"; + startInfo.Arguments = "app.py"; + startInfo.EnvironmentVariables["PORT"] = port.ToString(); + break; + + default: + throw new ArgumentOutOfRangeException(nameof(platform), platform, "Unsupported platform"); + } + + return startInfo; + } + + private static string GetRunGuidance(ProjectPlatform platform) + { + return platform switch + { + ProjectPlatform.DotNet => "Try running the app manually:\n" + + " dotnet run\n" + + "Verify it starts and responds on /api/messages.", + ProjectPlatform.NodeJs => "Try running the app manually:\n" + + " npm start\n" + + "Verify it starts and responds on /api/messages.", + ProjectPlatform.Python => "Try running the app manually:\n" + + " python app.py\n" + + "Verify it starts and responds on /api/messages.", + _ => "Try running the app manually and verify it responds on /api/messages." + }; + } + + private static string GetConversationGuidance(List turns, ProjectPlatform platform) + { + var failedTurns = turns.Where(t => !t.Ok).ToList(); + var hasAuthFailure = failedTurns.Any(t => t.StatusCode is 401 or 403); + + if (hasAuthFailure) + { + return platform switch + { + ProjectPlatform.DotNet => "Local conversation validation requires auth bypass.\n" + + " Ensure MapAgentApplicationEndpoints is called with requireAuth: false.", + _ => "Local conversation validation requires auth bypass.\n" + + " Use channelId 'emulator' or disable auth for local testing." + }; + } + + return "Check the agent is handling /api/messages correctly.\n" + + " Try testing with AgentsPlayground:\n" + + " agentsplayground -e \"http://localhost:/api/messages\" -c \"emulator\""; + } + + /// + /// Launches AgentsPlayground for interactive testing. Blocks until the user closes it. + /// + private async Task LaunchPlaygroundAsync( + string messagesUrl, + ILogger logger, + CancellationToken cancellationToken) + { + logger.LogInformation("Launching AgentsPlayground for interactive testing..."); + logger.LogInformation(" Endpoint: {MessagesUrl}", messagesUrl); + logger.LogInformation(" Close the playground window or press Ctrl+C to continue validation."); + + var pgStartInfo = new ProcessStartInfo + { + FileName = "agentsplayground", + Arguments = $"-e \"{messagesUrl}\" -c \"emulator\"", + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true, + CreateNoWindow = true + }; + + Process? playgroundProcess = null; + try + { + playgroundProcess = _processService.Start(pgStartInfo); + if (playgroundProcess is null) + { + logger.LogWarning( + "Could not start AgentsPlayground. Install with: npm install -g agentsplayground"); + return false; + } + + playgroundProcess.BeginOutputReadLine(); + playgroundProcess.BeginErrorReadLine(); + + await playgroundProcess.WaitForExitAsync(cancellationToken); + logger.LogInformation("AgentsPlayground exited, continuing validation."); + return true; + } + catch (OperationCanceledException) + { + logger.LogInformation("Playground session cancelled, continuing validation."); + return true; + } + catch (Exception ex) + { + logger.LogWarning("Could not launch AgentsPlayground: {Message}", ex.Message); + logger.LogWarning(" Install with: npm install -g agentsplayground"); + return false; + } + finally + { + if (playgroundProcess is not null) + { + try + { + if (!playgroundProcess.HasExited) + { + playgroundProcess.Kill(entireProcessTree: true); + } + } + catch + { + // Best effort + } + + playgroundProcess.Dispose(); + } + } + } + + private static string GetCapturedOutput( + LocalRuntimeRequirementCheck.BoundedLineBuffer outputLines, + LocalRuntimeRequirementCheck.BoundedLineBuffer errorLines) + { + var sb = new StringBuilder(); + var stdout = outputLines.GetLines(); + var stderr = errorLines.GetLines(); + + if (stdout.Length > 0) + { + sb.AppendLine(" [stdout]"); + foreach (var line in stdout) + sb.AppendLine($" {line}"); + } + + if (stderr.Length > 0) + { + sb.AppendLine(" [stderr]"); + foreach (var line in stderr) + sb.AppendLine($" {line}"); + } + + if (sb.Length == 0) + { + sb.Append(" (no output captured)"); + } + + return sb.ToString().TrimEnd(); + } + + private static string TruncateResponse(string response, int maxLength) + { + if (string.IsNullOrEmpty(response)) return string.Empty; + return response.Length <= maxLength + ? response + : response[..maxLength] + "..."; + } + + private static bool IsErrorResponse(string? responseText) + { + if (string.IsNullOrWhiteSpace(responseText)) + { + return false; + } + + var lower = responseText.ToLowerInvariant(); + return lower.Contains("error") || + lower.Contains("exception") || + lower.Contains("failed") || + lower.Contains("not found") || + lower.Contains("unauthorized") || + lower.Contains("forbidden") || + lower.Contains("internal server error") || + lower.Contains("unhandled") || + lower.Contains("stack trace") || + lower.Contains("timed out") || + lower.Contains("timeout") || + System.Text.RegularExpressions.Regex.IsMatch(lower, @"http\s*[45]\d{2}"); + } + + /// + /// Returns deploymentProjectPath if configured, otherwise falls back to the current directory. + /// + private static string ResolveProjectPath(Agent365Config config) + { + return string.IsNullOrWhiteSpace(config.DeploymentProjectPath) + ? Directory.GetCurrentDirectory() + : Path.GetFullPath(config.DeploymentProjectPath); + } + + private static readonly JsonSerializerOptions ActivitySerializerOptions = new() + { + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull + }; + + /// + /// Internal data structure for tracking turn results during execution. + /// + internal sealed class ConversationTurnData + { + public string Input { get; init; } = string.Empty; + public int? StatusCode { get; init; } + public string? ResponseSnippet { get; init; } + public long? LatencyMs { get; init; } + public bool Ok { get; init; } + public string? Error { get; init; } + public bool? AgentResponded { get; init; } + public string? AgentResponseText { get; init; } + } + + /// + /// Minimal Bot Framework Activity model for local validation. + /// + private sealed class BotActivity + { + [JsonPropertyName("type")] + public string Type { get; set; } = "message"; + + [JsonPropertyName("id")] + public string? Id { get; set; } + + [JsonPropertyName("text")] + public string? Text { get; set; } + + [JsonPropertyName("from")] + public BotChannelAccount? From { get; set; } + + [JsonPropertyName("recipient")] + public BotChannelAccount? Recipient { get; set; } + + [JsonPropertyName("conversation")] + public BotConversationAccount? Conversation { get; set; } + + [JsonPropertyName("channelId")] + public string? ChannelId { get; set; } + + [JsonPropertyName("serviceUrl")] + public string? ServiceUrl { get; set; } + + [JsonPropertyName("timestamp")] + public DateTimeOffset? Timestamp { get; set; } + } + + private sealed class BotChannelAccount + { + [JsonPropertyName("id")] + public string Id { get; set; } = string.Empty; + + [JsonPropertyName("name")] + public string? Name { get; set; } + } + + private sealed class BotConversationAccount + { + [JsonPropertyName("id")] + public string Id { get; set; } = string.Empty; + } +} diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/LocalRuntimeRequirementCheck.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/LocalRuntimeRequirementCheck.cs new file mode 100644 index 00000000..928a5909 --- /dev/null +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/LocalRuntimeRequirementCheck.cs @@ -0,0 +1,378 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Diagnostics; +using System.Net.Http; +using System.Text; +using Microsoft.Agents.A365.DevTools.Cli.Models; +using Microsoft.Extensions.Logging; + +namespace Microsoft.Agents.A365.DevTools.Cli.Services.Requirements.RequirementChecks; + +/// +/// Validates that the user's agent app starts locally and responds on a health endpoint. +/// Spawns the app process, polls /api/health, captures stdout/stderr, then stops the process. +/// +public class LocalRuntimeRequirementCheck : RequirementCheck +{ + private readonly PlatformDetector _platformDetector; + private readonly IProcessService _processService; + private readonly HttpClient _httpClient; + + /// + /// Default port used when no port can be inferred from configuration. + /// + internal const int DefaultPort = 5000; + + /// + /// Default health endpoint path to probe. + /// + internal const string DefaultHealthPath = "/api/health"; + + /// + /// Maximum time to wait for the app to start and respond. + /// + internal static readonly TimeSpan StartupTimeout = TimeSpan.FromSeconds(30); + + /// + /// Interval between health endpoint polls. + /// + internal static readonly TimeSpan PollInterval = TimeSpan.FromMilliseconds(500); + + /// + /// Maximum number of stdout/stderr lines to capture for diagnostics. + /// + internal const int MaxOutputLines = 50; + + public LocalRuntimeRequirementCheck( + PlatformDetector platformDetector, + IProcessService processService, + HttpClient? httpClient = null) + { + _platformDetector = platformDetector ?? throw new ArgumentNullException(nameof(platformDetector)); + _processService = processService ?? throw new ArgumentNullException(nameof(processService)); + _httpClient = httpClient ?? new HttpClient(); + } + + /// + public override string Name => "Local Runtime"; + + /// + public override string Description => "Validates that the agent app starts locally and responds on a health endpoint"; + + /// + public override string Category => "Code Health"; + + /// + public override async Task CheckAsync( + Agent365Config config, + ILogger logger, + CancellationToken cancellationToken = default) + { + return await ExecuteCheckWithLoggingAsync(config, logger, CheckImplementationAsync, cancellationToken); + } + + private async Task CheckImplementationAsync( + Agent365Config config, + ILogger logger, + CancellationToken cancellationToken) + { + var projectPath = ResolveProjectPath(config); + + if (!Directory.Exists(projectPath)) + { + return RequirementCheckResult.Failure( + $"Project path does not exist: {projectPath}", + "Ensure the project directory exists, or set deploymentProjectPath in a365.config.json"); + } + + var platform = _platformDetector.Detect(projectPath); + if (platform == ProjectPlatform.Unknown) + { + return RequirementCheckResult.Warning( + "Could not detect project platform, skipping local runtime validation", + details: $"No .NET, Node.js, or Python project detected in {projectPath}"); + } + + var port = ResolvePort(config.MessagingEndpoint); + var healthUrl = $"http://localhost:{port}{DefaultHealthPath}"; + + logger.LogDebug( + "Starting local runtime check: platform={Platform}, port={Port}, healthUrl={HealthUrl}, projectPath={ProjectPath}", + platform, port, healthUrl, projectPath); + + var startInfo = BuildProcessStartInfo(platform, projectPath, port); + return await SpawnAndProbeAsync(startInfo, healthUrl, platform, port, logger, cancellationToken); + } + + /// + /// Resolves the local port from a MessagingEndpoint URL. Only uses the port when the host + /// is localhost/127.0.0.1/[::1]. Otherwise returns the default port. + /// + internal static int ResolvePort(string? messagingEndpoint) + { + if (string.IsNullOrWhiteSpace(messagingEndpoint)) + { + return DefaultPort; + } + + if (Uri.TryCreate(messagingEndpoint, UriKind.Absolute, out var uri)) + { + var host = uri.Host.ToLowerInvariant(); + var isLocalhost = host is "localhost" or "127.0.0.1" or "[::1]" or "::1"; + + if (isLocalhost && !uri.IsDefaultPort) + { + return uri.Port; + } + } + + return DefaultPort; + } + + private static ProcessStartInfo BuildProcessStartInfo(ProjectPlatform platform, string projectPath, int port) + { + var startInfo = new ProcessStartInfo + { + WorkingDirectory = projectPath, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + }; + + switch (platform) + { + case ProjectPlatform.DotNet: + startInfo.FileName = "dotnet"; + startInfo.Arguments = "run --no-build"; + startInfo.EnvironmentVariables["ASPNETCORE_URLS"] = $"http://localhost:{port}"; + break; + + case ProjectPlatform.NodeJs: + startInfo.FileName = "npm"; + startInfo.Arguments = "start"; + startInfo.EnvironmentVariables["PORT"] = port.ToString(); + break; + + case ProjectPlatform.Python: + startInfo.FileName = "python"; + startInfo.Arguments = "app.py"; + startInfo.EnvironmentVariables["PORT"] = port.ToString(); + break; + + default: + throw new ArgumentOutOfRangeException(nameof(platform), platform, "Unsupported platform"); + } + + return startInfo; + } + + private async Task SpawnAndProbeAsync( + ProcessStartInfo startInfo, + string healthUrl, + ProjectPlatform platform, + int port, + ILogger logger, + CancellationToken cancellationToken) + { + var outputLines = new BoundedLineBuffer(MaxOutputLines); + var errorLines = new BoundedLineBuffer(MaxOutputLines); + Process? process = null; + var stopwatch = System.Diagnostics.Stopwatch.StartNew(); + + try + { + process = _processService.Start(startInfo); + if (process is null) + { + return RequirementCheckResult.Failure( + $"Failed to start {platform} process", + GetRunGuidance(platform)); + } + + process.OutputDataReceived += (_, args) => + { + if (args.Data is not null) outputLines.Add(args.Data); + }; + process.ErrorDataReceived += (_, args) => + { + if (args.Data is not null) errorLines.Add(args.Data); + }; + process.BeginOutputReadLine(); + process.BeginErrorReadLine(); + + using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + timeoutCts.CancelAfter(StartupTimeout); + + while (!timeoutCts.Token.IsCancellationRequested) + { + if (process.HasExited) + { + var exitOutput = GetCapturedOutput(outputLines, errorLines); + return RequirementCheckResult.Failure( + $"App exited early with code {process.ExitCode} before health endpoint responded:\n{exitOutput}", + GetRunGuidance(platform)); + } + + try + { + using var response = await _httpClient.GetAsync(healthUrl, timeoutCts.Token); + if (response.IsSuccessStatusCode) + { + stopwatch.Stop(); + logger.LogDebug("Health endpoint returned {StatusCode}", (int)response.StatusCode); + return new RequirementCheckResult + { + Passed = true, + Details = $"{platform} app running on port {port}, health endpoint returned HTTP {(int)response.StatusCode}", + Metadata = new RequirementCheckMetadata + { + Port = port, + BootMs = stopwatch.ElapsedMilliseconds, + Platform = platform.ToString() + } + }; + } + + logger.LogDebug("Health endpoint returned non-success status {StatusCode}", (int)response.StatusCode); + } + catch (HttpRequestException) + { + // App not ready yet, will retry + } + catch (TaskCanceledException) when (timeoutCts.Token.IsCancellationRequested && !cancellationToken.IsCancellationRequested) + { + // Timeout — fall through to failure below + break; + } + + try + { + await Task.Delay(PollInterval, timeoutCts.Token); + } + catch (TaskCanceledException) when (!cancellationToken.IsCancellationRequested) + { + break; + } + } + + cancellationToken.ThrowIfCancellationRequested(); + + var timeoutOutput = GetCapturedOutput(outputLines, errorLines); + return RequirementCheckResult.Failure( + $"App did not respond on {healthUrl} within {(int)StartupTimeout.TotalSeconds} seconds:\n{timeoutOutput}", + GetRunGuidance(platform)); + } + finally + { + if (process is not null) + { + try + { + if (!process.HasExited) + { + process.Kill(entireProcessTree: true); + } + } + catch (Exception ex) + { + logger.LogDebug(ex, "Failed to kill process during cleanup"); + } + + process.Dispose(); + } + } + } + + private static string GetCapturedOutput(BoundedLineBuffer outputLines, BoundedLineBuffer errorLines) + { + var sb = new StringBuilder(); + var stdout = outputLines.GetLines(); + var stderr = errorLines.GetLines(); + + if (stdout.Length > 0) + { + sb.AppendLine(" [stdout]"); + foreach (var line in stdout) + sb.AppendLine($" {line}"); + } + + if (stderr.Length > 0) + { + sb.AppendLine(" [stderr]"); + foreach (var line in stderr) + sb.AppendLine($" {line}"); + } + + if (sb.Length == 0) + { + sb.Append(" (no output captured)"); + } + + return sb.ToString().TrimEnd(); + } + + private static string GetRunGuidance(ProjectPlatform platform) + { + return platform switch + { + ProjectPlatform.DotNet => "Try running the app manually:\n" + + " dotnet run\n" + + "Verify it starts and exposes /api/health.", + ProjectPlatform.NodeJs => "Try running the app manually:\n" + + " npm start\n" + + "Verify it starts and exposes /api/health.", + ProjectPlatform.Python => "Try running the app manually:\n" + + " python app.py\n" + + "Verify it starts and exposes /api/health.", + _ => "Try running the app manually and verify it exposes /api/health." + }; + } + + /// + /// Thread-safe bounded buffer that keeps the last N lines. + /// + internal sealed class BoundedLineBuffer + { + private readonly Queue _lines; + private readonly int _maxLines; + private readonly object _lock = new(); + + public BoundedLineBuffer(int maxLines) + { + _maxLines = maxLines; + _lines = new Queue(maxLines); + } + + public void Add(string line) + { + lock (_lock) + { + if (_lines.Count >= _maxLines) + { + _lines.Dequeue(); + } + _lines.Enqueue(line); + } + } + + public string[] GetLines() + { + lock (_lock) + { + return _lines.ToArray(); + } + } + } + + /// + /// Returns deploymentProjectPath if configured, otherwise falls back to the current directory. + /// + private static string ResolveProjectPath(Agent365Config config) + { + return string.IsNullOrWhiteSpace(config.DeploymentProjectPath) + ? Directory.GetCurrentDirectory() + : Path.GetFullPath(config.DeploymentProjectPath); + } +} diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/PowerShellModulesRequirementCheck.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/PowerShellModulesRequirementCheck.cs index 5cf9b9c6..8292ef16 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/PowerShellModulesRequirementCheck.cs +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/PowerShellModulesRequirementCheck.cs @@ -4,7 +4,6 @@ using Microsoft.Agents.A365.DevTools.Cli.Models; using Microsoft.Extensions.Logging; using System.Diagnostics; -using System.Text.Json; namespace Microsoft.Agents.A365.DevTools.Cli.Services.Requirements.RequirementChecks; diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/ProjectBuildRequirementCheck.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/ProjectBuildRequirementCheck.cs new file mode 100644 index 00000000..3ec973d3 --- /dev/null +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/ProjectBuildRequirementCheck.cs @@ -0,0 +1,203 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.Agents.A365.DevTools.Cli.Models; +using Microsoft.Extensions.Logging; + +namespace Microsoft.Agents.A365.DevTools.Cli.Services.Requirements.RequirementChecks; + +/// +/// Validates that the user's project builds locally with warnings treated as errors. +/// Uses PlatformDetector to determine the project type and runs the appropriate build command. +/// +public class ProjectBuildRequirementCheck : RequirementCheck +{ + private readonly PlatformDetector _platformDetector; + private readonly CommandExecutor _commandExecutor; + + public ProjectBuildRequirementCheck(PlatformDetector platformDetector, CommandExecutor commandExecutor) + { + _platformDetector = platformDetector ?? throw new ArgumentNullException(nameof(platformDetector)); + _commandExecutor = commandExecutor ?? throw new ArgumentNullException(nameof(commandExecutor)); + } + + /// + public override string Name => "Project Build"; + + /// + public override string Description => "Validates that the project builds locally with warnings treated as errors"; + + /// + public override string Category => "Code Health"; + + /// + public override async Task CheckAsync( + Agent365Config config, + ILogger logger, + CancellationToken cancellationToken = default) + { + return await ExecuteCheckWithLoggingAsync(config, logger, CheckImplementationAsync, cancellationToken); + } + + private async Task CheckImplementationAsync( + Agent365Config config, + ILogger logger, + CancellationToken cancellationToken) + { + var projectPath = ResolveProjectPath(config); + + if (!Directory.Exists(projectPath)) + { + return RequirementCheckResult.Failure( + $"Project path does not exist: {projectPath}", + "Ensure the project directory exists, or set deploymentProjectPath in a365.config.json"); + } + + var platform = _platformDetector.Detect(projectPath); + + if (platform == ProjectPlatform.Unknown) + { + return RequirementCheckResult.Warning( + "Could not detect project platform, skipping build validation", + details: $"No .NET, Node.js, or Python project detected in {projectPath}"); + } + + var (command, arguments) = GetBuildCommand(platform); + + logger.LogDebug("Running build check: {Command} {Arguments} in {Path}", command, arguments, projectPath); + + var result = await _commandExecutor.ExecuteAsync( + command, + arguments, + workingDirectory: projectPath, + captureOutput: true, + suppressErrorLogging: true, + cancellationToken: cancellationToken); + + if (result.Success) + { + return new RequirementCheckResult + { + Passed = true, + Details = $"{platform} project builds with warnings as errors", + Metadata = new RequirementCheckMetadata + { + Platform = platform.ToString(), + ExitCode = result.ExitCode, + Log = TruncateLog(result.StandardOutput) + } + }; + } + + var errorSummary = ExtractBuildErrorSummary(result, platform); + + return new RequirementCheckResult + { + Passed = false, + ErrorMessage = $"Project build failed ({platform}):\n{errorSummary}", + ResolutionGuidance = GetResolutionGuidance(platform), + Metadata = new RequirementCheckMetadata + { + Platform = platform.ToString(), + ExitCode = result.ExitCode, + Log = TruncateLog(!string.IsNullOrWhiteSpace(result.StandardError) ? result.StandardError : result.StandardOutput) + } + }; + } + + private static (string Command, string Arguments) GetBuildCommand(ProjectPlatform platform) + { + return platform switch + { + ProjectPlatform.DotNet => ("dotnet", "build --no-restore /p:TreatWarningsAsErrors=true"), + ProjectPlatform.NodeJs => ("npm", "run build"), + ProjectPlatform.Python => ("python", "-m py_compile ."), + _ => throw new ArgumentOutOfRangeException(nameof(platform), platform, "Unsupported platform") + }; + } + + private static string GetResolutionGuidance(ProjectPlatform platform) + { + return platform switch + { + ProjectPlatform.DotNet => "Fix the build errors and warnings in your project.\n" + + "Run 'dotnet build /p:TreatWarningsAsErrors=true' locally to see the full output.", + ProjectPlatform.NodeJs => "Fix the build errors in your project.\n" + + "Run 'npm run build' locally to see the full output.", + ProjectPlatform.Python => "Fix the syntax errors in your Python files.\n" + + "Run 'python -m py_compile ' on each file to check for syntax errors.", + _ => "Fix the build errors in your project and try again." + }; + } + + /// + /// Extracts a concise summary from build output, limiting to the most relevant lines. + /// + private static string ExtractBuildErrorSummary(CommandResult result, ProjectPlatform platform) + { + var output = !string.IsNullOrWhiteSpace(result.StandardError) + ? result.StandardError + : result.StandardOutput; + + if (string.IsNullOrWhiteSpace(output)) + { + return $"Build exited with code {result.ExitCode} (no output captured)"; + } + + var lines = output.Split('\n', StringSplitOptions.RemoveEmptyEntries); + + if (platform == ProjectPlatform.DotNet) + { + // For .NET, extract lines containing "error" or "warning" (MSBuild output) + var diagnosticLines = lines + .Where(l => l.Contains(": error ", StringComparison.OrdinalIgnoreCase) || + l.Contains(": warning ", StringComparison.OrdinalIgnoreCase)) + .Select(l => l.Trim()) + .Take(10) + .ToArray(); + + if (diagnosticLines.Length > 0) + { + return string.Join("\n", diagnosticLines.Select(l => $" {l}")); + } + } + + // Fallback: return last 10 meaningful lines + var lastLines = lines + .Select(l => l.Trim()) + .Where(l => !string.IsNullOrWhiteSpace(l)) + .TakeLast(10) + .ToArray(); + + return string.Join("\n", lastLines.Select(l => $" {l}")); + } + + /// + /// Returns deploymentProjectPath if configured, otherwise falls back to the current directory. + /// + private static string ResolveProjectPath(Agent365Config config) + { + return string.IsNullOrWhiteSpace(config.DeploymentProjectPath) + ? Directory.GetCurrentDirectory() + : Path.GetFullPath(config.DeploymentProjectPath); + } + + /// + /// Truncates log output to the last 100 lines to keep the JSON report reasonable. + /// + private static string? TruncateLog(string? log, int maxLines = 100) + { + if (string.IsNullOrWhiteSpace(log)) + { + return null; + } + + var lines = log.Split('\n'); + if (lines.Length <= maxLines) + { + return log.TrimEnd(); + } + + return string.Join("\n", lines.TakeLast(maxLines)).TrimEnd(); + } +} diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/ToolingManifestRequirementCheck.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/ToolingManifestRequirementCheck.cs new file mode 100644 index 00000000..9e193749 --- /dev/null +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/ToolingManifestRequirementCheck.cs @@ -0,0 +1,91 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Text.Json; +using Microsoft.Agents.A365.DevTools.Cli.Constants; +using Microsoft.Agents.A365.DevTools.Cli.Models; +using Microsoft.Extensions.Logging; + +namespace Microsoft.Agents.A365.DevTools.Cli.Services.Requirements.RequirementChecks; + +/// +/// Validates the ToolingManifest.json schema when present. +/// Checks that the manifest is valid JSON, has required fields, and has no duplicate server names. +/// +public class ToolingManifestRequirementCheck : RequirementCheck +{ + /// + public override string Name => "Tooling Manifest"; + + /// + public override string Description => "Validates ToolingManifest.json schema and server configuration"; + + /// + public override string Category => "Configuration"; + + /// + public override async Task CheckAsync( + Agent365Config config, + ILogger logger, + CancellationToken cancellationToken = default) + { + return await ExecuteCheckWithLoggingAsync(config, logger, CheckImplementationAsync, cancellationToken); + } + + private async Task CheckImplementationAsync( + Agent365Config config, + ILogger logger, + CancellationToken cancellationToken) + { + var projectPath = ResolveProjectPath(config); + var manifestPath = Path.Combine(projectPath, McpConstants.ToolingManifestFileName); + + if (!File.Exists(manifestPath)) + { + return RequirementCheckResult.Success("ToolingManifest.json not present, skipping"); + } + + ToolingManifest? manifest; + try + { + var json = await File.ReadAllTextAsync(manifestPath, cancellationToken); + manifest = JsonSerializer.Deserialize(json); + } + catch (JsonException ex) + { + logger.LogDebug(ex, "Failed to parse ToolingManifest.json at {ManifestPath}", manifestPath); + return RequirementCheckResult.Failure( + "ToolingManifest.json contains invalid JSON", + "Fix the JSON syntax in ToolingManifest.json and try again"); + } + + if (manifest is null || manifest.McpServers is null) + { + return RequirementCheckResult.Failure( + "ToolingManifest.json is invalid: mcpServers must be an array", + "Ensure ToolingManifest.json contains a valid JSON object with an mcpServers array"); + } + + var errors = manifest.GetValidationErrors(); + if (errors.Length > 0) + { + return RequirementCheckResult.Failure( + $"ToolingManifest.json has {errors.Length} validation error(s):\n" + + string.Join("\n", errors.Select(e => $" - {e}")), + "Fix the reported issues in ToolingManifest.json and run 'a365 validate' again"); + } + + return RequirementCheckResult.Success( + $"{manifest.McpServers.Length} MCP server(s) configured"); + } + + /// + /// Returns deploymentProjectPath if configured, otherwise falls back to the current directory. + /// + private static string ResolveProjectPath(Agent365Config config) + { + return string.IsNullOrWhiteSpace(config.DeploymentProjectPath) + ? Directory.GetCurrentDirectory() + : Path.GetFullPath(config.DeploymentProjectPath); + } +} diff --git a/src/Microsoft.Agents.A365.DevTools.Validation/Microsoft.Agents.A365.DevTools.Validation.csproj b/src/Microsoft.Agents.A365.DevTools.Validation/Microsoft.Agents.A365.DevTools.Validation.csproj new file mode 100644 index 00000000..bef54f79 --- /dev/null +++ b/src/Microsoft.Agents.A365.DevTools.Validation/Microsoft.Agents.A365.DevTools.Validation.csproj @@ -0,0 +1,8 @@ + + + net8.0 + enable + enable + false + + \ No newline at end of file diff --git a/src/Microsoft.Agents.A365.DevTools.Validation/ValidateReport.cs b/src/Microsoft.Agents.A365.DevTools.Validation/ValidateReport.cs new file mode 100644 index 00000000..2de42122 --- /dev/null +++ b/src/Microsoft.Agents.A365.DevTools.Validation/ValidateReport.cs @@ -0,0 +1,254 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Text.Json.Serialization; + +namespace Microsoft.Agents.A365.DevTools.Validation; + +/// +/// Root model for the structured validation report written to a365.validate.json. +/// +public sealed class ValidateReport +{ + [JsonPropertyName("agent")] + public AgentInfo Agent { get; set; } = new(); + + [JsonPropertyName("tiers")] + public ValidationTiers Tiers { get; set; } = new(); + + [JsonPropertyName("repair")] + public RepairResult Repair { get; set; } = RepairResult.NotImplemented(); + + [JsonPropertyName("summary")] + public SummaryResult Summary { get; set; } = new(); +} + +/// +/// Metadata about the agent project being validated. +/// +public sealed class AgentInfo +{ + [JsonPropertyName("path")] + public string? Path { get; set; } + + [JsonPropertyName("language")] + public string? Language { get; set; } + + [JsonPropertyName("framework")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Framework { get; set; } + + [JsonPropertyName("capabilities")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public List? Capabilities { get; set; } +} + +/// +/// Container for all validation tiers. +/// +public sealed class ValidationTiers +{ + [JsonPropertyName("structural")] + public StructuralTierResult Structural { get; set; } = TierResult.CreateSkipped(); + + [JsonPropertyName("build")] + public BuildTierResult Build { get; set; } = TierResult.CreateSkipped(); + + [JsonPropertyName("boot")] + public BootTierResult Boot { get; set; } = TierResult.CreateSkipped(); + + [JsonPropertyName("conversation")] + public ConversationTierResult Conversation { get; set; } = TierResult.CreateSkipped("not yet implemented"); + + [JsonPropertyName("telemetry")] + public TierResult Telemetry { get; set; } = TierResult.CreateSkipped("not yet implemented"); + + [JsonPropertyName("blueprint")] + public TierResult Blueprint { get; set; } = TierResult.CreateSkipped("not yet implemented"); + + [JsonPropertyName("mac")] + public TierResult Mac { get; set; } = TierResult.CreateSkipped("not yet implemented"); + + [JsonPropertyName("m365")] + public TierResult M365 { get; set; } = TierResult.CreateSkipped("not yet implemented"); + + [JsonPropertyName("judge")] + public TierResult Judge { get; set; } = TierResult.CreateSkipped("not yet implemented"); +} + +/// +/// Base tier result. When skipped, ok is null. +/// +public class TierResult +{ + [JsonPropertyName("ok")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public bool? Ok { get; set; } + + [JsonPropertyName("skipped")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public bool Skipped { get; set; } + + [JsonPropertyName("reason")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Reason { get; set; } + + [JsonPropertyName("warning")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Warning { get; set; } + + public static TierResult CreateSkipped(string reason = "not yet implemented") + { + return new TierResult { Skipped = true, Reason = reason }; + } + + public static T CreateSkipped(string reason = "not yet implemented") where T : TierResult, new() + { + return new T { Skipped = true, Reason = reason }; + } +} + +/// +/// Structural tier: config and manifest validation checks. +/// +public sealed class StructuralTierResult : TierResult +{ + [JsonPropertyName("checks")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public List? Checks { get; set; } +} + +/// +/// Individual structural check result. +/// +public sealed class StructuralCheck +{ + [JsonPropertyName("name")] + public string Name { get; set; } = string.Empty; + + [JsonPropertyName("ok")] + public bool Ok { get; set; } + + [JsonPropertyName("message")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Message { get; set; } +} + +/// +/// Build tier: project compilation result. +/// +public sealed class BuildTierResult : TierResult +{ + [JsonPropertyName("log")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Log { get; set; } + + [JsonPropertyName("exitCode")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? ExitCode { get; set; } +} + +/// +/// Boot tier: local runtime health probe result. +/// +public sealed class BootTierResult : TierResult +{ + [JsonPropertyName("port")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? Port { get; set; } + + [JsonPropertyName("bootMs")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public long? BootMs { get; set; } +} + +/// +/// Conversation tier: multi-turn conversation validation result. +/// +public sealed class ConversationTierResult : TierResult +{ + [JsonPropertyName("turns")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public List? Turns { get; set; } + + [JsonPropertyName("playgroundLaunched")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public bool? PlaygroundLaunched { get; set; } +} + +/// +/// Result of a single conversation turn. +/// +public sealed class ConversationTurnResult +{ + [JsonPropertyName("input")] + public string Input { get; set; } = string.Empty; + + [JsonPropertyName("statusCode")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? StatusCode { get; set; } + + [JsonPropertyName("responseSnippet")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? ResponseSnippet { get; set; } + + [JsonPropertyName("latencyMs")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public long? LatencyMs { get; set; } + + [JsonPropertyName("ok")] + public bool Ok { get; set; } + + [JsonPropertyName("error")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Error { get; set; } + + [JsonPropertyName("agentResponded")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public bool? AgentResponded { get; set; } + + [JsonPropertyName("agentResponseText")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? AgentResponseText { get; set; } +} + +/// +/// Repair result (not yet implemented). +/// +public sealed class RepairResult +{ + [JsonPropertyName("skipped")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public bool Skipped { get; set; } + + [JsonPropertyName("reason")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Reason { get; set; } + + [JsonPropertyName("iterations")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? Iterations { get; set; } + + [JsonPropertyName("patches")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public List? Patches { get; set; } + + [JsonPropertyName("finalOk")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public bool? FinalOk { get; set; } + + public static RepairResult NotImplemented() => new() { Skipped = true, Reason = "not yet implemented" }; +} + +/// +/// Summary of the validation run. +/// +public sealed class SummaryResult +{ + [JsonPropertyName("ok")] + public bool Ok { get; set; } + + [JsonPropertyName("blocker")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Blocker { get; set; } +} diff --git a/src/Microsoft.Agents.A365.DevTools.Validation/ValidationContracts.cs b/src/Microsoft.Agents.A365.DevTools.Validation/ValidationContracts.cs new file mode 100644 index 00000000..96312621 --- /dev/null +++ b/src/Microsoft.Agents.A365.DevTools.Validation/ValidationContracts.cs @@ -0,0 +1,183 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.Agents.A365.DevTools.Validation; + +/// +/// Severity for a validation issue. +/// +public enum ValidationSeverity +{ + Info, + Warning, + Error +} + +/// +/// Represents a single validation issue. +/// +public sealed record ValidationIssue( + string Code, + string Message, + ValidationSeverity Severity = ValidationSeverity.Error); + +/// +/// Represents the outcome of a validation operation. +/// +public sealed class ValidationOutcome +{ + public bool IsValid { get; init; } + + public int ExitCode { get; init; } + + public IReadOnlyList Issues { get; init; } = []; + + public static ValidationOutcome Success() => new() + { + IsValid = true, + ExitCode = 0 + }; + + public static ValidationOutcome Failure(params ValidationIssue[] issues) => new() + { + IsValid = false, + ExitCode = 1, + Issues = issues + }; +} + +/// +/// Result of loading configuration for validation. +/// +public sealed record ValidationLoadResult +{ + public bool IsSuccess { get; init; } + + public TConfig? Value { get; init; } + + public int ExitCode { get; init; } + + public IReadOnlyList Issues { get; init; } = []; + + public static ValidationLoadResult Success(TConfig value) => new() + { + IsSuccess = true, + Value = value, + ExitCode = 0 + }; + + public static ValidationLoadResult Failure(int exitCode, params ValidationIssue[] issues) => new() + { + IsSuccess = false, + ExitCode = exitCode, + Issues = issues + }; +} + +/// +/// Orchestrates the CLI validation workflow using delegates supplied by the caller. +/// +public sealed class CliValidationCoordinator +{ + public required Func> ConfigExistsAsync { get; init; } + + public required Func>> LoadConfigAsync { get; init; } + + public required Func> ValidateConfig { get; init; } + + public required Func> RunSystemChecksAsync { get; init; } + + public required Func> RunConfigChecksAsync { get; init; } + + public required Action ReportIssue { get; init; } + + public async Task ExecuteAsync(CancellationToken cancellationToken = default) + { + var issues = new List(); + var exitCode = 0; + + var configExists = await ConfigExistsAsync(cancellationToken); + if (!configExists) + { + var issue = new ValidationIssue( + "CONFIG_FILE_NOT_FOUND", + "Configuration file not found. Run 'a365 setup all --agent-name ' to set up from scratch."); + issues.Add(issue); + ReportIssue(issue); + exitCode = 2; + } + + TConfig? config = default; + if (configExists) + { + var loadResult = await LoadConfigAsync(cancellationToken); + issues.AddRange(loadResult.Issues); + + foreach (var issue in loadResult.Issues) + { + ReportIssue(issue); + } + + if (!loadResult.IsSuccess || loadResult.Value is null) + { + exitCode = Math.Max(exitCode, loadResult.ExitCode == 0 ? 2 : loadResult.ExitCode); + + if (!await RunSystemChecksAsync(cancellationToken)) + { + exitCode = Math.Max(exitCode, 1); + } + + return new ValidationOutcome + { + IsValid = exitCode == 0, + ExitCode = exitCode, + Issues = issues + }; + } + + config = loadResult.Value; + + var configErrors = ValidateConfig(config); + foreach (var error in configErrors) + { + var issue = new ValidationIssue("CONFIG_VALIDATION_FAILED", error); + issues.Add(issue); + ReportIssue(issue); + } + + if (configErrors.Count > 0) + { + exitCode = Math.Max(exitCode, 2); + } + } + + if (!await RunSystemChecksAsync(cancellationToken)) + { + exitCode = Math.Max(exitCode, 1); + } + + if (config is not null && exitCode < 2) + { + var configChecksPassed = await RunConfigChecksAsync(config, cancellationToken); + if (!configChecksPassed) + { + exitCode = Math.Max(exitCode, 1); + } + } + + return new ValidationOutcome + { + IsValid = exitCode == 0, + ExitCode = exitCode, + Issues = issues + }; + } +} + +/// +/// Contract for validation components in the validation subproject. +/// +public interface IValidator +{ + Task ValidateAsync(T value, CancellationToken cancellationToken = default); +} \ No newline at end of file diff --git a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Commands/ValidateCommandTests.cs b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Commands/ValidateCommandTests.cs new file mode 100644 index 00000000..fa8d2095 --- /dev/null +++ b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Commands/ValidateCommandTests.cs @@ -0,0 +1,190 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using FluentAssertions; +using Microsoft.Agents.A365.DevTools.Cli.Commands; +using Microsoft.Agents.A365.DevTools.Cli.Models; +using Microsoft.Agents.A365.DevTools.Cli.Services; +using Microsoft.Agents.A365.DevTools.Cli.Services.Requirements; +using Microsoft.Agents.A365.DevTools.Cli.Tests.TestHelpers; +using Microsoft.Extensions.Logging; +using NSubstitute; +using System.CommandLine; +using System.Text.Json; +using Xunit; +using Microsoft.Agents.A365.DevTools.Validation; + +namespace Microsoft.Agents.A365.DevTools.Cli.Tests.Commands; + +[CollectionDefinition("ValidateCommandTests", DisableParallelization = true)] +public class ValidateCommandTestCollection { } + +[Collection("ValidateCommandTests")] +public class ValidateCommandTests : IDisposable +{ + private readonly ILogger _logger; + private readonly IConfigService _configService; + private readonly string _reportPath; + + public ValidateCommandTests() + { + _logger = Substitute.For>(); + _configService = Substitute.For(); + _reportPath = Path.Combine(Directory.GetCurrentDirectory(), ValidateCommand.ReportFileName); + } + + public void Dispose() + { + if (File.Exists(_reportPath)) + { + File.Delete(_reportPath); + } + } + + [Fact] + public void ValidateCommand_IsRegisteredWithExpectedNameAndDescription() + { + // Act + var command = ValidateCommand.CreateCommand(_logger, _configService, requirementChecksOverride: new List()); + + // Assert + command.Name.Should().Be("validate"); + command.Description.Should().Contain("Validate the local Agent 365 CLI configuration"); + } + + [Fact] + public async Task ValidateCommand_WithValidConfigAndPassingChecks_ReturnsExitCode0() + { + // Arrange + _configService.ConfigExistsAsync(Arg.Any()).Returns(true); + _configService.LoadAsync(Arg.Any(), Arg.Any()).Returns(new Agent365Config + { + TenantId = "12345678-1234-1234-1234-123456789012", + ClientAppId = "87654321-4321-4321-4321-210987654321", + AgentIdentityDisplayName = "Test Agent" + }); + + var root = new RootCommand(); + root.AddCommand(ValidateCommand.CreateCommand(_logger, _configService, requirementChecksOverride: [new AlwaysPassRequirementCheck()])); + + // Act + var exitCode = await root.InvokeAsync("validate"); + + // Assert + exitCode.Should().Be(0, because: "a valid config and passing validation checks should succeed"); + } + + [Fact] + public async Task ValidateCommand_WithMissingConfig_ReturnsExitCode1() + { + // Arrange + _configService.ConfigExistsAsync(Arg.Any()).Returns(false); + + var root = new RootCommand(); + root.AddCommand(ValidateCommand.CreateCommand(_logger, _configService, requirementChecksOverride: [new AlwaysPassRequirementCheck()])); + + // Act + var exitCode = await root.InvokeAsync("validate"); + + // Assert + exitCode.Should().Be(1, because: "missing config should fail immediately since setup must be run first"); + } + + [Fact] + public async Task ValidateCommand_WithMissingConfig_WritesReportWithStructuralBlocker() + { + // Arrange + _configService.ConfigExistsAsync(Arg.Any()).Returns(false); + + var root = new RootCommand(); + root.AddCommand(ValidateCommand.CreateCommand(_logger, _configService, requirementChecksOverride: [new AlwaysPassRequirementCheck()])); + + // Act + await root.InvokeAsync("validate"); + + // Assert + File.Exists(_reportPath).Should().BeTrue(because: "report should always be written even on failure"); + var report = JsonSerializer.Deserialize(await File.ReadAllTextAsync(_reportPath)); + report.Should().NotBeNull(); + report!.Summary.Ok.Should().BeFalse(); + report.Summary.Blocker.Should().Be("structural"); + report.Tiers.Structural.Ok.Should().BeFalse(); + } + + [Fact] + public async Task ValidateCommand_WithValidConfig_WritesReportWithSummaryOk() + { + // Arrange + _configService.ConfigExistsAsync(Arg.Any()).Returns(true); + _configService.LoadAsync(Arg.Any(), Arg.Any()).Returns(new Agent365Config + { + TenantId = "12345678-1234-1234-1234-123456789012", + ClientAppId = "87654321-4321-4321-4321-210987654321", + AgentIdentityDisplayName = "Test Agent" + }); + + var root = new RootCommand(); + root.AddCommand(ValidateCommand.CreateCommand(_logger, _configService, requirementChecksOverride: [new AlwaysPassRequirementCheck()])); + + // Act + await root.InvokeAsync("validate"); + + // Assert + File.Exists(_reportPath).Should().BeTrue(because: "report should be written on success"); + var report = JsonSerializer.Deserialize(await File.ReadAllTextAsync(_reportPath)); + report.Should().NotBeNull(); + report!.Summary.Ok.Should().BeTrue(); + report.Summary.Blocker.Should().BeNull(); + } + + [Fact] + public async Task ValidateCommand_WithFailingCheck_ReturnsExitCode1() + { + // Arrange + _configService.ConfigExistsAsync(Arg.Any()).Returns(true); + _configService.LoadAsync(Arg.Any(), Arg.Any()).Returns(new Agent365Config + { + TenantId = "12345678-1234-1234-1234-123456789012", + ClientAppId = "87654321-4321-4321-4321-210987654321", + AgentIdentityDisplayName = "Test Agent" + }); + + var root = new RootCommand(); + root.AddCommand(ValidateCommand.CreateCommand(_logger, _configService, requirementChecksOverride: [new AlwaysFailRequirementCheck()])); + + // Act + var exitCode = await root.InvokeAsync("validate"); + + // Assert + exitCode.Should().Be(1, because: "failing validation checks should return exit code 1"); + } + + [Fact] + public async Task ValidateCommand_Report_HasSkippedUnimplementedTiers() + { + // Arrange + _configService.ConfigExistsAsync(Arg.Any()).Returns(true); + _configService.LoadAsync(Arg.Any(), Arg.Any()).Returns(new Agent365Config + { + TenantId = "12345678-1234-1234-1234-123456789012", + ClientAppId = "87654321-4321-4321-4321-210987654321", + AgentIdentityDisplayName = "Test Agent" + }); + + var root = new RootCommand(); + root.AddCommand(ValidateCommand.CreateCommand(_logger, _configService, requirementChecksOverride: [new AlwaysPassRequirementCheck()])); + + // Act + await root.InvokeAsync("validate"); + + // Assert + var report = JsonSerializer.Deserialize(await File.ReadAllTextAsync(_reportPath)); + report!.Tiers.Conversation.Skipped.Should().BeTrue(); + report.Tiers.Telemetry.Skipped.Should().BeTrue(); + report.Tiers.Blueprint.Skipped.Should().BeTrue(); + report.Tiers.Mac.Skipped.Should().BeTrue(); + report.Tiers.M365.Skipped.Should().BeTrue(); + report.Tiers.Judge.Skipped.Should().BeTrue(); + report.Repair.Skipped.Should().BeTrue(); + } +} \ No newline at end of file diff --git a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Microsoft.Agents.A365.DevTools.Cli.Tests.csproj b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Microsoft.Agents.A365.DevTools.Cli.Tests.csproj index 00da7394..35add1fc 100644 --- a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Microsoft.Agents.A365.DevTools.Cli.Tests.csproj +++ b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Microsoft.Agents.A365.DevTools.Cli.Tests.csproj @@ -26,6 +26,7 @@ + diff --git a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/ConversationRequirementCheckTests.cs b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/ConversationRequirementCheckTests.cs new file mode 100644 index 00000000..1b9e36fd --- /dev/null +++ b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/ConversationRequirementCheckTests.cs @@ -0,0 +1,703 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Diagnostics; +using System.Net; +using FluentAssertions; +using Microsoft.Agents.A365.DevTools.Cli.Models; +using Microsoft.Agents.A365.DevTools.Cli.Services; +using Microsoft.Agents.A365.DevTools.Cli.Services.Requirements; +using Microsoft.Agents.A365.DevTools.Cli.Services.Requirements.RequirementChecks; +using Microsoft.Extensions.Logging; +using NSubstitute; +using Xunit; + +namespace Microsoft.Agents.A365.DevTools.Cli.Tests.Services.Requirements; + +public class ConversationRequirementCheckTests : IDisposable +{ + private readonly ILogger _logger; + private readonly PlatformDetector _platformDetector; + private readonly IProcessService _processService; + private readonly string _tempDir; + + public ConversationRequirementCheckTests() + { + _logger = Substitute.For(); + _platformDetector = new PlatformDetector(Substitute.For>()); + _processService = Substitute.For(); + _tempDir = Path.Combine(Path.GetTempPath(), $"a365-conversation-test-{Guid.NewGuid():N}"); + Directory.CreateDirectory(_tempDir); + } + + public void Dispose() + { + if (Directory.Exists(_tempDir)) + { + Directory.Delete(_tempDir, recursive: true); + } + } + + private ConversationRequirementCheck CreateCheck( + HttpMessageHandler? handler = null, + IBotCallbackReceiver? callbackReceiver = null, + bool launchPlayground = false) + { + var httpClient = handler is not null ? new HttpClient(handler) : new HttpClient(); + return new ConversationRequirementCheck( + _platformDetector, _processService, httpClient, callbackReceiver, launchPlayground); + } + + [Fact] + public void Check_HasExpectedMetadata() + { + var check = CreateCheck(); + check.Name.Should().Be("Conversation"); + check.Category.Should().Be("Code Health"); + check.Description.Should().Contain("/api/messages"); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + public async Task CheckAsync_WhenDeploymentProjectPathIsEmpty_FallsBackToCwd(string? path) + { + var check = CreateCheck(); + var config = new Agent365Config { DeploymentProjectPath = path ?? string.Empty }; + + var result = await check.CheckAsync(config, _logger); + + result.Should().NotBeNull(); + } + + [Fact] + public async Task CheckAsync_WhenDirectoryDoesNotExist_ReturnsFailure() + { + var check = CreateCheck(); + var config = new Agent365Config { DeploymentProjectPath = Path.Combine(_tempDir, "nonexistent") }; + + var result = await check.CheckAsync(config, _logger); + + result.Passed.Should().BeFalse(because: "a non-existent directory cannot run an app"); + result.ErrorMessage.Should().Contain("does not exist"); + } + + [Fact] + public async Task CheckAsync_WhenPlatformIsUnknown_ReturnsWarning() + { + var check = CreateCheck(); + var config = new Agent365Config { DeploymentProjectPath = _tempDir }; + + var result = await check.CheckAsync(config, _logger); + + result.Passed.Should().BeTrue(because: "unknown platform is a non-blocking warning"); + result.IsWarning.Should().BeTrue(); + } + + [Fact] + public async Task CheckAsync_WhenProcessFailsToStart_ReturnsFailure() + { + File.WriteAllText(Path.Combine(_tempDir, "test.csproj"), ""); + _processService.Start(Arg.Any()).Returns((Process?)null); + var check = CreateCheck(); + var config = new Agent365Config { DeploymentProjectPath = _tempDir }; + + var result = await check.CheckAsync(config, _logger); + + result.Passed.Should().BeFalse(because: "a process that fails to start cannot handle conversations"); + result.ErrorMessage.Should().Contain("Failed to start"); + } + + [Fact] + public async Task CheckAsync_WhenAllTurnsSucceed_ReturnsSuccess() + { + File.WriteAllText(Path.Combine(_tempDir, "test.csproj"), ""); + var fakeProcess = CreateFakeProcess(exitImmediately: false); + _processService.Start(Arg.Any()).Returns(fakeProcess); + + var handler = new ConversationHttpHandler( + healthStatusCode: HttpStatusCode.OK, + messagesStatusCode: HttpStatusCode.OK, + responseBody: "{\"type\":\"message\",\"text\":\"Hello!\"}"); + var fakeReceiver = new FakeBotCallbackReceiver( + new BotCallbackResponse("Hello!", "message")); + var check = CreateCheck(handler, fakeReceiver); + var config = new Agent365Config { DeploymentProjectPath = _tempDir }; + + var result = await check.CheckAsync(config, _logger); + + result.Passed.Should().BeTrue(because: "all conversation turns returned 200"); + result.Details.Should().Contain("3/3 turns succeeded"); + result.Metadata.Should().NotBeNull(); + result.Metadata!.Turns.Should().HaveCount(3); + result.Metadata.Turns!.Should().OnlyContain(t => t.Ok, because: "every turn should report success"); + } + + [Fact] + public async Task CheckAsync_WhenTurnReturnsAuthFailure_ReturnsFailureWithGuidance() + { + File.WriteAllText(Path.Combine(_tempDir, "test.csproj"), ""); + var fakeProcess = CreateFakeProcess(exitImmediately: false); + _processService.Start(Arg.Any()).Returns(fakeProcess); + + var handler = new ConversationHttpHandler( + healthStatusCode: HttpStatusCode.OK, + messagesStatusCode: HttpStatusCode.Unauthorized); + var check = CreateCheck(handler); + var config = new Agent365Config { DeploymentProjectPath = _tempDir }; + + var result = await check.CheckAsync(config, _logger); + + result.Passed.Should().BeFalse(because: "auth failures should block the conversation tier"); + result.Metadata!.Turns!.Should().Contain(t => t.Error != null && t.Error.Contains("Auth rejected"), + because: "auth failures should report targeted guidance"); + } + + [Fact] + public async Task CheckAsync_WhenTurnReturns500_ReturnsFailure() + { + File.WriteAllText(Path.Combine(_tempDir, "test.csproj"), ""); + var fakeProcess = CreateFakeProcess(exitImmediately: false); + _processService.Start(Arg.Any()).Returns(fakeProcess); + + var handler = new ConversationHttpHandler( + healthStatusCode: HttpStatusCode.OK, + messagesStatusCode: HttpStatusCode.InternalServerError); + var check = CreateCheck(handler); + var config = new Agent365Config { DeploymentProjectPath = _tempDir }; + + var result = await check.CheckAsync(config, _logger); + + result.Passed.Should().BeFalse(because: "server errors should fail the conversation tier"); + result.Metadata!.Turns!.Should().Contain(t => !t.Ok); + } + + [Fact] + public async Task CheckAsync_WhenProcessExitsDuringConversation_ReturnsFailure() + { + File.WriteAllText(Path.Combine(_tempDir, "test.csproj"), ""); + var fakeProcess = CreateFakeProcess(exitImmediately: false); + _processService.Start(Arg.Any()).Returns(fakeProcess); + + // Health returns OK, then process exits during message turn + var handler = new ConversationHttpHandler( + healthStatusCode: HttpStatusCode.OK, + messagesStatusCode: HttpStatusCode.OK, + killProcessOnMessage: fakeProcess); + var check = CreateCheck(handler); + var config = new Agent365Config { DeploymentProjectPath = _tempDir }; + + var result = await check.CheckAsync(config, _logger); + + result.Passed.Should().BeFalse(because: "the process crashed during conversation"); + result.ErrorMessage.Should().Contain("exited during conversation"); + } + + [Fact] + public async Task CheckAsync_WhenHealthTimesOut_ReturnsFailure() + { + File.WriteAllText(Path.Combine(_tempDir, "test.csproj"), ""); + var fakeProcess = CreateFakeProcess(exitImmediately: false); + _processService.Start(Arg.Any()).Returns(fakeProcess); + + // Health never returns success + var handler = new ConversationHttpHandler( + healthStatusCode: HttpStatusCode.ServiceUnavailable, + messagesStatusCode: HttpStatusCode.OK); + var check = CreateCheck(handler); + var config = new Agent365Config { DeploymentProjectPath = _tempDir }; + + var result = await check.CheckAsync(config, _logger); + + result.Passed.Should().BeFalse(because: "the app never became ready"); + result.ErrorMessage.Should().Contain("did not respond"); + } + + [Fact] + public async Task CheckAsync_SuccessfulTurns_IncludeLatencyAndSnippet() + { + File.WriteAllText(Path.Combine(_tempDir, "test.csproj"), ""); + var fakeProcess = CreateFakeProcess(exitImmediately: false); + _processService.Start(Arg.Any()).Returns(fakeProcess); + + var handler = new ConversationHttpHandler( + healthStatusCode: HttpStatusCode.OK, + messagesStatusCode: HttpStatusCode.OK, + responseBody: "{\"type\":\"message\",\"text\":\"I can help you\"}"); + var fakeReceiver = new FakeBotCallbackReceiver( + new BotCallbackResponse("I can help you", "message")); + var check = CreateCheck(handler, fakeReceiver); + var config = new Agent365Config { DeploymentProjectPath = _tempDir }; + + var result = await check.CheckAsync(config, _logger); + + result.Passed.Should().BeTrue(); + var turn = result.Metadata!.Turns!.First(); + turn.LatencyMs.Should().NotBeNull(because: "latency should always be captured"); + turn.StatusCode.Should().Be(200); + turn.ResponseSnippet.Should().Contain("I can help you"); + } + + [Fact] + public async Task CheckAsync_NodeJsProject_UsesNpmStart() + { + File.WriteAllText(Path.Combine(_tempDir, "package.json"), "{}"); + var fakeProcess = CreateFakeProcess(exitImmediately: false); + _processService.Start(Arg.Any()).Returns(fakeProcess); + + var handler = new ConversationHttpHandler( + healthStatusCode: HttpStatusCode.OK, + messagesStatusCode: HttpStatusCode.OK); + var check = CreateCheck(handler); + var config = new Agent365Config { DeploymentProjectPath = _tempDir }; + + await check.CheckAsync(config, _logger); + + _processService.Received(1).Start(Arg.Is(p => + p.FileName == "npm" && p.Arguments == "start")); + } + + [Fact] + public async Task CheckAsync_ContinuesAllTurnsEvenOnFailure() + { + File.WriteAllText(Path.Combine(_tempDir, "test.csproj"), ""); + var fakeProcess = CreateFakeProcess(exitImmediately: false); + _processService.Start(Arg.Any()).Returns(fakeProcess); + + // First turn fails, rest succeed — all 3 should be in the report + var handler = new ConversationHttpHandler( + healthStatusCode: HttpStatusCode.OK, + messagesStatusCode: HttpStatusCode.OK, + failFirstTurn: true); + var check = CreateCheck(handler); + var config = new Agent365Config { DeploymentProjectPath = _tempDir }; + + var result = await check.CheckAsync(config, _logger); + + result.Passed.Should().BeFalse(because: "at least one turn failed"); + result.Metadata!.Turns.Should().HaveCount(3, + because: "all turns should be attempted even if one fails for complete reporting"); + } + + [Fact] + public async Task CheckAsync_WhenCallbackReceiverProvided_TracksAgentResponded() + { + File.WriteAllText(Path.Combine(_tempDir, "test.csproj"), ""); + var fakeProcess = CreateFakeProcess(exitImmediately: false); + _processService.Start(Arg.Any()).Returns(fakeProcess); + + var handler = new ConversationHttpHandler( + healthStatusCode: HttpStatusCode.OK, + messagesStatusCode: HttpStatusCode.OK, + responseBody: "{\"type\":\"message\",\"text\":\"Hello!\"}"); + var fakeReceiver = new FakeBotCallbackReceiver( + new BotCallbackResponse("I can help you with that!", "message")); + var check = CreateCheck(handler, fakeReceiver); + var config = new Agent365Config { DeploymentProjectPath = _tempDir }; + + var result = await check.CheckAsync(config, _logger); + + result.Passed.Should().BeTrue(because: "all turns succeeded with agent responses"); + result.Metadata!.Turns.Should().OnlyContain( + t => t.AgentResponded == true, + because: "callback receiver reported agent responses for every turn"); + result.Metadata.Turns.Should().OnlyContain( + t => t.AgentResponseText == "I can help you with that!", + because: "agent response text should be captured from callback"); + } + + [Fact] + public async Task CheckAsync_WhenAgentDoesNotRespond_ReturnsFailure() + { + File.WriteAllText(Path.Combine(_tempDir, "test.csproj"), ""); + var fakeProcess = CreateFakeProcess(exitImmediately: false); + _processService.Start(Arg.Any()).Returns(fakeProcess); + + var handler = new ConversationHttpHandler( + healthStatusCode: HttpStatusCode.OK, + messagesStatusCode: HttpStatusCode.OK); + var fakeReceiver = new FakeBotCallbackReceiver(response: null); + var check = CreateCheck(handler, fakeReceiver); + var config = new Agent365Config { DeploymentProjectPath = _tempDir }; + + var result = await check.CheckAsync(config, _logger); + + result.Passed.Should().BeFalse( + because: "in non-playground mode, agent must respond for turn to pass"); + result.Metadata!.Turns.Should().OnlyContain( + t => t.AgentResponded == false, + because: "callback receiver returned no response"); + result.Metadata.Turns.Should().OnlyContain( + t => t.Ok == false && t.Error!.Contains("did not respond"), + because: "each turn should report agent did not respond"); + } + + [Fact] + public async Task CheckAsync_WhenNoCallbackReceiverInjected_AutoCreatesReceiverAndFailsWithoutResponse() + { + File.WriteAllText(Path.Combine(_tempDir, "test.csproj"), ""); + var fakeProcess = CreateFakeProcess(exitImmediately: false); + _processService.Start(Arg.Any()).Returns(fakeProcess); + + var handler = new ConversationHttpHandler( + healthStatusCode: HttpStatusCode.OK, + messagesStatusCode: HttpStatusCode.OK); + // No callback receiver injected — code auto-creates one internally. + // Since the mock handler does not call back, agentResponded will be false → failure. + var check = CreateCheck(handler, callbackReceiver: null); + var config = new Agent365Config { DeploymentProjectPath = _tempDir }; + + var result = await check.CheckAsync(config, _logger); + + result.Passed.Should().BeFalse( + because: "auto-created receiver gets no callback from mock handler so turns fail"); + result.Metadata!.Turns.Should().OnlyContain( + t => t.AgentResponded == false, + because: "auto-created receiver gets no callback from mock handler so agentResponded is false"); + } + + [Fact] + public async Task CheckAsync_WhenAgentReturnsErrorResponse_ReturnsFailure() + { + File.WriteAllText(Path.Combine(_tempDir, "test.csproj"), ""); + var fakeProcess = CreateFakeProcess(exitImmediately: false); + _processService.Start(Arg.Any()).Returns(fakeProcess); + + var handler = new ConversationHttpHandler( + healthStatusCode: HttpStatusCode.OK, + messagesStatusCode: HttpStatusCode.OK); + var fakeReceiver = new FakeBotCallbackReceiver( + new BotCallbackResponse("An internal error occurred while processing", "message")); + var check = CreateCheck(handler, fakeReceiver); + var config = new Agent365Config { DeploymentProjectPath = _tempDir }; + + var result = await check.CheckAsync(config, _logger); + + result.Passed.Should().BeFalse( + because: "agent responded with an error message"); + result.Metadata!.Turns.Should().OnlyContain( + t => t.Ok == false && t.Error!.Contains("error response"), + because: "each turn should report agent returned an error"); + } + + [Fact] + public async Task CheckAsync_InPlaygroundMode_PassesEvenWithoutAgentResponse() + { + File.WriteAllText(Path.Combine(_tempDir, "test.csproj"), ""); + var fakeProcess = CreateFakeProcess(exitImmediately: false); + var fakePlayground = CreateFakeProcess(exitImmediately: true); + _processService.Start(Arg.Any()) + .Returns(fakeProcess, fakePlayground); + + var handler = new ConversationHttpHandler( + healthStatusCode: HttpStatusCode.OK, + messagesStatusCode: HttpStatusCode.OK); + var fakeReceiver = new FakeBotCallbackReceiver(response: null); + var check = CreateCheck(handler, fakeReceiver, launchPlayground: true); + var config = new Agent365Config { DeploymentProjectPath = _tempDir }; + + var result = await check.CheckAsync(config, _logger); + + result.Passed.Should().BeTrue( + because: "in playground mode, missing agent response does not fail the turn"); + result.Metadata!.Turns.Should().OnlyContain( + t => t.Ok == true, + because: "playground mode is lenient about agent responses"); + } + + [Fact] + public async Task CheckAsync_DetailsSummaryIncludesAgentResponseCount() + { + File.WriteAllText(Path.Combine(_tempDir, "test.csproj"), ""); + var fakeProcess = CreateFakeProcess(exitImmediately: false); + _processService.Start(Arg.Any()).Returns(fakeProcess); + + var handler = new ConversationHttpHandler( + healthStatusCode: HttpStatusCode.OK, + messagesStatusCode: HttpStatusCode.OK); + var fakeReceiver = new FakeBotCallbackReceiver( + new BotCallbackResponse("Hi there", "message")); + var check = CreateCheck(handler, fakeReceiver); + var config = new Agent365Config { DeploymentProjectPath = _tempDir }; + + var result = await check.CheckAsync(config, _logger); + + result.Details.Should().Contain("agent responses received", + because: "details should summarize how many agent responses were captured"); + } + + [Fact] + public async Task CheckAsync_WhenAuthFailure_AgentRespondedIsFalse() + { + File.WriteAllText(Path.Combine(_tempDir, "test.csproj"), ""); + var fakeProcess = CreateFakeProcess(exitImmediately: false); + _processService.Start(Arg.Any()).Returns(fakeProcess); + + var handler = new ConversationHttpHandler( + healthStatusCode: HttpStatusCode.OK, + messagesStatusCode: HttpStatusCode.Unauthorized); + var fakeReceiver = new FakeBotCallbackReceiver( + new BotCallbackResponse("should not appear", "message")); + var check = CreateCheck(handler, fakeReceiver); + var config = new Agent365Config { DeploymentProjectPath = _tempDir }; + + var result = await check.CheckAsync(config, _logger); + + result.Passed.Should().BeFalse(); + result.Metadata!.Turns.Should().OnlyContain( + t => t.AgentResponded == false, + because: "auth failures should report agent did not respond, not attempt callback wait"); + } + + [Fact] + public async Task CheckAsync_ServiceUrlPointsToCallbackReceiver() + { + File.WriteAllText(Path.Combine(_tempDir, "test.csproj"), ""); + var fakeProcess = CreateFakeProcess(exitImmediately: false); + _processService.Start(Arg.Any()).Returns(fakeProcess); + + string? capturedServiceUrl = null; + var handler = new ConversationHttpHandler( + healthStatusCode: HttpStatusCode.OK, + messagesStatusCode: HttpStatusCode.OK, + captureServiceUrl: url => capturedServiceUrl = url); + var fakeReceiver = new FakeBotCallbackReceiver(response: null); + var check = CreateCheck(handler, fakeReceiver); + var config = new Agent365Config { DeploymentProjectPath = _tempDir }; + + await check.CheckAsync(config, _logger); + + capturedServiceUrl.Should().Be(fakeReceiver.ServiceUrl, + because: "activities should use the callback receiver's URL so the bot sends responses there"); + } + + [Fact] + public async Task CheckAsync_WhenPlaygroundEnabled_LaunchesAgentsPlayground() + { + File.WriteAllText(Path.Combine(_tempDir, "test.csproj"), ""); + var fakeAgentProcess = CreateFakeProcess(exitImmediately: false); + var fakePlaygroundProcess = CreateFakeProcess(exitImmediately: true); + + // First Start call returns the agent process, second returns the playground process + _processService.Start(Arg.Any()) + .Returns(fakeAgentProcess, fakePlaygroundProcess); + + var handler = new ConversationHttpHandler( + healthStatusCode: HttpStatusCode.OK, + messagesStatusCode: HttpStatusCode.OK); + var fakeReceiver = new FakeBotCallbackReceiver( + new BotCallbackResponse("Hello!", "message")); + var check = CreateCheck(handler, fakeReceiver, launchPlayground: true); + var config = new Agent365Config { DeploymentProjectPath = _tempDir }; + + var result = await check.CheckAsync(config, _logger); + + result.Passed.Should().BeTrue(); + result.Metadata!.PlaygroundLaunched.Should().BeTrue( + because: "playground was requested and started successfully"); + _processService.Received(2).Start(Arg.Any()); + _processService.Received(1).Start(Arg.Is(p => + p.FileName == "agentsplayground" && p.Arguments.Contains("-c \"emulator\""))); + } + + [Fact] + public async Task CheckAsync_WhenPlaygroundDisabled_DoesNotLaunchPlayground() + { + File.WriteAllText(Path.Combine(_tempDir, "test.csproj"), ""); + var fakeProcess = CreateFakeProcess(exitImmediately: false); + _processService.Start(Arg.Any()).Returns(fakeProcess); + + var handler = new ConversationHttpHandler( + healthStatusCode: HttpStatusCode.OK, + messagesStatusCode: HttpStatusCode.OK); + var fakeReceiver = new FakeBotCallbackReceiver( + new BotCallbackResponse("Hello!", "message")); + var check = CreateCheck(handler, fakeReceiver, launchPlayground: false); + var config = new Agent365Config { DeploymentProjectPath = _tempDir }; + + var result = await check.CheckAsync(config, _logger); + + result.Passed.Should().BeTrue(); + result.Metadata!.PlaygroundLaunched.Should().BeNull( + because: "playground was not requested"); + // Only one Start call for the agent process + _processService.Received(1).Start(Arg.Any()); + } + + [Fact] + public async Task CheckAsync_WhenPlaygroundFailsToStart_ReturnsSuccessWithoutPlayground() + { + File.WriteAllText(Path.Combine(_tempDir, "test.csproj"), ""); + var fakeAgentProcess = CreateFakeProcess(exitImmediately: false); + + // Agent process starts, playground fails to start + _processService.Start(Arg.Any()) + .Returns(fakeAgentProcess, (Process?)null); + + var handler = new ConversationHttpHandler( + healthStatusCode: HttpStatusCode.OK, + messagesStatusCode: HttpStatusCode.OK); + var fakeReceiver = new FakeBotCallbackReceiver( + new BotCallbackResponse("Hello!", "message")); + var check = CreateCheck(handler, fakeReceiver, launchPlayground: true); + var config = new Agent365Config { DeploymentProjectPath = _tempDir }; + + var result = await check.CheckAsync(config, _logger); + + result.Passed.Should().BeTrue( + because: "playground failure should not block conversation validation"); + result.Metadata!.PlaygroundLaunched.Should().BeNull( + because: "playground failed to start so it should not be reported as launched"); + } + + /// + /// Creates a fake Process for testing. + /// + private static Process CreateFakeProcess(bool exitImmediately, int exitCode = 0) + { + var startInfo = new ProcessStartInfo + { + FileName = OperatingSystem.IsWindows() ? "cmd.exe" : "/bin/sh", + Arguments = OperatingSystem.IsWindows() + ? (exitImmediately ? "/c exit 1" : "/c ping -n 60 127.0.0.1 >nul") + : (exitImmediately ? "-c 'exit 1'" : "-c 'sleep 60'"), + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + }; + + var process = Process.Start(startInfo)!; + + if (exitImmediately) + { + process.WaitForExit(5000); + } + + return process; + } + + /// + /// Fake callback receiver for testing. Returns a configurable response. + /// + private sealed class FakeBotCallbackReceiver : IBotCallbackReceiver + { + private readonly BotCallbackResponse? _response; + + public FakeBotCallbackReceiver(BotCallbackResponse? response = null) + { + _response = response; + } + + public string ServiceUrl => "http://localhost:39999"; + + public Task StartAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task WaitForResponseAsync( + TimeSpan timeout, + CancellationToken cancellationToken = default) + { + return Task.FromResult(_response); + } + + public void ClearResponses() { } + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + } + + /// + /// HTTP handler that simulates both health and /api/messages endpoints. + /// + private sealed class ConversationHttpHandler : HttpMessageHandler + { + private readonly HttpStatusCode _healthStatusCode; + private readonly HttpStatusCode _messagesStatusCode; + private readonly string? _responseBody; + private readonly Process? _killProcessOnMessage; + private readonly bool _failFirstTurn; + private readonly Action? _captureServiceUrl; + private int _messageCount; + + public ConversationHttpHandler( + HttpStatusCode healthStatusCode, + HttpStatusCode messagesStatusCode, + string? responseBody = null, + Process? killProcessOnMessage = null, + bool failFirstTurn = false, + Action? captureServiceUrl = null) + { + _healthStatusCode = healthStatusCode; + _messagesStatusCode = messagesStatusCode; + _responseBody = responseBody; + _killProcessOnMessage = killProcessOnMessage; + _failFirstTurn = failFirstTurn; + _captureServiceUrl = captureServiceUrl; + } + + protected override async Task SendAsync( + HttpRequestMessage request, + CancellationToken cancellationToken) + { + var path = request.RequestUri?.AbsolutePath ?? string.Empty; + + if (path.Contains("/api/health")) + { + return new HttpResponseMessage(_healthStatusCode); + } + + if (path.Contains("/api/messages")) + { + _messageCount++; + + // Capture serviceUrl from the activity body if requested + if (_captureServiceUrl is not null && request.Content is not null) + { + var body = await request.Content.ReadAsStringAsync(cancellationToken); + try + { + using var doc = System.Text.Json.JsonDocument.Parse(body); + if (doc.RootElement.TryGetProperty("serviceUrl", out var prop)) + { + _captureServiceUrl(prop.GetString()!); + } + } + catch + { + // Best effort + } + } + + if (_killProcessOnMessage is not null) + { + try + { + if (!_killProcessOnMessage.HasExited) + { + _killProcessOnMessage.Kill(entireProcessTree: true); + _killProcessOnMessage.WaitForExit(5000); + } + } + catch + { + // Best effort + } + } + + if (_failFirstTurn && _messageCount == 1) + { + return new HttpResponseMessage(HttpStatusCode.InternalServerError); + } + + var response = new HttpResponseMessage(_messagesStatusCode); + if (_responseBody is not null) + { + response.Content = new StringContent(_responseBody, System.Text.Encoding.UTF8, "application/json"); + } + return response; + } + + return new HttpResponseMessage(HttpStatusCode.NotFound); + } + } +} diff --git a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/LocalRuntimeRequirementCheckTests.cs b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/LocalRuntimeRequirementCheckTests.cs new file mode 100644 index 00000000..1aa236a0 --- /dev/null +++ b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/LocalRuntimeRequirementCheckTests.cs @@ -0,0 +1,281 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Diagnostics; +using System.Net; +using FluentAssertions; +using Microsoft.Agents.A365.DevTools.Cli.Models; +using Microsoft.Agents.A365.DevTools.Cli.Services; +using Microsoft.Agents.A365.DevTools.Cli.Services.Requirements; +using Microsoft.Agents.A365.DevTools.Cli.Services.Requirements.RequirementChecks; +using Microsoft.Extensions.Logging; +using NSubstitute; +using Xunit; + +namespace Microsoft.Agents.A365.DevTools.Cli.Tests.Services.Requirements; + +public class LocalRuntimeRequirementCheckTests : IDisposable +{ + private readonly ILogger _logger; + private readonly PlatformDetector _platformDetector; + private readonly IProcessService _processService; + private readonly string _tempDir; + + public LocalRuntimeRequirementCheckTests() + { + _logger = Substitute.For(); + _platformDetector = new PlatformDetector(Substitute.For>()); + _processService = Substitute.For(); + _tempDir = Path.Combine(Path.GetTempPath(), $"a365-runtime-test-{Guid.NewGuid():N}"); + Directory.CreateDirectory(_tempDir); + } + + public void Dispose() + { + if (Directory.Exists(_tempDir)) + { + Directory.Delete(_tempDir, recursive: true); + } + } + + private LocalRuntimeRequirementCheck CreateCheck(HttpMessageHandler? handler = null) + { + var httpClient = handler is not null ? new HttpClient(handler) : new HttpClient(); + return new LocalRuntimeRequirementCheck(_platformDetector, _processService, httpClient); + } + + [Fact] + public void Check_HasExpectedMetadata() + { + var check = CreateCheck(); + check.Name.Should().Be("Local Runtime"); + check.Category.Should().Be("Code Health"); + check.Description.Should().Contain("health endpoint"); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + public async Task CheckAsync_WhenDeploymentProjectPathIsEmpty_FallsBackToCwd(string? path) + { + // Arrange - when deploymentProjectPath is empty, falls back to CWD + var check = CreateCheck(); + var config = new Agent365Config { DeploymentProjectPath = path ?? string.Empty }; + + // Act + var result = await check.CheckAsync(config, _logger); + + // Assert - should not crash; CWD may or may not have a recognized project + result.Should().NotBeNull(); + } + + [Fact] + public async Task CheckAsync_WhenDeploymentProjectPathIsInvalid_ReturnsFailure() + { + // Arrange + var check = CreateCheck(); + var config = new Agent365Config { DeploymentProjectPath = "path\0with\0nulls" }; + + // Act + var result = await check.CheckAsync(config, _logger); + + // Assert - Path.GetFullPath throws, caught by ExecuteCheckWithLoggingAsync + result.Passed.Should().BeFalse(because: "an invalid path format should be reported as a failure"); + } + + [Fact] + public async Task CheckAsync_WhenDirectoryDoesNotExist_ReturnsFailure() + { + // Arrange + var check = CreateCheck(); + var config = new Agent365Config { DeploymentProjectPath = Path.Combine(_tempDir, "nonexistent") }; + + // Act + var result = await check.CheckAsync(config, _logger); + + // Assert + result.Passed.Should().BeFalse(because: "a non-existent directory cannot run an app"); + result.ErrorMessage.Should().Contain("does not exist"); + } + + [Fact] + public async Task CheckAsync_WhenPlatformIsUnknown_ReturnsWarning() + { + // Arrange - empty directory, PlatformDetector returns Unknown + var check = CreateCheck(); + var config = new Agent365Config { DeploymentProjectPath = _tempDir }; + + // Act + var result = await check.CheckAsync(config, _logger); + + // Assert + result.Passed.Should().BeTrue(because: "unknown platform is a non-blocking warning"); + result.IsWarning.Should().BeTrue(); + } + + [Fact] + public async Task CheckAsync_WhenProcessFailsToStart_ReturnsFailure() + { + // Arrange - create a .csproj so platform is detected as DotNet + File.WriteAllText(Path.Combine(_tempDir, "test.csproj"), ""); + _processService.Start(Arg.Any()).Returns((Process?)null); + var check = CreateCheck(); + var config = new Agent365Config { DeploymentProjectPath = _tempDir }; + + // Act + var result = await check.CheckAsync(config, _logger); + + // Assert + result.Passed.Should().BeFalse(because: "a process that fails to start cannot serve health requests"); + result.ErrorMessage.Should().Contain("Failed to start"); + } + + [Fact] + public async Task CheckAsync_WhenHealthEndpointResponds200_ReturnsSuccess() + { + // Arrange + File.WriteAllText(Path.Combine(_tempDir, "test.csproj"), ""); + var fakeProcess = CreateFakeProcess(exitImmediately: false); + _processService.Start(Arg.Any()).Returns(fakeProcess); + + var handler = new FakeHttpHandler(HttpStatusCode.OK); + var check = CreateCheck(handler); + var config = new Agent365Config { DeploymentProjectPath = _tempDir }; + + // Act + var result = await check.CheckAsync(config, _logger); + + // Assert + result.Passed.Should().BeTrue(because: "a health endpoint returning 200 means the app is running"); + result.Details.Should().Contain("200"); + } + + [Fact] + public async Task CheckAsync_WhenProcessExitsEarly_ReturnsFailure() + { + // Arrange + File.WriteAllText(Path.Combine(_tempDir, "test.csproj"), ""); + var fakeProcess = CreateFakeProcess(exitImmediately: true, exitCode: 1); + _processService.Start(Arg.Any()).Returns(fakeProcess); + + var handler = new FakeHttpHandler(HttpStatusCode.OK); + var check = CreateCheck(handler); + var config = new Agent365Config { DeploymentProjectPath = _tempDir }; + + // Act + var result = await check.CheckAsync(config, _logger); + + // Assert + result.Passed.Should().BeFalse(because: "an early exit means the app crashed before responding"); + result.ErrorMessage.Should().Contain("exited early"); + } + + [Theory] + [InlineData("https://localhost:3978/api/messages", 3978)] + [InlineData("https://127.0.0.1:8080/api/messages", 8080)] + [InlineData("https://myapp.azurewebsites.net/api/messages", 5000)] + [InlineData("https://localhost/api/messages", 5000)] + [InlineData("", 5000)] + [InlineData(null, 5000)] + public void ResolvePort_ReturnsExpectedPort(string? endpoint, int expected) + { + var port = LocalRuntimeRequirementCheck.ResolvePort(endpoint); + port.Should().Be(expected, because: "port resolution should respect localhost URLs and fall back to default"); + } + + [Fact] + public async Task CheckAsync_NodeJsProject_UsesNpmStart() + { + // Arrange + File.WriteAllText(Path.Combine(_tempDir, "package.json"), "{}"); + var fakeProcess = CreateFakeProcess(exitImmediately: false); + _processService.Start(Arg.Any()).Returns(fakeProcess); + + var handler = new FakeHttpHandler(HttpStatusCode.OK); + var check = CreateCheck(handler); + var config = new Agent365Config { DeploymentProjectPath = _tempDir }; + + // Act + var result = await check.CheckAsync(config, _logger); + + // Assert + result.Passed.Should().BeTrue(); + _processService.Received(1).Start(Arg.Is(p => + p.FileName == "npm" && p.Arguments == "start")); + } + + [Fact] + public async Task CheckAsync_DotNetProject_SetsAspNetCoreUrls() + { + // Arrange + File.WriteAllText(Path.Combine(_tempDir, "test.csproj"), ""); + var fakeProcess = CreateFakeProcess(exitImmediately: false); + _processService.Start(Arg.Any()).Returns(fakeProcess); + + var handler = new FakeHttpHandler(HttpStatusCode.OK); + var check = CreateCheck(handler); + var config = new Agent365Config + { + DeploymentProjectPath = _tempDir, + MessagingEndpoint = "https://localhost:3978/api/messages" + }; + + // Act + await check.CheckAsync(config, _logger); + + // Assert + _processService.Received(1).Start(Arg.Is(p => + p.FileName == "dotnet" && + p.EnvironmentVariables["ASPNETCORE_URLS"] == "http://localhost:3978")); + } + + /// + /// Creates a fake Process for testing. When exitImmediately is true, the process + /// appears to have already exited. + /// + private static Process CreateFakeProcess(bool exitImmediately, int exitCode = 0) + { + // Start a real but trivial process we can control + var startInfo = new ProcessStartInfo + { + FileName = OperatingSystem.IsWindows() ? "cmd.exe" : "/bin/sh", + Arguments = OperatingSystem.IsWindows() + ? (exitImmediately ? "/c exit 1" : "/c ping -n 60 127.0.0.1 >nul") + : (exitImmediately ? "-c 'exit 1'" : "-c 'sleep 60'"), + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + }; + + var process = Process.Start(startInfo)!; + + if (exitImmediately) + { + process.WaitForExit(5000); + } + + return process; + } + + /// + /// Fake HTTP handler that returns a configurable status code. + /// + private sealed class FakeHttpHandler : HttpMessageHandler + { + private readonly HttpStatusCode _statusCode; + + public FakeHttpHandler(HttpStatusCode statusCode) + { + _statusCode = statusCode; + } + + protected override Task SendAsync( + HttpRequestMessage request, + CancellationToken cancellationToken) + { + return Task.FromResult(new HttpResponseMessage(_statusCode)); + } + } +} diff --git a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/ProjectBuildRequirementCheckTests.cs b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/ProjectBuildRequirementCheckTests.cs new file mode 100644 index 00000000..9199c515 --- /dev/null +++ b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/ProjectBuildRequirementCheckTests.cs @@ -0,0 +1,308 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using FluentAssertions; +using Microsoft.Agents.A365.DevTools.Cli.Models; +using Microsoft.Agents.A365.DevTools.Cli.Services; +using Microsoft.Agents.A365.DevTools.Cli.Services.Requirements; +using Microsoft.Agents.A365.DevTools.Cli.Services.Requirements.RequirementChecks; +using Microsoft.Extensions.Logging; +using NSubstitute; +using Xunit; + +namespace Microsoft.Agents.A365.DevTools.Cli.Tests.Services.Requirements; + +public class ProjectBuildRequirementCheckTests : IDisposable +{ + private readonly ILogger _logger; + private readonly PlatformDetector _platformDetector; + private readonly CommandExecutor _commandExecutor; + private readonly ProjectBuildRequirementCheck _check; + private readonly string _tempDir; + + public ProjectBuildRequirementCheckTests() + { + _logger = Substitute.For(); + _platformDetector = new PlatformDetector(Substitute.For>()); + _commandExecutor = Substitute.For(Substitute.For>()); + _check = new ProjectBuildRequirementCheck(_platformDetector, _commandExecutor); + _tempDir = Path.Combine(Path.GetTempPath(), $"a365-build-test-{Guid.NewGuid():N}"); + Directory.CreateDirectory(_tempDir); + } + + public void Dispose() + { + if (Directory.Exists(_tempDir)) + { + Directory.Delete(_tempDir, recursive: true); + } + } + + [Fact] + public void Check_HasExpectedMetadata() + { + _check.Name.Should().Be("Project Build"); + _check.Category.Should().Be("Code Health"); + _check.Description.Should().Contain("warnings treated as errors"); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + public async Task CheckAsync_WhenDeploymentProjectPathIsEmpty_FallsBackToCwd(string? path) + { + // Arrange - when deploymentProjectPath is empty, falls back to CWD + var config = new Agent365Config { DeploymentProjectPath = path ?? string.Empty }; + + // Act + var result = await _check.CheckAsync(config, _logger); + + // Assert - should not crash; CWD may or may not have a recognized project + result.Should().NotBeNull(); + } + + [Fact] + public async Task CheckAsync_WhenDeploymentProjectPathIsInvalid_ReturnsFailure() + { + // Arrange + var config = new Agent365Config { DeploymentProjectPath = "path\0with\0nulls" }; + + // Act + var result = await _check.CheckAsync(config, _logger); + + // Assert - Path.GetFullPath throws, caught by ExecuteCheckWithLoggingAsync + result.Passed.Should().BeFalse(because: "an invalid path format should be reported as a failure"); + } + + [Fact] + public async Task CheckAsync_WhenDeploymentProjectPathDoesNotExist_ReturnsFailure() + { + // Arrange + var config = new Agent365Config { DeploymentProjectPath = Path.Combine(_tempDir, "nonexistent") }; + + // Act + var result = await _check.CheckAsync(config, _logger); + + // Assert + result.Passed.Should().BeFalse(because: "a non-existent directory cannot be built"); + result.ErrorMessage.Should().Contain("does not exist"); + } + + [Fact] + public async Task CheckAsync_WhenPlatformIsUnknown_ReturnsWarning() + { + // Arrange - empty directory has no project files, so PlatformDetector returns Unknown + var config = new Agent365Config { DeploymentProjectPath = _tempDir }; + + // Act + var result = await _check.CheckAsync(config, _logger); + + // Assert + result.Passed.Should().BeTrue(because: "unknown platform is a non-blocking warning"); + result.IsWarning.Should().BeTrue(); + result.ErrorMessage.Should().Contain("Could not detect project platform"); + } + + [Fact] + public async Task CheckAsync_WhenDotNetBuildSucceeds_ReturnsSuccess() + { + // Arrange - create a .csproj so PlatformDetector identifies DotNet + File.WriteAllText(Path.Combine(_tempDir, "test.csproj"), ""); + var config = new Agent365Config { DeploymentProjectPath = _tempDir }; + _commandExecutor.ExecuteAsync( + Arg.Is("dotnet"), + Arg.Is(a => a.Contains("TreatWarningsAsErrors")), + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any()) + .Returns(new CommandResult { ExitCode = 0, StandardOutput = "Build succeeded." }); + + // Act + var result = await _check.CheckAsync(config, _logger); + + // Assert + result.Passed.Should().BeTrue(because: "a successful build should pass the check"); + result.Details.Should().Contain("DotNet"); + } + + [Fact] + public async Task CheckAsync_WhenDotNetBuildFails_ReturnsFailure() + { + // Arrange + File.WriteAllText(Path.Combine(_tempDir, "test.csproj"), ""); + var config = new Agent365Config { DeploymentProjectPath = _tempDir }; + _commandExecutor.ExecuteAsync( + Arg.Is("dotnet"), + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any()) + .Returns(new CommandResult + { + ExitCode = 1, + StandardOutput = "Program.cs(10,5): error CS1002: ; expected\nBuild FAILED." + }); + + // Act + var result = await _check.CheckAsync(config, _logger); + + // Assert + result.Passed.Should().BeFalse(because: "a failed build must be reported as a check failure"); + result.ErrorMessage.Should().Contain("build failed", because: "the error message should identify the failure type"); + } + + [Fact] + public async Task CheckAsync_WhenDotNetBuildHasWarningsTreatedAsErrors_ReturnsFailure() + { + // Arrange + File.WriteAllText(Path.Combine(_tempDir, "test.csproj"), ""); + var config = new Agent365Config { DeploymentProjectPath = _tempDir }; + _commandExecutor.ExecuteAsync( + Arg.Is("dotnet"), + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any()) + .Returns(new CommandResult + { + ExitCode = 1, + StandardOutput = "Program.cs(5,1): error CS8600: Converting null literal or possible null value to non-nullable type. [Treated as error]\nBuild FAILED." + }); + + // Act + var result = await _check.CheckAsync(config, _logger); + + // Assert + result.Passed.Should().BeFalse(because: "warnings treated as errors should cause build failure"); + result.ErrorMessage.Should().Contain("DotNet"); + result.ResolutionGuidance.Should().Contain("TreatWarningsAsErrors", because: "guidance should tell the user how to reproduce locally"); + } + + [Fact] + public async Task CheckAsync_WhenNodeJsBuildSucceeds_ReturnsSuccess() + { + // Arrange - create a package.json so PlatformDetector identifies NodeJs + File.WriteAllText(Path.Combine(_tempDir, "package.json"), "{}"); + var config = new Agent365Config { DeploymentProjectPath = _tempDir }; + _commandExecutor.ExecuteAsync( + Arg.Is("npm"), + Arg.Is("run build"), + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any()) + .Returns(new CommandResult { ExitCode = 0, StandardOutput = "Build completed." }); + + // Act + var result = await _check.CheckAsync(config, _logger); + + // Assert + result.Passed.Should().BeTrue(because: "a successful Node.js build should pass"); + result.Details.Should().Contain("NodeJs"); + } + + [Fact] + public async Task CheckAsync_WhenNodeJsBuildFails_ReturnsFailure() + { + // Arrange + File.WriteAllText(Path.Combine(_tempDir, "package.json"), "{}"); + var config = new Agent365Config { DeploymentProjectPath = _tempDir }; + _commandExecutor.ExecuteAsync( + Arg.Is("npm"), + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any()) + .Returns(new CommandResult + { + ExitCode = 1, + StandardError = "Error: Cannot find module './config'" + }); + + // Act + var result = await _check.CheckAsync(config, _logger); + + // Assert + result.Passed.Should().BeFalse(because: "a failed Node.js build must be reported"); + result.ResolutionGuidance.Should().Contain("npm run build"); + } + + [Fact] + public async Task CheckAsync_WhenPythonBuildSucceeds_ReturnsSuccess() + { + // Arrange - create a .py file so PlatformDetector identifies Python + File.WriteAllText(Path.Combine(_tempDir, "app.py"), "print('hello')"); + var config = new Agent365Config { DeploymentProjectPath = _tempDir }; + _commandExecutor.ExecuteAsync( + Arg.Is("python"), + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any()) + .Returns(new CommandResult { ExitCode = 0, StandardOutput = "" }); + + // Act + var result = await _check.CheckAsync(config, _logger); + + // Assert + result.Passed.Should().BeTrue(because: "a successful Python syntax check should pass"); + result.Details.Should().Contain("Python"); + } + + [Fact] + public async Task CheckAsync_WhenBuildFailsWithNoOutput_ReportsExitCode() + { + // Arrange + File.WriteAllText(Path.Combine(_tempDir, "test.csproj"), ""); + var config = new Agent365Config { DeploymentProjectPath = _tempDir }; + _commandExecutor.ExecuteAsync( + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any()) + .Returns(new CommandResult { ExitCode = 1, StandardOutput = "", StandardError = "" }); + + // Act + var result = await _check.CheckAsync(config, _logger); + + // Assert + result.Passed.Should().BeFalse(); + result.ErrorMessage.Should().Contain("exited with code 1", because: "when there is no output, the exit code is the only diagnostic"); + } + + [Fact] + public async Task CheckAsync_DotNetBuild_PassesTreatWarningsAsErrors() + { + // Arrange + File.WriteAllText(Path.Combine(_tempDir, "test.csproj"), ""); + var config = new Agent365Config { DeploymentProjectPath = _tempDir }; + _commandExecutor.ExecuteAsync( + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any()) + .Returns(new CommandResult { ExitCode = 0 }); + + // Act + await _check.CheckAsync(config, _logger); + + // Assert - verify the correct build arguments were passed + await _commandExecutor.Received(1).ExecuteAsync( + "dotnet", + Arg.Is(a => a.Contains("/p:TreatWarningsAsErrors=true")), + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any()); + } +} diff --git a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/ToolingManifestRequirementCheckTests.cs b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/ToolingManifestRequirementCheckTests.cs new file mode 100644 index 00000000..80d96819 --- /dev/null +++ b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/ToolingManifestRequirementCheckTests.cs @@ -0,0 +1,224 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using FluentAssertions; +using Microsoft.Agents.A365.DevTools.Cli.Constants; +using Microsoft.Agents.A365.DevTools.Cli.Models; +using Microsoft.Agents.A365.DevTools.Cli.Services.Requirements; +using Microsoft.Agents.A365.DevTools.Cli.Services.Requirements.RequirementChecks; +using Microsoft.Extensions.Logging; +using NSubstitute; +using Xunit; + +namespace Microsoft.Agents.A365.DevTools.Cli.Tests.Services.Requirements; + +public class ToolingManifestRequirementCheckTests : IDisposable +{ + private readonly ILogger _logger; + private readonly ToolingManifestRequirementCheck _check; + private readonly string _tempDir; + + public ToolingManifestRequirementCheckTests() + { + _logger = Substitute.For(); + _check = new ToolingManifestRequirementCheck(); + _tempDir = Path.Combine(Path.GetTempPath(), $"a365-manifest-test-{Guid.NewGuid():N}"); + Directory.CreateDirectory(_tempDir); + } + + public void Dispose() + { + if (Directory.Exists(_tempDir)) + { + Directory.Delete(_tempDir, recursive: true); + } + } + + [Fact] + public void Check_HasExpectedMetadata() + { + _check.Name.Should().Be("Tooling Manifest"); + _check.Category.Should().Be("Configuration"); + _check.Description.Should().Contain("ToolingManifest.json"); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + public async Task CheckAsync_WhenDeploymentProjectPathIsEmpty_FallsBackToCwd(string? path) + { + // Arrange - when deploymentProjectPath is empty, falls back to CWD + // CWD likely has no ToolingManifest.json, so we expect a warning about missing file + var config = new Agent365Config { DeploymentProjectPath = path ?? string.Empty }; + + // Act + var result = await _check.CheckAsync(config, _logger); + + // Assert - should not fail, should either warn (no manifest found) or pass (if manifest exists in CWD) + result.Passed.Should().BeTrue(because: "missing manifest in CWD is a non-blocking warning"); + } + + [Fact] + public async Task CheckAsync_WhenManifestFileIsMissing_ReturnsPass() + { + // Arrange - use a directory that exists but has no manifest + var config = new Agent365Config { DeploymentProjectPath = _tempDir }; + + // Act + var result = await _check.CheckAsync(config, _logger); + + // Assert + result.Passed.Should().BeTrue(because: "manifest is optional, check passes when file is absent"); + result.IsWarning.Should().BeFalse(); + } + + [Fact] + public async Task CheckAsync_WhenManifestContainsInvalidJson_ReturnsFailure() + { + // Arrange + var manifestPath = Path.Combine(_tempDir, McpConstants.ToolingManifestFileName); + await File.WriteAllTextAsync(manifestPath, "{ not valid json }"); + var config = new Agent365Config { DeploymentProjectPath = _tempDir }; + + // Act + var result = await _check.CheckAsync(config, _logger); + + // Assert + result.Passed.Should().BeFalse(because: "invalid JSON is a parse error that must be fixed"); + result.ErrorMessage.Should().Contain("invalid JSON"); + } + + [Fact] + public async Task CheckAsync_WhenManifestIsJsonNull_ReturnsFailure() + { + // Arrange + var manifestPath = Path.Combine(_tempDir, McpConstants.ToolingManifestFileName); + await File.WriteAllTextAsync(manifestPath, "null"); + var config = new Agent365Config { DeploymentProjectPath = _tempDir }; + + // Act + var result = await _check.CheckAsync(config, _logger); + + // Assert + result.Passed.Should().BeFalse(because: "a null manifest is not a valid ToolingManifest"); + result.ErrorMessage.Should().Contain("mcpServers must be an array"); + } + + [Fact] + public async Task CheckAsync_WhenMcpServersIsNull_ReturnsFailure() + { + // Arrange + var manifestPath = Path.Combine(_tempDir, McpConstants.ToolingManifestFileName); + await File.WriteAllTextAsync(manifestPath, """{ "mcpServers": null }"""); + var config = new Agent365Config { DeploymentProjectPath = _tempDir }; + + // Act + var result = await _check.CheckAsync(config, _logger); + + // Assert + result.Passed.Should().BeFalse(because: "mcpServers null is not a valid manifest"); + result.ErrorMessage.Should().Contain("mcpServers must be an array"); + } + + [Fact] + public async Task CheckAsync_WhenManifestHasNoServers_ReturnsFailure() + { + // Arrange + var manifestPath = Path.Combine(_tempDir, McpConstants.ToolingManifestFileName); + await File.WriteAllTextAsync(manifestPath, """{ "mcpServers": [] }"""); + var config = new Agent365Config { DeploymentProjectPath = _tempDir }; + + // Act + var result = await _check.CheckAsync(config, _logger); + + // Assert + result.Passed.Should().BeFalse(because: "an empty mcpServers array is invalid per ToolingManifest validation rules"); + result.ErrorMessage.Should().Contain("validation error"); + } + + [Fact] + public async Task CheckAsync_WhenManifestHasDuplicateServerNames_ReturnsFailure() + { + // Arrange + var manifestPath = Path.Combine(_tempDir, McpConstants.ToolingManifestFileName); + var content = """ + { + "mcpServers": [ + { "mcpServerName": "MCP_MailTools", "url": "https://example.com/mail" }, + { "mcpServerName": "MCP_MailTools", "url": "https://example.com/mail2" } + ] + } + """; + await File.WriteAllTextAsync(manifestPath, content); + var config = new Agent365Config { DeploymentProjectPath = _tempDir }; + + // Act + var result = await _check.CheckAsync(config, _logger); + + // Assert + result.Passed.Should().BeFalse(because: "duplicate server names are not allowed"); + result.ErrorMessage.Should().Contain("Duplicate"); + } + + [Fact] + public async Task CheckAsync_WhenManifestHasServerMissingUrl_ReturnsFailure() + { + // Arrange + var manifestPath = Path.Combine(_tempDir, McpConstants.ToolingManifestFileName); + var content = """ + { + "mcpServers": [ + { "mcpServerName": "MCP_MailTools" } + ] + } + """; + await File.WriteAllTextAsync(manifestPath, content); + var config = new Agent365Config { DeploymentProjectPath = _tempDir }; + + // Act + var result = await _check.CheckAsync(config, _logger); + + // Assert + result.Passed.Should().BeFalse(because: "MCP server entries require a url field"); + result.ErrorMessage.Should().Contain("invalid"); + } + + [Fact] + public async Task CheckAsync_WhenManifestIsValid_ReturnsSuccess() + { + // Arrange + var manifestPath = Path.Combine(_tempDir, McpConstants.ToolingManifestFileName); + var content = """ + { + "mcpServers": [ + { "mcpServerName": "MCP_MailTools", "url": "https://example.com/mail" }, + { "mcpServerName": "MCP_CalendarTools", "url": "https://example.com/calendar" } + ] + } + """; + await File.WriteAllTextAsync(manifestPath, content); + var config = new Agent365Config { DeploymentProjectPath = _tempDir }; + + // Act + var result = await _check.CheckAsync(config, _logger); + + // Assert + result.Passed.Should().BeTrue(because: "a valid manifest with proper server entries should pass validation"); + result.IsWarning.Should().BeFalse(); + result.Details.Should().Contain("2 MCP server(s) configured"); + } + + [Fact] + public async Task CheckAsync_WhenDeploymentProjectPathIsInvalid_ReturnsFailure() + { + // Arrange - use characters that are invalid in a path + var config = new Agent365Config { DeploymentProjectPath = "path\0with\0nulls" }; + + // Act + var result = await _check.CheckAsync(config, _logger); + + // Assert - Path.GetFullPath throws, caught by ExecuteCheckWithLoggingAsync + result.Passed.Should().BeFalse(because: "an invalid path format should be reported as a failure"); + } +} diff --git a/src/Tests/Microsoft.Agents.A365.DevTools.Validation.Tests/Microsoft.Agents.A365.DevTools.Validation.Tests.csproj b/src/Tests/Microsoft.Agents.A365.DevTools.Validation.Tests/Microsoft.Agents.A365.DevTools.Validation.Tests.csproj new file mode 100644 index 00000000..3a746e90 --- /dev/null +++ b/src/Tests/Microsoft.Agents.A365.DevTools.Validation.Tests/Microsoft.Agents.A365.DevTools.Validation.Tests.csproj @@ -0,0 +1,18 @@ + + + net8.0 + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/Tests/Microsoft.Agents.A365.DevTools.Validation.Tests/Validation/ValidationContractsTests.cs b/src/Tests/Microsoft.Agents.A365.DevTools.Validation.Tests/Validation/ValidationContractsTests.cs new file mode 100644 index 00000000..59a88cf4 --- /dev/null +++ b/src/Tests/Microsoft.Agents.A365.DevTools.Validation.Tests/Validation/ValidationContractsTests.cs @@ -0,0 +1,40 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using FluentAssertions; +using Microsoft.Agents.A365.DevTools.Validation; +using Xunit; + +namespace Microsoft.Agents.A365.DevTools.Validation.Tests.Validation; + +public class ValidationContractsTests +{ + [Fact] + public void Success_ReturnsValidOutcomeWithoutIssues() + { + // Act + var outcome = ValidationOutcome.Success(); + + // Assert + outcome.IsValid.Should().BeTrue("a success outcome must indicate that validation passed"); + outcome.Issues.Should().BeEmpty("a successful validation should not carry issues"); + } + + [Fact] + public void Failure_ReturnsInvalidOutcomeWithIssues() + { + // Arrange + var issues = new[] + { + new ValidationIssue("MISSING_BLUEPRINT", "Blueprint ID is required"), + new ValidationIssue("MISSING_MANIFEST", "ToolingManifest.json not found", ValidationSeverity.Warning) + }; + + // Act + var outcome = ValidationOutcome.Failure(issues); + + // Assert + outcome.IsValid.Should().BeFalse("a failure outcome must indicate validation did not pass"); + outcome.Issues.Should().ContainInOrder(issues, "the failure helper should preserve the supplied issues verbatim"); + } +} \ No newline at end of file From 1e18b78b303b1b1c31a1df856a7ebc0049591ac7 Mon Sep 17 00:00:00 2001 From: Abbinayaa Subramanian Date: Fri, 22 May 2026 16:14:35 -0700 Subject: [PATCH 02/27] Fixing validation errors for py and nodejs --- .../Commands/ValidateCommand.cs | 275 +++++++++-- .../Services/CommandExecutor.cs | 31 +- .../Requirements/RequirementCheckResult.cs | 28 ++ .../ConversationRequirementCheck.cs | 300 +++++++++++- .../LocalRuntimeRequirementCheck.cs | 266 ++++++++++- .../ProjectBuildRequirementCheck.cs | 444 +++++++++++++++++- .../TelemetryRequirementCheck.cs | 245 ++++++++++ .../ValidateReport.cs | 40 +- .../Services/CommandExecutorTests.cs | 13 +- .../LocalRuntimeRequirementCheckTests.cs | 140 ++++++ .../ProjectBuildRequirementCheckTests.cs | 79 ++++ .../TelemetryRequirementCheckTests.cs | 322 +++++++++++++ 12 files changed, 2095 insertions(+), 88 deletions(-) create mode 100644 src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/TelemetryRequirementCheck.cs create mode 100644 src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/TelemetryRequirementCheckTests.cs diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Commands/ValidateCommand.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Commands/ValidateCommand.cs index 23f27ecf..67b8542e 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Commands/ValidateCommand.cs +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Commands/ValidateCommand.cs @@ -26,6 +26,7 @@ public sealed class ValidateCommand // Status markers — use characters supported across Windows/macOS/Linux terminals private const string PassMark = "\u221A"; // √ (square root, same as Windows renders for checkmark) private const string FailMark = "X"; + private const string WarnMark = "!"; private const string SkipMark = "-"; private static readonly JsonSerializerOptions ReportSerializerOptions = new() @@ -84,31 +85,74 @@ public static Command CreateCommand( : null }; - // Phase 2: Run requirement checks and map to tiers - var checks = requirementChecksOverride?.ToList() - ?? BuildValidationChecks(platformDetector, commandExecutor, processService, includeConversation: false); + // Phase 2: Run structural checks (manifest + build) + var structuralChecks = requirementChecksOverride?.ToList() + ?? BuildStructuralChecks(platformDetector, commandExecutor); - var results = await RunChecksDetailedAsync(checks, config, logger, ct); + var results = await RunChecksDetailedAsync(structuralChecks, config, logger, ct); MapResultsToTiers(results, report); - // Phase 2b: Run conversation check only if boot tier passed + // Extract resolved uv command from build step for boot and conversation steps + var buildResultEntry = results + .FirstOrDefault(r => r.Check is ProjectBuildRequirementCheck); + var resolvedUvCommand = buildResultEntry.Result?.Metadata?.ResolvedUvCommand; + + // Phase 2b: Run boot check only if build passed + var buildPassed = report.Tiers.Build is { Skipped: true } or { Ok: true }; + if (buildPassed && requirementChecksOverride is null) + { + var bootChecks = BuildBootChecks(platformDetector, processService, resolvedUvCommand); + if (bootChecks.Count > 0) + { + var bootResults = await RunChecksDetailedAsync(bootChecks, config, logger, ct); + MapResultsToTiers(bootResults, report); + results.AddRange(bootResults); + } + } + else if (!buildPassed) + { + report.Tiers.Boot = new BootTierResult + { + Skipped = true, + Reason = "build failed" + }; + } + + // Phase 2c: Run conversation check only if boot tier passed var bootPassed = report.Tiers.Boot is { Skipped: false, Ok: true }; if (bootPassed && requirementChecksOverride is null) { - var conversationChecks = BuildConversationChecks(platformDetector, processService, launchPlayground); + var conversationChecks = BuildConversationChecks(platformDetector, processService, launchPlayground, resolvedUvCommand); if (conversationChecks.Count > 0) { var conversationResults = await RunChecksDetailedAsync(conversationChecks, config, logger, ct); MapResultsToTiers(conversationResults, report); results.AddRange(conversationResults); + + // Phase 2c: Run telemetry check using agent's console log file + var conversationResult = conversationResults + .FirstOrDefault(r => r.Check is ConversationRequirementCheck); + var agentLogPath = conversationResult.Result?.Metadata?.AgentConsoleLogPath; + report.AgentConsoleLogFile = agentLogPath; + var telemetryCheck = new TelemetryRequirementCheck(agentLogPath); + var telemetryResults = await RunChecksDetailedAsync( + new List { telemetryCheck }, config, logger, ct); + MapResultsToTiers(telemetryResults, report); + results.AddRange(telemetryResults); } } - else if (!bootPassed && report.Tiers.Boot is not { Skipped: true }) + else if (!bootPassed) { + var skipReason = report.Tiers.Boot is { Skipped: true } ? "build failed" : "boot tier failed"; report.Tiers.Conversation = new ConversationTierResult { Skipped = true, - Reason = "boot tier failed" + Reason = skipReason + }; + report.Tiers.Telemetry = new TelemetryTierResult + { + Skipped = true, + Reason = skipReason }; } @@ -296,8 +340,9 @@ private static void MapResultsToTiers( report.Tiers.Build = new BuildTierResult { Ok = result.Passed, - Log = result.Metadata?.Log, - ExitCode = result.Metadata?.ExitCode + ExitCode = result.Metadata?.ExitCode, + ErrorSummary = result.Passed ? null : result.ErrorMessage, + BuildLogFile = result.Metadata?.BuildLogFile }; } break; @@ -317,7 +362,8 @@ private static void MapResultsToTiers( { Ok = result.Passed, Port = result.Metadata?.Port, - BootMs = result.Metadata?.BootMs + BootMs = result.Metadata?.BootMs, + BootLogFile = result.Metadata?.BootLogFile }; } break; @@ -337,6 +383,7 @@ private static void MapResultsToTiers( { Ok = result.Passed, PlaygroundLaunched = result.Metadata?.PlaygroundLaunched, + ConversationLogFile = result.Metadata?.ConversationLogFile, Turns = result.Metadata?.Turns?.Select(t => new ConversationTurnResult { Input = t.Input, @@ -351,10 +398,103 @@ private static void MapResultsToTiers( }; } break; + + case TelemetryRequirementCheck: + if (result.IsWarning) + { + report.Tiers.Telemetry = new TelemetryTierResult + { + Ok = true, + Warning = result.ErrorMessage, + ExportDetected = false + }; + } + else + { + report.Tiers.Telemetry = new TelemetryTierResult + { + Ok = result.Passed, + ExportDetected = result.Passed, + MatchedPatterns = ParseMatchedPatterns(result.Details) + }; + + if (!result.Passed) + { + report.Tiers.Telemetry.Reason = result.ErrorMessage; + } + } + break; } } } + private static List? ParseMatchedPatterns(string? details) + { + if (string.IsNullOrWhiteSpace(details)) + return null; + + // Extract pattern names from details like "Telemetry export evidence found: pattern1, pattern2." + var colonIndex = details.IndexOf(':'); + if (colonIndex < 0) + return null; + + var patternsText = details[(colonIndex + 1)..].Trim().TrimEnd('.'); + var dotIndex = patternsText.IndexOf('.'); + if (dotIndex >= 0) + patternsText = patternsText[..dotIndex]; + + var patterns = patternsText.Split(',', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries); + return patterns.Length > 0 ? patterns.ToList() : null; + } + + private static string GetTelemetrySuggestion(TelemetryTierResult telemetry) + { + var patterns = telemetry.MatchedPatterns; + if (patterns is not null) + { + var lower = patterns.Select(p => p.ToLowerInvariant()).ToHashSet(); + if (lower.Contains("missing tenant") || lower.Contains("missing agent id")) + return "configure tenant ID and agent ID for telemetry export"; + if (lower.Contains("nothing exported") || lower.Contains("spans skipped") || lower.Contains("spans filtered out")) + return "check agent identity configuration for telemetry export"; + if (lower.Contains("connection refused") || lower.Contains("unavailable") || lower.Contains("deadline_exceeded")) + return "check OTLP endpoint connectivity"; + if (lower.Contains("unauthenticated") || lower.Contains("permissiondenied")) + return "check OTLP endpoint credentials"; + } + + return "check agent console logs for telemetry export errors"; + } + + private static (string Description, string Suggestion) GetCodeHealthFailureInfo(ValidationTiers tiers) + { + var buildFailed = tiers.Build is { Skipped: false, Ok: false }; + var structuralFailed = tiers.Structural is { Skipped: false, Ok: false }; + + if (buildFailed) + { + var buildLogFile = tiers.Build is BuildTierResult bt ? bt.BuildLogFile : null; + var suggestion = buildLogFile is not null + ? $"fix build errors, see: {buildLogFile}" + : "fix build errors and re-run `a365 validate`"; + return ("build failed", suggestion); + } + + if (structuralFailed) + { + var failedChecks = tiers.Structural is StructuralTierResult st + ? st.Checks?.Where(c => !c.Ok).Select(c => c.Name).ToList() + : null; + + var desc = failedChecks is { Count: > 0 } + ? $"failed: {string.Join(", ", failedChecks)}" + : "structural checks failed"; + return (desc, "fix project structure issues and re-run `a365 validate`"); + } + + return ("code health check failed", "fix errors and re-run `a365 validate`"); + } + private static string? FindBlocker(ValidationTiers tiers) { if (tiers.Structural is { Skipped: false, Ok: false }) return "structural"; @@ -378,6 +518,7 @@ internal static void PrintSummary(ValidateReport report, ILogger logger) int passCount = 0; int failCount = 0; + int warnCount = 0; int localChecks = 0; foreach (var row in rows) @@ -387,6 +528,17 @@ internal static void PrintSummary(ValidateReport report, ILogger logger) var reason = row.Reason ?? "not configured"; logger.LogInformation(" {Skip} {Name,-20} skipped ({Reason})", SkipMark, row.Label, reason); } + else if (row.IsWarning) + { + warnCount++; + localChecks++; + logger.LogInformation(" {Warn} {Name,-20} {Description}", WarnMark, row.Label, row.Description); + + if (row.Suggestion is not null) + { + logger.LogInformation(" -> suggestion: {Suggestion}", row.Suggestion); + } + } else if (row.Ok) { passCount++; @@ -410,7 +562,8 @@ internal static void PrintSummary(ValidateReport report, ILogger logger) if (failCount == 0 && localChecks > 0) { - logger.LogInformation(" All {PassCount} checks passed.", passCount); + var warnSuffix = warnCount > 0 ? $" ({warnCount} warning(s))" : ""; + logger.LogInformation(" All {PassCount} checks passed.{WarnSuffix}", passCount, warnSuffix); } else if (failCount > 0) { @@ -433,12 +586,26 @@ private static List BuildDisplayRows(ValidateReport report) if (codeHealthActive.Count > 0) { var allOk = codeHealthActive.All(t => t.Ok == true); + string description; + string? suggestion = null; + + if (allOk) + { + description = "project structure, manifest, build"; + } + else + { + var (desc, sug) = GetCodeHealthFailureInfo(tiers); + description = desc; + suggestion = sug; + } + rows.Add(new DisplayRow { Label = "Code health", Ok = allOk, - Description = allOk ? "project structure, manifest, build" : "code health check failed", - Suggestion = allOk ? null : "fix build errors and re-run `a365 validate`" + Description = description, + Suggestion = suggestion }); } else @@ -499,10 +666,51 @@ private static List BuildDisplayRows(ValidateReport report) }); } - // Remaining individual tiers - rows.Add(CreateTierRow("Telemetry", tiers.Telemetry, - "tracing and observability", - "re-run \"instrument-observability\" skill")); + // Row 4: Telemetry + var telemetry = tiers.Telemetry; + if (!telemetry.Skipped) + { + var telOk = telemetry.Ok == true; + string telDesc; + string? telSuggestion = null; + + if (telOk && telemetry.Warning is not null) + { + // Warning state: SDK not detected + telDesc = telemetry.Warning; + telSuggestion = "configure OpenTelemetry to export traces to Agent365"; + } + else if (telOk) + { + telDesc = telemetry.ExportDetected == true + ? "trace export to Agent365 detected" + : "tracing and observability"; + } + else + { + telDesc = "trace export failures detected"; + telSuggestion = GetTelemetrySuggestion(telemetry); + } + + rows.Add(new DisplayRow + { + Label = "Telemetry", + Ok = telOk && telemetry.Warning is null, + Description = telDesc, + Suggestion = telSuggestion, + IsWarning = telemetry.Warning is not null + }); + } + else + { + rows.Add(new DisplayRow + { + Label = "Telemetry", + Skipped = true, + Reason = telemetry.Reason ?? "not yet run" + }); + } + rows.Add(CreateTierRow("Registered", tiers.Blueprint, "blueprint registration", null)); @@ -538,6 +746,7 @@ private sealed class DisplayRow public bool Skipped { get; init; } public string? Reason { get; init; } public bool Ok { get; init; } + public bool IsWarning { get; init; } public string? Description { get; init; } public string? Suggestion { get; init; } } @@ -564,11 +773,9 @@ private static string ResolveProjectPath(Agent365Config config) : Path.GetFullPath(config.DeploymentProjectPath); } - private static List BuildValidationChecks( + private static List BuildStructuralChecks( PlatformDetector? platformDetector, - CommandExecutor? commandExecutor, - IProcessService? processService, - bool includeConversation = false) + CommandExecutor? commandExecutor) { var checks = new List { @@ -580,14 +787,20 @@ private static List BuildValidationChecks( checks.Add(new ProjectBuildRequirementCheck(platformDetector, commandExecutor)); } + return checks; + } + + private static List BuildBootChecks( + PlatformDetector? platformDetector, + IProcessService? processService, + string? resolvedUvCommand = null) + { + var checks = new List(); + if (platformDetector is not null && processService is not null) { - checks.Add(new LocalRuntimeRequirementCheck(platformDetector, processService)); - - if (includeConversation) - { - checks.Add(new ConversationRequirementCheck(platformDetector, processService)); - } + checks.Add(new LocalRuntimeRequirementCheck(platformDetector, processService, + resolvedUvCommand: resolvedUvCommand)); } return checks; @@ -596,14 +809,16 @@ private static List BuildValidationChecks( private static List BuildConversationChecks( PlatformDetector? platformDetector, IProcessService? processService, - bool launchPlayground = false) + bool launchPlayground = false, + string? resolvedUvCommand = null) { var checks = new List(); if (platformDetector is not null && processService is not null) { checks.Add(new ConversationRequirementCheck( - platformDetector, processService, launchPlayground: launchPlayground)); + platformDetector, processService, launchPlayground: launchPlayground, + resolvedUvCommand: resolvedUvCommand)); } return checks; diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Services/CommandExecutor.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Services/CommandExecutor.cs index 4142964a..6aac4c23 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Services/CommandExecutor.cs +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Services/CommandExecutor.cs @@ -78,7 +78,20 @@ public virtual async Task ExecuteAsync( }; } - process.Start(); + try + { + process.Start(); + } + catch (System.ComponentModel.Win32Exception ex) + { + _logger.LogDebug(ex, "Process '{Command}' not found", command); + return new CommandResult + { + ExitCode = -1, + StandardOutput = string.Empty, + StandardError = $"'{command}' not found: {ex.Message}" + }; + } if (captureOutput) { @@ -202,7 +215,21 @@ public virtual async Task ExecuteWithStreamingAsync( } }; - process.Start(); + try + { + process.Start(); + } + catch (System.ComponentModel.Win32Exception ex) + { + _logger.LogDebug(ex, "Process '{Command}' not found", command); + return new CommandResult + { + ExitCode = -1, + StandardOutput = string.Empty, + StandardError = $"'{command}' not found: {ex.Message}" + }; + } + process.BeginOutputReadLine(); process.BeginErrorReadLine(); diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementCheckResult.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementCheckResult.cs index dc2217ee..dafb764d 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementCheckResult.cs +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementCheckResult.cs @@ -106,6 +106,34 @@ public sealed class RequirementCheckMetadata /// Whether AgentsPlayground was launched for interactive testing. public bool? PlaygroundLaunched { get; init; } + + /// + /// Path to the agent's captured console output log file. + /// Written during the conversation step; used by telemetry check and referenced in the report. + /// + public string? AgentConsoleLogPath { get; init; } + + /// + /// Path to the MSBuild file log written during project build validation. + /// + public string? BuildLogFile { get; init; } + + /// + /// Path to the boot log file written during local runtime validation. + /// + public string? BootLogFile { get; init; } + + /// + /// Path to the conversation log file written during conversation validation. + /// Contains HTTP request/response details for each turn. + /// + public string? ConversationLogFile { get; init; } + + /// + /// Resolved path to the uv command, set during build dependency install. + /// Used by the boot step to run Python agents in uv-managed projects. + /// + public string? ResolvedUvCommand { get; init; } } /// diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/ConversationRequirementCheck.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/ConversationRequirementCheck.cs index 5efe82cb..84cc0874 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/ConversationRequirementCheck.cs +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/ConversationRequirementCheck.cs @@ -23,6 +23,7 @@ public class ConversationRequirementCheck : RequirementCheck private readonly HttpClient _httpClient; private readonly IBotCallbackReceiver? _callbackReceiver; private readonly bool _launchPlayground; + private readonly string? _resolvedUvCommand; /// /// Maximum time to wait for the app to start and respond on the health endpoint. @@ -44,6 +45,22 @@ public class ConversationRequirementCheck : RequirementCheck /// internal static readonly TimeSpan PollInterval = TimeSpan.FromMilliseconds(500); + /// + /// Delay after health endpoint is ready before sending messages. + /// Agents often need additional time to initialize their message pipeline after the HTTP server is up. + /// + internal static readonly TimeSpan PostHealthWarmupDelay = TimeSpan.FromSeconds(2); + + /// + /// Maximum number of retries for a conversation turn that fails with a transient error. + /// + internal const int MaxTurnRetries = 2; + + /// + /// Delay between retry attempts for a failed conversation turn. + /// + internal static readonly TimeSpan TurnRetryDelay = TimeSpan.FromSeconds(2); + /// /// Maximum number of stdout/stderr lines to capture for diagnostics. /// @@ -69,13 +86,15 @@ public ConversationRequirementCheck( IProcessService processService, HttpClient? httpClient = null, IBotCallbackReceiver? callbackReceiver = null, - bool launchPlayground = false) + bool launchPlayground = false, + string? resolvedUvCommand = null) { _platformDetector = platformDetector ?? throw new ArgumentNullException(nameof(platformDetector)); _processService = processService ?? throw new ArgumentNullException(nameof(processService)); _httpClient = httpClient ?? new HttpClient(); _callbackReceiver = callbackReceiver; _launchPlayground = launchPlayground; + _resolvedUvCommand = resolvedUvCommand; } /// @@ -143,12 +162,34 @@ private async Task SpawnAndConverse( { var outputLines = new LocalRuntimeRequirementCheck.BoundedLineBuffer(MaxOutputLines); var errorLines = new LocalRuntimeRequirementCheck.BoundedLineBuffer(MaxOutputLines); + string? agentConsoleLogPath = null; + string? conversationLogPath = null; + StreamWriter? consoleLogWriter = null; + StreamWriter? conversationLogWriter = null; Process? process = null; IBotCallbackReceiver? receiver = _callbackReceiver; bool ownedReceiver = false; try { + // Create conversation log file for diagnostics + try + { + conversationLogPath = ConfigService.GetCommandLogPath("validate.conversation"); + conversationLogWriter = new StreamWriter(conversationLogPath, append: false, encoding: System.Text.Encoding.UTF8) + { + AutoFlush = true + }; + conversationLogWriter.WriteLine($"Conversation validation started at {DateTimeOffset.Now:O}"); + conversationLogWriter.WriteLine($"Platform: {platform}, Port: {port}"); + conversationLogWriter.WriteLine($"Command: {startInfo.FileName} {startInfo.Arguments}"); + conversationLogWriter.WriteLine(new string('-', 60)); + } + catch + { + conversationLogPath = null; + } + // Start callback receiver for agent response tracking if (receiver is null) { @@ -169,30 +210,79 @@ private async Task SpawnAndConverse( process = _processService.Start(startInfo); if (process is null) { - return RequirementCheckResult.Failure( - $"Failed to start {platform} process", - GetRunGuidance(platform)); + conversationLogWriter?.WriteLine("FAILED: Could not start process"); + return new RequirementCheckResult + { + Passed = false, + ErrorMessage = $"Failed to start {platform} process", + ResolutionGuidance = GetRunGuidance(platform), + Metadata = new RequirementCheckMetadata + { + Platform = platform.ToString(), + ConversationLogFile = conversationLogPath + } + }; + } + + // Create agent console log file for telemetry analysis + try + { + agentConsoleLogPath = ConfigService.GetCommandLogPath("validate.agent-console"); + consoleLogWriter = new StreamWriter(agentConsoleLogPath, append: false, encoding: System.Text.Encoding.UTF8) + { + AutoFlush = true + }; + logger.LogDebug("Writing agent console output to {LogPath}", agentConsoleLogPath); + } + catch (Exception ex) + { + logger.LogDebug(ex, "Could not create agent console log file, telemetry analysis will use in-memory buffer"); + agentConsoleLogPath = null; } process.OutputDataReceived += (_, args) => { - if (args.Data is not null) outputLines.Add(args.Data); + if (args.Data is not null) + { + outputLines.Add(args.Data); + try { consoleLogWriter?.WriteLine(args.Data); } catch { /* best-effort */ } + } }; process.ErrorDataReceived += (_, args) => { - if (args.Data is not null) errorLines.Add(args.Data); + if (args.Data is not null) + { + errorLines.Add(args.Data); + try { consoleLogWriter?.WriteLine(args.Data); } catch { /* best-effort */ } + } }; process.BeginOutputReadLine(); process.BeginErrorReadLine(); // Phase 1: Wait for health endpoint + conversationLogWriter?.WriteLine("\n[Phase 1] Waiting for health endpoint..."); var bootResult = await WaitForHealthAsync(process, healthUrl, platform, logger, cancellationToken); if (bootResult is not null) { + conversationLogWriter?.WriteLine($"FAILED: {bootResult.ErrorMessage}"); + WriteAgentOutputToConversationLog(conversationLogWriter, outputLines, errorLines); + bootResult.Metadata = new RequirementCheckMetadata + { + Platform = platform.ToString(), + ConversationLogFile = conversationLogPath, + AgentConsoleLogPath = agentConsoleLogPath + }; return bootResult; } + conversationLogWriter?.WriteLine("Health endpoint ready."); + + // Allow the agent message pipeline to finish initializing after the HTTP server is up. + conversationLogWriter?.WriteLine($"Waiting {(int)PostHealthWarmupDelay.TotalSeconds}s for message pipeline warmup..."); + await Task.Delay(PostHealthWarmupDelay, cancellationToken); + // Phase 2: Multi-turn conversation + conversationLogWriter?.WriteLine($"\n[Phase 2] Starting {ConversationPrompts.Length}-turn conversation..."); var turns = new List(); var allOk = true; @@ -201,13 +291,25 @@ private async Task SpawnAndConverse( if (process.HasExited) { var exitOutput = GetCapturedOutput(outputLines, errorLines); - return RequirementCheckResult.Failure( - $"Agent process exited during conversation (turn {i + 1}):\n{exitOutput}", - GetRunGuidance(platform)); + conversationLogWriter?.WriteLine($"\nFAILED: Agent process exited during turn {i + 1}"); + WriteAgentOutputToConversationLog(conversationLogWriter, outputLines, errorLines); + return new RequirementCheckResult + { + Passed = false, + ErrorMessage = $"Agent process exited during conversation (turn {i + 1}):\n{exitOutput}", + ResolutionGuidance = GetRunGuidance(platform), + Metadata = new RequirementCheckMetadata + { + Platform = platform.ToString(), + ConversationLogFile = conversationLogPath, + AgentConsoleLogPath = agentConsoleLogPath + } + }; } - var turnResult = await SendTurnAsync(messagesUrl, conversationId, ConversationPrompts[i], i, port, receiver, logger, cancellationToken); + var turnResult = await SendTurnWithRetryAsync(messagesUrl, conversationId, ConversationPrompts[i], i, port, receiver, logger, conversationLogWriter, cancellationToken); turns.Add(turnResult); + LogTurn(conversationLogWriter, i + 1, turnResult); if (!turnResult.Ok) { @@ -231,6 +333,10 @@ private async Task SpawnAndConverse( playgroundLaunched = await LaunchPlaygroundAsync(messagesUrl, logger, cancellationToken); } + conversationLogWriter?.WriteLine($"\n[Summary] {turnSummary}"); + conversationLogWriter?.WriteLine($"Result: {(allOk ? "PASSED" : "FAILED")}"); + WriteAgentOutputToConversationLog(conversationLogWriter, outputLines, errorLines); + return new RequirementCheckResult { Passed = allOk, @@ -242,6 +348,8 @@ private async Task SpawnAndConverse( Port = port, Platform = platform.ToString(), PlaygroundLaunched = playgroundLaunched ? true : null, + AgentConsoleLogPath = agentConsoleLogPath, + ConversationLogFile = conversationLogPath, Turns = turns.Select(t => new ConversationTurnMetadata { Input = t.Input, @@ -258,6 +366,9 @@ private async Task SpawnAndConverse( } finally { + conversationLogWriter?.Dispose(); + consoleLogWriter?.Dispose(); + if (process is not null) { try @@ -343,6 +454,71 @@ private async Task SpawnAndConverse( GetRunGuidance(platform)); } + /// + /// Sends a conversation turn with retry logic for transient failures. + /// Agents may not have their message pipeline fully ready even after the health endpoint responds. + /// + private async Task SendTurnWithRetryAsync( + string messagesUrl, + string conversationId, + string text, + int turnIndex, + int port, + IBotCallbackReceiver? callbackReceiver, + ILogger logger, + StreamWriter? conversationLogWriter, + CancellationToken cancellationToken) + { + ConversationTurnData? lastResult = null; + + for (int attempt = 0; attempt <= MaxTurnRetries; attempt++) + { + lastResult = await SendTurnAsync(messagesUrl, conversationId, text, turnIndex, port, callbackReceiver, logger, cancellationToken); + + if (lastResult.Ok || !IsTransientFailure(lastResult)) + { + return lastResult; + } + + if (attempt < MaxTurnRetries) + { + logger.LogDebug( + "Turn {Turn} failed with transient error, retrying in {Delay}s (attempt {Attempt}/{Max}): {Error}", + turnIndex + 1, (int)TurnRetryDelay.TotalSeconds, attempt + 1, MaxTurnRetries, lastResult.Error); + conversationLogWriter?.WriteLine( + $" [Retry] Turn {turnIndex + 1} failed ({lastResult.Error}), retrying in {(int)TurnRetryDelay.TotalSeconds}s..."); + await Task.Delay(TurnRetryDelay, cancellationToken); + } + } + + return lastResult!; + } + + /// + /// Determines if a turn failure is transient and worth retrying. + /// Connection failures and server errors (5xx) are transient. + /// Auth failures (401/403) and client errors (4xx) are not. + /// + private static bool IsTransientFailure(ConversationTurnData turnResult) + { + if (turnResult.Error?.StartsWith("Connection failed", StringComparison.Ordinal) == true) + { + return true; + } + + if (turnResult.Error?.StartsWith("Turn timed out", StringComparison.Ordinal) == true) + { + return true; + } + + if (turnResult.StatusCode is >= 500 and < 600) + { + return true; + } + + return false; + } + private async Task SendTurnAsync( string messagesUrl, string conversationId, @@ -491,7 +667,7 @@ private async Task SendTurnAsync( } } - private static ProcessStartInfo BuildProcessStartInfo(ProjectPlatform platform, string projectPath, int port) + private ProcessStartInfo BuildProcessStartInfo(ProjectPlatform platform, string projectPath, int port) { var startInfo = new ProcessStartInfo { @@ -502,8 +678,17 @@ private static ProcessStartInfo BuildProcessStartInfo(ProjectPlatform platform, CreateNoWindow = true }; - // Disable auth so the bot accepts unauthenticated local requests + // Disable auth so the bot accepts unauthenticated local requests. + // BYPASS_AUTH: used by .NET Agent SDK startInfo.EnvironmentVariables["BYPASS_AUTH"] = "true"; + // Clear credentials that trigger JWT middleware in Python/Node SDKs. + // When these are absent, agents run in anonymous mode. + startInfo.EnvironmentVariables["CLIENT_ID"] = ""; + startInfo.EnvironmentVariables["CLIENT_SECRET"] = ""; + startInfo.EnvironmentVariables["TENANT_ID"] = ""; + // Also clear .NET Bot Framework equivalents + startInfo.EnvironmentVariables["MicrosoftAppId"] = ""; + startInfo.EnvironmentVariables["MicrosoftAppPassword"] = ""; switch (platform) { @@ -514,14 +699,23 @@ private static ProcessStartInfo BuildProcessStartInfo(ProjectPlatform platform, break; case ProjectPlatform.NodeJs: - startInfo.FileName = "npm"; - startInfo.Arguments = "start"; + LocalRuntimeRequirementCheck.WrapForWindows(startInfo, "npm", "start"); startInfo.EnvironmentVariables["PORT"] = port.ToString(); break; case ProjectPlatform.Python: - startInfo.FileName = "python"; - startInfo.Arguments = "app.py"; + var entryPoint = LocalRuntimeRequirementCheck.ResolvePythonEntryPoint(projectPath); + var usesUv = ProjectBuildRequirementCheck.DetectPythonInstallCommand(projectPath) is ("uv", _); + if (usesUv) + { + startInfo.FileName = _resolvedUvCommand ?? "uv"; + startInfo.Arguments = $"run python {entryPoint}"; + } + else + { + startInfo.FileName = "python"; + startInfo.Arguments = entryPoint; + } startInfo.EnvironmentVariables["PORT"] = port.ToString(); break; @@ -543,7 +737,7 @@ private static string GetRunGuidance(ProjectPlatform platform) " npm start\n" + "Verify it starts and responds on /api/messages.", ProjectPlatform.Python => "Try running the app manually:\n" + - " python app.py\n" + + " python .py (or: uv run python .py)\n" + "Verify it starts and responds on /api/messages.", _ => "Try running the app manually and verify it responds on /api/messages." }; @@ -672,6 +866,78 @@ private static string GetCapturedOutput( return sb.ToString().TrimEnd(); } + private static void LogTurn(StreamWriter? writer, int turnNumber, ConversationTurnData turn) + { + if (writer is null) return; + + try + { + writer.WriteLine($"\n Turn {turnNumber}: \"{turn.Input}\""); + writer.WriteLine($" Status: HTTP {turn.StatusCode?.ToString() ?? "N/A"}, Latency: {turn.LatencyMs}ms, Ok: {turn.Ok}"); + + if (turn.ResponseSnippet is not null) + { + writer.WriteLine($" Response: {turn.ResponseSnippet}"); + } + + if (turn.AgentResponded is not null) + { + writer.WriteLine($" Agent responded: {turn.AgentResponded}"); + } + + if (turn.AgentResponseText is not null) + { + writer.WriteLine($" Agent response: {turn.AgentResponseText}"); + } + + if (turn.Error is not null) + { + writer.WriteLine($" Error: {turn.Error}"); + } + } + catch + { + // best-effort + } + } + + private static void WriteAgentOutputToConversationLog( + StreamWriter? writer, + LocalRuntimeRequirementCheck.BoundedLineBuffer outputLines, + LocalRuntimeRequirementCheck.BoundedLineBuffer errorLines) + { + if (writer is null) return; + + try + { + var stdout = outputLines.GetLines(); + var stderr = errorLines.GetLines(); + + if (stdout.Length > 0 || stderr.Length > 0) + { + writer.WriteLine($"\n{new string('-', 60)}"); + writer.WriteLine("[Agent console output]"); + } + + if (stdout.Length > 0) + { + foreach (var line in stdout) + writer.WriteLine(line); + } + + if (stderr.Length > 0) + { + writer.WriteLine("[stderr]"); + foreach (var line in stderr) + writer.WriteLine(line); + } + } + catch + { + // best-effort + } + } + private static string TruncateResponse(string response, int maxLength) { if (string.IsNullOrEmpty(response)) return string.Empty; diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/LocalRuntimeRequirementCheck.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/LocalRuntimeRequirementCheck.cs index 928a5909..18eda3cd 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/LocalRuntimeRequirementCheck.cs +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/LocalRuntimeRequirementCheck.cs @@ -3,6 +3,7 @@ using System.Diagnostics; using System.Net.Http; +using System.Runtime.InteropServices; using System.Text; using Microsoft.Agents.A365.DevTools.Cli.Models; using Microsoft.Extensions.Logging; @@ -18,6 +19,7 @@ public class LocalRuntimeRequirementCheck : RequirementCheck private readonly PlatformDetector _platformDetector; private readonly IProcessService _processService; private readonly HttpClient _httpClient; + private readonly string? _resolvedUvCommand; /// /// Default port used when no port can be inferred from configuration. @@ -47,11 +49,13 @@ public class LocalRuntimeRequirementCheck : RequirementCheck public LocalRuntimeRequirementCheck( PlatformDetector platformDetector, IProcessService processService, - HttpClient? httpClient = null) + HttpClient? httpClient = null, + string? resolvedUvCommand = null) { _platformDetector = platformDetector ?? throw new ArgumentNullException(nameof(platformDetector)); _processService = processService ?? throw new ArgumentNullException(nameof(processService)); _httpClient = httpClient ?? new HttpClient(); + _resolvedUvCommand = resolvedUvCommand; } /// @@ -130,7 +134,7 @@ internal static int ResolvePort(string? messagingEndpoint) return DefaultPort; } - private static ProcessStartInfo BuildProcessStartInfo(ProjectPlatform platform, string projectPath, int port) + private ProcessStartInfo BuildProcessStartInfo(ProjectPlatform platform, string projectPath, int port) { var startInfo = new ProcessStartInfo { @@ -150,14 +154,23 @@ private static ProcessStartInfo BuildProcessStartInfo(ProjectPlatform platform, break; case ProjectPlatform.NodeJs: - startInfo.FileName = "npm"; - startInfo.Arguments = "start"; + WrapForWindows(startInfo, "npm", "start"); startInfo.EnvironmentVariables["PORT"] = port.ToString(); break; case ProjectPlatform.Python: - startInfo.FileName = "python"; - startInfo.Arguments = "app.py"; + var entryPoint = ResolvePythonEntryPoint(projectPath); + var usesUv = ProjectBuildRequirementCheck.DetectPythonInstallCommand(projectPath) is ("uv", _); + if (usesUv) + { + startInfo.FileName = _resolvedUvCommand ?? "uv"; + startInfo.Arguments = $"run python {entryPoint}"; + } + else + { + startInfo.FileName = "python"; + startInfo.Arguments = entryPoint; + } startInfo.EnvironmentVariables["PORT"] = port.ToString(); break; @@ -168,6 +181,30 @@ private static ProcessStartInfo BuildProcessStartInfo(ProjectPlatform platform, return startInfo; } + /// + /// On Windows, wraps batch-file commands (npm, npx, node) with cmd.exe /c + /// so they can be started with UseShellExecute=false. + /// + internal static void WrapForWindows(ProcessStartInfo startInfo, string command, string arguments) + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && IsBatchCommand(command)) + { + startInfo.FileName = "cmd.exe"; + startInfo.Arguments = $"/c {command} {arguments}"; + } + else + { + startInfo.FileName = command; + startInfo.Arguments = arguments; + } + } + + private static bool IsBatchCommand(string command) + { + var name = Path.GetFileNameWithoutExtension(command).ToLowerInvariant(); + return name is "npm" or "npx" or "node"; + } + private async Task SpawnAndProbeAsync( ProcessStartInfo startInfo, string healthUrl, @@ -180,6 +217,7 @@ private async Task SpawnAndProbeAsync( var errorLines = new BoundedLineBuffer(MaxOutputLines); Process? process = null; var stopwatch = System.Diagnostics.Stopwatch.StartNew(); + var bootLogFile = ConfigService.GetCommandLogPath("validate.boot"); try { @@ -210,9 +248,19 @@ private async Task SpawnAndProbeAsync( if (process.HasExited) { var exitOutput = GetCapturedOutput(outputLines, errorLines); - return RequirementCheckResult.Failure( - $"App exited early with code {process.ExitCode} before health endpoint responded:\n{exitOutput}", - GetRunGuidance(platform)); + WriteBootLog(bootLogFile, outputLines, errorLines); + return new RequirementCheckResult + { + Passed = false, + ErrorMessage = $"App exited early with code {process.ExitCode} before health endpoint responded:\n{exitOutput}", + ResolutionGuidance = GetRunGuidance(platform), + Metadata = new RequirementCheckMetadata + { + Platform = platform.ToString(), + ExitCode = process.ExitCode, + BootLogFile = bootLogFile + } + }; } try @@ -260,9 +308,18 @@ private async Task SpawnAndProbeAsync( cancellationToken.ThrowIfCancellationRequested(); var timeoutOutput = GetCapturedOutput(outputLines, errorLines); - return RequirementCheckResult.Failure( - $"App did not respond on {healthUrl} within {(int)StartupTimeout.TotalSeconds} seconds:\n{timeoutOutput}", - GetRunGuidance(platform)); + WriteBootLog(bootLogFile, outputLines, errorLines); + return new RequirementCheckResult + { + Passed = false, + ErrorMessage = $"App did not respond on {healthUrl} within {(int)StartupTimeout.TotalSeconds} seconds:\n{timeoutOutput}", + ResolutionGuidance = GetRunGuidance(platform), + Metadata = new RequirementCheckMetadata + { + Platform = platform.ToString(), + BootLogFile = bootLogFile + } + }; } finally { @@ -285,6 +342,48 @@ private async Task SpawnAndProbeAsync( } } + /// + /// Writes captured stdout/stderr to a boot log file for post-mortem inspection. + /// + private static void WriteBootLog(string logPath, BoundedLineBuffer outputLines, BoundedLineBuffer errorLines) + { + try + { + var dir = Path.GetDirectoryName(logPath); + if (dir is not null) + { + Directory.CreateDirectory(dir); + } + + using var writer = new StreamWriter(logPath, append: false); + var stdout = outputLines.GetLines(); + var stderr = errorLines.GetLines(); + + if (stdout.Length > 0) + { + writer.WriteLine("[stdout]"); + foreach (var line in stdout) + writer.WriteLine(line); + } + + if (stderr.Length > 0) + { + writer.WriteLine("[stderr]"); + foreach (var line in stderr) + writer.WriteLine(line); + } + + if (stdout.Length == 0 && stderr.Length == 0) + { + writer.WriteLine("(no output captured)"); + } + } + catch + { + // Best-effort: don't fail the check if log writing fails. + } + } + private static string GetCapturedOutput(BoundedLineBuffer outputLines, BoundedLineBuffer errorLines) { var sb = new StringBuilder(); @@ -313,6 +412,149 @@ private static string GetCapturedOutput(BoundedLineBuffer outputLines, BoundedLi return sb.ToString().TrimEnd(); } + /// + /// Resolves the Python entry point by inspecting the project: + /// 1. Procfile (web: python <file>) — explicit, highest priority + /// 2. Scan top-level .py files for if __name__ == "__main__" guard + /// 3. Among matches, prefer well-known names (app.py, main.py) + /// 4. Falls back to app.py if nothing is found. + /// + internal static string ResolvePythonEntryPoint(string projectPath) + { + // Check Procfile for explicit command + var procfilePath = Path.Combine(projectPath, "Procfile"); + if (File.Exists(procfilePath)) + { + var entryFromProcfile = ParseProcfileEntryPoint(procfilePath); + if (entryFromProcfile is not null) + { + return entryFromProcfile; + } + } + + // Scan top-level .py files for entry point guard + var pyFiles = Directory.GetFiles(projectPath, "*.py", SearchOption.TopDirectoryOnly); + var filesWithMain = new List(); + + foreach (var pyFile in pyFiles) + { + if (HasMainGuard(pyFile)) + { + filesWithMain.Add(Path.GetFileName(pyFile)); + } + } + + if (filesWithMain.Count == 1) + { + return filesWithMain[0]; + } + + if (filesWithMain.Count > 1) + { + // Prefer well-known entry point names among matches + string[] preferred = ["app.py", "main.py", "__main__.py", "bot.py", "server.py"]; + foreach (var name in preferred) + { + if (filesWithMain.Contains(name, StringComparer.OrdinalIgnoreCase)) + { + return name; + } + } + + // Return the first match alphabetically + filesWithMain.Sort(StringComparer.OrdinalIgnoreCase); + return filesWithMain[0]; + } + + // No __main__ guard found — check if well-known files exist at all + string[] fallbackCandidates = ["app.py", "main.py", "__main__.py", "bot.py", "server.py"]; + foreach (var candidate in fallbackCandidates) + { + if (File.Exists(Path.Combine(projectPath, candidate))) + { + return candidate; + } + } + + // Default fallback + return "app.py"; + } + + /// + /// Checks whether a Python file contains an if __name__ == "__main__" guard, + /// indicating it is designed to be run directly. + /// + internal static bool HasMainGuard(string filePath) + { + try + { + foreach (var line in File.ReadLines(filePath)) + { + var trimmed = line.TrimStart(); + // Match: if __name__ == "__main__" or if __name__ == '__main__' + if (trimmed.StartsWith("if", StringComparison.Ordinal) && + trimmed.Contains("__name__") && + trimmed.Contains("__main__")) + { + return true; + } + } + } + catch + { + // Best-effort: unreadable files are skipped + } + + return false; + } + + /// + /// Parses a Procfile for the web process entry point. + /// Expects format: web: python <file.py> [args...] + /// + internal static string? ParseProcfileEntryPoint(string procfilePath) + { + try + { + var lines = File.ReadAllLines(procfilePath); + foreach (var line in lines) + { + var trimmed = line.Trim(); + if (!trimmed.StartsWith("web:", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + // Extract the command after "web:" + var command = trimmed["web:".Length..].Trim(); + + // Match patterns like "python app.py", "python -m module", "python3 main.py" + var parts = command.Split(' ', StringSplitOptions.RemoveEmptyEntries); + if (parts.Length < 2) + { + continue; + } + + // Skip the python executable (python, python3, etc.) + if (parts[0].StartsWith("python", StringComparison.OrdinalIgnoreCase)) + { + // Return everything after the python command as arguments + return string.Join(' ', parts[1..]); + } + + // If it's gunicorn/uvicorn, return the full command using -m approach + // e.g., "gunicorn app:app" -> we can't directly use this with "python" + // So skip and fall through to file detection + } + } + catch + { + // Best-effort: fall through to file detection + } + + return null; + } + private static string GetRunGuidance(ProjectPlatform platform) { return platform switch diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/ProjectBuildRequirementCheck.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/ProjectBuildRequirementCheck.cs index 3ec973d3..909b2fbf 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/ProjectBuildRequirementCheck.cs +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/ProjectBuildRequirementCheck.cs @@ -15,6 +15,9 @@ public class ProjectBuildRequirementCheck : RequirementCheck private readonly PlatformDetector _platformDetector; private readonly CommandExecutor _commandExecutor; + // Resolved uv command path — set during dependency install, used by build command. + private string? _resolvedUvCommand; + public ProjectBuildRequirementCheck(PlatformDetector platformDetector, CommandExecutor commandExecutor) { _platformDetector = platformDetector ?? throw new ArgumentNullException(nameof(platformDetector)); @@ -62,7 +65,33 @@ private async Task CheckImplementationAsync( details: $"No .NET, Node.js, or Python project detected in {projectPath}"); } - var (command, arguments) = GetBuildCommand(platform); + var buildLogFile = GetBuildLogPath(platform); + + // Install dependencies before building (Python: uv/pip, Node.js: npm install) + var (depFailure, depOutput) = await InstallDependenciesAsync(platform, projectPath, logger, cancellationToken); + + // Write dependency install output to the build log (even on success) + if (depOutput is not null) + { + WriteBuildLog(buildLogFile, depOutput); + } + + if (depFailure is not null) + { + if (depOutput is null) + { + WriteBuildLog(buildLogFile, null, depFailure.ErrorMessage); + } + + depFailure.Metadata = new RequirementCheckMetadata + { + Platform = platform.ToString(), + BuildLogFile = buildLogFile + }; + return depFailure; + } + + var (command, arguments) = GetBuildCommand(platform, buildLogFile, projectPath); logger.LogDebug("Running build check: {Command} {Arguments} in {Path}", command, arguments, projectPath); @@ -74,6 +103,13 @@ private async Task CheckImplementationAsync( suppressErrorLogging: true, cancellationToken: cancellationToken); + // For non-.NET platforms, append build output to the log file. + // .NET uses MSBuild's built-in file logger (-fl) so this is already handled. + if (platform != ProjectPlatform.DotNet) + { + WriteBuildLog(buildLogFile, result, append: depOutput is not null); + } + if (result.Success) { return new RequirementCheckResult @@ -84,7 +120,8 @@ private async Task CheckImplementationAsync( { Platform = platform.ToString(), ExitCode = result.ExitCode, - Log = TruncateLog(result.StandardOutput) + BuildLogFile = buildLogFile, + ResolvedUvCommand = _resolvedUvCommand } }; } @@ -100,22 +137,42 @@ private async Task CheckImplementationAsync( { Platform = platform.ToString(), ExitCode = result.ExitCode, - Log = TruncateLog(!string.IsNullOrWhiteSpace(result.StandardError) ? result.StandardError : result.StandardOutput) + BuildLogFile = buildLogFile, + ResolvedUvCommand = _resolvedUvCommand } }; } - private static (string Command, string Arguments) GetBuildCommand(ProjectPlatform platform) + private (string Command, string Arguments) GetBuildCommand(ProjectPlatform platform, string buildLogFile, string projectPath) { return platform switch { - ProjectPlatform.DotNet => ("dotnet", "build --no-restore /p:TreatWarningsAsErrors=true"), + ProjectPlatform.DotNet => ("dotnet", + $"build --no-restore /p:TreatWarningsAsErrors=true -fl -flp:logfile={buildLogFile};verbosity=normal"), ProjectPlatform.NodeJs => ("npm", "run build"), - ProjectPlatform.Python => ("python", "-m py_compile ."), + ProjectPlatform.Python => DetectPythonInstallCommand(projectPath) is ("uv", _) + ? (_resolvedUvCommand ?? "uv", "run python -m compileall -q .") + : ("python", "-m compileall -q ."), _ => throw new ArgumentOutOfRangeException(nameof(platform), platform, "Unsupported platform") }; } + /// + /// Returns the path for the build log file. + /// + private static string GetBuildLogPath(ProjectPlatform platform) + { + var suffix = platform switch + { + ProjectPlatform.DotNet => "build", + ProjectPlatform.NodeJs => "build.npm", + ProjectPlatform.Python => "build.python", + _ => "build" + }; + + return ConfigService.GetCommandLogPath($"validate.{suffix}"); + } + private static string GetResolutionGuidance(ProjectPlatform platform) { return platform switch @@ -125,7 +182,7 @@ private static string GetResolutionGuidance(ProjectPlatform platform) ProjectPlatform.NodeJs => "Fix the build errors in your project.\n" + "Run 'npm run build' locally to see the full output.", ProjectPlatform.Python => "Fix the syntax errors in your Python files.\n" + - "Run 'python -m py_compile ' on each file to check for syntax errors.", + "Run 'python -m compileall -q .' locally to see the full output.", _ => "Fix the build errors in your project and try again." }; } @@ -135,9 +192,20 @@ private static string GetResolutionGuidance(ProjectPlatform platform) /// private static string ExtractBuildErrorSummary(CommandResult result, ProjectPlatform platform) { - var output = !string.IsNullOrWhiteSpace(result.StandardError) - ? result.StandardError - : result.StandardOutput; + // Combine both streams — many tools write warnings (not errors) to stderr, + // so preferring stderr alone can surface a deprecation warning instead of the real error. + var parts = new List(); + if (!string.IsNullOrWhiteSpace(result.StandardOutput)) + { + parts.Add(result.StandardOutput); + } + + if (!string.IsNullOrWhiteSpace(result.StandardError)) + { + parts.Add(result.StandardError); + } + + var output = string.Join("\n", parts); if (string.IsNullOrWhiteSpace(output)) { @@ -173,31 +241,363 @@ private static string ExtractBuildErrorSummary(CommandResult result, ProjectPlat } /// - /// Returns deploymentProjectPath if configured, otherwise falls back to the current directory. + /// Writes captured build output to a log file for non-.NET platforms. /// - private static string ResolveProjectPath(Agent365Config config) + private static void WriteBuildLog(string logPath, CommandResult? result, string? errorMessage = null, bool append = false) { - return string.IsNullOrWhiteSpace(config.DeploymentProjectPath) - ? Directory.GetCurrentDirectory() - : Path.GetFullPath(config.DeploymentProjectPath); + try + { + var dir = Path.GetDirectoryName(logPath); + if (dir is not null) + { + Directory.CreateDirectory(dir); + } + + using var writer = new StreamWriter(logPath, append: append); + + if (errorMessage is not null) + { + writer.WriteLine(errorMessage); + } + + if (result is not null) + { + if (!string.IsNullOrEmpty(result.StandardOutput)) + { + writer.WriteLine(result.StandardOutput); + } + + if (!string.IsNullOrEmpty(result.StandardError)) + { + writer.WriteLine(result.StandardError); + } + } + } + catch + { + // Best-effort: don't fail the check if log writing fails. + } + } + + /// + /// Installs dependencies for the detected platform before building. + /// Returns null if no install is needed or deps installed successfully. + /// Returns a failure result if installation fails. + /// + private async Task<(RequirementCheckResult? FailureResult, CommandResult? Output)> InstallDependenciesAsync( + ProjectPlatform platform, + string projectPath, + ILogger logger, + CancellationToken cancellationToken) + { + return platform switch + { + ProjectPlatform.Python => await InstallPythonDependenciesAsync(projectPath, logger, cancellationToken), + ProjectPlatform.NodeJs => await InstallNodeDependenciesAsync(projectPath, logger, cancellationToken), + _ => (null, null) + }; + } + + /// + /// Runs npm install if a package.json exists and node_modules is missing. + /// + private async Task<(RequirementCheckResult? FailureResult, CommandResult? Output)> InstallNodeDependenciesAsync( + string projectPath, + ILogger logger, + CancellationToken cancellationToken) + { + var packageJson = Path.Combine(projectPath, "package.json"); + if (!File.Exists(packageJson)) + { + return (null, null); + } + + var nodeModules = Path.Combine(projectPath, "node_modules"); + if (Directory.Exists(nodeModules)) + { + logger.LogDebug("node_modules already exists, skipping npm install"); + return (null, null); + } + + logger.LogDebug("Running npm install in {Path}", projectPath); + + var result = await _commandExecutor.ExecuteAsync( + "npm", "install", + workingDirectory: projectPath, + captureOutput: true, + suppressErrorLogging: true, + cancellationToken: cancellationToken); + + if (result.Success) + { + logger.LogDebug("npm install completed successfully"); + return (null, result); + } + + var summary = ExtractBuildErrorSummary(result, ProjectPlatform.NodeJs); + + return (new RequirementCheckResult + { + Passed = false, + ErrorMessage = $"Failed to install Node.js dependencies (npm install):\n{summary}", + ResolutionGuidance = "Run 'npm install' manually to see the full output." + }, result); + } + + /// + /// Detects the Python package manager used by the project and installs dependencies. + /// Detection order: uv (uv.lock or pyproject.toml) -> pip (requirements.txt). + /// Returns null if no dependency file is found (no install needed) or deps installed successfully. + /// Returns a failure result if installation fails. + /// + private async Task<(RequirementCheckResult? FailureResult, CommandResult? Output)> InstallPythonDependenciesAsync( + string projectPath, + ILogger logger, + CancellationToken cancellationToken) + { + var (command, arguments) = DetectPythonInstallCommand(projectPath); + + if (command is null) + { + logger.LogDebug("No Python dependency file found, skipping dependency install"); + return (null, null); + } + + // If uv is needed but not installed, install it and resolve its path + if (command == "uv") + { + var uvCommand = await EnsureUvInstalledAsync(logger, cancellationToken); + if (uvCommand is null) + { + return (new RequirementCheckResult + { + Passed = false, + ErrorMessage = "uv is required but could not be installed", + ResolutionGuidance = "Install uv manually: pip install uv (or see https://docs.astral.sh/uv/getting-started/installation/)" + }, null); + } + + // Use the resolved uv path for the install command + command = uvCommand; + _resolvedUvCommand = uvCommand; + } + + logger.LogDebug("Installing Python dependencies: {Command} {Arguments}", command, arguments); + + var result = await _commandExecutor.ExecuteAsync( + command, + arguments!, + workingDirectory: projectPath, + captureOutput: true, + suppressErrorLogging: true, + cancellationToken: cancellationToken); + + if (result.Success) + { + logger.LogDebug("Python dependencies installed successfully"); + return (null, result); + } + + var summary = ExtractBuildErrorSummary(result, ProjectPlatform.Python); + + return (new RequirementCheckResult + { + Passed = false, + ErrorMessage = $"Failed to install Python dependencies ({command} {arguments}):\n{summary}", + ResolutionGuidance = $"Run '{command} {arguments}' manually to see the full output." + }, result); } /// - /// Truncates log output to the last 100 lines to keep the JSON report reasonable. + /// Checks if uv is available on PATH. If not, attempts to install it via pip. + /// Returns the resolved uv command (either "uv" if on PATH, or full path after pip install). + /// Returns null if uv cannot be made available. /// - private static string? TruncateLog(string? log, int maxLines = 100) + private async Task EnsureUvInstalledAsync(ILogger logger, CancellationToken cancellationToken) { - if (string.IsNullOrWhiteSpace(log)) + // Check if uv is already available on PATH + var checkResult = await _commandExecutor.ExecuteAsync( + "uv", "version", + captureOutput: true, + suppressErrorLogging: true, + cancellationToken: cancellationToken); + + if (checkResult.Success) { + logger.LogDebug("uv is available: {Version}", checkResult.StandardOutput.Trim()); + return "uv"; + } + + // Try to install uv via pip + logger.LogDebug("uv not found, attempting to install via pip"); + + var installResult = await _commandExecutor.ExecuteAsync( + "pip", "install uv", + captureOutput: true, + suppressErrorLogging: true, + cancellationToken: cancellationToken); + + if (!installResult.Success) + { + logger.LogWarning("Failed to install uv: {Error}", + !string.IsNullOrWhiteSpace(installResult.StandardError) + ? installResult.StandardError.Trim() + : installResult.StandardOutput.Trim()); return null; } - var lines = log.Split('\n'); - if (lines.Length <= maxLines) + logger.LogDebug("uv installed via pip, resolving path"); + + // After pip install, uv may not be on PATH for the current process. + // Resolve its location via pip show or python -c. + var resolveResult = await _commandExecutor.ExecuteAsync( + "python", "-c \"import shutil; p = shutil.which('uv'); print(p if p else '')\"", + captureOutput: true, + suppressErrorLogging: true, + cancellationToken: cancellationToken); + + var uvPath = resolveResult.Success ? resolveResult.StandardOutput.Trim() : null; + + if (!string.IsNullOrWhiteSpace(uvPath) && File.Exists(uvPath)) + { + logger.LogDebug("Resolved uv path: {Path}", uvPath); + return uvPath; + } + + // Fallback: try common Scripts directory + var scriptsUv = ResolveUvFromPythonScripts(); + if (scriptsUv is not null) + { + logger.LogDebug("Found uv in Python Scripts: {Path}", scriptsUv); + return scriptsUv; + } + + // Last resort: try "uv" again (pip may have added it to PATH) + var retryResult = await _commandExecutor.ExecuteAsync( + "uv", "version", + captureOutput: true, + suppressErrorLogging: true, + cancellationToken: cancellationToken); + + if (retryResult.Success) + { + return "uv"; + } + + logger.LogWarning("uv was installed via pip but could not be found on PATH"); + return null; + } + + /// + /// Attempts to find the uv executable in Python's Scripts directory. + /// + private static string? ResolveUvFromPythonScripts() + { + try + { + var pythonPath = Environment.GetEnvironmentVariable("VIRTUAL_ENV"); + if (!string.IsNullOrEmpty(pythonPath)) + { + var uvInVenv = Path.Combine(pythonPath, + OperatingSystem.IsWindows() ? "Scripts" : "bin", + OperatingSystem.IsWindows() ? "uv.exe" : "uv"); + if (File.Exists(uvInVenv)) + return uvInVenv; + } + + // Check user-level pip install location + var userBase = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile); + if (!string.IsNullOrEmpty(userBase)) + { + // Windows: %APPDATA%\Python\PythonXX\Scripts + var appData = Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData); + if (!string.IsNullOrEmpty(appData)) + { + var pythonDirs = Directory.Exists(Path.Combine(appData, "Python")) + ? Directory.GetDirectories(Path.Combine(appData, "Python"), "Python*") + : []; + + foreach (var pyDir in pythonDirs) + { + var candidate = Path.Combine(pyDir, "Scripts", + OperatingSystem.IsWindows() ? "uv.exe" : "uv"); + if (File.Exists(candidate)) + return candidate; + } + } + } + } + catch + { + // Best-effort + } + + return null; + } + + /// + /// Detects the appropriate install command for a Python project. + /// Returns (null, null) if no dependency file is found. + /// + internal static (string? Command, string? Arguments) DetectPythonInstallCommand(string projectPath) + { + // uv: check for uv.lock or pyproject.toml (uv can work with pyproject.toml directly) + if (File.Exists(Path.Combine(projectPath, "uv.lock"))) { - return log.TrimEnd(); + return ("uv", "sync"); } - return string.Join("\n", lines.TakeLast(maxLines)).TrimEnd(); + if (File.Exists(Path.Combine(projectPath, "pyproject.toml"))) + { + // pyproject.toml could be used by uv, pip, or poetry. + // Check if uv is the tool by looking for [tool.uv] section + if (HasUvConfig(Path.Combine(projectPath, "pyproject.toml"))) + { + return ("uv", "sync"); + } + + // Generic pyproject.toml — use pip install + return ("pip", "install -e ."); + } + + if (File.Exists(Path.Combine(projectPath, "requirements.txt"))) + { + return ("pip", "install -r requirements.txt"); + } + + return (null, null); + } + + /// + /// Checks if a pyproject.toml contains a [tool.uv] section, indicating uv is the package manager. + /// + internal static bool HasUvConfig(string pyprojectPath) + { + try + { + foreach (var line in File.ReadLines(pyprojectPath)) + { + if (line.TrimStart().StartsWith("[tool.uv", StringComparison.OrdinalIgnoreCase)) + { + return true; + } + } + } + catch + { + // Best-effort + } + + return false; + } + + /// + /// Returns deploymentProjectPath if configured, otherwise falls back to the current directory. + /// + private static string ResolveProjectPath(Agent365Config config) + { + return string.IsNullOrWhiteSpace(config.DeploymentProjectPath) + ? Directory.GetCurrentDirectory() + : Path.GetFullPath(config.DeploymentProjectPath); } } diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/TelemetryRequirementCheck.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/TelemetryRequirementCheck.cs new file mode 100644 index 00000000..c41585f7 --- /dev/null +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/TelemetryRequirementCheck.cs @@ -0,0 +1,245 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.Agents.A365.DevTools.Cli.Models; +using Microsoft.Extensions.Logging; + +namespace Microsoft.Agents.A365.DevTools.Cli.Services.Requirements.RequirementChecks; + +/// +/// Validates that the agent is exporting telemetry traces by analyzing the agent's +/// console output log file captured during the conversation validation step. +/// +public class TelemetryRequirementCheck : RequirementCheck +{ + private readonly string? _agentConsoleLogPath; + + /// + /// Maximum number of telemetry-relevant log lines to analyze. + /// + internal const int MaxTelemetryLines = 100; + + /// + /// Keywords that identify a log line as telemetry-related. + /// A line must contain at least one of these to be considered relevant. + /// + internal static readonly string[] TelemetryContextKeywords = new[] + { + "opentelemetry", + "otel", + "otlp", + "tracer", + "tracerprovider", + "activitysource", + "span", + "exporter", + "agent365observability", + "agent365.observability", + "agent365exporter", + "otelwrite", + "batchexportprocessor", + "traces exported", + "export completed", + "export succeeded" + }; + + /// + /// Patterns that indicate successful trace export when found in telemetry-relevant lines. + /// + internal static readonly string[] SuccessPatterns = new[] + { + "export completed", + "export succeeded", + "successfully exported", + "traces exported", + "span exported", + "tracerprovider built", + "tracerprovider started", + "otlpexporter", + "otlptraceexporter" + }; + + /// + /// Patterns that indicate trace export failure when found in telemetry-relevant lines. + /// + internal static readonly string[] FailurePatterns = new[] + { + "export failed", + "exporter error", + "connection refused", + "unavailable", + "deadline_exceeded", + "unauthenticated", + "permissiondenied", + "failed to export", + "exporter threw", + "dropped spans", + "nothing exported", + "spans skipped", + "spans filtered out", + "missing tenant", + "missing agent id" + }; + + public TelemetryRequirementCheck(string? agentConsoleLogPath) + { + _agentConsoleLogPath = agentConsoleLogPath; + } + + /// + public override string Name => "Telemetry"; + + /// + public override string Description => "Validates that telemetry traces are being exported to Agent365"; + + /// + public override string Category => "Observability"; + + /// + public override async Task CheckAsync( + Agent365Config config, + ILogger logger, + CancellationToken cancellationToken = default) + { + return await ExecuteCheckWithLoggingAsync(config, logger, CheckImplementationAsync, cancellationToken); + } + + private Task CheckImplementationAsync( + Agent365Config config, + ILogger logger, + CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(_agentConsoleLogPath) || !File.Exists(_agentConsoleLogPath)) + { + return Task.FromResult(RequirementCheckResult.Warning( + "No agent console log file available to analyze for telemetry", + details: "Telemetry check requires agent console output from the conversation step")); + } + + logger.LogDebug("Analyzing agent console log at {LogPath}", _agentConsoleLogPath); + + string[] logLines; + try + { + logLines = File.ReadAllLines(_agentConsoleLogPath); + } + catch (Exception ex) + { + logger.LogDebug(ex, "Failed to read agent console log file"); + return Task.FromResult(RequirementCheckResult.Warning( + "Could not read agent console log file", + details: $"Failed to read {_agentConsoleLogPath}: {ex.Message}")); + } + + var relevantLines = FilterTelemetryLines(logLines); + + if (relevantLines.Count == 0) + { + return Task.FromResult(RequirementCheckResult.Failure( + "No telemetry-related output detected in agent console logs", + "Configure OpenTelemetry in your agent to export traces to Agent365.", + details: "No OpenTelemetry, OTLP, or trace export evidence found in agent console output.")); + } + + logger.LogDebug("Found {Count} telemetry-relevant log lines", relevantLines.Count); + + var matchedSuccessPatterns = FindMatchingPatterns(relevantLines, SuccessPatterns); + var matchedFailurePatterns = FindMatchingPatterns(relevantLines, FailurePatterns); + + // Failure takes precedence over success + if (matchedFailurePatterns.Count > 0) + { + var failureDetails = string.Join(", ", matchedFailurePatterns); + var guidance = GetFailureGuidance(matchedFailurePatterns); + return Task.FromResult(RequirementCheckResult.Failure( + $"Telemetry export failures detected: {failureDetails}", + guidance, + details: $"Failure patterns found: {failureDetails}. " + + $"Success patterns found: {(matchedSuccessPatterns.Count > 0 ? string.Join(", ", matchedSuccessPatterns) : "none")}. " + + $"Analyzed {relevantLines.Count} telemetry-relevant log lines.")); + } + + if (matchedSuccessPatterns.Count > 0) + { + var successDetails = string.Join(", ", matchedSuccessPatterns); + return Task.FromResult(RequirementCheckResult.Success( + details: $"Telemetry export evidence found: {successDetails}. " + + $"Analyzed {relevantLines.Count} telemetry-relevant log lines.")); + } + + // Telemetry lines exist but no clear success or failure — treat as failure + return Task.FromResult(RequirementCheckResult.Failure( + "Telemetry SDK detected but no trace export evidence found", + "Ensure traces are being exported to the Agent365 OTLP endpoint.", + details: $"Found {relevantLines.Count} telemetry-related log lines but could not " + + "confirm successful trace export.")); + } + + /// + /// Filters log lines to only those containing telemetry-related keywords. + /// + internal static List FilterTelemetryLines(string[] logLines) + { + var result = new List(); + foreach (var line in logLines) + { + if (string.IsNullOrWhiteSpace(line)) + continue; + + var lower = line.ToLowerInvariant(); + foreach (var keyword in TelemetryContextKeywords) + { + if (lower.Contains(keyword)) + { + result.Add(line); + if (result.Count >= MaxTelemetryLines) + return result; + break; + } + } + } + + return result; + } + + /// + /// Finds which patterns from the given set appear in the relevant log lines. + /// + internal static List FindMatchingPatterns(List relevantLines, string[] patterns) + { + var matched = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var line in relevantLines) + { + var lower = line.ToLowerInvariant(); + foreach (var pattern in patterns) + { + if (lower.Contains(pattern) && matched.Add(pattern)) + { + // Found a new pattern match + } + } + } + + return matched.ToList(); + } + + private static string GetFailureGuidance(List failurePatterns) + { + var lower = failurePatterns.Select(p => p.ToLowerInvariant()).ToHashSet(); + + if (lower.Contains("missing tenant") || lower.Contains("missing agent id") || + lower.Contains("nothing exported") || lower.Contains("spans skipped")) + return "Configure tenant ID and agent ID in your agent's observability settings. " + + "The Agent365 exporter requires both to export spans."; + + if (lower.Contains("connection refused") || lower.Contains("unavailable") || lower.Contains("deadline_exceeded")) + return "Check that the OTLP endpoint is reachable from the agent. " + + "Verify the endpoint URL and network connectivity."; + + if (lower.Contains("unauthenticated") || lower.Contains("permissiondenied")) + return "Check OTLP endpoint credentials. " + + "Ensure the agent has valid authentication for the observability endpoint."; + + return "Check the agent console logs for telemetry export error details."; + } +} diff --git a/src/Microsoft.Agents.A365.DevTools.Validation/ValidateReport.cs b/src/Microsoft.Agents.A365.DevTools.Validation/ValidateReport.cs index 2de42122..bfe4348c 100644 --- a/src/Microsoft.Agents.A365.DevTools.Validation/ValidateReport.cs +++ b/src/Microsoft.Agents.A365.DevTools.Validation/ValidateReport.cs @@ -21,6 +21,10 @@ public sealed class ValidateReport [JsonPropertyName("summary")] public SummaryResult Summary { get; set; } = new(); + + [JsonPropertyName("agentConsoleLogFile")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? AgentConsoleLogFile { get; set; } } /// @@ -61,7 +65,7 @@ public sealed class ValidationTiers public ConversationTierResult Conversation { get; set; } = TierResult.CreateSkipped("not yet implemented"); [JsonPropertyName("telemetry")] - public TierResult Telemetry { get; set; } = TierResult.CreateSkipped("not yet implemented"); + public TelemetryTierResult Telemetry { get; set; } = TierResult.CreateSkipped("not yet run"); [JsonPropertyName("blueprint")] public TierResult Blueprint { get; set; } = TierResult.CreateSkipped("not yet implemented"); @@ -146,6 +150,14 @@ public sealed class BuildTierResult : TierResult [JsonPropertyName("exitCode")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public int? ExitCode { get; set; } + + [JsonPropertyName("errorSummary")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? ErrorSummary { get; set; } + + [JsonPropertyName("buildLogFile")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? BuildLogFile { get; set; } } /// @@ -160,6 +172,10 @@ public sealed class BootTierResult : TierResult [JsonPropertyName("bootMs")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public long? BootMs { get; set; } + + [JsonPropertyName("bootLogFile")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? BootLogFile { get; set; } } /// @@ -174,6 +190,28 @@ public sealed class ConversationTierResult : TierResult [JsonPropertyName("playgroundLaunched")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public bool? PlaygroundLaunched { get; set; } + + [JsonPropertyName("conversationLogFile")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? ConversationLogFile { get; set; } +} + +/// +/// Telemetry tier: trace export validation result. +/// +public sealed class TelemetryTierResult : TierResult +{ + [JsonPropertyName("exportDetected")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public bool? ExportDetected { get; set; } + + [JsonPropertyName("matchedPatterns")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public List? MatchedPatterns { get; set; } + + [JsonPropertyName("analyzedLineCount")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? AnalyzedLineCount { get; set; } } /// diff --git a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/CommandExecutorTests.cs b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/CommandExecutorTests.cs index c7f69d67..762fb342 100644 --- a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/CommandExecutorTests.cs +++ b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/CommandExecutorTests.cs @@ -45,15 +45,20 @@ public async Task ExecuteAsync_ValidCommand_ReturnsSuccess() } [Fact] - public async Task ExecuteAsync_InvalidCommand_ThrowsException() + public async Task ExecuteAsync_InvalidCommand_ReturnsFailedResult() { // Arrange var command = "nonexistent-command-12345"; var args = ""; - // Act & Assert - await Assert.ThrowsAsync(() => - _executor.ExecuteAsync(command, args, captureOutput: true)); + // Act + var result = await _executor.ExecuteAsync(command, args, captureOutput: true); + + // Assert + result.ExitCode.Should().Be(-1, + because: "a missing executable should return exit code -1 instead of throwing"); + result.StandardError.Should().Contain("not found", + because: "the error message should indicate the command was not found"); } [Fact] diff --git a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/LocalRuntimeRequirementCheckTests.cs b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/LocalRuntimeRequirementCheckTests.cs index 1aa236a0..503d3aef 100644 --- a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/LocalRuntimeRequirementCheckTests.cs +++ b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/LocalRuntimeRequirementCheckTests.cs @@ -259,6 +259,146 @@ private static Process CreateFakeProcess(bool exitImmediately, int exitCode = 0) return process; } + #region Python Entry Point Detection + + [Fact] + public void ResolvePythonEntryPoint_WhenSingleFileHasMainGuard_ReturnsThatFile() + { + File.WriteAllText(Path.Combine(_tempDir, "bot_runner.py"), + "import bot\nif __name__ == \"__main__\":\n bot.run()"); + + var result = LocalRuntimeRequirementCheck.ResolvePythonEntryPoint(_tempDir); + + result.Should().Be("bot_runner.py"); + } + + [Fact] + public void ResolvePythonEntryPoint_WhenMultipleFilesHaveMainGuard_PrefersWellKnownName() + { + File.WriteAllText(Path.Combine(_tempDir, "app.py"), + "if __name__ == '__main__':\n pass"); + File.WriteAllText(Path.Combine(_tempDir, "helper.py"), + "if __name__ == '__main__':\n pass"); + + var result = LocalRuntimeRequirementCheck.ResolvePythonEntryPoint(_tempDir); + + result.Should().Be("app.py", + because: "app.py is a preferred entry point name when multiple files have __main__ guards"); + } + + [Fact] + public void ResolvePythonEntryPoint_WhenNoMainGuard_FallsBackToExistingWellKnownFile() + { + File.WriteAllText(Path.Combine(_tempDir, "main.py"), "# no guard"); + File.WriteAllText(Path.Combine(_tempDir, "utils.py"), "# utility"); + + var result = LocalRuntimeRequirementCheck.ResolvePythonEntryPoint(_tempDir); + + result.Should().Be("main.py", + because: "main.py exists and is a known entry point name even without a __main__ guard"); + } + + [Fact] + public void ResolvePythonEntryPoint_WhenNoPyFiles_FallsBackToAppPy() + { + var result = LocalRuntimeRequirementCheck.ResolvePythonEntryPoint(_tempDir); + + result.Should().Be("app.py", + because: "app.py is the ultimate default when nothing else is found"); + } + + [Fact] + public void ResolvePythonEntryPoint_ProcfileTakesPriority_OverMainGuard() + { + File.WriteAllText(Path.Combine(_tempDir, "Procfile"), "web: python serve.py --host 0.0.0.0"); + File.WriteAllText(Path.Combine(_tempDir, "app.py"), + "if __name__ == '__main__':\n pass"); + + var result = LocalRuntimeRequirementCheck.ResolvePythonEntryPoint(_tempDir); + + result.Should().Be("serve.py --host 0.0.0.0", + because: "Procfile is the explicit user-declared entry point and takes highest priority"); + } + + [Fact] + public void ResolvePythonEntryPoint_WhenProcfileHasGunicorn_FallsToCodeScan() + { + File.WriteAllText(Path.Combine(_tempDir, "Procfile"), "web: gunicorn app:app"); + File.WriteAllText(Path.Combine(_tempDir, "bot.py"), + "if __name__ == '__main__':\n run()"); + + var result = LocalRuntimeRequirementCheck.ResolvePythonEntryPoint(_tempDir); + + result.Should().Be("bot.py", + because: "gunicorn is not a python command so Procfile is skipped and code scanning finds bot.py"); + } + + [Fact] + public void ResolvePythonEntryPoint_WhenProcfileHasDashM_ReturnsModuleArgs() + { + File.WriteAllText(Path.Combine(_tempDir, "Procfile"), "web: python -m uvicorn app:app"); + + var result = LocalRuntimeRequirementCheck.ResolvePythonEntryPoint(_tempDir); + + result.Should().Be("-m uvicorn app:app"); + } + + [Fact] + public void HasMainGuard_WithDoubleQuotes_ReturnsTrue() + { + var file = Path.Combine(_tempDir, "test.py"); + File.WriteAllText(file, "import os\nif __name__ == \"__main__\":\n main()"); + + LocalRuntimeRequirementCheck.HasMainGuard(file).Should().BeTrue(); + } + + [Fact] + public void HasMainGuard_WithSingleQuotes_ReturnsTrue() + { + var file = Path.Combine(_tempDir, "test.py"); + File.WriteAllText(file, "if __name__ == '__main__':\n main()"); + + LocalRuntimeRequirementCheck.HasMainGuard(file).Should().BeTrue(); + } + + [Fact] + public void HasMainGuard_WithoutGuard_ReturnsFalse() + { + var file = Path.Combine(_tempDir, "test.py"); + File.WriteAllText(file, "def main():\n pass\n\nmain()"); + + LocalRuntimeRequirementCheck.HasMainGuard(file).Should().BeFalse(); + } + + [Fact] + public void HasMainGuard_WithIndentation_ReturnsTrue() + { + var file = Path.Combine(_tempDir, "test.py"); + File.WriteAllText(file, "# entry\n if __name__ == '__main__':\n run()"); + + LocalRuntimeRequirementCheck.HasMainGuard(file).Should().BeTrue(); + } + + [Fact] + public void ParseProcfileEntryPoint_WhenEmpty_ReturnsNull() + { + var procfile = Path.Combine(_tempDir, "Procfile"); + File.WriteAllText(procfile, ""); + + LocalRuntimeRequirementCheck.ParseProcfileEntryPoint(procfile).Should().BeNull(); + } + + [Fact] + public void ParseProcfileEntryPoint_WhenNoWebProcess_ReturnsNull() + { + var procfile = Path.Combine(_tempDir, "Procfile"); + File.WriteAllText(procfile, "worker: python worker.py"); + + LocalRuntimeRequirementCheck.ParseProcfileEntryPoint(procfile).Should().BeNull(); + } + + #endregion + /// /// Fake HTTP handler that returns a configurable status code. /// diff --git a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/ProjectBuildRequirementCheckTests.cs b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/ProjectBuildRequirementCheckTests.cs index 9199c515..75d3d5bd 100644 --- a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/ProjectBuildRequirementCheckTests.cs +++ b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/ProjectBuildRequirementCheckTests.cs @@ -187,6 +187,7 @@ public async Task CheckAsync_WhenNodeJsBuildSucceeds_ReturnsSuccess() { // Arrange - create a package.json so PlatformDetector identifies NodeJs File.WriteAllText(Path.Combine(_tempDir, "package.json"), "{}"); + Directory.CreateDirectory(Path.Combine(_tempDir, "node_modules")); var config = new Agent365Config { DeploymentProjectPath = _tempDir }; _commandExecutor.ExecuteAsync( Arg.Is("npm"), @@ -210,6 +211,7 @@ public async Task CheckAsync_WhenNodeJsBuildFails_ReturnsFailure() { // Arrange File.WriteAllText(Path.Combine(_tempDir, "package.json"), "{}"); + Directory.CreateDirectory(Path.Combine(_tempDir, "node_modules")); var config = new Agent365Config { DeploymentProjectPath = _tempDir }; _commandExecutor.ExecuteAsync( Arg.Is("npm"), @@ -305,4 +307,81 @@ await _commandExecutor.Received(1).ExecuteAsync( Arg.Any(), Arg.Any()); } + + #region Python Dependency Detection + + [Fact] + public void DetectPythonInstallCommand_WhenUvLockExists_ReturnsUvSync() + { + File.WriteAllText(Path.Combine(_tempDir, "uv.lock"), "# uv lock"); + + var (command, arguments) = ProjectBuildRequirementCheck.DetectPythonInstallCommand(_tempDir); + + command.Should().Be("uv"); + arguments.Should().Be("sync"); + } + + [Fact] + public void DetectPythonInstallCommand_WhenPyprojectWithUvSection_ReturnsUvSync() + { + File.WriteAllText(Path.Combine(_tempDir, "pyproject.toml"), + "[project]\nname = \"mybot\"\n\n[tool.uv]\ndev-dependencies = []"); + + var (command, arguments) = ProjectBuildRequirementCheck.DetectPythonInstallCommand(_tempDir); + + command.Should().Be("uv"); + arguments.Should().Be("sync"); + } + + [Fact] + public void DetectPythonInstallCommand_WhenPyprojectWithoutUv_ReturnsPipInstall() + { + File.WriteAllText(Path.Combine(_tempDir, "pyproject.toml"), + "[project]\nname = \"mybot\"\n\n[build-system]\nrequires = [\"setuptools\"]"); + + var (command, arguments) = ProjectBuildRequirementCheck.DetectPythonInstallCommand(_tempDir); + + command.Should().Be("pip"); + arguments.Should().Be("install -e ."); + } + + [Fact] + public void DetectPythonInstallCommand_WhenRequirementsTxt_ReturnsPipInstall() + { + File.WriteAllText(Path.Combine(_tempDir, "requirements.txt"), "flask>=2.0\nbotbuilder-core"); + + var (command, arguments) = ProjectBuildRequirementCheck.DetectPythonInstallCommand(_tempDir); + + command.Should().Be("pip"); + arguments.Should().Be("install -r requirements.txt"); + } + + [Fact] + public void DetectPythonInstallCommand_WhenNoDependencyFile_ReturnsNull() + { + var (command, arguments) = ProjectBuildRequirementCheck.DetectPythonInstallCommand(_tempDir); + + command.Should().BeNull(); + arguments.Should().BeNull(); + } + + [Fact] + public void HasUvConfig_WithToolUvSection_ReturnsTrue() + { + var pyproject = Path.Combine(_tempDir, "pyproject.toml"); + File.WriteAllText(pyproject, "[project]\nname = \"bot\"\n\n[tool.uv]\ndev-dependencies = []"); + + ProjectBuildRequirementCheck.HasUvConfig(pyproject).Should().BeTrue(); + } + + [Fact] + public void HasUvConfig_WithoutToolUvSection_ReturnsFalse() + { + var pyproject = Path.Combine(_tempDir, "pyproject.toml"); + File.WriteAllText(pyproject, "[project]\nname = \"bot\"\n\n[build-system]\nrequires = [\"hatchling\"]"); + + ProjectBuildRequirementCheck.HasUvConfig(pyproject).Should().BeFalse(); + } + + #endregion } diff --git a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/TelemetryRequirementCheckTests.cs b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/TelemetryRequirementCheckTests.cs new file mode 100644 index 00000000..d07206d8 --- /dev/null +++ b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/TelemetryRequirementCheckTests.cs @@ -0,0 +1,322 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using FluentAssertions; +using Microsoft.Agents.A365.DevTools.Cli.Models; +using Microsoft.Agents.A365.DevTools.Cli.Services.Requirements; +using Microsoft.Agents.A365.DevTools.Cli.Services.Requirements.RequirementChecks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Xunit; + +namespace Microsoft.Agents.A365.DevTools.Cli.Tests.Services.Requirements; + +public class TelemetryRequirementCheckTests : IDisposable +{ + private readonly ILogger _logger = NullLoggerFactory.Instance.CreateLogger("test"); + private readonly Agent365Config _config = new(); + private readonly List _tempFiles = new(); + + private string CreateTempLogFile(string[] lines) + { + var path = Path.Combine(Path.GetTempPath(), $"telemetry-test-{Guid.NewGuid()}.log"); + File.WriteAllLines(path, lines); + _tempFiles.Add(path); + return path; + } + + public void Dispose() + { + foreach (var f in _tempFiles) + { + try { File.Delete(f); } catch { /* best-effort cleanup */ } + } + } + + [Fact] + public void Name_ReturnsTelemetry() + { + var check = new TelemetryRequirementCheck(null); + check.Name.Should().Be("Telemetry"); + } + + [Fact] + public void Category_ReturnsObservability() + { + var check = new TelemetryRequirementCheck(null); + check.Category.Should().Be("Observability"); + } + + [Fact] + public async Task CheckAsync_NullLogPath_ReturnsWarning() + { + var check = new TelemetryRequirementCheck(null); + + var result = await check.CheckAsync(_config, _logger); + + result.Passed.Should().BeTrue(because: "no log file is a warning, not a failure"); + result.IsWarning.Should().BeTrue(because: "missing log file means telemetry status is unknown"); + result.ErrorMessage.Should().Contain("No agent console log file available"); + } + + [Fact] + public async Task CheckAsync_NonExistentLogPath_ReturnsWarning() + { + var check = new TelemetryRequirementCheck("/nonexistent/path.log"); + + var result = await check.CheckAsync(_config, _logger); + + result.Passed.Should().BeTrue(because: "missing file is a warning, not a failure"); + result.IsWarning.Should().BeTrue(); + } + + [Fact] + public async Task CheckAsync_NoTelemetryLines_ReturnsWarning() + { + var logPath = CreateTempLogFile(new[] + { + "info: Application started", + "info: Listening on http://localhost:5000", + "info: Received message: Hello" + }); + + var check = new TelemetryRequirementCheck(logPath); + + var result = await check.CheckAsync(_config, _logger); + + result.Passed.Should().BeFalse(because: "no telemetry evidence means telemetry is not configured"); + result.IsWarning.Should().BeFalse(because: "missing telemetry is a failure, not a warning"); + result.ErrorMessage.Should().Contain("No telemetry-related output detected"); + } + + [Fact] + public async Task CheckAsync_SuccessPatterns_ReturnsPass() + { + var logPath = CreateTempLogFile(new[] + { + "info: Application started", + "info: OpenTelemetry TracerProvider built successfully", + "info: OtlpExporter configured for https://agent365.observability.endpoint", + "info: BatchExportProcessor started", + "info: Export completed - 5 spans exported" + }); + + var check = new TelemetryRequirementCheck(logPath); + + var result = await check.CheckAsync(_config, _logger); + + result.Passed.Should().BeTrue(because: "success patterns indicate traces are being exported"); + result.IsWarning.Should().BeFalse(); + result.Details.Should().Contain("Telemetry export evidence found"); + } + + [Fact] + public async Task CheckAsync_FailurePatterns_ReturnsFail() + { + var logPath = CreateTempLogFile(new[] + { + "info: Application started", + "info: OpenTelemetry TracerProvider built successfully", + "error: OTLP export failed: connection refused", + "warn: Dropped spans due to exporter error" + }); + + var check = new TelemetryRequirementCheck(logPath); + + var result = await check.CheckAsync(_config, _logger); + + result.Passed.Should().BeFalse(because: "export failures indicate telemetry is not working"); + result.ErrorMessage.Should().Contain("Telemetry export failures detected"); + result.ResolutionGuidance.Should().Contain("OTLP endpoint", because: "connection refused should suggest checking endpoint connectivity"); + } + + [Fact] + public async Task CheckAsync_MixedSuccessAndFailure_FailureTakesPrecedence() + { + var logPath = CreateTempLogFile(new[] + { + "info: OpenTelemetry TracerProvider built successfully", + "info: Export completed - 3 spans exported", + "error: OTLP exporter: UNAVAILABLE - endpoint unreachable", + "info: BatchExportProcessor: dropped spans" + }); + + var check = new TelemetryRequirementCheck(logPath); + + var result = await check.CheckAsync(_config, _logger); + + result.Passed.Should().BeFalse(because: "failure patterns take precedence over success patterns"); + result.Details.Should().Contain("Failure patterns found"); + result.Details.Should().Contain("Success patterns found"); + } + + [Fact] + public async Task CheckAsync_TelemetryContextButNoExportEvidence_ReturnsWarning() + { + var logPath = CreateTempLogFile(new[] + { + "info: Application started", + "dbug: OpenTelemetry SDK initialized", + "dbug: Adding OTLP exporter to pipeline" + }); + + var check = new TelemetryRequirementCheck(logPath); + + var result = await check.CheckAsync(_config, _logger); + + result.Passed.Should().BeFalse(because: "SDK detected without export evidence should fail"); + result.IsWarning.Should().BeFalse(because: "no confirmed export means telemetry is not working"); + result.ErrorMessage.Should().Contain("Telemetry SDK detected but no trace export evidence found"); + } + + [Fact] + public async Task CheckAsync_Agent365ObservabilityPattern_ReturnsPass() + { + var logPath = CreateTempLogFile(new[] + { + "info: Configuring Agent365.Observability.OtelWrite endpoint", + "info: TracerProvider started with OTLP exporter" + }); + + var check = new TelemetryRequirementCheck(logPath); + + var result = await check.CheckAsync(_config, _logger); + + result.Passed.Should().BeTrue(); + result.IsWarning.Should().BeFalse(); + } + + [Fact] + public async Task CheckAsync_CaseInsensitiveMatching() + { + var logPath = CreateTempLogFile(new[] + { + "INFO: OPENTELEMETRY TRACERPROVIDER BUILT successfully", + "INFO: OTLPEXPORTER EXPORT COMPLETED" + }); + + var check = new TelemetryRequirementCheck(logPath); + + var result = await check.CheckAsync(_config, _logger); + + result.Passed.Should().BeTrue(because: "pattern matching should be case-insensitive"); + result.IsWarning.Should().BeFalse(); + } + + [Fact] + public void FilterTelemetryLines_FiltersOnlyRelevantLines() + { + var logLines = new[] + { + "info: Application started", + "info: OpenTelemetry TracerProvider built", + "info: Listening on port 5000", + "error: OTLP export failed", + "info: Received message" + }; + + var result = TelemetryRequirementCheck.FilterTelemetryLines(logLines); + + result.Should().HaveCount(2); + result[0].Should().Contain("OpenTelemetry"); + result[1].Should().Contain("OTLP"); + } + + [Fact] + public void FilterTelemetryLines_RespectsMaxLimit() + { + var logLines = Enumerable.Range(0, 200) + .Select(i => $"info: OpenTelemetry span {i} exported") + .ToArray(); + + var result = TelemetryRequirementCheck.FilterTelemetryLines(logLines); + + result.Should().HaveCount(TelemetryRequirementCheck.MaxTelemetryLines); + } + + [Fact] + public void FilterTelemetryLines_SkipsEmptyAndWhitespace() + { + var logLines = new[] { "", " ", null!, "info: TracerProvider started" }; + + var result = TelemetryRequirementCheck.FilterTelemetryLines(logLines); + + result.Should().ContainSingle() + .Which.Should().Contain("TracerProvider"); + } + + [Fact] + public void FindMatchingPatterns_FindsAllMatches() + { + var lines = new List + { + "Export completed successfully", + "TracerProvider built and started", + "OtlpExporter configured" + }; + + var result = TelemetryRequirementCheck.FindMatchingPatterns( + lines, TelemetryRequirementCheck.SuccessPatterns); + + result.Should().Contain("export completed"); + result.Should().Contain("otlpexporter"); + } + + [Fact] + public void FindMatchingPatterns_ReturnsEmptyForNoMatches() + { + var lines = new List + { + "Application started", + "Listening on port 5000" + }; + + var result = TelemetryRequirementCheck.FindMatchingPatterns( + lines, TelemetryRequirementCheck.SuccessPatterns); + + result.Should().BeEmpty(); + } + + [Fact] + public async Task CheckAsync_UnrelatedConnectionRefused_NotFlaggedAsFailure() + { + var logPath = CreateTempLogFile(new[] + { + "info: Application started", + "error: Database connection refused on port 5432" + }); + + var check = new TelemetryRequirementCheck(logPath); + + var result = await check.CheckAsync(_config, _logger); + + result.Passed.Should().BeTrue(because: "unrelated connection errors should not trigger telemetry failure"); + result.IsWarning.Should().BeTrue(because: "no telemetry-relevant lines found"); + } + + [Fact] + public async Task CheckAsync_Agent365ExporterSpansSkipped_ReturnsFail() + { + var logPath = CreateTempLogFile(new[] + { + "dbug: Microsoft.Agents.A365.Observability.Runtime.Tracing.Exporters.Agent365Exporter[0]", + " Agent365Exporter: Exporting batch of 7 spans.", + "dbug: Microsoft.Agents.A365.Observability.Runtime.Tracing.Exporters.Agent365ExporterCore[0]", + " [Agent365Exporter] 5 non-genAI spans filtered out", + "dbug: Microsoft.Agents.A365.Observability.Runtime.Tracing.Exporters.Agent365ExporterCore[0]", + " [Agent365Exporter] 2 spans skipped due to missing tenant or agent ID", + "dbug: Microsoft.Agents.A365.Observability.Runtime.Tracing.Exporters.Agent365ExporterCore[0]", + " [Agent365Exporter] Partitioned into 0 identity groups (7 spans skipped)", + "dbug: Microsoft.Agents.A365.Observability.Runtime.Tracing.Exporters.Agent365Exporter[0]", + " Agent365Exporter: No spans with tenant/agent identity found; nothing exported." + }); + + var check = new TelemetryRequirementCheck(logPath); + + var result = await check.CheckAsync(_config, _logger); + + result.Passed.Should().BeFalse(because: "Agent365Exporter skipped all spans due to missing identity -- telemetry is not working"); + result.ErrorMessage.Should().Contain("Telemetry export failures detected"); + result.ResolutionGuidance.Should().Contain("tenant ID", because: "missing tenant/agent ID requires identity configuration guidance"); + } +} From c51dd5c40dfabd87061f5f0fab5c247512c06064 Mon Sep 17 00:00:00 2001 From: Abbinayaa Subramanian Date: Tue, 26 May 2026 14:55:59 -0700 Subject: [PATCH 03/27] Fixing telemetry check and remove anonymous auth --- .../Commands/ValidateCommand.cs | 94 ++- .../Program.cs | 2 +- .../BlueprintRegistrationRequirementCheck.cs | 151 ++++ .../ConversationRequirementCheck.cs | 127 +++- .../TelemetryRequirementCheck.cs | 454 +++++++++--- .../ValidateReport.cs | 32 +- ...eprintRegistrationRequirementCheckTests.cs | 295 ++++++++ .../ConversationRequirementCheckTests.cs | 195 +++++- .../LocalRuntimeRequirementCheckTests.cs | 13 +- .../TelemetryRequirementCheckTests.cs | 650 ++++++++++++++---- 10 files changed, 1702 insertions(+), 311 deletions(-) create mode 100644 src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/BlueprintRegistrationRequirementCheck.cs create mode 100644 src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/BlueprintRegistrationRequirementCheckTests.cs diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Commands/ValidateCommand.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Commands/ValidateCommand.cs index 67b8542e..537a0adc 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Commands/ValidateCommand.cs +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Commands/ValidateCommand.cs @@ -41,6 +41,7 @@ public static Command CreateCommand( PlatformDetector? platformDetector = null, CommandExecutor? commandExecutor = null, IProcessService? processService = null, + GraphApiService? graphApiService = null, IEnumerable? requirementChecksOverride = null) { var command = new Command(CommandNames.Validate, @@ -92,6 +93,16 @@ public static Command CreateCommand( var results = await RunChecksDetailedAsync(structuralChecks, config, logger, ct); MapResultsToTiers(results, report); + // Phase 2a: Run blueprint registration check (independent of build/boot) + if (requirementChecksOverride is null && graphApiService is not null) + { + var registrationCheck = new BlueprintRegistrationRequirementCheck(graphApiService); + var registrationResults = await RunChecksDetailedAsync( + new List { registrationCheck }, config, logger, ct); + MapResultsToTiers(registrationResults, report); + results.AddRange(registrationResults); + } + // Extract resolved uv command from build step for boot and conversation steps var buildResultEntry = results .FirstOrDefault(r => r.Check is ProjectBuildRequirementCheck); @@ -400,13 +411,18 @@ private static void MapResultsToTiers( break; case TelemetryRequirementCheck: + // ConsoleExporterActive is true if span blocks were found, even if some operations are missing. + // The Details field contains "Console exporter active" when spans were detected. + var exporterDetected = result.Details?.Contains("Console exporter active", StringComparison.OrdinalIgnoreCase) == true + || result.Details?.Contains("span(s)", StringComparison.OrdinalIgnoreCase) == true; + if (result.IsWarning) { report.Tiers.Telemetry = new TelemetryTierResult { Ok = true, Warning = result.ErrorMessage, - ExportDetected = false + ConsoleExporterActive = exporterDetected }; } else @@ -414,8 +430,7 @@ private static void MapResultsToTiers( report.Tiers.Telemetry = new TelemetryTierResult { Ok = result.Passed, - ExportDetected = result.Passed, - MatchedPatterns = ParseMatchedPatterns(result.Details) + ConsoleExporterActive = exporterDetected || result.Passed }; if (!result.Passed) @@ -424,46 +439,27 @@ private static void MapResultsToTiers( } } break; - } - } - } - - private static List? ParseMatchedPatterns(string? details) - { - if (string.IsNullOrWhiteSpace(details)) - return null; - - // Extract pattern names from details like "Telemetry export evidence found: pattern1, pattern2." - var colonIndex = details.IndexOf(':'); - if (colonIndex < 0) - return null; - - var patternsText = details[(colonIndex + 1)..].Trim().TrimEnd('.'); - var dotIndex = patternsText.IndexOf('.'); - if (dotIndex >= 0) - patternsText = patternsText[..dotIndex]; - - var patterns = patternsText.Split(',', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries); - return patterns.Length > 0 ? patterns.ToList() : null; - } - private static string GetTelemetrySuggestion(TelemetryTierResult telemetry) - { - var patterns = telemetry.MatchedPatterns; - if (patterns is not null) - { - var lower = patterns.Select(p => p.ToLowerInvariant()).ToHashSet(); - if (lower.Contains("missing tenant") || lower.Contains("missing agent id")) - return "configure tenant ID and agent ID for telemetry export"; - if (lower.Contains("nothing exported") || lower.Contains("spans skipped") || lower.Contains("spans filtered out")) - return "check agent identity configuration for telemetry export"; - if (lower.Contains("connection refused") || lower.Contains("unavailable") || lower.Contains("deadline_exceeded")) - return "check OTLP endpoint connectivity"; - if (lower.Contains("unauthenticated") || lower.Contains("permissiondenied")) - return "check OTLP endpoint credentials"; + case BlueprintRegistrationRequirementCheck: + if (result.IsWarning) + { + report.Tiers.Blueprint = new TierResult + { + Ok = true, + Warning = result.ErrorMessage + }; + } + else + { + report.Tiers.Blueprint = new TierResult + { + Ok = result.Passed, + Reason = result.Passed ? null : result.ErrorMessage + }; + } + break; + } } - - return "check agent console logs for telemetry export errors"; } private static (string Description, string Suggestion) GetCodeHealthFailureInfo(ValidationTiers tiers) @@ -676,20 +672,18 @@ private static List BuildDisplayRows(ValidateReport report) if (telOk && telemetry.Warning is not null) { - // Warning state: SDK not detected + // Warning state: console exporter not detected telDesc = telemetry.Warning; - telSuggestion = "configure OpenTelemetry to export traces to Agent365"; + telSuggestion = "configure OpenTelemetry console exporter to output traces"; } else if (telOk) { - telDesc = telemetry.ExportDetected == true - ? "trace export to Agent365 detected" - : "tracing and observability"; + telDesc = "console exporter active, all GenAI operation spans detected"; } else { - telDesc = "trace export failures detected"; - telSuggestion = GetTelemetrySuggestion(telemetry); + telDesc = telemetry.Reason ?? "telemetry check failed"; + telSuggestion = "ensure Agent365Sdk console exporter is enabled with invoke_agent, chat, and execute_tool spans"; } rows.Add(new DisplayRow @@ -712,8 +706,8 @@ private static List BuildDisplayRows(ValidateReport report) } rows.Add(CreateTierRow("Registered", tiers.Blueprint, - "blueprint registration", - null)); + "blueprint registered in Entra ID", + "run 'a365 setup blueprint' to register the blueprint")); rows.Add(CreateTierRow("Visible in MAC", tiers.Mac, "app compliance checks", null)); diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Program.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Program.cs index 06d8e029..26270e2b 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Program.cs +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Program.cs @@ -157,7 +157,7 @@ await Task.WhenAll( var confirmationProvider = serviceProvider.GetRequiredService(); rootCommand.AddCommand(SetupCommand.CreateCommand(setupLogger, configService, executor, backendConfigurator, azureAuthValidator, platformDetector, graphApiService, agentBlueprintService, blueprintLookupService, federatedCredentialService, clientAppValidator, confirmationProvider, armApiService, resolver: bootstrapResolver)); - rootCommand.AddCommand(ValidateCommand.CreateCommand(validateLogger, configService, platformDetector, executor, processService)); + rootCommand.AddCommand(ValidateCommand.CreateCommand(validateLogger, configService, platformDetector, executor, processService, graphApiService)); var manifestTemplateService = serviceProvider.GetRequiredService(); rootCommand.AddCommand(QueryEntraCommand.CreateCommand(queryEntraLogger, configService, executor, graphApiService, agentBlueprintService, resolver: bootstrapResolver)); rootCommand.AddCommand(CleanupCommand.CreateCommand(cleanupLogger, configService, backendConfigurator, executor, agentBlueprintService, confirmationProvider, federatedCredentialService, azureAuthValidator, graphApiService, resolver: bootstrapResolver)); diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/BlueprintRegistrationRequirementCheck.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/BlueprintRegistrationRequirementCheck.cs new file mode 100644 index 00000000..8dd39ae9 --- /dev/null +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/BlueprintRegistrationRequirementCheck.cs @@ -0,0 +1,151 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.Agents.A365.DevTools.Cli.Models; +using Microsoft.Extensions.Logging; + +namespace Microsoft.Agents.A365.DevTools.Cli.Services.Requirements.RequirementChecks; + +/// +/// Validates that the agent blueprint is registered in Microsoft Entra ID. +/// Checks that the blueprint application exists, has a service principal, and +/// (if configured) has an agent registration in the Microsoft Agent Registry. +/// Uses the same Graph API methods as query-entra. +/// +public class BlueprintRegistrationRequirementCheck : RequirementCheck +{ + private readonly GraphApiService _graphApiService; + + public BlueprintRegistrationRequirementCheck(GraphApiService graphApiService) + { + _graphApiService = graphApiService ?? throw new ArgumentNullException(nameof(graphApiService)); + } + + /// + public override string Name => "Blueprint Registration"; + + /// + public override string Description => "Validates that the agent blueprint is registered in Microsoft Entra ID"; + + /// + public override string Category => "Registration"; + + /// + public override async Task CheckAsync( + Agent365Config config, + ILogger logger, + CancellationToken cancellationToken = default) + { + return await ExecuteCheckWithLoggingAsync(config, logger, CheckImplementationAsync, cancellationToken); + } + + private async Task CheckImplementationAsync( + Agent365Config config, + ILogger logger, + CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(config.AgentBlueprintId)) + { + return RequirementCheckResult.Failure( + "Agent blueprint ID not found in configuration", + "Run 'a365 setup blueprint' to create and register a blueprint.", + details: "The agentBlueprintId must be set in a365.generated.config.json before registration can be verified."); + } + + if (string.IsNullOrWhiteSpace(config.TenantId)) + { + return RequirementCheckResult.Failure( + "Tenant ID not found in configuration", + "Run 'a365 setup all' to configure your tenant ID.", + details: "The tenantId must be set in a365.config.json before registration can be verified."); + } + + var blueprintId = config.AgentBlueprintId; + var tenantId = config.TenantId; + + // Check 1: Blueprint application exists in Entra + bool appExists; + try + { + appExists = await _graphApiService.ApplicationExistsByAppIdAsync(tenantId, blueprintId, cancellationToken); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + logger.LogDebug(ex, "Failed to query Entra for blueprint application"); + return RequirementCheckResult.Warning( + "Could not verify blueprint application in Entra ID", + details: $"Graph API query failed: {ex.Message}. Ensure you are authenticated with 'az login'."); + } + + if (!appExists) + { + return RequirementCheckResult.Failure( + $"Blueprint application '{blueprintId}' not found in Entra ID", + "Run 'a365 setup blueprint' to create the blueprint application, or verify the agentBlueprintId in your configuration.", + details: $"No Entra application with appId '{blueprintId}' exists in tenant '{tenantId}'."); + } + + // Check 2: Service principal exists for the blueprint + string? servicePrincipalId; + try + { + servicePrincipalId = await _graphApiService.LookupServicePrincipalByAppIdAsync(tenantId, blueprintId, cancellationToken); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + logger.LogDebug(ex, "Failed to query Entra for blueprint service principal"); + return RequirementCheckResult.Warning( + "Blueprint application exists but could not verify service principal", + details: $"Graph API query failed: {ex.Message}"); + } + + if (string.IsNullOrEmpty(servicePrincipalId)) + { + return RequirementCheckResult.Failure( + $"Service principal not found for blueprint '{blueprintId}'", + "Run 'a365 setup blueprint' to ensure the service principal is provisioned.", + details: $"Application '{blueprintId}' exists but has no service principal in tenant '{tenantId}'."); + } + + // Check 3: Agent registration exists (if registrationId is configured) + if (!string.IsNullOrWhiteSpace(config.AgentRegistrationId)) + { + bool? registrationExists; + try + { + registrationExists = await _graphApiService.AgentRegistrationExistsAsync( + tenantId, config.AgentRegistrationId, cancellationToken); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + logger.LogDebug(ex, "Failed to query agent registration"); + return RequirementCheckResult.Warning( + "Blueprint and service principal exist but could not verify agent registration", + details: $"Agent registry query failed: {ex.Message}"); + } + + if (registrationExists == false) + { + return RequirementCheckResult.Failure( + $"Agent registration '{config.AgentRegistrationId}' not found", + "Run 'a365 setup all' to register the agent, or verify the agentRegistrationId in your configuration.", + details: $"Blueprint '{blueprintId}' and service principal exist, but agent registration " + + $"'{config.AgentRegistrationId}' was not found in the agent registry."); + } + + if (registrationExists == null) + { + return RequirementCheckResult.Warning( + "Blueprint registered but agent registration status is unknown", + details: $"Application and service principal verified. Agent registration '{config.AgentRegistrationId}' " + + "could not be confirmed (insufficient permissions or transient error)."); + } + + return RequirementCheckResult.Success( + details: $"Blueprint '{blueprintId}' registered with service principal and agent registration '{config.AgentRegistrationId}'."); + } + + return RequirementCheckResult.Success( + details: $"Blueprint '{blueprintId}' registered with service principal '{servicePrincipalId}'."); + } +} diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/ConversationRequirementCheck.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/ConversationRequirementCheck.cs index 84cc0874..1fcc9cdb 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/ConversationRequirementCheck.cs +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/ConversationRequirementCheck.cs @@ -7,6 +7,7 @@ using System.Text; using System.Text.Json; using System.Text.Json.Serialization; +using Microsoft.Agents.A365.DevTools.Cli.Constants; using Microsoft.Agents.A365.DevTools.Cli.Models; using Microsoft.Extensions.Logging; @@ -72,15 +73,116 @@ public class ConversationRequirementCheck : RequirementCheck internal const int DefaultPort = 5000; /// - /// Multi-turn conversation prompts used for validation. + /// Default multi-turn conversation prompts used for validation. + /// The middle turn is replaced with a tool-specific prompt when ToolingManifest.json is available. /// - internal static readonly string[] ConversationPrompts = new[] + internal static readonly string[] DefaultConversationPrompts = new[] { "Hello", "What can you do?", "Thanks" }; + /// + /// Fallback prompt when no tools are discovered from ToolingManifest.json. + /// + internal const string FallbackToolPrompt = "What can you do?"; + + /// + /// Maps well-known MCP server names to natural-language questions that would trigger tool usage. + /// Keys are lowercase for case-insensitive matching. + /// + internal static readonly Dictionary KnownToolPrompts = new(StringComparer.OrdinalIgnoreCase) + { + ["mail"] = "Get me my recent emails", + ["calendar"] = "What meetings do I have today?", + ["sharepoint"] = "Get me my recent SharePoint files", + ["onedrive"] = "List my recent OneDrive files", + ["teams"] = "Show me my recent Teams messages", + ["planner"] = "What tasks are assigned to me?", + ["todo"] = "Show me my to-do items", + ["people"] = "Find my recent contacts", + ["search"] = "Search for recent documents", + ["files"] = "List my recent files", + }; + + /// + /// Builds a natural-language prompt that would trigger the agent to invoke a configured tool. + /// Looks up the first MCP server name in for a matching question, + /// falls back to a description-based prompt, then to a generic question. + /// + internal static string BuildToolInvocationPrompt(string projectPath, ILogger logger) + { + var manifestPath = Path.Combine(projectPath, McpConstants.ToolingManifestFileName); + if (!File.Exists(manifestPath)) + { + logger.LogDebug("No ToolingManifest.json found at {Path}, using default prompt", manifestPath); + return FallbackToolPrompt; + } + + try + { + var json = File.ReadAllText(manifestPath); + var manifest = JsonSerializer.Deserialize(json); + + if (manifest?.McpServers is null || manifest.McpServers.Length == 0) + { + logger.LogDebug("ToolingManifest.json has no MCP servers, using default prompt"); + return FallbackToolPrompt; + } + + var firstTool = manifest.McpServers[0]; + var toolName = firstTool.McpServerName; + + if (string.IsNullOrWhiteSpace(toolName)) + { + logger.LogDebug("First MCP server has no name, using default prompt"); + return FallbackToolPrompt; + } + + // Check for a well-known tool keyword in the server name (e.g. "SharePoint" matches "M365SharePoint") + var matchedPrompt = KnownToolPrompts + .FirstOrDefault(kvp => toolName.Contains(kvp.Key, StringComparison.OrdinalIgnoreCase)); + + if (matchedPrompt.Value is not null) + { + logger.LogDebug("Using known tool prompt for MCP server: {ToolName} (matched keyword: {Keyword})", toolName, matchedPrompt.Key); + return matchedPrompt.Value; + } + + // Fall back to a description-based prompt if available + if (!string.IsNullOrWhiteSpace(firstTool.Description)) + { + logger.LogDebug("Using description-based prompt for MCP server: {ToolName}", toolName); + return $"Help me with {firstTool.Description.TrimEnd('.')}"; + } + + // Generic prompt referencing the tool name + logger.LogDebug("Using generic prompt for MCP server: {ToolName}", toolName); + return $"Help me with {toolName}"; + } + catch (JsonException ex) + { + logger.LogDebug(ex, "Failed to parse ToolingManifest.json for tool discovery, using default prompt"); + return FallbackToolPrompt; + } + } + + /// + /// Builds conversation prompts, replacing the middle turn with a tool invocation prompt + /// when ToolingManifest.json is present and contains configured tools. + /// + internal static string[] BuildConversationPrompts(string projectPath, ILogger logger) + { + var toolPrompt = BuildToolInvocationPrompt(projectPath, logger); + return new[] + { + DefaultConversationPrompts[0], + toolPrompt, + DefaultConversationPrompts[2] + }; + } + public ConversationRequirementCheck( PlatformDetector platformDetector, IProcessService processService, @@ -147,7 +249,8 @@ private async Task CheckImplementationAsync( platform, port, projectPath); var startInfo = BuildProcessStartInfo(platform, projectPath, port); - return await SpawnAndConverse(startInfo, healthUrl, messagesUrl, conversationId, platform, port, logger, cancellationToken); + var prompts = BuildConversationPrompts(projectPath, logger); + return await SpawnAndConverse(startInfo, healthUrl, messagesUrl, conversationId, platform, port, prompts, logger, cancellationToken); } private async Task SpawnAndConverse( @@ -157,6 +260,7 @@ private async Task SpawnAndConverse( string conversationId, ProjectPlatform platform, int port, + string[] conversationPrompts, ILogger logger, CancellationToken cancellationToken) { @@ -282,11 +386,11 @@ private async Task SpawnAndConverse( await Task.Delay(PostHealthWarmupDelay, cancellationToken); // Phase 2: Multi-turn conversation - conversationLogWriter?.WriteLine($"\n[Phase 2] Starting {ConversationPrompts.Length}-turn conversation..."); + conversationLogWriter?.WriteLine($"\n[Phase 2] Starting {conversationPrompts.Length}-turn conversation..."); var turns = new List(); var allOk = true; - for (int i = 0; i < ConversationPrompts.Length; i++) + for (int i = 0; i < conversationPrompts.Length; i++) { if (process.HasExited) { @@ -307,7 +411,7 @@ private async Task SpawnAndConverse( }; } - var turnResult = await SendTurnWithRetryAsync(messagesUrl, conversationId, ConversationPrompts[i], i, port, receiver, logger, conversationLogWriter, cancellationToken); + var turnResult = await SendTurnWithRetryAsync(messagesUrl, conversationId, conversationPrompts[i], i, port, receiver, logger, conversationLogWriter, cancellationToken); turns.Add(turnResult); LogTurn(conversationLogWriter, i + 1, turnResult); @@ -678,17 +782,6 @@ private ProcessStartInfo BuildProcessStartInfo(ProjectPlatform platform, string CreateNoWindow = true }; - // Disable auth so the bot accepts unauthenticated local requests. - // BYPASS_AUTH: used by .NET Agent SDK - startInfo.EnvironmentVariables["BYPASS_AUTH"] = "true"; - // Clear credentials that trigger JWT middleware in Python/Node SDKs. - // When these are absent, agents run in anonymous mode. - startInfo.EnvironmentVariables["CLIENT_ID"] = ""; - startInfo.EnvironmentVariables["CLIENT_SECRET"] = ""; - startInfo.EnvironmentVariables["TENANT_ID"] = ""; - // Also clear .NET Bot Framework equivalents - startInfo.EnvironmentVariables["MicrosoftAppId"] = ""; - startInfo.EnvironmentVariables["MicrosoftAppPassword"] = ""; switch (platform) { diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/TelemetryRequirementCheck.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/TelemetryRequirementCheck.cs index c41585f7..c9a8b3cc 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/TelemetryRequirementCheck.cs +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/TelemetryRequirementCheck.cs @@ -7,8 +7,10 @@ namespace Microsoft.Agents.A365.DevTools.Cli.Services.Requirements.RequirementChecks; /// -/// Validates that the agent is exporting telemetry traces by analyzing the agent's -/// console output log file captured during the conversation validation step. +/// Validates that the agent is producing telemetry traces via the console exporter +/// by analyzing the agent's console output log file captured during the conversation step. +/// For local validation, this checks that the console exporter is active and that the +/// required GenAI semantic convention spans (invoke_agent, chat, execute_tool) appear. /// public class TelemetryRequirementCheck : RequirementCheck { @@ -17,70 +19,85 @@ public class TelemetryRequirementCheck : RequirementCheck /// /// Maximum number of telemetry-relevant log lines to analyze. /// - internal const int MaxTelemetryLines = 100; + internal const int MaxTelemetryLines = 200; /// - /// Keywords that identify a log line as telemetry-related. + /// Keywords that identify a log line as telemetry-related (console exporter output). /// A line must contain at least one of these to be considered relevant. /// internal static readonly string[] TelemetryContextKeywords = new[] { + // Console exporter span output fields + "traceid:", + "spanid:", + "tracestate:", + "parentspancontext:", + // OpenTelemetry SDK indicators "opentelemetry", "otel", - "otlp", - "tracer", - "tracerprovider", + "telemetry.sdk.name", + "telemetry.sdk.version", + "consoleexporter", + "consolespanexporter", + // .NET Activity-based console exporter + "activity.traceid", + "activity.displayname", + "activity.spanid", "activitysource", - "span", - "exporter", + // GenAI semantic convention attributes + "gen_ai.operation.name", + "gen_ai.request.model", + "gen_ai.usage.input_tokens", + "gen_ai.agent.name", + // Agent365 observability SDK + "a365observabilitysdk", "agent365observability", - "agent365.observability", - "agent365exporter", - "otelwrite", - "batchexportprocessor", - "traces exported", - "export completed", - "export succeeded" + "agent365.observability" }; /// - /// Patterns that indicate successful trace export when found in telemetry-relevant lines. + /// Instrumentation scope name to exclude from validation. + /// Spans from this scope are auto-generated by the agents-telemetry package + /// and should not count toward GenAI operation checks. /// - internal static readonly string[] SuccessPatterns = new[] + internal const string IgnoredInstrumentationScope = "@microsoft/agents-telemetry"; + + /// + /// Required GenAI semantic convention operation names that must ALL appear in traces. + /// These correspond to gen_ai.operation.name values for agent orchestration spans. + /// + internal static readonly string[] RequiredGenAiSpans = new[] + { + "invoke_agent", + "chat", + "execute_tool" + }; + + /// + /// Operation names that must have a non-empty parentId to verify proper trace hierarchy. + /// These are child spans — execute_tool and chat should be linked to a parent invoke_agent span. + /// + internal static readonly string[] ChildSpanOperations = new[] { - "export completed", - "export succeeded", - "successfully exported", - "traces exported", - "span exported", - "tracerprovider built", - "tracerprovider started", - "otlpexporter", - "otlptraceexporter" + "chat", + "execute_tool" }; /// - /// Patterns that indicate trace export failure when found in telemetry-relevant lines. + /// OTel semantic convention resource attributes that should be present in span output. /// - internal static readonly string[] FailurePatterns = new[] + internal static readonly string[] RequiredResourceAttributes = new[] { - "export failed", - "exporter error", - "connection refused", - "unavailable", - "deadline_exceeded", - "unauthenticated", - "permissiondenied", - "failed to export", - "exporter threw", - "dropped spans", - "nothing exported", - "spans skipped", - "spans filtered out", - "missing tenant", - "missing agent id" + "telemetry.sdk.name", + "telemetry.sdk.version", + "service.name" }; + /// + /// Keywords for detecting scope version in span output. + /// + internal const string ScopeVersionKey = "version"; + public TelemetryRequirementCheck(string? agentConsoleLogPath) { _agentConsoleLogPath = agentConsoleLogPath; @@ -131,115 +148,338 @@ private Task CheckImplementationAsync( details: $"Failed to read {_agentConsoleLogPath}: {ex.Message}")); } - var relevantLines = FilterTelemetryLines(logLines); + // Split log into span blocks and find Agent365Sdk spans + var spanBlocks = SplitIntoSpanBlocks(logLines); + if (spanBlocks.Count == 0) + { + return Task.FromResult(RequirementCheckResult.Failure( + "No console exporter span output detected in agent logs", + "Enable the OpenTelemetry console exporter in your agent so that spans are written to stdout.", + details: "Expected to find span blocks containing traceId in agent console logs.")); + } - if (relevantLines.Count == 0) + // Include all span blocks except those from the ignored instrumentation scope + var relevantBlocks = spanBlocks + .Where(block => !block.Any(line => + line.IndexOf(IgnoredInstrumentationScope, StringComparison.OrdinalIgnoreCase) >= 0)) + .ToList(); + + if (relevantBlocks.Count == 0) { + var ignoredCount = spanBlocks.Count - relevantBlocks.Count; return Task.FromResult(RequirementCheckResult.Failure( - "No telemetry-related output detected in agent console logs", - "Configure OpenTelemetry in your agent to export traces to Agent365.", - details: "No OpenTelemetry, OTLP, or trace export evidence found in agent console output.")); + "No relevant instrumentation scope spans found", + "Ensure your agent instruments GenAI spans with a named instrumentation scope.", + details: $"Found {spanBlocks.Count} span block(s) in console output, " + + $"but all were from ignored scope '{IgnoredInstrumentationScope}'.")); } - logger.LogDebug("Found {Count} telemetry-relevant log lines", relevantLines.Count); + logger.LogDebug("Found {Count} relevant span blocks (excluded {Ignored} from '{Scope}')", + relevantBlocks.Count, spanBlocks.Count - relevantBlocks.Count, IgnoredInstrumentationScope); - var matchedSuccessPatterns = FindMatchingPatterns(relevantLines, SuccessPatterns); - var matchedFailurePatterns = FindMatchingPatterns(relevantLines, FailurePatterns); + // Extract gen_ai.operation.name values from relevant spans + var foundOperations = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var block in relevantBlocks) + { + foreach (var op in ExtractOperationNames(block)) + { + foundOperations.Add(op); + } + } + + var missingOperations = RequiredGenAiSpans + .Where(op => !foundOperations.Contains(op)) + .ToList(); - // Failure takes precedence over success - if (matchedFailurePatterns.Count > 0) + if (missingOperations.Count > 0) { - var failureDetails = string.Join(", ", matchedFailurePatterns); - var guidance = GetFailureGuidance(matchedFailurePatterns); + var foundList = foundOperations.Count > 0 ? string.Join(", ", foundOperations) : "none"; return Task.FromResult(RequirementCheckResult.Failure( - $"Telemetry export failures detected: {failureDetails}", - guidance, - details: $"Failure patterns found: {failureDetails}. " + - $"Success patterns found: {(matchedSuccessPatterns.Count > 0 ? string.Join(", ", matchedSuccessPatterns) : "none")}. " + - $"Analyzed {relevantLines.Count} telemetry-relevant log lines.")); + $"Missing required GenAI operation spans: {string.Join(", ", missingOperations)}", + "Ensure your agent instruments invoke_agent, chat, and execute_tool operations with OpenTelemetry.", + details: $"Found {relevantBlocks.Count} relevant span(s). " + + $"Found operations: {foundList}. " + + $"Missing operations: {string.Join(", ", missingOperations)}. " + + $"All three gen_ai.operation.name values (invoke_agent, chat, execute_tool) are required.")); + } + + // Additional OTel semantic convention checks (collected as warnings) + var warnings = new List(); + + // Check scope version: instrumentationScope should include a version + var hasScopeVersion = relevantBlocks.Any(block => + HasInstrumentationScopeVersion(block)); + if (!hasScopeVersion) + { + warnings.Add("instrumentationScope is missing 'version' — OTel semantic conventions recommend including scope version"); + } + + // Check parent links: execute_tool and chat spans must have non-empty parentId + var childOpsWithoutParent = GetChildSpansMissingParent(relevantBlocks); + if (childOpsWithoutParent.Count > 0) + { + warnings.Add($"child spans missing parentId: {string.Join(", ", childOpsWithoutParent)} — " + + "these spans should be children of an invoke_agent span"); } - if (matchedSuccessPatterns.Count > 0) + // Check resource attributes: telemetry.sdk.name, telemetry.sdk.version, service.name + var missingResources = GetMissingResourceAttributes(logLines); + if (missingResources.Count > 0) { - var successDetails = string.Join(", ", matchedSuccessPatterns); - return Task.FromResult(RequirementCheckResult.Success( - details: $"Telemetry export evidence found: {successDetails}. " + - $"Analyzed {relevantLines.Count} telemetry-relevant log lines.")); + warnings.Add($"missing OTel resource attributes: {string.Join(", ", missingResources)} — " + + "ensure your SDK resource is configured per OTel semantic conventions"); } - // Telemetry lines exist but no clear success or failure — treat as failure - return Task.FromResult(RequirementCheckResult.Failure( - "Telemetry SDK detected but no trace export evidence found", - "Ensure traces are being exported to the Agent365 OTLP endpoint.", - details: $"Found {relevantLines.Count} telemetry-related log lines but could not " + - "confirm successful trace export.")); + var detailsBuilder = $"Console exporter active with {relevantBlocks.Count} relevant span(s). " + + $"All required GenAI operation spans detected: {string.Join(", ", RequiredGenAiSpans)}."; + + if (warnings.Count > 0) + { + detailsBuilder += $" Warnings: {string.Join("; ", warnings)}"; + return Task.FromResult(RequirementCheckResult.Warning( + "Telemetry spans detected but with OTel semantic convention gaps", + details: detailsBuilder)); + } + + return Task.FromResult(RequirementCheckResult.Success(details: detailsBuilder)); } /// - /// Filters log lines to only those containing telemetry-related keywords. + /// Splits console exporter output into span blocks. + /// Each span block is a group of lines belonging to one span. + /// Blocks are delimited by lines starting with '{' (span object boundary). + /// Falls back to traceId-based splitting if no '{' delimiters are found. /// - internal static List FilterTelemetryLines(string[] logLines) + internal static List> SplitIntoSpanBlocks(string[] logLines) { - var result = new List(); + // Try brace-delimited first (standard console exporter format) + var braceBlocks = SplitOnBraces(logLines); + if (braceBlocks.Count > 0) + return braceBlocks; + + // Fallback: split on traceId lines + return SplitOnTraceId(logLines); + } + + private static List> SplitOnBraces(string[] logLines) + { + var blocks = new List>(); + List? currentBlock = null; + foreach (var line in logLines) { if (string.IsNullOrWhiteSpace(line)) continue; - var lower = line.ToLowerInvariant(); - foreach (var keyword in TelemetryContextKeywords) + var trimmed = line.TrimStart(); + if (trimmed == "{") { - if (lower.Contains(keyword)) - { - result.Add(line); - if (result.Count >= MaxTelemetryLines) - return result; - break; - } + currentBlock = new List(); + blocks.Add(currentBlock); + } + + currentBlock?.Add(line); + } + + // Only return if we found blocks that look like spans (contain traceId) + var spanBlocks = blocks.Where(b => b.Any(l => + l.TrimStart().StartsWith("traceId:", StringComparison.OrdinalIgnoreCase) || + l.TrimStart().StartsWith("\"traceId\":", StringComparison.OrdinalIgnoreCase))).ToList(); + + return spanBlocks; + } + + private static List> SplitOnTraceId(string[] logLines) + { + var blocks = new List>(); + List? currentBlock = null; + var pendingLines = new List(); + + foreach (var line in logLines) + { + if (string.IsNullOrWhiteSpace(line)) + continue; + + var trimmed = line.TrimStart(); + bool isTraceIdLine = trimmed.StartsWith("traceId:", StringComparison.OrdinalIgnoreCase) + || trimmed.StartsWith("\"traceId\":", StringComparison.OrdinalIgnoreCase) + || trimmed.StartsWith("'traceId':", StringComparison.OrdinalIgnoreCase) + || trimmed.StartsWith("Activity.TraceId:", StringComparison.OrdinalIgnoreCase); + + if (isTraceIdLine) + { + // Include any pending lines (e.g., instrumentationScope before traceId) + currentBlock = new List(pendingLines) { line }; + blocks.Add(currentBlock); + pendingLines.Clear(); + } + else if (currentBlock is not null) + { + currentBlock.Add(line); + } + else + { + // Lines before the first traceId — may include instrumentationScope + pendingLines.Add(line); } } - return result; + return blocks; } /// - /// Finds which patterns from the given set appear in the relevant log lines. + /// Extracts gen_ai.operation.name values from a span block. /// - internal static List FindMatchingPatterns(List relevantLines, string[] patterns) + internal static List ExtractOperationNames(List spanBlock) { - var matched = new HashSet(StringComparer.OrdinalIgnoreCase); - foreach (var line in relevantLines) + var operations = new List(); + + foreach (var line in spanBlock) { - var lower = line.ToLowerInvariant(); - foreach (var pattern in patterns) + var idx = line.IndexOf("gen_ai.operation.name", StringComparison.OrdinalIgnoreCase); + if (idx < 0) + continue; + + // Extract the value after the key — supports formats like: + // 'gen_ai.operation.name': 'chat' + // "gen_ai.operation.name": "invoke_agent" + // gen_ai.operation.name=execute_tool + var afterKey = line.Substring(idx + "gen_ai.operation.name".Length).TrimStart(); + + // Skip separator characters (: = ' ") + var valueStart = 0; + while (valueStart < afterKey.Length && (afterKey[valueStart] == ':' || afterKey[valueStart] == '=' + || afterKey[valueStart] == '\'' || afterKey[valueStart] == '"' || afterKey[valueStart] == ' ')) { - if (lower.Contains(pattern) && matched.Add(pattern)) + valueStart++; + } + + if (valueStart >= afterKey.Length) + continue; + + // Read until the next delimiter + var valueEnd = valueStart; + while (valueEnd < afterKey.Length && afterKey[valueEnd] != '\'' + && afterKey[valueEnd] != '"' && afterKey[valueEnd] != ',' + && afterKey[valueEnd] != ' ' && afterKey[valueEnd] != '}') + { + valueEnd++; + } + + if (valueEnd > valueStart) + { + operations.Add(afterKey.Substring(valueStart, valueEnd - valueStart)); + } + } + + return operations; + } + + /// + /// Checks whether a span block's instrumentationScope section contains a version field. + /// In the console exporter output, this appears near the scope name, e.g.: + /// instrumentationScope: { name: 'Agent365Sdk', version: '1.0.0' } + /// + internal static bool HasInstrumentationScopeVersion(List spanBlock) + { + bool inScope = false; + foreach (var line in spanBlock) + { + var trimmed = line.TrimStart(); + + if (trimmed.IndexOf("instrumentationScope", StringComparison.OrdinalIgnoreCase) >= 0) + { + inScope = true; + // Check if version is on the same line + if (trimmed.IndexOf(ScopeVersionKey, StringComparison.OrdinalIgnoreCase) >= 0) + return true; + continue; + } + + if (inScope) + { + // Scope section ends at the next top-level field (no leading whitespace or a new section) + if (!trimmed.StartsWith("'") && !trimmed.StartsWith("\"") && !trimmed.StartsWith("}") && + trimmed.Length > 0 && !char.IsWhiteSpace(line[0]) && + !trimmed.StartsWith("name", StringComparison.OrdinalIgnoreCase) && + !trimmed.StartsWith("version", StringComparison.OrdinalIgnoreCase)) { - // Found a new pattern match + inScope = false; + continue; } + + if (trimmed.IndexOf(ScopeVersionKey, StringComparison.OrdinalIgnoreCase) >= 0) + return true; } } - return matched.ToList(); + return false; } - private static string GetFailureGuidance(List failurePatterns) + /// + /// Identifies child span operations (chat, execute_tool) that are missing a parentId/parentSpanId. + /// Returns the list of operation names that should have parent links but don't. + /// + internal static List GetChildSpansMissingParent(List> spanBlocks) { - var lower = failurePatterns.Select(p => p.ToLowerInvariant()).ToHashSet(); + var missingParent = new HashSet(StringComparer.OrdinalIgnoreCase); + + foreach (var block in spanBlocks) + { + var ops = ExtractOperationNames(block); + var isChildSpan = ops.Any(op => ChildSpanOperations.Contains(op, StringComparer.OrdinalIgnoreCase)); + if (!isChildSpan) + continue; - if (lower.Contains("missing tenant") || lower.Contains("missing agent id") || - lower.Contains("nothing exported") || lower.Contains("spans skipped")) - return "Configure tenant ID and agent ID in your agent's observability settings. " + - "The Agent365 exporter requires both to export spans."; + var hasParent = block.Any(line => + { + var trimmed = line.TrimStart(); + return (trimmed.StartsWith("parentId:", StringComparison.OrdinalIgnoreCase) || + trimmed.StartsWith("parentSpanId:", StringComparison.OrdinalIgnoreCase) || + trimmed.StartsWith("\"parentId\":", StringComparison.OrdinalIgnoreCase) || + trimmed.StartsWith("'parentId':", StringComparison.OrdinalIgnoreCase) || + trimmed.StartsWith("parentSpanContext:", StringComparison.OrdinalIgnoreCase)) && + HasNonEmptyValue(trimmed); + }); + + if (!hasParent) + { + foreach (var op in ops.Where(o => ChildSpanOperations.Contains(o, StringComparer.OrdinalIgnoreCase))) + { + missingParent.Add(op); + } + } + } - if (lower.Contains("connection refused") || lower.Contains("unavailable") || lower.Contains("deadline_exceeded")) - return "Check that the OTLP endpoint is reachable from the agent. " + - "Verify the endpoint URL and network connectivity."; + return missingParent.OrderBy(o => o).ToList(); + } - if (lower.Contains("unauthenticated") || lower.Contains("permissiondenied")) - return "Check OTLP endpoint credentials. " + - "Ensure the agent has valid authentication for the observability endpoint."; + /// + /// Checks that a line like "parentId: 'abc123'" has a non-empty value after the key. + /// Returns false for lines like "parentId: undefined" or "parentId: ''". + /// + internal static bool HasNonEmptyValue(string line) + { + var colonIdx = line.IndexOf(':'); + if (colonIdx < 0) + return false; + + var value = line.Substring(colonIdx + 1).Trim().Trim('\'', '"', ' '); + return !string.IsNullOrWhiteSpace(value) && + !value.Equals("undefined", StringComparison.OrdinalIgnoreCase) && + !value.Equals("null", StringComparison.OrdinalIgnoreCase); + } - return "Check the agent console logs for telemetry export error details."; + /// + /// Checks for missing OTel resource semantic convention attributes across all log lines. + /// Looks for telemetry.sdk.name, telemetry.sdk.version, and service.name + /// which should appear in the resource section of console exporter output. + /// + internal static List GetMissingResourceAttributes(string[] logLines) + { + var allText = string.Join("\n", logLines); + return RequiredResourceAttributes + .Where(attr => allText.IndexOf(attr, StringComparison.OrdinalIgnoreCase) < 0) + .ToList(); } } diff --git a/src/Microsoft.Agents.A365.DevTools.Validation/ValidateReport.cs b/src/Microsoft.Agents.A365.DevTools.Validation/ValidateReport.cs index bfe4348c..8ea60bbd 100644 --- a/src/Microsoft.Agents.A365.DevTools.Validation/ValidateReport.cs +++ b/src/Microsoft.Agents.A365.DevTools.Validation/ValidateReport.cs @@ -201,17 +201,37 @@ public sealed class ConversationTierResult : TierResult /// public sealed class TelemetryTierResult : TierResult { - [JsonPropertyName("exportDetected")] + [JsonPropertyName("consoleExporterActive")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public bool? ExportDetected { get; set; } + public bool? ConsoleExporterActive { get; set; } - [JsonPropertyName("matchedPatterns")] + [JsonPropertyName("foundOperations")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public List? MatchedPatterns { get; set; } + public List? FoundOperations { get; set; } - [JsonPropertyName("analyzedLineCount")] + [JsonPropertyName("missingOperations")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public int? AnalyzedLineCount { get; set; } + public List? MissingOperations { get; set; } + + [JsonPropertyName("scopeVersionPresent")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public bool? ScopeVersionPresent { get; set; } + + [JsonPropertyName("parentLinksValid")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public bool? ParentLinksValid { get; set; } + + [JsonPropertyName("childSpansMissingParent")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public List? ChildSpansMissingParent { get; set; } + + [JsonPropertyName("resourceAttributesPresent")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public bool? ResourceAttributesPresent { get; set; } + + [JsonPropertyName("missingResourceAttributes")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public List? MissingResourceAttributes { get; set; } } /// diff --git a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/BlueprintRegistrationRequirementCheckTests.cs b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/BlueprintRegistrationRequirementCheckTests.cs new file mode 100644 index 00000000..ecdb4872 --- /dev/null +++ b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/BlueprintRegistrationRequirementCheckTests.cs @@ -0,0 +1,295 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using FluentAssertions; +using Microsoft.Agents.A365.DevTools.Cli.Models; +using Microsoft.Agents.A365.DevTools.Cli.Services; +using Microsoft.Agents.A365.DevTools.Cli.Services.Requirements.RequirementChecks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using NSubstitute; +using NSubstitute.ExceptionExtensions; +using Xunit; + +namespace Microsoft.Agents.A365.DevTools.Cli.Tests.Services.Requirements; + +public class BlueprintRegistrationRequirementCheckTests +{ + private readonly GraphApiService _mockGraphApiService; + private readonly ILogger _logger = NullLoggerFactory.Instance.CreateLogger("test"); + + private const string TestTenantId = "00000000-0000-0000-0000-000000000001"; + private const string TestBlueprintId = "00000000-0000-0000-0000-000000000002"; + private const string TestServicePrincipalId = "00000000-0000-0000-0000-000000000003"; + private const string TestRegistrationId = "reg-12345"; + + public BlueprintRegistrationRequirementCheckTests() + { + _mockGraphApiService = Substitute.ForPartsOf(); + } + + // --- Metadata --- + + [Fact] + public void Name_ReturnsBlueprintRegistration() + { + var check = new BlueprintRegistrationRequirementCheck(_mockGraphApiService); + check.Name.Should().Be("Blueprint Registration"); + } + + [Fact] + public void Category_ReturnsRegistration() + { + var check = new BlueprintRegistrationRequirementCheck(_mockGraphApiService); + check.Category.Should().Be("Registration"); + } + + // --- Missing config --- + + [Fact] + public async Task CheckAsync_NoBlueprintId_ReturnsFail() + { + var config = new Agent365Config { TenantId = TestTenantId }; + var check = new BlueprintRegistrationRequirementCheck(_mockGraphApiService); + + var result = await check.CheckAsync(config, _logger); + + result.Passed.Should().BeFalse(because: "blueprintId is required"); + result.ErrorMessage.Should().Contain("blueprint ID not found", + because: "error should indicate missing blueprint ID"); + } + + [Fact] + public async Task CheckAsync_NoTenantId_ReturnsFail() + { + var config = new Agent365Config { AgentBlueprintId = TestBlueprintId }; + var check = new BlueprintRegistrationRequirementCheck(_mockGraphApiService); + + var result = await check.CheckAsync(config, _logger); + + result.Passed.Should().BeFalse(because: "tenantId is required"); + result.ErrorMessage.Should().Contain("Tenant ID not found", + because: "error should indicate missing tenant ID"); + } + + // --- Application check --- + + [Fact] + public async Task CheckAsync_AppNotFound_ReturnsFail() + { + var config = new Agent365Config + { + TenantId = TestTenantId, + AgentBlueprintId = TestBlueprintId + }; + + _mockGraphApiService.ApplicationExistsByAppIdAsync(TestTenantId, TestBlueprintId, Arg.Any()) + .Returns(false); + + var check = new BlueprintRegistrationRequirementCheck(_mockGraphApiService); + var result = await check.CheckAsync(config, _logger); + + result.Passed.Should().BeFalse(because: "application does not exist in Entra"); + result.ErrorMessage.Should().Contain("not found in Entra ID", + because: "error should indicate app not found"); + } + + [Fact] + public async Task CheckAsync_AppCheckThrows_ReturnsWarning() + { + var config = new Agent365Config + { + TenantId = TestTenantId, + AgentBlueprintId = TestBlueprintId + }; + + _mockGraphApiService.ApplicationExistsByAppIdAsync(TestTenantId, TestBlueprintId, Arg.Any()) + .ThrowsAsync(new HttpRequestException("Network error")); + + var check = new BlueprintRegistrationRequirementCheck(_mockGraphApiService); + var result = await check.CheckAsync(config, _logger); + + result.Passed.Should().BeTrue(because: "auth/network errors are warnings, not failures"); + result.IsWarning.Should().BeTrue(because: "Graph API errors should produce a warning"); + } + + // --- Service principal check --- + + [Fact] + public async Task CheckAsync_AppExistsButNoServicePrincipal_ReturnsFail() + { + var config = new Agent365Config + { + TenantId = TestTenantId, + AgentBlueprintId = TestBlueprintId + }; + + _mockGraphApiService.ApplicationExistsByAppIdAsync(TestTenantId, TestBlueprintId, Arg.Any()) + .Returns(true); + _mockGraphApiService.LookupServicePrincipalByAppIdAsync(TestTenantId, TestBlueprintId, Arg.Any(), Arg.Any?>()) + .Returns((string?)null); + + var check = new BlueprintRegistrationRequirementCheck(_mockGraphApiService); + var result = await check.CheckAsync(config, _logger); + + result.Passed.Should().BeFalse(because: "service principal must exist"); + result.ErrorMessage.Should().Contain("Service principal not found", + because: "error should indicate missing service principal"); + } + + [Fact] + public async Task CheckAsync_ServicePrincipalCheckThrows_ReturnsWarning() + { + var config = new Agent365Config + { + TenantId = TestTenantId, + AgentBlueprintId = TestBlueprintId + }; + + _mockGraphApiService.ApplicationExistsByAppIdAsync(TestTenantId, TestBlueprintId, Arg.Any()) + .Returns(true); + _mockGraphApiService.LookupServicePrincipalByAppIdAsync(TestTenantId, TestBlueprintId, Arg.Any(), Arg.Any?>()) + .ThrowsAsync(new HttpRequestException("Token expired")); + + var check = new BlueprintRegistrationRequirementCheck(_mockGraphApiService); + var result = await check.CheckAsync(config, _logger); + + result.Passed.Should().BeTrue(because: "network errors on service principal check are warnings"); + result.IsWarning.Should().BeTrue(); + } + + // --- Full success without registration --- + + [Fact] + public async Task CheckAsync_AppAndServicePrincipalExist_NoRegistrationId_ReturnsSuccess() + { + var config = new Agent365Config + { + TenantId = TestTenantId, + AgentBlueprintId = TestBlueprintId + }; + + _mockGraphApiService.ApplicationExistsByAppIdAsync(TestTenantId, TestBlueprintId, Arg.Any()) + .Returns(true); + _mockGraphApiService.LookupServicePrincipalByAppIdAsync(TestTenantId, TestBlueprintId, Arg.Any(), Arg.Any?>()) + .Returns(TestServicePrincipalId); + + var check = new BlueprintRegistrationRequirementCheck(_mockGraphApiService); + var result = await check.CheckAsync(config, _logger); + + result.Passed.Should().BeTrue(because: "app and service principal exist"); + result.IsWarning.Should().BeFalse(); + result.Details.Should().Contain(TestBlueprintId, + because: "details should mention the blueprint ID"); + } + + // --- Agent registration checks --- + + [Fact] + public async Task CheckAsync_RegistrationExists_ReturnsSuccess() + { + var config = new Agent365Config + { + TenantId = TestTenantId, + AgentBlueprintId = TestBlueprintId, + AgentRegistrationId = TestRegistrationId + }; + + _mockGraphApiService.ApplicationExistsByAppIdAsync(TestTenantId, TestBlueprintId, Arg.Any()) + .Returns(true); + _mockGraphApiService.LookupServicePrincipalByAppIdAsync(TestTenantId, TestBlueprintId, Arg.Any(), Arg.Any?>()) + .Returns(TestServicePrincipalId); + _mockGraphApiService.AgentRegistrationExistsAsync(TestTenantId, TestRegistrationId, Arg.Any()) + .Returns(true); + + var check = new BlueprintRegistrationRequirementCheck(_mockGraphApiService); + var result = await check.CheckAsync(config, _logger); + + result.Passed.Should().BeTrue(because: "all registration checks passed"); + result.IsWarning.Should().BeFalse(); + result.Details.Should().Contain(TestRegistrationId, + because: "details should mention the registration ID"); + } + + [Fact] + public async Task CheckAsync_RegistrationNotFound_ReturnsFail() + { + var config = new Agent365Config + { + TenantId = TestTenantId, + AgentBlueprintId = TestBlueprintId, + AgentRegistrationId = TestRegistrationId + }; + + _mockGraphApiService.ApplicationExistsByAppIdAsync(TestTenantId, TestBlueprintId, Arg.Any()) + .Returns(true); + _mockGraphApiService.LookupServicePrincipalByAppIdAsync(TestTenantId, TestBlueprintId, Arg.Any(), Arg.Any?>()) + .Returns(TestServicePrincipalId); + _mockGraphApiService.AgentRegistrationExistsAsync(TestTenantId, TestRegistrationId, Arg.Any()) + .Returns(false); + + var check = new BlueprintRegistrationRequirementCheck(_mockGraphApiService); + var result = await check.CheckAsync(config, _logger); + + result.Passed.Should().BeFalse(because: "agent registration does not exist"); + result.ErrorMessage.Should().Contain("not found", + because: "error should indicate registration not found"); + } + + [Fact] + public async Task CheckAsync_RegistrationUnknown_ReturnsWarning() + { + var config = new Agent365Config + { + TenantId = TestTenantId, + AgentBlueprintId = TestBlueprintId, + AgentRegistrationId = TestRegistrationId + }; + + _mockGraphApiService.ApplicationExistsByAppIdAsync(TestTenantId, TestBlueprintId, Arg.Any()) + .Returns(true); + _mockGraphApiService.LookupServicePrincipalByAppIdAsync(TestTenantId, TestBlueprintId, Arg.Any(), Arg.Any?>()) + .Returns(TestServicePrincipalId); + _mockGraphApiService.AgentRegistrationExistsAsync(TestTenantId, TestRegistrationId, Arg.Any()) + .Returns((bool?)null); + + var check = new BlueprintRegistrationRequirementCheck(_mockGraphApiService); + var result = await check.CheckAsync(config, _logger); + + result.Passed.Should().BeTrue(because: "unknown registration status is a warning, not a failure"); + result.IsWarning.Should().BeTrue(); + } + + [Fact] + public async Task CheckAsync_RegistrationCheckThrows_ReturnsWarning() + { + var config = new Agent365Config + { + TenantId = TestTenantId, + AgentBlueprintId = TestBlueprintId, + AgentRegistrationId = TestRegistrationId + }; + + _mockGraphApiService.ApplicationExistsByAppIdAsync(TestTenantId, TestBlueprintId, Arg.Any()) + .Returns(true); + _mockGraphApiService.LookupServicePrincipalByAppIdAsync(TestTenantId, TestBlueprintId, Arg.Any(), Arg.Any?>()) + .Returns(TestServicePrincipalId); + _mockGraphApiService.AgentRegistrationExistsAsync(TestTenantId, TestRegistrationId, Arg.Any()) + .ThrowsAsync(new HttpRequestException("Forbidden")); + + var check = new BlueprintRegistrationRequirementCheck(_mockGraphApiService); + var result = await check.CheckAsync(config, _logger); + + result.Passed.Should().BeTrue(because: "registration check errors are warnings"); + result.IsWarning.Should().BeTrue(); + } + + // --- Constructor validation --- + + [Fact] + public void Constructor_NullGraphApiService_Throws() + { + var act = () => new BlueprintRegistrationRequirementCheck(null!); + act.Should().Throw(); + } +} diff --git a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/ConversationRequirementCheckTests.cs b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/ConversationRequirementCheckTests.cs index 1b9e36fd..8d997461 100644 --- a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/ConversationRequirementCheckTests.cs +++ b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/ConversationRequirementCheckTests.cs @@ -3,6 +3,8 @@ using System.Diagnostics; using System.Net; +using System.Runtime.InteropServices; +using System.Text.Json; using FluentAssertions; using Microsoft.Agents.A365.DevTools.Cli.Models; using Microsoft.Agents.A365.DevTools.Cli.Services; @@ -254,8 +256,16 @@ public async Task CheckAsync_NodeJsProject_UsesNpmStart() await check.CheckAsync(config, _logger); - _processService.Received(1).Start(Arg.Is(p => - p.FileName == "npm" && p.Arguments == "start")); + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + _processService.Received(1).Start(Arg.Is(p => + p.FileName == "cmd.exe" && p.Arguments == "/c npm start")); + } + else + { + _processService.Received(1).Start(Arg.Is(p => + p.FileName == "npm" && p.Arguments == "start")); + } } [Fact] @@ -700,4 +710,185 @@ protected override async Task SendAsync( return new HttpResponseMessage(HttpStatusCode.NotFound); } } + + [Fact] + public void BuildToolInvocationPrompt_NoManifest_ReturnsFallback() + { + var prompt = ConversationRequirementCheck.BuildToolInvocationPrompt(_tempDir, _logger); + prompt.Should().Be(ConversationRequirementCheck.FallbackToolPrompt, + because: "no ToolingManifest.json means we fall back to the default prompt"); + } + + [Fact] + public void BuildToolInvocationPrompt_EmptyServers_ReturnsFallback() + { + var manifest = new { mcpServers = Array.Empty() }; + File.WriteAllText( + Path.Combine(_tempDir, "ToolingManifest.json"), + JsonSerializer.Serialize(manifest)); + + var prompt = ConversationRequirementCheck.BuildToolInvocationPrompt(_tempDir, _logger); + prompt.Should().Be(ConversationRequirementCheck.FallbackToolPrompt, + because: "an empty mcpServers array means no tools to invoke"); + } + + [Fact] + public void BuildToolInvocationPrompt_InvalidJson_ReturnsFallback() + { + File.WriteAllText( + Path.Combine(_tempDir, "ToolingManifest.json"), + "{ not valid json }}}"); + + var prompt = ConversationRequirementCheck.BuildToolInvocationPrompt(_tempDir, _logger); + prompt.Should().Be(ConversationRequirementCheck.FallbackToolPrompt, + because: "malformed JSON should not crash the check"); + } + + [Fact] + public void BuildToolInvocationPrompt_WithKnownTool_ReturnsNaturalQuestion() + { + var manifest = new + { + mcpServers = new[] + { + new { mcpServerName = "Mail", url = "https://example.com/mail" } + } + }; + File.WriteAllText( + Path.Combine(_tempDir, "ToolingManifest.json"), + JsonSerializer.Serialize(manifest)); + + var prompt = ConversationRequirementCheck.BuildToolInvocationPrompt(_tempDir, _logger); + prompt.Should().Be("Get me my recent emails", + because: "Mail is a known tool with a mapped natural-language question"); + } + + [Fact] + public void BuildToolInvocationPrompt_MultipleTools_UsesFirstTool() + { + var manifest = new + { + mcpServers = new[] + { + new { mcpServerName = "Calendar", url = "https://example.com/cal" }, + new { mcpServerName = "Mail", url = "https://example.com/mail" } + } + }; + File.WriteAllText( + Path.Combine(_tempDir, "ToolingManifest.json"), + JsonSerializer.Serialize(manifest)); + + var prompt = ConversationRequirementCheck.BuildToolInvocationPrompt(_tempDir, _logger); + prompt.Should().Be("What meetings do I have today?", + because: "Calendar is first and is a known tool"); + } + + [Fact] + public void BuildToolInvocationPrompt_UnknownToolWithDescription_UsesDescription() + { + var manifest = new + { + mcpServers = new[] + { + new { mcpServerName = "CustomCRM", url = "https://example.com/crm", description = "Manage customer relationships." } + } + }; + File.WriteAllText( + Path.Combine(_tempDir, "ToolingManifest.json"), + JsonSerializer.Serialize(manifest)); + + var prompt = ConversationRequirementCheck.BuildToolInvocationPrompt(_tempDir, _logger); + prompt.Should().Be("Help me with Manage customer relationships", + because: "unknown tools with a description fall back to description-based prompt"); + } + + [Fact] + public void BuildToolInvocationPrompt_UnknownToolNoDescription_UsesToolName() + { + var manifest = new + { + mcpServers = new[] + { + new { mcpServerName = "CustomCRM", url = "https://example.com/crm" } + } + }; + File.WriteAllText( + Path.Combine(_tempDir, "ToolingManifest.json"), + JsonSerializer.Serialize(manifest)); + + var prompt = ConversationRequirementCheck.BuildToolInvocationPrompt(_tempDir, _logger); + prompt.Should().Be("Help me with CustomCRM", + because: "unknown tools without a description fall back to name-based prompt"); + } + + [Fact] + public void BuildToolInvocationPrompt_KnownToolCaseInsensitive_ReturnsNaturalQuestion() + { + var manifest = new + { + mcpServers = new[] + { + new { mcpServerName = "SHAREPOINT", url = "https://example.com/sp" } + } + }; + File.WriteAllText( + Path.Combine(_tempDir, "ToolingManifest.json"), + JsonSerializer.Serialize(manifest)); + + var prompt = ConversationRequirementCheck.BuildToolInvocationPrompt(_tempDir, _logger); + prompt.Should().Be("Get me my recent SharePoint files", + because: "tool name matching should be case-insensitive"); + } + + [Fact] + public void BuildToolInvocationPrompt_ContainsKnownKeyword_ReturnsNaturalQuestion() + { + var manifest = new + { + mcpServers = new[] + { + new { mcpServerName = "M365SharePoint", url = "https://example.com/sp" } + } + }; + File.WriteAllText( + Path.Combine(_tempDir, "ToolingManifest.json"), + JsonSerializer.Serialize(manifest)); + + var prompt = ConversationRequirementCheck.BuildToolInvocationPrompt(_tempDir, _logger); + prompt.Should().Be("Get me my recent SharePoint files", + because: "tool name containing a known keyword should match via contains"); + } + + [Fact] + public void BuildConversationPrompts_NoManifest_ReturnsDefaults() + { + var prompts = ConversationRequirementCheck.BuildConversationPrompts(_tempDir, _logger); + prompts.Should().HaveCount(3); + prompts[0].Should().Be("Hello"); + prompts[1].Should().Be("What can you do?", + because: "without a manifest the fallback prompt is used"); + prompts[2].Should().Be("Thanks"); + } + + [Fact] + public void BuildConversationPrompts_WithManifest_ReplacesMiddleTurn() + { + var manifest = new + { + mcpServers = new[] + { + new { mcpServerName = "Mail", url = "https://example.com/mail" } + } + }; + File.WriteAllText( + Path.Combine(_tempDir, "ToolingManifest.json"), + JsonSerializer.Serialize(manifest)); + + var prompts = ConversationRequirementCheck.BuildConversationPrompts(_tempDir, _logger); + prompts.Should().HaveCount(3); + prompts[0].Should().Be("Hello"); + prompts[1].Should().Be("Get me my recent emails", + because: "the middle turn should be a natural question that triggers the Mail tool"); + prompts[2].Should().Be("Thanks"); + } } diff --git a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/LocalRuntimeRequirementCheckTests.cs b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/LocalRuntimeRequirementCheckTests.cs index 503d3aef..a91bc8fb 100644 --- a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/LocalRuntimeRequirementCheckTests.cs +++ b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/LocalRuntimeRequirementCheckTests.cs @@ -3,6 +3,7 @@ using System.Diagnostics; using System.Net; +using System.Runtime.InteropServices; using FluentAssertions; using Microsoft.Agents.A365.DevTools.Cli.Models; using Microsoft.Agents.A365.DevTools.Cli.Services; @@ -201,8 +202,16 @@ public async Task CheckAsync_NodeJsProject_UsesNpmStart() // Assert result.Passed.Should().BeTrue(); - _processService.Received(1).Start(Arg.Is(p => - p.FileName == "npm" && p.Arguments == "start")); + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + _processService.Received(1).Start(Arg.Is(p => + p.FileName == "cmd.exe" && p.Arguments == "/c npm start")); + } + else + { + _processService.Received(1).Start(Arg.Is(p => + p.FileName == "npm" && p.Arguments == "start")); + } } [Fact] diff --git a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/TelemetryRequirementCheckTests.cs b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/TelemetryRequirementCheckTests.cs index d07206d8..66103a69 100644 --- a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/TelemetryRequirementCheckTests.cs +++ b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/TelemetryRequirementCheckTests.cs @@ -3,7 +3,6 @@ using FluentAssertions; using Microsoft.Agents.A365.DevTools.Cli.Models; -using Microsoft.Agents.A365.DevTools.Cli.Services.Requirements; using Microsoft.Agents.A365.DevTools.Cli.Services.Requirements.RequirementChecks; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; @@ -25,6 +24,80 @@ private string CreateTempLogFile(string[] lines) return path; } + /// + /// Helper to build a console exporter span block with Agent365Sdk scope. + /// + private static string[] MakeAgent365Span(string operationName) => new[] + { + " traceId: '59ea028f0ee6a6cbb3b0e3c96ee96fa7',", + " instrumentationScope: {", + " name: 'Agent365Sdk',", + " },", + $" 'gen_ai.operation.name': '{operationName}'," + }; + + /// + /// Helper to build a fully-compliant span block with scope version and parentId. + /// + private static string[] MakeFullAgent365Span(string operationName, bool withParent = false) => withParent + ? new[] + { + " traceId: '59ea028f0ee6a6cbb3b0e3c96ee96fa7',", + " parentId: 'abc123def456',", + " instrumentationScope: {", + " name: 'Agent365Sdk',", + " version: '1.0.0',", + " },", + $" 'gen_ai.operation.name': '{operationName}'," + } + : new[] + { + " traceId: '59ea028f0ee6a6cbb3b0e3c96ee96fa7',", + " instrumentationScope: {", + " name: 'Agent365Sdk',", + " version: '1.0.0',", + " },", + $" 'gen_ai.operation.name': '{operationName}'," + }; + + /// + /// Resource lines that satisfy OTel semantic convention checks. + /// + private static readonly string[] ResourceLines = new[] + { + " resource: {", + " 'telemetry.sdk.name': 'opentelemetry',", + " 'telemetry.sdk.version': '1.25.0',", + " 'service.name': 'my-agent',", + " }," + }; + + /// + /// Helper to build a span block with a non-Agent365 scope (but not the ignored scope). + /// These spans SHOULD be accepted by the check. + /// + private static string[] MakeOtherSpan(string operationName) => new[] + { + " traceId: 'aaaa028f0ee6a6cbb3b0e3c96ee96fa7',", + " instrumentationScope: {", + " name: 'microsoft-otel-langchain',", + " },", + $" 'gen_ai.operation.name': '{operationName}'," + }; + + /// + /// Helper to build a span block from the ignored @microsoft/agents-telemetry scope. + /// These spans should be EXCLUDED from validation. + /// + private static string[] MakeIgnoredScopeSpan(string operationName) => new[] + { + " traceId: 'bbbb028f0ee6a6cbb3b0e3c96ee96fa7',", + " instrumentationScope: {", + " name: '@microsoft/agents-telemetry',", + " },", + $" 'gen_ai.operation.name': '{operationName}'," + }; + public void Dispose() { foreach (var f in _tempFiles) @@ -33,6 +106,8 @@ public void Dispose() } } + // --- Metadata --- + [Fact] public void Name_ReturnsTelemetry() { @@ -47,6 +122,8 @@ public void Category_ReturnsObservability() check.Category.Should().Be("Observability"); } + // --- No log file --- + [Fact] public async Task CheckAsync_NullLogPath_ReturnsWarning() { @@ -56,7 +133,6 @@ public async Task CheckAsync_NullLogPath_ReturnsWarning() result.Passed.Should().BeTrue(because: "no log file is a warning, not a failure"); result.IsWarning.Should().BeTrue(because: "missing log file means telemetry status is unknown"); - result.ErrorMessage.Should().Contain("No agent console log file available"); } [Fact] @@ -70,253 +146,575 @@ public async Task CheckAsync_NonExistentLogPath_ReturnsWarning() result.IsWarning.Should().BeTrue(); } + // --- No span output --- + [Fact] - public async Task CheckAsync_NoTelemetryLines_ReturnsWarning() + public async Task CheckAsync_NoSpanOutput_ReturnsFail() { var logPath = CreateTempLogFile(new[] { "info: Application started", - "info: Listening on http://localhost:5000", - "info: Received message: Hello" + "info: Listening on http://localhost:5000" }); var check = new TelemetryRequirementCheck(logPath); var result = await check.CheckAsync(_config, _logger); - result.Passed.Should().BeFalse(because: "no telemetry evidence means telemetry is not configured"); - result.IsWarning.Should().BeFalse(because: "missing telemetry is a failure, not a warning"); - result.ErrorMessage.Should().Contain("No telemetry-related output detected"); + result.Passed.Should().BeFalse(because: "no console exporter span output detected"); + result.ErrorMessage.Should().Contain("No console exporter span output detected"); } + // --- Ignored scope exclusion --- + [Fact] - public async Task CheckAsync_SuccessPatterns_ReturnsPass() + public async Task CheckAsync_SpansOnlyFromIgnoredScope_ReturnsFail() { - var logPath = CreateTempLogFile(new[] - { - "info: Application started", - "info: OpenTelemetry TracerProvider built successfully", - "info: OtlpExporter configured for https://agent365.observability.endpoint", - "info: BatchExportProcessor started", - "info: Export completed - 5 spans exported" - }); + var lines = new List(); + lines.AddRange(MakeIgnoredScopeSpan("invoke_agent")); + lines.AddRange(MakeIgnoredScopeSpan("chat")); + lines.AddRange(MakeIgnoredScopeSpan("execute_tool")); + var logPath = CreateTempLogFile(lines.ToArray()); var check = new TelemetryRequirementCheck(logPath); var result = await check.CheckAsync(_config, _logger); - result.Passed.Should().BeTrue(because: "success patterns indicate traces are being exported"); - result.IsWarning.Should().BeFalse(); - result.Details.Should().Contain("Telemetry export evidence found"); + result.Passed.Should().BeFalse(because: "spans from @microsoft/agents-telemetry scope should be excluded"); + result.Details.Should().Contain("@microsoft/agents-telemetry", + because: "details should indicate the ignored scope that caused all spans to be filtered out"); } + // --- All 3 GenAI spans from Agent365Sdk --- + [Fact] - public async Task CheckAsync_FailurePatterns_ReturnsFail() + public async Task CheckAsync_AllThreeSpansFromAgent365Sdk_ReturnsPass() { - var logPath = CreateTempLogFile(new[] - { - "info: Application started", - "info: OpenTelemetry TracerProvider built successfully", - "error: OTLP export failed: connection refused", - "warn: Dropped spans due to exporter error" - }); + var lines = new List(); + lines.AddRange(MakeAgent365Span("invoke_agent")); + lines.AddRange(MakeAgent365Span("chat")); + lines.AddRange(MakeAgent365Span("execute_tool")); + var logPath = CreateTempLogFile(lines.ToArray()); var check = new TelemetryRequirementCheck(logPath); var result = await check.CheckAsync(_config, _logger); - result.Passed.Should().BeFalse(because: "export failures indicate telemetry is not working"); - result.ErrorMessage.Should().Contain("Telemetry export failures detected"); - result.ResolutionGuidance.Should().Contain("OTLP endpoint", because: "connection refused should suggest checking endpoint connectivity"); + result.Passed.Should().BeTrue(because: "all 3 required GenAI spans are present from Agent365Sdk scope"); + result.Details.Should().Contain("All required GenAI operation spans detected"); } [Fact] - public async Task CheckAsync_MixedSuccessAndFailure_FailureTakesPrecedence() + public async Task CheckAsync_MixedScopes_ExcludesIgnoredScope() { - var logPath = CreateTempLogFile(new[] - { - "info: OpenTelemetry TracerProvider built successfully", - "info: Export completed - 3 spans exported", - "error: OTLP exporter: UNAVAILABLE - endpoint unreachable", - "info: BatchExportProcessor: dropped spans" - }); + var lines = new List(); + // Other scope has invoke_agent and chat — should count + lines.AddRange(MakeOtherSpan("invoke_agent")); + lines.AddRange(MakeOtherSpan("chat")); + // Ignored scope has execute_tool — should NOT count + lines.AddRange(MakeIgnoredScopeSpan("execute_tool")); + var logPath = CreateTempLogFile(lines.ToArray()); var check = new TelemetryRequirementCheck(logPath); var result = await check.CheckAsync(_config, _logger); - result.Passed.Should().BeFalse(because: "failure patterns take precedence over success patterns"); - result.Details.Should().Contain("Failure patterns found"); - result.Details.Should().Contain("Success patterns found"); + result.Passed.Should().BeFalse(because: "execute_tool from @microsoft/agents-telemetry scope should not count"); + result.ErrorMessage.Should().Contain("execute_tool"); } [Fact] - public async Task CheckAsync_TelemetryContextButNoExportEvidence_ReturnsWarning() + public async Task CheckAsync_OtherScopes_AllAccepted() { var logPath = CreateTempLogFile(new[] { - "info: Application started", - "dbug: OpenTelemetry SDK initialized", - "dbug: Adding OTLP exporter to pipeline" + " traceId: 'abc',", + " instrumentationScope: {", + " name: 'CustomSdk',", + " },", + " 'gen_ai.operation.name': 'invoke_agent',", + " traceId: 'def',", + " instrumentationScope: {", + " name: 'microsoft-otel-langchain',", + " },", + " 'gen_ai.operation.name': 'chat',", + " traceId: 'ghi',", + " instrumentationScope: {", + " name: 'Agent365Sdk',", + " },", + " 'gen_ai.operation.name': 'execute_tool'," }); var check = new TelemetryRequirementCheck(logPath); var result = await check.CheckAsync(_config, _logger); - result.Passed.Should().BeFalse(because: "SDK detected without export evidence should fail"); - result.IsWarning.Should().BeFalse(because: "no confirmed export means telemetry is not working"); - result.ErrorMessage.Should().Contain("Telemetry SDK detected but no trace export evidence found"); + result.Passed.Should().BeTrue(because: "all scopes except @microsoft/agents-telemetry should be accepted"); } [Fact] - public async Task CheckAsync_Agent365ObservabilityPattern_ReturnsPass() + public async Task CheckAsync_IgnoredScope_CaseInsensitiveExclusion() { - var logPath = CreateTempLogFile(new[] - { - "info: Configuring Agent365.Observability.OtelWrite endpoint", - "info: TracerProvider started with OTLP exporter" - }); + var lines = new List(); + // Ignored scope with different casing — should still be excluded + lines.Add(" traceId: 'abc',"); + lines.Add(" instrumentationScope: {"); + lines.Add(" name: '@Microsoft/Agents-Telemetry',"); + lines.Add(" },"); + lines.Add(" 'gen_ai.operation.name': 'invoke_agent',"); + lines.Add(" traceId: 'def',"); + lines.Add(" instrumentationScope: {"); + lines.Add(" name: '@MICROSOFT/AGENTS-TELEMETRY',"); + lines.Add(" },"); + lines.Add(" 'gen_ai.operation.name': 'chat',"); + lines.Add(" traceId: 'ghi',"); + lines.Add(" instrumentationScope: {"); + lines.Add(" name: '@microsoft/agents-telemetry',"); + lines.Add(" },"); + lines.Add(" 'gen_ai.operation.name': 'execute_tool',"); + var logPath = CreateTempLogFile(lines.ToArray()); var check = new TelemetryRequirementCheck(logPath); var result = await check.CheckAsync(_config, _logger); - result.Passed.Should().BeTrue(); - result.IsWarning.Should().BeFalse(); + result.Passed.Should().BeFalse(because: "ignored scope exclusion should be case-insensitive"); } + // --- Missing spans --- + [Fact] - public async Task CheckAsync_CaseInsensitiveMatching() + public async Task CheckAsync_MissingChat_ReturnsFail() { - var logPath = CreateTempLogFile(new[] - { - "INFO: OPENTELEMETRY TRACERPROVIDER BUILT successfully", - "INFO: OTLPEXPORTER EXPORT COMPLETED" - }); + var lines = new List(); + lines.AddRange(MakeAgent365Span("invoke_agent")); + lines.AddRange(MakeAgent365Span("execute_tool")); + var logPath = CreateTempLogFile(lines.ToArray()); var check = new TelemetryRequirementCheck(logPath); var result = await check.CheckAsync(_config, _logger); - result.Passed.Should().BeTrue(because: "pattern matching should be case-insensitive"); - result.IsWarning.Should().BeFalse(); + result.Passed.Should().BeFalse(because: "chat operation is missing"); + result.ErrorMessage.Should().Contain("chat"); } [Fact] - public void FilterTelemetryLines_FiltersOnlyRelevantLines() + public async Task CheckAsync_OnlyInvokeAgent_ReportsOtherTwoMissing() + { + var lines = new List(); + lines.AddRange(MakeAgent365Span("invoke_agent")); + var logPath = CreateTempLogFile(lines.ToArray()); + + var check = new TelemetryRequirementCheck(logPath); + + var result = await check.CheckAsync(_config, _logger); + + result.Passed.Should().BeFalse(); + result.ErrorMessage.Should().Contain("chat"); + result.ErrorMessage.Should().Contain("execute_tool"); + } + + // --- SplitIntoSpanBlocks --- + + [Fact] + public void SplitIntoSpanBlocks_SplitsOnTraceId() { - var logLines = new[] + var lines = new[] + { + " traceId: 'aaa',", + " name: 'span1',", + " traceId: 'bbb',", + " name: 'span2'," + }; + + var blocks = TelemetryRequirementCheck.SplitIntoSpanBlocks(lines); + + blocks.Should().HaveCount(2); + blocks[0].Should().Contain(l => l.Contains("span1")); + blocks[1].Should().Contain(l => l.Contains("span2")); + } + + [Fact] + public void SplitIntoSpanBlocks_IgnoresLinesBeforeFirstTraceId() + { + var lines = new[] { "info: Application started", - "info: OpenTelemetry TracerProvider built", - "info: Listening on port 5000", - "error: OTLP export failed", - "info: Received message" + "info: some noise", + " traceId: 'aaa',", + " name: 'span1'," }; - var result = TelemetryRequirementCheck.FilterTelemetryLines(logLines); + var blocks = TelemetryRequirementCheck.SplitIntoSpanBlocks(lines); - result.Should().HaveCount(2); - result[0].Should().Contain("OpenTelemetry"); - result[1].Should().Contain("OTLP"); + blocks.Should().HaveCount(1); + } + + [Fact] + public void SplitIntoSpanBlocks_EmptyInput_ReturnsEmpty() + { + var blocks = TelemetryRequirementCheck.SplitIntoSpanBlocks(Array.Empty()); + blocks.Should().BeEmpty(); } [Fact] - public void FilterTelemetryLines_RespectsMaxLimit() + public void SplitIntoSpanBlocks_IncludesLinesBeforeTraceId() { - var logLines = Enumerable.Range(0, 200) - .Select(i => $"info: OpenTelemetry span {i} exported") - .ToArray(); + // instrumentationScope appears before traceId in real output + var lines = new[] + { + " instrumentationScope: {", + " name: 'Agent365Sdk',", + " },", + " traceId: 'aaa',", + " 'gen_ai.operation.name': 'chat'," + }; - var result = TelemetryRequirementCheck.FilterTelemetryLines(logLines); + var blocks = TelemetryRequirementCheck.SplitIntoSpanBlocks(lines); - result.Should().HaveCount(TelemetryRequirementCheck.MaxTelemetryLines); + // The lines before the first traceId won't be in any block + // instrumentationScope needs to be AFTER traceId or we need to handle this + blocks.Should().HaveCount(1); } + // --- ExtractOperationNames --- + [Fact] - public void FilterTelemetryLines_SkipsEmptyAndWhitespace() + public void ExtractOperationNames_SingleQuoteFormat() { - var logLines = new[] { "", " ", null!, "info: TracerProvider started" }; + var block = new List { " 'gen_ai.operation.name': 'chat'," }; - var result = TelemetryRequirementCheck.FilterTelemetryLines(logLines); + var result = TelemetryRequirementCheck.ExtractOperationNames(block); - result.Should().ContainSingle() - .Which.Should().Contain("TracerProvider"); + result.Should().ContainSingle().Which.Should().Be("chat"); } [Fact] - public void FindMatchingPatterns_FindsAllMatches() + public void ExtractOperationNames_DoubleQuoteFormat() { - var lines = new List - { - "Export completed successfully", - "TracerProvider built and started", - "OtlpExporter configured" - }; + var block = new List { " \"gen_ai.operation.name\": \"invoke_agent\"," }; - var result = TelemetryRequirementCheck.FindMatchingPatterns( - lines, TelemetryRequirementCheck.SuccessPatterns); + var result = TelemetryRequirementCheck.ExtractOperationNames(block); - result.Should().Contain("export completed"); - result.Should().Contain("otlpexporter"); + result.Should().ContainSingle().Which.Should().Be("invoke_agent"); } [Fact] - public void FindMatchingPatterns_ReturnsEmptyForNoMatches() + public void ExtractOperationNames_EqualsFormat() { - var lines = new List - { - "Application started", - "Listening on port 5000" - }; + var block = new List { "gen_ai.operation.name=execute_tool" }; + + var result = TelemetryRequirementCheck.ExtractOperationNames(block); + + result.Should().ContainSingle().Which.Should().Be("execute_tool"); + } - var result = TelemetryRequirementCheck.FindMatchingPatterns( - lines, TelemetryRequirementCheck.SuccessPatterns); + [Fact] + public void ExtractOperationNames_NoMatch_ReturnsEmpty() + { + var block = new List { " name: 'some-span',", " duration: 123" }; + + var result = TelemetryRequirementCheck.ExtractOperationNames(block); result.Should().BeEmpty(); } + // --- Real-world console exporter output --- + [Fact] - public async Task CheckAsync_UnrelatedConnectionRefused_NotFlaggedAsFailure() + public async Task CheckAsync_RealWorldNodeConsoleExporter_ReturnsPass() { var logPath = CreateTempLogFile(new[] { - "info: Application started", - "error: Database connection refused on port 5432" + "{", + " resource: {", + " attributes: {", + " 'service.name': 'internal-docs-agent',", + " 'telemetry.sdk.name': 'opentelemetry',", + " }", + " },", + " instrumentationScope: {", + " name: 'Agent365Sdk',", + " version: '1.0.0',", + " },", + " traceId: '59ea028f0ee6a6cbb3b0e3c96ee96fa7',", + " name: 'invoke_agent Agent',", + " attributes: {", + " 'gen_ai.operation.name': 'invoke_agent',", + " },", + "}", + "{", + " instrumentationScope: {", + " name: 'Agent365Sdk',", + " },", + " traceId: '59ea028f0ee6a6cbb3b0e3c96ee96fa7',", + " name: 'chat gpt-4.1',", + " attributes: {", + " 'gen_ai.operation.name': 'chat',", + " 'gen_ai.request.model': 'gpt-4.1-2025-04-14',", + " },", + "}", + "{", + " instrumentationScope: {", + " name: 'Agent365Sdk',", + " },", + " traceId: '59ea028f0ee6a6cbb3b0e3c96ee96fa7',", + " name: 'execute_tool search_docs',", + " attributes: {", + " 'gen_ai.operation.name': 'execute_tool',", + " },", + "}" }); var check = new TelemetryRequirementCheck(logPath); var result = await check.CheckAsync(_config, _logger); - result.Passed.Should().BeTrue(because: "unrelated connection errors should not trigger telemetry failure"); - result.IsWarning.Should().BeTrue(because: "no telemetry-relevant lines found"); + result.Passed.Should().BeTrue(because: "real-world Node.js console exporter output with Agent365Sdk scope should pass"); + result.Details.Should().Contain("invoke_agent"); + result.Details.Should().Contain("chat"); + result.Details.Should().Contain("execute_tool"); } + // --- Scope version checks --- + [Fact] - public async Task CheckAsync_Agent365ExporterSpansSkipped_ReturnsFail() + public void HasInstrumentationScopeVersion_WithVersion_ReturnsTrue() { - var logPath = CreateTempLogFile(new[] + var block = new List { - "dbug: Microsoft.Agents.A365.Observability.Runtime.Tracing.Exporters.Agent365Exporter[0]", - " Agent365Exporter: Exporting batch of 7 spans.", - "dbug: Microsoft.Agents.A365.Observability.Runtime.Tracing.Exporters.Agent365ExporterCore[0]", - " [Agent365Exporter] 5 non-genAI spans filtered out", - "dbug: Microsoft.Agents.A365.Observability.Runtime.Tracing.Exporters.Agent365ExporterCore[0]", - " [Agent365Exporter] 2 spans skipped due to missing tenant or agent ID", - "dbug: Microsoft.Agents.A365.Observability.Runtime.Tracing.Exporters.Agent365ExporterCore[0]", - " [Agent365Exporter] Partitioned into 0 identity groups (7 spans skipped)", - "dbug: Microsoft.Agents.A365.Observability.Runtime.Tracing.Exporters.Agent365Exporter[0]", - " Agent365Exporter: No spans with tenant/agent identity found; nothing exported." - }); + " instrumentationScope: {", + " name: 'Agent365Sdk',", + " version: '1.0.0',", + " }," + }; + + TelemetryRequirementCheck.HasInstrumentationScopeVersion(block).Should().BeTrue(); + } + [Fact] + public void HasInstrumentationScopeVersion_SameLine_ReturnsTrue() + { + var block = new List + { + " instrumentationScope: { name: 'Agent365Sdk', version: '1.0.0' }," + }; + + TelemetryRequirementCheck.HasInstrumentationScopeVersion(block).Should().BeTrue(); + } + + [Fact] + public void HasInstrumentationScopeVersion_NoVersion_ReturnsFalse() + { + var block = new List + { + " instrumentationScope: {", + " name: 'Agent365Sdk',", + " }," + }; + + TelemetryRequirementCheck.HasInstrumentationScopeVersion(block).Should().BeFalse(); + } + + // --- Parent link checks --- + + [Fact] + public void GetChildSpansMissingParent_WithParentId_ReturnsEmpty() + { + var blocks = new List> + { + new(MakeFullAgent365Span("chat", withParent: true)), + new(MakeFullAgent365Span("execute_tool", withParent: true)) + }; + + TelemetryRequirementCheck.GetChildSpansMissingParent(blocks).Should().BeEmpty(); + } + + [Fact] + public void GetChildSpansMissingParent_MissingParent_ReturnsOperations() + { + var blocks = new List> + { + new(MakeAgent365Span("chat")), + new(MakeAgent365Span("execute_tool")) + }; + + var missing = TelemetryRequirementCheck.GetChildSpansMissingParent(blocks); + missing.Should().Contain("chat"); + missing.Should().Contain("execute_tool"); + } + + [Fact] + public void GetChildSpansMissingParent_InvokeAgentWithoutParent_IsIgnored() + { + var blocks = new List> + { + new(MakeAgent365Span("invoke_agent")) + }; + + TelemetryRequirementCheck.GetChildSpansMissingParent(blocks) + .Should().BeEmpty(because: "invoke_agent is a root span and does not need a parent"); + } + + [Fact] + public void HasNonEmptyValue_ValidValue_ReturnsTrue() + { + TelemetryRequirementCheck.HasNonEmptyValue(" parentId: 'abc123'").Should().BeTrue(); + } + + [Fact] + public void HasNonEmptyValue_Undefined_ReturnsFalse() + { + TelemetryRequirementCheck.HasNonEmptyValue(" parentId: undefined").Should().BeFalse(); + } + + [Fact] + public void HasNonEmptyValue_EmptyQuotes_ReturnsFalse() + { + TelemetryRequirementCheck.HasNonEmptyValue(" parentId: ''").Should().BeFalse(); + } + + // --- Resource attribute checks --- + + [Fact] + public void GetMissingResourceAttributes_AllPresent_ReturnsEmpty() + { + var lines = new[] + { + " 'telemetry.sdk.name': 'opentelemetry',", + " 'telemetry.sdk.version': '1.25.0',", + " 'service.name': 'my-agent'," + }; + + TelemetryRequirementCheck.GetMissingResourceAttributes(lines).Should().BeEmpty(); + } + + [Fact] + public void GetMissingResourceAttributes_MissingSdkVersion_ReportsIt() + { + var lines = new[] + { + " 'telemetry.sdk.name': 'opentelemetry',", + " 'service.name': 'my-agent'," + }; + + var missing = TelemetryRequirementCheck.GetMissingResourceAttributes(lines); + missing.Should().Contain("telemetry.sdk.version"); + missing.Should().NotContain("telemetry.sdk.name"); + missing.Should().NotContain("service.name"); + } + + [Fact] + public void GetMissingResourceAttributes_NonePresent_ReturnsAll() + { + var lines = new[] { "some unrelated log output" }; + + var missing = TelemetryRequirementCheck.GetMissingResourceAttributes(lines); + missing.Should().HaveCount(3); + } + + // --- End-to-end: fully compliant spans return success --- + + [Fact] + public async Task CheckAsync_FullyCompliantSpans_ReturnsSuccess() + { + var lines = new List(); + lines.AddRange(ResourceLines); + lines.Add("{"); + lines.AddRange(MakeFullAgent365Span("invoke_agent")); + lines.Add("}"); + lines.Add("{"); + lines.AddRange(MakeFullAgent365Span("chat", withParent: true)); + lines.Add("}"); + lines.Add("{"); + lines.AddRange(MakeFullAgent365Span("execute_tool", withParent: true)); + lines.Add("}"); + + var logPath = CreateTempLogFile(lines.ToArray()); + var check = new TelemetryRequirementCheck(logPath); + + var result = await check.CheckAsync(_config, _logger); + + result.Passed.Should().BeTrue(); + result.IsWarning.Should().BeFalse(because: "fully compliant spans should not produce warnings"); + } + + [Fact] + public async Task CheckAsync_MissingScopeVersion_ReturnsWarning() + { + var lines = new List(); + lines.AddRange(ResourceLines); + lines.Add("{"); + lines.AddRange(MakeAgent365Span("invoke_agent")); + lines.Add("}"); + lines.Add("{"); + // chat span without parent or version + lines.AddRange(MakeAgent365Span("chat")); + lines.Add("}"); + lines.Add("{"); + lines.AddRange(MakeAgent365Span("execute_tool")); + lines.Add("}"); + + var logPath = CreateTempLogFile(lines.ToArray()); + var check = new TelemetryRequirementCheck(logPath); + + var result = await check.CheckAsync(_config, _logger); + + result.Passed.Should().BeTrue(because: "scope version missing is a warning not a failure"); + result.IsWarning.Should().BeTrue(); + result.Details.Should().Contain("version", because: "warning should mention missing scope version"); + } + + [Fact] + public async Task CheckAsync_ChildSpansMissingParent_ReturnsWarning() + { + var lines = new List(); + lines.AddRange(ResourceLines); + lines.Add("{"); + lines.AddRange(MakeFullAgent365Span("invoke_agent")); + lines.Add("}"); + lines.Add("{"); + // chat without parentId + lines.AddRange(MakeFullAgent365Span("chat", withParent: false)); + lines.Add("}"); + lines.Add("{"); + lines.AddRange(MakeFullAgent365Span("execute_tool", withParent: true)); + lines.Add("}"); + + var logPath = CreateTempLogFile(lines.ToArray()); var check = new TelemetryRequirementCheck(logPath); var result = await check.CheckAsync(_config, _logger); - result.Passed.Should().BeFalse(because: "Agent365Exporter skipped all spans due to missing identity -- telemetry is not working"); - result.ErrorMessage.Should().Contain("Telemetry export failures detected"); - result.ResolutionGuidance.Should().Contain("tenant ID", because: "missing tenant/agent ID requires identity configuration guidance"); + result.Passed.Should().BeTrue(because: "missing parent is a warning not a failure"); + result.IsWarning.Should().BeTrue(); + result.Details.Should().Contain("parentId", because: "warning should mention missing parent links"); + result.Details.Should().Contain("chat"); + } + + [Fact] + public async Task CheckAsync_MissingResourceAttributes_ReturnsWarning() + { + var lines = new List(); + // No resource lines + lines.Add("{"); + lines.AddRange(MakeFullAgent365Span("invoke_agent")); + lines.Add("}"); + lines.Add("{"); + lines.AddRange(MakeFullAgent365Span("chat", withParent: true)); + lines.Add("}"); + lines.Add("{"); + lines.AddRange(MakeFullAgent365Span("execute_tool", withParent: true)); + lines.Add("}"); + + var logPath = CreateTempLogFile(lines.ToArray()); + var check = new TelemetryRequirementCheck(logPath); + + var result = await check.CheckAsync(_config, _logger); + + result.Passed.Should().BeTrue(because: "missing resource attributes is a warning not a failure"); + result.IsWarning.Should().BeTrue(); + result.Details.Should().Contain("service.name", because: "warning should list missing resource attributes"); } } From 4781aba54d13df516c0f02087e60b9d8cb4ecde2 Mon Sep 17 00:00:00 2001 From: Abbinayaa Subramanian Date: Wed, 27 May 2026 10:09:37 -0700 Subject: [PATCH 04/27] Fixing telemetry checks for python --- .../Commands/ValidateCommand.cs | 3 +- .../Program.cs | 2 +- .../BlueprintRegistrationRequirementCheck.cs | 100 +++++- .../TelemetryRequirementCheck.cs | 110 +++---- ...eprintRegistrationRequirementCheckTests.cs | 228 ++++++++++++++ .../TelemetryRequirementCheckTests.cs | 295 ++++++++++-------- 6 files changed, 520 insertions(+), 218 deletions(-) diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Commands/ValidateCommand.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Commands/ValidateCommand.cs index 537a0adc..c971b7d7 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Commands/ValidateCommand.cs +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Commands/ValidateCommand.cs @@ -42,6 +42,7 @@ public static Command CreateCommand( CommandExecutor? commandExecutor = null, IProcessService? processService = null, GraphApiService? graphApiService = null, + AgentBlueprintService? agentBlueprintService = null, IEnumerable? requirementChecksOverride = null) { var command = new Command(CommandNames.Validate, @@ -96,7 +97,7 @@ public static Command CreateCommand( // Phase 2a: Run blueprint registration check (independent of build/boot) if (requirementChecksOverride is null && graphApiService is not null) { - var registrationCheck = new BlueprintRegistrationRequirementCheck(graphApiService); + var registrationCheck = new BlueprintRegistrationRequirementCheck(graphApiService, agentBlueprintService); var registrationResults = await RunChecksDetailedAsync( new List { registrationCheck }, config, logger, ct); MapResultsToTiers(registrationResults, report); diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Program.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Program.cs index 26270e2b..5e75fc7f 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Program.cs +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Program.cs @@ -157,7 +157,7 @@ await Task.WhenAll( var confirmationProvider = serviceProvider.GetRequiredService(); rootCommand.AddCommand(SetupCommand.CreateCommand(setupLogger, configService, executor, backendConfigurator, azureAuthValidator, platformDetector, graphApiService, agentBlueprintService, blueprintLookupService, federatedCredentialService, clientAppValidator, confirmationProvider, armApiService, resolver: bootstrapResolver)); - rootCommand.AddCommand(ValidateCommand.CreateCommand(validateLogger, configService, platformDetector, executor, processService, graphApiService)); + rootCommand.AddCommand(ValidateCommand.CreateCommand(validateLogger, configService, platformDetector, executor, processService, graphApiService, agentBlueprintService)); var manifestTemplateService = serviceProvider.GetRequiredService(); rootCommand.AddCommand(QueryEntraCommand.CreateCommand(queryEntraLogger, configService, executor, graphApiService, agentBlueprintService, resolver: bootstrapResolver)); rootCommand.AddCommand(CleanupCommand.CreateCommand(cleanupLogger, configService, backendConfigurator, executor, agentBlueprintService, confirmationProvider, federatedCredentialService, azureAuthValidator, graphApiService, resolver: bootstrapResolver)); diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/BlueprintRegistrationRequirementCheck.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/BlueprintRegistrationRequirementCheck.cs index 8dd39ae9..be8ef153 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/BlueprintRegistrationRequirementCheck.cs +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/BlueprintRegistrationRequirementCheck.cs @@ -8,17 +8,19 @@ namespace Microsoft.Agents.A365.DevTools.Cli.Services.Requirements.RequirementCh /// /// Validates that the agent blueprint is registered in Microsoft Entra ID. -/// Checks that the blueprint application exists, has a service principal, and -/// (if configured) has an agent registration in the Microsoft Agent Registry. +/// Checks that the blueprint application exists, has a service principal, +/// (if configured) has an agent registration, and has inheritable permissions configured. /// Uses the same Graph API methods as query-entra. /// public class BlueprintRegistrationRequirementCheck : RequirementCheck { private readonly GraphApiService _graphApiService; + private readonly AgentBlueprintService? _blueprintService; - public BlueprintRegistrationRequirementCheck(GraphApiService graphApiService) + public BlueprintRegistrationRequirementCheck(GraphApiService graphApiService, AgentBlueprintService? blueprintService = null) { _graphApiService = graphApiService ?? throw new ArgumentNullException(nameof(graphApiService)); + _blueprintService = blueprintService; } /// @@ -141,11 +143,95 @@ private async Task CheckImplementationAsync( "could not be confirmed (insufficient permissions or transient error)."); } - return RequirementCheckResult.Success( - details: $"Blueprint '{blueprintId}' registered with service principal and agent registration '{config.AgentRegistrationId}'."); + return await BuildSuccessResult(config, blueprintId, tenantId, logger, + $"Blueprint '{blueprintId}' registered with service principal and agent registration '{config.AgentRegistrationId}'.", + cancellationToken); } - return RequirementCheckResult.Success( - details: $"Blueprint '{blueprintId}' registered with service principal '{servicePrincipalId}'."); + return await BuildSuccessResult(config, blueprintId, tenantId, logger, + $"Blueprint '{blueprintId}' registered with service principal '{servicePrincipalId}'.", + cancellationToken); + } + + /// + /// After core registration checks pass, verify inheritable permissions and consent status + /// by comparing config.ResourceConsents (expected) against what is actually in Entra. + /// Missing or mismatched permissions produce a warning (not a failure). + /// + private async Task BuildSuccessResult( + Agent365Config config, + string blueprintId, + string tenantId, + ILogger logger, + string baseDetails, + CancellationToken cancellationToken) + { + if (_blueprintService is null || config.ResourceConsents.Count == 0) + { + return RequirementCheckResult.Success(details: baseDetails); + } + + List<(string ResourceAppId, List Scopes)> actualPermissions; + try + { + actualPermissions = await _blueprintService.ListInheritablePermissionsAsync( + tenantId, blueprintId, ct: cancellationToken); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + logger.LogDebug(ex, "Failed to query inheritable permissions"); + return RequirementCheckResult.Warning( + "Blueprint registered but could not verify inheritable permissions", + details: $"{baseDetails} Permissions query failed: {ex.Message}"); + } + + var actualByResource = actualPermissions.ToDictionary( + p => p.ResourceAppId, + p => new HashSet(p.Scopes, StringComparer.OrdinalIgnoreCase), + StringComparer.OrdinalIgnoreCase); + + var warnings = new List(); + + foreach (var expected in config.ResourceConsents) + { + var resourceLabel = !string.IsNullOrWhiteSpace(expected.ResourceName) + ? expected.ResourceName + : expected.ResourceAppId; + + if (!actualByResource.TryGetValue(expected.ResourceAppId, out var actualScopes)) + { + warnings.Add($"{resourceLabel}: no inheritable permissions configured in Entra"); + continue; + } + + var missingScopes = expected.Scopes + .Where(s => !actualScopes.Contains(s)) + .ToList(); + + if (missingScopes.Count > 0) + { + warnings.Add($"{resourceLabel}: missing scopes: {string.Join(", ", missingScopes)}"); + } + + if (expected.ConsentGranted is false) + { + warnings.Add($"{resourceLabel}: admin consent not granted"); + } + } + + if (warnings.Count > 0) + { + return RequirementCheckResult.Warning( + "Blueprint registered but permissions/consent gaps detected", + details: $"{baseDetails} {string.Join(". ", warnings)}. " + + "Run 'a365 setup all' or grant consent in the Azure portal."); + } + + var scopeSummary = string.Join("; ", config.ResourceConsents.Select(r => + $"{(string.IsNullOrWhiteSpace(r.ResourceName) ? r.ResourceAppId : r.ResourceName)}: " + + $"{string.Join(", ", r.Scopes)}")); + + return RequirementCheckResult.Success( + details: $"{baseDetails} Permissions verified: {scopeSummary}"); } } diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/TelemetryRequirementCheck.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/TelemetryRequirementCheck.cs index c9a8b3cc..e867938e 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/TelemetryRequirementCheck.cs +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/TelemetryRequirementCheck.cs @@ -27,11 +27,14 @@ public class TelemetryRequirementCheck : RequirementCheck /// internal static readonly string[] TelemetryContextKeywords = new[] { - // Console exporter span output fields + // Console exporter span output fields (camelCase and snake_case variants) "traceid:", + "trace_id:", "spanid:", + "span_id:", "tracestate:", "parentspancontext:", + "parent_id:", // OpenTelemetry SDK indicators "opentelemetry", "otel", @@ -55,13 +58,6 @@ public class TelemetryRequirementCheck : RequirementCheck "agent365.observability" }; - /// - /// Instrumentation scope name to exclude from validation. - /// Spans from this scope are auto-generated by the agents-telemetry package - /// and should not count toward GenAI operation checks. - /// - internal const string IgnoredInstrumentationScope = "@microsoft/agents-telemetry"; - /// /// Required GenAI semantic convention operation names that must ALL appear in traces. /// These correspond to gen_ai.operation.name values for agent orchestration spans. @@ -73,9 +69,21 @@ public class TelemetryRequirementCheck : RequirementCheck "execute_tool" }; + /// + /// All recognized GenAI operation names used to filter relevant span blocks. + /// Only span blocks with one of these values in gen_ai.operation.name are considered. + /// + internal static readonly string[] RecognizedGenAiOperations = new[] + { + "invoke_agent", + "chat", + "execute_tool", + "output_messages" + }; + /// /// Operation names that must have a non-empty parentId to verify proper trace hierarchy. - /// These are child spans — execute_tool and chat should be linked to a parent invoke_agent span. + /// These are child spans that should be linked to a parent invoke_agent span. /// internal static readonly string[] ChildSpanOperations = new[] { @@ -93,10 +101,6 @@ public class TelemetryRequirementCheck : RequirementCheck "service.name" }; - /// - /// Keywords for detecting scope version in span output. - /// - internal const string ScopeVersionKey = "version"; public TelemetryRequirementCheck(string? agentConsoleLogPath) { @@ -148,7 +152,7 @@ private Task CheckImplementationAsync( details: $"Failed to read {_agentConsoleLogPath}: {ex.Message}")); } - // Split log into span blocks and find Agent365Sdk spans + // Split log into span blocks var spanBlocks = SplitIntoSpanBlocks(logLines); if (spanBlocks.Count == 0) { @@ -158,24 +162,23 @@ private Task CheckImplementationAsync( details: "Expected to find span blocks containing traceId in agent console logs.")); } - // Include all span blocks except those from the ignored instrumentation scope + // Filter to only span blocks that have a recognized gen_ai.operation.name var relevantBlocks = spanBlocks - .Where(block => !block.Any(line => - line.IndexOf(IgnoredInstrumentationScope, StringComparison.OrdinalIgnoreCase) >= 0)) + .Where(block => ExtractOperationNames(block) + .Any(op => RecognizedGenAiOperations.Contains(op, StringComparer.OrdinalIgnoreCase))) .ToList(); if (relevantBlocks.Count == 0) { - var ignoredCount = spanBlocks.Count - relevantBlocks.Count; return Task.FromResult(RequirementCheckResult.Failure( - "No relevant instrumentation scope spans found", - "Ensure your agent instruments GenAI spans with a named instrumentation scope.", + "No GenAI operation spans found", + "Ensure your agent instruments spans with gen_ai.operation.name set to invoke_agent, chat, or execute_tool.", details: $"Found {spanBlocks.Count} span block(s) in console output, " + - $"but all were from ignored scope '{IgnoredInstrumentationScope}'.")); + $"but none had a recognized gen_ai.operation.name value.")); } - logger.LogDebug("Found {Count} relevant span blocks (excluded {Ignored} from '{Scope}')", - relevantBlocks.Count, spanBlocks.Count - relevantBlocks.Count, IgnoredInstrumentationScope); + logger.LogDebug("Found {Count} relevant span blocks out of {Total} total", + relevantBlocks.Count, spanBlocks.Count); // Extract gen_ai.operation.name values from relevant spans var foundOperations = new HashSet(StringComparer.OrdinalIgnoreCase); @@ -206,14 +209,6 @@ private Task CheckImplementationAsync( // Additional OTel semantic convention checks (collected as warnings) var warnings = new List(); - // Check scope version: instrumentationScope should include a version - var hasScopeVersion = relevantBlocks.Any(block => - HasInstrumentationScopeVersion(block)); - if (!hasScopeVersion) - { - warnings.Add("instrumentationScope is missing 'version' — OTel semantic conventions recommend including scope version"); - } - // Check parent links: execute_tool and chat spans must have non-empty parentId var childOpsWithoutParent = GetChildSpansMissingParent(relevantBlocks); if (childOpsWithoutParent.Count > 0) @@ -284,7 +279,9 @@ private static List> SplitOnBraces(string[] logLines) // Only return if we found blocks that look like spans (contain traceId) var spanBlocks = blocks.Where(b => b.Any(l => l.TrimStart().StartsWith("traceId:", StringComparison.OrdinalIgnoreCase) || - l.TrimStart().StartsWith("\"traceId\":", StringComparison.OrdinalIgnoreCase))).ToList(); + l.TrimStart().StartsWith("\"traceId\":", StringComparison.OrdinalIgnoreCase) || + l.TrimStart().StartsWith("\"trace_id\":", StringComparison.OrdinalIgnoreCase) || + l.TrimStart().StartsWith("trace_id:", StringComparison.OrdinalIgnoreCase))).ToList(); return spanBlocks; } @@ -304,6 +301,9 @@ private static List> SplitOnTraceId(string[] logLines) bool isTraceIdLine = trimmed.StartsWith("traceId:", StringComparison.OrdinalIgnoreCase) || trimmed.StartsWith("\"traceId\":", StringComparison.OrdinalIgnoreCase) || trimmed.StartsWith("'traceId':", StringComparison.OrdinalIgnoreCase) + || trimmed.StartsWith("trace_id:", StringComparison.OrdinalIgnoreCase) + || trimmed.StartsWith("\"trace_id\":", StringComparison.OrdinalIgnoreCase) + || trimmed.StartsWith("'trace_id':", StringComparison.OrdinalIgnoreCase) || trimmed.StartsWith("Activity.TraceId:", StringComparison.OrdinalIgnoreCase); if (isTraceIdLine) @@ -375,47 +375,6 @@ internal static List ExtractOperationNames(List spanBlock) return operations; } - /// - /// Checks whether a span block's instrumentationScope section contains a version field. - /// In the console exporter output, this appears near the scope name, e.g.: - /// instrumentationScope: { name: 'Agent365Sdk', version: '1.0.0' } - /// - internal static bool HasInstrumentationScopeVersion(List spanBlock) - { - bool inScope = false; - foreach (var line in spanBlock) - { - var trimmed = line.TrimStart(); - - if (trimmed.IndexOf("instrumentationScope", StringComparison.OrdinalIgnoreCase) >= 0) - { - inScope = true; - // Check if version is on the same line - if (trimmed.IndexOf(ScopeVersionKey, StringComparison.OrdinalIgnoreCase) >= 0) - return true; - continue; - } - - if (inScope) - { - // Scope section ends at the next top-level field (no leading whitespace or a new section) - if (!trimmed.StartsWith("'") && !trimmed.StartsWith("\"") && !trimmed.StartsWith("}") && - trimmed.Length > 0 && !char.IsWhiteSpace(line[0]) && - !trimmed.StartsWith("name", StringComparison.OrdinalIgnoreCase) && - !trimmed.StartsWith("version", StringComparison.OrdinalIgnoreCase)) - { - inScope = false; - continue; - } - - if (trimmed.IndexOf(ScopeVersionKey, StringComparison.OrdinalIgnoreCase) >= 0) - return true; - } - } - - return false; - } - /// /// Identifies child span operations (chat, execute_tool) that are missing a parentId/parentSpanId. /// Returns the list of operation names that should have parent links but don't. @@ -436,8 +395,11 @@ internal static List GetChildSpansMissingParent(List> spanB var trimmed = line.TrimStart(); return (trimmed.StartsWith("parentId:", StringComparison.OrdinalIgnoreCase) || trimmed.StartsWith("parentSpanId:", StringComparison.OrdinalIgnoreCase) || + trimmed.StartsWith("parent_id:", StringComparison.OrdinalIgnoreCase) || trimmed.StartsWith("\"parentId\":", StringComparison.OrdinalIgnoreCase) || + trimmed.StartsWith("\"parent_id\":", StringComparison.OrdinalIgnoreCase) || trimmed.StartsWith("'parentId':", StringComparison.OrdinalIgnoreCase) || + trimmed.StartsWith("'parent_id':", StringComparison.OrdinalIgnoreCase) || trimmed.StartsWith("parentSpanContext:", StringComparison.OrdinalIgnoreCase)) && HasNonEmptyValue(trimmed); }); @@ -464,7 +426,7 @@ internal static bool HasNonEmptyValue(string line) if (colonIdx < 0) return false; - var value = line.Substring(colonIdx + 1).Trim().Trim('\'', '"', ' '); + var value = line.Substring(colonIdx + 1).Trim().Trim('\'', '"', ',', ' '); return !string.IsNullOrWhiteSpace(value) && !value.Equals("undefined", StringComparison.OrdinalIgnoreCase) && !value.Equals("null", StringComparison.OrdinalIgnoreCase); diff --git a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/BlueprintRegistrationRequirementCheckTests.cs b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/BlueprintRegistrationRequirementCheckTests.cs index ecdb4872..85ef089c 100644 --- a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/BlueprintRegistrationRequirementCheckTests.cs +++ b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/BlueprintRegistrationRequirementCheckTests.cs @@ -292,4 +292,232 @@ public void Constructor_NullGraphApiService_Throws() var act = () => new BlueprintRegistrationRequirementCheck(null!); act.Should().Throw(); } + + // --- Inheritable permissions checks --- + + private AgentBlueprintService CreateMockBlueprintService() + { + var bpLogger = NullLoggerFactory.Instance.CreateLogger(); + return Substitute.ForPartsOf(bpLogger, _mockGraphApiService); + } + + private void SetupAppAndSpExist() + { + _mockGraphApiService.ApplicationExistsByAppIdAsync(TestTenantId, TestBlueprintId, Arg.Any()) + .Returns(true); + _mockGraphApiService.LookupServicePrincipalByAppIdAsync(TestTenantId, TestBlueprintId, Arg.Any(), Arg.Any?>()) + .Returns(TestServicePrincipalId); + } + + [Fact] + public async Task CheckAsync_WithPermissions_IncludesScopesInDetails() + { + var config = new Agent365Config + { + TenantId = TestTenantId, + AgentBlueprintId = TestBlueprintId, + ResourceConsents = new List + { + new() + { + ResourceName = "Microsoft Graph", + ResourceAppId = "00000003-0000-0000-c000-000000000000", + ConsentGranted = true, + Scopes = new List { "User.Read", "Mail.Read" } + } + } + }; + + SetupAppAndSpExist(); + var mockBpService = CreateMockBlueprintService(); + mockBpService.ListInheritablePermissionsAsync(TestTenantId, TestBlueprintId, Arg.Any?>(), Arg.Any()) + .Returns(new List<(string ResourceAppId, List Scopes)> + { + ("00000003-0000-0000-c000-000000000000", new List { "User.Read", "Mail.Read" }) + }); + + var check = new BlueprintRegistrationRequirementCheck(_mockGraphApiService, mockBpService); + var result = await check.CheckAsync(config, _logger); + + result.Passed.Should().BeTrue(because: "all expected scopes are present in Entra"); + result.IsWarning.Should().BeFalse(); + result.Details.Should().Contain("User.Read", + because: "details should list the configured scopes"); + } + + [Fact] + public async Task CheckAsync_MissingScopes_ReturnsWarning() + { + var config = new Agent365Config + { + TenantId = TestTenantId, + AgentBlueprintId = TestBlueprintId, + ResourceConsents = new List + { + new() + { + ResourceName = "Microsoft Graph", + ResourceAppId = "00000003-0000-0000-c000-000000000000", + ConsentGranted = true, + Scopes = new List { "User.Read", "Mail.Read", "Mail.Send" } + } + } + }; + + SetupAppAndSpExist(); + var mockBpService = CreateMockBlueprintService(); + mockBpService.ListInheritablePermissionsAsync(TestTenantId, TestBlueprintId, Arg.Any?>(), Arg.Any()) + .Returns(new List<(string ResourceAppId, List Scopes)> + { + ("00000003-0000-0000-c000-000000000000", new List { "User.Read" }) + }); + + var check = new BlueprintRegistrationRequirementCheck(_mockGraphApiService, mockBpService); + var result = await check.CheckAsync(config, _logger); + + result.Passed.Should().BeTrue(because: "missing scopes is a warning, not a failure"); + result.IsWarning.Should().BeTrue(); + result.Details.Should().Contain("Mail.Read", + because: "warning should list the missing scopes"); + result.Details.Should().Contain("Mail.Send", + because: "warning should list all missing scopes"); + } + + [Fact] + public async Task CheckAsync_ResourceNotInEntra_ReturnsWarning() + { + var config = new Agent365Config + { + TenantId = TestTenantId, + AgentBlueprintId = TestBlueprintId, + ResourceConsents = new List + { + new() + { + ResourceName = "Agent 365 Tools", + ResourceAppId = "ea9ffc3e-8a23-4a7d-836d-234d7c7565c1", + ConsentGranted = true, + Scopes = new List { "McpServers.DASearch.All" } + } + } + }; + + SetupAppAndSpExist(); + var mockBpService = CreateMockBlueprintService(); + mockBpService.ListInheritablePermissionsAsync(TestTenantId, TestBlueprintId, Arg.Any?>(), Arg.Any()) + .Returns(new List<(string ResourceAppId, List Scopes)>()); + + var check = new BlueprintRegistrationRequirementCheck(_mockGraphApiService, mockBpService); + var result = await check.CheckAsync(config, _logger); + + result.Passed.Should().BeTrue(because: "missing resource permissions is a warning, not a failure"); + result.IsWarning.Should().BeTrue(); + result.Details.Should().Contain("Agent 365 Tools", + because: "warning should name the resource missing permissions"); + } + + [Fact] + public async Task CheckAsync_ConsentNotGranted_ReturnsWarning() + { + var config = new Agent365Config + { + TenantId = TestTenantId, + AgentBlueprintId = TestBlueprintId, + ResourceConsents = new List + { + new() + { + ResourceName = "Microsoft Graph", + ResourceAppId = "00000003-0000-0000-c000-000000000000", + ConsentGranted = false, + Scopes = new List { "User.Read" } + } + } + }; + + SetupAppAndSpExist(); + var mockBpService = CreateMockBlueprintService(); + mockBpService.ListInheritablePermissionsAsync(TestTenantId, TestBlueprintId, Arg.Any?>(), Arg.Any()) + .Returns(new List<(string ResourceAppId, List Scopes)> + { + ("00000003-0000-0000-c000-000000000000", new List { "User.Read" }) + }); + + var check = new BlueprintRegistrationRequirementCheck(_mockGraphApiService, mockBpService); + var result = await check.CheckAsync(config, _logger); + + result.Passed.Should().BeTrue(because: "consent issues are warnings, not failures"); + result.IsWarning.Should().BeTrue(); + result.Details.Should().Contain("admin consent not granted", + because: "warning should indicate consent is missing"); + } + + [Fact] + public async Task CheckAsync_NoResourceConsentsInConfig_SkipsPermissionsCheck() + { + var config = new Agent365Config + { + TenantId = TestTenantId, + AgentBlueprintId = TestBlueprintId + }; + + SetupAppAndSpExist(); + var mockBpService = CreateMockBlueprintService(); + + var check = new BlueprintRegistrationRequirementCheck(_mockGraphApiService, mockBpService); + var result = await check.CheckAsync(config, _logger); + + result.Passed.Should().BeTrue(because: "no resource consents in config means permissions check is skipped"); + result.IsWarning.Should().BeFalse(); + } + + [Fact] + public async Task CheckAsync_PermissionsCheckThrows_ReturnsWarning() + { + var config = new Agent365Config + { + TenantId = TestTenantId, + AgentBlueprintId = TestBlueprintId, + ResourceConsents = new List + { + new() + { + ResourceAppId = "00000003-0000-0000-c000-000000000000", + ConsentGranted = true, + Scopes = new List { "User.Read" } + } + } + }; + + SetupAppAndSpExist(); + var mockBpService = CreateMockBlueprintService(); + mockBpService.ListInheritablePermissionsAsync(TestTenantId, TestBlueprintId, Arg.Any?>(), Arg.Any()) + .ThrowsAsync(new HttpRequestException("Forbidden")); + + var check = new BlueprintRegistrationRequirementCheck(_mockGraphApiService, mockBpService); + var result = await check.CheckAsync(config, _logger); + + result.Passed.Should().BeTrue(because: "permissions query errors are warnings"); + result.IsWarning.Should().BeTrue(); + result.Details.Should().Contain("Permissions query failed", + because: "warning should indicate what went wrong"); + } + + [Fact] + public async Task CheckAsync_NoBlueprintService_SkipsPermissionsCheck() + { + var config = new Agent365Config + { + TenantId = TestTenantId, + AgentBlueprintId = TestBlueprintId + }; + + SetupAppAndSpExist(); + + var check = new BlueprintRegistrationRequirementCheck(_mockGraphApiService, blueprintService: null); + var result = await check.CheckAsync(config, _logger); + + result.Passed.Should().BeTrue(because: "without blueprint service, permissions check is skipped"); + result.IsWarning.Should().BeFalse(because: "skipping permissions check is not a warning"); + } } diff --git a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/TelemetryRequirementCheckTests.cs b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/TelemetryRequirementCheckTests.cs index 66103a69..b290c92e 100644 --- a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/TelemetryRequirementCheckTests.cs +++ b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/TelemetryRequirementCheckTests.cs @@ -73,23 +73,11 @@ private static string[] MakeFullAgent365Span(string operationName, bool withPare }; /// - /// Helper to build a span block with a non-Agent365 scope (but not the ignored scope). - /// These spans SHOULD be accepted by the check. + /// Helper to build a span block from the @microsoft/agents-telemetry scope. + /// With the operation-name-based filtering, these spans are now included + /// if they have a recognized gen_ai.operation.name. /// - private static string[] MakeOtherSpan(string operationName) => new[] - { - " traceId: 'aaaa028f0ee6a6cbb3b0e3c96ee96fa7',", - " instrumentationScope: {", - " name: 'microsoft-otel-langchain',", - " },", - $" 'gen_ai.operation.name': '{operationName}'," - }; - - /// - /// Helper to build a span block from the ignored @microsoft/agents-telemetry scope. - /// These spans should be EXCLUDED from validation. - /// - private static string[] MakeIgnoredScopeSpan(string operationName) => new[] + private static string[] MakeAgentsTelemetrySpan(string operationName) => new[] { " traceId: 'bbbb028f0ee6a6cbb3b0e3c96ee96fa7',", " instrumentationScope: {", @@ -165,66 +153,66 @@ public async Task CheckAsync_NoSpanOutput_ReturnsFail() result.ErrorMessage.Should().Contain("No console exporter span output detected"); } - // --- Ignored scope exclusion --- + // --- Operation-name-based filtering --- [Fact] - public async Task CheckAsync_SpansOnlyFromIgnoredScope_ReturnsFail() + public async Task CheckAsync_SpansWithNoRecognizedOperations_ReturnsFail() { - var lines = new List(); - lines.AddRange(MakeIgnoredScopeSpan("invoke_agent")); - lines.AddRange(MakeIgnoredScopeSpan("chat")); - lines.AddRange(MakeIgnoredScopeSpan("execute_tool")); - var logPath = CreateTempLogFile(lines.ToArray()); + var logPath = CreateTempLogFile(new[] + { + " traceId: 'abc',", + " instrumentationScope: {", + " name: 'SomeSdk',", + " },", + " 'gen_ai.operation.name': 'unknown_op'," + }); var check = new TelemetryRequirementCheck(logPath); var result = await check.CheckAsync(_config, _logger); - result.Passed.Should().BeFalse(because: "spans from @microsoft/agents-telemetry scope should be excluded"); - result.Details.Should().Contain("@microsoft/agents-telemetry", - because: "details should indicate the ignored scope that caused all spans to be filtered out"); + result.Passed.Should().BeFalse(because: "no spans have a recognized gen_ai.operation.name"); + result.ErrorMessage.Should().Contain("No GenAI operation spans found"); } - // --- All 3 GenAI spans from Agent365Sdk --- - [Fact] - public async Task CheckAsync_AllThreeSpansFromAgent365Sdk_ReturnsPass() + public async Task CheckAsync_AgentsTelemetryScope_IncludedWhenHasRecognizedOp() { var lines = new List(); - lines.AddRange(MakeAgent365Span("invoke_agent")); - lines.AddRange(MakeAgent365Span("chat")); - lines.AddRange(MakeAgent365Span("execute_tool")); + lines.AddRange(MakeAgentsTelemetrySpan("invoke_agent")); + lines.AddRange(MakeAgentsTelemetrySpan("chat")); + lines.AddRange(MakeAgentsTelemetrySpan("execute_tool")); var logPath = CreateTempLogFile(lines.ToArray()); var check = new TelemetryRequirementCheck(logPath); var result = await check.CheckAsync(_config, _logger); - result.Passed.Should().BeTrue(because: "all 3 required GenAI spans are present from Agent365Sdk scope"); - result.Details.Should().Contain("All required GenAI operation spans detected"); + result.Passed.Should().BeTrue( + because: "spans from any scope are accepted if they have recognized gen_ai.operation.name values"); } + // --- All 3 GenAI spans from Agent365Sdk --- + [Fact] - public async Task CheckAsync_MixedScopes_ExcludesIgnoredScope() + public async Task CheckAsync_AllThreeSpansFromAgent365Sdk_ReturnsPass() { var lines = new List(); - // Other scope has invoke_agent and chat — should count - lines.AddRange(MakeOtherSpan("invoke_agent")); - lines.AddRange(MakeOtherSpan("chat")); - // Ignored scope has execute_tool — should NOT count - lines.AddRange(MakeIgnoredScopeSpan("execute_tool")); + lines.AddRange(MakeAgent365Span("invoke_agent")); + lines.AddRange(MakeAgent365Span("chat")); + lines.AddRange(MakeAgent365Span("execute_tool")); var logPath = CreateTempLogFile(lines.ToArray()); var check = new TelemetryRequirementCheck(logPath); var result = await check.CheckAsync(_config, _logger); - result.Passed.Should().BeFalse(because: "execute_tool from @microsoft/agents-telemetry scope should not count"); - result.ErrorMessage.Should().Contain("execute_tool"); + result.Passed.Should().BeTrue(because: "all 3 required GenAI spans are present from Agent365Sdk scope"); + result.Details.Should().Contain("All required GenAI operation spans detected"); } [Fact] - public async Task CheckAsync_OtherScopes_AllAccepted() + public async Task CheckAsync_MixedScopes_AllAccepted() { var logPath = CreateTempLogFile(new[] { @@ -249,36 +237,7 @@ public async Task CheckAsync_OtherScopes_AllAccepted() var result = await check.CheckAsync(_config, _logger); - result.Passed.Should().BeTrue(because: "all scopes except @microsoft/agents-telemetry should be accepted"); - } - - [Fact] - public async Task CheckAsync_IgnoredScope_CaseInsensitiveExclusion() - { - var lines = new List(); - // Ignored scope with different casing — should still be excluded - lines.Add(" traceId: 'abc',"); - lines.Add(" instrumentationScope: {"); - lines.Add(" name: '@Microsoft/Agents-Telemetry',"); - lines.Add(" },"); - lines.Add(" 'gen_ai.operation.name': 'invoke_agent',"); - lines.Add(" traceId: 'def',"); - lines.Add(" instrumentationScope: {"); - lines.Add(" name: '@MICROSOFT/AGENTS-TELEMETRY',"); - lines.Add(" },"); - lines.Add(" 'gen_ai.operation.name': 'chat',"); - lines.Add(" traceId: 'ghi',"); - lines.Add(" instrumentationScope: {"); - lines.Add(" name: '@microsoft/agents-telemetry',"); - lines.Add(" },"); - lines.Add(" 'gen_ai.operation.name': 'execute_tool',"); - var logPath = CreateTempLogFile(lines.ToArray()); - - var check = new TelemetryRequirementCheck(logPath); - - var result = await check.CheckAsync(_config, _logger); - - result.Passed.Should().BeFalse(because: "ignored scope exclusion should be case-insensitive"); + result.Passed.Should().BeTrue(because: "spans from any scope are accepted when they have recognized operations"); } // --- Missing spans --- @@ -477,46 +436,6 @@ public async Task CheckAsync_RealWorldNodeConsoleExporter_ReturnsPass() result.Details.Should().Contain("execute_tool"); } - // --- Scope version checks --- - - [Fact] - public void HasInstrumentationScopeVersion_WithVersion_ReturnsTrue() - { - var block = new List - { - " instrumentationScope: {", - " name: 'Agent365Sdk',", - " version: '1.0.0',", - " }," - }; - - TelemetryRequirementCheck.HasInstrumentationScopeVersion(block).Should().BeTrue(); - } - - [Fact] - public void HasInstrumentationScopeVersion_SameLine_ReturnsTrue() - { - var block = new List - { - " instrumentationScope: { name: 'Agent365Sdk', version: '1.0.0' }," - }; - - TelemetryRequirementCheck.HasInstrumentationScopeVersion(block).Should().BeTrue(); - } - - [Fact] - public void HasInstrumentationScopeVersion_NoVersion_ReturnsFalse() - { - var block = new List - { - " instrumentationScope: {", - " name: 'Agent365Sdk',", - " }," - }; - - TelemetryRequirementCheck.HasInstrumentationScopeVersion(block).Should().BeFalse(); - } - // --- Parent link checks --- [Fact] @@ -641,19 +560,19 @@ public async Task CheckAsync_FullyCompliantSpans_ReturnsSuccess() } [Fact] - public async Task CheckAsync_MissingScopeVersion_ReturnsWarning() + public async Task CheckAsync_ChildSpansMissingParent_ReturnsWarning() { var lines = new List(); lines.AddRange(ResourceLines); lines.Add("{"); - lines.AddRange(MakeAgent365Span("invoke_agent")); + lines.AddRange(MakeFullAgent365Span("invoke_agent")); lines.Add("}"); lines.Add("{"); - // chat span without parent or version - lines.AddRange(MakeAgent365Span("chat")); + // chat without parentId + lines.AddRange(MakeFullAgent365Span("chat", withParent: false)); lines.Add("}"); lines.Add("{"); - lines.AddRange(MakeAgent365Span("execute_tool")); + lines.AddRange(MakeFullAgent365Span("execute_tool", withParent: true)); lines.Add("}"); var logPath = CreateTempLogFile(lines.ToArray()); @@ -661,22 +580,22 @@ public async Task CheckAsync_MissingScopeVersion_ReturnsWarning() var result = await check.CheckAsync(_config, _logger); - result.Passed.Should().BeTrue(because: "scope version missing is a warning not a failure"); + result.Passed.Should().BeTrue(because: "missing parent is a warning not a failure"); result.IsWarning.Should().BeTrue(); - result.Details.Should().Contain("version", because: "warning should mention missing scope version"); + result.Details.Should().Contain("parentId", because: "warning should mention missing parent links"); + result.Details.Should().Contain("chat"); } [Fact] - public async Task CheckAsync_ChildSpansMissingParent_ReturnsWarning() + public async Task CheckAsync_MissingResourceAttributes_ReturnsWarning() { var lines = new List(); - lines.AddRange(ResourceLines); + // No resource lines lines.Add("{"); lines.AddRange(MakeFullAgent365Span("invoke_agent")); lines.Add("}"); lines.Add("{"); - // chat without parentId - lines.AddRange(MakeFullAgent365Span("chat", withParent: false)); + lines.AddRange(MakeFullAgent365Span("chat", withParent: true)); lines.Add("}"); lines.Add("{"); lines.AddRange(MakeFullAgent365Span("execute_tool", withParent: true)); @@ -687,25 +606,73 @@ public async Task CheckAsync_ChildSpansMissingParent_ReturnsWarning() var result = await check.CheckAsync(_config, _logger); - result.Passed.Should().BeTrue(because: "missing parent is a warning not a failure"); + result.Passed.Should().BeTrue(because: "missing resource attributes is a warning not a failure"); result.IsWarning.Should().BeTrue(); - result.Details.Should().Contain("parentId", because: "warning should mention missing parent links"); - result.Details.Should().Contain("chat"); + result.Details.Should().Contain("service.name", because: "warning should list missing resource attributes"); } + // ── Python console exporter format tests ── + + /// + /// Helper to build a Python console exporter span block (JSON with snake_case keys). + /// + private static string[] MakePythonSpan(string operationName, bool withParent = false) => withParent + ? new[] + { + $" \"name\": \"{operationName} gpt-5.4-mini\",", + " \"context\": {", + $" \"trace_id\": \"0xdd1ed405c6970ac9a12f716d10348920\",", + " \"span_id\": \"0xfb587c5909a6c691\",", + " \"trace_state\": \"[]\"", + " },", + " \"kind\": \"SpanKind.INTERNAL\",", + $" \"parent_id\": \"0x9534d47ca25deef6\",", + " \"attributes\": {", + $" \"gen_ai.operation.name\": \"{operationName}\",", + " \"gen_ai.request.model\": \"gpt-5.4-mini\"", + " },", + } + : new[] + { + $" \"name\": \"{operationName} gpt-5.4-mini\",", + " \"context\": {", + $" \"trace_id\": \"0xdd1ed405c6970ac9a12f716d10348920\",", + " \"span_id\": \"0xfb587c5909a6c691\",", + " \"trace_state\": \"[]\"", + " },", + " \"kind\": \"SpanKind.INTERNAL\",", + " \"parent_id\": null,", + " \"attributes\": {", + $" \"gen_ai.operation.name\": \"{operationName}\",", + " \"gen_ai.request.model\": \"gpt-5.4-mini\"", + " },", + }; + + private static readonly string[] PythonResourceLines = new[] + { + " \"resource\": {", + " \"attributes\": {", + " \"telemetry.sdk.language\": \"python\",", + " \"telemetry.sdk.name\": \"opentelemetry\",", + " \"telemetry.sdk.version\": \"1.40.0\",", + " \"service.name\": \"pirate-agent\"", + " }", + " }" + }; + [Fact] - public async Task CheckAsync_MissingResourceAttributes_ReturnsWarning() + public async Task Python_ConsoleExporter_AllOpsPresent_Passes() { var lines = new List(); - // No resource lines + lines.AddRange(PythonResourceLines); lines.Add("{"); - lines.AddRange(MakeFullAgent365Span("invoke_agent")); + lines.AddRange(MakePythonSpan("invoke_agent")); lines.Add("}"); lines.Add("{"); - lines.AddRange(MakeFullAgent365Span("chat", withParent: true)); + lines.AddRange(MakePythonSpan("chat", withParent: true)); lines.Add("}"); lines.Add("{"); - lines.AddRange(MakeFullAgent365Span("execute_tool", withParent: true)); + lines.AddRange(MakePythonSpan("execute_tool", withParent: true)); lines.Add("}"); var logPath = CreateTempLogFile(lines.ToArray()); @@ -713,8 +680,66 @@ public async Task CheckAsync_MissingResourceAttributes_ReturnsWarning() var result = await check.CheckAsync(_config, _logger); - result.Passed.Should().BeTrue(because: "missing resource attributes is a warning not a failure"); - result.IsWarning.Should().BeTrue(); - result.Details.Should().Contain("service.name", because: "warning should list missing resource attributes"); + result.Passed.Should().BeTrue(because: "all three required GenAI operations are present in Python format"); + } + + [Fact] + public async Task Python_ConsoleExporter_MissingOps_Fails() + { + var lines = new List(); + lines.Add("{"); + lines.AddRange(MakePythonSpan("chat", withParent: true)); + lines.Add("}"); + + var logPath = CreateTempLogFile(lines.ToArray()); + var check = new TelemetryRequirementCheck(logPath); + + var result = await check.CheckAsync(_config, _logger); + + result.Passed.Should().BeFalse(because: "invoke_agent and execute_tool operations are missing"); + result.ErrorMessage.Should().Contain("invoke_agent"); + result.ErrorMessage.Should().Contain("execute_tool"); + } + + [Fact] + public async Task Python_ConsoleExporter_ChildSpan_WithoutParent_WarnsAboutMissingParent() + { + var lines = new List(); + lines.AddRange(PythonResourceLines); + lines.Add("{"); + lines.AddRange(MakePythonSpan("invoke_agent")); + lines.Add("}"); + lines.Add("{"); + // chat span without parent_id (null) + lines.AddRange(MakePythonSpan("chat")); + lines.Add("}"); + lines.Add("{"); + lines.AddRange(MakePythonSpan("execute_tool", withParent: true)); + lines.Add("}"); + + var logPath = CreateTempLogFile(lines.ToArray()); + var check = new TelemetryRequirementCheck(logPath); + + var result = await check.CheckAsync(_config, _logger); + + result.Passed.Should().BeTrue(because: "missing parent is a warning not a failure"); + result.IsWarning.Should().BeTrue(because: "chat span has null parent_id"); + result.Details.Should().Contain("chat", because: "chat span is missing parent link"); + } + + [Fact] + public void SplitIntoSpanBlocks_PythonFormat_SplitsCorrectly() + { + var lines = new List(); + lines.Add("{"); + lines.AddRange(MakePythonSpan("invoke_agent")); + lines.Add("}"); + lines.Add("{"); + lines.AddRange(MakePythonSpan("chat", withParent: true)); + lines.Add("}"); + + var blocks = TelemetryRequirementCheck.SplitIntoSpanBlocks(lines.ToArray()); + + blocks.Should().HaveCount(2, because: "two Python span JSON blocks were provided"); } } From 1d6f56a82ba42c986583edf649f8d24f00cb5500 Mon Sep 17 00:00:00 2001 From: Abbinayaa Subramanian Date: Wed, 27 May 2026 12:55:32 -0700 Subject: [PATCH 05/27] Update registration check --- .../Commands/ValidateCommand.cs | 64 +++- .../Requirements/RequirementCheckResult.cs | 39 +++ .../BlueprintRegistrationRequirementCheck.cs | 182 +++++++++-- .../ValidateReport.cs | 93 +++--- .../Commands/ValidateCommandTests.cs | 4 +- ...eprintRegistrationRequirementCheckTests.cs | 284 ++++++++++++------ 6 files changed, 484 insertions(+), 182 deletions(-) diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Commands/ValidateCommand.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Commands/ValidateCommand.cs index c971b7d7..78e8b534 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Commands/ValidateCommand.cs +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Commands/ValidateCommand.cs @@ -54,6 +54,11 @@ public static Command CreateCommand( "Launch AgentsPlayground after automated conversation turns for interactive testing"); command.AddOption(playgroundOption); + var withTenantOption = new Option( + "--with-tenant", + "Run tenant-level checks (blueprint registration, permissions, consent)"); + command.AddOption(withTenantOption); + command.SetHandler(async (InvocationContext context) => { var ct = context.GetCancellationToken(); @@ -61,6 +66,7 @@ public static Command CreateCommand( var configPath = Path.Combine(cwd, ConfigConstants.DefaultConfigFileName); var report = new ValidateReport(); var launchPlayground = context.ParseResult.GetValueForOption(playgroundOption); + var withTenant = context.ParseResult.GetValueForOption(withTenantOption); try { @@ -94,8 +100,8 @@ public static Command CreateCommand( var results = await RunChecksDetailedAsync(structuralChecks, config, logger, ct); MapResultsToTiers(results, report); - // Phase 2a: Run blueprint registration check (independent of build/boot) - if (requirementChecksOverride is null && graphApiService is not null) + // Phase 2a: Run blueprint registration check (requires --with-tenant) + if (withTenant && requirementChecksOverride is null && graphApiService is not null) { var registrationCheck = new BlueprintRegistrationRequirementCheck(graphApiService, agentBlueprintService); var registrationResults = await RunChecksDetailedAsync( @@ -103,6 +109,12 @@ public static Command CreateCommand( MapResultsToTiers(registrationResults, report); results.AddRange(registrationResults); } + else if (!withTenant && requirementChecksOverride is null) + { + report.Tiers.Blueprint = TierResult.CreateSkipped("use --with-tenant"); + report.Tiers.AgentMetrics = TierResult.CreateSkipped("use --with-tenant"); + report.Tiers.M365 = TierResult.CreateSkipped("use --with-tenant"); + } // Extract resolved uv command from build step for boot and conversation steps var buildResultEntry = results @@ -442,22 +454,41 @@ private static void MapResultsToTiers( break; case BlueprintRegistrationRequirementCheck: + var blueprintTier = new BlueprintTierResult(); if (result.IsWarning) { - report.Tiers.Blueprint = new TierResult - { - Ok = true, - Warning = result.ErrorMessage - }; + blueprintTier.Ok = true; + blueprintTier.Warning = result.ErrorMessage; } else { - report.Tiers.Blueprint = new TierResult + blueprintTier.Ok = result.Passed; + blueprintTier.Reason = result.Passed ? null : result.ErrorMessage; + } + + if (result.Metadata is not null) + { + blueprintTier.AppExists = result.Metadata.AppExists; + blueprintTier.ServicePrincipalExists = result.Metadata.ServicePrincipalExists; + blueprintTier.RegistrationExists = result.Metadata.RegistrationExists; + + if (result.Metadata.ResourcePermissions is { Count: > 0 }) { - Ok = result.Passed, - Reason = result.Passed ? null : result.ErrorMessage - }; + blueprintTier.Resources = result.Metadata.ResourcePermissions.Select(rp => + new BlueprintResourceResult + { + ResourceName = rp.ResourceName, + ResourceAppId = rp.ResourceAppId, + ExpectedScopes = rp.ExpectedScopes, + ActualScopes = rp.ActualScopes, + MissingScopes = rp.MissingScopes.Count > 0 ? rp.MissingScopes : null, + ConsentGranted = rp.ConsentGranted, + InheritablePermissionsConfigured = rp.InheritablePermissionsConfigured + }).ToList(); + } } + + report.Tiers.Blueprint = blueprintTier; break; } } @@ -500,9 +531,8 @@ private static (string Description, string Suggestion) GetCodeHealthFailureInfo( if (tiers.Conversation is { Skipped: false, Ok: false }) return "conversation"; if (tiers.Telemetry is { Skipped: false, Ok: false }) return "telemetry"; if (tiers.Blueprint is { Skipped: false, Ok: false }) return "blueprint"; - if (tiers.Mac is { Skipped: false, Ok: false }) return "mac"; + if (tiers.AgentMetrics is { Skipped: false, Ok: false }) return "agentMetrics"; if (tiers.M365 is { Skipped: false, Ok: false }) return "m365"; - if (tiers.Judge is { Skipped: false, Ok: false }) return "judge"; return null; } @@ -565,7 +595,7 @@ internal static void PrintSummary(ValidateReport report, ILogger logger) else if (failCount > 0) { logger.LogInformation( - " {FailCount} of {LocalChecks} checks failed. Run `a365 validate --fix` to attempt auto-repair.", + " {FailCount} of {LocalChecks} checks failed.", failCount, localChecks); } @@ -708,8 +738,10 @@ private static List BuildDisplayRows(ValidateReport report) rows.Add(CreateTierRow("Registered", tiers.Blueprint, "blueprint registered in Entra ID", - "run 'a365 setup blueprint' to register the blueprint")); - rows.Add(CreateTierRow("Visible in MAC", tiers.Mac, + tiers.Blueprint.Reason?.Contains("permissions/consent", StringComparison.OrdinalIgnoreCase) == true + ? "run 'a365 setup permissions' to configure inheritable permissions" + : "run 'a365 setup blueprint' to register the blueprint")); + rows.Add(CreateTierRow("Agent Metrics Visible", tiers.AgentMetrics, "app compliance checks", null)); rows.Add(CreateTierRow("Visible in M365", tiers.M365, diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementCheckResult.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementCheckResult.cs index dafb764d..8374fb33 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementCheckResult.cs +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementCheckResult.cs @@ -134,6 +134,18 @@ public sealed class RequirementCheckMetadata /// Used by the boot step to run Python agents in uv-managed projects. /// public string? ResolvedUvCommand { get; init; } + + /// Whether the blueprint application exists in Entra ID. + public bool? AppExists { get; init; } + + /// Whether a service principal exists for the blueprint. + public bool? ServicePrincipalExists { get; init; } + + /// Whether the agent registration exists (null if not configured). + public bool? RegistrationExists { get; init; } + + /// Resource permission results from comparing config vs Entra. + public List? ResourcePermissions { get; set; } } /// @@ -165,3 +177,30 @@ public sealed class ConversationTurnMetadata /// The text content of the agent's callback response, if any. public string? AgentResponseText { get; init; } } + +/// +/// Permission status for a single resource API in the blueprint registration check. +/// +public sealed class BlueprintResourcePermission +{ + /// Display name of the resource (e.g., "Microsoft Graph"). + public string ResourceName { get; init; } = string.Empty; + + /// Application ID of the resource. + public string ResourceAppId { get; init; } = string.Empty; + + /// Scopes expected from config. + public List ExpectedScopes { get; init; } = new(); + + /// Scopes actually found in Entra inheritable permissions. + public List ActualScopes { get; init; } = new(); + + /// Scopes in config but missing from Entra. + public List MissingScopes { get; init; } = new(); + + /// Whether admin consent has been granted (from config). + public bool? ConsentGranted { get; init; } + + /// Whether inheritable permissions are configured in Entra for this resource. + public bool InheritablePermissionsConfigured { get; init; } +} diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/BlueprintRegistrationRequirementCheck.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/BlueprintRegistrationRequirementCheck.cs index be8ef153..46c8d6d6 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/BlueprintRegistrationRequirementCheck.cs +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/BlueprintRegistrationRequirementCheck.cs @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +using Microsoft.Agents.A365.DevTools.Cli.Constants; +using Microsoft.Agents.A365.DevTools.Cli.Helpers; using Microsoft.Agents.A365.DevTools.Cli.Models; using Microsoft.Extensions.Logging; @@ -10,6 +12,7 @@ namespace Microsoft.Agents.A365.DevTools.Cli.Services.Requirements.RequirementCh /// Validates that the agent blueprint is registered in Microsoft Entra ID. /// Checks that the blueprint application exists, has a service principal, /// (if configured) has an agent registration, and has inheritable permissions configured. +/// Expected permissions come from a static baseline plus the tooling manifest (if present). /// Uses the same Graph API methods as query-entra. /// public class BlueprintRegistrationRequirementCheck : RequirementCheck @@ -17,6 +20,21 @@ public class BlueprintRegistrationRequirementCheck : RequirementCheck private readonly GraphApiService _graphApiService; private readonly AgentBlueprintService? _blueprintService; + /// + /// Static baseline permissions required for every agent blueprint. + /// + internal static readonly List<(string ResourceAppId, string ResourceName, string[] Scopes)> BaselinePermissions = + [ + (ConfigConstants.ObservabilityApiAppId, "Agent365 Observability", + new[] { ConfigConstants.ObservabilityApiOtelWriteScope }), + (AuthenticationConstants.MicrosoftGraphResourceAppId, "Microsoft Graph", + ConfigConstants.DefaultAgentApplicationScopes.ToArray()), + (PowerPlatformConstants.PowerPlatformApiResourceAppId, "Power Platform API", + new[] { PowerPlatformConstants.PermissionNames.ConnectivityConnectionsRead }), + (McpConstants.WorkIQToolsProdAppId, "Work IQ Tools", + new[] { "McpServersMetadata.Read.All" }), + ]; + public BlueprintRegistrationRequirementCheck(GraphApiService graphApiService, AgentBlueprintService? blueprintService = null) { _graphApiService = graphApiService ?? throw new ArgumentNullException(nameof(graphApiService)); @@ -81,10 +99,12 @@ private async Task CheckImplementationAsync( if (!appExists) { - return RequirementCheckResult.Failure( + var result = RequirementCheckResult.Failure( $"Blueprint application '{blueprintId}' not found in Entra ID", "Run 'a365 setup blueprint' to create the blueprint application, or verify the agentBlueprintId in your configuration.", details: $"No Entra application with appId '{blueprintId}' exists in tenant '{tenantId}'."); + result.Metadata = new RequirementCheckMetadata { AppExists = false }; + return result; } // Check 2: Service principal exists for the blueprint @@ -96,17 +116,21 @@ private async Task CheckImplementationAsync( catch (Exception ex) when (ex is not OperationCanceledException) { logger.LogDebug(ex, "Failed to query Entra for blueprint service principal"); - return RequirementCheckResult.Warning( + var result = RequirementCheckResult.Warning( "Blueprint application exists but could not verify service principal", details: $"Graph API query failed: {ex.Message}"); + result.Metadata = new RequirementCheckMetadata { AppExists = true }; + return result; } if (string.IsNullOrEmpty(servicePrincipalId)) { - return RequirementCheckResult.Failure( + var result = RequirementCheckResult.Failure( $"Service principal not found for blueprint '{blueprintId}'", "Run 'a365 setup blueprint' to ensure the service principal is provisioned.", details: $"Application '{blueprintId}' exists but has no service principal in tenant '{tenantId}'."); + result.Metadata = new RequirementCheckMetadata { AppExists = true, ServicePrincipalExists = false }; + return result; } // Check 3: Agent registration exists (if registrationId is configured) @@ -121,41 +145,54 @@ private async Task CheckImplementationAsync( catch (Exception ex) when (ex is not OperationCanceledException) { logger.LogDebug(ex, "Failed to query agent registration"); - return RequirementCheckResult.Warning( + var result = RequirementCheckResult.Warning( "Blueprint and service principal exist but could not verify agent registration", details: $"Agent registry query failed: {ex.Message}"); + result.Metadata = new RequirementCheckMetadata { AppExists = true, ServicePrincipalExists = true }; + return result; } if (registrationExists == false) { - return RequirementCheckResult.Failure( + var result = RequirementCheckResult.Failure( $"Agent registration '{config.AgentRegistrationId}' not found", "Run 'a365 setup all' to register the agent, or verify the agentRegistrationId in your configuration.", details: $"Blueprint '{blueprintId}' and service principal exist, but agent registration " + $"'{config.AgentRegistrationId}' was not found in the agent registry."); + result.Metadata = new RequirementCheckMetadata + { + AppExists = true, + ServicePrincipalExists = true, + RegistrationExists = false + }; + return result; } if (registrationExists == null) { - return RequirementCheckResult.Warning( + var result = RequirementCheckResult.Warning( "Blueprint registered but agent registration status is unknown", details: $"Application and service principal verified. Agent registration '{config.AgentRegistrationId}' " + "could not be confirmed (insufficient permissions or transient error)."); + result.Metadata = new RequirementCheckMetadata { AppExists = true, ServicePrincipalExists = true }; + return result; } return await BuildSuccessResult(config, blueprintId, tenantId, logger, $"Blueprint '{blueprintId}' registered with service principal and agent registration '{config.AgentRegistrationId}'.", + registrationExists: true, cancellationToken); } return await BuildSuccessResult(config, blueprintId, tenantId, logger, $"Blueprint '{blueprintId}' registered with service principal '{servicePrincipalId}'.", + registrationExists: null, cancellationToken); } /// - /// After core registration checks pass, verify inheritable permissions and consent status - /// by comparing config.ResourceConsents (expected) against what is actually in Entra. + /// After core registration checks pass, verify inheritable permissions + /// by comparing the static baseline + tooling manifest scopes against what is actually in Entra. /// Missing or mismatched permissions produce a warning (not a failure). /// private async Task BuildSuccessResult( @@ -164,13 +201,26 @@ private async Task BuildSuccessResult( string tenantId, ILogger logger, string baseDetails, + bool? registrationExists, CancellationToken cancellationToken) { - if (_blueprintService is null || config.ResourceConsents.Count == 0) + var baseMetadata = new RequirementCheckMetadata { - return RequirementCheckResult.Success(details: baseDetails); + AppExists = true, + ServicePrincipalExists = true, + RegistrationExists = registrationExists + }; + + if (_blueprintService is null) + { + var result = RequirementCheckResult.Success(details: baseDetails); + result.Metadata = baseMetadata; + return result; } + // Build expected permissions: static baseline + tooling manifest scopes + var expectedPermissions = BuildExpectedPermissions(config, logger); + List<(string ResourceAppId, List Scopes)> actualPermissions; try { @@ -180,9 +230,11 @@ private async Task BuildSuccessResult( catch (Exception ex) when (ex is not OperationCanceledException) { logger.LogDebug(ex, "Failed to query inheritable permissions"); - return RequirementCheckResult.Warning( + var result = RequirementCheckResult.Warning( "Blueprint registered but could not verify inheritable permissions", details: $"{baseDetails} Permissions query failed: {ex.Message}"); + result.Metadata = baseMetadata; + return result; } var actualByResource = actualPermissions.ToDictionary( @@ -191,16 +243,22 @@ private async Task BuildSuccessResult( StringComparer.OrdinalIgnoreCase); var warnings = new List(); + var resourcePermissionResults = new List(); - foreach (var expected in config.ResourceConsents) + foreach (var expected in expectedPermissions) { - var resourceLabel = !string.IsNullOrWhiteSpace(expected.ResourceName) - ? expected.ResourceName - : expected.ResourceAppId; - if (!actualByResource.TryGetValue(expected.ResourceAppId, out var actualScopes)) { - warnings.Add($"{resourceLabel}: no inheritable permissions configured in Entra"); + warnings.Add($"{expected.ResourceName}: no inheritable permissions configured in Entra"); + resourcePermissionResults.Add(new BlueprintResourcePermission + { + ResourceName = expected.ResourceName, + ResourceAppId = expected.ResourceAppId, + ExpectedScopes = expected.Scopes.ToList(), + ActualScopes = new List(), + MissingScopes = expected.Scopes.ToList(), + InheritablePermissionsConfigured = false + }); continue; } @@ -210,28 +268,96 @@ private async Task BuildSuccessResult( if (missingScopes.Count > 0) { - warnings.Add($"{resourceLabel}: missing scopes: {string.Join(", ", missingScopes)}"); + warnings.Add($"{expected.ResourceName}: missing scopes: {string.Join(", ", missingScopes)}"); } - if (expected.ConsentGranted is false) + resourcePermissionResults.Add(new BlueprintResourcePermission { - warnings.Add($"{resourceLabel}: admin consent not granted"); - } + ResourceName = expected.ResourceName, + ResourceAppId = expected.ResourceAppId, + ExpectedScopes = expected.Scopes.ToList(), + ActualScopes = actualScopes.ToList(), + MissingScopes = missingScopes, + InheritablePermissionsConfigured = true + }); } + baseMetadata.ResourcePermissions = resourcePermissionResults; + if (warnings.Count > 0) { - return RequirementCheckResult.Warning( + var result = RequirementCheckResult.Failure( "Blueprint registered but permissions/consent gaps detected", - details: $"{baseDetails} {string.Join(". ", warnings)}. " + - "Run 'a365 setup all' or grant consent in the Azure portal."); + "Run 'a365 setup all' or grant consent in the Azure portal.", + details: $"{baseDetails} {string.Join(". ", warnings)}."); + result.Metadata = baseMetadata; + return result; } - var scopeSummary = string.Join("; ", config.ResourceConsents.Select(r => - $"{(string.IsNullOrWhiteSpace(r.ResourceName) ? r.ResourceAppId : r.ResourceName)}: " + - $"{string.Join(", ", r.Scopes)}")); + var scopeSummary = string.Join("; ", expectedPermissions.Select(r => + $"{r.ResourceName}: {string.Join(", ", r.Scopes)}")); - return RequirementCheckResult.Success( + var successResult = RequirementCheckResult.Success( details: $"{baseDetails} Permissions verified: {scopeSummary}"); + successResult.Metadata = baseMetadata; + return successResult; + } + + /// + /// Builds the expected permission list from the static baseline plus tooling manifest scopes. + /// Scopes for the same resource app ID are merged. + /// + internal static List<(string ResourceAppId, string ResourceName, List Scopes)> BuildExpectedPermissions( + Agent365Config config, ILogger logger) + { + var merged = new Dictionary Scopes)>(StringComparer.OrdinalIgnoreCase); + + // Add static baseline + foreach (var (appId, name, scopes) in BaselinePermissions) + { + if (!merged.TryGetValue(appId, out var entry)) + { + entry = (name, new HashSet(StringComparer.OrdinalIgnoreCase)); + merged[appId] = entry; + } + + foreach (var scope in scopes) + { + entry.Scopes.Add(scope); + } + } + + // Add tooling manifest scopes (if manifest exists) + var manifestPath = Path.Combine( + config.DeploymentProjectPath ?? Directory.GetCurrentDirectory(), + McpConstants.ToolingManifestFileName); + + if (File.Exists(manifestPath)) + { + try + { + var scopesByAudience = ManifestHelper.GetScopesByAudienceAsync(manifestPath).GetAwaiter().GetResult(); + + foreach (var (audienceAppId, scopes) in scopesByAudience) + { + if (!merged.TryGetValue(audienceAppId, out var entry)) + { + entry = ("Agent 365 Tools", new HashSet(StringComparer.OrdinalIgnoreCase)); + merged[audienceAppId] = entry; + } + + foreach (var scope in scopes) + { + entry.Scopes.Add(scope); + } + } + } + catch (Exception ex) + { + logger.LogDebug(ex, "Failed to read tooling manifest at {Path}, skipping manifest scopes", manifestPath); + } + } + + return merged.Select(kvp => (kvp.Key, kvp.Value.ResourceName, kvp.Value.Scopes.OrderBy(s => s).ToList())).ToList(); } } diff --git a/src/Microsoft.Agents.A365.DevTools.Validation/ValidateReport.cs b/src/Microsoft.Agents.A365.DevTools.Validation/ValidateReport.cs index 8ea60bbd..f1bde58c 100644 --- a/src/Microsoft.Agents.A365.DevTools.Validation/ValidateReport.cs +++ b/src/Microsoft.Agents.A365.DevTools.Validation/ValidateReport.cs @@ -16,9 +16,6 @@ public sealed class ValidateReport [JsonPropertyName("tiers")] public ValidationTiers Tiers { get; set; } = new(); - [JsonPropertyName("repair")] - public RepairResult Repair { get; set; } = RepairResult.NotImplemented(); - [JsonPropertyName("summary")] public SummaryResult Summary { get; set; } = new(); @@ -68,16 +65,14 @@ public sealed class ValidationTiers public TelemetryTierResult Telemetry { get; set; } = TierResult.CreateSkipped("not yet run"); [JsonPropertyName("blueprint")] - public TierResult Blueprint { get; set; } = TierResult.CreateSkipped("not yet implemented"); + public BlueprintTierResult Blueprint { get; set; } = TierResult.CreateSkipped("not yet implemented"); - [JsonPropertyName("mac")] - public TierResult Mac { get; set; } = TierResult.CreateSkipped("not yet implemented"); + [JsonPropertyName("agentMetrics")] + public TierResult AgentMetrics { get; set; } = TierResult.CreateSkipped("not yet implemented"); [JsonPropertyName("m365")] public TierResult M365 { get; set; } = TierResult.CreateSkipped("not yet implemented"); - [JsonPropertyName("judge")] - public TierResult Judge { get; set; } = TierResult.CreateSkipped("not yet implemented"); } /// @@ -234,6 +229,60 @@ public sealed class TelemetryTierResult : TierResult public List? MissingResourceAttributes { get; set; } } +/// +/// Blueprint tier: Entra registration, permissions, and consent validation. +/// +public sealed class BlueprintTierResult : TierResult +{ + [JsonPropertyName("appExists")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public bool? AppExists { get; set; } + + [JsonPropertyName("servicePrincipalExists")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public bool? ServicePrincipalExists { get; set; } + + [JsonPropertyName("registrationExists")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public bool? RegistrationExists { get; set; } + + [JsonPropertyName("resources")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public List? Resources { get; set; } +} + +/// +/// Permission and consent status for a single resource API in the blueprint. +/// +public sealed class BlueprintResourceResult +{ + [JsonPropertyName("resourceName")] + public string ResourceName { get; set; } = string.Empty; + + [JsonPropertyName("resourceAppId")] + public string ResourceAppId { get; set; } = string.Empty; + + [JsonPropertyName("expectedScopes")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public List? ExpectedScopes { get; set; } + + [JsonPropertyName("actualScopes")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public List? ActualScopes { get; set; } + + [JsonPropertyName("missingScopes")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public List? MissingScopes { get; set; } + + [JsonPropertyName("consentGranted")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public bool? ConsentGranted { get; set; } + + [JsonPropertyName("inheritablePermissionsConfigured")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public bool? InheritablePermissionsConfigured { get; set; } +} + /// /// Result of a single conversation turn. /// @@ -270,34 +319,6 @@ public sealed class ConversationTurnResult public string? AgentResponseText { get; set; } } -/// -/// Repair result (not yet implemented). -/// -public sealed class RepairResult -{ - [JsonPropertyName("skipped")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] - public bool Skipped { get; set; } - - [JsonPropertyName("reason")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? Reason { get; set; } - - [JsonPropertyName("iterations")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public int? Iterations { get; set; } - - [JsonPropertyName("patches")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public List? Patches { get; set; } - - [JsonPropertyName("finalOk")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public bool? FinalOk { get; set; } - - public static RepairResult NotImplemented() => new() { Skipped = true, Reason = "not yet implemented" }; -} - /// /// Summary of the validation run. /// diff --git a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Commands/ValidateCommandTests.cs b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Commands/ValidateCommandTests.cs index fa8d2095..83c0cefd 100644 --- a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Commands/ValidateCommandTests.cs +++ b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Commands/ValidateCommandTests.cs @@ -182,9 +182,7 @@ public async Task ValidateCommand_Report_HasSkippedUnimplementedTiers() report!.Tiers.Conversation.Skipped.Should().BeTrue(); report.Tiers.Telemetry.Skipped.Should().BeTrue(); report.Tiers.Blueprint.Skipped.Should().BeTrue(); - report.Tiers.Mac.Skipped.Should().BeTrue(); + report.Tiers.AgentMetrics.Skipped.Should().BeTrue(); report.Tiers.M365.Skipped.Should().BeTrue(); - report.Tiers.Judge.Skipped.Should().BeTrue(); - report.Repair.Skipped.Should().BeTrue(); } } \ No newline at end of file diff --git a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/BlueprintRegistrationRequirementCheckTests.cs b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/BlueprintRegistrationRequirementCheckTests.cs index 85ef089c..2900fd6d 100644 --- a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/BlueprintRegistrationRequirementCheckTests.cs +++ b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/BlueprintRegistrationRequirementCheckTests.cs @@ -2,6 +2,7 @@ // Licensed under the MIT License. using FluentAssertions; +using Microsoft.Agents.A365.DevTools.Cli.Constants; using Microsoft.Agents.A365.DevTools.Cli.Models; using Microsoft.Agents.A365.DevTools.Cli.Services; using Microsoft.Agents.A365.DevTools.Cli.Services.Requirements.RequirementChecks; @@ -310,150 +311,212 @@ private void SetupAppAndSpExist() } [Fact] - public async Task CheckAsync_WithPermissions_IncludesScopesInDetails() + public async Task CheckAsync_AllBaselinePermissionsPresent_ReturnsSuccess() { var config = new Agent365Config { TenantId = TestTenantId, - AgentBlueprintId = TestBlueprintId, - ResourceConsents = new List - { - new() - { - ResourceName = "Microsoft Graph", - ResourceAppId = "00000003-0000-0000-c000-000000000000", - ConsentGranted = true, - Scopes = new List { "User.Read", "Mail.Read" } - } - } + AgentBlueprintId = TestBlueprintId }; SetupAppAndSpExist(); var mockBpService = CreateMockBlueprintService(); mockBpService.ListInheritablePermissionsAsync(TestTenantId, TestBlueprintId, Arg.Any?>(), Arg.Any()) - .Returns(new List<(string ResourceAppId, List Scopes)> - { - ("00000003-0000-0000-c000-000000000000", new List { "User.Read", "Mail.Read" }) - }); + .Returns(BlueprintRegistrationRequirementCheck.BaselinePermissions.Select(b => + (b.ResourceAppId, b.Scopes.ToList())).ToList()); var check = new BlueprintRegistrationRequirementCheck(_mockGraphApiService, mockBpService); var result = await check.CheckAsync(config, _logger); - result.Passed.Should().BeTrue(because: "all expected scopes are present in Entra"); + result.Passed.Should().BeTrue(because: "all baseline scopes are present in Entra"); result.IsWarning.Should().BeFalse(); - result.Details.Should().Contain("User.Read", - because: "details should list the configured scopes"); + result.Details.Should().Contain("Permissions verified", + because: "details should confirm permissions were verified"); } [Fact] - public async Task CheckAsync_MissingScopes_ReturnsWarning() + public async Task CheckAsync_MissingGraphScopes_ReturnsFail() { var config = new Agent365Config { TenantId = TestTenantId, - AgentBlueprintId = TestBlueprintId, - ResourceConsents = new List - { - new() - { - ResourceName = "Microsoft Graph", - ResourceAppId = "00000003-0000-0000-c000-000000000000", - ConsentGranted = true, - Scopes = new List { "User.Read", "Mail.Read", "Mail.Send" } - } - } + AgentBlueprintId = TestBlueprintId }; SetupAppAndSpExist(); var mockBpService = CreateMockBlueprintService(); + + // Return all baseline resources but Graph only has User.Read (missing other scopes) + var actual = BlueprintRegistrationRequirementCheck.BaselinePermissions.Select(b => + b.ResourceAppId == AuthenticationConstants.MicrosoftGraphResourceAppId + ? (b.ResourceAppId, new List { "User.Read.All" }) + : (b.ResourceAppId, b.Scopes.ToList())).ToList(); mockBpService.ListInheritablePermissionsAsync(TestTenantId, TestBlueprintId, Arg.Any?>(), Arg.Any()) - .Returns(new List<(string ResourceAppId, List Scopes)> - { - ("00000003-0000-0000-c000-000000000000", new List { "User.Read" }) - }); + .Returns(actual); var check = new BlueprintRegistrationRequirementCheck(_mockGraphApiService, mockBpService); var result = await check.CheckAsync(config, _logger); - result.Passed.Should().BeTrue(because: "missing scopes is a warning, not a failure"); - result.IsWarning.Should().BeTrue(); - result.Details.Should().Contain("Mail.Read", - because: "warning should list the missing scopes"); - result.Details.Should().Contain("Mail.Send", - because: "warning should list all missing scopes"); + result.Passed.Should().BeFalse(because: "missing scopes should fail the tier"); + result.ErrorMessage.Should().Contain("gaps detected", + because: "error should describe permission gaps"); + result.Details.Should().Contain("Mail.ReadWrite", + because: "details should list one of the missing Graph scopes"); } [Fact] - public async Task CheckAsync_ResourceNotInEntra_ReturnsWarning() + public async Task CheckAsync_ResourceNotInEntra_ReturnsFail() { var config = new Agent365Config { TenantId = TestTenantId, - AgentBlueprintId = TestBlueprintId, - ResourceConsents = new List - { - new() - { - ResourceName = "Agent 365 Tools", - ResourceAppId = "ea9ffc3e-8a23-4a7d-836d-234d7c7565c1", - ConsentGranted = true, - Scopes = new List { "McpServers.DASearch.All" } - } - } + AgentBlueprintId = TestBlueprintId }; SetupAppAndSpExist(); var mockBpService = CreateMockBlueprintService(); + // Return empty — no resources configured at all mockBpService.ListInheritablePermissionsAsync(TestTenantId, TestBlueprintId, Arg.Any?>(), Arg.Any()) .Returns(new List<(string ResourceAppId, List Scopes)>()); var check = new BlueprintRegistrationRequirementCheck(_mockGraphApiService, mockBpService); var result = await check.CheckAsync(config, _logger); - result.Passed.Should().BeTrue(because: "missing resource permissions is a warning, not a failure"); - result.IsWarning.Should().BeTrue(); - result.Details.Should().Contain("Agent 365 Tools", - because: "warning should name the resource missing permissions"); + result.Passed.Should().BeFalse(because: "missing resource permissions should fail the tier"); + result.Details.Should().Contain("no inheritable permissions configured in Entra", + because: "details should indicate resources are missing"); } [Fact] - public async Task CheckAsync_ConsentNotGranted_ReturnsWarning() + public async Task CheckAsync_PermissionsCheckThrows_ReturnsWarning() { var config = new Agent365Config { TenantId = TestTenantId, - AgentBlueprintId = TestBlueprintId, - ResourceConsents = new List - { - new() - { - ResourceName = "Microsoft Graph", - ResourceAppId = "00000003-0000-0000-c000-000000000000", - ConsentGranted = false, - Scopes = new List { "User.Read" } - } - } + AgentBlueprintId = TestBlueprintId }; SetupAppAndSpExist(); var mockBpService = CreateMockBlueprintService(); mockBpService.ListInheritablePermissionsAsync(TestTenantId, TestBlueprintId, Arg.Any?>(), Arg.Any()) - .Returns(new List<(string ResourceAppId, List Scopes)> - { - ("00000003-0000-0000-c000-000000000000", new List { "User.Read" }) - }); + .ThrowsAsync(new HttpRequestException("Forbidden")); var check = new BlueprintRegistrationRequirementCheck(_mockGraphApiService, mockBpService); var result = await check.CheckAsync(config, _logger); - result.Passed.Should().BeTrue(because: "consent issues are warnings, not failures"); + result.Passed.Should().BeTrue(because: "permissions query errors are warnings"); result.IsWarning.Should().BeTrue(); - result.Details.Should().Contain("admin consent not granted", - because: "warning should indicate consent is missing"); + result.Details.Should().Contain("Permissions query failed", + because: "warning should indicate what went wrong"); } [Fact] - public async Task CheckAsync_NoResourceConsentsInConfig_SkipsPermissionsCheck() + public async Task CheckAsync_NoBlueprintService_SkipsPermissionsCheck() + { + var config = new Agent365Config + { + TenantId = TestTenantId, + AgentBlueprintId = TestBlueprintId + }; + + SetupAppAndSpExist(); + + var check = new BlueprintRegistrationRequirementCheck(_mockGraphApiService, blueprintService: null); + var result = await check.CheckAsync(config, _logger); + + result.Passed.Should().BeTrue(because: "without blueprint service, permissions check is skipped"); + result.IsWarning.Should().BeFalse(because: "skipping permissions check is not a warning"); + } + + // --- Metadata population --- + + [Fact] + public async Task CheckAsync_AppNotFound_MetadataHasAppExistsFalse() + { + var config = new Agent365Config + { + TenantId = TestTenantId, + AgentBlueprintId = TestBlueprintId + }; + + _mockGraphApiService.ApplicationExistsByAppIdAsync(TestTenantId, TestBlueprintId, Arg.Any()) + .Returns(false); + + var check = new BlueprintRegistrationRequirementCheck(_mockGraphApiService); + var result = await check.CheckAsync(config, _logger); + + result.Metadata.Should().NotBeNull(because: "metadata should be set on failure results"); + result.Metadata!.AppExists.Should().BeFalse(because: "app does not exist in Entra"); + } + + [Fact] + public async Task CheckAsync_NoServicePrincipal_MetadataHasAppTrueSpFalse() + { + var config = new Agent365Config + { + TenantId = TestTenantId, + AgentBlueprintId = TestBlueprintId + }; + + _mockGraphApiService.ApplicationExistsByAppIdAsync(TestTenantId, TestBlueprintId, Arg.Any()) + .Returns(true); + _mockGraphApiService.LookupServicePrincipalByAppIdAsync(TestTenantId, TestBlueprintId, Arg.Any(), Arg.Any?>()) + .Returns((string?)null); + + var check = new BlueprintRegistrationRequirementCheck(_mockGraphApiService); + var result = await check.CheckAsync(config, _logger); + + result.Metadata.Should().NotBeNull(); + result.Metadata!.AppExists.Should().BeTrue(because: "app exists"); + result.Metadata.ServicePrincipalExists.Should().BeFalse(because: "SP does not exist"); + } + + [Fact] + public async Task CheckAsync_RegistrationNotFound_MetadataHasRegistrationFalse() + { + var config = new Agent365Config + { + TenantId = TestTenantId, + AgentBlueprintId = TestBlueprintId, + AgentRegistrationId = TestRegistrationId + }; + + SetupAppAndSpExist(); + _mockGraphApiService.AgentRegistrationExistsAsync(TestTenantId, TestRegistrationId, Arg.Any()) + .Returns(false); + + var check = new BlueprintRegistrationRequirementCheck(_mockGraphApiService); + var result = await check.CheckAsync(config, _logger); + + result.Metadata.Should().NotBeNull(); + result.Metadata!.AppExists.Should().BeTrue(); + result.Metadata.ServicePrincipalExists.Should().BeTrue(); + result.Metadata.RegistrationExists.Should().BeFalse(because: "registration was not found"); + } + + [Fact] + public async Task CheckAsync_Success_MetadataHasAllTrue() + { + var config = new Agent365Config + { + TenantId = TestTenantId, + AgentBlueprintId = TestBlueprintId + }; + + SetupAppAndSpExist(); + + var check = new BlueprintRegistrationRequirementCheck(_mockGraphApiService); + var result = await check.CheckAsync(config, _logger); + + result.Metadata.Should().NotBeNull(because: "metadata should be set on success results"); + result.Metadata!.AppExists.Should().BeTrue(); + result.Metadata.ServicePrincipalExists.Should().BeTrue(); + result.Metadata.RegistrationExists.Should().BeNull( + because: "no registration ID was configured, so registration check was skipped"); + } + + [Fact] + public async Task CheckAsync_WithPermissions_MetadataHasResourceDetails() { var config = new Agent365Config { @@ -463,48 +526,58 @@ public async Task CheckAsync_NoResourceConsentsInConfig_SkipsPermissionsCheck() SetupAppAndSpExist(); var mockBpService = CreateMockBlueprintService(); + // Return all baseline resources with their full scopes + mockBpService.ListInheritablePermissionsAsync(TestTenantId, TestBlueprintId, Arg.Any?>(), Arg.Any()) + .Returns(BlueprintRegistrationRequirementCheck.BaselinePermissions.Select(b => + (b.ResourceAppId, b.Scopes.ToList())).ToList()); var check = new BlueprintRegistrationRequirementCheck(_mockGraphApiService, mockBpService); var result = await check.CheckAsync(config, _logger); - result.Passed.Should().BeTrue(because: "no resource consents in config means permissions check is skipped"); - result.IsWarning.Should().BeFalse(); + result.Metadata.Should().NotBeNull(); + result.Metadata!.ResourcePermissions.Should().NotBeNull(); + + var graphResource = result.Metadata.ResourcePermissions! + .FirstOrDefault(r => r.ResourceAppId == AuthenticationConstants.MicrosoftGraphResourceAppId); + graphResource.Should().NotBeNull(because: "Microsoft Graph is a baseline resource"); + graphResource!.ResourceName.Should().Be("Microsoft Graph"); + graphResource.MissingScopes.Should().BeEmpty(because: "all expected scopes are present"); + graphResource.InheritablePermissionsConfigured.Should().BeTrue(); } [Fact] - public async Task CheckAsync_PermissionsCheckThrows_ReturnsWarning() + public async Task CheckAsync_MissingScopes_MetadataHasMissingScopesListed() { var config = new Agent365Config { TenantId = TestTenantId, - AgentBlueprintId = TestBlueprintId, - ResourceConsents = new List - { - new() - { - ResourceAppId = "00000003-0000-0000-c000-000000000000", - ConsentGranted = true, - Scopes = new List { "User.Read" } - } - } + AgentBlueprintId = TestBlueprintId }; SetupAppAndSpExist(); var mockBpService = CreateMockBlueprintService(); + // Return Graph with only User.Read.All — missing other baseline scopes + var actual = BlueprintRegistrationRequirementCheck.BaselinePermissions.Select(b => + b.ResourceAppId == AuthenticationConstants.MicrosoftGraphResourceAppId + ? (b.ResourceAppId, new List { "User.Read.All" }) + : (b.ResourceAppId, b.Scopes.ToList())).ToList(); mockBpService.ListInheritablePermissionsAsync(TestTenantId, TestBlueprintId, Arg.Any?>(), Arg.Any()) - .ThrowsAsync(new HttpRequestException("Forbidden")); + .Returns(actual); var check = new BlueprintRegistrationRequirementCheck(_mockGraphApiService, mockBpService); var result = await check.CheckAsync(config, _logger); - result.Passed.Should().BeTrue(because: "permissions query errors are warnings"); - result.IsWarning.Should().BeTrue(); - result.Details.Should().Contain("Permissions query failed", - because: "warning should indicate what went wrong"); + result.Metadata.Should().NotBeNull(); + var graphResource = result.Metadata!.ResourcePermissions! + .First(r => r.ResourceAppId == AuthenticationConstants.MicrosoftGraphResourceAppId); + graphResource.MissingScopes.Should().Contain("Mail.ReadWrite", + because: "Mail.ReadWrite is a baseline Graph scope not returned by Entra"); + graphResource.InheritablePermissionsConfigured.Should().BeTrue( + because: "the resource exists in Entra, just missing some scopes"); } [Fact] - public async Task CheckAsync_NoBlueprintService_SkipsPermissionsCheck() + public async Task CheckAsync_ResourceNotInEntra_MetadataShowsNotConfigured() { var config = new Agent365Config { @@ -513,11 +586,24 @@ public async Task CheckAsync_NoBlueprintService_SkipsPermissionsCheck() }; SetupAppAndSpExist(); + var mockBpService = CreateMockBlueprintService(); + // Return no resources at all + mockBpService.ListInheritablePermissionsAsync(TestTenantId, TestBlueprintId, Arg.Any?>(), Arg.Any()) + .Returns(new List<(string ResourceAppId, List Scopes)>()); - var check = new BlueprintRegistrationRequirementCheck(_mockGraphApiService, blueprintService: null); + var check = new BlueprintRegistrationRequirementCheck(_mockGraphApiService, mockBpService); var result = await check.CheckAsync(config, _logger); - result.Passed.Should().BeTrue(because: "without blueprint service, permissions check is skipped"); - result.IsWarning.Should().BeFalse(because: "skipping permissions check is not a warning"); + result.Metadata.Should().NotBeNull(); + result.Metadata!.ResourcePermissions.Should().NotBeNull() + .And.HaveCountGreaterOrEqualTo(BlueprintRegistrationRequirementCheck.BaselinePermissions.Count, + because: "all baseline resources should appear in metadata even when missing from Entra"); + + var graphResource = result.Metadata.ResourcePermissions! + .First(r => r.ResourceAppId == AuthenticationConstants.MicrosoftGraphResourceAppId); + graphResource.InheritablePermissionsConfigured.Should().BeFalse( + because: "the resource was not found in Entra at all"); + graphResource.MissingScopes.Should().Contain("Mail.ReadWrite", + because: "all expected scopes are missing when resource is not configured"); } } From cacf166ef8c58357c6ebe42db730597bf06255a7 Mon Sep 17 00:00:00 2001 From: Isha Arora Date: Wed, 27 May 2026 22:03:01 -0700 Subject: [PATCH 06/27] MAC visibility validation via mcp tool call Co-authored-by: Copilot --- .../Commands/ValidateCommand.cs | 81 +- .../Constants/McpConstants.cs | 5 + .../Models/Agent365Config.cs | 44 + .../Program.cs | 2 +- .../Requirements/RequirementCheckResult.cs | 55 ++ .../MacVisibilityRequirementCheck.cs | 855 ++++++++++++++++++ .../ValidateReport.cs | 60 ++ .../MacVisibilityRequirementCheckTests.cs | 511 +++++++++++ src/a365.config.example.json | 11 +- 9 files changed, 1619 insertions(+), 5 deletions(-) create mode 100644 src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/MacVisibilityRequirementCheck.cs create mode 100644 src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/MacVisibilityRequirementCheckTests.cs diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Commands/ValidateCommand.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Commands/ValidateCommand.cs index a642b87b..d11a2210 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Commands/ValidateCommand.cs +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Commands/ValidateCommand.cs @@ -11,7 +11,7 @@ using System.CommandLine; using System.CommandLine.Invocation; using System.Text.Json; -using Microsoft.Agents.A365.DevTools.Validation; +using Microsoft.Agents.A365.DevTools.Validation; namespace Microsoft.Agents.A365.DevTools.Cli.Commands; @@ -41,6 +41,7 @@ public static Command CreateCommand( PlatformDetector? platformDetector = null, CommandExecutor? commandExecutor = null, IProcessService? processService = null, + AuthenticationService? authService = null, GraphApiService? graphApiService = null, AgentBlueprintService? agentBlueprintService = null, IEnumerable? requirementChecksOverride = null) @@ -73,6 +74,8 @@ public static Command CreateCommand( var launchPlayground = context.ParseResult.GetValueForOption(playgroundOption); var withTenant = context.ParseResult.GetValueForOption(withTenantOption); var instanceName = context.ParseResult.GetValueForOption(instanceNameOption); + var macBaselineFilePath = Path.Combine(cwd, MacVisibilityRequirementCheck.BaselineFileName); + var macBaselineCaptureFailed = false; try { @@ -167,6 +170,30 @@ public static Command CreateCommand( var bootPassed = report.Tiers.Boot is { Skipped: false, Ok: true }; if (bootPassed && requirementChecksOverride is null) { + // Capture baseline metrics before conversation for MAC visibility comparison. + if (authService is not null) + { + var baselineCapture = await MacVisibilityRequirementCheck.CaptureInitialMetricsAsync( + config, + logger, + authService, + baselineFilePath: macBaselineFilePath, + cancellationToken: ct); + + if (!baselineCapture.Passed) + { + macBaselineCaptureFailed = true; + report.Tiers.Mac = new MacTierResult + { + Ok = false, + Reason = baselineCapture.ErrorMessage, + BaselineFile = macBaselineFilePath, + BaselineMetrics = baselineCapture.Metadata?.MacBaselineMetrics, + ConversationVerified = false + }; + } + } + var conversationChecks = BuildConversationChecks(platformDetector, processService, launchPlayground, resolvedUvCommand); if (conversationChecks.Count > 0) { @@ -184,6 +211,29 @@ public static Command CreateCommand( new List { telemetryCheck }, config, logger, ct); MapResultsToTiers(telemetryResults, report); results.AddRange(telemetryResults); + + // Phase 2d: Run MAC visibility check by comparing post-conversation metrics + // to pre-conversation baseline captured earlier. + if (authService is not null && !macBaselineCaptureFailed) + { + var conversationVerified = report.Tiers.Conversation is { Skipped: false, Ok: true }; + var macCheck = new MacVisibilityRequirementCheck( + authService, + macBaselineFilePath, + conversationVerified); + var macResults = await RunChecksDetailedAsync( + new List { macCheck }, config, logger, ct); + MapResultsToTiers(macResults, report); + results.AddRange(macResults); + } + else if (authService is null) + { + report.Tiers.Mac = new MacTierResult + { + Skipped = true, + Reason = "authentication service unavailable" + }; + } } } else if (!bootPassed) @@ -199,6 +249,11 @@ public static Command CreateCommand( Skipped = true, Reason = skipReason }; + report.Tiers.Mac = new MacTierResult + { + Skipped = true, + Reason = skipReason + }; } // For test overrides, also map conversation checks @@ -530,6 +585,30 @@ private static void MapResultsToTiers( } report.Tiers.AgentMetrics = metricsTier; break; + + case MacVisibilityRequirementCheck: + report.Tiers.Mac = new MacTierResult + { + Ok = result.Passed, + Reason = result.Passed ? null : result.ErrorMessage, + Warning = result.IsWarning ? result.ErrorMessage : null, + BaselineFile = result.Metadata?.MacMetricsBaselineFile, + BaselineMetrics = result.Metadata?.MacBaselineMetrics, + CurrentMetrics = result.Metadata?.MacCurrentMetrics, + ConversationVerified = result.Metadata?.ConversationStepVerified, + Comparisons = result.Metadata?.MacMetricComparisons?.Select(c => new MacMetricComparisonResult + { + MetricKey = c.MetricKey, + Before = c.Before, + After = c.After, + Delta = c.Delta, + Increased = c.Increased, + IsExceptionRate = c.IsExceptionRate, + Passed = c.Passed, + Reason = c.Reason + }).ToList() + }; + break; } } } diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Constants/McpConstants.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Constants/McpConstants.cs index 97cb87d5..bffddd43 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Constants/McpConstants.cs +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Constants/McpConstants.cs @@ -61,6 +61,11 @@ public static string BuildPpmiIdentifierUri(string environment, string tenantId, /// public const string ToolsCallMethod = "tools/call"; + /// + /// Method name for listing MCP tools + /// + public const string ToolsListMethod = "tools/list"; + /// /// Name of the ListToolServers tool /// diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Models/Agent365Config.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Models/Agent365Config.cs index b9323f71..fe27e616 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Models/Agent365Config.cs +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Models/Agent365Config.cs @@ -165,6 +165,12 @@ private static void ValidateAuthMode(string? value, List errors) [JsonPropertyName("clientAppId")] public string ClientAppId { get; init; } = string.Empty; + /// + /// Optional local overrides for observability MCP validation calls. + /// + [JsonPropertyName("agent365ObservabilityMcpOptions")] + public Agent365ObservabilityMcpOptions? Agent365ObservabilityMcpOptions { get; init; } + /// /// Authentication pattern for the agent identity (blueprint agents only). /// Accepted values: "obo" (default), "s2s", "both". @@ -688,6 +694,7 @@ public Agent365Config WithCustomBlueprintPermissions(List +/// Optional overrides for observability MCP calls used by local validation. +/// +public sealed class Agent365ObservabilityMcpOptions +{ + /// + /// Base URL for the observability service host. The MCP path is appended automatically. + /// + [JsonPropertyName("baseUrl")] + public string? BaseUrl { get; init; } + + /// + /// Tenant ID used to construct the observability MCP endpoint. + /// + [JsonPropertyName("tenantId")] + public string? TenantId { get; init; } + + /// + /// Optional observability identifier passed to getAgentMetrics as agentObservabilityId. + /// When provided, this takes precedence over AgentName. + /// + [JsonPropertyName("agentObservabilityId")] + public string? AgentObservabilityId { get; init; } + + /// + /// Agent name passed to getAgentMetrics when validating a specific agent. + /// + [JsonPropertyName("agentName")] + public string? AgentName { get; init; } + + /// + /// App ID used as token audience when calling observability MCP endpoints. + /// + [JsonPropertyName("appId")] + public string? AppId { get; init; } +} diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Program.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Program.cs index c186ad6a..f9f9e14d 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Program.cs +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Program.cs @@ -176,7 +176,7 @@ await Task.WhenAll( var confirmationProvider = serviceProvider.GetRequiredService(); rootCommand.AddCommand(SetupCommand.CreateCommand(setupLogger, configService, executor, backendConfigurator, azureAuthValidator, platformDetector, graphApiService, agentBlueprintService, blueprintLookupService, federatedCredentialService, clientAppValidator, confirmationProvider, armApiService, resolver: bootstrapResolver)); - rootCommand.AddCommand(ValidateCommand.CreateCommand(validateLogger, configService, platformDetector, executor, processService, graphApiService, agentBlueprintService)); + rootCommand.AddCommand(ValidateCommand.CreateCommand(validateLogger, configService, platformDetector, executor, processService, authService, graphApiService, agentBlueprintService)); var manifestTemplateService = serviceProvider.GetRequiredService(); rootCommand.AddCommand(QueryEntraCommand.CreateCommand(queryEntraLogger, configService, executor, graphApiService, agentBlueprintService, resolver: bootstrapResolver)); rootCommand.AddCommand(CleanupCommand.CreateCommand(cleanupLogger, configService, backendConfigurator, executor, agentBlueprintService, confirmationProvider, federatedCredentialService, azureAuthValidator, graphApiService, resolver: bootstrapResolver)); diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementCheckResult.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementCheckResult.cs index 4c70420d..1f486d7d 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementCheckResult.cs +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementCheckResult.cs @@ -146,6 +146,61 @@ public sealed class RequirementCheckMetadata /// Resource permission results from comparing config vs Entra. public List? ResourcePermissions { get; set; } + + /// + /// Path to the persisted pre-conversation MAC metrics baseline file. + /// + public string? MacMetricsBaselineFile { get; init; } + + /// + /// Flattened numeric metrics captured before conversation. + /// + public Dictionary? MacBaselineMetrics { get; init; } + + /// + /// Flattened numeric metrics captured after conversation. + /// + public Dictionary? MacCurrentMetrics { get; init; } + + /// + /// Per-metric comparison outcome between baseline and post-conversation snapshots. + /// + public List? MacMetricComparisons { get; init; } + + /// + /// Whether conversation simulation completion was verified before MAC comparison. + /// + public bool? ConversationStepVerified { get; init; } +} + +/// +/// Comparison details for a single MAC metric. +/// +public sealed class MacMetricComparisonMetadata +{ + /// Canonical metric key (e.g., kpi.invocations.rl7). + public string MetricKey { get; init; } = string.Empty; + + /// Baseline value. + public double Before { get; init; } + + /// Post-conversation value. + public double After { get; init; } + + /// After - Before. + public double Delta { get; init; } + + /// True when delta is positive. + public bool Increased { get; init; } + + /// True when this metric is the exception-rate metric. + public bool IsExceptionRate { get; init; } + + /// Final pass/fail for this metric after applying rule exceptions. + public bool Passed { get; init; } + + /// Human-readable reason for this comparison result. + public string? Reason { get; init; } } /// diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/MacVisibilityRequirementCheck.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/MacVisibilityRequirementCheck.cs new file mode 100644 index 00000000..8c5706bf --- /dev/null +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/MacVisibilityRequirementCheck.cs @@ -0,0 +1,855 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Globalization; +using System.Net.Http.Headers; +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; +using Microsoft.Agents.A365.DevTools.Cli.Constants; +using Microsoft.Agents.A365.DevTools.Cli.Models; +using Microsoft.Agents.A365.DevTools.Cli.Services.Helpers; +using Microsoft.Agents.A365.DevTools.Cli.Services.Internal; +using Microsoft.Agents.A365.DevTools.Cli.Services.Requirements; +using Microsoft.Extensions.Logging; + +namespace Microsoft.Agents.A365.DevTools.Cli.Services.Requirements.RequirementChecks; + +/// +/// Validates that MAC visibility metrics increase after conversation simulation by +/// calling the getAgentMetrics MCP tool before and after conversation. +/// +public sealed class MacVisibilityRequirementCheck : RequirementCheck +{ + public const string BaselineFileName = "a365.metrics.baseline.json"; + public const string GetAgentMetricsToolName = "getAgentMetrics"; + public const string ObservabilityMcpServerName = "observability-mcp"; + + private readonly AuthenticationService? _authService; + private readonly string _baselineFilePath; + private readonly bool _conversationStepVerified; + private readonly string _environment; + private readonly string? _baseUrlOverride; + private readonly string? _tenantIdOverride; + private readonly string? _agentNameOverride; + private readonly HttpMessageHandler? _httpHandler; + private readonly Func>? _tokenProviderOverride; + + public MacVisibilityRequirementCheck( + AuthenticationService? authService, + string baselineFilePath, + bool conversationStepVerified, + string environment = "prod", + string? baseUrlOverride = null, + string? tenantIdOverride = null, + string? agentNameOverride = null, + HttpMessageHandler? httpHandler = null, + Func>? tokenProviderOverride = null) + { + _authService = authService; + _baselineFilePath = baselineFilePath; + _conversationStepVerified = conversationStepVerified; + _environment = environment; + _baseUrlOverride = baseUrlOverride; + _tenantIdOverride = tenantIdOverride; + _agentNameOverride = agentNameOverride; + _httpHandler = httpHandler; + _tokenProviderOverride = tokenProviderOverride; + } + + /// + public override string Name => "Visible in MAC"; + + /// + public override string Description => "Validates MAC visibility by comparing getAgentMetrics before and after conversation"; + + /// + public override string Category => "Observability"; + + /// + /// Captures pre-conversation metrics and persists them to a baseline file. + /// + public static async Task CaptureInitialMetricsAsync( + Agent365Config config, + ILogger logger, + AuthenticationService? authService, + string environment = "prod", + string? baseUrlOverride = null, + string? tenantIdOverride = null, + string? agentNameOverride = null, + string? baselineFilePath = null, + HttpMessageHandler? httpHandler = null, + Func>? tokenProviderOverride = null, + CancellationToken cancellationToken = default) + { + var filePath = string.IsNullOrWhiteSpace(baselineFilePath) + ? Path.Combine(Directory.GetCurrentDirectory(), BaselineFileName) + : baselineFilePath; + + var check = new MacVisibilityRequirementCheck( + authService, + filePath, + conversationStepVerified: true, + environment, + baseUrlOverride, + tenantIdOverride, + agentNameOverride, + httpHandler, + tokenProviderOverride); + + try + { + var endpoint = check.ResolveEndpoint(config); + var metricArgument = check.ResolveMetricsArgument(config); + var toolText = await check.CallGetAgentMetricsToolAsync(config, endpoint, metricArgument, logger, cancellationToken); + var metrics = ParseNumericMetrics(toolText); + + if (metrics.Count == 0) + { + return RequirementCheckResult.Failure( + "Could not parse any numeric values from getAgentMetrics output", + "Ensure getAgentMetrics returns KPI and/or daily-series numeric values.", + details: "No numeric metrics were found in the MCP response text."); + } + + var snapshot = new MacMetricsSnapshotFile + { + CapturedAtUtc = DateTimeOffset.UtcNow, + Endpoint = endpoint, + ToolName = GetAgentMetricsToolName, + ServerName = ObservabilityMcpServerName, + NumericMetrics = metrics + }; + + await check.WriteBaselineAsync(snapshot, logger, cancellationToken); + + return new RequirementCheckResult + { + Passed = true, + IsWarning = false, + Details = $"Captured initial MAC metrics to {filePath}", + Metadata = new RequirementCheckMetadata + { + MacMetricsBaselineFile = filePath, + MacBaselineMetrics = metrics, + ConversationStepVerified = true + } + }; + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + logger.LogWarning(ex, "Failed to capture initial MAC metrics"); + return RequirementCheckResult.Failure( + "Failed to capture initial getAgentMetrics baseline", + "Check observability MCP endpoint settings and authentication, then retry validation.", + details: ex.Message); + } + } + + /// + public override async Task CheckAsync( + Agent365Config config, + ILogger logger, + CancellationToken cancellationToken = default) + { + return await ExecuteCheckWithLoggingAsync(config, logger, CheckImplementationAsync, cancellationToken); + } + + private async Task CheckImplementationAsync( + Agent365Config config, + ILogger logger, + CancellationToken cancellationToken) + { + if (!_conversationStepVerified) + { + return RequirementCheckResult.Failure( + "Conversation simulation step is not verified as complete", + "Ensure the designated teammate completes the Playwright mock conversation successfully before running MAC comparison.", + details: "TODO: artifact-level teammate verification signal will be added later; current check requires conversation tier success."); + } + + if (string.IsNullOrWhiteSpace(_baselineFilePath) || !File.Exists(_baselineFilePath)) + { + return RequirementCheckResult.Failure( + "Initial MAC metrics baseline file not found", + "Run initial getAgentMetrics capture before post-conversation comparison.", + details: $"Baseline file path: {_baselineFilePath}"); + } + + var baseline = await ReadBaselineAsync(logger, cancellationToken); + if (baseline?.NumericMetrics is null || baseline.NumericMetrics.Count == 0) + { + return RequirementCheckResult.Failure( + "Initial MAC metrics baseline is empty or invalid", + "Regenerate the baseline and rerun validation.", + details: $"Baseline file path: {_baselineFilePath}"); + } + + var endpoint = ResolveEndpoint(config); + var metricArgument = ResolveMetricsArgument(config); + var toolText = await CallGetAgentMetricsToolAsync(config, endpoint, metricArgument, logger, cancellationToken); + var currentMetrics = ParseNumericMetrics(toolText); + + if (currentMetrics.Count == 0) + { + return RequirementCheckResult.Failure( + "Could not parse any numeric values from post-conversation getAgentMetrics output", + "Ensure getAgentMetrics output includes numeric KPI values."); + } + + var comparisons = CompareMetrics(baseline.NumericMetrics, currentMetrics); + var blockingFailures = comparisons + .Where(c => !c.Passed) + .Select(c => c.MetricKey) + .ToList(); + + var details = $"Compared {comparisons.Count} KPI metrics (exception rate excluded from increase requirement)."; + + if (blockingFailures.Count > 0) + { + return new RequirementCheckResult + { + Passed = false, + IsWarning = false, + ErrorMessage = $"MAC metrics did not increase for required fields: {string.Join(", ", blockingFailures)}", + ResolutionGuidance = "Execute the conversation simulation successfully and rerun validation.", + Details = details, + Metadata = new RequirementCheckMetadata + { + MacMetricsBaselineFile = _baselineFilePath, + MacBaselineMetrics = baseline.NumericMetrics, + MacCurrentMetrics = currentMetrics, + MacMetricComparisons = comparisons, + ConversationStepVerified = _conversationStepVerified + } + }; + } + + return new RequirementCheckResult + { + Passed = true, + IsWarning = false, + Details = details, + Metadata = new RequirementCheckMetadata + { + MacMetricsBaselineFile = _baselineFilePath, + MacBaselineMetrics = baseline.NumericMetrics, + MacCurrentMetrics = currentMetrics, + MacMetricComparisons = comparisons, + ConversationStepVerified = _conversationStepVerified + } + }; + } + + private string ResolveEndpoint(Agent365Config config) + { + var baseUrl = FirstNonEmpty( + _baseUrlOverride, + config.Agent365ObservabilityMcpOptions?.BaseUrl, + Environment.GetEnvironmentVariable("A365_OBSERVABILITY_BASE_URL")) + ?? new Uri(ConfigConstants.GetDiscoverEndpointUrl(_environment)).GetLeftPart(UriPartial.Authority); + + var tenantId = FirstNonEmpty( + _tenantIdOverride, + config.Agent365ObservabilityMcpOptions?.TenantId, + Environment.GetEnvironmentVariable("A365_OBSERVABILITY_TENANT_ID"), + config.TenantId); + + if (string.IsNullOrWhiteSpace(tenantId)) + { + throw new InvalidOperationException("Tenant ID is required for observability MCP endpoint resolution."); + } + + return $"{baseUrl.TrimEnd('/')}/observability/tenants/{Uri.EscapeDataString(tenantId)}/mcp"; + } + + private KeyValuePair ResolveMetricsArgument(Agent365Config config) + { + var agentObservabilityId = FirstNonEmpty( + config.Agent365ObservabilityMcpOptions?.AgentObservabilityId, + Environment.GetEnvironmentVariable("A365_OBSERVABILITY_AGENT_OBSERVABILITY_ID")); + + if (!string.IsNullOrWhiteSpace(agentObservabilityId)) + { + return new KeyValuePair("agentObservabilityId", agentObservabilityId); + } + + var agentName = FirstNonEmpty( + _agentNameOverride, + config.Agent365ObservabilityMcpOptions?.AgentName, + Environment.GetEnvironmentVariable("A365_OBSERVABILITY_AGENT_NAME"), + config.AgentIdentityDisplayName, + config.AgentBlueprintDisplayName); + + if (string.IsNullOrWhiteSpace(agentName)) + { + throw new InvalidOperationException( + "Agent selector is required for getAgentMetrics. Configure agent365ObservabilityMcpOptions.agentObservabilityId or agent365ObservabilityMcpOptions.agentName, set A365_OBSERVABILITY_AGENT_OBSERVABILITY_ID / A365_OBSERVABILITY_AGENT_NAME, or configure agentIdentityDisplayName."); + } + + return new KeyValuePair("agentName", agentName); + } + + private async Task ResolveAccessTokenAsync(Agent365Config config, CancellationToken cancellationToken) + { + if (_tokenProviderOverride is not null) + { + var overridden = await _tokenProviderOverride(cancellationToken); + if (string.IsNullOrWhiteSpace(overridden)) + { + throw new InvalidOperationException("Overridden token provider returned an empty token."); + } + + return overridden; + } + + var envToken = Environment.GetEnvironmentVariable("A365_OBSERVABILITY_MCP_BEARER_TOKEN"); + if (!string.IsNullOrWhiteSpace(envToken)) + { + return envToken; + } + + if (_authService is null) + { + throw new InvalidOperationException( + "Authentication service is required when no explicit observability bearer token is configured."); + } + + var audience = FirstNonEmpty( + Environment.GetEnvironmentVariable("A365_OBSERVABILITY_MCP_APP_ID"), + config.Agent365ObservabilityMcpOptions?.AppId) + ?? ConfigConstants.GetAgent365ToolsResourceAppId(_environment); + var loginHint = await AzCliHelper.ResolveLoginHintAsync(); + var token = await _authService.GetAccessTokenAsync(audience, userId: loginHint, ct: cancellationToken); + if (string.IsNullOrWhiteSpace(token)) + { + throw new InvalidOperationException("Failed to acquire access token for observability MCP call."); + } + + return token; + } + + private async Task CallGetAgentMetricsToolAsync( + Agent365Config config, + string endpoint, + KeyValuePair metricArgument, + ILogger logger, + CancellationToken cancellationToken) + { + var token = await ResolveAccessTokenAsync(config, cancellationToken); + var correlationId = HttpClientFactory.GenerateCorrelationId(); + + using var httpClient = HttpClientFactory.CreateAuthenticatedClient( + token, + correlationId: correlationId, + handler: _httpHandler); + + var toolIsAdvertised = await ProbeToolsListAsync(httpClient, endpoint, correlationId, logger, cancellationToken); + if (!toolIsAdvertised) + { + throw new InvalidOperationException( + $"MCP tools/list did not advertise required tool '{GetAgentMetricsToolName}'."); + } + + var requestPayload = new + { + jsonrpc = McpConstants.JsonRpcVersion, + id = "1", + method = McpConstants.ToolsCallMethod, + @params = new + { + name = GetAgentMetricsToolName, + arguments = new Dictionary(StringComparer.Ordinal) + { + [metricArgument.Key] = metricArgument.Value + } + } + }; + + var payloadJson = JsonSerializer.Serialize(requestPayload); + + using var request = new HttpRequestMessage(HttpMethod.Post, endpoint) + { + Content = new StringContent(payloadJson, Encoding.UTF8, McpConstants.MediaTypes.ApplicationJson) + }; + + request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue(McpConstants.MediaTypes.ApplicationJson)); + request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue(McpConstants.MediaTypes.TextEventStream)); + + using var response = await httpClient.SendAsync( + request, + HttpCompletionOption.ResponseHeadersRead, + cancellationToken); + + var content = await response.Content.ReadAsStringAsync(cancellationToken); + if (!response.IsSuccessStatusCode) + { + logger.LogWarning( + "getAgentMetrics returned non-success status {StatusCode} from {Endpoint} (CorrelationId: {CorrelationId})", + (int)response.StatusCode, + endpoint, + correlationId); + throw new InvalidOperationException( + $"getAgentMetrics call failed with status {(int)response.StatusCode}: {content}"); + } + + return ExtractToolText(content); + } + + private static async Task ProbeToolsListAsync( + HttpClient httpClient, + string endpoint, + string correlationId, + ILogger logger, + CancellationToken cancellationToken) + { + try + { + var requestPayload = new + { + jsonrpc = McpConstants.JsonRpcVersion, + id = "tools-list", + method = McpConstants.ToolsListMethod + }; + + var payloadJson = JsonSerializer.Serialize(requestPayload); + using var request = new HttpRequestMessage(HttpMethod.Post, endpoint) + { + Content = new StringContent(payloadJson, Encoding.UTF8, McpConstants.MediaTypes.ApplicationJson) + }; + + request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue(McpConstants.MediaTypes.ApplicationJson)); + request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue(McpConstants.MediaTypes.TextEventStream)); + + using var response = await httpClient.SendAsync( + request, + HttpCompletionOption.ResponseHeadersRead, + cancellationToken); + + var content = await response.Content.ReadAsStringAsync(cancellationToken); + if (!response.IsSuccessStatusCode) + { + logger.LogWarning( + "MCP tools/list probe failed with status {StatusCode} from {Endpoint} (CorrelationId: {CorrelationId})", + (int)response.StatusCode, + endpoint, + correlationId); + return false; + } + + var advertised = IsToolAdvertisedInToolsListResponse(content, GetAgentMetricsToolName); + logger.LogInformation( + "MCP tools/list advertised {ToolName}: {Advertised}", + GetAgentMetricsToolName, + advertised); + return advertised; + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + logger.LogWarning(ex, "MCP tools/list probe failed unexpectedly."); + return false; + } + } + + internal static bool IsToolAdvertisedInToolsListResponse(string responseContent, string toolName) + { + if (string.IsNullOrWhiteSpace(responseContent) || string.IsNullOrWhiteSpace(toolName)) + { + return false; + } + + try + { + var dataJson = string.Concat( + responseContent + .Split(new[] { "\r\n", "\n" }, StringSplitOptions.None) + .Where(line => line.StartsWith("data:", StringComparison.OrdinalIgnoreCase)) + .Select(line => line.Substring(5).Trim())); + + var candidate = string.IsNullOrWhiteSpace(dataJson) ? responseContent : dataJson; + var root = JsonNode.Parse(candidate); + + // MCP canonical shape: result.tools[].name + var toolNodes = root?["result"]?["tools"]?.AsArray(); + if (toolNodes is not null) + { + return toolNodes.Any(n => + string.Equals( + n?["name"]?.GetValue(), + toolName, + StringComparison.OrdinalIgnoreCase)); + } + + // Fallback shape used by some servers: result.content[].text containing JSON. + var textPayload = root?["result"]?["content"]? + .AsArray() + .Select(n => n?["text"]?.GetValue()) + .FirstOrDefault(t => !string.IsNullOrWhiteSpace(t)); + + if (!string.IsNullOrWhiteSpace(textPayload)) + { + var inner = JsonNode.Parse(textPayload); + var innerTools = inner?["tools"]?.AsArray(); + if (innerTools is not null) + { + return innerTools.Any(n => + string.Equals( + n?["name"]?.GetValue(), + toolName, + StringComparison.OrdinalIgnoreCase)); + } + } + } + catch + { + // Best-effort probe; caller will proceed with tools/call. + } + + return false; + } + + private static string? FirstNonEmpty(params string?[] values) + { + foreach (var value in values) + { + if (!string.IsNullOrWhiteSpace(value)) + { + return value; + } + } + + return null; + } + + private async Task WriteBaselineAsync( + MacMetricsSnapshotFile snapshot, + ILogger logger, + CancellationToken cancellationToken) + { + var json = JsonSerializer.Serialize(snapshot, new JsonSerializerOptions + { + WriteIndented = true + }); + + await File.WriteAllTextAsync(_baselineFilePath, json, cancellationToken); + logger.LogInformation("Wrote MAC baseline metrics to {Path}", _baselineFilePath); + } + + private async Task ReadBaselineAsync(ILogger logger, CancellationToken cancellationToken) + { + try + { + var json = await File.ReadAllTextAsync(_baselineFilePath, cancellationToken); + return JsonSerializer.Deserialize(json); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + logger.LogWarning(ex, "Failed to read baseline file {Path}", _baselineFilePath); + return null; + } + } + + internal static string ExtractToolText(string responseContent) + { + // Handle SSE envelopes first. + var candidate = ExtractJsonRpcCandidate(responseContent); + + try + { + var root = JsonNode.Parse(candidate); + var text = root?["result"]?["content"]? + .AsArray() + .Select(n => n?["text"]?.GetValue()) + .FirstOrDefault(t => !string.IsNullOrWhiteSpace(t)); + + if (!string.IsNullOrWhiteSpace(text)) + { + return text; + } + } + catch + { + // If candidate is already plain text/markdown, return as-is below. + } + + return candidate; + } + + private static string ExtractJsonRpcCandidate(string responseContent) + { + var dataJson = string.Concat( + responseContent + .Split(new[] { "\r\n", "\n" }, StringSplitOptions.None) + .Where(line => line.StartsWith("data:", StringComparison.OrdinalIgnoreCase)) + .Select(line => line.Substring(5).Trim())); + + return string.IsNullOrWhiteSpace(dataJson) ? responseContent : dataJson; + } + + internal static Dictionary ParseNumericMetrics(string toolText) + { + var metrics = new Dictionary(StringComparer.OrdinalIgnoreCase); + var lines = toolText + .Split(new[] { "\r\n", "\n" }, StringSplitOptions.None) + .Select(l => l.Trim()) + .Where(l => !string.IsNullOrWhiteSpace(l)) + .ToList(); + + ParseKpiTable(lines, metrics); + ParseDailySeriesTable(lines, metrics); + + return metrics; + } + + internal static List CompareMetrics( + IReadOnlyDictionary baseline, + IReadOnlyDictionary current) + { + var comparisons = new List(); + + foreach (var key in baseline.Keys.Where(IsRelevantComparisonMetric)) + { + if (!current.TryGetValue(key, out var currentValue)) + { + comparisons.Add(new MacMetricComparisonMetadata + { + MetricKey = key, + Before = baseline[key], + After = double.NaN, + Delta = double.NaN, + Increased = false, + IsExceptionRate = key.Contains("exception_rate", StringComparison.OrdinalIgnoreCase), + Passed = false, + Reason = "metric missing in post-conversation snapshot" + }); + continue; + } + + var before = baseline[key]; + var delta = currentValue - before; + var increased = delta > 0; + var isExceptionRate = key.Contains("exception_rate", StringComparison.OrdinalIgnoreCase); + + comparisons.Add(new MacMetricComparisonMetadata + { + MetricKey = key, + Before = before, + After = currentValue, + Delta = delta, + Increased = increased, + IsExceptionRate = isExceptionRate, + Passed = isExceptionRate || increased, + Reason = isExceptionRate + ? "exception rate does not need to increase" + : (increased ? "increased" : "did not increase") + }); + } + + return comparisons; + } + + private static bool IsRelevantComparisonMetric(string key) + { + if (!key.StartsWith("kpi.", StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + return key.EndsWith(".rl7", StringComparison.OrdinalIgnoreCase) + || key.EndsWith(".rl30", StringComparison.OrdinalIgnoreCase); + } + + private static void ParseKpiTable(List lines, IDictionary metrics) + { + var headerIndex = lines.FindIndex(l => + l.Contains('|') + && l.Contains("Metric", StringComparison.OrdinalIgnoreCase) + && l.Contains("RL7", StringComparison.OrdinalIgnoreCase) + && l.Contains("RL30", StringComparison.OrdinalIgnoreCase)); + + if (headerIndex < 0) + { + return; + } + + for (var i = headerIndex + 1; i < lines.Count; i++) + { + var line = lines[i]; + if (!line.Contains('|')) + { + break; + } + + if (line.All(c => c is '|' or '-' or ':' or ' ')) + { + continue; + } + + var cells = line.Split('|', StringSplitOptions.RemoveEmptyEntries) + .Select(c => c.Trim()) + .ToList(); + + if (cells.Count < 4) + { + continue; + } + + var metricName = NormalizeMetricName(cells[0]); + if (TryParseMetricNumber(cells[1], out var rl7)) + { + metrics[$"kpi.{metricName}.rl7"] = rl7; + } + + if (TryParseMetricNumber(cells[2], out var rl30)) + { + metrics[$"kpi.{metricName}.rl30"] = rl30; + } + + if (TryParseMetricNumber(cells[3], out var wow)) + { + metrics[$"kpi.{metricName}.wow_change_percent"] = wow; + } + } + } + + private static void ParseDailySeriesTable(List lines, IDictionary metrics) + { + var headerIndex = lines.FindIndex(l => + l.Contains('|') + && l.Contains("Date", StringComparison.OrdinalIgnoreCase) + && l.Contains("Users", StringComparison.OrdinalIgnoreCase) + && l.Contains("Invocations", StringComparison.OrdinalIgnoreCase) + && l.Contains("Sessions", StringComparison.OrdinalIgnoreCase)); + + if (headerIndex < 0) + { + return; + } + + for (var i = headerIndex + 1; i < lines.Count; i++) + { + var line = lines[i]; + if (!line.Contains('|')) + { + break; + } + + if (line.All(c => c is '|' or '-' or ':' or ' ')) + { + continue; + } + + var cells = line.Split('|', StringSplitOptions.RemoveEmptyEntries) + .Select(c => c.Trim()) + .ToList(); + + if (cells.Count < 4) + { + continue; + } + + var dateKey = NormalizeMetricName(cells[0]); + if (TryParseMetricNumber(cells[1], out var users)) + { + metrics[$"daily.{dateKey}.users"] = users; + } + + if (TryParseMetricNumber(cells[2], out var invocations)) + { + metrics[$"daily.{dateKey}.invocations"] = invocations; + } + + if (TryParseMetricNumber(cells[3], out var sessions)) + { + metrics[$"daily.{dateKey}.sessions"] = sessions; + } + } + } + + internal static bool TryParseMetricNumber(string raw, out double value) + { + value = 0; + var cleaned = raw.Trim(); + if (cleaned is "-" or "--") + { + return false; + } + + var filtered = new string(cleaned + .Where(c => char.IsDigit(c) || c is '.' or '-' or ',') + .ToArray()) + .Replace(",", string.Empty, StringComparison.Ordinal); + + return double.TryParse(filtered, NumberStyles.Float, CultureInfo.InvariantCulture, out value); + } + + private static string NormalizeMetricName(string raw) + { + var input = raw.Trim().ToLowerInvariant(); + var builder = new StringBuilder(input.Length + 8); + var pendingUnderscore = false; + + for (var i = 0; i < input.Length; i++) + { + if (i + 4 < input.Length && input.AsSpan(i, 5).SequenceEqual("(hrs)")) + { + if (builder.Length > 0) + { + builder.Append('_'); + } + + builder.Append("hrs"); + i += 4; + pendingUnderscore = false; + continue; + } + + var c = input[i]; + if (c == '%') + { + if (builder.Length > 0) + { + builder.Append('_'); + } + + builder.Append("percent"); + pendingUnderscore = false; + continue; + } + + if (c is ' ' or '-' or '/') + { + pendingUnderscore = builder.Length > 0; + continue; + } + + if (c == '.') + { + continue; + } + + if (pendingUnderscore && builder.Length > 0) + { + builder.Append('_'); + } + + builder.Append(c); + pendingUnderscore = false; + } + + return builder.ToString().Trim('_'); + } + + private sealed class MacMetricsSnapshotFile + { + public DateTimeOffset CapturedAtUtc { get; set; } + + public string Endpoint { get; set; } = string.Empty; + + public string ToolName { get; set; } = string.Empty; + + public string ServerName { get; set; } = string.Empty; + + public Dictionary NumericMetrics { get; set; } = new(StringComparer.OrdinalIgnoreCase); + } +} diff --git a/src/Microsoft.Agents.A365.DevTools.Validation/ValidateReport.cs b/src/Microsoft.Agents.A365.DevTools.Validation/ValidateReport.cs index 8224e1fa..3c614711 100644 --- a/src/Microsoft.Agents.A365.DevTools.Validation/ValidateReport.cs +++ b/src/Microsoft.Agents.A365.DevTools.Validation/ValidateReport.cs @@ -70,6 +70,9 @@ public sealed class ValidationTiers [JsonPropertyName("agentMetrics")] public AgentMetricsTierResult AgentMetrics { get; set; } = TierResult.CreateSkipped("not yet implemented"); + [JsonPropertyName("mac")] + public MacTierResult Mac { get; set; } = TierResult.CreateSkipped("not yet implemented"); + [JsonPropertyName("m365")] public TierResult M365 { get; set; } = TierResult.CreateSkipped("not yet implemented"); @@ -337,6 +340,63 @@ public sealed class AgentMetricsSnapshotResult public double? AverageLatencyMs { get; set; } } +/// +/// MAC visibility tier: compares getAgentMetrics snapshots before and after conversation. +/// +public sealed class MacTierResult : TierResult +{ + [JsonPropertyName("baselineFile")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? BaselineFile { get; set; } + + [JsonPropertyName("conversationVerified")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public bool? ConversationVerified { get; set; } + + [JsonPropertyName("baselineMetrics")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public Dictionary? BaselineMetrics { get; set; } + + [JsonPropertyName("currentMetrics")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public Dictionary? CurrentMetrics { get; set; } + + [JsonPropertyName("comparisons")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public List? Comparisons { get; set; } +} + +/// +/// Result of comparing a single MAC metric between baseline and post-conversation snapshots. +/// +public sealed class MacMetricComparisonResult +{ + [JsonPropertyName("metricKey")] + public string MetricKey { get; set; } = string.Empty; + + [JsonPropertyName("before")] + public double Before { get; set; } + + [JsonPropertyName("after")] + public double After { get; set; } + + [JsonPropertyName("delta")] + public double Delta { get; set; } + + [JsonPropertyName("increased")] + public bool Increased { get; set; } + + [JsonPropertyName("isExceptionRate")] + public bool IsExceptionRate { get; set; } + + [JsonPropertyName("passed")] + public bool Passed { get; set; } + + [JsonPropertyName("reason")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Reason { get; set; } +} + /// /// Result of a single conversation turn. /// diff --git a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/MacVisibilityRequirementCheckTests.cs b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/MacVisibilityRequirementCheckTests.cs new file mode 100644 index 00000000..e6351558 --- /dev/null +++ b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/MacVisibilityRequirementCheckTests.cs @@ -0,0 +1,511 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Net; +using System.Text; +using System.Text.Json; +using FluentAssertions; +using Microsoft.Agents.A365.DevTools.Cli.Models; +using Microsoft.Agents.A365.DevTools.Cli.Services.Requirements.RequirementChecks; +using Microsoft.Extensions.Logging; +using NSubstitute; +using Xunit; + +namespace Microsoft.Agents.A365.DevTools.Cli.Tests.Services.Requirements; + +public class MacVisibilityRequirementCheckTests : IDisposable +{ + private readonly ILogger _logger; + private readonly string _tempDir; + + public MacVisibilityRequirementCheckTests() + { + _logger = Substitute.For(); + _tempDir = Path.Combine(Path.GetTempPath(), $"a365-mac-validate-{Guid.NewGuid():N}"); + Directory.CreateDirectory(_tempDir); + } + + public void Dispose() + { + if (Directory.Exists(_tempDir)) + { + Directory.Delete(_tempDir, recursive: true); + } + } + + [Fact] + public void ParseNumericMetrics_FromMarkdown_ParsesKpiAndDailyValues() + { + var text = BuildMarkdownMetrics( + activeUsersRl7: 9, + invocationsRl7: 47, + sessionsRl7: 17, + toolExecutionsRl7: 60, + inferenceCallsRl7: 0, + runtimeHrsRl7: 0.15, + exceptionRateRl7: 0, + activeUsersRl30: 18, + invocationsRl30: 362, + sessionsRl30: 118, + toolExecutionsRl30: 1452, + inferenceCallsRl30: 431, + runtimeHrsRl30: 13.8, + exceptionRateRl30: 0); + + var parsed = MacVisibilityRequirementCheck.ParseNumericMetrics(text); + + parsed["kpi.active_users.rl7"].Should().Be(9); + parsed["kpi.invocations.rl30"].Should().Be(362); + parsed["kpi.runtime_hrs.rl7"].Should().Be(0.15); + parsed["kpi.exception_rate.rl30"].Should().Be(0); + parsed["daily.may_26.users"].Should().Be(4); + parsed["daily.may_26.invocations"].Should().Be(12); + parsed["daily.may_26.sessions"].Should().Be(4); + } + + [Fact] + public void CompareMetrics_NonExceptionMetricNotIncreased_Fails() + { + var baseline = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["kpi.invocations.rl7"] = 10, + ["kpi.exception_rate.rl7"] = 1 + }; + var current = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["kpi.invocations.rl7"] = 10, + ["kpi.exception_rate.rl7"] = 0 + }; + + var comparisons = MacVisibilityRequirementCheck.CompareMetrics(baseline, current); + + comparisons.Should().ContainSingle(c => c.MetricKey == "kpi.invocations.rl7" && !c.Passed, + because: "required non-exception metrics must strictly increase"); + comparisons.Should().ContainSingle(c => c.MetricKey == "kpi.exception_rate.rl7" && c.Passed, + because: "exception rate is exempt from increase requirement"); + } + + [Fact] + public async Task CaptureInitialMetricsAsync_WritesBaselineAndCheckAsync_PassesOnIncrease() + { + var baselinePath = Path.Combine(_tempDir, MacVisibilityRequirementCheck.BaselineFileName); + var config = new Agent365Config + { + TenantId = "11111111-1111-1111-1111-111111111111", + AgentIdentityDisplayName = "TestAgent", + Agent365ObservabilityMcpOptions = new Agent365ObservabilityMcpOptions + { + AgentObservabilityId = "11111111-2222-3333-4444-555555555555" + } + }; + + var baselineText = BuildMarkdownMetrics( + activeUsersRl7: 9, + invocationsRl7: 47, + sessionsRl7: 17, + toolExecutionsRl7: 60, + inferenceCallsRl7: 0, + runtimeHrsRl7: 0.15, + exceptionRateRl7: 0, + activeUsersRl30: 18, + invocationsRl30: 362, + sessionsRl30: 118, + toolExecutionsRl30: 1452, + inferenceCallsRl30: 431, + runtimeHrsRl30: 13.8, + exceptionRateRl30: 0); + + var postText = BuildMarkdownMetrics( + activeUsersRl7: 10, + invocationsRl7: 48, + sessionsRl7: 18, + toolExecutionsRl7: 61, + inferenceCallsRl7: 1, + runtimeHrsRl7: 0.20, + exceptionRateRl7: 0, + activeUsersRl30: 19, + invocationsRl30: 363, + sessionsRl30: 119, + toolExecutionsRl30: 1453, + inferenceCallsRl30: 432, + runtimeHrsRl30: 13.9, + exceptionRateRl30: 0); + + var captureHandler = new StaticToolResponseHandler(CreateJsonRpcResponseWithText(baselineText)); + var capture = await MacVisibilityRequirementCheck.CaptureInitialMetricsAsync( + config, + _logger, + authService: null, + environment: "prod", + baseUrlOverride: "https://example.test", + baselineFilePath: baselinePath, + httpHandler: captureHandler, + tokenProviderOverride: _ => Task.FromResult("token")); + + capture.Passed.Should().BeTrue(); + File.Exists(baselinePath).Should().BeTrue(); + + var checkHandler = new StaticToolResponseHandler(CreateJsonRpcResponseWithText(postText)); + var check = new MacVisibilityRequirementCheck( + authService: null, + baselineFilePath: baselinePath, + conversationStepVerified: true, + environment: "prod", + baseUrlOverride: "https://example.test", + httpHandler: checkHandler, + tokenProviderOverride: _ => Task.FromResult("token")); + + var result = await check.CheckAsync(config, _logger); + + result.Passed.Should().BeTrue(because: "all required KPI metrics increased and exception rate is exempt"); + result.Metadata!.MacMetricComparisons.Should().NotBeNull(); + } + + [Fact] + public async Task CheckAsync_WhenConversationNotVerified_Fails() + { + var baselinePath = Path.Combine(_tempDir, MacVisibilityRequirementCheck.BaselineFileName); + var config = new Agent365Config + { + TenantId = "11111111-1111-1111-1111-111111111111", + AgentIdentityDisplayName = "TestAgent" + }; + + var text = BuildMarkdownMetrics( + activeUsersRl7: 1, + invocationsRl7: 1, + sessionsRl7: 1, + toolExecutionsRl7: 1, + inferenceCallsRl7: 1, + runtimeHrsRl7: 1, + exceptionRateRl7: 0, + activeUsersRl30: 1, + invocationsRl30: 1, + sessionsRl30: 1, + toolExecutionsRl30: 1, + inferenceCallsRl30: 1, + runtimeHrsRl30: 1, + exceptionRateRl30: 0); + + await File.WriteAllTextAsync( + baselinePath, + "{\"capturedAtUtc\":\"2026-01-01T00:00:00Z\",\"endpoint\":\"https://example\",\"toolName\":\"getAgentMetrics\",\"serverName\":\"observability-mcp\",\"numericMetrics\":{\"kpi.invocations.rl7\":1}}", + Encoding.UTF8); + + var check = new MacVisibilityRequirementCheck( + authService: null, + baselineFilePath: baselinePath, + conversationStepVerified: false, + baseUrlOverride: "https://example.test", + httpHandler: new StaticToolResponseHandler(CreateJsonRpcResponseWithText(text)), + tokenProviderOverride: _ => Task.FromResult("token")); + + var result = await check.CheckAsync(config, _logger); + + result.Passed.Should().BeFalse(); + result.ErrorMessage.Should().Contain("Conversation simulation step is not verified"); + } + + [Fact] + public async Task CaptureInitialMetricsAsync_BlankOverrides_FallsBackToConfigValues() + { + var baselinePath = Path.Combine(_tempDir, MacVisibilityRequirementCheck.BaselineFileName); + var config = new Agent365Config + { + TenantId = "11111111-1111-1111-1111-111111111111", + AgentIdentityDisplayName = "Fallback Agent", + Agent365ObservabilityMcpOptions = new Agent365ObservabilityMcpOptions + { + BaseUrl = "https://example.test/", + TenantId = "22222222-2222-2222-2222-222222222222", + AgentName = "Configured Agent" + } + }; + + var handler = new StaticToolResponseHandler(CreateJsonRpcResponseWithText(BuildMarkdownMetrics( + activeUsersRl7: 1, + invocationsRl7: 2, + sessionsRl7: 3, + toolExecutionsRl7: 4, + inferenceCallsRl7: 5, + runtimeHrsRl7: 0.5, + exceptionRateRl7: 0, + activeUsersRl30: 10, + invocationsRl30: 20, + sessionsRl30: 30, + toolExecutionsRl30: 40, + inferenceCallsRl30: 50, + runtimeHrsRl30: 5, + exceptionRateRl30: 0))); + + var result = await MacVisibilityRequirementCheck.CaptureInitialMetricsAsync( + config, + _logger, + authService: null, + environment: "prod", + baseUrlOverride: string.Empty, + tenantIdOverride: string.Empty, + agentNameOverride: string.Empty, + baselineFilePath: baselinePath, + httpHandler: handler, + tokenProviderOverride: _ => Task.FromResult("token")); + + result.Passed.Should().BeTrue(because: "blank overrides should not block fallback to configured observability MCP values"); + } + + [Fact] + public async Task CaptureInitialMetricsAsync_ProbesToolsListBeforeCallingGetAgentMetrics() + { + var baselinePath = Path.Combine(_tempDir, MacVisibilityRequirementCheck.BaselineFileName); + var config = new Agent365Config + { + TenantId = "11111111-1111-1111-1111-111111111111", + AgentIdentityDisplayName = "TestAgent" + }; + + var handler = new SequencedMcpResponseHandler( + CreateToolsListResponse(MacVisibilityRequirementCheck.GetAgentMetricsToolName, "otherTool"), + CreateJsonRpcResponseWithText(BuildMarkdownMetrics( + activeUsersRl7: 1, + invocationsRl7: 2, + sessionsRl7: 3, + toolExecutionsRl7: 4, + inferenceCallsRl7: 5, + runtimeHrsRl7: 0.5, + exceptionRateRl7: 0, + activeUsersRl30: 10, + invocationsRl30: 20, + sessionsRl30: 30, + toolExecutionsRl30: 40, + inferenceCallsRl30: 50, + runtimeHrsRl30: 5, + exceptionRateRl30: 0))); + + var result = await MacVisibilityRequirementCheck.CaptureInitialMetricsAsync( + config, + _logger, + authService: null, + environment: "prod", + baseUrlOverride: "https://example.test", + baselineFilePath: baselinePath, + httpHandler: handler, + tokenProviderOverride: _ => Task.FromResult("token")); + + result.Passed.Should().BeTrue(); + handler.Methods.Should().ContainInOrder("tools/list", "tools/call"); + handler.ToolCallArgumentNames.Should().Contain("agentName", + because: "when agentObservabilityId is not configured, the MAC check falls back to agentName"); + HasLogMessageContaining("advertised getAgentMetrics: True").Should().BeTrue( + because: "successful discovery should record whether the server advertises getAgentMetrics"); + } + + [Fact] + public async Task CaptureInitialMetricsAsync_ToolsListFailure_FailsWithoutCallingGetAgentMetrics() + { + var baselinePath = Path.Combine(_tempDir, MacVisibilityRequirementCheck.BaselineFileName); + var config = new Agent365Config + { + TenantId = "11111111-1111-1111-1111-111111111111", + AgentIdentityDisplayName = "TestAgent" + }; + + var handler = new SequencedMcpResponseHandler( + "{\"error\":\"boom\"}", + HttpStatusCode.InternalServerError, + CreateJsonRpcResponseWithText(BuildMarkdownMetrics( + activeUsersRl7: 1, + invocationsRl7: 2, + sessionsRl7: 3, + toolExecutionsRl7: 4, + inferenceCallsRl7: 5, + runtimeHrsRl7: 0.5, + exceptionRateRl7: 0, + activeUsersRl30: 10, + invocationsRl30: 20, + sessionsRl30: 30, + toolExecutionsRl30: 40, + inferenceCallsRl30: 50, + runtimeHrsRl30: 5, + exceptionRateRl30: 0))); + + var result = await MacVisibilityRequirementCheck.CaptureInitialMetricsAsync( + config, + _logger, + authService: null, + environment: "prod", + baseUrlOverride: "https://example.test", + baselineFilePath: baselinePath, + httpHandler: handler, + tokenProviderOverride: _ => Task.FromResult("token")); + + result.Passed.Should().BeFalse(because: "tools/list must advertise getAgentMetrics before tools/call is attempted"); + result.Details.Should().Contain("required tool 'getAgentMetrics'", because: "discovery should fail fast when required tool is not listed"); + handler.Methods.Should().Equal(new[] { "tools/list" }, + because: "tools/call should not run when required tool discovery fails"); + HasLogMessageContaining("tools/list probe failed").Should().BeTrue( + because: "discovery failures should be logged distinctly from getAgentMetrics invocation failures"); + } + + private static string CreateJsonRpcResponseWithText(string text) + { + var escaped = text.Replace("\\", "\\\\", StringComparison.Ordinal) + .Replace("\"", "\\\"", StringComparison.Ordinal) + .Replace("\r", "\\r", StringComparison.Ordinal) + .Replace("\n", "\\n", StringComparison.Ordinal); + + return + "{" + + "\"jsonrpc\":\"2.0\"," + + "\"id\":\"1\"," + + "\"result\":{" + + "\"content\":[{" + + "\"text\":\"" + escaped + "\"" + + "}]" + + "}" + + "}"; + } + + private static string CreateToolsListResponse(params string[] toolNames) + { + return JsonSerializer.Serialize(new + { + jsonrpc = "2.0", + id = "tools-list", + result = new + { + tools = toolNames.Select(name => new { name }).ToArray() + } + }); + } + + private static string BuildMarkdownMetrics( + double activeUsersRl7, + double invocationsRl7, + double sessionsRl7, + double toolExecutionsRl7, + double inferenceCallsRl7, + double runtimeHrsRl7, + double exceptionRateRl7, + double activeUsersRl30, + double invocationsRl30, + double sessionsRl30, + double toolExecutionsRl30, + double inferenceCallsRl30, + double runtimeHrsRl30, + double exceptionRateRl30) + { + return + "2. getAgentMetrics - KPIs + daily time series\n\n" + + "| Metric | RL7 | RL30 | WoW Change |\n" + + "|---|---:|---:|---:|\n" + + $"| Active Users | {activeUsersRl7} | {activeUsersRl30} | -10% |\n" + + $"| Invocations | {invocationsRl7} | {invocationsRl30} | -63% |\n" + + $"| Sessions | {sessionsRl7} | {sessionsRl30} | - |\n" + + $"| Tool Executions | {toolExecutionsRl7} | {toolExecutionsRl30} | - |\n" + + $"| Inference Calls | {inferenceCallsRl7} | {inferenceCallsRl30} | - |\n" + + $"| Runtime (hrs) | {runtimeHrsRl7} | {runtimeHrsRl30} | -93.9% |\n" + + $"| Exception Rate | {exceptionRateRl7}% | {exceptionRateRl30}% | 0% |\n\n" + + "Daily time series (last 5 days):\n" + + "| Date | Users | Invocations | Sessions |\n" + + "|---|---:|---:|---:|\n" + + "| May 26 | 4 | 12 | 4 |\n"; + } + + private bool HasLogMessageContaining(string expectedText) + { + return _logger.ReceivedCalls().Any(call => + { + if (!string.Equals(call.GetMethodInfo().Name, nameof(ILogger.Log), StringComparison.Ordinal)) + { + return false; + } + + var state = call.GetArguments()[2]; + return state?.ToString()?.Contains(expectedText, StringComparison.OrdinalIgnoreCase) == true; + }); + } + + private sealed class StaticToolResponseHandler : HttpMessageHandler + { + private readonly string _responseContent; + + public StaticToolResponseHandler(string responseContent) + { + _responseContent = responseContent; + } + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + var responseContent = _responseContent; + if (request.Content is not null) + { + var requestJson = request.Content.ReadAsStringAsync(cancellationToken).GetAwaiter().GetResult(); + using var document = JsonDocument.Parse(requestJson); + var method = document.RootElement.GetProperty("method").GetString(); + if (string.Equals(method, "tools/list", StringComparison.OrdinalIgnoreCase)) + { + responseContent = CreateToolsListResponse(MacVisibilityRequirementCheck.GetAgentMetricsToolName, "otherTool"); + } + } + + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(responseContent, Encoding.UTF8, "application/json") + }; + + return Task.FromResult(response); + } + } + + private sealed class SequencedMcpResponseHandler : HttpMessageHandler + { + private readonly Queue<(HttpStatusCode StatusCode, string Content)> _responses; + + public SequencedMcpResponseHandler(string responseContent, HttpStatusCode statusCode, params string[] additionalResponseContents) + { + _responses = new Queue<(HttpStatusCode StatusCode, string Content)>(); + _responses.Enqueue((statusCode, responseContent)); + + foreach (var content in additionalResponseContents) + { + _responses.Enqueue((HttpStatusCode.OK, content)); + } + } + + public SequencedMcpResponseHandler(params string[] responseContents) + { + _responses = new Queue<(HttpStatusCode StatusCode, string Content)>( + responseContents.Select(content => (HttpStatusCode.OK, content))); + } + + public List Methods { get; } = []; + public List ToolCallArgumentNames { get; } = []; + + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + request.Content.Should().NotBeNull(); + + var requestJson = await request.Content!.ReadAsStringAsync(cancellationToken); + using var document = JsonDocument.Parse(requestJson); + var method = document.RootElement.GetProperty("method").GetString() ?? string.Empty; + Methods.Add(method); + + if (string.Equals(method, "tools/call", StringComparison.OrdinalIgnoreCase) + && document.RootElement.TryGetProperty("params", out var paramsElement) + && paramsElement.TryGetProperty("arguments", out var argumentsElement) + && argumentsElement.ValueKind == JsonValueKind.Object) + { + foreach (var property in argumentsElement.EnumerateObject()) + { + ToolCallArgumentNames.Add(property.Name); + } + } + + var (statusCode, content) = _responses.Dequeue(); + return new HttpResponseMessage(statusCode) + { + Content = new StringContent(content, Encoding.UTF8, "application/json") + }; + } + } +} diff --git a/src/a365.config.example.json b/src/a365.config.example.json index 5b5bd051..2b92a710 100644 --- a/src/a365.config.example.json +++ b/src/a365.config.example.json @@ -5,13 +5,18 @@ "resourceGroup": "your-resource-group-name", "location": "westus", "environment": "preprod", + "agent365ObservabilityMcpOptions": { + "baseUrl": "https://your-observability-host", + "agentObservabilityId": "11111111-2222-3333-4444-555555555555", + "appId": "42edbbd6-cfc9-4637-9068-ebac6df46171" + }, "appServicePlanName": "your-app-service-plan-name", "appServicePlanSku": "B1", "webAppName": "your-unique-webapp-name", - "agentIdentityDisplayName": "Your Agent Display Name", - "agentBlueprintDisplayName": "Your Blueprint Display Name", + "agentIdentityDisplayName": "Your Agent Identity Display Name", + "agentBlueprintDisplayName": "Your Blueprint App Display Name", "agentUserPrincipalName": "agentuser@yourdomain.onmicrosoft.com", - "agentUserDisplayName": "Your Agent User Display Name", + "agentUserDisplayName": "Your Agent User Account Display Name", "managerEmail": "manager@yourdomain.onmicrosoft.com", "agentUserUsageLocation": "US", "deploymentProjectPath": "/path/to/your/agent/project", From 8c8c0707cc615c898813c2ace866adb780044c36 Mon Sep 17 00:00:00 2001 From: Isha Arora Date: Wed, 27 May 2026 22:19:42 -0700 Subject: [PATCH 07/27] integrate mac visibility validation changes with AgentMetricRequirementCheck Co-authored-by: Copilot --- .../AgentMetricsRequirementCheck.cs | 152 +++++++++++++++++- .../MacVisibilityRequirementCheck.cs | 26 ++- 2 files changed, 166 insertions(+), 12 deletions(-) diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/AgentMetricsRequirementCheck.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/AgentMetricsRequirementCheck.cs index 92a0a549..15cc1dc2 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/AgentMetricsRequirementCheck.cs +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/AgentMetricsRequirementCheck.cs @@ -3,7 +3,12 @@ using Microsoft.Agents.A365.DevTools.Cli.Models; using Microsoft.Agents.A365.DevTools.Cli.Services; +using Microsoft.Agents.A365.DevTools.Cli.Constants; +using Microsoft.Agents.A365.DevTools.Cli.Services.Internal; using Microsoft.Extensions.Logging; +using System.Net.Http.Headers; +using System.Text; +using System.Text.Json; namespace Microsoft.Agents.A365.DevTools.Cli.Services.Requirements.RequirementChecks; @@ -15,6 +20,8 @@ namespace Microsoft.Agents.A365.DevTools.Cli.Services.Requirements.RequirementCh /// public class AgentMetricsRequirementCheck : RequirementCheck { + private const string GetAgentMetricsToolName = "getAgentMetrics"; + private readonly CopilotChatPlaywrightService? _playwrightService; private readonly string? _instanceName; @@ -146,19 +153,150 @@ private async Task CheckImplementationAsync( /// /// Queries agent metrics via MCP tool call. - /// This is a placeholder — the actual implementation will call an MCP server tool - /// to retrieve agent telemetry/metrics from the observability backend. /// protected internal virtual Task GetAgentMetricsAsync( Agent365Config config, ILogger logger, CancellationToken cancellationToken) { - // TODO: Replace with actual MCP tool call to retrieve agent metrics - // Expected call: invoke MCP tool "get_agent_metrics" with agent app ID - // Returns: invocation count, error count, latency percentiles, etc. - logger.LogDebug("Agent metrics MCP tool call not yet implemented — returning null placeholder"); - return Task.FromResult(null); + return GetAgentMetricsInternalAsync(config, logger, cancellationToken); + } + + private static async Task GetAgentMetricsInternalAsync( + Agent365Config config, + ILogger logger, + CancellationToken cancellationToken) + { + var token = Environment.GetEnvironmentVariable("A365_OBSERVABILITY_MCP_BEARER_TOKEN"); + if (string.IsNullOrWhiteSpace(token)) + { + logger.LogWarning("Observability MCP bearer token is not set. Configure A365_OBSERVABILITY_MCP_BEARER_TOKEN."); + return null; + } + + var endpoint = MacVisibilityRequirementCheck.ResolveEndpointForObservability(config, config.Environment); + var metricArgument = MacVisibilityRequirementCheck.ResolveMetricsArgumentForObservability(config); + var correlationId = HttpClientFactory.GenerateCorrelationId(); + + using var httpClient = HttpClientFactory.CreateAuthenticatedClient(token, correlationId: correlationId); + + var toolIsAdvertised = await MacVisibilityRequirementCheck.ProbeToolsListAsync( + httpClient, + endpoint, + correlationId, + logger, + cancellationToken); + if (!toolIsAdvertised) + { + logger.LogWarning("MCP tools/list did not advertise required tool '{ToolName}'.", GetAgentMetricsToolName); + return null; + } + + var requestPayload = new + { + jsonrpc = McpConstants.JsonRpcVersion, + id = "agent-metrics", + method = McpConstants.ToolsCallMethod, + @params = new + { + name = GetAgentMetricsToolName, + arguments = new Dictionary(StringComparer.Ordinal) + { + [metricArgument.Key] = metricArgument.Value + } + } + }; + + var payloadJson = JsonSerializer.Serialize(requestPayload); + using var request = new HttpRequestMessage(HttpMethod.Post, endpoint) + { + Content = new StringContent(payloadJson, Encoding.UTF8, McpConstants.MediaTypes.ApplicationJson) + }; + + request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue(McpConstants.MediaTypes.ApplicationJson)); + request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue(McpConstants.MediaTypes.TextEventStream)); + + using var response = await httpClient.SendAsync( + request, + HttpCompletionOption.ResponseHeadersRead, + cancellationToken); + + var content = await response.Content.ReadAsStringAsync(cancellationToken); + if (!response.IsSuccessStatusCode) + { + logger.LogWarning( + "getAgentMetrics returned non-success status {StatusCode} from {Endpoint} (CorrelationId: {CorrelationId})", + (int)response.StatusCode, + endpoint, + correlationId); + return null; + } + + var toolText = MacVisibilityRequirementCheck.ExtractToolText(content); + var metrics = MacVisibilityRequirementCheck.ParseNumericMetrics(toolText); + if (metrics.Count == 0) + { + logger.LogWarning("getAgentMetrics response did not contain numeric metrics."); + return null; + } + + return new AgentMetricsSnapshot + { + InvocationCount = ConvertToLong(SumByMetricPrefix(metrics, "kpi.invocations.")), + ErrorCount = ConvertToLong(SumByMetricPrefix(metrics, "kpi.errors.")), + AverageLatencyMs = GetFirstMetricValue(metrics, + "kpi.latency.avg", + "kpi.latency.average", + "kpi.avgLatencyMs", + "kpi.averageLatencyMs") + }; + } + + private static double SumByMetricPrefix(IReadOnlyDictionary metrics, string prefix) + { + double sum = 0; + foreach (var pair in metrics) + { + if (pair.Key.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) + { + sum += pair.Value; + } + } + + return sum; + } + + private static double? GetFirstMetricValue(IReadOnlyDictionary metrics, params string[] metricKeys) + { + foreach (var metricKey in metricKeys) + { + if (metrics.TryGetValue(metricKey, out var value)) + { + return value; + } + } + + return null; + } + + private static long ConvertToLong(double value) + { + if (double.IsNaN(value) || double.IsInfinity(value)) + { + return 0; + } + + if (value > long.MaxValue) + { + return long.MaxValue; + } + + if (value < long.MinValue) + { + return long.MinValue; + } + + return (long)Math.Round(value, MidpointRounding.AwayFromZero); } /// diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/MacVisibilityRequirementCheck.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/MacVisibilityRequirementCheck.cs index 8c5706bf..da1aae91 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/MacVisibilityRequirementCheck.cs +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/MacVisibilityRequirementCheck.cs @@ -242,15 +242,24 @@ private async Task CheckImplementationAsync( } private string ResolveEndpoint(Agent365Config config) + { + return ResolveEndpointForObservability(config, _environment, _baseUrlOverride, _tenantIdOverride); + } + + internal static string ResolveEndpointForObservability( + Agent365Config config, + string environment, + string? baseUrlOverride = null, + string? tenantIdOverride = null) { var baseUrl = FirstNonEmpty( - _baseUrlOverride, + baseUrlOverride, config.Agent365ObservabilityMcpOptions?.BaseUrl, Environment.GetEnvironmentVariable("A365_OBSERVABILITY_BASE_URL")) - ?? new Uri(ConfigConstants.GetDiscoverEndpointUrl(_environment)).GetLeftPart(UriPartial.Authority); + ?? new Uri(ConfigConstants.GetDiscoverEndpointUrl(environment)).GetLeftPart(UriPartial.Authority); var tenantId = FirstNonEmpty( - _tenantIdOverride, + tenantIdOverride, config.Agent365ObservabilityMcpOptions?.TenantId, Environment.GetEnvironmentVariable("A365_OBSERVABILITY_TENANT_ID"), config.TenantId); @@ -264,6 +273,13 @@ private string ResolveEndpoint(Agent365Config config) } private KeyValuePair ResolveMetricsArgument(Agent365Config config) + { + return ResolveMetricsArgumentForObservability(config, _agentNameOverride); + } + + internal static KeyValuePair ResolveMetricsArgumentForObservability( + Agent365Config config, + string? agentNameOverride = null) { var agentObservabilityId = FirstNonEmpty( config.Agent365ObservabilityMcpOptions?.AgentObservabilityId, @@ -275,7 +291,7 @@ private KeyValuePair ResolveMetricsArgument(Agent365Config confi } var agentName = FirstNonEmpty( - _agentNameOverride, + agentNameOverride, config.Agent365ObservabilityMcpOptions?.AgentName, Environment.GetEnvironmentVariable("A365_OBSERVABILITY_AGENT_NAME"), config.AgentIdentityDisplayName, @@ -396,7 +412,7 @@ private async Task CallGetAgentMetricsToolAsync( return ExtractToolText(content); } - private static async Task ProbeToolsListAsync( + internal static async Task ProbeToolsListAsync( HttpClient httpClient, string endpoint, string correlationId, From c6fd796bbddf90e49d94a1601d75529769c784dc Mon Sep 17 00:00:00 2001 From: Abbinayaa Subramanian Date: Thu, 28 May 2026 12:49:38 -0700 Subject: [PATCH 08/27] Remove agent metrics check for demo --- .../Commands/ValidateCommand.cs | 210 +++---- .../Services/CopilotChatPlaywrightService.cs | 540 +++++++++++------- .../AgentMetricsRequirementCheck.cs | 336 ++++++++--- .../ConversationRequirementCheck.cs | 2 +- .../ValidateReport.cs | 9 - .../Commands/ValidateCommandTests.cs | 1 - 6 files changed, 690 insertions(+), 408 deletions(-) diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Commands/ValidateCommand.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Commands/ValidateCommand.cs index d11a2210..5e680848 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Commands/ValidateCommand.cs +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Commands/ValidateCommand.cs @@ -62,7 +62,7 @@ public static Command CreateCommand( var instanceNameOption = new Option( "--instance-name", - "Agent instance display name in Copilot Chat (used for agent metrics conversation test)"); + "Agent instance display name (reserved for future agent metrics validation)"); command.AddOption(instanceNameOption); command.SetHandler(async (InvocationContext context) => @@ -73,9 +73,6 @@ public static Command CreateCommand( var report = new ValidateReport(); var launchPlayground = context.ParseResult.GetValueForOption(playgroundOption); var withTenant = context.ParseResult.GetValueForOption(withTenantOption); - var instanceName = context.ParseResult.GetValueForOption(instanceNameOption); - var macBaselineFilePath = Path.Combine(cwd, MacVisibilityRequirementCheck.BaselineFileName); - var macBaselineCaptureFailed = false; try { @@ -102,44 +99,66 @@ public static Command CreateCommand( : null }; - // Phase 2: Run structural checks (manifest + build) - var structuralChecks = requirementChecksOverride?.ToList() - ?? BuildStructuralChecks(platformDetector, commandExecutor); - - var results = await RunChecksDetailedAsync(structuralChecks, config, logger, ct); - MapResultsToTiers(results, report); - - // Phase 2a: Run blueprint registration check (requires --with-tenant) - if (withTenant && requirementChecksOverride is null && graphApiService is not null) + if (withTenant) { - var registrationCheck = new BlueprintRegistrationRequirementCheck(graphApiService, agentBlueprintService); - var registrationResults = await RunChecksDetailedAsync( - new List { registrationCheck }, config, logger, ct); - MapResultsToTiers(registrationResults, report); - results.AddRange(registrationResults); - - // Phase 2a-ii: Run agent metrics check (after blueprint, requires --with-tenant + --instance-name) - if (!string.IsNullOrWhiteSpace(instanceName)) + // --with-tenant: load previous report, run only tenant checks, merge + var existingReport = await LoadExistingReportAsync(cwd, logger); + if (existingReport is not null) { - var playwrightService = new CopilotChatPlaywrightService(logger); - var metricsCheck = new AgentMetricsRequirementCheck(playwrightService, instanceName); - var metricsResults = await RunChecksDetailedAsync( - new List { metricsCheck }, config, logger, ct); - MapResultsToTiers(metricsResults, report); - results.AddRange(metricsResults); + // Carry forward all local tiers from previous run + report.Tiers.Structural = existingReport.Tiers.Structural; + report.Tiers.Build = existingReport.Tiers.Build; + report.Tiers.Boot = existingReport.Tiers.Boot; + report.Tiers.Conversation = existingReport.Tiers.Conversation; + report.Tiers.Telemetry = existingReport.Tiers.Telemetry; + report.Agent = existingReport.Agent ?? report.Agent; + report.AgentConsoleLogFile = existingReport.AgentConsoleLogFile; } else { - report.Tiers.AgentMetrics = TierResult.CreateSkipped("use --instance-name"); + logger.LogWarning("No previous {ReportFile} found. Run 'a365 validate' first, then 'a365 validate --with-tenant'.", ReportFileName); + context.ExitCode = 1; + return; } - } - else if (!withTenant && requirementChecksOverride is null) - { - report.Tiers.Blueprint = TierResult.CreateSkipped("use --with-tenant"); - report.Tiers.AgentMetrics = TierResult.CreateSkipped("use --with-tenant"); - report.Tiers.M365 = TierResult.CreateSkipped("use --with-tenant"); + + var tenantResults = new List<(IRequirementCheck Check, RequirementCheckResult Result)>(); + + // Blueprint registration check + if (requirementChecksOverride is null && graphApiService is not null) + { + var registrationCheck = new BlueprintRegistrationRequirementCheck(graphApiService, agentBlueprintService); + var registrationResults = await RunChecksDetailedAsync( + new List { registrationCheck }, config, logger, ct); + MapResultsToTiers(registrationResults, report); + tenantResults.AddRange(registrationResults); + } + + // Summary based on all tiers (existing + new tenant) + var tenantAnyFailed = tenantResults.Any(r => !r.Result.Passed); + var tenantBlocker = FindBlocker(report.Tiers); + report.Summary = new SummaryResult + { + Ok = !tenantAnyFailed && tenantBlocker is null, + Blocker = tenantBlocker + }; + + context.ExitCode = report.Summary.Ok ? 0 : 1; + PrintSummary(report, logger); + return; } + // --- Non-tenant flow: run all local checks --- + + // Phase 2: Run structural checks (manifest + build) + var structuralChecks = requirementChecksOverride?.ToList() + ?? BuildStructuralChecks(platformDetector, commandExecutor); + + var results = await RunChecksDetailedAsync(structuralChecks, config, logger, ct); + MapResultsToTiers(results, report); + + // Mark tenant-dependent tiers as skipped + report.Tiers.Blueprint = TierResult.CreateSkipped("use --with-tenant"); + // Extract resolved uv command from build step for boot and conversation steps var buildResultEntry = results .FirstOrDefault(r => r.Check is ProjectBuildRequirementCheck); @@ -170,30 +189,6 @@ public static Command CreateCommand( var bootPassed = report.Tiers.Boot is { Skipped: false, Ok: true }; if (bootPassed && requirementChecksOverride is null) { - // Capture baseline metrics before conversation for MAC visibility comparison. - if (authService is not null) - { - var baselineCapture = await MacVisibilityRequirementCheck.CaptureInitialMetricsAsync( - config, - logger, - authService, - baselineFilePath: macBaselineFilePath, - cancellationToken: ct); - - if (!baselineCapture.Passed) - { - macBaselineCaptureFailed = true; - report.Tiers.Mac = new MacTierResult - { - Ok = false, - Reason = baselineCapture.ErrorMessage, - BaselineFile = macBaselineFilePath, - BaselineMetrics = baselineCapture.Metadata?.MacBaselineMetrics, - ConversationVerified = false - }; - } - } - var conversationChecks = BuildConversationChecks(platformDetector, processService, launchPlayground, resolvedUvCommand); if (conversationChecks.Count > 0) { @@ -201,7 +196,7 @@ public static Command CreateCommand( MapResultsToTiers(conversationResults, report); results.AddRange(conversationResults); - // Phase 2c: Run telemetry check using agent's console log file + // Run telemetry check using agent's console log file var conversationResult = conversationResults .FirstOrDefault(r => r.Check is ConversationRequirementCheck); var agentLogPath = conversationResult.Result?.Metadata?.AgentConsoleLogPath; @@ -211,29 +206,6 @@ public static Command CreateCommand( new List { telemetryCheck }, config, logger, ct); MapResultsToTiers(telemetryResults, report); results.AddRange(telemetryResults); - - // Phase 2d: Run MAC visibility check by comparing post-conversation metrics - // to pre-conversation baseline captured earlier. - if (authService is not null && !macBaselineCaptureFailed) - { - var conversationVerified = report.Tiers.Conversation is { Skipped: false, Ok: true }; - var macCheck = new MacVisibilityRequirementCheck( - authService, - macBaselineFilePath, - conversationVerified); - var macResults = await RunChecksDetailedAsync( - new List { macCheck }, config, logger, ct); - MapResultsToTiers(macResults, report); - results.AddRange(macResults); - } - else if (authService is null) - { - report.Tiers.Mac = new MacTierResult - { - Skipped = true, - Reason = "authentication service unavailable" - }; - } } } else if (!bootPassed) @@ -249,11 +221,6 @@ public static Command CreateCommand( Skipped = true, Reason = skipReason }; - report.Tiers.Mac = new MacTierResult - { - Skipped = true, - Reason = skipReason - }; } // For test overrides, also map conversation checks @@ -570,45 +537,6 @@ private static void MapResultsToTiers( report.Tiers.Blueprint = blueprintTier; break; - - case AgentMetricsRequirementCheck: - var metricsTier = new AgentMetricsTierResult(); - if (result.IsWarning) - { - metricsTier.Ok = true; - metricsTier.Warning = result.ErrorMessage; - } - else - { - metricsTier.Ok = result.Passed; - metricsTier.Reason = result.Passed ? null : result.ErrorMessage; - } - report.Tiers.AgentMetrics = metricsTier; - break; - - case MacVisibilityRequirementCheck: - report.Tiers.Mac = new MacTierResult - { - Ok = result.Passed, - Reason = result.Passed ? null : result.ErrorMessage, - Warning = result.IsWarning ? result.ErrorMessage : null, - BaselineFile = result.Metadata?.MacMetricsBaselineFile, - BaselineMetrics = result.Metadata?.MacBaselineMetrics, - CurrentMetrics = result.Metadata?.MacCurrentMetrics, - ConversationVerified = result.Metadata?.ConversationStepVerified, - Comparisons = result.Metadata?.MacMetricComparisons?.Select(c => new MacMetricComparisonResult - { - MetricKey = c.MetricKey, - Before = c.Before, - After = c.After, - Delta = c.Delta, - Increased = c.Increased, - IsExceptionRate = c.IsExceptionRate, - Passed = c.Passed, - Reason = c.Reason - }).ToList() - }; - break; } } } @@ -650,8 +578,6 @@ private static (string Description, string Suggestion) GetCodeHealthFailureInfo( if (tiers.Conversation is { Skipped: false, Ok: false }) return "conversation"; if (tiers.Telemetry is { Skipped: false, Ok: false }) return "telemetry"; if (tiers.Blueprint is { Skipped: false, Ok: false }) return "blueprint"; - if (tiers.AgentMetrics is { Skipped: false, Ok: false }) return "agentMetrics"; - if (tiers.M365 is { Skipped: false, Ok: false }) return "m365"; return null; } @@ -860,12 +786,6 @@ private static List BuildDisplayRows(ValidateReport report) tiers.Blueprint.Reason?.Contains("permissions/consent", StringComparison.OrdinalIgnoreCase) == true ? "run 'a365 setup permissions' to configure inheritable permissions" : "run 'a365 setup blueprint' to register the blueprint")); - rows.Add(CreateTierRow("Agent Metrics Visible", tiers.AgentMetrics, - "app compliance checks", - null)); - rows.Add(CreateTierRow("Visible in M365", tiers.M365, - "Teams/M365 visibility", - null)); return rows; } @@ -912,6 +832,28 @@ private static async Task WriteReportAsync(ValidateReport report, string directo } } + private static async Task LoadExistingReportAsync(string directory, ILogger logger) + { + var reportPath = Path.Combine(directory, ReportFileName); + if (!File.Exists(reportPath)) + { + return null; + } + + try + { + var json = await File.ReadAllTextAsync(reportPath); + var report = JsonSerializer.Deserialize(json, ReportSerializerOptions); + logger.LogDebug("Loaded existing report from {ReportPath}", reportPath); + return report; + } + catch (Exception ex) + { + logger.LogWarning(ex, "Failed to load existing report from {ReportPath}", reportPath); + return null; + } + } + private static string ResolveProjectPath(Agent365Config config) { return string.IsNullOrWhiteSpace(config.DeploymentProjectPath) diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Services/CopilotChatPlaywrightService.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Services/CopilotChatPlaywrightService.cs index 86f899f1..5d253f7e 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Services/CopilotChatPlaywrightService.cs +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Services/CopilotChatPlaywrightService.cs @@ -7,28 +7,28 @@ namespace Microsoft.Agents.A365.DevTools.Cli.Services; /// -/// Automates a Copilot Chat conversation using Playwright. -/// Opens the M365 Chat page, selects an agent by name, sends a test message, -/// and waits for the agent to respond. +/// Automates a conversation with an agent in Microsoft Teams using Playwright. +/// Opens Teams web, searches for the agent by name in a new chat, sends a test +/// message, and waits for the agent to respond. /// /// Auth strategy: /// - Reuses the CLI's existing MSAL browser session. On Windows, WAM (Windows -/// Authentication Manager) automatically provides SSO for the M365 domain, so +/// Authentication Manager) automatically provides SSO for the Teams domain, so /// the browser launched by Playwright inherits the user's session without any /// manual login. /// - On non-Windows platforms (or if SSO is not available), a headed browser /// opens and waits for the user to log in manually. /// - Saves browser storage state to a local file for reuse across validate runs. -/// -/// The flow mirrors the Camp-AIR A365ObservabilityTests pattern: -/// auth/setup.ts for login, pages/chat-page.ts for interaction. /// public class CopilotChatPlaywrightService { private readonly ILogger _logger; - /// Base URL for M365 Chat. - internal const string ChatBaseUrl = "https://m365.cloud.microsoft/chat"; + /// Base URL for Microsoft Teams web. + internal const string ChatBaseUrl = "https://teams.microsoft.com"; + + /// URL pattern indicating the user has successfully authenticated into Teams. + internal const string AuthenticatedUrlPattern = "**/teams.microsoft.com/**"; /// Timeout for agent response in milliseconds (2 minutes). internal const int AgentResponseTimeoutMs = 120_000; @@ -50,25 +50,16 @@ public class CopilotChatPlaywrightService /// Auth state is reused if it is less than this many minutes old. private const int AuthStateTtlMinutes = 30; - /// Streaming indicator phrases that signal the agent is still generating. - private static readonly string[] StreamingPatterns = new[] - { - "Generating response", - "Lining things up", - "Working on it", - "Thinking" - }; - public CopilotChatPlaywrightService(ILogger logger) { _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } /// - /// Sends a test message to the specified agent in Copilot Chat and returns the + /// Sends a test message to the specified agent in Teams web and returns the /// agent's response text, or null if the conversation could not be completed. /// - /// Display name of the agent in M365 Chat. + /// Display name of the agent (bot) in Teams. /// Message to send to the agent. /// Cancellation token. /// The agent's response text, or null if the conversation failed. @@ -82,7 +73,6 @@ public CopilotChatPlaywrightService(ILogger logger) var authStatePath = GetAuthStatePath(); - // Install Playwright browsers if needed (first-run only) _logger.LogDebug("Ensuring Playwright browsers are installed..."); var installExitCode = Microsoft.Playwright.Program.Main(new[] { "install", "chromium" }); if (installExitCode != 0) @@ -92,21 +82,17 @@ public CopilotChatPlaywrightService(ILogger logger) using var playwright = await Microsoft.Playwright.Playwright.CreateAsync(); - // Determine if we can reuse saved auth state var hasFreshAuthState = HasFreshAuthState(authStatePath); - // Launch browser: headed if no saved state (user needs to log in), headless if reusing state var launchOptions = new BrowserTypeLaunchOptions { - Headless = hasFreshAuthState, - // Slow down actions slightly for stability with M365 UI - SlowMo = hasFreshAuthState ? 0 : 100 + Headless = false, + SlowMo = 100 }; - _logger.LogDebug("Launching Chromium (headless: {Headless})...", launchOptions.Headless); + _logger.LogDebug("Launching Chromium (headed)..."); await using var browser = await playwright.Chromium.LaunchAsync(launchOptions); - // Create context with saved state if available var contextOptions = new BrowserNewContextOptions { ViewportSize = new ViewportSize { Width = 1280, Height = 720 } @@ -122,24 +108,21 @@ public CopilotChatPlaywrightService(ILogger logger) var page = await context.NewPageAsync(); - // Step 1: Navigate and authenticate - _logger.LogDebug("Navigating to M365 Chat..."); + _logger.LogInformation("Navigating to Teams web..."); await page.GotoAsync(ChatBaseUrl, new PageGotoOptions { - WaitUntil = WaitUntilState.NetworkIdle, + WaitUntil = WaitUntilState.DOMContentLoaded, Timeout = NavigationTimeoutMs }); - // Check if we landed on the chat page or need to authenticate - var chatInput = page.GetByRole(AriaRole.Textbox, new PageGetByRoleOptions { Name = "Message Copilot" }); - var isAuthenticated = await chatInput.IsVisibleAsync().ConfigureAwait(false); + // Check if we landed on Teams or need to authenticate + var isAuthenticated = await IsTeamsLoadedAsync(page); if (!isAuthenticated) { if (hasFreshAuthState) { _logger.LogDebug("Saved auth state did not work. Re-launching as headed for login..."); - // Close headless browser, reopen headed await page.CloseAsync(); await context.CloseAsync(); await browser.CloseAsync(); @@ -147,35 +130,20 @@ public CopilotChatPlaywrightService(ILogger logger) return await RunWithInteractiveLoginAsync(playwright, agentName, testMessage, authStatePath, cancellationToken); } - // Wait for the user to log in manually - _logger.LogInformation("Please log in to M365 in the browser window."); + _logger.LogInformation("Please log in to Teams in the browser window."); _logger.LogInformation("The CLI will continue automatically once login completes."); - // Wait for redirect to M365 chat (user completes login) - await page.WaitForURLAsync("**/m365.cloud.microsoft/**", + await page.WaitForURLAsync(AuthenticatedUrlPattern, new PageWaitForURLOptions { Timeout = 300_000 }); - // Navigate to chat page to ensure all cookies are set - await page.GotoAsync(ChatBaseUrl, new PageGotoOptions - { - WaitUntil = WaitUntilState.NetworkIdle, - Timeout = NavigationTimeoutMs - }); - - // Wait for chat input to appear - await chatInput.WaitForAsync(new LocatorWaitForOptions - { - State = WaitForSelectorState.Visible, - Timeout = 30_000 - }); + // Wait for Teams to fully load after redirect + await WaitForTeamsReadyAsync(page); _logger.LogInformation("Login successful."); } - // Save auth state for next time await SaveAuthStateAsync(context, authStatePath); - // Step 2: Open agent chat and send message var responseText = await OpenAgentChatAndSendMessageAsync(page, agentName, testMessage, cancellationToken); return responseText; @@ -210,29 +178,17 @@ await chatInput.WaitForAsync(new LocatorWaitForOptions await page.GotoAsync(ChatBaseUrl, new PageGotoOptions { - WaitUntil = WaitUntilState.NetworkIdle, + WaitUntil = WaitUntilState.DOMContentLoaded, Timeout = NavigationTimeoutMs }); - _logger.LogInformation("Please log in to M365 in the browser window."); + _logger.LogInformation("Please log in to Teams in the browser window."); _logger.LogInformation("The CLI will continue automatically once login completes."); - // Wait for redirect to M365 - await page.WaitForURLAsync("**/m365.cloud.microsoft/**", + await page.WaitForURLAsync(AuthenticatedUrlPattern, new PageWaitForURLOptions { Timeout = 300_000 }); - await page.GotoAsync(ChatBaseUrl, new PageGotoOptions - { - WaitUntil = WaitUntilState.NetworkIdle, - Timeout = NavigationTimeoutMs - }); - - var chatInput = page.GetByRole(AriaRole.Textbox, new PageGetByRoleOptions { Name = "Message Copilot" }); - await chatInput.WaitForAsync(new LocatorWaitForOptions - { - State = WaitForSelectorState.Visible, - Timeout = 30_000 - }); + await WaitForTeamsReadyAsync(page); _logger.LogInformation("Login successful."); @@ -242,8 +198,51 @@ await chatInput.WaitForAsync(new LocatorWaitForOptions } /// - /// Opens a new chat with the specified agent and sends a test message. - /// Returns the agent's response text, or null if the response could not be extracted. + /// Checks if the Teams web app has loaded by looking for the left rail or chat UI. + /// + private async Task IsTeamsLoadedAsync(IPage page) + { + try + { + // Teams v2 uses a left app bar with Chat, Activity, etc. + var chatNavItem = page.Locator("[data-tid='app-bar-86fcd49b-61a2-4701-b771-54728cd291fb']") + .Or(page.GetByRole(AriaRole.Tab, new PageGetByRoleOptions { Name = "Chat" })) + .Or(page.Locator("[data-tid='app-bar-chat']")); + + // Wait briefly for the chat nav to appear, then check visibility + await chatNavItem.First.WaitForAsync(new LocatorWaitForOptions + { + State = WaitForSelectorState.Visible, + Timeout = 10_000 + }); + return true; + } + catch + { + return false; + } + } + + /// + /// Waits for the Teams web UI to become ready after authentication. + /// + private async Task WaitForTeamsReadyAsync(IPage page) + { + // Wait for the app bar or chat section to appear, indicating Teams has loaded + var chatNavItem = page.Locator("[data-tid='app-bar-86fcd49b-61a2-4701-b771-54728cd291fb']") + .Or(page.GetByRole(AriaRole.Tab, new PageGetByRoleOptions { Name = "Chat" })) + .Or(page.Locator("[data-tid='app-bar-chat']")); + + await chatNavItem.First.WaitForAsync(new LocatorWaitForOptions + { + State = WaitForSelectorState.Visible, + Timeout = 60_000 + }); + } + + /// + /// Opens a chat with the agent in Teams using the top search bar, sends a test + /// message, and returns the agent's response text. /// private async Task OpenAgentChatAndSendMessageAsync( IPage page, @@ -251,196 +250,351 @@ await chatInput.WaitForAsync(new LocatorWaitForOptions string testMessage, CancellationToken cancellationToken) { - // Click the agent button in the sidebar - _logger.LogDebug("Opening agent chat for '{AgentName}'...", agentName); - var agentButton = page.GetByRole(AriaRole.Button, new PageGetByRoleOptions { Name = agentName }); - await agentButton.First.WaitForAsync(new LocatorWaitForOptions - { - State = WaitForSelectorState.Visible, - Timeout = 15_000 - }); - await agentButton.First.ClickAsync(); - await page.WaitForTimeoutAsync(2000); + // Step 1: Use the top search bar to find the agent + _logger.LogInformation("Searching for agent '{AgentName}' via search bar...", agentName); + var searchBox = page.Locator("[data-tid='searchbox']") + .Or(page.GetByRole(AriaRole.Search)) + .Or(page.GetByPlaceholder("Search")); - // Wait for the chat input to be ready - var chatInput = page.GetByRole(AriaRole.Textbox, new PageGetByRoleOptions { Name = "Message Copilot" }); - await chatInput.WaitForAsync(new LocatorWaitForOptions + try { - State = WaitForSelectorState.Visible, - Timeout = 30_000 - }); + await searchBox.First.WaitForAsync(new LocatorWaitForOptions + { + State = WaitForSelectorState.Visible, + Timeout = 15_000 + }); + await searchBox.First.ClickAsync(); + await page.WaitForTimeoutAsync(500); + await page.Keyboard.TypeAsync(agentName, new KeyboardTypeOptions { Delay = 50 }); + await page.WaitForTimeoutAsync(3000); + } + catch (Exception ex) + { + _logger.LogWarning("Could not use the search bar: {Message}", ex.Message); + await CaptureScreenshotAsync(page, "teams-no-search-bar"); + return null; + } + + // Step 2: Click the matching result from the search dropdown + _logger.LogInformation("Selecting agent from search results..."); + var searchResult = page.GetByRole(AriaRole.Option).Filter(new LocatorFilterOptions { HasText = agentName }) + .Or(page.GetByRole(AriaRole.Listitem).Filter(new LocatorFilterOptions { HasText = agentName })); - // Click "New chat" to start a fresh conversation - // The header button is typically the second "New chat" button (first is sidebar menuitem) - var newChatButtons = page.GetByRole(AriaRole.Button, new PageGetByRoleOptions { Name = "New chat" }); - var buttonCount = await newChatButtons.CountAsync(); - if (buttonCount > 1) + try { - await newChatButtons.Nth(1).ClickAsync(); + await searchResult.First.WaitForAsync(new LocatorWaitForOptions + { + State = WaitForSelectorState.Visible, + Timeout = 10_000 + }); + await searchResult.First.ClickAsync(); } - else if (buttonCount == 1) + catch (Exception ex) { - await newChatButtons.First.ClickAsync(); + _logger.LogWarning("Agent '{AgentName}' was not found in search results: {Message}", agentName, ex.Message); + await CaptureScreenshotAsync(page, "teams-agent-not-found"); + return null; } - await page.WaitForTimeoutAsync(2000); - // Re-focus the chat input - await chatInput.WaitForAsync(new LocatorWaitForOptions - { - State = WaitForSelectorState.Visible, - Timeout = 15_000 - }); - await chatInput.ClickAsync(); - await page.WaitForTimeoutAsync(500); + await page.WaitForTimeoutAsync(3000); - // Type and send message - _logger.LogDebug("Sending test message to agent..."); + // Step 3: Type and send the message in the compose box + _logger.LogInformation("Sending test message to agent..."); + var composeBox = page.GetByRole(AriaRole.Textbox, new PageGetByRoleOptions { Name = "Type a message" }) + .Or(page.Locator("[data-tid='ckeditor-replyConversation']")) + .Or(page.GetByRole(AriaRole.Textbox, new PageGetByRoleOptions { Name = "Type a new message" })); - // Click the paragraph inside the textbox to ensure focus (M365 Chat pattern) - var paragraph = chatInput.Locator("p, [role='paragraph']"); try { - await paragraph.First.ClickAsync(new LocatorClickOptions { Timeout = 5000 }); + await composeBox.First.WaitForAsync(new LocatorWaitForOptions + { + State = WaitForSelectorState.Visible, + Timeout = 15_000 + }); + await composeBox.First.ClickAsync(); + await page.WaitForTimeoutAsync(500); + + await composeBox.First.FillAsync(testMessage); + await page.WaitForTimeoutAsync(500); } - catch + catch (Exception ex) + { + _logger.LogWarning("Could not find or fill the compose box: {Message}", ex.Message); + await CaptureScreenshotAsync(page, "teams-no-compose-box"); + return null; + } + + // Record message count before sending so we can detect new messages + var messagesBefore = await CountVisibleMessagesAsync(page); + _logger.LogDebug("Messages visible before sending: {Count}", messagesBefore); + + // Click the send button or press Enter + var sendButton = page.Locator("[data-tid='newMessageCommands-send']") + .Or(page.GetByRole(AriaRole.Button, new PageGetByRoleOptions { Name = "Send" })); + + if (await sendButton.First.IsVisibleAsync().ConfigureAwait(false)) { - await chatInput.ClickAsync(); + await sendButton.First.ClickAsync(); + } + else + { + await composeBox.First.PressAsync("Enter"); } - await chatInput.FillAsync(testMessage); - await page.WaitForTimeoutAsync(500); - await chatInput.PressAsync("Enter"); + _logger.LogInformation("Message sent, waiting for agent response..."); - // Wait for agent response - _logger.LogDebug("Waiting for agent response..."); - var responseText = await WaitForAgentResponseAsync(page, cancellationToken); + // Step 4: Wait for agent response (new message must appear) + var responseText = await WaitForNewMessageAsync(page, messagesBefore, cancellationToken); if (string.IsNullOrWhiteSpace(responseText)) { _logger.LogWarning("Agent response was empty."); + await CaptureScreenshotAsync(page, "teams-empty-response"); return null; } - _logger.LogDebug("Agent responded with {Length} characters.", responseText.Length); + _logger.LogInformation("Agent responded with {Length} characters.", responseText.Length); return responseText; } /// - /// Waits for the agent to finish responding and extracts the response text. - /// Strategy (from Camp-AIR): - /// 1. Wait for "Copy Response" button or feedback group (response has started) - /// 2. Poll until streaming indicators ("Generating response", etc.) clear - /// 3. Stabilize: wait for text to stop changing - /// 4. Extract final text from the last non-user article element + /// Captures a screenshot for debugging when a step fails. + /// Saved to the CLI's local data directory. /// - private async Task WaitForAgentResponseAsync(IPage page, CancellationToken cancellationToken) + private async Task CaptureScreenshotAsync(IPage page, string stepName) { - var timeout = AgentResponseTimeoutMs; + try + { + var screenshotDir = Path.Combine( + Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), + Constants.AuthenticationConstants.ApplicationName, + "screenshots"); + Directory.CreateDirectory(screenshotDir); - // Step 1: Wait for response to start - var copyResponseButton = page.GetByRole(AriaRole.Button, new PageGetByRoleOptions { Name = "Copy Response" }); - var feedbackGroup = page.GetByRole(AriaRole.Group, new PageGetByRoleOptions { NameRegex = new System.Text.RegularExpressions.Regex("feedback", System.Text.RegularExpressions.RegexOptions.IgnoreCase) }); - var completionIndicator = copyResponseButton.Or(feedbackGroup); + var fileName = $"{stepName}-{DateTime.UtcNow:yyyyMMdd-HHmmss}.png"; + var filePath = Path.Combine(screenshotDir, fileName); - await completionIndicator.First.WaitForAsync(new LocatorWaitForOptions + await page.ScreenshotAsync(new PageScreenshotOptions { Path = filePath, FullPage = true }); + _logger.LogInformation("Screenshot saved: {Path}", filePath); + } + catch (Exception ex) { - State = WaitForSelectorState.Visible, - Timeout = timeout - }); + _logger.LogDebug(ex, "Failed to capture screenshot for step '{Step}'.", stepName); + } + } + + /// + /// Counts visible message elements in the chat pane. + /// Uses multiple selector strategies to find message containers in Teams. + /// + private static async Task CountVisibleMessagesAsync(IPage page) + { + // Teams renders messages in divs with data-tid="chat-pane-message" or similar + var selectors = new[] + { + "[data-tid='chat-pane-message']", + "[data-tid='messageListItem']", + ".message-body-content", + }; + + foreach (var selector in selectors) + { + var count = await page.Locator(selector).CountAsync(); + if (count > 0) + { + return count; + } + } + + return 0; + } - // Step 2: Poll until streaming indicators clear + /// + /// Waits for a new message to appear after the user's message was sent. + /// Polls the page for new message elements and waits for the response text to stabilize. + /// Returns null if no new message appears within the timeout. + /// + /// + /// Common placeholder patterns that agents send before the real response. + /// These are short acknowledgements that should be ignored. + /// + private static readonly string[] PlaceholderPatterns = new[] + { + "got it", "working on it", "let me", "thinking", "one moment", + "just a moment", "hold on", "processing", "looking into", + "give me a moment", "i'm on it", "sure thing" + }; + + private async Task WaitForNewMessageAsync( + IPage page, + int messageCountBefore, + CancellationToken cancellationToken) + { + var timeout = AgentResponseTimeoutMs; var pollStart = Environment.TickCount64; + var pollIntervalMs = 3000; + + // Phase 1: Wait for any new message to appear beyond our sent message + _logger.LogDebug("Waiting for new messages (had {Before} before)...", messageCountBefore); + + var newMessageDetected = false; while (Environment.TickCount64 - pollStart < timeout) { cancellationToken.ThrowIfCancellationRequested(); - await page.WaitForTimeoutAsync(3000); - var allArticles = page.GetByRole(AriaRole.Article); - var count = await allArticles.CountAsync(); - var stillStreaming = false; + var currentCount = await CountVisibleMessagesAsync(page); + if (currentCount >= messageCountBefore + 2) + { + newMessageDetected = true; + _logger.LogDebug("New message detected (count: {Before} -> {Current}).", messageCountBefore, currentCount); + break; + } + + var elapsed = (Environment.TickCount64 - pollStart) / 1000; + _logger.LogInformation("Waiting for agent response... ({Elapsed}s)", elapsed); + await page.WaitForTimeoutAsync(pollIntervalMs); + } + + if (!newMessageDetected) + { + _logger.LogWarning("No new message appeared within {Timeout}s timeout.", timeout / 1000); + await CaptureScreenshotAsync(page, "teams-no-response"); + return null; + } + + // Phase 2: Wait for agent to finish responding. + // The agent may send a placeholder first (e.g. "Got it...working on it"), + // then replace or follow up with the actual response. We need to wait for: + // (a) typing indicator to clear, (b) text to not be a placeholder, + // (c) text to stabilize. + var previousText = string.Empty; + var stableCount = 0; + var phase2Start = Environment.TickCount64; + var phase2TimeoutMs = 120_000; // 2 minutes for the real response + + while (Environment.TickCount64 - phase2Start < phase2TimeoutMs) + { + cancellationToken.ThrowIfCancellationRequested(); - for (var i = 0; i < count; i++) + // Check typing indicator + var isTyping = false; + try { - string text; - try - { - text = await allArticles.Nth(i).InnerTextAsync() ?? string.Empty; - } - catch - { - text = string.Empty; - } + var typingIndicator = page.Locator("[data-tid='chat-typing-indicator']") + .Or(page.Locator(".typing-indicator")); + isTyping = await typingIndicator.First.IsVisibleAsync().ConfigureAwait(false); + } + catch + { + // No typing indicator found + } + + if (isTyping) + { + _logger.LogDebug("Agent still typing..."); + stableCount = 0; + await page.WaitForTimeoutAsync(2000); + continue; + } + + var currentText = await ExtractLatestMessageTextAsync(page); - if (StreamingPatterns.Any(p => text.Contains(p, StringComparison.Ordinal))) + // Check if current text looks like a placeholder + if (IsPlaceholderResponse(currentText)) + { + var elapsed = (Environment.TickCount64 - phase2Start) / 1000; + _logger.LogDebug("Detected placeholder response, waiting for real response... ({Elapsed}s)", elapsed); + stableCount = 0; + await page.WaitForTimeoutAsync(3000); + continue; + } + + // Check if text has stabilized (same non-placeholder text 3 times in a row) + if (!string.IsNullOrEmpty(currentText) && currentText == previousText) + { + stableCount++; + if (stableCount >= 3) { - stillStreaming = true; - break; + _logger.LogDebug("Response text stabilized ({Length} chars).", currentText.Length); + return currentText; } } - - if (!stillStreaming) + else { - break; + stableCount = 0; } - _logger.LogDebug("Agent still streaming, waiting..."); + previousText = currentText; + await page.WaitForTimeoutAsync(2000); } - // Step 3: Stabilize -- wait for text to stop changing - var previousText = string.Empty; - for (var i = 0; i < 3; i++) + // Return whatever we have after timeout + if (!string.IsNullOrEmpty(previousText) && !IsPlaceholderResponse(previousText)) { - await page.WaitForTimeoutAsync(2000); - var currentText = await ExtractLastAgentResponseAsync(page); - if (currentText == previousText && !string.IsNullOrEmpty(currentText)) - { - break; - } - previousText = currentText; + return previousText; } - // Step 4: Extract final response - return await ExtractLastAgentResponseAsync(page); + _logger.LogWarning("Agent response did not stabilize within timeout."); + await CaptureScreenshotAsync(page, "teams-response-timeout"); + return null; } /// - /// Extracts the text from the last non-user article element on the page. - /// User messages start with "You said:"; agent responses do not. + /// Checks if a response looks like a short placeholder/acknowledgement + /// that the agent sends before the real answer. /// - private static async Task ExtractLastAgentResponseAsync(IPage page) + private static bool IsPlaceholderResponse(string? text) { - var allArticles = page.GetByRole(AriaRole.Article); - var count = await allArticles.CountAsync(); + if (string.IsNullOrWhiteSpace(text)) + { + return true; + } - for (var i = count - 1; i >= 0; i--) + // Very short responses (under 30 chars) that match placeholder patterns + var trimmed = text.Trim(); + if (trimmed.Length > 80) { - string text; - try - { - text = await allArticles.Nth(i).InnerTextAsync() ?? string.Empty; - } - catch - { - continue; - } + return false; + } + + var lower = trimmed.ToLowerInvariant(); + return PlaceholderPatterns.Any(p => lower.Contains(p, StringComparison.Ordinal)); + } + + /// + /// Extracts the text content from the last message element in the chat pane. + /// + private static async Task ExtractLatestMessageTextAsync(IPage page) + { + var selectors = new[] + { + "[data-tid='chat-pane-message']", + "[data-tid='messageListItem']", + ".message-body-content", + }; - if (text.StartsWith("You said:", StringComparison.OrdinalIgnoreCase)) + foreach (var selector in selectors) + { + var messages = page.Locator(selector); + var count = await messages.CountAsync(); + if (count == 0) { continue; } - // Extract text after "said:" heading if present - var saidIndex = text.IndexOf("said:", StringComparison.OrdinalIgnoreCase); - if (saidIndex >= 0) + // Get the last message's text + try { - var afterSaid = text[(saidIndex + 5)..].Trim(); - if (!string.IsNullOrEmpty(afterSaid)) + var text = await messages.Nth(count - 1).InnerTextAsync() ?? string.Empty; + if (!string.IsNullOrWhiteSpace(text)) { - return afterSaid; + return text.Trim(); } } - - return text.Trim(); + catch + { + continue; + } } return string.Empty; diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/AgentMetricsRequirementCheck.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/AgentMetricsRequirementCheck.cs index 15cc1dc2..d64df1a3 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/AgentMetricsRequirementCheck.cs +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/AgentMetricsRequirementCheck.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +using System.Diagnostics; using Microsoft.Agents.A365.DevTools.Cli.Models; using Microsoft.Agents.A365.DevTools.Cli.Services; using Microsoft.Agents.A365.DevTools.Cli.Constants; @@ -14,9 +15,11 @@ namespace Microsoft.Agents.A365.DevTools.Cli.Services.Requirements.RequirementCh /// /// Validates that agent metrics are visible and incrementing by: -/// 1. Querying baseline metrics via MCP tool call -/// 2. Generating a conversation with the agent in Copilot Chat (via Playwright) -/// 3. Re-querying metrics to verify they incremented +/// 1. Starting the agent locally +/// 2. Querying baseline metrics via MCP tool call +/// 3. Generating a conversation with the agent in Teams Chat (via Playwright) +/// 4. Re-querying metrics to verify they incremented +/// 5. Stopping the agent /// public class AgentMetricsRequirementCheck : RequirementCheck { @@ -24,22 +27,37 @@ public class AgentMetricsRequirementCheck : RequirementCheck private readonly CopilotChatPlaywrightService? _playwrightService; private readonly string? _instanceName; + private readonly PlatformDetector? _platformDetector; + private readonly IProcessService? _processService; + private readonly string? _resolvedUvCommand; /// Default test message sent to the agent during the metrics check. internal const string DefaultTestMessage = ConversationRequirementCheck.FallbackToolPrompt; + /// Maximum time to wait for the agent to start and respond on the health endpoint. + private static readonly TimeSpan AgentStartupTimeout = TimeSpan.FromSeconds(30); + + /// Interval between health endpoint polls during startup. + private static readonly TimeSpan HealthPollInterval = TimeSpan.FromMilliseconds(500); + public AgentMetricsRequirementCheck( CopilotChatPlaywrightService? playwrightService = null, - string? instanceName = null) + string? instanceName = null, + PlatformDetector? platformDetector = null, + IProcessService? processService = null, + string? resolvedUvCommand = null) { _playwrightService = playwrightService; _instanceName = instanceName; + _platformDetector = platformDetector; + _processService = processService; + _resolvedUvCommand = resolvedUvCommand; } /// public override string Name => "AgentMetrics"; /// - public override string Description => "Verifies agent metrics are visible and incrementing after a Copilot Chat conversation"; + public override string Description => "Verifies agent metrics are visible and incrementing after a Teams Chat conversation"; /// public override string Category => "Observability"; @@ -59,96 +77,276 @@ private async Task CheckImplementationAsync( CancellationToken cancellationToken) { var metadata = new AgentMetricsMetadata(); + Process? agentProcess = null; - // Step 1: Get baseline metrics via MCP tool call (best-effort, does not block step 2) - logger.LogDebug("Step 1: Querying baseline agent metrics..."); - AgentMetricsSnapshot? baselineMetrics = null; try { - baselineMetrics = await GetAgentMetricsAsync(config, logger, cancellationToken); + // Step 0: Start the agent locally + logger.LogInformation("Starting agent locally for metrics validation..."); + agentProcess = await StartAgentLocallyAsync(config, logger, cancellationToken); + if (agentProcess is null) + { + return RequirementCheckResult.Warning( + "Could not start the agent locally for metrics validation", + details: "Ensure the project builds and runs successfully. Run 'a365 validate' without --with-tenant first."); + } + + // Step 1: Get baseline metrics via MCP tool call (best-effort, does not block step 2) + logger.LogInformation("Step 1: Querying baseline agent metrics..."); + AgentMetricsSnapshot? baselineMetrics = null; + try + { + baselineMetrics = await GetAgentMetricsAsync(config, logger, cancellationToken); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + logger.LogDebug(ex, "Failed to query baseline agent metrics: {Message}", ex.Message); + } + + metadata.BaselineMetrics = baselineMetrics; + if (baselineMetrics is null) + { + logger.LogDebug("Baseline metrics not available -- will still attempt conversation."); + } + + // Step 2: Generate a conversation with the agent in Teams Chat (via Playwright) + logger.LogInformation("Step 2: Generating conversation with agent in Teams Chat..."); + bool conversationGenerated; + try + { + conversationGenerated = await GenerateCopilotChatConversationAsync(config, logger, cancellationToken); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + logger.LogWarning(ex, "Failed to generate Teams Chat conversation: {Message}", ex.Message); + return RequirementCheckResult.Warning( + "Could not generate Teams Chat conversation for metrics validation", + details: $"Playwright test failed: {ex.Message}"); + } + + metadata.ConversationGenerated = conversationGenerated; + + if (!conversationGenerated) + { + return RequirementCheckResult.Warning( + "Teams Chat conversation could not be generated", + details: "Playwright was unable to complete a conversation with the agent. Metrics increment check skipped."); + } + + if (baselineMetrics is null) + { + return RequirementCheckResult.Failure( + "Agent metrics endpoint not available", + "Ensure the agent is deployed and metrics are configured. The MCP metrics tool must be reachable.", + details: "Teams Chat conversation completed successfully but baseline metrics could not be retrieved."); + } + + // Step 3: Re-query metrics and verify they incremented + logger.LogInformation("Step 3: Re-querying agent metrics to verify increment..."); + AgentMetricsSnapshot? postConversationMetrics = null; + try + { + postConversationMetrics = await GetAgentMetricsAsync(config, logger, cancellationToken); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + logger.LogDebug(ex, "Failed to query post-conversation agent metrics: {Message}", ex.Message); + } + + metadata.PostConversationMetrics = postConversationMetrics; + + if (postConversationMetrics is null) + { + return RequirementCheckResult.Failure( + "Agent metrics endpoint not available after conversation", + "Ensure the metrics endpoint remains reachable. The MCP metrics tool must return data.", + details: "Teams Chat conversation completed successfully but post-conversation metrics could not be retrieved."); + } + + var incremented = postConversationMetrics.InvocationCount > baselineMetrics.InvocationCount; + metadata.MetricsIncremented = incremented; + + if (!incremented) + { + return RequirementCheckResult.Failure( + "Agent metrics did not increment after Teams Chat conversation", + "Verify that the agent is instrumented with Agent365 observability and that metrics are flowing to the backend.", + details: $"Baseline invocations: {baselineMetrics.InvocationCount}, " + + $"Post-conversation invocations: {postConversationMetrics.InvocationCount}"); + } + + return RequirementCheckResult.Success( + details: $"Agent metrics incremented from {baselineMetrics.InvocationCount} to " + + $"{postConversationMetrics.InvocationCount} after Teams Chat conversation."); } - catch (Exception ex) when (ex is not OperationCanceledException) + finally { - logger.LogDebug(ex, "Failed to query baseline agent metrics: {Message}", ex.Message); + if (agentProcess is not null && !agentProcess.HasExited) + { + logger.LogDebug("Stopping local agent process..."); + try + { + agentProcess.Kill(entireProcessTree: true); + await agentProcess.WaitForExitAsync(cancellationToken).WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + } + catch + { + // Best-effort cleanup + } + finally + { + agentProcess.Dispose(); + } + } } + } - metadata.BaselineMetrics = baselineMetrics; - if (baselineMetrics is null) + /// + /// Starts the agent locally and waits for the health endpoint to respond. + /// Returns the agent process, or null if the agent could not be started. + /// + protected internal virtual async Task StartAgentLocallyAsync( + Agent365Config config, + ILogger logger, + CancellationToken cancellationToken) + { + if (_platformDetector is null || _processService is null) { - logger.LogDebug("Baseline metrics not available -- will still attempt conversation."); + logger.LogWarning("Platform detector or process service not available. Cannot start agent locally."); + return null; } - // Step 2: Generate a conversation with the agent in Copilot Chat (via Playwright) - logger.LogDebug("Step 2: Generating conversation with agent in Copilot Chat..."); - bool conversationGenerated; - try + var projectPath = ConversationRequirementCheck.ResolveProjectPath(config); + if (!Directory.Exists(projectPath)) { - conversationGenerated = await GenerateCopilotChatConversationAsync(config, logger, cancellationToken); + logger.LogWarning("Project path does not exist: {Path}", projectPath); + return null; } - catch (Exception ex) when (ex is not OperationCanceledException) + + var platform = _platformDetector.Detect(projectPath); + if (platform == ProjectPlatform.Unknown) { - logger.LogDebug(ex, "Failed to generate Copilot Chat conversation"); - return RequirementCheckResult.Warning( - "Could not generate Copilot Chat conversation for metrics validation", - details: $"Playwright test failed: {ex.Message}"); + logger.LogWarning("Could not detect project platform in {Path}", projectPath); + return null; } - metadata.ConversationGenerated = conversationGenerated; + var port = LocalRuntimeRequirementCheck.ResolvePort(config.MessagingEndpoint); + var healthUrl = $"http://localhost:{port}{LocalRuntimeRequirementCheck.DefaultHealthPath}"; - if (!conversationGenerated) + logger.LogInformation("Starting agent locally ({Platform} on port {Port})...", platform, port); + + var startInfo = BuildProcessStartInfo(platform, projectPath, port); + var process = _processService.Start(startInfo); + if (process is null) { - return RequirementCheckResult.Warning( - "Copilot Chat conversation could not be generated", - details: "Playwright was unable to complete a conversation with the agent. Metrics increment check skipped."); + logger.LogWarning("Failed to start {Platform} process.", platform); + return null; } - // If baseline metrics were not available, fail — metrics must be reachable - if (baselineMetrics is null) + var healthResult = await WaitForHealthAsync(process, healthUrl, logger, cancellationToken); + if (!healthResult) { - return RequirementCheckResult.Failure( - "Agent metrics endpoint not available", - "Ensure the agent is deployed and metrics are configured. The MCP metrics tool must be reachable.", - details: "Copilot Chat conversation completed successfully but baseline metrics could not be retrieved."); + logger.LogWarning("Agent did not respond on health endpoint within timeout."); + try { process.Kill(entireProcessTree: true); } catch { } + process.Dispose(); + return null; } - // Step 3: Re-query metrics and verify they incremented - logger.LogDebug("Step 3: Re-querying agent metrics to verify increment..."); - AgentMetricsSnapshot? postConversationMetrics = null; - try + logger.LogInformation("Agent is running and healthy on port {Port}.", port); + return process; + } + + private ProcessStartInfo BuildProcessStartInfo(ProjectPlatform platform, string projectPath, int port) + { + var startInfo = new ProcessStartInfo { - postConversationMetrics = await GetAgentMetricsAsync(config, logger, cancellationToken); - } - catch (Exception ex) when (ex is not OperationCanceledException) + WorkingDirectory = projectPath, + RedirectStandardOutput = false, + RedirectStandardError = false, + UseShellExecute = false, + CreateNoWindow = true + }; + + switch (platform) { - logger.LogDebug(ex, "Failed to query post-conversation agent metrics: {Message}", ex.Message); + case ProjectPlatform.DotNet: + startInfo.FileName = "dotnet"; + startInfo.Arguments = "run --no-build"; + startInfo.EnvironmentVariables["ASPNETCORE_URLS"] = $"http://localhost:{port}"; + break; + + case ProjectPlatform.NodeJs: + LocalRuntimeRequirementCheck.WrapForWindows(startInfo, "npm", "start"); + startInfo.EnvironmentVariables["PORT"] = port.ToString(); + break; + + case ProjectPlatform.Python: + var entryPoint = LocalRuntimeRequirementCheck.ResolvePythonEntryPoint(projectPath); + var usesUv = ProjectBuildRequirementCheck.DetectPythonInstallCommand(projectPath) is ("uv", _); + if (usesUv) + { + startInfo.FileName = _resolvedUvCommand ?? "uv"; + startInfo.Arguments = $"run python {entryPoint}"; + } + else + { + startInfo.FileName = "python"; + startInfo.Arguments = entryPoint; + } + startInfo.EnvironmentVariables["PORT"] = port.ToString(); + break; + + default: + throw new ArgumentOutOfRangeException(nameof(platform), platform, "Unsupported platform"); } - metadata.PostConversationMetrics = postConversationMetrics; + return startInfo; + } + + private static async Task WaitForHealthAsync( + Process process, + string healthUrl, + ILogger logger, + CancellationToken cancellationToken) + { + using var httpClient = new HttpClient(); + using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + timeoutCts.CancelAfter(AgentStartupTimeout); - if (postConversationMetrics is null) + while (!timeoutCts.Token.IsCancellationRequested) { - return RequirementCheckResult.Failure( - "Agent metrics endpoint not available after conversation", - "Ensure the metrics endpoint remains reachable. The MCP metrics tool must return data.", - details: "Copilot Chat conversation completed successfully but post-conversation metrics could not be retrieved."); - } + if (process.HasExited) + { + logger.LogWarning("Agent exited with code {ExitCode} before health endpoint responded.", process.ExitCode); + return false; + } - // Compare baseline vs post-conversation - var incremented = postConversationMetrics.InvocationCount > baselineMetrics.InvocationCount; - metadata.MetricsIncremented = incremented; + try + { + using var response = await httpClient.GetAsync(healthUrl, timeoutCts.Token); + if (response.IsSuccessStatusCode) + { + return true; + } + } + catch (HttpRequestException) { } + catch (TaskCanceledException) when (timeoutCts.Token.IsCancellationRequested && !cancellationToken.IsCancellationRequested) + { + break; + } - if (!incremented) - { - return RequirementCheckResult.Failure( - "Agent metrics did not increment after Copilot Chat conversation", - "Verify that the agent is instrumented with Agent365 observability and that metrics are flowing to the backend.", - details: $"Baseline invocations: {baselineMetrics.InvocationCount}, " + - $"Post-conversation invocations: {postConversationMetrics.InvocationCount}"); + try + { + await Task.Delay(HealthPollInterval, timeoutCts.Token); + } + catch (TaskCanceledException) when (!cancellationToken.IsCancellationRequested) + { + break; + } } - return RequirementCheckResult.Success( - details: $"Agent metrics incremented from {baselineMetrics.InvocationCount} to " + - $"{postConversationMetrics.InvocationCount} after Copilot Chat conversation."); + cancellationToken.ThrowIfCancellationRequested(); + return false; } /// @@ -300,13 +498,13 @@ private static long ConvertToLong(double value) } /// - /// Generates a conversation with the agent in Copilot Chat using Playwright. + /// Generates a conversation with the agent in Microsoft Teams using Playwright. /// Reuses the CLI's existing MSAL authentication context (WAM on Windows, /// browser auth on other platforms) so the user is not prompted to log in again. /// /// Uses to: /// 1. Launch a Chromium browser (headless if saved auth state is fresh, headed otherwise) - /// 2. Navigate to M365 Chat, select the agent, send a test message + /// 2. Navigate to Teams web, search for the agent by name, send a test message /// 3. Wait for the agent to respond /// 4. Save browser auth state for future runs /// @@ -321,17 +519,15 @@ protected internal virtual async Task GenerateCopilotChatConversationAsync return false; } - // Instance name is required (provided via --instance-name) var agentName = _instanceName; if (string.IsNullOrWhiteSpace(agentName)) { - logger.LogWarning("Instance name not provided. Use --instance-name to specify the agent name in Copilot Chat."); + logger.LogWarning("Instance name not provided. Use --instance-name to specify the agent name in Teams."); return false; } - logger.LogInformation("Opening Copilot Chat conversation with agent '{AgentName}'...", agentName); + logger.LogInformation("Opening Teams chat conversation with agent '{AgentName}'...", agentName); - // Use the same tool-specific prompt as the conversation check var projectPath = Directory.GetCurrentDirectory(); var testMessage = ConversationRequirementCheck.BuildToolInvocationPrompt(projectPath, logger); logger.LogDebug("Using test message: {Message}", testMessage); @@ -343,7 +539,7 @@ protected internal virtual async Task GenerateCopilotChatConversationAsync if (string.IsNullOrWhiteSpace(response)) { - logger.LogWarning("Agent did not respond to the test message."); + logger.LogWarning("Agent did not respond to the test message in Teams."); return false; } diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/ConversationRequirementCheck.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/ConversationRequirementCheck.cs index 1fcc9cdb..4f2ae11c 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/ConversationRequirementCheck.cs +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/ConversationRequirementCheck.cs @@ -1064,7 +1064,7 @@ private static bool IsErrorResponse(string? responseText) /// /// Returns deploymentProjectPath if configured, otherwise falls back to the current directory. /// - private static string ResolveProjectPath(Agent365Config config) + internal static string ResolveProjectPath(Agent365Config config) { return string.IsNullOrWhiteSpace(config.DeploymentProjectPath) ? Directory.GetCurrentDirectory() diff --git a/src/Microsoft.Agents.A365.DevTools.Validation/ValidateReport.cs b/src/Microsoft.Agents.A365.DevTools.Validation/ValidateReport.cs index 3c614711..f406888d 100644 --- a/src/Microsoft.Agents.A365.DevTools.Validation/ValidateReport.cs +++ b/src/Microsoft.Agents.A365.DevTools.Validation/ValidateReport.cs @@ -67,15 +67,6 @@ public sealed class ValidationTiers [JsonPropertyName("blueprint")] public BlueprintTierResult Blueprint { get; set; } = TierResult.CreateSkipped("not yet implemented"); - [JsonPropertyName("agentMetrics")] - public AgentMetricsTierResult AgentMetrics { get; set; } = TierResult.CreateSkipped("not yet implemented"); - - [JsonPropertyName("mac")] - public MacTierResult Mac { get; set; } = TierResult.CreateSkipped("not yet implemented"); - - [JsonPropertyName("m365")] - public TierResult M365 { get; set; } = TierResult.CreateSkipped("not yet implemented"); - } /// diff --git a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Commands/ValidateCommandTests.cs b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Commands/ValidateCommandTests.cs index 83c0cefd..7f7b4676 100644 --- a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Commands/ValidateCommandTests.cs +++ b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Commands/ValidateCommandTests.cs @@ -183,6 +183,5 @@ public async Task ValidateCommand_Report_HasSkippedUnimplementedTiers() report.Tiers.Telemetry.Skipped.Should().BeTrue(); report.Tiers.Blueprint.Skipped.Should().BeTrue(); report.Tiers.AgentMetrics.Skipped.Should().BeTrue(); - report.Tiers.M365.Skipped.Should().BeTrue(); } } \ No newline at end of file From 0a7ac063a854f2a566f3569b8f1673075ffd0e7c Mon Sep 17 00:00:00 2001 From: Abbinayaa Subramanian Date: Thu, 11 Jun 2026 13:58:45 -0700 Subject: [PATCH 09/27] Cleaning up validate to include only local checks --- CHANGELOG.md | 3 + .../Commands/ValidateCommand.cs | 100 +- .../Models/Agent365Config.cs | 44 - .../Services/CopilotChatPlaywrightService.cs | 659 ------------- .../HttpListenerBotCallbackReceiver.cs | 159 +++- .../AgentMetricsRequirementCheck.cs | 582 ------------ .../MacVisibilityRequirementCheck.cs | 871 ------------------ .../TelemetryRequirementCheck.cs | 30 - .../ValidateReport.cs | 95 -- .../Commands/ValidateCommandTests.cs | 1 - .../HttpListenerBotCallbackReceiverTests.cs | 110 +++ .../AgentMetricsRequirementCheckTests.cs | 207 ----- .../MacVisibilityRequirementCheckTests.cs | 511 ---------- .../TelemetryRequirementCheckTests.cs | 47 +- 14 files changed, 276 insertions(+), 3143 deletions(-) delete mode 100644 src/Microsoft.Agents.A365.DevTools.Cli/Services/CopilotChatPlaywrightService.cs delete mode 100644 src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/AgentMetricsRequirementCheck.cs delete mode 100644 src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/MacVisibilityRequirementCheck.cs create mode 100644 src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/HttpListenerBotCallbackReceiverTests.cs delete mode 100644 src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/AgentMetricsRequirementCheckTests.cs delete mode 100644 src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/MacVisibilityRequirementCheckTests.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 71416e57..8b94564d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,9 @@ Agents provisioned before this release need `Agent365.Observability.OtelWrite` g ### Added - `a365 validate` — validates the local `a365.config.json` plus prerequisite checks without making changes. Reports missing or invalid config, then runs the existing setup prerequisite checks so users can catch problems before starting a setup workflow. + +### Changed +- `a365 validate` now runs blueprint registration checks (Entra app, service principal, permissions) as part of the normal flow. The `--with-tenant` option has been removed; all checks run in a single invocation. - New `Microsoft.Agents.A365.DevTools.Validation` subproject for reusable validation contracts and helpers. - `logs export [command] [--output ]` — exports a redacted copy of a CLI diagnostic log safe to share with Microsoft support. Redacts JWT tokens, email addresses, OS-path usernames, and tenant-specific GUIDs; replaces identical values with consistent aliases so log correlation is preserved. Preserves diagnostic IDs that aren't sensitive but are useful for debugging — `TraceId`, `CorrelationId`, Microsoft Graph `request-id` and `client-request-id` values, and well-known public Microsoft / Agent 365 resource appIds (such as the Microsoft Graph appId `00000003-0000-0000-c000-000000000000`). Omit `[command]` to export all available logs at once. - `setup blueprint --show-secret` — displays the blueprint client secret stored in `a365.generated.config.json` in plaintext without re-running any setup steps. On Windows, decryption requires the same machine and user account that ran setup (DPAPI). When no secret is found, the command prints instructions to run `a365 setup blueprint --agent-name `. diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Commands/ValidateCommand.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Commands/ValidateCommand.cs index 5e680848..0177cd27 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Commands/ValidateCommand.cs +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Commands/ValidateCommand.cs @@ -48,23 +48,13 @@ public static Command CreateCommand( { var command = new Command(CommandNames.Validate, "Validate the local Agent 365 CLI configuration and prerequisite state\n" + - "Checks config validity and code health. Run 'a365 setup all' before using this command."); + "Checks config validity, code health, and blueprint registration. Run 'a365 setup all' before using this command."); var playgroundOption = new Option( "--playground", "Launch AgentsPlayground after automated conversation turns for interactive testing"); command.AddOption(playgroundOption); - var withTenantOption = new Option( - "--with-tenant", - "Run tenant-level checks (blueprint registration, permissions, consent)"); - command.AddOption(withTenantOption); - - var instanceNameOption = new Option( - "--instance-name", - "Agent instance display name (reserved for future agent metrics validation)"); - command.AddOption(instanceNameOption); - command.SetHandler(async (InvocationContext context) => { var ct = context.GetCancellationToken(); @@ -72,7 +62,6 @@ public static Command CreateCommand( var configPath = Path.Combine(cwd, ConfigConstants.DefaultConfigFileName); var report = new ValidateReport(); var launchPlayground = context.ParseResult.GetValueForOption(playgroundOption); - var withTenant = context.ParseResult.GetValueForOption(withTenantOption); try { @@ -99,55 +88,7 @@ public static Command CreateCommand( : null }; - if (withTenant) - { - // --with-tenant: load previous report, run only tenant checks, merge - var existingReport = await LoadExistingReportAsync(cwd, logger); - if (existingReport is not null) - { - // Carry forward all local tiers from previous run - report.Tiers.Structural = existingReport.Tiers.Structural; - report.Tiers.Build = existingReport.Tiers.Build; - report.Tiers.Boot = existingReport.Tiers.Boot; - report.Tiers.Conversation = existingReport.Tiers.Conversation; - report.Tiers.Telemetry = existingReport.Tiers.Telemetry; - report.Agent = existingReport.Agent ?? report.Agent; - report.AgentConsoleLogFile = existingReport.AgentConsoleLogFile; - } - else - { - logger.LogWarning("No previous {ReportFile} found. Run 'a365 validate' first, then 'a365 validate --with-tenant'.", ReportFileName); - context.ExitCode = 1; - return; - } - - var tenantResults = new List<(IRequirementCheck Check, RequirementCheckResult Result)>(); - - // Blueprint registration check - if (requirementChecksOverride is null && graphApiService is not null) - { - var registrationCheck = new BlueprintRegistrationRequirementCheck(graphApiService, agentBlueprintService); - var registrationResults = await RunChecksDetailedAsync( - new List { registrationCheck }, config, logger, ct); - MapResultsToTiers(registrationResults, report); - tenantResults.AddRange(registrationResults); - } - - // Summary based on all tiers (existing + new tenant) - var tenantAnyFailed = tenantResults.Any(r => !r.Result.Passed); - var tenantBlocker = FindBlocker(report.Tiers); - report.Summary = new SummaryResult - { - Ok = !tenantAnyFailed && tenantBlocker is null, - Blocker = tenantBlocker - }; - - context.ExitCode = report.Summary.Ok ? 0 : 1; - PrintSummary(report, logger); - return; - } - - // --- Non-tenant flow: run all local checks --- + // --- Run all checks --- // Phase 2: Run structural checks (manifest + build) var structuralChecks = requirementChecksOverride?.ToList() @@ -156,9 +97,6 @@ public static Command CreateCommand( var results = await RunChecksDetailedAsync(structuralChecks, config, logger, ct); MapResultsToTiers(results, report); - // Mark tenant-dependent tiers as skipped - report.Tiers.Blueprint = TierResult.CreateSkipped("use --with-tenant"); - // Extract resolved uv command from build step for boot and conversation steps var buildResultEntry = results .FirstOrDefault(r => r.Check is ProjectBuildRequirementCheck); @@ -229,7 +167,17 @@ public static Command CreateCommand( // Conversation checks from override are already in results via MapResultsToTiers } - // Phase 3: Build summary — any failed check is a blocker + // Phase 3: Blueprint registration check (tenant-level) + if (requirementChecksOverride is null && graphApiService is not null) + { + var registrationCheck = new BlueprintRegistrationRequirementCheck(graphApiService, agentBlueprintService); + var registrationResults = await RunChecksDetailedAsync( + new List { registrationCheck }, config, logger, ct); + MapResultsToTiers(registrationResults, report); + results.AddRange(registrationResults); + } + + // Phase 4: Build summary — any failed check is a blocker var anyFailed = results.Any(r => !r.Result.Passed); var blocker = FindBlocker(report.Tiers); report.Summary = new SummaryResult @@ -832,28 +780,6 @@ private static async Task WriteReportAsync(ValidateReport report, string directo } } - private static async Task LoadExistingReportAsync(string directory, ILogger logger) - { - var reportPath = Path.Combine(directory, ReportFileName); - if (!File.Exists(reportPath)) - { - return null; - } - - try - { - var json = await File.ReadAllTextAsync(reportPath); - var report = JsonSerializer.Deserialize(json, ReportSerializerOptions); - logger.LogDebug("Loaded existing report from {ReportPath}", reportPath); - return report; - } - catch (Exception ex) - { - logger.LogWarning(ex, "Failed to load existing report from {ReportPath}", reportPath); - return null; - } - } - private static string ResolveProjectPath(Agent365Config config) { return string.IsNullOrWhiteSpace(config.DeploymentProjectPath) diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Models/Agent365Config.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Models/Agent365Config.cs index fe27e616..b9323f71 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Models/Agent365Config.cs +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Models/Agent365Config.cs @@ -165,12 +165,6 @@ private static void ValidateAuthMode(string? value, List errors) [JsonPropertyName("clientAppId")] public string ClientAppId { get; init; } = string.Empty; - /// - /// Optional local overrides for observability MCP validation calls. - /// - [JsonPropertyName("agent365ObservabilityMcpOptions")] - public Agent365ObservabilityMcpOptions? Agent365ObservabilityMcpOptions { get; init; } - /// /// Authentication pattern for the agent identity (blueprint agents only). /// Accepted values: "obo" (default), "s2s", "both". @@ -694,7 +688,6 @@ public Agent365Config WithCustomBlueprintPermissions(List -/// Optional overrides for observability MCP calls used by local validation. -/// -public sealed class Agent365ObservabilityMcpOptions -{ - /// - /// Base URL for the observability service host. The MCP path is appended automatically. - /// - [JsonPropertyName("baseUrl")] - public string? BaseUrl { get; init; } - - /// - /// Tenant ID used to construct the observability MCP endpoint. - /// - [JsonPropertyName("tenantId")] - public string? TenantId { get; init; } - - /// - /// Optional observability identifier passed to getAgentMetrics as agentObservabilityId. - /// When provided, this takes precedence over AgentName. - /// - [JsonPropertyName("agentObservabilityId")] - public string? AgentObservabilityId { get; init; } - - /// - /// Agent name passed to getAgentMetrics when validating a specific agent. - /// - [JsonPropertyName("agentName")] - public string? AgentName { get; init; } - - /// - /// App ID used as token audience when calling observability MCP endpoints. - /// - [JsonPropertyName("appId")] - public string? AppId { get; init; } -} diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Services/CopilotChatPlaywrightService.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Services/CopilotChatPlaywrightService.cs deleted file mode 100644 index 5d253f7e..00000000 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Services/CopilotChatPlaywrightService.cs +++ /dev/null @@ -1,659 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -using Microsoft.Extensions.Logging; -using Microsoft.Playwright; - -namespace Microsoft.Agents.A365.DevTools.Cli.Services; - -/// -/// Automates a conversation with an agent in Microsoft Teams using Playwright. -/// Opens Teams web, searches for the agent by name in a new chat, sends a test -/// message, and waits for the agent to respond. -/// -/// Auth strategy: -/// - Reuses the CLI's existing MSAL browser session. On Windows, WAM (Windows -/// Authentication Manager) automatically provides SSO for the Teams domain, so -/// the browser launched by Playwright inherits the user's session without any -/// manual login. -/// - On non-Windows platforms (or if SSO is not available), a headed browser -/// opens and waits for the user to log in manually. -/// - Saves browser storage state to a local file for reuse across validate runs. -/// -public class CopilotChatPlaywrightService -{ - private readonly ILogger _logger; - - /// Base URL for Microsoft Teams web. - internal const string ChatBaseUrl = "https://teams.microsoft.com"; - - /// URL pattern indicating the user has successfully authenticated into Teams. - internal const string AuthenticatedUrlPattern = "**/teams.microsoft.com/**"; - - /// Timeout for agent response in milliseconds (2 minutes). - internal const int AgentResponseTimeoutMs = 120_000; - - /// Timeout for page navigation in milliseconds (60 seconds). - internal const int NavigationTimeoutMs = 60_000; - - /// - /// Directory name under LocalApplicationData for storing auth state. - /// Reuses the same directory as the CLI's MSAL token cache. - /// - private static readonly string AuthStateDirectory = Path.Combine( - Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), - Constants.AuthenticationConstants.ApplicationName); - - /// File name for the Playwright browser auth state. - private const string AuthStateFileName = "playwright-auth-state.json"; - - /// Auth state is reused if it is less than this many minutes old. - private const int AuthStateTtlMinutes = 30; - - public CopilotChatPlaywrightService(ILogger logger) - { - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - } - - /// - /// Sends a test message to the specified agent in Teams web and returns the - /// agent's response text, or null if the conversation could not be completed. - /// - /// Display name of the agent (bot) in Teams. - /// Message to send to the agent. - /// Cancellation token. - /// The agent's response text, or null if the conversation failed. - public virtual async Task SendMessageToAgentAsync( - string agentName, - string testMessage, - CancellationToken cancellationToken = default) - { - ArgumentException.ThrowIfNullOrWhiteSpace(agentName); - ArgumentException.ThrowIfNullOrWhiteSpace(testMessage); - - var authStatePath = GetAuthStatePath(); - - _logger.LogDebug("Ensuring Playwright browsers are installed..."); - var installExitCode = Microsoft.Playwright.Program.Main(new[] { "install", "chromium" }); - if (installExitCode != 0) - { - _logger.LogWarning("Playwright browser install returned exit code {ExitCode}. Attempting to continue.", installExitCode); - } - - using var playwright = await Microsoft.Playwright.Playwright.CreateAsync(); - - var hasFreshAuthState = HasFreshAuthState(authStatePath); - - var launchOptions = new BrowserTypeLaunchOptions - { - Headless = false, - SlowMo = 100 - }; - - _logger.LogDebug("Launching Chromium (headed)..."); - await using var browser = await playwright.Chromium.LaunchAsync(launchOptions); - - var contextOptions = new BrowserNewContextOptions - { - ViewportSize = new ViewportSize { Width = 1280, Height = 720 } - }; - if (hasFreshAuthState) - { - contextOptions.StorageStatePath = authStatePath; - } - - await using var context = await browser.NewContextAsync(contextOptions); - context.SetDefaultTimeout(30_000); - context.SetDefaultNavigationTimeout(NavigationTimeoutMs); - - var page = await context.NewPageAsync(); - - _logger.LogInformation("Navigating to Teams web..."); - await page.GotoAsync(ChatBaseUrl, new PageGotoOptions - { - WaitUntil = WaitUntilState.DOMContentLoaded, - Timeout = NavigationTimeoutMs - }); - - // Check if we landed on Teams or need to authenticate - var isAuthenticated = await IsTeamsLoadedAsync(page); - - if (!isAuthenticated) - { - if (hasFreshAuthState) - { - _logger.LogDebug("Saved auth state did not work. Re-launching as headed for login..."); - await page.CloseAsync(); - await context.CloseAsync(); - await browser.CloseAsync(); - - return await RunWithInteractiveLoginAsync(playwright, agentName, testMessage, authStatePath, cancellationToken); - } - - _logger.LogInformation("Please log in to Teams in the browser window."); - _logger.LogInformation("The CLI will continue automatically once login completes."); - - await page.WaitForURLAsync(AuthenticatedUrlPattern, - new PageWaitForURLOptions { Timeout = 300_000 }); - - // Wait for Teams to fully load after redirect - await WaitForTeamsReadyAsync(page); - - _logger.LogInformation("Login successful."); - } - - await SaveAuthStateAsync(context, authStatePath); - - var responseText = await OpenAgentChatAndSendMessageAsync(page, agentName, testMessage, cancellationToken); - - return responseText; - } - - /// - /// Fallback path: launch a headed browser for interactive login, then send message. - /// - private async Task RunWithInteractiveLoginAsync( - IPlaywright playwright, - string agentName, - string testMessage, - string authStatePath, - CancellationToken cancellationToken) - { - var launchOptions = new BrowserTypeLaunchOptions - { - Headless = false, - SlowMo = 100 - }; - - await using var browser = await playwright.Chromium.LaunchAsync(launchOptions); - var contextOptions = new BrowserNewContextOptions - { - ViewportSize = new ViewportSize { Width = 1280, Height = 720 } - }; - await using var context = await browser.NewContextAsync(contextOptions); - context.SetDefaultTimeout(30_000); - context.SetDefaultNavigationTimeout(NavigationTimeoutMs); - - var page = await context.NewPageAsync(); - - await page.GotoAsync(ChatBaseUrl, new PageGotoOptions - { - WaitUntil = WaitUntilState.DOMContentLoaded, - Timeout = NavigationTimeoutMs - }); - - _logger.LogInformation("Please log in to Teams in the browser window."); - _logger.LogInformation("The CLI will continue automatically once login completes."); - - await page.WaitForURLAsync(AuthenticatedUrlPattern, - new PageWaitForURLOptions { Timeout = 300_000 }); - - await WaitForTeamsReadyAsync(page); - - _logger.LogInformation("Login successful."); - - await SaveAuthStateAsync(context, authStatePath); - - return await OpenAgentChatAndSendMessageAsync(page, agentName, testMessage, cancellationToken); - } - - /// - /// Checks if the Teams web app has loaded by looking for the left rail or chat UI. - /// - private async Task IsTeamsLoadedAsync(IPage page) - { - try - { - // Teams v2 uses a left app bar with Chat, Activity, etc. - var chatNavItem = page.Locator("[data-tid='app-bar-86fcd49b-61a2-4701-b771-54728cd291fb']") - .Or(page.GetByRole(AriaRole.Tab, new PageGetByRoleOptions { Name = "Chat" })) - .Or(page.Locator("[data-tid='app-bar-chat']")); - - // Wait briefly for the chat nav to appear, then check visibility - await chatNavItem.First.WaitForAsync(new LocatorWaitForOptions - { - State = WaitForSelectorState.Visible, - Timeout = 10_000 - }); - return true; - } - catch - { - return false; - } - } - - /// - /// Waits for the Teams web UI to become ready after authentication. - /// - private async Task WaitForTeamsReadyAsync(IPage page) - { - // Wait for the app bar or chat section to appear, indicating Teams has loaded - var chatNavItem = page.Locator("[data-tid='app-bar-86fcd49b-61a2-4701-b771-54728cd291fb']") - .Or(page.GetByRole(AriaRole.Tab, new PageGetByRoleOptions { Name = "Chat" })) - .Or(page.Locator("[data-tid='app-bar-chat']")); - - await chatNavItem.First.WaitForAsync(new LocatorWaitForOptions - { - State = WaitForSelectorState.Visible, - Timeout = 60_000 - }); - } - - /// - /// Opens a chat with the agent in Teams using the top search bar, sends a test - /// message, and returns the agent's response text. - /// - private async Task OpenAgentChatAndSendMessageAsync( - IPage page, - string agentName, - string testMessage, - CancellationToken cancellationToken) - { - // Step 1: Use the top search bar to find the agent - _logger.LogInformation("Searching for agent '{AgentName}' via search bar...", agentName); - var searchBox = page.Locator("[data-tid='searchbox']") - .Or(page.GetByRole(AriaRole.Search)) - .Or(page.GetByPlaceholder("Search")); - - try - { - await searchBox.First.WaitForAsync(new LocatorWaitForOptions - { - State = WaitForSelectorState.Visible, - Timeout = 15_000 - }); - await searchBox.First.ClickAsync(); - await page.WaitForTimeoutAsync(500); - await page.Keyboard.TypeAsync(agentName, new KeyboardTypeOptions { Delay = 50 }); - await page.WaitForTimeoutAsync(3000); - } - catch (Exception ex) - { - _logger.LogWarning("Could not use the search bar: {Message}", ex.Message); - await CaptureScreenshotAsync(page, "teams-no-search-bar"); - return null; - } - - // Step 2: Click the matching result from the search dropdown - _logger.LogInformation("Selecting agent from search results..."); - var searchResult = page.GetByRole(AriaRole.Option).Filter(new LocatorFilterOptions { HasText = agentName }) - .Or(page.GetByRole(AriaRole.Listitem).Filter(new LocatorFilterOptions { HasText = agentName })); - - try - { - await searchResult.First.WaitForAsync(new LocatorWaitForOptions - { - State = WaitForSelectorState.Visible, - Timeout = 10_000 - }); - await searchResult.First.ClickAsync(); - } - catch (Exception ex) - { - _logger.LogWarning("Agent '{AgentName}' was not found in search results: {Message}", agentName, ex.Message); - await CaptureScreenshotAsync(page, "teams-agent-not-found"); - return null; - } - - await page.WaitForTimeoutAsync(3000); - - // Step 3: Type and send the message in the compose box - _logger.LogInformation("Sending test message to agent..."); - var composeBox = page.GetByRole(AriaRole.Textbox, new PageGetByRoleOptions { Name = "Type a message" }) - .Or(page.Locator("[data-tid='ckeditor-replyConversation']")) - .Or(page.GetByRole(AriaRole.Textbox, new PageGetByRoleOptions { Name = "Type a new message" })); - - try - { - await composeBox.First.WaitForAsync(new LocatorWaitForOptions - { - State = WaitForSelectorState.Visible, - Timeout = 15_000 - }); - await composeBox.First.ClickAsync(); - await page.WaitForTimeoutAsync(500); - - await composeBox.First.FillAsync(testMessage); - await page.WaitForTimeoutAsync(500); - } - catch (Exception ex) - { - _logger.LogWarning("Could not find or fill the compose box: {Message}", ex.Message); - await CaptureScreenshotAsync(page, "teams-no-compose-box"); - return null; - } - - // Record message count before sending so we can detect new messages - var messagesBefore = await CountVisibleMessagesAsync(page); - _logger.LogDebug("Messages visible before sending: {Count}", messagesBefore); - - // Click the send button or press Enter - var sendButton = page.Locator("[data-tid='newMessageCommands-send']") - .Or(page.GetByRole(AriaRole.Button, new PageGetByRoleOptions { Name = "Send" })); - - if (await sendButton.First.IsVisibleAsync().ConfigureAwait(false)) - { - await sendButton.First.ClickAsync(); - } - else - { - await composeBox.First.PressAsync("Enter"); - } - - _logger.LogInformation("Message sent, waiting for agent response..."); - - // Step 4: Wait for agent response (new message must appear) - var responseText = await WaitForNewMessageAsync(page, messagesBefore, cancellationToken); - - if (string.IsNullOrWhiteSpace(responseText)) - { - _logger.LogWarning("Agent response was empty."); - await CaptureScreenshotAsync(page, "teams-empty-response"); - return null; - } - - _logger.LogInformation("Agent responded with {Length} characters.", responseText.Length); - return responseText; - } - - /// - /// Captures a screenshot for debugging when a step fails. - /// Saved to the CLI's local data directory. - /// - private async Task CaptureScreenshotAsync(IPage page, string stepName) - { - try - { - var screenshotDir = Path.Combine( - Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), - Constants.AuthenticationConstants.ApplicationName, - "screenshots"); - Directory.CreateDirectory(screenshotDir); - - var fileName = $"{stepName}-{DateTime.UtcNow:yyyyMMdd-HHmmss}.png"; - var filePath = Path.Combine(screenshotDir, fileName); - - await page.ScreenshotAsync(new PageScreenshotOptions { Path = filePath, FullPage = true }); - _logger.LogInformation("Screenshot saved: {Path}", filePath); - } - catch (Exception ex) - { - _logger.LogDebug(ex, "Failed to capture screenshot for step '{Step}'.", stepName); - } - } - - /// - /// Counts visible message elements in the chat pane. - /// Uses multiple selector strategies to find message containers in Teams. - /// - private static async Task CountVisibleMessagesAsync(IPage page) - { - // Teams renders messages in divs with data-tid="chat-pane-message" or similar - var selectors = new[] - { - "[data-tid='chat-pane-message']", - "[data-tid='messageListItem']", - ".message-body-content", - }; - - foreach (var selector in selectors) - { - var count = await page.Locator(selector).CountAsync(); - if (count > 0) - { - return count; - } - } - - return 0; - } - - /// - /// Waits for a new message to appear after the user's message was sent. - /// Polls the page for new message elements and waits for the response text to stabilize. - /// Returns null if no new message appears within the timeout. - /// - /// - /// Common placeholder patterns that agents send before the real response. - /// These are short acknowledgements that should be ignored. - /// - private static readonly string[] PlaceholderPatterns = new[] - { - "got it", "working on it", "let me", "thinking", "one moment", - "just a moment", "hold on", "processing", "looking into", - "give me a moment", "i'm on it", "sure thing" - }; - - private async Task WaitForNewMessageAsync( - IPage page, - int messageCountBefore, - CancellationToken cancellationToken) - { - var timeout = AgentResponseTimeoutMs; - var pollStart = Environment.TickCount64; - var pollIntervalMs = 3000; - - // Phase 1: Wait for any new message to appear beyond our sent message - _logger.LogDebug("Waiting for new messages (had {Before} before)...", messageCountBefore); - - var newMessageDetected = false; - while (Environment.TickCount64 - pollStart < timeout) - { - cancellationToken.ThrowIfCancellationRequested(); - - var currentCount = await CountVisibleMessagesAsync(page); - if (currentCount >= messageCountBefore + 2) - { - newMessageDetected = true; - _logger.LogDebug("New message detected (count: {Before} -> {Current}).", messageCountBefore, currentCount); - break; - } - - var elapsed = (Environment.TickCount64 - pollStart) / 1000; - _logger.LogInformation("Waiting for agent response... ({Elapsed}s)", elapsed); - await page.WaitForTimeoutAsync(pollIntervalMs); - } - - if (!newMessageDetected) - { - _logger.LogWarning("No new message appeared within {Timeout}s timeout.", timeout / 1000); - await CaptureScreenshotAsync(page, "teams-no-response"); - return null; - } - - // Phase 2: Wait for agent to finish responding. - // The agent may send a placeholder first (e.g. "Got it...working on it"), - // then replace or follow up with the actual response. We need to wait for: - // (a) typing indicator to clear, (b) text to not be a placeholder, - // (c) text to stabilize. - var previousText = string.Empty; - var stableCount = 0; - var phase2Start = Environment.TickCount64; - var phase2TimeoutMs = 120_000; // 2 minutes for the real response - - while (Environment.TickCount64 - phase2Start < phase2TimeoutMs) - { - cancellationToken.ThrowIfCancellationRequested(); - - // Check typing indicator - var isTyping = false; - try - { - var typingIndicator = page.Locator("[data-tid='chat-typing-indicator']") - .Or(page.Locator(".typing-indicator")); - isTyping = await typingIndicator.First.IsVisibleAsync().ConfigureAwait(false); - } - catch - { - // No typing indicator found - } - - if (isTyping) - { - _logger.LogDebug("Agent still typing..."); - stableCount = 0; - await page.WaitForTimeoutAsync(2000); - continue; - } - - var currentText = await ExtractLatestMessageTextAsync(page); - - // Check if current text looks like a placeholder - if (IsPlaceholderResponse(currentText)) - { - var elapsed = (Environment.TickCount64 - phase2Start) / 1000; - _logger.LogDebug("Detected placeholder response, waiting for real response... ({Elapsed}s)", elapsed); - stableCount = 0; - await page.WaitForTimeoutAsync(3000); - continue; - } - - // Check if text has stabilized (same non-placeholder text 3 times in a row) - if (!string.IsNullOrEmpty(currentText) && currentText == previousText) - { - stableCount++; - if (stableCount >= 3) - { - _logger.LogDebug("Response text stabilized ({Length} chars).", currentText.Length); - return currentText; - } - } - else - { - stableCount = 0; - } - - previousText = currentText; - await page.WaitForTimeoutAsync(2000); - } - - // Return whatever we have after timeout - if (!string.IsNullOrEmpty(previousText) && !IsPlaceholderResponse(previousText)) - { - return previousText; - } - - _logger.LogWarning("Agent response did not stabilize within timeout."); - await CaptureScreenshotAsync(page, "teams-response-timeout"); - return null; - } - - /// - /// Checks if a response looks like a short placeholder/acknowledgement - /// that the agent sends before the real answer. - /// - private static bool IsPlaceholderResponse(string? text) - { - if (string.IsNullOrWhiteSpace(text)) - { - return true; - } - - // Very short responses (under 30 chars) that match placeholder patterns - var trimmed = text.Trim(); - if (trimmed.Length > 80) - { - return false; - } - - var lower = trimmed.ToLowerInvariant(); - return PlaceholderPatterns.Any(p => lower.Contains(p, StringComparison.Ordinal)); - } - - /// - /// Extracts the text content from the last message element in the chat pane. - /// - private static async Task ExtractLatestMessageTextAsync(IPage page) - { - var selectors = new[] - { - "[data-tid='chat-pane-message']", - "[data-tid='messageListItem']", - ".message-body-content", - }; - - foreach (var selector in selectors) - { - var messages = page.Locator(selector); - var count = await messages.CountAsync(); - if (count == 0) - { - continue; - } - - // Get the last message's text - try - { - var text = await messages.Nth(count - 1).InnerTextAsync() ?? string.Empty; - if (!string.IsNullOrWhiteSpace(text)) - { - return text.Trim(); - } - } - catch - { - continue; - } - } - - return string.Empty; - } - - /// - /// Checks if saved auth state exists and is less than old. - /// - private bool HasFreshAuthState(string authStatePath) - { - if (!File.Exists(authStatePath)) - { - _logger.LogDebug("No saved auth state found at {Path}.", authStatePath); - return false; - } - - var fileInfo = new FileInfo(authStatePath); - var ageMinutes = (DateTime.UtcNow - fileInfo.LastWriteTimeUtc).TotalMinutes; - if (ageMinutes < AuthStateTtlMinutes) - { - _logger.LogDebug("Auth state is {Age:F0} min old (< {Ttl} min). Reusing.", ageMinutes, AuthStateTtlMinutes); - return true; - } - - _logger.LogDebug("Auth state is {Age:F0} min old (>= {Ttl} min). Will re-authenticate.", ageMinutes, AuthStateTtlMinutes); - return false; - } - - /// - /// Saves browser context storage state for reuse. - /// - private async Task SaveAuthStateAsync(IBrowserContext context, string authStatePath) - { - try - { - var directory = Path.GetDirectoryName(authStatePath); - if (!string.IsNullOrEmpty(directory)) - { - Directory.CreateDirectory(directory); - } - - await context.StorageStateAsync(new BrowserContextStorageStateOptions - { - Path = authStatePath - }); - _logger.LogDebug("Auth state saved to {Path}.", authStatePath); - } - catch (Exception ex) - { - // Non-fatal: failing to save just means the user re-authenticates next time - _logger.LogDebug(ex, "Failed to save auth state: {Message}", ex.Message); - } - } - - /// - /// Gets the path where Playwright auth state is stored. - /// - internal static string GetAuthStatePath() - { - return Path.Combine(AuthStateDirectory, AuthStateFileName); - } -} diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Services/HttpListenerBotCallbackReceiver.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Services/HttpListenerBotCallbackReceiver.cs index adf070f1..4857dd5b 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Services/HttpListenerBotCallbackReceiver.cs +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Services/HttpListenerBotCallbackReceiver.cs @@ -23,11 +23,40 @@ internal sealed class HttpListenerBotCallbackReceiver : IBotCallbackReceiver private Task? _listenTask; /// - /// After the first callback arrives, continue collecting for this long to capture - /// the actual response (agents often send an acknowledgment before the real reply). + /// After a non-interim callback arrives, continue collecting for this long to capture + /// any follow-up responses. /// internal static readonly TimeSpan GracePeriod = TimeSpan.FromSeconds(5); + /// + /// Extended timeout used when only interim/acknowledgment responses have been received. + /// Agents that send "Got it..working on it" often need 10-30s to produce the real response. + /// + internal static readonly TimeSpan InterimExtendedTimeout = TimeSpan.FromSeconds(30); + + /// + /// Patterns that indicate an interim/acknowledgment message rather than a real response. + /// These are anchored to avoid matching error messages (e.g. "processing failed" is NOT interim). + /// + internal static readonly string[] InterimPatterns = new[] + { + "got it", + "working on", + "work on it", + "processing your", + "thinking", + "one moment", + "please wait", + "hold on", + "looking into", + "let me check", + "let me look", + "let me find", + "let me get", + "just a moment", + "just a sec", + }; + public string ServiceUrl => $"http://localhost:{_port}"; public HttpListenerBotCallbackReceiver() @@ -62,13 +91,65 @@ public Task StartAsync(CancellationToken cancellationToken = default) using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); timeoutCts.CancelAfter(timeout); + bool receivedInterim = false; + try { // Wait for the first callback activity await _responseReceived.WaitAsync(timeoutCts.Token); - // Grace period: keep collecting to capture the actual response - // after an initial acknowledgment (e.g., "Got it - working on it...") + // Check if the first response is an interim message + bool firstIsFinal; + lock (_lock) + { + var latest = _responses.Count > 0 ? _responses[^1] : null; + firstIsFinal = latest is not null && IsFinalMessage(latest); + if (!firstIsFinal) + { + receivedInterim = true; + } + } + + if (receivedInterim) + { + // Interim message received — agent is alive but still processing. + // Extend the timeout to allow the real response to arrive. + var extendedDeadline = DateTime.UtcNow + InterimExtendedTimeout; + + while (DateTime.UtcNow < extendedDeadline && !timeoutCts.Token.IsCancellationRequested) + { + var remaining = extendedDeadline - DateTime.UtcNow; + if (remaining <= TimeSpan.Zero) + { + break; + } + + try + { + if (!await _responseReceived.WaitAsync(remaining, timeoutCts.Token)) + { + break; + } + + // Check if we now have a final response + lock (_lock) + { + var latest = _responses.Count > 0 ? _responses[^1] : null; + if (latest is not null && IsFinalMessage(latest)) + { + // Got a real response — start the short grace period for any follow-ups + break; + } + } + } + catch (OperationCanceledException) + { + break; + } + } + } + + // Grace period: collect any remaining follow-up responses var graceDeadline = DateTime.UtcNow + GracePeriod; while (DateTime.UtcNow < graceDeadline && !timeoutCts.Token.IsCancellationRequested) @@ -190,8 +271,8 @@ private async Task HandleRequestAsync(HttpListenerContext context) /// /// Selects the best response from collected callbacks. - /// Prefers the last message-type response with substantive text, - /// falling back to the last response of any type. + /// Returns null if only interim/typing responses were collected (agent did not produce a final answer). + /// Prefers the last message-type response with substantive non-interim text. /// private BotCallbackResponse? SelectBestResponse() { @@ -200,15 +281,69 @@ private async Task HandleRequestAsync(HttpListenerContext context) return null; } - // Prefer the last message with non-trivial text (skip short acknowledgments) + // Prefer the last final message (non-interim, non-typing, with substantive text) var bestMessage = _responses - .LastOrDefault(r => r.Type == "message" && !string.IsNullOrWhiteSpace(r.Text) && r.Text.Length > 30); + .LastOrDefault(r => IsFinalMessage(r)); + + if (bestMessage is not null) + { + return bestMessage; + } + + // No final message found — all responses were interim or typing. + // Return null so the caller treats this as "agent did not respond with a final answer". + return null; + } + + /// + /// Determines whether a response is a final (non-interim) message from the agent. + /// + internal static bool IsFinalMessage(BotCallbackResponse response) + { + // Typing activities are never final + if (string.Equals(response.Type, "typing", StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + // Must be a message with text + if (response.Type != "message" || string.IsNullOrWhiteSpace(response.Text)) + { + return false; + } + + return !IsInterimMessage(response.Text); + } - // Fall back to last message with any text - bestMessage ??= _responses.LastOrDefault(r => r.Type == "message" && !string.IsNullOrWhiteSpace(r.Text)); + /// + /// Detects interim/acknowledgment messages that agents send while processing. + /// Only matches short messages (under 60 chars) containing known interim phrases. + /// Longer messages are assumed to be real responses even if they contain interim-like words. + /// + internal static bool IsInterimMessage(string? text) + { + if (string.IsNullOrWhiteSpace(text)) + { + return false; + } + + // Long messages are unlikely to be interim acknowledgments + if (text.Length > 60) + { + return false; + } + + var lower = text.ToLowerInvariant(); + + foreach (var pattern in InterimPatterns) + { + if (lower.Contains(pattern, StringComparison.Ordinal)) + { + return true; + } + } - // Fall back to last response of any kind - return bestMessage ?? _responses[^1]; + return false; } public async ValueTask DisposeAsync() diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/AgentMetricsRequirementCheck.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/AgentMetricsRequirementCheck.cs deleted file mode 100644 index d64df1a3..00000000 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/AgentMetricsRequirementCheck.cs +++ /dev/null @@ -1,582 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -using System.Diagnostics; -using Microsoft.Agents.A365.DevTools.Cli.Models; -using Microsoft.Agents.A365.DevTools.Cli.Services; -using Microsoft.Agents.A365.DevTools.Cli.Constants; -using Microsoft.Agents.A365.DevTools.Cli.Services.Internal; -using Microsoft.Extensions.Logging; -using System.Net.Http.Headers; -using System.Text; -using System.Text.Json; - -namespace Microsoft.Agents.A365.DevTools.Cli.Services.Requirements.RequirementChecks; - -/// -/// Validates that agent metrics are visible and incrementing by: -/// 1. Starting the agent locally -/// 2. Querying baseline metrics via MCP tool call -/// 3. Generating a conversation with the agent in Teams Chat (via Playwright) -/// 4. Re-querying metrics to verify they incremented -/// 5. Stopping the agent -/// -public class AgentMetricsRequirementCheck : RequirementCheck -{ - private const string GetAgentMetricsToolName = "getAgentMetrics"; - - private readonly CopilotChatPlaywrightService? _playwrightService; - private readonly string? _instanceName; - private readonly PlatformDetector? _platformDetector; - private readonly IProcessService? _processService; - private readonly string? _resolvedUvCommand; - - /// Default test message sent to the agent during the metrics check. - internal const string DefaultTestMessage = ConversationRequirementCheck.FallbackToolPrompt; - - /// Maximum time to wait for the agent to start and respond on the health endpoint. - private static readonly TimeSpan AgentStartupTimeout = TimeSpan.FromSeconds(30); - - /// Interval between health endpoint polls during startup. - private static readonly TimeSpan HealthPollInterval = TimeSpan.FromMilliseconds(500); - - public AgentMetricsRequirementCheck( - CopilotChatPlaywrightService? playwrightService = null, - string? instanceName = null, - PlatformDetector? platformDetector = null, - IProcessService? processService = null, - string? resolvedUvCommand = null) - { - _playwrightService = playwrightService; - _instanceName = instanceName; - _platformDetector = platformDetector; - _processService = processService; - _resolvedUvCommand = resolvedUvCommand; - } - /// - public override string Name => "AgentMetrics"; - - /// - public override string Description => "Verifies agent metrics are visible and incrementing after a Teams Chat conversation"; - - /// - public override string Category => "Observability"; - - /// - public override Task CheckAsync( - Agent365Config config, - ILogger logger, - CancellationToken cancellationToken = default) - { - return ExecuteCheckWithLoggingAsync(config, logger, CheckImplementationAsync, cancellationToken); - } - - private async Task CheckImplementationAsync( - Agent365Config config, - ILogger logger, - CancellationToken cancellationToken) - { - var metadata = new AgentMetricsMetadata(); - Process? agentProcess = null; - - try - { - // Step 0: Start the agent locally - logger.LogInformation("Starting agent locally for metrics validation..."); - agentProcess = await StartAgentLocallyAsync(config, logger, cancellationToken); - if (agentProcess is null) - { - return RequirementCheckResult.Warning( - "Could not start the agent locally for metrics validation", - details: "Ensure the project builds and runs successfully. Run 'a365 validate' without --with-tenant first."); - } - - // Step 1: Get baseline metrics via MCP tool call (best-effort, does not block step 2) - logger.LogInformation("Step 1: Querying baseline agent metrics..."); - AgentMetricsSnapshot? baselineMetrics = null; - try - { - baselineMetrics = await GetAgentMetricsAsync(config, logger, cancellationToken); - } - catch (Exception ex) when (ex is not OperationCanceledException) - { - logger.LogDebug(ex, "Failed to query baseline agent metrics: {Message}", ex.Message); - } - - metadata.BaselineMetrics = baselineMetrics; - if (baselineMetrics is null) - { - logger.LogDebug("Baseline metrics not available -- will still attempt conversation."); - } - - // Step 2: Generate a conversation with the agent in Teams Chat (via Playwright) - logger.LogInformation("Step 2: Generating conversation with agent in Teams Chat..."); - bool conversationGenerated; - try - { - conversationGenerated = await GenerateCopilotChatConversationAsync(config, logger, cancellationToken); - } - catch (Exception ex) when (ex is not OperationCanceledException) - { - logger.LogWarning(ex, "Failed to generate Teams Chat conversation: {Message}", ex.Message); - return RequirementCheckResult.Warning( - "Could not generate Teams Chat conversation for metrics validation", - details: $"Playwright test failed: {ex.Message}"); - } - - metadata.ConversationGenerated = conversationGenerated; - - if (!conversationGenerated) - { - return RequirementCheckResult.Warning( - "Teams Chat conversation could not be generated", - details: "Playwright was unable to complete a conversation with the agent. Metrics increment check skipped."); - } - - if (baselineMetrics is null) - { - return RequirementCheckResult.Failure( - "Agent metrics endpoint not available", - "Ensure the agent is deployed and metrics are configured. The MCP metrics tool must be reachable.", - details: "Teams Chat conversation completed successfully but baseline metrics could not be retrieved."); - } - - // Step 3: Re-query metrics and verify they incremented - logger.LogInformation("Step 3: Re-querying agent metrics to verify increment..."); - AgentMetricsSnapshot? postConversationMetrics = null; - try - { - postConversationMetrics = await GetAgentMetricsAsync(config, logger, cancellationToken); - } - catch (Exception ex) when (ex is not OperationCanceledException) - { - logger.LogDebug(ex, "Failed to query post-conversation agent metrics: {Message}", ex.Message); - } - - metadata.PostConversationMetrics = postConversationMetrics; - - if (postConversationMetrics is null) - { - return RequirementCheckResult.Failure( - "Agent metrics endpoint not available after conversation", - "Ensure the metrics endpoint remains reachable. The MCP metrics tool must return data.", - details: "Teams Chat conversation completed successfully but post-conversation metrics could not be retrieved."); - } - - var incremented = postConversationMetrics.InvocationCount > baselineMetrics.InvocationCount; - metadata.MetricsIncremented = incremented; - - if (!incremented) - { - return RequirementCheckResult.Failure( - "Agent metrics did not increment after Teams Chat conversation", - "Verify that the agent is instrumented with Agent365 observability and that metrics are flowing to the backend.", - details: $"Baseline invocations: {baselineMetrics.InvocationCount}, " + - $"Post-conversation invocations: {postConversationMetrics.InvocationCount}"); - } - - return RequirementCheckResult.Success( - details: $"Agent metrics incremented from {baselineMetrics.InvocationCount} to " + - $"{postConversationMetrics.InvocationCount} after Teams Chat conversation."); - } - finally - { - if (agentProcess is not null && !agentProcess.HasExited) - { - logger.LogDebug("Stopping local agent process..."); - try - { - agentProcess.Kill(entireProcessTree: true); - await agentProcess.WaitForExitAsync(cancellationToken).WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); - } - catch - { - // Best-effort cleanup - } - finally - { - agentProcess.Dispose(); - } - } - } - } - - /// - /// Starts the agent locally and waits for the health endpoint to respond. - /// Returns the agent process, or null if the agent could not be started. - /// - protected internal virtual async Task StartAgentLocallyAsync( - Agent365Config config, - ILogger logger, - CancellationToken cancellationToken) - { - if (_platformDetector is null || _processService is null) - { - logger.LogWarning("Platform detector or process service not available. Cannot start agent locally."); - return null; - } - - var projectPath = ConversationRequirementCheck.ResolveProjectPath(config); - if (!Directory.Exists(projectPath)) - { - logger.LogWarning("Project path does not exist: {Path}", projectPath); - return null; - } - - var platform = _platformDetector.Detect(projectPath); - if (platform == ProjectPlatform.Unknown) - { - logger.LogWarning("Could not detect project platform in {Path}", projectPath); - return null; - } - - var port = LocalRuntimeRequirementCheck.ResolvePort(config.MessagingEndpoint); - var healthUrl = $"http://localhost:{port}{LocalRuntimeRequirementCheck.DefaultHealthPath}"; - - logger.LogInformation("Starting agent locally ({Platform} on port {Port})...", platform, port); - - var startInfo = BuildProcessStartInfo(platform, projectPath, port); - var process = _processService.Start(startInfo); - if (process is null) - { - logger.LogWarning("Failed to start {Platform} process.", platform); - return null; - } - - var healthResult = await WaitForHealthAsync(process, healthUrl, logger, cancellationToken); - if (!healthResult) - { - logger.LogWarning("Agent did not respond on health endpoint within timeout."); - try { process.Kill(entireProcessTree: true); } catch { } - process.Dispose(); - return null; - } - - logger.LogInformation("Agent is running and healthy on port {Port}.", port); - return process; - } - - private ProcessStartInfo BuildProcessStartInfo(ProjectPlatform platform, string projectPath, int port) - { - var startInfo = new ProcessStartInfo - { - WorkingDirectory = projectPath, - RedirectStandardOutput = false, - RedirectStandardError = false, - UseShellExecute = false, - CreateNoWindow = true - }; - - switch (platform) - { - case ProjectPlatform.DotNet: - startInfo.FileName = "dotnet"; - startInfo.Arguments = "run --no-build"; - startInfo.EnvironmentVariables["ASPNETCORE_URLS"] = $"http://localhost:{port}"; - break; - - case ProjectPlatform.NodeJs: - LocalRuntimeRequirementCheck.WrapForWindows(startInfo, "npm", "start"); - startInfo.EnvironmentVariables["PORT"] = port.ToString(); - break; - - case ProjectPlatform.Python: - var entryPoint = LocalRuntimeRequirementCheck.ResolvePythonEntryPoint(projectPath); - var usesUv = ProjectBuildRequirementCheck.DetectPythonInstallCommand(projectPath) is ("uv", _); - if (usesUv) - { - startInfo.FileName = _resolvedUvCommand ?? "uv"; - startInfo.Arguments = $"run python {entryPoint}"; - } - else - { - startInfo.FileName = "python"; - startInfo.Arguments = entryPoint; - } - startInfo.EnvironmentVariables["PORT"] = port.ToString(); - break; - - default: - throw new ArgumentOutOfRangeException(nameof(platform), platform, "Unsupported platform"); - } - - return startInfo; - } - - private static async Task WaitForHealthAsync( - Process process, - string healthUrl, - ILogger logger, - CancellationToken cancellationToken) - { - using var httpClient = new HttpClient(); - using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); - timeoutCts.CancelAfter(AgentStartupTimeout); - - while (!timeoutCts.Token.IsCancellationRequested) - { - if (process.HasExited) - { - logger.LogWarning("Agent exited with code {ExitCode} before health endpoint responded.", process.ExitCode); - return false; - } - - try - { - using var response = await httpClient.GetAsync(healthUrl, timeoutCts.Token); - if (response.IsSuccessStatusCode) - { - return true; - } - } - catch (HttpRequestException) { } - catch (TaskCanceledException) when (timeoutCts.Token.IsCancellationRequested && !cancellationToken.IsCancellationRequested) - { - break; - } - - try - { - await Task.Delay(HealthPollInterval, timeoutCts.Token); - } - catch (TaskCanceledException) when (!cancellationToken.IsCancellationRequested) - { - break; - } - } - - cancellationToken.ThrowIfCancellationRequested(); - return false; - } - - /// - /// Queries agent metrics via MCP tool call. - /// - protected internal virtual Task GetAgentMetricsAsync( - Agent365Config config, - ILogger logger, - CancellationToken cancellationToken) - { - return GetAgentMetricsInternalAsync(config, logger, cancellationToken); - } - - private static async Task GetAgentMetricsInternalAsync( - Agent365Config config, - ILogger logger, - CancellationToken cancellationToken) - { - var token = Environment.GetEnvironmentVariable("A365_OBSERVABILITY_MCP_BEARER_TOKEN"); - if (string.IsNullOrWhiteSpace(token)) - { - logger.LogWarning("Observability MCP bearer token is not set. Configure A365_OBSERVABILITY_MCP_BEARER_TOKEN."); - return null; - } - - var endpoint = MacVisibilityRequirementCheck.ResolveEndpointForObservability(config, config.Environment); - var metricArgument = MacVisibilityRequirementCheck.ResolveMetricsArgumentForObservability(config); - var correlationId = HttpClientFactory.GenerateCorrelationId(); - - using var httpClient = HttpClientFactory.CreateAuthenticatedClient(token, correlationId: correlationId); - - var toolIsAdvertised = await MacVisibilityRequirementCheck.ProbeToolsListAsync( - httpClient, - endpoint, - correlationId, - logger, - cancellationToken); - if (!toolIsAdvertised) - { - logger.LogWarning("MCP tools/list did not advertise required tool '{ToolName}'.", GetAgentMetricsToolName); - return null; - } - - var requestPayload = new - { - jsonrpc = McpConstants.JsonRpcVersion, - id = "agent-metrics", - method = McpConstants.ToolsCallMethod, - @params = new - { - name = GetAgentMetricsToolName, - arguments = new Dictionary(StringComparer.Ordinal) - { - [metricArgument.Key] = metricArgument.Value - } - } - }; - - var payloadJson = JsonSerializer.Serialize(requestPayload); - using var request = new HttpRequestMessage(HttpMethod.Post, endpoint) - { - Content = new StringContent(payloadJson, Encoding.UTF8, McpConstants.MediaTypes.ApplicationJson) - }; - - request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue(McpConstants.MediaTypes.ApplicationJson)); - request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue(McpConstants.MediaTypes.TextEventStream)); - - using var response = await httpClient.SendAsync( - request, - HttpCompletionOption.ResponseHeadersRead, - cancellationToken); - - var content = await response.Content.ReadAsStringAsync(cancellationToken); - if (!response.IsSuccessStatusCode) - { - logger.LogWarning( - "getAgentMetrics returned non-success status {StatusCode} from {Endpoint} (CorrelationId: {CorrelationId})", - (int)response.StatusCode, - endpoint, - correlationId); - return null; - } - - var toolText = MacVisibilityRequirementCheck.ExtractToolText(content); - var metrics = MacVisibilityRequirementCheck.ParseNumericMetrics(toolText); - if (metrics.Count == 0) - { - logger.LogWarning("getAgentMetrics response did not contain numeric metrics."); - return null; - } - - return new AgentMetricsSnapshot - { - InvocationCount = ConvertToLong(SumByMetricPrefix(metrics, "kpi.invocations.")), - ErrorCount = ConvertToLong(SumByMetricPrefix(metrics, "kpi.errors.")), - AverageLatencyMs = GetFirstMetricValue(metrics, - "kpi.latency.avg", - "kpi.latency.average", - "kpi.avgLatencyMs", - "kpi.averageLatencyMs") - }; - } - - private static double SumByMetricPrefix(IReadOnlyDictionary metrics, string prefix) - { - double sum = 0; - foreach (var pair in metrics) - { - if (pair.Key.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) - { - sum += pair.Value; - } - } - - return sum; - } - - private static double? GetFirstMetricValue(IReadOnlyDictionary metrics, params string[] metricKeys) - { - foreach (var metricKey in metricKeys) - { - if (metrics.TryGetValue(metricKey, out var value)) - { - return value; - } - } - - return null; - } - - private static long ConvertToLong(double value) - { - if (double.IsNaN(value) || double.IsInfinity(value)) - { - return 0; - } - - if (value > long.MaxValue) - { - return long.MaxValue; - } - - if (value < long.MinValue) - { - return long.MinValue; - } - - return (long)Math.Round(value, MidpointRounding.AwayFromZero); - } - - /// - /// Generates a conversation with the agent in Microsoft Teams using Playwright. - /// Reuses the CLI's existing MSAL authentication context (WAM on Windows, - /// browser auth on other platforms) so the user is not prompted to log in again. - /// - /// Uses to: - /// 1. Launch a Chromium browser (headless if saved auth state is fresh, headed otherwise) - /// 2. Navigate to Teams web, search for the agent by name, send a test message - /// 3. Wait for the agent to respond - /// 4. Save browser auth state for future runs - /// - protected internal virtual async Task GenerateCopilotChatConversationAsync( - Agent365Config config, - ILogger logger, - CancellationToken cancellationToken) - { - if (_playwrightService is null) - { - logger.LogDebug("CopilotChatPlaywrightService not provided -- returning false placeholder."); - return false; - } - - var agentName = _instanceName; - if (string.IsNullOrWhiteSpace(agentName)) - { - logger.LogWarning("Instance name not provided. Use --instance-name to specify the agent name in Teams."); - return false; - } - - logger.LogInformation("Opening Teams chat conversation with agent '{AgentName}'...", agentName); - - var projectPath = Directory.GetCurrentDirectory(); - var testMessage = ConversationRequirementCheck.BuildToolInvocationPrompt(projectPath, logger); - logger.LogDebug("Using test message: {Message}", testMessage); - - var response = await _playwrightService.SendMessageToAgentAsync( - agentName, - testMessage, - cancellationToken); - - if (string.IsNullOrWhiteSpace(response)) - { - logger.LogWarning("Agent did not respond to the test message in Teams."); - return false; - } - - logger.LogInformation("Agent responded successfully ({Length} chars).", response.Length); - return true; - } -} - -/// -/// Snapshot of agent metrics at a point in time. -/// -public class AgentMetricsSnapshot -{ - /// Total number of agent invocations recorded. - public long InvocationCount { get; set; } - - /// Total number of errors recorded. - public long ErrorCount { get; set; } - - /// Average latency in milliseconds (if available). - public double? AverageLatencyMs { get; set; } -} - -/// -/// Metadata for agent metrics check results, used for structured report output. -/// -public class AgentMetricsMetadata -{ - /// Metrics snapshot before the conversation. - public AgentMetricsSnapshot? BaselineMetrics { get; set; } - - /// Whether a Copilot Chat conversation was successfully generated. - public bool? ConversationGenerated { get; set; } - - /// Metrics snapshot after the conversation. - public AgentMetricsSnapshot? PostConversationMetrics { get; set; } - - /// Whether metrics incremented after the conversation. - public bool? MetricsIncremented { get; set; } -} diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/MacVisibilityRequirementCheck.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/MacVisibilityRequirementCheck.cs deleted file mode 100644 index da1aae91..00000000 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/MacVisibilityRequirementCheck.cs +++ /dev/null @@ -1,871 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -using System.Globalization; -using System.Net.Http.Headers; -using System.Text; -using System.Text.Json; -using System.Text.Json.Nodes; -using Microsoft.Agents.A365.DevTools.Cli.Constants; -using Microsoft.Agents.A365.DevTools.Cli.Models; -using Microsoft.Agents.A365.DevTools.Cli.Services.Helpers; -using Microsoft.Agents.A365.DevTools.Cli.Services.Internal; -using Microsoft.Agents.A365.DevTools.Cli.Services.Requirements; -using Microsoft.Extensions.Logging; - -namespace Microsoft.Agents.A365.DevTools.Cli.Services.Requirements.RequirementChecks; - -/// -/// Validates that MAC visibility metrics increase after conversation simulation by -/// calling the getAgentMetrics MCP tool before and after conversation. -/// -public sealed class MacVisibilityRequirementCheck : RequirementCheck -{ - public const string BaselineFileName = "a365.metrics.baseline.json"; - public const string GetAgentMetricsToolName = "getAgentMetrics"; - public const string ObservabilityMcpServerName = "observability-mcp"; - - private readonly AuthenticationService? _authService; - private readonly string _baselineFilePath; - private readonly bool _conversationStepVerified; - private readonly string _environment; - private readonly string? _baseUrlOverride; - private readonly string? _tenantIdOverride; - private readonly string? _agentNameOverride; - private readonly HttpMessageHandler? _httpHandler; - private readonly Func>? _tokenProviderOverride; - - public MacVisibilityRequirementCheck( - AuthenticationService? authService, - string baselineFilePath, - bool conversationStepVerified, - string environment = "prod", - string? baseUrlOverride = null, - string? tenantIdOverride = null, - string? agentNameOverride = null, - HttpMessageHandler? httpHandler = null, - Func>? tokenProviderOverride = null) - { - _authService = authService; - _baselineFilePath = baselineFilePath; - _conversationStepVerified = conversationStepVerified; - _environment = environment; - _baseUrlOverride = baseUrlOverride; - _tenantIdOverride = tenantIdOverride; - _agentNameOverride = agentNameOverride; - _httpHandler = httpHandler; - _tokenProviderOverride = tokenProviderOverride; - } - - /// - public override string Name => "Visible in MAC"; - - /// - public override string Description => "Validates MAC visibility by comparing getAgentMetrics before and after conversation"; - - /// - public override string Category => "Observability"; - - /// - /// Captures pre-conversation metrics and persists them to a baseline file. - /// - public static async Task CaptureInitialMetricsAsync( - Agent365Config config, - ILogger logger, - AuthenticationService? authService, - string environment = "prod", - string? baseUrlOverride = null, - string? tenantIdOverride = null, - string? agentNameOverride = null, - string? baselineFilePath = null, - HttpMessageHandler? httpHandler = null, - Func>? tokenProviderOverride = null, - CancellationToken cancellationToken = default) - { - var filePath = string.IsNullOrWhiteSpace(baselineFilePath) - ? Path.Combine(Directory.GetCurrentDirectory(), BaselineFileName) - : baselineFilePath; - - var check = new MacVisibilityRequirementCheck( - authService, - filePath, - conversationStepVerified: true, - environment, - baseUrlOverride, - tenantIdOverride, - agentNameOverride, - httpHandler, - tokenProviderOverride); - - try - { - var endpoint = check.ResolveEndpoint(config); - var metricArgument = check.ResolveMetricsArgument(config); - var toolText = await check.CallGetAgentMetricsToolAsync(config, endpoint, metricArgument, logger, cancellationToken); - var metrics = ParseNumericMetrics(toolText); - - if (metrics.Count == 0) - { - return RequirementCheckResult.Failure( - "Could not parse any numeric values from getAgentMetrics output", - "Ensure getAgentMetrics returns KPI and/or daily-series numeric values.", - details: "No numeric metrics were found in the MCP response text."); - } - - var snapshot = new MacMetricsSnapshotFile - { - CapturedAtUtc = DateTimeOffset.UtcNow, - Endpoint = endpoint, - ToolName = GetAgentMetricsToolName, - ServerName = ObservabilityMcpServerName, - NumericMetrics = metrics - }; - - await check.WriteBaselineAsync(snapshot, logger, cancellationToken); - - return new RequirementCheckResult - { - Passed = true, - IsWarning = false, - Details = $"Captured initial MAC metrics to {filePath}", - Metadata = new RequirementCheckMetadata - { - MacMetricsBaselineFile = filePath, - MacBaselineMetrics = metrics, - ConversationStepVerified = true - } - }; - } - catch (Exception ex) when (ex is not OperationCanceledException) - { - logger.LogWarning(ex, "Failed to capture initial MAC metrics"); - return RequirementCheckResult.Failure( - "Failed to capture initial getAgentMetrics baseline", - "Check observability MCP endpoint settings and authentication, then retry validation.", - details: ex.Message); - } - } - - /// - public override async Task CheckAsync( - Agent365Config config, - ILogger logger, - CancellationToken cancellationToken = default) - { - return await ExecuteCheckWithLoggingAsync(config, logger, CheckImplementationAsync, cancellationToken); - } - - private async Task CheckImplementationAsync( - Agent365Config config, - ILogger logger, - CancellationToken cancellationToken) - { - if (!_conversationStepVerified) - { - return RequirementCheckResult.Failure( - "Conversation simulation step is not verified as complete", - "Ensure the designated teammate completes the Playwright mock conversation successfully before running MAC comparison.", - details: "TODO: artifact-level teammate verification signal will be added later; current check requires conversation tier success."); - } - - if (string.IsNullOrWhiteSpace(_baselineFilePath) || !File.Exists(_baselineFilePath)) - { - return RequirementCheckResult.Failure( - "Initial MAC metrics baseline file not found", - "Run initial getAgentMetrics capture before post-conversation comparison.", - details: $"Baseline file path: {_baselineFilePath}"); - } - - var baseline = await ReadBaselineAsync(logger, cancellationToken); - if (baseline?.NumericMetrics is null || baseline.NumericMetrics.Count == 0) - { - return RequirementCheckResult.Failure( - "Initial MAC metrics baseline is empty or invalid", - "Regenerate the baseline and rerun validation.", - details: $"Baseline file path: {_baselineFilePath}"); - } - - var endpoint = ResolveEndpoint(config); - var metricArgument = ResolveMetricsArgument(config); - var toolText = await CallGetAgentMetricsToolAsync(config, endpoint, metricArgument, logger, cancellationToken); - var currentMetrics = ParseNumericMetrics(toolText); - - if (currentMetrics.Count == 0) - { - return RequirementCheckResult.Failure( - "Could not parse any numeric values from post-conversation getAgentMetrics output", - "Ensure getAgentMetrics output includes numeric KPI values."); - } - - var comparisons = CompareMetrics(baseline.NumericMetrics, currentMetrics); - var blockingFailures = comparisons - .Where(c => !c.Passed) - .Select(c => c.MetricKey) - .ToList(); - - var details = $"Compared {comparisons.Count} KPI metrics (exception rate excluded from increase requirement)."; - - if (blockingFailures.Count > 0) - { - return new RequirementCheckResult - { - Passed = false, - IsWarning = false, - ErrorMessage = $"MAC metrics did not increase for required fields: {string.Join(", ", blockingFailures)}", - ResolutionGuidance = "Execute the conversation simulation successfully and rerun validation.", - Details = details, - Metadata = new RequirementCheckMetadata - { - MacMetricsBaselineFile = _baselineFilePath, - MacBaselineMetrics = baseline.NumericMetrics, - MacCurrentMetrics = currentMetrics, - MacMetricComparisons = comparisons, - ConversationStepVerified = _conversationStepVerified - } - }; - } - - return new RequirementCheckResult - { - Passed = true, - IsWarning = false, - Details = details, - Metadata = new RequirementCheckMetadata - { - MacMetricsBaselineFile = _baselineFilePath, - MacBaselineMetrics = baseline.NumericMetrics, - MacCurrentMetrics = currentMetrics, - MacMetricComparisons = comparisons, - ConversationStepVerified = _conversationStepVerified - } - }; - } - - private string ResolveEndpoint(Agent365Config config) - { - return ResolveEndpointForObservability(config, _environment, _baseUrlOverride, _tenantIdOverride); - } - - internal static string ResolveEndpointForObservability( - Agent365Config config, - string environment, - string? baseUrlOverride = null, - string? tenantIdOverride = null) - { - var baseUrl = FirstNonEmpty( - baseUrlOverride, - config.Agent365ObservabilityMcpOptions?.BaseUrl, - Environment.GetEnvironmentVariable("A365_OBSERVABILITY_BASE_URL")) - ?? new Uri(ConfigConstants.GetDiscoverEndpointUrl(environment)).GetLeftPart(UriPartial.Authority); - - var tenantId = FirstNonEmpty( - tenantIdOverride, - config.Agent365ObservabilityMcpOptions?.TenantId, - Environment.GetEnvironmentVariable("A365_OBSERVABILITY_TENANT_ID"), - config.TenantId); - - if (string.IsNullOrWhiteSpace(tenantId)) - { - throw new InvalidOperationException("Tenant ID is required for observability MCP endpoint resolution."); - } - - return $"{baseUrl.TrimEnd('/')}/observability/tenants/{Uri.EscapeDataString(tenantId)}/mcp"; - } - - private KeyValuePair ResolveMetricsArgument(Agent365Config config) - { - return ResolveMetricsArgumentForObservability(config, _agentNameOverride); - } - - internal static KeyValuePair ResolveMetricsArgumentForObservability( - Agent365Config config, - string? agentNameOverride = null) - { - var agentObservabilityId = FirstNonEmpty( - config.Agent365ObservabilityMcpOptions?.AgentObservabilityId, - Environment.GetEnvironmentVariable("A365_OBSERVABILITY_AGENT_OBSERVABILITY_ID")); - - if (!string.IsNullOrWhiteSpace(agentObservabilityId)) - { - return new KeyValuePair("agentObservabilityId", agentObservabilityId); - } - - var agentName = FirstNonEmpty( - agentNameOverride, - config.Agent365ObservabilityMcpOptions?.AgentName, - Environment.GetEnvironmentVariable("A365_OBSERVABILITY_AGENT_NAME"), - config.AgentIdentityDisplayName, - config.AgentBlueprintDisplayName); - - if (string.IsNullOrWhiteSpace(agentName)) - { - throw new InvalidOperationException( - "Agent selector is required for getAgentMetrics. Configure agent365ObservabilityMcpOptions.agentObservabilityId or agent365ObservabilityMcpOptions.agentName, set A365_OBSERVABILITY_AGENT_OBSERVABILITY_ID / A365_OBSERVABILITY_AGENT_NAME, or configure agentIdentityDisplayName."); - } - - return new KeyValuePair("agentName", agentName); - } - - private async Task ResolveAccessTokenAsync(Agent365Config config, CancellationToken cancellationToken) - { - if (_tokenProviderOverride is not null) - { - var overridden = await _tokenProviderOverride(cancellationToken); - if (string.IsNullOrWhiteSpace(overridden)) - { - throw new InvalidOperationException("Overridden token provider returned an empty token."); - } - - return overridden; - } - - var envToken = Environment.GetEnvironmentVariable("A365_OBSERVABILITY_MCP_BEARER_TOKEN"); - if (!string.IsNullOrWhiteSpace(envToken)) - { - return envToken; - } - - if (_authService is null) - { - throw new InvalidOperationException( - "Authentication service is required when no explicit observability bearer token is configured."); - } - - var audience = FirstNonEmpty( - Environment.GetEnvironmentVariable("A365_OBSERVABILITY_MCP_APP_ID"), - config.Agent365ObservabilityMcpOptions?.AppId) - ?? ConfigConstants.GetAgent365ToolsResourceAppId(_environment); - var loginHint = await AzCliHelper.ResolveLoginHintAsync(); - var token = await _authService.GetAccessTokenAsync(audience, userId: loginHint, ct: cancellationToken); - if (string.IsNullOrWhiteSpace(token)) - { - throw new InvalidOperationException("Failed to acquire access token for observability MCP call."); - } - - return token; - } - - private async Task CallGetAgentMetricsToolAsync( - Agent365Config config, - string endpoint, - KeyValuePair metricArgument, - ILogger logger, - CancellationToken cancellationToken) - { - var token = await ResolveAccessTokenAsync(config, cancellationToken); - var correlationId = HttpClientFactory.GenerateCorrelationId(); - - using var httpClient = HttpClientFactory.CreateAuthenticatedClient( - token, - correlationId: correlationId, - handler: _httpHandler); - - var toolIsAdvertised = await ProbeToolsListAsync(httpClient, endpoint, correlationId, logger, cancellationToken); - if (!toolIsAdvertised) - { - throw new InvalidOperationException( - $"MCP tools/list did not advertise required tool '{GetAgentMetricsToolName}'."); - } - - var requestPayload = new - { - jsonrpc = McpConstants.JsonRpcVersion, - id = "1", - method = McpConstants.ToolsCallMethod, - @params = new - { - name = GetAgentMetricsToolName, - arguments = new Dictionary(StringComparer.Ordinal) - { - [metricArgument.Key] = metricArgument.Value - } - } - }; - - var payloadJson = JsonSerializer.Serialize(requestPayload); - - using var request = new HttpRequestMessage(HttpMethod.Post, endpoint) - { - Content = new StringContent(payloadJson, Encoding.UTF8, McpConstants.MediaTypes.ApplicationJson) - }; - - request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue(McpConstants.MediaTypes.ApplicationJson)); - request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue(McpConstants.MediaTypes.TextEventStream)); - - using var response = await httpClient.SendAsync( - request, - HttpCompletionOption.ResponseHeadersRead, - cancellationToken); - - var content = await response.Content.ReadAsStringAsync(cancellationToken); - if (!response.IsSuccessStatusCode) - { - logger.LogWarning( - "getAgentMetrics returned non-success status {StatusCode} from {Endpoint} (CorrelationId: {CorrelationId})", - (int)response.StatusCode, - endpoint, - correlationId); - throw new InvalidOperationException( - $"getAgentMetrics call failed with status {(int)response.StatusCode}: {content}"); - } - - return ExtractToolText(content); - } - - internal static async Task ProbeToolsListAsync( - HttpClient httpClient, - string endpoint, - string correlationId, - ILogger logger, - CancellationToken cancellationToken) - { - try - { - var requestPayload = new - { - jsonrpc = McpConstants.JsonRpcVersion, - id = "tools-list", - method = McpConstants.ToolsListMethod - }; - - var payloadJson = JsonSerializer.Serialize(requestPayload); - using var request = new HttpRequestMessage(HttpMethod.Post, endpoint) - { - Content = new StringContent(payloadJson, Encoding.UTF8, McpConstants.MediaTypes.ApplicationJson) - }; - - request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue(McpConstants.MediaTypes.ApplicationJson)); - request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue(McpConstants.MediaTypes.TextEventStream)); - - using var response = await httpClient.SendAsync( - request, - HttpCompletionOption.ResponseHeadersRead, - cancellationToken); - - var content = await response.Content.ReadAsStringAsync(cancellationToken); - if (!response.IsSuccessStatusCode) - { - logger.LogWarning( - "MCP tools/list probe failed with status {StatusCode} from {Endpoint} (CorrelationId: {CorrelationId})", - (int)response.StatusCode, - endpoint, - correlationId); - return false; - } - - var advertised = IsToolAdvertisedInToolsListResponse(content, GetAgentMetricsToolName); - logger.LogInformation( - "MCP tools/list advertised {ToolName}: {Advertised}", - GetAgentMetricsToolName, - advertised); - return advertised; - } - catch (Exception ex) when (ex is not OperationCanceledException) - { - logger.LogWarning(ex, "MCP tools/list probe failed unexpectedly."); - return false; - } - } - - internal static bool IsToolAdvertisedInToolsListResponse(string responseContent, string toolName) - { - if (string.IsNullOrWhiteSpace(responseContent) || string.IsNullOrWhiteSpace(toolName)) - { - return false; - } - - try - { - var dataJson = string.Concat( - responseContent - .Split(new[] { "\r\n", "\n" }, StringSplitOptions.None) - .Where(line => line.StartsWith("data:", StringComparison.OrdinalIgnoreCase)) - .Select(line => line.Substring(5).Trim())); - - var candidate = string.IsNullOrWhiteSpace(dataJson) ? responseContent : dataJson; - var root = JsonNode.Parse(candidate); - - // MCP canonical shape: result.tools[].name - var toolNodes = root?["result"]?["tools"]?.AsArray(); - if (toolNodes is not null) - { - return toolNodes.Any(n => - string.Equals( - n?["name"]?.GetValue(), - toolName, - StringComparison.OrdinalIgnoreCase)); - } - - // Fallback shape used by some servers: result.content[].text containing JSON. - var textPayload = root?["result"]?["content"]? - .AsArray() - .Select(n => n?["text"]?.GetValue()) - .FirstOrDefault(t => !string.IsNullOrWhiteSpace(t)); - - if (!string.IsNullOrWhiteSpace(textPayload)) - { - var inner = JsonNode.Parse(textPayload); - var innerTools = inner?["tools"]?.AsArray(); - if (innerTools is not null) - { - return innerTools.Any(n => - string.Equals( - n?["name"]?.GetValue(), - toolName, - StringComparison.OrdinalIgnoreCase)); - } - } - } - catch - { - // Best-effort probe; caller will proceed with tools/call. - } - - return false; - } - - private static string? FirstNonEmpty(params string?[] values) - { - foreach (var value in values) - { - if (!string.IsNullOrWhiteSpace(value)) - { - return value; - } - } - - return null; - } - - private async Task WriteBaselineAsync( - MacMetricsSnapshotFile snapshot, - ILogger logger, - CancellationToken cancellationToken) - { - var json = JsonSerializer.Serialize(snapshot, new JsonSerializerOptions - { - WriteIndented = true - }); - - await File.WriteAllTextAsync(_baselineFilePath, json, cancellationToken); - logger.LogInformation("Wrote MAC baseline metrics to {Path}", _baselineFilePath); - } - - private async Task ReadBaselineAsync(ILogger logger, CancellationToken cancellationToken) - { - try - { - var json = await File.ReadAllTextAsync(_baselineFilePath, cancellationToken); - return JsonSerializer.Deserialize(json); - } - catch (Exception ex) when (ex is not OperationCanceledException) - { - logger.LogWarning(ex, "Failed to read baseline file {Path}", _baselineFilePath); - return null; - } - } - - internal static string ExtractToolText(string responseContent) - { - // Handle SSE envelopes first. - var candidate = ExtractJsonRpcCandidate(responseContent); - - try - { - var root = JsonNode.Parse(candidate); - var text = root?["result"]?["content"]? - .AsArray() - .Select(n => n?["text"]?.GetValue()) - .FirstOrDefault(t => !string.IsNullOrWhiteSpace(t)); - - if (!string.IsNullOrWhiteSpace(text)) - { - return text; - } - } - catch - { - // If candidate is already plain text/markdown, return as-is below. - } - - return candidate; - } - - private static string ExtractJsonRpcCandidate(string responseContent) - { - var dataJson = string.Concat( - responseContent - .Split(new[] { "\r\n", "\n" }, StringSplitOptions.None) - .Where(line => line.StartsWith("data:", StringComparison.OrdinalIgnoreCase)) - .Select(line => line.Substring(5).Trim())); - - return string.IsNullOrWhiteSpace(dataJson) ? responseContent : dataJson; - } - - internal static Dictionary ParseNumericMetrics(string toolText) - { - var metrics = new Dictionary(StringComparer.OrdinalIgnoreCase); - var lines = toolText - .Split(new[] { "\r\n", "\n" }, StringSplitOptions.None) - .Select(l => l.Trim()) - .Where(l => !string.IsNullOrWhiteSpace(l)) - .ToList(); - - ParseKpiTable(lines, metrics); - ParseDailySeriesTable(lines, metrics); - - return metrics; - } - - internal static List CompareMetrics( - IReadOnlyDictionary baseline, - IReadOnlyDictionary current) - { - var comparisons = new List(); - - foreach (var key in baseline.Keys.Where(IsRelevantComparisonMetric)) - { - if (!current.TryGetValue(key, out var currentValue)) - { - comparisons.Add(new MacMetricComparisonMetadata - { - MetricKey = key, - Before = baseline[key], - After = double.NaN, - Delta = double.NaN, - Increased = false, - IsExceptionRate = key.Contains("exception_rate", StringComparison.OrdinalIgnoreCase), - Passed = false, - Reason = "metric missing in post-conversation snapshot" - }); - continue; - } - - var before = baseline[key]; - var delta = currentValue - before; - var increased = delta > 0; - var isExceptionRate = key.Contains("exception_rate", StringComparison.OrdinalIgnoreCase); - - comparisons.Add(new MacMetricComparisonMetadata - { - MetricKey = key, - Before = before, - After = currentValue, - Delta = delta, - Increased = increased, - IsExceptionRate = isExceptionRate, - Passed = isExceptionRate || increased, - Reason = isExceptionRate - ? "exception rate does not need to increase" - : (increased ? "increased" : "did not increase") - }); - } - - return comparisons; - } - - private static bool IsRelevantComparisonMetric(string key) - { - if (!key.StartsWith("kpi.", StringComparison.OrdinalIgnoreCase)) - { - return false; - } - - return key.EndsWith(".rl7", StringComparison.OrdinalIgnoreCase) - || key.EndsWith(".rl30", StringComparison.OrdinalIgnoreCase); - } - - private static void ParseKpiTable(List lines, IDictionary metrics) - { - var headerIndex = lines.FindIndex(l => - l.Contains('|') - && l.Contains("Metric", StringComparison.OrdinalIgnoreCase) - && l.Contains("RL7", StringComparison.OrdinalIgnoreCase) - && l.Contains("RL30", StringComparison.OrdinalIgnoreCase)); - - if (headerIndex < 0) - { - return; - } - - for (var i = headerIndex + 1; i < lines.Count; i++) - { - var line = lines[i]; - if (!line.Contains('|')) - { - break; - } - - if (line.All(c => c is '|' or '-' or ':' or ' ')) - { - continue; - } - - var cells = line.Split('|', StringSplitOptions.RemoveEmptyEntries) - .Select(c => c.Trim()) - .ToList(); - - if (cells.Count < 4) - { - continue; - } - - var metricName = NormalizeMetricName(cells[0]); - if (TryParseMetricNumber(cells[1], out var rl7)) - { - metrics[$"kpi.{metricName}.rl7"] = rl7; - } - - if (TryParseMetricNumber(cells[2], out var rl30)) - { - metrics[$"kpi.{metricName}.rl30"] = rl30; - } - - if (TryParseMetricNumber(cells[3], out var wow)) - { - metrics[$"kpi.{metricName}.wow_change_percent"] = wow; - } - } - } - - private static void ParseDailySeriesTable(List lines, IDictionary metrics) - { - var headerIndex = lines.FindIndex(l => - l.Contains('|') - && l.Contains("Date", StringComparison.OrdinalIgnoreCase) - && l.Contains("Users", StringComparison.OrdinalIgnoreCase) - && l.Contains("Invocations", StringComparison.OrdinalIgnoreCase) - && l.Contains("Sessions", StringComparison.OrdinalIgnoreCase)); - - if (headerIndex < 0) - { - return; - } - - for (var i = headerIndex + 1; i < lines.Count; i++) - { - var line = lines[i]; - if (!line.Contains('|')) - { - break; - } - - if (line.All(c => c is '|' or '-' or ':' or ' ')) - { - continue; - } - - var cells = line.Split('|', StringSplitOptions.RemoveEmptyEntries) - .Select(c => c.Trim()) - .ToList(); - - if (cells.Count < 4) - { - continue; - } - - var dateKey = NormalizeMetricName(cells[0]); - if (TryParseMetricNumber(cells[1], out var users)) - { - metrics[$"daily.{dateKey}.users"] = users; - } - - if (TryParseMetricNumber(cells[2], out var invocations)) - { - metrics[$"daily.{dateKey}.invocations"] = invocations; - } - - if (TryParseMetricNumber(cells[3], out var sessions)) - { - metrics[$"daily.{dateKey}.sessions"] = sessions; - } - } - } - - internal static bool TryParseMetricNumber(string raw, out double value) - { - value = 0; - var cleaned = raw.Trim(); - if (cleaned is "-" or "--") - { - return false; - } - - var filtered = new string(cleaned - .Where(c => char.IsDigit(c) || c is '.' or '-' or ',') - .ToArray()) - .Replace(",", string.Empty, StringComparison.Ordinal); - - return double.TryParse(filtered, NumberStyles.Float, CultureInfo.InvariantCulture, out value); - } - - private static string NormalizeMetricName(string raw) - { - var input = raw.Trim().ToLowerInvariant(); - var builder = new StringBuilder(input.Length + 8); - var pendingUnderscore = false; - - for (var i = 0; i < input.Length; i++) - { - if (i + 4 < input.Length && input.AsSpan(i, 5).SequenceEqual("(hrs)")) - { - if (builder.Length > 0) - { - builder.Append('_'); - } - - builder.Append("hrs"); - i += 4; - pendingUnderscore = false; - continue; - } - - var c = input[i]; - if (c == '%') - { - if (builder.Length > 0) - { - builder.Append('_'); - } - - builder.Append("percent"); - pendingUnderscore = false; - continue; - } - - if (c is ' ' or '-' or '/') - { - pendingUnderscore = builder.Length > 0; - continue; - } - - if (c == '.') - { - continue; - } - - if (pendingUnderscore && builder.Length > 0) - { - builder.Append('_'); - } - - builder.Append(c); - pendingUnderscore = false; - } - - return builder.ToString().Trim('_'); - } - - private sealed class MacMetricsSnapshotFile - { - public DateTimeOffset CapturedAtUtc { get; set; } - - public string Endpoint { get; set; } = string.Empty; - - public string ToolName { get; set; } = string.Empty; - - public string ServerName { get; set; } = string.Empty; - - public Dictionary NumericMetrics { get; set; } = new(StringComparer.OrdinalIgnoreCase); - } -} diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/TelemetryRequirementCheck.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/TelemetryRequirementCheck.cs index e867938e..1dbf431c 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/TelemetryRequirementCheck.cs +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/TelemetryRequirementCheck.cs @@ -91,16 +91,6 @@ public class TelemetryRequirementCheck : RequirementCheck "execute_tool" }; - /// - /// OTel semantic convention resource attributes that should be present in span output. - /// - internal static readonly string[] RequiredResourceAttributes = new[] - { - "telemetry.sdk.name", - "telemetry.sdk.version", - "service.name" - }; - public TelemetryRequirementCheck(string? agentConsoleLogPath) { @@ -217,14 +207,6 @@ private Task CheckImplementationAsync( "these spans should be children of an invoke_agent span"); } - // Check resource attributes: telemetry.sdk.name, telemetry.sdk.version, service.name - var missingResources = GetMissingResourceAttributes(logLines); - if (missingResources.Count > 0) - { - warnings.Add($"missing OTel resource attributes: {string.Join(", ", missingResources)} — " + - "ensure your SDK resource is configured per OTel semantic conventions"); - } - var detailsBuilder = $"Console exporter active with {relevantBlocks.Count} relevant span(s). " + $"All required GenAI operation spans detected: {string.Join(", ", RequiredGenAiSpans)}."; @@ -432,16 +414,4 @@ internal static bool HasNonEmptyValue(string line) !value.Equals("null", StringComparison.OrdinalIgnoreCase); } - /// - /// Checks for missing OTel resource semantic convention attributes across all log lines. - /// Looks for telemetry.sdk.name, telemetry.sdk.version, and service.name - /// which should appear in the resource section of console exporter output. - /// - internal static List GetMissingResourceAttributes(string[] logLines) - { - var allText = string.Join("\n", logLines); - return RequiredResourceAttributes - .Where(attr => allText.IndexOf(attr, StringComparison.OrdinalIgnoreCase) < 0) - .ToList(); - } } diff --git a/src/Microsoft.Agents.A365.DevTools.Validation/ValidateReport.cs b/src/Microsoft.Agents.A365.DevTools.Validation/ValidateReport.cs index f406888d..dafeb351 100644 --- a/src/Microsoft.Agents.A365.DevTools.Validation/ValidateReport.cs +++ b/src/Microsoft.Agents.A365.DevTools.Validation/ValidateReport.cs @@ -293,101 +293,6 @@ public sealed class BlueprintResourceResult public bool? EffectiveInheritance { get; set; } } -/// -/// Agent metrics tier result with baseline/post-conversation snapshots and increment status. -/// -public sealed class AgentMetricsTierResult : TierResult -{ - [JsonPropertyName("baselineMetrics")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public AgentMetricsSnapshotResult? BaselineMetrics { get; set; } - - [JsonPropertyName("conversationGenerated")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public bool? ConversationGenerated { get; set; } - - [JsonPropertyName("postConversationMetrics")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public AgentMetricsSnapshotResult? PostConversationMetrics { get; set; } - - [JsonPropertyName("metricsIncremented")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public bool? MetricsIncremented { get; set; } -} - -/// -/// Snapshot of agent metrics at a point in time. -/// -public sealed class AgentMetricsSnapshotResult -{ - [JsonPropertyName("invocationCount")] - public long InvocationCount { get; set; } - - [JsonPropertyName("errorCount")] - public long ErrorCount { get; set; } - - [JsonPropertyName("averageLatencyMs")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public double? AverageLatencyMs { get; set; } -} - -/// -/// MAC visibility tier: compares getAgentMetrics snapshots before and after conversation. -/// -public sealed class MacTierResult : TierResult -{ - [JsonPropertyName("baselineFile")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? BaselineFile { get; set; } - - [JsonPropertyName("conversationVerified")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public bool? ConversationVerified { get; set; } - - [JsonPropertyName("baselineMetrics")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public Dictionary? BaselineMetrics { get; set; } - - [JsonPropertyName("currentMetrics")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public Dictionary? CurrentMetrics { get; set; } - - [JsonPropertyName("comparisons")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public List? Comparisons { get; set; } -} - -/// -/// Result of comparing a single MAC metric between baseline and post-conversation snapshots. -/// -public sealed class MacMetricComparisonResult -{ - [JsonPropertyName("metricKey")] - public string MetricKey { get; set; } = string.Empty; - - [JsonPropertyName("before")] - public double Before { get; set; } - - [JsonPropertyName("after")] - public double After { get; set; } - - [JsonPropertyName("delta")] - public double Delta { get; set; } - - [JsonPropertyName("increased")] - public bool Increased { get; set; } - - [JsonPropertyName("isExceptionRate")] - public bool IsExceptionRate { get; set; } - - [JsonPropertyName("passed")] - public bool Passed { get; set; } - - [JsonPropertyName("reason")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? Reason { get; set; } -} - /// /// Result of a single conversation turn. /// diff --git a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Commands/ValidateCommandTests.cs b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Commands/ValidateCommandTests.cs index 7f7b4676..5a1b78a9 100644 --- a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Commands/ValidateCommandTests.cs +++ b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Commands/ValidateCommandTests.cs @@ -182,6 +182,5 @@ public async Task ValidateCommand_Report_HasSkippedUnimplementedTiers() report!.Tiers.Conversation.Skipped.Should().BeTrue(); report.Tiers.Telemetry.Skipped.Should().BeTrue(); report.Tiers.Blueprint.Skipped.Should().BeTrue(); - report.Tiers.AgentMetrics.Skipped.Should().BeTrue(); } } \ No newline at end of file diff --git a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/HttpListenerBotCallbackReceiverTests.cs b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/HttpListenerBotCallbackReceiverTests.cs new file mode 100644 index 00000000..5fd6bac8 --- /dev/null +++ b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/HttpListenerBotCallbackReceiverTests.cs @@ -0,0 +1,110 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using FluentAssertions; +using Microsoft.Agents.A365.DevTools.Cli.Services; + +namespace Microsoft.Agents.A365.DevTools.Cli.Tests.Services; + +public class HttpListenerBotCallbackReceiverTests +{ + [Theory] + [InlineData("Got it..work on it")] + [InlineData("Got it - working on it...")] + [InlineData("Working on your request")] + [InlineData("Thinking...")] + [InlineData("One moment please")] + [InlineData("Please wait...")] + [InlineData("Hold on, let me check")] + [InlineData("Let me check that for you")] + [InlineData("Let me look into that")] + [InlineData("Let me find that information")] + [InlineData("Let me get that for you")] + [InlineData("Just a moment...")] + [InlineData("Just a sec")] + [InlineData("Looking into it")] + [InlineData("Processing your request...")] + public void IsInterimMessage_KnownInterimPatterns_ReturnsTrue(string text) + { + HttpListenerBotCallbackReceiver.IsInterimMessage(text) + .Should().BeTrue(because: "'{0}' is an interim acknowledgment, not a final response", text); + } + + [Theory] + [InlineData("Here are your recent emails: 1. Meeting reminder from John...")] + [InlineData("I found 3 documents matching your search criteria. The first one is about project planning and was created last week.")] + [InlineData("Processing failed due to an authentication error")] + [InlineData("I can help you with email, calendar, and file management")] + [InlineData("Hello! How can I help you today?")] + [InlineData("The meeting is scheduled for 3 PM tomorrow")] + [InlineData("Error: Unable to connect to the service")] + public void IsInterimMessage_RealResponses_ReturnsFalse(string text) + { + HttpListenerBotCallbackReceiver.IsInterimMessage(text) + .Should().BeFalse(because: "'{0}' is a real response, not an interim acknowledgment", text); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + public void IsInterimMessage_NullOrWhitespace_ReturnsFalse(string? text) + { + HttpListenerBotCallbackReceiver.IsInterimMessage(text) + .Should().BeFalse(because: "null/empty text is not an interim message"); + } + + [Fact] + public void IsInterimMessage_LongTextWithInterimWord_ReturnsFalse() + { + var longText = "I'm working on analyzing your data. Here are the preliminary results that I found across multiple sources in your organization's SharePoint sites."; + + HttpListenerBotCallbackReceiver.IsInterimMessage(longText) + .Should().BeFalse(because: "messages over 60 chars are treated as real responses even if they contain interim-like words"); + } + + [Fact] + public void IsFinalMessage_TypingActivity_ReturnsFalse() + { + var response = new BotCallbackResponse("typing indicator", "typing"); + + HttpListenerBotCallbackReceiver.IsFinalMessage(response) + .Should().BeFalse(because: "typing activities are never final responses"); + } + + [Fact] + public void IsFinalMessage_MessageWithSubstantiveText_ReturnsTrue() + { + var response = new BotCallbackResponse("Here are your recent emails from today", "message"); + + HttpListenerBotCallbackReceiver.IsFinalMessage(response) + .Should().BeTrue(because: "a message with substantive non-interim text is a final response"); + } + + [Fact] + public void IsFinalMessage_InterimMessage_ReturnsFalse() + { + var response = new BotCallbackResponse("Got it..work on it", "message"); + + HttpListenerBotCallbackReceiver.IsFinalMessage(response) + .Should().BeFalse(because: "interim acknowledgment messages are not final responses"); + } + + [Fact] + public void IsFinalMessage_MessageWithNullText_ReturnsFalse() + { + var response = new BotCallbackResponse(null, "message"); + + HttpListenerBotCallbackReceiver.IsFinalMessage(response) + .Should().BeFalse(because: "a message with no text is not a final response"); + } + + [Fact] + public void IsFinalMessage_NullType_WithText_ReturnsFalse() + { + var response = new BotCallbackResponse("some text", null); + + HttpListenerBotCallbackReceiver.IsFinalMessage(response) + .Should().BeFalse(because: "only message-type activities can be final responses"); + } +} diff --git a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/AgentMetricsRequirementCheckTests.cs b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/AgentMetricsRequirementCheckTests.cs deleted file mode 100644 index 97b94730..00000000 --- a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/AgentMetricsRequirementCheckTests.cs +++ /dev/null @@ -1,207 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -using FluentAssertions; -using Microsoft.Agents.A365.DevTools.Cli.Models; -using Microsoft.Agents.A365.DevTools.Cli.Services.Requirements.RequirementChecks; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; - -namespace Microsoft.Agents.A365.DevTools.Cli.Tests.Services.Requirements; - -public class AgentMetricsRequirementCheckTests -{ - private readonly ILogger _logger = NullLoggerFactory.Instance.CreateLogger("test"); - - [Fact] - public void Name_ReturnsAgentMetrics() - { - var check = new AgentMetricsRequirementCheck(); - check.Name.Should().Be("AgentMetrics"); - } - - [Fact] - public void Category_ReturnsObservability() - { - var check = new AgentMetricsRequirementCheck(); - check.Category.Should().Be("Observability"); - } - - [Fact] - public async Task CheckAsync_BaselineMetricsNull_ConversationSucceeds_ReturnsFail() - { - var check = new TestableAgentMetricsCheck( - baselineMetrics: null, - conversationResult: true); - var result = await check.CheckAsync(new Agent365Config(), _logger); - - result.Passed.Should().BeFalse(because: "metrics endpoint must be reachable even if conversation succeeded"); - result.ErrorMessage.Should().Contain("not available", - because: "error should indicate metrics endpoint was unreachable"); - } - - [Fact] - public async Task CheckAsync_BaselineMetricsNull_ConversationFails_ReturnsWarning() - { - var check = new TestableAgentMetricsCheck( - baselineMetrics: null, - conversationResult: false); - var result = await check.CheckAsync(new Agent365Config(), _logger); - - result.Passed.Should().BeTrue(because: "conversation failure is a warning"); - result.IsWarning.Should().BeTrue(); - result.ErrorMessage.Should().Contain("could not be generated", - because: "warning should indicate conversation generation failed"); - } - - [Fact] - public async Task CheckAsync_BaselineMetricsThrows_ConversationSucceeds_ReturnsFail() - { - var check = new TestableAgentMetricsCheck( - metricsException: new HttpRequestException("Connection refused"), - conversationResult: true); - var result = await check.CheckAsync(new Agent365Config(), _logger); - - result.Passed.Should().BeFalse(because: "metrics endpoint must be reachable"); - result.ErrorMessage.Should().Contain("not available", - because: "error should indicate metrics endpoint was unreachable"); - } - - [Fact] - public async Task CheckAsync_ConversationFails_ReturnsWarning() - { - var check = new TestableAgentMetricsCheck( - baselineMetrics: new AgentMetricsSnapshot { InvocationCount = 5 }, - conversationResult: false); - var result = await check.CheckAsync(new Agent365Config(), _logger); - - result.Passed.Should().BeTrue(because: "conversation failure is a warning"); - result.IsWarning.Should().BeTrue(); - result.ErrorMessage.Should().Contain("could not be generated", - because: "warning should indicate conversation generation failed"); - } - - [Fact] - public async Task CheckAsync_ConversationThrows_ReturnsWarning() - { - var check = new TestableAgentMetricsCheck( - baselineMetrics: new AgentMetricsSnapshot { InvocationCount = 5 }, - conversationException: new InvalidOperationException("Browser not found")); - var result = await check.CheckAsync(new Agent365Config(), _logger); - - result.Passed.Should().BeTrue(because: "Playwright errors are warnings"); - result.IsWarning.Should().BeTrue(); - result.Details.Should().Contain("Browser not found", - because: "warning details should contain the Playwright error"); - } - - [Fact] - public async Task CheckAsync_MetricsNotIncremented_ReturnsFail() - { - var snapshot = new AgentMetricsSnapshot { InvocationCount = 5 }; - var check = new TestableAgentMetricsCheck( - baselineMetrics: snapshot, - postConversationMetrics: snapshot, - conversationResult: true); - var result = await check.CheckAsync(new Agent365Config(), _logger); - - result.Passed.Should().BeFalse(because: "metrics should have incremented after conversation"); - result.ErrorMessage.Should().Contain("did not increment", - because: "error should describe the metrics did not change"); - result.Details.Should().Contain("Baseline invocations: 5", - because: "details should show the baseline count"); - } - - [Fact] - public async Task CheckAsync_MetricsIncremented_ReturnsSuccess() - { - var baseline = new AgentMetricsSnapshot { InvocationCount = 5 }; - var postConversation = new AgentMetricsSnapshot { InvocationCount = 8 }; - var check = new TestableAgentMetricsCheck( - baselineMetrics: baseline, - postConversationMetrics: postConversation, - conversationResult: true); - var result = await check.CheckAsync(new Agent365Config(), _logger); - - result.Passed.Should().BeTrue(because: "metrics incremented from 5 to 8"); - result.IsWarning.Should().BeFalse(); - result.Details.Should().Contain("incremented from 5 to 8", - because: "details should show the increment"); - } - - [Fact] - public async Task CheckAsync_PostConversationMetricsNull_ReturnsFail() - { - var baseline = new AgentMetricsSnapshot { InvocationCount = 5 }; - var check = new TestableAgentMetricsCheck( - baselineMetrics: baseline, - postConversationMetrics: null, - conversationResult: true, - postConversationMetricsExplicitNull: true); - var result = await check.CheckAsync(new Agent365Config(), _logger); - - result.Passed.Should().BeFalse(because: "post-conversation metrics must be retrievable"); - result.ErrorMessage.Should().Contain("not available after conversation", - because: "error should indicate post-conversation metrics failed"); - } - - [Fact] - public async Task CheckAsync_DefaultPlaceholder_ReturnsWarning() - { - // Default check has no playwright service, so conversation returns false - var check = new AgentMetricsRequirementCheck(); - var result = await check.CheckAsync(new Agent365Config(), _logger); - - result.Passed.Should().BeTrue(because: "conversation failure is a warning"); - result.IsWarning.Should().BeTrue(); - } - - /// - /// Testable subclass that overrides the virtual methods for controlled test behavior. - /// - private sealed class TestableAgentMetricsCheck : AgentMetricsRequirementCheck - { - private readonly AgentMetricsSnapshot? _baselineMetrics; - private readonly AgentMetricsSnapshot? _postConversationMetrics; - private readonly bool _postConversationMetricsExplicitNull; - private readonly bool _conversationResult; - private readonly Exception? _metricsException; - private readonly Exception? _conversationException; - private int _metricsCallCount; - - public TestableAgentMetricsCheck( - AgentMetricsSnapshot? baselineMetrics = null, - AgentMetricsSnapshot? postConversationMetrics = null, - bool conversationResult = false, - Exception? metricsException = null, - Exception? conversationException = null, - bool postConversationMetricsExplicitNull = false) - { - _baselineMetrics = baselineMetrics; - _postConversationMetrics = postConversationMetricsExplicitNull ? null : (postConversationMetrics ?? baselineMetrics); - _postConversationMetricsExplicitNull = postConversationMetricsExplicitNull; - _conversationResult = conversationResult; - _metricsException = metricsException; - _conversationException = conversationException; - } - - protected internal override Task GetAgentMetricsAsync( - Agent365Config config, ILogger logger, CancellationToken cancellationToken) - { - if (_metricsException is not null) - throw _metricsException; - - _metricsCallCount++; - return Task.FromResult(_metricsCallCount == 1 ? _baselineMetrics : _postConversationMetrics); - } - - protected internal override Task GenerateCopilotChatConversationAsync( - Agent365Config config, ILogger logger, CancellationToken cancellationToken) - { - if (_conversationException is not null) - throw _conversationException; - - return Task.FromResult(_conversationResult); - } - } -} diff --git a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/MacVisibilityRequirementCheckTests.cs b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/MacVisibilityRequirementCheckTests.cs deleted file mode 100644 index e6351558..00000000 --- a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/MacVisibilityRequirementCheckTests.cs +++ /dev/null @@ -1,511 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -using System.Net; -using System.Text; -using System.Text.Json; -using FluentAssertions; -using Microsoft.Agents.A365.DevTools.Cli.Models; -using Microsoft.Agents.A365.DevTools.Cli.Services.Requirements.RequirementChecks; -using Microsoft.Extensions.Logging; -using NSubstitute; -using Xunit; - -namespace Microsoft.Agents.A365.DevTools.Cli.Tests.Services.Requirements; - -public class MacVisibilityRequirementCheckTests : IDisposable -{ - private readonly ILogger _logger; - private readonly string _tempDir; - - public MacVisibilityRequirementCheckTests() - { - _logger = Substitute.For(); - _tempDir = Path.Combine(Path.GetTempPath(), $"a365-mac-validate-{Guid.NewGuid():N}"); - Directory.CreateDirectory(_tempDir); - } - - public void Dispose() - { - if (Directory.Exists(_tempDir)) - { - Directory.Delete(_tempDir, recursive: true); - } - } - - [Fact] - public void ParseNumericMetrics_FromMarkdown_ParsesKpiAndDailyValues() - { - var text = BuildMarkdownMetrics( - activeUsersRl7: 9, - invocationsRl7: 47, - sessionsRl7: 17, - toolExecutionsRl7: 60, - inferenceCallsRl7: 0, - runtimeHrsRl7: 0.15, - exceptionRateRl7: 0, - activeUsersRl30: 18, - invocationsRl30: 362, - sessionsRl30: 118, - toolExecutionsRl30: 1452, - inferenceCallsRl30: 431, - runtimeHrsRl30: 13.8, - exceptionRateRl30: 0); - - var parsed = MacVisibilityRequirementCheck.ParseNumericMetrics(text); - - parsed["kpi.active_users.rl7"].Should().Be(9); - parsed["kpi.invocations.rl30"].Should().Be(362); - parsed["kpi.runtime_hrs.rl7"].Should().Be(0.15); - parsed["kpi.exception_rate.rl30"].Should().Be(0); - parsed["daily.may_26.users"].Should().Be(4); - parsed["daily.may_26.invocations"].Should().Be(12); - parsed["daily.may_26.sessions"].Should().Be(4); - } - - [Fact] - public void CompareMetrics_NonExceptionMetricNotIncreased_Fails() - { - var baseline = new Dictionary(StringComparer.OrdinalIgnoreCase) - { - ["kpi.invocations.rl7"] = 10, - ["kpi.exception_rate.rl7"] = 1 - }; - var current = new Dictionary(StringComparer.OrdinalIgnoreCase) - { - ["kpi.invocations.rl7"] = 10, - ["kpi.exception_rate.rl7"] = 0 - }; - - var comparisons = MacVisibilityRequirementCheck.CompareMetrics(baseline, current); - - comparisons.Should().ContainSingle(c => c.MetricKey == "kpi.invocations.rl7" && !c.Passed, - because: "required non-exception metrics must strictly increase"); - comparisons.Should().ContainSingle(c => c.MetricKey == "kpi.exception_rate.rl7" && c.Passed, - because: "exception rate is exempt from increase requirement"); - } - - [Fact] - public async Task CaptureInitialMetricsAsync_WritesBaselineAndCheckAsync_PassesOnIncrease() - { - var baselinePath = Path.Combine(_tempDir, MacVisibilityRequirementCheck.BaselineFileName); - var config = new Agent365Config - { - TenantId = "11111111-1111-1111-1111-111111111111", - AgentIdentityDisplayName = "TestAgent", - Agent365ObservabilityMcpOptions = new Agent365ObservabilityMcpOptions - { - AgentObservabilityId = "11111111-2222-3333-4444-555555555555" - } - }; - - var baselineText = BuildMarkdownMetrics( - activeUsersRl7: 9, - invocationsRl7: 47, - sessionsRl7: 17, - toolExecutionsRl7: 60, - inferenceCallsRl7: 0, - runtimeHrsRl7: 0.15, - exceptionRateRl7: 0, - activeUsersRl30: 18, - invocationsRl30: 362, - sessionsRl30: 118, - toolExecutionsRl30: 1452, - inferenceCallsRl30: 431, - runtimeHrsRl30: 13.8, - exceptionRateRl30: 0); - - var postText = BuildMarkdownMetrics( - activeUsersRl7: 10, - invocationsRl7: 48, - sessionsRl7: 18, - toolExecutionsRl7: 61, - inferenceCallsRl7: 1, - runtimeHrsRl7: 0.20, - exceptionRateRl7: 0, - activeUsersRl30: 19, - invocationsRl30: 363, - sessionsRl30: 119, - toolExecutionsRl30: 1453, - inferenceCallsRl30: 432, - runtimeHrsRl30: 13.9, - exceptionRateRl30: 0); - - var captureHandler = new StaticToolResponseHandler(CreateJsonRpcResponseWithText(baselineText)); - var capture = await MacVisibilityRequirementCheck.CaptureInitialMetricsAsync( - config, - _logger, - authService: null, - environment: "prod", - baseUrlOverride: "https://example.test", - baselineFilePath: baselinePath, - httpHandler: captureHandler, - tokenProviderOverride: _ => Task.FromResult("token")); - - capture.Passed.Should().BeTrue(); - File.Exists(baselinePath).Should().BeTrue(); - - var checkHandler = new StaticToolResponseHandler(CreateJsonRpcResponseWithText(postText)); - var check = new MacVisibilityRequirementCheck( - authService: null, - baselineFilePath: baselinePath, - conversationStepVerified: true, - environment: "prod", - baseUrlOverride: "https://example.test", - httpHandler: checkHandler, - tokenProviderOverride: _ => Task.FromResult("token")); - - var result = await check.CheckAsync(config, _logger); - - result.Passed.Should().BeTrue(because: "all required KPI metrics increased and exception rate is exempt"); - result.Metadata!.MacMetricComparisons.Should().NotBeNull(); - } - - [Fact] - public async Task CheckAsync_WhenConversationNotVerified_Fails() - { - var baselinePath = Path.Combine(_tempDir, MacVisibilityRequirementCheck.BaselineFileName); - var config = new Agent365Config - { - TenantId = "11111111-1111-1111-1111-111111111111", - AgentIdentityDisplayName = "TestAgent" - }; - - var text = BuildMarkdownMetrics( - activeUsersRl7: 1, - invocationsRl7: 1, - sessionsRl7: 1, - toolExecutionsRl7: 1, - inferenceCallsRl7: 1, - runtimeHrsRl7: 1, - exceptionRateRl7: 0, - activeUsersRl30: 1, - invocationsRl30: 1, - sessionsRl30: 1, - toolExecutionsRl30: 1, - inferenceCallsRl30: 1, - runtimeHrsRl30: 1, - exceptionRateRl30: 0); - - await File.WriteAllTextAsync( - baselinePath, - "{\"capturedAtUtc\":\"2026-01-01T00:00:00Z\",\"endpoint\":\"https://example\",\"toolName\":\"getAgentMetrics\",\"serverName\":\"observability-mcp\",\"numericMetrics\":{\"kpi.invocations.rl7\":1}}", - Encoding.UTF8); - - var check = new MacVisibilityRequirementCheck( - authService: null, - baselineFilePath: baselinePath, - conversationStepVerified: false, - baseUrlOverride: "https://example.test", - httpHandler: new StaticToolResponseHandler(CreateJsonRpcResponseWithText(text)), - tokenProviderOverride: _ => Task.FromResult("token")); - - var result = await check.CheckAsync(config, _logger); - - result.Passed.Should().BeFalse(); - result.ErrorMessage.Should().Contain("Conversation simulation step is not verified"); - } - - [Fact] - public async Task CaptureInitialMetricsAsync_BlankOverrides_FallsBackToConfigValues() - { - var baselinePath = Path.Combine(_tempDir, MacVisibilityRequirementCheck.BaselineFileName); - var config = new Agent365Config - { - TenantId = "11111111-1111-1111-1111-111111111111", - AgentIdentityDisplayName = "Fallback Agent", - Agent365ObservabilityMcpOptions = new Agent365ObservabilityMcpOptions - { - BaseUrl = "https://example.test/", - TenantId = "22222222-2222-2222-2222-222222222222", - AgentName = "Configured Agent" - } - }; - - var handler = new StaticToolResponseHandler(CreateJsonRpcResponseWithText(BuildMarkdownMetrics( - activeUsersRl7: 1, - invocationsRl7: 2, - sessionsRl7: 3, - toolExecutionsRl7: 4, - inferenceCallsRl7: 5, - runtimeHrsRl7: 0.5, - exceptionRateRl7: 0, - activeUsersRl30: 10, - invocationsRl30: 20, - sessionsRl30: 30, - toolExecutionsRl30: 40, - inferenceCallsRl30: 50, - runtimeHrsRl30: 5, - exceptionRateRl30: 0))); - - var result = await MacVisibilityRequirementCheck.CaptureInitialMetricsAsync( - config, - _logger, - authService: null, - environment: "prod", - baseUrlOverride: string.Empty, - tenantIdOverride: string.Empty, - agentNameOverride: string.Empty, - baselineFilePath: baselinePath, - httpHandler: handler, - tokenProviderOverride: _ => Task.FromResult("token")); - - result.Passed.Should().BeTrue(because: "blank overrides should not block fallback to configured observability MCP values"); - } - - [Fact] - public async Task CaptureInitialMetricsAsync_ProbesToolsListBeforeCallingGetAgentMetrics() - { - var baselinePath = Path.Combine(_tempDir, MacVisibilityRequirementCheck.BaselineFileName); - var config = new Agent365Config - { - TenantId = "11111111-1111-1111-1111-111111111111", - AgentIdentityDisplayName = "TestAgent" - }; - - var handler = new SequencedMcpResponseHandler( - CreateToolsListResponse(MacVisibilityRequirementCheck.GetAgentMetricsToolName, "otherTool"), - CreateJsonRpcResponseWithText(BuildMarkdownMetrics( - activeUsersRl7: 1, - invocationsRl7: 2, - sessionsRl7: 3, - toolExecutionsRl7: 4, - inferenceCallsRl7: 5, - runtimeHrsRl7: 0.5, - exceptionRateRl7: 0, - activeUsersRl30: 10, - invocationsRl30: 20, - sessionsRl30: 30, - toolExecutionsRl30: 40, - inferenceCallsRl30: 50, - runtimeHrsRl30: 5, - exceptionRateRl30: 0))); - - var result = await MacVisibilityRequirementCheck.CaptureInitialMetricsAsync( - config, - _logger, - authService: null, - environment: "prod", - baseUrlOverride: "https://example.test", - baselineFilePath: baselinePath, - httpHandler: handler, - tokenProviderOverride: _ => Task.FromResult("token")); - - result.Passed.Should().BeTrue(); - handler.Methods.Should().ContainInOrder("tools/list", "tools/call"); - handler.ToolCallArgumentNames.Should().Contain("agentName", - because: "when agentObservabilityId is not configured, the MAC check falls back to agentName"); - HasLogMessageContaining("advertised getAgentMetrics: True").Should().BeTrue( - because: "successful discovery should record whether the server advertises getAgentMetrics"); - } - - [Fact] - public async Task CaptureInitialMetricsAsync_ToolsListFailure_FailsWithoutCallingGetAgentMetrics() - { - var baselinePath = Path.Combine(_tempDir, MacVisibilityRequirementCheck.BaselineFileName); - var config = new Agent365Config - { - TenantId = "11111111-1111-1111-1111-111111111111", - AgentIdentityDisplayName = "TestAgent" - }; - - var handler = new SequencedMcpResponseHandler( - "{\"error\":\"boom\"}", - HttpStatusCode.InternalServerError, - CreateJsonRpcResponseWithText(BuildMarkdownMetrics( - activeUsersRl7: 1, - invocationsRl7: 2, - sessionsRl7: 3, - toolExecutionsRl7: 4, - inferenceCallsRl7: 5, - runtimeHrsRl7: 0.5, - exceptionRateRl7: 0, - activeUsersRl30: 10, - invocationsRl30: 20, - sessionsRl30: 30, - toolExecutionsRl30: 40, - inferenceCallsRl30: 50, - runtimeHrsRl30: 5, - exceptionRateRl30: 0))); - - var result = await MacVisibilityRequirementCheck.CaptureInitialMetricsAsync( - config, - _logger, - authService: null, - environment: "prod", - baseUrlOverride: "https://example.test", - baselineFilePath: baselinePath, - httpHandler: handler, - tokenProviderOverride: _ => Task.FromResult("token")); - - result.Passed.Should().BeFalse(because: "tools/list must advertise getAgentMetrics before tools/call is attempted"); - result.Details.Should().Contain("required tool 'getAgentMetrics'", because: "discovery should fail fast when required tool is not listed"); - handler.Methods.Should().Equal(new[] { "tools/list" }, - because: "tools/call should not run when required tool discovery fails"); - HasLogMessageContaining("tools/list probe failed").Should().BeTrue( - because: "discovery failures should be logged distinctly from getAgentMetrics invocation failures"); - } - - private static string CreateJsonRpcResponseWithText(string text) - { - var escaped = text.Replace("\\", "\\\\", StringComparison.Ordinal) - .Replace("\"", "\\\"", StringComparison.Ordinal) - .Replace("\r", "\\r", StringComparison.Ordinal) - .Replace("\n", "\\n", StringComparison.Ordinal); - - return - "{" + - "\"jsonrpc\":\"2.0\"," + - "\"id\":\"1\"," + - "\"result\":{" + - "\"content\":[{" + - "\"text\":\"" + escaped + "\"" + - "}]" + - "}" + - "}"; - } - - private static string CreateToolsListResponse(params string[] toolNames) - { - return JsonSerializer.Serialize(new - { - jsonrpc = "2.0", - id = "tools-list", - result = new - { - tools = toolNames.Select(name => new { name }).ToArray() - } - }); - } - - private static string BuildMarkdownMetrics( - double activeUsersRl7, - double invocationsRl7, - double sessionsRl7, - double toolExecutionsRl7, - double inferenceCallsRl7, - double runtimeHrsRl7, - double exceptionRateRl7, - double activeUsersRl30, - double invocationsRl30, - double sessionsRl30, - double toolExecutionsRl30, - double inferenceCallsRl30, - double runtimeHrsRl30, - double exceptionRateRl30) - { - return - "2. getAgentMetrics - KPIs + daily time series\n\n" + - "| Metric | RL7 | RL30 | WoW Change |\n" + - "|---|---:|---:|---:|\n" + - $"| Active Users | {activeUsersRl7} | {activeUsersRl30} | -10% |\n" + - $"| Invocations | {invocationsRl7} | {invocationsRl30} | -63% |\n" + - $"| Sessions | {sessionsRl7} | {sessionsRl30} | - |\n" + - $"| Tool Executions | {toolExecutionsRl7} | {toolExecutionsRl30} | - |\n" + - $"| Inference Calls | {inferenceCallsRl7} | {inferenceCallsRl30} | - |\n" + - $"| Runtime (hrs) | {runtimeHrsRl7} | {runtimeHrsRl30} | -93.9% |\n" + - $"| Exception Rate | {exceptionRateRl7}% | {exceptionRateRl30}% | 0% |\n\n" + - "Daily time series (last 5 days):\n" + - "| Date | Users | Invocations | Sessions |\n" + - "|---|---:|---:|---:|\n" + - "| May 26 | 4 | 12 | 4 |\n"; - } - - private bool HasLogMessageContaining(string expectedText) - { - return _logger.ReceivedCalls().Any(call => - { - if (!string.Equals(call.GetMethodInfo().Name, nameof(ILogger.Log), StringComparison.Ordinal)) - { - return false; - } - - var state = call.GetArguments()[2]; - return state?.ToString()?.Contains(expectedText, StringComparison.OrdinalIgnoreCase) == true; - }); - } - - private sealed class StaticToolResponseHandler : HttpMessageHandler - { - private readonly string _responseContent; - - public StaticToolResponseHandler(string responseContent) - { - _responseContent = responseContent; - } - - protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) - { - var responseContent = _responseContent; - if (request.Content is not null) - { - var requestJson = request.Content.ReadAsStringAsync(cancellationToken).GetAwaiter().GetResult(); - using var document = JsonDocument.Parse(requestJson); - var method = document.RootElement.GetProperty("method").GetString(); - if (string.Equals(method, "tools/list", StringComparison.OrdinalIgnoreCase)) - { - responseContent = CreateToolsListResponse(MacVisibilityRequirementCheck.GetAgentMetricsToolName, "otherTool"); - } - } - - var response = new HttpResponseMessage(HttpStatusCode.OK) - { - Content = new StringContent(responseContent, Encoding.UTF8, "application/json") - }; - - return Task.FromResult(response); - } - } - - private sealed class SequencedMcpResponseHandler : HttpMessageHandler - { - private readonly Queue<(HttpStatusCode StatusCode, string Content)> _responses; - - public SequencedMcpResponseHandler(string responseContent, HttpStatusCode statusCode, params string[] additionalResponseContents) - { - _responses = new Queue<(HttpStatusCode StatusCode, string Content)>(); - _responses.Enqueue((statusCode, responseContent)); - - foreach (var content in additionalResponseContents) - { - _responses.Enqueue((HttpStatusCode.OK, content)); - } - } - - public SequencedMcpResponseHandler(params string[] responseContents) - { - _responses = new Queue<(HttpStatusCode StatusCode, string Content)>( - responseContents.Select(content => (HttpStatusCode.OK, content))); - } - - public List Methods { get; } = []; - public List ToolCallArgumentNames { get; } = []; - - protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) - { - request.Content.Should().NotBeNull(); - - var requestJson = await request.Content!.ReadAsStringAsync(cancellationToken); - using var document = JsonDocument.Parse(requestJson); - var method = document.RootElement.GetProperty("method").GetString() ?? string.Empty; - Methods.Add(method); - - if (string.Equals(method, "tools/call", StringComparison.OrdinalIgnoreCase) - && document.RootElement.TryGetProperty("params", out var paramsElement) - && paramsElement.TryGetProperty("arguments", out var argumentsElement) - && argumentsElement.ValueKind == JsonValueKind.Object) - { - foreach (var property in argumentsElement.EnumerateObject()) - { - ToolCallArgumentNames.Add(property.Name); - } - } - - var (statusCode, content) = _responses.Dequeue(); - return new HttpResponseMessage(statusCode) - { - Content = new StringContent(content, Encoding.UTF8, "application/json") - }; - } - } -} diff --git a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/TelemetryRequirementCheckTests.cs b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/TelemetryRequirementCheckTests.cs index b290c92e..99a49346 100644 --- a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/TelemetryRequirementCheckTests.cs +++ b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/TelemetryRequirementCheckTests.cs @@ -494,45 +494,6 @@ public void HasNonEmptyValue_EmptyQuotes_ReturnsFalse() TelemetryRequirementCheck.HasNonEmptyValue(" parentId: ''").Should().BeFalse(); } - // --- Resource attribute checks --- - - [Fact] - public void GetMissingResourceAttributes_AllPresent_ReturnsEmpty() - { - var lines = new[] - { - " 'telemetry.sdk.name': 'opentelemetry',", - " 'telemetry.sdk.version': '1.25.0',", - " 'service.name': 'my-agent'," - }; - - TelemetryRequirementCheck.GetMissingResourceAttributes(lines).Should().BeEmpty(); - } - - [Fact] - public void GetMissingResourceAttributes_MissingSdkVersion_ReportsIt() - { - var lines = new[] - { - " 'telemetry.sdk.name': 'opentelemetry',", - " 'service.name': 'my-agent'," - }; - - var missing = TelemetryRequirementCheck.GetMissingResourceAttributes(lines); - missing.Should().Contain("telemetry.sdk.version"); - missing.Should().NotContain("telemetry.sdk.name"); - missing.Should().NotContain("service.name"); - } - - [Fact] - public void GetMissingResourceAttributes_NonePresent_ReturnsAll() - { - var lines = new[] { "some unrelated log output" }; - - var missing = TelemetryRequirementCheck.GetMissingResourceAttributes(lines); - missing.Should().HaveCount(3); - } - // --- End-to-end: fully compliant spans return success --- [Fact] @@ -587,10 +548,9 @@ public async Task CheckAsync_ChildSpansMissingParent_ReturnsWarning() } [Fact] - public async Task CheckAsync_MissingResourceAttributes_ReturnsWarning() + public async Task CheckAsync_AllSpansPresent_NoResourceAttributes_ReturnsSuccess() { var lines = new List(); - // No resource lines lines.Add("{"); lines.AddRange(MakeFullAgent365Span("invoke_agent")); lines.Add("}"); @@ -606,9 +566,8 @@ public async Task CheckAsync_MissingResourceAttributes_ReturnsWarning() var result = await check.CheckAsync(_config, _logger); - result.Passed.Should().BeTrue(because: "missing resource attributes is a warning not a failure"); - result.IsWarning.Should().BeTrue(); - result.Details.Should().Contain("service.name", because: "warning should list missing resource attributes"); + result.Passed.Should().BeTrue(because: "all required spans are present with parent links"); + result.IsWarning.Should().BeFalse(because: "resource attributes are no longer checked"); } // ── Python console exporter format tests ── From 01148a4fff61d45966311236002fee088755cc8e Mon Sep 17 00:00:00 2001 From: Abbinayaa Subramanian Date: Thu, 11 Jun 2026 16:03:23 -0700 Subject: [PATCH 10/27] Clean up change in cli and move checks to validate --- CHANGELOG.md | 5 +- src/Directory.Packages.props | 1 - .../Commands/ValidateCommand.cs | 35 +-- .../Constants/McpConstants.cs | 5 - .../Microsoft.Agents.A365.DevTools.Cli.csproj | 3 - .../Requirements/RequirementCheckResult.cs | 199 +----------------- .../BlueprintRegistrationRequirementCheck.cs | 1 + .../ConversationRequirementCheck.cs | 1 + .../LocalRuntimeRequirementCheck.cs | 1 + .../PowerShellModulesRequirementCheck.cs | 1 + .../ProjectBuildRequirementCheck.cs | 1 + .../HttpListenerBotCallbackReceiver.cs | 10 +- .../IBotCallbackReceiver.cs | 2 +- .../RequirementCheckMetadata.cs | 199 ++++++++++++++++++ ...eprintRegistrationRequirementCheckTests.cs | 39 ++-- .../ConversationRequirementCheckTests.cs | 38 ++-- .../HttpListenerBotCallbackReceiverTests.cs | 5 +- src/a365.config.example.json | 11 +- 18 files changed, 285 insertions(+), 272 deletions(-) rename src/{Microsoft.Agents.A365.DevTools.Cli/Services => Microsoft.Agents.A365.DevTools.Validation}/HttpListenerBotCallbackReceiver.cs (94%) rename src/{Microsoft.Agents.A365.DevTools.Cli/Services => Microsoft.Agents.A365.DevTools.Validation}/IBotCallbackReceiver.cs (93%) create mode 100644 src/Microsoft.Agents.A365.DevTools.Validation/RequirementCheckMetadata.cs rename src/Tests/{Microsoft.Agents.A365.DevTools.Cli.Tests/Services => Microsoft.Agents.A365.DevTools.Validation.Tests/Validation}/HttpListenerBotCallbackReceiverTests.cs (94%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b94564d..dd942616 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,11 +23,10 @@ Agents provisioned before this release need `Agent365.Observability.OtelWrite` g **Option B — CLI** (`a365 setup admin`) has been removed in this release. Use Option A above, or copy the PowerShell instructions printed in the `a365 setup all` summary output. ### Added -- `a365 validate` — validates the local `a365.config.json` plus prerequisite checks without making changes. Reports missing or invalid config, then runs the existing setup prerequisite checks so users can catch problems before starting a setup workflow. +- `a365 validate` — validates the local `a365.config.json` plus prerequisite checks without making changes. Reports missing or invalid config, then runs the existing setup prerequisite checks so users can catch problems before starting a publish workflow. +- New `Microsoft.Agents.A365.DevTools.Validation` subproject for reusable validation contracts and helpers. ### Changed -- `a365 validate` now runs blueprint registration checks (Entra app, service principal, permissions) as part of the normal flow. The `--with-tenant` option has been removed; all checks run in a single invocation. -- New `Microsoft.Agents.A365.DevTools.Validation` subproject for reusable validation contracts and helpers. - `logs export [command] [--output ]` — exports a redacted copy of a CLI diagnostic log safe to share with Microsoft support. Redacts JWT tokens, email addresses, OS-path usernames, and tenant-specific GUIDs; replaces identical values with consistent aliases so log correlation is preserved. Preserves diagnostic IDs that aren't sensitive but are useful for debugging — `TraceId`, `CorrelationId`, Microsoft Graph `request-id` and `client-request-id` values, and well-known public Microsoft / Agent 365 resource appIds (such as the Microsoft Graph appId `00000003-0000-0000-c000-000000000000`). Omit `[command]` to export all available logs at once. - `setup blueprint --show-secret` — displays the blueprint client secret stored in `a365.generated.config.json` in plaintext without re-running any setup steps. On Windows, decryption requires the same machine and user account that ran setup (DPAPI). When no secret is found, the command prints instructions to run `a365 setup blueprint --agent-name `. - Blueprint client secret is now printed to the terminal at creation time with a "copy this value now" warning. Use `a365 setup blueprint --show-secret` to retrieve it afterwards. diff --git a/src/Directory.Packages.props b/src/Directory.Packages.props index 4383fa98..f3fccf3a 100644 --- a/src/Directory.Packages.props +++ b/src/Directory.Packages.props @@ -13,7 +13,6 @@ - diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Commands/ValidateCommand.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Commands/ValidateCommand.cs index 0177cd27..abead4f9 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Commands/ValidateCommand.cs +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Commands/ValidateCommand.cs @@ -100,7 +100,7 @@ public static Command CreateCommand( // Extract resolved uv command from build step for boot and conversation steps var buildResultEntry = results .FirstOrDefault(r => r.Check is ProjectBuildRequirementCheck); - var resolvedUvCommand = buildResultEntry.Result?.Metadata?.ResolvedUvCommand; + var resolvedUvCommand = (buildResultEntry.Result?.Metadata as RequirementCheckMetadata)?.ResolvedUvCommand; // Phase 2b: Run boot check only if build passed var buildPassed = report.Tiers.Build is { Skipped: true } or { Ok: true }; @@ -137,7 +137,7 @@ public static Command CreateCommand( // Run telemetry check using agent's console log file var conversationResult = conversationResults .FirstOrDefault(r => r.Check is ConversationRequirementCheck); - var agentLogPath = conversationResult.Result?.Metadata?.AgentConsoleLogPath; + var agentLogPath = (conversationResult.Result?.Metadata as RequirementCheckMetadata)?.AgentConsoleLogPath; report.AgentConsoleLogFile = agentLogPath; var telemetryCheck = new TelemetryRequirementCheck(agentLogPath); var telemetryResults = await RunChecksDetailedAsync( @@ -352,12 +352,13 @@ private static void MapResultsToTiers( } else { + var buildMeta = result.Metadata as RequirementCheckMetadata; report.Tiers.Build = new BuildTierResult { Ok = result.Passed, - ExitCode = result.Metadata?.ExitCode, + ExitCode = buildMeta?.ExitCode, ErrorSummary = result.Passed ? null : result.ErrorMessage, - BuildLogFile = result.Metadata?.BuildLogFile + BuildLogFile = buildMeta?.BuildLogFile }; } break; @@ -373,12 +374,13 @@ private static void MapResultsToTiers( } else { + var bootMeta = result.Metadata as RequirementCheckMetadata; report.Tiers.Boot = new BootTierResult { Ok = result.Passed, - Port = result.Metadata?.Port, - BootMs = result.Metadata?.BootMs, - BootLogFile = result.Metadata?.BootLogFile + Port = bootMeta?.Port, + BootMs = bootMeta?.BootMs, + BootLogFile = bootMeta?.BootLogFile }; } break; @@ -394,12 +396,13 @@ private static void MapResultsToTiers( } else { + var convMeta = result.Metadata as RequirementCheckMetadata; report.Tiers.Conversation = new ConversationTierResult { Ok = result.Passed, - PlaygroundLaunched = result.Metadata?.PlaygroundLaunched, - ConversationLogFile = result.Metadata?.ConversationLogFile, - Turns = result.Metadata?.Turns?.Select(t => new ConversationTurnResult + PlaygroundLaunched = convMeta?.PlaygroundLaunched, + ConversationLogFile = convMeta?.ConversationLogFile, + Turns = convMeta?.Turns?.Select(t => new ConversationTurnResult { Input = t.Input, StatusCode = t.StatusCode, @@ -457,15 +460,15 @@ private static void MapResultsToTiers( blueprintTier.Reason = result.Passed ? null : result.ErrorMessage; } - if (result.Metadata is not null) + if (result.Metadata is RequirementCheckMetadata bpMeta) { - blueprintTier.AppExists = result.Metadata.AppExists; - blueprintTier.ServicePrincipalExists = result.Metadata.ServicePrincipalExists; - blueprintTier.RegistrationExists = result.Metadata.RegistrationExists; + blueprintTier.AppExists = bpMeta.AppExists; + blueprintTier.ServicePrincipalExists = bpMeta.ServicePrincipalExists; + blueprintTier.RegistrationExists = bpMeta.RegistrationExists; - if (result.Metadata.ResourcePermissions is { Count: > 0 }) + if (bpMeta.ResourcePermissions is { Count: > 0 }) { - blueprintTier.Resources = result.Metadata.ResourcePermissions.Select(rp => + blueprintTier.Resources = bpMeta.ResourcePermissions.Select(rp => new BlueprintResourceResult { ResourceName = rp.ResourceName, diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Constants/McpConstants.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Constants/McpConstants.cs index bffddd43..97cb87d5 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Constants/McpConstants.cs +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Constants/McpConstants.cs @@ -61,11 +61,6 @@ public static string BuildPpmiIdentifierUri(string environment, string tenantId, /// public const string ToolsCallMethod = "tools/call"; - /// - /// Method name for listing MCP tools - /// - public const string ToolsListMethod = "tools/list"; - /// /// Name of the ListToolServers tool /// diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Microsoft.Agents.A365.DevTools.Cli.csproj b/src/Microsoft.Agents.A365.DevTools.Cli/Microsoft.Agents.A365.DevTools.Cli.csproj index e719c0d2..e24be59f 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Microsoft.Agents.A365.DevTools.Cli.csproj +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Microsoft.Agents.A365.DevTools.Cli.csproj @@ -45,9 +45,6 @@ - - - diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementCheckResult.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementCheckResult.cs index 1f486d7d..02f8bc0b 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementCheckResult.cs +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementCheckResult.cs @@ -34,9 +34,10 @@ public class RequirementCheckResult public string? Details { get; set; } /// - /// Optional typed metadata for structured report output. + /// Optional metadata for structured report output (validate-specific). + /// Consumers should cast to the appropriate metadata type. /// - public RequirementCheckMetadata? Metadata { get; set; } + public object? Metadata { get; set; } /// /// Creates a successful result @@ -80,197 +81,3 @@ public static RequirementCheckResult Failure(string errorMessage, string resolut }; } } - -/// -/// Typed metadata for structured validation report output. -/// -public sealed class RequirementCheckMetadata -{ - /// Port the app is running on (boot tier). - public int? Port { get; init; } - - /// Time in milliseconds for the app to respond (boot tier). - public long? BootMs { get; init; } - - /// Build or runtime log output (build/boot tier). - public string? Log { get; init; } - - /// Process exit code (build tier). - public int? ExitCode { get; init; } - - /// Detected platform name (build/boot tier). - public string? Platform { get; init; } - - /// Conversation turn results (conversation tier). - public List? Turns { get; init; } - - /// Whether AgentsPlayground was launched for interactive testing. - public bool? PlaygroundLaunched { get; init; } - - /// - /// Path to the agent's captured console output log file. - /// Written during the conversation step; used by telemetry check and referenced in the report. - /// - public string? AgentConsoleLogPath { get; init; } - - /// - /// Path to the MSBuild file log written during project build validation. - /// - public string? BuildLogFile { get; init; } - - /// - /// Path to the boot log file written during local runtime validation. - /// - public string? BootLogFile { get; init; } - - /// - /// Path to the conversation log file written during conversation validation. - /// Contains HTTP request/response details for each turn. - /// - public string? ConversationLogFile { get; init; } - - /// - /// Resolved path to the uv command, set during build dependency install. - /// Used by the boot step to run Python agents in uv-managed projects. - /// - public string? ResolvedUvCommand { get; init; } - - /// Whether the blueprint application exists in Entra ID. - public bool? AppExists { get; init; } - - /// Whether a service principal exists for the blueprint. - public bool? ServicePrincipalExists { get; init; } - - /// Whether the agent registration exists (null if not configured). - public bool? RegistrationExists { get; init; } - - /// Resource permission results from comparing config vs Entra. - public List? ResourcePermissions { get; set; } - - /// - /// Path to the persisted pre-conversation MAC metrics baseline file. - /// - public string? MacMetricsBaselineFile { get; init; } - - /// - /// Flattened numeric metrics captured before conversation. - /// - public Dictionary? MacBaselineMetrics { get; init; } - - /// - /// Flattened numeric metrics captured after conversation. - /// - public Dictionary? MacCurrentMetrics { get; init; } - - /// - /// Per-metric comparison outcome between baseline and post-conversation snapshots. - /// - public List? MacMetricComparisons { get; init; } - - /// - /// Whether conversation simulation completion was verified before MAC comparison. - /// - public bool? ConversationStepVerified { get; init; } -} - -/// -/// Comparison details for a single MAC metric. -/// -public sealed class MacMetricComparisonMetadata -{ - /// Canonical metric key (e.g., kpi.invocations.rl7). - public string MetricKey { get; init; } = string.Empty; - - /// Baseline value. - public double Before { get; init; } - - /// Post-conversation value. - public double After { get; init; } - - /// After - Before. - public double Delta { get; init; } - - /// True when delta is positive. - public bool Increased { get; init; } - - /// True when this metric is the exception-rate metric. - public bool IsExceptionRate { get; init; } - - /// Final pass/fail for this metric after applying rule exceptions. - public bool Passed { get; init; } - - /// Human-readable reason for this comparison result. - public string? Reason { get; init; } -} - -/// -/// Metadata for a single conversation turn. -/// -public sealed class ConversationTurnMetadata -{ - /// The message sent to the agent. - public string Input { get; init; } = string.Empty; - - /// HTTP status code returned by /api/messages. - public int? StatusCode { get; init; } - - /// Truncated response body snippet. - public string? ResponseSnippet { get; init; } - - /// Round-trip latency in milliseconds. - public long? LatencyMs { get; init; } - - /// Whether this turn succeeded. - public bool Ok { get; init; } - - /// Error description if the turn failed. - public string? Error { get; init; } - - /// Whether the agent sent a response via the serviceUrl callback. Null if tracking was unavailable. - public bool? AgentResponded { get; init; } - - /// The text content of the agent's callback response, if any. - public string? AgentResponseText { get; init; } -} - -/// -/// Permission status for a single resource API in the blueprint registration check. -/// -public sealed class BlueprintResourcePermission -{ - /// Display name of the resource (e.g., "Microsoft Graph"). - public string ResourceName { get; init; } = string.Empty; - - /// Application ID of the resource. - public string ResourceAppId { get; init; } = string.Empty; - - /// Scopes expected from config. - public List ExpectedScopes { get; init; } = new(); - - /// Scopes actually found in Entra inheritable permissions. - public List ActualScopes { get; init; } = new(); - - /// Scopes in config but missing from Entra. - public List MissingScopes { get; init; } = new(); - - /// Whether admin consent has been granted (from config). - public bool? ConsentGranted { get; init; } - - /// Whether inheritable permissions are configured in Entra for this resource. - public bool InheritablePermissionsConfigured { get; init; } - - /// Whether kind=allAllowed is set for delegated scopes on this resource. - public bool ScopesAllAllowed { get; init; } - - /// Whether kind=allAllowed is set for app roles on this resource. - public bool RolesAllAllowed { get; init; } - - /// App roles actually granted on the blueprint SP for this resource. - public List ActualAppRoles { get; init; } = new(); - - /// - /// Effective inheritance status: true when kind=allAllowed on both sides AND at least one - /// permission is granted on the blueprint SP for this resource. - /// - public bool EffectiveInheritance { get; init; } -} diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/BlueprintRegistrationRequirementCheck.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/BlueprintRegistrationRequirementCheck.cs index 8fcf5577..bd34afa0 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/BlueprintRegistrationRequirementCheck.cs +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/BlueprintRegistrationRequirementCheck.cs @@ -4,6 +4,7 @@ using Microsoft.Agents.A365.DevTools.Cli.Constants; using Microsoft.Agents.A365.DevTools.Cli.Helpers; using Microsoft.Agents.A365.DevTools.Cli.Models; +using Microsoft.Agents.A365.DevTools.Validation; using Microsoft.Extensions.Logging; namespace Microsoft.Agents.A365.DevTools.Cli.Services.Requirements.RequirementChecks; diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/ConversationRequirementCheck.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/ConversationRequirementCheck.cs index 4f2ae11c..c3042b71 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/ConversationRequirementCheck.cs +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/ConversationRequirementCheck.cs @@ -8,6 +8,7 @@ using System.Text.Json; using System.Text.Json.Serialization; using Microsoft.Agents.A365.DevTools.Cli.Constants; +using Microsoft.Agents.A365.DevTools.Validation; using Microsoft.Agents.A365.DevTools.Cli.Models; using Microsoft.Extensions.Logging; diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/LocalRuntimeRequirementCheck.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/LocalRuntimeRequirementCheck.cs index 18eda3cd..9178be55 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/LocalRuntimeRequirementCheck.cs +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/LocalRuntimeRequirementCheck.cs @@ -6,6 +6,7 @@ using System.Runtime.InteropServices; using System.Text; using Microsoft.Agents.A365.DevTools.Cli.Models; +using Microsoft.Agents.A365.DevTools.Validation; using Microsoft.Extensions.Logging; namespace Microsoft.Agents.A365.DevTools.Cli.Services.Requirements.RequirementChecks; diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/PowerShellModulesRequirementCheck.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/PowerShellModulesRequirementCheck.cs index 8292ef16..5cf9b9c6 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/PowerShellModulesRequirementCheck.cs +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/PowerShellModulesRequirementCheck.cs @@ -4,6 +4,7 @@ using Microsoft.Agents.A365.DevTools.Cli.Models; using Microsoft.Extensions.Logging; using System.Diagnostics; +using System.Text.Json; namespace Microsoft.Agents.A365.DevTools.Cli.Services.Requirements.RequirementChecks; diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/ProjectBuildRequirementCheck.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/ProjectBuildRequirementCheck.cs index 909b2fbf..72be9922 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/ProjectBuildRequirementCheck.cs +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/ProjectBuildRequirementCheck.cs @@ -2,6 +2,7 @@ // Licensed under the MIT License. using Microsoft.Agents.A365.DevTools.Cli.Models; +using Microsoft.Agents.A365.DevTools.Validation; using Microsoft.Extensions.Logging; namespace Microsoft.Agents.A365.DevTools.Cli.Services.Requirements.RequirementChecks; diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Services/HttpListenerBotCallbackReceiver.cs b/src/Microsoft.Agents.A365.DevTools.Validation/HttpListenerBotCallbackReceiver.cs similarity index 94% rename from src/Microsoft.Agents.A365.DevTools.Cli/Services/HttpListenerBotCallbackReceiver.cs rename to src/Microsoft.Agents.A365.DevTools.Validation/HttpListenerBotCallbackReceiver.cs index 4857dd5b..8d69c796 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Services/HttpListenerBotCallbackReceiver.cs +++ b/src/Microsoft.Agents.A365.DevTools.Validation/HttpListenerBotCallbackReceiver.cs @@ -5,14 +5,14 @@ using System.Net.Sockets; using System.Text.Json; -namespace Microsoft.Agents.A365.DevTools.Cli.Services; +namespace Microsoft.Agents.A365.DevTools.Validation; /// /// HttpListener-based implementation of . /// Listens on a random available localhost port for Bot Framework callback activities /// sent by the agent to POST /v3/conversations/{id}/activities. /// -internal sealed class HttpListenerBotCallbackReceiver : IBotCallbackReceiver +public sealed class HttpListenerBotCallbackReceiver : IBotCallbackReceiver { private readonly HttpListener _listener; private readonly int _port; @@ -69,7 +69,7 @@ public HttpListenerBotCallbackReceiver() /// /// Initializes with a specific port (for testing). /// - internal HttpListenerBotCallbackReceiver(int port) + public HttpListenerBotCallbackReceiver(int port) { _port = port; _listener = new HttpListener(); @@ -298,7 +298,7 @@ private async Task HandleRequestAsync(HttpListenerContext context) /// /// Determines whether a response is a final (non-interim) message from the agent. /// - internal static bool IsFinalMessage(BotCallbackResponse response) + public static bool IsFinalMessage(BotCallbackResponse response) { // Typing activities are never final if (string.Equals(response.Type, "typing", StringComparison.OrdinalIgnoreCase)) @@ -320,7 +320,7 @@ internal static bool IsFinalMessage(BotCallbackResponse response) /// Only matches short messages (under 60 chars) containing known interim phrases. /// Longer messages are assumed to be real responses even if they contain interim-like words. /// - internal static bool IsInterimMessage(string? text) + public static bool IsInterimMessage(string? text) { if (string.IsNullOrWhiteSpace(text)) { diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Services/IBotCallbackReceiver.cs b/src/Microsoft.Agents.A365.DevTools.Validation/IBotCallbackReceiver.cs similarity index 93% rename from src/Microsoft.Agents.A365.DevTools.Cli/Services/IBotCallbackReceiver.cs rename to src/Microsoft.Agents.A365.DevTools.Validation/IBotCallbackReceiver.cs index 77bef734..0a5f51bb 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Services/IBotCallbackReceiver.cs +++ b/src/Microsoft.Agents.A365.DevTools.Validation/IBotCallbackReceiver.cs @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -namespace Microsoft.Agents.A365.DevTools.Cli.Services; +namespace Microsoft.Agents.A365.DevTools.Validation; /// /// Receives Bot Framework callback activities sent by the agent to its serviceUrl. diff --git a/src/Microsoft.Agents.A365.DevTools.Validation/RequirementCheckMetadata.cs b/src/Microsoft.Agents.A365.DevTools.Validation/RequirementCheckMetadata.cs new file mode 100644 index 00000000..66dbe79f --- /dev/null +++ b/src/Microsoft.Agents.A365.DevTools.Validation/RequirementCheckMetadata.cs @@ -0,0 +1,199 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.Agents.A365.DevTools.Validation; + +/// +/// Typed metadata for structured validation report output. +/// Attached to RequirementCheckResult.Metadata by validate-specific checks. +/// +public sealed class RequirementCheckMetadata +{ + /// Port the app is running on (boot tier). + public int? Port { get; init; } + + /// Time in milliseconds for the app to respond (boot tier). + public long? BootMs { get; init; } + + /// Build or runtime log output (build/boot tier). + public string? Log { get; init; } + + /// Process exit code (build tier). + public int? ExitCode { get; init; } + + /// Detected platform name (build/boot tier). + public string? Platform { get; init; } + + /// Conversation turn results (conversation tier). + public List? Turns { get; init; } + + /// Whether AgentsPlayground was launched for interactive testing. + public bool? PlaygroundLaunched { get; init; } + + /// + /// Path to the agent's captured console output log file. + /// Written during the conversation step; used by telemetry check and referenced in the report. + /// + public string? AgentConsoleLogPath { get; init; } + + /// + /// Path to the MSBuild file log written during project build validation. + /// + public string? BuildLogFile { get; init; } + + /// + /// Path to the boot log file written during local runtime validation. + /// + public string? BootLogFile { get; init; } + + /// + /// Path to the conversation log file written during conversation validation. + /// Contains HTTP request/response details for each turn. + /// + public string? ConversationLogFile { get; init; } + + /// + /// Resolved path to the uv command, set during build dependency install. + /// Used by the boot step to run Python agents in uv-managed projects. + /// + public string? ResolvedUvCommand { get; init; } + + /// Whether the blueprint application exists in Entra ID. + public bool? AppExists { get; init; } + + /// Whether a service principal exists for the blueprint. + public bool? ServicePrincipalExists { get; init; } + + /// Whether the agent registration exists (null if not configured). + public bool? RegistrationExists { get; init; } + + /// Resource permission results from comparing config vs Entra. + public List? ResourcePermissions { get; set; } + + /// + /// Path to the persisted pre-conversation MAC metrics baseline file. + /// + public string? MacMetricsBaselineFile { get; init; } + + /// + /// Flattened numeric metrics captured before conversation. + /// + public Dictionary? MacBaselineMetrics { get; init; } + + /// + /// Flattened numeric metrics captured after conversation. + /// + public Dictionary? MacCurrentMetrics { get; init; } + + /// + /// Per-metric comparison outcome between baseline and post-conversation snapshots. + /// + public List? MacMetricComparisons { get; init; } + + /// + /// Whether conversation simulation completion was verified before MAC comparison. + /// + public bool? ConversationStepVerified { get; init; } +} + +/// +/// Comparison details for a single MAC metric. +/// +public sealed class MacMetricComparisonMetadata +{ + /// Canonical metric key (e.g., kpi.invocations.rl7). + public string MetricKey { get; init; } = string.Empty; + + /// Baseline value. + public double Before { get; init; } + + /// Post-conversation value. + public double After { get; init; } + + /// After - Before. + public double Delta { get; init; } + + /// True when delta is positive. + public bool Increased { get; init; } + + /// True when this metric is the exception-rate metric. + public bool IsExceptionRate { get; init; } + + /// Final pass/fail for this metric after applying rule exceptions. + public bool Passed { get; init; } + + /// Human-readable reason for this comparison result. + public string? Reason { get; init; } +} + +/// +/// Metadata for a single conversation turn. +/// +public sealed class ConversationTurnMetadata +{ + /// The message sent to the agent. + public string Input { get; init; } = string.Empty; + + /// HTTP status code returned by /api/messages. + public int? StatusCode { get; init; } + + /// Truncated response body snippet. + public string? ResponseSnippet { get; init; } + + /// Round-trip latency in milliseconds. + public long? LatencyMs { get; init; } + + /// Whether this turn succeeded. + public bool Ok { get; init; } + + /// Error description if the turn failed. + public string? Error { get; init; } + + /// Whether the agent sent a response via the serviceUrl callback. Null if tracking was unavailable. + public bool? AgentResponded { get; init; } + + /// The text content of the agent's callback response, if any. + public string? AgentResponseText { get; init; } +} + +/// +/// Permission status for a single resource API in the blueprint registration check. +/// +public sealed class BlueprintResourcePermission +{ + /// Display name of the resource (e.g., "Microsoft Graph"). + public string ResourceName { get; init; } = string.Empty; + + /// Application ID of the resource. + public string ResourceAppId { get; init; } = string.Empty; + + /// Scopes expected from config. + public List ExpectedScopes { get; init; } = new(); + + /// Scopes actually found in Entra inheritable permissions. + public List ActualScopes { get; init; } = new(); + + /// Scopes in config but missing from Entra. + public List MissingScopes { get; init; } = new(); + + /// Whether admin consent has been granted (from config). + public bool? ConsentGranted { get; init; } + + /// Whether inheritable permissions are configured in Entra for this resource. + public bool InheritablePermissionsConfigured { get; init; } + + /// Whether kind=allAllowed is set for delegated scopes on this resource. + public bool ScopesAllAllowed { get; init; } + + /// Whether kind=allAllowed is set for app roles on this resource. + public bool RolesAllAllowed { get; init; } + + /// App roles actually granted on the blueprint SP for this resource. + public List ActualAppRoles { get; init; } = new(); + + /// + /// Effective inheritance status: true when kind=allAllowed on both sides AND at least one + /// permission is granted on the blueprint SP for this resource. + /// + public bool EffectiveInheritance { get; init; } +} diff --git a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/BlueprintRegistrationRequirementCheckTests.cs b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/BlueprintRegistrationRequirementCheckTests.cs index 2a54074b..fe99fa50 100644 --- a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/BlueprintRegistrationRequirementCheckTests.cs +++ b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/BlueprintRegistrationRequirementCheckTests.cs @@ -6,6 +6,7 @@ using Microsoft.Agents.A365.DevTools.Cli.Models; using Microsoft.Agents.A365.DevTools.Cli.Services; using Microsoft.Agents.A365.DevTools.Cli.Services.Requirements.RequirementChecks; +using Microsoft.Agents.A365.DevTools.Validation; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using NSubstitute; @@ -449,7 +450,8 @@ public async Task CheckAsync_AppNotFound_MetadataHasAppExistsFalse() var result = await check.CheckAsync(config, _logger); result.Metadata.Should().NotBeNull(because: "metadata should be set on failure results"); - result.Metadata!.AppExists.Should().BeFalse(because: "app does not exist in Entra"); + var meta = (RequirementCheckMetadata)result.Metadata!; + meta.AppExists.Should().BeFalse(because: "app does not exist in Entra"); } [Fact] @@ -470,8 +472,9 @@ public async Task CheckAsync_NoServicePrincipal_MetadataHasAppTrueSpFalse() var result = await check.CheckAsync(config, _logger); result.Metadata.Should().NotBeNull(); - result.Metadata!.AppExists.Should().BeTrue(because: "app exists"); - result.Metadata.ServicePrincipalExists.Should().BeFalse(because: "SP does not exist"); + var meta = (RequirementCheckMetadata)result.Metadata!; + meta.AppExists.Should().BeTrue(because: "app exists"); + meta.ServicePrincipalExists.Should().BeFalse(because: "SP does not exist"); } [Fact] @@ -492,9 +495,10 @@ public async Task CheckAsync_RegistrationNotFound_MetadataHasRegistrationFalse() var result = await check.CheckAsync(config, _logger); result.Metadata.Should().NotBeNull(); - result.Metadata!.AppExists.Should().BeTrue(); - result.Metadata.ServicePrincipalExists.Should().BeTrue(); - result.Metadata.RegistrationExists.Should().BeFalse(because: "registration was not found"); + var meta = (RequirementCheckMetadata)result.Metadata!; + meta.AppExists.Should().BeTrue(); + meta.ServicePrincipalExists.Should().BeTrue(); + meta.RegistrationExists.Should().BeFalse(because: "registration was not found"); } [Fact] @@ -512,9 +516,10 @@ public async Task CheckAsync_Success_MetadataHasAllTrue() var result = await check.CheckAsync(config, _logger); result.Metadata.Should().NotBeNull(because: "metadata should be set on success results"); - result.Metadata!.AppExists.Should().BeTrue(); - result.Metadata.ServicePrincipalExists.Should().BeTrue(); - result.Metadata.RegistrationExists.Should().BeNull( + var meta = (RequirementCheckMetadata)result.Metadata!; + meta.AppExists.Should().BeTrue(); + meta.ServicePrincipalExists.Should().BeTrue(); + meta.RegistrationExists.Should().BeNull( because: "no registration ID was configured, so registration check was skipped"); } @@ -535,9 +540,10 @@ public async Task CheckAsync_WithPermissions_MetadataHasResourceDetails() var result = await check.CheckAsync(config, _logger); result.Metadata.Should().NotBeNull(); - result.Metadata!.ResourcePermissions.Should().NotBeNull(); + var meta = (RequirementCheckMetadata)result.Metadata!; + meta.ResourcePermissions.Should().NotBeNull(); - var graphResource = result.Metadata.ResourcePermissions! + var graphResource = meta.ResourcePermissions! .FirstOrDefault(r => r.ResourceAppId == AuthenticationConstants.MicrosoftGraphResourceAppId); graphResource.Should().NotBeNull(because: "Microsoft Graph is a baseline resource"); graphResource!.ResourceName.Should().Be("Microsoft Graph"); @@ -572,7 +578,8 @@ public async Task CheckAsync_MissingScopes_MetadataHasMissingScopesListed() var result = await check.CheckAsync(config, _logger); result.Metadata.Should().NotBeNull(); - var graphResource = result.Metadata!.ResourcePermissions! + var meta = (RequirementCheckMetadata)result.Metadata!; + var graphResource = meta.ResourcePermissions! .First(r => r.ResourceAppId == AuthenticationConstants.MicrosoftGraphResourceAppId); graphResource.MissingScopes.Should().Contain("Mail.ReadWrite", because: "Mail.ReadWrite is a baseline Graph scope not returned by Entra"); @@ -601,11 +608,12 @@ public async Task CheckAsync_ResourceNotInEntra_MetadataShowsNotConfigured() var result = await check.CheckAsync(config, _logger); result.Metadata.Should().NotBeNull(); - result.Metadata!.ResourcePermissions.Should().NotBeNull() + var meta = (RequirementCheckMetadata)result.Metadata!; + meta.ResourcePermissions.Should().NotBeNull() .And.HaveCountGreaterOrEqualTo(BlueprintRegistrationRequirementCheck.BaselinePermissions.Count, because: "all baseline resources should appear in metadata even when missing from Entra"); - var graphResource = result.Metadata.ResourcePermissions! + var graphResource = meta.ResourcePermissions! .First(r => r.ResourceAppId == AuthenticationConstants.MicrosoftGraphResourceAppId); graphResource.InheritablePermissionsConfigured.Should().BeFalse( because: "the resource was not found in Entra at all"); @@ -688,7 +696,8 @@ public async Task CheckAsync_EffectiveInheritance_MetadataReflectsStatus() var result = await check.CheckAsync(config, _logger); result.Metadata.Should().NotBeNull(); - var graphResource = result.Metadata!.ResourcePermissions! + var meta = (RequirementCheckMetadata)result.Metadata!; + var graphResource = meta.ResourcePermissions! .First(r => r.ResourceAppId == AuthenticationConstants.MicrosoftGraphResourceAppId); graphResource.ScopesAllAllowed.Should().BeTrue( because: "scopes kind=allAllowed was configured"); diff --git a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/ConversationRequirementCheckTests.cs b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/ConversationRequirementCheckTests.cs index 8d997461..710e9014 100644 --- a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/ConversationRequirementCheckTests.cs +++ b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/ConversationRequirementCheckTests.cs @@ -10,6 +10,7 @@ using Microsoft.Agents.A365.DevTools.Cli.Services; using Microsoft.Agents.A365.DevTools.Cli.Services.Requirements; using Microsoft.Agents.A365.DevTools.Cli.Services.Requirements.RequirementChecks; +using Microsoft.Agents.A365.DevTools.Validation; using Microsoft.Extensions.Logging; using NSubstitute; using Xunit; @@ -132,8 +133,9 @@ public async Task CheckAsync_WhenAllTurnsSucceed_ReturnsSuccess() result.Passed.Should().BeTrue(because: "all conversation turns returned 200"); result.Details.Should().Contain("3/3 turns succeeded"); result.Metadata.Should().NotBeNull(); - result.Metadata!.Turns.Should().HaveCount(3); - result.Metadata.Turns!.Should().OnlyContain(t => t.Ok, because: "every turn should report success"); + var meta = (RequirementCheckMetadata)result.Metadata!; + meta.Turns.Should().HaveCount(3); + meta.Turns!.Should().OnlyContain(t => t.Ok, because: "every turn should report success"); } [Fact] @@ -152,7 +154,7 @@ public async Task CheckAsync_WhenTurnReturnsAuthFailure_ReturnsFailureWithGuidan var result = await check.CheckAsync(config, _logger); result.Passed.Should().BeFalse(because: "auth failures should block the conversation tier"); - result.Metadata!.Turns!.Should().Contain(t => t.Error != null && t.Error.Contains("Auth rejected"), + ((RequirementCheckMetadata)result.Metadata!).Turns!.Should().Contain(t => t.Error != null && t.Error.Contains("Auth rejected"), because: "auth failures should report targeted guidance"); } @@ -172,7 +174,7 @@ public async Task CheckAsync_WhenTurnReturns500_ReturnsFailure() var result = await check.CheckAsync(config, _logger); result.Passed.Should().BeFalse(because: "server errors should fail the conversation tier"); - result.Metadata!.Turns!.Should().Contain(t => !t.Ok); + ((RequirementCheckMetadata)result.Metadata!).Turns!.Should().Contain(t => !t.Ok); } [Fact] @@ -235,7 +237,7 @@ public async Task CheckAsync_SuccessfulTurns_IncludeLatencyAndSnippet() var result = await check.CheckAsync(config, _logger); result.Passed.Should().BeTrue(); - var turn = result.Metadata!.Turns!.First(); + var turn = ((RequirementCheckMetadata)result.Metadata!).Turns!.First(); turn.LatencyMs.Should().NotBeNull(because: "latency should always be captured"); turn.StatusCode.Should().Be(200); turn.ResponseSnippet.Should().Contain("I can help you"); @@ -286,7 +288,7 @@ public async Task CheckAsync_ContinuesAllTurnsEvenOnFailure() var result = await check.CheckAsync(config, _logger); result.Passed.Should().BeFalse(because: "at least one turn failed"); - result.Metadata!.Turns.Should().HaveCount(3, + ((RequirementCheckMetadata)result.Metadata!).Turns.Should().HaveCount(3, because: "all turns should be attempted even if one fails for complete reporting"); } @@ -309,10 +311,11 @@ public async Task CheckAsync_WhenCallbackReceiverProvided_TracksAgentResponded() var result = await check.CheckAsync(config, _logger); result.Passed.Should().BeTrue(because: "all turns succeeded with agent responses"); - result.Metadata!.Turns.Should().OnlyContain( + var meta = (RequirementCheckMetadata)result.Metadata!; + meta.Turns.Should().OnlyContain( t => t.AgentResponded == true, because: "callback receiver reported agent responses for every turn"); - result.Metadata.Turns.Should().OnlyContain( + meta.Turns.Should().OnlyContain( t => t.AgentResponseText == "I can help you with that!", because: "agent response text should be captured from callback"); } @@ -335,10 +338,11 @@ public async Task CheckAsync_WhenAgentDoesNotRespond_ReturnsFailure() result.Passed.Should().BeFalse( because: "in non-playground mode, agent must respond for turn to pass"); - result.Metadata!.Turns.Should().OnlyContain( + var meta = (RequirementCheckMetadata)result.Metadata!; + meta.Turns.Should().OnlyContain( t => t.AgentResponded == false, because: "callback receiver returned no response"); - result.Metadata.Turns.Should().OnlyContain( + meta.Turns.Should().OnlyContain( t => t.Ok == false && t.Error!.Contains("did not respond"), because: "each turn should report agent did not respond"); } @@ -362,7 +366,7 @@ public async Task CheckAsync_WhenNoCallbackReceiverInjected_AutoCreatesReceiverA result.Passed.Should().BeFalse( because: "auto-created receiver gets no callback from mock handler so turns fail"); - result.Metadata!.Turns.Should().OnlyContain( + ((RequirementCheckMetadata)result.Metadata!).Turns.Should().OnlyContain( t => t.AgentResponded == false, because: "auto-created receiver gets no callback from mock handler so agentResponded is false"); } @@ -386,7 +390,7 @@ public async Task CheckAsync_WhenAgentReturnsErrorResponse_ReturnsFailure() result.Passed.Should().BeFalse( because: "agent responded with an error message"); - result.Metadata!.Turns.Should().OnlyContain( + ((RequirementCheckMetadata)result.Metadata!).Turns.Should().OnlyContain( t => t.Ok == false && t.Error!.Contains("error response"), because: "each turn should report agent returned an error"); } @@ -411,7 +415,7 @@ public async Task CheckAsync_InPlaygroundMode_PassesEvenWithoutAgentResponse() result.Passed.Should().BeTrue( because: "in playground mode, missing agent response does not fail the turn"); - result.Metadata!.Turns.Should().OnlyContain( + ((RequirementCheckMetadata)result.Metadata!).Turns.Should().OnlyContain( t => t.Ok == true, because: "playground mode is lenient about agent responses"); } @@ -455,7 +459,7 @@ public async Task CheckAsync_WhenAuthFailure_AgentRespondedIsFalse() var result = await check.CheckAsync(config, _logger); result.Passed.Should().BeFalse(); - result.Metadata!.Turns.Should().OnlyContain( + ((RequirementCheckMetadata)result.Metadata!).Turns.Should().OnlyContain( t => t.AgentResponded == false, because: "auth failures should report agent did not respond, not attempt callback wait"); } @@ -504,7 +508,7 @@ public async Task CheckAsync_WhenPlaygroundEnabled_LaunchesAgentsPlayground() var result = await check.CheckAsync(config, _logger); result.Passed.Should().BeTrue(); - result.Metadata!.PlaygroundLaunched.Should().BeTrue( + ((RequirementCheckMetadata)result.Metadata!).PlaygroundLaunched.Should().BeTrue( because: "playground was requested and started successfully"); _processService.Received(2).Start(Arg.Any()); _processService.Received(1).Start(Arg.Is(p => @@ -529,7 +533,7 @@ public async Task CheckAsync_WhenPlaygroundDisabled_DoesNotLaunchPlayground() var result = await check.CheckAsync(config, _logger); result.Passed.Should().BeTrue(); - result.Metadata!.PlaygroundLaunched.Should().BeNull( + ((RequirementCheckMetadata)result.Metadata!).PlaygroundLaunched.Should().BeNull( because: "playground was not requested"); // Only one Start call for the agent process _processService.Received(1).Start(Arg.Any()); @@ -557,7 +561,7 @@ public async Task CheckAsync_WhenPlaygroundFailsToStart_ReturnsSuccessWithoutPla result.Passed.Should().BeTrue( because: "playground failure should not block conversation validation"); - result.Metadata!.PlaygroundLaunched.Should().BeNull( + ((RequirementCheckMetadata)result.Metadata!).PlaygroundLaunched.Should().BeNull( because: "playground failed to start so it should not be reported as launched"); } diff --git a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/HttpListenerBotCallbackReceiverTests.cs b/src/Tests/Microsoft.Agents.A365.DevTools.Validation.Tests/Validation/HttpListenerBotCallbackReceiverTests.cs similarity index 94% rename from src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/HttpListenerBotCallbackReceiverTests.cs rename to src/Tests/Microsoft.Agents.A365.DevTools.Validation.Tests/Validation/HttpListenerBotCallbackReceiverTests.cs index 5fd6bac8..e2c4eea8 100644 --- a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/HttpListenerBotCallbackReceiverTests.cs +++ b/src/Tests/Microsoft.Agents.A365.DevTools.Validation.Tests/Validation/HttpListenerBotCallbackReceiverTests.cs @@ -2,9 +2,10 @@ // Licensed under the MIT License. using FluentAssertions; -using Microsoft.Agents.A365.DevTools.Cli.Services; +using Microsoft.Agents.A365.DevTools.Validation; +using Xunit; -namespace Microsoft.Agents.A365.DevTools.Cli.Tests.Services; +namespace Microsoft.Agents.A365.DevTools.Validation.Tests; public class HttpListenerBotCallbackReceiverTests { diff --git a/src/a365.config.example.json b/src/a365.config.example.json index 2b92a710..5b5bd051 100644 --- a/src/a365.config.example.json +++ b/src/a365.config.example.json @@ -5,18 +5,13 @@ "resourceGroup": "your-resource-group-name", "location": "westus", "environment": "preprod", - "agent365ObservabilityMcpOptions": { - "baseUrl": "https://your-observability-host", - "agentObservabilityId": "11111111-2222-3333-4444-555555555555", - "appId": "42edbbd6-cfc9-4637-9068-ebac6df46171" - }, "appServicePlanName": "your-app-service-plan-name", "appServicePlanSku": "B1", "webAppName": "your-unique-webapp-name", - "agentIdentityDisplayName": "Your Agent Identity Display Name", - "agentBlueprintDisplayName": "Your Blueprint App Display Name", + "agentIdentityDisplayName": "Your Agent Display Name", + "agentBlueprintDisplayName": "Your Blueprint Display Name", "agentUserPrincipalName": "agentuser@yourdomain.onmicrosoft.com", - "agentUserDisplayName": "Your Agent User Account Display Name", + "agentUserDisplayName": "Your Agent User Display Name", "managerEmail": "manager@yourdomain.onmicrosoft.com", "agentUserUsageLocation": "US", "deploymentProjectPath": "/path/to/your/agent/project", From 88c25e181b25ff445a9115b26ed52aff155ffd56 Mon Sep 17 00:00:00 2001 From: Abbinayaa Subramanian Date: Thu, 11 Jun 2026 16:44:03 -0700 Subject: [PATCH 11/27] Removing interim message dependency --- .../HttpListenerBotCallbackReceiver.cs | 166 +++--------------- .../HttpListenerBotCallbackReceiverTests.cs | 100 +++-------- 2 files changed, 47 insertions(+), 219 deletions(-) diff --git a/src/Microsoft.Agents.A365.DevTools.Validation/HttpListenerBotCallbackReceiver.cs b/src/Microsoft.Agents.A365.DevTools.Validation/HttpListenerBotCallbackReceiver.cs index 8d69c796..06a63658 100644 --- a/src/Microsoft.Agents.A365.DevTools.Validation/HttpListenerBotCallbackReceiver.cs +++ b/src/Microsoft.Agents.A365.DevTools.Validation/HttpListenerBotCallbackReceiver.cs @@ -23,39 +23,16 @@ public sealed class HttpListenerBotCallbackReceiver : IBotCallbackReceiver private Task? _listenTask; /// - /// After a non-interim callback arrives, continue collecting for this long to capture - /// any follow-up responses. + /// After each received message, wait this long for another message before settling. + /// Each new arrival resets this timer, so fast bursts are collected together. /// - internal static readonly TimeSpan GracePeriod = TimeSpan.FromSeconds(5); + internal static readonly TimeSpan SettlePeriod = TimeSpan.FromSeconds(5); /// - /// Extended timeout used when only interim/acknowledgment responses have been received. - /// Agents that send "Got it..working on it" often need 10-30s to produce the real response. + /// Maximum total time to collect messages after the first one arrives. + /// Prevents indefinite waiting when an agent keeps sending messages. /// - internal static readonly TimeSpan InterimExtendedTimeout = TimeSpan.FromSeconds(30); - - /// - /// Patterns that indicate an interim/acknowledgment message rather than a real response. - /// These are anchored to avoid matching error messages (e.g. "processing failed" is NOT interim). - /// - internal static readonly string[] InterimPatterns = new[] - { - "got it", - "working on", - "work on it", - "processing your", - "thinking", - "one moment", - "please wait", - "hold on", - "looking into", - "let me check", - "let me look", - "let me find", - "let me get", - "just a moment", - "just a sec", - }; + internal static readonly TimeSpan MaxCollectionWindow = TimeSpan.FromSeconds(30); public string ServiceUrl => $"http://localhost:{_port}"; @@ -91,82 +68,32 @@ public Task StartAsync(CancellationToken cancellationToken = default) using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); timeoutCts.CancelAfter(timeout); - bool receivedInterim = false; - try { // Wait for the first callback activity await _responseReceived.WaitAsync(timeoutCts.Token); - // Check if the first response is an interim message - bool firstIsFinal; - lock (_lock) - { - var latest = _responses.Count > 0 ? _responses[^1] : null; - firstIsFinal = latest is not null && IsFinalMessage(latest); - if (!firstIsFinal) - { - receivedInterim = true; - } - } + // First message received — start the collection window. + // Each new message resets the settle timer; collection caps at MaxCollectionWindow. + var collectionDeadline = DateTime.UtcNow + MaxCollectionWindow; - if (receivedInterim) + while (DateTime.UtcNow < collectionDeadline && !timeoutCts.Token.IsCancellationRequested) { - // Interim message received — agent is alive but still processing. - // Extend the timeout to allow the real response to arrive. - var extendedDeadline = DateTime.UtcNow + InterimExtendedTimeout; + var untilDeadline = collectionDeadline - DateTime.UtcNow; + var settleWait = untilDeadline < SettlePeriod ? untilDeadline : SettlePeriod; - while (DateTime.UtcNow < extendedDeadline && !timeoutCts.Token.IsCancellationRequested) - { - var remaining = extendedDeadline - DateTime.UtcNow; - if (remaining <= TimeSpan.Zero) - { - break; - } - - try - { - if (!await _responseReceived.WaitAsync(remaining, timeoutCts.Token)) - { - break; - } - - // Check if we now have a final response - lock (_lock) - { - var latest = _responses.Count > 0 ? _responses[^1] : null; - if (latest is not null && IsFinalMessage(latest)) - { - // Got a real response — start the short grace period for any follow-ups - break; - } - } - } - catch (OperationCanceledException) - { - break; - } - } - } - - // Grace period: collect any remaining follow-up responses - var graceDeadline = DateTime.UtcNow + GracePeriod; - - while (DateTime.UtcNow < graceDeadline && !timeoutCts.Token.IsCancellationRequested) - { - var remaining = graceDeadline - DateTime.UtcNow; - if (remaining <= TimeSpan.Zero) + if (settleWait <= TimeSpan.Zero) { break; } try { - if (!await _responseReceived.WaitAsync(remaining, timeoutCts.Token)) + if (!await _responseReceived.WaitAsync(settleWait, timeoutCts.Token)) { - break; // No more responses within grace period + break; // No new message within settle period — done collecting } - // Got another response — continue collecting + // New message arrived — loop resets the settle timer } catch (OperationCanceledException) { @@ -271,8 +198,8 @@ private async Task HandleRequestAsync(HttpListenerContext context) /// /// Selects the best response from collected callbacks. - /// Returns null if only interim/typing responses were collected (agent did not produce a final answer). - /// Prefers the last message-type response with substantive non-interim text. + /// Returns the last message-type response with text, skipping typing indicators. + /// Returns null if no substantive message was received. /// private BotCallbackResponse? SelectBestResponse() { @@ -281,69 +208,20 @@ private async Task HandleRequestAsync(HttpListenerContext context) return null; } - // Prefer the last final message (non-interim, non-typing, with substantive text) - var bestMessage = _responses - .LastOrDefault(r => IsFinalMessage(r)); - - if (bestMessage is not null) - { - return bestMessage; - } - - // No final message found — all responses were interim or typing. - // Return null so the caller treats this as "agent did not respond with a final answer". - return null; + return _responses.LastOrDefault(r => IsSubstantiveMessage(r)); } /// - /// Determines whether a response is a final (non-interim) message from the agent. + /// Determines whether a response is a substantive message (not a typing indicator). /// - public static bool IsFinalMessage(BotCallbackResponse response) + public static bool IsSubstantiveMessage(BotCallbackResponse response) { - // Typing activities are never final if (string.Equals(response.Type, "typing", StringComparison.OrdinalIgnoreCase)) { return false; } - // Must be a message with text - if (response.Type != "message" || string.IsNullOrWhiteSpace(response.Text)) - { - return false; - } - - return !IsInterimMessage(response.Text); - } - - /// - /// Detects interim/acknowledgment messages that agents send while processing. - /// Only matches short messages (under 60 chars) containing known interim phrases. - /// Longer messages are assumed to be real responses even if they contain interim-like words. - /// - public static bool IsInterimMessage(string? text) - { - if (string.IsNullOrWhiteSpace(text)) - { - return false; - } - - // Long messages are unlikely to be interim acknowledgments - if (text.Length > 60) - { - return false; - } - - var lower = text.ToLowerInvariant(); - - foreach (var pattern in InterimPatterns) - { - if (lower.Contains(pattern, StringComparison.Ordinal)) - { - return true; - } - } - - return false; + return response.Type == "message" && !string.IsNullOrWhiteSpace(response.Text); } public async ValueTask DisposeAsync() diff --git a/src/Tests/Microsoft.Agents.A365.DevTools.Validation.Tests/Validation/HttpListenerBotCallbackReceiverTests.cs b/src/Tests/Microsoft.Agents.A365.DevTools.Validation.Tests/Validation/HttpListenerBotCallbackReceiverTests.cs index e2c4eea8..44e0f041 100644 --- a/src/Tests/Microsoft.Agents.A365.DevTools.Validation.Tests/Validation/HttpListenerBotCallbackReceiverTests.cs +++ b/src/Tests/Microsoft.Agents.A365.DevTools.Validation.Tests/Validation/HttpListenerBotCallbackReceiverTests.cs @@ -9,103 +9,53 @@ namespace Microsoft.Agents.A365.DevTools.Validation.Tests; public class HttpListenerBotCallbackReceiverTests { - [Theory] - [InlineData("Got it..work on it")] - [InlineData("Got it - working on it...")] - [InlineData("Working on your request")] - [InlineData("Thinking...")] - [InlineData("One moment please")] - [InlineData("Please wait...")] - [InlineData("Hold on, let me check")] - [InlineData("Let me check that for you")] - [InlineData("Let me look into that")] - [InlineData("Let me find that information")] - [InlineData("Let me get that for you")] - [InlineData("Just a moment...")] - [InlineData("Just a sec")] - [InlineData("Looking into it")] - [InlineData("Processing your request...")] - public void IsInterimMessage_KnownInterimPatterns_ReturnsTrue(string text) - { - HttpListenerBotCallbackReceiver.IsInterimMessage(text) - .Should().BeTrue(because: "'{0}' is an interim acknowledgment, not a final response", text); - } - - [Theory] - [InlineData("Here are your recent emails: 1. Meeting reminder from John...")] - [InlineData("I found 3 documents matching your search criteria. The first one is about project planning and was created last week.")] - [InlineData("Processing failed due to an authentication error")] - [InlineData("I can help you with email, calendar, and file management")] - [InlineData("Hello! How can I help you today?")] - [InlineData("The meeting is scheduled for 3 PM tomorrow")] - [InlineData("Error: Unable to connect to the service")] - public void IsInterimMessage_RealResponses_ReturnsFalse(string text) - { - HttpListenerBotCallbackReceiver.IsInterimMessage(text) - .Should().BeFalse(because: "'{0}' is a real response, not an interim acknowledgment", text); - } - - [Theory] - [InlineData(null)] - [InlineData("")] - [InlineData(" ")] - public void IsInterimMessage_NullOrWhitespace_ReturnsFalse(string? text) - { - HttpListenerBotCallbackReceiver.IsInterimMessage(text) - .Should().BeFalse(because: "null/empty text is not an interim message"); - } - [Fact] - public void IsInterimMessage_LongTextWithInterimWord_ReturnsFalse() - { - var longText = "I'm working on analyzing your data. Here are the preliminary results that I found across multiple sources in your organization's SharePoint sites."; - - HttpListenerBotCallbackReceiver.IsInterimMessage(longText) - .Should().BeFalse(because: "messages over 60 chars are treated as real responses even if they contain interim-like words"); - } - - [Fact] - public void IsFinalMessage_TypingActivity_ReturnsFalse() + public void IsSubstantiveMessage_TypingActivity_ReturnsFalse() { var response = new BotCallbackResponse("typing indicator", "typing"); - HttpListenerBotCallbackReceiver.IsFinalMessage(response) - .Should().BeFalse(because: "typing activities are never final responses"); + HttpListenerBotCallbackReceiver.IsSubstantiveMessage(response) + .Should().BeFalse(because: "typing activities are never substantive responses"); } - [Fact] - public void IsFinalMessage_MessageWithSubstantiveText_ReturnsTrue() + [Theory] + [InlineData("Here are your recent emails from today")] + [InlineData("Got it..working on it")] + [InlineData("Thinking...")] + [InlineData("Hello! How can I help you today?")] + public void IsSubstantiveMessage_MessageWithText_ReturnsTrue(string text) { - var response = new BotCallbackResponse("Here are your recent emails from today", "message"); + var response = new BotCallbackResponse(text, "message"); - HttpListenerBotCallbackReceiver.IsFinalMessage(response) - .Should().BeTrue(because: "a message with substantive non-interim text is a final response"); + HttpListenerBotCallbackReceiver.IsSubstantiveMessage(response) + .Should().BeTrue( + because: "any message with text is substantive — interim detection is handled by the settle window"); } [Fact] - public void IsFinalMessage_InterimMessage_ReturnsFalse() + public void IsSubstantiveMessage_MessageWithNullText_ReturnsFalse() { - var response = new BotCallbackResponse("Got it..work on it", "message"); + var response = new BotCallbackResponse(null, "message"); - HttpListenerBotCallbackReceiver.IsFinalMessage(response) - .Should().BeFalse(because: "interim acknowledgment messages are not final responses"); + HttpListenerBotCallbackReceiver.IsSubstantiveMessage(response) + .Should().BeFalse(because: "a message with no text is not substantive"); } [Fact] - public void IsFinalMessage_MessageWithNullText_ReturnsFalse() + public void IsSubstantiveMessage_NullType_WithText_ReturnsFalse() { - var response = new BotCallbackResponse(null, "message"); + var response = new BotCallbackResponse("some text", null); - HttpListenerBotCallbackReceiver.IsFinalMessage(response) - .Should().BeFalse(because: "a message with no text is not a final response"); + HttpListenerBotCallbackReceiver.IsSubstantiveMessage(response) + .Should().BeFalse(because: "only message-type activities are substantive"); } [Fact] - public void IsFinalMessage_NullType_WithText_ReturnsFalse() + public void IsSubstantiveMessage_MessageWithWhitespaceText_ReturnsFalse() { - var response = new BotCallbackResponse("some text", null); + var response = new BotCallbackResponse(" ", "message"); - HttpListenerBotCallbackReceiver.IsFinalMessage(response) - .Should().BeFalse(because: "only message-type activities can be final responses"); + HttpListenerBotCallbackReceiver.IsSubstantiveMessage(response) + .Should().BeFalse(because: "whitespace-only text is not substantive"); } } From 0b268bdc9b5d1d807f48147f26a7689e9ce64d1e Mon Sep 17 00:00:00 2001 From: Abbinayaa Subramanian Date: Thu, 11 Jun 2026 17:05:57 -0700 Subject: [PATCH 12/27] Fixing requirement check log levels --- .../Commands/ValidateCommand.cs | 7 ++++++- .../Services/Requirements/RequirementCheck.cs | 14 ++++++++------ 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Commands/ValidateCommand.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Commands/ValidateCommand.cs index abead4f9..65486cd4 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Commands/ValidateCommand.cs +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Commands/ValidateCommand.cs @@ -8,6 +8,7 @@ using Microsoft.Agents.A365.DevTools.Cli.Services.Requirements; using Microsoft.Agents.A365.DevTools.Cli.Services.Requirements.RequirementChecks; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; using System.CommandLine; using System.CommandLine.Invocation; using System.Text.Json; @@ -296,9 +297,13 @@ public static Command CreateCommand( logger.LogDebug("Checking requirements..."); + // Pass NullLogger to checks to suppress their built-in logging. + // Validate handles its own structured output via the report and PrintSummary. + var checkLogger = NullLogger.Instance; + foreach (var check in checks) { - var result = await check.CheckAsync(config, logger, ct); + var result = await check.CheckAsync(config, checkLogger, ct); results.Add((check, result)); } diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementCheck.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementCheck.cs index 0d0dae90..0ee8fac7 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementCheck.cs +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementCheck.cs @@ -28,7 +28,7 @@ public abstract class RequirementCheck : IRequirementCheck /// protected virtual void LogCheckSuccess(ILogger logger, string? details = null) { - logger.LogDebug("Pass: {Name}{Details}", Name, + logger.LogInformation("Pass: {Name}{Details}", Name, string.IsNullOrWhiteSpace(details) ? "" : $" ({details})"); } @@ -37,7 +37,7 @@ protected virtual void LogCheckSuccess(ILogger logger, string? details = null) /// protected virtual void LogCheckWarning(ILogger logger, string? message = null) { - logger.LogDebug("Warn: {Name}{Details}", Name, + logger.LogWarning("Warn: {Name}{Details}", Name, string.IsNullOrWhiteSpace(message) ? "" : $" - {message}"); } @@ -46,16 +46,18 @@ protected virtual void LogCheckWarning(ILogger logger, string? message = null) /// protected virtual void LogCheckFailure(ILogger logger, string errorMessage, string resolutionGuidance) { - logger.LogDebug("Fail: {Name}", Name); + // Single red line — AZ CLI convention: one ERROR line per failure, not per detail + logger.LogError("Fail: {Name}", Name); + // Error details and resolution guidance in white — they describe and guide, not error foreach (var line in errorMessage.Split('\n', StringSplitOptions.RemoveEmptyEntries)) - logger.LogDebug(" {Line}", line.TrimEnd()); + logger.LogInformation(" {Line}", line.TrimEnd()); if (!string.IsNullOrWhiteSpace(resolutionGuidance)) { - logger.LogDebug(""); + logger.LogInformation(""); foreach (var step in resolutionGuidance.Split('\n', StringSplitOptions.RemoveEmptyEntries)) - logger.LogDebug(" {Step}", step.TrimEnd()); + logger.LogInformation(" {Step}", step.TrimEnd()); } } From f8ef5138f041c2b8d88adb1b4f5498abc9bc9986 Mon Sep 17 00:00:00 2001 From: Abbinayaa Subramanian Date: Mon, 15 Jun 2026 16:14:22 -0700 Subject: [PATCH 13/27] Adding bearer token check for tool validation --- .../Commands/ValidateCommand.cs | 36 ++- .../BearerTokenRequirementCheck.cs | 175 +++++++++++++ .../ConversationRequirementCheck.cs | 2 +- .../LocalRuntimeRequirementCheck.cs | 109 +++++++- .../TelemetryRequirementCheck.cs | 40 +-- .../BearerTokenRequirementCheckTests.cs | 241 ++++++++++++++++++ .../LocalRuntimeRequirementCheckTests.cs | 50 +++- .../TelemetryRequirementCheckTests.cs | 30 +-- 8 files changed, 606 insertions(+), 77 deletions(-) create mode 100644 src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/BearerTokenRequirementCheck.cs create mode 100644 src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/BearerTokenRequirementCheckTests.cs diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Commands/ValidateCommand.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Commands/ValidateCommand.cs index 65486cd4..1926924b 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Commands/ValidateCommand.cs +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Commands/ValidateCommand.cs @@ -91,21 +91,23 @@ public static Command CreateCommand( // --- Run all checks --- - // Phase 2: Run structural checks (manifest + build) + // Phase 2: Run structural checks (manifest + bearer token + build) var structuralChecks = requirementChecksOverride?.ToList() ?? BuildStructuralChecks(platformDetector, commandExecutor); var results = await RunChecksDetailedAsync(structuralChecks, config, logger, ct); MapResultsToTiers(results, report); + var structuralPassed = report.Tiers.Structural is { Skipped: true } or { Ok: true }; + // Extract resolved uv command from build step for boot and conversation steps var buildResultEntry = results .FirstOrDefault(r => r.Check is ProjectBuildRequirementCheck); var resolvedUvCommand = (buildResultEntry.Result?.Metadata as RequirementCheckMetadata)?.ResolvedUvCommand; - // Phase 2b: Run boot check only if build passed + // Phase 2b: Run boot check only if structural and build passed var buildPassed = report.Tiers.Build is { Skipped: true } or { Ok: true }; - if (buildPassed && requirementChecksOverride is null) + if (structuralPassed && buildPassed && requirementChecksOverride is null) { var bootChecks = BuildBootChecks(platformDetector, processService, resolvedUvCommand); if (bootChecks.Count > 0) @@ -115,12 +117,13 @@ public static Command CreateCommand( results.AddRange(bootResults); } } - else if (!buildPassed) + else if (!structuralPassed || !buildPassed) { + var skipReason = !structuralPassed ? "structural checks failed" : "build failed"; report.Tiers.Boot = new BootTierResult { Skipped = true, - Reason = "build failed" + Reason = skipReason }; } @@ -149,7 +152,7 @@ public static Command CreateCommand( } else if (!bootPassed) { - var skipReason = report.Tiers.Boot is { Skipped: true } ? "build failed" : "boot tier failed"; + var skipReason = report.Tiers.Boot?.Reason ?? "boot tier failed"; report.Tiers.Conversation = new ConversationTierResult { Skipped = true, @@ -326,6 +329,7 @@ private static void MapResultsToTiers( switch (check) { case ToolingManifestRequirementCheck: + case BearerTokenRequirementCheck: // Add to structural tier var structural = report.Tiers.Structural; if (structural.Skipped) @@ -334,9 +338,15 @@ private static void MapResultsToTiers( report.Tiers.Structural = structural; } structural.Checks ??= new List(); + var checkName = check switch + { + ToolingManifestRequirementCheck => "tooling-manifest", + BearerTokenRequirementCheck => "bearer-token", + _ => check.Name.ToLowerInvariant().Replace(' ', '-') + }; structural.Checks.Add(new StructuralCheck { - Name = "tooling-manifest", + Name = checkName, Ok = result.Passed, Message = result.Passed ? result.Details : result.ErrorMessage }); @@ -520,7 +530,12 @@ private static (string Description, string Suggestion) GetCodeHealthFailureInfo( var desc = failedChecks is { Count: > 0 } ? $"failed: {string.Join(", ", failedChecks)}" : "structural checks failed"; - return (desc, "fix project structure issues and re-run `a365 validate`"); + + var suggestion = failedChecks?.Contains("bearer-token") == true + ? "run `a365 develop get-token` to retrieve a bearer token and add it to your launch settings" + : "fix project structure issues and re-run `a365 validate`"; + + return (desc, suggestion); } return ("code health check failed", "fix errors and re-run `a365 validate`"); @@ -804,6 +819,11 @@ private static List BuildStructuralChecks( new ToolingManifestRequirementCheck() }; + if (platformDetector is not null) + { + checks.Add(new BearerTokenRequirementCheck(platformDetector)); + } + if (platformDetector is not null && commandExecutor is not null) { checks.Add(new ProjectBuildRequirementCheck(platformDetector, commandExecutor)); diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/BearerTokenRequirementCheck.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/BearerTokenRequirementCheck.cs new file mode 100644 index 00000000..75e6e966 --- /dev/null +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/BearerTokenRequirementCheck.cs @@ -0,0 +1,175 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Text.Json; +using Microsoft.Agents.A365.DevTools.Cli.Constants; +using Microsoft.Agents.A365.DevTools.Cli.Models; +using Microsoft.Extensions.Logging; + +namespace Microsoft.Agents.A365.DevTools.Cli.Services.Requirements.RequirementChecks; + +/// +/// Validates that a bearer token is configured in the agent's launch settings. +/// Checks launchSettings.json (for .NET) or .env files (for Node.js/Python) +/// for the BEARER_TOKEN environment variable required for MCP tool authentication. +/// +public class BearerTokenRequirementCheck : RequirementCheck +{ + private readonly PlatformDetector _platformDetector; + + public BearerTokenRequirementCheck(PlatformDetector platformDetector) + { + _platformDetector = platformDetector; + } + + /// + public override string Name => "Bearer Token"; + + /// + public override string Description => "Validates that a bearer token is configured for MCP tool authentication"; + + /// + public override string Category => "Configuration"; + + /// + public override async Task CheckAsync( + Agent365Config config, + ILogger logger, + CancellationToken cancellationToken = default) + { + return await ExecuteCheckWithLoggingAsync(config, logger, CheckImplementationAsync, cancellationToken); + } + + private Task CheckImplementationAsync( + Agent365Config config, + ILogger logger, + CancellationToken cancellationToken) + { + var projectPath = ResolveProjectPath(config); + + if (!Directory.Exists(projectPath)) + { + return Task.FromResult(RequirementCheckResult.Failure( + $"Project path does not exist: {projectPath}", + "Ensure the project directory exists, or set deploymentProjectPath in a365.config.json")); + } + + var platform = _platformDetector.Detect(projectPath); + + var tokenEnvVar = AuthenticationConstants.BearerTokenEnvironmentVariable; + + return platform switch + { + ProjectPlatform.DotNet => Task.FromResult(CheckLaunchSettings(projectPath, tokenEnvVar)), + ProjectPlatform.NodeJs => Task.FromResult(CheckEnvFile(projectPath, tokenEnvVar)), + ProjectPlatform.Python => Task.FromResult(CheckEnvFile(projectPath, tokenEnvVar)), + _ => Task.FromResult( + CheckLaunchSettings(projectPath, tokenEnvVar) is { Passed: true } launchResult + ? launchResult + : CheckEnvFile(projectPath, tokenEnvVar)) + }; + } + + /// + /// Checks Properties/launchSettings.json for the bearer token in environmentVariables. + /// + internal static RequirementCheckResult CheckLaunchSettings(string projectPath, string envVarName) + { + var launchSettingsPath = Path.Combine(projectPath, "Properties", "launchSettings.json"); + if (!File.Exists(launchSettingsPath)) + { + return RequirementCheckResult.Failure( + "No launchSettings.json found", + $"Create Properties/launchSettings.json with {envVarName} in environmentVariables. " + + $"Run 'a365 develop get-token' to retrieve a token."); + } + + try + { + var json = File.ReadAllText(launchSettingsPath); + using var doc = JsonDocument.Parse(json); + + if (!doc.RootElement.TryGetProperty("profiles", out var profiles)) + { + return RequirementCheckResult.Failure( + "launchSettings.json has no profiles section", + $"Add a profile with {envVarName} in environmentVariables."); + } + + foreach (var profile in profiles.EnumerateObject()) + { + if (profile.Value.TryGetProperty("environmentVariables", out var envVars) && + envVars.TryGetProperty(envVarName, out var tokenValue)) + { + var token = tokenValue.GetString(); + if (!string.IsNullOrWhiteSpace(token)) + { + return RequirementCheckResult.Success( + details: $"Found {envVarName} in launchSettings.json profile '{profile.Name}'"); + } + } + } + } + catch (JsonException) + { + return RequirementCheckResult.Failure( + "launchSettings.json is not valid JSON", + "Fix the JSON syntax in Properties/launchSettings.json."); + } + + return RequirementCheckResult.Failure( + $"{envVarName} not found in launchSettings.json", + $"Add {envVarName} to environmentVariables in a launchSettings.json profile. " + + $"Run 'a365 develop get-token' to retrieve a token."); + } + + /// + /// Checks .env file for the bearer token variable. + /// + internal static RequirementCheckResult CheckEnvFile(string projectPath, string envVarName) + { + var envPath = Path.Combine(projectPath, ".env"); + if (!File.Exists(envPath)) + { + return RequirementCheckResult.Failure( + "No .env file found", + $"Create a .env file with {envVarName}=. " + + $"Run 'a365 develop get-token' to retrieve a token."); + } + + try + { + foreach (var line in File.ReadLines(envPath)) + { + var trimmed = line.Trim(); + if (trimmed.StartsWith($"{envVarName}=", StringComparison.OrdinalIgnoreCase)) + { + var value = trimmed.Substring(envVarName.Length + 1).Trim().Trim('"', '\''); + if (!string.IsNullOrWhiteSpace(value)) + { + return RequirementCheckResult.Success( + details: $"Found {envVarName} in .env file"); + } + } + } + } + catch (IOException) + { + return RequirementCheckResult.Failure( + "Could not read .env file", + "Ensure the .env file is accessible and not locked by another process."); + } + + return RequirementCheckResult.Failure( + $"{envVarName} not found in .env file", + $"Add {envVarName}= to your .env file. " + + $"Run 'a365 develop get-token' to retrieve a token."); + } + + private static string ResolveProjectPath(Agent365Config config) + { + return !string.IsNullOrWhiteSpace(config.DeploymentProjectPath) + ? config.DeploymentProjectPath + : Directory.GetCurrentDirectory(); + } +} diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/ConversationRequirementCheck.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/ConversationRequirementCheck.cs index c3042b71..71de9c8a 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/ConversationRequirementCheck.cs +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/ConversationRequirementCheck.cs @@ -240,7 +240,7 @@ private async Task CheckImplementationAsync( details: $"No .NET, Node.js, or Python project detected in {projectPath}"); } - var port = LocalRuntimeRequirementCheck.ResolvePort(config.MessagingEndpoint); + var port = LocalRuntimeRequirementCheck.ResolvePort(projectPath, platform); var healthUrl = $"http://localhost:{port}{LocalRuntimeRequirementCheck.DefaultHealthPath}"; var messagesUrl = $"http://localhost:{port}/api/messages"; var conversationId = $"validate-{Guid.NewGuid():N}"; diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/LocalRuntimeRequirementCheck.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/LocalRuntimeRequirementCheck.cs index 9178be55..933785bd 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/LocalRuntimeRequirementCheck.cs +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/LocalRuntimeRequirementCheck.cs @@ -99,7 +99,7 @@ private async Task CheckImplementationAsync( details: $"No .NET, Node.js, or Python project detected in {projectPath}"); } - var port = ResolvePort(config.MessagingEndpoint); + var port = ResolvePort(projectPath, platform); var healthUrl = $"http://localhost:{port}{DefaultHealthPath}"; logger.LogDebug( @@ -111,28 +111,113 @@ private async Task CheckImplementationAsync( } /// - /// Resolves the local port from a MessagingEndpoint URL. Only uses the port when the host - /// is localhost/127.0.0.1/[::1]. Otherwise returns the default port. + /// Resolves the local port from the agent's project launch settings. + /// Checks launchSettings.json (for .NET) or .env files (for Node.js/Python). + /// Falls back to the default port when no setting is found. /// - internal static int ResolvePort(string? messagingEndpoint) + internal static int ResolvePort(string? projectPath, ProjectPlatform platform = ProjectPlatform.Unknown) { - if (string.IsNullOrWhiteSpace(messagingEndpoint)) + if (!string.IsNullOrWhiteSpace(projectPath) && Directory.Exists(projectPath)) { - return DefaultPort; + var portFromSettings = platform switch + { + ProjectPlatform.DotNet => ResolvePortFromLaunchSettings(projectPath), + ProjectPlatform.NodeJs => ResolvePortFromEnvFile(projectPath), + ProjectPlatform.Python => ResolvePortFromEnvFile(projectPath), + _ => ResolvePortFromLaunchSettings(projectPath) ?? ResolvePortFromEnvFile(projectPath) + }; + + if (portFromSettings.HasValue) + { + return portFromSettings.Value; + } + } + + return DefaultPort; + } + + /// + /// Reads the port from Properties/launchSettings.json (first profile's applicationUrl). + /// + internal static int? ResolvePortFromLaunchSettings(string projectPath) + { + var launchSettingsPath = Path.Combine(projectPath, "Properties", "launchSettings.json"); + if (!File.Exists(launchSettingsPath)) + { + return null; } - if (Uri.TryCreate(messagingEndpoint, UriKind.Absolute, out var uri)) + try { - var host = uri.Host.ToLowerInvariant(); - var isLocalhost = host is "localhost" or "127.0.0.1" or "[::1]" or "::1"; + var json = File.ReadAllText(launchSettingsPath); + using var doc = System.Text.Json.JsonDocument.Parse(json); + + if (!doc.RootElement.TryGetProperty("profiles", out var profiles)) + { + return null; + } - if (isLocalhost && !uri.IsDefaultPort) + foreach (var profile in profiles.EnumerateObject()) { - return uri.Port; + if (profile.Value.TryGetProperty("applicationUrl", out var urlProp)) + { + var urls = urlProp.GetString(); + if (string.IsNullOrWhiteSpace(urls)) + { + continue; + } + + // applicationUrl can be semicolon-separated; prefer HTTP for local validation + foreach (var url in urls.Split(';', StringSplitOptions.RemoveEmptyEntries)) + { + if (Uri.TryCreate(url.Trim(), UriKind.Absolute, out var uri) && !uri.IsDefaultPort) + { + return uri.Port; + } + } + } } } + catch + { + // Malformed launchSettings — fall through + } - return DefaultPort; + return null; + } + + /// + /// Reads the PORT variable from a .env file in the project directory. + /// + internal static int? ResolvePortFromEnvFile(string projectPath) + { + var envPath = Path.Combine(projectPath, ".env"); + if (!File.Exists(envPath)) + { + return null; + } + + try + { + foreach (var line in File.ReadLines(envPath)) + { + var trimmed = line.Trim(); + if (trimmed.StartsWith("PORT=", StringComparison.OrdinalIgnoreCase)) + { + var value = trimmed.Substring("PORT=".Length).Trim().Trim('"', '\''); + if (int.TryParse(value, out var port) && port > 0 && port <= 65535) + { + return port; + } + } + } + } + catch + { + // Malformed .env — fall through + } + + return null; } private ProcessStartInfo BuildProcessStartInfo(ProjectPlatform platform, string projectPath, int port) diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/TelemetryRequirementCheck.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/TelemetryRequirementCheck.cs index 1dbf431c..fde9559b 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/TelemetryRequirementCheck.cs +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/TelemetryRequirementCheck.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +using System.Text.RegularExpressions; using Microsoft.Agents.A365.DevTools.Cli.Models; using Microsoft.Extensions.Logging; @@ -91,6 +92,15 @@ public class TelemetryRequirementCheck : RequirementCheck "execute_tool" }; + /// + /// Matches any line containing a parent span identifier key (parentId, parentSpanId, parent_id, etc.) + /// followed by a separator and a non-empty hex value. Handles all known exporter formats: + /// Activity.ParentSpanId, JSON quoted keys, YAML-style, and equals-sign separators. + /// + internal static readonly Regex ParentSpanPattern = new( + @"(?:parent[\._]?(?:span)?[\._]?(?:id|context))\s*[=:]\s*[""']?\s*(?:0x)?([0-9a-f]{2,})", + RegexOptions.IgnoreCase | RegexOptions.Compiled); + public TelemetryRequirementCheck(string? agentConsoleLogPath) { @@ -372,19 +382,7 @@ internal static List GetChildSpansMissingParent(List> spanB if (!isChildSpan) continue; - var hasParent = block.Any(line => - { - var trimmed = line.TrimStart(); - return (trimmed.StartsWith("parentId:", StringComparison.OrdinalIgnoreCase) || - trimmed.StartsWith("parentSpanId:", StringComparison.OrdinalIgnoreCase) || - trimmed.StartsWith("parent_id:", StringComparison.OrdinalIgnoreCase) || - trimmed.StartsWith("\"parentId\":", StringComparison.OrdinalIgnoreCase) || - trimmed.StartsWith("\"parent_id\":", StringComparison.OrdinalIgnoreCase) || - trimmed.StartsWith("'parentId':", StringComparison.OrdinalIgnoreCase) || - trimmed.StartsWith("'parent_id':", StringComparison.OrdinalIgnoreCase) || - trimmed.StartsWith("parentSpanContext:", StringComparison.OrdinalIgnoreCase)) && - HasNonEmptyValue(trimmed); - }); + var hasParent = block.Any(line => ParentSpanPattern.IsMatch(line)); if (!hasParent) { @@ -398,20 +396,4 @@ internal static List GetChildSpansMissingParent(List> spanB return missingParent.OrderBy(o => o).ToList(); } - /// - /// Checks that a line like "parentId: 'abc123'" has a non-empty value after the key. - /// Returns false for lines like "parentId: undefined" or "parentId: ''". - /// - internal static bool HasNonEmptyValue(string line) - { - var colonIdx = line.IndexOf(':'); - if (colonIdx < 0) - return false; - - var value = line.Substring(colonIdx + 1).Trim().Trim('\'', '"', ',', ' '); - return !string.IsNullOrWhiteSpace(value) && - !value.Equals("undefined", StringComparison.OrdinalIgnoreCase) && - !value.Equals("null", StringComparison.OrdinalIgnoreCase); - } - } diff --git a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/BearerTokenRequirementCheckTests.cs b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/BearerTokenRequirementCheckTests.cs new file mode 100644 index 00000000..7cbe4b28 --- /dev/null +++ b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/BearerTokenRequirementCheckTests.cs @@ -0,0 +1,241 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using FluentAssertions; +using Microsoft.Agents.A365.DevTools.Cli.Constants; +using Microsoft.Agents.A365.DevTools.Cli.Models; +using Microsoft.Agents.A365.DevTools.Cli.Services; +using Microsoft.Agents.A365.DevTools.Cli.Services.Requirements.RequirementChecks; +using Microsoft.Extensions.Logging; +using NSubstitute; +using Xunit; + +namespace Microsoft.Agents.A365.DevTools.Cli.Tests.Services.Requirements; + +public class BearerTokenRequirementCheckTests : IDisposable +{ + private readonly string _tempDir; + private readonly ILogger _logger = Substitute.For(); + + public BearerTokenRequirementCheckTests() + { + _tempDir = Path.Combine(Path.GetTempPath(), $"a365-bearer-test-{Guid.NewGuid():N}"); + Directory.CreateDirectory(_tempDir); + } + + public void Dispose() + { + if (Directory.Exists(_tempDir)) + { + Directory.Delete(_tempDir, recursive: true); + } + } + + private static PlatformDetector CreateDetector() => + new(Substitute.For>()); + + [Fact] + public async Task CheckAsync_DotNet_WithBearerToken_ReturnsSuccess() + { + // Arrange + File.WriteAllText(Path.Combine(_tempDir, "Program.cs"), ""); + File.WriteAllText(Path.Combine(_tempDir, "test.csproj"), ""); + var propsDir = Directory.CreateDirectory(Path.Combine(_tempDir, "Properties")); + File.WriteAllText(Path.Combine(propsDir.FullName, "launchSettings.json"), $$""" + { + "profiles": { + "MyApp": { + "environmentVariables": { + "{{AuthenticationConstants.BearerTokenEnvironmentVariable}}": "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9" + } + } + } + } + """); + + var check = new BearerTokenRequirementCheck(CreateDetector()); + var config = new Agent365Config { DeploymentProjectPath = _tempDir }; + + // Act + var result = await check.CheckAsync(config, _logger); + + // Assert + result.Passed.Should().BeTrue(because: "bearer token is present in launchSettings.json"); + result.Details.Should().Contain("launchSettings.json"); + } + + [Fact] + public async Task CheckAsync_DotNet_WithoutBearerToken_ReturnsFailure() + { + // Arrange + File.WriteAllText(Path.Combine(_tempDir, "Program.cs"), ""); + File.WriteAllText(Path.Combine(_tempDir, "test.csproj"), ""); + var propsDir = Directory.CreateDirectory(Path.Combine(_tempDir, "Properties")); + File.WriteAllText(Path.Combine(propsDir.FullName, "launchSettings.json"), """ + { + "profiles": { + "MyApp": { + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + } + } + } + """); + + var check = new BearerTokenRequirementCheck(CreateDetector()); + var config = new Agent365Config { DeploymentProjectPath = _tempDir }; + + // Act + var result = await check.CheckAsync(config, _logger); + + // Assert + result.Passed.Should().BeFalse(because: "BEARER_TOKEN is not set in launchSettings.json"); + result.ErrorMessage.Should().Contain("BEARER_TOKEN"); + result.ResolutionGuidance.Should().Contain("a365 develop get-token"); + } + + [Fact] + public async Task CheckAsync_DotNet_NoLaunchSettings_ReturnsFailure() + { + // Arrange + File.WriteAllText(Path.Combine(_tempDir, "Program.cs"), ""); + File.WriteAllText(Path.Combine(_tempDir, "test.csproj"), ""); + + var check = new BearerTokenRequirementCheck(CreateDetector()); + var config = new Agent365Config { DeploymentProjectPath = _tempDir }; + + // Act + var result = await check.CheckAsync(config, _logger); + + // Assert + result.Passed.Should().BeFalse(because: "launchSettings.json does not exist"); + result.ErrorMessage.Should().Contain("launchSettings.json"); + } + + [Fact] + public async Task CheckAsync_NodeJs_WithBearerToken_ReturnsSuccess() + { + // Arrange + File.WriteAllText(Path.Combine(_tempDir, "package.json"), "{}"); + File.WriteAllText(Path.Combine(_tempDir, ".env"), + $"{AuthenticationConstants.BearerTokenEnvironmentVariable}=eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9\n"); + + var check = new BearerTokenRequirementCheck(CreateDetector()); + var config = new Agent365Config { DeploymentProjectPath = _tempDir }; + + // Act + var result = await check.CheckAsync(config, _logger); + + // Assert + result.Passed.Should().BeTrue(because: "bearer token is present in .env file"); + result.Details.Should().Contain(".env"); + } + + [Fact] + public async Task CheckAsync_NodeJs_WithoutBearerToken_ReturnsFailure() + { + // Arrange + File.WriteAllText(Path.Combine(_tempDir, "package.json"), "{}"); + File.WriteAllText(Path.Combine(_tempDir, ".env"), "PORT=3978\n"); + + var check = new BearerTokenRequirementCheck(CreateDetector()); + var config = new Agent365Config { DeploymentProjectPath = _tempDir }; + + // Act + var result = await check.CheckAsync(config, _logger); + + // Assert + result.Passed.Should().BeFalse(because: "BEARER_TOKEN is not in .env file"); + result.ResolutionGuidance.Should().Contain("a365 develop get-token"); + } + + [Fact] + public async Task CheckAsync_NodeJs_NoEnvFile_ReturnsFailure() + { + // Arrange + File.WriteAllText(Path.Combine(_tempDir, "package.json"), "{}"); + + var check = new BearerTokenRequirementCheck(CreateDetector()); + var config = new Agent365Config { DeploymentProjectPath = _tempDir }; + + // Act + var result = await check.CheckAsync(config, _logger); + + // Assert + result.Passed.Should().BeFalse(because: ".env file does not exist"); + result.ErrorMessage.Should().Contain(".env"); + } + + [Fact] + public async Task CheckAsync_Python_WithBearerToken_ReturnsSuccess() + { + // Arrange + File.WriteAllText(Path.Combine(_tempDir, "requirements.txt"), ""); + File.WriteAllText(Path.Combine(_tempDir, ".env"), + $"{AuthenticationConstants.BearerTokenEnvironmentVariable}=\"eyJtoken\"\n"); + + var check = new BearerTokenRequirementCheck(CreateDetector()); + var config = new Agent365Config { DeploymentProjectPath = _tempDir }; + + // Act + var result = await check.CheckAsync(config, _logger); + + // Assert + result.Passed.Should().BeTrue(because: "bearer token is present in .env file for Python"); + } + + [Fact] + public async Task CheckAsync_EmptyTokenValue_ReturnsFailure() + { + // Arrange + File.WriteAllText(Path.Combine(_tempDir, "package.json"), "{}"); + File.WriteAllText(Path.Combine(_tempDir, ".env"), + $"{AuthenticationConstants.BearerTokenEnvironmentVariable}=\n"); + + var check = new BearerTokenRequirementCheck(CreateDetector()); + var config = new Agent365Config { DeploymentProjectPath = _tempDir }; + + // Act + var result = await check.CheckAsync(config, _logger); + + // Assert + result.Passed.Should().BeFalse(because: "empty token value should not be accepted"); + } + + [Fact] + public async Task CheckAsync_ProjectPathDoesNotExist_ReturnsFailure() + { + // Arrange + var check = new BearerTokenRequirementCheck(CreateDetector()); + var config = new Agent365Config { DeploymentProjectPath = Path.Combine(_tempDir, "nonexistent") }; + + // Act + var result = await check.CheckAsync(config, _logger); + + // Assert + result.Passed.Should().BeFalse(because: "project path does not exist"); + } + + [Fact] + public void CheckLaunchSettings_EmptyBearerToken_ReturnsFailure() + { + var propsDir = Directory.CreateDirectory(Path.Combine(_tempDir, "Properties")); + File.WriteAllText(Path.Combine(propsDir.FullName, "launchSettings.json"), """ + { + "profiles": { + "MyApp": { + "environmentVariables": { + "BEARER_TOKEN": "" + } + } + } + } + """); + + var result = BearerTokenRequirementCheck.CheckLaunchSettings( + _tempDir, AuthenticationConstants.BearerTokenEnvironmentVariable); + + result.Passed.Should().BeFalse(because: "empty token value should not pass"); + } +} diff --git a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/LocalRuntimeRequirementCheckTests.cs b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/LocalRuntimeRequirementCheckTests.cs index a91bc8fb..ba5df506 100644 --- a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/LocalRuntimeRequirementCheckTests.cs +++ b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/LocalRuntimeRequirementCheckTests.cs @@ -172,17 +172,45 @@ public async Task CheckAsync_WhenProcessExitsEarly_ReturnsFailure() result.ErrorMessage.Should().Contain("exited early"); } - [Theory] - [InlineData("https://localhost:3978/api/messages", 3978)] - [InlineData("https://127.0.0.1:8080/api/messages", 8080)] - [InlineData("https://myapp.azurewebsites.net/api/messages", 5000)] - [InlineData("https://localhost/api/messages", 5000)] - [InlineData("", 5000)] - [InlineData(null, 5000)] - public void ResolvePort_ReturnsExpectedPort(string? endpoint, int expected) - { - var port = LocalRuntimeRequirementCheck.ResolvePort(endpoint); - port.Should().Be(expected, because: "port resolution should respect localhost URLs and fall back to default"); + [Fact] + public void ResolvePort_WithLaunchSettings_ReturnsConfiguredPort() + { + var propsDir = Directory.CreateDirectory(Path.Combine(_tempDir, "Properties")); + File.WriteAllText(Path.Combine(propsDir.FullName, "launchSettings.json"), """ + { + "profiles": { + "MyApp": { + "applicationUrl": "http://localhost:3978" + } + } + } + """); + + var port = LocalRuntimeRequirementCheck.ResolvePort(_tempDir, ProjectPlatform.DotNet); + port.Should().Be(3978, because: "port should be read from launchSettings.json applicationUrl"); + } + + [Fact] + public void ResolvePort_WithEnvFile_ReturnsConfiguredPort() + { + File.WriteAllText(Path.Combine(_tempDir, ".env"), "PORT=8080\n"); + + var port = LocalRuntimeRequirementCheck.ResolvePort(_tempDir, ProjectPlatform.NodeJs); + port.Should().Be(8080, because: "port should be read from .env PORT variable"); + } + + [Fact] + public void ResolvePort_WithNoSettings_ReturnsDefault() + { + var port = LocalRuntimeRequirementCheck.ResolvePort(_tempDir, ProjectPlatform.DotNet); + port.Should().Be(5000, because: "default port is used when no launch settings are found"); + } + + [Fact] + public void ResolvePort_WithNullPath_ReturnsDefault() + { + var port = LocalRuntimeRequirementCheck.ResolvePort(null); + port.Should().Be(5000, because: "default port is used when project path is null"); } [Fact] diff --git a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/TelemetryRequirementCheckTests.cs b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/TelemetryRequirementCheckTests.cs index 99a49346..6587c3ef 100644 --- a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/TelemetryRequirementCheckTests.cs +++ b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/TelemetryRequirementCheckTests.cs @@ -476,22 +476,20 @@ public void GetChildSpansMissingParent_InvokeAgentWithoutParent_IsIgnored() .Should().BeEmpty(because: "invoke_agent is a root span and does not need a parent"); } - [Fact] - public void HasNonEmptyValue_ValidValue_ReturnsTrue() - { - TelemetryRequirementCheck.HasNonEmptyValue(" parentId: 'abc123'").Should().BeTrue(); - } - - [Fact] - public void HasNonEmptyValue_Undefined_ReturnsFalse() - { - TelemetryRequirementCheck.HasNonEmptyValue(" parentId: undefined").Should().BeFalse(); - } - - [Fact] - public void HasNonEmptyValue_EmptyQuotes_ReturnsFalse() - { - TelemetryRequirementCheck.HasNonEmptyValue(" parentId: ''").Should().BeFalse(); + [Theory] + [InlineData(" parentId: 'abc123def456',", true, "JS/Node camelCase format")] + [InlineData(" Activity.ParentSpanId: 63a4d021ceef33ed", true, ".NET console exporter format")] + [InlineData(" \"parent_id\": \"0x9534d47ca25deef6\",", true, "Python JSON format with 0x prefix")] + [InlineData(" parentSpanId: 'abc123',", true, "camelCase spanId variant")] + [InlineData(" parent_id=abc123", true, "equals-sign separator")] + [InlineData(" parentId: undefined", false, "undefined is not a valid hex span ID")] + [InlineData(" parentId: ''", false, "empty value has no hex digits")] + [InlineData(" parentId: null", false, "null is not a valid hex span ID")] + [InlineData(" some unrelated line", false, "no parent key present")] + public void ParentSpanPattern_MatchesExpectedFormats(string line, bool shouldMatch, string because) + { + TelemetryRequirementCheck.ParentSpanPattern.IsMatch(line) + .Should().Be(shouldMatch, because: because); } // --- End-to-end: fully compliant spans return success --- From e0f7b4635e7252650a4ba7c4ad7e2ce6fc358bf9 Mon Sep 17 00:00:00 2001 From: Code Review Bot Date: Tue, 23 Jun 2026 12:10:34 -0700 Subject: [PATCH 14/27] fix(validate): replace Unicode checkmark with plain ASCII status markers Address review comment: the command summary used a Unicode square root character that may not render consistently across terminals and log files. Replaced with descriptive plain-text markers (PASS/FAIL/WARN/SKIP). Also added IDisposable tracking for requirement checks. --- .../Commands/ValidateCommand.cs | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Commands/ValidateCommand.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Commands/ValidateCommand.cs index 1926924b..f9c8612c 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Commands/ValidateCommand.cs +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Commands/ValidateCommand.cs @@ -24,11 +24,11 @@ public sealed class ValidateCommand { internal const string ReportFileName = "a365.validate.json"; - // Status markers — use characters supported across Windows/macOS/Linux terminals - private const string PassMark = "\u221A"; // √ (square root, same as Windows renders for checkmark) - private const string FailMark = "X"; - private const string WarnMark = "!"; - private const string SkipMark = "-"; + // Status markers — plain ASCII for consistent rendering across terminals and log files + private const string PassMark = "PASS"; + private const string FailMark = "FAIL"; + private const string WarnMark = "WARN"; + private const string SkipMark = "SKIP"; private static readonly JsonSerializerOptions ReportSerializerOptions = new() { @@ -63,6 +63,7 @@ public static Command CreateCommand( var configPath = Path.Combine(cwd, ConfigConstants.DefaultConfigFileName); var report = new ValidateReport(); var launchPlayground = context.ParseResult.GetValueForOption(playgroundOption); + var disposableChecks = new List(); try { @@ -110,6 +111,7 @@ public static Command CreateCommand( if (structuralPassed && buildPassed && requirementChecksOverride is null) { var bootChecks = BuildBootChecks(platformDetector, processService, resolvedUvCommand); + disposableChecks.AddRange(bootChecks.OfType()); if (bootChecks.Count > 0) { var bootResults = await RunChecksDetailedAsync(bootChecks, config, logger, ct); @@ -132,6 +134,7 @@ public static Command CreateCommand( if (bootPassed && requirementChecksOverride is null) { var conversationChecks = BuildConversationChecks(platformDetector, processService, launchPlayground, resolvedUvCommand); + disposableChecks.AddRange(conversationChecks.OfType()); if (conversationChecks.Count > 0) { var conversationResults = await RunChecksDetailedAsync(conversationChecks, config, logger, ct); @@ -198,6 +201,11 @@ public static Command CreateCommand( finally { await WriteReportAsync(report, cwd, logger); + + foreach (var disposable in disposableChecks) + { + disposable.Dispose(); + } } }); From 0e8e523e8d82e37b738861bae4010b9f99ef44fa Mon Sep 17 00:00:00 2001 From: Code Review Bot Date: Tue, 23 Jun 2026 12:10:46 -0700 Subject: [PATCH 15/27] fix(blueprint): address multiple review comments on BlueprintRegistrationRequirementCheck - Fix XML doc/behavior mismatch: updated comment to say 'failure' not 'warning' for missing/mismatched permissions (matches actual behavior at line 350) - Convert BuildExpectedPermissions to async (BuildExpectedPermissionsAsync) to eliminate sync-over-async .GetAwaiter().GetResult() call - Add OperationCanceledException filter to catch block for manifest read, matching the pattern used elsewhere in the file - Fix DeploymentProjectPath empty/whitespace handling: treat empty/whitespace as 'use CWD' instead of relying on null-coalescing alone --- .../BlueprintRegistrationRequirementCheck.cs | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/BlueprintRegistrationRequirementCheck.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/BlueprintRegistrationRequirementCheck.cs index bd34afa0..62d76f5a 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/BlueprintRegistrationRequirementCheck.cs +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/BlueprintRegistrationRequirementCheck.cs @@ -194,7 +194,7 @@ private async Task CheckImplementationAsync( /// /// After core registration checks pass, verify inheritable permissions /// by comparing the static baseline + tooling manifest scopes against what is actually in Entra. - /// Missing or mismatched permissions produce a warning (not a failure). + /// Missing or mismatched permissions produce a failure. /// private async Task BuildSuccessResult( Agent365Config config, @@ -220,7 +220,7 @@ private async Task BuildSuccessResult( } // Build expected permissions: static baseline + tooling manifest scopes - var expectedPermissions = BuildExpectedPermissions(config, logger); + var expectedPermissions = await BuildExpectedPermissionsAsync(config, logger, cancellationToken); List<(string ResourceAppId, bool ScopesAllAllowed, bool RolesAllAllowed)> inheritableEntries; try @@ -368,8 +368,8 @@ private async Task BuildSuccessResult( /// Builds the expected permission list from the static baseline plus tooling manifest scopes. /// Scopes for the same resource app ID are merged. /// - internal static List<(string ResourceAppId, string ResourceName, List Scopes)> BuildExpectedPermissions( - Agent365Config config, ILogger logger) + internal static async Task Scopes)>> BuildExpectedPermissionsAsync( + Agent365Config config, ILogger logger, CancellationToken cancellationToken = default) { var merged = new Dictionary Scopes)>(StringComparer.OrdinalIgnoreCase); @@ -389,15 +389,18 @@ private async Task BuildSuccessResult( } // Add tooling manifest scopes (if manifest exists) + var projectPath = string.IsNullOrWhiteSpace(config.DeploymentProjectPath) + ? Directory.GetCurrentDirectory() + : config.DeploymentProjectPath; var manifestPath = Path.Combine( - config.DeploymentProjectPath ?? Directory.GetCurrentDirectory(), + projectPath, McpConstants.ToolingManifestFileName); if (File.Exists(manifestPath)) { try { - var scopesByAudience = ManifestHelper.GetScopesByAudienceAsync(manifestPath).GetAwaiter().GetResult(); + var scopesByAudience = await ManifestHelper.GetScopesByAudienceAsync(manifestPath); foreach (var (audienceAppId, scopes) in scopesByAudience) { @@ -413,7 +416,7 @@ private async Task BuildSuccessResult( } } } - catch (Exception ex) + catch (Exception ex) when (ex is not OperationCanceledException) { logger.LogDebug(ex, "Failed to read tooling manifest at {Path}, skipping manifest scopes", manifestPath); } From e2ab4739f4feca724cdd0f2669a6fa9397ed2261 Mon Sep 17 00:00:00 2001 From: Code Review Bot Date: Tue, 23 Jun 2026 12:10:54 -0700 Subject: [PATCH 16/27] fix(build): quote MSBuild file-logger path to handle spaces in user profile paths The -flp:logfile= argument was unquoted, causing dotnet build to fail with MSB1009 when the user's profile path contains spaces (e.g. 'First Last'). Wrapped the switch in quotes to handle this correctly. --- .../RequirementChecks/ProjectBuildRequirementCheck.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/ProjectBuildRequirementCheck.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/ProjectBuildRequirementCheck.cs index 72be9922..625a53bc 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/ProjectBuildRequirementCheck.cs +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/ProjectBuildRequirementCheck.cs @@ -149,7 +149,7 @@ private async Task CheckImplementationAsync( return platform switch { ProjectPlatform.DotNet => ("dotnet", - $"build --no-restore /p:TreatWarningsAsErrors=true -fl -flp:logfile={buildLogFile};verbosity=normal"), + $"build --no-restore /p:TreatWarningsAsErrors=true -fl \"-flp:logfile={buildLogFile};verbosity=normal\""), ProjectPlatform.NodeJs => ("npm", "run build"), ProjectPlatform.Python => DetectPythonInstallCommand(projectPath) is ("uv", _) ? (_resolvedUvCommand ?? "uv", "run python -m compileall -q .") From 1b8b8047d2ca15966f001cf55998726a8491241c Mon Sep 17 00:00:00 2001 From: Code Review Bot Date: Tue, 23 Jun 2026 12:11:07 -0700 Subject: [PATCH 17/27] fix(conversation): narrow IsErrorResponse to structural signals to reduce false positives Replaced broad keyword matching (error/exception/failed/not found/etc.) with structural signals: stack traces, exception type headers, HTTP 4xx/5xx codes, and 'internal server error'. This prevents false failures on legitimate agent replies like 'the file was not found in SharePoint'. Also implemented IDisposable pattern to properly dispose internally-created HttpClient instances. Updated test to use 'internal server error' text that matches the new structural detection. --- .../ConversationRequirementCheck.cs | 56 ++++++++++++++----- .../ConversationRequirementCheckTests.cs | 2 +- 2 files changed, 44 insertions(+), 14 deletions(-) diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/ConversationRequirementCheck.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/ConversationRequirementCheck.cs index 71de9c8a..f09622f5 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/ConversationRequirementCheck.cs +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/ConversationRequirementCheck.cs @@ -18,11 +18,12 @@ namespace Microsoft.Agents.A365.DevTools.Cli.Services.Requirements.RequirementCh /// Validates that the agent can hold a multi-turn conversation by spawning the agent locally, /// waiting for readiness via /api/health, then POSTing Bot Framework Activity messages to /api/messages. /// -public class ConversationRequirementCheck : RequirementCheck +public class ConversationRequirementCheck : RequirementCheck, IDisposable { private readonly PlatformDetector _platformDetector; private readonly IProcessService _processService; private readonly HttpClient _httpClient; + private readonly bool _ownsHttpClient; private readonly IBotCallbackReceiver? _callbackReceiver; private readonly bool _launchPlayground; private readonly string? _resolvedUvCommand; @@ -194,12 +195,22 @@ public ConversationRequirementCheck( { _platformDetector = platformDetector ?? throw new ArgumentNullException(nameof(platformDetector)); _processService = processService ?? throw new ArgumentNullException(nameof(processService)); + _ownsHttpClient = httpClient is null; _httpClient = httpClient ?? new HttpClient(); _callbackReceiver = callbackReceiver; _launchPlayground = launchPlayground; _resolvedUvCommand = resolvedUvCommand; } + /// + public void Dispose() + { + if (_ownsHttpClient) + { + _httpClient.Dispose(); + } + } + /// public override string Name => "Conversation"; @@ -1040,6 +1051,11 @@ private static string TruncateResponse(string response, int maxLength) : response[..maxLength] + "..."; } + /// + /// Detects error responses using structural signals (stack traces, HTTP error codes) + /// rather than keyword matching, to avoid false positives on legitimate agent replies + /// that mention words like "error" or "not found" in conversational context. + /// private static bool IsErrorResponse(string? responseText) { if (string.IsNullOrWhiteSpace(responseText)) @@ -1048,18 +1064,32 @@ private static bool IsErrorResponse(string? responseText) } var lower = responseText.ToLowerInvariant(); - return lower.Contains("error") || - lower.Contains("exception") || - lower.Contains("failed") || - lower.Contains("not found") || - lower.Contains("unauthorized") || - lower.Contains("forbidden") || - lower.Contains("internal server error") || - lower.Contains("unhandled") || - lower.Contains("stack trace") || - lower.Contains("timed out") || - lower.Contains("timeout") || - System.Text.RegularExpressions.Regex.IsMatch(lower, @"http\s*[45]\d{2}"); + + // Detect stack traces (e.g., " at Namespace.Class.Method()") + if (System.Text.RegularExpressions.Regex.IsMatch(responseText, @"^\s+at\s+\S+\.\S+\(", System.Text.RegularExpressions.RegexOptions.Multiline)) + { + return true; + } + + // Detect unhandled exception headers (e.g., "System.InvalidOperationException:") + if (System.Text.RegularExpressions.Regex.IsMatch(responseText, @"\b\w+Exception\s*:", System.Text.RegularExpressions.RegexOptions.Multiline)) + { + return true; + } + + // Detect HTTP 4xx/5xx status codes + if (System.Text.RegularExpressions.Regex.IsMatch(lower, @"http\s*[45]\d{2}")) + { + return true; + } + + // Detect "internal server error" as a specific, unambiguous signal + if (lower.Contains("internal server error")) + { + return true; + } + + return false; } /// diff --git a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/ConversationRequirementCheckTests.cs b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/ConversationRequirementCheckTests.cs index 710e9014..730653aa 100644 --- a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/ConversationRequirementCheckTests.cs +++ b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/ConversationRequirementCheckTests.cs @@ -382,7 +382,7 @@ public async Task CheckAsync_WhenAgentReturnsErrorResponse_ReturnsFailure() healthStatusCode: HttpStatusCode.OK, messagesStatusCode: HttpStatusCode.OK); var fakeReceiver = new FakeBotCallbackReceiver( - new BotCallbackResponse("An internal error occurred while processing", "message")); + new BotCallbackResponse("An internal server error occurred while processing your request", "message")); var check = CreateCheck(handler, fakeReceiver); var config = new Agent365Config { DeploymentProjectPath = _tempDir }; From 48a4e8fa75b7c78180327149b9fa5dc50eea3da5 Mon Sep 17 00:00:00 2001 From: Code Review Bot Date: Tue, 23 Jun 2026 12:11:14 -0700 Subject: [PATCH 18/27] fix(local-runtime): implement IDisposable for HttpClient ownership tracking Added _ownsHttpClient flag and IDisposable implementation to properly dispose internally-created HttpClient instances, following CLAUDE.md disposal rules. When an HttpClient is injected from outside, it is not disposed. --- .../LocalRuntimeRequirementCheck.cs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/LocalRuntimeRequirementCheck.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/LocalRuntimeRequirementCheck.cs index 933785bd..fde44da7 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/LocalRuntimeRequirementCheck.cs +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/LocalRuntimeRequirementCheck.cs @@ -15,11 +15,12 @@ namespace Microsoft.Agents.A365.DevTools.Cli.Services.Requirements.RequirementCh /// Validates that the user's agent app starts locally and responds on a health endpoint. /// Spawns the app process, polls /api/health, captures stdout/stderr, then stops the process. /// -public class LocalRuntimeRequirementCheck : RequirementCheck +public class LocalRuntimeRequirementCheck : RequirementCheck, IDisposable { private readonly PlatformDetector _platformDetector; private readonly IProcessService _processService; private readonly HttpClient _httpClient; + private readonly bool _ownsHttpClient; private readonly string? _resolvedUvCommand; /// @@ -55,10 +56,20 @@ public LocalRuntimeRequirementCheck( { _platformDetector = platformDetector ?? throw new ArgumentNullException(nameof(platformDetector)); _processService = processService ?? throw new ArgumentNullException(nameof(processService)); + _ownsHttpClient = httpClient is null; _httpClient = httpClient ?? new HttpClient(); _resolvedUvCommand = resolvedUvCommand; } + /// + public void Dispose() + { + if (_ownsHttpClient) + { + _httpClient.Dispose(); + } + } + /// public override string Name => "Local Runtime"; From 5e9a0c23a7c882231793b0b4a964e11f1d496f31 Mon Sep 17 00:00:00 2001 From: Code Review Bot Date: Tue, 23 Jun 2026 12:11:23 -0700 Subject: [PATCH 19/27] fix(telemetry): remove dead TelemetryContextKeywords constant and enforce MaxTelemetryLines cap Removed unused TelemetryContextKeywords array (never referenced beyond declaration). Applied MaxTelemetryLines cap by analyzing only the last 200 lines of the agent console log, preventing unbounded memory usage for large log files. --- .../TelemetryRequirementCheck.cs | 44 +++---------------- 1 file changed, 5 insertions(+), 39 deletions(-) diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/TelemetryRequirementCheck.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/TelemetryRequirementCheck.cs index fde9559b..d3a1102a 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/TelemetryRequirementCheck.cs +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/TelemetryRequirementCheck.cs @@ -18,47 +18,10 @@ public class TelemetryRequirementCheck : RequirementCheck private readonly string? _agentConsoleLogPath; /// - /// Maximum number of telemetry-relevant log lines to analyze. + /// Maximum number of log lines to analyze from the end of the agent console output. /// internal const int MaxTelemetryLines = 200; - /// - /// Keywords that identify a log line as telemetry-related (console exporter output). - /// A line must contain at least one of these to be considered relevant. - /// - internal static readonly string[] TelemetryContextKeywords = new[] - { - // Console exporter span output fields (camelCase and snake_case variants) - "traceid:", - "trace_id:", - "spanid:", - "span_id:", - "tracestate:", - "parentspancontext:", - "parent_id:", - // OpenTelemetry SDK indicators - "opentelemetry", - "otel", - "telemetry.sdk.name", - "telemetry.sdk.version", - "consoleexporter", - "consolespanexporter", - // .NET Activity-based console exporter - "activity.traceid", - "activity.displayname", - "activity.spanid", - "activitysource", - // GenAI semantic convention attributes - "gen_ai.operation.name", - "gen_ai.request.model", - "gen_ai.usage.input_tokens", - "gen_ai.agent.name", - // Agent365 observability SDK - "a365observabilitysdk", - "agent365observability", - "agent365.observability" - }; - /// /// Required GenAI semantic convention operation names that must ALL appear in traces. /// These correspond to gen_ai.operation.name values for agent orchestration spans. @@ -142,7 +105,10 @@ private Task CheckImplementationAsync( string[] logLines; try { - logLines = File.ReadAllLines(_agentConsoleLogPath); + var allLines = File.ReadAllLines(_agentConsoleLogPath); + logLines = allLines.Length > MaxTelemetryLines + ? allLines[^MaxTelemetryLines..] + : allLines; } catch (Exception ex) { From f306622be07f630e4af807aa7bc48a086a55bf87 Mon Sep 17 00:00:00 2001 From: Code Review Bot Date: Tue, 23 Jun 2026 12:11:32 -0700 Subject: [PATCH 20/27] fix(callback): use case-insensitive comparison for message type in IsSubstantiveMessage The 'typing' check already used OrdinalIgnoreCase but the 'message' check used case-sensitive == comparison. This inconsistency could cause legitimate message activities to be ignored if the type casing differs. --- .../HttpListenerBotCallbackReceiver.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.Agents.A365.DevTools.Validation/HttpListenerBotCallbackReceiver.cs b/src/Microsoft.Agents.A365.DevTools.Validation/HttpListenerBotCallbackReceiver.cs index 06a63658..07dac4df 100644 --- a/src/Microsoft.Agents.A365.DevTools.Validation/HttpListenerBotCallbackReceiver.cs +++ b/src/Microsoft.Agents.A365.DevTools.Validation/HttpListenerBotCallbackReceiver.cs @@ -221,7 +221,7 @@ public static bool IsSubstantiveMessage(BotCallbackResponse response) return false; } - return response.Type == "message" && !string.IsNullOrWhiteSpace(response.Text); + return string.Equals(response.Type, "message", StringComparison.OrdinalIgnoreCase) && !string.IsNullOrWhiteSpace(response.Text); } public async ValueTask DisposeAsync() From 46cc28f1dabeac5e9fbdc670c72643a6a7e444dd Mon Sep 17 00:00:00 2001 From: Code Review Bot Date: Tue, 23 Jun 2026 12:11:39 -0700 Subject: [PATCH 21/27] docs(tooling-manifest): add clarifying comment for optional manifest behavior Added comment explaining that returning Success for a missing ToolingManifest.json is intentional since agents that do not use MCP tool servers will not have one. --- .../RequirementChecks/ToolingManifestRequirementCheck.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/ToolingManifestRequirementCheck.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/ToolingManifestRequirementCheck.cs index 9e193749..04c6e054 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/ToolingManifestRequirementCheck.cs +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/ToolingManifestRequirementCheck.cs @@ -42,6 +42,8 @@ private async Task CheckImplementationAsync( if (!File.Exists(manifestPath)) { + // ToolingManifest.json is optional — agents that do not use MCP tool servers + // will not have one. Returning Success here is intentional. return RequirementCheckResult.Success("ToolingManifest.json not present, skipping"); } From 0a7ec9feb99713126ad975548a113cbf671f2b21 Mon Sep 17 00:00:00 2001 From: Code Review Bot Date: Tue, 23 Jun 2026 12:11:51 -0700 Subject: [PATCH 22/27] refactor(validation): reorganize types into Models/ folder with one type per file Moved all DTO types from flat root files (ValidateReport.cs, RequirementCheckMetadata.cs, ValidationContracts.cs) into individual files under Models/, following the CLI project's convention of one public type per file. Removed unused CliValidationCoordinator and IValidator types. Namespace remains unchanged (no breaking change for consumers). --- .../Models/AgentInfo.cs | 26 ++ .../Models/BlueprintResourcePermission.cs | 46 +++ .../Models/BlueprintResourceResult.cs | 54 +++ .../Models/BlueprintTierResult.cs | 28 ++ .../Models/BootTierResult.cs | 24 ++ .../Models/BuildTierResult.cs | 28 ++ .../Models/ConversationTierResult.cs | 24 ++ .../Models/ConversationTurnMetadata.cs | 34 ++ .../Models/ConversationTurnResult.cs | 42 +++ .../Models/MacMetricComparisonMetadata.cs | 34 ++ .../Models/RequirementCheckMetadata.cs | 97 +++++ .../Models/StructuralCheck.cs | 22 ++ .../Models/StructuralTierResult.cs | 16 + .../Models/SummaryResult.cs | 19 + .../Models/TelemetryTierResult.cs | 44 +++ .../Models/TierResult.cs | 38 ++ .../Models/ValidateReport.cs | 25 ++ .../Models/ValidationIssue.cs | 12 + .../Models/ValidationLoadResult.cs | 32 ++ .../Models/ValidationOutcome.cs | 29 ++ .../Models/ValidationSeverity.cs | 14 + .../Models/ValidationTiers.cs | 30 ++ .../RequirementCheckMetadata.cs | 199 ---------- .../ValidateReport.cs | 343 ------------------ .../ValidationContracts.cs | 183 ---------- 25 files changed, 718 insertions(+), 725 deletions(-) create mode 100644 src/Microsoft.Agents.A365.DevTools.Validation/Models/AgentInfo.cs create mode 100644 src/Microsoft.Agents.A365.DevTools.Validation/Models/BlueprintResourcePermission.cs create mode 100644 src/Microsoft.Agents.A365.DevTools.Validation/Models/BlueprintResourceResult.cs create mode 100644 src/Microsoft.Agents.A365.DevTools.Validation/Models/BlueprintTierResult.cs create mode 100644 src/Microsoft.Agents.A365.DevTools.Validation/Models/BootTierResult.cs create mode 100644 src/Microsoft.Agents.A365.DevTools.Validation/Models/BuildTierResult.cs create mode 100644 src/Microsoft.Agents.A365.DevTools.Validation/Models/ConversationTierResult.cs create mode 100644 src/Microsoft.Agents.A365.DevTools.Validation/Models/ConversationTurnMetadata.cs create mode 100644 src/Microsoft.Agents.A365.DevTools.Validation/Models/ConversationTurnResult.cs create mode 100644 src/Microsoft.Agents.A365.DevTools.Validation/Models/MacMetricComparisonMetadata.cs create mode 100644 src/Microsoft.Agents.A365.DevTools.Validation/Models/RequirementCheckMetadata.cs create mode 100644 src/Microsoft.Agents.A365.DevTools.Validation/Models/StructuralCheck.cs create mode 100644 src/Microsoft.Agents.A365.DevTools.Validation/Models/StructuralTierResult.cs create mode 100644 src/Microsoft.Agents.A365.DevTools.Validation/Models/SummaryResult.cs create mode 100644 src/Microsoft.Agents.A365.DevTools.Validation/Models/TelemetryTierResult.cs create mode 100644 src/Microsoft.Agents.A365.DevTools.Validation/Models/TierResult.cs create mode 100644 src/Microsoft.Agents.A365.DevTools.Validation/Models/ValidateReport.cs create mode 100644 src/Microsoft.Agents.A365.DevTools.Validation/Models/ValidationIssue.cs create mode 100644 src/Microsoft.Agents.A365.DevTools.Validation/Models/ValidationLoadResult.cs create mode 100644 src/Microsoft.Agents.A365.DevTools.Validation/Models/ValidationOutcome.cs create mode 100644 src/Microsoft.Agents.A365.DevTools.Validation/Models/ValidationSeverity.cs create mode 100644 src/Microsoft.Agents.A365.DevTools.Validation/Models/ValidationTiers.cs delete mode 100644 src/Microsoft.Agents.A365.DevTools.Validation/RequirementCheckMetadata.cs delete mode 100644 src/Microsoft.Agents.A365.DevTools.Validation/ValidateReport.cs delete mode 100644 src/Microsoft.Agents.A365.DevTools.Validation/ValidationContracts.cs diff --git a/src/Microsoft.Agents.A365.DevTools.Validation/Models/AgentInfo.cs b/src/Microsoft.Agents.A365.DevTools.Validation/Models/AgentInfo.cs new file mode 100644 index 00000000..3af5c528 --- /dev/null +++ b/src/Microsoft.Agents.A365.DevTools.Validation/Models/AgentInfo.cs @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Text.Json.Serialization; + +namespace Microsoft.Agents.A365.DevTools.Validation; + +/// +/// Metadata about the agent project being validated. +/// +public sealed class AgentInfo +{ + [JsonPropertyName("path")] + public string? Path { get; set; } + + [JsonPropertyName("language")] + public string? Language { get; set; } + + [JsonPropertyName("framework")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Framework { get; set; } + + [JsonPropertyName("capabilities")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public List? Capabilities { get; set; } +} diff --git a/src/Microsoft.Agents.A365.DevTools.Validation/Models/BlueprintResourcePermission.cs b/src/Microsoft.Agents.A365.DevTools.Validation/Models/BlueprintResourcePermission.cs new file mode 100644 index 00000000..60463ae6 --- /dev/null +++ b/src/Microsoft.Agents.A365.DevTools.Validation/Models/BlueprintResourcePermission.cs @@ -0,0 +1,46 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.Agents.A365.DevTools.Validation; + +/// +/// Permission status for a single resource API in the blueprint registration check. +/// +public sealed class BlueprintResourcePermission +{ + /// Display name of the resource (e.g., "Microsoft Graph"). + public string ResourceName { get; init; } = string.Empty; + + /// Application ID of the resource. + public string ResourceAppId { get; init; } = string.Empty; + + /// Scopes expected from config. + public List ExpectedScopes { get; init; } = new(); + + /// Scopes actually found in Entra inheritable permissions. + public List ActualScopes { get; init; } = new(); + + /// Scopes in config but missing from Entra. + public List MissingScopes { get; init; } = new(); + + /// Whether admin consent has been granted (from config). + public bool? ConsentGranted { get; init; } + + /// Whether inheritable permissions are configured in Entra for this resource. + public bool InheritablePermissionsConfigured { get; init; } + + /// Whether kind=allAllowed is set for delegated scopes on this resource. + public bool ScopesAllAllowed { get; init; } + + /// Whether kind=allAllowed is set for app roles on this resource. + public bool RolesAllAllowed { get; init; } + + /// App roles actually granted on the blueprint SP for this resource. + public List ActualAppRoles { get; init; } = new(); + + /// + /// Effective inheritance status: true when kind=allAllowed on both sides AND at least one + /// permission is granted on the blueprint SP for this resource. + /// + public bool EffectiveInheritance { get; init; } +} diff --git a/src/Microsoft.Agents.A365.DevTools.Validation/Models/BlueprintResourceResult.cs b/src/Microsoft.Agents.A365.DevTools.Validation/Models/BlueprintResourceResult.cs new file mode 100644 index 00000000..3a345a6c --- /dev/null +++ b/src/Microsoft.Agents.A365.DevTools.Validation/Models/BlueprintResourceResult.cs @@ -0,0 +1,54 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Text.Json.Serialization; + +namespace Microsoft.Agents.A365.DevTools.Validation; + +/// +/// Permission and consent status for a single resource API in the blueprint. +/// +public sealed class BlueprintResourceResult +{ + [JsonPropertyName("resourceName")] + public string ResourceName { get; set; } = string.Empty; + + [JsonPropertyName("resourceAppId")] + public string ResourceAppId { get; set; } = string.Empty; + + [JsonPropertyName("expectedScopes")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public List? ExpectedScopes { get; set; } + + [JsonPropertyName("actualScopes")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public List? ActualScopes { get; set; } + + [JsonPropertyName("missingScopes")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public List? MissingScopes { get; set; } + + [JsonPropertyName("consentGranted")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public bool? ConsentGranted { get; set; } + + [JsonPropertyName("inheritablePermissionsConfigured")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public bool? InheritablePermissionsConfigured { get; set; } + + [JsonPropertyName("scopesAllAllowed")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public bool? ScopesAllAllowed { get; set; } + + [JsonPropertyName("rolesAllAllowed")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public bool? RolesAllAllowed { get; set; } + + [JsonPropertyName("actualAppRoles")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public List? ActualAppRoles { get; set; } + + [JsonPropertyName("effectiveInheritance")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public bool? EffectiveInheritance { get; set; } +} diff --git a/src/Microsoft.Agents.A365.DevTools.Validation/Models/BlueprintTierResult.cs b/src/Microsoft.Agents.A365.DevTools.Validation/Models/BlueprintTierResult.cs new file mode 100644 index 00000000..e13b5506 --- /dev/null +++ b/src/Microsoft.Agents.A365.DevTools.Validation/Models/BlueprintTierResult.cs @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Text.Json.Serialization; + +namespace Microsoft.Agents.A365.DevTools.Validation; + +/// +/// Blueprint tier: Entra registration, permissions, and consent validation. +/// +public sealed class BlueprintTierResult : TierResult +{ + [JsonPropertyName("appExists")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public bool? AppExists { get; set; } + + [JsonPropertyName("servicePrincipalExists")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public bool? ServicePrincipalExists { get; set; } + + [JsonPropertyName("registrationExists")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public bool? RegistrationExists { get; set; } + + [JsonPropertyName("resources")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public List? Resources { get; set; } +} diff --git a/src/Microsoft.Agents.A365.DevTools.Validation/Models/BootTierResult.cs b/src/Microsoft.Agents.A365.DevTools.Validation/Models/BootTierResult.cs new file mode 100644 index 00000000..9fd6b95e --- /dev/null +++ b/src/Microsoft.Agents.A365.DevTools.Validation/Models/BootTierResult.cs @@ -0,0 +1,24 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Text.Json.Serialization; + +namespace Microsoft.Agents.A365.DevTools.Validation; + +/// +/// Boot tier: local runtime health probe result. +/// +public sealed class BootTierResult : TierResult +{ + [JsonPropertyName("port")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? Port { get; set; } + + [JsonPropertyName("bootMs")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public long? BootMs { get; set; } + + [JsonPropertyName("bootLogFile")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? BootLogFile { get; set; } +} diff --git a/src/Microsoft.Agents.A365.DevTools.Validation/Models/BuildTierResult.cs b/src/Microsoft.Agents.A365.DevTools.Validation/Models/BuildTierResult.cs new file mode 100644 index 00000000..50f4cc37 --- /dev/null +++ b/src/Microsoft.Agents.A365.DevTools.Validation/Models/BuildTierResult.cs @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Text.Json.Serialization; + +namespace Microsoft.Agents.A365.DevTools.Validation; + +/// +/// Build tier: project compilation result. +/// +public sealed class BuildTierResult : TierResult +{ + [JsonPropertyName("log")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Log { get; set; } + + [JsonPropertyName("exitCode")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? ExitCode { get; set; } + + [JsonPropertyName("errorSummary")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? ErrorSummary { get; set; } + + [JsonPropertyName("buildLogFile")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? BuildLogFile { get; set; } +} diff --git a/src/Microsoft.Agents.A365.DevTools.Validation/Models/ConversationTierResult.cs b/src/Microsoft.Agents.A365.DevTools.Validation/Models/ConversationTierResult.cs new file mode 100644 index 00000000..76c6386a --- /dev/null +++ b/src/Microsoft.Agents.A365.DevTools.Validation/Models/ConversationTierResult.cs @@ -0,0 +1,24 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Text.Json.Serialization; + +namespace Microsoft.Agents.A365.DevTools.Validation; + +/// +/// Conversation tier: multi-turn conversation validation result. +/// +public sealed class ConversationTierResult : TierResult +{ + [JsonPropertyName("turns")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public List? Turns { get; set; } + + [JsonPropertyName("playgroundLaunched")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public bool? PlaygroundLaunched { get; set; } + + [JsonPropertyName("conversationLogFile")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? ConversationLogFile { get; set; } +} diff --git a/src/Microsoft.Agents.A365.DevTools.Validation/Models/ConversationTurnMetadata.cs b/src/Microsoft.Agents.A365.DevTools.Validation/Models/ConversationTurnMetadata.cs new file mode 100644 index 00000000..44115590 --- /dev/null +++ b/src/Microsoft.Agents.A365.DevTools.Validation/Models/ConversationTurnMetadata.cs @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.Agents.A365.DevTools.Validation; + +/// +/// Metadata for a single conversation turn. +/// +public sealed class ConversationTurnMetadata +{ + /// The message sent to the agent. + public string Input { get; init; } = string.Empty; + + /// HTTP status code returned by /api/messages. + public int? StatusCode { get; init; } + + /// Truncated response body snippet. + public string? ResponseSnippet { get; init; } + + /// Round-trip latency in milliseconds. + public long? LatencyMs { get; init; } + + /// Whether this turn succeeded. + public bool Ok { get; init; } + + /// Error description if the turn failed. + public string? Error { get; init; } + + /// Whether the agent sent a response via the serviceUrl callback. Null if tracking was unavailable. + public bool? AgentResponded { get; init; } + + /// The text content of the agent's callback response, if any. + public string? AgentResponseText { get; init; } +} diff --git a/src/Microsoft.Agents.A365.DevTools.Validation/Models/ConversationTurnResult.cs b/src/Microsoft.Agents.A365.DevTools.Validation/Models/ConversationTurnResult.cs new file mode 100644 index 00000000..a0b476a8 --- /dev/null +++ b/src/Microsoft.Agents.A365.DevTools.Validation/Models/ConversationTurnResult.cs @@ -0,0 +1,42 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Text.Json.Serialization; + +namespace Microsoft.Agents.A365.DevTools.Validation; + +/// +/// Result of a single conversation turn. +/// +public sealed class ConversationTurnResult +{ + [JsonPropertyName("input")] + public string Input { get; set; } = string.Empty; + + [JsonPropertyName("statusCode")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? StatusCode { get; set; } + + [JsonPropertyName("responseSnippet")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? ResponseSnippet { get; set; } + + [JsonPropertyName("latencyMs")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public long? LatencyMs { get; set; } + + [JsonPropertyName("ok")] + public bool Ok { get; set; } + + [JsonPropertyName("error")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Error { get; set; } + + [JsonPropertyName("agentResponded")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public bool? AgentResponded { get; set; } + + [JsonPropertyName("agentResponseText")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? AgentResponseText { get; set; } +} diff --git a/src/Microsoft.Agents.A365.DevTools.Validation/Models/MacMetricComparisonMetadata.cs b/src/Microsoft.Agents.A365.DevTools.Validation/Models/MacMetricComparisonMetadata.cs new file mode 100644 index 00000000..a979498b --- /dev/null +++ b/src/Microsoft.Agents.A365.DevTools.Validation/Models/MacMetricComparisonMetadata.cs @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.Agents.A365.DevTools.Validation; + +/// +/// Comparison details for a single MAC metric. +/// +public sealed class MacMetricComparisonMetadata +{ + /// Canonical metric key (e.g., kpi.invocations.rl7). + public string MetricKey { get; init; } = string.Empty; + + /// Baseline value. + public double Before { get; init; } + + /// Post-conversation value. + public double After { get; init; } + + /// After - Before. + public double Delta { get; init; } + + /// True when delta is positive. + public bool Increased { get; init; } + + /// True when this metric is the exception-rate metric. + public bool IsExceptionRate { get; init; } + + /// Final pass/fail for this metric after applying rule exceptions. + public bool Passed { get; init; } + + /// Human-readable reason for this comparison result. + public string? Reason { get; init; } +} diff --git a/src/Microsoft.Agents.A365.DevTools.Validation/Models/RequirementCheckMetadata.cs b/src/Microsoft.Agents.A365.DevTools.Validation/Models/RequirementCheckMetadata.cs new file mode 100644 index 00000000..135ecf91 --- /dev/null +++ b/src/Microsoft.Agents.A365.DevTools.Validation/Models/RequirementCheckMetadata.cs @@ -0,0 +1,97 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.Agents.A365.DevTools.Validation; + +/// +/// Typed metadata for structured validation report output. +/// Attached to RequirementCheckResult.Metadata by validate-specific checks. +/// +public sealed class RequirementCheckMetadata +{ + /// Port the app is running on (boot tier). + public int? Port { get; init; } + + /// Time in milliseconds for the app to respond (boot tier). + public long? BootMs { get; init; } + + /// Build or runtime log output (build/boot tier). + public string? Log { get; init; } + + /// Process exit code (build tier). + public int? ExitCode { get; init; } + + /// Detected platform name (build/boot tier). + public string? Platform { get; init; } + + /// Conversation turn results (conversation tier). + public List? Turns { get; init; } + + /// Whether AgentsPlayground was launched for interactive testing. + public bool? PlaygroundLaunched { get; init; } + + /// + /// Path to the agent's captured console output log file. + /// Written during the conversation step; used by telemetry check and referenced in the report. + /// + public string? AgentConsoleLogPath { get; init; } + + /// + /// Path to the MSBuild file log written during project build validation. + /// + public string? BuildLogFile { get; init; } + + /// + /// Path to the boot log file written during local runtime validation. + /// + public string? BootLogFile { get; init; } + + /// + /// Path to the conversation log file written during conversation validation. + /// Contains HTTP request/response details for each turn. + /// + public string? ConversationLogFile { get; init; } + + /// + /// Resolved path to the uv command, set during build dependency install. + /// Used by the boot step to run Python agents in uv-managed projects. + /// + public string? ResolvedUvCommand { get; init; } + + /// Whether the blueprint application exists in Entra ID. + public bool? AppExists { get; init; } + + /// Whether a service principal exists for the blueprint. + public bool? ServicePrincipalExists { get; init; } + + /// Whether the agent registration exists (null if not configured). + public bool? RegistrationExists { get; init; } + + /// Resource permission results from comparing config vs Entra. + public List? ResourcePermissions { get; set; } + + /// + /// Path to the persisted pre-conversation MAC metrics baseline file. + /// + public string? MacMetricsBaselineFile { get; init; } + + /// + /// Flattened numeric metrics captured before conversation. + /// + public Dictionary? MacBaselineMetrics { get; init; } + + /// + /// Flattened numeric metrics captured after conversation. + /// + public Dictionary? MacCurrentMetrics { get; init; } + + /// + /// Per-metric comparison outcome between baseline and post-conversation snapshots. + /// + public List? MacMetricComparisons { get; init; } + + /// + /// Whether conversation simulation completion was verified before MAC comparison. + /// + public bool? ConversationStepVerified { get; init; } +} diff --git a/src/Microsoft.Agents.A365.DevTools.Validation/Models/StructuralCheck.cs b/src/Microsoft.Agents.A365.DevTools.Validation/Models/StructuralCheck.cs new file mode 100644 index 00000000..2fbc2f02 --- /dev/null +++ b/src/Microsoft.Agents.A365.DevTools.Validation/Models/StructuralCheck.cs @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Text.Json.Serialization; + +namespace Microsoft.Agents.A365.DevTools.Validation; + +/// +/// Individual structural check result. +/// +public sealed class StructuralCheck +{ + [JsonPropertyName("name")] + public string Name { get; set; } = string.Empty; + + [JsonPropertyName("ok")] + public bool Ok { get; set; } + + [JsonPropertyName("message")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Message { get; set; } +} diff --git a/src/Microsoft.Agents.A365.DevTools.Validation/Models/StructuralTierResult.cs b/src/Microsoft.Agents.A365.DevTools.Validation/Models/StructuralTierResult.cs new file mode 100644 index 00000000..516fe032 --- /dev/null +++ b/src/Microsoft.Agents.A365.DevTools.Validation/Models/StructuralTierResult.cs @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Text.Json.Serialization; + +namespace Microsoft.Agents.A365.DevTools.Validation; + +/// +/// Structural tier: config and manifest validation checks. +/// +public sealed class StructuralTierResult : TierResult +{ + [JsonPropertyName("checks")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public List? Checks { get; set; } +} diff --git a/src/Microsoft.Agents.A365.DevTools.Validation/Models/SummaryResult.cs b/src/Microsoft.Agents.A365.DevTools.Validation/Models/SummaryResult.cs new file mode 100644 index 00000000..a6f69db6 --- /dev/null +++ b/src/Microsoft.Agents.A365.DevTools.Validation/Models/SummaryResult.cs @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Text.Json.Serialization; + +namespace Microsoft.Agents.A365.DevTools.Validation; + +/// +/// Summary of the validation run. +/// +public sealed class SummaryResult +{ + [JsonPropertyName("ok")] + public bool Ok { get; set; } + + [JsonPropertyName("blocker")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Blocker { get; set; } +} diff --git a/src/Microsoft.Agents.A365.DevTools.Validation/Models/TelemetryTierResult.cs b/src/Microsoft.Agents.A365.DevTools.Validation/Models/TelemetryTierResult.cs new file mode 100644 index 00000000..cda31c49 --- /dev/null +++ b/src/Microsoft.Agents.A365.DevTools.Validation/Models/TelemetryTierResult.cs @@ -0,0 +1,44 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Text.Json.Serialization; + +namespace Microsoft.Agents.A365.DevTools.Validation; + +/// +/// Telemetry tier: trace export validation result. +/// +public sealed class TelemetryTierResult : TierResult +{ + [JsonPropertyName("consoleExporterActive")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public bool? ConsoleExporterActive { get; set; } + + [JsonPropertyName("foundOperations")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public List? FoundOperations { get; set; } + + [JsonPropertyName("missingOperations")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public List? MissingOperations { get; set; } + + [JsonPropertyName("scopeVersionPresent")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public bool? ScopeVersionPresent { get; set; } + + [JsonPropertyName("parentLinksValid")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public bool? ParentLinksValid { get; set; } + + [JsonPropertyName("childSpansMissingParent")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public List? ChildSpansMissingParent { get; set; } + + [JsonPropertyName("resourceAttributesPresent")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public bool? ResourceAttributesPresent { get; set; } + + [JsonPropertyName("missingResourceAttributes")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public List? MissingResourceAttributes { get; set; } +} diff --git a/src/Microsoft.Agents.A365.DevTools.Validation/Models/TierResult.cs b/src/Microsoft.Agents.A365.DevTools.Validation/Models/TierResult.cs new file mode 100644 index 00000000..1bd4f455 --- /dev/null +++ b/src/Microsoft.Agents.A365.DevTools.Validation/Models/TierResult.cs @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Text.Json.Serialization; + +namespace Microsoft.Agents.A365.DevTools.Validation; + +/// +/// Base tier result. When skipped, ok is null. +/// +public class TierResult +{ + [JsonPropertyName("ok")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public bool? Ok { get; set; } + + [JsonPropertyName("skipped")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public bool Skipped { get; set; } + + [JsonPropertyName("reason")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Reason { get; set; } + + [JsonPropertyName("warning")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Warning { get; set; } + + public static TierResult CreateSkipped(string reason = "not yet implemented") + { + return new TierResult { Skipped = true, Reason = reason }; + } + + public static T CreateSkipped(string reason = "not yet implemented") where T : TierResult, new() + { + return new T { Skipped = true, Reason = reason }; + } +} diff --git a/src/Microsoft.Agents.A365.DevTools.Validation/Models/ValidateReport.cs b/src/Microsoft.Agents.A365.DevTools.Validation/Models/ValidateReport.cs new file mode 100644 index 00000000..ed7452b1 --- /dev/null +++ b/src/Microsoft.Agents.A365.DevTools.Validation/Models/ValidateReport.cs @@ -0,0 +1,25 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Text.Json.Serialization; + +namespace Microsoft.Agents.A365.DevTools.Validation; + +/// +/// Root model for the structured validation report written to a365.validate.json. +/// +public sealed class ValidateReport +{ + [JsonPropertyName("agent")] + public AgentInfo Agent { get; set; } = new(); + + [JsonPropertyName("tiers")] + public ValidationTiers Tiers { get; set; } = new(); + + [JsonPropertyName("summary")] + public SummaryResult Summary { get; set; } = new(); + + [JsonPropertyName("agentConsoleLogFile")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? AgentConsoleLogFile { get; set; } +} diff --git a/src/Microsoft.Agents.A365.DevTools.Validation/Models/ValidationIssue.cs b/src/Microsoft.Agents.A365.DevTools.Validation/Models/ValidationIssue.cs new file mode 100644 index 00000000..6eee5c4a --- /dev/null +++ b/src/Microsoft.Agents.A365.DevTools.Validation/Models/ValidationIssue.cs @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.Agents.A365.DevTools.Validation; + +/// +/// Represents a single validation issue. +/// +public sealed record ValidationIssue( + string Code, + string Message, + ValidationSeverity Severity = ValidationSeverity.Error); diff --git a/src/Microsoft.Agents.A365.DevTools.Validation/Models/ValidationLoadResult.cs b/src/Microsoft.Agents.A365.DevTools.Validation/Models/ValidationLoadResult.cs new file mode 100644 index 00000000..b085790d --- /dev/null +++ b/src/Microsoft.Agents.A365.DevTools.Validation/Models/ValidationLoadResult.cs @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.Agents.A365.DevTools.Validation; + +/// +/// Result of loading configuration for validation. +/// +public sealed record ValidationLoadResult +{ + public bool IsSuccess { get; init; } + + public TConfig? Value { get; init; } + + public int ExitCode { get; init; } + + public IReadOnlyList Issues { get; init; } = []; + + public static ValidationLoadResult Success(TConfig value) => new() + { + IsSuccess = true, + Value = value, + ExitCode = 0 + }; + + public static ValidationLoadResult Failure(int exitCode, params ValidationIssue[] issues) => new() + { + IsSuccess = false, + ExitCode = exitCode, + Issues = issues + }; +} diff --git a/src/Microsoft.Agents.A365.DevTools.Validation/Models/ValidationOutcome.cs b/src/Microsoft.Agents.A365.DevTools.Validation/Models/ValidationOutcome.cs new file mode 100644 index 00000000..ed7484db --- /dev/null +++ b/src/Microsoft.Agents.A365.DevTools.Validation/Models/ValidationOutcome.cs @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.Agents.A365.DevTools.Validation; + +/// +/// Represents the outcome of a validation operation. +/// +public sealed class ValidationOutcome +{ + public bool IsValid { get; init; } + + public int ExitCode { get; init; } + + public IReadOnlyList Issues { get; init; } = []; + + public static ValidationOutcome Success() => new() + { + IsValid = true, + ExitCode = 0 + }; + + public static ValidationOutcome Failure(params ValidationIssue[] issues) => new() + { + IsValid = false, + ExitCode = 1, + Issues = issues + }; +} diff --git a/src/Microsoft.Agents.A365.DevTools.Validation/Models/ValidationSeverity.cs b/src/Microsoft.Agents.A365.DevTools.Validation/Models/ValidationSeverity.cs new file mode 100644 index 00000000..b9c42c21 --- /dev/null +++ b/src/Microsoft.Agents.A365.DevTools.Validation/Models/ValidationSeverity.cs @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.Agents.A365.DevTools.Validation; + +/// +/// Severity for a validation issue. +/// +public enum ValidationSeverity +{ + Info, + Warning, + Error +} diff --git a/src/Microsoft.Agents.A365.DevTools.Validation/Models/ValidationTiers.cs b/src/Microsoft.Agents.A365.DevTools.Validation/Models/ValidationTiers.cs new file mode 100644 index 00000000..b7093cf1 --- /dev/null +++ b/src/Microsoft.Agents.A365.DevTools.Validation/Models/ValidationTiers.cs @@ -0,0 +1,30 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Text.Json.Serialization; + +namespace Microsoft.Agents.A365.DevTools.Validation; + +/// +/// Container for all validation tiers. +/// +public sealed class ValidationTiers +{ + [JsonPropertyName("structural")] + public StructuralTierResult Structural { get; set; } = TierResult.CreateSkipped(); + + [JsonPropertyName("build")] + public BuildTierResult Build { get; set; } = TierResult.CreateSkipped(); + + [JsonPropertyName("boot")] + public BootTierResult Boot { get; set; } = TierResult.CreateSkipped(); + + [JsonPropertyName("conversation")] + public ConversationTierResult Conversation { get; set; } = TierResult.CreateSkipped("not yet implemented"); + + [JsonPropertyName("telemetry")] + public TelemetryTierResult Telemetry { get; set; } = TierResult.CreateSkipped("not yet run"); + + [JsonPropertyName("blueprint")] + public BlueprintTierResult Blueprint { get; set; } = TierResult.CreateSkipped("not yet implemented"); +} diff --git a/src/Microsoft.Agents.A365.DevTools.Validation/RequirementCheckMetadata.cs b/src/Microsoft.Agents.A365.DevTools.Validation/RequirementCheckMetadata.cs deleted file mode 100644 index 66dbe79f..00000000 --- a/src/Microsoft.Agents.A365.DevTools.Validation/RequirementCheckMetadata.cs +++ /dev/null @@ -1,199 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -namespace Microsoft.Agents.A365.DevTools.Validation; - -/// -/// Typed metadata for structured validation report output. -/// Attached to RequirementCheckResult.Metadata by validate-specific checks. -/// -public sealed class RequirementCheckMetadata -{ - /// Port the app is running on (boot tier). - public int? Port { get; init; } - - /// Time in milliseconds for the app to respond (boot tier). - public long? BootMs { get; init; } - - /// Build or runtime log output (build/boot tier). - public string? Log { get; init; } - - /// Process exit code (build tier). - public int? ExitCode { get; init; } - - /// Detected platform name (build/boot tier). - public string? Platform { get; init; } - - /// Conversation turn results (conversation tier). - public List? Turns { get; init; } - - /// Whether AgentsPlayground was launched for interactive testing. - public bool? PlaygroundLaunched { get; init; } - - /// - /// Path to the agent's captured console output log file. - /// Written during the conversation step; used by telemetry check and referenced in the report. - /// - public string? AgentConsoleLogPath { get; init; } - - /// - /// Path to the MSBuild file log written during project build validation. - /// - public string? BuildLogFile { get; init; } - - /// - /// Path to the boot log file written during local runtime validation. - /// - public string? BootLogFile { get; init; } - - /// - /// Path to the conversation log file written during conversation validation. - /// Contains HTTP request/response details for each turn. - /// - public string? ConversationLogFile { get; init; } - - /// - /// Resolved path to the uv command, set during build dependency install. - /// Used by the boot step to run Python agents in uv-managed projects. - /// - public string? ResolvedUvCommand { get; init; } - - /// Whether the blueprint application exists in Entra ID. - public bool? AppExists { get; init; } - - /// Whether a service principal exists for the blueprint. - public bool? ServicePrincipalExists { get; init; } - - /// Whether the agent registration exists (null if not configured). - public bool? RegistrationExists { get; init; } - - /// Resource permission results from comparing config vs Entra. - public List? ResourcePermissions { get; set; } - - /// - /// Path to the persisted pre-conversation MAC metrics baseline file. - /// - public string? MacMetricsBaselineFile { get; init; } - - /// - /// Flattened numeric metrics captured before conversation. - /// - public Dictionary? MacBaselineMetrics { get; init; } - - /// - /// Flattened numeric metrics captured after conversation. - /// - public Dictionary? MacCurrentMetrics { get; init; } - - /// - /// Per-metric comparison outcome between baseline and post-conversation snapshots. - /// - public List? MacMetricComparisons { get; init; } - - /// - /// Whether conversation simulation completion was verified before MAC comparison. - /// - public bool? ConversationStepVerified { get; init; } -} - -/// -/// Comparison details for a single MAC metric. -/// -public sealed class MacMetricComparisonMetadata -{ - /// Canonical metric key (e.g., kpi.invocations.rl7). - public string MetricKey { get; init; } = string.Empty; - - /// Baseline value. - public double Before { get; init; } - - /// Post-conversation value. - public double After { get; init; } - - /// After - Before. - public double Delta { get; init; } - - /// True when delta is positive. - public bool Increased { get; init; } - - /// True when this metric is the exception-rate metric. - public bool IsExceptionRate { get; init; } - - /// Final pass/fail for this metric after applying rule exceptions. - public bool Passed { get; init; } - - /// Human-readable reason for this comparison result. - public string? Reason { get; init; } -} - -/// -/// Metadata for a single conversation turn. -/// -public sealed class ConversationTurnMetadata -{ - /// The message sent to the agent. - public string Input { get; init; } = string.Empty; - - /// HTTP status code returned by /api/messages. - public int? StatusCode { get; init; } - - /// Truncated response body snippet. - public string? ResponseSnippet { get; init; } - - /// Round-trip latency in milliseconds. - public long? LatencyMs { get; init; } - - /// Whether this turn succeeded. - public bool Ok { get; init; } - - /// Error description if the turn failed. - public string? Error { get; init; } - - /// Whether the agent sent a response via the serviceUrl callback. Null if tracking was unavailable. - public bool? AgentResponded { get; init; } - - /// The text content of the agent's callback response, if any. - public string? AgentResponseText { get; init; } -} - -/// -/// Permission status for a single resource API in the blueprint registration check. -/// -public sealed class BlueprintResourcePermission -{ - /// Display name of the resource (e.g., "Microsoft Graph"). - public string ResourceName { get; init; } = string.Empty; - - /// Application ID of the resource. - public string ResourceAppId { get; init; } = string.Empty; - - /// Scopes expected from config. - public List ExpectedScopes { get; init; } = new(); - - /// Scopes actually found in Entra inheritable permissions. - public List ActualScopes { get; init; } = new(); - - /// Scopes in config but missing from Entra. - public List MissingScopes { get; init; } = new(); - - /// Whether admin consent has been granted (from config). - public bool? ConsentGranted { get; init; } - - /// Whether inheritable permissions are configured in Entra for this resource. - public bool InheritablePermissionsConfigured { get; init; } - - /// Whether kind=allAllowed is set for delegated scopes on this resource. - public bool ScopesAllAllowed { get; init; } - - /// Whether kind=allAllowed is set for app roles on this resource. - public bool RolesAllAllowed { get; init; } - - /// App roles actually granted on the blueprint SP for this resource. - public List ActualAppRoles { get; init; } = new(); - - /// - /// Effective inheritance status: true when kind=allAllowed on both sides AND at least one - /// permission is granted on the blueprint SP for this resource. - /// - public bool EffectiveInheritance { get; init; } -} diff --git a/src/Microsoft.Agents.A365.DevTools.Validation/ValidateReport.cs b/src/Microsoft.Agents.A365.DevTools.Validation/ValidateReport.cs deleted file mode 100644 index dafeb351..00000000 --- a/src/Microsoft.Agents.A365.DevTools.Validation/ValidateReport.cs +++ /dev/null @@ -1,343 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -using System.Text.Json.Serialization; - -namespace Microsoft.Agents.A365.DevTools.Validation; - -/// -/// Root model for the structured validation report written to a365.validate.json. -/// -public sealed class ValidateReport -{ - [JsonPropertyName("agent")] - public AgentInfo Agent { get; set; } = new(); - - [JsonPropertyName("tiers")] - public ValidationTiers Tiers { get; set; } = new(); - - [JsonPropertyName("summary")] - public SummaryResult Summary { get; set; } = new(); - - [JsonPropertyName("agentConsoleLogFile")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? AgentConsoleLogFile { get; set; } -} - -/// -/// Metadata about the agent project being validated. -/// -public sealed class AgentInfo -{ - [JsonPropertyName("path")] - public string? Path { get; set; } - - [JsonPropertyName("language")] - public string? Language { get; set; } - - [JsonPropertyName("framework")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? Framework { get; set; } - - [JsonPropertyName("capabilities")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public List? Capabilities { get; set; } -} - -/// -/// Container for all validation tiers. -/// -public sealed class ValidationTiers -{ - [JsonPropertyName("structural")] - public StructuralTierResult Structural { get; set; } = TierResult.CreateSkipped(); - - [JsonPropertyName("build")] - public BuildTierResult Build { get; set; } = TierResult.CreateSkipped(); - - [JsonPropertyName("boot")] - public BootTierResult Boot { get; set; } = TierResult.CreateSkipped(); - - [JsonPropertyName("conversation")] - public ConversationTierResult Conversation { get; set; } = TierResult.CreateSkipped("not yet implemented"); - - [JsonPropertyName("telemetry")] - public TelemetryTierResult Telemetry { get; set; } = TierResult.CreateSkipped("not yet run"); - - [JsonPropertyName("blueprint")] - public BlueprintTierResult Blueprint { get; set; } = TierResult.CreateSkipped("not yet implemented"); - -} - -/// -/// Base tier result. When skipped, ok is null. -/// -public class TierResult -{ - [JsonPropertyName("ok")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public bool? Ok { get; set; } - - [JsonPropertyName("skipped")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] - public bool Skipped { get; set; } - - [JsonPropertyName("reason")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? Reason { get; set; } - - [JsonPropertyName("warning")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? Warning { get; set; } - - public static TierResult CreateSkipped(string reason = "not yet implemented") - { - return new TierResult { Skipped = true, Reason = reason }; - } - - public static T CreateSkipped(string reason = "not yet implemented") where T : TierResult, new() - { - return new T { Skipped = true, Reason = reason }; - } -} - -/// -/// Structural tier: config and manifest validation checks. -/// -public sealed class StructuralTierResult : TierResult -{ - [JsonPropertyName("checks")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public List? Checks { get; set; } -} - -/// -/// Individual structural check result. -/// -public sealed class StructuralCheck -{ - [JsonPropertyName("name")] - public string Name { get; set; } = string.Empty; - - [JsonPropertyName("ok")] - public bool Ok { get; set; } - - [JsonPropertyName("message")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? Message { get; set; } -} - -/// -/// Build tier: project compilation result. -/// -public sealed class BuildTierResult : TierResult -{ - [JsonPropertyName("log")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? Log { get; set; } - - [JsonPropertyName("exitCode")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public int? ExitCode { get; set; } - - [JsonPropertyName("errorSummary")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? ErrorSummary { get; set; } - - [JsonPropertyName("buildLogFile")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? BuildLogFile { get; set; } -} - -/// -/// Boot tier: local runtime health probe result. -/// -public sealed class BootTierResult : TierResult -{ - [JsonPropertyName("port")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public int? Port { get; set; } - - [JsonPropertyName("bootMs")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public long? BootMs { get; set; } - - [JsonPropertyName("bootLogFile")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? BootLogFile { get; set; } -} - -/// -/// Conversation tier: multi-turn conversation validation result. -/// -public sealed class ConversationTierResult : TierResult -{ - [JsonPropertyName("turns")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public List? Turns { get; set; } - - [JsonPropertyName("playgroundLaunched")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public bool? PlaygroundLaunched { get; set; } - - [JsonPropertyName("conversationLogFile")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? ConversationLogFile { get; set; } -} - -/// -/// Telemetry tier: trace export validation result. -/// -public sealed class TelemetryTierResult : TierResult -{ - [JsonPropertyName("consoleExporterActive")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public bool? ConsoleExporterActive { get; set; } - - [JsonPropertyName("foundOperations")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public List? FoundOperations { get; set; } - - [JsonPropertyName("missingOperations")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public List? MissingOperations { get; set; } - - [JsonPropertyName("scopeVersionPresent")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public bool? ScopeVersionPresent { get; set; } - - [JsonPropertyName("parentLinksValid")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public bool? ParentLinksValid { get; set; } - - [JsonPropertyName("childSpansMissingParent")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public List? ChildSpansMissingParent { get; set; } - - [JsonPropertyName("resourceAttributesPresent")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public bool? ResourceAttributesPresent { get; set; } - - [JsonPropertyName("missingResourceAttributes")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public List? MissingResourceAttributes { get; set; } -} - -/// -/// Blueprint tier: Entra registration, permissions, and consent validation. -/// -public sealed class BlueprintTierResult : TierResult -{ - [JsonPropertyName("appExists")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public bool? AppExists { get; set; } - - [JsonPropertyName("servicePrincipalExists")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public bool? ServicePrincipalExists { get; set; } - - [JsonPropertyName("registrationExists")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public bool? RegistrationExists { get; set; } - - [JsonPropertyName("resources")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public List? Resources { get; set; } -} - -/// -/// Permission and consent status for a single resource API in the blueprint. -/// -public sealed class BlueprintResourceResult -{ - [JsonPropertyName("resourceName")] - public string ResourceName { get; set; } = string.Empty; - - [JsonPropertyName("resourceAppId")] - public string ResourceAppId { get; set; } = string.Empty; - - [JsonPropertyName("expectedScopes")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public List? ExpectedScopes { get; set; } - - [JsonPropertyName("actualScopes")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public List? ActualScopes { get; set; } - - [JsonPropertyName("missingScopes")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public List? MissingScopes { get; set; } - - [JsonPropertyName("consentGranted")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public bool? ConsentGranted { get; set; } - - [JsonPropertyName("inheritablePermissionsConfigured")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public bool? InheritablePermissionsConfigured { get; set; } - - [JsonPropertyName("scopesAllAllowed")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public bool? ScopesAllAllowed { get; set; } - - [JsonPropertyName("rolesAllAllowed")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public bool? RolesAllAllowed { get; set; } - - [JsonPropertyName("actualAppRoles")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public List? ActualAppRoles { get; set; } - - [JsonPropertyName("effectiveInheritance")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public bool? EffectiveInheritance { get; set; } -} - -/// -/// Result of a single conversation turn. -/// -public sealed class ConversationTurnResult -{ - [JsonPropertyName("input")] - public string Input { get; set; } = string.Empty; - - [JsonPropertyName("statusCode")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public int? StatusCode { get; set; } - - [JsonPropertyName("responseSnippet")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? ResponseSnippet { get; set; } - - [JsonPropertyName("latencyMs")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public long? LatencyMs { get; set; } - - [JsonPropertyName("ok")] - public bool Ok { get; set; } - - [JsonPropertyName("error")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? Error { get; set; } - - [JsonPropertyName("agentResponded")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public bool? AgentResponded { get; set; } - - [JsonPropertyName("agentResponseText")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? AgentResponseText { get; set; } -} - -/// -/// Summary of the validation run. -/// -public sealed class SummaryResult -{ - [JsonPropertyName("ok")] - public bool Ok { get; set; } - - [JsonPropertyName("blocker")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? Blocker { get; set; } -} diff --git a/src/Microsoft.Agents.A365.DevTools.Validation/ValidationContracts.cs b/src/Microsoft.Agents.A365.DevTools.Validation/ValidationContracts.cs deleted file mode 100644 index 96312621..00000000 --- a/src/Microsoft.Agents.A365.DevTools.Validation/ValidationContracts.cs +++ /dev/null @@ -1,183 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -namespace Microsoft.Agents.A365.DevTools.Validation; - -/// -/// Severity for a validation issue. -/// -public enum ValidationSeverity -{ - Info, - Warning, - Error -} - -/// -/// Represents a single validation issue. -/// -public sealed record ValidationIssue( - string Code, - string Message, - ValidationSeverity Severity = ValidationSeverity.Error); - -/// -/// Represents the outcome of a validation operation. -/// -public sealed class ValidationOutcome -{ - public bool IsValid { get; init; } - - public int ExitCode { get; init; } - - public IReadOnlyList Issues { get; init; } = []; - - public static ValidationOutcome Success() => new() - { - IsValid = true, - ExitCode = 0 - }; - - public static ValidationOutcome Failure(params ValidationIssue[] issues) => new() - { - IsValid = false, - ExitCode = 1, - Issues = issues - }; -} - -/// -/// Result of loading configuration for validation. -/// -public sealed record ValidationLoadResult -{ - public bool IsSuccess { get; init; } - - public TConfig? Value { get; init; } - - public int ExitCode { get; init; } - - public IReadOnlyList Issues { get; init; } = []; - - public static ValidationLoadResult Success(TConfig value) => new() - { - IsSuccess = true, - Value = value, - ExitCode = 0 - }; - - public static ValidationLoadResult Failure(int exitCode, params ValidationIssue[] issues) => new() - { - IsSuccess = false, - ExitCode = exitCode, - Issues = issues - }; -} - -/// -/// Orchestrates the CLI validation workflow using delegates supplied by the caller. -/// -public sealed class CliValidationCoordinator -{ - public required Func> ConfigExistsAsync { get; init; } - - public required Func>> LoadConfigAsync { get; init; } - - public required Func> ValidateConfig { get; init; } - - public required Func> RunSystemChecksAsync { get; init; } - - public required Func> RunConfigChecksAsync { get; init; } - - public required Action ReportIssue { get; init; } - - public async Task ExecuteAsync(CancellationToken cancellationToken = default) - { - var issues = new List(); - var exitCode = 0; - - var configExists = await ConfigExistsAsync(cancellationToken); - if (!configExists) - { - var issue = new ValidationIssue( - "CONFIG_FILE_NOT_FOUND", - "Configuration file not found. Run 'a365 setup all --agent-name ' to set up from scratch."); - issues.Add(issue); - ReportIssue(issue); - exitCode = 2; - } - - TConfig? config = default; - if (configExists) - { - var loadResult = await LoadConfigAsync(cancellationToken); - issues.AddRange(loadResult.Issues); - - foreach (var issue in loadResult.Issues) - { - ReportIssue(issue); - } - - if (!loadResult.IsSuccess || loadResult.Value is null) - { - exitCode = Math.Max(exitCode, loadResult.ExitCode == 0 ? 2 : loadResult.ExitCode); - - if (!await RunSystemChecksAsync(cancellationToken)) - { - exitCode = Math.Max(exitCode, 1); - } - - return new ValidationOutcome - { - IsValid = exitCode == 0, - ExitCode = exitCode, - Issues = issues - }; - } - - config = loadResult.Value; - - var configErrors = ValidateConfig(config); - foreach (var error in configErrors) - { - var issue = new ValidationIssue("CONFIG_VALIDATION_FAILED", error); - issues.Add(issue); - ReportIssue(issue); - } - - if (configErrors.Count > 0) - { - exitCode = Math.Max(exitCode, 2); - } - } - - if (!await RunSystemChecksAsync(cancellationToken)) - { - exitCode = Math.Max(exitCode, 1); - } - - if (config is not null && exitCode < 2) - { - var configChecksPassed = await RunConfigChecksAsync(config, cancellationToken); - if (!configChecksPassed) - { - exitCode = Math.Max(exitCode, 1); - } - } - - return new ValidationOutcome - { - IsValid = exitCode == 0, - ExitCode = exitCode, - Issues = issues - }; - } -} - -/// -/// Contract for validation components in the validation subproject. -/// -public interface IValidator -{ - Task ValidateAsync(T value, CancellationToken cancellationToken = default); -} \ No newline at end of file From bc37d7e5be42ae504f507b2c24cea928e70d103f Mon Sep 17 00:00:00 2001 From: Code Review Bot Date: Tue, 23 Jun 2026 12:19:08 -0700 Subject: [PATCH 23/27] fix(test): update SetsAspNetCoreUrls to use launchSettings.json instead of MessagingEndpoint The test was still using config.MessagingEndpoint which was replaced by file-based port resolution (launchSettings.json / .env). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../LocalRuntimeRequirementCheckTests.cs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/LocalRuntimeRequirementCheckTests.cs b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/LocalRuntimeRequirementCheckTests.cs index ba5df506..dad40b56 100644 --- a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/LocalRuntimeRequirementCheckTests.cs +++ b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/LocalRuntimeRequirementCheckTests.cs @@ -247,6 +247,16 @@ public async Task CheckAsync_DotNetProject_SetsAspNetCoreUrls() { // Arrange File.WriteAllText(Path.Combine(_tempDir, "test.csproj"), ""); + var propsDir = Directory.CreateDirectory(Path.Combine(_tempDir, "Properties")); + File.WriteAllText(Path.Combine(propsDir.FullName, "launchSettings.json"), """ + { + "profiles": { + "MyApp": { + "applicationUrl": "http://localhost:3978" + } + } + } + """); var fakeProcess = CreateFakeProcess(exitImmediately: false); _processService.Start(Arg.Any()).Returns(fakeProcess); @@ -254,8 +264,7 @@ public async Task CheckAsync_DotNetProject_SetsAspNetCoreUrls() var check = CreateCheck(handler); var config = new Agent365Config { - DeploymentProjectPath = _tempDir, - MessagingEndpoint = "https://localhost:3978/api/messages" + DeploymentProjectPath = _tempDir }; // Act From ed8fe6c6d06ef8bbc27e1fb290de071fca11628f Mon Sep 17 00:00:00 2001 From: Code Review Bot Date: Tue, 23 Jun 2026 12:28:28 -0700 Subject: [PATCH 24/27] fix(telemetry): restore ParentSpanPattern regex to handle JSON-quoted keys Add optional closing quote after key name to match formats like parent_id from Python console exporter. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Requirements/RequirementChecks/TelemetryRequirementCheck.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/TelemetryRequirementCheck.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/TelemetryRequirementCheck.cs index d3a1102a..d6e8eeb1 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/TelemetryRequirementCheck.cs +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/TelemetryRequirementCheck.cs @@ -61,7 +61,7 @@ public class TelemetryRequirementCheck : RequirementCheck /// Activity.ParentSpanId, JSON quoted keys, YAML-style, and equals-sign separators. /// internal static readonly Regex ParentSpanPattern = new( - @"(?:parent[\._]?(?:span)?[\._]?(?:id|context))\s*[=:]\s*[""']?\s*(?:0x)?([0-9a-f]{2,})", + @"(?:parent[\._]?(?:span)?[\._]?(?:id|context))[""']?\s*[=:]\s*[""']?\s*(?:0x)?([0-9a-f]{2,})", RegexOptions.IgnoreCase | RegexOptions.Compiled); From cb658bd42de626b35596e8aa6a8b4219f386b194 Mon Sep 17 00:00:00 2001 From: Code Review Bot Date: Tue, 23 Jun 2026 13:36:28 -0700 Subject: [PATCH 25/27] Address PR #456 review comments: dispose, streaming, pip - Dispose check instances in LocalRuntimeRequirementCheckTests via tracked _disposables list disposed in test class Dispose() - Stream telemetry log tail with File.ReadLines().TakeLast() instead of File.ReadAllLines() to avoid loading entire log into memory - Use python -m pip instead of bare pip for uv install to target the active Python interpreter - Update CHANGELOG to accurately describe validate command behavior (build step installs dependencies) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CHANGELOG.md | 2 +- .../RequirementChecks/ProjectBuildRequirementCheck.cs | 6 +++--- .../RequirementChecks/TelemetryRequirementCheck.cs | 7 +++---- .../Requirements/LocalRuntimeRequirementCheckTests.cs | 11 ++++++++++- 4 files changed, 17 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f48e28dc..0fb9ab72 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,7 +23,7 @@ Agents provisioned before this release need `Agent365.Observability.OtelWrite` g **Option B — CLI** (`a365 setup admin`) has been removed in this release. Use Option A above, or copy the PowerShell instructions printed in the `a365 setup all` summary output. ### Added -- `a365 validate` — validates the local `a365.config.json` plus prerequisite checks without making changes. Reports missing or invalid config, then runs the existing setup prerequisite checks so users can catch problems before starting a publish workflow. +- `a365 validate` — validates the local agent project configuration and readiness. Runs structural checks (config, manifest, bearer token), builds the project (which may install dependencies such as npm packages or Python packages), boots the agent locally, sends test conversation turns, verifies telemetry spans, and checks blueprint registration. Reports results as a tiered JSON report (`a365.validate.json`). - New `Microsoft.Agents.A365.DevTools.Validation` subproject for reusable validation contracts and helpers. - Log separator written at the start of each CLI invocation now redacts values for secret-bearing options (e.g. `--idp-client-secret`) so they are not written to the log file in plain text. - Authentication context (tenant and user) is now logged at the `Information` level whenever the resolved sign-in identity changes, giving operators a clear audit trail in the log file of who the CLI is acting as, without exposing credentials. diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/ProjectBuildRequirementCheck.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/ProjectBuildRequirementCheck.cs index 625a53bc..6cf7165b 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/ProjectBuildRequirementCheck.cs +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/ProjectBuildRequirementCheck.cs @@ -429,11 +429,11 @@ private static void WriteBuildLog(string logPath, CommandResult? result, string? return "uv"; } - // Try to install uv via pip - logger.LogDebug("uv not found, attempting to install via pip"); + // Try to install uv via pip (use python -m pip to target the active interpreter) + logger.LogDebug("uv not found, attempting to install via python -m pip"); var installResult = await _commandExecutor.ExecuteAsync( - "pip", "install uv", + "python", "-m pip install uv", captureOutput: true, suppressErrorLogging: true, cancellationToken: cancellationToken); diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/TelemetryRequirementCheck.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/TelemetryRequirementCheck.cs index d6e8eeb1..d962f0c1 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/TelemetryRequirementCheck.cs +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/TelemetryRequirementCheck.cs @@ -105,10 +105,9 @@ private Task CheckImplementationAsync( string[] logLines; try { - var allLines = File.ReadAllLines(_agentConsoleLogPath); - logLines = allLines.Length > MaxTelemetryLines - ? allLines[^MaxTelemetryLines..] - : allLines; + logLines = File.ReadLines(_agentConsoleLogPath) + .TakeLast(MaxTelemetryLines) + .ToArray(); } catch (Exception ex) { diff --git a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/LocalRuntimeRequirementCheckTests.cs b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/LocalRuntimeRequirementCheckTests.cs index dad40b56..8ba78755 100644 --- a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/LocalRuntimeRequirementCheckTests.cs +++ b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/LocalRuntimeRequirementCheckTests.cs @@ -21,6 +21,7 @@ public class LocalRuntimeRequirementCheckTests : IDisposable private readonly PlatformDetector _platformDetector; private readonly IProcessService _processService; private readonly string _tempDir; + private readonly List _disposables = new(); public LocalRuntimeRequirementCheckTests() { @@ -33,6 +34,11 @@ public LocalRuntimeRequirementCheckTests() public void Dispose() { + foreach (var disposable in _disposables) + { + disposable.Dispose(); + } + if (Directory.Exists(_tempDir)) { Directory.Delete(_tempDir, recursive: true); @@ -42,7 +48,10 @@ public void Dispose() private LocalRuntimeRequirementCheck CreateCheck(HttpMessageHandler? handler = null) { var httpClient = handler is not null ? new HttpClient(handler) : new HttpClient(); - return new LocalRuntimeRequirementCheck(_platformDetector, _processService, httpClient); + _disposables.Add(httpClient); + var check = new LocalRuntimeRequirementCheck(_platformDetector, _processService, httpClient); + _disposables.Add(check); + return check; } [Fact] From 51fa6c725f83b3fbade6ac379af03453527a989f Mon Sep 17 00:00:00 2001 From: Code Review Bot Date: Tue, 23 Jun 2026 14:32:15 -0700 Subject: [PATCH 26/27] Address PR #456 round 2: restore, await handler, boot log, test fixes - Add dotnet restore step before --no-restore build so fresh clones don't fail with missing assets file - Await HandleRequestAsync in ListenLoopAsync instead of fire-and-forget for deterministic error handling - Write boot log and include BootLogFile in metadata on success path (not just failure) for diagnosing flaky startups - Fix misleading test comment about SplitIntoSpanBlocks behavior - Update ProjectBuildRequirementCheckTests to mock dotnet restore separately from build Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../LocalRuntimeRequirementCheck.cs | 4 ++- .../ProjectBuildRequirementCheck.cs | 34 +++++++++++++++++++ .../HttpListenerBotCallbackReceiver.cs | 2 +- .../ProjectBuildRequirementCheckTests.cs | 34 ++++++++++++++++++- .../TelemetryRequirementCheckTests.cs | 4 +-- 5 files changed, 73 insertions(+), 5 deletions(-) diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/LocalRuntimeRequirementCheck.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/LocalRuntimeRequirementCheck.cs index fde44da7..3f8fd58b 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/LocalRuntimeRequirementCheck.cs +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/LocalRuntimeRequirementCheck.cs @@ -367,6 +367,7 @@ private async Task SpawnAndProbeAsync( { stopwatch.Stop(); logger.LogDebug("Health endpoint returned {StatusCode}", (int)response.StatusCode); + WriteBootLog(bootLogFile, outputLines, errorLines); return new RequirementCheckResult { Passed = true, @@ -375,7 +376,8 @@ private async Task SpawnAndProbeAsync( { Port = port, BootMs = stopwatch.ElapsedMilliseconds, - Platform = platform.ToString() + Platform = platform.ToString(), + BootLogFile = bootLogFile } }; } diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/ProjectBuildRequirementCheck.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/ProjectBuildRequirementCheck.cs index 6cf7165b..aa1f62f7 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/ProjectBuildRequirementCheck.cs +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Services/Requirements/RequirementChecks/ProjectBuildRequirementCheck.cs @@ -295,10 +295,44 @@ private static void WriteBuildLog(string logPath, CommandResult? result, string? { ProjectPlatform.Python => await InstallPythonDependenciesAsync(projectPath, logger, cancellationToken), ProjectPlatform.NodeJs => await InstallNodeDependenciesAsync(projectPath, logger, cancellationToken), + ProjectPlatform.DotNet => await RestoreDotNetDependenciesAsync(projectPath, logger, cancellationToken), _ => (null, null) }; } + /// + /// Runs dotnet restore so that the subsequent --no-restore build has a valid assets file. + /// + private async Task<(RequirementCheckResult? FailureResult, CommandResult? Output)> RestoreDotNetDependenciesAsync( + string projectPath, + ILogger logger, + CancellationToken cancellationToken) + { + logger.LogDebug("Running dotnet restore in {Path}", projectPath); + + var result = await _commandExecutor.ExecuteAsync( + "dotnet", "restore", + workingDirectory: projectPath, + captureOutput: true, + suppressErrorLogging: true, + cancellationToken: cancellationToken); + + if (result.Success) + { + logger.LogDebug("dotnet restore completed successfully"); + return (null, result); + } + + var summary = ExtractBuildErrorSummary(result, ProjectPlatform.DotNet); + + return (new RequirementCheckResult + { + Passed = false, + ErrorMessage = $"Package restore failed (.NET):\n{summary}", + ResolutionGuidance = "Run 'dotnet restore' manually and fix any dependency issues." + }, null); + } + /// /// Runs npm install if a package.json exists and node_modules is missing. /// diff --git a/src/Microsoft.Agents.A365.DevTools.Validation/HttpListenerBotCallbackReceiver.cs b/src/Microsoft.Agents.A365.DevTools.Validation/HttpListenerBotCallbackReceiver.cs index 07dac4df..a51a38d0 100644 --- a/src/Microsoft.Agents.A365.DevTools.Validation/HttpListenerBotCallbackReceiver.cs +++ b/src/Microsoft.Agents.A365.DevTools.Validation/HttpListenerBotCallbackReceiver.cs @@ -132,7 +132,7 @@ private async Task ListenLoopAsync(CancellationToken cancellationToken) try { var context = await _listener.GetContextAsync(); - _ = HandleRequestAsync(context); + await HandleRequestAsync(context); } catch (HttpListenerException) when (cancellationToken.IsCancellationRequested) { diff --git a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/ProjectBuildRequirementCheckTests.cs b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/ProjectBuildRequirementCheckTests.cs index 75d3d5bd..0a77a3be 100644 --- a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/ProjectBuildRequirementCheckTests.cs +++ b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/ProjectBuildRequirementCheckTests.cs @@ -110,6 +110,14 @@ public async Task CheckAsync_WhenDotNetBuildSucceeds_ReturnsSuccess() // Arrange - create a .csproj so PlatformDetector identifies DotNet File.WriteAllText(Path.Combine(_tempDir, "test.csproj"), ""); var config = new Agent365Config { DeploymentProjectPath = _tempDir }; + _commandExecutor.ExecuteAsync( + Arg.Is("dotnet"), + Arg.Is(a => a.Contains("restore")), + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any()) + .Returns(new CommandResult { ExitCode = 0, StandardOutput = "Restore succeeded." }); _commandExecutor.ExecuteAsync( Arg.Is("dotnet"), Arg.Is(a => a.Contains("TreatWarningsAsErrors")), @@ -135,7 +143,15 @@ public async Task CheckAsync_WhenDotNetBuildFails_ReturnsFailure() var config = new Agent365Config { DeploymentProjectPath = _tempDir }; _commandExecutor.ExecuteAsync( Arg.Is("dotnet"), + Arg.Is(a => a.Contains("restore")), Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any()) + .Returns(new CommandResult { ExitCode = 0, StandardOutput = "Restore succeeded." }); + _commandExecutor.ExecuteAsync( + Arg.Is("dotnet"), + Arg.Is(a => a.Contains("TreatWarningsAsErrors")), Arg.Any(), Arg.Any(), Arg.Any(), @@ -162,7 +178,15 @@ public async Task CheckAsync_WhenDotNetBuildHasWarningsTreatedAsErrors_ReturnsFa var config = new Agent365Config { DeploymentProjectPath = _tempDir }; _commandExecutor.ExecuteAsync( Arg.Is("dotnet"), + Arg.Is(a => a.Contains("restore")), Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any()) + .Returns(new CommandResult { ExitCode = 0, StandardOutput = "Restore succeeded." }); + _commandExecutor.ExecuteAsync( + Arg.Is("dotnet"), + Arg.Is(a => a.Contains("TreatWarningsAsErrors")), Arg.Any(), Arg.Any(), Arg.Any(), @@ -264,8 +288,16 @@ public async Task CheckAsync_WhenBuildFailsWithNoOutput_ReportsExitCode() File.WriteAllText(Path.Combine(_tempDir, "test.csproj"), ""); var config = new Agent365Config { DeploymentProjectPath = _tempDir }; _commandExecutor.ExecuteAsync( + Arg.Is("dotnet"), + Arg.Is(a => a.Contains("restore")), Arg.Any(), - Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any()) + .Returns(new CommandResult { ExitCode = 0, StandardOutput = "Restore succeeded." }); + _commandExecutor.ExecuteAsync( + Arg.Is("dotnet"), + Arg.Is(a => a.Contains("TreatWarningsAsErrors")), Arg.Any(), Arg.Any(), Arg.Any(), diff --git a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/TelemetryRequirementCheckTests.cs b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/TelemetryRequirementCheckTests.cs index 6587c3ef..beb509ff 100644 --- a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/TelemetryRequirementCheckTests.cs +++ b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/TelemetryRequirementCheckTests.cs @@ -332,8 +332,8 @@ public void SplitIntoSpanBlocks_IncludesLinesBeforeTraceId() var blocks = TelemetryRequirementCheck.SplitIntoSpanBlocks(lines); - // The lines before the first traceId won't be in any block - // instrumentationScope needs to be AFTER traceId or we need to handle this + // SplitIntoSpanBlocks carries lines before the first traceId into the first block, + // so instrumentationScope preceding traceId is preserved in block[0] blocks.Should().HaveCount(1); } From 21c41bb7efb6befc531bb9a97eef3567dee6f82f Mon Sep 17 00:00:00 2001 From: Code Review Bot Date: Tue, 23 Jun 2026 16:45:25 -0700 Subject: [PATCH 27/27] fix(tests): use double quotes for Linux shell commands in CreateFakeProcess Single quotes in ProcessStartInfo.Arguments are passed literally to /bin/sh on Linux, causing 'sleep 60' to be interpreted as a malformed command (exit code 2). Use escaped double quotes instead, which .NET passes correctly to the shell via execve. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Requirements/ConversationRequirementCheckTests.cs | 2 +- .../Requirements/LocalRuntimeRequirementCheckTests.cs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/ConversationRequirementCheckTests.cs b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/ConversationRequirementCheckTests.cs index 730653aa..39d95024 100644 --- a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/ConversationRequirementCheckTests.cs +++ b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/ConversationRequirementCheckTests.cs @@ -575,7 +575,7 @@ private static Process CreateFakeProcess(bool exitImmediately, int exitCode = 0) FileName = OperatingSystem.IsWindows() ? "cmd.exe" : "/bin/sh", Arguments = OperatingSystem.IsWindows() ? (exitImmediately ? "/c exit 1" : "/c ping -n 60 127.0.0.1 >nul") - : (exitImmediately ? "-c 'exit 1'" : "-c 'sleep 60'"), + : (exitImmediately ? $"-c \"exit {exitCode}\"" : "-c \"sleep 60\""), RedirectStandardOutput = true, RedirectStandardError = true, UseShellExecute = false, diff --git a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/LocalRuntimeRequirementCheckTests.cs b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/LocalRuntimeRequirementCheckTests.cs index 8ba78755..ceb4e155 100644 --- a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/LocalRuntimeRequirementCheckTests.cs +++ b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Services/Requirements/LocalRuntimeRequirementCheckTests.cs @@ -238,7 +238,7 @@ public async Task CheckAsync_NodeJsProject_UsesNpmStart() var result = await check.CheckAsync(config, _logger); // Assert - result.Passed.Should().BeTrue(); + result.Passed.Should().BeTrue(because: "Node.js project with health endpoint responding should pass"); if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { _processService.Received(1).Start(Arg.Is(p => @@ -297,7 +297,7 @@ private static Process CreateFakeProcess(bool exitImmediately, int exitCode = 0) FileName = OperatingSystem.IsWindows() ? "cmd.exe" : "/bin/sh", Arguments = OperatingSystem.IsWindows() ? (exitImmediately ? "/c exit 1" : "/c ping -n 60 127.0.0.1 >nul") - : (exitImmediately ? "-c 'exit 1'" : "-c 'sleep 60'"), + : (exitImmediately ? $"-c \"exit {exitCode}\"" : "-c \"sleep 60\""), RedirectStandardOutput = true, RedirectStandardError = true, UseShellExecute = false,