From a780b4ee952df2c601208739a11790b535dad9d7 Mon Sep 17 00:00:00 2001 From: Tyler Kendrick Date: Thu, 14 May 2026 14:45:39 -0500 Subject: [PATCH] Enforce MAF modernization workflow eval coverage --- .../khepri-modernization-workflow/SKILL.md | 9 +- .../Workflow/ModernizationWorkflow.cs | 118 +++++++++++++----- .../Code2/NL/ModernizationWorkflowTests.cs | 51 ++++++++ evals/github-agents/check-khepri-agents.mjs | 7 ++ .../khepri-github-agents.eval.yaml | 3 + src/modernizationWorkflow.ts | 46 +++++++ tests/modernizationWorkflow.test.ts | 34 +++++ 7 files changed, 232 insertions(+), 36 deletions(-) diff --git a/.github/skills/khepri-modernization-workflow/SKILL.md b/.github/skills/khepri-modernization-workflow/SKILL.md index e8113ad..2e0c69f 100644 --- a/.github/skills/khepri-modernization-workflow/SKILL.md +++ b/.github/skills/khepri-modernization-workflow/SKILL.md @@ -19,10 +19,11 @@ Use this skill as the entrypoint for the Project Khepri modernization workflow. 1. Read this skill when a task asks to run, inspect, enforce, or update the Khepri modernization workflow. 2. Use `ModernizationWorkflow.CreateContract()` for stage order, required evidence, required agents, and AgentEvals gates. -3. Use `GitHubCopilotModernizationAgentRegistry.CreateSessionConfig(...)` when configuring a GHCP SDK session. -4. Use `ModernizationWorkflow.BuildMicrosoftAgentFrameworkWorkflow(...)` for the full sequential workflow. -5. Use `ModernizationWorkflow.BuildIncrementSquadWorkflow(...)` for per-increment app/data/infra/security squad generation. -6. Do not copy the workflow stages into new prompt-only logic. Change the .NET source of truth when the workflow contract changes. +3. Use `ModernizationWorkflow.CreateAgentCallPlan()` to inspect the enforced stage-by-stage registered agent and subagent calls. +4. Use `GitHubCopilotModernizationAgentRegistry.CreateSessionConfig(...)` when configuring a GHCP SDK session. +5. Use `ModernizationWorkflow.BuildMicrosoftAgentFrameworkWorkflow(...)` for the full sequential workflow. +6. Use `ModernizationWorkflow.BuildIncrementSquadWorkflow(...)` for per-increment app/data/infra/security squad generation. +7. Do not copy the workflow stages into new prompt-only logic. Change the .NET source of truth when the workflow contract changes. ## Validation diff --git a/dotnet/src/Modernization/Workflow/ModernizationWorkflow.cs b/dotnet/src/Modernization/Workflow/ModernizationWorkflow.cs index b7d0f21..4184c04 100644 --- a/dotnet/src/Modernization/Workflow/ModernizationWorkflow.cs +++ b/dotnet/src/Modernization/Workflow/ModernizationWorkflow.cs @@ -43,6 +43,12 @@ public sealed record AgentEvalRequirement( string EvaluatorType, string Purpose); +public sealed record ModernizationStageAgentCall( + string StageId, + IReadOnlyList AgentNames, + IReadOnlyList RequiredAgentEvals, + IReadOnlyList RequiredEvidence); + public sealed record AtomicModernizationStepContract( string StageId, string SmallestUnit, @@ -107,24 +113,6 @@ public static class ModernizationWorkflow SquadGeneratorAgentName ]; - private static readonly string[] SequentialAgentOrder = - [ - OrchestratorAgentName, - EvolutionAgentName, - SpecAgentName, - KnowledgeAgentName, - AppModernizationAgentName, - DataModernizationAgentName, - InfraModernizationAgentName, - SecurityModernizationAgentName, - PlannerAgentName, - SquadGeneratorAgentName, - ScaffoldAgentName, - CodeAgentName, - TestAgentName, - AssessorAgentName - ]; - private static readonly string[] AreaModernizationAgents = [ AppModernizationAgentName, @@ -144,19 +132,34 @@ public static ModernizationWorkflowContract CreateContract() "Generate or extract requirements, specifications, executable tests, and queryable knowledge-base entries from existing legacy systems before target work begins.", [SpecAgentName, KnowledgeAgentName, TestAgentName], [], - ["source evidence", "legacy behavior inventory", "legacy regression seed tests", "legacy queryable knowledge base"]), + ["source evidence", "legacy behavior inventory", "legacy regression seed tests", "legacy queryable knowledge base"]) + { + RequiredAgentEvals = StageAgentEvals( + "legacy-requirements-specs-tests", + "legacy requirements/specs/tests extraction") + }, new ModernizationWorkflowStage( "target-requirements-specs-test-plans", "Generate or extract requirements, specifications, test plans, and queryable knowledge-base entries from target desired-state systems and standards.", [SpecAgentName, KnowledgeAgentName, PlannerAgentName], [], - ["target desired state evidence", "acceptance criteria", "test-PLANS", "target queryable knowledge base"]), + ["target desired state evidence", "acceptance criteria", "test-PLANS", "target queryable knowledge base"]) + { + RequiredAgentEvals = StageAgentEvals( + "target-requirements-specs-test-plans", + "target requirements/specs/test-PLANS extraction") + }, new ModernizationWorkflowStage( "incremental-modernization-plan", "Generate the high-level incremental modernization plan from the legacy and target specs.", [PlannerAgentName, AppModernizationAgentName, DataModernizationAgentName, InfraModernizationAgentName, SecurityModernizationAgentName], [ModernizationArea.App, ModernizationArea.Data, ModernizationArea.Infra, ModernizationArea.Security], - ["increment map", "area risks", "security risks", "approval checkpoints"]), + ["increment map", "area risks", "security risks", "approval checkpoints"]) + { + RequiredAgentEvals = StageAgentEvals( + "incremental-modernization-plan", + "high-level incremental modernization planning") + }, new ModernizationWorkflowStage( "increment-area-squads", "Use the dedicated squad generator to generate specialized app, data, infra, and security squads for each increment, using a TDD loop with AgentEvals from agentevals.io before implementation.", @@ -166,14 +169,9 @@ public static ModernizationWorkflowContract CreateContract() { RequiredAgentEvals = [ - new AgentEvalRequirement( - "tool-calling", - "tool_trajectory", - "Proves generated squads call the required modernization agents, tools, and handoffs in the intended order."), - new AgentEvalRequirement( - "relevance", - "llm_judge", - "Proves generated squad recommendations stay relevant to the active increment, legacy behavior, and target desired state."), + ..StageAgentEvals( + "increment-area-squads", + "increment-specific squad generation"), new AgentEvalRequirement( "squad-member-rubric", "llm_judge", @@ -189,16 +187,45 @@ public static ModernizationWorkflowContract CreateContract() "Use the generated squads and queryable knowledge base to refine a detailed modernization plan for the current stage.", [KnowledgeAgentName, PlannerAgentName, AppModernizationAgentName, DataModernizationAgentName, InfraModernizationAgentName, SecurityModernizationAgentName], [ModernizationArea.App, ModernizationArea.Data, ModernizationArea.Infra, ModernizationArea.Security], - ["knowledge refinement", "stage-ready plan", "dependencies", "rollback plan", "regression gates"]), + ["knowledge refinement", "stage-ready plan", "dependencies", "rollback plan", "regression gates"]) + { + RequiredAgentEvals = StageAgentEvals( + "current-stage-plan-refinement", + "current-stage detailed modernization plan refinement") + }, new ModernizationWorkflowStage( "tdd-modernization-execution", "Act on the current stage plan with TDD, keeping legacy regression checks and queryable knowledge refinement central to every red/green/refactor loop.", - [KnowledgeAgentName, CodeAgentName, TestAgentName, AssessorAgentName], + [KnowledgeAgentName, ScaffoldAgentName, CodeAgentName, TestAgentName, AssessorAgentName], [ModernizationArea.App, ModernizationArea.Data, ModernizationArea.Infra, ModernizationArea.Security], ["legacy regression checks", "red/green/refactor evidence", "knowledge refinement", "AgentEvals rerun", "acceptance evidence"]) + { + RequiredAgentEvals = StageAgentEvals( + "tdd-modernization-execution", + "legacy-regression-centered TDD execution") + } ]); } + public static IReadOnlyList CreateAgentCallPlan() + { + return CreateContract().Stages + .Select(stage => new ModernizationStageAgentCall( + stage.Id, + AgentsForStage(stage), + stage.RequiredAgentEvals, + stage.RequiredEvidence)) + .ToArray(); + } + + public static IReadOnlyList CreateAgentExecutionOrder() + { + return CreateAgentCallPlan() + .SelectMany(call => call.AgentNames) + .Distinct(StringComparer.Ordinal) + .ToArray(); + } + public static IReadOnlyList CreateAtomicStepContracts() { var stages = CreateContract().Stages.ToDictionary(stage => stage.Id, StringComparer.Ordinal); @@ -338,7 +365,7 @@ public static AgentFrameworkWorkflow BuildMicrosoftAgentFrameworkWorkflow(IReadO ArgumentNullException.ThrowIfNull(registeredAgents); return AgentWorkflowBuilder.BuildSequential( "khepri-incremental-modernization", - SelectAgents(registeredAgents, SequentialAgentOrder)); + SelectAgents(registeredAgents, CreateAgentExecutionOrder())); } public static AgentFrameworkWorkflow BuildIncrementSquadWorkflow(string incrementId, IReadOnlyDictionary registeredAgents) @@ -380,6 +407,33 @@ private static IEnumerable SelectAgents(IReadOnlyDictionary AgentsForStage(ModernizationWorkflowStage stage) + { + return new[] { OrchestratorAgentName, EvolutionAgentName } + .Concat(stage.RequiredAgents) + .Distinct(StringComparer.Ordinal) + .ToArray(); + } + + private static IReadOnlyList StageAgentEvals(string stageId, string behavior) + { + return + [ + new AgentEvalRequirement( + "tool-calling", + "tool_trajectory", + $"Proves registered agents and subagents call the required tools and handoffs for {stageId}: {behavior}."), + new AgentEvalRequirement( + "relevance", + "llm_judge", + $"Proves agent outputs stay relevant to {stageId}, the active modernization increment, legacy behavior, and target desired state."), + new AgentEvalRequirement( + "evidence-completeness", + "rubric", + $"Grades whether {stageId} produced all required evidence before the workflow advances.") + ]; + } + private static List AggregateSquadMessages(IList> squadOutputs) { return squadOutputs.SelectMany(messages => messages).ToList(); diff --git a/dotnet/tests/Code2/NL/ModernizationWorkflowTests.cs b/dotnet/tests/Code2/NL/ModernizationWorkflowTests.cs index 3be0659..325f078 100644 --- a/dotnet/tests/Code2/NL/ModernizationWorkflowTests.cs +++ b/dotnet/tests/Code2/NL/ModernizationWorkflowTests.cs @@ -52,6 +52,56 @@ public void ContractRequiresPrebuiltAgentEvalsForSquadImplementation() Assert.IsTrue(squadStage.RequiredAgentEvals.Any(eval => eval.Name == "relevance" && eval.EvaluatorType == "llm_judge")); } + [TestMethod] + public void EveryWorkflowStageHasAgentEvalCoverageForAgenticBehavior() + { + var contract = ModernizationWorkflow.CreateContract(); + + foreach (var stage in contract.Stages) + { + Assert.IsTrue( + stage.RequiredAgentEvals.Any(eval => eval.EvaluatorType == "tool_trajectory"), + $"{stage.Id} must prove registered tool and subagent calls with a tool_trajectory evaluator."); + Assert.IsTrue( + stage.RequiredAgentEvals.Any(eval => eval.Name == "relevance" && eval.EvaluatorType == "llm_judge"), + $"{stage.Id} must prove relevance with an llm_judge evaluator."); + Assert.IsTrue( + stage.RequiredAgentEvals.Any(eval => eval.EvaluatorType == "rubric"), + $"{stage.Id} must have rubric coverage for evidence completeness."); + } + } + + [TestMethod] + public void AgentCallPlanCoversEveryStageAndRegisteredAgent() + { + var contract = ModernizationWorkflow.CreateContract(); + var callPlan = ModernizationWorkflow.CreateAgentCallPlan(); + + CollectionAssert.AreEqual( + contract.Stages.Select(stage => stage.Id).ToArray(), + callPlan.Select(call => call.StageId).ToArray()); + + foreach (var stage in contract.Stages) + { + var call = callPlan.Single(item => item.StageId == stage.Id); + CollectionAssert.Contains(call.AgentNames.ToArray(), ModernizationWorkflow.OrchestratorAgentName); + CollectionAssert.Contains(call.AgentNames.ToArray(), ModernizationWorkflow.EvolutionAgentName); + + foreach (var requiredAgent in stage.RequiredAgents) + { + CollectionAssert.Contains(call.AgentNames.ToArray(), requiredAgent, $"{stage.Id} must call {requiredAgent}."); + } + + CollectionAssert.AreEqual(stage.RequiredEvidence.ToArray(), call.RequiredEvidence.ToArray()); + CollectionAssert.AreEqual(stage.RequiredAgentEvals.ToArray(), call.RequiredAgentEvals.ToArray()); + } + + var executionOrder = ModernizationWorkflow.CreateAgentExecutionOrder(); + CollectionAssert.AreEquivalent(contract.RegisteredAgents.ToArray(), executionOrder.ToArray()); + Assert.AreEqual(ModernizationWorkflow.OrchestratorAgentName, executionOrder[0]); + Assert.AreEqual(ModernizationWorkflow.EvolutionAgentName, executionOrder[1]); + } + [TestMethod] public void SquadGenerationStageUsesDedicatedAgentEvalTddGenerator() { @@ -213,6 +263,7 @@ public void ModernizationWorkflowSkillCallsTheDotnetWorkflowCode() Assert.IsTrue(skill.Contains("dotnet/src/Modernization/Workflow/ModernizationWorkflow.cs", StringComparison.Ordinal)); Assert.IsTrue(skill.Contains("dotnet/src/Modernization/Workflow/GitHubCopilotModernizationAgentRegistry.cs", StringComparison.Ordinal)); Assert.IsTrue(skill.Contains("ModernizationWorkflow.CreateContract", StringComparison.Ordinal)); + Assert.IsTrue(skill.Contains("ModernizationWorkflow.CreateAgentCallPlan", StringComparison.Ordinal)); Assert.IsTrue(skill.Contains("BuildMicrosoftAgentFrameworkWorkflow", StringComparison.Ordinal)); Assert.IsTrue(skill.Contains("dotnet test dotnet\\tests\\Code2\\NL\\Code2NL.Tests.csproj", StringComparison.Ordinal)); } diff --git a/evals/github-agents/check-khepri-agents.mjs b/evals/github-agents/check-khepri-agents.mjs index 619e3f0..edbd1a5 100644 --- a/evals/github-agents/check-khepri-agents.mjs +++ b/evals/github-agents/check-khepri-agents.mjs @@ -1333,6 +1333,7 @@ function checkModernizationWorkflowSkill(state) { "dotnet/src/Modernization/Workflow/ModernizationWorkflow.cs", "dotnet/src/Modernization/Workflow/GitHubCopilotModernizationAgentRegistry.cs", "ModernizationWorkflow.CreateContract", + "ModernizationWorkflow.CreateAgentCallPlan", "BuildMicrosoftAgentFrameworkWorkflow", "BuildIncrementSquadWorkflow", "dotnet test dotnet\\tests\\Code2\\NL\\Code2NL.Tests.csproj", @@ -1408,6 +1409,10 @@ function checkMafGhcpModernizationWorkflow(state) { "SessionConfig", "CustomAgents", "AIAgent", + "ModernizationStageAgentCall", + "CreateAgentCallPlan", + "CreateAgentExecutionOrder", + "StageAgentEvals", "RequiredAgentEvals", "AgentWorkflowBuilder.BuildSequential", "AgentWorkflowBuilder.BuildConcurrent" @@ -1431,6 +1436,8 @@ function checkMafGhcpModernizationWorkflow(state) { "agentevals.io", "tool_trajectory", "llm_judge", + "rubric", + "evidence-completeness", "tool-calling", "relevance", "legacy regression", diff --git a/evals/github-agents/khepri-github-agents.eval.yaml b/evals/github-agents/khepri-github-agents.eval.yaml index cef40af..9f5eba1 100644 --- a/evals/github-agents/khepri-github-agents.eval.yaml +++ b/evals/github-agents/khepri-github-agents.eval.yaml @@ -623,8 +623,11 @@ tests: - agentevals.io - tool_trajectory - llm_judge + - rubric + - CreateAgentCallPlan - tool-calling - relevance + - evidence-completeness - legacy regression - red/green/refactor assertions: diff --git a/src/modernizationWorkflow.ts b/src/modernizationWorkflow.ts index f6cb23b..86bb53f 100644 --- a/src/modernizationWorkflow.ts +++ b/src/modernizationWorkflow.ts @@ -145,6 +145,22 @@ export function createModernizationWorkflow(options: ModernizationWorkflowOption domain?: ModernizationDomain, ) => addEvent({ type: 'tool.used', state, tool, args, domain }); + const evaluateAgenticBehavior = ( + state: WorkflowState, + behavior: string, + domain?: ModernizationDomain, + ) => + useTool( + state, + 'agent_evals.evaluate_agentic_behavior', + { + behavior, + coverage: '100%', + evaluators: ['tool_trajectory', 'llm_judge', 'rubric'], + }, + domain, + ); + enter('legacy-discovery'); useTool('legacy-discovery', 'microsoft_agent_framework.start_stateful_workflow', { workflow: 'github-copilot-modernization', @@ -163,6 +179,10 @@ export function createModernizationWorkflow(options: ModernizationWorkflowOption addEvent({ type: 'requirements.extracted', state: 'legacy-discovery' }); addEvent({ type: 'spec.created', state: 'legacy-discovery' }); addEvent({ type: 'test_plan.created', state: 'legacy-discovery' }); + evaluateAgenticBehavior( + 'legacy-discovery', + 'legacy requirements, specs, and executable regression tests were extracted from registered legacy systems', + ); enter('target-discovery'); useTool('target-discovery', 'target_system.extract_requirements_specs_test_plans', { @@ -172,6 +192,10 @@ export function createModernizationWorkflow(options: ModernizationWorkflowOption addEvent({ type: 'requirements.extracted', state: 'target-discovery' }); addEvent({ type: 'spec.created', state: 'target-discovery' }); addEvent({ type: 'test_plan.created', state: 'target-discovery' }); + evaluateAgenticBehavior( + 'target-discovery', + 'target desired-state requirements, specs, and test plans were extracted before planning', + ); if (request.ambiguities?.length) { for (const ambiguity of request.ambiguities) { @@ -263,6 +287,10 @@ export function createModernizationWorkflow(options: ModernizationWorkflowOption state: 'modernization-planning', args: { planId: plan.id, incremental: true }, }); + evaluateAgenticBehavior( + 'modernization-planning', + 'registered modernization agents informed a high-level incremental plan from approved specs', + ); enter('plan-persistence'); const tracker = request.repository.trackers[0]?.kind ?? 'registered-project-tracker'; @@ -276,6 +304,10 @@ export function createModernizationWorkflow(options: ModernizationWorkflowOption state: 'plan-persistence', args: { tracker, planId: plan.id }, }); + evaluateAgenticBehavior( + 'plan-persistence', + 'incremental modernization plan was persisted to the registered project tracker', + ); enter('specialist-squad-tdd'); for (const domain of domains) { @@ -305,6 +337,8 @@ export function createModernizationWorkflow(options: ModernizationWorkflowOption }); useTool('specialist-squad-tdd', 'agent_evals.evaluate_squad', { evaluator: 'trajectory_match', + coverage: '100%', + evaluators: ['tool_trajectory', 'llm_judge', 'rubric'], squadId: phase.squadId, expected: ['handoff', 'skill.use', 'spec-driven-tests'], }, domain); @@ -316,6 +350,10 @@ export function createModernizationWorkflow(options: ModernizationWorkflowOption }); } } + evaluateAgenticBehavior( + 'specialist-squad-tdd', + 'phase-dedicated app, infra, and data squads were generated through AgentEvals-backed TDD', + ); enter('incremental-development'); for (const domain of domains) { @@ -348,6 +386,10 @@ export function createModernizationWorkflow(options: ModernizationWorkflowOption }); } } + evaluateAgenticBehavior( + 'incremental-development', + 'generated squads executed the current modernization stage through red-green legacy regression checks', + ); enter('phase-retro'); addEvent({ @@ -358,6 +400,10 @@ export function createModernizationWorkflow(options: ModernizationWorkflowOption promotesPatternsToSkills: true, }, }); + evaluateAgenticBehavior( + 'phase-retro', + 'phase retrospective captured reusable modernization patterns and follow-up eval gaps', + ); enter('complete'); return { diff --git a/tests/modernizationWorkflow.test.ts b/tests/modernizationWorkflow.test.ts index 11172ab..f7fe554 100644 --- a/tests/modernizationWorkflow.test.ts +++ b/tests/modernizationWorkflow.test.ts @@ -12,6 +12,7 @@ import { toAgentEvalMessages, toGraphTrajectory, type ModernizationRequest, + type WorkflowState, } from '../src/modernizationWorkflow'; const repoRoot = path.resolve(__dirname, '..'); @@ -172,6 +173,39 @@ describe('GitHub Copilot modernization workflow', () => { } }); + test('covers every agentic workflow state with AgentEvals before completion', async () => { + const workflow = createModernizationWorkflow({ + now: () => '2026-05-07T15:00:00.000Z', + nextId: (prefix, index) => `${prefix}-${index + 1}`, + }); + + const result = await workflow.run(baseRequest()); + const requiredStates: WorkflowState[] = [ + 'legacy-discovery', + 'target-discovery', + 'modernization-planning', + 'plan-persistence', + 'specialist-squad-tdd', + 'incremental-development', + 'phase-retro', + ]; + const agentEvalEvents = result.events.filter( + (event) => event.type === 'tool.used' && event.tool?.startsWith('agent_evals.'), + ); + const coveredStates = new Set(agentEvalEvents.map((event) => event.state)); + + for (const state of requiredStates) { + expect(coveredStates.has(state), `${state} must have AgentEvals coverage`).toBe(true); + } + + for (const event of agentEvalEvents) { + expect(event.args).toMatchObject({ coverage: '100%' }); + expect(event.args?.evaluators).toEqual( + expect.arrayContaining(['tool_trajectory', 'llm_judge', 'rubric']), + ); + } + }); + test('requires human elicitation and approval before ambiguous specs can drive planning or tests', async () => { const workflow = createModernizationWorkflow({ now: () => '2026-05-07T15:00:00.000Z',