[codex] Enforce MAF modernization workflow eval coverage

.github/skills/khepri-modernization-workflow/SKILL.md

@@ -19,10 +19,11 @@ Use this skill as the entrypoint for the Project Khepri modernization workflow.
 
 1. Read this skill when a task asks to run, inspect, enforce, or update the Khepri modernization workflow.
 2. Use `ModernizationWorkflow.CreateContract()` for stage order, required evidence, required agents, and AgentEvals gates.
-3. Use `GitHubCopilotModernizationAgentRegistry.CreateSessionConfig(...)` when configuring a GHCP SDK session.
-4. Use `ModernizationWorkflow.BuildMicrosoftAgentFrameworkWorkflow(...)` for the full sequential workflow.
-5. Use `ModernizationWorkflow.BuildIncrementSquadWorkflow(...)` for per-increment app/data/infra/security squad generation.
-6. Do not copy the workflow stages into new prompt-only logic. Change the .NET source of truth when the workflow contract changes.
+3. Use `ModernizationWorkflow.CreateAgentCallPlan()` to inspect the enforced stage-by-stage registered agent and subagent calls.
+4. Use `GitHubCopilotModernizationAgentRegistry.CreateSessionConfig(...)` when configuring a GHCP SDK session.
+5. Use `ModernizationWorkflow.BuildMicrosoftAgentFrameworkWorkflow(...)` for the full sequential workflow.
+6. Use `ModernizationWorkflow.BuildIncrementSquadWorkflow(...)` for per-increment app/data/infra/security squad generation.
+7. Do not copy the workflow stages into new prompt-only logic. Change the .NET source of truth when the workflow contract changes.
 
 ## Validation
 

dotnet/src/Modernization/Workflow/ModernizationWorkflow.cs

@@ -43,6 +43,12 @@ public sealed record AgentEvalRequirement(
     string EvaluatorType,
     string Purpose);
 
+public sealed record ModernizationStageAgentCall(
+    string StageId,
+    IReadOnlyList<string> AgentNames,
+    IReadOnlyList<AgentEvalRequirement> RequiredAgentEvals,
+    IReadOnlyList<string> RequiredEvidence);
+
 public sealed record AtomicModernizationStepContract(
     string StageId,
     string SmallestUnit,
@@ -107,24 +113,6 @@ public static class ModernizationWorkflow
         SquadGeneratorAgentName
     ];
 
-    private static readonly string[] SequentialAgentOrder =
-    [
-        OrchestratorAgentName,
-        EvolutionAgentName,
-        SpecAgentName,
-        KnowledgeAgentName,
-        AppModernizationAgentName,
-        DataModernizationAgentName,
-        InfraModernizationAgentName,
-        SecurityModernizationAgentName,
-        PlannerAgentName,
-        SquadGeneratorAgentName,
-        ScaffoldAgentName,
-        CodeAgentName,
-        TestAgentName,
-        AssessorAgentName
-    ];
-
     private static readonly string[] AreaModernizationAgents =
     [
         AppModernizationAgentName,
@@ -144,19 +132,34 @@ public static ModernizationWorkflowContract CreateContract()
                     "Generate or extract requirements, specifications, executable tests, and queryable knowledge-base entries from existing legacy systems before target work begins.",
                     [SpecAgentName, KnowledgeAgentName, TestAgentName],
                     [],
-                    ["source evidence", "legacy behavior inventory", "legacy regression seed tests", "legacy queryable knowledge base"]),
+                    ["source evidence", "legacy behavior inventory", "legacy regression seed tests", "legacy queryable knowledge base"])
+                {
+                    RequiredAgentEvals = StageAgentEvals(
+                        "legacy-requirements-specs-tests",
+                        "legacy requirements/specs/tests extraction")
+                },
                 new ModernizationWorkflowStage(
                     "target-requirements-specs-test-plans",
                     "Generate or extract requirements, specifications, test plans, and queryable knowledge-base entries from target desired-state systems and standards.",
                     [SpecAgentName, KnowledgeAgentName, PlannerAgentName],
                     [],
-                    ["target desired state evidence", "acceptance criteria", "test-PLANS", "target queryable knowledge base"]),
+                    ["target desired state evidence", "acceptance criteria", "test-PLANS", "target queryable knowledge base"])
+                {
+                    RequiredAgentEvals = StageAgentEvals(
+                        "target-requirements-specs-test-plans",
+                        "target requirements/specs/test-PLANS extraction")
+                },
                 new ModernizationWorkflowStage(
                     "incremental-modernization-plan",
                     "Generate the high-level incremental modernization plan from the legacy and target specs.",
                     [PlannerAgentName, AppModernizationAgentName, DataModernizationAgentName, InfraModernizationAgentName, SecurityModernizationAgentName],
                     [ModernizationArea.App, ModernizationArea.Data, ModernizationArea.Infra, ModernizationArea.Security],
-                    ["increment map", "area risks", "security risks", "approval checkpoints"]),
+                    ["increment map", "area risks", "security risks", "approval checkpoints"])
+                {
+                    RequiredAgentEvals = StageAgentEvals(
+                        "incremental-modernization-plan",
+                        "high-level incremental modernization planning")
+                },
                 new ModernizationWorkflowStage(
                     "increment-area-squads",
                     "Use the dedicated squad generator to generate specialized app, data, infra, and security squads for each increment, using a TDD loop with AgentEvals from agentevals.io before implementation.",
@@ -166,14 +169,9 @@ public static ModernizationWorkflowContract CreateContract()
                 {
                     RequiredAgentEvals =
                     [
-                        new AgentEvalRequirement(
-                            "tool-calling",
-                            "tool_trajectory",
-                            "Proves generated squads call the required modernization agents, tools, and handoffs in the intended order."),
-                        new AgentEvalRequirement(
-                            "relevance",
-                            "llm_judge",
-                            "Proves generated squad recommendations stay relevant to the active increment, legacy behavior, and target desired state."),
+                        ..StageAgentEvals(
+                            "increment-area-squads",
+                            "increment-specific squad generation"),
                         new AgentEvalRequirement(
                             "squad-member-rubric",
                             "llm_judge",
@@ -189,16 +187,45 @@ public static ModernizationWorkflowContract CreateContract()
                     "Use the generated squads and queryable knowledge base to refine a detailed modernization plan for the current stage.",
                     [KnowledgeAgentName, PlannerAgentName, AppModernizationAgentName, DataModernizationAgentName, InfraModernizationAgentName, SecurityModernizationAgentName],
                     [ModernizationArea.App, ModernizationArea.Data, ModernizationArea.Infra, ModernizationArea.Security],
-                    ["knowledge refinement", "stage-ready plan", "dependencies", "rollback plan", "regression gates"]),
+                    ["knowledge refinement", "stage-ready plan", "dependencies", "rollback plan", "regression gates"])
+                {
+                    RequiredAgentEvals = StageAgentEvals(
+                        "current-stage-plan-refinement",
+                        "current-stage detailed modernization plan refinement")
+                },
                 new ModernizationWorkflowStage(
                     "tdd-modernization-execution",
                     "Act on the current stage plan with TDD, keeping legacy regression checks and queryable knowledge refinement central to every red/green/refactor loop.",
-                    [KnowledgeAgentName, CodeAgentName, TestAgentName, AssessorAgentName],
+                    [KnowledgeAgentName, ScaffoldAgentName, CodeAgentName, TestAgentName, AssessorAgentName],
                     [ModernizationArea.App, ModernizationArea.Data, ModernizationArea.Infra, ModernizationArea.Security],
                     ["legacy regression checks", "red/green/refactor evidence", "knowledge refinement", "AgentEvals rerun", "acceptance evidence"])
+                {
+                    RequiredAgentEvals = StageAgentEvals(
+                        "tdd-modernization-execution",
+                        "legacy-regression-centered TDD execution")
+                }
             ]);
     }
 
+    public static IReadOnlyList<ModernizationStageAgentCall> CreateAgentCallPlan()
+    {
+        return CreateContract().Stages
+            .Select(stage => new ModernizationStageAgentCall(
+                stage.Id,
+                AgentsForStage(stage),
+                stage.RequiredAgentEvals,
+                stage.RequiredEvidence))
+            .ToArray();
+    }
+
+    public static IReadOnlyList<string> CreateAgentExecutionOrder()
+    {
+        return CreateAgentCallPlan()
+            .SelectMany(call => call.AgentNames)
+            .Distinct(StringComparer.Ordinal)
+            .ToArray();
+    }
+
     public static IReadOnlyList<AtomicModernizationStepContract> CreateAtomicStepContracts()
     {
         var stages = CreateContract().Stages.ToDictionary(stage => stage.Id, StringComparer.Ordinal);
@@ -338,7 +365,7 @@ public static AgentFrameworkWorkflow BuildMicrosoftAgentFrameworkWorkflow(IReadO
         ArgumentNullException.ThrowIfNull(registeredAgents);
         return AgentWorkflowBuilder.BuildSequential(
             "khepri-incremental-modernization",
-            SelectAgents(registeredAgents, SequentialAgentOrder));
+            SelectAgents(registeredAgents, CreateAgentExecutionOrder()));
     }
 
     public static AgentFrameworkWorkflow BuildIncrementSquadWorkflow(string incrementId, IReadOnlyDictionary<string, AIAgent> registeredAgents)
@@ -380,6 +407,33 @@ private static IEnumerable<AIAgent> SelectAgents(IReadOnlyDictionary<string, AIA
         }
     }
 
+    private static IReadOnlyList<string> AgentsForStage(ModernizationWorkflowStage stage)
+    {
+        return new[] { OrchestratorAgentName, EvolutionAgentName }
+            .Concat(stage.RequiredAgents)
+            .Distinct(StringComparer.Ordinal)
+            .ToArray();
+    }
+
+    private static IReadOnlyList<AgentEvalRequirement> StageAgentEvals(string stageId, string behavior)
+    {
+        return
+        [
+            new AgentEvalRequirement(
+                "tool-calling",
+                "tool_trajectory",
+                $"Proves registered agents and subagents call the required tools and handoffs for {stageId}: {behavior}."),
+            new AgentEvalRequirement(
+                "relevance",
+                "llm_judge",
+                $"Proves agent outputs stay relevant to {stageId}, the active modernization increment, legacy behavior, and target desired state."),
+            new AgentEvalRequirement(
+                "evidence-completeness",
+                "rubric",
+                $"Grades whether {stageId} produced all required evidence before the workflow advances.")
+        ];
+    }
+
     private static List<ChatMessage> AggregateSquadMessages(IList<List<ChatMessage>> squadOutputs)
     {
         return squadOutputs.SelectMany(messages => messages).ToList();

dotnet/tests/Code2/NL/ModernizationWorkflowTests.cs

@@ -52,6 +52,56 @@ public void ContractRequiresPrebuiltAgentEvalsForSquadImplementation()
         Assert.IsTrue(squadStage.RequiredAgentEvals.Any(eval => eval.Name == "relevance" && eval.EvaluatorType == "llm_judge"));
     }
 
+    [TestMethod]
+    public void EveryWorkflowStageHasAgentEvalCoverageForAgenticBehavior()
+    {
+        var contract = ModernizationWorkflow.CreateContract();
+
+        foreach (var stage in contract.Stages)
+        {
+            Assert.IsTrue(
+                stage.RequiredAgentEvals.Any(eval => eval.EvaluatorType == "tool_trajectory"),
+                $"{stage.Id} must prove registered tool and subagent calls with a tool_trajectory evaluator.");
+            Assert.IsTrue(
+                stage.RequiredAgentEvals.Any(eval => eval.Name == "relevance" && eval.EvaluatorType == "llm_judge"),
+                $"{stage.Id} must prove relevance with an llm_judge evaluator.");
+            Assert.IsTrue(
+                stage.RequiredAgentEvals.Any(eval => eval.EvaluatorType == "rubric"),
+                $"{stage.Id} must have rubric coverage for evidence completeness.");
+        }
+    }
+
+    [TestMethod]
+    public void AgentCallPlanCoversEveryStageAndRegisteredAgent()
+    {
+        var contract = ModernizationWorkflow.CreateContract();
+        var callPlan = ModernizationWorkflow.CreateAgentCallPlan();
+
+        CollectionAssert.AreEqual(
+            contract.Stages.Select(stage => stage.Id).ToArray(),
+            callPlan.Select(call => call.StageId).ToArray());
+
+        foreach (var stage in contract.Stages)
+        {
+            var call = callPlan.Single(item => item.StageId == stage.Id);
+            CollectionAssert.Contains(call.AgentNames.ToArray(), ModernizationWorkflow.OrchestratorAgentName);
+            CollectionAssert.Contains(call.AgentNames.ToArray(), ModernizationWorkflow.EvolutionAgentName);
+
+            foreach (var requiredAgent in stage.RequiredAgents)
+            {
+                CollectionAssert.Contains(call.AgentNames.ToArray(), requiredAgent, $"{stage.Id} must call {requiredAgent}.");
+            }
+
+            CollectionAssert.AreEqual(stage.RequiredEvidence.ToArray(), call.RequiredEvidence.ToArray());
+            CollectionAssert.AreEqual(stage.RequiredAgentEvals.ToArray(), call.RequiredAgentEvals.ToArray());
+        }
+
+        var executionOrder = ModernizationWorkflow.CreateAgentExecutionOrder();
+        CollectionAssert.AreEquivalent(contract.RegisteredAgents.ToArray(), executionOrder.ToArray());
+        Assert.AreEqual(ModernizationWorkflow.OrchestratorAgentName, executionOrder[0]);
+        Assert.AreEqual(ModernizationWorkflow.EvolutionAgentName, executionOrder[1]);
+    }
+
     [TestMethod]
     public void SquadGenerationStageUsesDedicatedAgentEvalTddGenerator()
     {
@@ -213,6 +263,7 @@ public void ModernizationWorkflowSkillCallsTheDotnetWorkflowCode()
         Assert.IsTrue(skill.Contains("dotnet/src/Modernization/Workflow/ModernizationWorkflow.cs", StringComparison.Ordinal));
         Assert.IsTrue(skill.Contains("dotnet/src/Modernization/Workflow/GitHubCopilotModernizationAgentRegistry.cs", StringComparison.Ordinal));
         Assert.IsTrue(skill.Contains("ModernizationWorkflow.CreateContract", StringComparison.Ordinal));
+        Assert.IsTrue(skill.Contains("ModernizationWorkflow.CreateAgentCallPlan", StringComparison.Ordinal));
         Assert.IsTrue(skill.Contains("BuildMicrosoftAgentFrameworkWorkflow", StringComparison.Ordinal));
         Assert.IsTrue(skill.Contains("dotnet test dotnet\\tests\\Code2\\NL\\Code2NL.Tests.csproj", StringComparison.Ordinal));
     }

evals/github-agents/check-khepri-agents.mjs

@@ -1333,6 +1333,7 @@ function checkModernizationWorkflowSkill(state) {
       "dotnet/src/Modernization/Workflow/ModernizationWorkflow.cs",
       "dotnet/src/Modernization/Workflow/GitHubCopilotModernizationAgentRegistry.cs",
       "ModernizationWorkflow.CreateContract",
+      "ModernizationWorkflow.CreateAgentCallPlan",
       "BuildMicrosoftAgentFrameworkWorkflow",
       "BuildIncrementSquadWorkflow",
       "dotnet test dotnet\\tests\\Code2\\NL\\Code2NL.Tests.csproj",
@@ -1408,6 +1409,10 @@ function checkMafGhcpModernizationWorkflow(state) {
     "SessionConfig",
     "CustomAgents",
     "AIAgent",
+    "ModernizationStageAgentCall",
+    "CreateAgentCallPlan",
+    "CreateAgentExecutionOrder",
+    "StageAgentEvals",
     "RequiredAgentEvals",
     "AgentWorkflowBuilder.BuildSequential",
     "AgentWorkflowBuilder.BuildConcurrent"
@@ -1431,6 +1436,8 @@ function checkMafGhcpModernizationWorkflow(state) {
     "agentevals.io",
     "tool_trajectory",
     "llm_judge",
+    "rubric",
+    "evidence-completeness",
     "tool-calling",
     "relevance",
     "legacy regression",

evals/github-agents/khepri-github-agents.eval.yaml

@@ -623,8 +623,11 @@ tests:
         - agentevals.io
         - tool_trajectory
         - llm_judge
+        - rubric
+        - CreateAgentCallPlan
         - tool-calling
         - relevance
+        - evidence-completeness
         - legacy regression
         - red/green/refactor
     assertions: