From 2b4ddfc4c632c01035c2582916babe33f43f8cbb Mon Sep 17 00:00:00 2001
From: karimad <19360713+karimad@users.noreply.github.com>
Date: Thu, 6 Aug 2026 21:24:08 +0200
Subject: [PATCH] Add eval scenarios for the agent-skill target
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

#47 added the "agent-skill" (generic) architecture but only covered it
with structural/manifest tests (architecture-registry.test.ts,
catalogue-registry.test.ts) — nothing exercises the real builder LLM
against the new catalogue the way the skillbuilder eval suite does for
scout/cowork.

Adds two scenarios:

- github-issue-triage-agent-skill: clones github-issue-triage-skill
  onto architecture: "agent-skill". Requires the gh CLI (a universal
  primitive) while forbidding any vendor-specific tool name. Ran
  against the real Copilot CLI builder: 100% pass — generalizes to gh
  CLI shell steps with zero product-specific references.

- teams-digest-agent-skill: clones cowork-teams-digest onto
  architecture: "agent-skill" — a harder case, since there's no CLI at
  all for Teams. Confirms the builder doesn't invent a plausible-
  sounding native tool name when none exists in the catalogue; instead
  it correctly falls back to documented Microsoft Graph HTTP calls via
  curl. Ran against the real builder: 100% pass.

Both scenarios are additive only; no production code changed.
---
 evals/skillbuilder/scenarios.ts | 45 +++++++++++++++++++++++++++++++++
 1 file changed, 45 insertions(+)

diff --git a/evals/skillbuilder/scenarios.ts b/evals/skillbuilder/scenarios.ts
index 0c655a5..606b022 100644
--- a/evals/skillbuilder/scenarios.ts
+++ b/evals/skillbuilder/scenarios.ts
@@ -177,6 +177,32 @@ const githubIssueTriage: SkillBuilderScenario = {
   },
 };
 
+/** Same recording as githubIssueTriage, targeting the agent-skill (generic) catalogue: must
+ *  still reach for the gh CLI (a universal primitive — the shell), but the plan must name NO
+ *  product-specific tool ID (no workiq_*, no m365_*, no named browser-tool suite) since the
+ *  destination agent is unknown. */
+const githubIssueTriageAgentSkill: SkillBuilderScenario = {
+  ...githubIssueTriage,
+  id: "github-issue-triage-agent-skill",
+  title: "Triage new bug issues on GitHub (agent skill)",
+  architecture: "agent-skill",
+  rubric: {
+    mustUseAny: [["gh "], ["gh issue", "gh api"]],
+    forbidden: [
+      "playwright",
+      "browser_",
+      "workiq",
+      "m365_",
+      "click",
+      "navigate to github",
+      "github.com/acme",
+    ],
+    minValues: 1,
+    minCalculations: 1,
+    minActions: 1,
+  },
+};
+
 /* --- Cowork (Microsoft 365 Copilot) scenarios ----------------------------- */
 
 /** Read a Teams channel in the web app, summarize, post a digest — must use the m365_teams tool, never the browser. */
@@ -254,6 +280,23 @@ const coworkTeamsDigest: SkillBuilderScenario = {
   },
 };
 
+/** Same Teams recording as coworkTeamsDigest, retargeted to agent-skill — a harder case than
+ *  the gh-CLI scenario since there's no CLI at all for Teams. No proprietary tool exists in the
+ *  agent-skill catalogue for this, so the right generalization is a documented HTTP API call
+ *  (Microsoft Graph) rather than inventing a plausible-sounding native tool name. */
+const teamsDigestAgentSkill: SkillBuilderScenario = {
+  ...coworkTeamsDigest,
+  id: "teams-digest-agent-skill",
+  title: "Post a morning digest to the leads Teams channel (agent skill)",
+  architecture: "agent-skill",
+  rubric: {
+    mustUseAny: [["curl", "web_fetch", "fetch"]],
+    forbidden: ["m365_teams", "workiq", "postchannelmessage", "listchannelmessages", "playwright", "browser_"],
+    minCalculations: 1,
+    minActions: 1,
+  },
+};
+
 /** Triage unread mail in Outlook web and reply — must use the outlook tool, never the browser. */
 const coworkOutlookReply: SkillBuilderScenario = {
   id: "cowork-outlook-reply",
@@ -399,6 +442,8 @@ const coworkCalendarSchedule: SkillBuilderScenario = {
 export const skillScenarios: SkillBuilderScenario[] = [
   priceTracker,
   githubIssueTriage,
+  githubIssueTriageAgentSkill,
+  teamsDigestAgentSkill,
   coworkTeamsDigest,
   coworkOutlookReply,
   coworkCalendarSchedule,