test(e2e): split model and world suites

.changeset/eval-exclude-tag.md

@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+`eve eval` gains a repeatable `--exclude-tag <tag...>` flag that skips evals carrying a tag. Exclusion applies after `--tag` inclusion, and a run where exclusion removes every matching eval now exits successfully with nothing executed. `--list` reports the post-filter selection — `--list --json` prints `[]` when exclusion removes everything — so suite runners can probe whether anything would run.

.github/scripts/discover-e2e-fixtures.mjs

@@ -0,0 +1,100 @@
+// Discover e2e fixture directories for the CI matrices.
+//
+// Models and worlds live in e2e/matrix.json — edit that file to add either.
+//
+// A fixture qualifies when it has an `evals/` directory under one of the
+// fixture roots. Emits one GitHub Actions output per matrix:
+//
+//   model_matrix          `{ name, dir, model_name, model_id }` entries for
+//                         the model suite (e2e-local). Fixtures marked
+//                         `"e2e": { "modelMatrix": "full" }` in package.json
+//                         run on every registry model; all other fixtures run
+//                         once on the default (first) model.
+//   world_matrix_<world>  `{ name, dir[, world_package] }` entries for that
+//                         world's suite workflow, which runs every fixture
+//                         once with mock models (EVE_E2E_MODEL=mock).
+import { appendFileSync, existsSync, readdirSync, readFileSync, statSync } from "node:fs";
+import { join } from "node:path";
+
+const roots = ["e2e/fixtures", "apps/fixtures"];
+
+const registry = JSON.parse(readFileSync("e2e/matrix.json", "utf8"));
+const models = validateNamedEntries(registry.models, "models", ["id"]);
+const worlds = validateNamedEntries(registry.worlds, "worlds", []);
+
+const fixtures = [];
+for (const root of roots) {
+  if (!existsSync(root)) continue;
+  for (const entry of readdirSync(root).sort()) {
+    const dir = join(root, entry);
+    if (!statSync(dir).isDirectory() || !existsSync(join(dir, "evals"))) continue;
+
+    let modelMatrix = "default";
+    const packageJsonPath = join(dir, "package.json");
+    if (existsSync(packageJsonPath)) {
+      const pkg = JSON.parse(readFileSync(packageJsonPath, "utf8"));
+      modelMatrix = pkg.e2e?.modelMatrix ?? "default";
+    }
+    if (modelMatrix !== "default" && modelMatrix !== "full") {
+      throw new Error(`${packageJsonPath}: e2e.modelMatrix must be "default" or "full".`);
+    }
+
+    fixtures.push({ name: entry, dir, modelMatrix });
+  }
+}
+
+if (fixtures.length === 0) {
+  console.error("No e2e fixtures with an evals/ directory were found.");
+  process.exit(1);
+}
+
+const modelMatrix = fixtures.flatMap(({ name, dir, modelMatrix }) =>
+  (modelMatrix === "full" ? models : models.slice(0, 1)).map((model) => ({
+    name,
+    dir,
+    model_name: model.name,
+    model_id: model.id,
+  })),
+);
+
+const outputs = [`model_matrix=${JSON.stringify(modelMatrix)}`];
+for (const world of worlds) {
+  const legs = fixtures.map(({ name, dir }) =>
+    world.package === undefined ? { name, dir } : { name, dir, world_package: world.package },
+  );
+  outputs.push(`world_matrix_${world.name}=${JSON.stringify(legs)}`);
+}
+
+console.error(
+  `Discovered ${fixtures.length} fixtures (${modelMatrix.length} model-suite jobs, ${worlds.length} worlds).`,
+);
+const lines = `${outputs.join("\n")}\n`;
+if (process.env.GITHUB_OUTPUT) {
+  appendFileSync(process.env.GITHUB_OUTPUT, lines);
+} else {
+  process.stdout.write(lines);
+}
+
+function validateNamedEntries(entries, key, requiredFields) {
+  if (!Array.isArray(entries) || entries.length === 0) {
+    throw new Error(`e2e/matrix.json: "${key}" must be a non-empty array.`);
+  }
+  const names = new Set();
+  for (const entry of entries) {
+    for (const field of ["name", ...requiredFields]) {
+      if (typeof entry[field] !== "string" || entry[field].length === 0) {
+        throw new Error(`e2e/matrix.json: every "${key}" entry needs a non-empty "${field}".`);
+      }
+    }
+    if (!/^[a-z0-9-]+$/.test(entry.name)) {
+      throw new Error(
+        `e2e/matrix.json: "${key}" name "${entry.name}" must be lowercase alphanumerics and dashes (it becomes a check identifier).`,
+      );
+    }
+    if (names.has(entry.name)) {
+      throw new Error(`e2e/matrix.json: duplicate "${key}" name "${entry.name}".`);
+    }
+    names.add(entry.name);
+  }
+  return entries;
+}

.github/workflows/e2e-local.yml

@@ -36,7 +36,7 @@ jobs:
           fi
           # HEAD is the PR merge commit; HEAD^1 is the base branch tip, so
           # this diff is exactly the PR's changed files.
-          if git diff --name-only HEAD^1 HEAD | grep -qE '^(packages/eve/|apps/fixtures/|e2e/|pnpm-workspace\.yaml$|pnpm-lock\.yaml$|\.github/workflows/e2e-local\.yml$|\.github/scripts/discover-e2e-fixtures\.sh$)'; then
+          if git diff --name-only HEAD^1 HEAD | grep -qE '^(packages/eve/|apps/fixtures/|e2e/|pnpm-workspace\.yaml$|pnpm-lock\.yaml$|\.github/workflows/e2e-local\.yml$|\.github/scripts/discover-e2e-fixtures\.mjs$)'; then
             echo "relevant=true" >> "$GITHUB_OUTPUT"
           else
             echo "relevant=false" >> "$GITHUB_OUTPUT"
@@ -46,7 +46,7 @@ jobs:
     name: discover-fixtures
     runs-on: ubuntu-latest
     outputs:
-      matrix: ${{ steps.discover.outputs.matrix }}
+      model_matrix: ${{ steps.discover.outputs.model_matrix }}
     steps:
       - name: Checkout code
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
@@ -55,10 +55,15 @@ jobs:
 
       - name: Discover eval fixtures
         id: discover
-        run: .github/scripts/discover-e2e-fixtures.sh
+        run: node .github/scripts/discover-e2e-fixtures.mjs
 
+  # The model suite: every fixture runs here against the local world with
+  # real matrix models. Model-sensitive fixtures (`"e2e": { "modelMatrix":
+  # "full" }` in package.json) run on every matrix model; the rest run once
+  # on the default model. The world suites (e2e-vercel, e2e-postgres) cover
+  # the other workflow worlds with deterministic mock models.
   e2e:
-    name: e2e-local (${{ matrix.fixture.name }}, ${{ matrix.model.name }})
+    name: e2e-local (${{ matrix.name }}, ${{ matrix.model_name }})
     runs-on: ubuntu-latest
     needs: [changes, discover-fixtures]
     # No job-level `if`: the stable aggregate check below requires a successful
@@ -67,16 +72,11 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        fixture: ${{ fromJSON(needs.discover-fixtures.outputs.matrix) }}
-        model:
-          - name: openai-sol
-            id: openai/gpt-5.6-sol
-          - name: anthropic-opus
-            id: anthropic/claude-opus-5
+        include: ${{ fromJSON(needs.discover-fixtures.outputs.model_matrix) }}
 
     env:
       AI_GATEWAY_API_KEY: ${{ secrets.AI_GATEWAY_API_KEY }}
-      EVE_E2E_MODEL: ${{ matrix.model.id }}
+      EVE_E2E_MODEL: ${{ matrix.model_id }}
       EVE_EVAL_JUNIT_DIR: ${{ github.workspace }}/.junit
 
     steps:
@@ -131,15 +131,15 @@ jobs:
         if: needs.changes.outputs.relevant == 'true'
         run: |
           mkdir -p "$EVE_EVAL_JUNIT_DIR"
-          cd "${{ matrix.fixture.dir }}"
+          cd "${{ matrix.dir }}"
           pnpm exec eve eval --strict --verbose \
-            --junit "$EVE_EVAL_JUNIT_DIR/${{ matrix.fixture.name }}-${{ matrix.model.name }}.xml"
+            --junit "$EVE_EVAL_JUNIT_DIR/${{ matrix.name }}-${{ matrix.model_name }}.xml"
 
       - name: Upload eval artifacts
         if: (failure()) && needs.changes.outputs.relevant == 'true'
         uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v6
         with:
-          name: eval-artifacts-${{ matrix.fixture.name }}-${{ matrix.model.name }}
+          name: eval-artifacts-${{ matrix.name }}-${{ matrix.model_name }}
           path: |
             .junit
             e2e/fixtures/*/.eve/evals