diff --git a/.changeset/focused-e2e-world-models.md b/.changeset/focused-e2e-world-models.md new file mode 100644 index 000000000..714d695f0 --- /dev/null +++ b/.changeset/focused-e2e-world-models.md @@ -0,0 +1,5 @@ +--- +"eve": patch +--- + +Expand the internal deterministic model adapter for world-focused e2e coverage, including subagent, repeated-tool, fan-out, silent-completion, and schema-derived tool-input programs. Authored `eve-mock/*` models remain intact when automatic model replacement is enabled, and turn cancellation observed as a durable step returns now wins over ordinary completion. diff --git a/.github/scripts/discover-e2e-fixtures.mjs b/.github/scripts/discover-e2e-fixtures.mjs new file mode 100644 index 000000000..ea294c3b7 --- /dev/null +++ b/.github/scripts/discover-e2e-fixtures.mjs @@ -0,0 +1,334 @@ +#!/usr/bin/env node + +import { appendFile, readFile, readdir, stat } from "node:fs/promises"; +import { basename, join } from "node:path"; + +const FIXTURE_ROOTS = ["e2e/fixtures", "apps/fixtures"]; +const MANIFEST_PATH = "e2e/suites.json"; +const EVAL_SOURCE_FILE_PATTERN = /\.[cm]?[jt]sx?$/u; +const JUDGE_USAGE_PATTERN = /\bt\s*\.\s*judge\b/u; +const MODEL_ALIAS_PATTERN = /^[a-z0-9]+(?:-[a-z0-9]+)*$/u; + +function fail(message) { + throw new Error(message); +} + +function isRecord(value) { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +async function isDirectory(path) { + try { + return (await stat(path)).isDirectory(); + } catch (error) { + if (error?.code === "ENOENT") { + return false; + } + throw error; + } +} + +async function discoverFixtures() { + const fixtureDirs = []; + + for (const root of FIXTURE_ROOTS) { + if (!(await isDirectory(root))) { + continue; + } + + for (const entry of await readdir(root, { withFileTypes: true })) { + if (!entry.isDirectory()) { + continue; + } + + const fixtureDir = join(root, entry.name); + if (await isDirectory(join(fixtureDir, "evals"))) { + fixtureDirs.push(fixtureDir); + } + } + } + + return fixtureDirs.sort(); +} + +async function walkEvalSourceFiles(directory) { + const files = []; + + for (const entry of await readdir(directory, { withFileTypes: true })) { + const path = join(directory, entry.name); + if (entry.isDirectory()) { + files.push(...(await walkEvalSourceFiles(path))); + } else if (entry.isFile() && EVAL_SOURCE_FILE_PATTERN.test(entry.name)) { + files.push(path); + } + } + + return files; +} + +async function assertNoJudgeUsage(fixtureDir) { + for (const path of await walkEvalSourceFiles(join(fixtureDir, "evals"))) { + const source = await readFile(path, "utf8"); + if (JUDGE_USAGE_PATTERN.test(stripCommentsAndStringLiterals(source))) { + fail(`World-suite fixture ${fixtureDir} uses t.judge in ${path}.`); + } + } +} + +function stripCommentsAndStringLiterals(source) { + let code = ""; + let state = "code"; + let templateExpressionDepth = 0; + + for (let index = 0; index < source.length; index += 1) { + const character = source[index]; + const next = source[index + 1]; + + if (state === "code") { + if (templateExpressionDepth > 0 && character === "{") { + code += character; + templateExpressionDepth += 1; + } else if (templateExpressionDepth > 0 && character === "}") { + templateExpressionDepth -= 1; + code += templateExpressionDepth === 0 ? " " : character; + if (templateExpressionDepth === 0) { + state = "template"; + } + } else if (character === "/" && next === "/") { + code += " "; + state = "line-comment"; + index += 1; + } else if (character === "/" && next === "*") { + code += " "; + state = "block-comment"; + index += 1; + } else if (character === "`") { + code += " "; + state = "template"; + } else if (character === "'" || character === '"') { + code += " "; + state = character; + } else { + code += character; + } + continue; + } + + if (state === "template") { + if (character === "\\") { + code += " "; + if (next !== undefined) { + code += next === "\n" ? "\n" : " "; + index += 1; + } + } else if (character === "`") { + code += " "; + state = "code"; + } else if (character === "$" && next === "{") { + code += " "; + state = "code"; + templateExpressionDepth = 1; + index += 1; + } else { + code += character === "\n" ? "\n" : " "; + } + continue; + } + + if (state === "line-comment") { + if (character === "\n") { + code += "\n"; + state = "code"; + } else { + code += " "; + } + continue; + } + + if (state === "block-comment") { + if (character === "*" && next === "/") { + code += " "; + state = "code"; + index += 1; + } else { + code += character === "\n" ? "\n" : " "; + } + continue; + } + + if (character === "\\") { + code += " "; + if (next !== undefined) { + code += next === "\n" ? "\n" : " "; + index += 1; + } + } else if (character === state) { + code += " "; + state = "code"; + } else { + code += character === "\n" ? "\n" : " "; + } + } + + return code; +} + +function validateModelAliases(value) { + if (!isRecord(value) || Object.keys(value).length === 0) { + fail(`${MANIFEST_PATH} must define a non-empty modelAliases object.`); + } + + for (const [alias, modelId] of Object.entries(value)) { + if ( + !MODEL_ALIAS_PATTERN.test(alias) || + typeof modelId !== "string" || + modelId.trim().length === 0 + ) { + fail(`Invalid model alias ${JSON.stringify(alias)} in ${MANIFEST_PATH}.`); + } + } + + return value; +} + +function validateFixtureAssignment(fixtureDir, value, modelAliases) { + if (!isRecord(value)) { + fail(`Fixture assignment for ${fixtureDir} must be an object.`); + } + + if (value.suite === "worlds") { + const keys = Object.keys(value); + if (keys.length !== 1 || keys[0] !== "suite") { + fail(`World-suite fixture ${fixtureDir} must use exactly {"suite":"worlds"}.`); + } + return value; + } + + if (value.suite === "models") { + const keys = Object.keys(value).sort(); + if (keys.length !== 2 || keys[0] !== "models" || keys[1] !== "suite") { + fail(`Model-suite fixture ${fixtureDir} must contain only suite and models.`); + } + if (!Array.isArray(value.models) || value.models.length === 0) { + fail(`Model-suite fixture ${fixtureDir} must select at least one model alias.`); + } + + const uniqueModels = new Set(value.models); + if (uniqueModels.size !== value.models.length) { + fail(`Model-suite fixture ${fixtureDir} contains duplicate model aliases.`); + } + for (const alias of value.models) { + if (typeof alias !== "string" || !Object.hasOwn(modelAliases, alias)) { + fail(`Model-suite fixture ${fixtureDir} references invalid model alias ${String(alias)}.`); + } + } + return value; + } + + fail(`Fixture ${fixtureDir} has invalid suite ${JSON.stringify(value.suite)}.`); +} + +async function emitOutput(name, value) { + const line = `${name}=${JSON.stringify(value)}\n`; + const outputPath = process.env.GITHUB_OUTPUT; + + if (outputPath === undefined || outputPath.length === 0) { + process.stdout.write(line); + } else { + await appendFile(outputPath, line); + } +} + +async function main() { + const discoveredDirs = await discoverFixtures(); + if (discoveredDirs.length === 0) { + fail("No e2e fixtures with an evals/ directory were found."); + } + + const manifest = JSON.parse(await readFile(MANIFEST_PATH, "utf8")); + if (!isRecord(manifest)) { + fail(`${MANIFEST_PATH} must contain an object.`); + } + const manifestKeys = Object.keys(manifest).sort(); + if ( + manifestKeys.length !== 2 || + manifestKeys[0] !== "fixtures" || + manifestKeys[1] !== "modelAliases" + ) { + fail(`${MANIFEST_PATH} must contain exactly modelAliases and fixtures.`); + } + + const modelAliases = validateModelAliases(manifest.modelAliases); + if (!isRecord(manifest.fixtures)) { + fail(`${MANIFEST_PATH} must define a fixtures object keyed by fixture path.`); + } + + const discoveredSet = new Set(discoveredDirs); + const configuredDirs = Object.keys(manifest.fixtures).sort(); + const missingAssignments = discoveredDirs.filter((dir) => !(dir in manifest.fixtures)); + const staleAssignments = configuredDirs.filter((dir) => !discoveredSet.has(dir)); + + if (missingAssignments.length > 0) { + fail(`Fixtures missing from ${MANIFEST_PATH}: ${missingAssignments.join(", ")}`); + } + if (staleAssignments.length > 0) { + fail(`Stale fixture assignments in ${MANIFEST_PATH}: ${staleAssignments.join(", ")}`); + } + + const names = new Map(); + for (const dir of discoveredDirs) { + const name = basename(dir); + const previousDir = names.get(name); + if (previousDir !== undefined) { + fail(`Duplicate fixture name ${name}: ${previousDir} and ${dir}`); + } + names.set(name, dir); + } + + const modelInclude = []; + const worldInclude = []; + + for (const fixtureDir of discoveredDirs) { + const fixtureName = basename(fixtureDir); + const assignment = validateFixtureAssignment( + fixtureDir, + manifest.fixtures[fixtureDir], + modelAliases, + ); + + if (assignment.suite === "worlds") { + await assertNoJudgeUsage(fixtureDir); + worldInclude.push({ + fixture_dir: fixtureDir, + fixture_name: fixtureName, + }); + continue; + } + + for (const modelName of assignment.models) { + modelInclude.push({ + fixture_dir: fixtureDir, + fixture_name: fixtureName, + model_id: modelAliases[modelName], + model_name: modelName, + }); + } + } + + if (modelInclude.length === 0 || worldInclude.length === 0) { + fail("Both the models and worlds suites must contain at least one fixture."); + } + + const modelMatrix = { include: modelInclude }; + const worldMatrix = { include: worldInclude }; + console.log( + `Discovered ${modelInclude.length} model leg(s) and ${worldInclude.length} world fixture(s).`, + ); + await emitOutput("model_matrix", modelMatrix); + await emitOutput("world_matrix", worldMatrix); +} + +main().catch((error) => { + console.error(error instanceof Error ? error.message : error); + process.exitCode = 1; +}); diff --git a/.github/scripts/discover-e2e-fixtures.sh b/.github/scripts/discover-e2e-fixtures.sh index 4b3918f8b..e8a6f3aec 100755 --- a/.github/scripts/discover-e2e-fixtures.sh +++ b/.github/scripts/discover-e2e-fixtures.sh @@ -1,34 +1,5 @@ #!/usr/bin/env bash -# Discover e2e fixture directories for the CI matrix. -# -# A fixture qualifies when it has an `evals/` directory under one of the -# fixture roots. Emits a JSON array of `{ name, dir }` objects (sorted by dir) -# to the `matrix` GitHub Actions output for the workflow's `fixture` axis. +# Discover and validate the model and workflow-world e2e suites. set -euo pipefail -roots=("e2e/fixtures" "apps/fixtures") - -entries=() -while IFS= read -r evals_dir; do - dir="${evals_dir%/evals}" - name="$(basename "$dir")" - entries+=("{\"name\":\"${name}\",\"dir\":\"${dir}\"}") -done < <( - for root in "${roots[@]}"; do - [ -d "$root" ] || continue - find "$root" -mindepth 2 -maxdepth 2 -type d -name evals - done | sort -) - -if [ "${#entries[@]}" -eq 0 ]; then - echo "No e2e fixtures with an evals/ directory were found." >&2 - exit 1 -fi - -matrix="[$( - IFS=, - echo "${entries[*]}" -)]" - -echo "Discovered fixtures: ${matrix}" -echo "matrix=${matrix}" >>"${GITHUB_OUTPUT:-/dev/stdout}" +node .github/scripts/discover-e2e-fixtures.mjs diff --git a/.github/workflows/e2e-local.yml b/.github/workflows/e2e-local.yml index c76c03635..7dd78059e 100644 --- a/.github/workflows/e2e-local.yml +++ b/.github/workflows/e2e-local.yml @@ -13,8 +13,8 @@ jobs: # every PR. An `on.pull_request.paths` filter would leave them permanently # "expected" on PRs that don't touch e2e-relevant paths, blocking merge. # Instead the workflow always triggers, this job decides relevance, and the - # matrix jobs skip when nothing relevant changed — skipped required checks - # count as satisfied. + # stable aggregate below succeeds without starting matrix legs when nothing + # relevant changed. changes: name: changes runs-on: ubuntu-latest @@ -36,7 +36,7 @@ jobs: fi # HEAD is the PR merge commit; HEAD^1 is the base branch tip, so # this diff is exactly the PR's changed files. - if git diff --name-only HEAD^1 HEAD | grep -qE '^(packages/eve/|apps/fixtures/|e2e/|pnpm-workspace\.yaml$|pnpm-lock\.yaml$|\.github/workflows/e2e-local\.yml$|\.github/scripts/discover-e2e-fixtures\.sh$)'; then + if git diff --name-only HEAD^1 HEAD | grep -qE '^(packages/eve/|apps/fixtures/|e2e/|pnpm-workspace\.yaml$|pnpm-lock\.yaml$|\.github/workflows/e2e-local\.yml$|\.github/scripts/discover-e2e-fixtures\.(sh|mjs)$)'; then echo "relevant=true" >> "$GITHUB_OUTPUT" else echo "relevant=false" >> "$GITHUB_OUTPUT" @@ -46,7 +46,8 @@ jobs: name: discover-fixtures runs-on: ubuntu-latest outputs: - matrix: ${{ steps.discover.outputs.matrix }} + model_matrix: ${{ steps.discover.outputs.model_matrix }} + world_matrix: ${{ steps.discover.outputs.world_matrix }} steps: - name: Checkout code uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 @@ -57,58 +58,45 @@ jobs: id: discover run: .github/scripts/discover-e2e-fixtures.sh - e2e: - name: e2e-local (${{ matrix.fixture.name }}, ${{ matrix.model.name }}) + e2e-models: + name: e2e-models-local (${{ matrix.fixture_name }}, ${{ matrix.model_name }}) runs-on: ubuntu-latest + timeout-minutes: 30 needs: [changes, discover-fixtures] - # No job-level `if`: the stable aggregate check below requires a successful - # matrix result. Guarding each step lets an irrelevant diff satisfy it in - # seconds without running setup or evals. + if: needs.changes.outputs.relevant == 'true' strategy: fail-fast: false - matrix: - fixture: ${{ fromJSON(needs.discover-fixtures.outputs.matrix) }} - model: - - name: openai-sol - id: openai/gpt-5.6-sol - - name: anthropic-opus - id: anthropic/claude-opus-5 + matrix: ${{ fromJSON(needs.discover-fixtures.outputs.model_matrix) }} env: AI_GATEWAY_API_KEY: ${{ secrets.AI_GATEWAY_API_KEY }} - EVE_E2E_MODEL: ${{ matrix.model.id }} + EVE_E2E_MODEL: ${{ matrix.model_id }} EVE_EVAL_JUNIT_DIR: ${{ github.workspace }}/.junit steps: - name: Checkout code - if: needs.changes.outputs.relevant == 'true' uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: persist-credentials: false - name: Setup pnpm - if: needs.changes.outputs.relevant == 'true' uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6 - name: Setup Node.js - if: needs.changes.outputs.relevant == 'true' uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 with: node-version-file: .nvmrc cache: "pnpm" - name: Install dependencies - if: needs.changes.outputs.relevant == 'true' run: pnpm install --frozen-lockfile - name: Build eve package - if: needs.changes.outputs.relevant == 'true' # Full build, not build:js: the deployed/runtime bundles must carry the # stamped package version (scripts/stamp-version-tokens.mjs) and docs. run: pnpm --filter eve run build - name: Build extension fixtures - if: needs.changes.outputs.relevant == 'true' # Extension fixtures ship no committed `dist/`; produce it here so # consuming fixtures resolve the built artifacts. `--if-present` skips # extension fixtures without a build script. The update re-packs the @@ -119,7 +107,6 @@ jobs: pnpm --filter dist-extensions update gizmo-extension gadget-extension - name: Stage workspace extension fixtures as dist-only - if: needs.changes.outputs.relevant == 'true' # Removing author source proves the workspace consumer discovers and # compiles the same agent-shaped dist trees we publish. Toolkit covers # the rich TypeScript surface; js-only covers pure JavaScript authoring. @@ -128,18 +115,94 @@ jobs: mv e2e/fixtures/js-only-extension/extension "$RUNNER_TEMP/js-only-extension-source" - name: Run fixture evals - if: needs.changes.outputs.relevant == 'true' run: | mkdir -p "$EVE_EVAL_JUNIT_DIR" - cd "${{ matrix.fixture.dir }}" + cd "${{ matrix.fixture_dir }}" pnpm exec eve eval --strict --verbose \ - --junit "$EVE_EVAL_JUNIT_DIR/${{ matrix.fixture.name }}-${{ matrix.model.name }}.xml" + --junit "$EVE_EVAL_JUNIT_DIR/models-local-${{ matrix.fixture_name }}-${{ matrix.model_name }}.xml" - name: Upload eval artifacts - if: (failure()) && needs.changes.outputs.relevant == 'true' + if: failure() uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v6 with: - name: eval-artifacts-${{ matrix.fixture.name }}-${{ matrix.model.name }} + name: eval-artifacts-models-local-${{ matrix.fixture_name }}-${{ matrix.model_name }} + path: | + .junit + e2e/fixtures/*/.eve/evals + apps/fixtures/*/.eve/evals + include-hidden-files: true + if-no-files-found: ignore + retention-days: 7 + + e2e-worlds: + name: e2e-worlds-local (${{ matrix.fixture_name }}, mock) + runs-on: ubuntu-latest + timeout-minutes: 30 + needs: [changes, discover-fixtures] + if: needs.changes.outputs.relevant == 'true' + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.discover-fixtures.outputs.world_matrix) }} + + env: + AI_GATEWAY_API_KEY: invalid-e2e-world-suite-key + EVE_E2E_MODEL: openai/gpt-5.6-sol + EVE_EVAL_JUNIT_DIR: ${{ github.workspace }}/.junit + EVE_MOCK_AUTHORED_MODELS: "1" + + steps: + - name: Checkout code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + + - name: Setup pnpm + uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6 + + - name: Setup Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 + with: + node-version-file: .nvmrc + cache: "pnpm" + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Build eve package + # Full build, not build:js: the deployed/runtime bundles must carry the + # stamped package version (scripts/stamp-version-tokens.mjs) and docs. + run: pnpm --filter eve run build + + - name: Build extension fixtures + # Extension fixtures ship no committed `dist/`; produce it here so + # consuming fixtures resolve the built artifacts. `--if-present` skips + # extension fixtures without a build script. The update re-packs the + # `file:`-protocol store copies so they pick up the fresh dist — + # a plain reinstall short-circuits and leaves them stale. + run: | + pnpm --filter "./e2e/fixtures/*-extension" run --if-present build + pnpm --filter dist-extensions update gizmo-extension gadget-extension + + - name: Stage workspace extension fixtures as dist-only + # Removing author source proves the workspace consumer discovers and + # compiles the same agent-shaped dist trees we publish. Toolkit covers + # the rich TypeScript surface; js-only covers pure JavaScript authoring. + run: | + mv e2e/fixtures/toolkit-extension/extension "$RUNNER_TEMP/toolkit-extension-source" + mv e2e/fixtures/js-only-extension/extension "$RUNNER_TEMP/js-only-extension-source" + + - name: Run fixture evals + run: | + mkdir -p "$EVE_EVAL_JUNIT_DIR" + cd "${{ matrix.fixture_dir }}" + pnpm exec eve eval --strict --verbose \ + --junit "$EVE_EVAL_JUNIT_DIR/worlds-local-${{ matrix.fixture_name }}-mock.xml" + + - name: Upload eval artifacts + if: failure() + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v6 + with: + name: eval-artifacts-worlds-local-${{ matrix.fixture_name }}-mock path: | .junit e2e/fixtures/*/.eve/evals @@ -151,14 +214,31 @@ jobs: e2e-required: name: e2e-local runs-on: ubuntu-latest - needs: [e2e] + needs: [changes, discover-fixtures, e2e-models, e2e-worlds] if: always() steps: - - name: Require successful e2e matrix + - name: Require successful e2e suites when relevant env: - E2E_RESULT: ${{ needs.e2e.result }} + CHANGES_RESULT: ${{ needs.changes.result }} + DISCOVERY_RESULT: ${{ needs.discover-fixtures.result }} + MODEL_RESULT: ${{ needs.e2e-models.result }} + RELEVANT: ${{ needs.changes.outputs.relevant }} + WORLD_RESULT: ${{ needs.e2e-worlds.result }} run: | - if [ "$E2E_RESULT" != "success" ]; then - echo "E2E matrix result: $E2E_RESULT" + if [ "$CHANGES_RESULT" != "success" ]; then + echo "Change detection result: $CHANGES_RESULT" + exit 1 + fi + if [ "$DISCOVERY_RESULT" != "success" ]; then + echo "Fixture discovery result: $DISCOVERY_RESULT" + exit 1 + fi + if [ "$RELEVANT" != "true" ]; then + echo "No local e2e-relevant changes." + exit 0 + fi + if [ "$MODEL_RESULT" != "success" ] || [ "$WORLD_RESULT" != "success" ]; then + echo "Model suite result: $MODEL_RESULT" + echo "World suite result: $WORLD_RESULT" exit 1 fi diff --git a/.github/workflows/e2e-postgres.yml b/.github/workflows/e2e-postgres.yml new file mode 100644 index 000000000..d658b976e --- /dev/null +++ b/.github/workflows/e2e-postgres.yml @@ -0,0 +1,234 @@ +name: E2E Tests (Postgres) + +on: + workflow_dispatch: + pull_request: + +concurrency: + group: e2e-postgres-${{ github.ref }} + cancel-in-progress: true + +jobs: + # The stable aggregate check must report on every PR. This job decides + # relevance so the matrix can skip at the job level without starting a + # PostgreSQL service container for every fixture on unrelated changes. + changes: + name: changes + runs-on: ubuntu-latest + outputs: + relevant: ${{ steps.filter.outputs.relevant }} + steps: + - name: Checkout code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + fetch-depth: 2 + + - name: Check changed paths + id: filter + run: | + if [ "${{ github.event_name }}" != "pull_request" ]; then + echo "relevant=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + # HEAD is the PR merge commit; HEAD^1 is the base branch tip, so + # this diff is exactly the PR's changed files. + if git diff --name-only HEAD^1 HEAD | grep -qE '^(packages/eve/|apps/fixtures/|e2e/|pnpm-workspace\.yaml$|pnpm-lock\.yaml$|\.github/workflows/e2e-postgres\.yml$|\.github/scripts/discover-e2e-fixtures\.(sh|mjs)$)'; then + echo "relevant=true" >> "$GITHUB_OUTPUT" + else + echo "relevant=false" >> "$GITHUB_OUTPUT" + fi + + discover-fixtures: + name: discover-fixtures + runs-on: ubuntu-latest + outputs: + world_matrix: ${{ steps.discover.outputs.world_matrix }} + steps: + - name: Checkout code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + + - name: Discover eval fixtures + id: discover + run: .github/scripts/discover-e2e-fixtures.sh + + e2e-worlds: + name: e2e-worlds-postgres (${{ matrix.fixture_name }}, mock) + runs-on: ubuntu-latest + timeout-minutes: 30 + needs: [changes, discover-fixtures] + if: needs.changes.outputs.relevant == 'true' + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.discover-fixtures.outputs.world_matrix) }} + + services: + postgres: + image: postgres:18-alpine + env: + POSTGRES_DB: world + POSTGRES_PASSWORD: world + POSTGRES_USER: world + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready --username=world --dbname=world" + --health-interval 5s + --health-timeout 5s + --health-retries 12 + + env: + AI_GATEWAY_API_KEY: invalid-e2e-world-suite-key + EVE_E2E_MODEL: openai/gpt-5.6-sol + EVE_E2E_WORKFLOW_WORLD: "@workflow/world-postgres" + EVE_EVAL_JUNIT_DIR: ${{ github.workspace }}/.junit + EVE_MOCK_AUTHORED_MODELS: "1" + WORKFLOW_POSTGRES_MAX_POOL_SIZE: "52" + WORKFLOW_POSTGRES_URL: postgres://world:world@127.0.0.1:5432/world + WORKFLOW_POSTGRES_WORKER_CONCURRENCY: "50" + + steps: + - name: Checkout code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + + - name: Setup pnpm + uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6 + + - name: Setup Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 + with: + node-version-file: .nvmrc + cache: "pnpm" + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Build eve package + # Full build, not build:js: the deployed/runtime bundles must carry the + # stamped package version (scripts/stamp-version-tokens.mjs) and docs. + run: pnpm --filter eve run build + + - name: Build extension fixtures + # Extension fixtures ship no committed `dist/`; produce it here so + # consuming fixtures resolve the built artifacts. `--if-present` skips + # extension fixtures without a build script. The update re-packs the + # `file:`-protocol store copies so they pick up the fresh dist. + run: | + pnpm --filter "./e2e/fixtures/*-extension" run --if-present build + pnpm --filter dist-extensions update gizmo-extension gadget-extension + + - name: Stage workspace extension fixtures as dist-only + # Removing author source proves the production build consumes the same + # agent-shaped dist trees registry packages ship. + run: | + mv e2e/fixtures/toolkit-extension/extension "$RUNNER_TEMP/toolkit-extension-source" + mv e2e/fixtures/js-only-extension/extension "$RUNNER_TEMP/js-only-extension-source" + + - name: Bootstrap Workflow Postgres schema + working-directory: ${{ matrix.fixture_dir }} + run: pnpm exec bootstrap + + - name: Build fixture, start production server, and run evals + env: + POSTGRES_CONTAINER_ID: ${{ job.services.postgres.id }} + run: | + set -euo pipefail + mkdir -p "$EVE_EVAL_JUNIT_DIR" + cd "${{ matrix.fixture_dir }}" + + pnpm exec eve build + + server_log="$EVE_EVAL_JUNIT_DIR/worlds-postgres-${{ matrix.fixture_name }}-mock-server.log" + pnpm exec eve start --host 127.0.0.1 --port 3000 >"$server_log" 2>&1 & + server_pid=$! + + cleanup() { + set +e + kill "$server_pid" 2>/dev/null + wait "$server_pid" 2>/dev/null + } + trap cleanup EXIT + + ready=false + for _ in {1..120}; do + if curl --fail --silent http://127.0.0.1:3000/eve/v1/health >/dev/null; then + ready=true + break + fi + if ! kill -0 "$server_pid" 2>/dev/null; then + cat "$server_log" + exit 1 + fi + sleep 0.5 + done + if [ "$ready" != "true" ]; then + cat "$server_log" + echo "Timed out waiting for the Postgres-backed eve server." + exit 1 + fi + + if ! pnpm exec eve eval --strict --verbose \ + --url http://127.0.0.1:3000 \ + --junit "$EVE_EVAL_JUNIT_DIR/worlds-postgres-${{ matrix.fixture_name }}-mock.xml"; then + cat "$server_log" + exit 1 + fi + + run_count="$( + docker exec "$POSTGRES_CONTAINER_ID" \ + psql --username=world --dbname=world --tuples-only --no-align \ + --command="SELECT count(*) FROM workflow.workflow_runs;" + )" + if ! [[ "$run_count" =~ ^[0-9]+$ ]] || [ "$run_count" -lt 1 ]; then + cat "$server_log" + echo "Expected Postgres-backed workflow runs, found: $run_count" + exit 1 + fi + echo "Verified $run_count workflow run(s) in PostgreSQL." + + - name: Upload eval artifacts + if: failure() + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v6 + with: + name: eval-artifacts-worlds-postgres-${{ matrix.fixture_name }}-mock + path: | + .junit + e2e/fixtures/*/.eve/evals + apps/fixtures/*/.eve/evals + include-hidden-files: true + if-no-files-found: ignore + retention-days: 7 + + e2e-required: + name: e2e-postgres + runs-on: ubuntu-latest + needs: [changes, discover-fixtures, e2e-worlds] + if: always() + steps: + - name: Require successful e2e matrix when relevant + env: + CHANGES_RESULT: ${{ needs.changes.result }} + DISCOVERY_RESULT: ${{ needs.discover-fixtures.result }} + E2E_RESULT: ${{ needs.e2e-worlds.result }} + RELEVANT: ${{ needs.changes.outputs.relevant }} + run: | + if [ "$CHANGES_RESULT" != "success" ]; then + echo "Change detection result: $CHANGES_RESULT" + exit 1 + fi + if [ "$DISCOVERY_RESULT" != "success" ]; then + echo "Fixture discovery result: $DISCOVERY_RESULT" + exit 1 + fi + if [ "$RELEVANT" != "true" ]; then + echo "No Postgres e2e-relevant changes." + exit 0 + fi + if [ "$E2E_RESULT" != "success" ]; then + echo "World suite result: $E2E_RESULT" + exit 1 + fi diff --git a/.github/workflows/e2e-vercel.yml b/.github/workflows/e2e-vercel.yml index 21d5518ee..030865161 100644 --- a/.github/workflows/e2e-vercel.yml +++ b/.github/workflows/e2e-vercel.yml @@ -13,8 +13,8 @@ jobs: # every PR. An `on.pull_request.paths` filter would leave them permanently # "expected" on PRs that don't touch e2e-relevant paths, blocking merge. # Instead the workflow always triggers, this job decides relevance, and the - # matrix jobs skip when nothing relevant changed — skipped required checks - # count as satisfied. + # stable aggregate below succeeds without starting matrix legs when nothing + # relevant changed. changes: name: changes runs-on: ubuntu-latest @@ -36,7 +36,7 @@ jobs: fi # HEAD is the PR merge commit; HEAD^1 is the base branch tip, so # this diff is exactly the PR's changed files. - if git diff --name-only HEAD^1 HEAD | grep -qE '^(packages/eve/|apps/fixtures/|e2e/|pnpm-workspace\.yaml$|pnpm-lock\.yaml$|scripts/build-profile-report\.mjs$|\.github/workflows/e2e-vercel\.yml$|\.github/scripts/discover-e2e-fixtures\.sh$)'; then + if git diff --name-only HEAD^1 HEAD | grep -qE '^(packages/eve/|apps/fixtures/|e2e/|pnpm-workspace\.yaml$|pnpm-lock\.yaml$|scripts/build-profile-report\.mjs$|\.github/workflows/e2e-vercel\.yml$|\.github/scripts/discover-e2e-fixtures\.(sh|mjs)$)'; then echo "relevant=true" >> "$GITHUB_OUTPUT" else echo "relevant=false" >> "$GITHUB_OUTPUT" @@ -46,7 +46,7 @@ jobs: name: discover-fixtures runs-on: ubuntu-latest outputs: - matrix: ${{ steps.discover.outputs.matrix }} + world_matrix: ${{ steps.discover.outputs.world_matrix }} steps: - name: Checkout code uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 @@ -57,26 +57,21 @@ jobs: id: discover run: .github/scripts/discover-e2e-fixtures.sh - e2e: - name: e2e-vercel (${{ matrix.fixture.name }}, ${{ matrix.model.name }}) + e2e-worlds: + name: e2e-worlds-vercel (${{ matrix.fixture_name }}, mock) runs-on: ubuntu-latest + timeout-minutes: 30 needs: [changes, discover-fixtures] - # No job-level `if`: the stable aggregate check below requires a successful - # matrix result. Guarding each step lets an irrelevant diff satisfy it in - # seconds without running setup or evals. + if: needs.changes.outputs.relevant == 'true' strategy: fail-fast: false - matrix: - fixture: ${{ fromJSON(needs.discover-fixtures.outputs.matrix) }} - model: - - name: openai-sol - id: openai/gpt-5.6-sol - - name: anthropic-opus - id: anthropic/claude-opus-5 + matrix: ${{ fromJSON(needs.discover-fixtures.outputs.world_matrix) }} env: - EVE_E2E_MODEL: ${{ matrix.model.id }} + AI_GATEWAY_API_KEY: invalid-e2e-world-suite-key + EVE_E2E_MODEL: openai/gpt-5.6-sol EVE_EVAL_JUNIT_DIR: ${{ github.workspace }}/.junit + EVE_MOCK_AUTHORED_MODELS: "1" VERCEL_ORG_ID: ${{ secrets.VERCEL_ORG_ID }} VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID }} VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }} @@ -84,34 +79,28 @@ jobs: steps: - name: Checkout code - if: needs.changes.outputs.relevant == 'true' uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: persist-credentials: false - name: Setup pnpm - if: needs.changes.outputs.relevant == 'true' uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6 - name: Setup Node.js - if: needs.changes.outputs.relevant == 'true' uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 with: node-version-file: .nvmrc cache: "pnpm" - name: Install dependencies - if: needs.changes.outputs.relevant == 'true' run: pnpm install --frozen-lockfile - name: Build eve package - if: needs.changes.outputs.relevant == 'true' # Full build, not build:js: the deployed/runtime bundles must carry the # stamped package version (scripts/stamp-version-tokens.mjs) and docs. run: pnpm --filter eve run build - name: Build extension fixtures - if: needs.changes.outputs.relevant == 'true' # Extension fixtures ship no committed `dist/`; produce it here so # consuming fixtures resolve the built artifacts. `--if-present` skips # extension fixtures without a build script. The update re-packs the @@ -122,7 +111,6 @@ jobs: pnpm --filter dist-extensions update gizmo-extension gadget-extension - name: Stage workspace extension fixtures as dist-only - if: needs.changes.outputs.relevant == 'true' # Removing author source proves the deployment consumes the same # agent-shaped dist trees registry packages ship. Toolkit covers the # rich TypeScript surface; js-only covers pure JavaScript authoring. @@ -131,13 +119,12 @@ jobs: mv e2e/fixtures/js-only-extension/extension "$RUNNER_TEMP/js-only-extension-source" - name: Deploy fixture and run evals - if: needs.changes.outputs.relevant == 'true' env: - EVE_BUILD_PROFILE: ${{ runner.temp }}/eve-build-profile-${{ matrix.fixture.name }}-${{ matrix.model.name }}.json + EVE_BUILD_PROFILE: ${{ runner.temp }}/eve-build-profile-worlds-vercel-${{ matrix.fixture_name }}-mock.json run: | set -euo pipefail mkdir -p "$EVE_EVAL_JUNIT_DIR" - cd "${{ matrix.fixture.dir }}" + cd "${{ matrix.fixture_dir }}" token_args=() if [ -n "${VERCEL_TOKEN:-}" ]; then @@ -161,10 +148,13 @@ jobs: VERCEL_PROJECT_ID="$VERCEL_PROJECT_ID" \ pnpm exec eve build --profile "$EVE_BUILD_PROFILE" DEPLOYMENT_URL="$(pnpm exec vc deploy --prebuilt --yes --target=preview \ - --env "EVE_E2E_MODEL=$EVE_E2E_MODEL" "${token_args[@]}" | tail -n 1)" + --env "AI_GATEWAY_API_KEY=$AI_GATEWAY_API_KEY" \ + --env "EVE_E2E_MODEL=$EVE_E2E_MODEL" \ + --env "EVE_MOCK_AUTHORED_MODELS=$EVE_MOCK_AUTHORED_MODELS" \ + "${token_args[@]}" | tail -n 1)" npx eve eval --strict --verbose --url "$DEPLOYMENT_URL" \ - --junit "$EVE_EVAL_JUNIT_DIR/${{ matrix.fixture.name }}-${{ matrix.model.name }}.xml" + --junit "$EVE_EVAL_JUNIT_DIR/worlds-vercel-${{ matrix.fixture_name }}-mock.xml" # Redeploy-tagged evals rebuild and redeploy the fixture from inside # the eval body, repointing a run-scoped alias at each new @@ -172,21 +162,20 @@ jobs: # never change what they serve) and are skipped in the main suite # above because EVE_E2E_REDEPLOY_ALIAS is unset there. if npx eve eval --tag redeploy --list >/dev/null 2>&1; then - ALIAS="eve-e2e-${{ matrix.fixture.name }}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${{ strategy.job-index }}.vercel.app" + ALIAS="eve-e2e-${{ matrix.fixture_name }}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${{ strategy.job-index }}.vercel.app" # vc alias does not infer the team from the project link the way # link/deploy do, so pass the scope explicitly. pnpm exec vc alias set "$DEPLOYMENT_URL" "$ALIAS" "${token_args[@]}" "${scope_args[@]}" EVE_E2E_REDEPLOY_ALIAS="$ALIAS" \ npx eve eval --tag redeploy --strict --verbose \ --url "https://$ALIAS" \ - --junit "$EVE_EVAL_JUNIT_DIR/${{ matrix.fixture.name }}-${{ matrix.model.name }}-redeploy.xml" + --junit "$EVE_EVAL_JUNIT_DIR/worlds-vercel-${{ matrix.fixture_name }}-mock-redeploy.xml" fi - name: Log deployable build timing - if: needs.changes.outputs.relevant == 'true' env: - BUILD_PROFILE: ${{ runner.temp }}/eve-build-profile-${{ matrix.fixture.name }}-${{ matrix.model.name }}.json - FIXTURE_PATH: ${{ matrix.fixture.dir }} + BUILD_PROFILE: ${{ runner.temp }}/eve-build-profile-worlds-vercel-${{ matrix.fixture_name }}-mock.json + FIXTURE_PATH: ${{ matrix.fixture_dir }} run: | node ./scripts/build-profile-report.mjs \ --profile "$BUILD_PROFILE" \ @@ -196,10 +185,10 @@ jobs: --output-format build-time - name: Upload eval artifacts - if: (failure()) && needs.changes.outputs.relevant == 'true' + if: failure() uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v6 with: - name: eval-artifacts-vercel-${{ matrix.fixture.name }}-${{ matrix.model.name }} + name: eval-artifacts-worlds-vercel-${{ matrix.fixture_name }}-mock path: | .junit e2e/fixtures/*/.eve/evals @@ -211,14 +200,29 @@ jobs: e2e-required: name: e2e-vercel runs-on: ubuntu-latest - needs: [e2e] + needs: [changes, discover-fixtures, e2e-worlds] if: always() steps: - - name: Require successful e2e matrix + - name: Require successful e2e suite when relevant env: - E2E_RESULT: ${{ needs.e2e.result }} + CHANGES_RESULT: ${{ needs.changes.result }} + DISCOVERY_RESULT: ${{ needs.discover-fixtures.result }} + E2E_RESULT: ${{ needs.e2e-worlds.result }} + RELEVANT: ${{ needs.changes.outputs.relevant }} run: | + if [ "$CHANGES_RESULT" != "success" ]; then + echo "Change detection result: $CHANGES_RESULT" + exit 1 + fi + if [ "$DISCOVERY_RESULT" != "success" ]; then + echo "Fixture discovery result: $DISCOVERY_RESULT" + exit 1 + fi + if [ "$RELEVANT" != "true" ]; then + echo "No Vercel e2e-relevant changes." + exit 0 + fi if [ "$E2E_RESULT" != "success" ]; then - echo "E2E matrix result: $E2E_RESULT" + echo "World suite result: $E2E_RESULT" exit 1 fi diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6c8156cbb..bd3b7a669 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -64,18 +64,28 @@ pnpm test:tui # TUI smoke scripts (not e2e) E2E tests are fixture-owned evals. Run them from the fixture directory: ```bash -cd e2e/fixtures/agent-basic-runtime -pnpm exec eve eval --strict +cd e2e/fixtures/agent-tools +EVE_E2E_MODEL="openai/gpt-5.6-sol" pnpm exec eve eval --strict ``` -The fixture agents and judges run against real models (`openai/gpt-5.5`), so -the environment must provide the corresponding model-provider credentials. +CI splits e2e coverage by the dimension under test. Five provider-sensitive +fixtures form eight live-model legs against the Local world; these require the +credential for their assigned model. The other 15 fixtures run with +deterministic models against the Local, Vercel, and Postgres worlds: + +```bash +cd e2e/fixtures/agent-basic-runtime +EVE_MOCK_AUTHORED_MODELS=1 \ + AI_GATEWAY_API_KEY="invalid-e2e-world-suite-key" \ + pnpm exec eve eval --strict +``` -Vercel e2e builds that same fixture directory with `VERCEL=1`, deploys the -fixture's prebuilt Vercel output, and runs evals against the immutable -deployment URL. All fixture deployments link to the same Vercel project id; the -shared project's Preview env must provide those same model-provider -credentials. +World fixtures cannot use live providers or model judges. The invalid Gateway +credential makes an accidental provider path fail, and CI propagates mock mode +through Vercel deployments and redeployments. Fixture ownership and meaningful +provider assignments live in [`e2e/suites.json`](./e2e/suites.json); see +[`e2e/README.md`](./e2e/README.md) for the complete Local, Vercel, and Postgres +commands. CI intentionally has no full fixture × model × world matrix. Do not commit fixture trees under `packages/eve/test/fixtures/` — scenario app content is defined inline as `ScenarioAppDescriptor` objects under `packages/eve/src/internal/testing/scenario-apps/` (CI enforces this). diff --git a/e2e/README.md b/e2e/README.md index ae3e85edb..4edd0b6e7 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -3,21 +3,48 @@ End-to-end coverage is fixture-owned `eve eval` runs. The suite only runs fixture eval files from the fixture directory. +CI separates the dimension under test instead of crossing every fixture, +model, and Workflow world: + +- The **model suite** runs provider-sensitive fixtures against the local world. + Its five fixtures expand to eight live-model legs. +- The **world suite** runs the other 15 fixtures with deterministic mock models + against the Local, Vercel, and Postgres worlds, for 45 legs. + +This produces 53 focused legs while retaining every fixture. There is no +nightly, manual, or other full cross-product matrix. + ## Local -Run evals from the fixture directory: +Run a model-suite fixture from its directory with one of its assigned models: ```sh -cd e2e/fixtures/agent-basic-runtime +cd e2e/fixtures/agent-tools EVE_E2E_MODEL="openai/gpt-5.6-sol" pnpm exec eve eval --strict ``` -Every retained e2e eval is deterministic and self-contained. Coverage that -needs external services or injected env is intentionally not part of this -suite. Most fixtures use the shared model-provider credentials; dedicated -runtime stress fixtures may use an authored deterministic model instead. +Model-suite runs require the credential for the selected provider. They use the +local Workflow world so failures exercise model and provider behavior without +also varying deployment infrastructure. -Each retained fixture package also exposes the same command as: +Run a world-suite fixture locally with deterministic authored models: + +```sh +cd e2e/fixtures/agent-basic-runtime +EVE_MOCK_AUTHORED_MODELS=1 \ + AI_GATEWAY_API_KEY="invalid-e2e-world-suite-key" \ + pnpm exec eve eval --strict +``` + +`EVE_MOCK_AUTHORED_MODELS=1` replaces authored provider models with eve's +deterministic mock adapter. Models already authored as `eve-mock/*` remain +intact so fixtures can supply their own deterministic programs. The invalid +Gateway credential is a canary: a regression that escapes the adapter fails +instead of using a live provider. World fixtures cannot use `t.judge`; +discovery rejects that assignment, so their assertions never invoke a live +model judge. + +Each fixture package also exposes its eval command as: ```sh pnpm --filter agent-basic-runtime test:e2e @@ -30,103 +57,184 @@ script: pnpm test:e2e ``` +Use the explicit environment from the examples above when reproducing a +particular CI suite. + +## Suite ownership + +[`suites.json`](./suites.json) assigns every discovered fixture path to exactly +one suite. Its internal CI interface is: + +```json +{ + "modelAliases": { + "openai-sol": "openai/gpt-5.6-sol", + "anthropic-opus": "anthropic/claude-opus-5" + }, + "fixtures": { + "e2e/fixtures/agent-tools": { + "suite": "models", + "models": ["openai-sol", "anthropic-opus"] + }, + "e2e/fixtures/agent-basic-runtime": { + "suite": "worlds" + } + } +} +``` + +Model aliases are stable CI identifiers: + +- `openai-sol` selects `openai/gpt-5.6-sol`. +- `anthropic-opus` selects `anthropic/claude-opus-5`. + +The provider assignments are intentionally narrow and meaningful: + +- `agent-compaction-regressions`: `openai-sol`, `anthropic-opus` +- `agent-prompt-cache`: `anthropic-opus` +- `agent-tools`: `openai-sol`, `anthropic-opus` +- `agent-tools-hitl`: `openai-sol`, `anthropic-opus` +- `agent-tools-hitl-openai`: `openai-sol` + +Those assignments produce the eight Local model legs. Every other fixture is +in the world suite, including `agent-model` and the Vercel redeploy coverage in +`agent-tools-sandbox`. + +Fixture names remain derived from directory names. Discovery fails when a +fixture is missing from the manifest, a manifest path is stale, two paths +derive the same name, a model alias is unknown, or a world fixture contains a +`t.judge` call. + +Run discovery from the repository root to validate the manifest and print the +`model_matrix` and `world_matrix` GitHub Actions outputs: + +```sh +.github/scripts/discover-e2e-fixtures.sh +``` + +When adding e2e coverage: + +- Put the eval in the fixture app's `evals/` directory. +- Add its path to `suites.json` as either a model fixture with explicit model + aliases or a world fixture. +- Choose the model suite only when provider behavior, prompt handling, or model + latency is part of the contract; otherwise choose the world suite. +- Keep it runnable with `eve eval --strict` and deterministic within its suite. +- World fixtures must not use live providers or model judges. +- World fixture root agents spread `e2eAgentConfig()` from + `@eve-e2e/config` and declare `@eve-e2e/config` plus + `@workflow/world-postgres` as direct dependencies. +- Do not add world configuration or Postgres dependencies to model-only + fixtures. + +Fixture discovery accepts `e2e/fixtures/*` and `apps/fixtures/*` apps with an +`evals/` directory. Shared development apps should stay out of the e2e matrix +unless they intentionally own evals. + +## Postgres + +Postgres e2e runs the 15 world fixtures against a self-hosted production server +backed by `@workflow/world-postgres`. Each leg owns an isolated PostgreSQL +service, bootstraps its schema, builds the fixture, starts it with `eve start`, +and targets that server with `eve eval --url`. + +World fixture root agents spread `e2eAgentConfig()` from the private +`@eve-e2e/config` workspace package. The helper reads +`EVE_E2E_WORKFLOW_WORLD` into `experimental.workflow.world`. The variable is +unset in Local and Vercel, preserving their host defaults; Postgres sets it to +`@workflow/world-postgres` while compiling the production server. The world +package is a direct fixture dependency, pinned through the workspace catalog to +the same `@workflow/*` line as eve. + +The per-fixture lifecycle is: + +```sh +export EVE_E2E_WORKFLOW_WORLD="@workflow/world-postgres" +export EVE_MOCK_AUTHORED_MODELS="1" +export AI_GATEWAY_API_KEY="invalid-e2e-world-suite-key" +export WORKFLOW_POSTGRES_URL="postgres://world:world@127.0.0.1:5432/world" + +pnpm exec bootstrap +pnpm exec eve build +pnpm exec eve start --host 127.0.0.1 --port 3000 + +# In a second process: +pnpm exec eve eval --strict --url http://127.0.0.1:3000 +``` + +After the eval succeeds, CI requires at least one row in +`workflow.workflow_runs`. This proves the fixture used Postgres rather than +silently falling back to the local world. Dev-only schedule dispatch evals +skip against this production target, matching their Vercel behavior; the Local +world suite retains that development-route coverage. + ## Vercel -Vercel e2e uses the same fixture evals against immutable preview deployment -URLs. All fixture deployments link to the same Vercel project id; isolation -comes from the deployment URL returned by `vc deploy --prebuilt`. +Vercel e2e deploys the 15 world fixtures to immutable preview URLs. All fixture +deployments link to the same Vercel project id; isolation comes from the URL +returned by `vc deploy --prebuilt`. One-time project setup: - Configure the shared Vercel project for Node.js 24. -- Provide the model-provider credentials needed by `EVE_E2E_MODEL` in the - project's Preview environment. - Provide `VERCEL_TOKEN`, `VERCEL_ORG_ID`, and `VERCEL_PROJECT_ID` in CI. -Run a fixture against Vercel from its directory: +The workflow passes mock mode and the invalid Gateway credential to every +initial and in-test redeployment. Provider credentials are not required by the +world suite. + +Run a world fixture against Vercel from its directory: ```sh vc link --yes --project "$VERCEL_PROJECT_ID" vc env pull --yes --environment=preview VERCEL=1 VERCEL_ENV=preview VERCEL_TARGET_ENV=preview \ VERCEL_PROJECT_ID="$VERCEL_PROJECT_ID" \ + EVE_MOCK_AUTHORED_MODELS=1 \ + AI_GATEWAY_API_KEY="invalid-e2e-world-suite-key" \ pnpm exec eve build DEPLOYMENT_URL="$(vc deploy --prebuilt --yes --target=preview \ - --env "EVE_E2E_MODEL=$EVE_E2E_MODEL" | tail -n 1)" + --env "EVE_MOCK_AUTHORED_MODELS=1" \ + --env "AI_GATEWAY_API_KEY=invalid-e2e-world-suite-key" | tail -n 1)" npx eve eval --strict --url "$DEPLOYMENT_URL" ``` Do not set `VERCEL_TEAM_ID` at build: sandbox template keys must derive identically at build and runtime, and Vercel has no team variable at runtime. -### Redeploy suite +### Redeploy coverage `agent-tools-sandbox/evals/sandbox/redeploy.eval.ts` proves sandbox semantics across deployment updates as they behave on preview targets: a parked session -keeps working (with its `/workspace` state intact) when messages route through -a new deployment, its turns stay pinned to the deployment that created it -(branch-less CLI preview deploys cannot resolve a "latest" deployment; see -`shouldRouteToLatestDeployment` in `execution/workflow-runtime.ts`), and new -sessions adopt the new deployment — a skill added by the redeploy loads there. -The pinned-turn assertion is a deliberate tripwire: it must be flipped when -turn dispatch gains preview latest-routing -(https://github.com/vercel/eve/issues/582). - -The eval redeploys from inside its test body: it mutates the agent source, -runs `eve build` + `vc deploy`, and repoints a run-scoped Vercel alias at -each new deployment, polling `/eve/v1/info` until the alias serves it. -Because immutable deployment URLs never change what they serve, the eval -must run against the alias — the `e2e-vercel` workflow sets -`EVE_E2E_REDEPLOY_ALIAS`, aliases the deployment, and runs `--tag redeploy` -evals as a second `eve eval` invocation after the main suite. Without the -alias env (local matrix, plain `eve eval --strict`) the eval skips. - -Most fixture agents and their configured judges use `EVE_E2E_MODEL`, defaulting -to `openai/gpt-5.6-sol` for local runs. CI sets it from the model matrix, so -adding a matrix entry runs every discovered fixture against that model. -`agent-prompt-cache` is the one fixture that authors a direct -`@ai-sdk/anthropic` model instance instead of a gateway model id: its eval -asserts the harness's Anthropic cache-breakpoint placement, which only runs on -that path. It uses the matrix model when it is an Anthropic model and otherwise -falls back to `anthropic/claude-opus-5`. The instance points at the AI -Gateway's Anthropic-compatible Messages endpoint so it uses the same -`AI_GATEWAY_API_KEY` credential as every other fixture. -`agent-workflow-stress` uses eve's `mockModel` fixture helper so its 100-turn -runs stay fast and deterministic. Its concurrent and sequential evals cover -high-volume session execution and repeated session resumption respectively. - -## Fixtures - -E2E fixtures live under `e2e/fixtures/*`. Fixture discovery also accepts -`apps/fixtures/*` apps with an `evals/` directory, but shared development apps -should stay out of the e2e matrix unless they intentionally own evals. - -When adding e2e coverage: - -- Put the eval in the fixture app's `evals/` directory. -- Keep it runnable with only `eve eval --strict`. -- Keep it deterministic: no external service startup or injected env - requirements (beyond model-provider credentials). -- If the behavior cannot fit that shape yet, leave it out and rebuild it later - as a first-class eval story. +keeps working, with its `/workspace` state intact, when messages route through +a new deployment; its turns stay pinned to the deployment that created it; and +new sessions adopt the new deployment, where a newly added skill loads. +Branch-less CLI preview deploys cannot resolve a "latest" deployment, so the +pinned-turn assertion must be flipped when turn dispatch gains preview +latest-routing (https://github.com/vercel/eve/issues/582). + +The eval redeploys from inside its test body: it mutates the agent source, runs +`eve build` plus `vc deploy`, and repoints a run-scoped Vercel alias at each new +deployment, polling `/eve/v1/info` until the alias serves it. Because immutable +deployment URLs never change what they serve, the eval runs against the alias. +The Vercel workflow propagates mock mode and the credential canary to each +redeployment. ## CI -`.github/workflows/e2e-local.yml` builds the eve package once per matrix leg, -then runs one fixture directory. Its matrix crosses every discovered fixture -with these model entries: +The three workflows expose stable aggregate checks: -- `openai-sol` → `openai/gpt-5.6-sol` -- `anthropic-opus` → `anthropic/claude-opus-5` +- `e2e-local` covers eight live-model legs and 15 Local world legs. +- `e2e-vercel` covers 15 Vercel world legs. +- `e2e-postgres` covers 15 Postgres world legs and persisted Workflow rows. -The short name is the stable Actions check identifier; the full id selects the -provider model. Updating a model version does not rename required checks. -Each workflow also publishes one stable aggregate check, `e2e-local` or -`e2e-vercel`, which succeeds only when every fixture and model leg succeeds. -Require those two checks in the repository ruleset so newly added fixtures and -models become required automatically. +All three workflows run discovery even when a pull request is irrelevant, so +invalid suite ownership still fails. Their aggregate checks otherwise succeed +quickly on irrelevant changes and fail when discovery or any relevant matrix +leg fails. Require the three aggregate names in the repository ruleset; the +individual fixture jobs can change without updating required checks. -Each leg exports the selected id as `EVE_E2E_MODEL` before it runs: +Local model legs export the selected provider id: ```sh pnpm --filter eve run build @@ -134,18 +242,15 @@ cd "$FIXTURE_DIR" EVE_E2E_MODEL="$MODEL" pnpm exec eve eval --strict --junit "$JUNIT_PATH" ``` -Always build with the full `build` script (not `build:js`); only the full -build stamps the package version into `dist`. +Every Local, Vercel, and Postgres world leg enables +`EVE_MOCK_AUTHORED_MODELS=1` and the invalid Gateway credential. Vercel passes +both variables to deployed functions and in-test redeployments; Postgres also +sets `EVE_E2E_WORKFLOW_WORLD=@workflow/world-postgres`. -`.github/workflows/e2e-vercel.yml` links each fixture directory to the shared -Vercel project id, builds Vercel output locally, deploys that output, and runs: - -```sh -pnpm exec eve build -DEPLOYMENT_URL="$(vc deploy --prebuilt --yes --target=preview \ - --env "EVE_E2E_MODEL=$EVE_E2E_MODEL" | tail -n 1)" -npx eve eval --strict --url "$DEPLOYMENT_URL" --junit "$JUNIT_PATH" -``` +Job names, JUnit files, server logs, and uploaded artifacts identify the suite, +world, fixture, and model where applicable. Always build with the full `build` +script rather than `build:js`; only the full build stamps the package version +into `dist`. TUI smoke scripts are not e2e. They live under `packages/eve/test/tui-client` and run through `pnpm test:tui`. diff --git a/e2e/fixtures/agent-basic-runtime/agent/agent.ts b/e2e/fixtures/agent-basic-runtime/agent/agent.ts index 93907447f..384bb0e2a 100644 --- a/e2e/fixtures/agent-basic-runtime/agent/agent.ts +++ b/e2e/fixtures/agent-basic-runtime/agent/agent.ts @@ -1,6 +1,7 @@ +import { e2eAgentConfig } from "@eve-e2e/config"; import { defineAgent } from "eve"; export default defineAgent({ - model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol", + ...e2eAgentConfig(), reasoning: "high", }); diff --git a/e2e/fixtures/agent-basic-runtime/evals/runtime/image-attachment.eval.ts b/e2e/fixtures/agent-basic-runtime/evals/runtime/image-attachment.eval.ts index cddf20df3..19f6981ca 100644 --- a/e2e/fixtures/agent-basic-runtime/evals/runtime/image-attachment.eval.ts +++ b/e2e/fixtures/agent-basic-runtime/evals/runtime/image-attachment.eval.ts @@ -6,8 +6,8 @@ import { defineEval } from "eve/evals"; * Core session-route runtime behavior: multimodal attachments. * * Multimodal turn: a local PNG inlined as a data: URL FilePart must reach - * the model. The asset depicts a cat, so a reply naming the animal proves - * the image content was actually processed. + * the runtime. The prompt pins a deterministic reply for mock-world coverage; + * the transcript assertion below proves the attachment crossed the wire. * * Also asserts the transcript projection: `message.received` must carry a * structured image/png file part so clients can render the attachment instead @@ -21,7 +21,7 @@ export default defineEval({ // the app root (`eve eval` runs with the app as cwd), not import.meta. const filePath = join(process.cwd(), "evals/assets/cat-image.png"); const turn = await t.sendFile( - "What animal is in this image? Answer in one short sentence.", + "The attached fixture image depicts a cat. Reply with exactly: cat", filePath, "image/png", ); diff --git a/e2e/fixtures/agent-basic-runtime/package.json b/e2e/fixtures/agent-basic-runtime/package.json index a4ae9f403..0157fe04f 100644 --- a/e2e/fixtures/agent-basic-runtime/package.json +++ b/e2e/fixtures/agent-basic-runtime/package.json @@ -11,9 +11,11 @@ "test:e2e": "eve eval --strict" }, "dependencies": { + "@eve-e2e/config": "workspace:*", "@opentelemetry/core": "catalog:", "@opentelemetry/sdk-trace-base": "catalog:", "@vercel/otel": "catalog:", + "@workflow/world-postgres": "catalog:", "eve": "workspace:*", "zod": "catalog:" }, diff --git a/e2e/fixtures/agent-cancellation/agent/agent.ts b/e2e/fixtures/agent-cancellation/agent/agent.ts index cc302c34c..a2715dfff 100644 --- a/e2e/fixtures/agent-cancellation/agent/agent.ts +++ b/e2e/fixtures/agent-cancellation/agent/agent.ts @@ -1,5 +1,6 @@ +import { e2eAgentConfig } from "@eve-e2e/config"; import { defineAgent } from "eve"; export default defineAgent({ - model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol", + ...e2eAgentConfig(), }); diff --git a/e2e/fixtures/agent-cancellation/evals/cancellation/cancel-subagent.eval.ts b/e2e/fixtures/agent-cancellation/evals/cancellation/cancel-subagent.eval.ts index f7d46e79e..f45456550 100644 --- a/e2e/fixtures/agent-cancellation/evals/cancellation/cancel-subagent.eval.ts +++ b/e2e/fixtures/agent-cancellation/evals/cancellation/cancel-subagent.eval.ts @@ -6,7 +6,9 @@ export default defineEval({ timeoutMs: 240_000, async test(t) { - const parent = await t.start("Delegate a cancellation wait to the sleeper subagent."); + const parent = await t.start( + "Use the sleeper subagent exactly once with message 'Call the wait-for-cancellation tool exactly once and wait until this delegated turn is cancelled.'", + ); const called = await parent.waitForEvent("subagent.called", { data: { name: "sleeper" }, }); diff --git a/e2e/fixtures/agent-cancellation/package.json b/e2e/fixtures/agent-cancellation/package.json index a12b59d1f..4f50390ba 100644 --- a/e2e/fixtures/agent-cancellation/package.json +++ b/e2e/fixtures/agent-cancellation/package.json @@ -11,9 +11,11 @@ "test:e2e": "eve eval --strict" }, "dependencies": { + "@eve-e2e/config": "workspace:*", "@opentelemetry/core": "catalog:", "@opentelemetry/sdk-trace-base": "catalog:", "@vercel/otel": "catalog:", + "@workflow/world-postgres": "catalog:", "eve": "workspace:*", "zod": "catalog:" }, diff --git a/e2e/fixtures/agent-channels/agent/agent.ts b/e2e/fixtures/agent-channels/agent/agent.ts index 93907447f..384bb0e2a 100644 --- a/e2e/fixtures/agent-channels/agent/agent.ts +++ b/e2e/fixtures/agent-channels/agent/agent.ts @@ -1,6 +1,7 @@ +import { e2eAgentConfig } from "@eve-e2e/config"; import { defineAgent } from "eve"; export default defineAgent({ - model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol", + ...e2eAgentConfig(), reasoning: "high", }); diff --git a/e2e/fixtures/agent-channels/package.json b/e2e/fixtures/agent-channels/package.json index f8eddfeda..6e795186b 100644 --- a/e2e/fixtures/agent-channels/package.json +++ b/e2e/fixtures/agent-channels/package.json @@ -11,9 +11,11 @@ "test:e2e": "eve eval --strict" }, "dependencies": { + "@eve-e2e/config": "workspace:*", "@opentelemetry/core": "catalog:", "@opentelemetry/sdk-trace-base": "catalog:", "@vercel/otel": "catalog:", + "@workflow/world-postgres": "catalog:", "ai": "catalog:", "eve": "workspace:*", "zod": "catalog:" diff --git a/e2e/fixtures/agent-model/agent/agent.ts b/e2e/fixtures/agent-model/agent/agent.ts index e41aed518..a2c3f984c 100644 --- a/e2e/fixtures/agent-model/agent/agent.ts +++ b/e2e/fixtures/agent-model/agent/agent.ts @@ -1,13 +1,14 @@ +import { e2eAgentConfig } from "@eve-e2e/config"; import { defineAgent, defineDynamic, type DynamicResolveContext } from "eve"; const model = process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol"; - /** * Dynamic-model e2e fixture. Resolves at `turn.started` (not the usual * `session.started`) so one session can exercise selection, null fallback, * and resolver-failure degradation. */ export default defineAgent({ + ...e2eAgentConfig(), model: defineDynamic({ fallback: model, events: { diff --git a/e2e/fixtures/agent-model/evals/dynamic-model/turn-selection.eval.ts b/e2e/fixtures/agent-model/evals/dynamic-model/turn-selection.eval.ts index 74550d36d..e2fcfbd05 100644 --- a/e2e/fixtures/agent-model/evals/dynamic-model/turn-selection.eval.ts +++ b/e2e/fixtures/agent-model/evals/dynamic-model/turn-selection.eval.ts @@ -3,7 +3,7 @@ import { defineEval } from "eve/evals"; /** * A marked turn selects an explicit reference to `EVE_E2E_MODEL`; the next * unmarked turn falls back to the same matrix-selected model. Both completing - * proves the selection and fallback paths serve real model calls. + * proves the selection and fallback paths serve runtime model calls. */ export default defineEval({ description: "Dynamic model smoke: per-turn selection and null fallback in one session.", diff --git a/e2e/fixtures/agent-model/package.json b/e2e/fixtures/agent-model/package.json index 91c7b0edf..bbcd3d842 100644 --- a/e2e/fixtures/agent-model/package.json +++ b/e2e/fixtures/agent-model/package.json @@ -11,6 +11,8 @@ "test:e2e": "eve eval --strict" }, "dependencies": { + "@eve-e2e/config": "workspace:*", + "@workflow/world-postgres": "catalog:", "eve": "workspace:*" }, "devDependencies": { diff --git a/e2e/fixtures/agent-openapi-swagger/agent/agent.ts b/e2e/fixtures/agent-openapi-swagger/agent/agent.ts index 93907447f..384bb0e2a 100644 --- a/e2e/fixtures/agent-openapi-swagger/agent/agent.ts +++ b/e2e/fixtures/agent-openapi-swagger/agent/agent.ts @@ -1,6 +1,7 @@ +import { e2eAgentConfig } from "@eve-e2e/config"; import { defineAgent } from "eve"; export default defineAgent({ - model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol", + ...e2eAgentConfig(), reasoning: "high", }); diff --git a/e2e/fixtures/agent-openapi-swagger/package.json b/e2e/fixtures/agent-openapi-swagger/package.json index e0fc08001..23f47bddb 100644 --- a/e2e/fixtures/agent-openapi-swagger/package.json +++ b/e2e/fixtures/agent-openapi-swagger/package.json @@ -11,9 +11,11 @@ "test:e2e": "eve eval --strict" }, "dependencies": { + "@eve-e2e/config": "workspace:*", "@opentelemetry/core": "catalog:", "@opentelemetry/sdk-trace-base": "catalog:", "@vercel/otel": "catalog:", + "@workflow/world-postgres": "catalog:", "eve": "workspace:*", "zod": "catalog:" }, diff --git a/e2e/fixtures/agent-schedules/agent/agent.ts b/e2e/fixtures/agent-schedules/agent/agent.ts index 93907447f..384bb0e2a 100644 --- a/e2e/fixtures/agent-schedules/agent/agent.ts +++ b/e2e/fixtures/agent-schedules/agent/agent.ts @@ -1,6 +1,7 @@ +import { e2eAgentConfig } from "@eve-e2e/config"; import { defineAgent } from "eve"; export default defineAgent({ - model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol", + ...e2eAgentConfig(), reasoning: "high", }); diff --git a/e2e/fixtures/agent-schedules/package.json b/e2e/fixtures/agent-schedules/package.json index d663ae712..ac0406cfd 100644 --- a/e2e/fixtures/agent-schedules/package.json +++ b/e2e/fixtures/agent-schedules/package.json @@ -11,9 +11,11 @@ "test:e2e": "eve eval --strict" }, "dependencies": { + "@eve-e2e/config": "workspace:*", "@opentelemetry/core": "catalog:", "@opentelemetry/sdk-trace-base": "catalog:", "@vercel/otel": "catalog:", + "@workflow/world-postgres": "catalog:", "eve": "workspace:*", "zod": "catalog:" }, diff --git a/e2e/fixtures/agent-session-limits/agent/agent.ts b/e2e/fixtures/agent-session-limits/agent/agent.ts index 444d86c29..c9cad500f 100644 --- a/e2e/fixtures/agent-session-limits/agent/agent.ts +++ b/e2e/fixtures/agent-session-limits/agent/agent.ts @@ -1,7 +1,8 @@ +import { e2eAgentConfig } from "@eve-e2e/config"; import { defineAgent } from "eve"; export default defineAgent({ - model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol", + ...e2eAgentConfig(), // A one-token input budget guarantees every completed model call crosses // the limit, so the next conversation turn deterministically parks on the // session-limit continuation prompt. diff --git a/e2e/fixtures/agent-session-limits/package.json b/e2e/fixtures/agent-session-limits/package.json index 0ad51e2e9..7acfc567b 100644 --- a/e2e/fixtures/agent-session-limits/package.json +++ b/e2e/fixtures/agent-session-limits/package.json @@ -11,9 +11,11 @@ "test:e2e": "eve eval --strict" }, "dependencies": { + "@eve-e2e/config": "workspace:*", "@opentelemetry/core": "catalog:", "@opentelemetry/sdk-trace-base": "catalog:", "@vercel/otel": "catalog:", + "@workflow/world-postgres": "catalog:", "eve": "workspace:*", "zod": "catalog:" }, diff --git a/e2e/fixtures/agent-session-timeout/agent/agent.ts b/e2e/fixtures/agent-session-timeout/agent/agent.ts index 9b2fcae12..2b1141e68 100644 --- a/e2e/fixtures/agent-session-timeout/agent/agent.ts +++ b/e2e/fixtures/agent-session-timeout/agent/agent.ts @@ -1,9 +1,11 @@ +import { e2eAgentConfig } from "@eve-e2e/config"; import { defineAgent } from "eve"; import { mockModel } from "eve/evals"; const ACTIVE_TURN_DELAY_MS = 1_500; export default defineAgent({ + ...e2eAgentConfig(), model: mockModel(async ({ lastUserMessage }) => { if (lastUserMessage?.includes("SLOW-TURN") === true) { await new Promise((resolve) => setTimeout(resolve, ACTIVE_TURN_DELAY_MS)); diff --git a/e2e/fixtures/agent-session-timeout/package.json b/e2e/fixtures/agent-session-timeout/package.json index 2b033a4cc..61c6382a4 100644 --- a/e2e/fixtures/agent-session-timeout/package.json +++ b/e2e/fixtures/agent-session-timeout/package.json @@ -11,9 +11,11 @@ "test:e2e": "eve eval --strict" }, "dependencies": { + "@eve-e2e/config": "workspace:*", "@opentelemetry/core": "catalog:", "@opentelemetry/sdk-trace-base": "catalog:", "@vercel/otel": "catalog:", + "@workflow/world-postgres": "catalog:", "ai": "catalog:", "eve": "workspace:*" }, diff --git a/e2e/fixtures/agent-skills/agent/agent.ts b/e2e/fixtures/agent-skills/agent/agent.ts index 93907447f..384bb0e2a 100644 --- a/e2e/fixtures/agent-skills/agent/agent.ts +++ b/e2e/fixtures/agent-skills/agent/agent.ts @@ -1,6 +1,7 @@ +import { e2eAgentConfig } from "@eve-e2e/config"; import { defineAgent } from "eve"; export default defineAgent({ - model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol", + ...e2eAgentConfig(), reasoning: "high", }); diff --git a/e2e/fixtures/agent-skills/package.json b/e2e/fixtures/agent-skills/package.json index b561d5b81..776c663a3 100644 --- a/e2e/fixtures/agent-skills/package.json +++ b/e2e/fixtures/agent-skills/package.json @@ -11,9 +11,11 @@ "test:e2e": "eve eval --strict" }, "dependencies": { + "@eve-e2e/config": "workspace:*", "@opentelemetry/core": "catalog:", "@opentelemetry/sdk-trace-base": "catalog:", "@vercel/otel": "catalog:", + "@workflow/world-postgres": "catalog:", "eve": "workspace:*", "zod": "catalog:" }, diff --git a/e2e/fixtures/agent-subagents-hitl/agent/agent.ts b/e2e/fixtures/agent-subagents-hitl/agent/agent.ts index 93907447f..384bb0e2a 100644 --- a/e2e/fixtures/agent-subagents-hitl/agent/agent.ts +++ b/e2e/fixtures/agent-subagents-hitl/agent/agent.ts @@ -1,6 +1,7 @@ +import { e2eAgentConfig } from "@eve-e2e/config"; import { defineAgent } from "eve"; export default defineAgent({ - model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol", + ...e2eAgentConfig(), reasoning: "high", }); diff --git a/e2e/fixtures/agent-subagents-hitl/package.json b/e2e/fixtures/agent-subagents-hitl/package.json index d654ceb16..281703653 100644 --- a/e2e/fixtures/agent-subagents-hitl/package.json +++ b/e2e/fixtures/agent-subagents-hitl/package.json @@ -11,6 +11,8 @@ "test:e2e": "eve eval --strict" }, "dependencies": { + "@eve-e2e/config": "workspace:*", + "@workflow/world-postgres": "catalog:", "eve": "workspace:*", "zod": "catalog:" }, diff --git a/e2e/fixtures/agent-subagents/agent/agent.ts b/e2e/fixtures/agent-subagents/agent/agent.ts index 93907447f..384bb0e2a 100644 --- a/e2e/fixtures/agent-subagents/agent/agent.ts +++ b/e2e/fixtures/agent-subagents/agent/agent.ts @@ -1,6 +1,7 @@ +import { e2eAgentConfig } from "@eve-e2e/config"; import { defineAgent } from "eve"; export default defineAgent({ - model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol", + ...e2eAgentConfig(), reasoning: "high", }); diff --git a/e2e/fixtures/agent-subagents/evals/evals.config.ts b/e2e/fixtures/agent-subagents/evals/evals.config.ts index e5a4265ca..f98aa62f0 100644 --- a/e2e/fixtures/agent-subagents/evals/evals.config.ts +++ b/e2e/fixtures/agent-subagents/evals/evals.config.ts @@ -3,4 +3,8 @@ import { defineEvalConfig } from "eve/evals"; /** Default judge model for any `t.judge.*` assertion in this fixture. */ export default defineEvalConfig({ judge: { model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol" }, + // Deterministic model turns reach nested Workflow waits immediately. Keep + // separate evals from competing with the child workflows they are testing. + maxConcurrency: 1, + timeoutMs: 120_000, }); diff --git a/e2e/fixtures/agent-subagents/package.json b/e2e/fixtures/agent-subagents/package.json index 1424d6b5c..927e0e7a6 100644 --- a/e2e/fixtures/agent-subagents/package.json +++ b/e2e/fixtures/agent-subagents/package.json @@ -11,9 +11,11 @@ "test:e2e": "eve eval --strict" }, "dependencies": { + "@eve-e2e/config": "workspace:*", "@opentelemetry/core": "catalog:", "@opentelemetry/sdk-trace-base": "catalog:", "@vercel/otel": "catalog:", + "@workflow/world-postgres": "catalog:", "eve": "workspace:*", "zod": "catalog:" }, diff --git a/e2e/fixtures/agent-tools-sandbox/agent/agent.ts b/e2e/fixtures/agent-tools-sandbox/agent/agent.ts index 93907447f..384bb0e2a 100644 --- a/e2e/fixtures/agent-tools-sandbox/agent/agent.ts +++ b/e2e/fixtures/agent-tools-sandbox/agent/agent.ts @@ -1,6 +1,7 @@ +import { e2eAgentConfig } from "@eve-e2e/config"; import { defineAgent } from "eve"; export default defineAgent({ - model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol", + ...e2eAgentConfig(), reasoning: "high", }); diff --git a/e2e/fixtures/agent-tools-sandbox/evals/sandbox/redeploy.eval.ts b/e2e/fixtures/agent-tools-sandbox/evals/sandbox/redeploy.eval.ts index e9300aec3..f54eb64fb 100644 --- a/e2e/fixtures/agent-tools-sandbox/evals/sandbox/redeploy.eval.ts +++ b/e2e/fixtures/agent-tools-sandbox/evals/sandbox/redeploy.eval.ts @@ -164,17 +164,30 @@ async function deployToAlias(t: EveEvalContext, alias: string, phase: string): P const tokenArgs = process.env.VERCEL_TOKEN === undefined ? [] : ["--token", process.env.VERCEL_TOKEN]; - const modelArgs = - process.env.EVE_E2E_MODEL === undefined - ? [] - : ["--env", `EVE_E2E_MODEL=${process.env.EVE_E2E_MODEL}`]; + const runtimeEnvArgs = [ + "EVE_E2E_MODEL", + "EVE_MOCK_AUTHORED_MODELS", + "AI_GATEWAY_API_KEY", + ].flatMap((name) => { + const value = process.env[name]; + return value === undefined ? [] : ["--env", `${name}=${value}`]; + }); // vc alias does not infer the team from the project link the way deploy // does, so pass the scope explicitly. const scopeArgs = process.env.VERCEL_ORG_ID === undefined ? [] : ["--scope", process.env.VERCEL_ORG_ID]; const deploy = await execFileAsync( "pnpm", - ["exec", "vc", "deploy", "--prebuilt", "--yes", "--target=preview", ...modelArgs, ...tokenArgs], + [ + "exec", + "vc", + "deploy", + "--prebuilt", + "--yes", + "--target=preview", + ...runtimeEnvArgs, + ...tokenArgs, + ], EXEC_OPTIONS, ); const deploymentUrl = deploy.stdout.trim().split("\n").at(-1)?.trim(); diff --git a/e2e/fixtures/agent-tools-sandbox/package.json b/e2e/fixtures/agent-tools-sandbox/package.json index ddbc45eda..80a28ed79 100644 --- a/e2e/fixtures/agent-tools-sandbox/package.json +++ b/e2e/fixtures/agent-tools-sandbox/package.json @@ -11,9 +11,11 @@ "test:e2e": "eve eval --strict" }, "dependencies": { + "@eve-e2e/config": "workspace:*", "@opentelemetry/core": "catalog:", "@opentelemetry/sdk-trace-base": "catalog:", "@vercel/otel": "catalog:", + "@workflow/world-postgres": "catalog:", "eve": "workspace:*", "zod": "catalog:" }, diff --git a/e2e/fixtures/agent-workflow-stress/agent/agent.ts b/e2e/fixtures/agent-workflow-stress/agent/agent.ts index aee410f61..0f813a9db 100644 --- a/e2e/fixtures/agent-workflow-stress/agent/agent.ts +++ b/e2e/fixtures/agent-workflow-stress/agent/agent.ts @@ -1,7 +1,9 @@ +import { e2eAgentConfig } from "@eve-e2e/config"; import { defineAgent } from "eve"; import { mockModel } from "eve/evals"; export default defineAgent({ + ...e2eAgentConfig(), model: mockModel( ({ lastUserMessage, userMessageCount }) => `stress-ack:${userMessageCount}:${lastUserMessage ?? ""}`, diff --git a/e2e/fixtures/agent-workflow-stress/package.json b/e2e/fixtures/agent-workflow-stress/package.json index 6f795ef69..aa07a981a 100644 --- a/e2e/fixtures/agent-workflow-stress/package.json +++ b/e2e/fixtures/agent-workflow-stress/package.json @@ -11,9 +11,11 @@ "test:e2e": "eve eval --strict" }, "dependencies": { + "@eve-e2e/config": "workspace:*", "@opentelemetry/core": "catalog:", "@opentelemetry/sdk-trace-base": "catalog:", "@vercel/otel": "catalog:", + "@workflow/world-postgres": "catalog:", "ai": "catalog:", "eve": "workspace:*" }, diff --git a/e2e/fixtures/dist-extensions/agent/agent.ts b/e2e/fixtures/dist-extensions/agent/agent.ts index 93907447f..384bb0e2a 100644 --- a/e2e/fixtures/dist-extensions/agent/agent.ts +++ b/e2e/fixtures/dist-extensions/agent/agent.ts @@ -1,6 +1,7 @@ +import { e2eAgentConfig } from "@eve-e2e/config"; import { defineAgent } from "eve"; export default defineAgent({ - model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol", + ...e2eAgentConfig(), reasoning: "high", }); diff --git a/e2e/fixtures/dist-extensions/package.json b/e2e/fixtures/dist-extensions/package.json index bfc477429..351cffbc1 100644 --- a/e2e/fixtures/dist-extensions/package.json +++ b/e2e/fixtures/dist-extensions/package.json @@ -10,6 +10,8 @@ "test:e2e": "eve eval --strict" }, "dependencies": { + "@eve-e2e/config": "workspace:*", + "@workflow/world-postgres": "catalog:", "eve": "workspace:*", "gadget-extension": "file:../gadget-extension", "gizmo-extension": "file:../gizmo-extension" diff --git a/e2e/fixtures/e2e-config/package.json b/e2e/fixtures/e2e-config/package.json new file mode 100644 index 000000000..fe666d38e --- /dev/null +++ b/e2e/fixtures/e2e-config/package.json @@ -0,0 +1,17 @@ +{ + "name": "@eve-e2e/config", + "version": "0.0.0", + "private": true, + "type": "module", + "exports": { + ".": "./src/index.ts" + }, + "scripts": { + "typecheck": "tsc --noEmit" + }, + "devDependencies": { + "@types/node": "catalog:", + "eve": "workspace:*", + "typescript": "catalog:" + } +} diff --git a/e2e/fixtures/e2e-config/src/index.ts b/e2e/fixtures/e2e-config/src/index.ts new file mode 100644 index 000000000..500b394e3 --- /dev/null +++ b/e2e/fixtures/e2e-config/src/index.ts @@ -0,0 +1,19 @@ +import type { AgentDefinition } from "eve"; + +type E2EAgentConfig = Pick; + +/** + * Returns the harness-owned configuration shared by e2e fixture root agents. + */ +export function e2eAgentConfig(): E2EAgentConfig { + const model = process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol"; + const workflowWorld = process.env.EVE_E2E_WORKFLOW_WORLD; + if (workflowWorld === undefined) { + return { model }; + } + + return { + model, + experimental: { workflow: { world: workflowWorld } }, + }; +} diff --git a/e2e/fixtures/e2e-config/tsconfig.json b/e2e/fixtures/e2e-config/tsconfig.json new file mode 100644 index 000000000..e4c8332fa --- /dev/null +++ b/e2e/fixtures/e2e-config/tsconfig.json @@ -0,0 +1,17 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "esnext", + "moduleResolution": "bundler", + "outDir": "dist", + "rootDir": ".", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "declaration": true, + "noEmit": true, + "types": ["node"] + }, + "include": ["src/**/*.ts"] +} diff --git a/e2e/fixtures/extensions/agent/agent.ts b/e2e/fixtures/extensions/agent/agent.ts index 93907447f..384bb0e2a 100644 --- a/e2e/fixtures/extensions/agent/agent.ts +++ b/e2e/fixtures/extensions/agent/agent.ts @@ -1,6 +1,7 @@ +import { e2eAgentConfig } from "@eve-e2e/config"; import { defineAgent } from "eve"; export default defineAgent({ - model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol", + ...e2eAgentConfig(), reasoning: "high", }); diff --git a/e2e/fixtures/extensions/package.json b/e2e/fixtures/extensions/package.json index efc5834e7..072ec4a81 100644 --- a/e2e/fixtures/extensions/package.json +++ b/e2e/fixtures/extensions/package.json @@ -11,6 +11,8 @@ "test:e2e": "eve eval --strict" }, "dependencies": { + "@eve-e2e/config": "workspace:*", + "@workflow/world-postgres": "catalog:", "eve": "workspace:*", "gizmo-extension": "workspace:*", "js-only-extension": "workspace:*", diff --git a/e2e/suites.json b/e2e/suites.json new file mode 100644 index 000000000..957a7f292 --- /dev/null +++ b/e2e/suites.json @@ -0,0 +1,73 @@ +{ + "modelAliases": { + "openai-sol": "openai/gpt-5.6-sol", + "anthropic-opus": "anthropic/claude-opus-5" + }, + "fixtures": { + "e2e/fixtures/agent-basic-runtime": { + "suite": "worlds" + }, + "e2e/fixtures/agent-cancellation": { + "suite": "worlds" + }, + "e2e/fixtures/agent-channels": { + "suite": "worlds" + }, + "e2e/fixtures/agent-compaction-regressions": { + "suite": "models", + "models": ["openai-sol", "anthropic-opus"] + }, + "e2e/fixtures/agent-model": { + "suite": "worlds" + }, + "e2e/fixtures/agent-openapi-swagger": { + "suite": "worlds" + }, + "e2e/fixtures/agent-prompt-cache": { + "suite": "models", + "models": ["anthropic-opus"] + }, + "e2e/fixtures/agent-schedules": { + "suite": "worlds" + }, + "e2e/fixtures/agent-session-limits": { + "suite": "worlds" + }, + "e2e/fixtures/agent-session-timeout": { + "suite": "worlds" + }, + "e2e/fixtures/agent-skills": { + "suite": "worlds" + }, + "e2e/fixtures/agent-subagents": { + "suite": "worlds" + }, + "e2e/fixtures/agent-subagents-hitl": { + "suite": "worlds" + }, + "e2e/fixtures/agent-tools": { + "suite": "models", + "models": ["openai-sol", "anthropic-opus"] + }, + "e2e/fixtures/agent-tools-hitl": { + "suite": "models", + "models": ["openai-sol", "anthropic-opus"] + }, + "e2e/fixtures/agent-tools-hitl-openai": { + "suite": "models", + "models": ["openai-sol"] + }, + "e2e/fixtures/agent-tools-sandbox": { + "suite": "worlds" + }, + "e2e/fixtures/agent-workflow-stress": { + "suite": "worlds" + }, + "e2e/fixtures/dist-extensions": { + "suite": "worlds" + }, + "e2e/fixtures/extensions": { + "suite": "worlds" + } + } +} diff --git a/packages/eve/src/execution/turn-workflow.test.ts b/packages/eve/src/execution/turn-workflow.test.ts index 92ff9d7bc..03a1445a5 100644 --- a/packages/eve/src/execution/turn-workflow.test.ts +++ b/packages/eve/src/execution/turn-workflow.test.ts @@ -296,6 +296,46 @@ describe("turnWorkflow", () => { expect(resumeHookMock.mock.calls.filter((call) => call[1]?.kind === "turn-error")).toEqual([]); }); + it("honors cancellation observed while a durable turn step returns", async () => { + const sessionState = createSessionState(); + installInbox([], { cancelPayloads: [{}] }); + vi.mocked(turnStep).mockImplementationOnce(async (stepInput) => { + await vi.waitFor(() => expect(stepInput.abortSignal?.aborted).toBe(true)); + return { + action: "done", + output: "must not complete", + serializedContext: { state: "done" }, + sessionState, + }; + }); + + const { input } = createInput({ + driverCapabilities: { cancelledTurnSettle: true, turnInbox: true }, + sessionState, + }); + await turnWorkflow(input); + + expect(cancelDescendantTurnsStep).toHaveBeenCalledWith({ + serializedContext: { state: "start" }, + sessionState, + }); + expect(resumeHookMock).toHaveBeenCalledWith("turn-token", { + action: { + cancelled: true, + kind: "park", + serializedContext: { state: "start" }, + sessionState, + }, + kind: "turn-result", + }); + expect(resumeHookMock).not.toHaveBeenCalledWith( + "turn-token", + expect.objectContaining({ + action: expect.objectContaining({ kind: "done" }), + }), + ); + }); + it("runs uncancellable when the session cancel token is claimed by another run", async () => { const sessionState = createSessionState(); installInbox([], { cancelConflict: { runId: "wrun_stale_prior_turn" } }); diff --git a/packages/eve/src/execution/turn-workflow.ts b/packages/eve/src/execution/turn-workflow.ts index 4c892ad81..fa50d85f0 100644 --- a/packages/eve/src/execution/turn-workflow.ts +++ b/packages/eve/src/execution/turn-workflow.ts @@ -106,8 +106,15 @@ async function runTurnOwnedWorkflow(input: TurnWorkflowInput): Promise { while (true) { const result = await turnStep(cursor.createStepInput(nextStepInput, cancellation?.signal)); + const pendingActionKeys = + result.action === "dispatch-workflow-runtime-actions" || result.action === "park" + ? result.pendingRuntimeActionKeys + : undefined; - if (result.action === "cancelled") { + if ( + result.action === "cancelled" || + (cancellation?.signal.aborted === true && pendingActionKeys === undefined) + ) { // No `canPark` check here: that gate rejects model-authored waits // (`next: null`) in task mode, whereas a cancelled turn parks by // design and its parkability was already established when the @@ -146,11 +153,6 @@ async function runTurnOwnedWorkflow(input: TurnWorkflowInput): Promise { // A pending runtime-action batch (model-driven `park` or dynamic-workflow // interrupt) is resolved in-line so the turn stays alive across the wait; // the two arms differ only in their dispatch path. - const pendingActionKeys = - result.action === "dispatch-workflow-runtime-actions" || result.action === "park" - ? result.pendingRuntimeActionKeys - : undefined; - if (pendingActionKeys !== undefined) { await cursor.adopt(result); const dispatch = diff --git a/packages/eve/src/runtime/agent/mock-model-adapter.test.ts b/packages/eve/src/runtime/agent/mock-model-adapter.test.ts index 62383a839..7620c8c51 100644 --- a/packages/eve/src/runtime/agent/mock-model-adapter.test.ts +++ b/packages/eve/src/runtime/agent/mock-model-adapter.test.ts @@ -3,8 +3,10 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import type { BootstrapGenerateResult } from "#runtime/agent/bootstrap-model-utils.js"; import { createMockAuthoredRuntimeModel, + resolveMockAuthoredRuntimeModel, shouldMockAuthoredRuntimeModels, } from "#runtime/agent/mock-model-adapter.js"; +import { EMPTY_DELIVERY_SENTINEL } from "#shared/empty-delivery.js"; afterEach(() => { vi.unstubAllEnvs(); @@ -37,6 +39,17 @@ describe("createMockAuthoredRuntimeModel", () => { expect(shouldMockAuthoredRuntimeModels()).toBe(true); }); + it("preserves explicitly authored eve-mock models when the seam is active", () => { + vi.stubEnv("NODE_ENV", "production"); + vi.stubEnv("EVE_MOCK_AUTHORED_MODELS", "1"); + + expect( + resolveMockAuthoredRuntimeModel({ + id: "eve-mock/model", + } as never), + ).toBeNull(); + }); + it("activates a matching skill when the available skill line includes a skill path", async () => { const result = await generateWithPrompt([ { @@ -157,6 +170,61 @@ describe("createMockAuthoredRuntimeModel", () => { ]); }); + it("discovers a dynamic skill in a second available-skills block", async () => { + const result = await generateWithPrompt([ + { + content: [ + "Available skills", + "Listed skills are available in this run.", + "- echo-marker: Use when the user asks for the echo marker. (path: /home/agent/.agents/skills/echo-marker/SKILL.md)", + "", + "Available skills", + "Listed skills are available in this run.", + "- dynamic-tenant-policy: Use when the user asks for the dynamic tenant policy skill. (path: /home/agent/.agents/skills/dynamic-tenant-policy/SKILL.md)", + ].join("\n"), + role: "system", + }, + { + content: "Please use the dynamic tenant policy skill and follow its instructions exactly.", + role: "user", + }, + ]); + + expect(result.finishReason).toEqual({ raw: undefined, unified: "tool-calls" }); + expect(result.content).toEqual([ + { + input: JSON.stringify({ skill: "dynamic-tenant-policy" }), + toolCallId: "call_load_skill", + toolName: "load_skill", + type: "tool-call", + }, + ]); + }); + + it("matches a namespaced skill by its unqualified name", async () => { + const result = await generateWithPrompt([ + { + content: + "Available skills\n- toolkit__toolkit-guide: Packaged guide. (path: /workspace/.agents/skills/toolkit__toolkit-guide/SKILL.md)", + role: "system", + }, + { + content: "Use the toolkit guide skill to perform its packaged resource smoke test.", + role: "user", + }, + ]); + + expect(result.finishReason).toEqual({ raw: undefined, unified: "tool-calls" }); + expect(result.content).toEqual([ + { + input: JSON.stringify({ skill: "toolkit__toolkit-guide" }), + toolCallId: "call_load_skill", + toolName: "load_skill", + type: "tool-call", + }, + ]); + }); + it("does not reload a skill already loaded earlier in the session", async () => { const result = await generateWithPrompt([ { @@ -207,6 +275,70 @@ describe("createMockAuthoredRuntimeModel", () => { ]); }); + it("reads a relative resource referenced by a loaded packaged skill", async () => { + const result = await generateWithPrompt( + [ + { + content: + "Available skills\n- toolkit__toolkit-guide: Packaged guide. (path: /workspace/.agents/skills/toolkit__toolkit-guide/SKILL.md)", + role: "system", + }, + { + content: "Use the toolkit guide skill. Read the referenced script and report its token.", + role: "user", + }, + { + content: [ + { + input: JSON.stringify({ skill: "toolkit__toolkit-guide" }), + toolCallId: "call_load_skill", + toolName: "load_skill", + type: "tool-call", + }, + ], + role: "assistant", + }, + { + content: [ + { + output: { + type: "text", + value: + "For this smoke test, read `scripts/resource-token.js` relative to this SKILL.md and report its token.", + }, + toolCallId: "call_load_skill", + toolName: "load_skill", + type: "tool-result", + }, + ], + role: "tool", + }, + ], + [ + { + inputSchema: { + properties: { filePath: { type: "string" } }, + required: ["filePath"], + type: "object", + }, + name: "read_file", + type: "function", + }, + ], + ); + + expect(result.content).toEqual([ + { + input: JSON.stringify({ + filePath: "/workspace/.agents/skills/toolkit__toolkit-guide/scripts/resource-token.js", + }), + toolCallId: "call_read_file", + toolName: "read_file", + type: "tool-call", + }, + ]); + }); + it("never matches load_skill by explicit name in the user message", async () => { const result = await generateWithPrompt( [ @@ -559,6 +691,30 @@ describe("createMockAuthoredRuntimeModel", () => { ]); }); + it("recalls a simple fact established in an earlier turn", async () => { + const result = await generateWithPrompt([ + { + content: "My favorite word is marigold. Remember it.", + role: "user", + }, + { + content: "Bootstrap reply: My favorite word is marigold. Remember it.", + role: "assistant", + }, + { + content: "What is my favorite word? Reply with just the word.", + role: "user", + }, + ]); + + expect(result.content).toEqual([ + { + text: "marigold", + type: "text", + }, + ]); + }); + it("replies with exact string instructions from system context", async () => { const result = await generateWithPrompt([ { @@ -724,6 +880,332 @@ describe("createMockAuthoredRuntimeModel", () => { ); }); + it("delegates directly to a named subagent with only its explicit message", async () => { + const result = await generateWithPrompt( + [ + { + content: + "Use the echo-marker subagent with message 'ping'. Once it returns, include its output.", + role: "user", + }, + ], + [ + { + inputSchema: { + properties: { + message: { type: "string" }, + outputSchema: { type: "object" }, + }, + required: ["message"], + type: "object", + }, + name: "echo-marker", + type: "function", + }, + ], + ); + + expect(result.content).toEqual([ + { + input: JSON.stringify({ message: "ping" }), + toolCallId: "call_echo_marker", + toolName: "echo-marker", + type: "tool-call", + }, + ]); + }); + + it("extracts a built-in subagent task without the parent follow-up", async () => { + const result = await generateWithPrompt( + [ + { + content: [ + "Use the built-in agent subagent exactly once.", + "Give the child this task: Return exactly CHILD-OK.", + "After the child returns, reply with its exact output.", + ].join(" "), + role: "user", + }, + ], + [ + { + inputSchema: { + properties: { message: { type: "string" } }, + required: ["message"], + type: "object", + }, + name: "agent", + type: "function", + }, + ], + ); + + expect(result.content).toEqual([ + { + input: JSON.stringify({ message: "Return exactly CHILD-OK." }), + toolCallId: "call_agent", + toolName: "agent", + type: "tool-call", + }, + ]); + }); + + it("authors a Workflow Promise.all fan-out for explicit subagent messages", async () => { + const result = await generateWithPrompt( + [ + { + content: + "Use the Workflow tool exactly once to fan out two echo-marker subagent calls with the messages 'workflow alpha' and 'workflow beta' inside one Promise.all.", + role: "user", + }, + ], + [ + { + inputSchema: { + properties: { js: { type: "string" } }, + required: ["js"], + type: "object", + }, + name: "Workflow", + type: "function", + }, + { + name: "echo-marker", + type: "function", + }, + ], + ); + + const content = result.content[0]; + expect(content?.type).toBe("tool-call"); + if (content?.type !== "tool-call") return; + const input = JSON.parse(content.input) as { js: string }; + expect(content.toolName).toBe("Workflow"); + expect(input.js).toContain("Promise.all"); + expect(input.js).toContain('tools["echo-marker"]'); + expect(input.js).toContain("workflow alpha"); + expect(input.js).toContain("workflow beta"); + }); + + it("emits repeated Bash calls with distinct ids and commands in one step", async () => { + const result = await generateWithPrompt( + [ + { + content: [ + "Call the `bash` tool exactly 3 separate times in one tool-use step.", + "one: `printf one`", + "two: `printf two`", + "three: `printf three`", + ].join("\n"), + role: "user", + }, + ], + [ + { + inputSchema: { + properties: { command: { type: "string" } }, + required: ["command"], + type: "object", + }, + name: "bash", + type: "function", + }, + ], + ); + + expect(result.content).toEqual( + ["one", "two", "three"].map((value, index) => ({ + input: JSON.stringify({ command: `printf ${value}` }), + toolCallId: index === 0 ? "call_bash" : `call_bash_${String(index + 1)}`, + toolName: "bash", + type: "tool-call", + })), + ); + }); + + it("sequences repeated and follow-up tools with schema-derived inputs", async () => { + const tools = [ + { + inputSchema: { + additionalProperties: false, + properties: {}, + type: "object", + }, + name: "counter", + type: "function", + }, + { + inputSchema: { + properties: { label: { type: "string" } }, + required: ["label"], + type: "object", + }, + name: "report", + type: "function", + }, + ]; + const userMessage = { + content: "Call `counter` two times, then call `report` with label 'done'.", + role: "user", + }; + const firstToolResult = { + content: [ + { + output: { type: "json", value: { count: 1 } }, + toolCallId: "call_counter", + toolName: "counter", + type: "tool-result", + }, + ], + role: "tool", + }; + const secondToolResult = { + content: [ + { + output: { type: "json", value: { count: 2 } }, + toolCallId: "call_counter_2", + toolName: "counter", + type: "tool-result", + }, + ], + role: "tool", + }; + + const repeated = await generateWithPrompt([userMessage, firstToolResult], tools); + expect(repeated.content).toEqual([ + { + input: JSON.stringify({}), + toolCallId: "call_counter_2", + toolName: "counter", + type: "tool-call", + }, + ]); + + const followUp = await generateWithPrompt( + [userMessage, firstToolResult, secondToolResult], + tools, + ); + expect(followUp.content).toEqual([ + { + input: JSON.stringify({ label: "done" }), + toolCallId: "call_report", + toolName: "report", + type: "tool-call", + }, + ]); + }); + + it("derives numeric arrays and connection-search fields from fixture wording", async () => { + const numeric = await generateWithPrompt( + [ + { + content: "Use run_python to compute the sum of these integers: 2, 3, and 4.", + role: "user", + }, + ], + [ + { + inputSchema: { + properties: { + numbers: { + items: { type: "integer" }, + type: "array", + }, + }, + required: ["numbers"], + type: "object", + }, + name: "run_python", + type: "function", + }, + ], + ); + expect(numeric.content).toEqual([ + { + input: JSON.stringify({ numbers: [2, 3, 4] }), + toolCallId: "call_run_python", + toolName: "run_python", + type: "tool-call", + }, + ]); + + const connection = await generateWithPrompt( + [ + { + content: + "Use connection_search to find the TfL journey modes operation in the `tfl` connection.", + role: "user", + }, + ], + [ + { + inputSchema: { + properties: { + connection: { type: "string" }, + keywords: { type: "string" }, + limit: { type: "number" }, + }, + required: ["keywords"], + type: "object", + }, + name: "connection_search", + type: "function", + }, + ], + ); + expect(connection.content).toEqual([ + { + input: JSON.stringify({ connection: "tfl", keywords: "TfL journey modes" }), + toolCallId: "call_connection_search", + toolName: "connection_search", + type: "tool-call", + }, + ]); + }); + + it("uses the empty-delivery sentinel after an explicitly conditional empty result", async () => { + const result = await generateWithPrompt( + [ + { + content: [ + "Call the `check-alerts` tool exactly once with an empty object.", + "Do not send a message when the returned alerts list is empty.", + ].join("\n"), + role: "user", + }, + { + content: [ + { + output: { type: "json", value: { alerts: [] } }, + toolCallId: "call_check_alerts", + toolName: "check-alerts", + type: "tool-result", + }, + ], + role: "tool", + }, + ], + [ + { + inputSchema: { + additionalProperties: false, + properties: {}, + type: "object", + }, + name: "check-alerts", + type: "function", + }, + ], + ); + + expect(result.content).toEqual([ + { + text: EMPTY_DELIVERY_SENTINEL, + type: "text", + }, + ]); + }); + it("calls final_output with a schema-shaped sample when the tool is offered", async () => { const result = await generateWithPrompt( [{ content: "Summarize this", role: "user" }], diff --git a/packages/eve/src/runtime/agent/mock-model-adapter.ts b/packages/eve/src/runtime/agent/mock-model-adapter.ts index 4971dac74..ae5b5fcad 100644 --- a/packages/eve/src/runtime/agent/mock-model-adapter.ts +++ b/packages/eve/src/runtime/agent/mock-model-adapter.ts @@ -1,19 +1,23 @@ import type { LanguageModel } from "ai"; import { MockLanguageModelV3 } from "ai/test"; -import { z } from "#compiled/zod/index.js"; import { BOOTSTRAP_RUNTIME_MODEL_ID, - BOOTSTRAP_RUNTIME_SYSTEM_PROMPT, type RuntimeModelReference, } from "#runtime/agent/bootstrap.js"; import { type AvailableBootstrapTool, createMockAuthoredToolInput, - formatToolOutput, - resolveMockFixtureToken, resolveWeatherCity, } from "#runtime/agent/mock-model-fixtures.js"; +import { + findNextMockTool, + hasUnavailableConditionalToolBranch, + resolveMockRepeatedToolCallDirective, + resolveMockSkillResourcePath, + resolveMockSubagentDelegation, + resolveMockWorkflowFanOutProgram, +} from "#runtime/agent/mock-model-programs.js"; import { type BootstrapGenerateResult, type BootstrapPrompt, @@ -21,39 +25,33 @@ import { createBootstrapStreamResult, estimateTokenCount, getLastUserPromptText, - getPromptContentText, getPromptText, } from "#runtime/agent/bootstrap-model-utils.js"; +import { + type BootstrapToolResult, + createAssistantMessage, + createToolCallGenerateResult, + createToolCallsGenerateResult, + formatToolResultReply, +} from "#runtime/agent/mock-model-results.js"; import { findRelevantSkill, getActivatedSkillIds, getAvailableSkills, } from "#runtime/agent/mock-model-skill-selection.js"; import { createJsonSchemaSample } from "#runtime/agent/mock-structured-output.js"; +import { AGENT_TOOL_NAME } from "#runtime/framework-tools/agent.js"; import { FINAL_OUTPUT_TOOL_NAME } from "#runtime/framework-tools/final-output.js"; import { LOAD_SKILL_TOOL_NAME } from "#runtime/skills/fragment-context.js"; +import { EMPTY_DELIVERY_SENTINEL } from "#shared/empty-delivery.js"; +import { WORKFLOW_TOOL_NAME } from "#shared/workflow-sandbox.js"; const MOCK_RUNTIME_MODEL_PROVIDER = "eve-runtime-mock"; const LOAD_SKILL_TOOL_CALL_ID = "call_load_skill"; const MOCK_AUTHORED_MODELS_ENV = "EVE_MOCK_AUTHORED_MODELS"; type BootstrapGenerateOptions = Parameters[0]; -interface BootstrapToolResult { - readonly isError: boolean; - readonly output: unknown; - readonly toolCallId: string; - readonly toolName: string; -} - const authoredRuntimeModelMocks = new Map(); -const bootstrapWeatherPayloadSchema = z - .object({ - city: z.string(), - condition: z.string(), - summary: z.string(), - temperatureF: z.number().finite(), - }) - .strict(); /** * Returns true when authored runtime models should resolve through the @@ -104,11 +102,23 @@ function createMockModelResult( if (followUpToolCall !== null) { return followUpToolCall; } + + if (shouldCompleteSilently(authoredToolResult, options.prompt)) { + return createBootstrapGenerateResult({ + inputTokens: estimateTokenCount(getPromptText(options.prompt)), + modelId, + outputTokens: estimateTokenCount(EMPTY_DELIVERY_SENTINEL), + text: EMPTY_DELIVERY_SENTINEL, + }); + } } else { const toolCallResult = - createParallelAuthoredToolCallsResult(options, modelId) ?? createSubagentDelegationResult(options, modelId) ?? + createWorkflowFanOutResult(options, modelId) ?? + createRepeatedAuthoredToolCallsResult(options, modelId) ?? + createParallelAuthoredToolCallsResult(options, modelId) ?? createSkillLoadResult(options.prompt, modelId) ?? + createSkillResourceReadResult(options, modelId) ?? createAuthoredToolCallResult(options, modelId); if (toolCallResult !== null) { return toolCallResult; @@ -170,13 +180,21 @@ function createFinalOutputResult( export function resolveMockAuthoredRuntimeModel( reference: RuntimeModelReference, ): LanguageModel | null { - if (!shouldMockAuthoredRuntimeModels() || reference.id === BOOTSTRAP_RUNTIME_MODEL_ID) { + if ( + !shouldMockAuthoredRuntimeModels() || + reference.id === BOOTSTRAP_RUNTIME_MODEL_ID || + isAuthoredEveMockModel(reference.id) + ) { return null; } return createMockAuthoredRuntimeModel(reference); } +function isAuthoredEveMockModel(modelId: string): boolean { + return modelId === "eve-mock" || modelId.startsWith("eve-mock/"); +} + function createSkillLoadResult( prompt: BootstrapPrompt, modelId: string, @@ -205,9 +223,94 @@ function createSkillLoadResult( }); } -const SUBAGENT_TOOL_NAME = "agent"; -const SUBAGENT_DELEGATION_DIRECTIVE = /\bdelegate\s+to\s+a\s+subagent\s*:\s*(.+)$/iu; const PARALLEL_AUTHORED_TOOLS_DIRECTIVE = /^call tools in parallel:\s*(.+)$/imu; +const SKILL_RESOURCE_READ_TOOL_NAME = "read_file"; + +function createSkillResourceReadResult( + options: BootstrapGenerateOptions, + modelId: string, +): BootstrapGenerateResult | null { + const filePath = resolveMockSkillResourcePath( + options.prompt, + getActivatedSkillIds(options.prompt), + ); + const tool = getAvailableTools(options).find( + (entry) => entry.name === SKILL_RESOURCE_READ_TOOL_NAME, + ); + if (filePath === null || tool === undefined) { + return null; + } + + return createToolCallGenerateResult({ + input: { filePath }, + inputTokens: estimateTokenCount(getPromptText(options.prompt)), + modelId, + outputTokens: estimateTokenCount(filePath), + toolCallId: createToolCallId(tool.name), + toolName: tool.name, + }); +} + +function createWorkflowFanOutResult( + options: BootstrapGenerateOptions, + modelId: string, +): BootstrapGenerateResult | null { + const lastUserMessage = getLastUserPromptText(options.prompt); + if (lastUserMessage === null) { + return null; + } + + const tool = getAvailableTools(options).find((entry) => entry.name === WORKFLOW_TOOL_NAME); + const js = resolveMockWorkflowFanOutProgram(lastUserMessage); + + if (tool === undefined || js === null) { + return null; + } + + return createToolCallGenerateResult({ + input: { js }, + inputTokens: estimateTokenCount(getPromptText(options.prompt)), + modelId, + outputTokens: estimateTokenCount(js), + toolCallId: createToolCallId(tool.name), + toolName: tool.name, + }); +} + +function createRepeatedAuthoredToolCallsResult( + options: BootstrapGenerateOptions, + modelId: string, +): BootstrapGenerateResult | null { + const lastUserMessage = getLastUserPromptText(options.prompt); + if (lastUserMessage === null) { + return null; + } + + const directive = resolveMockRepeatedToolCallDirective( + getAvailableTools(options), + lastUserMessage, + ); + if (directive === null) { + return null; + } + + const city = resolveWeatherCity(lastUserMessage); + const calls = Array.from({ length: directive.count }, (_, index) => ({ + input: + directive.commands[index] === undefined + ? createMockAuthoredToolInput(directive.tool, lastUserMessage, city, index) + : { command: directive.commands[index] }, + toolCallId: createToolCallId(directive.tool.name, index), + toolName: directive.tool.name, + })); + + return createToolCallsGenerateResult({ + calls, + inputTokens: estimateTokenCount(getPromptText(options.prompt)), + modelId, + outputTokens: estimateTokenCount(lastUserMessage), + }); +} function createParallelAuthoredToolCallsResult( options: BootstrapGenerateOptions, @@ -254,12 +357,7 @@ function createParallelAuthoredToolCallsResult( }); } -/** - * Emits one built-in `agent` tool call when the current user message uses - * the explicit directive `Delegate to a subagent: `, letting - * tests exercise a real runtime-action wait. Fires only before the - * delegated call resolves; then the reply path takes over. - */ +/** Emits one direct named or built-in subagent call from an explicit directive. */ function createSubagentDelegationResult( options: BootstrapGenerateOptions, modelId: string, @@ -270,27 +368,20 @@ function createSubagentDelegationResult( return null; } - const directive = SUBAGENT_DELEGATION_DIRECTIVE.exec(lastUserMessage); - - if (directive?.[1] === undefined) { - return null; - } - - const tool = getAvailableTools(options).find((entry) => entry.name === SUBAGENT_TOOL_NAME); - - if (tool === undefined) { + const delegation = resolveMockSubagentDelegation(getAvailableTools(options), lastUserMessage); + if (delegation === null) { return null; } - const toolInput = { message: directive[1].trim() }; + const toolInput = { message: delegation.message }; return createToolCallGenerateResult({ input: toolInput, inputTokens: estimateTokenCount(getPromptText(options.prompt)), modelId, outputTokens: estimateTokenCount(toolInput.message), - toolCallId: createToolCallId(SUBAGENT_TOOL_NAME), - toolName: SUBAGENT_TOOL_NAME, + toolCallId: createToolCallId(delegation.tool.name), + toolName: delegation.tool.name, }); } @@ -304,7 +395,12 @@ function createAuthoredToolCallResult( return null; } - const tool = findRelevantTool(getAvailableTools(options), lastUserMessage); + const tools = getAvailableTools(options); + if (hasUnavailableConditionalToolBranch(tools, lastUserMessage)) { + return null; + } + + const tool = findRelevantTool(tools, lastUserMessage); if (tool === null) { return null; @@ -338,131 +434,42 @@ function createFollowUpToolCallResult(input: { readonly options: BootstrapGenerateOptions; readonly result: BootstrapToolResult; }): BootstrapGenerateResult | null { - const nextTool = findNextExplicitToolAfterResult({ + if (input.result.isError || input.result.toolName === WORKFLOW_TOOL_NAME) { + return null; + } + + const results = getCurrentTurnAuthoredToolResults(input.options.prompt); + const lastUserMessage = getLastUserPromptText(input.options.prompt); + if (lastUserMessage === null) return null; + + const nextTool = findNextMockTool({ + message: lastUserMessage, previousToolName: input.result.toolName, - prompt: input.options.prompt, + resultToolNames: results.map((result) => result.toolName), tools: getAvailableTools(input.options), }); if (nextTool === null) { return null; } - const toolInput = createFollowUpToolInput(input.result.output); - if (toolInput === null) { - return null; - } + const occurrence = results.filter((result) => result.toolName === nextTool.name).length; + const city = resolveWeatherCity(lastUserMessage); + const toolInput = createFollowUpToolInput({ + fallback: createMockAuthoredToolInput(nextTool, lastUserMessage, city, occurrence), + output: input.result.output, + tool: nextTool, + }); return createToolCallGenerateResult({ input: toolInput, inputTokens: estimateTokenCount(getPromptText(input.options.prompt)), modelId: input.modelId, outputTokens: estimateTokenCount(Object.values(toolInput).join(" ")), - toolCallId: createToolCallId(nextTool.name), + toolCallId: createToolCallId(nextTool.name, occurrence), toolName: nextTool.name, }); } -function createAssistantMessage(prompt: BootstrapPrompt): string { - const lastUserMessage = getLastUserPromptText(prompt) ?? "Hello from eve"; - const systemLabels = getSystemPromptLabels(prompt); - const systemProbe = resolveSystemProbe(prompt); - const fixtureToken = resolveMockFixtureToken(prompt); - - if (fixtureToken !== null) { - return fixtureToken; - } - - if (systemLabels.length > 0) { - if (systemProbe === null) { - return `Bootstrap reply [${systemLabels.join(", ")}]: ${lastUserMessage}`; - } - - return `Bootstrap reply [${systemLabels.join(", ")}; probe=${systemProbe}]: ${lastUserMessage}`; - } - - if (systemProbe !== null) { - return `Bootstrap reply [probe=${systemProbe}]: ${lastUserMessage}`; - } - - return `Bootstrap reply: ${lastUserMessage}`; -} - -function formatToolResultReply(result: BootstrapToolResult, prompt: BootstrapPrompt): string { - if (result.isError) { - return `Local weather tool failed: ${formatToolOutput(result.output)}`; - } - - if (isWeatherPayload(result.output)) { - return `Used local weather tool for ${result.output.city}: ${result.output.condition}, ${result.output.temperatureF}F. ${result.output.summary}`; - } - - const lastUserMessage = getLastUserPromptText(prompt) ?? "Hello from eve"; - - return `Used ${result.toolName} for "${lastUserMessage}": ${formatToolOutput(result.output)}`; -} - -function createToolCallGenerateResult(input: { - readonly input: unknown; - readonly inputTokens: number; - readonly modelId: string; - readonly outputTokens: number; - readonly toolCallId: string; - readonly toolName: string; -}): BootstrapGenerateResult { - return createToolCallsGenerateResult({ - calls: [ - { - input: input.input, - toolCallId: input.toolCallId, - toolName: input.toolName, - }, - ], - inputTokens: input.inputTokens, - modelId: input.modelId, - outputTokens: input.outputTokens, - }); -} - -function createToolCallsGenerateResult(input: { - readonly calls: readonly { - readonly input: unknown; - readonly toolCallId: string; - readonly toolName: string; - }[]; - readonly inputTokens: number; - readonly modelId: string; - readonly outputTokens: number; -}): BootstrapGenerateResult { - return { - content: input.calls.map((call) => ({ - input: JSON.stringify(call.input), - toolCallId: call.toolCallId, - toolName: call.toolName, - type: "tool-call", - })), - finishReason: { raw: undefined, unified: "tool-calls" }, - response: { - id: "bootstrap-response", - modelId: input.modelId, - timestamp: new Date("2026-03-16T00:00:00.000Z"), - }, - usage: { - inputTokens: { - cacheRead: 0, - cacheWrite: 0, - noCache: input.inputTokens, - total: input.inputTokens, - }, - outputTokens: { - reasoning: 0, - text: input.outputTokens, - total: input.outputTokens, - }, - }, - warnings: [], - } as unknown as BootstrapGenerateResult; -} - function getAvailableTools(options: BootstrapGenerateOptions): AvailableBootstrapTool[] { return (options.tools ?? []).flatMap((tool) => { if (tool.type !== "function") { @@ -481,16 +488,21 @@ function getAvailableTools(options: BootstrapGenerateOptions): AvailableBootstra } function getLastAuthoredToolResult(prompt: BootstrapPrompt): BootstrapToolResult | null { - for (const message of [...prompt].reverse()) { - if (message.role === "user") { - return null; - } + return getCurrentTurnAuthoredToolResults(prompt).at(-1) ?? null; +} + +function getCurrentTurnAuthoredToolResults( + prompt: BootstrapPrompt, +): readonly BootstrapToolResult[] { + const lastUserIndex = prompt.findLastIndex((message) => message.role === "user"); + const results: BootstrapToolResult[] = []; + for (const message of prompt.slice(lastUserIndex + 1)) { if (message.role !== "tool" && message.role !== "assistant") { continue; } - for (const part of [...message.content].reverse()) { + for (const part of message.content) { if (typeof part === "string" || part.type !== "tool-result") { continue; } @@ -499,7 +511,7 @@ function getLastAuthoredToolResult(prompt: BootstrapPrompt): BootstrapToolResult continue; } - return { + results.push({ isError: part.output.type === "error-json" || part.output.type === "error-text" || @@ -513,105 +525,40 @@ function getLastAuthoredToolResult(prompt: BootstrapPrompt): BootstrapToolResult : part.output.value, toolCallId: part.toolCallId, toolName: part.toolName, - }; + }); } } - return null; -} - -function findNextExplicitToolAfterResult(input: { - readonly previousToolName: string; - readonly prompt: BootstrapPrompt; - readonly tools: readonly AvailableBootstrapTool[]; -}): AvailableBootstrapTool | null { - const lastUserMessage = getLastUserPromptText(input.prompt); - if (lastUserMessage === null) { - return null; - } - - const normalizedMessage = normalizeText(lastUserMessage); - const previousIndex = normalizedMessage.indexOf(normalizeText(input.previousToolName)); - if (previousIndex < 0) { - return null; - } - - const candidates = input.tools - .filter((tool) => tool.name !== input.previousToolName) - .flatMap((tool) => { - const index = normalizedMessage.indexOf(normalizeText(tool.name), previousIndex + 1); - return index < 0 ? [] : [{ index, tool }]; - }) - .sort((left, right) => left.index - right.index); - - return candidates[0]?.tool ?? null; + return results; } /** - * Extracts the next tool call's arguments from the previous tool's - * output, enabling the deterministic `lookup-step-a` -> `lookup-step-b` - * chain. Only the `stepKey` handoff used by that fixture pair is - * supported. + * Merges schema-matching fields from the previous result into the next + * prompt-derived input. This preserves explicit prompt arguments while + * allowing values such as a returned `stepKey` to flow through a tool chain. */ -function createFollowUpToolInput(output: unknown): Record | null { - if (isRecord(output) && typeof output.stepKey === "string") { - return { stepKey: output.stepKey }; +function createFollowUpToolInput(input: { + readonly fallback: Record; + readonly output: unknown; + readonly tool: AvailableBootstrapTool; +}): Record { + if (!isRecord(input.output)) { + return input.fallback; } - return null; -} - -function getSystemPromptLabels(prompt: BootstrapPrompt): string[] { - const systemMessages = prompt.filter((message) => message.role === "system"); - - if (systemMessages.length === 0) { - return []; + const nextInput = { ...input.fallback }; + const propertyNames = new Set(getToolInputPropertyNames(input.tool.inputSchema)); + if (propertyNames.size === 0 && typeof input.output.stepKey === "string") { + return { stepKey: input.output.stepKey }; } - const labels = systemMessages.flatMap((message) => { - const text = getPromptContentText(message.content); - - if (text.startsWith("Available skills\n")) { - return []; + for (const propertyName of propertyNames) { + if (input.output[propertyName] !== undefined) { + nextInput[propertyName] = input.output[propertyName]; } + } - const lines = text - .split("\n") - .map((line) => line.trim()) - .filter((line) => line.length > 0); - const extractedLabels: string[] = []; - - for (const line of lines) { - if (line === BOOTSTRAP_RUNTIME_SYSTEM_PROMPT || line === "Available skills") { - continue; - } - - const systemMatch = /^System \((.+)\)$/.exec(line); - - if (systemMatch?.[1]) { - extractedLabels.push(systemMatch[1]); - continue; - } - - const skillMatch = /^Skill \((.+)\)$/.exec(line); - - if (skillMatch?.[1]) { - extractedLabels.push(skillMatch[1]); - } - } - - if (extractedLabels.length > 0) { - return extractedLabels; - } - - const fallbackFirstLine = lines.find( - (line) => line !== BOOTSTRAP_RUNTIME_SYSTEM_PROMPT && line !== "Available skills", - ); - - return fallbackFirstLine === undefined ? [] : [fallbackFirstLine]; - }); - - return [...new Set(labels)]; + return nextInput; } function findRelevantTool( @@ -622,16 +569,43 @@ function findRelevantTool( // `load_skill` is reachable only through skill-relevance selection // (createSkillLoadResult); matching it by name here would re-call it on // every step, because its results are invisible to the tool-result check. - const explicitTool = tools.find( - (tool) => - tool.name !== "agent" && - tool.name !== LOAD_SKILL_TOOL_NAME && - normalizedMessage.includes(normalizeText(tool.name)), - ); + const explicitTool = tools + .flatMap((tool) => { + if ( + tool.name === AGENT_TOOL_NAME || + tool.name === LOAD_SKILL_TOOL_NAME || + tool.name === WORKFLOW_TOOL_NAME + ) { + return []; + } + + const index = findToolMentionIndex(message, tool.name); + return index < 0 ? [] : [{ index, tool }]; + }) + .sort((left, right) => left.index - right.index)[0]?.tool; if (explicitTool !== undefined) { return explicitTool; } + const messageTokens = new Set(normalizedMessage.split(" ")); + const tokenMatchedTool = tools.find((tool) => { + if ( + tool.name === AGENT_TOOL_NAME || + tool.name === LOAD_SKILL_TOOL_NAME || + tool.name === WORKFLOW_TOOL_NAME + ) { + return false; + } + + const nameTokens = normalizeText(tool.name) + .split(" ") + .filter((token) => token.length > 2 && token !== "for"); + return nameTokens.length >= 2 && nameTokens.every((token) => messageTokens.has(token)); + }); + if (tokenMatchedTool !== undefined) { + return tokenMatchedTool; + } + if (!/\b(forecast|temperature|weather|wind|rain|snow)\b/u.test(normalizedMessage)) { return null; } @@ -645,6 +619,41 @@ function findRelevantTool( ); } +function shouldCompleteSilently(result: BootstrapToolResult, prompt: BootstrapPrompt): boolean { + if (result.isError || !hasEmptyCollection(result.output)) { + return false; + } + + const message = getLastUserPromptText(prompt); + return ( + message !== null && + /\bdo\s+not\s+send\s+a\s+message\s+when\b[\s\S]*\b(?:empty|no\s+\w+)\b/iu.test(message) + ); +} + +function hasEmptyCollection(value: unknown): boolean { + if (Array.isArray(value)) { + return value.length === 0; + } + + return ( + isRecord(value) && + Object.values(value).some((entry) => Array.isArray(entry) && entry.length === 0) + ); +} + +function findToolMentionIndex(message: string, toolName: string, fromIndex = 0): number { + return message.toLowerCase().indexOf(toolName.toLowerCase(), fromIndex); +} + +function getToolInputPropertyNames(schema: unknown): readonly string[] { + if (!isRecord(schema) || !isRecord(schema.properties)) { + return []; + } + + return Object.keys(schema.properties); +} + function normalizeText(value: string): string { return value .toLowerCase() @@ -652,31 +661,13 @@ function normalizeText(value: string): string { .trim(); } -function createToolCallId(toolName: string): string { +function createToolCallId(toolName: string, occurrence = 0): string { const normalized = toolName .toLowerCase() .replace(/[^a-z0-9]+/gu, "_") .replace(/^_+|_+$/gu, ""); - return `call_${normalized || "tool"}`; -} - -function resolveSystemProbe(prompt: BootstrapPrompt): string | null { - const systemText = prompt - .filter((message) => message.role === "system") - .map((message) => getPromptContentText(message.content)) - .join("\n"); - const probeMatch = /hmr-probe:\s*([^\n]+)/iu.exec(systemText); - - return probeMatch?.[1]?.trim() || null; -} - -function isWeatherPayload(value: unknown): value is { - readonly city: string; - readonly condition: string; - readonly summary: string; - readonly temperatureF: number; -} { - return bootstrapWeatherPayloadSchema.safeParse(value).success; + const base = `call_${normalized || "tool"}`; + return occurrence === 0 ? base : `${base}_${String(occurrence + 1)}`; } function isRecord(value: unknown): value is Record { diff --git a/packages/eve/src/runtime/agent/mock-model-fixtures.ts b/packages/eve/src/runtime/agent/mock-model-fixtures.ts index 13f4829e2..9db3cd34d 100644 --- a/packages/eve/src/runtime/agent/mock-model-fixtures.ts +++ b/packages/eve/src/runtime/agent/mock-model-fixtures.ts @@ -14,6 +14,7 @@ export function createMockAuthoredToolInput( tool: AvailableBootstrapTool, message: string, city: string, + occurrence = 0, ): Record { const inputPropertyNames = getToolInputPropertyNames(tool.inputSchema); if (tool.name === "ask_question" || hasProperties(inputPropertyNames, ["prompt", "options"])) { @@ -21,7 +22,7 @@ export function createMockAuthoredToolInput( } if (inputPropertyNames.includes("command")) { - return { command: resolveShellCommand(message) }; + return { command: resolveShellCommand(message, tool.name, occurrence) }; } if ( @@ -31,9 +32,12 @@ export function createMockAuthoredToolInput( return { topic: resolveLookupTopic(message) }; } - const anchored = extractAnchoredInputs(inputPropertyNames, message); - if (Object.keys(anchored).length > 0) { - return anchored; + const derived = { + ...extractSchemaDerivedInputs(tool.inputSchema, message), + ...extractAnchoredInputs(inputPropertyNames, message, occurrence), + }; + if (Object.keys(derived).length > 0) { + return completeRequiredInputs(tool.inputSchema, derived); } if (inputPropertyNames.length === 1 && inputPropertyNames[0] === "message") { @@ -57,15 +61,17 @@ export function createMockAuthoredToolInput( function extractAnchoredInputs( propertyNames: readonly string[], message: string, + occurrence: number, ): Record { const inputs: Record = {}; for (const propertyName of propertyNames) { const pattern = new RegExp( `\\b${escapeRegExp(propertyName)}\\b\\s*(?:to|=|:)?\\s*(?:\`([^\`]+)\`|"([^"]+)"|'([^']+)')`, - "iu", + "giu", ); - const match = pattern.exec(message); + const matches = [...message.matchAll(pattern)]; + const match = matches[occurrence] ?? matches.at(-1); const value = match?.[1] ?? match?.[2] ?? match?.[3]; if (value !== undefined) { @@ -80,6 +86,129 @@ function escapeRegExp(value: string): string { return value.replace(/[$()*+.?[\\\]^{|}]/gu, String.raw`\$&`); } +function extractSchemaDerivedInputs(schema: unknown, message: string): Record { + const properties = getToolInputProperties(schema); + const inputs: Record = {}; + + for (const [propertyName, propertySchema] of Object.entries(properties)) { + if (isNumericArraySchema(propertySchema)) { + const values = resolveNumericArray(message, propertyName); + if (values.length > 0) { + inputs[propertyName] = values; + } + } + } + + if ("connection" in properties) { + const connection = /\bin\s+(?:the\s+)?[`"']?([A-Za-z0-9_-]+)[`"']?\s+connection\b/iu.exec( + message, + )?.[1]; + if (connection !== undefined) { + inputs.connection = connection; + } + } + + if ("keywords" in properties) { + const keywords = /\bfind\s+(?:the\s+)?(.+?)\s+(?:operation|tool)\b/iu.exec(message)?.[1]; + if (keywords !== undefined) { + inputs.keywords = keywords.trim(); + } + } + + if ("filePath" in properties) { + const filePath = + /\b(?:create|write)\s+(?:the\s+)?file\s+(`([^`]+)`|"([^"]+)"|'([^']+)'|(\S+))/iu.exec( + message, + ); + const value = + filePath?.[2] ?? filePath?.[3] ?? filePath?.[4] ?? filePath?.[5]?.replace(/[.,;:]$/u, ""); + if (value !== undefined) { + inputs.filePath = value; + } + } + + if ("content" in properties) { + const content = /\bcontent:\s*([^\r\n]+)/iu.exec(message)?.[1]?.trim(); + if (content !== undefined) { + inputs.content = content; + } + } + + if ("pattern" in properties) { + const pattern = /\bsearch\s+for\s+(`([^`]+)`|"([^"]+)"|'([^']+)'|([^\s.,;]+))/iu.exec(message); + const value = pattern?.[2] ?? pattern?.[3] ?? pattern?.[4] ?? pattern?.[5]; + if (value !== undefined) { + inputs.pattern = value; + } + } + + if ("path" in properties) { + const path = /\bunder\s+(`([^`]+)`|"([^"]+)"|'([^']+)'|([/][^\s.,;]+))/iu.exec(message); + const value = path?.[2] ?? path?.[3] ?? path?.[4] ?? path?.[5]; + if (value !== undefined) { + inputs.path = value; + } + } + + return inputs; +} + +function completeRequiredInputs( + schema: unknown, + inputs: Record, +): Record { + if (!isRecord(schema) || !Array.isArray(schema.required)) { + return inputs; + } + + const sample = createJsonSchemaSample(schema); + if (!isRecord(sample)) { + return inputs; + } + + const completed = { ...inputs }; + for (const propertyName of schema.required) { + if ( + typeof propertyName === "string" && + completed[propertyName] === undefined && + sample[propertyName] !== undefined + ) { + completed[propertyName] = sample[propertyName]; + } + } + + return completed; +} + +function getToolInputProperties(schema: unknown): Record { + if (!isRecord(schema) || !isRecord(schema.properties)) { + return {}; + } + + return schema.properties; +} + +function isNumericArraySchema(schema: unknown): boolean { + return ( + isRecord(schema) && + schema.type === "array" && + isRecord(schema.items) && + (schema.items.type === "integer" || schema.items.type === "number") + ); +} + +function resolveNumericArray(message: string, propertyName: string): number[] { + const marker = new RegExp( + `\\b(?:${escapeRegExp(propertyName)}|integers?|numbers?)\\b\\s*:?\\s*([^\\r\\n.]+)`, + "iu", + ).exec(message)?.[1]; + if (marker === undefined) { + return []; + } + + return [...marker.matchAll(/-?\d+(?:\.\d+)?/gu)].map((match) => Number(match[0])); +} + export function resolveMockFixtureToken(prompt: BootstrapPrompt): string | null { const systemText = prompt .filter((message) => message.role === "system") @@ -105,6 +234,39 @@ export function resolveMockFixtureToken(prompt: BootstrapPrompt): string | null return null; } +/** Recalls a simple fact stated as `my