diff --git a/.changeset/eval-exclude-tag.md b/.changeset/eval-exclude-tag.md new file mode 100644 index 000000000..17d90dcba --- /dev/null +++ b/.changeset/eval-exclude-tag.md @@ -0,0 +1,5 @@ +--- +"eve": patch +--- + +`eve eval` gains a repeatable `--exclude-tag ` flag that skips evals carrying a tag. Exclusion applies after `--tag` inclusion, and a run where exclusion removes every matching eval now exits successfully with nothing executed. `--list` reports the post-filter selection — `--list --json` prints `[]` when exclusion removes everything — so suite runners can probe whether anything would run. diff --git a/.github/scripts/discover-e2e-fixtures.mjs b/.github/scripts/discover-e2e-fixtures.mjs new file mode 100644 index 000000000..ebfb20197 --- /dev/null +++ b/.github/scripts/discover-e2e-fixtures.mjs @@ -0,0 +1,100 @@ +// Discover e2e fixture directories for the CI matrices. +// +// Models and worlds live in e2e/matrix.json — edit that file to add either. +// +// A fixture qualifies when it has an `evals/` directory under one of the +// fixture roots. Emits one GitHub Actions output per matrix: +// +// model_matrix `{ name, dir, model_name, model_id }` entries for +// the model suite (e2e-local). Fixtures marked +// `"e2e": { "modelMatrix": "full" }` in package.json +// run on every registry model; all other fixtures run +// once on the default (first) model. +// world_matrix_ `{ name, dir[, world_package] }` entries for that +// world's suite workflow, which runs every fixture +// once with mock models (EVE_E2E_MODEL=mock). +import { appendFileSync, existsSync, readdirSync, readFileSync, statSync } from "node:fs"; +import { join } from "node:path"; + +const roots = ["e2e/fixtures", "apps/fixtures"]; + +const registry = JSON.parse(readFileSync("e2e/matrix.json", "utf8")); +const models = validateNamedEntries(registry.models, "models", ["id"]); +const worlds = validateNamedEntries(registry.worlds, "worlds", []); + +const fixtures = []; +for (const root of roots) { + if (!existsSync(root)) continue; + for (const entry of readdirSync(root).sort()) { + const dir = join(root, entry); + if (!statSync(dir).isDirectory() || !existsSync(join(dir, "evals"))) continue; + + let modelMatrix = "default"; + const packageJsonPath = join(dir, "package.json"); + if (existsSync(packageJsonPath)) { + const pkg = JSON.parse(readFileSync(packageJsonPath, "utf8")); + modelMatrix = pkg.e2e?.modelMatrix ?? "default"; + } + if (modelMatrix !== "default" && modelMatrix !== "full") { + throw new Error(`${packageJsonPath}: e2e.modelMatrix must be "default" or "full".`); + } + + fixtures.push({ name: entry, dir, modelMatrix }); + } +} + +if (fixtures.length === 0) { + console.error("No e2e fixtures with an evals/ directory were found."); + process.exit(1); +} + +const modelMatrix = fixtures.flatMap(({ name, dir, modelMatrix }) => + (modelMatrix === "full" ? models : models.slice(0, 1)).map((model) => ({ + name, + dir, + model_name: model.name, + model_id: model.id, + })), +); + +const outputs = [`model_matrix=${JSON.stringify(modelMatrix)}`]; +for (const world of worlds) { + const legs = fixtures.map(({ name, dir }) => + world.package === undefined ? { name, dir } : { name, dir, world_package: world.package }, + ); + outputs.push(`world_matrix_${world.name}=${JSON.stringify(legs)}`); +} + +console.error( + `Discovered ${fixtures.length} fixtures (${modelMatrix.length} model-suite jobs, ${worlds.length} worlds).`, +); +const lines = `${outputs.join("\n")}\n`; +if (process.env.GITHUB_OUTPUT) { + appendFileSync(process.env.GITHUB_OUTPUT, lines); +} else { + process.stdout.write(lines); +} + +function validateNamedEntries(entries, key, requiredFields) { + if (!Array.isArray(entries) || entries.length === 0) { + throw new Error(`e2e/matrix.json: "${key}" must be a non-empty array.`); + } + const names = new Set(); + for (const entry of entries) { + for (const field of ["name", ...requiredFields]) { + if (typeof entry[field] !== "string" || entry[field].length === 0) { + throw new Error(`e2e/matrix.json: every "${key}" entry needs a non-empty "${field}".`); + } + } + if (!/^[a-z0-9-]+$/.test(entry.name)) { + throw new Error( + `e2e/matrix.json: "${key}" name "${entry.name}" must be lowercase alphanumerics and dashes (it becomes a check identifier).`, + ); + } + if (names.has(entry.name)) { + throw new Error(`e2e/matrix.json: duplicate "${key}" name "${entry.name}".`); + } + names.add(entry.name); + } + return entries; +} diff --git a/.github/scripts/discover-e2e-fixtures.sh b/.github/scripts/discover-e2e-fixtures.sh deleted file mode 100755 index 4b3918f8b..000000000 --- a/.github/scripts/discover-e2e-fixtures.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash -# Discover e2e fixture directories for the CI matrix. -# -# A fixture qualifies when it has an `evals/` directory under one of the -# fixture roots. Emits a JSON array of `{ name, dir }` objects (sorted by dir) -# to the `matrix` GitHub Actions output for the workflow's `fixture` axis. -set -euo pipefail - -roots=("e2e/fixtures" "apps/fixtures") - -entries=() -while IFS= read -r evals_dir; do - dir="${evals_dir%/evals}" - name="$(basename "$dir")" - entries+=("{\"name\":\"${name}\",\"dir\":\"${dir}\"}") -done < <( - for root in "${roots[@]}"; do - [ -d "$root" ] || continue - find "$root" -mindepth 2 -maxdepth 2 -type d -name evals - done | sort -) - -if [ "${#entries[@]}" -eq 0 ]; then - echo "No e2e fixtures with an evals/ directory were found." >&2 - exit 1 -fi - -matrix="[$( - IFS=, - echo "${entries[*]}" -)]" - -echo "Discovered fixtures: ${matrix}" -echo "matrix=${matrix}" >>"${GITHUB_OUTPUT:-/dev/stdout}" diff --git a/.github/workflows/e2e-local.yml b/.github/workflows/e2e-local.yml index c76c03635..d02a746eb 100644 --- a/.github/workflows/e2e-local.yml +++ b/.github/workflows/e2e-local.yml @@ -36,7 +36,7 @@ jobs: fi # HEAD is the PR merge commit; HEAD^1 is the base branch tip, so # this diff is exactly the PR's changed files. - if git diff --name-only HEAD^1 HEAD | grep -qE '^(packages/eve/|apps/fixtures/|e2e/|pnpm-workspace\.yaml$|pnpm-lock\.yaml$|\.github/workflows/e2e-local\.yml$|\.github/scripts/discover-e2e-fixtures\.sh$)'; then + if git diff --name-only HEAD^1 HEAD | grep -qE '^(packages/eve/|apps/fixtures/|e2e/|pnpm-workspace\.yaml$|pnpm-lock\.yaml$|\.github/workflows/e2e-local\.yml$|\.github/scripts/discover-e2e-fixtures\.mjs$)'; then echo "relevant=true" >> "$GITHUB_OUTPUT" else echo "relevant=false" >> "$GITHUB_OUTPUT" @@ -46,7 +46,7 @@ jobs: name: discover-fixtures runs-on: ubuntu-latest outputs: - matrix: ${{ steps.discover.outputs.matrix }} + model_matrix: ${{ steps.discover.outputs.model_matrix }} steps: - name: Checkout code uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 @@ -55,10 +55,15 @@ jobs: - name: Discover eval fixtures id: discover - run: .github/scripts/discover-e2e-fixtures.sh + run: node .github/scripts/discover-e2e-fixtures.mjs + # The model suite: every fixture runs here against the local world with + # real matrix models. Model-sensitive fixtures (`"e2e": { "modelMatrix": + # "full" }` in package.json) run on every matrix model; the rest run once + # on the default model. The world suites (e2e-vercel, e2e-postgres) cover + # the other workflow worlds with deterministic mock models. e2e: - name: e2e-local (${{ matrix.fixture.name }}, ${{ matrix.model.name }}) + name: e2e-local (${{ matrix.name }}, ${{ matrix.model_name }}) runs-on: ubuntu-latest needs: [changes, discover-fixtures] # No job-level `if`: the stable aggregate check below requires a successful @@ -67,16 +72,11 @@ jobs: strategy: fail-fast: false matrix: - fixture: ${{ fromJSON(needs.discover-fixtures.outputs.matrix) }} - model: - - name: openai-sol - id: openai/gpt-5.6-sol - - name: anthropic-opus - id: anthropic/claude-opus-5 + include: ${{ fromJSON(needs.discover-fixtures.outputs.model_matrix) }} env: AI_GATEWAY_API_KEY: ${{ secrets.AI_GATEWAY_API_KEY }} - EVE_E2E_MODEL: ${{ matrix.model.id }} + EVE_E2E_MODEL: ${{ matrix.model_id }} EVE_EVAL_JUNIT_DIR: ${{ github.workspace }}/.junit steps: @@ -131,15 +131,15 @@ jobs: if: needs.changes.outputs.relevant == 'true' run: | mkdir -p "$EVE_EVAL_JUNIT_DIR" - cd "${{ matrix.fixture.dir }}" + cd "${{ matrix.dir }}" pnpm exec eve eval --strict --verbose \ - --junit "$EVE_EVAL_JUNIT_DIR/${{ matrix.fixture.name }}-${{ matrix.model.name }}.xml" + --junit "$EVE_EVAL_JUNIT_DIR/${{ matrix.name }}-${{ matrix.model_name }}.xml" - name: Upload eval artifacts if: (failure()) && needs.changes.outputs.relevant == 'true' uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v6 with: - name: eval-artifacts-${{ matrix.fixture.name }}-${{ matrix.model.name }} + name: eval-artifacts-${{ matrix.name }}-${{ matrix.model_name }} path: | .junit e2e/fixtures/*/.eve/evals diff --git a/.github/workflows/e2e-postgres.yml b/.github/workflows/e2e-postgres.yml new file mode 100644 index 000000000..b74547373 --- /dev/null +++ b/.github/workflows/e2e-postgres.yml @@ -0,0 +1,261 @@ +name: E2E Tests (Postgres) + +on: + workflow_dispatch: + pull_request: + +concurrency: + group: e2e-postgres-${{ github.ref }} + cancel-in-progress: true + +jobs: + # The stable aggregate check must report on every PR. This job decides + # relevance so the matrix can skip at the job level without starting a + # PostgreSQL service container for every fixture on unrelated changes; the + # aggregate below treats an irrelevant diff as satisfied. + changes: + name: changes + runs-on: ubuntu-latest + outputs: + relevant: ${{ steps.filter.outputs.relevant }} + steps: + - name: Checkout code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + fetch-depth: 2 + + - name: Check changed paths + id: filter + run: | + if [ "${{ github.event_name }}" != "pull_request" ]; then + echo "relevant=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + # HEAD is the PR merge commit; HEAD^1 is the base branch tip, so + # this diff is exactly the PR's changed files. + if git diff --name-only HEAD^1 HEAD | grep -qE '^(packages/eve/|apps/fixtures/|e2e/|pnpm-workspace\.yaml$|pnpm-lock\.yaml$|\.github/workflows/e2e-postgres\.yml$|\.github/scripts/discover-e2e-fixtures\.mjs$)'; then + echo "relevant=true" >> "$GITHUB_OUTPUT" + else + echo "relevant=false" >> "$GITHUB_OUTPUT" + fi + + discover-fixtures: + name: discover-fixtures + runs-on: ubuntu-latest + outputs: + world_matrix: ${{ steps.discover.outputs.world_matrix_postgres }} + steps: + - name: Checkout code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + + - name: Discover eval fixtures + id: discover + run: node .github/scripts/discover-e2e-fixtures.mjs + + # The Postgres world suite: every fixture builds and runs once against a + # production server backed by @workflow/world-postgres, with deterministic + # mock models (EVE_E2E_MODEL=mock). Evals that need a real model carry the + # `real-model` tag and run only in the model suite (e2e-local). + e2e: + name: e2e-postgres (${{ matrix.name }}) + runs-on: ubuntu-latest + needs: [changes, discover-fixtures] + if: needs.changes.outputs.relevant == 'true' + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.discover-fixtures.outputs.world_matrix) }} + + services: + postgres: + image: postgres:18-alpine + env: + POSTGRES_DB: world + POSTGRES_PASSWORD: world + POSTGRES_USER: world + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready --username=world --dbname=world" + --health-interval 5s + --health-timeout 5s + --health-retries 12 + + env: + # Mock models never reach the gateway; the placeholder keeps any + # credential-presence checks satisfied without granting live access. + AI_GATEWAY_API_KEY: invalid-e2e-world-suite-key + EVE_E2E_MODEL: mock + # The world package comes from this world's e2e/matrix.json entry. + EVE_E2E_WORKFLOW_WORLD: ${{ matrix.world_package }} + EVE_EVAL_JUNIT_DIR: ${{ github.workspace }}/.junit + WORKFLOW_POSTGRES_MAX_POOL_SIZE: "52" + WORKFLOW_POSTGRES_URL: postgres://world:world@127.0.0.1:5432/world + WORKFLOW_POSTGRES_WORKER_CONCURRENCY: "50" + + steps: + - name: Checkout code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + + - name: Setup pnpm + uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6 + + - name: Setup Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 + with: + node-version-file: .nvmrc + cache: "pnpm" + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Build eve package + # Full build, not build:js: the deployed/runtime bundles must carry the + # stamped package version (scripts/stamp-version-tokens.mjs) and docs. + run: pnpm --filter eve run build + + - name: Build extension fixtures + # Extension fixtures ship no committed `dist/`; produce it here so + # consuming fixtures resolve the built artifacts. `--if-present` skips + # extension fixtures without a build script. The update re-packs the + # `file:`-protocol store copies so they pick up the fresh dist — + # a plain reinstall short-circuits and leaves them stale. + run: | + pnpm --filter "./e2e/fixtures/*-extension" run --if-present build + pnpm --filter dist-extensions update gizmo-extension gadget-extension + + - name: Stage workspace extension fixtures as dist-only + # Removing author source proves the production build consumes the same + # agent-shaped dist trees registry packages ship. + run: | + mv e2e/fixtures/toolkit-extension/extension "$RUNNER_TEMP/toolkit-extension-source" + mv e2e/fixtures/js-only-extension/extension "$RUNNER_TEMP/js-only-extension-source" + + - name: Bootstrap Workflow Postgres schema + working-directory: ${{ matrix.dir }} + run: pnpm exec bootstrap + + - name: Build fixture, start production server, and run evals + env: + POSTGRES_CONTAINER_ID: ${{ job.services.postgres.id }} + run: | + set -euo pipefail + mkdir -p "$EVE_EVAL_JUNIT_DIR" + cd "${{ matrix.dir }}" + + pnpm exec eve build + + server_log="$EVE_EVAL_JUNIT_DIR/${{ matrix.name }}-server.log" + pnpm exec eve start --host 127.0.0.1 --port 3000 >"$server_log" 2>&1 & + server_pid=$! + + cleanup() { + set +e + kill "$server_pid" 2>/dev/null + wait "$server_pid" 2>/dev/null + } + trap cleanup EXIT + + ready=false + for _ in {1..120}; do + if curl --fail --silent http://127.0.0.1:3000/eve/v1/health >/dev/null; then + ready=true + break + fi + if ! kill -0 "$server_pid" 2>/dev/null; then + cat "$server_log" + exit 1 + fi + sleep 0.5 + done + if [ "$ready" != "true" ]; then + cat "$server_log" + echo "Timed out waiting for the Postgres-backed eve server." + exit 1 + fi + + if ! pnpm exec eve eval --strict --verbose --exclude-tag real-model \ + --url http://127.0.0.1:3000 \ + --junit "$EVE_EVAL_JUNIT_DIR/${{ matrix.name }}.xml"; then + cat "$server_log" + exit 1 + fi + + # Durability proof: when any eval actually executed, its traffic + # must have produced Postgres-backed workflow runs, not silently + # fallen back to the local world. A fixture can execute nothing + # here — all evals tagged `real-model` (no junit written), or + # runtime self-skips (junit reports tests == skipped) — and its + # leg still proves the fixture builds, boots, and reports healthy + # on this world. + junit_file="$EVE_EVAL_JUNIT_DIR/${{ matrix.name }}.xml" + executed_evals=0 + if [ -f "$junit_file" ]; then + total="$(grep -o 'tests="[0-9]*"' "$junit_file" | head -1 | tr -dc '0-9')" + skipped="$(grep -o 'skipped="[0-9]*"' "$junit_file" | head -1 | tr -dc '0-9')" + executed_evals=$(( ${total:-0} - ${skipped:-0} )) + fi + if [ "$executed_evals" -eq 0 ]; then + echo "No evals executed on this world; build/boot/health verified, persistence assertion skipped." + exit 0 + fi + + run_count="$( + docker exec "$POSTGRES_CONTAINER_ID" \ + psql --username=world --dbname=world --tuples-only --no-align \ + --command="SELECT count(*) FROM workflow.workflow_runs;" + )" + if ! [[ "$run_count" =~ ^[0-9]+$ ]] || [ "$run_count" -lt 1 ]; then + cat "$server_log" + echo "Expected Postgres-backed workflow runs, found: $run_count" + exit 1 + fi + echo "Verified $run_count workflow run(s) in PostgreSQL." + + - name: Upload eval artifacts + if: failure() + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v6 + with: + name: eval-artifacts-postgres-${{ matrix.name }} + path: | + .junit + e2e/fixtures/*/.eve/evals + apps/fixtures/*/.eve/evals + include-hidden-files: true + if-no-files-found: ignore + retention-days: 7 + + e2e-required: + name: e2e-postgres + runs-on: ubuntu-latest + needs: [changes, discover-fixtures, e2e] + if: always() + steps: + - name: Require successful e2e matrix when relevant + env: + CHANGES_RESULT: ${{ needs.changes.result }} + DISCOVERY_RESULT: ${{ needs.discover-fixtures.result }} + E2E_RESULT: ${{ needs.e2e.result }} + RELEVANT: ${{ needs.changes.outputs.relevant }} + run: | + if [ "$CHANGES_RESULT" != "success" ]; then + echo "Change detection result: $CHANGES_RESULT" + exit 1 + fi + if [ "$DISCOVERY_RESULT" != "success" ]; then + echo "Fixture discovery result: $DISCOVERY_RESULT" + exit 1 + fi + if [ "$RELEVANT" != "true" ]; then + echo "No Postgres e2e-relevant changes." + exit 0 + fi + if [ "$E2E_RESULT" != "success" ]; then + echo "World suite result: $E2E_RESULT" + exit 1 + fi diff --git a/.github/workflows/e2e-vercel.yml b/.github/workflows/e2e-vercel.yml index 21d5518ee..ff6a624d0 100644 --- a/.github/workflows/e2e-vercel.yml +++ b/.github/workflows/e2e-vercel.yml @@ -36,7 +36,7 @@ jobs: fi # HEAD is the PR merge commit; HEAD^1 is the base branch tip, so # this diff is exactly the PR's changed files. - if git diff --name-only HEAD^1 HEAD | grep -qE '^(packages/eve/|apps/fixtures/|e2e/|pnpm-workspace\.yaml$|pnpm-lock\.yaml$|scripts/build-profile-report\.mjs$|\.github/workflows/e2e-vercel\.yml$|\.github/scripts/discover-e2e-fixtures\.sh$)'; then + if git diff --name-only HEAD^1 HEAD | grep -qE '^(packages/eve/|apps/fixtures/|e2e/|pnpm-workspace\.yaml$|pnpm-lock\.yaml$|scripts/build-profile-report\.mjs$|\.github/workflows/e2e-vercel\.yml$|\.github/scripts/discover-e2e-fixtures\.mjs$)'; then echo "relevant=true" >> "$GITHUB_OUTPUT" else echo "relevant=false" >> "$GITHUB_OUTPUT" @@ -46,7 +46,7 @@ jobs: name: discover-fixtures runs-on: ubuntu-latest outputs: - matrix: ${{ steps.discover.outputs.matrix }} + world_matrix: ${{ steps.discover.outputs.world_matrix_vercel }} steps: - name: Checkout code uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 @@ -55,10 +55,15 @@ jobs: - name: Discover eval fixtures id: discover - run: .github/scripts/discover-e2e-fixtures.sh + run: node .github/scripts/discover-e2e-fixtures.mjs + # The Vercel world suite: every fixture deploys and runs once with + # deterministic mock models (EVE_E2E_MODEL=mock), so this matrix proves the + # deployed world's infrastructure — build, deploy, boot, streaming, + # durability — without live-model flake. Evals that need a real model carry + # the `real-model` tag and run only in the model suite (e2e-local). e2e: - name: e2e-vercel (${{ matrix.fixture.name }}, ${{ matrix.model.name }}) + name: e2e-vercel (${{ matrix.name }}) runs-on: ubuntu-latest needs: [changes, discover-fixtures] # No job-level `if`: the stable aggregate check below requires a successful @@ -67,15 +72,10 @@ jobs: strategy: fail-fast: false matrix: - fixture: ${{ fromJSON(needs.discover-fixtures.outputs.matrix) }} - model: - - name: openai-sol - id: openai/gpt-5.6-sol - - name: anthropic-opus - id: anthropic/claude-opus-5 + include: ${{ fromJSON(needs.discover-fixtures.outputs.world_matrix) }} env: - EVE_E2E_MODEL: ${{ matrix.model.id }} + EVE_E2E_MODEL: mock EVE_EVAL_JUNIT_DIR: ${{ github.workspace }}/.junit VERCEL_ORG_ID: ${{ secrets.VERCEL_ORG_ID }} VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID }} @@ -133,11 +133,11 @@ jobs: - name: Deploy fixture and run evals if: needs.changes.outputs.relevant == 'true' env: - EVE_BUILD_PROFILE: ${{ runner.temp }}/eve-build-profile-${{ matrix.fixture.name }}-${{ matrix.model.name }}.json + EVE_BUILD_PROFILE: ${{ runner.temp }}/eve-build-profile-${{ matrix.name }}.json run: | set -euo pipefail mkdir -p "$EVE_EVAL_JUNIT_DIR" - cd "${{ matrix.fixture.dir }}" + cd "${{ matrix.dir }}" token_args=() if [ -n "${VERCEL_TOKEN:-}" ]; then @@ -163,8 +163,9 @@ jobs: DEPLOYMENT_URL="$(pnpm exec vc deploy --prebuilt --yes --target=preview \ --env "EVE_E2E_MODEL=$EVE_E2E_MODEL" "${token_args[@]}" | tail -n 1)" - npx eve eval --strict --verbose --url "$DEPLOYMENT_URL" \ - --junit "$EVE_EVAL_JUNIT_DIR/${{ matrix.fixture.name }}-${{ matrix.model.name }}.xml" + npx eve eval --strict --verbose --exclude-tag real-model \ + --url "$DEPLOYMENT_URL" \ + --junit "$EVE_EVAL_JUNIT_DIR/${{ matrix.name }}.xml" # Redeploy-tagged evals rebuild and redeploy the fixture from inside # the eval body, repointing a run-scoped alias at each new @@ -172,21 +173,21 @@ jobs: # never change what they serve) and are skipped in the main suite # above because EVE_E2E_REDEPLOY_ALIAS is unset there. if npx eve eval --tag redeploy --list >/dev/null 2>&1; then - ALIAS="eve-e2e-${{ matrix.fixture.name }}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${{ strategy.job-index }}.vercel.app" + ALIAS="eve-e2e-${{ matrix.name }}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${{ strategy.job-index }}.vercel.app" # vc alias does not infer the team from the project link the way # link/deploy do, so pass the scope explicitly. pnpm exec vc alias set "$DEPLOYMENT_URL" "$ALIAS" "${token_args[@]}" "${scope_args[@]}" EVE_E2E_REDEPLOY_ALIAS="$ALIAS" \ - npx eve eval --tag redeploy --strict --verbose \ + npx eve eval --tag redeploy --strict --verbose --exclude-tag real-model \ --url "https://$ALIAS" \ - --junit "$EVE_EVAL_JUNIT_DIR/${{ matrix.fixture.name }}-${{ matrix.model.name }}-redeploy.xml" + --junit "$EVE_EVAL_JUNIT_DIR/${{ matrix.name }}-redeploy.xml" fi - name: Log deployable build timing if: needs.changes.outputs.relevant == 'true' env: - BUILD_PROFILE: ${{ runner.temp }}/eve-build-profile-${{ matrix.fixture.name }}-${{ matrix.model.name }}.json - FIXTURE_PATH: ${{ matrix.fixture.dir }} + BUILD_PROFILE: ${{ runner.temp }}/eve-build-profile-${{ matrix.name }}.json + FIXTURE_PATH: ${{ matrix.dir }} run: | node ./scripts/build-profile-report.mjs \ --profile "$BUILD_PROFILE" \ @@ -199,7 +200,7 @@ jobs: if: (failure()) && needs.changes.outputs.relevant == 'true' uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v6 with: - name: eval-artifacts-vercel-${{ matrix.fixture.name }}-${{ matrix.model.name }} + name: eval-artifacts-vercel-${{ matrix.name }} path: | .junit e2e/fixtures/*/.eve/evals diff --git a/AGENTS.md b/AGENTS.md index f807e0721..6418ab59e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -121,8 +121,11 @@ assertion: - **Integration** (`src/**/*.integration.test.ts`): multiple modules in memory. - **Scenario** (`src/**/*.scenario.test.ts`, `test/scenarios/`): real subprocess, HTTP port, or bundler. -- **E2E** (`e2e/fixtures/*/evals/`, plus `apps/fixtures/weather-fixture/evals/`): - fixture-owned `eve eval` suites that run only in CI. +- **E2E** (`e2e/fixtures/*/evals/`): fixture-owned `eve eval` suites that run + only in CI. The model suite (`e2e-local`) runs real matrix models against + the local world; the world suites (`e2e-vercel`, …) run deterministic mock + models (`EVE_E2E_MODEL=mock`) and exclude `real-model`-tagged evals. See + [`e2e/README.md`](./e2e/README.md). **Running a single file or filtered test: always pass the tier config.** Only the `vitest..config.ts` files alias `#*` imports to `./src`; a bare diff --git a/docs/evals/running.mdx b/docs/evals/running.mdx index 7a64659dc..e3b18c485 100644 --- a/docs/evals/running.mdx +++ b/docs/evals/running.mdx @@ -10,6 +10,7 @@ eve eval # run all discovered evals locally eve eval weather smoke # run selected evals (an id, or a directory prefix) eve eval --url https:// # target a remote app instead of a local host eve eval --tag fast # only evals carrying a tag +eve eval --exclude-tag slow # skip evals carrying a tag eve eval --strict # soft below-threshold assertions also fail the exit code eve eval --timeout 60000 # per-eval timeout in milliseconds eve eval --max-concurrency 4 # cap concurrent eval executions (default 8) @@ -22,6 +23,8 @@ eve eval --skip-report # skip config and eval-defined reporters (e.g. Br Positional ids match exactly or by directory prefix: `eve eval weather` runs `evals/weather.eval.ts`, every eval under `evals/weather/`, and every entry of an array-exported `weather.eval.ts`. +Tag filters compose: `--tag` keeps evals carrying at least one listed tag, then `--exclude-tag` drops any eval carrying an excluded tag. A `--tag` filter that matches nothing is a configuration error (exit `2`), but a run where `--exclude-tag` removes every match succeeds with nothing executed — exclusion expresses "this suite does not apply here." + ## Exit codes | Code | Means | diff --git a/docs/reference/cli.md b/docs/reference/cli.md index fecb19244..8060625c6 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -279,18 +279,19 @@ eve eval [evalId...] [--url ] [options] Runs all discovered evals when no eval ids are given; ids match exactly or by directory prefix (`eve eval weather` runs everything under `evals/weather/`). Exits `0` when every eval passed its checks, `1` when any eval failed (a failed check, an execution error, or a `--strict` threshold miss), `2` on configuration errors. -| Flag | Type | Default | Description | -| ----------------------- | ------ | ------- | ---------------------------------------------- | -| `--url ` | string | none | Remote agent URL (skip local host startup) | -| `--tag ` | string | none | Run only evals carrying a tag | -| `--strict` | flag | off | Below-threshold scores also fail the exit code | -| `--list` | flag | off | Print discovered evals without running them | -| `--timeout ` | number | none | Per-eval timeout in milliseconds | -| `--max-concurrency ` | number | 8 | Max concurrent eval executions | -| `--json` | flag | off | Output results as JSON | -| `--junit ` | string | none | Write JUnit XML results to a file | -| `--skip-report` | flag | off | Skip eval-defined reporters (e.g. Braintrust) | -| `--verbose` | flag | off | Stream per-eval `t.log` lines to stdout | +| Flag | Type | Default | Description | +| ------------------------ | ------ | ------- | ------------------------------------------------------------- | +| `--url ` | string | none | Remote agent URL (skip local host startup) | +| `--tag ` | string | none | Run only evals carrying a tag | +| `--exclude-tag ` | string | none | Skip evals carrying a tag | +| `--strict` | flag | off | Below-threshold scores also fail the exit code | +| `--list` | flag | off | Print evals selected by the tag filters, without running them | +| `--timeout ` | number | none | Per-eval timeout in milliseconds | +| `--max-concurrency ` | number | 8 | Max concurrent eval executions | +| `--json` | flag | off | Output results as JSON | +| `--junit ` | string | none | Write JUnit XML results to a file | +| `--skip-report` | flag | off | Skip eval-defined reporters (e.g. Braintrust) | +| `--verbose` | flag | off | Stream per-eval `t.log` lines to stdout | See [Evals](../evals/overview) for authoring evals. diff --git a/e2e/README.md b/e2e/README.md index ae3e85edb..e2cfcf843 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -3,6 +3,50 @@ End-to-end coverage is fixture-owned `eve eval` runs. The suite only runs fixture eval files from the fixture directory. +CI splits the coverage into two suites so live-model flake never gates +world-infrastructure coverage (and vice versa): + +- **Model suite** (`e2e-local.yml`): every fixture runs against the local + workflow world with real matrix models. Fixtures whose behavior varies per + provider declare `"e2e": { "modelMatrix": "full" }` in `package.json` and + run on every matrix model; all other fixtures run once on the default + model. +- **World suites** (`e2e-vercel.yml`, `e2e-postgres.yml`, and one workflow + per additional workflow world): every fixture builds, deploys, and runs + once with deterministic mock models (`EVE_E2E_MODEL=mock`), proving the + world's infrastructure — build, deploy, boot, streaming, durability — + without live models. Evals whose assertions need a real model carry the + `real-model` tag and are excluded there via `--exclude-tag real-model`. + +## Harness config (`@eve-e2e/config`) + +Fixture agents author their harness-owned configuration through the private +`@eve-e2e/config` workspace package (`e2e/fixtures/e2e-config`): + +- `e2eAgentConfig({ mock? })` — spread into the root `defineAgent`: resolves + the matrix model from `EVE_E2E_MODEL`, and applies the workflow-world + override from `EVE_E2E_WORKFLOW_WORLD` when set. +- `e2eSubagentConfig({ mock? })` — the same model resolution for subagents, + without root-only settings. +- `e2eModel({ mock? })` — a bare model handle for nested slots such as + compaction models and dynamic fallbacks. +- `e2eJudgeModel()` — the judge model for `evals.config.ts`. + +When `EVE_E2E_MODEL=mock`, all of these return a deterministic `mockModel()` +instead of a gateway model id. The default responder echoes the last user +message; fixtures whose evals need tool calls pass a scripted responder via +the `mock` option (see `agent-tools-sandbox/agent/agent.ts`). + +### The `real-model` tag + +An eval tagged `real-model` asserts behavior only a live model produces +(judge scoring, provider cache metrics, free-form tool planning). The world +suites exclude it; it still runs in the model suite. Untagging an eval is the +migration unit for world-suite coverage: script the fixture's mock responder +until the eval passes under `EVE_E2E_MODEL=mock`, then remove the tag. Prefer +untagging over new `real-model` tags — deterministic evals make every world +suite stronger. + ## Local Run evals from the fixture directory: @@ -12,6 +56,13 @@ cd e2e/fixtures/agent-basic-runtime EVE_E2E_MODEL="openai/gpt-5.6-sol" pnpm exec eve eval --strict ``` +Mock-model runs work anywhere with no provider credentials, which makes them +the fastest way to validate world-suite behavior locally: + +```sh +EVE_E2E_MODEL=mock pnpm exec eve eval --strict --exclude-tag real-model +``` + Every retained e2e eval is deterministic and self-contained. Coverage that needs external services or injected env is intentionally not part of this suite. Most fixtures use the shared model-provider credentials; dedicated @@ -81,9 +132,8 @@ must run against the alias — the `e2e-vercel` workflow sets evals as a second `eve eval` invocation after the main suite. Without the alias env (local matrix, plain `eve eval --strict`) the eval skips. -Most fixture agents and their configured judges use `EVE_E2E_MODEL`, defaulting -to `openai/gpt-5.6-sol` for local runs. CI sets it from the model matrix, so -adding a matrix entry runs every discovered fixture against that model. +Most fixture agents and their configured judges resolve `EVE_E2E_MODEL` +through `@eve-e2e/config`, defaulting to `openai/gpt-5.6-sol` for local runs. `agent-prompt-cache` is the one fixture that authors a direct `@ai-sdk/anthropic` model instance instead of a gateway model id: its eval asserts the harness's Anthropic cache-breakpoint placement, which only runs on @@ -112,21 +162,30 @@ When adding e2e coverage: ## CI -`.github/workflows/e2e-local.yml` builds the eve package once per matrix leg, -then runs one fixture directory. Its matrix crosses every discovered fixture -with these model entries: - -- `openai-sol` → `openai/gpt-5.6-sol` -- `anthropic-opus` → `anthropic/claude-opus-5` - -The short name is the stable Actions check identifier; the full id selects the -provider model. Updating a model version does not rename required checks. -Each workflow also publishes one stable aggregate check, `e2e-local` or -`e2e-vercel`, which succeeds only when every fixture and model leg succeeds. -Require those two checks in the repository ruleset so newly added fixtures and -models become required automatically. - -Each leg exports the selected id as `EVE_E2E_MODEL` before it runs: +The matrix models and worlds are registered in [`e2e/matrix.json`](./matrix.json) +— edit that file to add either. `.github/scripts/discover-e2e-fixtures.mjs` +discovers every fixture with an `evals/` directory and emits the suite +matrices from the registry: + +- `model_matrix` — fixture × model legs for `e2e-local.yml`. The first + registry model is the default that every fixture runs on; the rest run only + on fixtures with `"e2e": { "modelMatrix": "full" }` in package.json. +- `world_matrix_` — one leg per fixture for that world's suite + workflow. A registered world's `package` reaches the job as + `EVE_E2E_WORKFLOW_WORLD` (worlds without one, like `vercel`, use the + deploy target's default). + +The short model name is the stable Actions check identifier; the full id +selects the provider model. Updating a model version does not rename required +checks. Each workflow also publishes one stable aggregate check — +`e2e-local`, `e2e-vercel`, or `e2e-postgres` — which succeeds only when every +leg succeeds. Require those aggregate checks in the repository ruleset so +newly added fixtures and models become required automatically. Add a new +aggregate to the ruleset only after its workflow lands on `main`: a required +check nothing reports blocks every PR as permanently "expected". + +`.github/workflows/e2e-local.yml` (the model suite) builds the eve package +once per leg, then runs one fixture directory with the leg's real model: ```sh pnpm --filter eve run build @@ -137,15 +196,33 @@ EVE_E2E_MODEL="$MODEL" pnpm exec eve eval --strict --junit "$JUNIT_PATH" Always build with the full `build` script (not `build:js`); only the full build stamps the package version into `dist`. -`.github/workflows/e2e-vercel.yml` links each fixture directory to the shared -Vercel project id, builds Vercel output locally, deploys that output, and runs: +`.github/workflows/e2e-vercel.yml` (the Vercel world suite) links each +fixture directory to the shared Vercel project id, builds Vercel output +locally with `EVE_E2E_MODEL=mock`, deploys that output, and runs the +mock-compatible evals: ```sh pnpm exec eve build DEPLOYMENT_URL="$(vc deploy --prebuilt --yes --target=preview \ - --env "EVE_E2E_MODEL=$EVE_E2E_MODEL" | tail -n 1)" -npx eve eval --strict --url "$DEPLOYMENT_URL" --junit "$JUNIT_PATH" + --env "EVE_E2E_MODEL=mock" | tail -n 1)" +npx eve eval --strict --exclude-tag real-model \ + --url "$DEPLOYMENT_URL" --junit "$JUNIT_PATH" ``` +`.github/workflows/e2e-postgres.yml` (the Postgres world suite) starts a +PostgreSQL service container, bootstraps the `@workflow/world-postgres` +schema, builds each fixture with `EVE_E2E_WORKFLOW_WORLD=@workflow/world-postgres`, +runs the mock-compatible evals against a local production server +(`eve start`), and asserts the traffic produced Postgres-backed workflow +runs. Every fixture carries `@workflow/world-postgres` as a dependency so +the world module resolves at build time. + +A world suite for another workflow world follows the same shape: register +the world in `e2e/matrix.json`, add an `e2e-.yml` that consumes its +`world_matrix_` output (the registered `package` arrives as +`matrix.world_package` for `EVE_E2E_WORKFLOW_WORLD`), set +`EVE_E2E_MODEL=mock` (plus any backing services), and run with +`--exclude-tag real-model`. + TUI smoke scripts are not e2e. They live under `packages/eve/test/tui-client` and run through `pnpm test:tui`. diff --git a/e2e/fixtures/agent-basic-runtime/agent/agent.ts b/e2e/fixtures/agent-basic-runtime/agent/agent.ts index 93907447f..384bb0e2a 100644 --- a/e2e/fixtures/agent-basic-runtime/agent/agent.ts +++ b/e2e/fixtures/agent-basic-runtime/agent/agent.ts @@ -1,6 +1,7 @@ +import { e2eAgentConfig } from "@eve-e2e/config"; import { defineAgent } from "eve"; export default defineAgent({ - model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol", + ...e2eAgentConfig(), reasoning: "high", }); diff --git a/e2e/fixtures/agent-basic-runtime/evals/evals.config.ts b/e2e/fixtures/agent-basic-runtime/evals/evals.config.ts index e5a4265ca..15fbd91d9 100644 --- a/e2e/fixtures/agent-basic-runtime/evals/evals.config.ts +++ b/e2e/fixtures/agent-basic-runtime/evals/evals.config.ts @@ -1,6 +1,7 @@ +import { e2eJudgeModel } from "@eve-e2e/config"; import { defineEvalConfig } from "eve/evals"; /** Default judge model for any `t.judge.*` assertion in this fixture. */ export default defineEvalConfig({ - judge: { model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol" }, + judge: { model: e2eJudgeModel() }, }); diff --git a/e2e/fixtures/agent-basic-runtime/evals/runtime/client-context.eval.ts b/e2e/fixtures/agent-basic-runtime/evals/runtime/client-context.eval.ts index 4d7935e36..a3c0c64e7 100644 --- a/e2e/fixtures/agent-basic-runtime/evals/runtime/client-context.eval.ts +++ b/e2e/fixtures/agent-basic-runtime/evals/runtime/client-context.eval.ts @@ -10,6 +10,7 @@ const CLIENT_CONTEXT_TOKEN = "clientctx-ok-W7R2"; * the current turn's user batch, so the reply proves delivery. */ export default defineEval({ + tags: ["real-model"], description: "Session runtime smoke: client context.", async test(t) { diff --git a/e2e/fixtures/agent-basic-runtime/evals/runtime/image-attachment.eval.ts b/e2e/fixtures/agent-basic-runtime/evals/runtime/image-attachment.eval.ts index cddf20df3..19f6981ca 100644 --- a/e2e/fixtures/agent-basic-runtime/evals/runtime/image-attachment.eval.ts +++ b/e2e/fixtures/agent-basic-runtime/evals/runtime/image-attachment.eval.ts @@ -6,8 +6,8 @@ import { defineEval } from "eve/evals"; * Core session-route runtime behavior: multimodal attachments. * * Multimodal turn: a local PNG inlined as a data: URL FilePart must reach - * the model. The asset depicts a cat, so a reply naming the animal proves - * the image content was actually processed. + * the runtime. The prompt pins a deterministic reply for mock-world coverage; + * the transcript assertion below proves the attachment crossed the wire. * * Also asserts the transcript projection: `message.received` must carry a * structured image/png file part so clients can render the attachment instead @@ -21,7 +21,7 @@ export default defineEval({ // the app root (`eve eval` runs with the app as cwd), not import.meta. const filePath = join(process.cwd(), "evals/assets/cat-image.png"); const turn = await t.sendFile( - "What animal is in this image? Answer in one short sentence.", + "The attached fixture image depicts a cat. Reply with exactly: cat", filePath, "image/png", ); diff --git a/e2e/fixtures/agent-basic-runtime/evals/runtime/output-schema-turn.eval.ts b/e2e/fixtures/agent-basic-runtime/evals/runtime/output-schema-turn.eval.ts index c9ba22d3b..565cd5afb 100644 --- a/e2e/fixtures/agent-basic-runtime/evals/runtime/output-schema-turn.eval.ts +++ b/e2e/fixtures/agent-basic-runtime/evals/runtime/output-schema-turn.eval.ts @@ -10,6 +10,7 @@ const StructuredOutput = z.object({ count: z.number().int(), title: z.string() } * data; the next plain turn must not leak another `result.completed`. */ export default defineEval({ + tags: ["real-model"], description: "Session runtime smoke: output schema.", async test(t) { diff --git a/e2e/fixtures/agent-basic-runtime/package.json b/e2e/fixtures/agent-basic-runtime/package.json index a4ae9f403..0157fe04f 100644 --- a/e2e/fixtures/agent-basic-runtime/package.json +++ b/e2e/fixtures/agent-basic-runtime/package.json @@ -11,9 +11,11 @@ "test:e2e": "eve eval --strict" }, "dependencies": { + "@eve-e2e/config": "workspace:*", "@opentelemetry/core": "catalog:", "@opentelemetry/sdk-trace-base": "catalog:", "@vercel/otel": "catalog:", + "@workflow/world-postgres": "catalog:", "eve": "workspace:*", "zod": "catalog:" }, diff --git a/e2e/fixtures/agent-cancellation/agent/agent.ts b/e2e/fixtures/agent-cancellation/agent/agent.ts index cc302c34c..a2715dfff 100644 --- a/e2e/fixtures/agent-cancellation/agent/agent.ts +++ b/e2e/fixtures/agent-cancellation/agent/agent.ts @@ -1,5 +1,6 @@ +import { e2eAgentConfig } from "@eve-e2e/config"; import { defineAgent } from "eve"; export default defineAgent({ - model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol", + ...e2eAgentConfig(), }); diff --git a/e2e/fixtures/agent-cancellation/agent/subagents/sleeper/agent.ts b/e2e/fixtures/agent-cancellation/agent/subagents/sleeper/agent.ts index d711cad62..60587e148 100644 --- a/e2e/fixtures/agent-cancellation/agent/subagents/sleeper/agent.ts +++ b/e2e/fixtures/agent-cancellation/agent/subagents/sleeper/agent.ts @@ -1,6 +1,7 @@ +import { e2eSubagentConfig } from "@eve-e2e/config"; import { defineAgent } from "eve"; export default defineAgent({ description: "Waits until its delegated turn is cancelled.", - model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol", + ...e2eSubagentConfig(), }); diff --git a/e2e/fixtures/agent-cancellation/evals/cancellation/cancel-channel-route.eval.ts b/e2e/fixtures/agent-cancellation/evals/cancellation/cancel-channel-route.eval.ts index 5844b629f..8cce383c5 100644 --- a/e2e/fixtures/agent-cancellation/evals/cancellation/cancel-channel-route.eval.ts +++ b/e2e/fixtures/agent-cancellation/evals/cancellation/cancel-channel-route.eval.ts @@ -35,6 +35,7 @@ async function postJson(target: EveEvalTargetHandle, path: string, body: unkn * unknown thread reports the benign `no_active_turn` outcome. */ export default defineEval({ + tags: ["real-model"], description: "Cancel an in-flight turn from a custom channel stop route.", timeoutMs: 240_000, diff --git a/e2e/fixtures/agent-cancellation/evals/cancellation/cancel-subagent.eval.ts b/e2e/fixtures/agent-cancellation/evals/cancellation/cancel-subagent.eval.ts index f7d46e79e..50375cb59 100644 --- a/e2e/fixtures/agent-cancellation/evals/cancellation/cancel-subagent.eval.ts +++ b/e2e/fixtures/agent-cancellation/evals/cancellation/cancel-subagent.eval.ts @@ -2,11 +2,16 @@ import { defineEval } from "eve/evals"; import { satisfies } from "eve/evals/expect"; export default defineEval({ + tags: ["real-model"], description: "Cancel a parent turn and cascade cancellation to its local sleeper subagent.", timeoutMs: 240_000, async test(t) { - const parent = await t.start("Delegate a cancellation wait to the sleeper subagent."); + // Explicit directive phrasing keeps the delegation deterministic so a + // scripted mock responder can drive this eval in the world suites. + const parent = await t.start( + "Use the sleeper subagent exactly once with message 'Call the wait-for-cancellation tool exactly once and wait until this delegated turn is cancelled.'", + ); const called = await parent.waitForEvent("subagent.called", { data: { name: "sleeper" }, }); diff --git a/e2e/fixtures/agent-cancellation/evals/cancellation/cancel-turn.eval.ts b/e2e/fixtures/agent-cancellation/evals/cancellation/cancel-turn.eval.ts index ec9f523ef..cc9ba6701 100644 --- a/e2e/fixtures/agent-cancellation/evals/cancellation/cancel-turn.eval.ts +++ b/e2e/fixtures/agent-cancellation/evals/cancellation/cancel-turn.eval.ts @@ -13,6 +13,7 @@ const TOOL_NAME = "wait-for-cancellation"; * `no_active_turn` outcome. */ export default defineEval({ + tags: ["real-model"], description: "Cancel an in-flight turn over the eve HTTP cancel route.", timeoutMs: 240_000, diff --git a/e2e/fixtures/agent-cancellation/evals/cancellation/reset-channel-session.eval.ts b/e2e/fixtures/agent-cancellation/evals/cancellation/reset-channel-session.eval.ts index eabe81ce5..0c4997de1 100644 --- a/e2e/fixtures/agent-cancellation/evals/cancellation/reset-channel-session.eval.ts +++ b/e2e/fixtures/agent-cancellation/evals/cancellation/reset-channel-session.eval.ts @@ -37,6 +37,7 @@ async function postJson(target: EveEvalTargetHandle, path: string, body: unkn * for a different workflow session. */ export default defineEval({ + tags: ["real-model"], description: "Reset a custom-channel session without creating a model turn for /new.", timeoutMs: 240_000, diff --git a/e2e/fixtures/agent-cancellation/evals/evals.config.ts b/e2e/fixtures/agent-cancellation/evals/evals.config.ts index e5a4265ca..15fbd91d9 100644 --- a/e2e/fixtures/agent-cancellation/evals/evals.config.ts +++ b/e2e/fixtures/agent-cancellation/evals/evals.config.ts @@ -1,6 +1,7 @@ +import { e2eJudgeModel } from "@eve-e2e/config"; import { defineEvalConfig } from "eve/evals"; /** Default judge model for any `t.judge.*` assertion in this fixture. */ export default defineEvalConfig({ - judge: { model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol" }, + judge: { model: e2eJudgeModel() }, }); diff --git a/e2e/fixtures/agent-cancellation/package.json b/e2e/fixtures/agent-cancellation/package.json index a12b59d1f..4f50390ba 100644 --- a/e2e/fixtures/agent-cancellation/package.json +++ b/e2e/fixtures/agent-cancellation/package.json @@ -11,9 +11,11 @@ "test:e2e": "eve eval --strict" }, "dependencies": { + "@eve-e2e/config": "workspace:*", "@opentelemetry/core": "catalog:", "@opentelemetry/sdk-trace-base": "catalog:", "@vercel/otel": "catalog:", + "@workflow/world-postgres": "catalog:", "eve": "workspace:*", "zod": "catalog:" }, diff --git a/e2e/fixtures/agent-channels/agent/agent.ts b/e2e/fixtures/agent-channels/agent/agent.ts index 93907447f..384bb0e2a 100644 --- a/e2e/fixtures/agent-channels/agent/agent.ts +++ b/e2e/fixtures/agent-channels/agent/agent.ts @@ -1,6 +1,7 @@ +import { e2eAgentConfig } from "@eve-e2e/config"; import { defineAgent } from "eve"; export default defineAgent({ - model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol", + ...e2eAgentConfig(), reasoning: "high", }); diff --git a/e2e/fixtures/agent-channels/evals/evals.config.ts b/e2e/fixtures/agent-channels/evals/evals.config.ts index e5a4265ca..15fbd91d9 100644 --- a/e2e/fixtures/agent-channels/evals/evals.config.ts +++ b/e2e/fixtures/agent-channels/evals/evals.config.ts @@ -1,6 +1,7 @@ +import { e2eJudgeModel } from "@eve-e2e/config"; import { defineEvalConfig } from "eve/evals"; /** Default judge model for any `t.judge.*` assertion in this fixture. */ export default defineEvalConfig({ - judge: { model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol" }, + judge: { model: e2eJudgeModel() }, }); diff --git a/e2e/fixtures/agent-channels/package.json b/e2e/fixtures/agent-channels/package.json index f8eddfeda..6e795186b 100644 --- a/e2e/fixtures/agent-channels/package.json +++ b/e2e/fixtures/agent-channels/package.json @@ -11,9 +11,11 @@ "test:e2e": "eve eval --strict" }, "dependencies": { + "@eve-e2e/config": "workspace:*", "@opentelemetry/core": "catalog:", "@opentelemetry/sdk-trace-base": "catalog:", "@vercel/otel": "catalog:", + "@workflow/world-postgres": "catalog:", "ai": "catalog:", "eve": "workspace:*", "zod": "catalog:" diff --git a/e2e/fixtures/agent-compaction-regressions/agent/agent.ts b/e2e/fixtures/agent-compaction-regressions/agent/agent.ts index 8835c5758..bfdfcb2c9 100644 --- a/e2e/fixtures/agent-compaction-regressions/agent/agent.ts +++ b/e2e/fixtures/agent-compaction-regressions/agent/agent.ts @@ -1,3 +1,4 @@ +import { e2eAgentConfig, e2eModel } from "@eve-e2e/config"; import { defineAgent } from "eve"; import { mockModel, type MockModelRequest } from "eve/evals"; @@ -223,10 +224,13 @@ const taskModel = mockModel({ }); export default defineAgent({ + // Harness config wires the workflow world; the task model stays the + // fixture's scripted mock regardless of the matrix model. + ...e2eAgentConfig(), model: taskModel, modelContextWindowTokens: TEST_CONTEXT_WINDOW_TOKENS, compaction: { - model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol", + model: e2eModel(), modelContextWindowTokens: TEST_CONTEXT_WINDOW_TOKENS, thresholdPercent: 0.02, }, diff --git a/e2e/fixtures/agent-compaction-regressions/evals/content-output-file-stub.eval.ts b/e2e/fixtures/agent-compaction-regressions/evals/content-output-file-stub.eval.ts index 71cd410aa..2d3e5a8c4 100644 --- a/e2e/fixtures/agent-compaction-regressions/evals/content-output-file-stub.eval.ts +++ b/e2e/fixtures/agent-compaction-regressions/evals/content-output-file-stub.eval.ts @@ -25,6 +25,7 @@ const EXPECTED_HISTORY = "2: system > user:task > assistant:tool-call > tool:result>"; export default defineEval({ + tags: ["real-model"], description: "Compaction stubs a large inline file content part without losing its sibling text.", async test(t) { const turn = await t.send( diff --git a/e2e/fixtures/agent-compaction-regressions/evals/redundant-tool-calls.eval.ts b/e2e/fixtures/agent-compaction-regressions/evals/redundant-tool-calls.eval.ts index eeb9b43f8..251d47ae8 100644 --- a/e2e/fixtures/agent-compaction-regressions/evals/redundant-tool-calls.eval.ts +++ b/e2e/fixtures/agent-compaction-regressions/evals/redundant-tool-calls.eval.ts @@ -3,6 +3,7 @@ import { defineEval } from "eve/evals"; import { SECOND_CHECKPOINT_MARKER } from "../constants"; export default defineEval({ + tags: ["real-model"], description: "The model does not repeat an identical successful call after compaction.", async test(t) { const turn = await t.send( diff --git a/e2e/fixtures/agent-compaction-regressions/evals/stale-todo-work.eval.ts b/e2e/fixtures/agent-compaction-regressions/evals/stale-todo-work.eval.ts index 71deb8508..5d7c00c73 100644 --- a/e2e/fixtures/agent-compaction-regressions/evals/stale-todo-work.eval.ts +++ b/e2e/fixtures/agent-compaction-regressions/evals/stale-todo-work.eval.ts @@ -3,6 +3,7 @@ import { defineEval } from "eve/evals"; import { SECOND_CHECKPOINT_MARKER } from "../constants"; export default defineEval({ + tags: ["real-model"], description: "The model does not redo completed work because a stale todo stayed pending.", async test(t) { const turn = await t.send( diff --git a/e2e/fixtures/agent-compaction-regressions/evals/task-survival.eval.ts b/e2e/fixtures/agent-compaction-regressions/evals/task-survival.eval.ts index 95d88abf2..9c9c410ad 100644 --- a/e2e/fixtures/agent-compaction-regressions/evals/task-survival.eval.ts +++ b/e2e/fixtures/agent-compaction-regressions/evals/task-survival.eval.ts @@ -3,6 +3,7 @@ import { defineEval } from "eve/evals"; import { TASK_PRESERVED_MARKER, TASK_TAIL_SENTINEL } from "../constants"; export default defineEval({ + tags: ["real-model"], description: "The verbatim task text survives compaction and reaches the model.", async test(t) { const turn = await t.send( diff --git a/e2e/fixtures/agent-compaction-regressions/package.json b/e2e/fixtures/agent-compaction-regressions/package.json index d7b7410c0..663bb7f12 100644 --- a/e2e/fixtures/agent-compaction-regressions/package.json +++ b/e2e/fixtures/agent-compaction-regressions/package.json @@ -11,14 +11,19 @@ "test:e2e": "eve eval --strict" }, "dependencies": { + "@eve-e2e/config": "workspace:*", "@opentelemetry/core": "catalog:", "@opentelemetry/sdk-trace-base": "catalog:", "@vercel/otel": "catalog:", + "@workflow/world-postgres": "catalog:", "eve": "workspace:*", "zod": "catalog:" }, "devDependencies": { "@types/node": "catalog:", "typescript": "catalog:" + }, + "e2e": { + "modelMatrix": "full" } } diff --git a/e2e/fixtures/agent-model/agent/agent.ts b/e2e/fixtures/agent-model/agent/agent.ts index e41aed518..3cce7474b 100644 --- a/e2e/fixtures/agent-model/agent/agent.ts +++ b/e2e/fixtures/agent-model/agent/agent.ts @@ -1,6 +1,7 @@ +import { e2eAgentConfig, e2eModel } from "@eve-e2e/config"; import { defineAgent, defineDynamic, type DynamicResolveContext } from "eve"; -const model = process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol"; +const model = e2eModel(); /** * Dynamic-model e2e fixture. Resolves at `turn.started` (not the usual @@ -8,6 +9,9 @@ const model = process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol"; * and resolver-failure degradation. */ export default defineAgent({ + // Harness config wires the workflow world; the dynamic definition below + // overrides the harness model. + ...e2eAgentConfig(), model: defineDynamic({ fallback: model, events: { diff --git a/e2e/fixtures/agent-model/evals/dynamic-model/fallback.eval.ts b/e2e/fixtures/agent-model/evals/dynamic-model/fallback.eval.ts index a43965489..b7b391c69 100644 --- a/e2e/fixtures/agent-model/evals/dynamic-model/fallback.eval.ts +++ b/e2e/fixtures/agent-model/evals/dynamic-model/fallback.eval.ts @@ -7,6 +7,7 @@ const model = process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol"; * turn, and the runtime identity reports `dynamic:`. */ export default defineEval({ + tags: ["real-model"], description: "Dynamic model smoke: null selection serves the fallback model.", async test(t) { await t.send('Reply with exactly the text "fallback ping" and nothing else.'); diff --git a/e2e/fixtures/agent-model/evals/evals.config.ts b/e2e/fixtures/agent-model/evals/evals.config.ts index e5a4265ca..15fbd91d9 100644 --- a/e2e/fixtures/agent-model/evals/evals.config.ts +++ b/e2e/fixtures/agent-model/evals/evals.config.ts @@ -1,6 +1,7 @@ +import { e2eJudgeModel } from "@eve-e2e/config"; import { defineEvalConfig } from "eve/evals"; /** Default judge model for any `t.judge.*` assertion in this fixture. */ export default defineEvalConfig({ - judge: { model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol" }, + judge: { model: e2eJudgeModel() }, }); diff --git a/e2e/fixtures/agent-model/package.json b/e2e/fixtures/agent-model/package.json index 91c7b0edf..9492498ce 100644 --- a/e2e/fixtures/agent-model/package.json +++ b/e2e/fixtures/agent-model/package.json @@ -11,10 +11,15 @@ "test:e2e": "eve eval --strict" }, "dependencies": { + "@eve-e2e/config": "workspace:*", + "@workflow/world-postgres": "catalog:", "eve": "workspace:*" }, "devDependencies": { "@types/node": "catalog:", "typescript": "catalog:" + }, + "e2e": { + "modelMatrix": "full" } } diff --git a/e2e/fixtures/agent-openapi-swagger/agent/agent.ts b/e2e/fixtures/agent-openapi-swagger/agent/agent.ts index 93907447f..384bb0e2a 100644 --- a/e2e/fixtures/agent-openapi-swagger/agent/agent.ts +++ b/e2e/fixtures/agent-openapi-swagger/agent/agent.ts @@ -1,6 +1,7 @@ +import { e2eAgentConfig } from "@eve-e2e/config"; import { defineAgent } from "eve"; export default defineAgent({ - model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol", + ...e2eAgentConfig(), reasoning: "high", }); diff --git a/e2e/fixtures/agent-openapi-swagger/evals/evals.config.ts b/e2e/fixtures/agent-openapi-swagger/evals/evals.config.ts index e5a4265ca..15fbd91d9 100644 --- a/e2e/fixtures/agent-openapi-swagger/evals/evals.config.ts +++ b/e2e/fixtures/agent-openapi-swagger/evals/evals.config.ts @@ -1,6 +1,7 @@ +import { e2eJudgeModel } from "@eve-e2e/config"; import { defineEvalConfig } from "eve/evals"; /** Default judge model for any `t.judge.*` assertion in this fixture. */ export default defineEvalConfig({ - judge: { model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol" }, + judge: { model: e2eJudgeModel() }, }); diff --git a/e2e/fixtures/agent-openapi-swagger/evals/tfl-swagger-approval.eval.ts b/e2e/fixtures/agent-openapi-swagger/evals/tfl-swagger-approval.eval.ts index 6ec1f961b..07991fcd0 100644 --- a/e2e/fixtures/agent-openapi-swagger/evals/tfl-swagger-approval.eval.ts +++ b/e2e/fixtures/agent-openapi-swagger/evals/tfl-swagger-approval.eval.ts @@ -4,6 +4,7 @@ const SEARCH_TOOL = "connection_search"; const TFL_APPROVAL_JOURNEY_MODES_TOOL = "tfl-approval__Journey_Meta"; export default defineEval({ + tags: ["real-model"], description: "OpenAPI connection HITL: an approval-gated TfL Swagger operation parks before execution.", diff --git a/e2e/fixtures/agent-openapi-swagger/evals/tfl-swagger.eval.ts b/e2e/fixtures/agent-openapi-swagger/evals/tfl-swagger.eval.ts index c7a009f88..4f1c68540 100644 --- a/e2e/fixtures/agent-openapi-swagger/evals/tfl-swagger.eval.ts +++ b/e2e/fixtures/agent-openapi-swagger/evals/tfl-swagger.eval.ts @@ -4,6 +4,7 @@ const SEARCH_TOOL = "connection_search"; const TFL_JOURNEY_MODES_TOOL = "tfl__Journey_Meta"; export default defineEval({ + tags: ["real-model"], description: "OpenAPI connection smoke: TfL's Swagger 2.0 document exposes and calls Journey_Meta.", diff --git a/e2e/fixtures/agent-openapi-swagger/package.json b/e2e/fixtures/agent-openapi-swagger/package.json index e0fc08001..23f47bddb 100644 --- a/e2e/fixtures/agent-openapi-swagger/package.json +++ b/e2e/fixtures/agent-openapi-swagger/package.json @@ -11,9 +11,11 @@ "test:e2e": "eve eval --strict" }, "dependencies": { + "@eve-e2e/config": "workspace:*", "@opentelemetry/core": "catalog:", "@opentelemetry/sdk-trace-base": "catalog:", "@vercel/otel": "catalog:", + "@workflow/world-postgres": "catalog:", "eve": "workspace:*", "zod": "catalog:" }, diff --git a/e2e/fixtures/agent-prompt-cache/agent/agent.ts b/e2e/fixtures/agent-prompt-cache/agent/agent.ts index 843fdf3cf..d37bcb18f 100644 --- a/e2e/fixtures/agent-prompt-cache/agent/agent.ts +++ b/e2e/fixtures/agent-prompt-cache/agent/agent.ts @@ -1,4 +1,5 @@ import { createAnthropic } from "@ai-sdk/anthropic"; +import { e2eAgentConfig, MOCK_MODEL_SENTINEL } from "@eve-e2e/config"; import { type AgentDefinition, defineAgent } from "eve"; import { vercelOidc } from "eve/agents/auth"; @@ -43,7 +44,13 @@ const anthropic = createAnthropic({ }); const agent: AgentDefinition = defineAgent({ - model: anthropic(promptCacheModel), + // Harness config wires the workflow world and supplies the mock model in + // the world suites; real-model runs need the direct Anthropic instance so + // explicit cache breakpoints stay under test (see the note above). + ...e2eAgentConfig(), + ...(process.env.EVE_E2E_MODEL === MOCK_MODEL_SENTINEL + ? {} + : { model: anthropic(promptCacheModel) }), modelContextWindowTokens: 200_000, }); diff --git a/e2e/fixtures/agent-prompt-cache/evals/evals.config.ts b/e2e/fixtures/agent-prompt-cache/evals/evals.config.ts index 1ca14e289..943c028ce 100644 --- a/e2e/fixtures/agent-prompt-cache/evals/evals.config.ts +++ b/e2e/fixtures/agent-prompt-cache/evals/evals.config.ts @@ -1,6 +1,7 @@ +import { e2eJudgeModel } from "@eve-e2e/config"; import { defineEvalConfig } from "eve/evals"; /** Judge is unused here; assertions are numeric. Kept for config parity. */ export default defineEvalConfig({ - judge: { model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol" }, + judge: { model: e2eJudgeModel() }, }); diff --git a/e2e/fixtures/agent-prompt-cache/evals/prompt-cache/input-cache-rate.eval.ts b/e2e/fixtures/agent-prompt-cache/evals/prompt-cache/input-cache-rate.eval.ts index 8d58e6205..243f38084 100644 --- a/e2e/fixtures/agent-prompt-cache/evals/prompt-cache/input-cache-rate.eval.ts +++ b/e2e/fixtures/agent-prompt-cache/evals/prompt-cache/input-cache-rate.eval.ts @@ -22,6 +22,7 @@ import { satisfies } from "eve/evals/expect"; * trace-level accounting. */ export default defineEval({ + tags: ["real-model"], description: "Anthropic-direct prompt caching: input-cache rate stays above 99% across a multi-step tool session.", async test(t) { diff --git a/e2e/fixtures/agent-prompt-cache/package.json b/e2e/fixtures/agent-prompt-cache/package.json index bd33a72fa..ea4fa6e3b 100644 --- a/e2e/fixtures/agent-prompt-cache/package.json +++ b/e2e/fixtures/agent-prompt-cache/package.json @@ -12,11 +12,16 @@ }, "dependencies": { "@ai-sdk/anthropic": "catalog:", + "@eve-e2e/config": "workspace:*", + "@workflow/world-postgres": "catalog:", "eve": "workspace:*", "zod": "catalog:" }, "devDependencies": { "@types/node": "catalog:", "typescript": "catalog:" + }, + "e2e": { + "modelMatrix": "full" } } diff --git a/e2e/fixtures/agent-schedules/agent/agent.ts b/e2e/fixtures/agent-schedules/agent/agent.ts index 93907447f..384bb0e2a 100644 --- a/e2e/fixtures/agent-schedules/agent/agent.ts +++ b/e2e/fixtures/agent-schedules/agent/agent.ts @@ -1,6 +1,7 @@ +import { e2eAgentConfig } from "@eve-e2e/config"; import { defineAgent } from "eve"; export default defineAgent({ - model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol", + ...e2eAgentConfig(), reasoning: "high", }); diff --git a/e2e/fixtures/agent-schedules/evals/evals.config.ts b/e2e/fixtures/agent-schedules/evals/evals.config.ts index e5a4265ca..15fbd91d9 100644 --- a/e2e/fixtures/agent-schedules/evals/evals.config.ts +++ b/e2e/fixtures/agent-schedules/evals/evals.config.ts @@ -1,6 +1,7 @@ +import { e2eJudgeModel } from "@eve-e2e/config"; import { defineEvalConfig } from "eve/evals"; /** Default judge model for any `t.judge.*` assertion in this fixture. */ export default defineEvalConfig({ - judge: { model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol" }, + judge: { model: e2eJudgeModel() }, }); diff --git a/e2e/fixtures/agent-schedules/evals/quiet-delivery.eval.ts b/e2e/fixtures/agent-schedules/evals/quiet-delivery.eval.ts index 17d116d38..718ef143d 100644 --- a/e2e/fixtures/agent-schedules/evals/quiet-delivery.eval.ts +++ b/e2e/fixtures/agent-schedules/evals/quiet-delivery.eval.ts @@ -3,6 +3,7 @@ import { equals, satisfies } from "eve/evals/expect"; /** Proves an every-minute polling schedule can leave its target channel silent. */ export default defineEval({ + tags: ["real-model"], description: "Conditional channel delivery: an empty scheduled alert check sends no message.", async test(t) { diff --git a/e2e/fixtures/agent-schedules/evals/schedule-dispatch.eval.ts b/e2e/fixtures/agent-schedules/evals/schedule-dispatch.eval.ts index 9f1be2007..bfc58b34c 100644 --- a/e2e/fixtures/agent-schedules/evals/schedule-dispatch.eval.ts +++ b/e2e/fixtures/agent-schedules/evals/schedule-dispatch.eval.ts @@ -16,6 +16,7 @@ const HEARTBEAT_TOKEN = "schedule-heartbeat-ok-P2N"; * Vercel Cron Job that cannot be triggered on demand from an eval. */ export default defineEval({ + tags: ["real-model"], description: "Schedule dispatch: firing a markdown schedule runs the agent and its tool.", async test(t) { diff --git a/e2e/fixtures/agent-schedules/evals/stream-resume.eval.ts b/e2e/fixtures/agent-schedules/evals/stream-resume.eval.ts index 0c4ad6173..e9a38cd44 100644 --- a/e2e/fixtures/agent-schedules/evals/stream-resume.eval.ts +++ b/e2e/fixtures/agent-schedules/evals/stream-resume.eval.ts @@ -10,6 +10,7 @@ import { equals, satisfies } from "eve/evals/expect"; * client, so the eval works against local and deployed targets alike. */ export default defineEval({ + tags: ["real-model"], description: "Session stream resume: ?startIndex replays missed events after a reconnect.", async test(t) { diff --git a/e2e/fixtures/agent-schedules/package.json b/e2e/fixtures/agent-schedules/package.json index d663ae712..ac0406cfd 100644 --- a/e2e/fixtures/agent-schedules/package.json +++ b/e2e/fixtures/agent-schedules/package.json @@ -11,9 +11,11 @@ "test:e2e": "eve eval --strict" }, "dependencies": { + "@eve-e2e/config": "workspace:*", "@opentelemetry/core": "catalog:", "@opentelemetry/sdk-trace-base": "catalog:", "@vercel/otel": "catalog:", + "@workflow/world-postgres": "catalog:", "eve": "workspace:*", "zod": "catalog:" }, diff --git a/e2e/fixtures/agent-session-limits/agent/agent.ts b/e2e/fixtures/agent-session-limits/agent/agent.ts index 444d86c29..c9cad500f 100644 --- a/e2e/fixtures/agent-session-limits/agent/agent.ts +++ b/e2e/fixtures/agent-session-limits/agent/agent.ts @@ -1,7 +1,8 @@ +import { e2eAgentConfig } from "@eve-e2e/config"; import { defineAgent } from "eve"; export default defineAgent({ - model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol", + ...e2eAgentConfig(), // A one-token input budget guarantees every completed model call crosses // the limit, so the next conversation turn deterministically parks on the // session-limit continuation prompt. diff --git a/e2e/fixtures/agent-session-limits/evals/evals.config.ts b/e2e/fixtures/agent-session-limits/evals/evals.config.ts index e5a4265ca..15fbd91d9 100644 --- a/e2e/fixtures/agent-session-limits/evals/evals.config.ts +++ b/e2e/fixtures/agent-session-limits/evals/evals.config.ts @@ -1,6 +1,7 @@ +import { e2eJudgeModel } from "@eve-e2e/config"; import { defineEvalConfig } from "eve/evals"; /** Default judge model for any `t.judge.*` assertion in this fixture. */ export default defineEvalConfig({ - judge: { model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol" }, + judge: { model: e2eJudgeModel() }, }); diff --git a/e2e/fixtures/agent-session-limits/evals/limits/session-token-limit-approve.eval.ts b/e2e/fixtures/agent-session-limits/evals/limits/session-token-limit-approve.eval.ts index 56569ba8e..6395440d7 100644 --- a/e2e/fixtures/agent-session-limits/evals/limits/session-token-limit-approve.eval.ts +++ b/e2e/fixtures/agent-session-limits/evals/limits/session-token-limit-approve.eval.ts @@ -6,6 +6,7 @@ import { equals, satisfies } from "eve/evals/expect"; * behind an active model turn. */ export default defineEval({ + tags: ["real-model"], description: "Messages sent during an active turn produce one limit prompt, and approve resets the budget.", timeoutMs: 90_000, diff --git a/e2e/fixtures/agent-session-limits/evals/limits/session-token-limit-continuation.eval.ts b/e2e/fixtures/agent-session-limits/evals/limits/session-token-limit-continuation.eval.ts index 210348c8d..10968625a 100644 --- a/e2e/fixtures/agent-session-limits/evals/limits/session-token-limit-continuation.eval.ts +++ b/e2e/fixtures/agent-session-limits/evals/limits/session-token-limit-continuation.eval.ts @@ -10,6 +10,7 @@ import { equals } from "eve/evals/expect"; * a user decision, not an error and not a session end. */ export default defineEval({ + tags: ["real-model"], description: "Session token limit parks on a continuation prompt; approve resumes, stop cancels.", async test(t) { // The 1-token budget lets this first call finish (limits are checked diff --git a/e2e/fixtures/agent-session-limits/package.json b/e2e/fixtures/agent-session-limits/package.json index 0ad51e2e9..7acfc567b 100644 --- a/e2e/fixtures/agent-session-limits/package.json +++ b/e2e/fixtures/agent-session-limits/package.json @@ -11,9 +11,11 @@ "test:e2e": "eve eval --strict" }, "dependencies": { + "@eve-e2e/config": "workspace:*", "@opentelemetry/core": "catalog:", "@opentelemetry/sdk-trace-base": "catalog:", "@vercel/otel": "catalog:", + "@workflow/world-postgres": "catalog:", "eve": "workspace:*", "zod": "catalog:" }, diff --git a/e2e/fixtures/agent-session-timeout/agent/agent.ts b/e2e/fixtures/agent-session-timeout/agent/agent.ts index 9b2fcae12..e6125391d 100644 --- a/e2e/fixtures/agent-session-timeout/agent/agent.ts +++ b/e2e/fixtures/agent-session-timeout/agent/agent.ts @@ -1,9 +1,13 @@ +import { e2eAgentConfig } from "@eve-e2e/config"; import { defineAgent } from "eve"; import { mockModel } from "eve/evals"; const ACTIVE_TURN_DELAY_MS = 1_500; export default defineAgent({ + // Harness config wires the workflow world; the model is always this + // fixture's scripted mock. + ...e2eAgentConfig(), model: mockModel(async ({ lastUserMessage }) => { if (lastUserMessage?.includes("SLOW-TURN") === true) { await new Promise((resolve) => setTimeout(resolve, ACTIVE_TURN_DELAY_MS)); diff --git a/e2e/fixtures/agent-session-timeout/package.json b/e2e/fixtures/agent-session-timeout/package.json index 2b033a4cc..61c6382a4 100644 --- a/e2e/fixtures/agent-session-timeout/package.json +++ b/e2e/fixtures/agent-session-timeout/package.json @@ -11,9 +11,11 @@ "test:e2e": "eve eval --strict" }, "dependencies": { + "@eve-e2e/config": "workspace:*", "@opentelemetry/core": "catalog:", "@opentelemetry/sdk-trace-base": "catalog:", "@vercel/otel": "catalog:", + "@workflow/world-postgres": "catalog:", "ai": "catalog:", "eve": "workspace:*" }, diff --git a/e2e/fixtures/agent-skills/agent/agent.ts b/e2e/fixtures/agent-skills/agent/agent.ts index 93907447f..384bb0e2a 100644 --- a/e2e/fixtures/agent-skills/agent/agent.ts +++ b/e2e/fixtures/agent-skills/agent/agent.ts @@ -1,6 +1,7 @@ +import { e2eAgentConfig } from "@eve-e2e/config"; import { defineAgent } from "eve"; export default defineAgent({ - model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol", + ...e2eAgentConfig(), reasoning: "high", }); diff --git a/e2e/fixtures/agent-skills/evals/evals.config.ts b/e2e/fixtures/agent-skills/evals/evals.config.ts index e5a4265ca..15fbd91d9 100644 --- a/e2e/fixtures/agent-skills/evals/evals.config.ts +++ b/e2e/fixtures/agent-skills/evals/evals.config.ts @@ -1,6 +1,7 @@ +import { e2eJudgeModel } from "@eve-e2e/config"; import { defineEvalConfig } from "eve/evals"; /** Default judge model for any `t.judge.*` assertion in this fixture. */ export default defineEvalConfig({ - judge: { model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol" }, + judge: { model: e2eJudgeModel() }, }); diff --git a/e2e/fixtures/agent-skills/evals/skills/auto-invocation.eval.ts b/e2e/fixtures/agent-skills/evals/skills/auto-invocation.eval.ts index 2d23d3762..e06a1de11 100644 --- a/e2e/fixtures/agent-skills/evals/skills/auto-invocation.eval.ts +++ b/e2e/fixtures/agent-skills/evals/skills/auto-invocation.eval.ts @@ -9,6 +9,7 @@ const ECHO_MARKER_TOKEN = "skill-echo-marker-ok-V8Y2"; * the reply: the skill instructs an exact-token response. */ export default defineEval({ + tags: ["real-model"], description: "Skills smoke: markdown skill auto-invocation via load_skill.", async test(t) { await t.send("Please use the echo marker skill and follow its instructions exactly."); diff --git a/e2e/fixtures/agent-skills/evals/skills/dynamic-instructions.eval.ts b/e2e/fixtures/agent-skills/evals/skills/dynamic-instructions.eval.ts index 2288f93f4..f2cb3fa51 100644 --- a/e2e/fixtures/agent-skills/evals/skills/dynamic-instructions.eval.ts +++ b/e2e/fixtures/agent-skills/evals/skills/dynamic-instructions.eval.ts @@ -9,6 +9,7 @@ const DYNAMIC_INSTRUCTIONS_TOKEN = "dynamic-instructions-ok-M3K8"; * reply honors its exact-token directive, proving delivery. */ export default defineEval({ + tags: ["real-model"], description: "Skills smoke: dynamic instructions injection at session start.", async test(t) { await t.send("Acknowledge this message."); diff --git a/e2e/fixtures/agent-skills/evals/skills/dynamic-skill-map.eval.ts b/e2e/fixtures/agent-skills/evals/skills/dynamic-skill-map.eval.ts index 0139dbda1..18446fa02 100644 --- a/e2e/fixtures/agent-skills/evals/skills/dynamic-skill-map.eval.ts +++ b/e2e/fixtures/agent-skills/evals/skills/dynamic-skill-map.eval.ts @@ -9,6 +9,7 @@ const DYNAMIC_MULTI_ALPHA_TOKEN = "dynamic-multi-alpha-Q8V3"; * must land in the load_skill result and the reply. */ export default defineEval({ + tags: ["real-model"], description: "Skills smoke: dynamic skill-map resolution.", async test(t) { await t.send("Please use the dynamic multi alpha skill and follow its instructions exactly."); diff --git a/e2e/fixtures/agent-skills/evals/skills/dynamic-skill.eval.ts b/e2e/fixtures/agent-skills/evals/skills/dynamic-skill.eval.ts index b6b98a3c6..4c427ab90 100644 --- a/e2e/fixtures/agent-skills/evals/skills/dynamic-skill.eval.ts +++ b/e2e/fixtures/agent-skills/evals/skills/dynamic-skill.eval.ts @@ -9,6 +9,7 @@ const DYNAMIC_SKILL_TOKEN = "dynamic-skill-ok-P4K9"; * markdown body, which then shapes the reply. */ export default defineEval({ + tags: ["real-model"], description: "Skills smoke: dynamic single-skill resolution.", async test(t) { await t.send("Please use the dynamic tenant policy skill and follow its instructions exactly."); diff --git a/e2e/fixtures/agent-skills/evals/skills/skill-override.eval.ts b/e2e/fixtures/agent-skills/evals/skills/skill-override.eval.ts index 580475cc7..a3746cf2a 100644 --- a/e2e/fixtures/agent-skills/evals/skills/skill-override.eval.ts +++ b/e2e/fixtures/agent-skills/evals/skills/skill-override.eval.ts @@ -10,6 +10,7 @@ const HOUSE_RULES_OVERRIDE_TOKEN = "house-rules-dynamic-ok-M5T8"; * authored one. */ export default defineEval({ + tags: ["real-model"], description: "Skills smoke: a dynamic skill overrides a same-named authored skill.", async test(t) { await t.send("Please use the house rules skill and follow its instructions exactly."); diff --git a/e2e/fixtures/agent-skills/package.json b/e2e/fixtures/agent-skills/package.json index b561d5b81..776c663a3 100644 --- a/e2e/fixtures/agent-skills/package.json +++ b/e2e/fixtures/agent-skills/package.json @@ -11,9 +11,11 @@ "test:e2e": "eve eval --strict" }, "dependencies": { + "@eve-e2e/config": "workspace:*", "@opentelemetry/core": "catalog:", "@opentelemetry/sdk-trace-base": "catalog:", "@vercel/otel": "catalog:", + "@workflow/world-postgres": "catalog:", "eve": "workspace:*", "zod": "catalog:" }, diff --git a/e2e/fixtures/agent-subagents-hitl/agent/agent.ts b/e2e/fixtures/agent-subagents-hitl/agent/agent.ts index 93907447f..384bb0e2a 100644 --- a/e2e/fixtures/agent-subagents-hitl/agent/agent.ts +++ b/e2e/fixtures/agent-subagents-hitl/agent/agent.ts @@ -1,6 +1,7 @@ +import { e2eAgentConfig } from "@eve-e2e/config"; import { defineAgent } from "eve"; export default defineAgent({ - model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol", + ...e2eAgentConfig(), reasoning: "high", }); diff --git a/e2e/fixtures/agent-subagents-hitl/agent/subagents/stock-price/agent.ts b/e2e/fixtures/agent-subagents-hitl/agent/subagents/stock-price/agent.ts index 749d35016..a281a6122 100644 --- a/e2e/fixtures/agent-subagents-hitl/agent/subagents/stock-price/agent.ts +++ b/e2e/fixtures/agent-subagents-hitl/agent/subagents/stock-price/agent.ts @@ -1,8 +1,9 @@ +import { e2eSubagentConfig } from "@eve-e2e/config"; import { defineAgent } from "eve"; export default defineAgent({ description: 'Look up the current stock price for a given ticker symbol. Pass the ticker symbol you want to look up in the message (e.g. "AAPL", "GOOG", or "TSLA").', - model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol", + ...e2eSubagentConfig(), reasoning: "high", }); diff --git a/e2e/fixtures/agent-subagents-hitl/evals/evals.config.ts b/e2e/fixtures/agent-subagents-hitl/evals/evals.config.ts index e5a4265ca..15fbd91d9 100644 --- a/e2e/fixtures/agent-subagents-hitl/evals/evals.config.ts +++ b/e2e/fixtures/agent-subagents-hitl/evals/evals.config.ts @@ -1,6 +1,7 @@ +import { e2eJudgeModel } from "@eve-e2e/config"; import { defineEvalConfig } from "eve/evals"; /** Default judge model for any `t.judge.*` assertion in this fixture. */ export default defineEvalConfig({ - judge: { model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol" }, + judge: { model: e2eJudgeModel() }, }); diff --git a/e2e/fixtures/agent-subagents-hitl/evals/hitl.eval.ts b/e2e/fixtures/agent-subagents-hitl/evals/hitl.eval.ts index 4c89788d4..236d67805 100644 --- a/e2e/fixtures/agent-subagents-hitl/evals/hitl.eval.ts +++ b/e2e/fixtures/agent-subagents-hitl/evals/hitl.eval.ts @@ -10,6 +10,7 @@ const GOOG_PRICE = "178.92"; * Parking is server-side. */ export default defineEval({ + tags: ["real-model"], description: "Subagent tool approval proxied through the parent session.", async test(t) { diff --git a/e2e/fixtures/agent-subagents-hitl/package.json b/e2e/fixtures/agent-subagents-hitl/package.json index d654ceb16..281703653 100644 --- a/e2e/fixtures/agent-subagents-hitl/package.json +++ b/e2e/fixtures/agent-subagents-hitl/package.json @@ -11,6 +11,8 @@ "test:e2e": "eve eval --strict" }, "dependencies": { + "@eve-e2e/config": "workspace:*", + "@workflow/world-postgres": "catalog:", "eve": "workspace:*", "zod": "catalog:" }, diff --git a/e2e/fixtures/agent-subagents/agent/agent.ts b/e2e/fixtures/agent-subagents/agent/agent.ts index 93907447f..384bb0e2a 100644 --- a/e2e/fixtures/agent-subagents/agent/agent.ts +++ b/e2e/fixtures/agent-subagents/agent/agent.ts @@ -1,6 +1,7 @@ +import { e2eAgentConfig } from "@eve-e2e/config"; import { defineAgent } from "eve"; export default defineAgent({ - model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol", + ...e2eAgentConfig(), reasoning: "high", }); diff --git a/e2e/fixtures/agent-subagents/agent/subagents/echo-marker/agent.ts b/e2e/fixtures/agent-subagents/agent/subagents/echo-marker/agent.ts index 028180a63..73788e6af 100644 --- a/e2e/fixtures/agent-subagents/agent/subagents/echo-marker/agent.ts +++ b/e2e/fixtures/agent-subagents/agent/subagents/echo-marker/agent.ts @@ -1,3 +1,4 @@ +import { e2eSubagentConfig } from "@eve-e2e/config"; import { defineAgent } from "eve"; /** @@ -12,6 +13,6 @@ import { defineAgent } from "eve"; export default defineAgent({ description: "Smoke-test echo subagent. Call this whenever the user mentions the phrase 'echo marker subagent'. Pass any short text as the input; the subagent replies with a fixed marker token.", - model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol", + ...e2eSubagentConfig(), reasoning: "high", }); diff --git a/e2e/fixtures/agent-subagents/agent/subagents/limited-worker/agent.ts b/e2e/fixtures/agent-subagents/agent/subagents/limited-worker/agent.ts index 8e30edce5..b4eb97fe5 100644 --- a/e2e/fixtures/agent-subagents/agent/subagents/limited-worker/agent.ts +++ b/e2e/fixtures/agent-subagents/agent/subagents/limited-worker/agent.ts @@ -1,3 +1,4 @@ +import { e2eSubagentConfig } from "@eve-e2e/config"; import { defineAgent } from "eve"; export default defineAgent({ @@ -6,6 +7,6 @@ export default defineAgent({ limits: { maxInputTokensPerSession: 1, }, - model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol", + ...e2eSubagentConfig(), reasoning: "high", }); diff --git a/e2e/fixtures/agent-subagents/evals/descendant-session-limit.eval.ts b/e2e/fixtures/agent-subagents/evals/descendant-session-limit.eval.ts index 6d2e13176..d6e60be67 100644 --- a/e2e/fixtures/agent-subagents/evals/descendant-session-limit.eval.ts +++ b/e2e/fixtures/agent-subagents/evals/descendant-session-limit.eval.ts @@ -16,6 +16,7 @@ const DELEGATE_PROMPT = [ * route back to the child that minted it. */ export default defineEval({ + tags: ["real-model"], description: "A descendant session-limit prompt reaches the root; continue resumes the child and stop leaves the root session reusable.", timeoutMs: 90_000, diff --git a/e2e/fixtures/agent-subagents/evals/dynamic-workflow.eval.ts b/e2e/fixtures/agent-subagents/evals/dynamic-workflow.eval.ts index bbb4075f7..afad986e0 100644 --- a/e2e/fixtures/agent-subagents/evals/dynamic-workflow.eval.ts +++ b/e2e/fixtures/agent-subagents/evals/dynamic-workflow.eval.ts @@ -17,6 +17,7 @@ function isFanOutProgram(input: unknown): boolean { /** Dynamic Workflow smoke: sandboxed JavaScript fans out durable children. */ export default defineEval({ + tags: ["real-model"], description: "Dynamic Workflow smoke: model-authored JavaScript fans out two local subagent calls and combines their results.", async test(t) { diff --git a/e2e/fixtures/agent-subagents/evals/evals.config.ts b/e2e/fixtures/agent-subagents/evals/evals.config.ts index e5a4265ca..acb8837a7 100644 --- a/e2e/fixtures/agent-subagents/evals/evals.config.ts +++ b/e2e/fixtures/agent-subagents/evals/evals.config.ts @@ -1,6 +1,11 @@ +import { e2eJudgeModel } from "@eve-e2e/config"; import { defineEvalConfig } from "eve/evals"; /** Default judge model for any `t.judge.*` assertion in this fixture. */ export default defineEvalConfig({ - judge: { model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol" }, + judge: { model: e2eJudgeModel() }, + // Deterministic model turns reach nested Workflow waits immediately. Keep + // separate evals from competing with the child workflows they are testing. + maxConcurrency: 1, + timeoutMs: 120_000, }); diff --git a/e2e/fixtures/agent-subagents/evals/local-delegation.eval.ts b/e2e/fixtures/agent-subagents/evals/local-delegation.eval.ts index 12ae3629f..5823dff40 100644 --- a/e2e/fixtures/agent-subagents/evals/local-delegation.eval.ts +++ b/e2e/fixtures/agent-subagents/evals/local-delegation.eval.ts @@ -8,6 +8,7 @@ const SUBAGENT_TOKEN = "SUBAGENT_TOKEN=echo-marker-9F2X"; * message proves the child's output was spliced back into the conversation. */ export default defineEval({ + tags: ["real-model"], description: "Local subagent delegation smoke: child output reaches the parent reply verbatim.", async test(t) { await t.send( diff --git a/e2e/fixtures/agent-subagents/evals/recursive-root-only.eval.ts b/e2e/fixtures/agent-subagents/evals/recursive-root-only.eval.ts index 7d671b2f2..e1bbc5b2d 100644 --- a/e2e/fixtures/agent-subagents/evals/recursive-root-only.eval.ts +++ b/e2e/fixtures/agent-subagents/evals/recursive-root-only.eval.ts @@ -4,6 +4,7 @@ const CHILD_TOKEN = "RECURSIVE_AGENT_NOT_AVAILABLE"; /** Runtime copies do not receive the root-only built-in `agent` tool. */ export default defineEval({ + tags: ["real-model"], description: "The built-in recursive agent tool is exposed only to the root session.", async test(t) { await t.send( diff --git a/e2e/fixtures/agent-subagents/evals/remote-principal-forwarding.eval.ts b/e2e/fixtures/agent-subagents/evals/remote-principal-forwarding.eval.ts index 0b100e469..1bfac3979 100644 --- a/e2e/fixtures/agent-subagents/evals/remote-principal-forwarding.eval.ts +++ b/e2e/fixtures/agent-subagents/evals/remote-principal-forwarding.eval.ts @@ -22,6 +22,7 @@ const FORWARDED_MARKER = "WHOAMI current=user:e2e-user-2 initiator=user:e2e-user forwarded-by=router-app"; export default defineEval({ + tags: ["real-model"], description: "Remote-agent principal forwarding: the child session runs as the parent's end user, with the distinct initiator preserved and the forwarder stamped.", async test(t) { diff --git a/e2e/fixtures/agent-subagents/evals/runtime-subagent-workflow.eval.ts b/e2e/fixtures/agent-subagents/evals/runtime-subagent-workflow.eval.ts index d726d34c3..e55876e6e 100644 --- a/e2e/fixtures/agent-subagents/evals/runtime-subagent-workflow.eval.ts +++ b/e2e/fixtures/agent-subagents/evals/runtime-subagent-workflow.eval.ts @@ -7,6 +7,7 @@ const CHILD_TOKEN = "CHILD_WORKFLOW_TOOL_NOT_AVAILABLE"; * `agent` child, which should not receive the root-only `Workflow` wrapper. */ export default defineEval({ + tags: ["real-model"], description: "Runtime subagent sessions do not receive the root-only Workflow tool.", async test(t) { await t.send( diff --git a/e2e/fixtures/agent-subagents/evals/workflow-subagent-limit.eval.ts b/e2e/fixtures/agent-subagents/evals/workflow-subagent-limit.eval.ts index 1689f1332..23188a8aa 100644 --- a/e2e/fixtures/agent-subagents/evals/workflow-subagent-limit.eval.ts +++ b/e2e/fixtures/agent-subagents/evals/workflow-subagent-limit.eval.ts @@ -6,6 +6,7 @@ import { defineEval } from "eve/evals"; * resolves inside the program with a `WORKFLOW_SUBAGENT_LIMIT_REACHED` error. */ export default defineEval({ + tags: ["real-model"], description: "Workflow calls beyond the tool's maxSubagents config resolve with WORKFLOW_SUBAGENT_LIMIT_REACHED instead of spawning children.", async test(t) { diff --git a/e2e/fixtures/agent-subagents/package.json b/e2e/fixtures/agent-subagents/package.json index 1424d6b5c..927e0e7a6 100644 --- a/e2e/fixtures/agent-subagents/package.json +++ b/e2e/fixtures/agent-subagents/package.json @@ -11,9 +11,11 @@ "test:e2e": "eve eval --strict" }, "dependencies": { + "@eve-e2e/config": "workspace:*", "@opentelemetry/core": "catalog:", "@opentelemetry/sdk-trace-base": "catalog:", "@vercel/otel": "catalog:", + "@workflow/world-postgres": "catalog:", "eve": "workspace:*", "zod": "catalog:" }, diff --git a/e2e/fixtures/agent-tools-hitl-openai/agent/agent.ts b/e2e/fixtures/agent-tools-hitl-openai/agent/agent.ts index 006ed0c4d..3b81e2012 100644 --- a/e2e/fixtures/agent-tools-hitl-openai/agent/agent.ts +++ b/e2e/fixtures/agent-tools-hitl-openai/agent/agent.ts @@ -1,3 +1,4 @@ +import { e2eAgentConfig } from "@eve-e2e/config"; import { defineAgent } from "eve"; /** @@ -7,5 +8,5 @@ import { defineAgent } from "eve"; * validated by OpenAI's `function_call` / `function_call_output` pairing. */ export default defineAgent({ - model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol", + ...e2eAgentConfig(), }); diff --git a/e2e/fixtures/agent-tools-hitl-openai/evals/evals.config.ts b/e2e/fixtures/agent-tools-hitl-openai/evals/evals.config.ts index e5a4265ca..15fbd91d9 100644 --- a/e2e/fixtures/agent-tools-hitl-openai/evals/evals.config.ts +++ b/e2e/fixtures/agent-tools-hitl-openai/evals/evals.config.ts @@ -1,6 +1,7 @@ +import { e2eJudgeModel } from "@eve-e2e/config"; import { defineEvalConfig } from "eve/evals"; /** Default judge model for any `t.judge.*` assertion in this fixture. */ export default defineEvalConfig({ - judge: { model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol" }, + judge: { model: e2eJudgeModel() }, }); diff --git a/e2e/fixtures/agent-tools-hitl-openai/evals/hitl/approve-resume.eval.ts b/e2e/fixtures/agent-tools-hitl-openai/evals/hitl/approve-resume.eval.ts index 350200a2d..19838cb88 100644 --- a/e2e/fixtures/agent-tools-hitl-openai/evals/hitl/approve-resume.eval.ts +++ b/e2e/fixtures/agent-tools-hitl-openai/evals/hitl/approve-resume.eval.ts @@ -13,6 +13,7 @@ const GUARDED_ECHO_OPENAI_TOKEN = "guarded-echo-openai-ok-R2D7"; * `No tool output found for function call call_`. */ export default defineEval({ + tags: ["real-model"], description: "HITL regression (#236): approve-resume executes and replays on OpenAI Responses.", async test(t) { const parked = await t.send('Call the guarded-echo tool with note "openai-approve".'); diff --git a/e2e/fixtures/agent-tools-hitl-openai/package.json b/e2e/fixtures/agent-tools-hitl-openai/package.json index 9d0ece5a1..cc6b941e9 100644 --- a/e2e/fixtures/agent-tools-hitl-openai/package.json +++ b/e2e/fixtures/agent-tools-hitl-openai/package.json @@ -11,9 +11,11 @@ "test:e2e": "eve eval --strict" }, "dependencies": { + "@eve-e2e/config": "workspace:*", "@opentelemetry/core": "catalog:", "@opentelemetry/sdk-trace-base": "catalog:", "@vercel/otel": "catalog:", + "@workflow/world-postgres": "catalog:", "eve": "workspace:*", "zod": "catalog:" }, diff --git a/e2e/fixtures/agent-tools-hitl/agent/agent.ts b/e2e/fixtures/agent-tools-hitl/agent/agent.ts index 93907447f..384bb0e2a 100644 --- a/e2e/fixtures/agent-tools-hitl/agent/agent.ts +++ b/e2e/fixtures/agent-tools-hitl/agent/agent.ts @@ -1,6 +1,7 @@ +import { e2eAgentConfig } from "@eve-e2e/config"; import { defineAgent } from "eve"; export default defineAgent({ - model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol", + ...e2eAgentConfig(), reasoning: "high", }); diff --git a/e2e/fixtures/agent-tools-hitl/evals/evals.config.ts b/e2e/fixtures/agent-tools-hitl/evals/evals.config.ts index e5a4265ca..15fbd91d9 100644 --- a/e2e/fixtures/agent-tools-hitl/evals/evals.config.ts +++ b/e2e/fixtures/agent-tools-hitl/evals/evals.config.ts @@ -1,6 +1,7 @@ +import { e2eJudgeModel } from "@eve-e2e/config"; import { defineEvalConfig } from "eve/evals"; /** Default judge model for any `t.judge.*` assertion in this fixture. */ export default defineEvalConfig({ - judge: { model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol" }, + judge: { model: e2eJudgeModel() }, }); diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/approve-then-no-regate.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/approve-then-no-regate.eval.ts index 02b55aea0..bde31e6be 100644 --- a/e2e/fixtures/agent-tools-hitl/evals/hitl/approve-then-no-regate.eval.ts +++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/approve-then-no-regate.eval.ts @@ -8,6 +8,7 @@ import { GUARDED_ECHO_TOKEN } from "./shared"; * park/resume here is deterministic. */ export default defineEval({ + tags: ["real-model"], description: "HITL smoke: an approved once() grant persists for the session.", async test(t) { const parked = await t.send('Call the guarded-echo tool with note "first-call".'); diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/ask-question-select.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/ask-question-select.eval.ts index fc45c3dde..248659428 100644 --- a/e2e/fixtures/agent-tools-hitl/evals/hitl/ask-question-select.eval.ts +++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/ask-question-select.eval.ts @@ -6,6 +6,7 @@ import { defineEval } from "eve/evals"; * here is deterministic. */ export default defineEval({ + tags: ["real-model"], description: "HITL smoke: ask-question select parks and resumes with the chosen option.", async test(t) { await t.send( diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/authored-always-approve-resume.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/authored-always-approve-resume.eval.ts index 5de7b5f44..31acbfa2c 100644 --- a/e2e/fixtures/agent-tools-hitl/evals/hitl/authored-always-approve-resume.eval.ts +++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/authored-always-approve-resume.eval.ts @@ -5,6 +5,7 @@ const TOOL_NAME = "gate"; /** Regression reproduction for https://github.com/vercel/eve/issues/533. */ export default defineEval({ + tags: ["real-model"], description: "HITL repro (#533): a separate approval response executes an authored always-gated tool.", async test(t) { diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/authored-always-unrelated-input.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/authored-always-unrelated-input.eval.ts index 4887add5f..696955dda 100644 --- a/e2e/fixtures/agent-tools-hitl/evals/hitl/authored-always-unrelated-input.eval.ts +++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/authored-always-unrelated-input.eval.ts @@ -5,6 +5,7 @@ const TOOL_NAME = "gate"; /** Regression reproduction for https://github.com/vercel/eve/issues/533. */ export default defineEval({ + tags: ["real-model"], description: "HITL repro (#533): unrelated input does not replay an unresolved authored tool call.", async test(t) { diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/deny-then-regate.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/deny-then-regate.eval.ts index 0a01180d5..9db60b964 100644 --- a/e2e/fixtures/agent-tools-hitl/evals/hitl/deny-then-regate.eval.ts +++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/deny-then-regate.eval.ts @@ -6,6 +6,7 @@ import { defineEval } from "eve/evals"; * park/resume here is deterministic. */ export default defineEval({ + tags: ["real-model"], description: "HITL smoke: a denied once() call does not execute and re-gates the next call.", async test(t) { await t.send('Call the guarded-echo tool with note "denied-call".'); diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/dynamic-approve-then-followup.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/dynamic-approve-then-followup.eval.ts index 95ece2901..70b16faf0 100644 --- a/e2e/fixtures/agent-tools-hitl/evals/hitl/dynamic-approve-then-followup.eval.ts +++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/dynamic-approve-then-followup.eval.ts @@ -14,6 +14,7 @@ const TOOL_NAME = "dynamic_guarded_echo"; * 400 lands, turning `session.waiting` into a terminal `session.failed`. */ export default defineEval({ + tags: ["real-model"], description: "HITL regression (#533): a resolved approval park replays on the next turn.", async test(t) { const parked = await t.send(`Call the \`${TOOL_NAME}\` tool with note "replay-probe".`); diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/dynamic-guarded-replay.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/dynamic-guarded-replay.eval.ts index fa3f2c3ca..46a5c5471 100644 --- a/e2e/fixtures/agent-tools-hitl/evals/hitl/dynamic-guarded-replay.eval.ts +++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/dynamic-guarded-replay.eval.ts @@ -9,6 +9,7 @@ const TOOL_NAME = "dynamic_guarded_echo"; * this eval fails before approval. */ export default defineEval({ + tags: ["real-model"], description: "HITL smoke: replayed dynamic tools preserve approval.", async test(t) { const parked = await t.send(`Call the \`${TOOL_NAME}\` tool with note "before-approval".`); diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/slow-once-approve-replay.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/slow-once-approve-replay.eval.ts index 189e12afd..787bb86b0 100644 --- a/e2e/fixtures/agent-tools-hitl/evals/hitl/slow-once-approve-replay.eval.ts +++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/slow-once-approve-replay.eval.ts @@ -15,6 +15,7 @@ const TOOL_NAME = "guarded-slow-echo"; * without a `tool_result` and the provider rejects every later turn. */ export default defineEval({ + tags: ["real-model"], description: "HITL regression (#460): approved slow tool result survives replay.", async test(t) { const parked = await t.send( diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/stale-ask-question-selection-idle.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/stale-ask-question-selection-idle.eval.ts index 3f0fa2094..16fc7b947 100644 --- a/e2e/fixtures/agent-tools-hitl/evals/hitl/stale-ask-question-selection-idle.eval.ts +++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/stale-ask-question-selection-idle.eval.ts @@ -7,6 +7,7 @@ import { defineEval } from "eve/evals"; * the request. */ export default defineEval({ + tags: ["real-model"], description: "HITL smoke: a stale ask-question selection becomes a new user turn when nothing is pending.", async test(t) { diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/stale-ask-question-selection.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/stale-ask-question-selection.eval.ts index 56185cb3c..78d1372a5 100644 --- a/e2e/fixtures/agent-tools-hitl/evals/hitl/stale-ask-question-selection.eval.ts +++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/stale-ask-question-selection.eval.ts @@ -7,6 +7,7 @@ import { defineEval } from "eve/evals"; * old selection is still relevant. */ export default defineEval({ + tags: ["real-model"], description: "HITL smoke: a stale ask-question selection becomes a new user turn while another question is pending.", async test(t) { diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/text-approve.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/text-approve.eval.ts index 5ad2ff85c..ad8bc008d 100644 --- a/e2e/fixtures/agent-tools-hitl/evals/hitl/text-approve.eval.ts +++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/text-approve.eval.ts @@ -7,6 +7,7 @@ import { GUARDED_ECHO_TOKEN } from "./shared"; * resolves the pending approval the same way as structured inputResponses. */ export default defineEval({ + tags: ["real-model"], description: "HITL smoke: text approve resolves a pending tool approval.", async test(t) { const parked = await t.send('Call the guarded-echo tool with note "text-approve".'); diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/unrelated-message-queued.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/unrelated-message-queued.eval.ts index 2325bd472..74ff12b74 100644 --- a/e2e/fixtures/agent-tools-hitl/evals/hitl/unrelated-message-queued.eval.ts +++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/unrelated-message-queued.eval.ts @@ -8,6 +8,7 @@ import { GUARDED_ECHO_TOKEN } from "./shared"; * receives an explicit terminal answer. */ export default defineEval({ + tags: ["real-model"], description: "HITL smoke: unrelated message during approval is queued.", async test(t) { const parked = await t.send('Call the guarded-echo tool with note "queued-approval".'); diff --git a/e2e/fixtures/agent-tools-hitl/package.json b/e2e/fixtures/agent-tools-hitl/package.json index 20fc21652..53f095904 100644 --- a/e2e/fixtures/agent-tools-hitl/package.json +++ b/e2e/fixtures/agent-tools-hitl/package.json @@ -11,9 +11,11 @@ "test:e2e": "eve eval --strict" }, "dependencies": { + "@eve-e2e/config": "workspace:*", "@opentelemetry/core": "catalog:", "@opentelemetry/sdk-trace-base": "catalog:", "@vercel/otel": "catalog:", + "@workflow/world-postgres": "catalog:", "eve": "workspace:*", "zod": "catalog:" }, diff --git a/e2e/fixtures/agent-tools-sandbox/agent/agent.ts b/e2e/fixtures/agent-tools-sandbox/agent/agent.ts index 93907447f..d3292ac9e 100644 --- a/e2e/fixtures/agent-tools-sandbox/agent/agent.ts +++ b/e2e/fixtures/agent-tools-sandbox/agent/agent.ts @@ -1,6 +1,68 @@ +import { e2eAgentConfig } from "@eve-e2e/config"; import { defineAgent } from "eve"; +import type { MockModelRequest, MockModelResponse } from "eve/evals"; + +const BASH_DIRECTIVE = /run the bash command `([^`]+)`/iu; +const SKILL_DIRECTIVE = /load the `([^`]+)` skill/iu; + +/** + * Scripted mock for the world suites: sandbox evals phrase every prompt as an + * explicit directive, so the responder executes exactly the requested tool + * and replies from its output. Turn state derives from the prompt: a tool + * message after the latest user message means this turn's call already ran. + */ +function respond(request: MockModelRequest): MockModelResponse | string { + const message = request.lastUserMessage ?? ""; + const roles = request.messages.map((entry) => entry.role); + const turnHasToolResult = roles.lastIndexOf("tool") > roles.lastIndexOf("user"); + + const bash = BASH_DIRECTIVE.exec(message); + if (bash?.[1] !== undefined) { + if (!turnHasToolResult) { + return { toolCalls: [{ input: { command: bash[1] }, name: "bash" }] }; + } + if (/reply with the single word:\s*done/iu.test(message)) { + return "done"; + } + return bashStdout(request); + } + + const skill = SKILL_DIRECTIVE.exec(message); + if (skill?.[1] !== undefined) { + if (!turnHasToolResult) { + return { toolCalls: [{ input: { skill: skill[1] }, name: "load_skill" }] }; + } + // Loaded skills instruct an exact reply whose text is the skill body's + // final line (see the redeploy eval's deploy-note skill). + return lastNonEmptyLine(formatOutput(request.toolResults.at(-1)?.output)); + } + + return `Mock reply: ${message}`; +} + +function bashStdout(request: MockModelRequest): string { + const output = [...request.toolResults] + .reverse() + .find((result) => result.name === "bash")?.output; + if (typeof output === "object" && output !== null && "stdout" in output) { + return String((output as { stdout: unknown }).stdout).trim(); + } + return formatOutput(output); +} + +function formatOutput(output: unknown): string { + return typeof output === "string" ? output : JSON.stringify(output ?? ""); +} + +function lastNonEmptyLine(text: string): string { + const lines = text + .split("\n") + .map((line) => line.trim()) + .filter((line) => line.length > 0); + return lines.at(-1) ?? text; +} export default defineAgent({ - model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol", + ...e2eAgentConfig({ mock: respond }), reasoning: "high", }); diff --git a/e2e/fixtures/agent-tools-sandbox/evals/evals.config.ts b/e2e/fixtures/agent-tools-sandbox/evals/evals.config.ts index e5a4265ca..15fbd91d9 100644 --- a/e2e/fixtures/agent-tools-sandbox/evals/evals.config.ts +++ b/e2e/fixtures/agent-tools-sandbox/evals/evals.config.ts @@ -1,6 +1,7 @@ +import { e2eJudgeModel } from "@eve-e2e/config"; import { defineEvalConfig } from "eve/evals"; /** Default judge model for any `t.judge.*` assertion in this fixture. */ export default defineEvalConfig({ - judge: { model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol" }, + judge: { model: e2eJudgeModel() }, }); diff --git a/e2e/fixtures/agent-tools-sandbox/evals/sandbox/bash-curl-fanout.eval.ts b/e2e/fixtures/agent-tools-sandbox/evals/sandbox/bash-curl-fanout.eval.ts index 33ff985d5..f87f117e0 100644 --- a/e2e/fixtures/agent-tools-sandbox/evals/sandbox/bash-curl-fanout.eval.ts +++ b/e2e/fixtures/agent-tools-sandbox/evals/sandbox/bash-curl-fanout.eval.ts @@ -25,6 +25,7 @@ interface CurlBarrierResult { } export default defineEval({ + tags: ["real-model"], description: "Sandbox Bash: at least ten curls reach a concurrency barrier.", async test(t) { const turn = await t.send( diff --git a/e2e/fixtures/agent-tools-sandbox/evals/sandbox/bootstrap.eval.ts b/e2e/fixtures/agent-tools-sandbox/evals/sandbox/bootstrap.eval.ts index e1f7956fb..54fd503ce 100644 --- a/e2e/fixtures/agent-tools-sandbox/evals/sandbox/bootstrap.eval.ts +++ b/e2e/fixtures/agent-tools-sandbox/evals/sandbox/bootstrap.eval.ts @@ -6,6 +6,7 @@ import { BOOTSTRAP_MARKER_PATH, BOOTSTRAP_MARKER_TOKEN } from "./shared"; // non-error result containing the marker token proves the bootstrap-written // file is visible inside the sandbox. export default defineEval({ + tags: ["real-model"], description: "Sandbox smoke: `defineSandbox({ bootstrap })` runs before the first bash call.", async test(t) { await t.send( diff --git a/e2e/fixtures/agent-tools-sandbox/evals/sandbox/cli-install.eval.ts b/e2e/fixtures/agent-tools-sandbox/evals/sandbox/cli-install.eval.ts index c1c88b27c..4323c0b89 100644 --- a/e2e/fixtures/agent-tools-sandbox/evals/sandbox/cli-install.eval.ts +++ b/e2e/fixtures/agent-tools-sandbox/evals/sandbox/cli-install.eval.ts @@ -7,6 +7,7 @@ import { SANDBOX_CLI_NAME, SANDBOX_CLI_TOKEN } from "./shared"; // tooling is both on the PATH and executable in later sessions, and that the // base image's Python runtime ran the bootstrap-authored script. export default defineEval({ + tags: ["real-model"], description: "Sandbox: a custom CLI installed in `bootstrap` is on the PATH for later sessions.", async test(t) { await t.send( diff --git a/e2e/fixtures/agent-tools-sandbox/evals/sandbox/file-tools.eval.ts b/e2e/fixtures/agent-tools-sandbox/evals/sandbox/file-tools.eval.ts index ee5ece0b0..8d8080c0f 100644 --- a/e2e/fixtures/agent-tools-sandbox/evals/sandbox/file-tools.eval.ts +++ b/e2e/fixtures/agent-tools-sandbox/evals/sandbox/file-tools.eval.ts @@ -8,6 +8,7 @@ const FILE_TOOLS_TOKEN = "sandbox-file-tools-ok-Q2H"; const FILE_TOOLS_PATH = "/workspace/file-tools-note.txt"; export default defineEval({ + tags: ["real-model"], description: "Sandbox: built-in write_file/grep tools operate on the sandbox filesystem.", async test(t) { await t.send( diff --git a/e2e/fixtures/agent-tools-sandbox/evals/sandbox/network-policy.eval.ts b/e2e/fixtures/agent-tools-sandbox/evals/sandbox/network-policy.eval.ts index 5527bb50e..66ccb58ac 100644 --- a/e2e/fixtures/agent-tools-sandbox/evals/sandbox/network-policy.eval.ts +++ b/e2e/fixtures/agent-tools-sandbox/evals/sandbox/network-policy.eval.ts @@ -21,6 +21,7 @@ function isBlocked(value: unknown): boolean { // then attempts egress. Asserting the probe came back blocked proves // `setNetworkPolicy("deny-all")` actually severs egress on the active backend. export default defineEval({ + tags: ["real-model"], description: "Sandbox: setNetworkPolicy('deny-all') blocks sandbox egress mid-turn.", async test(t) { await t.send( diff --git a/e2e/fixtures/agent-tools-sandbox/evals/sandbox/python-tool.eval.ts b/e2e/fixtures/agent-tools-sandbox/evals/sandbox/python-tool.eval.ts index 1baaef5d2..79c5e1485 100644 --- a/e2e/fixtures/agent-tools-sandbox/evals/sandbox/python-tool.eval.ts +++ b/e2e/fixtures/agent-tools-sandbox/evals/sandbox/python-tool.eval.ts @@ -5,6 +5,7 @@ import { defineEval } from "eve/evals"; // A correct sum proves the full authored-sandbox path works end-to-end with a // real Python interpreter, not a simulated shell. export default defineEval({ + tags: ["real-model"], description: "Sandbox: an authored tool runs real Python via ctx.getSandbox().", async test(t) { await t.send( diff --git a/e2e/fixtures/agent-tools-sandbox/evals/sandbox/session-persistence.eval.ts b/e2e/fixtures/agent-tools-sandbox/evals/sandbox/session-persistence.eval.ts index 37e9e5c4f..324f9ce17 100644 --- a/e2e/fixtures/agent-tools-sandbox/evals/sandbox/session-persistence.eval.ts +++ b/e2e/fixtures/agent-tools-sandbox/evals/sandbox/session-persistence.eval.ts @@ -9,6 +9,7 @@ const PERSIST_TOKEN = "sandbox-persist-ok-D6L"; const PERSIST_PATH = "/workspace/persist-note.txt"; export default defineEval({ + tags: ["real-model"], description: "Sandbox: workspace filesystem persists across turns in the same session.", async test(t) { const first = await t.send( diff --git a/e2e/fixtures/agent-tools-sandbox/evals/sandbox/session-setup.eval.ts b/e2e/fixtures/agent-tools-sandbox/evals/sandbox/session-setup.eval.ts index 5b5f7fd9f..2ec5ea892 100644 --- a/e2e/fixtures/agent-tools-sandbox/evals/sandbox/session-setup.eval.ts +++ b/e2e/fixtures/agent-tools-sandbox/evals/sandbox/session-setup.eval.ts @@ -13,6 +13,7 @@ import { // A single `cat` of both files proving both tokens appear shows session-scoped // setup and workspace seeding both landed on top of the shared template. export default defineEval({ + tags: ["real-model"], description: "Sandbox: onSession marker and seeded workspace file are both present per session.", async test(t) { await t.send( diff --git a/e2e/fixtures/agent-tools-sandbox/package.json b/e2e/fixtures/agent-tools-sandbox/package.json index ddbc45eda..80a28ed79 100644 --- a/e2e/fixtures/agent-tools-sandbox/package.json +++ b/e2e/fixtures/agent-tools-sandbox/package.json @@ -11,9 +11,11 @@ "test:e2e": "eve eval --strict" }, "dependencies": { + "@eve-e2e/config": "workspace:*", "@opentelemetry/core": "catalog:", "@opentelemetry/sdk-trace-base": "catalog:", "@vercel/otel": "catalog:", + "@workflow/world-postgres": "catalog:", "eve": "workspace:*", "zod": "catalog:" }, diff --git a/e2e/fixtures/agent-tools/agent/agent.ts b/e2e/fixtures/agent-tools/agent/agent.ts index 93907447f..384bb0e2a 100644 --- a/e2e/fixtures/agent-tools/agent/agent.ts +++ b/e2e/fixtures/agent-tools/agent/agent.ts @@ -1,6 +1,7 @@ +import { e2eAgentConfig } from "@eve-e2e/config"; import { defineAgent } from "eve"; export default defineAgent({ - model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol", + ...e2eAgentConfig(), reasoning: "high", }); diff --git a/e2e/fixtures/agent-tools/evals/channel-events/batched-delivery.eval.ts b/e2e/fixtures/agent-tools/evals/channel-events/batched-delivery.eval.ts index c63093e40..2b92bf9f8 100644 --- a/e2e/fixtures/agent-tools/evals/channel-events/batched-delivery.eval.ts +++ b/e2e/fixtures/agent-tools/evals/channel-events/batched-delivery.eval.ts @@ -5,6 +5,7 @@ import { equals } from "eve/evals/expect"; import { coalesceDeliverPayloads } from "../../../../../packages/eve/src/execution/deliver-payloads.js"; export default defineEval({ + tags: ["real-model"], description: "Delivery batching smoke: queued messages and context coalesce in arrival order.", async test(t) { const firstMessage = "Remember synthetic marker BATCH-FIRST."; diff --git a/e2e/fixtures/agent-tools/evals/channel-events/pre-tool-narration.eval.ts b/e2e/fixtures/agent-tools/evals/channel-events/pre-tool-narration.eval.ts index 7998ff4db..20112d6c0 100644 --- a/e2e/fixtures/agent-tools/evals/channel-events/pre-tool-narration.eval.ts +++ b/e2e/fixtures/agent-tools/evals/channel-events/pre-tool-narration.eval.ts @@ -101,6 +101,7 @@ async function postChannel( * before the matching action request and result. */ export default defineEval({ + tags: ["real-model"], description: "Channel event smoke: pre-tool narration is visible when an action is requested.", async test(t) { const token = `channel-narration-${randomBytes(4).toString("hex")}`; diff --git a/e2e/fixtures/agent-tools/evals/channel-metadata/anchored-no-topic.eval.ts b/e2e/fixtures/agent-tools/evals/channel-metadata/anchored-no-topic.eval.ts index 09479bb45..7c2148a54 100644 --- a/e2e/fixtures/agent-tools/evals/channel-metadata/anchored-no-topic.eval.ts +++ b/e2e/fixtures/agent-tools/evals/channel-metadata/anchored-no-topic.eval.ts @@ -8,6 +8,7 @@ import { METADATA_TOOL, PROMPT, startChannelSession } from "./shared"; // returns null and no tool registers: the turn completes without a // dynamic-channel-metadata result and without failures. export default defineEval({ + tags: ["real-model"], description: "Channel metadata smoke: missing topic metadata takes the null-resolve path.", async test(t) { const threadId = `thread-${randomBytes(4).toString("hex")}`; diff --git a/e2e/fixtures/agent-tools/evals/dynamic-tools/authored-override.eval.ts b/e2e/fixtures/agent-tools/evals/dynamic-tools/authored-override.eval.ts index 05b55fda3..a8707eabb 100644 --- a/e2e/fixtures/agent-tools/evals/dynamic-tools/authored-override.eval.ts +++ b/e2e/fixtures/agent-tools/evals/dynamic-tools/authored-override.eval.ts @@ -7,6 +7,7 @@ import { defineEval } from "eve/evals"; const OVERRIDE_TOKEN = "dynamic-override-ok-K2P7"; export default defineEval({ + tags: ["real-model"], description: "Dynamic tools smoke: a dynamic tool overrides a same-named authored tool.", async test(t) { await t.send("Call the `override-target` tool and report the `source` value it returns."); diff --git a/e2e/fixtures/agent-tools/evals/dynamic-tools/conditional.eval.ts b/e2e/fixtures/agent-tools/evals/dynamic-tools/conditional.eval.ts index e03c76bb2..3cd24287e 100644 --- a/e2e/fixtures/agent-tools/evals/dynamic-tools/conditional.eval.ts +++ b/e2e/fixtures/agent-tools/evals/dynamic-tools/conditional.eval.ts @@ -4,6 +4,7 @@ import { defineEval } from "eve/evals"; // truly runs once per session, both turns see { branch: "first" }; a // re-run would surface { branch: "reran" }. export default defineEval({ + tags: ["real-model"], description: "Dynamic tools smoke: the resolver runs once per session, not per turn.", async test(t) { const first = await t.send( diff --git a/e2e/fixtures/agent-tools/evals/dynamic-tools/echo.eval.ts b/e2e/fixtures/agent-tools/evals/dynamic-tools/echo.eval.ts index 8090cd550..ac8157d38 100644 --- a/e2e/fixtures/agent-tools/evals/dynamic-tools/echo.eval.ts +++ b/e2e/fixtures/agent-tools/evals/dynamic-tools/echo.eval.ts @@ -5,6 +5,7 @@ import { DYNAMIC_ECHO_TOKEN, ECHO_TOOL } from "./shared"; // defineDynamic resolves at session.started; the tool is called and // returns the fixture token. export default defineEval({ + tags: ["real-model"], description: "Dynamic tools smoke: resolver registers the echo tool and it returns the token.", async test(t) { await t.send( diff --git a/e2e/fixtures/agent-tools/evals/dynamic-tools/io-once.eval.ts b/e2e/fixtures/agent-tools/evals/dynamic-tools/io-once.eval.ts index 43748af9a..ab4bf6c34 100644 --- a/e2e/fixtures/agent-tools/evals/dynamic-tools/io-once.eval.ts +++ b/e2e/fixtures/agent-tools/evals/dynamic-tools/io-once.eval.ts @@ -3,6 +3,7 @@ import { defineEval } from "eve/evals"; // The resolver's simulated I/O runs once; the durable cache replays its // result on the second turn so ioCallCount stays at 1. export default defineEval({ + tags: ["real-model"], description: "Dynamic tools smoke: resolver I/O runs once and replays from the durable cache.", async test(t) { const first = await t.send( diff --git a/e2e/fixtures/agent-tools/evals/dynamic-tools/loop.eval.ts b/e2e/fixtures/agent-tools/evals/dynamic-tools/loop.eval.ts index a95073d85..15fe1332b 100644 --- a/e2e/fixtures/agent-tools/evals/dynamic-tools/loop.eval.ts +++ b/e2e/fixtures/agent-tools/evals/dynamic-tools/loop.eval.ts @@ -2,6 +2,7 @@ import { defineEval } from "eve/evals"; // Tools generated inside a for loop keep their per-iteration closures. export default defineEval({ + tags: ["real-model"], description: "Dynamic tools smoke: loop-generated tools keep per-iteration closures.", async test(t) { await t.send("Call the `alpha` tool and tell me the name and index it returned."); diff --git a/e2e/fixtures/agent-tools/evals/dynamic-tools/messages.eval.ts b/e2e/fixtures/agent-tools/evals/dynamic-tools/messages.eval.ts index 440ff6b38..d8422b71d 100644 --- a/e2e/fixtures/agent-tools/evals/dynamic-tools/messages.eval.ts +++ b/e2e/fixtures/agent-tools/evals/dynamic-tools/messages.eval.ts @@ -4,6 +4,7 @@ import { satisfies } from "eve/evals/expect"; // The step.started resolver sees the accumulated message history: the // second turn's count must exceed the first. export default defineEval({ + tags: ["real-model"], description: "Dynamic tools smoke: step.started resolver sees accumulated message history.", async test(t) { const first = await t.send( diff --git a/e2e/fixtures/agent-tools/evals/dynamic-tools/model-output.eval.ts b/e2e/fixtures/agent-tools/evals/dynamic-tools/model-output.eval.ts index 1544a6fc0..cb4534497 100644 --- a/e2e/fixtures/agent-tools/evals/dynamic-tools/model-output.eval.ts +++ b/e2e/fixtures/agent-tools/evals/dynamic-tools/model-output.eval.ts @@ -3,6 +3,7 @@ import { defineEval } from "eve/evals"; // The action.result stream event carries the RAW execute output // (including the secret field), not the toModelOutput projection. export default defineEval({ + tags: ["real-model"], description: "Dynamic tools smoke: action.result carries the raw execute output.", async test(t) { await t.send( diff --git a/e2e/fixtures/agent-tools/evals/dynamic-tools/multi-turn.eval.ts b/e2e/fixtures/agent-tools/evals/dynamic-tools/multi-turn.eval.ts index 32c82db01..4ef697308 100644 --- a/e2e/fixtures/agent-tools/evals/dynamic-tools/multi-turn.eval.ts +++ b/e2e/fixtures/agent-tools/evals/dynamic-tools/multi-turn.eval.ts @@ -5,6 +5,7 @@ import { DYNAMIC_ECHO_TOKEN, ECHO_TOOL } from "./shared"; // The dynamic tool must survive serialization/deserialization (lazy // replay of the resolver): both turns call it and see the token. export default defineEval({ + tags: ["real-model"], description: "Dynamic tools smoke: the dynamic tool survives serialization across turns.", async test(t) { const first = await t.send( diff --git a/e2e/fixtures/agent-tools/evals/dynamic-tools/nested.eval.ts b/e2e/fixtures/agent-tools/evals/dynamic-tools/nested.eval.ts index c20256c3d..9ff33572e 100644 --- a/e2e/fixtures/agent-tools/evals/dynamic-tools/nested.eval.ts +++ b/e2e/fixtures/agent-tools/evals/dynamic-tools/nested.eval.ts @@ -3,6 +3,7 @@ import { defineEval } from "eve/evals"; // One resolver returns a helper-built tool (closing over the helper // param and handler vars) and an inline tool; both survive replay. export default defineEval({ + tags: ["real-model"], description: "Dynamic tools smoke: helper-built and inline tools from one resolver survive replay.", async test(t) { diff --git a/e2e/fixtures/agent-tools/evals/dynamic-tools/null.eval.ts b/e2e/fixtures/agent-tools/evals/dynamic-tools/null.eval.ts index 58df218c8..836fdb623 100644 --- a/e2e/fixtures/agent-tools/evals/dynamic-tools/null.eval.ts +++ b/e2e/fixtures/agent-tools/evals/dynamic-tools/null.eval.ts @@ -3,6 +3,7 @@ import { defineEval } from "eve/evals"; // A resolver returning null must register no tools: the turn completes // without any "dynamic-null" call. export default defineEval({ + tags: ["real-model"], description: "Dynamic tools smoke: a null-returning resolver registers no tools.", async test(t) { await t.send( diff --git a/e2e/fixtures/agent-tools/evals/dynamic-tools/overwrite.eval.ts b/e2e/fixtures/agent-tools/evals/dynamic-tools/overwrite.eval.ts index b46e1c14c..ffd94526c 100644 --- a/e2e/fixtures/agent-tools/evals/dynamic-tools/overwrite.eval.ts +++ b/e2e/fixtures/agent-tools/evals/dynamic-tools/overwrite.eval.ts @@ -4,6 +4,7 @@ import { defineEval } from "eve/evals"; // `shared` wins over the session-scoped one, while `session_only` // remains available from session scope. export default defineEval({ + tags: ["real-model"], description: "Dynamic tools smoke: turn-scoped resolver tools win merges over session-scoped.", async test(t) { const first = await t.send("Call the `shared` tool and report the source and turn values."); diff --git a/e2e/fixtures/agent-tools/evals/evals.config.ts b/e2e/fixtures/agent-tools/evals/evals.config.ts index e5a4265ca..15fbd91d9 100644 --- a/e2e/fixtures/agent-tools/evals/evals.config.ts +++ b/e2e/fixtures/agent-tools/evals/evals.config.ts @@ -1,6 +1,7 @@ +import { e2eJudgeModel } from "@eve-e2e/config"; import { defineEvalConfig } from "eve/evals"; /** Default judge model for any `t.judge.*` assertion in this fixture. */ export default defineEvalConfig({ - judge: { model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol" }, + judge: { model: e2eJudgeModel() }, }); diff --git a/e2e/fixtures/agent-tools/evals/static-tools/dynamic-dependency-import.eval.ts b/e2e/fixtures/agent-tools/evals/static-tools/dynamic-dependency-import.eval.ts index 0d138ccdb..e3ce3213f 100644 --- a/e2e/fixtures/agent-tools/evals/static-tools/dynamic-dependency-import.eval.ts +++ b/e2e/fixtures/agent-tools/evals/static-tools/dynamic-dependency-import.eval.ts @@ -1,6 +1,7 @@ import { defineEval } from "eve/evals"; export default defineEval({ + tags: ["real-model"], description: "Static tools smoke: authored modules and dependencies can use dynamic imports.", async test(t) { await t.send( diff --git a/e2e/fixtures/agent-tools/evals/static-tools/fanout-authored.eval.ts b/e2e/fixtures/agent-tools/evals/static-tools/fanout-authored.eval.ts index 8e298be1a..56d672ed5 100644 --- a/e2e/fixtures/agent-tools/evals/static-tools/fanout-authored.eval.ts +++ b/e2e/fixtures/agent-tools/evals/static-tools/fanout-authored.eval.ts @@ -17,6 +17,7 @@ const LABELS = [ ] as const; export default defineEval({ + tags: ["real-model"], description: "Static tools smoke: ten authored tool calls begin concurrently.", async test(t) { const turn = await t.send( diff --git a/e2e/fixtures/agent-tools/evals/static-tools/multi-step-loop.eval.ts b/e2e/fixtures/agent-tools/evals/static-tools/multi-step-loop.eval.ts index acbf951e9..b1d02a96d 100644 --- a/e2e/fixtures/agent-tools/evals/static-tools/multi-step-loop.eval.ts +++ b/e2e/fixtures/agent-tools/evals/static-tools/multi-step-loop.eval.ts @@ -6,6 +6,7 @@ const MULTI_STEP_FINAL_VALUE = "phoenix-rising-9F2X"; // lookup-step-b in order, and the final value flows back into the // user-visible reply. export default defineEval({ + tags: ["real-model"], description: "Static tools smoke: deterministic two-step tool loop.", async test(t) { await t.send( diff --git a/e2e/fixtures/agent-tools/evals/static-tools/streamed-action.eval.ts b/e2e/fixtures/agent-tools/evals/static-tools/streamed-action.eval.ts index 88c231ced..e1231924c 100644 --- a/e2e/fixtures/agent-tools/evals/static-tools/streamed-action.eval.ts +++ b/e2e/fixtures/agent-tools/evals/static-tools/streamed-action.eval.ts @@ -65,6 +65,7 @@ function readExecutionCompletedAt(output: unknown): number | undefined { // consumed. The tool waits before completing, so post-execution batch emission // still cannot satisfy this relation. export default defineEval({ + tags: ["real-model"], description: "Static tools smoke: a local action request streams while execution is pending.", async test(t) { const turn = await t.send( diff --git a/e2e/fixtures/agent-tools/evals/static-tools/structured-echo-narrowing.eval.ts b/e2e/fixtures/agent-tools/evals/static-tools/structured-echo-narrowing.eval.ts index 5938fb97e..47e89f77a 100644 --- a/e2e/fixtures/agent-tools/evals/static-tools/structured-echo-narrowing.eval.ts +++ b/e2e/fixtures/agent-tools/evals/static-tools/structured-echo-narrowing.eval.ts @@ -5,6 +5,7 @@ import { defineEval } from "eve/evals"; // toolResultFrom: a symbol-identity miss throws inside the hook, which // surfaces as turn.failed and trips the succeeded check. export default defineEval({ + tags: ["real-model"], description: "Static tools smoke: structured tool-result narrowing via toolResultFrom.", async test(t) { await t.send( diff --git a/e2e/fixtures/agent-tools/evals/static-tools/throw-recover.eval.ts b/e2e/fixtures/agent-tools/evals/static-tools/throw-recover.eval.ts index 1ffc58e74..26eb7e0b2 100644 --- a/e2e/fixtures/agent-tools/evals/static-tools/throw-recover.eval.ts +++ b/e2e/fixtures/agent-tools/evals/static-tools/throw-recover.eval.ts @@ -3,6 +3,7 @@ import { defineEval } from "eve/evals"; // An authored tool throw surfaces as a failed action result (no turn.failed), // and the session stays responsive for a follow-up. export default defineEval({ + tags: ["real-model"], description: "Static tools smoke: tool throw surfaces as failed and the session recovers.", async test(t) { const first = await t.send( diff --git a/e2e/fixtures/agent-tools/evals/static-tools/to-model-output-content-parts.eval.ts b/e2e/fixtures/agent-tools/evals/static-tools/to-model-output-content-parts.eval.ts index 77f54483c..e24185c07 100644 --- a/e2e/fixtures/agent-tools/evals/static-tools/to-model-output-content-parts.eval.ts +++ b/e2e/fixtures/agent-tools/evals/static-tools/to-model-output-content-parts.eval.ts @@ -9,6 +9,7 @@ const TOOL_NAME = "render-stripes"; // model exclusively through `toModelOutput` content parts. Color recognition // remains tracked rather than gated because live vision quality varies. export default defineEval({ + tags: ["real-model"], description: "Static tools smoke: toModelOutput content parts deliver an image to the model.", async test(t) { await t.send( diff --git a/e2e/fixtures/agent-tools/evals/static-tools/web-fetch.eval.ts b/e2e/fixtures/agent-tools/evals/static-tools/web-fetch.eval.ts index 586702be8..77a3f7faf 100644 --- a/e2e/fixtures/agent-tools/evals/static-tools/web-fetch.eval.ts +++ b/e2e/fixtures/agent-tools/evals/static-tools/web-fetch.eval.ts @@ -5,6 +5,7 @@ const TARGET_URL = "https://example.com/"; const CONTENT_MARKER = "Example Domain"; export default defineEval({ + tags: ["real-model"], description: "Framework tools smoke: web_fetch retrieves an HTTPS page.", async test(t) { const turn = await t.send( diff --git a/e2e/fixtures/agent-tools/evals/static-tools/web-search-narrated-ordering.eval.ts b/e2e/fixtures/agent-tools/evals/static-tools/web-search-narrated-ordering.eval.ts index 0963d47a9..976341dee 100644 --- a/e2e/fixtures/agent-tools/evals/static-tools/web-search-narrated-ordering.eval.ts +++ b/e2e/fixtures/agent-tools/evals/static-tools/web-search-narrated-ordering.eval.ts @@ -2,6 +2,7 @@ import { defineEval } from "eve/evals"; import { narratedWebSearchOrder, WEB_SEARCH_TOOL_NAME } from "./web-search-ordering"; export default defineEval({ + tags: ["real-model"], description: "Provider tools: narrated web searches preserve event order.", async test(t) { const turn = await t.send( diff --git a/e2e/fixtures/agent-tools/evals/static-tools/web-search-parallel.eval.ts b/e2e/fixtures/agent-tools/evals/static-tools/web-search-parallel.eval.ts index e826d8136..d351e8e77 100644 --- a/e2e/fixtures/agent-tools/evals/static-tools/web-search-parallel.eval.ts +++ b/e2e/fixtures/agent-tools/evals/static-tools/web-search-parallel.eval.ts @@ -25,6 +25,7 @@ function completedToolResultCount(events: readonly MessageStreamEvent[], toolNam } export default defineEval({ + tags: ["real-model"], description: "Provider tools smoke: eight parallel gateway web searches complete successfully.", async test(t) { const turn = await t.send( diff --git a/e2e/fixtures/agent-tools/evals/static-tools/web-search-unnarrated-ordering.eval.ts b/e2e/fixtures/agent-tools/evals/static-tools/web-search-unnarrated-ordering.eval.ts index 06f57bedd..ffa646569 100644 --- a/e2e/fixtures/agent-tools/evals/static-tools/web-search-unnarrated-ordering.eval.ts +++ b/e2e/fixtures/agent-tools/evals/static-tools/web-search-unnarrated-ordering.eval.ts @@ -2,6 +2,7 @@ import { defineEval } from "eve/evals"; import { unNarratedWebSearchOrder, WEB_SEARCH_TOOL_NAME } from "./web-search-ordering"; export default defineEval({ + tags: ["real-model"], description: "Provider tools: un-narrated web searches preserve event order.", async test(t) { const turn = await t.send( diff --git a/e2e/fixtures/agent-tools/evals/static-tools/web-search.eval.ts b/e2e/fixtures/agent-tools/evals/static-tools/web-search.eval.ts index ecf4e2189..c70319ee2 100644 --- a/e2e/fixtures/agent-tools/evals/static-tools/web-search.eval.ts +++ b/e2e/fixtures/agent-tools/evals/static-tools/web-search.eval.ts @@ -41,6 +41,7 @@ function providerRequestsPrecedeResults(events: readonly MessageStreamEvent[]): } export default defineEval({ + tags: ["real-model"], description: "Provider tools smoke: gateway web search answers a current-events question.", async test(t) { const turn = await t.send( diff --git a/e2e/fixtures/agent-tools/package.json b/e2e/fixtures/agent-tools/package.json index 339edee71..0036f5f3e 100644 --- a/e2e/fixtures/agent-tools/package.json +++ b/e2e/fixtures/agent-tools/package.json @@ -11,15 +11,20 @@ "test:e2e": "eve eval --strict" }, "dependencies": { + "@eve-e2e/config": "workspace:*", "@eve-e2e/dynamic-import-dependency": "file:./fixtures/dynamic-import-dependency", "@opentelemetry/core": "catalog:", "@opentelemetry/sdk-trace-base": "catalog:", "@vercel/otel": "catalog:", + "@workflow/world-postgres": "catalog:", "eve": "workspace:*", "zod": "catalog:" }, "devDependencies": { "@types/node": "catalog:", "typescript": "catalog:" + }, + "e2e": { + "modelMatrix": "full" } } diff --git a/e2e/fixtures/agent-workflow-stress/agent/agent.ts b/e2e/fixtures/agent-workflow-stress/agent/agent.ts index aee410f61..377b38e64 100644 --- a/e2e/fixtures/agent-workflow-stress/agent/agent.ts +++ b/e2e/fixtures/agent-workflow-stress/agent/agent.ts @@ -1,7 +1,11 @@ +import { e2eAgentConfig } from "@eve-e2e/config"; import { defineAgent } from "eve"; import { mockModel } from "eve/evals"; export default defineAgent({ + // Harness config wires the workflow world; the model is always this + // fixture's scripted mock. + ...e2eAgentConfig(), model: mockModel( ({ lastUserMessage, userMessageCount }) => `stress-ack:${userMessageCount}:${lastUserMessage ?? ""}`, diff --git a/e2e/fixtures/agent-workflow-stress/package.json b/e2e/fixtures/agent-workflow-stress/package.json index 6f795ef69..aa07a981a 100644 --- a/e2e/fixtures/agent-workflow-stress/package.json +++ b/e2e/fixtures/agent-workflow-stress/package.json @@ -11,9 +11,11 @@ "test:e2e": "eve eval --strict" }, "dependencies": { + "@eve-e2e/config": "workspace:*", "@opentelemetry/core": "catalog:", "@opentelemetry/sdk-trace-base": "catalog:", "@vercel/otel": "catalog:", + "@workflow/world-postgres": "catalog:", "ai": "catalog:", "eve": "workspace:*" }, diff --git a/e2e/fixtures/dist-extensions/agent/agent.ts b/e2e/fixtures/dist-extensions/agent/agent.ts index 93907447f..384bb0e2a 100644 --- a/e2e/fixtures/dist-extensions/agent/agent.ts +++ b/e2e/fixtures/dist-extensions/agent/agent.ts @@ -1,6 +1,7 @@ +import { e2eAgentConfig } from "@eve-e2e/config"; import { defineAgent } from "eve"; export default defineAgent({ - model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol", + ...e2eAgentConfig(), reasoning: "high", }); diff --git a/e2e/fixtures/dist-extensions/evals/evals.config.ts b/e2e/fixtures/dist-extensions/evals/evals.config.ts index 3e9bed0f7..7a61a5b55 100644 --- a/e2e/fixtures/dist-extensions/evals/evals.config.ts +++ b/e2e/fixtures/dist-extensions/evals/evals.config.ts @@ -1,5 +1,6 @@ +import { e2eJudgeModel } from "@eve-e2e/config"; import { defineEvalConfig } from "eve/evals"; export default defineEvalConfig({ - judge: { model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol" }, + judge: { model: e2eJudgeModel() }, }); diff --git a/e2e/fixtures/dist-extensions/evals/gadget.eval.ts b/e2e/fixtures/dist-extensions/evals/gadget.eval.ts index b32939f78..567df0f87 100644 --- a/e2e/fixtures/dist-extensions/evals/gadget.eval.ts +++ b/e2e/fixtures/dist-extensions/evals/gadget.eval.ts @@ -1,6 +1,7 @@ import { defineEval } from "eve/evals"; export default defineEval({ + tags: ["real-model"], description: "An agent-shaped dist extension with a registry-style store layout loads and its tool runs.", async test(t) { diff --git a/e2e/fixtures/dist-extensions/evals/gizmo.eval.ts b/e2e/fixtures/dist-extensions/evals/gizmo.eval.ts index d6af70d05..5858782ce 100644 --- a/e2e/fixtures/dist-extensions/evals/gizmo.eval.ts +++ b/e2e/fixtures/dist-extensions/evals/gizmo.eval.ts @@ -1,6 +1,7 @@ import { defineEval } from "eve/evals"; export default defineEval({ + tags: ["real-model"], description: "A dist-only extension installed with a registry-style store layout loads and its tool runs.", async test(t) { diff --git a/e2e/fixtures/dist-extensions/package.json b/e2e/fixtures/dist-extensions/package.json index bfc477429..351cffbc1 100644 --- a/e2e/fixtures/dist-extensions/package.json +++ b/e2e/fixtures/dist-extensions/package.json @@ -10,6 +10,8 @@ "test:e2e": "eve eval --strict" }, "dependencies": { + "@eve-e2e/config": "workspace:*", + "@workflow/world-postgres": "catalog:", "eve": "workspace:*", "gadget-extension": "file:../gadget-extension", "gizmo-extension": "file:../gizmo-extension" diff --git a/e2e/fixtures/e2e-config/package.json b/e2e/fixtures/e2e-config/package.json new file mode 100644 index 000000000..fe666d38e --- /dev/null +++ b/e2e/fixtures/e2e-config/package.json @@ -0,0 +1,17 @@ +{ + "name": "@eve-e2e/config", + "version": "0.0.0", + "private": true, + "type": "module", + "exports": { + ".": "./src/index.ts" + }, + "scripts": { + "typecheck": "tsc --noEmit" + }, + "devDependencies": { + "@types/node": "catalog:", + "eve": "workspace:*", + "typescript": "catalog:" + } +} diff --git a/e2e/fixtures/e2e-config/src/index.ts b/e2e/fixtures/e2e-config/src/index.ts new file mode 100644 index 000000000..32f960bea --- /dev/null +++ b/e2e/fixtures/e2e-config/src/index.ts @@ -0,0 +1,106 @@ +import type { AgentDefinition } from "eve"; +import { mockModel, type MockModelResponder } from "eve/evals"; + +/** + * Sentinel value for `EVE_E2E_MODEL` that makes fixtures author a + * deterministic `mockModel()` instead of a real gateway model. The world + * suites (Vercel, Postgres) set it so infrastructure coverage never depends + * on live model behavior. + */ +export const MOCK_MODEL_SENTINEL = "mock"; + +const DEFAULT_MODEL = "openai/gpt-5.6-sol"; + +// Mock models carry no AI Gateway metadata, so mock-mode configs must state a +// context window explicitly for the compiler. Fixtures that need a specific +// window override it after the spread. +const MOCK_MODEL_CONTEXT_WINDOW_TOKENS = 1_000_000; + +/** Harness-owned agent configuration shared by e2e fixture root agents. */ +export type E2EAgentConfig = Pick< + AgentDefinition, + "experimental" | "model" | "modelContextWindowTokens" +>; + +/** Harness-owned configuration for fixture subagents: model identity only. */ +export type E2ESubagentConfig = Pick; + +/** + * A static model handle: a gateway model id string or a deterministic mock + * instance. Assignable anywhere fixtures author a non-dynamic model + * (agent `model`, compaction `model`, dynamic fallbacks, subagents). + */ +export type E2EModel = string | ReturnType; + +/** Options for {@link e2eAgentConfig} and {@link e2eModel}. */ +export interface E2EModelOptions { + /** + * Responder (or static reply) used when the harness requests mock models + * via `EVE_E2E_MODEL=mock`. Defaults to a deterministic echo of the last + * user message. + */ + readonly mock?: MockModelResponder | string; +} + +/** + * Resolves the model a fixture agent (or subagent) should author: + * `EVE_E2E_MODEL=mock` yields a deterministic `mockModel()`, any other value + * is used as a gateway model id, and the harness default applies when unset. + */ +export function e2eModel(options?: E2EModelOptions): E2EModel { + const requested = process.env.EVE_E2E_MODEL; + + if (requested === MOCK_MODEL_SENTINEL) { + return mockModel(options?.mock ?? defaultMockResponder); + } + + return requested ?? DEFAULT_MODEL; +} + +/** + * Returns the harness-owned configuration shared by e2e fixture root agents: + * the matrix model from `EVE_E2E_MODEL` (or a mock when the world suite + * requests one) and the workflow world override from `EVE_E2E_WORKFLOW_WORLD`. + */ +export function e2eAgentConfig(options?: E2EModelOptions): E2EAgentConfig { + const base = e2eSubagentConfig(options); + const workflowWorld = process.env.EVE_E2E_WORKFLOW_WORLD; + if (workflowWorld === undefined) { + return base; + } + + return { ...base, experimental: { workflow: { world: workflowWorld } } }; +} + +/** + * Returns the harness-owned configuration for fixture subagents: the matrix + * model (or a mock plus its explicit context window in the world suites), + * without root-only settings like the workflow world. + */ +export function e2eSubagentConfig(options?: E2EModelOptions): E2ESubagentConfig { + const model = e2eModel(options); + + if (typeof model === "string") { + return { model }; + } + + return { model, modelContextWindowTokens: MOCK_MODEL_CONTEXT_WINDOW_TOKENS }; +} + +/** + * Resolves the judge model for a fixture's `evals.config.ts`. Judge + * assertions only run in the model suite, so the mock sentinel falls back to + * the harness default rather than leaking `"mock"` as a gateway model id. + */ +export function e2eJudgeModel(): string { + const requested = process.env.EVE_E2E_MODEL; + + if (requested === undefined || requested === MOCK_MODEL_SENTINEL) { + return DEFAULT_MODEL; + } + + return requested; +} + +const defaultMockResponder: MockModelResponder = ({ lastUserMessage }) => + `Mock reply: ${lastUserMessage ?? ""}`; diff --git a/e2e/fixtures/e2e-config/tsconfig.json b/e2e/fixtures/e2e-config/tsconfig.json new file mode 100644 index 000000000..e4c8332fa --- /dev/null +++ b/e2e/fixtures/e2e-config/tsconfig.json @@ -0,0 +1,17 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "esnext", + "moduleResolution": "bundler", + "outDir": "dist", + "rootDir": ".", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "declaration": true, + "noEmit": true, + "types": ["node"] + }, + "include": ["src/**/*.ts"] +} diff --git a/e2e/fixtures/extensions/agent/agent.ts b/e2e/fixtures/extensions/agent/agent.ts index 93907447f..384bb0e2a 100644 --- a/e2e/fixtures/extensions/agent/agent.ts +++ b/e2e/fixtures/extensions/agent/agent.ts @@ -1,6 +1,7 @@ +import { e2eAgentConfig } from "@eve-e2e/config"; import { defineAgent } from "eve"; export default defineAgent({ - model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol", + ...e2eAgentConfig(), reasoning: "high", }); diff --git a/e2e/fixtures/extensions/evals/config-binding.eval.ts b/e2e/fixtures/extensions/evals/config-binding.eval.ts index fd19a4bed..8dc50b303 100644 --- a/e2e/fixtures/extensions/evals/config-binding.eval.ts +++ b/e2e/fixtures/extensions/evals/config-binding.eval.ts @@ -1,6 +1,7 @@ import { defineEval } from "eve/evals"; export default defineEval({ + tags: ["real-model"], description: "Mounted extension tool returns the config bound at the mount site.", async test(t) { await t.send( diff --git a/e2e/fixtures/extensions/evals/dynamic-capability.eval.ts b/e2e/fixtures/extensions/evals/dynamic-capability.eval.ts index 768ea8a0f..1661ee8f1 100644 --- a/e2e/fixtures/extensions/evals/dynamic-capability.eval.ts +++ b/e2e/fixtures/extensions/evals/dynamic-capability.eval.ts @@ -4,6 +4,7 @@ import { defineEval } from "eve/evals"; const TOOLKIT_FORECAST_TOKEN = "toolkit-forecast-ok-9F4Q"; export default defineEval({ + tags: ["real-model"], description: "Dynamic tool authored inside an extension resolves and runs when mounted.", async test(t) { await t.send("Call the `toolkit__toolkit_forecast` tool and report the token it returned."); diff --git a/e2e/fixtures/extensions/evals/dynamic-skill.eval.ts b/e2e/fixtures/extensions/evals/dynamic-skill.eval.ts index b910f9288..177bacde3 100644 --- a/e2e/fixtures/extensions/evals/dynamic-skill.eval.ts +++ b/e2e/fixtures/extensions/evals/dynamic-skill.eval.ts @@ -10,6 +10,7 @@ const TOOLKIT_INCIDENT_TOKEN = "toolkit-incident-dynamic-ok-7T2X"; * dynamic tools. */ export default defineEval({ + tags: ["real-model"], description: "Map-produced dynamic skill from an extension is namespaced (toolkit__incident).", async test(t) { await t.send( diff --git a/e2e/fixtures/extensions/evals/evals.config.ts b/e2e/fixtures/extensions/evals/evals.config.ts index 3e9bed0f7..7a61a5b55 100644 --- a/e2e/fixtures/extensions/evals/evals.config.ts +++ b/e2e/fixtures/extensions/evals/evals.config.ts @@ -1,5 +1,6 @@ +import { e2eJudgeModel } from "@eve-e2e/config"; import { defineEvalConfig } from "eve/evals"; export default defineEvalConfig({ - judge: { model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol" }, + judge: { model: e2eJudgeModel() }, }); diff --git a/e2e/fixtures/extensions/evals/js-only-extension.eval.ts b/e2e/fixtures/extensions/evals/js-only-extension.eval.ts index 96b7beb38..09348f039 100644 --- a/e2e/fixtures/extensions/evals/js-only-extension.eval.ts +++ b/e2e/fixtures/extensions/evals/js-only-extension.eval.ts @@ -3,6 +3,7 @@ import { defineEval } from "eve/evals"; const JS_ONLY_EXTENSION_TOKEN = "js-only-extension-dist-ok-4N7Q"; export default defineEval({ + tags: ["real-model"], description: "A JavaScript-only extension runs from its publisher-built dist tree.", async test(t) { await t.send("Call the `javascript__js_ping` tool and report exactly what it returned."); diff --git a/e2e/fixtures/extensions/evals/mixed.eval.ts b/e2e/fixtures/extensions/evals/mixed.eval.ts index 8a128db37..24711441c 100644 --- a/e2e/fixtures/extensions/evals/mixed.eval.ts +++ b/e2e/fixtures/extensions/evals/mixed.eval.ts @@ -1,6 +1,7 @@ import { defineEval } from "eve/evals"; export default defineEval({ + tags: ["real-model"], description: "Consumer-authored and mounted-extension tools coexist and both run in one turn.", async test(t) { await t.send( diff --git a/e2e/fixtures/extensions/evals/override.eval.ts b/e2e/fixtures/extensions/evals/override.eval.ts index aa40c8f05..33d602a58 100644 --- a/e2e/fixtures/extensions/evals/override.eval.ts +++ b/e2e/fixtures/extensions/evals/override.eval.ts @@ -1,6 +1,7 @@ import { defineEval } from "eve/evals"; export default defineEval({ + tags: ["real-model"], description: "A directory-mount override shadows a mounted extension tool of the same name.", async test(t) { await t.send("Call the `toolkit__toolkit_ping` tool and report exactly what it returned."); diff --git a/e2e/fixtures/extensions/evals/skill-script-resource.eval.ts b/e2e/fixtures/extensions/evals/skill-script-resource.eval.ts index 274c4215e..a33f8434d 100644 --- a/e2e/fixtures/extensions/evals/skill-script-resource.eval.ts +++ b/e2e/fixtures/extensions/evals/skill-script-resource.eval.ts @@ -3,6 +3,7 @@ import { defineEval } from "eve/evals"; const TOOLKIT_SKILL_SCRIPT_TOKEN = "toolkit-skill-script-ok-8M3P"; export default defineEval({ + tags: ["real-model"], description: "A dist-only extension preserves JavaScript files inside packaged skills.", async test(t) { await t.send( diff --git a/e2e/fixtures/extensions/evals/state-isolation.eval.ts b/e2e/fixtures/extensions/evals/state-isolation.eval.ts index 21e40c867..2f83f6b31 100644 --- a/e2e/fixtures/extensions/evals/state-isolation.eval.ts +++ b/e2e/fixtures/extensions/evals/state-isolation.eval.ts @@ -3,6 +3,7 @@ import { defineEval } from "eve/evals"; // Both extensions author defineState("budget"). If package scoping regressed, // the counters would collapse onto one slot and gizmo would read 3, not 1. export default defineEval({ + tags: ["real-model"], description: "Two extensions' identically-named defineState do not collide within a session.", async test(t) { await t.send( diff --git a/e2e/fixtures/extensions/package.json b/e2e/fixtures/extensions/package.json index efc5834e7..072ec4a81 100644 --- a/e2e/fixtures/extensions/package.json +++ b/e2e/fixtures/extensions/package.json @@ -11,6 +11,8 @@ "test:e2e": "eve eval --strict" }, "dependencies": { + "@eve-e2e/config": "workspace:*", + "@workflow/world-postgres": "catalog:", "eve": "workspace:*", "gizmo-extension": "workspace:*", "js-only-extension": "workspace:*", diff --git a/e2e/matrix.json b/e2e/matrix.json new file mode 100644 index 000000000..7c95cbbda --- /dev/null +++ b/e2e/matrix.json @@ -0,0 +1,8 @@ +{ + "$comment": "Registry for the e2e CI matrices. `models` drives the model suite (e2e-local): the first entry is the default every fixture runs on; fixtures with `\"e2e\": { \"modelMatrix\": \"full\" }` in package.json run on all of them. The short model name is the stable Actions check identifier, so updating a model version does not rename required checks. `worlds` drives the world suites: each entry backs one `world_matrix_` output consumed by the matching `e2e-.yml`, and `package` (when set) is exported to the job as EVE_E2E_WORKFLOW_WORLD. See e2e/README.md.", + "models": [ + { "name": "openai-sol", "id": "openai/gpt-5.6-sol" }, + { "name": "anthropic-opus", "id": "anthropic/claude-opus-5" } + ], + "worlds": [{ "name": "vercel" }, { "name": "postgres", "package": "@workflow/world-postgres" }] +} diff --git a/packages/eve/src/cli/run.ts b/packages/eve/src/cli/run.ts index 7d3292930..6a4285270 100644 --- a/packages/eve/src/cli/run.ts +++ b/packages/eve/src/cli/run.ts @@ -22,6 +22,7 @@ import { type DevelopmentRequestHeaders, } from "#cli/dev/url-target.js"; import type { RunDevelopmentTuiInput } from "#cli/dev/tui/tui.js"; +import type { EvalCliOptions } from "#evals/cli/eval.js"; import { registerRuntimeInvokeCommand, type InvokeCliRuntimeDependencies, @@ -128,19 +129,6 @@ function createDevBootProgressReporter( }; } -interface EvalCliOptions { - json?: boolean; - junit?: string; - list?: boolean; - maxConcurrency?: string; - skipReport?: boolean; - strict?: boolean; - tag?: string[]; - timeout?: string; - url?: string; - verbose?: boolean; -} - async function loadPrintApplicationInfo(): Promise { return (await import("#cli/commands/info.js")).printApplicationInfo; } @@ -642,6 +630,7 @@ function createCliProgram(logger: CliLogger, runtime: CliRuntimeOverrides): Comm ) .option("--url ", "Remote agent URL (skip local host startup)", parseDevelopmentServerUrl) .option("--tag ", "Run only evals carrying a tag") + .option("--exclude-tag ", "Skip evals carrying a tag") .option("--strict", "Fail the exit code when any score falls below its threshold") .option("--list", "Print discovered evals without running them") .option("--timeout ", "Per-eval timeout in milliseconds") diff --git a/packages/eve/src/evals/cli/eval.test.ts b/packages/eve/src/evals/cli/eval.test.ts new file mode 100644 index 000000000..81c7a046f --- /dev/null +++ b/packages/eve/src/evals/cli/eval.test.ts @@ -0,0 +1,87 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { runEvalCommand } from "#evals/cli/eval.js"; +import type { EveEval } from "#evals/types.js"; + +vi.mock("#internal/application/paths.js", () => ({ + resolveApplicationRoot: vi.fn(() => "/app"), +})); +vi.mock("#cli/dev/environment.js", () => ({ + loadDevelopmentEnvironmentFiles: vi.fn(), +})); +vi.mock("#evals/runner/discover.js", () => ({ + discoverAndImportEvals: vi.fn(async () => []), + discoverEvalConfig: vi.fn(async () => ({})), + findMisplacedEvalDirs: vi.fn(async () => []), +})); + +const { discoverAndImportEvals } = vi.mocked(await import("#evals/runner/discover.js"), { + partial: true, +}); + +function createEval(id: string, tags?: readonly string[]): EveEval { + return { _tag: "EveEval", id, tags, test: async () => {} }; +} + +function createLogger() { + return { error: vi.fn(), log: vi.fn() }; +} + +describe("runEvalCommand list mode", () => { + beforeEach(() => { + process.exitCode = undefined; + }); + + afterEach(() => { + process.exitCode = undefined; + vi.clearAllMocks(); + }); + + it("lists an empty JSON array when exclusion removes every eval", async () => { + discoverAndImportEvals.mockResolvedValueOnce([createEval("cancel", ["real-model"])]); + const logger = createLogger(); + + await runEvalCommand([], { excludeTag: ["real-model"], json: true, list: true }, logger); + + expect(process.exitCode).toBeUndefined(); + expect(logger.error).not.toHaveBeenCalled(); + expect(logger.log).toHaveBeenCalledTimes(1); + expect(JSON.parse(logger.log.mock.calls[0]![0] as string)).toEqual([]); + }); + + it("lists only the evals that survive exclusion", async () => { + discoverAndImportEvals.mockResolvedValueOnce([ + createEval("judged", ["real-model"]), + createEval("mock-capable"), + ]); + const logger = createLogger(); + + await runEvalCommand([], { excludeTag: ["real-model"], json: true, list: true }, logger); + + expect(process.exitCode).toBeUndefined(); + const listing = JSON.parse(logger.log.mock.calls[0]![0] as string) as { id: string }[]; + expect(listing.map((entry) => entry.id)).toEqual(["mock-capable"]); + }); + + it("still succeeds with the exclusion notice on a non-list run", async () => { + discoverAndImportEvals.mockResolvedValueOnce([createEval("cancel", ["real-model"])]); + const logger = createLogger(); + + await runEvalCommand([], { excludeTag: ["real-model"] }, logger); + + expect(process.exitCode).toBeUndefined(); + expect(logger.log).toHaveBeenCalledWith( + "All 1 matching evals are excluded by tags (real-model); nothing to run.", + ); + }); + + it("keeps exit 2 when an include tag matches nothing, even in list mode", async () => { + discoverAndImportEvals.mockResolvedValueOnce([createEval("mock-capable")]); + const logger = createLogger(); + + await runEvalCommand([], { json: true, list: true, tag: ["nonexistent"] }, logger); + + expect(process.exitCode).toBe(2); + expect(logger.error).toHaveBeenCalledWith("No evals matched the provided tags (nonexistent)."); + }); +}); diff --git a/packages/eve/src/evals/cli/eval.ts b/packages/eve/src/evals/cli/eval.ts index f8ae00869..26a709075 100644 --- a/packages/eve/src/evals/cli/eval.ts +++ b/packages/eve/src/evals/cli/eval.ts @@ -6,6 +6,7 @@ import { shutdownActiveSandboxHandles } from "#execution/sandbox/active-handles. import { resolveApplicationRoot } from "#internal/application/paths.js"; import { createDevelopmentServer, type DevelopmentServer } from "#internal/nitro/host.js"; import { createEvalClient } from "#evals/cli/eval-client.js"; +import { filterEvalsByTags } from "#evals/cli/filter.js"; import { discoverAndImportEvals, discoverEvalConfig, @@ -18,7 +19,8 @@ import type { EvalReporter } from "#evals/runner/reporters/types.js"; import { resolveEvalTargetHandle } from "#evals/target.js"; import type { EveEval, EveEvalTargetHandle } from "#evals/types.js"; -interface EvalCliOptions { +/** Parsed Commander options accepted by {@link runEvalCommand}. */ +export interface EvalCliOptions { url?: string; timeout?: string; maxConcurrency?: string; @@ -28,6 +30,7 @@ interface EvalCliOptions { strict?: boolean; list?: boolean; tag?: string[]; + excludeTag?: string[]; verbose?: boolean; } @@ -38,7 +41,9 @@ type EvalCliLogger = { log(message: string): void; error(message: string): void * * Exit codes: `0` when every executed eval passed its gate assertions (and * soft thresholds under `--strict`), `1` when any eval failed, `2` for runner - * or configuration errors (no evals discovered, no evals matching filters). + * or configuration errors (no evals discovered, no evals matching `--tag`). + * When `--exclude-tag` removes every matching eval the run succeeds with + * nothing executed. */ export async function runEvalCommand( evalIds: readonly string[], @@ -70,13 +75,34 @@ export async function runEvalCommand( return; } - const evaluations = filterEvalsByTag(discovered, options.tag ?? []); - if (evaluations.length === 0) { - logger.error(`No evals matched the provided tags (${(options.tag ?? []).join(", ")}).`); + const includeTags = options.tag ?? []; + const excludeTags = options.excludeTag ?? []; + const included = filterEvalsByTags({ evaluations: discovered, includeTags, excludeTags: [] }); + if (included.length === 0) { + logger.error(`No evals matched the provided tags (${includeTags.join(", ")}).`); process.exitCode = 2; return; } + const evaluations = filterEvalsByTags({ evaluations: included, includeTags: [], excludeTags }); + + // List mode reports the post-exclusion set — even when it is empty — so + // suite runners can probe "does anything run here?" with `--list --json`. + if (options.list === true) { + printEvalList(evaluations, options.json === true, logger); + return; + } + + if (evaluations.length === 0) { + // Every matching eval was excluded. Unlike an include filter with no + // matches, this is a legitimate "nothing applies to this run" outcome + // (e.g. a world suite excluding real-model evals), so it succeeds. + logger.log( + `All ${included.length} matching evals are excluded by tags (${excludeTags.join(", ")}); nothing to run.`, + ); + return; + } + let maxConcurrency: number | undefined; let timeoutMs: number | undefined; try { @@ -88,11 +114,6 @@ export async function runEvalCommand( return; } - if (options.list === true) { - printEvalList(evaluations, options.json === true, logger); - return; - } - let config; try { config = await discoverEvalConfig(appRoot); @@ -201,18 +222,6 @@ function parseNonNegativeInteger(value: string | undefined, flag: string): numbe return parsed; } -// --------------------------------------------------------------------------- -// Eval filtering -// --------------------------------------------------------------------------- - -/** Applies `--tag` filtering: an eval runs when it carries any requested tag. */ -function filterEvalsByTag(evaluations: readonly EveEval[], tags: readonly string[]): EveEval[] { - if (tags.length === 0) return [...evaluations]; - return evaluations.filter( - (evaluation) => evaluation.tags?.some((tag) => tags.includes(tag)) ?? false, - ); -} - // --------------------------------------------------------------------------- // --list // --------------------------------------------------------------------------- diff --git a/packages/eve/src/evals/cli/filter.test.ts b/packages/eve/src/evals/cli/filter.test.ts new file mode 100644 index 000000000..826f645ed --- /dev/null +++ b/packages/eve/src/evals/cli/filter.test.ts @@ -0,0 +1,62 @@ +import { describe, expect, it } from "vitest"; + +import { filterEvalsByTags } from "#evals/cli/filter.js"; +import type { EveEval } from "#evals/types.js"; + +function createEval(id: string, tags?: readonly string[]): EveEval { + return { _tag: "EveEval", id, tags, test: async () => {} }; +} + +const untagged = createEval("untagged"); +const fast = createEval("fast", ["fast"]); +const realModel = createEval("real-model-only", ["real-model"]); +const fastRealModel = createEval("fast-real-model", ["fast", "real-model"]); +const evaluations = [untagged, fast, realModel, fastRealModel]; + +describe("filterEvalsByTags", () => { + it("returns every eval when no filters are given", () => { + const result = filterEvalsByTags({ evaluations, includeTags: [], excludeTags: [] }); + expect(result).toEqual(evaluations); + }); + + it("keeps only evals carrying an include tag", () => { + const result = filterEvalsByTags({ evaluations, includeTags: ["fast"], excludeTags: [] }); + expect(result.map((entry) => entry.id)).toEqual(["fast", "fast-real-model"]); + }); + + it("drops evals carrying an exclude tag", () => { + const result = filterEvalsByTags({ + evaluations, + excludeTags: ["real-model"], + includeTags: [], + }); + expect(result.map((entry) => entry.id)).toEqual(["untagged", "fast"]); + }); + + it("applies exclusion after inclusion", () => { + const result = filterEvalsByTags({ + evaluations, + excludeTags: ["real-model"], + includeTags: ["fast"], + }); + expect(result.map((entry) => entry.id)).toEqual(["fast"]); + }); + + it("returns an empty list when exclusion removes every match", () => { + const result = filterEvalsByTags({ + evaluations: [realModel], + excludeTags: ["real-model"], + includeTags: [], + }); + expect(result).toEqual([]); + }); + + it("treats untagged evals as excluded by include filters only", () => { + const result = filterEvalsByTags({ + evaluations: [untagged], + excludeTags: ["real-model"], + includeTags: [], + }); + expect(result).toEqual([untagged]); + }); +}); diff --git a/packages/eve/src/evals/cli/filter.ts b/packages/eve/src/evals/cli/filter.ts new file mode 100644 index 000000000..d58ed6b46 --- /dev/null +++ b/packages/eve/src/evals/cli/filter.ts @@ -0,0 +1,25 @@ +import type { EveEval } from "#evals/types.js"; + +/** + * Applies `--tag` and `--exclude-tag` filtering for the `eve eval` command. + * + * An eval runs when it carries at least one `includeTags` entry (or no + * include filter is given) and carries none of the `excludeTags` entries. + * Exclusion wins over inclusion so suite-level runs (e.g. a mock-model world + * suite excluding `real-model` evals) can never pull in an excluded eval. + */ +export function filterEvalsByTags(input: { + readonly evaluations: readonly EveEval[]; + readonly includeTags: readonly string[]; + readonly excludeTags: readonly string[]; +}): EveEval[] { + return input.evaluations.filter((evaluation) => { + const tags = evaluation.tags ?? []; + + if (input.includeTags.length > 0 && !tags.some((tag) => input.includeTags.includes(tag))) { + return false; + } + + return !tags.some((tag) => input.excludeTags.includes(tag)); + }); +} diff --git a/packages/eve/src/execution/workflow-entry.integration.test.ts b/packages/eve/src/execution/workflow-entry.integration.test.ts index ad106cb5b..803d95c6f 100644 --- a/packages/eve/src/execution/workflow-entry.integration.test.ts +++ b/packages/eve/src/execution/workflow-entry.integration.test.ts @@ -328,8 +328,11 @@ describe("workflowEntry integration", () => { expect(firstTurn.every((event) => isEventId(event.meta.id))).toBe(true); expect(new Set(firstTurn.map((event) => event.meta.id)).size).toBe(firstTurn.length); + // No stream-order assertion on the ids: they sort in mint order per + // process, but a turn's events are appended by separate steps whose + // writes can interleave behind minting (see #protocol/event-id.js), + // so append order is not contractually sorted. const ids = firstTurn.map((event) => event.meta.id); - expect(ids).toEqual([...ids].sort()); // Re-reading the durable stream returns the same ids. const workflowRuntime = createWorkflowRuntime({ diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d1381c7fd..eaf7c900e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -51,6 +51,9 @@ catalogs: '@vercel/sandbox': specifier: 2.8.0 version: 2.8.0 + '@workflow/world-postgres': + specifier: 5.0.0-beta.29 + version: 5.0.0-beta.29 ai: specifier: ^7.0.38 version: 7.0.38 @@ -142,7 +145,7 @@ importers: version: 1.1.1(react@19.2.6) '@vercel/analytics': specifier: 2.0.1 - version: 2.0.1(2e1d1ea39f306c545b71c35841ac134e) + version: 2.0.1(@sveltejs/kit@2.61.1(@opentelemetry/api@1.9.1)(@sveltejs/vite-plugin-svelte@7.1.2(svelte@5.56.1(@typescript-eslint/types@8.59.4))(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)))(svelte@5.56.1(@typescript-eslint/types@8.59.4))(typescript@6.0.3)(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)))(next@16.3.0-preview.6(@babel/core@7.29.0(supports-color@10.2.2))(@opentelemetry/api@1.9.1)(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(nuxt@4.4.6(41ce06a5c11c989752f5b4fadc9496f0))(react@19.2.6)(svelte@5.56.1(@typescript-eslint/types@8.59.4))(vue@3.5.35(typescript@6.0.3)) '@vercel/eve-catalog': specifier: workspace:* version: link:../../packages/eve-catalog @@ -151,7 +154,7 @@ importers: version: 1.15.2(@svta/cml-cta@1.0.1(@svta/cml-structured-field-values@1.0.1(@svta/cml-utils@1.0.1))(@svta/cml-utils@1.0.1))(@svta/cml-structured-field-values@1.0.1(@svta/cml-utils@1.0.1))(@svta/cml-utils@1.0.1)(@types/mdast@4.0.4)(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(micromark-util-types@2.0.2)(micromark@4.0.2(supports-color@10.2.2))(next@16.3.0-preview.6(@babel/core@7.29.0(supports-color@10.2.2))(@opentelemetry/api@1.9.1)(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(supports-color@10.2.2)(tailwindcss@4.3.0)(unified@11.0.5)(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)) '@vercel/speed-insights': specifier: 2.0.0 - version: 2.0.0(2e1d1ea39f306c545b71c35841ac134e) + version: 2.0.0(@sveltejs/kit@2.61.1(@opentelemetry/api@1.9.1)(@sveltejs/vite-plugin-svelte@7.1.2(svelte@5.56.1(@typescript-eslint/types@8.59.4))(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)))(svelte@5.56.1(@typescript-eslint/types@8.59.4))(typescript@6.0.3)(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)))(next@16.3.0-preview.6(@babel/core@7.29.0(supports-color@10.2.2))(@opentelemetry/api@1.9.1)(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(nuxt@4.4.6(41ce06a5c11c989752f5b4fadc9496f0))(react@19.2.6)(svelte@5.56.1(@typescript-eslint/types@8.59.4))(vue@3.5.35(typescript@6.0.3)) '@vgpu/core': specifier: 0.0.7 version: 0.0.7 @@ -473,7 +476,7 @@ importers: version: link:../../../packages/eve nuxt: specifier: ^4.0.0 - version: 4.4.6(@babel/plugin-syntax-jsx@7.29.7(@babel/core@7.29.0(supports-color@10.2.2)))(@babel/plugin-syntax-typescript@7.29.7(@babel/core@7.29.0(supports-color@10.2.2)))(@parcel/watcher@2.5.6)(@types/node@25.9.1)(@upstash/redis@1.38.0)(@vercel/blob@2.4.0)(@vercel/functions@3.7.5(@aws-sdk/credential-provider-web-identity@3.972.49)(ws@8.21.1(bufferutil@4.1.0)))(@vue/compiler-sfc@3.5.39)(aws4fetch@1.0.20)(bufferutil@4.1.0)(cac@6.7.14)(commander@14.0.3)(db0@0.3.4)(esbuild@0.28.1)(eslint@10.4.0(jiti@2.7.0)(supports-color@10.2.2))(ioredis@5.11.1(supports-color@10.2.2))(lightningcss@1.32.0)(magicast@0.5.3)(optionator@0.9.4)(oxlint@1.73.0)(rolldown@1.1.5)(rollup-plugin-visualizer@7.0.1(rolldown@1.1.5)(rollup@4.62.2))(rollup@4.62.2)(srvx@0.11.22)(supports-color@10.2.2)(terser@5.49.0)(tsx@4.21.0)(typescript@6.0.3)(vite@8.1.5(@types/node@25.9.1)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0))(vue-tsc@3.3.6(typescript@6.0.3))(yaml@2.9.0) + version: 4.4.6(f32df07d5233da3a29c17b72e1467465) vue: specifier: ^3.5.0 version: 3.5.35(typescript@6.0.3) @@ -521,6 +524,9 @@ importers: e2e/fixtures/agent-basic-runtime: dependencies: + '@eve-e2e/config': + specifier: workspace:* + version: link:../e2e-config '@opentelemetry/core': specifier: 'catalog:' version: 2.6.1(@opentelemetry/api@1.9.1) @@ -530,6 +536,9 @@ importers: '@vercel/otel': specifier: 'catalog:' version: 2.1.3(@opentelemetry/api-logs@0.221.0)(@opentelemetry/api@1.9.1)(@opentelemetry/instrumentation@0.219.0(@opentelemetry/api@1.9.1)(supports-color@10.2.2))(@opentelemetry/resources@2.10.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-logs@0.221.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-metrics@2.10.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-trace-base@2.6.1(@opentelemetry/api@1.9.1)) + '@workflow/world-postgres': + specifier: 'catalog:' + version: 5.0.0-beta.29(@opentelemetry/api@1.9.1)(@types/pg@8.20.0)(@upstash/redis@1.38.0)(sql.js@1.14.1)(supports-color@10.2.2)(typescript@7.0.2) eve: specifier: workspace:* version: link:../../../packages/eve @@ -546,6 +555,9 @@ importers: e2e/fixtures/agent-cancellation: dependencies: + '@eve-e2e/config': + specifier: workspace:* + version: link:../e2e-config '@opentelemetry/core': specifier: 'catalog:' version: 2.6.1(@opentelemetry/api@1.9.1) @@ -555,6 +567,9 @@ importers: '@vercel/otel': specifier: 'catalog:' version: 2.1.3(@opentelemetry/api-logs@0.221.0)(@opentelemetry/api@1.9.1)(@opentelemetry/instrumentation@0.219.0(@opentelemetry/api@1.9.1)(supports-color@10.2.2))(@opentelemetry/resources@2.10.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-logs@0.221.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-metrics@2.10.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-trace-base@2.6.1(@opentelemetry/api@1.9.1)) + '@workflow/world-postgres': + specifier: 'catalog:' + version: 5.0.0-beta.29(@opentelemetry/api@1.9.1)(@types/pg@8.20.0)(@upstash/redis@1.38.0)(sql.js@1.14.1)(supports-color@10.2.2)(typescript@7.0.2) eve: specifier: workspace:* version: link:../../../packages/eve @@ -571,6 +586,9 @@ importers: e2e/fixtures/agent-channels: dependencies: + '@eve-e2e/config': + specifier: workspace:* + version: link:../e2e-config '@opentelemetry/core': specifier: 'catalog:' version: 2.6.1(@opentelemetry/api@1.9.1) @@ -580,6 +598,9 @@ importers: '@vercel/otel': specifier: 'catalog:' version: 2.1.3(@opentelemetry/api-logs@0.221.0)(@opentelemetry/api@1.9.1)(@opentelemetry/instrumentation@0.219.0(@opentelemetry/api@1.9.1)(supports-color@10.2.2))(@opentelemetry/resources@2.10.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-logs@0.221.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-metrics@2.10.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-trace-base@2.6.1(@opentelemetry/api@1.9.1)) + '@workflow/world-postgres': + specifier: 'catalog:' + version: 5.0.0-beta.29(@opentelemetry/api@1.9.1)(@types/pg@8.20.0)(@upstash/redis@1.38.0)(sql.js@1.14.1)(supports-color@10.2.2)(typescript@7.0.2) ai: specifier: 'catalog:' version: 7.0.38(zod@4.4.3) @@ -599,6 +620,9 @@ importers: e2e/fixtures/agent-compaction-regressions: dependencies: + '@eve-e2e/config': + specifier: workspace:* + version: link:../e2e-config '@opentelemetry/core': specifier: 'catalog:' version: 2.6.1(@opentelemetry/api@1.9.1) @@ -608,6 +632,9 @@ importers: '@vercel/otel': specifier: 'catalog:' version: 2.1.3(@opentelemetry/api-logs@0.221.0)(@opentelemetry/api@1.9.1)(@opentelemetry/instrumentation@0.219.0(@opentelemetry/api@1.9.1)(supports-color@10.2.2))(@opentelemetry/resources@2.10.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-logs@0.221.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-metrics@2.10.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-trace-base@2.6.1(@opentelemetry/api@1.9.1)) + '@workflow/world-postgres': + specifier: 'catalog:' + version: 5.0.0-beta.29(@opentelemetry/api@1.9.1)(@types/pg@8.20.0)(@upstash/redis@1.38.0)(sql.js@1.14.1)(supports-color@10.2.2)(typescript@7.0.2) eve: specifier: workspace:* version: link:../../../packages/eve @@ -624,6 +651,12 @@ importers: e2e/fixtures/agent-model: dependencies: + '@eve-e2e/config': + specifier: workspace:* + version: link:../e2e-config + '@workflow/world-postgres': + specifier: 'catalog:' + version: 5.0.0-beta.29(@opentelemetry/api@1.9.1)(@types/pg@8.20.0)(@upstash/redis@1.38.0)(sql.js@1.14.1)(supports-color@10.2.2)(typescript@7.0.2) eve: specifier: workspace:* version: link:../../../packages/eve @@ -637,6 +670,9 @@ importers: e2e/fixtures/agent-openapi-swagger: dependencies: + '@eve-e2e/config': + specifier: workspace:* + version: link:../e2e-config '@opentelemetry/core': specifier: 'catalog:' version: 2.6.1(@opentelemetry/api@1.9.1) @@ -646,6 +682,9 @@ importers: '@vercel/otel': specifier: 'catalog:' version: 2.1.3(@opentelemetry/api-logs@0.221.0)(@opentelemetry/api@1.9.1)(@opentelemetry/instrumentation@0.219.0(@opentelemetry/api@1.9.1)(supports-color@10.2.2))(@opentelemetry/resources@2.10.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-logs@0.221.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-metrics@2.10.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-trace-base@2.6.1(@opentelemetry/api@1.9.1)) + '@workflow/world-postgres': + specifier: 'catalog:' + version: 5.0.0-beta.29(@opentelemetry/api@1.9.1)(@types/pg@8.20.0)(@upstash/redis@1.38.0)(sql.js@1.14.1)(supports-color@10.2.2)(typescript@7.0.2) eve: specifier: workspace:* version: link:../../../packages/eve @@ -665,6 +704,12 @@ importers: '@ai-sdk/anthropic': specifier: 'catalog:' version: 4.0.18(zod@4.4.3) + '@eve-e2e/config': + specifier: workspace:* + version: link:../e2e-config + '@workflow/world-postgres': + specifier: 'catalog:' + version: 5.0.0-beta.29(@opentelemetry/api@1.9.1)(@types/pg@8.20.0)(@upstash/redis@1.38.0)(sql.js@1.14.1)(supports-color@10.2.2)(typescript@7.0.2) eve: specifier: workspace:* version: link:../../../packages/eve @@ -681,6 +726,9 @@ importers: e2e/fixtures/agent-schedules: dependencies: + '@eve-e2e/config': + specifier: workspace:* + version: link:../e2e-config '@opentelemetry/core': specifier: 'catalog:' version: 2.6.1(@opentelemetry/api@1.9.1) @@ -690,6 +738,9 @@ importers: '@vercel/otel': specifier: 'catalog:' version: 2.1.3(@opentelemetry/api-logs@0.221.0)(@opentelemetry/api@1.9.1)(@opentelemetry/instrumentation@0.219.0(@opentelemetry/api@1.9.1)(supports-color@10.2.2))(@opentelemetry/resources@2.10.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-logs@0.221.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-metrics@2.10.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-trace-base@2.6.1(@opentelemetry/api@1.9.1)) + '@workflow/world-postgres': + specifier: 'catalog:' + version: 5.0.0-beta.29(@opentelemetry/api@1.9.1)(@types/pg@8.20.0)(@upstash/redis@1.38.0)(sql.js@1.14.1)(supports-color@10.2.2)(typescript@7.0.2) eve: specifier: workspace:* version: link:../../../packages/eve @@ -706,6 +757,9 @@ importers: e2e/fixtures/agent-session-limits: dependencies: + '@eve-e2e/config': + specifier: workspace:* + version: link:../e2e-config '@opentelemetry/core': specifier: 'catalog:' version: 2.6.1(@opentelemetry/api@1.9.1) @@ -715,6 +769,9 @@ importers: '@vercel/otel': specifier: 'catalog:' version: 2.1.3(@opentelemetry/api-logs@0.221.0)(@opentelemetry/api@1.9.1)(@opentelemetry/instrumentation@0.219.0(@opentelemetry/api@1.9.1)(supports-color@10.2.2))(@opentelemetry/resources@2.10.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-logs@0.221.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-metrics@2.10.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-trace-base@2.6.1(@opentelemetry/api@1.9.1)) + '@workflow/world-postgres': + specifier: 'catalog:' + version: 5.0.0-beta.29(@opentelemetry/api@1.9.1)(@types/pg@8.20.0)(@upstash/redis@1.38.0)(sql.js@1.14.1)(supports-color@10.2.2)(typescript@7.0.2) eve: specifier: workspace:* version: link:../../../packages/eve @@ -731,6 +788,9 @@ importers: e2e/fixtures/agent-session-timeout: dependencies: + '@eve-e2e/config': + specifier: workspace:* + version: link:../e2e-config '@opentelemetry/core': specifier: 'catalog:' version: 2.6.1(@opentelemetry/api@1.9.1) @@ -740,6 +800,9 @@ importers: '@vercel/otel': specifier: 'catalog:' version: 2.1.3(@opentelemetry/api-logs@0.221.0)(@opentelemetry/api@1.9.1)(@opentelemetry/instrumentation@0.219.0(@opentelemetry/api@1.9.1)(supports-color@10.2.2))(@opentelemetry/resources@2.10.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-logs@0.221.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-metrics@2.10.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-trace-base@2.6.1(@opentelemetry/api@1.9.1)) + '@workflow/world-postgres': + specifier: 'catalog:' + version: 5.0.0-beta.29(@opentelemetry/api@1.9.1)(@types/pg@8.20.0)(@upstash/redis@1.38.0)(sql.js@1.14.1)(supports-color@10.2.2)(typescript@7.0.2) ai: specifier: 'catalog:' version: 7.0.38(zod@4.4.3) @@ -756,6 +819,9 @@ importers: e2e/fixtures/agent-skills: dependencies: + '@eve-e2e/config': + specifier: workspace:* + version: link:../e2e-config '@opentelemetry/core': specifier: 'catalog:' version: 2.6.1(@opentelemetry/api@1.9.1) @@ -765,6 +831,9 @@ importers: '@vercel/otel': specifier: 'catalog:' version: 2.1.3(@opentelemetry/api-logs@0.221.0)(@opentelemetry/api@1.9.1)(@opentelemetry/instrumentation@0.219.0(@opentelemetry/api@1.9.1)(supports-color@10.2.2))(@opentelemetry/resources@2.10.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-logs@0.221.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-metrics@2.10.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-trace-base@2.6.1(@opentelemetry/api@1.9.1)) + '@workflow/world-postgres': + specifier: 'catalog:' + version: 5.0.0-beta.29(@opentelemetry/api@1.9.1)(@types/pg@8.20.0)(@upstash/redis@1.38.0)(sql.js@1.14.1)(supports-color@10.2.2)(typescript@7.0.2) eve: specifier: workspace:* version: link:../../../packages/eve @@ -781,6 +850,9 @@ importers: e2e/fixtures/agent-subagents: dependencies: + '@eve-e2e/config': + specifier: workspace:* + version: link:../e2e-config '@opentelemetry/core': specifier: 'catalog:' version: 2.6.1(@opentelemetry/api@1.9.1) @@ -790,6 +862,9 @@ importers: '@vercel/otel': specifier: 'catalog:' version: 2.1.3(@opentelemetry/api-logs@0.221.0)(@opentelemetry/api@1.9.1)(@opentelemetry/instrumentation@0.219.0(@opentelemetry/api@1.9.1)(supports-color@10.2.2))(@opentelemetry/resources@2.10.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-logs@0.221.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-metrics@2.10.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-trace-base@2.6.1(@opentelemetry/api@1.9.1)) + '@workflow/world-postgres': + specifier: 'catalog:' + version: 5.0.0-beta.29(@opentelemetry/api@1.9.1)(@types/pg@8.20.0)(@upstash/redis@1.38.0)(sql.js@1.14.1)(supports-color@10.2.2)(typescript@7.0.2) eve: specifier: workspace:* version: link:../../../packages/eve @@ -806,6 +881,12 @@ importers: e2e/fixtures/agent-subagents-hitl: dependencies: + '@eve-e2e/config': + specifier: workspace:* + version: link:../e2e-config + '@workflow/world-postgres': + specifier: 'catalog:' + version: 5.0.0-beta.29(@opentelemetry/api@1.9.1)(@types/pg@8.20.0)(@upstash/redis@1.38.0)(sql.js@1.14.1)(supports-color@10.2.2)(typescript@7.0.2) eve: specifier: workspace:* version: link:../../../packages/eve @@ -822,6 +903,9 @@ importers: e2e/fixtures/agent-tools: dependencies: + '@eve-e2e/config': + specifier: workspace:* + version: link:../e2e-config '@eve-e2e/dynamic-import-dependency': specifier: file:./fixtures/dynamic-import-dependency version: file:e2e/fixtures/agent-tools/fixtures/dynamic-import-dependency @@ -834,6 +918,9 @@ importers: '@vercel/otel': specifier: 'catalog:' version: 2.1.3(@opentelemetry/api-logs@0.221.0)(@opentelemetry/api@1.9.1)(@opentelemetry/instrumentation@0.219.0(@opentelemetry/api@1.9.1)(supports-color@10.2.2))(@opentelemetry/resources@2.10.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-logs@0.221.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-metrics@2.10.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-trace-base@2.6.1(@opentelemetry/api@1.9.1)) + '@workflow/world-postgres': + specifier: 'catalog:' + version: 5.0.0-beta.29(@opentelemetry/api@1.9.1)(@types/pg@8.20.0)(@upstash/redis@1.38.0)(sql.js@1.14.1)(supports-color@10.2.2)(typescript@7.0.2) eve: specifier: workspace:* version: link:../../../packages/eve @@ -850,6 +937,9 @@ importers: e2e/fixtures/agent-tools-hitl: dependencies: + '@eve-e2e/config': + specifier: workspace:* + version: link:../e2e-config '@opentelemetry/core': specifier: 'catalog:' version: 2.6.1(@opentelemetry/api@1.9.1) @@ -859,6 +949,9 @@ importers: '@vercel/otel': specifier: 'catalog:' version: 2.1.3(@opentelemetry/api-logs@0.221.0)(@opentelemetry/api@1.9.1)(@opentelemetry/instrumentation@0.219.0(@opentelemetry/api@1.9.1)(supports-color@10.2.2))(@opentelemetry/resources@2.10.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-logs@0.221.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-metrics@2.10.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-trace-base@2.6.1(@opentelemetry/api@1.9.1)) + '@workflow/world-postgres': + specifier: 'catalog:' + version: 5.0.0-beta.29(@opentelemetry/api@1.9.1)(@types/pg@8.20.0)(@upstash/redis@1.38.0)(sql.js@1.14.1)(supports-color@10.2.2)(typescript@7.0.2) eve: specifier: workspace:* version: link:../../../packages/eve @@ -875,6 +968,9 @@ importers: e2e/fixtures/agent-tools-hitl-openai: dependencies: + '@eve-e2e/config': + specifier: workspace:* + version: link:../e2e-config '@opentelemetry/core': specifier: 'catalog:' version: 2.6.1(@opentelemetry/api@1.9.1) @@ -884,6 +980,9 @@ importers: '@vercel/otel': specifier: 'catalog:' version: 2.1.3(@opentelemetry/api-logs@0.221.0)(@opentelemetry/api@1.9.1)(@opentelemetry/instrumentation@0.219.0(@opentelemetry/api@1.9.1)(supports-color@10.2.2))(@opentelemetry/resources@2.10.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-logs@0.221.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-metrics@2.10.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-trace-base@2.6.1(@opentelemetry/api@1.9.1)) + '@workflow/world-postgres': + specifier: 'catalog:' + version: 5.0.0-beta.29(@opentelemetry/api@1.9.1)(@types/pg@8.20.0)(@upstash/redis@1.38.0)(sql.js@1.14.1)(supports-color@10.2.2)(typescript@7.0.2) eve: specifier: workspace:* version: link:../../../packages/eve @@ -900,6 +999,9 @@ importers: e2e/fixtures/agent-tools-sandbox: dependencies: + '@eve-e2e/config': + specifier: workspace:* + version: link:../e2e-config '@opentelemetry/core': specifier: 'catalog:' version: 2.6.1(@opentelemetry/api@1.9.1) @@ -909,6 +1011,9 @@ importers: '@vercel/otel': specifier: 'catalog:' version: 2.1.3(@opentelemetry/api-logs@0.221.0)(@opentelemetry/api@1.9.1)(@opentelemetry/instrumentation@0.219.0(@opentelemetry/api@1.9.1)(supports-color@10.2.2))(@opentelemetry/resources@2.10.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-logs@0.221.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-metrics@2.10.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-trace-base@2.6.1(@opentelemetry/api@1.9.1)) + '@workflow/world-postgres': + specifier: 'catalog:' + version: 5.0.0-beta.29(@opentelemetry/api@1.9.1)(@types/pg@8.20.0)(@upstash/redis@1.38.0)(sql.js@1.14.1)(supports-color@10.2.2)(typescript@7.0.2) eve: specifier: workspace:* version: link:../../../packages/eve @@ -925,6 +1030,9 @@ importers: e2e/fixtures/agent-workflow-stress: dependencies: + '@eve-e2e/config': + specifier: workspace:* + version: link:../e2e-config '@opentelemetry/core': specifier: 'catalog:' version: 2.6.1(@opentelemetry/api@1.9.1) @@ -934,6 +1042,9 @@ importers: '@vercel/otel': specifier: 'catalog:' version: 2.1.3(@opentelemetry/api-logs@0.221.0)(@opentelemetry/api@1.9.1)(@opentelemetry/instrumentation@0.219.0(@opentelemetry/api@1.9.1)(supports-color@10.2.2))(@opentelemetry/resources@2.10.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-logs@0.221.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-metrics@2.10.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-trace-base@2.6.1(@opentelemetry/api@1.9.1)) + '@workflow/world-postgres': + specifier: 'catalog:' + version: 5.0.0-beta.29(@opentelemetry/api@1.9.1)(@types/pg@8.20.0)(@upstash/redis@1.38.0)(sql.js@1.14.1)(supports-color@10.2.2)(typescript@7.0.2) ai: specifier: 'catalog:' version: 7.0.38(zod@4.4.3) @@ -950,6 +1061,12 @@ importers: e2e/fixtures/dist-extensions: dependencies: + '@eve-e2e/config': + specifier: workspace:* + version: link:../e2e-config + '@workflow/world-postgres': + specifier: 'catalog:' + version: 5.0.0-beta.29(@opentelemetry/api@1.9.1)(@types/pg@8.20.0)(@upstash/redis@1.38.0)(sql.js@1.14.1)(supports-color@10.2.2)(typescript@7.0.2) eve: specifier: workspace:* version: link:../../../packages/eve @@ -967,8 +1084,26 @@ importers: specifier: 'catalog:' version: 7.0.2 + e2e/fixtures/e2e-config: + devDependencies: + '@types/node': + specifier: 'catalog:' + version: 25.9.1 + eve: + specifier: workspace:* + version: link:../../../packages/eve + typescript: + specifier: 'catalog:' + version: 7.0.2 + e2e/fixtures/extensions: dependencies: + '@eve-e2e/config': + specifier: workspace:* + version: link:../e2e-config + '@workflow/world-postgres': + specifier: 'catalog:' + version: 5.0.0-beta.29(@opentelemetry/api@1.9.1)(@types/pg@8.20.0)(@upstash/redis@1.38.0)(sql.js@1.14.1)(supports-color@10.2.2)(typescript@7.0.2) eve: specifier: workspace:* version: link:../../../packages/eve @@ -1069,7 +1204,7 @@ importers: version: 3.24.0(@aws-sdk/credential-provider-web-identity@3.972.49)(supports-color@10.2.2)(zod@4.4.3) nitro: specifier: 3.0.260610-beta - version: 3.0.260610-beta(@upstash/redis@1.38.0)(@vercel/blob@2.4.0)(@vercel/functions@3.7.5(@aws-sdk/credential-provider-web-identity@3.972.49)(ws@8.21.1(bufferutil@4.1.0)))(aws4fetch@1.0.20)(chokidar@5.0.0)(dotenv@17.4.2)(giget@3.3.0)(ioredis@5.11.1(supports-color@10.2.2))(jiti@2.7.0)(lru-cache@11.5.2)(rollup@4.62.2)(vite@8.1.5(@types/node@25.9.1)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)) + version: 3.0.260610-beta(@upstash/redis@1.38.0)(@vercel/blob@2.4.0)(@vercel/functions@3.7.5(@aws-sdk/credential-provider-web-identity@3.972.49)(ws@8.21.1(bufferutil@4.1.0)))(aws4fetch@1.0.20)(chokidar@5.0.0)(dotenv@17.4.2)(drizzle-orm@0.45.2(@opentelemetry/api@1.9.1)(@types/pg@8.20.0)(@upstash/redis@1.38.0)(pg@8.20.0)(sql.js@1.14.1))(giget@3.3.0)(ioredis@5.11.1(supports-color@10.2.2))(jiti@2.7.0)(lru-cache@11.5.2)(rollup@4.62.2)(vite@8.1.5(@types/node@25.9.1)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)) undici: specifier: 8.9.0 version: 8.9.0 @@ -2738,6 +2873,9 @@ packages: resolution: {integrity: sha512-9tU0h6O/Ao9k53TQSgLs6D/mN++wKTHPIpT8tbPaayhOL3e4OhtJ3t+OD42avMe2pBmFcdRUVOSpZvqytxvWEg==} engines: {node: '>=12.0.0'} + '@graphile/logger@0.2.0': + resolution: {integrity: sha512-jjcWBokl9eb1gVJ85QmoaQ73CQ52xAaOCF29ukRbYNl6lY+ts0ErTaDYOBlejcbUs2OpaiqYLO5uDhyLFzWw4w==} + '@grpc/grpc-js@1.14.4': resolution: {integrity: sha512-k9Dj3DV/itK9D06Y8f190Qgop7/Ui+D0njFV3LHMPwPT75DpXLQohE9Wmz0QElrJnzsjB7KPWiKJbOl7IPDArQ==} engines: {node: '>=12.10.0'} @@ -6434,6 +6572,9 @@ packages: '@types/hast@3.0.4': resolution: {integrity: sha512-WPs+bbQw5aCj+x6laNGWLH3wviHtoCv/P3+otBhbOhJgG8qtpdAMlTCxLtsTWA7LH1Oh/bFCHsBn0TPS5m30EQ==} + '@types/interpret@1.1.4': + resolution: {integrity: sha512-r+tPKWHYqaxJOYA3Eik0mMi+SEREqOXLmsooRFmc6GHv7nWUDixFtKN+cegvsPlDcEZd9wxsdp041v2imQuvag==} + '@types/istanbul-lib-coverage@2.0.6': resolution: {integrity: sha512-2QF/t/auWm0lsy8XtKVPG19v3sSOQlJe/YHZgfjb/KBBHOGSV+J2q/S671rcq9uTBrLAXmZpqJiaQbMT+zNU1w==} @@ -6473,9 +6614,15 @@ packages: '@types/node@20.11.0': resolution: {integrity: sha512-o9bjXmDNcF7GbM4CNQpmi+TutCgap/K3w1JyKgxAjqx41zp9qlIAVFi0IhCNsJcXolEqLWhbFbEeL0PvYm4pcQ==} + '@types/node@22.20.1': + resolution: {integrity: sha512-EANqOCF9QFyra+4pfxUcX9STKJpCLjMbObVzljIJomAWSnuSIEAvyzEU53GaajbXJEgdh0iEcPL+DGvpUd4k1Q==} + '@types/node@25.9.1': resolution: {integrity: sha512-xfrlY7UD5rMJk3ZVJP8BNzS28J36YJg+xp+LPXV1TdWxr8uMH5A860QNxYDGQe/ylDSgjxE52Q9VnO7p75tJxg==} + '@types/pg@8.20.0': + resolution: {integrity: sha512-bEPFOaMAHTEP1EzpvHTbmwR8UsFyHSKsRisLIHVMXnpNefSbGA1bD6CVy+qKjGSqmZqNqBDV2azOBo8TgkcVow==} + '@types/pngjs@6.0.5': resolution: {integrity: sha512-0k5eKfrA83JOZPppLtS2C7OUtyNAl2wKNxfyYl9Q5g9lPkgBl/9hNyAu6HuEH2J4XmIv2znEpkDd0SaZVxW6iQ==} @@ -6496,6 +6643,9 @@ packages: '@types/retry@0.12.0': resolution: {integrity: sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==} + '@types/semver@7.7.1': + resolution: {integrity: sha512-FmgJfu+MOcQ370SD0ev7EI8TlCAfKYU+B4m5T3yXc1CiRN94g/SZPtsCkk506aUDtlMnFZvasDwHHUcZUEaYuA==} + '@types/trusted-types@2.0.7': resolution: {integrity: sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==} @@ -7290,6 +7440,9 @@ packages: '@opentelemetry/api': optional: true + '@workflow/errors@5.0.0-beta.13': + resolution: {integrity: sha512-xRiG9MEunLxkb7hw7n17+/cKKH4QfFvnQly4cAF2ImxPRDnL0HvWot9eHOPS3S41LEyZ/LSs/sYrIN07XUZbgw==} + '@workflow/errors@5.0.0-beta.14': resolution: {integrity: sha512-1RwF7LG7JbIp78zD+eb1A8ifDacI12wzpdMAFbzMODc/Ok0u+1IXbT1DIVKLtBLnUOPrWLxnXBYDnkvkvG9tAw==} @@ -7302,9 +7455,20 @@ packages: '@workflow/serde@5.0.0-beta.2': resolution: {integrity: sha512-INB+FcEKQkkFZa+s53sEMTNRFiBa2g9vzjsNuEDagvWxtI0t/cnCyoc57i7kS7nj+3/vGeRjO4Yn5ccbT0hyBQ==} + '@workflow/utils@5.0.0-beta.7': + resolution: {integrity: sha512-3vDiA8xea4USslIbz6m9HAhxDhvqLbETaXFxTNqVm/m8eji0rLKwnfF6JLaH0Qr1GsovysoOUlvyibokkNHZrg==} + '@workflow/utils@5.0.0-beta.8': resolution: {integrity: sha512-dDiQ/LT05g9HyOicqWNNX/Gw/fmO7Ma2kZ+ddUXXpm/EFTMRS9eiI69suSe1+1NnhMzdV6ObI8wrO5lvQqPIbg==} + '@workflow/world-local@5.0.0-beta.31': + resolution: {integrity: sha512-Z9nq5lzjXnig4ONfLnVwTqx6EbxJjhTB5SBKT8ERcKe6/SFm30s1BIZ03bZBJox2Y4jJAgsNgvey29xWGnzMAg==} + peerDependencies: + '@opentelemetry/api': '1' + peerDependenciesMeta: + '@opentelemetry/api': + optional: true + '@workflow/world-local@5.0.0-beta.32': resolution: {integrity: sha512-D8kb1IHHrkDbKnpUw3cqZnlkis3YCYPF2BoV6vWpGUGTeSGsnTZjMsrqsh5luKeZSUqK5BCO/YC27S9AdkBO0w==} peerDependencies: @@ -7313,6 +7477,10 @@ packages: '@opentelemetry/api': optional: true + '@workflow/world-postgres@5.0.0-beta.29': + resolution: {integrity: sha512-OaqDbvtM1mfsQtzOndyepyPARYXkWWZc53wcV3U3l65K8rMFjwX3TG+721wpEnIvWiFBVD3MJ8HjgnIIQG/1Hw==} + hasBin: true + '@workflow/world-vercel@5.0.0-beta.34': resolution: {integrity: sha512-bSBxUcs90wbhY8NlZYojoPAL7UwDuiTxr1R6eCb3ZCHCly5pTFXnkMIsM2hScPFM6ea/KImMl1KQVJc6OCPjIQ==} peerDependencies: @@ -8245,6 +8413,15 @@ packages: cose-base@2.2.0: resolution: {integrity: sha512-AzlgcsCbUMymkADOJtQm3wO9S3ltPfYOFD5033keQn9NJzIbtnZj+UdBJe7DYml/8TdbtHJW3j58SOnKhWY/5g==} + cosmiconfig@8.3.6: + resolution: {integrity: sha512-kcZ6+W5QzcJ3P1Mt+83OUv/oHFqZHIx8DuxG6eZ5RGMERoLqp4BuGjhHLYGK+Kf5XVkQvqBSmAy/nGWN3qDgEA==} + engines: {node: '>=14'} + peerDependencies: + typescript: '>=4.9.5' + peerDependenciesMeta: + typescript: + optional: true + cosmiconfig@9.0.2: resolution: {integrity: sha512-gtTZxTDau1wL7Y7zifc2dd8jHSK/k6BTx/2Xp/BpdlAdnlYWFVt7qhJqgwi7637yRwRQ3qL4ZidbB4I8tA5VOg==} engines: {node: '>=14'} @@ -8749,10 +8926,106 @@ packages: resolution: {integrity: sha512-uBq4egWHTcTt33a72vpSG0z3HnPuIl6NqYcTrKEg2azoEyl2hpW0zqlxysq2pK9HlDIHyHyakeYaYnSAwd8bow==} engines: {node: '>=12'} + dotenv@17.3.1: + resolution: {integrity: sha512-IO8C/dzEb6O3F9/twg6ZLXz164a2fhTnEWb95H23Dm4OuN+92NmEAlTrupP9VW6Jm3sO26tQlqyvyi4CsnY9GA==} + engines: {node: '>=12'} + dotenv@17.4.2: resolution: {integrity: sha512-nI4U3TottKAcAD9LLud4Cb7b2QztQMUEfHbvhTH09bqXTxnSie8WnjPALV/WMCrJZ6UV/qHJ6L03OqO3LcdYZw==} engines: {node: '>=12'} + drizzle-orm@0.45.2: + resolution: {integrity: sha512-kY0BSaTNYWnoDMVoyY8uxmyHjpJW1geOmBMdSSicKo9CIIWkSxMIj2rkeSR51b8KAPB7m+qysjuHme5nKP+E5Q==} + peerDependencies: + '@aws-sdk/client-rds-data': '>=3' + '@cloudflare/workers-types': '>=4' + '@electric-sql/pglite': '>=0.2.0' + '@libsql/client': '>=0.10.0' + '@libsql/client-wasm': '>=0.10.0' + '@neondatabase/serverless': '>=0.10.0' + '@op-engineering/op-sqlite': '>=2' + '@opentelemetry/api': ^1.4.1 + '@planetscale/database': '>=1.13' + '@prisma/client': '*' + '@tidbcloud/serverless': '*' + '@types/better-sqlite3': '*' + '@types/pg': '*' + '@types/sql.js': '*' + '@upstash/redis': '>=1.34.7' + '@vercel/postgres': '>=0.8.0' + '@xata.io/client': '*' + better-sqlite3: '>=7' + bun-types: '*' + expo-sqlite: '>=14.0.0' + gel: '>=2' + knex: '*' + kysely: '*' + mysql2: '>=2' + pg: '>=8' + postgres: '>=3' + prisma: '*' + sql.js: '>=1' + sqlite3: '>=5' + peerDependenciesMeta: + '@aws-sdk/client-rds-data': + optional: true + '@cloudflare/workers-types': + optional: true + '@electric-sql/pglite': + optional: true + '@libsql/client': + optional: true + '@libsql/client-wasm': + optional: true + '@neondatabase/serverless': + optional: true + '@op-engineering/op-sqlite': + optional: true + '@opentelemetry/api': + optional: true + '@planetscale/database': + optional: true + '@prisma/client': + optional: true + '@tidbcloud/serverless': + optional: true + '@types/better-sqlite3': + optional: true + '@types/pg': + optional: true + '@types/sql.js': + optional: true + '@upstash/redis': + optional: true + '@vercel/postgres': + optional: true + '@xata.io/client': + optional: true + better-sqlite3: + optional: true + bun-types: + optional: true + expo-sqlite: + optional: true + gel: + optional: true + knex: + optional: true + kysely: + optional: true + mysql2: + optional: true + pg: + optional: true + postgres: + optional: true + prisma: + optional: true + sql.js: + optional: true + sqlite3: + optional: true + dunder-proto@1.0.1: resolution: {integrity: sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==} engines: {node: '>= 0.4'} @@ -9790,6 +10063,15 @@ packages: graceful-fs@4.2.11: resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==} + graphile-config@0.0.1-beta.18: + resolution: {integrity: sha512-uMdF9Rt8/NwT1wVXNleYgM5ro2hHDodHiKA3efJhgdU8iP+r/hksnghOHreMva0sF5tV73f4TpiELPUR0g7O9w==} + engines: {node: '>=16'} + + graphile-worker@0.16.6: + resolution: {integrity: sha512-e7gGYDmGqzju2l83MpzX8vNG/lOtVJiSzI3eZpAFubSxh/cxs7sRrRGBGjzBP1kNG0H+c95etPpNRNlH65PYhw==} + engines: {node: '>=14.0.0'} + hasBin: true + gray-matter@4.0.3: resolution: {integrity: sha512-5v6yZd4JK3eMI3FqqCouswVqwugaA9r4dNZB1wwcmrD02QkV5H0y7XBQW8QwQqEaZY1pM9aqORSORhJRdNK44Q==} engines: {node: '>=6.0'} @@ -10113,6 +10395,10 @@ packages: resolution: {integrity: sha512-5Hh7Y1wQbvY5ooGgPbDaL5iYLAPzMTUrjMulskHLH6wnv/A+1q5rgEaiuqEjB+oxGXIVZs1FF+R/KPN3ZSQYYg==} engines: {node: '>=12'} + interpret@3.1.1: + resolution: {integrity: sha512-6xwYfHbajpoF0xLW+iwLkhwgvLoZDfjYfoFNu8ftMoXINzwuymNLd9u/KmwtdT2GbR+/Cz66otEGEVVUHX9QLQ==} + engines: {node: '>=10.13.0'} + invariant@2.2.4: resolution: {integrity: sha512-phJfQVBuaJM5raOpJjSfkiD6BpbCE4Ns//LaXl6wGYtUBY83nWS6Rf9tXm2e8VaK60JEjYldbPif/A2B1C2gNA==} @@ -11999,6 +12285,40 @@ packages: perfect-debounce@2.1.0: resolution: {integrity: sha512-LjgdTytVFXeUgtHZr9WYViYSM/g8MkcTPYDlPa3cDqMirHjKiSZPYd6DoL7pK8AJQr+uWkQvCjHNdiMqsrJs+g==} + pg-cloudflare@1.4.0: + resolution: {integrity: sha512-Vo7z/6rrQYxpNRylp4Tlob2elzbh+N/MOQbxFVWCxS7oEx6jF53GTJFxK2WWpKuBRkmiin4Mt+xofFDjx09R0A==} + + pg-connection-string@2.14.0: + resolution: {integrity: sha512-XwWDGcLRGCXAR8F/AM5bG7Q+A3Wm2s6QeEjlOKZLlH3UYcguiqCWKyWXVag5TLTIjR7oOJUY8kcADaZgWPyLeg==} + + pg-int8@1.0.1: + resolution: {integrity: sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==} + engines: {node: '>=4.0.0'} + + pg-pool@3.14.0: + resolution: {integrity: sha512-gKtPkFdQPU3DksooVLi9LsjZxrsBUZIpa+7aVx+LV5pNh0KzP4Zleud2po+ConrxbuXGBJ6Hfer6hdgpIBpBaw==} + peerDependencies: + pg: '>=8.0' + + pg-protocol@1.15.0: + resolution: {integrity: sha512-cq9sECI5s0+uPUXjbz8ioyPJni6RzsRib0US67i5IoTZKw8fNeYlVE7u8F4dG7vEJJtc5wdD1K189lCCUwqWTQ==} + + pg-types@2.2.0: + resolution: {integrity: sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==} + engines: {node: '>=4'} + + pg@8.20.0: + resolution: {integrity: sha512-ldhMxz2r8fl/6QkXnBD3CR9/xg694oT6DZQ2s6c/RI28OjtSOpxnPrUCGOBJ46RCUxcWdx3p6kw/xnDHjKvaRA==} + engines: {node: '>= 16.0.0'} + peerDependencies: + pg-native: '>=3.0.1' + peerDependenciesMeta: + pg-native: + optional: true + + pgpass@1.0.5: + resolution: {integrity: sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==} + phase@0.0.1: resolution: {integrity: sha512-QTeTClFj700SPwdgfVckQ8uO/dquPO5MdxHpSLCOtKpJbLrpUrFnEVhoCuQKXMSWXNIMlnwPsKQ7fqfXCX6BJg==} engines: {node: 24.x} @@ -12266,6 +12586,22 @@ packages: resolution: {integrity: sha512-lW616l85ucIQL+FocMmL7pQFPqBmwejrCMg+iPxyImlrANNJG9NHq/RkyCZopDhd8C3LA03PHRJDjkbGu8vvug==} engines: {node: ^10 || ^12 || >=14} + postgres-array@2.0.0: + resolution: {integrity: sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==} + engines: {node: '>=4'} + + postgres-bytea@1.0.1: + resolution: {integrity: sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==} + engines: {node: '>=0.10.0'} + + postgres-date@1.0.7: + resolution: {integrity: sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==} + engines: {node: '>=0.10.0'} + + postgres-interval@1.2.0: + resolution: {integrity: sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==} + engines: {node: '>=0.10.0'} + powershell-utils@0.1.0: resolution: {integrity: sha512-dM0jVuXJPsDN6DvRpea484tCUaMiXWjuCn++HGTqUWzGDjv5tZkEZldAJ/UMlqRYGFrD/etByo4/xOuC/snX2A==} engines: {node: '>=20'} @@ -13716,6 +14052,9 @@ packages: undici-types@5.26.5: resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==} + undici-types@6.21.0: + resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==} + undici-types@7.24.6: resolution: {integrity: sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==} @@ -14513,6 +14852,10 @@ packages: resolution: {integrity: sha512-k8KO9hrMyNk6tUWqUfkTEZbezRRpONVOzUTnc97VnCvyj6Tf9lyUR9EDAIeiVLv56jsMcoXEwjW8Kv5yPY52lw==} engines: {node: '>=16.0.0'} + xtend@4.0.2: + resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==} + engines: {node: '>=0.4'} + y18n@5.0.8: resolution: {integrity: sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==} engines: {node: '>=10'} @@ -16115,6 +16458,8 @@ snapshots: transitivePeerDependencies: - supports-color + '@graphile/logger@0.2.0': {} + '@grpc/grpc-js@1.14.4': dependencies: '@grpc/proto-loader': 0.8.1 @@ -16931,7 +17276,7 @@ snapshots: transitivePeerDependencies: - magicast - '@nuxt/nitro-server@4.4.6(5af45ff6ce88146737a84f10936700d8)': + '@nuxt/nitro-server@4.4.6(@babel/plugin-syntax-typescript@7.29.7(@babel/core@7.29.0(supports-color@10.2.2)))(@upstash/redis@1.38.0)(@vercel/blob@2.4.0)(@vercel/functions@3.7.5(@aws-sdk/credential-provider-web-identity@3.972.49)(ws@8.21.1(bufferutil@4.1.0)))(aws4fetch@1.0.20)(db0@0.3.4(drizzle-orm@0.45.2(@opentelemetry/api@1.9.1)(@types/pg@8.20.0)(@upstash/redis@1.38.0)(pg@8.20.0)(sql.js@1.14.1)))(drizzle-orm@0.45.2(@opentelemetry/api@1.9.1)(@types/pg@8.20.0)(@upstash/redis@1.38.0)(pg@8.20.0)(sql.js@1.14.1))(esbuild@0.28.1)(ioredis@5.11.1(supports-color@10.2.2))(magicast@0.5.3)(nuxt@4.4.6(41ce06a5c11c989752f5b4fadc9496f0))(oxc-parser@0.131.0)(rolldown@1.1.5)(rollup@4.62.2)(srvx@0.11.22)(supports-color@10.2.2)(typescript@6.0.3)(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0))': dependencies: '@nuxt/devalue': 2.0.2 '@nuxt/kit': 4.4.6(magicast@0.5.3) @@ -16948,8 +17293,8 @@ snapshots: impound: 1.1.5(esbuild@0.28.1)(rolldown@1.1.5)(rollup@4.62.2)(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)) klona: 2.0.6 mocked-exports: 0.1.1 - nitropack: 2.13.4(@upstash/redis@1.38.0)(@vercel/blob@2.4.0)(@vercel/functions@3.7.5(@aws-sdk/credential-provider-web-identity@3.972.49)(ws@8.21.1(bufferutil@4.1.0)))(aws4fetch@1.0.20)(oxc-parser@0.131.0)(rolldown@1.1.5)(srvx@0.11.22)(supports-color@10.2.2)(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)) - nuxt: 4.4.6(@babel/plugin-syntax-jsx@7.29.7(@babel/core@7.29.0(supports-color@10.2.2)))(@babel/plugin-syntax-typescript@7.29.7(@babel/core@7.29.0(supports-color@10.2.2)))(@parcel/watcher@2.5.6)(@types/node@25.9.1)(@upstash/redis@1.38.0)(@vercel/blob@2.4.0)(@vercel/functions@3.7.5(@aws-sdk/credential-provider-web-identity@3.972.49)(ws@8.21.1(bufferutil@4.1.0)))(@vue/compiler-sfc@3.5.39)(aws4fetch@1.0.20)(bufferutil@4.1.0)(cac@6.7.14)(commander@14.0.3)(db0@0.3.4)(esbuild@0.28.1)(eslint@10.4.0(jiti@2.7.0)(supports-color@10.2.2))(ioredis@5.11.1(supports-color@10.2.2))(lightningcss@1.32.0)(magicast@0.5.3)(optionator@0.9.4)(oxlint@1.73.0)(rolldown@1.1.5)(rollup-plugin-visualizer@7.0.1(rolldown@1.1.5)(rollup@4.62.2))(rollup@4.62.2)(srvx@0.11.22)(supports-color@10.2.2)(terser@5.49.0)(tsx@4.21.0)(typescript@6.0.3)(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0))(vue-tsc@3.3.6(typescript@6.0.3))(yaml@2.9.0) + nitropack: 2.13.4(@upstash/redis@1.38.0)(@vercel/blob@2.4.0)(@vercel/functions@3.7.5(@aws-sdk/credential-provider-web-identity@3.972.49)(ws@8.21.1(bufferutil@4.1.0)))(aws4fetch@1.0.20)(drizzle-orm@0.45.2(@opentelemetry/api@1.9.1)(@types/pg@8.20.0)(@upstash/redis@1.38.0)(pg@8.20.0)(sql.js@1.14.1))(oxc-parser@0.131.0)(rolldown@1.1.5)(srvx@0.11.22)(supports-color@10.2.2)(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)) + nuxt: 4.4.6(41ce06a5c11c989752f5b4fadc9496f0) nypm: 0.6.8 ohash: 2.0.11 pathe: 2.0.3 @@ -16957,7 +17302,7 @@ snapshots: std-env: 4.2.0 ufo: 1.6.4 unctx: 2.5.0 - unstorage: 1.17.5(@upstash/redis@1.38.0)(@vercel/blob@2.4.0)(@vercel/functions@3.7.5(@aws-sdk/credential-provider-web-identity@3.972.49)(ws@8.21.1(bufferutil@4.1.0)))(aws4fetch@1.0.20)(db0@0.3.4)(ioredis@5.11.1(supports-color@10.2.2)) + unstorage: 1.17.5(@upstash/redis@1.38.0)(@vercel/blob@2.4.0)(@vercel/functions@3.7.5(@aws-sdk/credential-provider-web-identity@3.972.49)(ws@8.21.1(bufferutil@4.1.0)))(aws4fetch@1.0.20)(db0@0.3.4(drizzle-orm@0.45.2(@opentelemetry/api@1.9.1)(@types/pg@8.20.0)(@upstash/redis@1.38.0)(pg@8.20.0)(sql.js@1.14.1)))(ioredis@5.11.1(supports-color@10.2.2)) vue: 3.5.35(typescript@6.0.3) vue-bundle-renderer: 2.3.1 vue-devtools-stub: 0.1.0 @@ -17010,7 +17355,7 @@ snapshots: - xml2js optional: true - '@nuxt/nitro-server@4.4.6(d7661b23bdde68362333e80e2569dc53)': + '@nuxt/nitro-server@4.4.6(@babel/plugin-syntax-typescript@7.29.7(@babel/core@7.29.0(supports-color@10.2.2)))(@upstash/redis@1.38.0)(@vercel/blob@2.4.0)(@vercel/functions@3.7.5(@aws-sdk/credential-provider-web-identity@3.972.49)(ws@8.21.1(bufferutil@4.1.0)))(aws4fetch@1.0.20)(db0@0.3.4(drizzle-orm@0.45.2(@opentelemetry/api@1.9.1)(@types/pg@8.20.0)(@upstash/redis@1.38.0)(pg@8.20.0)(sql.js@1.14.1)))(drizzle-orm@0.45.2(@opentelemetry/api@1.9.1)(@types/pg@8.20.0)(@upstash/redis@1.38.0)(pg@8.20.0)(sql.js@1.14.1))(esbuild@0.28.1)(ioredis@5.11.1(supports-color@10.2.2))(magicast@0.5.3)(nuxt@4.4.6(f32df07d5233da3a29c17b72e1467465))(oxc-parser@0.131.0)(rolldown@1.1.5)(rollup@4.62.2)(srvx@0.11.22)(supports-color@10.2.2)(typescript@6.0.3)(vite@8.1.5(@types/node@25.9.1)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0))': dependencies: '@nuxt/devalue': 2.0.2 '@nuxt/kit': 4.4.6(magicast@0.5.3) @@ -17027,8 +17372,8 @@ snapshots: impound: 1.1.5(esbuild@0.28.1)(rolldown@1.1.5)(rollup@4.62.2)(vite@8.1.5(@types/node@25.9.1)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)) klona: 2.0.6 mocked-exports: 0.1.1 - nitropack: 2.13.4(@upstash/redis@1.38.0)(@vercel/blob@2.4.0)(@vercel/functions@3.7.5(@aws-sdk/credential-provider-web-identity@3.972.49)(ws@8.21.1(bufferutil@4.1.0)))(aws4fetch@1.0.20)(oxc-parser@0.131.0)(rolldown@1.1.5)(srvx@0.11.22)(supports-color@10.2.2)(vite@8.1.5(@types/node@25.9.1)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)) - nuxt: 4.4.6(@babel/plugin-syntax-jsx@7.29.7(@babel/core@7.29.0(supports-color@10.2.2)))(@babel/plugin-syntax-typescript@7.29.7(@babel/core@7.29.0(supports-color@10.2.2)))(@parcel/watcher@2.5.6)(@types/node@25.9.1)(@upstash/redis@1.38.0)(@vercel/blob@2.4.0)(@vercel/functions@3.7.5(@aws-sdk/credential-provider-web-identity@3.972.49)(ws@8.21.1(bufferutil@4.1.0)))(@vue/compiler-sfc@3.5.39)(aws4fetch@1.0.20)(bufferutil@4.1.0)(cac@6.7.14)(commander@14.0.3)(db0@0.3.4)(esbuild@0.28.1)(eslint@10.4.0(jiti@2.7.0)(supports-color@10.2.2))(ioredis@5.11.1(supports-color@10.2.2))(lightningcss@1.32.0)(magicast@0.5.3)(optionator@0.9.4)(oxlint@1.73.0)(rolldown@1.1.5)(rollup-plugin-visualizer@7.0.1(rolldown@1.1.5)(rollup@4.62.2))(rollup@4.62.2)(srvx@0.11.22)(supports-color@10.2.2)(terser@5.49.0)(tsx@4.21.0)(typescript@6.0.3)(vite@8.1.5(@types/node@25.9.1)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0))(vue-tsc@3.3.6(typescript@6.0.3))(yaml@2.9.0) + nitropack: 2.13.4(@upstash/redis@1.38.0)(@vercel/blob@2.4.0)(@vercel/functions@3.7.5(@aws-sdk/credential-provider-web-identity@3.972.49)(ws@8.21.1(bufferutil@4.1.0)))(aws4fetch@1.0.20)(drizzle-orm@0.45.2(@opentelemetry/api@1.9.1)(@types/pg@8.20.0)(@upstash/redis@1.38.0)(pg@8.20.0)(sql.js@1.14.1))(oxc-parser@0.131.0)(rolldown@1.1.5)(srvx@0.11.22)(supports-color@10.2.2)(vite@8.1.5(@types/node@25.9.1)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)) + nuxt: 4.4.6(f32df07d5233da3a29c17b72e1467465) nypm: 0.6.8 ohash: 2.0.11 pathe: 2.0.3 @@ -17036,7 +17381,7 @@ snapshots: std-env: 4.2.0 ufo: 1.6.4 unctx: 2.5.0 - unstorage: 1.17.5(@upstash/redis@1.38.0)(@vercel/blob@2.4.0)(@vercel/functions@3.7.5(@aws-sdk/credential-provider-web-identity@3.972.49)(ws@8.21.1(bufferutil@4.1.0)))(aws4fetch@1.0.20)(db0@0.3.4)(ioredis@5.11.1(supports-color@10.2.2)) + unstorage: 1.17.5(@upstash/redis@1.38.0)(@vercel/blob@2.4.0)(@vercel/functions@3.7.5(@aws-sdk/credential-provider-web-identity@3.972.49)(ws@8.21.1(bufferutil@4.1.0)))(aws4fetch@1.0.20)(db0@0.3.4(drizzle-orm@0.45.2(@opentelemetry/api@1.9.1)(@types/pg@8.20.0)(@upstash/redis@1.38.0)(pg@8.20.0)(sql.js@1.14.1)))(ioredis@5.11.1(supports-color@10.2.2)) vue: 3.5.35(typescript@6.0.3) vue-bundle-renderer: 2.3.1 vue-devtools-stub: 0.1.0 @@ -17105,7 +17450,7 @@ snapshots: rc9: 3.0.1 std-env: 4.2.0 - '@nuxt/vite-builder@4.4.6(65326fc791d4735547fd3a6298d65608)': + '@nuxt/vite-builder@4.4.6(@babel/plugin-syntax-jsx@7.29.7(@babel/core@7.29.0(supports-color@10.2.2)))(@types/node@25.9.1)(eslint@10.4.0(jiti@2.7.0)(supports-color@10.2.2))(lightningcss@1.32.0)(magicast@0.5.3)(nuxt@4.4.6(41ce06a5c11c989752f5b4fadc9496f0))(optionator@0.9.4)(oxlint@1.73.0)(rolldown@1.1.5)(rollup-plugin-visualizer@7.0.1(rolldown@1.1.5)(rollup@4.62.2))(rollup@4.62.2)(supports-color@10.2.2)(terser@5.49.0)(tsx@4.21.0)(typescript@6.0.3)(vue-tsc@3.3.6(typescript@6.0.3))(vue@3.5.35(typescript@6.0.3))(yaml@2.9.0)': dependencies: '@nuxt/kit': 4.4.6(magicast@0.5.3) '@rollup/plugin-replace': 6.0.3(rollup@4.62.2) @@ -17123,7 +17468,7 @@ snapshots: magic-string: 0.30.21 mlly: 1.8.2 mocked-exports: 0.1.1 - nuxt: 4.4.6(@babel/plugin-syntax-jsx@7.29.7(@babel/core@7.29.0(supports-color@10.2.2)))(@babel/plugin-syntax-typescript@7.29.7(@babel/core@7.29.0(supports-color@10.2.2)))(@parcel/watcher@2.5.6)(@types/node@25.9.1)(@upstash/redis@1.38.0)(@vercel/blob@2.4.0)(@vercel/functions@3.7.5(@aws-sdk/credential-provider-web-identity@3.972.49)(ws@8.21.1(bufferutil@4.1.0)))(@vue/compiler-sfc@3.5.39)(aws4fetch@1.0.20)(bufferutil@4.1.0)(cac@6.7.14)(commander@14.0.3)(db0@0.3.4)(esbuild@0.28.1)(eslint@10.4.0(jiti@2.7.0)(supports-color@10.2.2))(ioredis@5.11.1(supports-color@10.2.2))(lightningcss@1.32.0)(magicast@0.5.3)(optionator@0.9.4)(oxlint@1.73.0)(rolldown@1.1.5)(rollup-plugin-visualizer@7.0.1(rolldown@1.1.5)(rollup@4.62.2))(rollup@4.62.2)(srvx@0.11.22)(supports-color@10.2.2)(terser@5.49.0)(tsx@4.21.0)(typescript@6.0.3)(vite@8.1.5(@types/node@25.9.1)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0))(vue-tsc@3.3.6(typescript@6.0.3))(yaml@2.9.0) + nuxt: 4.4.6(41ce06a5c11c989752f5b4fadc9496f0) nypm: 0.6.8 pathe: 2.0.3 pkg-types: 2.3.1 @@ -17165,8 +17510,9 @@ snapshots: - vti - vue-tsc - yaml + optional: true - '@nuxt/vite-builder@4.4.6(bd1799bb420f4f18d539b3aab18f78ee)': + '@nuxt/vite-builder@4.4.6(@babel/plugin-syntax-jsx@7.29.7(@babel/core@7.29.0(supports-color@10.2.2)))(@types/node@25.9.1)(eslint@10.4.0(jiti@2.7.0)(supports-color@10.2.2))(lightningcss@1.32.0)(magicast@0.5.3)(nuxt@4.4.6(f32df07d5233da3a29c17b72e1467465))(optionator@0.9.4)(oxlint@1.73.0)(rolldown@1.1.5)(rollup-plugin-visualizer@7.0.1(rolldown@1.1.5)(rollup@4.62.2))(rollup@4.62.2)(supports-color@10.2.2)(terser@5.49.0)(tsx@4.21.0)(typescript@6.0.3)(vue-tsc@3.3.6(typescript@6.0.3))(vue@3.5.35(typescript@6.0.3))(yaml@2.9.0)': dependencies: '@nuxt/kit': 4.4.6(magicast@0.5.3) '@rollup/plugin-replace': 6.0.3(rollup@4.62.2) @@ -17184,7 +17530,7 @@ snapshots: magic-string: 0.30.21 mlly: 1.8.2 mocked-exports: 0.1.1 - nuxt: 4.4.6(@babel/plugin-syntax-jsx@7.29.7(@babel/core@7.29.0(supports-color@10.2.2)))(@babel/plugin-syntax-typescript@7.29.7(@babel/core@7.29.0(supports-color@10.2.2)))(@parcel/watcher@2.5.6)(@types/node@25.9.1)(@upstash/redis@1.38.0)(@vercel/blob@2.4.0)(@vercel/functions@3.7.5(@aws-sdk/credential-provider-web-identity@3.972.49)(ws@8.21.1(bufferutil@4.1.0)))(@vue/compiler-sfc@3.5.39)(aws4fetch@1.0.20)(bufferutil@4.1.0)(cac@6.7.14)(commander@14.0.3)(db0@0.3.4)(esbuild@0.28.1)(eslint@10.4.0(jiti@2.7.0)(supports-color@10.2.2))(ioredis@5.11.1(supports-color@10.2.2))(lightningcss@1.32.0)(magicast@0.5.3)(optionator@0.9.4)(oxlint@1.73.0)(rolldown@1.1.5)(rollup-plugin-visualizer@7.0.1(rolldown@1.1.5)(rollup@4.62.2))(rollup@4.62.2)(srvx@0.11.22)(supports-color@10.2.2)(terser@5.49.0)(tsx@4.21.0)(typescript@6.0.3)(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0))(vue-tsc@3.3.6(typescript@6.0.3))(yaml@2.9.0) + nuxt: 4.4.6(f32df07d5233da3a29c17b72e1467465) nypm: 0.6.8 pathe: 2.0.3 pkg-types: 2.3.1 @@ -17226,7 +17572,6 @@ snapshots: - vti - vue-tsc - yaml - optional: true '@octokit/app@16.1.2': dependencies: @@ -19923,6 +20268,10 @@ snapshots: dependencies: '@types/unist': 3.0.3 + '@types/interpret@1.1.4': + dependencies: + '@types/node': 25.9.1 + '@types/istanbul-lib-coverage@2.0.6': {} '@types/istanbul-lib-report@3.0.3': @@ -19962,10 +20311,20 @@ snapshots: dependencies: undici-types: 5.26.5 + '@types/node@22.20.1': + dependencies: + undici-types: 6.21.0 + '@types/node@25.9.1': dependencies: undici-types: 7.24.6 + '@types/pg@8.20.0': + dependencies: + '@types/node': 25.9.1 + pg-protocol: 1.15.0 + pg-types: 2.2.0 + '@types/pngjs@6.0.5': dependencies: '@types/node': 25.9.1 @@ -19986,6 +20345,8 @@ snapshots: '@types/retry@0.12.0': {} + '@types/semver@7.7.1': {} + '@types/trusted-types@2.0.7': {} '@types/unist@2.0.11': {} @@ -20282,11 +20643,11 @@ snapshots: optionalDependencies: next: 16.3.0-preview.6(@babel/core@7.29.0(supports-color@10.2.2))(@opentelemetry/api@1.9.1)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) - '@vercel/analytics@2.0.1(2e1d1ea39f306c545b71c35841ac134e)': + '@vercel/analytics@2.0.1(@sveltejs/kit@2.61.1(@opentelemetry/api@1.9.1)(@sveltejs/vite-plugin-svelte@7.1.2(svelte@5.56.1(@typescript-eslint/types@8.59.4))(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)))(svelte@5.56.1(@typescript-eslint/types@8.59.4))(typescript@6.0.3)(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)))(next@16.3.0-preview.6(@babel/core@7.29.0(supports-color@10.2.2))(@opentelemetry/api@1.9.1)(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(nuxt@4.4.6(41ce06a5c11c989752f5b4fadc9496f0))(react@19.2.6)(svelte@5.56.1(@typescript-eslint/types@8.59.4))(vue@3.5.35(typescript@6.0.3))': optionalDependencies: '@sveltejs/kit': 2.61.1(@opentelemetry/api@1.9.1)(@sveltejs/vite-plugin-svelte@7.1.2(svelte@5.56.1(@typescript-eslint/types@8.59.4))(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)))(svelte@5.56.1(@typescript-eslint/types@8.59.4))(typescript@6.0.3)(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)) next: 16.3.0-preview.6(@babel/core@7.29.0(supports-color@10.2.2))(@opentelemetry/api@1.9.1)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) - nuxt: 4.4.6(@babel/plugin-syntax-jsx@7.29.7(@babel/core@7.29.0(supports-color@10.2.2)))(@babel/plugin-syntax-typescript@7.29.7(@babel/core@7.29.0(supports-color@10.2.2)))(@parcel/watcher@2.5.6)(@types/node@25.9.1)(@upstash/redis@1.38.0)(@vercel/blob@2.4.0)(@vercel/functions@3.7.5(@aws-sdk/credential-provider-web-identity@3.972.49)(ws@8.21.1(bufferutil@4.1.0)))(@vue/compiler-sfc@3.5.39)(aws4fetch@1.0.20)(bufferutil@4.1.0)(cac@6.7.14)(commander@14.0.3)(db0@0.3.4)(esbuild@0.28.1)(eslint@10.4.0(jiti@2.7.0)(supports-color@10.2.2))(ioredis@5.11.1(supports-color@10.2.2))(lightningcss@1.32.0)(magicast@0.5.3)(optionator@0.9.4)(oxlint@1.73.0)(rolldown@1.1.5)(rollup-plugin-visualizer@7.0.1(rolldown@1.1.5)(rollup@4.62.2))(rollup@4.62.2)(srvx@0.11.22)(supports-color@10.2.2)(terser@5.49.0)(tsx@4.21.0)(typescript@6.0.3)(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0))(vue-tsc@3.3.6(typescript@6.0.3))(yaml@2.9.0) + nuxt: 4.4.6(41ce06a5c11c989752f5b4fadc9496f0) react: 19.2.6 svelte: 5.56.1(@typescript-eslint/types@8.59.4) vue: 3.5.35(typescript@6.0.3) @@ -20755,11 +21116,11 @@ snapshots: dependencies: zod: 4.4.3 - '@vercel/speed-insights@2.0.0(2e1d1ea39f306c545b71c35841ac134e)': + '@vercel/speed-insights@2.0.0(@sveltejs/kit@2.61.1(@opentelemetry/api@1.9.1)(@sveltejs/vite-plugin-svelte@7.1.2(svelte@5.56.1(@typescript-eslint/types@8.59.4))(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)))(svelte@5.56.1(@typescript-eslint/types@8.59.4))(typescript@6.0.3)(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)))(next@16.3.0-preview.6(@babel/core@7.29.0(supports-color@10.2.2))(@opentelemetry/api@1.9.1)(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(nuxt@4.4.6(41ce06a5c11c989752f5b4fadc9496f0))(react@19.2.6)(svelte@5.56.1(@typescript-eslint/types@8.59.4))(vue@3.5.35(typescript@6.0.3))': optionalDependencies: '@sveltejs/kit': 2.61.1(@opentelemetry/api@1.9.1)(@sveltejs/vite-plugin-svelte@7.1.2(svelte@5.56.1(@typescript-eslint/types@8.59.4))(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)))(svelte@5.56.1(@typescript-eslint/types@8.59.4))(typescript@6.0.3)(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)) next: 16.3.0-preview.6(@babel/core@7.29.0(supports-color@10.2.2))(@opentelemetry/api@1.9.1)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) - nuxt: 4.4.6(@babel/plugin-syntax-jsx@7.29.7(@babel/core@7.29.0(supports-color@10.2.2)))(@babel/plugin-syntax-typescript@7.29.7(@babel/core@7.29.0(supports-color@10.2.2)))(@parcel/watcher@2.5.6)(@types/node@25.9.1)(@upstash/redis@1.38.0)(@vercel/blob@2.4.0)(@vercel/functions@3.7.5(@aws-sdk/credential-provider-web-identity@3.972.49)(ws@8.21.1(bufferutil@4.1.0)))(@vue/compiler-sfc@3.5.39)(aws4fetch@1.0.20)(bufferutil@4.1.0)(cac@6.7.14)(commander@14.0.3)(db0@0.3.4)(esbuild@0.28.1)(eslint@10.4.0(jiti@2.7.0)(supports-color@10.2.2))(ioredis@5.11.1(supports-color@10.2.2))(lightningcss@1.32.0)(magicast@0.5.3)(optionator@0.9.4)(oxlint@1.73.0)(rolldown@1.1.5)(rollup-plugin-visualizer@7.0.1(rolldown@1.1.5)(rollup@4.62.2))(rollup@4.62.2)(srvx@0.11.22)(supports-color@10.2.2)(terser@5.49.0)(tsx@4.21.0)(typescript@6.0.3)(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0))(vue-tsc@3.3.6(typescript@6.0.3))(yaml@2.9.0) + nuxt: 4.4.6(41ce06a5c11c989752f5b4fadc9496f0) react: 19.2.6 svelte: 5.56.1(@typescript-eslint/types@8.59.4) vue: 3.5.35(typescript@6.0.3) @@ -21062,6 +21423,11 @@ snapshots: - supports-color - ws + '@workflow/errors@5.0.0-beta.13': + dependencies: + '@workflow/utils': 5.0.0-beta.7 + ms: 2.1.3 + '@workflow/errors@5.0.0-beta.14': dependencies: '@workflow/utils': 5.0.0-beta.8 @@ -21073,10 +21439,27 @@ snapshots: '@workflow/serde@5.0.0-beta.2': {} + '@workflow/utils@5.0.0-beta.7': + dependencies: + ms: 2.1.3 + '@workflow/utils@5.0.0-beta.8': dependencies: ms: 2.1.3 + '@workflow/world-local@5.0.0-beta.31(@opentelemetry/api@1.9.1)': + dependencies: + '@vercel/queue': 0.4.0(@opentelemetry/api@1.9.1) + '@workflow/errors': 5.0.0-beta.13 + '@workflow/utils': 5.0.0-beta.7 + '@workflow/world': 5.0.0-beta.23 + async-sema: 3.1.1 + ulid: 3.0.2 + undici: 7.28.0 + zod: 4.3.6 + optionalDependencies: + '@opentelemetry/api': 1.9.1 + '@workflow/world-local@5.0.0-beta.32(@opentelemetry/api@1.9.1)': dependencies: '@vercel/queue': 0.4.0(@opentelemetry/api@1.9.1) @@ -21090,6 +21473,53 @@ snapshots: optionalDependencies: '@opentelemetry/api': 1.9.1 + '@workflow/world-postgres@5.0.0-beta.29(@opentelemetry/api@1.9.1)(@types/pg@8.20.0)(@upstash/redis@1.38.0)(sql.js@1.14.1)(supports-color@10.2.2)(typescript@7.0.2)': + dependencies: + '@vercel/queue': 0.4.0(@opentelemetry/api@1.9.1) + '@workflow/errors': 5.0.0-beta.13 + '@workflow/utils': 5.0.0-beta.7 + '@workflow/world': 5.0.0-beta.23 + '@workflow/world-local': 5.0.0-beta.31(@opentelemetry/api@1.9.1) + cbor-x: 1.6.0 + dotenv: 17.3.1 + drizzle-orm: 0.45.2(@opentelemetry/api@1.9.1)(@types/pg@8.20.0)(@upstash/redis@1.38.0)(pg@8.20.0)(sql.js@1.14.1) + graphile-worker: 0.16.6(supports-color@10.2.2)(typescript@7.0.2) + pg: 8.20.0 + ulid: 3.0.2 + zod: 4.3.6 + transitivePeerDependencies: + - '@aws-sdk/client-rds-data' + - '@cloudflare/workers-types' + - '@electric-sql/pglite' + - '@libsql/client' + - '@libsql/client-wasm' + - '@neondatabase/serverless' + - '@op-engineering/op-sqlite' + - '@opentelemetry/api' + - '@planetscale/database' + - '@prisma/client' + - '@tidbcloud/serverless' + - '@types/better-sqlite3' + - '@types/pg' + - '@types/sql.js' + - '@upstash/redis' + - '@vercel/postgres' + - '@xata.io/client' + - better-sqlite3 + - bun-types + - expo-sqlite + - gel + - knex + - kysely + - mysql2 + - pg-native + - postgres + - prisma + - sql.js + - sqlite3 + - supports-color + - typescript + '@workflow/world-vercel@5.0.0-beta.34(@opentelemetry/api@1.9.1)': dependencies: '@vercel/oidc': 3.2.0 @@ -22118,6 +22548,15 @@ snapshots: dependencies: layout-base: 2.0.1 + cosmiconfig@8.3.6(typescript@7.0.2): + dependencies: + import-fresh: 3.3.1 + js-yaml: 4.1.1 + parse-json: 5.2.0 + path-type: 4.0.0 + optionalDependencies: + typescript: 7.0.2 + cosmiconfig@9.0.2(typescript@6.0.3): dependencies: env-paths: 2.2.1 @@ -22468,7 +22907,9 @@ snapshots: dayjs@1.11.20: {} - db0@0.3.4: {} + db0@0.3.4(drizzle-orm@0.45.2(@opentelemetry/api@1.9.1)(@types/pg@8.20.0)(@upstash/redis@1.38.0)(pg@8.20.0)(sql.js@1.14.1)): + optionalDependencies: + drizzle-orm: 0.45.2(@opentelemetry/api@1.9.1)(@types/pg@8.20.0)(@upstash/redis@1.38.0)(pg@8.20.0)(sql.js@1.14.1) dc-browser@1.0.4: {} @@ -22636,8 +23077,18 @@ snapshots: dotenv@16.6.1: {} + dotenv@17.3.1: {} + dotenv@17.4.2: {} + drizzle-orm@0.45.2(@opentelemetry/api@1.9.1)(@types/pg@8.20.0)(@upstash/redis@1.38.0)(pg@8.20.0)(sql.js@1.14.1): + optionalDependencies: + '@opentelemetry/api': 1.9.1 + '@types/pg': 8.20.0 + '@upstash/redis': 1.38.0 + pg: 8.20.0 + sql.js: 1.14.1 + dunder-proto@1.0.1: dependencies: call-bind-apply-helpers: 1.0.2 @@ -24072,6 +24523,36 @@ snapshots: graceful-fs@4.2.11: {} + graphile-config@0.0.1-beta.18(supports-color@10.2.2): + dependencies: + '@types/interpret': 1.1.4 + '@types/node': 22.20.1 + '@types/semver': 7.7.1 + chalk: 4.1.2 + debug: 4.4.3(supports-color@10.2.2) + interpret: 3.1.1 + semver: 7.8.4 + tslib: 2.8.1 + yargs: 17.7.3 + transitivePeerDependencies: + - supports-color + + graphile-worker@0.16.6(supports-color@10.2.2)(typescript@7.0.2): + dependencies: + '@graphile/logger': 0.2.0 + '@types/debug': 4.1.13 + '@types/pg': 8.20.0 + cosmiconfig: 8.3.6(typescript@7.0.2) + graphile-config: 0.0.1-beta.18(supports-color@10.2.2) + json5: 2.2.3 + pg: 8.20.0 + tslib: 2.8.1 + yargs: 17.7.3 + transitivePeerDependencies: + - pg-native + - supports-color + - typescript + gray-matter@4.0.3: dependencies: js-yaml: 3.14.2 @@ -24547,6 +25028,8 @@ snapshots: internmap@2.0.3: {} + interpret@3.1.1: {} + invariant@2.2.4: dependencies: loose-envify: 1.4.0 @@ -26194,11 +26677,11 @@ snapshots: abort-controller-x: 0.5.0 nice-grpc-common: 2.0.3 - nitro@3.0.260610-beta(@upstash/redis@1.38.0)(@vercel/blob@2.4.0)(@vercel/functions@3.7.5(@aws-sdk/credential-provider-web-identity@3.972.49)(ws@8.21.1(bufferutil@4.1.0)))(aws4fetch@1.0.20)(chokidar@5.0.0)(dotenv@17.4.2)(giget@3.3.0)(ioredis@5.11.1(supports-color@10.2.2))(jiti@2.7.0)(lru-cache@11.5.2)(rollup@4.62.2)(vite@8.1.5(@types/node@25.9.1)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)): + nitro@3.0.260610-beta(@upstash/redis@1.38.0)(@vercel/blob@2.4.0)(@vercel/functions@3.7.5(@aws-sdk/credential-provider-web-identity@3.972.49)(ws@8.21.1(bufferutil@4.1.0)))(aws4fetch@1.0.20)(chokidar@5.0.0)(dotenv@17.4.2)(drizzle-orm@0.45.2(@opentelemetry/api@1.9.1)(@types/pg@8.20.0)(@upstash/redis@1.38.0)(pg@8.20.0)(sql.js@1.14.1))(giget@3.3.0)(ioredis@5.11.1(supports-color@10.2.2))(jiti@2.7.0)(lru-cache@11.5.2)(rollup@4.62.2)(vite@8.1.5(@types/node@25.9.1)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)): dependencies: consola: 3.4.2 crossws: 0.4.10(srvx@0.11.22) - db0: 0.3.4 + db0: 0.3.4(drizzle-orm@0.45.2(@opentelemetry/api@1.9.1)(@types/pg@8.20.0)(@upstash/redis@1.38.0)(pg@8.20.0)(sql.js@1.14.1)) env-runner: 0.1.16 h3: 2.0.1-rc.22(crossws@0.4.10(srvx@0.11.22)) hookable: 6.1.1 @@ -26209,7 +26692,7 @@ snapshots: rolldown: 1.1.5 srvx: 0.11.22 unenv: 2.0.0-rc.24 - unstorage: 2.0.0-alpha.7(@upstash/redis@1.38.0)(@vercel/blob@2.4.0)(@vercel/functions@3.7.5(@aws-sdk/credential-provider-web-identity@3.972.49)(ws@8.21.1(bufferutil@4.1.0)))(aws4fetch@1.0.20)(chokidar@5.0.0)(db0@0.3.4)(ioredis@5.11.1(supports-color@10.2.2))(lru-cache@11.5.2)(ofetch@2.0.0-alpha.3) + unstorage: 2.0.0-alpha.7(@upstash/redis@1.38.0)(@vercel/blob@2.4.0)(@vercel/functions@3.7.5(@aws-sdk/credential-provider-web-identity@3.972.49)(ws@8.21.1(bufferutil@4.1.0)))(aws4fetch@1.0.20)(chokidar@5.0.0)(db0@0.3.4(drizzle-orm@0.45.2(@opentelemetry/api@1.9.1)(@types/pg@8.20.0)(@upstash/redis@1.38.0)(pg@8.20.0)(sql.js@1.14.1)))(ioredis@5.11.1(supports-color@10.2.2))(lru-cache@11.5.2)(ofetch@2.0.0-alpha.3) optionalDependencies: dotenv: 17.4.2 giget: 3.3.0 @@ -26248,7 +26731,7 @@ snapshots: - uploadthing - wrangler - nitropack@2.13.4(@upstash/redis@1.38.0)(@vercel/blob@2.4.0)(@vercel/functions@3.7.5(@aws-sdk/credential-provider-web-identity@3.972.49)(ws@8.21.1(bufferutil@4.1.0)))(aws4fetch@1.0.20)(oxc-parser@0.131.0)(rolldown@1.1.5)(srvx@0.11.22)(supports-color@10.2.2)(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)): + nitropack@2.13.4(@upstash/redis@1.38.0)(@vercel/blob@2.4.0)(@vercel/functions@3.7.5(@aws-sdk/credential-provider-web-identity@3.972.49)(ws@8.21.1(bufferutil@4.1.0)))(aws4fetch@1.0.20)(drizzle-orm@0.45.2(@opentelemetry/api@1.9.1)(@types/pg@8.20.0)(@upstash/redis@1.38.0)(pg@8.20.0)(sql.js@1.14.1))(oxc-parser@0.131.0)(rolldown@1.1.5)(srvx@0.11.22)(supports-color@10.2.2)(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)): dependencies: '@cloudflare/kv-asset-handler': 0.4.2 '@rollup/plugin-alias': 6.0.0(rollup@4.62.2) @@ -26269,7 +26752,7 @@ snapshots: cookie-es: 2.0.1 croner: 10.0.1 crossws: 0.3.5 - db0: 0.3.4 + db0: 0.3.4(drizzle-orm@0.45.2(@opentelemetry/api@1.9.1)(@types/pg@8.20.0)(@upstash/redis@1.38.0)(pg@8.20.0)(sql.js@1.14.1)) defu: 6.1.7 destr: 2.0.5 dot-prop: 10.1.0 @@ -26315,7 +26798,7 @@ snapshots: unenv: 2.0.0-rc.24 unimport: 6.3.0(esbuild@0.28.1)(oxc-parser@0.131.0)(rolldown@1.1.5)(rollup@4.62.2)(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)) unplugin-utils: 0.3.2 - unstorage: 1.17.5(@upstash/redis@1.38.0)(@vercel/blob@2.4.0)(@vercel/functions@3.7.5(@aws-sdk/credential-provider-web-identity@3.972.49)(ws@8.21.1(bufferutil@4.1.0)))(aws4fetch@1.0.20)(db0@0.3.4)(ioredis@5.11.1(supports-color@10.2.2)) + unstorage: 1.17.5(@upstash/redis@1.38.0)(@vercel/blob@2.4.0)(@vercel/functions@3.7.5(@aws-sdk/credential-provider-web-identity@3.972.49)(ws@8.21.1(bufferutil@4.1.0)))(aws4fetch@1.0.20)(db0@0.3.4(drizzle-orm@0.45.2(@opentelemetry/api@1.9.1)(@types/pg@8.20.0)(@upstash/redis@1.38.0)(pg@8.20.0)(sql.js@1.14.1)))(ioredis@5.11.1(supports-color@10.2.2)) untyped: 2.0.0 unwasm: 0.5.3 youch: 4.1.1 @@ -26360,7 +26843,7 @@ snapshots: - webpack optional: true - nitropack@2.13.4(@upstash/redis@1.38.0)(@vercel/blob@2.4.0)(@vercel/functions@3.7.5(@aws-sdk/credential-provider-web-identity@3.972.49)(ws@8.21.1(bufferutil@4.1.0)))(aws4fetch@1.0.20)(oxc-parser@0.131.0)(rolldown@1.1.5)(srvx@0.11.22)(supports-color@10.2.2)(vite@8.1.5(@types/node@25.9.1)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)): + nitropack@2.13.4(@upstash/redis@1.38.0)(@vercel/blob@2.4.0)(@vercel/functions@3.7.5(@aws-sdk/credential-provider-web-identity@3.972.49)(ws@8.21.1(bufferutil@4.1.0)))(aws4fetch@1.0.20)(drizzle-orm@0.45.2(@opentelemetry/api@1.9.1)(@types/pg@8.20.0)(@upstash/redis@1.38.0)(pg@8.20.0)(sql.js@1.14.1))(oxc-parser@0.131.0)(rolldown@1.1.5)(srvx@0.11.22)(supports-color@10.2.2)(vite@8.1.5(@types/node@25.9.1)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)): dependencies: '@cloudflare/kv-asset-handler': 0.4.2 '@rollup/plugin-alias': 6.0.0(rollup@4.62.2) @@ -26381,7 +26864,7 @@ snapshots: cookie-es: 2.0.1 croner: 10.0.1 crossws: 0.3.5 - db0: 0.3.4 + db0: 0.3.4(drizzle-orm@0.45.2(@opentelemetry/api@1.9.1)(@types/pg@8.20.0)(@upstash/redis@1.38.0)(pg@8.20.0)(sql.js@1.14.1)) defu: 6.1.7 destr: 2.0.5 dot-prop: 10.1.0 @@ -26427,7 +26910,7 @@ snapshots: unenv: 2.0.0-rc.24 unimport: 6.3.0(esbuild@0.28.1)(oxc-parser@0.131.0)(rolldown@1.1.5)(rollup@4.62.2)(vite@8.1.5(@types/node@25.9.1)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)) unplugin-utils: 0.3.2 - unstorage: 1.17.5(@upstash/redis@1.38.0)(@vercel/blob@2.4.0)(@vercel/functions@3.7.5(@aws-sdk/credential-provider-web-identity@3.972.49)(ws@8.21.1(bufferutil@4.1.0)))(aws4fetch@1.0.20)(db0@0.3.4)(ioredis@5.11.1(supports-color@10.2.2)) + unstorage: 1.17.5(@upstash/redis@1.38.0)(@vercel/blob@2.4.0)(@vercel/functions@3.7.5(@aws-sdk/credential-provider-web-identity@3.972.49)(ws@8.21.1(bufferutil@4.1.0)))(aws4fetch@1.0.20)(db0@0.3.4(drizzle-orm@0.45.2(@opentelemetry/api@1.9.1)(@types/pg@8.20.0)(@upstash/redis@1.38.0)(pg@8.20.0)(sql.js@1.14.1)))(ioredis@5.11.1(supports-color@10.2.2)) untyped: 2.0.0 unwasm: 0.5.3 youch: 4.1.1 @@ -26565,16 +27048,16 @@ snapshots: nullthrows@1.1.1: {} - nuxt@4.4.6(@babel/plugin-syntax-jsx@7.29.7(@babel/core@7.29.0(supports-color@10.2.2)))(@babel/plugin-syntax-typescript@7.29.7(@babel/core@7.29.0(supports-color@10.2.2)))(@parcel/watcher@2.5.6)(@types/node@25.9.1)(@upstash/redis@1.38.0)(@vercel/blob@2.4.0)(@vercel/functions@3.7.5(@aws-sdk/credential-provider-web-identity@3.972.49)(ws@8.21.1(bufferutil@4.1.0)))(@vue/compiler-sfc@3.5.39)(aws4fetch@1.0.20)(bufferutil@4.1.0)(cac@6.7.14)(commander@14.0.3)(db0@0.3.4)(esbuild@0.28.1)(eslint@10.4.0(jiti@2.7.0)(supports-color@10.2.2))(ioredis@5.11.1(supports-color@10.2.2))(lightningcss@1.32.0)(magicast@0.5.3)(optionator@0.9.4)(oxlint@1.73.0)(rolldown@1.1.5)(rollup-plugin-visualizer@7.0.1(rolldown@1.1.5)(rollup@4.62.2))(rollup@4.62.2)(srvx@0.11.22)(supports-color@10.2.2)(terser@5.49.0)(tsx@4.21.0)(typescript@6.0.3)(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0))(vue-tsc@3.3.6(typescript@6.0.3))(yaml@2.9.0): + nuxt@4.4.6(41ce06a5c11c989752f5b4fadc9496f0): dependencies: '@dxup/nuxt': 0.4.1(magicast@0.5.3)(typescript@6.0.3) '@nuxt/cli': 3.37.0(@nuxt/schema@4.4.6)(cac@6.7.14)(commander@14.0.3)(magicast@0.5.3)(supports-color@10.2.2) '@nuxt/devtools': 3.2.4(bufferutil@4.1.0)(supports-color@10.2.2)(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0))(vue@3.5.35(typescript@6.0.3)) '@nuxt/kit': 4.4.6(magicast@0.5.3) - '@nuxt/nitro-server': 4.4.6(5af45ff6ce88146737a84f10936700d8) + '@nuxt/nitro-server': 4.4.6(@babel/plugin-syntax-typescript@7.29.7(@babel/core@7.29.0(supports-color@10.2.2)))(@upstash/redis@1.38.0)(@vercel/blob@2.4.0)(@vercel/functions@3.7.5(@aws-sdk/credential-provider-web-identity@3.972.49)(ws@8.21.1(bufferutil@4.1.0)))(aws4fetch@1.0.20)(db0@0.3.4(drizzle-orm@0.45.2(@opentelemetry/api@1.9.1)(@types/pg@8.20.0)(@upstash/redis@1.38.0)(pg@8.20.0)(sql.js@1.14.1)))(drizzle-orm@0.45.2(@opentelemetry/api@1.9.1)(@types/pg@8.20.0)(@upstash/redis@1.38.0)(pg@8.20.0)(sql.js@1.14.1))(esbuild@0.28.1)(ioredis@5.11.1(supports-color@10.2.2))(magicast@0.5.3)(nuxt@4.4.6(41ce06a5c11c989752f5b4fadc9496f0))(oxc-parser@0.131.0)(rolldown@1.1.5)(rollup@4.62.2)(srvx@0.11.22)(supports-color@10.2.2)(typescript@6.0.3)(vite@7.3.3(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)) '@nuxt/schema': 4.4.6 '@nuxt/telemetry': 2.8.0(@nuxt/kit@4.4.6(magicast@0.5.3)) - '@nuxt/vite-builder': 4.4.6(bd1799bb420f4f18d539b3aab18f78ee) + '@nuxt/vite-builder': 4.4.6(@babel/plugin-syntax-jsx@7.29.7(@babel/core@7.29.0(supports-color@10.2.2)))(@types/node@25.9.1)(eslint@10.4.0(jiti@2.7.0)(supports-color@10.2.2))(lightningcss@1.32.0)(magicast@0.5.3)(nuxt@4.4.6(41ce06a5c11c989752f5b4fadc9496f0))(optionator@0.9.4)(oxlint@1.73.0)(rolldown@1.1.5)(rollup-plugin-visualizer@7.0.1(rolldown@1.1.5)(rollup@4.62.2))(rollup@4.62.2)(supports-color@10.2.2)(terser@5.49.0)(tsx@4.21.0)(typescript@6.0.3)(vue-tsc@3.3.6(typescript@6.0.3))(vue@3.5.35(typescript@6.0.3))(yaml@2.9.0) '@unhead/vue': 2.1.15(vue@3.5.35(typescript@6.0.3)) '@vue/shared': 3.5.39 chokidar: 5.0.0 @@ -26702,16 +27185,16 @@ snapshots: - yaml optional: true - nuxt@4.4.6(@babel/plugin-syntax-jsx@7.29.7(@babel/core@7.29.0(supports-color@10.2.2)))(@babel/plugin-syntax-typescript@7.29.7(@babel/core@7.29.0(supports-color@10.2.2)))(@parcel/watcher@2.5.6)(@types/node@25.9.1)(@upstash/redis@1.38.0)(@vercel/blob@2.4.0)(@vercel/functions@3.7.5(@aws-sdk/credential-provider-web-identity@3.972.49)(ws@8.21.1(bufferutil@4.1.0)))(@vue/compiler-sfc@3.5.39)(aws4fetch@1.0.20)(bufferutil@4.1.0)(cac@6.7.14)(commander@14.0.3)(db0@0.3.4)(esbuild@0.28.1)(eslint@10.4.0(jiti@2.7.0)(supports-color@10.2.2))(ioredis@5.11.1(supports-color@10.2.2))(lightningcss@1.32.0)(magicast@0.5.3)(optionator@0.9.4)(oxlint@1.73.0)(rolldown@1.1.5)(rollup-plugin-visualizer@7.0.1(rolldown@1.1.5)(rollup@4.62.2))(rollup@4.62.2)(srvx@0.11.22)(supports-color@10.2.2)(terser@5.49.0)(tsx@4.21.0)(typescript@6.0.3)(vite@8.1.5(@types/node@25.9.1)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0))(vue-tsc@3.3.6(typescript@6.0.3))(yaml@2.9.0): + nuxt@4.4.6(f32df07d5233da3a29c17b72e1467465): dependencies: '@dxup/nuxt': 0.4.1(magicast@0.5.3)(typescript@6.0.3) '@nuxt/cli': 3.37.0(@nuxt/schema@4.4.6)(cac@6.7.14)(commander@14.0.3)(magicast@0.5.3)(supports-color@10.2.2) '@nuxt/devtools': 3.2.4(bufferutil@4.1.0)(supports-color@10.2.2)(vite@8.1.5(@types/node@25.9.1)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0))(vue@3.5.35(typescript@6.0.3)) '@nuxt/kit': 4.4.6(magicast@0.5.3) - '@nuxt/nitro-server': 4.4.6(d7661b23bdde68362333e80e2569dc53) + '@nuxt/nitro-server': 4.4.6(@babel/plugin-syntax-typescript@7.29.7(@babel/core@7.29.0(supports-color@10.2.2)))(@upstash/redis@1.38.0)(@vercel/blob@2.4.0)(@vercel/functions@3.7.5(@aws-sdk/credential-provider-web-identity@3.972.49)(ws@8.21.1(bufferutil@4.1.0)))(aws4fetch@1.0.20)(db0@0.3.4(drizzle-orm@0.45.2(@opentelemetry/api@1.9.1)(@types/pg@8.20.0)(@upstash/redis@1.38.0)(pg@8.20.0)(sql.js@1.14.1)))(drizzle-orm@0.45.2(@opentelemetry/api@1.9.1)(@types/pg@8.20.0)(@upstash/redis@1.38.0)(pg@8.20.0)(sql.js@1.14.1))(esbuild@0.28.1)(ioredis@5.11.1(supports-color@10.2.2))(magicast@0.5.3)(nuxt@4.4.6(f32df07d5233da3a29c17b72e1467465))(oxc-parser@0.131.0)(rolldown@1.1.5)(rollup@4.62.2)(srvx@0.11.22)(supports-color@10.2.2)(typescript@6.0.3)(vite@8.1.5(@types/node@25.9.1)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.49.0)(tsx@4.21.0)(yaml@2.9.0)) '@nuxt/schema': 4.4.6 '@nuxt/telemetry': 2.8.0(@nuxt/kit@4.4.6(magicast@0.5.3)) - '@nuxt/vite-builder': 4.4.6(65326fc791d4735547fd3a6298d65608) + '@nuxt/vite-builder': 4.4.6(@babel/plugin-syntax-jsx@7.29.7(@babel/core@7.29.0(supports-color@10.2.2)))(@types/node@25.9.1)(eslint@10.4.0(jiti@2.7.0)(supports-color@10.2.2))(lightningcss@1.32.0)(magicast@0.5.3)(nuxt@4.4.6(f32df07d5233da3a29c17b72e1467465))(optionator@0.9.4)(oxlint@1.73.0)(rolldown@1.1.5)(rollup-plugin-visualizer@7.0.1(rolldown@1.1.5)(rollup@4.62.2))(rollup@4.62.2)(supports-color@10.2.2)(terser@5.49.0)(tsx@4.21.0)(typescript@6.0.3)(vue-tsc@3.3.6(typescript@6.0.3))(vue@3.5.35(typescript@6.0.3))(yaml@2.9.0) '@unhead/vue': 2.1.15(vue@3.5.35(typescript@6.0.3)) '@vue/shared': 3.5.39 chokidar: 5.0.0 @@ -27413,6 +27896,41 @@ snapshots: perfect-debounce@2.1.0: {} + pg-cloudflare@1.4.0: + optional: true + + pg-connection-string@2.14.0: {} + + pg-int8@1.0.1: {} + + pg-pool@3.14.0(pg@8.20.0): + dependencies: + pg: 8.20.0 + + pg-protocol@1.15.0: {} + + pg-types@2.2.0: + dependencies: + pg-int8: 1.0.1 + postgres-array: 2.0.0 + postgres-bytea: 1.0.1 + postgres-date: 1.0.7 + postgres-interval: 1.2.0 + + pg@8.20.0: + dependencies: + pg-connection-string: 2.14.0 + pg-pool: 3.14.0(pg@8.20.0) + pg-protocol: 1.15.0 + pg-types: 2.2.0 + pgpass: 1.0.5 + optionalDependencies: + pg-cloudflare: 1.4.0 + + pgpass@1.0.5: + dependencies: + split2: 4.2.0 + phase@0.0.1(react@19.2.6): optionalDependencies: react: 19.2.6 @@ -27684,6 +28202,16 @@ snapshots: picocolors: 1.1.1 source-map-js: 1.2.1 + postgres-array@2.0.0: {} + + postgres-bytea@1.0.1: {} + + postgres-date@1.0.7: {} + + postgres-interval@1.2.0: + dependencies: + xtend: 4.0.2 + powershell-utils@0.1.0: {} preact-render-to-string@6.5.11(preact@10.24.3): @@ -29624,6 +30152,8 @@ snapshots: undici-types@5.26.5: {} + undici-types@6.21.0: {} + undici-types@7.24.6: {} undici@5.28.4: @@ -29833,7 +30363,7 @@ snapshots: '@unrs/resolver-binding-win32-ia32-msvc': 1.12.2 '@unrs/resolver-binding-win32-x64-msvc': 1.12.2 - unstorage@1.17.5(@upstash/redis@1.38.0)(@vercel/blob@2.4.0)(@vercel/functions@3.7.5(@aws-sdk/credential-provider-web-identity@3.972.49)(ws@8.21.1(bufferutil@4.1.0)))(aws4fetch@1.0.20)(db0@0.3.4)(ioredis@5.11.1(supports-color@10.2.2)): + unstorage@1.17.5(@upstash/redis@1.38.0)(@vercel/blob@2.4.0)(@vercel/functions@3.7.5(@aws-sdk/credential-provider-web-identity@3.972.49)(ws@8.21.1(bufferutil@4.1.0)))(aws4fetch@1.0.20)(db0@0.3.4(drizzle-orm@0.45.2(@opentelemetry/api@1.9.1)(@types/pg@8.20.0)(@upstash/redis@1.38.0)(pg@8.20.0)(sql.js@1.14.1)))(ioredis@5.11.1(supports-color@10.2.2)): dependencies: anymatch: 3.1.3 chokidar: 5.0.0 @@ -29848,17 +30378,17 @@ snapshots: '@vercel/blob': 2.4.0 '@vercel/functions': 3.7.5(@aws-sdk/credential-provider-web-identity@3.972.49)(ws@8.21.1(bufferutil@4.1.0)) aws4fetch: 1.0.20 - db0: 0.3.4 + db0: 0.3.4(drizzle-orm@0.45.2(@opentelemetry/api@1.9.1)(@types/pg@8.20.0)(@upstash/redis@1.38.0)(pg@8.20.0)(sql.js@1.14.1)) ioredis: 5.11.1(supports-color@10.2.2) - unstorage@2.0.0-alpha.7(@upstash/redis@1.38.0)(@vercel/blob@2.4.0)(@vercel/functions@3.7.5(@aws-sdk/credential-provider-web-identity@3.972.49)(ws@8.21.1(bufferutil@4.1.0)))(aws4fetch@1.0.20)(chokidar@5.0.0)(db0@0.3.4)(ioredis@5.11.1(supports-color@10.2.2))(lru-cache@11.5.2)(ofetch@2.0.0-alpha.3): + unstorage@2.0.0-alpha.7(@upstash/redis@1.38.0)(@vercel/blob@2.4.0)(@vercel/functions@3.7.5(@aws-sdk/credential-provider-web-identity@3.972.49)(ws@8.21.1(bufferutil@4.1.0)))(aws4fetch@1.0.20)(chokidar@5.0.0)(db0@0.3.4(drizzle-orm@0.45.2(@opentelemetry/api@1.9.1)(@types/pg@8.20.0)(@upstash/redis@1.38.0)(pg@8.20.0)(sql.js@1.14.1)))(ioredis@5.11.1(supports-color@10.2.2))(lru-cache@11.5.2)(ofetch@2.0.0-alpha.3): optionalDependencies: '@upstash/redis': 1.38.0 '@vercel/blob': 2.4.0 '@vercel/functions': 3.7.5(@aws-sdk/credential-provider-web-identity@3.972.49)(ws@8.21.1(bufferutil@4.1.0)) aws4fetch: 1.0.20 chokidar: 5.0.0 - db0: 0.3.4 + db0: 0.3.4(drizzle-orm@0.45.2(@opentelemetry/api@1.9.1)(@types/pg@8.20.0)(@upstash/redis@1.38.0)(pg@8.20.0)(sql.js@1.14.1)) ioredis: 5.11.1(supports-color@10.2.2) lru-cache: 11.5.2 ofetch: 2.0.0-alpha.3 @@ -30489,6 +31019,8 @@ snapshots: xml-naming@0.1.0: {} + xtend@4.0.2: {} + y18n@5.0.8: {} yallist@3.1.1: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 302fa1eb2..707282ee8 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -39,6 +39,7 @@ catalog: "@vercel/connect": "0.4.2" "@vercel/otel": "2.1.3" "@vercel/sandbox": "2.8.0" + "@workflow/world-postgres": "5.0.0-beta.29" ai: "^7.0.38" marked: "17.0.6" next: "16.3.0-preview.6"