diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 0be6e0f..459fde5 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -5,17 +5,60 @@ on: branches: [ main, develop ] pull_request: branches: [ main, develop ] + # No `schedule:` trigger yet — deliberately. The tier:nightly suites spend model + # tokens on every run, so they stay opt-in via workflow_dispatch until someone + # owns watching them. To enable, uncomment below and drop the workflow_dispatch + # condition on the e2e-nightly job: + # + # schedule: + # - cron: '17 3 * * *' workflow_dispatch: + inputs: + run_enterprise: + description: 'Run E2E against the remote Enterprise server' + type: boolean + default: true + run_local_server: + description: 'Run E2E against a local OSS Conductor server' + type: boolean + default: true + run_nightly: + description: 'Run the nightly tier (live LLM + agentspan deploy tests) — spends model tokens' + type: boolean + default: false permissions: contents: read checks: write pull-requests: write +env: + # The server is built from source because releases are cut from conductor-oss + # main and no artifact is published from it: Maven's newest is a tagged RC and + # the S3 'latest' jar is months stale. Building means the CLI is tested against + # what will actually ship. + # + # TODO(#105): revisit pinning to the newest published RC instead. RCs are cut + # from main, so an RC is a main snapshot, and pulling a cached jar is faster and + # decouples this repo's CI from the server repo's build health. See the tracking + # issue linked in ADR-0001. + CONDUCTOR_SERVER_REF: 'main' + # Scratch directory the local server is started from. `conductor server start` + # writes its SQLite database relative to the working directory with no flag to + # override it (see #104), so it must not run from the repo root. + SERVER_WORKDIR: /tmp/conductor-e2e + jobs: - e2e-test: + # --------------------------------------------------------------------------- + # Remote Enterprise server. Covers the Orkes-only surface (secret, webhook, + # api-gateway) that a local OSS server cannot exercise. + # --------------------------------------------------------------------------- + e2e-enterprise: + name: E2E (Enterprise) runs-on: ubuntu-latest - + if: >- + github.event_name != 'workflow_dispatch' || + inputs.run_enterprise steps: - uses: actions/checkout@v4 @@ -36,27 +79,321 @@ jobs: - name: Setup bats uses: bats-core/bats-action@2.0.0 - - name: Run E2E tests (no credentials) + - name: Verify bats supports tag filtering + run: | + bats --version + # --filter-tags requires bats >= 1.8.0; the whole selection scheme depends + # on it, so fail loudly rather than silently running the wrong tests. + bats --help 2>&1 | grep -q -- '--filter-tags' || { + echo "::error::installed bats does not support --filter-tags (needs >= 1.8.0)" + exit 1 + } + + - name: Run unauthenticated E2E tests env: CONDUCTOR_SERVER_URL: ${{ secrets.CONDUCTOR_SERVER_URL }} + CONDUCTOR_SERVER_TYPE: Enterprise run: | - bats test/e2e/auth.bats --show-output-of-passing-tests + # Deliberately no CONDUCTOR_AUTH_KEY/SECRET: these tests assert that a + # secured server rejects anonymous calls with actionable guidance. + bats --filter-tags 'unauthenticated' test/e2e/ --show-output-of-passing-tests - - name: Run other E2E tests (with credentials) + - name: Run authenticated E2E tests env: CONDUCTOR_SERVER_URL: ${{ secrets.CONDUCTOR_SERVER_URL }} CONDUCTOR_AUTH_KEY: ${{ secrets.CONDUCTOR_AUTH_KEY }} CONDUCTOR_AUTH_SECRET: ${{ secrets.CONDUCTOR_AUTH_SECRET }} CONDUCTOR_SERVER_TYPE: Enterprise run: | - bats test/e2e/workflow.bats test/e2e/rerun.bats test/e2e/schedule.bats test/e2e/webhook.bats test/e2e/secret.bats test/e2e/task.bats test/e2e/search.bats test/e2e/whoami.bats --show-output-of-passing-tests + # Tag-selected rather than a hand-maintained file list, so new suites are + # picked up automatically. This is what recovers api_gateway.bats, which + # existed but was in no job's file list. + bats --filter-tags 'tier:pr,!oss-only,!unauthenticated' test/e2e/ --show-output-of-passing-tests + + - name: Upload test artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: e2e-artifacts-enterprise + path: | + test/e2e/*.log + test/e2e/*.json + if-no-files-found: ignore + + # --------------------------------------------------------------------------- + # Local OSS server built from conductor-oss main. Covers the OSS code paths, the + # `server` command, and validates the CLI against the code that will ship — + # none of which the Enterprise job can do. + # --------------------------------------------------------------------------- + e2e-local-server: + name: E2E (local OSS server) + runs-on: ubuntu-latest + if: >- + github.event_name != 'workflow_dispatch' || + inputs.run_local_server + steps: + - uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version: 1.23 + + - name: Set up Java + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: '21' + + - name: Build CLI + run: | + VERSION="${GITHUB_REF_NAME:-dev}" + COMMIT="${GITHUB_SHA:-none}" + DATE=$(date -u +%Y-%m-%dT%H:%M:%SZ) + go build -o conductor -v -ldflags "-X github.com/conductor-oss/conductor-cli/cmd.Version=${VERSION} -X github.com/conductor-oss/conductor-cli/cmd.Commit=${COMMIT} -X github.com/conductor-oss/conductor-cli/cmd.Date=${DATE}" . + chmod +x conductor + ./conductor --version + + - name: Check out Conductor server source + uses: actions/checkout@v4 + with: + repository: conductor-oss/conductor + ref: ${{ env.CONDUCTOR_SERVER_REF }} + path: conductor-server-src + + - name: Set up Gradle + uses: gradle/actions/setup-gradle@v4 + + - name: Build Conductor server from source + working-directory: conductor-server-src + run: | + # Module names are prefixed in settings.gradle, so the task is + # :conductor-server:bootJar, not :server:bootJar. + ./gradlew :conductor-server:bootJar -x test --console=plain + ls -la server/build/libs/*-boot.jar + + - name: Setup bats + uses: bats-core/bats-action@2.0.0 + + - name: Verify bats supports tag filtering + run: | + bats --version + bats --help 2>&1 | grep -q -- '--filter-tags' || { + echo "::error::installed bats does not support --filter-tags (needs >= 1.8.0)" + exit 1 + } + + - name: Start local Conductor server + run: | + mkdir -p "$SERVER_WORKDIR" + JAR=$(ls "$GITHUB_WORKSPACE"/conductor-server-src/server/build/libs/*-boot.jar | head -1) + echo "starting $JAR" + # Run from the scratch dir so the SQLite database is not written into the + # repository (see #104). The AI flags mirror what `conductor server start` + # passes, so agent workflows behave the same as under the CLI. + # + # NOTE: a source-built jar cannot be launched via `conductor server start`, + # which only downloads published versions. Consequently no CLI-managed pid + # file exists and the 6 server-dependent tests in server.bats skip rather + # than run. They skip loudly with a reason; see the tracking issue in + # ADR-0001 for closing that gap. + cd "$SERVER_WORKDIR" + nohup java -jar "$JAR" \ + --conductor.integrations.ai.enabled=true \ + --agentspan.embedded=true \ + > /tmp/conductor-server.log 2>&1 & + echo $! > /tmp/conductor-server.pid + + - name: Wait for server health + run: | + for i in $(seq 1 60); do + if curl -sf http://localhost:8080/health | grep -q '"healthy":true'; then + echo "server healthy after ${i}s" + exit 0 + fi + sleep 1 + done + echo "::error::server did not become healthy within 60s" + tail -n 100 /tmp/conductor-server.log || true + exit 1 + + - name: Run OSS E2E tests + env: + CONDUCTOR_SERVER_URL: http://localhost:8080/api + CONDUCTOR_SERVER_TYPE: OSS + run: | + # Excludes orkes-only; includes oss-only (the server suite). + bats --filter-tags 'tier:pr,!orkes-only' test/e2e/ --show-output-of-passing-tests + + - name: Dump server logs on failure + if: failure() + run: tail -n 200 /tmp/conductor-server.log || true + + - name: Collect server log for artifacts + if: always() + # upload-artifact does not expand '~', so copy the log into the workspace. + run: | + mkdir -p e2e-logs + cp /tmp/conductor-server.log e2e-logs/ 2>/dev/null || true + + - name: Stop local Conductor server + if: always() + run: | + [ -f /tmp/conductor-server.pid ] && kill "$(cat /tmp/conductor-server.pid)" 2>/dev/null || true + + - name: Upload test artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: e2e-artifacts-local-server + path: | + test/e2e/*.log + test/e2e/*.json + e2e-logs/conductor.log + if-no-files-found: ignore + + # --------------------------------------------------------------------------- + # Nightly tier: live LLM runs and the agentspan-dependent deploy tests. Kept off + # the PR path because it spends provider tokens, depends on PyPI, and is + # non-deterministic by nature. + # + # Currently manual-only: there is no `schedule:` trigger, so this runs solely + # when someone dispatches the workflow with run_nightly=true. The tier is still + # named "nightly" because that is its intended cadence once someone owns the + # results; renaming the tag would churn 12 tests for no gain. + # --------------------------------------------------------------------------- + e2e-nightly: + name: E2E (nightly tier - LLM + deploy, manual) + runs-on: ubuntu-latest + if: >- + github.event_name == 'workflow_dispatch' && inputs.run_nightly + env: + # The `secrets` context is not available in step-level `if`, but it is in + # job-level `env`. Reduce the secret to a boolean here so steps can gate on + # it without leaking the value. + HAS_LLM_KEY: ${{ secrets.ANTHROPIC_API_KEY != '' }} + steps: + - uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version: 1.23 + + - name: Set up Java + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: '21' + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Build CLI + run: | + VERSION="${GITHUB_REF_NAME:-dev}" + COMMIT="${GITHUB_SHA:-none}" + DATE=$(date -u +%Y-%m-%dT%H:%M:%SZ) + go build -o conductor -v -ldflags "-X github.com/conductor-oss/conductor-cli/cmd.Version=${VERSION} -X github.com/conductor-oss/conductor-cli/cmd.Commit=${COMMIT} -X github.com/conductor-oss/conductor-cli/cmd.Date=${DATE}" . + chmod +x conductor + ./conductor --version + + - name: Check out Conductor server source + uses: actions/checkout@v4 + with: + repository: conductor-oss/conductor + ref: ${{ env.CONDUCTOR_SERVER_REF }} + path: conductor-server-src + + - name: Set up Gradle + uses: gradle/actions/setup-gradle@v4 + + - name: Build Conductor server from source + working-directory: conductor-server-src + run: | + ./gradlew :conductor-server:bootJar -x test --console=plain + ls -la server/build/libs/*-boot.jar + + - name: Setup bats + uses: bats-core/bats-action@2.0.0 + + - name: Start local Conductor server + env: + # Agent executions run *inside* the server, not in the CLI, so the provider + # credential must be present in the server process's environment. Setting + # it only on the bats step would leave the server unable to reach the + # provider, and agent runs would fail with a provider auth error. + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + run: | + mkdir -p "$SERVER_WORKDIR" + JAR=$(ls "$GITHUB_WORKSPACE"/conductor-server-src/server/build/libs/*-boot.jar | head -1) + cd "$SERVER_WORKDIR" + nohup java -jar "$JAR" \ + --conductor.integrations.ai.enabled=true \ + --agentspan.embedded=true \ + > /tmp/conductor-server.log 2>&1 & + echo $! > /tmp/conductor-server.pid + + - name: Wait for server health + run: | + for i in $(seq 1 60); do + if curl -sf http://localhost:8080/health | grep -q '"healthy":true'; then + echo "server healthy after ${i}s" + exit 0 + fi + sleep 1 + done + echo "::error::server did not become healthy within 60s" + exit 1 + + - name: Run agentspan deploy tests + env: + CONDUCTOR_SERVER_URL: http://localhost:8080/api + CONDUCTOR_SERVER_TYPE: OSS + run: | + # deploy.bats provisions its own fixture venv from the pinned + # requirements.txt and skips (rather than fails) if that is not possible. + bats --filter-tags 'tier:nightly,needs:agentspan' test/e2e/ --show-output-of-passing-tests + + - name: Run live LLM agent tests + # Skipped rather than failed when no provider credential is configured, so a + # fork or a repo without the secret does not report a spurious failure. + if: env.HAS_LLM_KEY == 'true' + env: + CONDUCTOR_SERVER_URL: http://localhost:8080/api + CONDUCTOR_SERVER_TYPE: OSS + run: | + bats --filter-tags 'tier:nightly,needs:llm' test/e2e/ --show-output-of-passing-tests + + - name: Note when LLM tests were skipped + if: env.HAS_LLM_KEY != 'true' + run: echo "::warning::ANTHROPIC_API_KEY not configured — needs:llm tests were not run" + + - name: Dump server logs on failure + if: failure() + run: tail -n 200 /tmp/conductor-server.log || true + + - name: Collect server log for artifacts + if: always() + # upload-artifact does not expand '~', so copy the log into the workspace. + run: | + mkdir -p e2e-logs + cp /tmp/conductor-server.log e2e-logs/ 2>/dev/null || true + + - name: Stop local Conductor server + if: always() + run: | + [ -f /tmp/conductor-server.pid ] && kill "$(cat /tmp/conductor-server.pid)" 2>/dev/null || true - name: Upload test artifacts if: always() uses: actions/upload-artifact@v4 with: - name: e2e-test-artifacts + name: e2e-artifacts-nightly path: | test/e2e/*.log test/e2e/*.json + e2e-logs/conductor.log if-no-files-found: ignore diff --git a/.gitignore b/.gitignore index a405d09..abf19bd 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,16 @@ go.work .idea/ conductorosstest.db conductor-cli + +# E2E deploy fixture virtualenv (provisioned by test/e2e/deploy.bats) +test/e2e/fixtures/**/.venv/ + +# Local Conductor server SQLite database. `conductor server start` writes this +# relative to the current working directory with no flag to override — see #104. +c123.db +c123.db-shm +c123.db-wal + +# Python bytecode from the E2E deploy fixture +__pycache__/ +*.pyc diff --git a/CLAUDE.md b/CLAUDE.md index 646ff42..c669cd8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -87,9 +87,9 @@ by hand. ## Command Reference Commands are organized into three help groups: -- **Conductor Management** — `workflow`, `task`, `schedule`, `webhook`, `secret`, `api-gateway`, `agent`, `worker` +- **Conductor Management** — `workflow`, `task`, `schedule`, `webhook`, `secret`, `api-gateway`, `agent` - **CLI Configuration** — `config`, `whoami`, `update`, `completion` -- **Development** — `server`, `code`, `deploy`, `doctor` +- **Development** — `server`, `deploy`, `doctor`, `worker` ### Server Commands @@ -442,18 +442,10 @@ protocols. | Command | Description | Required Args | Optional Flags | Example | |---------|-------------|---------------|----------------|---------| -| `code` | Generate a project from a template (interactive) | None | `--lang`/`-l`, `--framework`/`-f`, `--template`/`-t`, `--name`/`-n` | `conductor code --lang python --template hello-world` | -| `code list` | List available templates | None | | `conductor code list` | | `deploy` | Deploy agents from your project to the server | None | `--agents`/`-a`, `--language`/`-l`, `--package`/`-p`, `--json` | `conductor deploy --language python` | | `doctor` | Check runtime and AI provider configuration | None | | `conductor doctor` | | `whoami` | Display information about the current user | None | | `conductor whoami` | -**`code` flags:** -- `--lang`, `-l` - Programming language -- `--framework`, `-f` - Framework (defaults to `core`) -- `--template`, `-t` - Template name -- `--name`, `-n` - Project name - **`deploy` flags:** - `--agents`, `-a` - Comma-separated agent names to deploy (default: all discovered) - `--language`, `-l` - Project language: `python` or `typescript` (auto-detected if omitted) diff --git a/CONTEXT.md b/CONTEXT.md new file mode 100644 index 0000000..5cd83ff --- /dev/null +++ b/CONTEXT.md @@ -0,0 +1,94 @@ +# Conductor CLI + +A command-line client for Conductor. It talks to remote Conductor servers, runs a +Conductor server locally for development, and manages the workflow, task and agent +resources those servers hold. + +## Language + +### Server + +**OSS Conductor**: +The open-source Conductor distribution. Accepts anonymous requests and does not +implement the commercial resource APIs. +_Avoid_: community edition, open Conductor, vanilla Conductor + +**Orkes Conductor**: +The commercial Conductor distribution. Requires authentication and serves +resources absent from OSS Conductor. +_Avoid_: Enterprise Conductor (except as the literal `Enterprise` server-type +value), Orkes Cloud, hosted Conductor + +**Server type**: +Which distribution an invocation is addressing. It selects client behaviour, not +merely an address — the same command can be valid against one distribution and +refused by the other. +_Avoid_: mode, flavour, edition + +**Orkes-only**: +A command or capability that exists solely on Orkes Conductor and is refused +outright by OSS Conductor. +_Avoid_: enterprise-only, premium, paid + +**Local server**: +A Conductor server that the CLI downloads and runs as a background process on the +user's machine for development and testing. +_Avoid_: embedded server, dev server, test server + +### Configuration + +**Profile**: +A named, persisted set of server and authentication settings, selected per +invocation. +_Avoid_: config, environment, context, target + +**Default profile**: +The unnamed profile read when no profile is selected. +_Avoid_: global config, base config + +### Resources + +**Definition**: +The registered, versioned template for a workflow or task. +_Avoid_: spec, schema, metadata, template + +**Execution**: +A single running or completed instance of a definition. +_Avoid_: run, instance, job, invocation + +**Agent**: +A model-backed unit of work whose turns are carried out by the server rather than +by the CLI. Consequently the *server* needs the model provider credential; the CLI +only starts and observes. +_Avoid_: assistant, bot, LLM + +**Worker**: +A process that polls the server for tasks of a given type and executes them on the +machine where it runs. The counterpart to an Agent: a Worker runs locally, an Agent +runs server-side. +_Avoid_: consumer, poller, runner + +### Testing + +**Venue**: +The kind of server a test runs against — OSS Conductor or Orkes Conductor. It +determines which tests are *valid*, not merely where they execute: a test asserting +that anonymous access is rejected is meaningless against a server that permits it. +_Avoid_: environment, target, stage + +**Tier**: +When a test is intended to run — on every pull request, or on the slower nightly +cadence. Tests that cost money, depend on a third party, or are non-deterministic +belong to the nightly tier. The name states intent, not current wiring: the nightly +tier is presently triggered manually rather than on a schedule. +_Avoid_: suite, level, category, stage + +**OSS-safe**: +A test that is valid against a local OSS Conductor server. +_Avoid_: local test, offline test + +**Known-broken guard**: +A test that asserts the *correct* behaviour of an already-filed defect and is +skipped with that issue's reference, so the gap is executable documentation and +un-skipping it is the whole fix-verification step. +_Avoid_: expected failure, xfail, pending test diff --git a/docs/adr/0001-e2e-tests-are-tag-selected-bats-suites.md b/docs/adr/0001-e2e-tests-are-tag-selected-bats-suites.md new file mode 100644 index 0000000..e40c15d --- /dev/null +++ b/docs/adr/0001-e2e-tests-are-tag-selected-bats-suites.md @@ -0,0 +1,85 @@ +--- +status: accepted +--- + +# E2E tests are bats suites selected by tier and venue tags + +The CLI's end-to-end tests are bats suites under `test/e2e/`, and CI chooses which +of them to run using bats' native tag filtering (`# bats file_tags=` / +`--filter-tags`) rather than per-job lists of filenames. Tests are tagged by +**tier** (when they run) and, where it matters, by **venue** (which Conductor +distribution they are valid against). We chose this because the E2E surface has to +span two server distributions with genuinely different capabilities, and because a +hand-maintained file list had already silently orphaned an entire suite. + +## Considered options + +**k6.** Explicitly proposed, and rejected on capability rather than preference: +k6's JavaScript runtime has no subprocess API — no `child_process`, no `os.exec` — +so it cannot invoke a CLI binary at all. Testing `conductor` with it would require +building a custom k6 with the `xk6-exec` extension and then driving a shell from +inside a load-testing VM. k6 remains the right tool if we ever want to load-test the +Conductor *server's* HTTP API, which is a different project. + +**Go integration tests driving the binary via `os/exec`.** Genuinely attractive: +one toolchain, structured JSON assertions instead of `grep`, and idiomatic gating +via build tags. Rejected because 129 bats tests already existed and worked; adding +a second E2E idiom means two things to maintain and teach, for a benefit that is +real but not decisive. + +**Positive tagging (tag every test with its venue).** Rejected after discovering +that bats `file_tags` can be *added to* by `test_tags` but never *subtracted from*. +Expressing "this file is valid everywhere except these two tests" positively would +mean tagging all ~150 remaining tests explicitly. We tag only the exceptions +(`orkes-only`, `oss-only`, `unauthenticated`) and select with negation, so the +default is "runs everywhere". + +**Separate workflow files per tier.** Clear separation and independent scheduling, +but it duplicates checkout/build/bats-setup across files and drifts. + +## Consequences + +CI runs three jobs. Two run per pull request — one against a remote Orkes server for +the Orkes-only surface, one against a pinned local OSS server — and a third holds the +tests that spend model tokens or depend on PyPI. A test's venue is a property of the +test, so adding a suite requires no CI edit; this is what recovered +`api_gateway.bats`, whose 18 tests existed but appeared in no job's file list and so +ran nowhere. + +The third job is **manual-only for now**: no `schedule:` trigger is configured, so it +runs only on an explicit `workflow_dispatch` with `run_nightly=true`. The tier keeps +the name `nightly` because that is the intended cadence, but scheduling was +deliberately deferred until someone owns watching the results — an unattended cron +that spends model tokens and reports to nobody is worse than no coverage. The cron +line is present but commented, with the one other change needed to enable it. + +The local-server job exists because the remote-only arrangement could not, even in +principle, catch certain classes of defect: it pinned `CONDUCTOR_SERVER_TYPE` to +`Enterprise`, so OSS code paths were never exercised, and it tested against a server +of unknown version rather than the code being released. The first run of the new job +found `schedule pause`/`resume` completely broken on OSS (#101). + +That job **builds the server from `conductor-oss/conductor` at `main`** rather than +downloading a published jar, because releases are cut from `main` and nothing is +published from it: Maven Central carries only tagged RCs, and the S3 `latest` jar has +not moved since June. Building is the only way to test what will actually ship. + +The costs are real and tracked in #105: a Gradle build on every PR run, and this +repo's CI becoming sensitive to the server repo's build health. That choice, and the +reasons pinning a published version was preferred but not possible, are recorded +separately in [ADR-0002](./0002-e2e-builds-the-conductor-server-from-source.md). + +One consequence is worth knowing: `conductor server start` can only download +published versions, so a source-built jar must be launched with `java -jar` directly. +No CLI-managed pid file exists, and the six server-dependent tests in `server.bats` +skip in CI. They skip with a stated reason rather than failing, and #105 proposes a +`--jar` flag that would close the gap. + +The scheme depends on bats >= 1.8.0. Both server-backed jobs assert that +`--filter-tags` is supported before running, because a bats that ignored the flag +would silently run the wrong set of tests — a worse failure than an error. + +Known-broken behaviour is recorded as a test of the correct behaviour plus a `skip` +naming the issue. This keeps CI green while making each gap executable +documentation, at the cost of requiring someone to remove the skip when the fix +lands; a stale skip is a lie, so removing it belongs in the fix. diff --git a/docs/adr/0002-e2e-builds-the-conductor-server-from-source.md b/docs/adr/0002-e2e-builds-the-conductor-server-from-source.md new file mode 100644 index 0000000..9e15977 --- /dev/null +++ b/docs/adr/0002-e2e-builds-the-conductor-server-from-source.md @@ -0,0 +1,89 @@ +--- +status: accepted +--- + +# E2E builds the Conductor server from source rather than pinning a published version + +The E2E jobs check out `conductor-oss/conductor` at `main` and run +`:conductor-server:bootJar`, instead of pinning a published version via +`conductor server start --version `. We chose this because releases are cut from +`main` and **no artifact is published from `main`**, so a pinned version cannot +validate the code that will actually ship. This is a deliberate trade of speed, +reproducibility and CI isolation for fidelity, and it is expected to be revisited — +see [#105](https://github.com/conductor-oss/conductor-cli/issues/105). + +## Why pinning was the preferred option, and why we did not take it + +Pinning is the better engineering default and was the original plan. `@mp-orkes` +raised it in review of [#106](https://github.com/conductor-oss/conductor-cli/pull/106): + +> I really don't think we should be doing the gradle build from `main`. We should do +> something like `conductor server start --version 3.32.0-rc.23` + +It is faster (a cached jar versus a Gradle build), byte-for-byte reproducible between +runs, and it keeps this repo's CI independent of the server repo's build health. It +also uses the CLI's own `server start`, which means the `server` command is exercised +rather than bypassed. + +What blocked it is narrow and factual: **there is no published artifact built from +`main`.** + +- Maven Central's `org.conductoross:conductor-server` publishes tagged releases only. + At the time of writing its newest was `3.32.0-rc.23`. +- The S3 jar that `conductor server start` downloads as `latest` had + `Last-Modified: 3 June 2026` — months stale, and apparently no longer published. + +Release validation for this cycle was explicitly scoped to `main`, because that is +what the release is cut from. Pinning `3.32.0-rc.23` would have tested a snapshot 8 +commits behind `main`. In this instance those 8 commits were UI fixes, a CI fix, +provider API-key trimming and agent token-usage aggregation — nothing CLI-facing — so +pinning would have been *adequate*. But that is a property of this particular gap, not +a guarantee, and it is only knowable after the fact. + +## Consequences + +**A Gradle build runs on every PR.** Measured at 3m56s for the whole job on a cold +runner, which is acceptable but not free, and it will grow with the server. + +**This repo's CI becomes sensitive to the server repo.** A broken `main` in +`conductor-oss/conductor` turns conductor-cli PRs red for reasons unrelated to the +CLI. This is the most significant cost and the most likely trigger for reverting to a +pin. + +**Runs are not reproducible.** Two runs of the same CLI commit can test different +server code. A failure may not reproduce later. + +**`server start` is bypassed, so six tests skip.** It can only download published +versions, so a source-built jar must be launched with `java -jar`. With no CLI-managed +pid file, the six server-dependent tests in `server.bats` skip with a stated reason. +#105 proposes a `--jar` flag to close that. + +## When to revisit + +Any of these should prompt switching back to a pin: + +- A red CI run traced to the server build rather than the CLI. +- A `main`-tracking snapshot becoming available — either a fixed S3 `latest` or + published snapshots. That removes the reason for this decision entirely. +- Release validation no longer being scoped to unreleased `main`. + +The workflow keeps a single knob for this: `CONDUCTOR_SERVER_REF`. Reverting means +replacing the checkout-and-build steps with `conductor server start --version `, +which the earlier revision of #106 already implemented, so the change is recoverable +from git history rather than needing redesign. + +## Alternatives considered + +**Pin the newest published RC.** Covered above. Rejected for this cycle only because +it cannot test `main`; preferred on every other axis. + +**Ask for `main` snapshots to be published.** The best outcome — it would make this +ADR obsolete and give CI a cheap, current artifact. Depends on another team, so it was +not available for this release. Raised in #105, together with the apparently stalled +S3 `latest` publish. + +**Test only against the remote Enterprise server, as CI did before.** Rejected: it +pins `CONDUCTOR_SERVER_TYPE=Enterprise`, so OSS code paths were never exercised, and it +targets a server of unknown version. That arrangement is what allowed +[#101](https://github.com/conductor-oss/conductor-cli/issues/101) — +`schedule pause`/`resume` completely broken on OSS — to go unnoticed. diff --git a/test/e2e/README.md b/test/e2e/README.md new file mode 100644 index 0000000..79966cb --- /dev/null +++ b/test/e2e/README.md @@ -0,0 +1,134 @@ +# E2E tests + +Bats suites that drive the built `conductor` binary against a real Conductor +server. See [ADR-0001](../../docs/adr/0001-e2e-tests-are-tag-selected-bats-suites.md) +for why bats and why tags; see [CONTEXT.md](../../CONTEXT.md) for the meaning of +*tier*, *venue* and *Orkes-only*. + +## Prerequisites + +```bash +brew install bats-core # >= 1.8.0, required for --filter-tags +go build -o conductor . # suites invoke ./conductor from the repo root +``` + +Run from the **repository root**, not from this directory. + +## Running + +Against a local OSS server: + +```bash +conductor server start --version 3.32.0-rc.23 # from a scratch dir, see note below +export CONDUCTOR_SERVER_URL=http://localhost:8080/api +export CONDUCTOR_SERVER_TYPE=OSS +bats --filter-tags 'tier:pr,!orkes-only' test/e2e/ +``` + +To test against the code that will actually ship, build the server from source +instead — releases are cut from `conductor-oss/conductor` `main`, and no artifact is +published from it: + +```bash +cd ../conductor && git checkout main && git pull +./gradlew :conductor-server:bootJar -x test # note the conductor- prefix +mkdir -p /tmp/conductor-e2e && cd /tmp/conductor-e2e +java -jar ../../conductor/server/build/libs/*-boot.jar \ + --conductor.integrations.ai.enabled=true --agentspan.embedded=true +``` + +This is what CI does. Because `conductor server start` can only download published +versions, a source-built jar has no CLI-managed pid file, so the six server-dependent +tests in `server.bats` skip. See #105. + +Against an Orkes server: + +```bash +export CONDUCTOR_SERVER_URL=https://your-server/api +export CONDUCTOR_AUTH_KEY=... CONDUCTOR_AUTH_SECRET=... +export CONDUCTOR_SERVER_TYPE=Enterprise +bats --filter-tags 'tier:pr,!oss-only,!unauthenticated' test/e2e/ +bats --filter-tags 'unauthenticated' test/e2e/ # run with the key/secret UNSET +``` + +Nightly tier (spends model tokens, installs `agentspan` from PyPI): + +```bash +bats --filter-tags 'tier:nightly' test/e2e/ +``` + +> In CI the nightly tier is **manual-only**: there is no cron trigger, so it runs +> only via *Run workflow* on the Actions tab with `run_nightly` checked. The tag is +> named for its intended cadence, not its current wiring. Enabling the schedule means +> uncommenting the `schedule:` block in `.github/workflows/e2e.yml` and dropping the +> `workflow_dispatch` condition on the `e2e-nightly` job. + +A single suite, or a count without running anything: + +```bash +bats test/e2e/agent.bats +bats --count --filter-tags 'tier:pr,!orkes-only' test/e2e/ +``` + +> Start the local server from a scratch directory (e.g. `/tmp/conductor-e2e`). +> `conductor server start` writes its SQLite database relative to the working +> directory with no flag to override it, so starting it from the repo drops a +> multi-hundred-MB `c123.db` here. See issue #104. + +## Tags + +| Tag | Meaning | +|-----|---------| +| `tier:pr` | Runs on every pull request. Free and deterministic. | +| `tier:nightly` | Costs money, or depends on a third party, or is non-deterministic. Intended for a nightly cadence; currently run manually only. | +| `orkes-only` | Requires Orkes Conductor. OSS refuses the operation. | +| `oss-only` | Requires a local OSS server; meaningless against a remote Orkes server. | +| `unauthenticated` | Needs a *secured* server reached *without* credentials. Must be excluded from authenticated runs. | +| `needs:llm` | Requires a model provider credential. | +| `needs:agentspan` | Requires the `agentspan` Python package. | +| `needs:timeout` | Requires GNU `timeout(1)` (absent on stock macOS). | + +Selection uses **negation**, so untagged tests run everywhere. Tag only the +exceptions. `bats` `file_tags` can be added to by `test_tags` but never subtracted, +which is why the default has to be permissive. + +## Conventions + +- One suite per command, named `.bats`. +- Numbered test names (`"1. Create workflow definition"`), so ordering is legible in + output. +- `setup_file` checks for `./conductor` and cleans up leftovers from prior runs; + `teardown_file` cleans up again. Tests must be re-runnable without manual reset. +- Assert on `$status` as well as `$output`. The CLI exits non-zero on failure, so + status assertions are meaningful. +- `echo "Output: $output"` before assertions, so failures are diagnosable from CI + logs. +- Suites are self-contained: helpers are duplicated per file rather than shared, in + keeping with the existing suites. + +### Tests for known defects + +Assert the **correct** behaviour, then `skip` with the issue number: + +```bash +@test "12. Agent compile returns an execution plan" { + skip "known broken: #96 — CLI sends a bare config, server rejects it" + ... +} +``` + +CI stays green, the gap is executable documentation, and verifying a fix is just +removing the line. A skip that outlives its defect is a lie — delete it as part of +the fix. + +### Mixed-tier suites + +Put the tier in `test_tags` per test rather than in `file_tags`, since `file_tags` +applies to every test in the file and cannot be overridden. `agent.bats` does this. + +## Fixtures + +`fixtures/python_agents/` is a minimal project used by `deploy.bats` to exercise +agent discovery. `deploy.bats` provisions its `.venv` on first run from the pinned +`requirements.txt`, and *skips* rather than fails when that is not possible, so a +PyPI outage cannot turn into a red build. diff --git a/test/e2e/agent.bats b/test/e2e/agent.bats new file mode 100644 index 0000000..96a459f --- /dev/null +++ b/test/e2e/agent.bats @@ -0,0 +1,276 @@ +#!/usr/bin/env bats + +# E2E tests for agent commands +# Covers config scaffolding, definition CRUD, execution search, and (in the +# nightly tier) live agent runs that call a real LLM. +# +# Tier is declared per test rather than at file level: the offline/CRUD tests are +# free and deterministic (tier:pr), while runs that invoke a provider cost tokens +# and are non-deterministic (tier:nightly,needs:llm). Assertions on LLM tests are +# structural only — execution ids and terminal status, never generated text. + +# Tier is set per test; no file-level tier, because bats file_tags can only be +# added to by test_tags, never subtracted. + +AGENT_NAME="e2e_agent_probe" + +# A model verified to exist; the `agent init` default (openai/gpt-4o) is not +# usable unless an OpenAI key happens to be configured — see #103. +LLM_MODEL="anthropic/claude-haiku-4-5-20251001" + +setup_file() { + # Ensure the CLI binary exists + if [ ! -f "./conductor" ]; then + echo "ERROR: conductor binary not found. Please build it first." + exit 1 + fi + + ./conductor agent delete "$AGENT_NAME" -y 2>/dev/null || true +} + +teardown_file() { + ./conductor agent delete "$AGENT_NAME" -y 2>/dev/null || true +} + +# Helper: skip when no LLM provider credential is available. +require_llm() { + if [ -z "$ANTHROPIC_API_KEY" ]; then + skip "no ANTHROPIC_API_KEY configured" + fi +} + +# Helper: skip when the server has no Agents API. Not every Conductor deployment +# enables it — including the Enterprise server CI targets, which answers +# "Agents API is not available on this Conductor server". That is a deployment fact, +# not a CLI defect, so skip rather than fail. Applied per test rather than in +# setup() so the offline `agent init` tests still run everywhere. +require_agents_api() { + if ./conductor agent list 2>&1 | grep -q 'Agents API is not available'; then + skip "server has no Agents API enabled" + fi +} + +# Helper: write an agent config that uses a known-good model. +write_agent_config() { + local path="$1" + cat > "$path" <"$BATS_TEST_TMPDIR/bounded.out" 2>&1 & + local pid=$! + local i=0 + while [ "$i" -lt "$secs" ]; do + kill -0 "$pid" 2>/dev/null || { wait "$pid"; return $?; } + sleep 1 + i=$((i + 1)) + done + kill "$pid" 2>/dev/null + wait "$pid" 2>/dev/null + return 124 +} + +# ---- definition scaffolding and CRUD (tier:pr) ---- + +# bats test_tags=tier:pr +@test "1. Agent init creates a YAML config" { + run bash -c "cd '$BATS_TEST_TMPDIR' && '$PWD/conductor' agent init inittest 2>&1" + echo "Output: $output" + [ "$status" -eq 0 ] + [ -f "$BATS_TEST_TMPDIR/inittest.yaml" ] +} + +# bats test_tags=tier:pr +@test "2. Agent init --format json creates a JSON config" { + run bash -c "cd '$BATS_TEST_TMPDIR' && '$PWD/conductor' agent init jsontest --format json 2>&1" + echo "Output: $output" + [ "$status" -eq 0 ] + [ -f "$BATS_TEST_TMPDIR/jsontest.json" ] +} + +# bats test_tags=tier:pr +@test "3. Agent init --strategy records the strategy" { + run bash -c "cd '$BATS_TEST_TMPDIR' && '$PWD/conductor' agent init strattest --strategy handoff 2>&1" + echo "Output: $output" + [ "$status" -eq 0 ] + grep -q 'handoff' "$BATS_TEST_TMPDIR/strattest.yaml" +} + +# bats test_tags=tier:pr +@test "4. Agent list succeeds" { + require_agents_api + run bash -c "./conductor agent list 2>&1" + echo "Output: $output" + [ "$status" -eq 0 ] +} + +# bats test_tags=tier:pr +@test "5. Agent list --json produces valid JSON" { + require_agents_api + run bash -c "./conductor agent list --json 2>/dev/null" + echo "Output: $output" + [ "$status" -eq 0 ] + echo "$output" | python3 -c 'import json,sys; json.load(sys.stdin)' +} + +# bats test_tags=tier:pr +@test "6. Agent get for an unknown name fails" { + require_agents_api + run bash -c "./conductor agent get e2e_agent_definitely_absent 2>&1" + echo "Output: $output" + [ "$status" -ne 0 ] +} + +# bats test_tags=tier:pr +@test "7. Agent delete for an unknown name fails" { + require_agents_api + run bash -c "./conductor agent delete e2e_agent_definitely_absent -y 2>&1" + echo "Output: $output" + [ "$status" -ne 0 ] +} + +# ---- execution search (tier:pr) ---- + +# bats test_tags=tier:pr +@test "8. Agent execution search succeeds" { + require_agents_api + run bash -c "./conductor agent execution 2>&1" + echo "Output: $output" + [ "$status" -eq 0 ] +} + +# bats test_tags=tier:pr +@test "9. Agent execution --name filter succeeds" { + require_agents_api + run bash -c "./conductor agent execution --name e2e_agent_definitely_absent 2>&1" + echo "Output: $output" + [ "$status" -eq 0 ] + [[ "$output" == *"No executions found"* ]] +} + +# bats test_tags=tier:pr +@test "10. Agent execution --status filter succeeds" { + require_agents_api + run bash -c "./conductor agent execution --status FAILED 2>&1" + echo "Output: $output" + [ "$status" -eq 0 ] +} + +# bats test_tags=tier:pr +@test "11. Agent prune --dry-run does not delete" { + require_agents_api + run bash -c "./conductor agent prune --older-than 3650 --dry-run 2>&1" + echo "Output: $output" + [ "$status" -eq 0 ] + [[ "$output" == *"Dry run"* ]] +} + +# ---- known-broken guards ---- + +# Regression guard for #96: the CLI posts a bare config instead of +# {"agentConfig": ...}, so compile fails for every input. +# bats test_tags=tier:pr +@test "12. Agent compile returns an execution plan" { + skip "known broken: #96 — CLI sends a bare config, server rejects it" + write_agent_config "$BATS_TEST_TMPDIR/compile.yaml" + run bash -c "./conductor agent compile '$BATS_TEST_TMPDIR/compile.yaml' 2>&1" + echo "Output: $output" + [ "$status" -eq 0 ] +} + +# ---- live LLM runs (tier:nightly) ---- + +# bats test_tags=tier:nightly,needs:llm +@test "13. Agent run --config completes and returns an execution id" { + require_agents_api + require_llm + write_agent_config "$BATS_TEST_TMPDIR/run.yaml" + + run bash -c "./conductor agent run --config '$BATS_TEST_TMPDIR/run.yaml' 'Reply with one word' --no-stream 2>&1" + echo "Output: $output" + [ "$status" -eq 0 ] + # Structural assertion only: a UUID execution id must be present. + [[ "$output" =~ [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} ]] +} + +# bats test_tags=tier:nightly,needs:llm +@test "14. Agent run registers the agent, then run --name works" { + require_agents_api + require_llm + write_agent_config "$BATS_TEST_TMPDIR/run2.yaml" + ./conductor agent run --config "$BATS_TEST_TMPDIR/run2.yaml" "Reply with one word" --no-stream >/dev/null 2>&1 + + run bash -c "./conductor agent list 2>/dev/null" + echo "List: $output" + [[ "$output" == *"$AGENT_NAME"* ]] + + run bash -c "./conductor agent run --name $AGENT_NAME 'Reply with one word' --no-stream 2>&1" + echo "Output: $output" + [ "$status" -eq 0 ] +} + +# bats test_tags=tier:nightly,needs:llm +@test "15. Agent status reports a terminal state for a finished run" { + require_agents_api + require_llm + write_agent_config "$BATS_TEST_TMPDIR/run3.yaml" + out=$(./conductor agent run --config "$BATS_TEST_TMPDIR/run3.yaml" "Reply with one word" --no-stream 2>&1) + eid=$(echo "$out" | grep -oE '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}' | head -1) + echo "Execution: $eid" + [ -n "$eid" ] + + # Poll until terminal, bounded. + status_val="" + for _ in $(seq 1 30); do + status_val=$(./conductor agent status "$eid" 2>/dev/null | grep '"status"' | cut -d'"' -f4) + [ "$status_val" != "RUNNING" ] && break + sleep 2 + done + echo "Final status: $status_val" + [ -n "$status_val" ] + [ "$status_val" != "RUNNING" ] +} + +# Regression guard for #97: --since and --window return nothing even when +# matching executions exist. +# bats test_tags=tier:nightly,needs:llm +@test "16. Agent execution --since finds a just-created execution" { + skip "known broken: #97 — --since/--window always return no results" + require_llm + write_agent_config "$BATS_TEST_TMPDIR/run4.yaml" + ./conductor agent run --config "$BATS_TEST_TMPDIR/run4.yaml" "Reply with one word" --no-stream >/dev/null 2>&1 + + run bash -c "./conductor agent execution --since 1d 2>&1" + echo "Output: $output" + [ "$status" -eq 0 ] + [[ "$output" != *"No executions found"* ]] +} + +# Regression guard for #102: agent stream never exits once the execution is +# terminal. run_bounded keeps this from hanging CI if the skip is removed before +# the fix lands. +# bats test_tags=tier:nightly,needs:llm +@test "17. Agent stream exits after the terminal event" { + skip "known broken: #102 — agent stream hangs on a terminal execution" + require_llm + write_agent_config "$BATS_TEST_TMPDIR/run5.yaml" + out=$(./conductor agent run --config "$BATS_TEST_TMPDIR/run5.yaml" "Reply with one word" 2>&1) + eid=$(echo "$out" | grep -oE '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}' | head -1) + [ -n "$eid" ] + + run_bounded 30 ./conductor agent stream "$eid" + rc=$? + echo "stream exit: $rc" + cat "$BATS_TEST_TMPDIR/bounded.out" || true + [ "$rc" -ne 124 ] +} diff --git a/test/e2e/api_gateway.bats b/test/e2e/api_gateway.bats index c39ebbb..5ad65b7 100644 --- a/test/e2e/api_gateway.bats +++ b/test/e2e/api_gateway.bats @@ -3,6 +3,8 @@ # E2E tests for API Gateway functionality # Tests service, auth config, and route management +# bats file_tags=tier:pr,orkes-only + SERVICE_ID="e2e_test_service" AUTH_CONFIG_ID="e2e_test_auth" WORKFLOW_NAME="cli_e2e_test_workflow_2" @@ -14,6 +16,15 @@ setup() { exit 1 fi + # API Gateway is an opt-in Orkes capability, not present on every Enterprise + # deployment. Where the endpoints are absent the server answers 404 with + # "No static resource api/gateway/...", which is a deployment fact rather than a + # CLI defect — so skip loudly instead of reporting 18 spurious failures. + if ./conductor api-gateway service list 2>&1 | + grep -qE 'No static resource api/gateway|API Gateway management is only available'; then + skip "server does not have API Gateway enabled" + fi + # Ensure test workflow exists if [ ! -f "test/e2e/test-workflow-2.json" ]; then echo "ERROR: test-workflow-2.json not found" diff --git a/test/e2e/auth.bats b/test/e2e/auth.bats index cfd0d6a..bf43d1a 100755 --- a/test/e2e/auth.bats +++ b/test/e2e/auth.bats @@ -3,6 +3,16 @@ # E2E tests for authentication error handling # These tests verify that the CLI provides helpful error messages when authentication fails # This test assumes no credentials are set. +# +# Tagged both orkes-only and unauthenticated, because it needs a *secured* server +# reached *without* credentials: +# - orkes-only: against OSS, anonymous access is legitimate, so commands +# succeed and the "should fail" assertions are invalid. +# - unauthenticated: it must be excluded from any run that supplies +# CONDUCTOR_AUTH_KEY/SECRET, or the calls succeed and the +# same assertions fail for the opposite reason. + +# bats file_tags=tier:pr,orkes-only,unauthenticated setup() { # Ensure the CLI binary exists diff --git a/test/e2e/config.bats b/test/e2e/config.bats new file mode 100644 index 0000000..f7b10a2 --- /dev/null +++ b/test/e2e/config.bats @@ -0,0 +1,118 @@ +#!/usr/bin/env bats + +# E2E tests for config profile management +# Tests save / list / delete and profile selection precedence. +# +# config operates purely on ~/.conductor-cli/ and needs no server, so it is +# venue-agnostic. Tests use a dedicated profile name and clean up after +# themselves so they never disturb a developer's real profiles. + +# bats file_tags=tier:pr + +PROFILE="e2e_cfg_probe" +PROFILE_2="e2e_cfg_probe_2" + +setup_file() { + # Ensure the CLI binary exists + if [ ! -f "./conductor" ]; then + echo "ERROR: conductor binary not found. Please build it first." + exit 1 + fi + + # Remove leftovers from previous runs + ./conductor config delete "$PROFILE" -y 2>/dev/null || true + ./conductor config delete "$PROFILE_2" -y 2>/dev/null || true +} + +teardown_file() { + ./conductor config delete "$PROFILE" -y 2>/dev/null || true + ./conductor config delete "$PROFILE_2" -y 2>/dev/null || true +} + +# Helper: create a profile non-interactively by accepting every prompt default. +save_profile() { + local name="$1" + printf '\n\n\n\n\n' | ./conductor config save --profile "$name" >/dev/null 2>&1 +} + +@test "1. Save a named profile" { + run bash -c "printf '\n\n\n\n\n' | ./conductor config save --profile $PROFILE 2>&1" + echo "Output: $output" + [ "$status" -eq 0 ] + [[ "$output" == *"config-$PROFILE.yaml"* ]] +} + +@test "2. List shows the saved profile" { + save_profile "$PROFILE" + + run bash -c "./conductor config list 2>/dev/null" + echo "Output: $output" + [ "$status" -eq 0 ] + [[ "$output" == *"$PROFILE"* ]] +} + +@test "3. Saved profile is usable via --profile" { + save_profile "$PROFILE" + + run bash -c "./conductor --profile $PROFILE config list 2>/dev/null" + echo "Output: $output" + [ "$status" -eq 0 ] +} + +@test "4. Selecting a nonexistent profile fails with a helpful message" { + run bash -c "./conductor --profile e2e_definitely_absent workflow list 2>&1" + echo "Output: $output" + [ "$status" -ne 0 ] + [[ "$output" == *"e2e_definitely_absent"* ]] +} + +@test "5. Multiple profiles coexist" { + save_profile "$PROFILE" + save_profile "$PROFILE_2" + + run bash -c "./conductor config list 2>/dev/null" + echo "Output: $output" + [ "$status" -eq 0 ] + [[ "$output" == *"$PROFILE"* ]] + [[ "$output" == *"$PROFILE_2"* ]] +} + +@test "6. Delete a profile with -y" { + save_profile "$PROFILE_2" + + run bash -c "./conductor config delete $PROFILE_2 -y 2>&1" + echo "Output: $output" + [ "$status" -eq 0 ] + + run bash -c "./conductor config list 2>/dev/null" + echo "After delete: $output" + [[ "$output" != *"$PROFILE_2"* ]] +} + +@test "7. Delete accepts the profile via --profile flag" { + save_profile "$PROFILE_2" + + run bash -c "./conductor config delete --profile $PROFILE_2 -y 2>&1" + echo "Output: $output" + [ "$status" -eq 0 ] +} + +@test "8. Save without a profile name is rejected" { + # config save always targets config-.yaml; with no --profile it prompts, + # and empty input is an error rather than a default-profile write. + run bash -c "printf '\n' | ./conductor config save 2>&1" + echo "Output: $output" + [ "$status" -ne 0 ] + [[ "$output" == *"profile name is required"* ]] +} + +# Regression guard for #98: the default profile (~/.conductor-cli/config.yaml) is +# still read but cannot be created or updated through the CLI. Asserts the +# desired end state. +@test "9. Default profile can be managed through the CLI" { + skip "known broken: #98 — config save cannot write the default config.yaml" + run bash -c "printf '\n\n\n\n\n' | ./conductor config save 2>&1" + echo "Output: $output" + [ "$status" -eq 0 ] + [[ "$output" == *"config.yaml"* ]] +} diff --git a/test/e2e/deploy.bats b/test/e2e/deploy.bats new file mode 100644 index 0000000..7fe36f0 --- /dev/null +++ b/test/e2e/deploy.bats @@ -0,0 +1,131 @@ +#!/usr/bin/env bats + +# E2E tests for the deploy command +# Covers agent discovery, deployment, name filtering, JSON output and error paths +# against the fixture project in test/e2e/fixtures/python_agents. +# +# deploy shells out to `python -m agentspan.cli.discover`, so this suite depends +# on the external `agentspan` package (pinned in the fixture's requirements.txt). +# That external dependency is why it sits in the nightly tier behind +# needs:agentspan rather than running on every PR — a PyPI outage or an SDK API +# change must not block merges. +# +# Only the Python path is covered; TypeScript deploy is deliberately out of scope. + +# bats file_tags=tier:nightly,needs:agentspan + +FIXTURE="test/e2e/fixtures/python_agents" +PKG="fixture_agents" +AGENT_1="e2e_fixture_greeter" +AGENT_2="e2e_fixture_echo" + +setup_file() { + # Ensure the CLI binary exists + if [ ! -f "./conductor" ]; then + echo "ERROR: conductor binary not found. Please build it first." + exit 1 + fi + + if [ ! -d "$FIXTURE" ]; then + echo "ERROR: fixture project not found at $FIXTURE" + exit 1 + fi + + # Provision the fixture venv once per file. conductor deploy auto-detects + # /.venv/bin/python. + if [ ! -x "$FIXTURE/.venv/bin/python" ]; then + python3 -m venv "$FIXTURE/.venv" >/dev/null 2>&1 || return 0 + "$FIXTURE/.venv/bin/pip" install -q -r "$FIXTURE/requirements.txt" >/dev/null 2>&1 || return 0 + fi + + ./conductor agent delete "$AGENT_1" -y 2>/dev/null || true + ./conductor agent delete "$AGENT_2" -y 2>/dev/null || true +} + +teardown_file() { + ./conductor agent delete "$AGENT_1" -y 2>/dev/null || true + ./conductor agent delete "$AGENT_2" -y 2>/dev/null || true +} + +# Helper: skip when the fixture venv could not be provisioned (offline CI, PyPI +# outage). Better to skip loudly than to fail for an unrelated reason. +require_agentspan() { + if [ ! -x "$FIXTURE/.venv/bin/python" ]; then + skip "fixture venv not provisioned (agentspan unavailable)" + fi + if ! "$FIXTURE/.venv/bin/python" -c 'import agentspan.cli.discover' 2>/dev/null; then + skip "agentspan.cli.discover not importable in fixture venv" + fi +} + +@test "1. Deploy discovers both fixture agents" { + require_agentspan + + run bash -c "cd '$FIXTURE' && '$PWD/conductor' deploy --language python --package $PKG -y 2>&1" + echo "Output: $output" + [ "$status" -eq 0 ] + [[ "$output" == *"$AGENT_1"* ]] + [[ "$output" == *"$AGENT_2"* ]] +} + +@test "2. Deploy reports the discovered count" { + require_agentspan + + run bash -c "cd '$FIXTURE' && '$PWD/conductor' deploy --language python --package $PKG -y 2>&1" + echo "Output: $output" + [ "$status" -eq 0 ] + [[ "$output" == *"Discovered 2 agent(s)"* ]] +} + +@test "3. Deployed agents appear in agent list" { + require_agentspan + bash -c "cd '$FIXTURE' && '$PWD/conductor' deploy --language python --package $PKG -y" >/dev/null 2>&1 || true + + run bash -c "./conductor agent list 2>/dev/null" + echo "Output: $output" + [ "$status" -eq 0 ] + [[ "$output" == *"$AGENT_1"* ]] + [[ "$output" == *"$AGENT_2"* ]] +} + +@test "4. Deploy --agents filters to a single agent" { + require_agentspan + + run bash -c "cd '$FIXTURE' && '$PWD/conductor' deploy --language python --package $PKG --agents $AGENT_2 -y 2>&1" + echo "Output: $output" + [ "$status" -eq 0 ] + [[ "$output" == *"All 1 agent(s) deployed successfully"* ]] +} + +@test "5. Deploy --json emits valid JSON with a summary" { + require_agentspan + + run bash -c "cd '$FIXTURE' && '$PWD/conductor' deploy --language python --package $PKG --json -y 2>/dev/null" + echo "Output: $output" + [ "$status" -eq 0 ] + echo "$output" | python3 -c ' +import json,sys +d = json.load(sys.stdin) +assert "summary" in d, "missing summary" +assert d["summary"]["total"] == 2, d["summary"] +assert d["summary"]["failed"] == 0, d["summary"] +' +} + +@test "6. Deploy with an unknown agent name fails and lists the known ones" { + require_agentspan + + run bash -c "cd '$FIXTURE' && '$PWD/conductor' deploy --language python --package $PKG --agents e2e_not_a_real_agent -y 2>&1" + echo "Output: $output" + [ "$status" -ne 0 ] + [[ "$output" == *"not found"* ]] + [[ "$output" == *"$AGENT_1"* ]] +} + +@test "7. Deploy with an unknown package fails" { + require_agentspan + + run bash -c "cd '$FIXTURE' && '$PWD/conductor' deploy --language python --package definitely_not_a_package -y 2>&1" + echo "Output: $output" + [ "$status" -ne 0 ] +} diff --git a/test/e2e/doctor.bats b/test/e2e/doctor.bats new file mode 100644 index 0000000..96853e3 --- /dev/null +++ b/test/e2e/doctor.bats @@ -0,0 +1,77 @@ +#!/usr/bin/env bats + +# E2E tests for the doctor command +# Tests runtime detection, server reporting, and AI provider detection. +# +# doctor is read-only and venue-agnostic: it reports configuration rather than +# calling server APIs, so it runs against OSS and Enterprise alike. + +# bats file_tags=tier:pr + +setup_file() { + # Ensure the CLI binary exists + if [ ! -f "./conductor" ]; then + echo "ERROR: conductor binary not found. Please build it first." + exit 1 + fi +} + +@test "1. Doctor runs and reports the three sections" { + run bash -c "./conductor doctor 2>&1" + echo "Output: $output" + [ "$status" -eq 0 ] + [[ "$output" == *"Runtime"* ]] + [[ "$output" == *"Conductor server"* ]] + [[ "$output" == *"AI Providers"* ]] +} + +@test "2. Doctor reports the configured server URL" { + run bash -c "./conductor doctor 2>&1" + echo "Output: $output" + [ "$status" -eq 0 ] + [[ "$output" == *"Server:"* ]] +} + +@test "3. Doctor reports Java presence or absence explicitly" { + run bash -c "./conductor doctor 2>&1" + echo "Output: $output" + [ "$status" -eq 0 ] + # Either "ok Java " or the "-- Java not found" advisory. + [[ "$output" == *"Java"* ]] +} + +@test "4. Doctor summarises the AI provider count" { + run bash -c "./conductor doctor 2>&1" + echo "Output: $output" + [ "$status" -eq 0 ] + [[ "$output" == *"AI provider(s) configured"* ]] +} + +@test "5. Doctor honours an explicit --server flag" { + run bash -c "./conductor --server http://example.invalid:9999/api doctor 2>&1" + echo "Output: $output" + # doctor reports configuration; it must not fail merely because the URL is unreachable. + [ "$status" -eq 0 ] + [[ "$output" == *"example.invalid:9999"* ]] +} + +@test "6. Doctor lists a known provider env var name" { + run bash -c "./conductor doctor 2>&1" + echo "Output: $output" + [ "$status" -eq 0 ] + # Provider rows name their env vars whether configured or not. + [[ "$output" == *"ANTHROPIC_API_KEY"* ]] + [[ "$output" == *"OPENAI_API_KEY"* ]] +} + +# Regression guard for #103: doctor advertises specific model strings that have +# drifted out of date (two Anthropic models return 404 from the provider API). +# Asserts the desired end state — that doctor does not print known-dead models. +@test "7. Doctor does not advertise retired model identifiers" { + skip "known broken: #103 — doctor hardcodes stale model strings" + run bash -c "./conductor doctor 2>&1" + echo "Output: $output" + [ "$status" -eq 0 ] + [[ "$output" != *"claude-sonnet-4-20250514"* ]] + [[ "$output" != *"claude-3-5-sonnet-20241022"* ]] +} diff --git a/test/e2e/fixtures/python_agents/README.md b/test/e2e/fixtures/python_agents/README.md new file mode 100644 index 0000000..832524c --- /dev/null +++ b/test/e2e/fixtures/python_agents/README.md @@ -0,0 +1,17 @@ +# Python deploy fixture + +Minimal project used by `test/e2e/deploy.bats` to exercise `conductor deploy` +discovery and deployment against a live server. + +`conductor deploy` shells out to `python -m agentspan.cli.discover`, so the +`agentspan` package must be importable. The suite creates a `.venv` here on first +run; `conductor deploy` auto-detects `/.venv/bin/python`. + +Setup performed by the suite (or run manually): + +```bash +python3 -m venv .venv +./.venv/bin/pip install -r requirements.txt +``` + +Note `--path` discovery skips `__init__.py`, so the agents live in `agents.py`. diff --git a/test/e2e/fixtures/python_agents/fixture_agents/__init__.py b/test/e2e/fixtures/python_agents/fixture_agents/__init__.py new file mode 100644 index 0000000..e90bb82 --- /dev/null +++ b/test/e2e/fixtures/python_agents/fixture_agents/__init__.py @@ -0,0 +1 @@ +"""Fixture package for exercising `conductor deploy` agent discovery.""" diff --git a/test/e2e/fixtures/python_agents/fixture_agents/agents.py b/test/e2e/fixtures/python_agents/fixture_agents/agents.py new file mode 100644 index 0000000..c9796de --- /dev/null +++ b/test/e2e/fixtures/python_agents/fixture_agents/agents.py @@ -0,0 +1,21 @@ +"""Module-level Agent variables are what `agentspan.cli.discover` scans for. + +Kept deliberately minimal: these agents are only ever discovered and deployed by +the E2E suite, never executed, so the model is nominal. +""" + +from agentspan.agents import Agent + +e2e_fixture_greeter = Agent( + name="e2e_fixture_greeter", + model="anthropic/claude-haiku-4-5-20251001", + instructions="You greet the user in exactly one short word.", + max_turns=2, +) + +e2e_fixture_echo = Agent( + name="e2e_fixture_echo", + model="anthropic/claude-haiku-4-5-20251001", + instructions="You echo the user's input verbatim.", + max_turns=2, +) diff --git a/test/e2e/fixtures/python_agents/requirements.txt b/test/e2e/fixtures/python_agents/requirements.txt new file mode 100644 index 0000000..8c62a2f --- /dev/null +++ b/test/e2e/fixtures/python_agents/requirements.txt @@ -0,0 +1,2 @@ +# Pinned so SDK drift cannot silently break CI. Bump deliberately. +agentspan==0.2.0 diff --git a/test/e2e/rerun.bats b/test/e2e/rerun.bats index b0e9db2..9db43af 100755 --- a/test/e2e/rerun.bats +++ b/test/e2e/rerun.bats @@ -2,6 +2,8 @@ # E2E tests for workflow rerun functionality +# bats file_tags=tier:pr + WORKFLOW_NAME="cli_e2e_test_workflow_2" WORKFLOW_FILE="test/e2e/test-workflow-2.json" WORKFLOW_ID="" diff --git a/test/e2e/schedule.bats b/test/e2e/schedule.bats index 46e20a6..e09837a 100644 --- a/test/e2e/schedule.bats +++ b/test/e2e/schedule.bats @@ -3,6 +3,8 @@ # E2E tests for schedule commands # Tests schedule create, list, get, and delete functionality +# bats file_tags=tier:pr + setup_file() { # Ensure the CLI binary exists if [ ! -f "./conductor" ]; then @@ -181,7 +183,17 @@ ensure_schedule() { [[ "$output" == *"no such schedule"* ]] } +# Helper: pause/resume are broken against OSS Conductor — the SDK issues a GET +# where the server requires PUT, so both return 405. See #101. The operations work +# on Orkes, so these tests still run there; remove this guard once #101 is fixed. +skip_if_oss_101() { + if [ "${CONDUCTOR_SERVER_TYPE:-OSS}" != "Enterprise" ]; then + skip "known broken on OSS: #101 — schedule pause/resume send GET, server requires PUT" + fi +} + @test "15. Pause schedule" { + skip_if_oss_101 # Ensure schedule exists ensure_schedule e2e_test_schedule @@ -196,6 +208,7 @@ ensure_schedule() { } @test "16. Resume schedule" { + skip_if_oss_101 # Ensure schedule exists and is paused ensure_schedule e2e_test_schedule ./conductor schedule pause e2e_test_schedule 2>/dev/null || true @@ -253,7 +266,15 @@ ensure_schedule() { [[ "$output" == *"e2e_test_schedule_2"* ]] } +# Requires GNU timeout(1), which is absent on stock macOS (coreutils installs it +# as gtimeout). Tagged so local runs can exclude it with '!needs:timeout'; Ubuntu +# CI runners have it. +# bats test_tags=needs:timeout @test "21. Delete without -y flag prompts for confirmation" { + if ! command -v timeout >/dev/null 2>&1; then + skip "GNU timeout(1) not available" + fi + # Ensure schedule exists ensure_schedule e2e_test_schedule diff --git a/test/e2e/search.bats b/test/e2e/search.bats index 94d4336..2d39f8c 100644 --- a/test/e2e/search.bats +++ b/test/e2e/search.bats @@ -3,6 +3,8 @@ # E2E tests for workflow search functionality # Tests must be run in order as they depend on each other +# bats file_tags=tier:pr + WORKFLOW_NAME="cli_e2e_test_workflow_2" WORKFLOW_FILE="test/e2e/test-workflow-2.json" diff --git a/test/e2e/secret.bats b/test/e2e/secret.bats index 8c8405e..673ca33 100755 --- a/test/e2e/secret.bats +++ b/test/e2e/secret.bats @@ -2,6 +2,8 @@ # E2E tests for secret management functionality +# bats file_tags=tier:pr,orkes-only + SECRET_KEY="e2e_test_secret" SECRET_KEY_2="e2e_test_secret_2" SECRET_VALUE="test_secret_value_12345" diff --git a/test/e2e/server.bats b/test/e2e/server.bats new file mode 100644 index 0000000..8a48976 --- /dev/null +++ b/test/e2e/server.bats @@ -0,0 +1,113 @@ +#!/usr/bin/env bats + +# E2E tests for local Conductor server management +# +# Scope note: this suite deliberately does NOT exercise the start/stop lifecycle. +# The CLI tracks one server instance in ~/.conductor-cli/server/server-state.json +# (single pid/port), so starting or stopping a server here would clobber the +# instance the rest of the E2E run depends on. Instead it asserts the read-only +# commands plus the mutual-exclusion guards, which are non-destructive and are the +# behaviours most likely to regress. +# +# Full lifecycle coverage needs an exclusive server and a way to isolate the +# datasource (see #104 — `server start` writes c123.db into the current working +# directory with no flag to override). +# +# venue:oss only — these commands manage a local OSS jar and are meaningless +# against a remote Enterprise server. + +# bats file_tags=tier:pr,oss-only,needs:server + +setup_file() { + # Ensure the CLI binary exists + if [ ! -f "./conductor" ]; then + echo "ERROR: conductor binary not found. Please build it first." + exit 1 + fi +} + +# Helper: skip unless a CLI-managed local server is running. These tests assert +# behaviour relative to a running instance; without one they are meaningless. +require_running_server() { + if ! ./conductor server status 2>/dev/null | grep -q 'is running'; then + skip "no CLI-managed local server is running" + fi +} + +@test "1. Server status reports a running server with its PID" { + require_running_server + + run bash -c "./conductor server status 2>&1" + echo "Output: $output" + [ "$status" -eq 0 ] + [[ "$output" == *"is running"* ]] + [[ "$output" == *"PID"* ]] +} + +@test "2. Server status reports health and endpoints" { + require_running_server + + run bash -c "./conductor server status 2>&1" + echo "Output: $output" + [ "$status" -eq 0 ] + [[ "$output" == *"healthy"* ]] + [[ "$output" == *"API:"* ]] +} + +@test "3. Server logs returns output" { + require_running_server + + run bash -c "./conductor server logs 2>&1" + echo "Output length: ${#output}" + [ "$status" -eq 0 ] + [ "${#output}" -gt 0 ] +} + +@test "4. Server logs -n bounds the number of lines" { + require_running_server + + run bash -c "./conductor server logs -n 5 2>/dev/null | wc -l | tr -d ' '" + echo "Line count: $output" + [ "$status" -eq 0 ] + [ "$output" -le 5 ] +} + +@test "5. Server start refuses while an instance is already running" { + require_running_server + + # Non-destructive: the guard returns before touching the running instance. + run bash -c "./conductor server start 2>&1" + echo "Output: $output" + [ "$status" -ne 0 ] + [[ "$output" == *"already running"* ]] +} + +@test "6. Server update refuses while an instance is already running" { + require_running_server + + # Non-destructive, and importantly avoids triggering a ~435 MB download. + run bash -c "./conductor server update 2>&1" + echo "Output: $output" + [ "$status" -ne 0 ] + [[ "$output" == *"currently running"* ]] +} + +@test "7. Server help lists the lifecycle subcommands" { + run bash -c "./conductor server --help 2>&1" + echo "Output: $output" + [ "$status" -eq 0 ] + [[ "$output" == *"start"* ]] + [[ "$output" == *"stop"* ]] + [[ "$output" == *"status"* ]] + [[ "$output" == *"logs"* ]] + [[ "$output" == *"update"* ]] +} + +@test "8. Server start --help documents the version and port flags" { + run bash -c "./conductor server start --help 2>&1" + echo "Output: $output" + [ "$status" -eq 0 ] + [[ "$output" == *"--port"* ]] + [[ "$output" == *"--version"* ]] + [[ "$output" == *"--foreground"* ]] +} diff --git a/test/e2e/task.bats b/test/e2e/task.bats index a9a05b4..8b2c6de 100644 --- a/test/e2e/task.bats +++ b/test/e2e/task.bats @@ -3,6 +3,8 @@ # E2E tests for task definition CRUD and task execution operations # Tests must be run in order as they depend on each other +# bats file_tags=tier:pr + TASK_NAME="cli_e2e_test_task" TASK_FILE="test/e2e/test-task.json" WORKFLOW_NAME="cli_e2e_test_workflow" @@ -115,6 +117,8 @@ EOF [[ "$output" == "RUNNING" ]] } +# task signal is Orkes-only (see the note on test 13). +# bats test_tags=orkes-only @test "9. Signal WAIT task asynchronously" { WORKFLOW_ID=$(cat /tmp/task_signal_workflow_id.txt) [ -n "$WORKFLOW_ID" ] @@ -125,6 +129,8 @@ EOF echo "$output" | grep -q "Task signal sent asynchronously" } +# Depends on test 9 having delivered the signal, so Orkes-only too. +# bats test_tags=orkes-only @test "10. Verify workflow completes after async signal" { WORKFLOW_ID=$(cat /tmp/task_signal_workflow_id.txt) [ -n "$WORKFLOW_ID" ] @@ -169,6 +175,9 @@ EOF echo "Started workflow UUID: $WORKFLOW_ID" } +# task signal / signal-sync are Orkes-only; OSS Conductor rejects them with +# "signal operations are only available in Orkes Conductor (Enterprise)". +# bats test_tags=orkes-only @test "13. Signal WAIT task synchronously" { WORKFLOW_ID=$(cat /tmp/task_signal_sync_workflow_id.txt) [ -n "$WORKFLOW_ID" ] @@ -181,6 +190,8 @@ EOF [[ "$output" == *"\"status\""* ]] } +# Depends on test 13 having delivered the signal, so it is Orkes-only too. +# bats test_tags=orkes-only @test "14. Verify workflow completes after sync signal" { WORKFLOW_ID=$(cat /tmp/task_signal_sync_workflow_id.txt) [ -n "$WORKFLOW_ID" ] diff --git a/test/e2e/webhook.bats b/test/e2e/webhook.bats index ab66aa2..62b21f2 100644 --- a/test/e2e/webhook.bats +++ b/test/e2e/webhook.bats @@ -2,6 +2,8 @@ # E2E tests for webhook functionality +# bats file_tags=tier:pr,orkes-only + WEBHOOK_FILE="test/e2e/webhook.json" WEBHOOK_NAME="custom_webhook_1" WEBHOOK_ID="" diff --git a/test/e2e/whoami.bats b/test/e2e/whoami.bats index bc8b2a3..1925747 100644 --- a/test/e2e/whoami.bats +++ b/test/e2e/whoami.bats @@ -2,6 +2,8 @@ # E2E tests for whoami command +# bats file_tags=tier:pr + setup() { # Ensure the CLI binary exists if [ ! -f "./conductor" ]; then diff --git a/test/e2e/workflow.bats b/test/e2e/workflow.bats index d417d28..f71448e 100644 --- a/test/e2e/workflow.bats +++ b/test/e2e/workflow.bats @@ -3,6 +3,8 @@ # E2E tests for conductor-cli # Tests must be run in order as they depend on each other +# bats file_tags=tier:pr + WORKFLOW_NAME="cli_e2e_test_workflow" WORKFLOW_FILE="test/e2e/test-workflow.json" WORKFLOW_NAME_2="cli_e2e_test_workflow_2"