From 3a61aea09139f58669e9fdb688678ff00e317c45 Mon Sep 17 00:00:00 2001 From: openhands Date: Mon, 1 Jun 2026 13:01:40 -0600 Subject: [PATCH 1/6] chore: bump version to 1.0.0-beta.1 --- README.md | 4 ++-- package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 96073282..906104c2 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ agent-canvas ### Option 2: With a Docker Sandbox ```sh -docker pull ghcr.io/openhands/agent-canvas:1.0.0-alpha.10 +docker pull ghcr.io/openhands/agent-canvas:1.0.0-beta.1 export PROJECTS_PATH=~/projects # directory containing your project folders @@ -69,7 +69,7 @@ docker run -it --rm \ -p 8000:8000 \ -v ~/.openhands:/home/openhands/.openhands \ -v ${PROJECTS_PATH}:/projects \ - ghcr.io/openhands/agent-canvas:1.0.0-alpha.10 + ghcr.io/openhands/agent-canvas:1.0.0-beta.1 ``` The agent will be able to access any project under `PROJECTS_PATH`. diff --git a/package-lock.json b/package-lock.json index 630524a9..a9b54018 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@openhands/agent-canvas", - "version": "1.0.0-alpha.10", + "version": "1.0.0-beta.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@openhands/agent-canvas", - "version": "1.0.0-alpha.10", + "version": "1.0.0-beta.1", "license": "MIT", "dependencies": { "@heroui/react": "2.8.10", diff --git a/package.json b/package.json index 3ce4579d..4010a61b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@openhands/agent-canvas", - "version": "1.0.0-alpha.10", + "version": "1.0.0-beta.1", "description": "Agent Canvas UI for OpenHands - run AI coding agents with a visual interface", "license": "MIT", "private": false, From bb02177a5a7c8a132565c9fb322f11ffef13d336 Mon Sep 17 00:00:00 2001 From: openhands Date: Mon, 1 Jun 2026 13:23:52 -0600 Subject: [PATCH 2/6] chore: publish beta and rc versions as 'latest' dist-tag --- .github/workflows/npm-publish.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/npm-publish.yml b/.github/workflows/npm-publish.yml index bae15a48..54354d1d 100644 --- a/.github/workflows/npm-publish.yml +++ b/.github/workflows/npm-publish.yml @@ -92,9 +92,11 @@ jobs: # Resolve the npm dist-tag from the version's pre-release identifier: # alpha → --tag alpha (e.g. 1.0.0-alpha.1) - # beta → --tag beta (e.g. 1.0.0-beta.1) - # rc → --tag rc (e.g. 1.0.0-rc.1) + # beta → --tag latest (e.g. 1.0.0-beta.1) + # rc → --tag latest (e.g. 1.0.0-rc.1) # stable → --tag latest (e.g. 1.0.0) + # Beta and RC publish as `latest` so `npm install @openhands/agent-canvas` + # resolves to the most recent release until the first stable version ships. # Note: OIDC trusted-publishing tokens cover only the `npm publish` call # itself; a separate `npm dist-tag add` would fail with E401, so the tag # is resolved and passed directly in one step. @@ -104,10 +106,6 @@ jobs: VERSION=$(node -p "require('./package.json').version") if [[ "$VERSION" == *-alpha* ]]; then DIST_TAG="alpha" - elif [[ "$VERSION" == *-beta* ]]; then - DIST_TAG="beta" - elif [[ "$VERSION" == *-rc* ]]; then - DIST_TAG="rc" else DIST_TAG="latest" fi From 660efffe7f6c675cb09d23cfaf2d0b07dd00b2ed Mon Sep 17 00:00:00 2001 From: openhands Date: Mon, 1 Jun 2026 13:29:38 -0600 Subject: [PATCH 3/6] chore: bump version to 1.0.0-beta.2 --- README.md | 4 ++-- package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 906104c2..4c4daa29 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ agent-canvas ### Option 2: With a Docker Sandbox ```sh -docker pull ghcr.io/openhands/agent-canvas:1.0.0-beta.1 +docker pull ghcr.io/openhands/agent-canvas:1.0.0-beta.2 export PROJECTS_PATH=~/projects # directory containing your project folders @@ -69,7 +69,7 @@ docker run -it --rm \ -p 8000:8000 \ -v ~/.openhands:/home/openhands/.openhands \ -v ${PROJECTS_PATH}:/projects \ - ghcr.io/openhands/agent-canvas:1.0.0-beta.1 + ghcr.io/openhands/agent-canvas:1.0.0-beta.2 ``` The agent will be able to access any project under `PROJECTS_PATH`. diff --git a/package-lock.json b/package-lock.json index a9b54018..3e1ef92a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@openhands/agent-canvas", - "version": "1.0.0-beta.1", + "version": "1.0.0-beta.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@openhands/agent-canvas", - "version": "1.0.0-beta.1", + "version": "1.0.0-beta.2", "license": "MIT", "dependencies": { "@heroui/react": "2.8.10", diff --git a/package.json b/package.json index 4010a61b..ef48b205 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@openhands/agent-canvas", - "version": "1.0.0-beta.1", + "version": "1.0.0-beta.2", "description": "Agent Canvas UI for OpenHands - run AI coding agents with a visual interface", "license": "MIT", "private": false, From 20894d9baea5b61567b53f305e1aab67d3ef691f Mon Sep 17 00:00:00 2001 From: Tim O'Farrell Date: Mon, 1 Jun 2026 15:10:30 -0600 Subject: [PATCH 4/6] fix: use X-Session-API-Key for local automation auth in prompts and RUNTIME_SERVICES (#999) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #980 The agent prompt in recommended-automations-launcher and the RUNTIME_SERVICES block in agent-server-adapter both advertised X-API-Key as the auth header for the local automation backend. The automation service (openhands-automation) does not accept X-API-Key — it accepts Authorization: Bearer and X-Session-API-Key. X-Session-API-Key is the established local convention: the agent server uses it, the frontend automation API client uses it (with an explicit comment that both backends share the same header), and auth.py describes it as matching that convention. Update both call sites and the corresponding test assertion to use X-Session-API-Key. Co-authored-by: openhands --- AGENTS.md | 8 ++++---- __tests__/api/agent-server-adapter.test.ts | 3 ++- scripts/dev-safe.mjs | 2 +- src/api/agent-server-adapter.ts | 4 +++- .../automations/recommended-automations-launcher.tsx | 2 +- 5 files changed, 11 insertions(+), 8 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index fcded59e..1717463b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -25,7 +25,7 @@ - The block lists URLs **from the agent's point of view**: - The Agent Server is always reachable as `http://localhost:` from inside the sandbox — but that is _you_, not the automation backend. - Host-side services (ingress, Vite, automation) are reachable as `http://localhost:`. -- Agents should treat the `` block as authoritative: don't hardcode `localhost:8000` for "the automation server", and don't probe random ports trying to discover services. If the block says automation is not running, skip `/api/automation` calls; otherwise use the listed `url_from_agent` + `api_prefix` (default `/api/automation`) and the `X-API-Key: $OPENHANDS_AUTOMATION_API_KEY` header. +- Agents should treat the `` block as authoritative: don't hardcode `localhost:8000` for "the automation server", and don't probe random ports trying to discover services. If the block says automation is not running, skip `/api/automation` calls; otherwise use the listed `url_from_agent` + `api_prefix` (default `/api/automation`) and the `X-Session-API-Key: $OPENHANDS_AUTOMATION_API_KEY` header. - The launcher → frontend → suffix plumbing is: - `scripts/dev-safe.mjs::buildRuntimeServicesInfo()` — pure helper that constructs the info object. - `scripts/dev-with-automation.mjs::buildAutomationRuntimeServicesInfo()` — wraps it with automation details; called from both Vite spawn (`startVite`) and the static build (`static-build.mjs`). @@ -53,7 +53,7 @@ The env var is a JSON string of: "url_from_agent": "http://localhost:3001" }, "automation": { - "description": "OpenHands Automations service. All routes are mounted under '/api/automation'. Authenticate with header 'X-API-Key: $OPENHANDS_AUTOMATION_API_KEY'.", + "description": "OpenHands Automations service. All routes are mounted under '/api/automation'. Authenticate with header 'X-Session-API-Key: $OPENHANDS_AUTOMATION_API_KEY'.", "url_from_agent": "http://localhost:18001", "api_prefix": "/api/automation", "docs_url": "http://localhost:18001/api/automation/docs", @@ -81,10 +81,10 @@ from your point of view (i.e., as you should curl/fetch them). * Frontend: http://localhost:3001 Vite dev server hosting the agent-canvas frontend. * Automation backend: http://localhost:18001 - OpenHands Automations service. All routes are mounted under '/api/automation'. Authenticate with header 'X-API-Key: $OPENHANDS_AUTOMATION_API_KEY'. + OpenHands Automations service. All routes are mounted under '/api/automation'. Authenticate with header 'X-Session-API-Key: $OPENHANDS_AUTOMATION_API_KEY'. Docs: http://localhost:18001/api/automation/docs OpenAPI: http://localhost:18001/api/automation/openapi.json - Auth: header 'X-API-Key: $OPENHANDS_AUTOMATION_API_KEY' + Auth: header 'X-Session-API-Key: $OPENHANDS_AUTOMATION_API_KEY' Trust this block over guessing: do not assume any other URLs are running. In particular, http://localhost:18000 inside your sandbox is the Agent Server diff --git a/__tests__/api/agent-server-adapter.test.ts b/__tests__/api/agent-server-adapter.test.ts index c82028b5..f628d539 100644 --- a/__tests__/api/agent-server-adapter.test.ts +++ b/__tests__/api/agent-server-adapter.test.ts @@ -795,7 +795,8 @@ describe("buildRuntimeServicesSystemSuffix", () => { expect(suffix).toContain("http://localhost:18000"); expect(suffix).toContain("http://localhost:18001"); expect(suffix).toContain("http://localhost:18001/api/automation/docs"); - expect(suffix).toContain("X-API-Key: $OPENHANDS_AUTOMATION_API_KEY"); + expect(suffix).toContain("X-Session-API-Key: $OPENHANDS_AUTOMATION_API_KEY"); + expect(suffix).not.toContain("X-API-Key: $OPENHANDS_AUTOMATION_API_KEY"); expect(suffix).toContain(""); // The "don't guess" line should reference the actual agent-server URL // for this stack, not a hardcoded port. The assertion anchors on the URL diff --git a/scripts/dev-safe.mjs b/scripts/dev-safe.mjs index 322a63d1..c85d40fc 100644 --- a/scripts/dev-safe.mjs +++ b/scripts/dev-safe.mjs @@ -782,7 +782,7 @@ export function buildRuntimeServicesInfo(options) { description: "OpenHands Automations service. All routes are mounted under " + `'${apiPrefix}'. Authenticate with header ` + - `'X-API-Key: $${authEnvVar}'.`, + `'X-Session-API-Key: $${authEnvVar}'.`, url_from_agent: baseUrl, api_prefix: apiPrefix, docs_url: `${baseUrl}${apiPrefix}/docs`, diff --git a/src/api/agent-server-adapter.ts b/src/api/agent-server-adapter.ts index 075cf394..d8ee2e36 100644 --- a/src/api/agent-server-adapter.ts +++ b/src/api/agent-server-adapter.ts @@ -199,8 +199,10 @@ export function buildRuntimeServicesSystemSuffix(): string | undefined { lines.push(` OpenAPI: ${automation.openapi_url}`); } if (automation.auth_env_var) { + // X-Session-API-Key is the local convention shared by the agent-server + // and automation backend (see openhands-automation auth.py). lines.push( - ` Auth: header 'X-API-Key: $${automation.auth_env_var}'`, + ` Auth: header 'X-Session-API-Key: $${automation.auth_env_var}'`, ); } } else { diff --git a/src/components/features/automations/recommended-automations-launcher.tsx b/src/components/features/automations/recommended-automations-launcher.tsx index 5ab90710..9d457d58 100644 --- a/src/components/features/automations/recommended-automations-launcher.tsx +++ b/src/components/features/automations/recommended-automations-launcher.tsx @@ -73,7 +73,7 @@ export function buildAutomationPrompt( "**Which API to use:** Create this automation using the **local** OpenHands Automations API that is running alongside this agent.", "- Read the Automation backend URL from the `` block in your system context.", "- Endpoint path: `POST /api/automation/v1/preset/prompt`", - "- Auth: `X-API-Key: $OPENHANDS_AUTOMATION_API_KEY`", + "- Auth: `X-Session-API-Key: $OPENHANDS_AUTOMATION_API_KEY`", "- If no local Automation backend is listed in ``, stop and ask me to start the full local automation stack instead of using any remote/cloud automation API.", ].join("\n"); } From 0f860f5577b98dcc9ffd73304bba8a3e3e69265a Mon Sep 17 00:00:00 2001 From: Rohit Malhotra Date: Mon, 1 Jun 2026 15:09:01 -0400 Subject: [PATCH 5/6] feat: reuse mock-LLM E2E tests for Docker image validation (#992) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: reuse mock-LLM E2E tests for Docker image validation Add a Docker-specific Playwright config (playwright.mock-llm-docker.config.ts) that runs the exact same test specs and helpers against the agent-canvas Docker image instead of the npm build path (bin/agent-canvas.mjs + uvx). Key changes: - Split MOCK_LLM_BASE_URL into two constants in mock-llm-helpers.ts: - MOCK_LLM_BASE_URL: always host-local, used by tests for admin API - MOCK_LLM_AGENT_URL: env-overridable, used when configuring the LLM profile (the URL the agent-server uses for inference). Defaults to MOCK_LLM_BASE_URL for backward compatibility with the npm path. - New playwright.mock-llm-docker.config.ts: - Starts the mock LLM server on the host (same as npm path) - Runs the Docker container with --network host (Linux CI) - Points to the same testDir (tests/e2e/mock-llm/) and specs - Separate output dirs to avoid collision with npm path results - New CI workflow (.github/workflows/mock-llm-docker-e2e.yml): - Builds the Docker image from current code (or uses a pre-built image) - Runs the same specs against the container - Posts PR comment with differentiated report title - render-mock-llm-report.mjs: accept --title flag for Docker vs npm reports - npm run test:e2e:mock-llm:docker script added - .gitignore updated for docker test output dirs The npm path (test:e2e:mock-llm) is fully backward-compatible — no env var override needed since MOCK_LLM_AGENT_URL defaults to MOCK_LLM_BASE_URL. Co-authored-by: openhands * refactor: chain Docker E2E off existing Docker CI via workflow_run Instead of rebuilding the Docker image in the E2E workflow (duplicating ~10-15 min of Docker build time), use workflow_run to trigger automatically after the existing 'Docker' workflow completes successfully. The workflow now: - Triggers on: workflow_run (Docker completed) + workflow_dispatch (manual) - Derives the image tag from the Docker build's commit SHA (ghcr.io/openhands/agent-canvas:sha--amd64) - Pulls the already-built image from GHCR — no rebuild needed - Checks out code at the same SHA as the Docker build - Extracts PR number from workflow_run.pull_requests[] for comments Removed: Docker build steps, Buildx setup, build-arg resolution. All image building stays in docker.yml where it belongs. Co-authored-by: openhands * fix: replace flaky 1s timeout with polling for Active badge assertion The 'Active badge' check in step 2 used a hardcoded 1-second waitForTimeout before reloading. On a loaded CI runner the profile activation mutation may not persist in time, causing the reload to show stale state. This is a pre-existing flake (identical test code passed on the first push and failed on the second). Replace with expect.poll() that retries the reload+check cycle with increasing intervals (1s, 2s, 3s) up to 15 seconds total. Co-authored-by: openhands * fix: add pull_request trigger for Docker E2E (workflow_run bootstrap) workflow_run only fires when the workflow file exists on the default branch (main). Since mock-llm-docker-e2e.yml is new and only on the PR branch, GitHub doesn't recognize it as a workflow_run listener yet. Add pull_request trigger (gated by 'e2e-tests' label, skip forks) that polls the Docker workflow via gh API until it completes for the PR's head SHA, then pulls the already-built image from GHCR and runs tests. After merge, workflow_run takes over as the primary automatic trigger. The pull_request path remains as a fallback for label-gated runs. Co-authored-by: openhands * fix: add FILE_STORE, AUTOMATION_BASE_URL, AUTOMATION_WORKSPACE_BASE to Docker entrypoint The Docker entrypoint was missing several environment variables that the npm path (dev-with-automation.mjs) sets for the automation backend: - FILE_STORE=local — without this, the automation backend may fall back to cloud storage (S3/GCS) which fails without credentials, causing tarball- based presets (preset/prompt, preset/plugin) to silently error - LOCAL_STORAGE_PATH — where to store files on the local filesystem - AUTOMATION_BASE_URL — publicly-reachable base URL for callback URLs - AUTOMATION_WORKSPACE_BASE — where automation runs unpack tarballs This explains the Docker E2E failure: the agent's curl to create an automation via /api/automation/v1/preset/prompt returned an error (likely 500 from missing storage config), but the mock LLM doesn't care about terminal output and proceeded to return the scripted final reply. The test then found 0 automations. Co-authored-by: openhands * fix: exclude auth-modes spec from Docker E2E tests The mock-llm-auth-modes.spec.ts tests npm-binary-specific --auth-required behaviour (a second static-server instance on port 18301). The Docker image doesn't provide this second server — it has its own auth handling. Exclude the spec from the Docker test run via testIgnore. Co-authored-by: openhands * feat: run auth-modes tests inside Docker via PUBLIC_MODE_PORT Instead of excluding the auth-modes spec from the Docker E2E run or spinning up a host-side static server with a duplicate build/ directory, the Docker entrypoint now supports an optional PUBLIC_MODE_PORT env var. When set, entrypoint.sh starts a second static-server instance from the same baked-in frontend assets with --auth-required (no session key injected). This tests the actual Docker image's auth gate behaviour — not a host-side approximation. The Playwright Docker config passes -e PUBLIC_MODE_PORT=18301 to the container and exports MOCK_LLM_PUBLIC_MODE_URL so the auth-modes spec can reach it. With --network host the port is accessible from the host. Co-authored-by: openhands * address review feedback: drop unlabeled trigger, improve error messages, document env vars - Drop 'unlabeled' from pull_request trigger types to avoid wasted workflow runs when any label is removed (the job-level if: condition would skip immediately anyway) - Distinguish 'no Docker run found' vs 'didn't complete in time' in the polling loop's final error message - Add comment explaining /api/automation/v1 probe returns 200 without auth so the readiness check won't spin for 180s - Document FILE_STORE, LOCAL_STORAGE_PATH, AUTOMATION_BASE_URL, and AUTOMATION_WORKSPACE_BASE in the entrypoint header — these affect production deployments, not just E2E tests Co-authored-by: openhands --------- Co-authored-by: openhands --- .github/workflows/mock-llm-docker-e2e.yml | 321 ++++++++++++++++++ .gitignore | 2 + AGENTS.md | 11 + docker/entrypoint.sh | 59 +++- package.json | 1 + playwright.mock-llm-docker.config.ts | 171 ++++++++++ .../mock-llm/mock-llm-conversation.spec.ts | 63 ++-- .../scripts/render-mock-llm-report.mjs | 5 +- tests/e2e/mock-llm/utils/mock-llm-helpers.ts | 18 +- 9 files changed, 615 insertions(+), 36 deletions(-) create mode 100644 .github/workflows/mock-llm-docker-e2e.yml create mode 100644 playwright.mock-llm-docker.config.ts diff --git a/.github/workflows/mock-llm-docker-e2e.yml b/.github/workflows/mock-llm-docker-e2e.yml new file mode 100644 index 00000000..1915e629 --- /dev/null +++ b/.github/workflows/mock-llm-docker-e2e.yml @@ -0,0 +1,321 @@ +name: Mock-LLM Docker E2E Tests + +# Runs the same mock-LLM E2E test specs as mock-llm-e2e.yml, but against +# the Docker image instead of the npm build path (bin/agent-canvas.mjs). +# +# Trigger chain: +# 1. workflow_run — fires automatically after the "Docker" workflow +# completes on main. The image is already built/pushed to GHCR. +# 2. pull_request — fires on PRs with the 'e2e-tests' label. Waits for +# the Docker workflow to finish, then pulls the image from GHCR. +# (workflow_run doesn't fire for new workflow files until they're on +# the default branch, so pull_request is needed for first-run PRs.) +# 3. workflow_dispatch — manual trigger with a custom image tag. + +on: + workflow_run: + workflows: ["Docker"] + types: [completed] + pull_request: + types: [opened, synchronize, reopened, labeled] + workflow_dispatch: + inputs: + docker_image: + description: 'Docker image to test (e.g., ghcr.io/openhands/agent-canvas:sha-abc1234-amd64)' + type: string + default: "" + +concurrency: + group: mock-llm-docker-e2e-${{ github.event.workflow_run.id || github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + contents: read + packages: read + pull-requests: write + actions: read + +jobs: + mock-llm-docker-e2e: + # workflow_run: only run if the Docker build succeeded. + # pull_request: only run with the 'e2e-tests' label, skip fork PRs (no GHCR push). + # workflow_dispatch: always run. + if: >- + github.event_name == 'workflow_dispatch' || + (github.event_name == 'workflow_run' && + github.event.workflow_run.conclusion == 'success') || + (github.event_name == 'pull_request' && + contains(github.event.pull_request.labels.*.name, 'e2e-tests') && + !github.event.pull_request.head.repo.fork) + runs-on: ubuntu-24.04 + timeout-minutes: 15 + + env: + MOCK_LLM_REPORT_PATH: mock-llm-docker-report.md + MOCK_LLM_WORKFLOW_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + + steps: + # ── Resolve which commit / PR to test ────────────────────────────── + - name: Resolve source context + id: ctx + run: | + if [ "${{ github.event_name }}" = "workflow_run" ]; then + echo "sha=${{ github.event.workflow_run.head_sha }}" >> "$GITHUB_OUTPUT" + echo "ref=${{ github.event.workflow_run.head_sha }}" >> "$GITHUB_OUTPUT" + PR_NUMBER=$(echo '${{ toJSON(github.event.workflow_run.pull_requests) }}' \ + | jq -r '.[0].number // empty') + echo "pr_number=${PR_NUMBER}" >> "$GITHUB_OUTPUT" + elif [ "${{ github.event_name }}" = "pull_request" ]; then + echo "sha=${{ github.event.pull_request.head.sha }}" >> "$GITHUB_OUTPUT" + echo "ref=${{ github.event.pull_request.head.sha }}" >> "$GITHUB_OUTPUT" + echo "pr_number=${{ github.event.pull_request.number }}" >> "$GITHUB_OUTPUT" + else + echo "sha=${{ github.sha }}" >> "$GITHUB_OUTPUT" + echo "ref=${{ github.ref }}" >> "$GITHUB_OUTPUT" + echo "pr_number=" >> "$GITHUB_OUTPUT" + fi + + - name: Check out repository + uses: actions/checkout@v6 + with: + ref: ${{ steps.ctx.outputs.ref }} + + - name: Read defaults from config/defaults.json + id: defaults + run: | + echo "agent_server_version=$(node -p "require('./config/defaults.json').versions.agentServer")" >> "$GITHUB_OUTPUT" + + # ── Wait for Docker workflow (pull_request trigger only) ──────────── + # When triggered by pull_request, the Docker image may still be + # building. Poll the Docker workflow until it completes for this SHA. + - name: Wait for Docker workflow to complete + if: github.event_name == 'pull_request' + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + SHA="${{ steps.ctx.outputs.sha }}" + echo "Waiting for Docker workflow to complete for SHA ${SHA}..." + + for i in $(seq 1 60); do + # Find the Docker workflow run for this exact commit + RUN=$(gh api \ + "/repos/${{ github.repository }}/actions/workflows/docker.yml/runs?head_sha=${SHA}&per_page=1" \ + --jq '.workflow_runs[0] // empty' 2>/dev/null || echo "") + + if [ -z "$RUN" ]; then + echo " Attempt $i: No Docker workflow run found yet for ${SHA}..." + sleep 15 + continue + fi + + STATUS=$(echo "$RUN" | jq -r '.status') + CONCLUSION=$(echo "$RUN" | jq -r '.conclusion // empty') + RUN_URL=$(echo "$RUN" | jq -r '.html_url') + + if [ "$STATUS" = "completed" ]; then + if [ "$CONCLUSION" = "success" ]; then + echo "Docker workflow completed successfully: $RUN_URL" + break + else + echo "::error::Docker workflow finished with conclusion '$CONCLUSION': $RUN_URL" + exit 1 + fi + fi + + echo " Attempt $i: Docker workflow status=$STATUS (${RUN_URL})" + sleep 15 + done + + # Final check — if we exhausted retries + if [ -z "${STATUS:-}" ]; then + echo "::error::No Docker workflow run found for SHA ${SHA} after 15 minutes" + exit 1 + elif [ "$STATUS" != "completed" ]; then + echo "::error::Docker workflow did not complete within 15 minutes (last status: $STATUS)" + exit 1 + fi + + # ── Resolve Docker image tag ─────────────────────────────────────── + - name: Resolve Docker image + id: image + run: | + if [ -n "${{ inputs.docker_image }}" ]; then + echo "tag=${{ inputs.docker_image }}" >> "$GITHUB_OUTPUT" + else + SHORT_SHA=$(echo "${{ steps.ctx.outputs.sha }}" | cut -c1-7) + # Use the amd64-specific tag (always pushed by the Docker workflow). + echo "tag=ghcr.io/openhands/agent-canvas:sha-${SHORT_SHA}-amd64" >> "$GITHUB_OUTPUT" + fi + + - name: Log in to GHCR + uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Pull Docker image + run: | + echo "Pulling ${{ steps.image.outputs.tag }}..." + docker pull "${{ steps.image.outputs.tag }}" + + # ── Test infrastructure setup ────────────────────────────────────── + - name: Set up Node.js + uses: actions/setup-node@v6 + with: + # Pin to 24.15.x — Node 24.16.0 has a zip-extraction regression + # (nodejs/node#63487) that hangs `playwright install` for Playwright + # < 1.60.0. Remove this pin after upgrading to Playwright >= 1.60.0. + node-version: "24.15" + cache: npm + + - name: Install npm dependencies + run: npm ci + + - name: Get Playwright version + id: pw_version + run: echo "version=$(npx playwright --version | awk '{print $2}')" >> "$GITHUB_OUTPUT" + + - name: Cache Playwright browsers + id: pw_cache + uses: actions/cache@v4 + with: + path: ~/.cache/ms-playwright + key: playwright-${{ runner.os }}-${{ steps.pw_version.outputs.version }} + + - name: Install Playwright Chromium + if: steps.pw_cache.outputs.cache-hit != 'true' + run: npx playwright install chromium + + - name: Install Playwright system deps + run: npx playwright install-deps chromium + + - name: Install uv + run: | + curl -LsSf https://astral.sh/uv/install.sh | sh + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + + - name: Install openhands-sdk (for mock LLM server) + run: | + uv venv .mock-llm-venv + uv pip install -p .mock-llm-venv openhands-sdk==${{ steps.defaults.outputs.agent_server_version }} + + - name: Verify mock LLM server starts + run: | + .mock-llm-venv/bin/python3 tests/e2e/mock-llm/scripts/mock-llm-server.py --port 9998 & + SERVER_PID=$! + for i in $(seq 1 30); do + if curl -sf http://127.0.0.1:9998/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{"model":"test","messages":[]}' > /dev/null 2>&1; then + echo "Mock LLM server responded on attempt $i" + break + fi + sleep 1 + done + curl -sf http://127.0.0.1:9998/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{"model":"test","messages":[]}' | python3 -m json.tool + kill $SERVER_PID + + # ── Run tests ────────────────────────────────────────────────────── + - name: Run mock-LLM Docker E2E tests + id: run_tests + env: + MOCK_LLM_PYTHON: .mock-llm-venv/bin/python3 + MOCK_LLM_DOCKER_IMAGE: ${{ steps.image.outputs.tag }} + run: | + set +e + MARKER_DIR=".mock-llm-markers" + DONE_MARKER="$MARKER_DIR/.tests-done" + PASS_MARKER="$MARKER_DIR/.all-passed" + rm -rf "$MARKER_DIR" + + # Run Playwright in background so our shell survives if we have + # to kill it (the Docker container teardown can hang). + npm run test:e2e:mock-llm:docker & + PW_PID=$! + + # Wait up to 5 min for tests to complete. + deadline=$((SECONDS + 300)) + while [ "$SECONDS" -lt "$deadline" ]; do + if ! kill -0 "$PW_PID" 2>/dev/null; then + break + fi + if [ -f "$DONE_MARKER" ]; then + echo "Tests completed: $(cat "$DONE_MARKER")" + break + fi + sleep 2 + done + + # If Playwright is still running (teardown hang), give it 5s + # grace then force-kill. + if kill -0 "$PW_PID" 2>/dev/null; then + sleep 5 + if kill -0 "$PW_PID" 2>/dev/null; then + echo "::warning::Killing lingering Playwright process (teardown hung)" + kill "$PW_PID" 2>/dev/null + sleep 5 + kill -9 "$PW_PID" 2>/dev/null + fi + wait "$PW_PID" 2>/dev/null + pw_exit=124 + else + wait "$PW_PID" + pw_exit=$? + fi + + echo "Playwright exited with code $pw_exit" + + # When killed during teardown, the exit code is non-zero but + # tests may have passed. + if [ "$pw_exit" -ne 0 ] && [ -f "$PASS_MARKER" ]; then + echo "::notice::All tests passed (marker file present); non-zero exit was teardown-related" + pw_exit=0 + fi + + # Clean up the Docker container (belt-and-suspenders) + docker ps -q --filter "name=agent-canvas-mock-llm" | xargs -r docker stop 2>/dev/null || true + + echo "exit_code=$pw_exit" >> "$GITHUB_OUTPUT" + exit 0 + + # ── Reporting ────────────────────────────────────────────────────── + - name: Upload test artifacts + id: upload_artifacts + if: always() + uses: actions/upload-artifact@v7 + with: + name: mock-llm-docker-e2e-results + if-no-files-found: ignore + retention-days: 14 + path: | + playwright-report-mock-llm-docker/ + test-results-mock-llm-docker/ + + - name: Render test report + if: always() + run: | + node tests/e2e/mock-llm/scripts/render-mock-llm-report.mjs \ + --results "test-results-mock-llm-docker/results.json" \ + --output "$MOCK_LLM_REPORT_PATH" \ + --workflow-url "$MOCK_LLM_WORKFLOW_URL" \ + --commit "${{ steps.ctx.outputs.sha }}" \ + --artifact-url "${{ steps.upload_artifacts.outputs.artifact-url || '' }}" \ + --title "Mock-LLM Docker E2E Test Results" + cat "$MOCK_LLM_REPORT_PATH" >> "$GITHUB_STEP_SUMMARY" + + - name: Post PR comment + if: always() && steps.ctx.outputs.pr_number + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + gh pr comment "${{ steps.ctx.outputs.pr_number }}" \ + --body-file "$MOCK_LLM_REPORT_PATH" + + - name: Fail job when tests fail + if: always() + run: | + exit_code="${{ steps.run_tests.outputs.exit_code }}" + exit "${exit_code:-1}" diff --git a/.gitignore b/.gitignore index 97c97dcb..ea10421a 100644 --- a/.gitignore +++ b/.gitignore @@ -15,10 +15,12 @@ __pycache__ /test-results/ /test-results-live/ /test-results-mock-llm/ +/test-results-mock-llm-docker/ /.mock-llm-markers/ /playwright-report/ /playwright-report-live/ /playwright-report-mock-llm/ +/playwright-report-mock-llm-docker/ /blob-report/ /playwright/.cache/ # Snapshot baselines are stored as GitHub Actions artifacts — not in git. diff --git a/AGENTS.md b/AGENTS.md index 1717463b..52d123b1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -164,6 +164,17 @@ you are running inside of — NOT the automation backend. - CI workflow: `.github/workflows/mock-llm-e2e.yml` runs on PRs with the `e2e-tests` label or on manual dispatch. It builds the frontend, starts the mock LLM server, runs the tests, and posts a PR comment with results. - The custom `DoneMarkerReporter` writes `.mock-llm-markers/.tests-done` after all tests complete (before webServer teardown) so the CI wrapper can detect completion and kill the lingering teardown process. +### Docker Image Testing (Shared Specs) + +- The same test specs and helpers are reused to validate the Docker image via `playwright.mock-llm-docker.config.ts`. Run locally with `npm run test:e2e:mock-llm:docker` (requires Docker daemon and a built image). +- **Architecture**: The Docker config replaces the npm path's `bin/agent-canvas.mjs` webServer with a `docker run --network host` command. The mock LLM server still runs on the host. On Linux (including CI), `--network host` lets the container share the host's network stack so all `127.0.0.1` URLs work identically. On macOS/Windows Docker Desktop (bridge networking), set `MOCK_LLM_AGENT_URL=http://host.docker.internal:` so the agent-server inside Docker can reach the host-side mock LLM server. +- **URL split**: `mock-llm-helpers.ts` exports two mock LLM URL constants: + - `MOCK_LLM_BASE_URL` — always `http://127.0.0.1:`, used by tests for the mock LLM admin API (register/activate/reset trajectories). + - `MOCK_LLM_AGENT_URL` — defaults to `MOCK_LLM_BASE_URL`, overridable via `MOCK_LLM_AGENT_URL` env var. Used when configuring the LLM profile (`base_url` field) — this is the URL the agent-server uses for inference calls. The npm path and Docker-with-`--network host` path use the same value; Docker on macOS needs the override. +- **Docker image**: Set `MOCK_LLM_DOCKER_IMAGE` to the image tag (default: `ghcr.io/openhands/agent-canvas:latest`). The container is started with `--rm --network host` and a unique `--name` for cleanup. +- **State isolation**: The Docker container uses its internal state directory (no host mount needed for tests). Each test run starts a fresh container. +- CI workflow: `.github/workflows/mock-llm-docker-e2e.yml` has three triggers — all pull the already-built image from GHCR (no rebuild): (1) `workflow_run` fires automatically after the `Docker` workflow completes on main; (2) `pull_request` with the `e2e-tests` label polls the Docker workflow until it finishes for the PR's head SHA, then pulls the image (needed because `workflow_run` only fires for workflow files already on the default branch); (3) `workflow_dispatch` accepts a custom `docker_image` input. The image tag is derived from the commit SHA (`ghcr.io/openhands/agent-canvas:sha--amd64`). Fork PRs are skipped (no GHCR push). Report artifacts go to `test-results-mock-llm-docker/` and `playwright-report-mock-llm-docker/`. + ## Additional Notes - **Published binary auth fix**: When users install the npm package globally (`npm install -g @openhands/agent-canvas`) and run `agent-canvas`, the pre-built static frontend has a `VITE_SESSION_API_KEY` baked in at publish time that differs from the user's persisted runtime key (`~/.openhands/agent-canvas/session-api-key.txt`). The fix is to inject the runtime session key into `index.html` responses at serve time (not build time). `scripts/static-server.mjs` accepts a `--session-api-key ` flag and injects a tiny inline `