From 0e40272879a0bbbda38f8e727b9e7f38710e5892 Mon Sep 17 00:00:00 2001 From: openhands Date: Fri, 29 May 2026 19:02:09 +0000 Subject: [PATCH 1/4] test(snapshots): add LLM profiles visual and behavioral E2E tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cover the core LLM profiles lifecycle with 8 snapshot tests: 1. Empty state — no profiles, shows Add button 2. Profile list — 3 seeded profiles with active badge 3. Create form (blank) — empty name + LLM fields 4. Create form (filled) — name + model + API key, Save enabled 5. Delete modal — confirmation dialog 6. Rename active — badge follows the renamed profile 7. Delete active — no active badge remains after deletion 8. Activate — badge moves to the newly activated profile Profiles are seeded via fetch calls intercepted by MSW's service worker (settings-handlers.ts has full CRUD + activate handlers). Closes part of #511 (LLM Profiles section). Co-authored-by: openhands --- .../snapshots/llm-profiles.snapshot.spec.ts | 397 ++++++++++++++++++ 1 file changed, 397 insertions(+) create mode 100644 tests/e2e/snapshots/llm-profiles.snapshot.spec.ts diff --git a/tests/e2e/snapshots/llm-profiles.snapshot.spec.ts b/tests/e2e/snapshots/llm-profiles.snapshot.spec.ts new file mode 100644 index 000000000..9d2c9a7fb --- /dev/null +++ b/tests/e2e/snapshots/llm-profiles.snapshot.spec.ts @@ -0,0 +1,397 @@ +import { test, expect, Page } from "@playwright/test"; +import { seedLocalStorage } from "./support/seed-local-storage"; + +/** + * Visual snapshot tests for the LLM Profiles management UI. + * + * Covers: + * 1. Empty state (no profiles saved) + * 2. Profile list with active badge and action menu + * 3. Create profile — blank form + * 4. Create profile — filled form with Save enabled + * 5. Delete confirmation modal + * 6. Activated profile — active badge moves after activation + * + * The test server runs with VITE_MOCK_API=true (npm run dev:mock). + * MSW handles /api/profiles CRUD (settings-handlers.ts). + */ + +/** Seed profiles into MSW state via fetch calls intercepted by the service worker. */ +async function seedProfiles( + page: Page, + profiles: { name: string; model: string; apiKey?: string }[], + activeProfile?: string, +) { + for (const p of profiles) { + await page.evaluate( + async ({ name, model, apiKey }) => { + const llm: Record = { model }; + if (apiKey) llm.api_key = apiKey; + await fetch(`/api/profiles/${encodeURIComponent(name)}`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ llm }), + }); + }, + { name: p.name, model: p.model, apiKey: p.apiKey }, + ); + } + + if (activeProfile) { + await page.evaluate(async (name) => { + await fetch(`/api/profiles/${encodeURIComponent(name)}/activate`, { + method: "POST", + }); + }, activeProfile); + } +} + +async function dismissConsentModal(page: Page) { + await page + .getByRole("button", { name: "Confirm preferences" }) + .click({ timeout: 3_000 }) + .catch(() => undefined); +} + +async function setupMocks(page: Page) { + await seedLocalStorage(page); + + await page.route("**/api/conversations/search**", async (route) => { + await route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify({ results: [] }), + }); + }); +} + +/** Navigate to the LLM settings page and wait for it to stabilise. */ +async function navigateToLlmSettings(page: Page) { + await page.goto("/settings/llm", { waitUntil: "domcontentloaded" }); + await dismissConsentModal(page); + await page.waitForLoadState("networkidle"); + await expect(page.getByTestId("root-layout")).toBeVisible({ timeout: 15_000 }); +} + +const SCREENSHOT_OPTS = { animations: "disabled" as const, maxDiffPixelRatio: 0.01 }; + +test.describe("LLM Profiles Visual Snapshots", () => { + test.setTimeout(60_000); + + // ── 1. Empty state ────────────────────────────────────────────────── + + test("empty profile list shows empty state and Add button", async ({ page }) => { + await setupMocks(page); + await navigateToLlmSettings(page); + + // Verify the Add button and empty state text are visible + await expect(page.getByTestId("add-llm-profile")).toBeVisible({ timeout: 10_000 }); + + const rootLayout = page.getByTestId("root-layout"); + await expect(rootLayout).toHaveScreenshot("llm-profiles-empty.png", SCREENSHOT_OPTS); + }); + + // ── 2. Profile list with active badge ─────────────────────────────── + + test("profile list with 3 profiles and active badge renders correctly", async ({ + page, + }) => { + await setupMocks(page); + + // Seed profiles before navigating (MSW is loaded on any page) + await page.goto("about:blank"); + await seedProfiles( + page, + [ + { name: "gpt4-main", model: "openai/gpt-4o", apiKey: "sk-test-key" }, + { name: "claude-sonnet", model: "anthropic/claude-sonnet-4-20250514" }, + { name: "local-llama", model: "ollama/llama3.2" }, + ], + "gpt4-main", + ); + + await navigateToLlmSettings(page); + + // Verify profiles loaded + const profileRows = page.getByTestId("profile-row"); + await expect(profileRows).toHaveCount(3, { timeout: 10_000 }); + await expect(page.getByTestId("profile-active-badge")).toBeVisible(); + + const rootLayout = page.getByTestId("root-layout"); + await expect(rootLayout).toHaveScreenshot("llm-profiles-list.png", SCREENSHOT_OPTS); + }); + + // ── 3. Create profile — blank form ────────────────────────────────── + + test("blank create profile form renders correctly", async ({ page }) => { + await setupMocks(page); + await navigateToLlmSettings(page); + + // Click the Add button to enter create mode + await page.getByTestId("add-llm-profile").click(); + + // Wait for the create form to appear + await expect(page.getByTestId("profile-editor-title")).toBeVisible({ timeout: 10_000 }); + await expect(page.getByTestId("profile-name-input")).toBeVisible(); + await expect(page.getByTestId("save-profile-btn")).toBeVisible(); + + const rootLayout = page.getByTestId("root-layout"); + await expect(rootLayout).toHaveScreenshot( + "llm-profiles-create-empty.png", + SCREENSHOT_OPTS, + ); + }); + + // ── 4. Create profile — filled form ───────────────────────────────── + + test("filled create profile form with Save enabled renders correctly", async ({ + page, + }) => { + await setupMocks(page); + await navigateToLlmSettings(page); + + await page.getByTestId("add-llm-profile").click(); + await expect(page.getByTestId("profile-editor-title")).toBeVisible({ timeout: 10_000 }); + + // Fill in the profile name + const nameInput = page.getByTestId("profile-name-input"); + await nameInput.click(); + await nameInput.fill("my-new-profile"); + + // Fill in the model field using the custom model input (advanced view) + // The form starts in basic view with ModelSelector; type a model string + // into the custom model input if available, or use the basic model field. + const customModelInput = page.getByTestId("llm-custom-model-input"); + if (await customModelInput.isVisible({ timeout: 3_000 }).catch(() => false)) { + await customModelInput.click(); + await customModelInput.fill("openai/gpt-4o"); + } else { + // Basic view — type into the model selector combobox + const modelCombobox = page.getByRole("combobox").first(); + await modelCombobox.click(); + await modelCombobox.fill("openai/gpt-4o"); + // Select the first matching option + const option = page.getByRole("option").first(); + if (await option.isVisible({ timeout: 3_000 }).catch(() => false)) { + await option.click(); + } + } + + // Fill in the API key + const apiKeyInput = page.getByTestId("llm-api-key-input"); + await apiKeyInput.click(); + await apiKeyInput.fill("sk-test-key-123"); + + // Verify Save is enabled (not disabled) + const saveBtn = page.getByTestId("save-profile-btn"); + await expect(saveBtn).toBeEnabled({ timeout: 5_000 }); + + const rootLayout = page.getByTestId("root-layout"); + await expect(rootLayout).toHaveScreenshot( + "llm-profiles-create-filled.png", + SCREENSHOT_OPTS, + ); + }); + + // ── 5. Delete confirmation modal ──────────────────────────────────── + + test("delete profile confirmation modal renders correctly", async ({ + page, + }) => { + await setupMocks(page); + + // Seed a profile to delete + await page.goto("about:blank"); + await seedProfiles(page, [ + { name: "profile-to-delete", model: "openai/gpt-4o" }, + ]); + + await navigateToLlmSettings(page); + + // Wait for the profile row + const profileRow = page.getByTestId("profile-row"); + await expect(profileRow).toBeVisible({ timeout: 10_000 }); + + // Open the actions menu + await page.getByTestId("profile-menu-trigger").click(); + await expect(page.getByTestId("profile-actions-menu")).toBeVisible({ timeout: 5_000 }); + + // Click Delete + await page.getByTestId("profile-delete").click(); + + // Wait for the delete confirmation modal + await expect(page.getByTestId("delete-profile-confirm")).toBeVisible({ + timeout: 5_000, + }); + + const rootLayout = page.getByTestId("root-layout"); + await expect(rootLayout).toHaveScreenshot( + "llm-profiles-delete-modal.png", + SCREENSHOT_OPTS, + ); + }); + + // ── 6. Rename active profile — badge follows the new name ──────────── + + test("renaming the active profile keeps the active badge on the renamed entry", async ({ + page, + }) => { + await setupMocks(page); + + await page.goto("about:blank"); + await seedProfiles( + page, + [ + { name: "my-profile", model: "openai/gpt-4o", apiKey: "sk-key" }, + { name: "other-profile", model: "anthropic/claude-sonnet-4-20250514" }, + ], + "my-profile", + ); + + await navigateToLlmSettings(page); + + const profileRows = page.getByTestId("profile-row"); + await expect(profileRows).toHaveCount(2, { timeout: 10_000 }); + + // Verify "my-profile" row has the active badge + const firstRow = profileRows.nth(0); + await expect(firstRow.getByTestId("profile-active-badge")).toBeVisible(); + + // Open actions menu on the active profile and click Rename + await firstRow.getByTestId("profile-menu-trigger").click(); + await expect(page.getByTestId("profile-actions-menu")).toBeVisible({ timeout: 5_000 }); + await page.getByTestId("profile-rename").click(); + + // Wait for the rename modal + await expect(page.getByTestId("rename-profile-modal")).toBeVisible({ timeout: 5_000 }); + + // Clear and type the new name + const renameInput = page.getByTestId("rename-profile-input"); + await renameInput.clear(); + await renameInput.fill("my-profile-renamed"); + + // Submit + await page.getByTestId("rename-profile-submit").click(); + + // Wait for the modal to close and the list to update + await expect(page.getByTestId("rename-profile-modal")).toBeHidden({ timeout: 10_000 }); + + // The renamed profile should still have the active badge + const updatedRows = page.getByTestId("profile-row"); + await expect(updatedRows).toHaveCount(2, { timeout: 10_000 }); + + // Find the row containing the new name and assert its badge + const renamedRow = updatedRows.filter({ hasText: "my-profile-renamed" }); + await expect(renamedRow).toBeVisible({ timeout: 10_000 }); + await expect(renamedRow.getByTestId("profile-active-badge")).toBeVisible(); + + // The old name should be gone + await expect(page.getByText("my-profile", { exact: true })).toBeHidden(); + + // The other profile should NOT have the badge + const otherRow = updatedRows.filter({ hasText: "other-profile" }); + await expect(otherRow.getByTestId("profile-active-badge")).toBeHidden(); + + const rootLayout = page.getByTestId("root-layout"); + await expect(rootLayout).toHaveScreenshot( + "llm-profiles-rename-active.png", + SCREENSHOT_OPTS, + ); + }); + + // ── 7. Delete active profile — no active badge left ───────────────── + + test("deleting the active profile leaves no active badge", async ({ page }) => { + await setupMocks(page); + + await page.goto("about:blank"); + await seedProfiles( + page, + [ + { name: "active-one", model: "openai/gpt-4o", apiKey: "sk-key-a" }, + { name: "inactive-two", model: "anthropic/claude-sonnet-4-20250514" }, + ], + "active-one", + ); + + await navigateToLlmSettings(page); + + const profileRows = page.getByTestId("profile-row"); + await expect(profileRows).toHaveCount(2, { timeout: 10_000 }); + + // Verify "active-one" has the badge + const activeRow = profileRows.filter({ hasText: "active-one" }); + await expect(activeRow.getByTestId("profile-active-badge")).toBeVisible(); + + // Open actions menu on the active profile and click Delete + await activeRow.getByTestId("profile-menu-trigger").click(); + await expect(page.getByTestId("profile-actions-menu")).toBeVisible({ timeout: 5_000 }); + await page.getByTestId("profile-delete").click(); + + // Confirm deletion + await expect(page.getByTestId("delete-profile-confirm")).toBeVisible({ timeout: 5_000 }); + await page.getByTestId("delete-profile-confirm").click(); + + // Wait for the modal to close and the list to update + await expect(page.getByTestId("delete-profile-confirm")).toBeHidden({ timeout: 10_000 }); + + // Only one profile should remain + const remainingRows = page.getByTestId("profile-row"); + await expect(remainingRows).toHaveCount(1, { timeout: 10_000 }); + await expect(remainingRows.filter({ hasText: "inactive-two" })).toBeVisible(); + + // No profile should have the active badge + await expect(page.getByTestId("profile-active-badge")).toBeHidden(); + + const rootLayout = page.getByTestId("root-layout"); + await expect(rootLayout).toHaveScreenshot( + "llm-profiles-delete-active.png", + SCREENSHOT_OPTS, + ); + }); + + // ── 8. Activated profile — badge moves ────────────────────────────── + + test("activating a profile moves the active badge", async ({ page }) => { + await setupMocks(page); + + // Seed two profiles, first one active + await page.goto("about:blank"); + await seedProfiles( + page, + [ + { name: "profile-a", model: "openai/gpt-4o", apiKey: "sk-key-a" }, + { name: "profile-b", model: "anthropic/claude-sonnet-4-20250514", apiKey: "sk-key-b" }, + ], + "profile-a", + ); + + await navigateToLlmSettings(page); + + // Verify profile-a has the active badge + const profileRows = page.getByTestId("profile-row"); + await expect(profileRows).toHaveCount(2, { timeout: 10_000 }); + await expect(page.getByTestId("profile-active-badge")).toBeVisible(); + + // Open the actions menu on profile-b (the second row) + const secondRow = profileRows.nth(1); + await secondRow.getByTestId("profile-menu-trigger").click(); + await expect(page.getByTestId("profile-actions-menu")).toBeVisible({ timeout: 5_000 }); + + // Click "Set Active" + await page.getByTestId("profile-set-active").click(); + + // Wait for the badge to move — profile-b should now have it + // The activate call goes through MSW, invalidates the query, and + // the list re-renders with the new active_profile. + await expect( + secondRow.getByTestId("profile-active-badge"), + ).toBeVisible({ timeout: 10_000 }); + + const rootLayout = page.getByTestId("root-layout"); + await expect(rootLayout).toHaveScreenshot( + "llm-profiles-activated.png", + SCREENSHOT_OPTS, + ); + }); +}); From 25d012c1a7e0bac3bd13c31fd2367ea81fc54a52 Mon Sep 17 00:00:00 2001 From: openhands Date: Fri, 29 May 2026 19:31:07 +0000 Subject: [PATCH 2/4] test(mock-llm): add LLM profile lifecycle E2E tests Replace snapshot tests with proper mock-LLM E2E tests that run against the real agent-server. 4 serial steps cover the full profile lifecycle: 1. Create two profiles via UI, verify they appear in list + via API 2. Activate profile-beta, verify badge moves + API confirms active_profile 3. Rename the active profile, verify badge follows the new name + old name gone + API reflects new name 4. Delete the active profile, verify no badge remains + API returns active_profile: null Each step includes both UI assertions and API verification to confirm the agent-server persisted the expected state. Closes part of #511 (LLM Profiles section). Co-authored-by: openhands --- tests/e2e/mock-llm/mock-llm-profiles.spec.ts | 348 +++++++++++++++ .../snapshots/llm-profiles.snapshot.spec.ts | 397 ------------------ 2 files changed, 348 insertions(+), 397 deletions(-) create mode 100644 tests/e2e/mock-llm/mock-llm-profiles.spec.ts delete mode 100644 tests/e2e/snapshots/llm-profiles.snapshot.spec.ts diff --git a/tests/e2e/mock-llm/mock-llm-profiles.spec.ts b/tests/e2e/mock-llm/mock-llm-profiles.spec.ts new file mode 100644 index 000000000..338380d6f --- /dev/null +++ b/tests/e2e/mock-llm/mock-llm-profiles.spec.ts @@ -0,0 +1,348 @@ +/** + * Mock-LLM E2E test: LLM profile lifecycle management. + * + * Exercises profile CRUD, activation, rename, and delete behaviors against + * the real agent-server. No LLM responses are needed — profiles are a + * settings-layer feature managed entirely by the agent-server API. + * + * Flow (serial): + * 1. Navigate to /settings/llm, verify empty state, create two profiles + * 2. Activate the second profile, verify badge moves + * 3. Rename the active profile, verify badge follows the new name + * 4. Delete the active profile, verify no badge remains + */ + +import { test, expect, type APIRequestContext } from "@playwright/test"; +import { + BACKEND_URL, + SESSION_API_KEY, + seedLocalStorage, + routeSessionApiKey, + dismissAnalyticsModal, + waitForTestId, +} from "./utils/mock-llm-helpers"; + +const PROFILE_A = "profile-alpha"; +const PROFILE_B = "profile-beta"; +const PROFILE_B_RENAMED = "profile-beta-renamed"; +const MODEL_A = "openai/gpt-4o"; +const MODEL_B = "anthropic/claude-sonnet-4-20250514"; + +/** Delete all profiles via the API so tests start clean. */ +async function cleanupProfiles(request: APIRequestContext) { + const resp = await request.get(`${BACKEND_URL}/api/profiles`, { + headers: { "X-Session-API-Key": SESSION_API_KEY }, + }); + if (!resp.ok()) return; + const body = (await resp.json()) as { + profiles?: { name: string }[]; + }; + for (const p of body.profiles ?? []) { + await request + .delete( + `${BACKEND_URL}/api/profiles/${encodeURIComponent(p.name)}`, + { headers: { "X-Session-API-Key": SESSION_API_KEY } }, + ) + .catch(() => {}); + } +} + +test.describe.configure({ mode: "serial" }); + +test.describe("mock-LLM profile lifecycle", () => { + test.beforeEach(async ({ page }) => { + await seedLocalStorage(page); + }); + + // Clean up all profiles after the suite so subsequent test files start fresh. + test.afterAll(async ({ request }) => { + await cleanupProfiles(request); + }); + + // ── Step 1: Create two profiles ───────────────────────────────────── + + test("step 1: create two LLM profiles and verify they appear in the list", async ({ + page, + request, + }) => { + // Start clean + await cleanupProfiles(request); + + await routeSessionApiKey(page); + await page.goto("/settings/llm", { waitUntil: "domcontentloaded" }); + await dismissAnalyticsModal(page); + await waitForTestId(page, "add-llm-profile"); + + // ── Verify empty state ── + await test.step("verify empty state shows no profiles", async () => { + const profileRows = page.getByTestId("profile-row"); + await expect(profileRows).toHaveCount(0); + }); + + // ── Create profile A ── + await test.step("create profile-alpha", async () => { + await page.getByTestId("add-llm-profile").click(); + await waitForTestId(page, "profile-editor-title"); + + const nameInput = page.getByTestId("profile-name-input"); + await nameInput.click(); + await nameInput.fill(PROFILE_A); + + // Switch to advanced view for the model text input + await page.getByTestId("sdk-section-all-toggle").click(); + await waitForTestId(page, "llm-settings-form-advanced"); + + await page.getByTestId("llm-custom-model-input").click(); + await page.getByTestId("llm-custom-model-input").fill(MODEL_A); + await page.getByTestId("llm-api-key-input").click(); + await page.getByTestId("llm-api-key-input").fill("sk-test-a"); + + await page.getByTestId("save-profile-btn").click(); + await waitForTestId(page, "add-llm-profile"); + }); + + // ── Create profile B ── + await test.step("create profile-beta", async () => { + await page.getByTestId("add-llm-profile").click(); + await waitForTestId(page, "profile-editor-title"); + + const nameInput = page.getByTestId("profile-name-input"); + await nameInput.click(); + await nameInput.fill(PROFILE_B); + + await page.getByTestId("sdk-section-all-toggle").click(); + await waitForTestId(page, "llm-settings-form-advanced"); + + await page.getByTestId("llm-custom-model-input").click(); + await page.getByTestId("llm-custom-model-input").fill(MODEL_B); + await page.getByTestId("llm-api-key-input").click(); + await page.getByTestId("llm-api-key-input").fill("sk-test-b"); + + await page.getByTestId("save-profile-btn").click(); + await waitForTestId(page, "add-llm-profile"); + }); + + // ── Verify both profiles appear ── + await test.step("verify both profiles appear in the list", async () => { + const profileRows = page.getByTestId("profile-row"); + await expect(profileRows).toHaveCount(2, { timeout: 10_000 }); + + const allText = await profileRows.allTextContents(); + expect( + allText.some((t) => t.includes(PROFILE_A)), + `Expected "${PROFILE_A}" in profile list`, + ).toBe(true); + expect( + allText.some((t) => t.includes(PROFILE_B)), + `Expected "${PROFILE_B}" in profile list`, + ).toBe(true); + }); + + // ── Verify via API ── + await test.step("verify profiles exist via API", async () => { + const resp = await request.get(`${BACKEND_URL}/api/profiles`, { + headers: { "X-Session-API-Key": SESSION_API_KEY }, + }); + expect(resp.ok()).toBe(true); + + const body = (await resp.json()) as { + profiles: { name: string; model: string }[]; + active_profile: string | null; + }; + const names = body.profiles.map((p) => p.name); + expect(names).toContain(PROFILE_A); + expect(names).toContain(PROFILE_B); + }); + }); + + // ── Step 2: Activate profile B, verify badge moves ────────────────── + + test("step 2: activate profile-beta and verify the active badge", async ({ + page, + }) => { + await routeSessionApiKey(page); + await page.goto("/settings/llm", { waitUntil: "domcontentloaded" }); + await dismissAnalyticsModal(page); + await waitForTestId(page, "add-llm-profile"); + + const profileRows = page.getByTestId("profile-row"); + await expect(profileRows).toHaveCount(2, { timeout: 10_000 }); + + // Find profile-beta's row and open the actions menu + const betaRow = profileRows.filter({ hasText: PROFILE_B }); + await expect(betaRow).toBeVisible({ timeout: 5_000 }); + await betaRow.getByTestId("profile-menu-trigger").click(); + await expect(page.getByTestId("profile-actions-menu")).toBeVisible({ + timeout: 5_000, + }); + + // Click "Set Active" + await page.getByTestId("profile-set-active").click(); + + await test.step("verify active badge is on profile-beta", async () => { + // Wait for the badge to appear on profile-beta + await expect( + betaRow.getByTestId("profile-active-badge"), + ).toBeVisible({ timeout: 10_000 }); + + // profile-alpha should NOT have the badge + const alphaRow = profileRows.filter({ hasText: PROFILE_A }); + await expect( + alphaRow.getByTestId("profile-active-badge"), + ).toBeHidden(); + }); + + await test.step("verify active_profile via API", async () => { + // Use page.evaluate to call the API (session key routed by page.route) + const apiResult = await page.evaluate(async () => { + const resp = await fetch("/api/profiles"); + return resp.json(); + }); + expect(apiResult.active_profile).toBe(PROFILE_B); + }); + }); + + // ── Step 3: Rename the active profile, verify badge follows ───────── + + test("step 3: rename the active profile and verify badge follows the new name", async ({ + page, + }) => { + await routeSessionApiKey(page); + await page.goto("/settings/llm", { waitUntil: "domcontentloaded" }); + await dismissAnalyticsModal(page); + await waitForTestId(page, "add-llm-profile"); + + const profileRows = page.getByTestId("profile-row"); + await expect(profileRows).toHaveCount(2, { timeout: 10_000 }); + + // profile-beta should be active from step 2 + const betaRow = profileRows.filter({ hasText: PROFILE_B }); + await expect( + betaRow.getByTestId("profile-active-badge"), + ).toBeVisible({ timeout: 5_000 }); + + // Open actions menu and click Rename + await betaRow.getByTestId("profile-menu-trigger").click(); + await expect(page.getByTestId("profile-actions-menu")).toBeVisible({ + timeout: 5_000, + }); + await page.getByTestId("profile-rename").click(); + + // Wait for the rename modal + await expect(page.getByTestId("rename-profile-modal")).toBeVisible({ + timeout: 5_000, + }); + + // Clear and type the new name + const renameInput = page.getByTestId("rename-profile-input"); + await renameInput.clear(); + await renameInput.fill(PROFILE_B_RENAMED); + await page.getByTestId("rename-profile-submit").click(); + + // Wait for the modal to close + await expect(page.getByTestId("rename-profile-modal")).toBeHidden({ + timeout: 10_000, + }); + + await test.step("verify renamed profile has the active badge", async () => { + const updatedRows = page.getByTestId("profile-row"); + await expect(updatedRows).toHaveCount(2, { timeout: 10_000 }); + + const renamedRow = updatedRows.filter({ hasText: PROFILE_B_RENAMED }); + await expect(renamedRow).toBeVisible({ timeout: 10_000 }); + await expect( + renamedRow.getByTestId("profile-active-badge"), + ).toBeVisible(); + }); + + await test.step("verify old name is gone from the list", async () => { + // The old name should no longer appear as a standalone profile name. + // Use a strict check: no profile row should contain the old name + // without also containing the renamed suffix. + const updatedRows = page.getByTestId("profile-row"); + const allText = await updatedRows.allTextContents(); + const hasOldNameOnly = allText.some( + (t) => t.includes(PROFILE_B) && !t.includes(PROFILE_B_RENAMED), + ); + expect( + hasOldNameOnly, + `Old profile name "${PROFILE_B}" should not appear without the renamed suffix`, + ).toBe(false); + }); + + await test.step("verify other profile does NOT have the badge", async () => { + const alphaRow = page + .getByTestId("profile-row") + .filter({ hasText: PROFILE_A }); + await expect( + alphaRow.getByTestId("profile-active-badge"), + ).toBeHidden(); + }); + + await test.step("verify active_profile via API reflects the new name", async () => { + const apiResult = await page.evaluate(async () => { + const resp = await fetch("/api/profiles"); + return resp.json(); + }); + expect(apiResult.active_profile).toBe(PROFILE_B_RENAMED); + }); + }); + + // ── Step 4: Delete the active profile, verify no badge remains ────── + + test("step 4: delete the active profile and verify no active badge remains", async ({ + page, + }) => { + await routeSessionApiKey(page); + await page.goto("/settings/llm", { waitUntil: "domcontentloaded" }); + await dismissAnalyticsModal(page); + await waitForTestId(page, "add-llm-profile"); + + const profileRows = page.getByTestId("profile-row"); + await expect(profileRows).toHaveCount(2, { timeout: 10_000 }); + + // profile-beta-renamed should be active from step 3 + const activeRow = profileRows.filter({ hasText: PROFILE_B_RENAMED }); + await expect( + activeRow.getByTestId("profile-active-badge"), + ).toBeVisible({ timeout: 5_000 }); + + // Open actions menu and click Delete + await activeRow.getByTestId("profile-menu-trigger").click(); + await expect(page.getByTestId("profile-actions-menu")).toBeVisible({ + timeout: 5_000, + }); + await page.getByTestId("profile-delete").click(); + + // Confirm deletion + await expect(page.getByTestId("delete-profile-confirm")).toBeVisible({ + timeout: 5_000, + }); + await page.getByTestId("delete-profile-confirm").click(); + + // Wait for the modal to close and list to update + await expect(page.getByTestId("delete-profile-confirm")).toBeHidden({ + timeout: 10_000, + }); + + await test.step("verify only profile-alpha remains", async () => { + const remainingRows = page.getByTestId("profile-row"); + await expect(remainingRows).toHaveCount(1, { timeout: 10_000 }); + await expect( + remainingRows.filter({ hasText: PROFILE_A }), + ).toBeVisible(); + }); + + await test.step("verify no active badge is visible", async () => { + await expect(page.getByTestId("profile-active-badge")).toBeHidden(); + }); + + await test.step("verify active_profile is null via API", async () => { + const apiResult = await page.evaluate(async () => { + const resp = await fetch("/api/profiles"); + return resp.json(); + }); + expect(apiResult.active_profile).toBeNull(); + }); + }); +}); diff --git a/tests/e2e/snapshots/llm-profiles.snapshot.spec.ts b/tests/e2e/snapshots/llm-profiles.snapshot.spec.ts deleted file mode 100644 index 9d2c9a7fb..000000000 --- a/tests/e2e/snapshots/llm-profiles.snapshot.spec.ts +++ /dev/null @@ -1,397 +0,0 @@ -import { test, expect, Page } from "@playwright/test"; -import { seedLocalStorage } from "./support/seed-local-storage"; - -/** - * Visual snapshot tests for the LLM Profiles management UI. - * - * Covers: - * 1. Empty state (no profiles saved) - * 2. Profile list with active badge and action menu - * 3. Create profile — blank form - * 4. Create profile — filled form with Save enabled - * 5. Delete confirmation modal - * 6. Activated profile — active badge moves after activation - * - * The test server runs with VITE_MOCK_API=true (npm run dev:mock). - * MSW handles /api/profiles CRUD (settings-handlers.ts). - */ - -/** Seed profiles into MSW state via fetch calls intercepted by the service worker. */ -async function seedProfiles( - page: Page, - profiles: { name: string; model: string; apiKey?: string }[], - activeProfile?: string, -) { - for (const p of profiles) { - await page.evaluate( - async ({ name, model, apiKey }) => { - const llm: Record = { model }; - if (apiKey) llm.api_key = apiKey; - await fetch(`/api/profiles/${encodeURIComponent(name)}`, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ llm }), - }); - }, - { name: p.name, model: p.model, apiKey: p.apiKey }, - ); - } - - if (activeProfile) { - await page.evaluate(async (name) => { - await fetch(`/api/profiles/${encodeURIComponent(name)}/activate`, { - method: "POST", - }); - }, activeProfile); - } -} - -async function dismissConsentModal(page: Page) { - await page - .getByRole("button", { name: "Confirm preferences" }) - .click({ timeout: 3_000 }) - .catch(() => undefined); -} - -async function setupMocks(page: Page) { - await seedLocalStorage(page); - - await page.route("**/api/conversations/search**", async (route) => { - await route.fulfill({ - status: 200, - contentType: "application/json", - body: JSON.stringify({ results: [] }), - }); - }); -} - -/** Navigate to the LLM settings page and wait for it to stabilise. */ -async function navigateToLlmSettings(page: Page) { - await page.goto("/settings/llm", { waitUntil: "domcontentloaded" }); - await dismissConsentModal(page); - await page.waitForLoadState("networkidle"); - await expect(page.getByTestId("root-layout")).toBeVisible({ timeout: 15_000 }); -} - -const SCREENSHOT_OPTS = { animations: "disabled" as const, maxDiffPixelRatio: 0.01 }; - -test.describe("LLM Profiles Visual Snapshots", () => { - test.setTimeout(60_000); - - // ── 1. Empty state ────────────────────────────────────────────────── - - test("empty profile list shows empty state and Add button", async ({ page }) => { - await setupMocks(page); - await navigateToLlmSettings(page); - - // Verify the Add button and empty state text are visible - await expect(page.getByTestId("add-llm-profile")).toBeVisible({ timeout: 10_000 }); - - const rootLayout = page.getByTestId("root-layout"); - await expect(rootLayout).toHaveScreenshot("llm-profiles-empty.png", SCREENSHOT_OPTS); - }); - - // ── 2. Profile list with active badge ─────────────────────────────── - - test("profile list with 3 profiles and active badge renders correctly", async ({ - page, - }) => { - await setupMocks(page); - - // Seed profiles before navigating (MSW is loaded on any page) - await page.goto("about:blank"); - await seedProfiles( - page, - [ - { name: "gpt4-main", model: "openai/gpt-4o", apiKey: "sk-test-key" }, - { name: "claude-sonnet", model: "anthropic/claude-sonnet-4-20250514" }, - { name: "local-llama", model: "ollama/llama3.2" }, - ], - "gpt4-main", - ); - - await navigateToLlmSettings(page); - - // Verify profiles loaded - const profileRows = page.getByTestId("profile-row"); - await expect(profileRows).toHaveCount(3, { timeout: 10_000 }); - await expect(page.getByTestId("profile-active-badge")).toBeVisible(); - - const rootLayout = page.getByTestId("root-layout"); - await expect(rootLayout).toHaveScreenshot("llm-profiles-list.png", SCREENSHOT_OPTS); - }); - - // ── 3. Create profile — blank form ────────────────────────────────── - - test("blank create profile form renders correctly", async ({ page }) => { - await setupMocks(page); - await navigateToLlmSettings(page); - - // Click the Add button to enter create mode - await page.getByTestId("add-llm-profile").click(); - - // Wait for the create form to appear - await expect(page.getByTestId("profile-editor-title")).toBeVisible({ timeout: 10_000 }); - await expect(page.getByTestId("profile-name-input")).toBeVisible(); - await expect(page.getByTestId("save-profile-btn")).toBeVisible(); - - const rootLayout = page.getByTestId("root-layout"); - await expect(rootLayout).toHaveScreenshot( - "llm-profiles-create-empty.png", - SCREENSHOT_OPTS, - ); - }); - - // ── 4. Create profile — filled form ───────────────────────────────── - - test("filled create profile form with Save enabled renders correctly", async ({ - page, - }) => { - await setupMocks(page); - await navigateToLlmSettings(page); - - await page.getByTestId("add-llm-profile").click(); - await expect(page.getByTestId("profile-editor-title")).toBeVisible({ timeout: 10_000 }); - - // Fill in the profile name - const nameInput = page.getByTestId("profile-name-input"); - await nameInput.click(); - await nameInput.fill("my-new-profile"); - - // Fill in the model field using the custom model input (advanced view) - // The form starts in basic view with ModelSelector; type a model string - // into the custom model input if available, or use the basic model field. - const customModelInput = page.getByTestId("llm-custom-model-input"); - if (await customModelInput.isVisible({ timeout: 3_000 }).catch(() => false)) { - await customModelInput.click(); - await customModelInput.fill("openai/gpt-4o"); - } else { - // Basic view — type into the model selector combobox - const modelCombobox = page.getByRole("combobox").first(); - await modelCombobox.click(); - await modelCombobox.fill("openai/gpt-4o"); - // Select the first matching option - const option = page.getByRole("option").first(); - if (await option.isVisible({ timeout: 3_000 }).catch(() => false)) { - await option.click(); - } - } - - // Fill in the API key - const apiKeyInput = page.getByTestId("llm-api-key-input"); - await apiKeyInput.click(); - await apiKeyInput.fill("sk-test-key-123"); - - // Verify Save is enabled (not disabled) - const saveBtn = page.getByTestId("save-profile-btn"); - await expect(saveBtn).toBeEnabled({ timeout: 5_000 }); - - const rootLayout = page.getByTestId("root-layout"); - await expect(rootLayout).toHaveScreenshot( - "llm-profiles-create-filled.png", - SCREENSHOT_OPTS, - ); - }); - - // ── 5. Delete confirmation modal ──────────────────────────────────── - - test("delete profile confirmation modal renders correctly", async ({ - page, - }) => { - await setupMocks(page); - - // Seed a profile to delete - await page.goto("about:blank"); - await seedProfiles(page, [ - { name: "profile-to-delete", model: "openai/gpt-4o" }, - ]); - - await navigateToLlmSettings(page); - - // Wait for the profile row - const profileRow = page.getByTestId("profile-row"); - await expect(profileRow).toBeVisible({ timeout: 10_000 }); - - // Open the actions menu - await page.getByTestId("profile-menu-trigger").click(); - await expect(page.getByTestId("profile-actions-menu")).toBeVisible({ timeout: 5_000 }); - - // Click Delete - await page.getByTestId("profile-delete").click(); - - // Wait for the delete confirmation modal - await expect(page.getByTestId("delete-profile-confirm")).toBeVisible({ - timeout: 5_000, - }); - - const rootLayout = page.getByTestId("root-layout"); - await expect(rootLayout).toHaveScreenshot( - "llm-profiles-delete-modal.png", - SCREENSHOT_OPTS, - ); - }); - - // ── 6. Rename active profile — badge follows the new name ──────────── - - test("renaming the active profile keeps the active badge on the renamed entry", async ({ - page, - }) => { - await setupMocks(page); - - await page.goto("about:blank"); - await seedProfiles( - page, - [ - { name: "my-profile", model: "openai/gpt-4o", apiKey: "sk-key" }, - { name: "other-profile", model: "anthropic/claude-sonnet-4-20250514" }, - ], - "my-profile", - ); - - await navigateToLlmSettings(page); - - const profileRows = page.getByTestId("profile-row"); - await expect(profileRows).toHaveCount(2, { timeout: 10_000 }); - - // Verify "my-profile" row has the active badge - const firstRow = profileRows.nth(0); - await expect(firstRow.getByTestId("profile-active-badge")).toBeVisible(); - - // Open actions menu on the active profile and click Rename - await firstRow.getByTestId("profile-menu-trigger").click(); - await expect(page.getByTestId("profile-actions-menu")).toBeVisible({ timeout: 5_000 }); - await page.getByTestId("profile-rename").click(); - - // Wait for the rename modal - await expect(page.getByTestId("rename-profile-modal")).toBeVisible({ timeout: 5_000 }); - - // Clear and type the new name - const renameInput = page.getByTestId("rename-profile-input"); - await renameInput.clear(); - await renameInput.fill("my-profile-renamed"); - - // Submit - await page.getByTestId("rename-profile-submit").click(); - - // Wait for the modal to close and the list to update - await expect(page.getByTestId("rename-profile-modal")).toBeHidden({ timeout: 10_000 }); - - // The renamed profile should still have the active badge - const updatedRows = page.getByTestId("profile-row"); - await expect(updatedRows).toHaveCount(2, { timeout: 10_000 }); - - // Find the row containing the new name and assert its badge - const renamedRow = updatedRows.filter({ hasText: "my-profile-renamed" }); - await expect(renamedRow).toBeVisible({ timeout: 10_000 }); - await expect(renamedRow.getByTestId("profile-active-badge")).toBeVisible(); - - // The old name should be gone - await expect(page.getByText("my-profile", { exact: true })).toBeHidden(); - - // The other profile should NOT have the badge - const otherRow = updatedRows.filter({ hasText: "other-profile" }); - await expect(otherRow.getByTestId("profile-active-badge")).toBeHidden(); - - const rootLayout = page.getByTestId("root-layout"); - await expect(rootLayout).toHaveScreenshot( - "llm-profiles-rename-active.png", - SCREENSHOT_OPTS, - ); - }); - - // ── 7. Delete active profile — no active badge left ───────────────── - - test("deleting the active profile leaves no active badge", async ({ page }) => { - await setupMocks(page); - - await page.goto("about:blank"); - await seedProfiles( - page, - [ - { name: "active-one", model: "openai/gpt-4o", apiKey: "sk-key-a" }, - { name: "inactive-two", model: "anthropic/claude-sonnet-4-20250514" }, - ], - "active-one", - ); - - await navigateToLlmSettings(page); - - const profileRows = page.getByTestId("profile-row"); - await expect(profileRows).toHaveCount(2, { timeout: 10_000 }); - - // Verify "active-one" has the badge - const activeRow = profileRows.filter({ hasText: "active-one" }); - await expect(activeRow.getByTestId("profile-active-badge")).toBeVisible(); - - // Open actions menu on the active profile and click Delete - await activeRow.getByTestId("profile-menu-trigger").click(); - await expect(page.getByTestId("profile-actions-menu")).toBeVisible({ timeout: 5_000 }); - await page.getByTestId("profile-delete").click(); - - // Confirm deletion - await expect(page.getByTestId("delete-profile-confirm")).toBeVisible({ timeout: 5_000 }); - await page.getByTestId("delete-profile-confirm").click(); - - // Wait for the modal to close and the list to update - await expect(page.getByTestId("delete-profile-confirm")).toBeHidden({ timeout: 10_000 }); - - // Only one profile should remain - const remainingRows = page.getByTestId("profile-row"); - await expect(remainingRows).toHaveCount(1, { timeout: 10_000 }); - await expect(remainingRows.filter({ hasText: "inactive-two" })).toBeVisible(); - - // No profile should have the active badge - await expect(page.getByTestId("profile-active-badge")).toBeHidden(); - - const rootLayout = page.getByTestId("root-layout"); - await expect(rootLayout).toHaveScreenshot( - "llm-profiles-delete-active.png", - SCREENSHOT_OPTS, - ); - }); - - // ── 8. Activated profile — badge moves ────────────────────────────── - - test("activating a profile moves the active badge", async ({ page }) => { - await setupMocks(page); - - // Seed two profiles, first one active - await page.goto("about:blank"); - await seedProfiles( - page, - [ - { name: "profile-a", model: "openai/gpt-4o", apiKey: "sk-key-a" }, - { name: "profile-b", model: "anthropic/claude-sonnet-4-20250514", apiKey: "sk-key-b" }, - ], - "profile-a", - ); - - await navigateToLlmSettings(page); - - // Verify profile-a has the active badge - const profileRows = page.getByTestId("profile-row"); - await expect(profileRows).toHaveCount(2, { timeout: 10_000 }); - await expect(page.getByTestId("profile-active-badge")).toBeVisible(); - - // Open the actions menu on profile-b (the second row) - const secondRow = profileRows.nth(1); - await secondRow.getByTestId("profile-menu-trigger").click(); - await expect(page.getByTestId("profile-actions-menu")).toBeVisible({ timeout: 5_000 }); - - // Click "Set Active" - await page.getByTestId("profile-set-active").click(); - - // Wait for the badge to move — profile-b should now have it - // The activate call goes through MSW, invalidates the query, and - // the list re-renders with the new active_profile. - await expect( - secondRow.getByTestId("profile-active-badge"), - ).toBeVisible({ timeout: 10_000 }); - - const rootLayout = page.getByTestId("root-layout"); - await expect(rootLayout).toHaveScreenshot( - "llm-profiles-activated.png", - SCREENSHOT_OPTS, - ); - }); -}); From dae3e750aef38d4a8332a144cf73f6b893d2fb4f Mon Sep 17 00:00:00 2001 From: openhands Date: Mon, 1 Jun 2026 14:45:53 +0000 Subject: [PATCH 3/4] fix(mock-llm): make profile cleanup robust against settings-level LLM config The profiles test's cleanupProfiles() only deleted named profiles via DELETE /api/profiles/{name}. The automation test's ensureMockLLMProfile() PATCHes /api/settings with agent_settings_diff.llm.*, which persists LLM config at the settings layer. After deleting named profiles, the server may still surface a "default" profile derived from those lingering settings, causing the 'verify empty state' assertion to find 1 profile instead of 0. Fix: - Also PATCH /api/settings to clear LLM model/api_key/base_url after deleting named profiles - Add diagnostic logging for non-OK API responses instead of silently swallowing errors - Add a verification step that re-fetches profiles after cleanup and warns if any remain - Increase empty-state assertion timeout to 10s for slower CI Co-authored-by: openhands --- tests/e2e/mock-llm/mock-llm-profiles.spec.ts | 84 ++++++++++++++++---- 1 file changed, 70 insertions(+), 14 deletions(-) diff --git a/tests/e2e/mock-llm/mock-llm-profiles.spec.ts b/tests/e2e/mock-llm/mock-llm-profiles.spec.ts index 338380d6f..2f47ffcf6 100644 --- a/tests/e2e/mock-llm/mock-llm-profiles.spec.ts +++ b/tests/e2e/mock-llm/mock-llm-profiles.spec.ts @@ -28,22 +28,78 @@ const PROFILE_B_RENAMED = "profile-beta-renamed"; const MODEL_A = "openai/gpt-4o"; const MODEL_B = "anthropic/claude-sonnet-4-20250514"; -/** Delete all profiles via the API so tests start clean. */ +/** + * Delete all profiles AND reset settings-level LLM config so tests start + * truly clean. + * + * The automation test's `ensureMockLLMProfile()` PATCHes `/api/settings` + * with `agent_settings_diff.llm.*`, which stores LLM config at the + * settings layer. Deleting named profiles alone is not enough — the + * server may still surface a "default" profile derived from those + * settings. We therefore also PATCH settings to clear the LLM model so + * the server reports an empty profile list. + */ async function cleanupProfiles(request: APIRequestContext) { - const resp = await request.get(`${BACKEND_URL}/api/profiles`, { - headers: { "X-Session-API-Key": SESSION_API_KEY }, + const headers = { "X-Session-API-Key": SESSION_API_KEY }; + + // 1. Delete all named profiles + const listResp = await request.get(`${BACKEND_URL}/api/profiles`, { + headers, }); - if (!resp.ok()) return; - const body = (await resp.json()) as { - profiles?: { name: string }[]; - }; - for (const p of body.profiles ?? []) { - await request - .delete( + if (listResp.ok()) { + const body = (await listResp.json()) as { + profiles?: { name: string }[]; + }; + for (const p of body.profiles ?? []) { + const delResp = await request.delete( `${BACKEND_URL}/api/profiles/${encodeURIComponent(p.name)}`, - { headers: { "X-Session-API-Key": SESSION_API_KEY } }, - ) - .catch(() => {}); + { headers }, + ); + if (!delResp.ok()) { + console.warn( + `DELETE /api/profiles/${p.name} returned ${delResp.status()}`, + ); + } + } + } else { + console.warn( + `GET /api/profiles returned ${listResp.status()} — skipping profile cleanup`, + ); + } + + // 2. Clear settings-level LLM config so no implicit profile lingers. + // The automation test's ensureMockLLMProfile() PATCHes agent_settings + // directly; without this reset, the server may derive a "default" + // profile from the lingering settings even after all named profiles + // are deleted. + const patchResp = await request.patch(`${BACKEND_URL}/api/settings`, { + headers: { ...headers, "Content-Type": "application/json" }, + data: { + agent_settings_diff: { + llm: { model: "", api_key: "", base_url: "" }, + }, + }, + }); + if (!patchResp.ok()) { + console.warn( + `PATCH /api/settings (clear LLM) returned ${patchResp.status()}`, + ); + } + + // 3. Verify the cleanup actually worked + const verifyResp = await request.get(`${BACKEND_URL}/api/profiles`, { + headers, + }); + if (verifyResp.ok()) { + const verifyBody = (await verifyResp.json()) as { + profiles?: { name: string }[]; + }; + const remaining = verifyBody.profiles ?? []; + if (remaining.length > 0) { + console.warn( + `After cleanup, ${remaining.length} profile(s) still exist: ${remaining.map((p) => p.name).join(", ")}`, + ); + } } } @@ -76,7 +132,7 @@ test.describe("mock-LLM profile lifecycle", () => { // ── Verify empty state ── await test.step("verify empty state shows no profiles", async () => { const profileRows = page.getByTestId("profile-row"); - await expect(profileRows).toHaveCount(0); + await expect(profileRows).toHaveCount(0, { timeout: 10_000 }); }); // ── Create profile A ── From 1f5784c11ece56e23873f6bb85a5a9a8fd065faa Mon Sep 17 00:00:00 2001 From: openhands Date: Mon, 1 Jun 2026 15:14:38 +0000 Subject: [PATCH 4/4] =?UTF-8?q?test(mock-llm):=20add=20folder=20browser=20?= =?UTF-8?q?=E2=86=92=20workspace=20=E2=86=92=20conversation=20E2E=20test?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Exercise the full flow: open folder browser → navigate to a test directory → 'Use this folder' → select in dropdown → confirm → type message → submit → verify POST /api/conversations payload has correct workspace.working_dir → verify selected_workspace persisted in localStorage. Covers two 'I can' statements from #511: - 'I can browse local files and folders to choose where to begin' - 'I can start a conversation against a local Git repo without typing the path' The test creates /tmp/e2e-folder-workspace-test/my-test-project before running and cleans it up after. The folder browser navigates the real filesystem served by the agent-server's /api/file/ endpoints. Co-authored-by: openhands --- .../mock-llm-folder-workspace.spec.ts | 270 ++++++++++++++++++ 1 file changed, 270 insertions(+) create mode 100644 tests/e2e/mock-llm/mock-llm-folder-workspace.spec.ts diff --git a/tests/e2e/mock-llm/mock-llm-folder-workspace.spec.ts b/tests/e2e/mock-llm/mock-llm-folder-workspace.spec.ts new file mode 100644 index 000000000..f78c248fb --- /dev/null +++ b/tests/e2e/mock-llm/mock-llm-folder-workspace.spec.ts @@ -0,0 +1,270 @@ +/** + * Mock-LLM E2E test: folder browsing → workspace selection → conversation creation. + * + * Covers two "I can" statements from issue #511: + * - "I can browse local files and folders to choose where to begin" + * - "I can start a conversation against a local Git repo without typing the path" + * + * Flow (serial): + * 1. Open the folder browser, navigate to a known test directory, click + * "Use this folder" — verify the workspace appears in the dropdown + * 2. Select the workspace, type a message, submit — intercept + * POST /api/conversations and assert workspace.working_dir matches + * the selected folder path + * 3. After conversation creation, verify selected_workspace is persisted + * in localStorage under the conversation's metadata key + */ + +import { test, expect } from "@playwright/test"; +import { + seedLocalStorage, + routeSessionApiKey, + dismissAnalyticsModal, + waitForTestId, + waitForPath, + ensureMockLLMProfile, + resetMockLLM, + deleteConversation, +} from "./utils/mock-llm-helpers"; +import * as fs from "fs"; +import * as path from "path"; + +/** A unique directory that is created before the test and cleaned up after. */ +const TEST_DIR_BASE = "/tmp/e2e-folder-workspace-test"; +const TEST_DIR_NAME = "my-test-project"; +const TEST_DIR = path.join(TEST_DIR_BASE, TEST_DIR_NAME); + +const METADATA_STORAGE_KEY = "openhands-agent-server-conversation-metadata"; + +test.describe.configure({ mode: "serial" }); + +test.describe("mock-LLM folder browser → workspace → conversation", () => { + const conversationIds = new Set(); + + test.beforeAll(async ({ request }) => { + // Create the test directory hierarchy + fs.mkdirSync(TEST_DIR, { recursive: true }); + + // Ensure the mock LLM profile is configured so conversations can start + await ensureMockLLMProfile(request); + }); + + test.beforeEach(async ({ page }) => { + await seedLocalStorage(page); + }); + + test.afterEach(async ({ request }) => { + await resetMockLLM(request); + + // Cleanup conversations created during the test + for (const id of conversationIds) { + try { + await deleteConversation(request, id); + } catch { + // best-effort + } + } + conversationIds.clear(); + }); + + test.afterAll(async () => { + // Remove the test directory + try { + fs.rmSync(TEST_DIR_BASE, { recursive: true, force: true }); + } catch { + // best-effort + } + }); + + // ── Step 1: Browse to a folder and add it as a workspace ──────────── + + test("step 1: browse to a folder, add it as a workspace, and launch a conversation with the correct working_dir", async ({ + page, + }) => { + // Set up passive listener for POST /api/conversations BEFORE navigation. + // Uses page.on('request') (not page.route) to avoid conflicts with + // routeSessionApiKey — only one handler can call continue() per request. + let capturedPayload: Record | null = null; + const captureConversationPayload = ( + req: import("@playwright/test").Request, + ) => { + if ( + req.method() === "POST" && + new URL(req.url()).pathname === "/api/conversations" + ) { + try { + capturedPayload = req.postDataJSON(); + } catch { + // non-JSON body + } + } + }; + page.on("request", captureConversationPayload); + + await routeSessionApiKey(page); + await page.goto("/", { waitUntil: "domcontentloaded" }); + await dismissAnalyticsModal(page); + await waitForTestId(page, "home-chat-launcher"); + + // ── Open the "Open Workspace" dialog ── + await test.step("open workspace dialog", async () => { + await page.getByTestId("open-workspace-button").click(); + await expect( + page.getByTestId("open-workspace-dialog-body"), + ).toBeVisible({ timeout: 10_000 }); + }); + + // ── Browse to the test directory using the folder browser UI ── + await test.step("open folder browser and navigate to test directory", async () => { + await page.getByTestId("add-workspaces-button").click(); + await expect(page.getByTestId("folder-browser-modal")).toBeVisible({ + timeout: 10_000, + }); + + // Navigate up to root first — click the "up" button repeatedly + // until we reach "/" (path shows "/" or up button is disabled). + const upBtn = page.getByTestId("folder-browser-up"); + const currentPathEl = page.getByTestId("folder-browser-current-path"); + + // Keep clicking up until disabled (at root) + for (let i = 0; i < 10; i++) { + if (await upBtn.isDisabled()) break; + await upBtn.click(); + // Small wait for the listing to load + await page.waitForTimeout(300); + } + + // Now navigate down: / → tmp → e2e-folder-workspace-test → my-test-project + const tmpEntry = page.getByTestId("folder-browser-entry-tmp"); + await expect(tmpEntry).toBeVisible({ timeout: 10_000 }); + await tmpEntry.click(); + await expect(currentPathEl).toHaveText(/\/tmp/, { timeout: 5_000 }); + + const baseEntry = page.getByTestId( + `folder-browser-entry-${path.basename(TEST_DIR_BASE)}`, + ); + await expect(baseEntry).toBeVisible({ timeout: 10_000 }); + await baseEntry.click(); + + const projectEntry = page.getByTestId( + `folder-browser-entry-${TEST_DIR_NAME}`, + ); + await expect(projectEntry).toBeVisible({ timeout: 10_000 }); + await projectEntry.click(); + + // Verify we're at the correct path + await expect(currentPathEl).toHaveText(TEST_DIR, { timeout: 5_000 }); + + // Click "Use this folder" + await page.getByTestId("folder-browser-use").click(); + + // Modal should close + await expect(page.getByTestId("folder-browser-modal")).toBeHidden({ + timeout: 5_000, + }); + }); + + // ── Select the workspace in the dropdown and confirm ── + // The workspace dialog is still open after the folder browser closed. + // The folder browser's onAdd adds the workspace to the store, so the + // dropdown should now include it. + await test.step("select the workspace in the dropdown and confirm", async () => { + // The workspace dialog should still be visible + await expect( + page.getByTestId("open-workspace-dialog-body"), + ).toBeVisible({ timeout: 10_000 }); + + // The workspace dropdown should contain our test directory. + const dropdown = page.getByTestId("workspace-dropdown"); + await expect(dropdown).toBeVisible({ timeout: 10_000 }); + await dropdown.click(); + + // Find and click the entry for our test workspace + const workspaceOption = page.getByRole("option", { + name: new RegExp(TEST_DIR_NAME), + }); + await expect(workspaceOption).toBeVisible({ timeout: 10_000 }); + await workspaceOption.click(); + + // Click "Confirm" to accept the workspace selection + const confirmBtn = page.getByRole("button", { name: /confirm/i }); + await confirmBtn.click(); + + // The dialog should close + await expect( + page.getByTestId("open-workspace-dialog-body"), + ).toBeHidden({ timeout: 5_000 }); + }); + + // ── Type a message and submit to create a conversation ── + await test.step("submit a message to create a conversation", async () => { + // Type into the home-page chat input (contentEditable div) + const chatInput = page.getByTestId("home-chat-launcher").locator( + '[contenteditable="true"]', + ); + await expect(chatInput).toBeVisible({ timeout: 10_000 }); + await chatInput.click(); + + await page.evaluate((msg: string) => { + const el = document.querySelector( + '[data-testid="home-chat-launcher"] [contenteditable="true"]', + ); + if (el) { + el.textContent = msg; + el.dispatchEvent(new Event("input", { bubbles: true })); + } + }, "Hello from the workspace test"); + + // Submit with Enter + await chatInput.press("Enter"); + + // Wait for navigation to a conversation page + await waitForPath(page, /\/conversations\/.+/, 30_000); + }); + + // Track the conversation for cleanup + const match = page.url().match(/\/conversations\/([^/?#]+)/); + const conversationId = match?.[1] ? decodeURIComponent(match[1]) : null; + expect(conversationId, "Should be on a conversation page").toBeTruthy(); + conversationIds.add(conversationId!); + + // ── Verify: POST /api/conversations payload has correct working_dir ── + await test.step("verify working_dir in POST /api/conversations payload", async () => { + expect( + capturedPayload, + "POST /api/conversations payload was not captured", + ).not.toBeNull(); + + const workspace = capturedPayload?.workspace as + | Record + | undefined; + expect(workspace, "payload should have a workspace object").toBeTruthy(); + expect(workspace?.working_dir).toBe(TEST_DIR); + }); + + // ── Verify: selected_workspace in localStorage ── + await test.step("verify selected_workspace in localStorage", async () => { + const metadata = await page.evaluate( + ({ key, convId }) => { + const raw = window.localStorage.getItem(key); + if (!raw) return null; + try { + const parsed = JSON.parse(raw); + return parsed[convId] ?? null; + } catch { + return null; + } + }, + { key: METADATA_STORAGE_KEY, convId: conversationId! }, + ); + + expect( + metadata, + `localStorage metadata for conversation ${conversationId} should exist`, + ).not.toBeNull(); + expect(metadata?.selected_workspace).toBe(TEST_DIR); + }); + + page.off("request", captureConversationPayload); + }); +});