diff --git a/tests/e2e/mock-llm/mock-llm-folder-workspace.spec.ts b/tests/e2e/mock-llm/mock-llm-folder-workspace.spec.ts new file mode 100644 index 00000000..f78c248f --- /dev/null +++ b/tests/e2e/mock-llm/mock-llm-folder-workspace.spec.ts @@ -0,0 +1,270 @@ +/** + * Mock-LLM E2E test: folder browsing → workspace selection → conversation creation. + * + * Covers two "I can" statements from issue #511: + * - "I can browse local files and folders to choose where to begin" + * - "I can start a conversation against a local Git repo without typing the path" + * + * Flow (serial): + * 1. Open the folder browser, navigate to a known test directory, click + * "Use this folder" — verify the workspace appears in the dropdown + * 2. Select the workspace, type a message, submit — intercept + * POST /api/conversations and assert workspace.working_dir matches + * the selected folder path + * 3. After conversation creation, verify selected_workspace is persisted + * in localStorage under the conversation's metadata key + */ + +import { test, expect } from "@playwright/test"; +import { + seedLocalStorage, + routeSessionApiKey, + dismissAnalyticsModal, + waitForTestId, + waitForPath, + ensureMockLLMProfile, + resetMockLLM, + deleteConversation, +} from "./utils/mock-llm-helpers"; +import * as fs from "fs"; +import * as path from "path"; + +/** A unique directory that is created before the test and cleaned up after. */ +const TEST_DIR_BASE = "/tmp/e2e-folder-workspace-test"; +const TEST_DIR_NAME = "my-test-project"; +const TEST_DIR = path.join(TEST_DIR_BASE, TEST_DIR_NAME); + +const METADATA_STORAGE_KEY = "openhands-agent-server-conversation-metadata"; + +test.describe.configure({ mode: "serial" }); + +test.describe("mock-LLM folder browser → workspace → conversation", () => { + const conversationIds = new Set(); + + test.beforeAll(async ({ request }) => { + // Create the test directory hierarchy + fs.mkdirSync(TEST_DIR, { recursive: true }); + + // Ensure the mock LLM profile is configured so conversations can start + await ensureMockLLMProfile(request); + }); + + test.beforeEach(async ({ page }) => { + await seedLocalStorage(page); + }); + + test.afterEach(async ({ request }) => { + await resetMockLLM(request); + + // Cleanup conversations created during the test + for (const id of conversationIds) { + try { + await deleteConversation(request, id); + } catch { + // best-effort + } + } + conversationIds.clear(); + }); + + test.afterAll(async () => { + // Remove the test directory + try { + fs.rmSync(TEST_DIR_BASE, { recursive: true, force: true }); + } catch { + // best-effort + } + }); + + // ── Step 1: Browse to a folder and add it as a workspace ──────────── + + test("step 1: browse to a folder, add it as a workspace, and launch a conversation with the correct working_dir", async ({ + page, + }) => { + // Set up passive listener for POST /api/conversations BEFORE navigation. + // Uses page.on('request') (not page.route) to avoid conflicts with + // routeSessionApiKey — only one handler can call continue() per request. + let capturedPayload: Record | null = null; + const captureConversationPayload = ( + req: import("@playwright/test").Request, + ) => { + if ( + req.method() === "POST" && + new URL(req.url()).pathname === "/api/conversations" + ) { + try { + capturedPayload = req.postDataJSON(); + } catch { + // non-JSON body + } + } + }; + page.on("request", captureConversationPayload); + + await routeSessionApiKey(page); + await page.goto("/", { waitUntil: "domcontentloaded" }); + await dismissAnalyticsModal(page); + await waitForTestId(page, "home-chat-launcher"); + + // ── Open the "Open Workspace" dialog ── + await test.step("open workspace dialog", async () => { + await page.getByTestId("open-workspace-button").click(); + await expect( + page.getByTestId("open-workspace-dialog-body"), + ).toBeVisible({ timeout: 10_000 }); + }); + + // ── Browse to the test directory using the folder browser UI ── + await test.step("open folder browser and navigate to test directory", async () => { + await page.getByTestId("add-workspaces-button").click(); + await expect(page.getByTestId("folder-browser-modal")).toBeVisible({ + timeout: 10_000, + }); + + // Navigate up to root first — click the "up" button repeatedly + // until we reach "/" (path shows "/" or up button is disabled). + const upBtn = page.getByTestId("folder-browser-up"); + const currentPathEl = page.getByTestId("folder-browser-current-path"); + + // Keep clicking up until disabled (at root) + for (let i = 0; i < 10; i++) { + if (await upBtn.isDisabled()) break; + await upBtn.click(); + // Small wait for the listing to load + await page.waitForTimeout(300); + } + + // Now navigate down: / → tmp → e2e-folder-workspace-test → my-test-project + const tmpEntry = page.getByTestId("folder-browser-entry-tmp"); + await expect(tmpEntry).toBeVisible({ timeout: 10_000 }); + await tmpEntry.click(); + await expect(currentPathEl).toHaveText(/\/tmp/, { timeout: 5_000 }); + + const baseEntry = page.getByTestId( + `folder-browser-entry-${path.basename(TEST_DIR_BASE)}`, + ); + await expect(baseEntry).toBeVisible({ timeout: 10_000 }); + await baseEntry.click(); + + const projectEntry = page.getByTestId( + `folder-browser-entry-${TEST_DIR_NAME}`, + ); + await expect(projectEntry).toBeVisible({ timeout: 10_000 }); + await projectEntry.click(); + + // Verify we're at the correct path + await expect(currentPathEl).toHaveText(TEST_DIR, { timeout: 5_000 }); + + // Click "Use this folder" + await page.getByTestId("folder-browser-use").click(); + + // Modal should close + await expect(page.getByTestId("folder-browser-modal")).toBeHidden({ + timeout: 5_000, + }); + }); + + // ── Select the workspace in the dropdown and confirm ── + // The workspace dialog is still open after the folder browser closed. + // The folder browser's onAdd adds the workspace to the store, so the + // dropdown should now include it. + await test.step("select the workspace in the dropdown and confirm", async () => { + // The workspace dialog should still be visible + await expect( + page.getByTestId("open-workspace-dialog-body"), + ).toBeVisible({ timeout: 10_000 }); + + // The workspace dropdown should contain our test directory. + const dropdown = page.getByTestId("workspace-dropdown"); + await expect(dropdown).toBeVisible({ timeout: 10_000 }); + await dropdown.click(); + + // Find and click the entry for our test workspace + const workspaceOption = page.getByRole("option", { + name: new RegExp(TEST_DIR_NAME), + }); + await expect(workspaceOption).toBeVisible({ timeout: 10_000 }); + await workspaceOption.click(); + + // Click "Confirm" to accept the workspace selection + const confirmBtn = page.getByRole("button", { name: /confirm/i }); + await confirmBtn.click(); + + // The dialog should close + await expect( + page.getByTestId("open-workspace-dialog-body"), + ).toBeHidden({ timeout: 5_000 }); + }); + + // ── Type a message and submit to create a conversation ── + await test.step("submit a message to create a conversation", async () => { + // Type into the home-page chat input (contentEditable div) + const chatInput = page.getByTestId("home-chat-launcher").locator( + '[contenteditable="true"]', + ); + await expect(chatInput).toBeVisible({ timeout: 10_000 }); + await chatInput.click(); + + await page.evaluate((msg: string) => { + const el = document.querySelector( + '[data-testid="home-chat-launcher"] [contenteditable="true"]', + ); + if (el) { + el.textContent = msg; + el.dispatchEvent(new Event("input", { bubbles: true })); + } + }, "Hello from the workspace test"); + + // Submit with Enter + await chatInput.press("Enter"); + + // Wait for navigation to a conversation page + await waitForPath(page, /\/conversations\/.+/, 30_000); + }); + + // Track the conversation for cleanup + const match = page.url().match(/\/conversations\/([^/?#]+)/); + const conversationId = match?.[1] ? decodeURIComponent(match[1]) : null; + expect(conversationId, "Should be on a conversation page").toBeTruthy(); + conversationIds.add(conversationId!); + + // ── Verify: POST /api/conversations payload has correct working_dir ── + await test.step("verify working_dir in POST /api/conversations payload", async () => { + expect( + capturedPayload, + "POST /api/conversations payload was not captured", + ).not.toBeNull(); + + const workspace = capturedPayload?.workspace as + | Record + | undefined; + expect(workspace, "payload should have a workspace object").toBeTruthy(); + expect(workspace?.working_dir).toBe(TEST_DIR); + }); + + // ── Verify: selected_workspace in localStorage ── + await test.step("verify selected_workspace in localStorage", async () => { + const metadata = await page.evaluate( + ({ key, convId }) => { + const raw = window.localStorage.getItem(key); + if (!raw) return null; + try { + const parsed = JSON.parse(raw); + return parsed[convId] ?? null; + } catch { + return null; + } + }, + { key: METADATA_STORAGE_KEY, convId: conversationId! }, + ); + + expect( + metadata, + `localStorage metadata for conversation ${conversationId} should exist`, + ).not.toBeNull(); + expect(metadata?.selected_workspace).toBe(TEST_DIR); + }); + + page.off("request", captureConversationPayload); + }); +}); diff --git a/tests/e2e/mock-llm/mock-llm-profiles.spec.ts b/tests/e2e/mock-llm/mock-llm-profiles.spec.ts new file mode 100644 index 00000000..2f47ffcf --- /dev/null +++ b/tests/e2e/mock-llm/mock-llm-profiles.spec.ts @@ -0,0 +1,404 @@ +/** + * Mock-LLM E2E test: LLM profile lifecycle management. + * + * Exercises profile CRUD, activation, rename, and delete behaviors against + * the real agent-server. No LLM responses are needed — profiles are a + * settings-layer feature managed entirely by the agent-server API. + * + * Flow (serial): + * 1. Navigate to /settings/llm, verify empty state, create two profiles + * 2. Activate the second profile, verify badge moves + * 3. Rename the active profile, verify badge follows the new name + * 4. Delete the active profile, verify no badge remains + */ + +import { test, expect, type APIRequestContext } from "@playwright/test"; +import { + BACKEND_URL, + SESSION_API_KEY, + seedLocalStorage, + routeSessionApiKey, + dismissAnalyticsModal, + waitForTestId, +} from "./utils/mock-llm-helpers"; + +const PROFILE_A = "profile-alpha"; +const PROFILE_B = "profile-beta"; +const PROFILE_B_RENAMED = "profile-beta-renamed"; +const MODEL_A = "openai/gpt-4o"; +const MODEL_B = "anthropic/claude-sonnet-4-20250514"; + +/** + * Delete all profiles AND reset settings-level LLM config so tests start + * truly clean. + * + * The automation test's `ensureMockLLMProfile()` PATCHes `/api/settings` + * with `agent_settings_diff.llm.*`, which stores LLM config at the + * settings layer. Deleting named profiles alone is not enough — the + * server may still surface a "default" profile derived from those + * settings. We therefore also PATCH settings to clear the LLM model so + * the server reports an empty profile list. + */ +async function cleanupProfiles(request: APIRequestContext) { + const headers = { "X-Session-API-Key": SESSION_API_KEY }; + + // 1. Delete all named profiles + const listResp = await request.get(`${BACKEND_URL}/api/profiles`, { + headers, + }); + if (listResp.ok()) { + const body = (await listResp.json()) as { + profiles?: { name: string }[]; + }; + for (const p of body.profiles ?? []) { + const delResp = await request.delete( + `${BACKEND_URL}/api/profiles/${encodeURIComponent(p.name)}`, + { headers }, + ); + if (!delResp.ok()) { + console.warn( + `DELETE /api/profiles/${p.name} returned ${delResp.status()}`, + ); + } + } + } else { + console.warn( + `GET /api/profiles returned ${listResp.status()} — skipping profile cleanup`, + ); + } + + // 2. Clear settings-level LLM config so no implicit profile lingers. + // The automation test's ensureMockLLMProfile() PATCHes agent_settings + // directly; without this reset, the server may derive a "default" + // profile from the lingering settings even after all named profiles + // are deleted. + const patchResp = await request.patch(`${BACKEND_URL}/api/settings`, { + headers: { ...headers, "Content-Type": "application/json" }, + data: { + agent_settings_diff: { + llm: { model: "", api_key: "", base_url: "" }, + }, + }, + }); + if (!patchResp.ok()) { + console.warn( + `PATCH /api/settings (clear LLM) returned ${patchResp.status()}`, + ); + } + + // 3. Verify the cleanup actually worked + const verifyResp = await request.get(`${BACKEND_URL}/api/profiles`, { + headers, + }); + if (verifyResp.ok()) { + const verifyBody = (await verifyResp.json()) as { + profiles?: { name: string }[]; + }; + const remaining = verifyBody.profiles ?? []; + if (remaining.length > 0) { + console.warn( + `After cleanup, ${remaining.length} profile(s) still exist: ${remaining.map((p) => p.name).join(", ")}`, + ); + } + } +} + +test.describe.configure({ mode: "serial" }); + +test.describe("mock-LLM profile lifecycle", () => { + test.beforeEach(async ({ page }) => { + await seedLocalStorage(page); + }); + + // Clean up all profiles after the suite so subsequent test files start fresh. + test.afterAll(async ({ request }) => { + await cleanupProfiles(request); + }); + + // ── Step 1: Create two profiles ───────────────────────────────────── + + test("step 1: create two LLM profiles and verify they appear in the list", async ({ + page, + request, + }) => { + // Start clean + await cleanupProfiles(request); + + await routeSessionApiKey(page); + await page.goto("/settings/llm", { waitUntil: "domcontentloaded" }); + await dismissAnalyticsModal(page); + await waitForTestId(page, "add-llm-profile"); + + // ── Verify empty state ── + await test.step("verify empty state shows no profiles", async () => { + const profileRows = page.getByTestId("profile-row"); + await expect(profileRows).toHaveCount(0, { timeout: 10_000 }); + }); + + // ── Create profile A ── + await test.step("create profile-alpha", async () => { + await page.getByTestId("add-llm-profile").click(); + await waitForTestId(page, "profile-editor-title"); + + const nameInput = page.getByTestId("profile-name-input"); + await nameInput.click(); + await nameInput.fill(PROFILE_A); + + // Switch to advanced view for the model text input + await page.getByTestId("sdk-section-all-toggle").click(); + await waitForTestId(page, "llm-settings-form-advanced"); + + await page.getByTestId("llm-custom-model-input").click(); + await page.getByTestId("llm-custom-model-input").fill(MODEL_A); + await page.getByTestId("llm-api-key-input").click(); + await page.getByTestId("llm-api-key-input").fill("sk-test-a"); + + await page.getByTestId("save-profile-btn").click(); + await waitForTestId(page, "add-llm-profile"); + }); + + // ── Create profile B ── + await test.step("create profile-beta", async () => { + await page.getByTestId("add-llm-profile").click(); + await waitForTestId(page, "profile-editor-title"); + + const nameInput = page.getByTestId("profile-name-input"); + await nameInput.click(); + await nameInput.fill(PROFILE_B); + + await page.getByTestId("sdk-section-all-toggle").click(); + await waitForTestId(page, "llm-settings-form-advanced"); + + await page.getByTestId("llm-custom-model-input").click(); + await page.getByTestId("llm-custom-model-input").fill(MODEL_B); + await page.getByTestId("llm-api-key-input").click(); + await page.getByTestId("llm-api-key-input").fill("sk-test-b"); + + await page.getByTestId("save-profile-btn").click(); + await waitForTestId(page, "add-llm-profile"); + }); + + // ── Verify both profiles appear ── + await test.step("verify both profiles appear in the list", async () => { + const profileRows = page.getByTestId("profile-row"); + await expect(profileRows).toHaveCount(2, { timeout: 10_000 }); + + const allText = await profileRows.allTextContents(); + expect( + allText.some((t) => t.includes(PROFILE_A)), + `Expected "${PROFILE_A}" in profile list`, + ).toBe(true); + expect( + allText.some((t) => t.includes(PROFILE_B)), + `Expected "${PROFILE_B}" in profile list`, + ).toBe(true); + }); + + // ── Verify via API ── + await test.step("verify profiles exist via API", async () => { + const resp = await request.get(`${BACKEND_URL}/api/profiles`, { + headers: { "X-Session-API-Key": SESSION_API_KEY }, + }); + expect(resp.ok()).toBe(true); + + const body = (await resp.json()) as { + profiles: { name: string; model: string }[]; + active_profile: string | null; + }; + const names = body.profiles.map((p) => p.name); + expect(names).toContain(PROFILE_A); + expect(names).toContain(PROFILE_B); + }); + }); + + // ── Step 2: Activate profile B, verify badge moves ────────────────── + + test("step 2: activate profile-beta and verify the active badge", async ({ + page, + }) => { + await routeSessionApiKey(page); + await page.goto("/settings/llm", { waitUntil: "domcontentloaded" }); + await dismissAnalyticsModal(page); + await waitForTestId(page, "add-llm-profile"); + + const profileRows = page.getByTestId("profile-row"); + await expect(profileRows).toHaveCount(2, { timeout: 10_000 }); + + // Find profile-beta's row and open the actions menu + const betaRow = profileRows.filter({ hasText: PROFILE_B }); + await expect(betaRow).toBeVisible({ timeout: 5_000 }); + await betaRow.getByTestId("profile-menu-trigger").click(); + await expect(page.getByTestId("profile-actions-menu")).toBeVisible({ + timeout: 5_000, + }); + + // Click "Set Active" + await page.getByTestId("profile-set-active").click(); + + await test.step("verify active badge is on profile-beta", async () => { + // Wait for the badge to appear on profile-beta + await expect( + betaRow.getByTestId("profile-active-badge"), + ).toBeVisible({ timeout: 10_000 }); + + // profile-alpha should NOT have the badge + const alphaRow = profileRows.filter({ hasText: PROFILE_A }); + await expect( + alphaRow.getByTestId("profile-active-badge"), + ).toBeHidden(); + }); + + await test.step("verify active_profile via API", async () => { + // Use page.evaluate to call the API (session key routed by page.route) + const apiResult = await page.evaluate(async () => { + const resp = await fetch("/api/profiles"); + return resp.json(); + }); + expect(apiResult.active_profile).toBe(PROFILE_B); + }); + }); + + // ── Step 3: Rename the active profile, verify badge follows ───────── + + test("step 3: rename the active profile and verify badge follows the new name", async ({ + page, + }) => { + await routeSessionApiKey(page); + await page.goto("/settings/llm", { waitUntil: "domcontentloaded" }); + await dismissAnalyticsModal(page); + await waitForTestId(page, "add-llm-profile"); + + const profileRows = page.getByTestId("profile-row"); + await expect(profileRows).toHaveCount(2, { timeout: 10_000 }); + + // profile-beta should be active from step 2 + const betaRow = profileRows.filter({ hasText: PROFILE_B }); + await expect( + betaRow.getByTestId("profile-active-badge"), + ).toBeVisible({ timeout: 5_000 }); + + // Open actions menu and click Rename + await betaRow.getByTestId("profile-menu-trigger").click(); + await expect(page.getByTestId("profile-actions-menu")).toBeVisible({ + timeout: 5_000, + }); + await page.getByTestId("profile-rename").click(); + + // Wait for the rename modal + await expect(page.getByTestId("rename-profile-modal")).toBeVisible({ + timeout: 5_000, + }); + + // Clear and type the new name + const renameInput = page.getByTestId("rename-profile-input"); + await renameInput.clear(); + await renameInput.fill(PROFILE_B_RENAMED); + await page.getByTestId("rename-profile-submit").click(); + + // Wait for the modal to close + await expect(page.getByTestId("rename-profile-modal")).toBeHidden({ + timeout: 10_000, + }); + + await test.step("verify renamed profile has the active badge", async () => { + const updatedRows = page.getByTestId("profile-row"); + await expect(updatedRows).toHaveCount(2, { timeout: 10_000 }); + + const renamedRow = updatedRows.filter({ hasText: PROFILE_B_RENAMED }); + await expect(renamedRow).toBeVisible({ timeout: 10_000 }); + await expect( + renamedRow.getByTestId("profile-active-badge"), + ).toBeVisible(); + }); + + await test.step("verify old name is gone from the list", async () => { + // The old name should no longer appear as a standalone profile name. + // Use a strict check: no profile row should contain the old name + // without also containing the renamed suffix. + const updatedRows = page.getByTestId("profile-row"); + const allText = await updatedRows.allTextContents(); + const hasOldNameOnly = allText.some( + (t) => t.includes(PROFILE_B) && !t.includes(PROFILE_B_RENAMED), + ); + expect( + hasOldNameOnly, + `Old profile name "${PROFILE_B}" should not appear without the renamed suffix`, + ).toBe(false); + }); + + await test.step("verify other profile does NOT have the badge", async () => { + const alphaRow = page + .getByTestId("profile-row") + .filter({ hasText: PROFILE_A }); + await expect( + alphaRow.getByTestId("profile-active-badge"), + ).toBeHidden(); + }); + + await test.step("verify active_profile via API reflects the new name", async () => { + const apiResult = await page.evaluate(async () => { + const resp = await fetch("/api/profiles"); + return resp.json(); + }); + expect(apiResult.active_profile).toBe(PROFILE_B_RENAMED); + }); + }); + + // ── Step 4: Delete the active profile, verify no badge remains ────── + + test("step 4: delete the active profile and verify no active badge remains", async ({ + page, + }) => { + await routeSessionApiKey(page); + await page.goto("/settings/llm", { waitUntil: "domcontentloaded" }); + await dismissAnalyticsModal(page); + await waitForTestId(page, "add-llm-profile"); + + const profileRows = page.getByTestId("profile-row"); + await expect(profileRows).toHaveCount(2, { timeout: 10_000 }); + + // profile-beta-renamed should be active from step 3 + const activeRow = profileRows.filter({ hasText: PROFILE_B_RENAMED }); + await expect( + activeRow.getByTestId("profile-active-badge"), + ).toBeVisible({ timeout: 5_000 }); + + // Open actions menu and click Delete + await activeRow.getByTestId("profile-menu-trigger").click(); + await expect(page.getByTestId("profile-actions-menu")).toBeVisible({ + timeout: 5_000, + }); + await page.getByTestId("profile-delete").click(); + + // Confirm deletion + await expect(page.getByTestId("delete-profile-confirm")).toBeVisible({ + timeout: 5_000, + }); + await page.getByTestId("delete-profile-confirm").click(); + + // Wait for the modal to close and list to update + await expect(page.getByTestId("delete-profile-confirm")).toBeHidden({ + timeout: 10_000, + }); + + await test.step("verify only profile-alpha remains", async () => { + const remainingRows = page.getByTestId("profile-row"); + await expect(remainingRows).toHaveCount(1, { timeout: 10_000 }); + await expect( + remainingRows.filter({ hasText: PROFILE_A }), + ).toBeVisible(); + }); + + await test.step("verify no active badge is visible", async () => { + await expect(page.getByTestId("profile-active-badge")).toBeHidden(); + }); + + await test.step("verify active_profile is null via API", async () => { + const apiResult = await page.evaluate(async () => { + const resp = await fetch("/api/profiles"); + return resp.json(); + }); + expect(apiResult.active_profile).toBeNull(); + }); + }); +});