diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 580b920..5eae89d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,6 +50,32 @@ jobs: - run: for f in static/js/*.js; do node --check "$f"; done - run: for f in tests/js/*.test.mjs; do node "$f"; done + # Browser tests for static/js (#339). node --check above is a syntax parse; + # this drives the real UI against the real backend, including the Tauri code + # path, which is where #335 hid because it was invisible in a browser. + frontend-e2e: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7.0.1 + - uses: actions/setup-node@v7.0.0 + with: + node-version: "20" + cache: npm + - run: curl -LsSf https://astral.sh/uv/install.sh | sh + - run: echo "$HOME/.local/bin" >> "$GITHUB_PATH" + - run: uv sync --frozen --all-extras + - run: npm ci + - run: npx playwright install --with-deps chromium + - run: npx playwright test + - if: failure() + uses: actions/upload-artifact@v7.0.1 + with: + name: playwright-report + path: | + playwright-report/ + test-results/ + retention-days: 7 + # The Linux installer only ever runs on a user's machine, so nothing else # would catch a regression in it. Runs on a real Linux image rather than the # macOS bash used during development. diff --git a/.gitignore b/.gitignore index 731ba93..dbc488b 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,12 @@ static/version.json # Desktop dependencies and build outputs desktop/node_modules/ +node_modules/ + +# Playwright browser-test output (tests/e2e) +test-results/ +playwright-report/ +blob-report/ desktop/src-tauri/target/ desktop/src-tauri/gen/ diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..1da0087 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,76 @@ +{ + "name": "stemdeck-frontend-tests", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "stemdeck-frontend-tests", + "devDependencies": { + "@playwright/test": "^1.49.0" + } + }, + "node_modules/@playwright/test": { + "version": "1.62.1", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.62.1.tgz", + "integrity": "sha512-DTcUc8qii+cpHvtOwggMtBRMjKZHXYWdw8syRYu2vtzuq4Wxphqq4NfCs5Zt44L6mA8rfDfj+PHnxFc/FeK6mQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.62.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/playwright": { + "version": "1.62.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.62.1.tgz", + "integrity": "sha512-0M+L3LAD8/nm554LOla9Ayx0j0tmFZ0FBcoQ7F1VuVHpM/XpiC8RcDzBQB8W5+hA8L22THxELzeF+2WcUzvcLg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.62.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.62.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.62.1.tgz", + "integrity": "sha512-wPYSwEBJY9GHraISXqyqtx0na0LpO3XEX7jNDhntbex7tzUS7kLnZsOlFruFJB4Hi/rhDMjXGqHewDZ68nYZVw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=20" + } + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..50645eb --- /dev/null +++ b/package.json @@ -0,0 +1,14 @@ +{ + "name": "stemdeck-frontend-tests", + "private": true, + "type": "module", + "description": "Browser tests for static/js. The app itself has no build step and no runtime dependencies; this exists only so CI can drive a real browser (#339).", + "scripts": { + "test:e2e": "playwright test", + "test:e2e:headed": "playwright test --headed", + "test:js": "for f in tests/js/*.test.mjs; do node \"$f\"; done" + }, + "devDependencies": { + "@playwright/test": "^1.49.0" + } +} diff --git a/playwright.config.mjs b/playwright.config.mjs new file mode 100644 index 0000000..c1e724f --- /dev/null +++ b/playwright.config.mjs @@ -0,0 +1,39 @@ +import { defineConfig, devices } from "@playwright/test"; + +// The backend is real: these tests exercise the actual endpoints, with only the +// separation pipeline skipped (tests/e2e/seed.py writes a finished job instead +// of running demucs to test a menu). +// +// serve.sh seeds a throwaway jobs directory and execs uvicorn against it, so a +// run can never see or touch a developer's real library. +const PORT = process.env.STEMDECK_E2E_PORT || "8123"; + +export default defineConfig({ + testDir: "tests/e2e", + testMatch: /.*\.spec\.mjs/, + // A stuck export used to hang for 15 minutes by design (EXPORT_BUSY_MAX_MS), + // so a generous per-test timeout would hide exactly the bug these tests exist + // to catch. + timeout: 45_000, + expect: { timeout: 10_000 }, + fullyParallel: false, + workers: 1, + forbidOnly: Boolean(process.env.CI), + retries: process.env.CI ? 1 : 0, + reporter: process.env.CI ? [["list"], ["github"]] : [["list"]], + use: { + baseURL: `http://127.0.0.1:${PORT}`, + trace: process.env.CI ? "retain-on-failure" : "off", + screenshot: "only-on-failure", + video: "off", + }, + projects: [{ name: "chromium", use: { ...devices["Desktop Chrome"] } }], + webServer: { + command: `bash tests/e2e/serve.sh ${PORT}`, + url: `http://127.0.0.1:${PORT}/api/health`, + reuseExistingServer: !process.env.CI, + timeout: 120_000, + stdout: "pipe", + stderr: "pipe", + }, +}); diff --git a/tests/e2e/export-menu.spec.mjs b/tests/e2e/export-menu.spec.mjs new file mode 100644 index 0000000..9056afc --- /dev/null +++ b/tests/e2e/export-menu.spec.mjs @@ -0,0 +1,172 @@ +// Regression tests for the export menu (#335, #337). +// +// #335: "Export All Stems" became permanently unclickable after one export, for +// every track, until the app restarted. It shipped in alpha 15 and a user found +// it. It reproduced only in the desktop build, because in a browser the +// synthetic .click() closes the chip panel before the busy state is applied +// and the bug hides -- so the Tauri-mode cases below are the ones that matter. +// +// #337 fixed it and turned up three more defects in the same state machine, +// each covered here. + +import { test, expect } from "@playwright/test"; +import { openStudio, exportUi } from "./helpers.mjs"; + +test.describe("export menu, desktop (Tauri) mode", () => { + test("every row is usable again after an export completes", async ({ page }) => { + await openStudio(page, { tauri: true }); + const ui = exportUi(page); + + await ui.open(); + await ui.stems.click(); + + // Mid-export: the menu is busy and says so. + await expect(ui.label).toHaveText(/Exporting/); + await expect(ui.stems).toHaveAttribute("aria-disabled", "true"); + expect(await page.evaluate(() => window.__e2e.savePending())).toBe(true); + + await page.evaluate(() => window.__e2e.finishSave()); + + // #335 itself: without a symmetric reset this row stays disabled forever. + await expect(ui.label).toHaveText("Export Mix"); + await expect(ui.stems).not.toHaveAttribute("aria-disabled", "true"); + await expect(ui.mix).not.toHaveAttribute("aria-disabled", "true"); + }); + + test("a second export still works after the first", async ({ page }) => { + await openStudio(page, { tauri: true }); + const ui = exportUi(page); + + for (const pass of [1, 2]) { + await ui.open(); + await ui.stems.click(); + await expect(ui.label).toHaveText(/Exporting/, { timeout: 5000 }); + await page.evaluate(() => window.__e2e.finishSave()); + await expect(ui.label).toHaveText("Export Mix"); + expect( + await page.evaluate(() => window.__e2e.callsFor("save_audio_file").length), + `save_audio_file should have fired on pass ${pass}`, + ).toBe(pass); + } + }); + + test("the busy state waits for the save, not a fixed timer", async ({ page }) => { + // #337: the reset used to run on a timer, so a slow save looked finished + // while it was still writing, and a failed one looked identical to success. + await openStudio(page, { tauri: true }); + const ui = exportUi(page); + + await ui.open(); + await ui.mix.click(); + await expect(ui.label).toHaveText(/Exporting/); + + await page.waitForTimeout(3000); // comfortably past the 1200 ms guess + await expect(ui.label).toHaveText(/Exporting/); + expect(await page.evaluate(() => window.__e2e.savePending())).toBe(true); + + await page.evaluate(() => window.__e2e.finishSave()); + await expect(ui.label).toHaveText("Export Mix"); + }); + + test("a failed export says so and leaves the menu usable", async ({ page }) => { + await openStudio(page, { tauri: true }); + const ui = exportUi(page); + + await ui.open(); + await ui.mix.click(); + await expect(ui.label).toHaveText(/Exporting/); + + await page.evaluate(() => window.__e2e.failSave("disk full")); + + await expect(ui.error).toBeVisible(); + await expect(ui.error).toContainText(/disk full/i); + // The state machine has to recover from the failure, not just report it. + await expect(ui.label).toHaveText("Export Mix"); + await expect(ui.mix).not.toHaveAttribute("aria-disabled", "true"); + }); + + test("an export failure does not offer to retry the import", async ({ page }) => { + // #337: export errors reused the import error box, whose "Try again" button + // sends the user to the URL field -- which has nothing to do with a failed + // save and loses the studio they were working in. + await openStudio(page, { tauri: true }); + const ui = exportUi(page); + + await ui.open(); + await ui.mix.click(); + await page.evaluate(() => window.__e2e.failSave("nope")); + + await expect(ui.error).toBeVisible(); + await expect(ui.error).toContainText("Dismiss"); + await expect(ui.error).not.toContainText("Try again"); + }); +}); + +test.describe("export menu, browser mode", () => { + test("rows recover after the fire-and-forget download path", async ({ page }) => { + // No Tauri bridge: _triggerDownload falls back to a synthetic , + // which reports nothing back, so the reset runs on the timer instead. + await openStudio(page); + const ui = exportUi(page); + + await ui.open(); + await ui.stems.click(); + + await expect(ui.label).toHaveText("Export Mix", { timeout: 8000 }); + await ui.open(); + await expect(ui.stems).not.toHaveAttribute("aria-disabled", "true"); + }); +}); + +test.describe("format switching", () => { + test("picking a format updates the radio group", async ({ page }) => { + await openStudio(page, { tauri: true }); + const ui = exportUi(page); + + await ui.open(); + await expect(ui.fmt("wav")).toHaveAttribute("aria-checked", "true"); + + await ui.fmt("flac").click(); + await expect(ui.fmt("flac")).toHaveAttribute("aria-checked", "true"); + await expect(ui.fmt("wav")).toHaveAttribute("aria-checked", "false"); + await expect(ui.fmt("flac")).toHaveClass(/active/); + }); + + test("MP4 is not offered for a track with no video", async ({ page }) => { + // The video format only appears once the track actually has one + // (#footer-export-wrap.has-video). Offering it otherwise produces an export + // that cannot succeed. + await openStudio(page, { tauri: true }); + const ui = exportUi(page); + + await ui.open(); + await expect(ui.fmt("mp4")).toBeHidden(); + await expect(ui.stems).toBeVisible(); + }); +}); + +test.describe("busy state", () => { + test("the menu cannot be reopened mid-export", async ({ page }) => { + // flashBusy closes the panel and the button ignores clicks while busy, so + // there is no way to change format or fire a second export underneath the + // first. This is what makes the "hidden row left disabled" case unreachable + // from the UI; the reset clears every row regardless. + await openStudio(page, { tauri: true }); + const ui = exportUi(page); + + await ui.open(); + await ui.mix.click(); + await expect(ui.label).toHaveText(/Exporting/); + await expect(ui.panel).toHaveClass(/hidden/); + + await ui.button.click(); + await expect(ui.panel).toHaveClass(/hidden/); + + await page.evaluate(() => window.__e2e.finishSave()); + await expect(ui.label).toHaveText("Export Mix"); + + // ...and it works again immediately afterwards. + await ui.open(); + await expect(ui.panel).not.toHaveClass(/hidden/); + }); +}); diff --git a/tests/e2e/helpers.mjs b/tests/e2e/helpers.mjs new file mode 100644 index 0000000..f7620dd --- /dev/null +++ b/tests/e2e/helpers.mjs @@ -0,0 +1,146 @@ +// Shared setup for the browser tests. +// +// Two things here are load-bearing and were both learned the hard way: +// +// 1. The sidebar renders from the library store, not from /api/jobs. A job that +// exists on disk but is absent from the store is invisible in the UI, and a +// test that clicks nothing passes for the wrong reason. seedLibrary writes +// that store before any script runs. +// +// 2. The desktop and browser download paths genuinely diverge, which is why +// #335 was invisible in a browser. stubTauri installs a controllable +// window.__TAURI__ so the desktop branch runs, and so the test decides when +// an export finishes rather than racing a real one. + +export const JOB_ID = "e2e0deadbeef"; +export const TRACK_TITLE = "E2E Fixture Track"; + +const STORAGE_KEY = "stemdeck.folders"; +const STORAGE_VERSION = 2; + +/** Put the fixture track in the library so the sidebar renders it. */ +export async function seedLibrary(page) { + const state = { + v: STORAGE_VERSION, + folders: [ + { id: "f-unsorted", name: "Unsorted", items: [JOB_ID], color: null }, + { id: "trash", name: "Trash", items: [], color: null }, + ], + tracks: { + [JOB_ID]: { + id: JOB_ID, + title: TRACK_TITLE, + status: "done", + stems: ["vocals", "drums", "bass", "other"], + sourceUrl: "local:e2e-fixture.wav", + createdAt: 1700000000, + favorite: false, + }, + }, + }; + await page.addInitScript( + ([key, value]) => window.localStorage.setItem(key, JSON.stringify(value)), + [STORAGE_KEY, state], + ); +} + +/** + * Install a fake Tauri bridge so the app takes its desktop code path. + * + * `save_audio_file` is left pending until the test resolves or rejects it, + * which is the whole point: the export busy state is a promise state machine, + * and #335 was a stuck one. Tests drive it through window.__e2e. + */ +export async function stubTauri(page) { + await page.addInitScript(() => { + const calls = []; + let pendingResolve = null; + let pendingReject = null; + + window.__e2e = { + calls, + // Settle the export that is currently in flight. + finishSave: (value) => pendingResolve && pendingResolve(value ?? null), + failSave: (message) => pendingReject && pendingReject(message ?? "save failed"), + savePending: () => Boolean(pendingResolve), + callsFor: (cmd) => calls.filter((c) => c.cmd === cmd), + }; + + window.__TAURI__ = { + core: { + invoke: (cmd, args) => { + calls.push({ cmd, args }); + switch (cmd) { + case "save_audio_file": + return new Promise((resolve, reject) => { + pendingResolve = (v) => { pendingResolve = null; pendingReject = null; resolve(v); }; + pendingReject = (e) => { pendingResolve = null; pendingReject = null; reject(e); }; + }); + // The library store lives in the Tauri store on desktop. Back it + // with localStorage so seedLibrary works in this mode too. + case "store_get": { + const raw = window.localStorage.getItem(args?.key); + return Promise.resolve(raw === null ? null : JSON.parse(raw)); + } + case "store_set": + window.localStorage.setItem(args?.key, JSON.stringify(args?.value)); + return Promise.resolve(null); + case "get_setup_status": + return Promise.resolve({ ready: true, data_dir: "/tmp/e2e", ffmpeg: "/usr/bin/ffmpeg" }); + default: + return Promise.resolve(null); + } + }, + }, + event: { listen: () => Promise.resolve(() => {}) }, + }; + }); +} + +/** + * Keep export requests off the real backend. + * + * A browser-mode export is an pointed at a mixdown endpoint that + * shells out to ffmpeg. These tests are about the menu's state machine, so the + * bytes are irrelevant and the render time is not worth paying. + */ +export async function stubExportEndpoints(page) { + await page.route("**/api/jobs/*/mix**", (route) => + route.fulfill({ status: 200, contentType: "audio/wav", body: Buffer.from("RIFF") })); + await page.route("**/api/jobs/*/stems.zip**", (route) => + route.fulfill({ status: 200, contentType: "application/zip", body: Buffer.from("PK") })); + await page.route("**/api/jobs/*/render**", (route) => + route.fulfill({ status: 200, contentType: "audio/wav", body: Buffer.from("RIFF") })); +} + +/** Open the fixture track in the studio and wait until the transport is live. */ +export async function openStudio(page, { tauri = false } = {}) { + await seedLibrary(page); + if (tauri) await stubTauri(page); + await stubExportEndpoints(page); + + await page.goto("/", { waitUntil: "domcontentloaded" }); + await page.locator(`.cat-item[data-id="${JOB_ID}"]`).first().click(); + // The transport total only leaves 00:00 once an engine has reported a + // duration, so it doubles as "the studio is actually ready". + await page.waitForFunction( + () => !/\/\s*00:00\s*$/.test(document.querySelector("#t-time")?.textContent || "00:00 / 00:00"), + null, + { timeout: 20000 }, + ); +} + +export const exportUi = (page) => ({ + button: page.locator("#t-export-btn"), + panel: page.locator("#t-export-panel"), + label: page.locator("#t-export-label"), + mix: page.locator("#t-export-mix"), + stems: page.locator("#t-export-stems"), + region: page.locator("#t-export-region"), + fmt: (name) => page.locator(`#t-fmt-${name}`), + error: page.locator("#error:not(.hidden)"), + open: async () => { + await page.locator("#t-export-btn").click(); + await page.locator("#t-export-panel:not(.hidden)").waitFor({ timeout: 5000 }); + }, +}); diff --git a/tests/e2e/seed.py b/tests/e2e/seed.py new file mode 100644 index 0000000..5bf3481 --- /dev/null +++ b/tests/e2e/seed.py @@ -0,0 +1,119 @@ +"""Build a jobs directory containing one finished track, for the browser tests. + +The point is that the tests talk to the real backend: real Range requests for +stems, the real registry, the real endpoints. Only the pipeline is skipped, +because running demucs to test a menu would be absurd. + +The stems are genuine PCM16 WAVs rather than placeholder bytes. The chunked +audio engine parses WAV containers itself, so a file that is not really a WAV +loads with no duration and the studio comes up without playback -- which is the +#358 failure mode, and it would make every test here fail for the wrong reason. + +Usage: python tests/e2e/seed.py +""" + +from __future__ import annotations + +import json +import math +import struct +import sys +import time +from pathlib import Path + +JOB_ID = "e2e0deadbeef" +TITLE = "E2E Fixture Track" +STEMS = ["vocals", "drums", "bass", "other"] +SAMPLE_RATE = 44100 +CHANNELS = 2 +DURATION_SEC = 6 + + +def _wav_bytes(freq: float, seconds: int = DURATION_SEC) -> bytes: + """A short stereo PCM16 tone. Audible content matters: silence would make a + broken mix indistinguishable from a working one if these tests ever grow + real audio assertions.""" + frames = SAMPLE_RATE * seconds + body = bytearray() + for i in range(frames): + value = int(12000 * math.sin(2 * math.pi * freq * i / SAMPLE_RATE)) + body += struct.pack(" list[list[float]]: + """Matches what the backend writes: min/max pairs per bucket, so the studio + renders overview waveforms from peaks instead of falling back to decoding + every stem in the browser.""" + out = [] + for i in range(points): + amp = abs(math.sin(i / 18.0)) * 0.8 + out.append([round(-amp, 4), round(amp, 4)]) + return out + + +def seed(jobs_dir: Path) -> str: + job_dir = jobs_dir / JOB_ID + stems_dir = job_dir / "stems" + stems_dir.mkdir(parents=True, exist_ok=True) + + for index, name in enumerate(STEMS): + (stems_dir / f"{name}.wav").write_bytes(_wav_bytes(220.0 * (index + 1))) + + (job_dir / "peaks.json").write_text( + json.dumps({name: _peaks() for name in STEMS}), encoding="utf-8" + ) + (job_dir / "beats.json").write_text( + json.dumps( + { + "bpm": 120.0, + "beats": [round(i * 0.5, 3) for i in range(DURATION_SEC * 2)], + "downbeats": [round(i * 2.0, 3) for i in range(DURATION_SEC // 2)], + } + ), + encoding="utf-8", + ) + + # Field names are the dataclass's, not the API's: from_record filters on + # Job's own fields, so "stage" or "duration" would be silently dropped and + # the track would load without a duration. + record = { + "id": JOB_ID, + "status": "done", + "progress": 1.0, + "stage_message": "Done", + "title": TITLE, + "duration_sec": float(DURATION_SEC), + "source_url": "local:e2e-fixture.wav", + # Now, not a fixed date. The hourly sweep deletes job directories older + # than JOB_TTL_SECONDS, and it runs at startup: a fixture with a + # hardcoded timestamp is reaped before the first test opens the page, + # leaving a registry entry pointing at nothing. + "created_at": time.time(), + "bpm": 120, + "key": "C maj", + "scale": "Major", + # Per-stem RMS as 0-100 ints, which is what drives the presence cards. + "stem_presence": {name: 80 for name in STEMS}, + # Same shape the pipeline writes (runner.py): entries, not bare names. + # A list of strings deserialises without error and then leaves the + # studio with nothing to play. + "stems": [{"name": name, "url": f"/api/jobs/{JOB_ID}/stems/{name}.wav"} for name in STEMS], + } + (jobs_dir / "registry.json").write_text( + json.dumps({"version": 1, "jobs": [record]}, indent=2) + "\n", encoding="utf-8" + ) + return JOB_ID + + +if __name__ == "__main__": + target = Path(sys.argv[1]).expanduser().resolve() + target.mkdir(parents=True, exist_ok=True) + print(seed(target)) diff --git a/tests/e2e/serve.sh b/tests/e2e/serve.sh new file mode 100755 index 0000000..76e51ce --- /dev/null +++ b/tests/e2e/serve.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# +# Boot a StemDeck backend for the browser tests, against a throwaway jobs +# directory seeded with one finished track. +# +# Everything the app writes is redirected into that directory, so a test run can +# never read, modify or delete a developer's real library. The directory is +# recreated on every run, so state cannot leak between runs either. + +set -euo pipefail + +PORT="${1:-8123}" +REPO_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd)" +WORK_DIR="${STEMDECK_E2E_DIR:-${TMPDIR:-/tmp}/stemdeck-e2e}" + +rm -rf "$WORK_DIR" +mkdir -p "$WORK_DIR/jobs" "$WORK_DIR/data" + +uv run python "${REPO_ROOT}/tests/e2e/seed.py" "$WORK_DIR/jobs" >/dev/null + +cd "$REPO_ROOT" +exec env \ + STEMDECK_JOBS_DIR="$WORK_DIR/jobs" \ + STEMDECK_DATA_DIR="$WORK_DIR/data" \ + STEMDECK_CACHE_DIR="$WORK_DIR/data/cache" \ + STEMDECK_LOGS_DIR="$WORK_DIR/data/logs" \ + STEMDECK_MODELS_DIR="$WORK_DIR/data/models" \ + STEMDECK_DOWNLOADS_DIR="$WORK_DIR/data/downloads" \ + uv run uvicorn app.main:app \ + --host 127.0.0.1 \ + --port "$PORT" \ + --log-level warning \ + --timeout-graceful-shutdown 2