diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 48a97425..738f151a 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -8,6 +8,7 @@ ### Added +- **CLI binary e2e smoke tests (`packages/cli/__tests__/e2e/`).** New `spawn.ts` helper spawns the *built* `dist/index.js` as a separate Node process with a tmpdir HOME and captures stdout/stderr/exit. The exemplar `cli-smoke.e2e.test.ts` covers `--help` / `-h` / `--version` / `-v` / unknown-command exit codes — testing what the user actually sees rather than calling `main()` in-process. Catches build-pipeline regressions, argv-parser breakage, and exit-code drift that in-process tests can't see. Foundation for the deferred full push-flow e2e (which needs ccusage stubbing infrastructure or a real local stack). - **Real-Supabase integration tests for API routes (`bun run --cwd apps/web test:integration`).** New `apps/web/__tests__/integration/` directory + `vitest.integration.config.ts` with a globalSetup that asserts a local Supabase stack is reachable, reads its ephemeral keys via `bunx supabase status -o env`, and exposes them to test workers. The exemplar `usage-submit.test.ts` calls the real `POST /api/usage/submit` handler with a real Request, mints a real CLI JWT via `createCliToken`, and asserts on rows actually written to Postgres — no mocks of the Supabase client, the auth helper, or the chained queries. Catches bug classes the existing mocked `__tests__/api/usage-submit.test.ts` cannot: missing columns (the `collector_meta` cache-mismatch class), real CHECK constraints, FK behavior, NUMERIC→JS roundtrip precision, and end-to-end JWT signing with real `CLI_JWT_SECRET`. CI step added: `supabase/setup-cli@v1` plus `bunx supabase start` before `test:integration`. Existing mocked tests stay for now; future PRs can migrate cases incrementally to the integration directory. - **CLI activation tracking events: `cli_first_run` and `cli_authenticated`.** PostHog showed 103 users tried Straude in the last 7 days but only 48 (47%) ever pushed once successfully — the missing event was a clean install→activate funnel. The CLI now writes `~/.straude/.first-run` on the first invocation per machine and captures `cli_first_run` (with `platform`, `node_version`, `command`) before any other code runs, so even `npx straude --help` counts as install. Every subsequent invocation that loads a stored config also captures `cli_authenticated`. Saved insight `DV22QC1d` ([URL](https://us.posthog.com/project/374497/insights/DV22QC1d)) tracks the funnel; goal is ≥75% activation on `cli_version` ≥ 0.1.24. diff --git a/packages/cli/__tests__/e2e/cli-smoke.e2e.test.ts b/packages/cli/__tests__/e2e/cli-smoke.e2e.test.ts new file mode 100644 index 00000000..8489789c --- /dev/null +++ b/packages/cli/__tests__/e2e/cli-smoke.e2e.test.ts @@ -0,0 +1,96 @@ +import { describe, it, expect, beforeAll, beforeEach, afterEach } from "vitest"; +import { spawnSync } from "node:child_process"; +import { existsSync, mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { spawnCli, rmDir, CLI_DIST_ENTRY } from "./spawn"; + +/** + * Real-binary e2e smoke tests. The previous CLI tests all import functions + * from src/ and exercise them in-process. These tests do something none of + * those can: they spawn the *built* `dist/index.js` as a separate Node + * process with a controlled HOME, capture stdout/stderr/exit, and assert + * on the observable behavior a user actually sees. + * + * What this catches that in-process tests cannot: + * - Build pipeline regressions (tsc emits broken JS, missing imports). + * - argv parsing breakage that doesn't surface when calling main() directly. + * - Real exit codes for help/version/error paths. + * - Real stdout output the user reads. + * - Future regressions to the CLI's startup-time behavior under real Node. + * + * Out of scope for this exemplar suite (tracked as follow-ups): + * - Full `straude push` flow — needs ccusage on PATH or a stub binary. + * - `straude login` — needs an HTTP listener mocking the auth poll. + * - Auto-push / hooks — needs Claude Code's settings.json scaffold. + */ + +const PKG_DIR = resolve(dirname(fileURLToPath(import.meta.url)), "../.."); + +beforeAll(() => { + // The CLI binary is what we're testing, so make sure it exists. If the + // suite is run cold (no `bun run build` first), build it now. tsc takes + // ~3s; the alternative is a confusing "ENOENT dist/index.js" failure. + if (!existsSync(CLI_DIST_ENTRY)) { + const built = spawnSync("bun", ["run", "build"], { cwd: PKG_DIR, stdio: "inherit" }); + if (built.status !== 0) { + throw new Error(`failed to build CLI before e2e suite (exit ${built.status})`); + } + } +}); + +let home: string; +beforeEach(() => { + home = mkdtempSync(join(tmpdir(), "straude-e2e-")); +}); +afterEach(() => { + rmDir(home); +}); + +describe("straude binary — smoke", () => { + it("--help prints usage and exits 0", async () => { + const r = await spawnCli({ args: ["--help"], home }); + expect(r.exitCode).toBe(0); + expect(r.stderr).toBe(""); + expect(r.stdout).toMatch(/Usage:/); + expect(r.stdout).toMatch(/straude\s*/); + }); + + it("-h is an alias for --help", async () => { + const r = await spawnCli({ args: ["-h"], home }); + expect(r.exitCode).toBe(0); + expect(r.stdout).toMatch(/Usage:/); + }); + + it("--version prints the package version and exits 0", async () => { + // Read the pinned version straight from package.json — no source-of-truth + // drift between this assertion and what the build embeds. + const pkg = await import(join(PKG_DIR, "package.json"), { with: { type: "json" } }); + const version = (pkg as unknown as { default: { version: string } }).default.version; + + const r = await spawnCli({ args: ["--version"], home }); + expect(r.exitCode).toBe(0); + expect(r.stdout.trim()).toBe(`straude v${version}`); + }); + + it("-v is an alias for --version", async () => { + const r = await spawnCli({ args: ["-v"], home }); + expect(r.exitCode).toBe(0); + expect(r.stdout).toMatch(/^straude v/); + }); + + it("unknown command prints the help and exits non-zero", async () => { + const r = await spawnCli({ args: ["nonsense"], home }); + expect(r.exitCode).not.toBe(0); + expect(r.stderr).toMatch(/Unknown command/); + }); + + it("--version does not create an authenticated config", async () => { + // First-run telemetry may create ~/.straude metadata, but version reads + // must never create an auth config or require a logged-in account. + const r = await spawnCli({ args: ["--version"], home }); + expect(r.exitCode).toBe(0); + expect(existsSync(join(home, ".straude", "config.json"))).toBe(false); + }); +}); diff --git a/packages/cli/__tests__/e2e/spawn.ts b/packages/cli/__tests__/e2e/spawn.ts new file mode 100644 index 00000000..f294350a --- /dev/null +++ b/packages/cli/__tests__/e2e/spawn.ts @@ -0,0 +1,111 @@ +import { spawn, type SpawnOptions } from "node:child_process"; +import { mkdtempSync, rmSync, mkdirSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +/** + * Locate the built CLI binary. Tests assume `bun run --cwd packages/cli build` + * has run — see `e2e/setup.ts`. We don't rebuild from inside each test + * because tsc takes ~3s and the e2e suite shares one binary. + */ +const __dirname = dirname(fileURLToPath(import.meta.url)); +export const CLI_DIST_ENTRY = resolve(__dirname, "../../dist/index.js"); + +export interface SpawnResult { + /** Full captured stdout (utf-8). */ + stdout: string; + /** Full captured stderr (utf-8). */ + stderr: string; + /** Process exit code. `null` if the process was killed by a signal. */ + exitCode: number | null; + /** Signal that killed the process, or `null` if it exited normally. */ + signal: NodeJS.Signals | null; +} + +export interface SpawnCliOptions { + /** Args passed to `node dist/index.js`. */ + args: string[]; + /** + * Directory used as $HOME — config and machine_id live under + * `${home}/.straude`. A fresh tmpdir is created if not supplied so + * tests don't accidentally read or mutate the real user's config. + */ + home?: string; + /** Environment overrides merged on top of a minimal scrubbed base. */ + env?: NodeJS.ProcessEnv; + /** + * If true, child stdout is piped to a child of /dev/null-equivalent so + * EPIPE behaves like `straude … | head` would. Default false. + */ + truncateStdout?: boolean; + /** Timeout in ms before killing the child. Default 15s. */ + timeoutMs?: number; +} + +/** + * Spawn the real CLI binary in a controlled environment and return + * exit/output. No mocks of fs, env, or fetch — this is the real argv + * parser running the real entry point against the real filesystem + * (in a tmpdir HOME) and the real Node runtime. + */ +export function spawnCli(opts: SpawnCliOptions): Promise { + const home = opts.home ?? mkdtempSync(join(tmpdir(), "straude-e2e-")); + // Make sure ~/.straude exists if the test wants it; otherwise the CLI's + // own first-run path creates it. Either is fine. + mkdirSync(home, { recursive: true }); + + const env: NodeJS.ProcessEnv = { + PATH: process.env.PATH, + // Scrub anything that could make the CLI behave differently in tests. + STRAUDE_TELEMETRY_DISABLED: "1", + NODE_ENV: "test", + ...opts.env, + HOME: home, + }; + + return new Promise((resolveResult, reject) => { + const spawnOpts: SpawnOptions = { + env, + stdio: opts.truncateStdout ? ["ignore", "pipe", "pipe"] : "pipe", + }; + const child = spawn(process.execPath, [CLI_DIST_ENTRY, ...opts.args], spawnOpts); + + let stdout = ""; + let stderr = ""; + child.stdout?.on("data", (chunk: Buffer) => { + stdout += chunk.toString("utf-8"); + if (opts.truncateStdout) { + // Close the pipe after the first chunk so the writer hits EPIPE on + // the next write — same shape as `straude --help | head -1`. + child.stdout?.destroy(); + } + }); + child.stderr?.on("data", (chunk: Buffer) => { + stderr += chunk.toString("utf-8"); + }); + + const timeout = setTimeout(() => { + child.kill("SIGKILL"); + reject(new Error(`spawnCli timed out after ${opts.timeoutMs ?? 15_000}ms`)); + }, opts.timeoutMs ?? 15_000); + + child.on("error", (err) => { + clearTimeout(timeout); + reject(err); + }); + child.on("close", (exitCode, signal) => { + clearTimeout(timeout); + resolveResult({ stdout, stderr, exitCode, signal }); + }); + }); +} + +/** Cleanup helper — removes a tmpdir HOME from `spawnCli`. Best effort. */ +export function rmDir(dir: string): void { + try { + rmSync(dir, { recursive: true, force: true }); + } catch { + // ignore + } +}