From c957b1a7b57b79106e14ddd30851f33ae88dc65f Mon Sep 17 00:00:00 2001 From: Vladimir Rogojin Date: Sun, 3 May 2026 23:06:55 +0200 Subject: [PATCH 1/6] test(e2e-live): HMA-orchestrated tenant lifecycle (3 spawns via sphere host) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the architectural foundation for live e2e tests that spawn every tenant THROUGH the host-manager agent (HMA) instead of via direct docker run. This is the production architecture: HMA owns lifecycle, controllers reach the HMA via HMCP DMs, and trade ops (subsequent PR-C) go controller→tenant directly via ACP DMs. What's new in this PR: test/e2e-live/helpers/manager-process.ts Pre-provisions a Sphere wallet for the manager, then spawns dist/host-manager.js from the agentic-hosting checkout (default /home/vrogojin/agentic_hosting, override AGENTIC_HOSTING_PATH). Resolves once `host_manager_started` lands in stdout. Drift-guard env vars (MANAGER_PUBKEY/MANAGER_DIRECT_ADDRESS) are wired from the pre-created wallet so the manager loads it cleanly. test/e2e-live/helpers/sphere-cli.ts Locates the sphere binary (default /home/vrogojin/sphere-cli-work/sphere-cli/bin/sphere.mjs, override SPHERE_CLI_BIN), probes via `sphere --help`, returns ok/skip reason. Includes createSphereCliEnv() for an isolated CWD with .sphere-cli/config.json, and bootstrapControllerWallet() that runs `sphere wallet init` and parses chainPubkey from output. test/e2e-live/helpers/hma-spawn.ts hostSpawn / hostStop / hostList wrappers around sphere-cli's `sphere host …` subcommands. Parses --json output, returns typed payloads (SpawnedTenant with tenantPubkey, tenantDirectAddress, tenantNametag). Throws on hm.error / hm.spawn_failed so callers don't need defensive parsing. test/e2e-live/hma-orchestrated.e2e-live.test.ts Foundation test: bootstraps controller, boots manager, spawns 1 escrow + 2 traders via `sphere host spawn`, asserts each returns hm.spawn_ready RUNNING with a valid tenant pubkey, verifies all three appear in `sphere host list`. Cleans up each tenant via `sphere host stop` then stops the manager. test/e2e-live/{preflight.ts,global-setup.ts,infra-probe.d.ts} @unicitylabs/infra-probe preflight (mirrors agentic-hosting's pattern). Aborts the run up-front if testnet Nostr/aggregator/ IPFS/Fulcrum/Market is unreachable. Bypass via TRADER_E2E_SKIP_PREFLIGHT=1. vitest.e2e-live.config.ts Wires globalSetup; refreshes the stale comment block (the prior "tests are NOT runnable in trader-service standalone" note no longer applied since direct-docker tests already work). Tightens include glob to `*.e2e-live.test.ts` (helper unit tests stay in the default suite). Skip semantics: describe.skipIf gates the new test file when sphere-cli isn't runnable OR the agentic-hosting binary is missing. Both repos move independently; an upstream regression in either should not red-CI this branch. The skip message names which prerequisite is missing so the operator can fix it. Verified live (2026-05-03 testnet, all 5 services HEALTHY): TRADER_E2E_SKIP_PREFLIGHT=1 npx vitest run --config vitest.e2e-live.config.ts \ test/e2e-live/hma-orchestrated.e2e-live.test.ts → 1/1 PASS in 38s. Manager booted in ~5s (Sphere.init + nametag mint). Three real Docker spawns × ~10s each (HMCP request → docker create → tenant Sphere.init in container → acp.hello → spawn_ready). Default `vitest run`: 651 tests still pass. Existing direct-docker e2e-live tests remain untouched — the migration from direct-docker to HMA-orchestrated is incremental (this PR lands the foundation; subsequent PRs migrate scenarios one at a time). Adds @unicitylabs/infra-probe@^0.3.0 as devDependency, plus npm scripts: `preflight`, `preflight:json`. Depends on: - agentic-hosting PR #22 (Phase 5 DM transport) — already merged. - sphere-cli PR #6 (encrypt/decrypt L1 namespace fix) — open. --- package-lock.json | 69 +++++ package.json | 3 + test/e2e-live/global-setup.ts | 14 + test/e2e-live/helpers/hma-spawn.ts | 214 +++++++++++++ test/e2e-live/helpers/manager-process.ts | 280 ++++++++++++++++++ test/e2e-live/helpers/sphere-cli.ts | 163 ++++++++++ .../hma-orchestrated.e2e-live.test.ts | 214 +++++++++++++ test/e2e-live/infra-probe.d.ts | 17 ++ test/e2e-live/preflight.ts | 151 ++++++++++ vitest.e2e-live.config.ts | 50 +++- 10 files changed, 1161 insertions(+), 14 deletions(-) create mode 100644 test/e2e-live/global-setup.ts create mode 100644 test/e2e-live/helpers/hma-spawn.ts create mode 100644 test/e2e-live/helpers/manager-process.ts create mode 100644 test/e2e-live/helpers/sphere-cli.ts create mode 100644 test/e2e-live/hma-orchestrated.e2e-live.test.ts create mode 100644 test/e2e-live/infra-probe.d.ts create mode 100644 test/e2e-live/preflight.ts diff --git a/package-lock.json b/package-lock.json index 213e642..0d77f7b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -20,6 +20,7 @@ "@types/uuid": "^10.0.0", "@typescript-eslint/eslint-plugin": "^8.24.0", "@typescript-eslint/parser": "^8.24.0", + "@unicitylabs/infra-probe": "^0.3.0", "@vitest/coverage-v8": "^3.0.0", "eslint": "^9.20.0", "tsup": "^8.4.0", @@ -955,6 +956,35 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "node_modules/@noble/curves": { + "version": "1.9.7", + "resolved": "https://registry.npmjs.org/@noble/curves/-/curves-1.9.7.tgz", + "integrity": "sha512-gbKGcRUYIjA3/zCCNaWDciTMFI0dCkvou3TL8Zmy5Nc7sJ47a0jtOeZoTaMxkuqRo9cRhjOdZJXegxYE5FN/xw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@noble/hashes": "1.8.0" + }, + "engines": { + "node": "^14.21.3 || >=16" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/@noble/hashes": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/@noble/hashes/-/hashes-1.8.0.tgz", + "integrity": "sha512-jCs9ldd7NwzpgXDIf6P3+NrHh9/sD6CQdxHyjQI+h/6rDNo88ypBxxz45UDuZHz9r3tNz7N/VInSVoVdtXEI4A==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^14.21.3 || >=16" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, "node_modules/@pkgjs/parseargs": { "version": "0.11.0", "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", @@ -1598,6 +1628,23 @@ "url": "https://opencollective.com/eslint" } }, + "node_modules/@unicitylabs/infra-probe": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@unicitylabs/infra-probe/-/infra-probe-0.3.0.tgz", + "integrity": "sha512-lul/bpMmJmguDchCxmJGaO/rBWjOs+dHbSP0brnxRbc/QvLF/tJqqZpvJIgqRMwQziWB4fWPvYu1cwqDFNcE6w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@noble/curves": "^1.6.0", + "ws": "^8.18.0" + }, + "bin": { + "unicity-infra-probe": "bin/unicity-infra-probe.mjs" + }, + "engines": { + "node": ">=20.0.0" + } + }, "node_modules/@unicitylabs/sphere-sdk": { "resolved": "../sphere-sdk", "link": true @@ -4237,6 +4284,28 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, + "node_modules/ws": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.20.0.tgz", + "integrity": "sha512-sAt8BhgNbzCtgGbt2OxmpuryO63ZoDk/sqaB/znQm94T4fCEsy/yV+7CdC1kJhOU9lboAEU7R3kquuycDoibVA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, "node_modules/yocto-queue": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", diff --git a/package.json b/package.json index f289a51..80f0586 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,8 @@ "test:watch": "vitest", "test:coverage": "vitest run --coverage", "test:e2e-live": "vitest run --config vitest.e2e-live.config.ts", + "preflight": "unicity-infra-probe --network testnet", + "preflight:json": "unicity-infra-probe --network testnet --format json", "lint": "eslint .", "typecheck": "tsc --noEmit && tsc --noEmit -p tsconfig.test.json" }, @@ -28,6 +30,7 @@ "@types/uuid": "^10.0.0", "@typescript-eslint/eslint-plugin": "^8.24.0", "@typescript-eslint/parser": "^8.24.0", + "@unicitylabs/infra-probe": "^0.3.0", "@vitest/coverage-v8": "^3.0.0", "eslint": "^9.20.0", "tsup": "^8.4.0", diff --git a/test/e2e-live/global-setup.ts b/test/e2e-live/global-setup.ts new file mode 100644 index 0000000..40a6fb3 --- /dev/null +++ b/test/e2e-live/global-setup.ts @@ -0,0 +1,14 @@ +/** + * Vitest globalSetup for the e2e-live suite. + * + * Runs ONCE before any test file. If the preflight throws, vitest aborts + * the entire run before booting the host-manager binary or spawning any + * tenant containers — saving the multi-minute round-trip we'd otherwise + * eat on a relay outage or unreachable aggregator. + */ + +import { runPreflight } from './preflight.js'; + +export async function setup(): Promise { + await runPreflight(); +} diff --git a/test/e2e-live/helpers/hma-spawn.ts b/test/e2e-live/helpers/hma-spawn.ts new file mode 100644 index 0000000..2d5eebf --- /dev/null +++ b/test/e2e-live/helpers/hma-spawn.ts @@ -0,0 +1,214 @@ +/** + * Helper: spawn / list / stop tenants via sphere-cli's `sphere host …` + * subcommand against a live host-manager (HMA). + * + * Bridges `manager-process.ts` (which provides the manager's address and + * controller wallet binding) and `sphere-cli.ts` (which knows how to + * invoke the binary). Each function here issues a single HMCP DM + * round-trip via sphere-cli, parses the JSON response, and returns a + * typed payload. + * + * The architectural intent (matching agentic-hosting's + * SPHERE-CLI-EXTRACTION-PLAN §6.4): + * - `sphere host spawn` for tenant lifecycle (spawn, stop, list, + * inspect) — the controller talks to the HMA. + * - `sphere trader …` (or `trader-ctl`) for trade ops — the + * controller talks to the tenant directly, host-agnostic. + * + * This module covers the first half. The second half remains in + * `trader-ctl-driver.ts` (and will gain a `sphere trader` mode in PR-C). + */ + +import { runSphere, type SphereRunResult } from './sphere-cli.js'; + +/** Timeout passed to sphere-cli's `--timeout` flag (DM request budget). */ +const DEFAULT_HMCP_TIMEOUT_MS = 120_000; + +/** + * Shape of `hm.spawn_ready` payload as emitted by sphere-cli's `--json` + * mode. Mirrors the agentic-hosting protocol type + * `HmSpawnReadyPayload` (instance_id, tenant_pubkey, tenant_direct_address, + * tenant_nametag, state). + */ +export interface SpawnedTenant { + readonly instanceId: string; + readonly instanceName: string; + /** secp256k1 chain pubkey of the spawned tenant's wallet. */ + readonly tenantPubkey: string; + /** Canonical Unicity DIRECT://... address for ACP/trade-ops DMs. */ + readonly tenantDirectAddress: string; + /** `@nametag` registration if the tenant succeeded; null if registration failed. */ + readonly tenantNametag: string | null; + /** State per HMCP — always 'RUNNING' on success. */ + readonly state: string; +} + +interface HmcpResponseEnvelope { + readonly hmcp_version: string; + readonly type: string; + readonly in_reply_to: string; + readonly payload: Record; +} + +function parseSpawnResponses(stdout: string): HmcpResponseEnvelope[] { + // sphere-cli's --json mode prints the array of collected responses + // pretty-printed. Strip surrounding noise (commander adds nothing, + // but be defensive against future banners) by locating the first + // `[` and last `]`. + const start = stdout.indexOf('['); + const end = stdout.lastIndexOf(']'); + if (start < 0 || end < 0 || end <= start) { + throw new Error( + `sphere host spawn --json: could not find JSON array in stdout (start=${start} end=${end}). Raw: ${stdout.slice(0, 500)}`, + ); + } + const slice = stdout.slice(start, end + 1); + let parsed: unknown; + try { + parsed = JSON.parse(slice); + } catch (err) { + throw new Error( + `sphere host spawn --json: failed to parse JSON: ${err instanceof Error ? err.message : String(err)}\n` + + `slice (first 800): ${slice.slice(0, 800)}`, + ); + } + if (!Array.isArray(parsed)) { + throw new Error(`sphere host spawn --json: expected array, got ${typeof parsed}`); + } + return parsed as HmcpResponseEnvelope[]; +} + +export interface HostSpawnOpts { + /** Path to the built sphere-cli binary. */ + cliPath: string; + /** Test's controller-wallet CWD (passed to runSphere as cwd). */ + cliHome: string; + /** Manager address — `@nametag`, `DIRECT://hex`, or raw hex pubkey. */ + managerAddress: string; + /** Template ID, must exist in the HMA's templates.json. */ + templateId: string; + /** Instance name (Docker label component; alphanumeric, _, ., - up to 63 chars). */ + instanceName: string; + /** Per-DM timeout in ms. Default 120s — tenant Sphere.init can be slow on testnet. */ + timeoutMs?: number; + /** Optional env overrides for the spawned tenant container. */ + env?: Record; +} + +/** + * Issue `sphere host spawn` and return the typed `hm.spawn_ready` + * payload. Throws if the spawn streamed `hm.spawn_failed` or + * `hm.error`, or if the response payload is structurally invalid — + * caller doesn't have to repeat the parse defensively. + */ +export function hostSpawn(opts: HostSpawnOpts): SpawnedTenant { + const args = [ + 'host', + 'spawn', + opts.instanceName, + '--manager', opts.managerAddress, + '--template', opts.templateId, + '--json', + '--timeout', String(opts.timeoutMs ?? DEFAULT_HMCP_TIMEOUT_MS), + ]; + for (const [k, v] of Object.entries(opts.env ?? {})) { + args.push('--env', `${k}=${v}`); + } + const result = runSphere(opts.cliPath, opts.cliHome, args, { + timeoutMs: (opts.timeoutMs ?? DEFAULT_HMCP_TIMEOUT_MS) + 30_000, + }); + if (result.status !== 0) { + throw new Error( + `sphere host spawn failed (status=${result.status}, signal=${result.signal}). ` + + `stderr: ${result.stderr.slice(0, 800)}\nstdout: ${result.stdout.slice(0, 800)}`, + ); + } + const responses = parseSpawnResponses(result.stdout); + const ready = responses.find((r) => r.type === 'hm.spawn_ready'); + if (!ready) { + const failed = responses.find((r) => r.type === 'hm.spawn_failed' || r.type === 'hm.error'); + throw new Error( + `sphere host spawn did not produce hm.spawn_ready. ` + + `last response: ${JSON.stringify(failed ?? responses[responses.length - 1])}`, + ); + } + const p = ready.payload; + return { + instanceId: String(p['instance_id'] ?? ''), + instanceName: String(p['instance_name'] ?? ''), + tenantPubkey: String(p['tenant_pubkey'] ?? ''), + tenantDirectAddress: String(p['tenant_direct_address'] ?? ''), + tenantNametag: typeof p['tenant_nametag'] === 'string' ? p['tenant_nametag'] : null, + state: String(p['state'] ?? ''), + }; +} + +export interface HostStopOpts { + cliPath: string; + cliHome: string; + managerAddress: string; + /** Either instanceName or instanceId works — sphere-cli accepts both. */ + target: string; + timeoutMs?: number; +} + +/** + * Issue `sphere host stop` for a tenant. Best-effort: tolerates + * already-stopped tenants and missing-instance errors so it's safe to + * call from afterAll() without precise lifecycle bookkeeping. + */ +export function hostStop(opts: HostStopOpts): SphereRunResult { + return runSphere( + opts.cliPath, + opts.cliHome, + [ + 'host', + 'stop', + opts.target, + '--manager', opts.managerAddress, + '--timeout', String(opts.timeoutMs ?? DEFAULT_HMCP_TIMEOUT_MS), + ], + { timeoutMs: (opts.timeoutMs ?? DEFAULT_HMCP_TIMEOUT_MS) + 15_000 }, + ); +} + +export interface HostListInstance { + readonly instance_id: string; + readonly instance_name: string; + readonly template_id: string; + readonly state: string; + readonly tenant_pubkey: string | null; + readonly tenant_direct_address: string | null; + readonly tenant_nametag: string | null; +} + +/** + * Issue `sphere host list --json` and return the parsed tenant list. + * Useful for assertions like "after 3 spawns, list returns 3 + * RUNNING instances" without repeating the JSON-parse boilerplate. + */ +export function hostList( + cliPath: string, + cliHome: string, + managerAddress: string, + timeoutMs = DEFAULT_HMCP_TIMEOUT_MS, +): readonly HostListInstance[] { + const result = runSphere( + cliPath, + cliHome, + ['host', 'list', '--manager', managerAddress, '--json', '--timeout', String(timeoutMs)], + { timeoutMs: timeoutMs + 15_000 }, + ); + if (result.status !== 0) { + throw new Error( + `sphere host list failed (status=${result.status}). stderr: ${result.stderr.slice(0, 500)}`, + ); + } + const start = result.stdout.indexOf('{'); + const end = result.stdout.lastIndexOf('}'); + if (start < 0 || end <= start) { + throw new Error(`sphere host list --json: no JSON object found. stdout: ${result.stdout.slice(0, 500)}`); + } + const obj = JSON.parse(result.stdout.slice(start, end + 1)) as { payload?: { instances?: HostListInstance[] } }; + return obj.payload?.instances ?? []; +} diff --git a/test/e2e-live/helpers/manager-process.ts b/test/e2e-live/helpers/manager-process.ts new file mode 100644 index 0000000..a81be2a --- /dev/null +++ b/test/e2e-live/helpers/manager-process.ts @@ -0,0 +1,280 @@ +/** + * Helper: pre-provision a Sphere wallet for the host-manager and spawn the + * compiled `dist/host-manager.js` binary from the agentic-hosting repo as + * a child process. + * + * Why this lives in trader-service instead of being imported from + * agentic-hosting: the trader-service repo doesn't depend on + * agentic-hosting as an npm package — they're sibling projects in the + * Unicity ecosystem. This helper reads the manager binary from a path + * resolved at runtime (default + * `/home/vrogojin/agentic_hosting/dist/host-manager.js`, override via + * `AGENTIC_HOSTING_PATH`). When agentic-hosting eventually publishes + * its manager binary as a versioned npm package, this resolver can be + * swapped to use the package path without touching the test layer. + * + * Why pre-create the wallet? + * The manager validates `MANAGER_PUBKEY`/`MANAGER_DIRECT_ADDRESS` env + * vars against the loaded wallet's identity (drift guard added in + * agentic-hosting Phase 5). Auto-generating a wallet at boot would + * mint a random keypair that doesn't match the env vars we set, so + * the manager would refuse to start. By pre-creating the wallet here + * we know the pubkey before spawn and wire env vars accordingly. + * + * The HMA itself runs as a Sphere wallet identity on testnet — same + * relay (`wss://nostr-relay.testnet.unicity.network`) that traders and + * escrow use. Tests using this helper consume one extra HMA wallet + * registration per run. + */ + +import { spawn, type ChildProcess } from 'node:child_process'; +import { mkdir, writeFile, rm, access } from 'node:fs/promises'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { randomUUID } from 'node:crypto'; +import { Sphere } from '@unicitylabs/sphere-sdk'; +import { createNodeProviders } from '@unicitylabs/sphere-sdk/impl/nodejs'; + +const TRUSTBASE_URL = + 'https://raw.githubusercontent.com/unicitynetwork/unicity-ids/refs/heads/main/bft-trustbase.testnet.json'; + +const DEFAULT_AGENTIC_HOSTING_PATH = '/home/vrogojin/agentic_hosting'; + +export interface HostManagerProcess { + /** Manager's chainPubkey, hex secp256k1. */ + readonly pubkey: string; + /** Canonical Unicity DIRECT://... address. */ + readonly directAddress: string; + /** `@nametag` registration if successful; null if registration failed. */ + readonly nametag: string | null; + /** Wallet directory the manager loaded its identity from. */ + readonly dataDir: string; + /** templates.json path the manager is configured against. */ + readonly templatesPath: string; + /** Authorized controller pubkey baked into AUTHORIZED_CONTROLLERS. */ + readonly controllerPubkey: string; + /** Process handle — null after `stop()`. */ + child: ChildProcess | null; + /** Accumulated stdout+stderr (JSON Lines + plain). */ + readonly logs: string[]; + /** Resolves when the manager logs `host_manager_started`. */ + readonly ready: Promise; + stop(): Promise; +} + +/** + * Resolve the path to the agentic-hosting checkout. Used to find + * `dist/host-manager.js`. Override the default via + * `AGENTIC_HOSTING_PATH`. Existence is checked at spawn time so the + * error message points at the env-var the operator can fix. + */ +function resolveAgenticHostingPath(): string { + return (process.env['AGENTIC_HOSTING_PATH'] ?? '').trim() || DEFAULT_AGENTIC_HOSTING_PATH; +} + +async function ensureTrustbase(dataDir: string): Promise { + await mkdir(dataDir, { recursive: true }); + const trustbasePath = join(dataDir, 'trustbase.json'); + const res = await fetch(TRUSTBASE_URL); + if (!res.ok) { + throw new Error(`Failed to download trustbase: HTTP ${res.status}`); + } + await writeFile(trustbasePath, await res.text(), 'utf-8'); + return trustbasePath; +} + +/** + * Pre-create the manager's Sphere wallet so the spawned binary loads it + * (instead of auto-generating a fresh one) and the env-var drift guard + * passes. Returns the identity that must be wired into MANAGER_PUBKEY / + * MANAGER_DIRECT_ADDRESS. + */ +async function provisionManagerWallet(dataDir: string, hostId: string): Promise<{ + pubkey: string; + directAddress: string; + nametag: string | null; +}> { + const trustbasePath = await ensureTrustbase(dataDir); + // Forward UNICITY_API_KEY when set; the SDK falls back to its public + // placeholder otherwise. + const apiKey = process.env['UNICITY_API_KEY']?.trim() || undefined; + const providers = createNodeProviders({ + network: 'testnet', + dataDir, + tokensDir: join(dataDir, 'tokens'), + oracle: { + trustBasePath: trustbasePath, + ...(apiKey ? { apiKey } : {}), + }, + }); + const nametag = `m-${hostId.replace(/[^a-z0-9]/gi, '').slice(0, 12).toLowerCase()}`; + const { sphere } = await Sphere.init({ + ...providers, + autoGenerate: true, + nametag, + }); + const identity = sphere.identity; + if (!identity) { + throw new Error('manager wallet provisioning returned no identity'); + } + const pubkey = identity.chainPubkey; + const directAddress = identity.directAddress ?? `DIRECT://${pubkey}`; + const resolvedNametag = identity.nametag ?? null; + // Tear down the helper-side wallet — the spawned manager re-loads + // from dataDir on its own. Two Sphere instances pointing at the same + // storage simultaneously would race relay subscriptions. + sphere.destroy(); + return { pubkey, directAddress, nametag: resolvedNametag }; +} + +export interface SpawnHostManagerOptions { + /** Test-supplied controller pubkey (hex secp256k1) added to AUTHORIZED_CONTROLLERS. */ + controllerPubkey: string; + /** Path to a templates.json file — defaults to agentic-hosting's config/templates.json. */ + templatesPath?: string; + /** HOST_ID override. Default: random `e2elive-<8chars>`. */ + hostId?: string; + /** Health/metrics port (diagnostics-only HTTP). Default 19401. */ + healthPort?: number; + /** Hello-handshake timeout in ms (default 60000 — generous for testnet boot). */ + helloTimeoutMs?: number; +} + +/** + * Provision a fresh Sphere wallet for the manager, then spawn the + * compiled host-manager binary against it. Resolves once the manager + * has logged `host_manager_started` (DM listener active, ready to + * accept HMCP). Rejects if the binary exits before that point with the + * last 30 log lines for diagnosis. + * + * Caller MUST call `.stop()` even if a test fails — the manager holds + * the persistence-path lock and a relay connection. + */ +export async function spawnHostManager(opts: SpawnHostManagerOptions): Promise { + const agenticPath = resolveAgenticHostingPath(); + const binPath = join(agenticPath, 'dist', 'host-manager.js'); + try { + await access(binPath); + } catch { + throw new Error( + `host-manager binary not found at ${binPath}. ` + + `Build it first (cd ${agenticPath} && npm run build), or set ` + + `AGENTIC_HOSTING_PATH to a checkout that has dist/host-manager.js.`, + ); + } + + const templatesPath = opts.templatesPath ?? join(agenticPath, 'config', 'templates.json'); + try { + await access(templatesPath); + } catch { + throw new Error( + `templates.json not found at ${templatesPath}. ` + + `Pass templatesPath explicitly or check AGENTIC_HOSTING_PATH.`, + ); + } + + const hostId = opts.hostId ?? `e2elive-${randomUUID().slice(0, 8)}`; + const sessionDir = join(tmpdir(), `trader-e2e-hma-${randomUUID()}`); + const dataDir = join(sessionDir, 'wallet'); + const tenantsDir = join(sessionDir, 'tenants'); + const stateDir = join(sessionDir, 'state'); + await mkdir(sessionDir, { recursive: true }); + await mkdir(tenantsDir, { recursive: true }); + await mkdir(stateDir, { recursive: true }); + + const identity = await provisionManagerWallet(dataDir, hostId); + + const persistencePath = join(stateDir, 'state.json'); + const env: Record = { + PATH: process.env['PATH'] ?? '', + HOST_ID: hostId, + MANAGER_PUBKEY: identity.pubkey, + MANAGER_DIRECT_ADDRESS: identity.directAddress, + AUTHORIZED_CONTROLLERS: opts.controllerPubkey, + TEMPLATES_PATH: templatesPath, + TENANTS_DIR: tenantsDir, + SPHERE_MANAGER_DATA_DIR: dataDir, + PERSISTENCE_PATH: persistencePath, + UNICITY_HEALTH_PORT: String(opts.healthPort ?? 19401), + UNICITY_NETWORK: 'testnet', + HELLO_TIMEOUT_MS: String(opts.helloTimeoutMs ?? 60_000), + LOG_LEVEL: 'info', + }; + if (process.env['UNICITY_API_KEY']) env['UNICITY_API_KEY'] = process.env['UNICITY_API_KEY']; + + const child = spawn('node', [binPath], { env, cwd: sessionDir, stdio: ['ignore', 'pipe', 'pipe'] }); + + const logs: string[] = []; + child.stdout?.setEncoding('utf-8'); + child.stderr?.setEncoding('utf-8'); + child.stdout?.on('data', (chunk: string) => { + for (const line of chunk.split('\n')) if (line) logs.push(line); + }); + child.stderr?.on('data', (chunk: string) => { + for (const line of chunk.split('\n')) if (line) logs.push(line); + }); + + // Resolve when `host_manager_started` log line appears. Reject if + // the process exits before that (boot failure surfaces here, not as + // an opaque hang during the first hm.spawn). + const ready = new Promise((resolve, reject) => { + let settled = false; + const finishOk = (): void => { if (!settled) { settled = true; resolve(); } }; + const finishErr = (e: Error): void => { if (!settled) { settled = true; reject(e); } }; + + const watcher = setInterval(() => { + if (logs.some((l) => l.includes('"host_manager_started"'))) { + clearInterval(watcher); + finishOk(); + } + }, 250); + + child.once('exit', (code) => { + clearInterval(watcher); + finishErr(new Error( + `host-manager exited prematurely (code=${code}) before host_manager_started. ` + + `last logs:\n${logs.slice(-30).join('\n')}`, + )); + }); + child.once('error', (err) => { + clearInterval(watcher); + finishErr(err); + }); + setTimeout(() => { + clearInterval(watcher); + finishErr(new Error( + `host-manager did not reach host_manager_started within 240s. last logs:\n${logs.slice(-40).join('\n')}`, + )); + }, 240_000).unref(); + }); + + const proc: HostManagerProcess = { + pubkey: identity.pubkey, + directAddress: identity.directAddress, + nametag: identity.nametag, + dataDir, + templatesPath, + controllerPubkey: opts.controllerPubkey, + child, + logs, + ready, + async stop(): Promise { + const c = proc.child; + if (!c) return; + proc.child = null; + if (c.exitCode === null) { + c.kill('SIGTERM'); + await new Promise((resolve) => { + const killTimer = setTimeout(() => { + try { c.kill('SIGKILL'); } catch { /* already dead */ } + }, 12_000); + killTimer.unref(); + c.once('exit', () => { clearTimeout(killTimer); resolve(); }); + }); + } + await rm(sessionDir, { recursive: true, force: true }).catch(() => { /* ignore */ }); + }, + }; + + return proc; +} diff --git a/test/e2e-live/helpers/sphere-cli.ts b/test/e2e-live/helpers/sphere-cli.ts new file mode 100644 index 0000000..6c34ecb --- /dev/null +++ b/test/e2e-live/helpers/sphere-cli.ts @@ -0,0 +1,163 @@ +/** + * Helper: locate, probe, and invoke the `sphere-cli` binary against a + * live host-manager. + * + * sphere-cli lives in its own repo (github.com/unicity-sphere/sphere-cli). + * This helper resolves the binary at runtime via `SPHERE_CLI_BIN` + * (default: `/home/vrogojin/sphere-cli-work/sphere-cli/bin/sphere.mjs`). + * The probe runs `sphere --help` with a 10s budget and reports the exit + * code + stderr so an upstream regression can be diagnosed without + * reading container logs. + * + * Tests use `probeSphereCli()` once in `beforeAll` and gate the suite + * with `describe.skipIf(!probe.ok)` if sphere-cli isn't runnable. + */ + +import { spawnSync, type SpawnSyncReturns } from 'node:child_process'; +import { existsSync, mkdirSync, writeFileSync } from 'node:fs'; +import { mkdtempSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +const DEFAULT_CLI_PATH = '/home/vrogojin/sphere-cli-work/sphere-cli/bin/sphere.mjs'; + +export type SphereCliProbe = + | { readonly ok: true; readonly path: string } + | { readonly ok: false; readonly reason: string }; + +export function probeSphereCli(): SphereCliProbe { + const cliPath = process.env['SPHERE_CLI_BIN']?.trim() || DEFAULT_CLI_PATH; + if (!existsSync(cliPath)) { + return { + ok: false, + reason: `sphere-cli binary not found at ${cliPath}. ` + + `Set SPHERE_CLI_BIN to override, or build sphere-cli ` + + `(github.com/unicity-sphere/sphere-cli).`, + }; + } + const r = spawnSync('node', [cliPath, '--help'], { + encoding: 'utf8', + timeout: 10_000, + killSignal: 'SIGKILL', + }); + if (r.error) { + return { ok: false, reason: `sphere-cli launch error: ${r.error.message}` }; + } + if (r.status !== 0) { + const stderr = (r.stderr || '').slice(0, 500).trim(); + return { + ok: false, + reason: `sphere-cli --help exited ${r.status}. stderr: ${stderr || '(empty)'}`, + }; + } + return { ok: true, path: cliPath }; +} + +/** + * Create an isolated CWD with a pre-seeded `.sphere-cli/config.json`. + * sphere-cli reads its config relative to cwd, so spawning the CLI with + * `cwd: home` isolates each test's wallet directory. + * + * Caller MUST eventually `rmSync(home, { recursive: true })` — the + * wallet contains a testnet mnemonic; leaving it on disk after the run + * is a leak even on testnet. + */ +export function createSphereCliEnv(label: string): { home: string } { + const safeLabel = label.replace(/[^a-zA-Z0-9-_]/g, '-').slice(0, 24); + const home = mkdtempSync(join(tmpdir(), `trader-e2e-sphere-${safeLabel}-`)); + const cfgDir = join(home, '.sphere-cli'); + mkdirSync(cfgDir, { recursive: true }); + writeFileSync( + join(cfgDir, 'config.json'), + JSON.stringify({ + network: 'testnet', + dataDir: cfgDir, + tokensDir: join(cfgDir, 'tokens'), + }), + 'utf8', + ); + return { home }; +} + +export interface SphereRunResult { + readonly stdout: string; + readonly stderr: string; + readonly status: number | null; + readonly signal: NodeJS.Signals | null; +} + +/** + * Invoke the sphere CLI with `cwd` set to a SphereCliEnv home so the + * wallet/config directory is isolated. Returns stdout/stderr/status. + * + * Non-zero exit codes are NORMAL for sphere-cli (e.g. timeout, hm.error + * response, manager rejected). Caller decides whether to assert or + * inspect. + * + * `extraEnv` overlays on top of the minimal env we forward. The default + * is intentionally minimal (PATH + UNICITY_API_KEY if set) — we don't + * leak the parent's env into the CLI's logs. + */ +export function runSphere( + cliPath: string, + cwd: string, + args: readonly string[], + opts?: { timeoutMs?: number; extraEnv?: Record }, +): SphereRunResult { + const r: SpawnSyncReturns = spawnSync('node', [cliPath, ...args], { + cwd, + encoding: 'utf8', + timeout: opts?.timeoutMs ?? 180_000, + killSignal: 'SIGKILL', + env: { + PATH: process.env['PATH'] ?? '', + HOME: process.env['HOME'] ?? cwd, + ...(process.env['UNICITY_API_KEY'] ? { UNICITY_API_KEY: process.env['UNICITY_API_KEY'] } : {}), + CI: '1', + FORCE_COLOR: '0', + ...(opts?.extraEnv ?? {}), + }, + }); + return { + stdout: r.stdout?.toString() ?? '', + stderr: r.stderr?.toString() ?? '', + status: r.status, + signal: r.signal, + }; +} + +/** + * Bootstrap a fresh wallet via `sphere wallet init --network testnet`. + * Captures the chainPubkey and directAddress from the JSON identity + * block emitted to stdout. Used to set up a controller wallet for HMA + * tests. + * + * Timeout: 240s — wallet creation does an aggregator round-trip (nametag + * mint) plus IPFS publish; typical happy path is ~60-90s but we give + * generous slack for a slow relay. + */ +export function bootstrapControllerWallet(cliPath: string, home: string): { + pubkey: string; + directAddress: string; +} { + const init = runSphere(cliPath, home, ['wallet', 'init', '--network', 'testnet'], { + timeoutMs: 240_000, + }); + if (init.status !== 0) { + throw new Error( + `sphere wallet init failed (status=${init.status}, signal=${init.signal}). ` + + `stderr (first 500): ${init.stderr.slice(0, 500)}\n` + + `stdout (last 500): ${init.stdout.slice(-500)}`, + ); + } + // Lenient regex matching either chainPubkey or directAddress JSON fields. + const pkMatch = init.stdout.match(/"chainPubkey":\s*"([0-9a-fA-F]{64,130})"/); + const addrMatch = init.stdout.match(/"directAddress":\s*"(DIRECT:\/\/[0-9a-fA-F]+)"/); + if (!pkMatch || !pkMatch[1]) { + throw new Error(`chainPubkey not found in sphere wallet init output:\n${init.stdout.slice(-1500)}`); + } + return { + pubkey: pkMatch[1], + directAddress: addrMatch?.[1] ?? `DIRECT://${pkMatch[1]}`, + }; +} diff --git a/test/e2e-live/hma-orchestrated.e2e-live.test.ts b/test/e2e-live/hma-orchestrated.e2e-live.test.ts new file mode 100644 index 0000000..31b26d2 --- /dev/null +++ b/test/e2e-live/hma-orchestrated.e2e-live.test.ts @@ -0,0 +1,214 @@ +/** + * Live e2e: HMA-orchestrated tenant lifecycle (escrow + 2 traders). + * + * Demonstrates the architecture the user actually wants for partner + * demos and production: a Host Manager Agent (HMA) spawns and manages + * every tenant container, controllers talk to the HMA via HMCP DMs, + * and trade ops go controller→tenant directly via ACP DMs (host- + * agnostic). NO direct `docker run` calls — the HMA owns lifecycle. + * + * Flow: + * 1. Probe sphere-cli's binary and the agentic-hosting build. Skip + * gracefully if either is missing — both repos move + * independently and an upstream regression shouldn't fail this + * branch's CI. + * 2. Bootstrap a controller wallet via `sphere wallet init`. + * 3. Boot the host-manager binary with the controller pubkey in + * AUTHORIZED_CONTROLLERS. + * 4. `sphere host spawn` for the escrow agent. + * 5. `sphere host spawn` for trader Alice. + * 6. `sphere host spawn` for trader Bob. + * 7. `sphere host list` returns 3 RUNNING instances. + * 8. Cleanup: `sphere host stop` for each, then stop the manager. + * + * This is the foundation test for the HMA-orchestrated e2e suite. + * Subsequent PRs (PR-C in the migration plan) layer trade ops on top: + * `sphere trader create-intent`, portfolio assertions, swap settlement. + * + * Performance target: ~3-4 minutes end-to-end on a healthy testnet + * (manager boot ~30s, three spawn round-trips ~30s each including + * `acp.hello`, three teardowns ~5s each). + * + * Skip semantics: + * describe.skipIf(!preconditions) — if either sphere-cli or + * agentic-hosting is unavailable, the suite skips with a clear + * diagnostic. The DM transport itself is covered by agentic-hosting's + * own live tests; this file specifically covers the trader-service + * integration seam. + */ + +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { rmSync, existsSync } from 'node:fs'; +import { join } from 'node:path'; +import { randomUUID } from 'node:crypto'; +import { + probeSphereCli, + createSphereCliEnv, + bootstrapControllerWallet, + type SphereCliProbe, +} from './helpers/sphere-cli.js'; +import { spawnHostManager, type HostManagerProcess } from './helpers/manager-process.js'; +import { + hostSpawn, + hostStop, + hostList, + type SpawnedTenant, +} from './helpers/hma-spawn.js'; + +// --------------------------------------------------------------------------- +// Preconditions: sphere-cli runnable + agentic-hosting binary built. +// Evaluated at module-load time so describe.skipIf can short-circuit +// the entire suite without spending a second on Sphere.init. +// --------------------------------------------------------------------------- + +const cliProbe: SphereCliProbe = probeSphereCli(); + +const agenticPath = (process.env['AGENTIC_HOSTING_PATH']?.trim() + || '/home/vrogojin/agentic_hosting'); +const managerBinPath = join(agenticPath, 'dist', 'host-manager.js'); +const agenticReady = existsSync(managerBinPath); + +const skip = !cliProbe.ok || !agenticReady; +const skipReason = !cliProbe.ok + ? `sphere-cli not runnable: ${cliProbe.ok ? '' : cliProbe.reason}` + : !agenticReady + ? `agentic-hosting binary missing at ${managerBinPath}. Build it (cd ${agenticPath} && npm run build) or set AGENTIC_HOSTING_PATH.` + : ''; + +interface SuiteState { + cliPath: string; + cliHome: string; + controllerPubkey: string; + manager: HostManagerProcess; + spawned: SpawnedTenant[]; +} + +describe.skipIf(skip)('HMA-orchestrated tenant lifecycle (live testnet)', () => { + if (skip) { + + console.warn(`[hma-orchestrated] SKIPPED: ${skipReason}`); + } + + let state: SuiteState | null = null; + + beforeAll(async () => { + if (skip) return; + if (!cliProbe.ok) throw new Error('precondition gate inverted'); // type narrowing + const cliPath = cliProbe.path; + const { home: cliHome } = createSphereCliEnv('hma-test'); + + console.log('[hma-orchestrated] bootstrapping controller wallet...'); + const controller = bootstrapControllerWallet(cliPath, cliHome); + + console.log(`[hma-orchestrated] controller pubkey ${controller.pubkey.slice(0, 16)}...`); + + + console.log('[hma-orchestrated] booting host-manager...'); + const manager = await spawnHostManager({ controllerPubkey: controller.pubkey }); + await manager.ready; + + console.log(`[hma-orchestrated] manager ready @ ${manager.nametag ?? manager.directAddress}`); + + state = { + cliPath, + cliHome, + controllerPubkey: controller.pubkey, + manager, + spawned: [], + }; + }, 540_000); // 9 min — wallet bootstrap + manager boot can each be 60-90s on slow testnet + + afterAll(async () => { + if (!state) return; + // Stop all tenants in parallel — best-effort. The manager will + // also force-remove on its own shutdown, but explicit stop keeps + // tenants from leaking into the next run if the manager hangs. + await Promise.allSettled( + state.spawned.map((t) => + Promise.resolve(hostStop({ + cliPath: state!.cliPath, + cliHome: state!.cliHome, + managerAddress: state!.manager.pubkey, + target: t.instanceName, + timeoutMs: 60_000, + })), + ), + ); + await state.manager.stop(); + try { rmSync(state.cliHome, { recursive: true, force: true }); } + catch { /* best effort */ } + }, 240_000); + + it('spawns escrow + 2 traders through the HMA and lists them as RUNNING', () => { + if (!state) throw new Error('beforeAll did not initialize state'); + const s = state; + + // Use the manager's @nametag if available — otherwise fall back to + // raw pubkey. sphere-cli accepts either. + const managerAddr = s.manager.nametag ? `@${s.manager.nametag}` : s.manager.pubkey; + + // Unique suffix so concurrent runs don't collide on Docker labels. + const runId = randomUUID().slice(0, 6); + + + console.log('[hma-orchestrated] spawning escrow...'); + const escrow = hostSpawn({ + cliPath: s.cliPath, + cliHome: s.cliHome, + managerAddress: managerAddr, + templateId: 'escrow-service', + instanceName: `escrow-${runId}`, + timeoutMs: 180_000, + }); + s.spawned.push(escrow); + expect(escrow.state).toBe('RUNNING'); + expect(escrow.tenantPubkey).toMatch(/^[0-9a-fA-F]{64,130}$/); + expect(escrow.tenantDirectAddress).toMatch(/^DIRECT:\/\//); + + + console.log(`[hma-orchestrated] escrow up: ${escrow.tenantNametag ?? escrow.tenantDirectAddress}`); + + + console.log('[hma-orchestrated] spawning trader Alice...'); + // Note: not configuring TRUSTED_ESCROWS at spawn-time. The HMA + // forbids UNICITY_* prefixes in controller-supplied env (security + // policy: only the manager itself injects UNICITY_* boot vars). + // Strategy/trust configuration belongs to runtime trade-ops via + // `sphere trader set-strategy` — covered by PR-C, not the + // lifecycle foundation test. + const alice = hostSpawn({ + cliPath: s.cliPath, + cliHome: s.cliHome, + managerAddress: managerAddr, + templateId: 'trader-agent', + instanceName: `alice-${runId}`, + timeoutMs: 180_000, + }); + s.spawned.push(alice); + expect(alice.state).toBe('RUNNING'); + + + console.log('[hma-orchestrated] spawning trader Bob...'); + const bob = hostSpawn({ + cliPath: s.cliPath, + cliHome: s.cliHome, + managerAddress: managerAddr, + templateId: 'trader-agent', + instanceName: `bob-${runId}`, + timeoutMs: 180_000, + }); + s.spawned.push(bob); + expect(bob.state).toBe('RUNNING'); + + // Verify all three are listed by the HMA as RUNNING. + + console.log('[hma-orchestrated] verifying via sphere host list...'); + const listed = hostList(s.cliPath, s.cliHome, managerAddr); + const ourSpawns = listed.filter((i) => [escrow.instanceId, alice.instanceId, bob.instanceId].includes(i.instance_id)); + expect(ourSpawns).toHaveLength(3); + for (const inst of ourSpawns) { + expect(inst.state).toBe('RUNNING'); + expect(inst.tenant_pubkey).toMatch(/^[0-9a-fA-F]{64,130}$/); + } + }, 720_000); // 12 min — three real spawns × ~3min each on slow testnet +}); diff --git a/test/e2e-live/infra-probe.d.ts b/test/e2e-live/infra-probe.d.ts new file mode 100644 index 0000000..da3b901 --- /dev/null +++ b/test/e2e-live/infra-probe.d.ts @@ -0,0 +1,17 @@ +/** + * Ambient declaration for @unicitylabs/infra-probe — the package ships + * pure-ESM .mjs without bundled .d.ts. Only typed to the surface our + * preflight uses; the full report shape is re-typed locally in preflight.ts + * with `as Report` so changes upstream surface as type errors there. + */ +declare module '@unicitylabs/infra-probe' { + export interface ProbeOptions { + network?: 'testnet' | 'mainnet' | 'dev'; + only?: string[]; + timeoutMs?: number; + aggregatorApiKey?: string; + } + export function runProbes(options?: ProbeOptions): Promise; + export function exitCodeForReport(report: unknown): number; + export const SERVICES: readonly string[]; +} diff --git a/test/e2e-live/preflight.ts b/test/e2e-live/preflight.ts new file mode 100644 index 0000000..c1c4bcb --- /dev/null +++ b/test/e2e-live/preflight.ts @@ -0,0 +1,151 @@ +/** + * Preflight infrastructure check for the trader-service e2e-live suite. + * + * Wraps `@unicitylabs/infra-probe` to verify that every Unicity Network + * service the live tests depend on (Nostr relay, L3 Aggregator, IPFS + * gateway, L1 Fulcrum, Market API) is reachable and functional BEFORE + * booting the host-manager binary or spawning any tenant containers. + * + * Failure modes the probe catches that the e2e suite would otherwise hit + * as opaque timeouts deep into a run: + * - Nostr relay silently dropping kind:1059 publishes — controller's + * HMCP DM never reaches the manager; we'd time out on `hm.spawn`. + * - Aggregator API key rejection / rate-limit — Sphere.init hangs + * during nametag registration. + * - IPFS gateway 5xx — token-create races during nametag mint. + * - Fulcrum chain tip stale — L1-side identity publishes never resolve. + * - Market API down — traders post intents but no peer can find them. + * + * Environment knobs: + * TRADER_E2E_SKIP_PREFLIGHT=1 — bypass the gate entirely (escape hatch) + * TRADER_E2E_PREFLIGHT_STRICT=1 — also fail on `degraded` (default: warn-only) + * TRADER_E2E_PREFLIGHT_NETWORK — override the network (default: testnet) + * TRADER_E2E_PREFLIGHT_TIMEOUT_MS — per-probe ceiling (default: 30000) + */ + +import { runProbes } from '@unicitylabs/infra-probe'; + +interface Check { + readonly name: string; + readonly status: 'pass' | 'warn' | 'fail'; + readonly latencyMs: number; + readonly message: string; +} + +interface Service { + readonly service: string; + readonly endpoint: string; + readonly status: 'healthy' | 'degraded' | 'unreachable' | 'error'; + readonly latencyMs: number; + readonly checks: Check[]; + readonly error?: string; +} + +interface Report { + readonly services: Service[]; + readonly summary: { + readonly total: number; + readonly healthy: number; + readonly degraded: number; + readonly unreachable: number; + }; +} + +function statusIcon(status: Service['status']): string { + if (status === 'healthy') return '✓'; + if (status === 'degraded') return '⚠'; + return '✗'; +} + +function checkIcon(status: Check['status']): string { + if (status === 'pass') return '✓'; + if (status === 'warn') return '⚠'; + return '✗'; +} + +function logReport(report: Report): void { + for (const svc of report.services) { + console.log( + `[preflight] ${statusIcon(svc.status)} ${svc.service.padEnd(11)} ${svc.endpoint} (${svc.status}, ${svc.latencyMs}ms)`, + ); + if (svc.error) { + console.log(`[preflight] error: ${svc.error}`); + } + for (const c of svc.checks) { + const icon = checkIcon(c.status); + console.log(`[preflight] ${icon} ${c.name.padEnd(20)} ${c.latencyMs}ms ${c.message}`); + } + } + const { total, healthy, degraded, unreachable } = report.summary; + console.log( + `[preflight] summary: ${healthy}/${total} healthy, ${degraded} degraded, ${unreachable} unreachable`, + ); +} + +export async function runPreflight(): Promise { + if (process.env['TRADER_E2E_SKIP_PREFLIGHT'] === '1') { + console.log('[preflight] SKIPPED (TRADER_E2E_SKIP_PREFLIGHT=1)'); + return; + } + + const VALID_NETWORKS = ['testnet', 'mainnet', 'dev'] as const; + type Network = (typeof VALID_NETWORKS)[number]; + const rawNetwork = process.env['TRADER_E2E_PREFLIGHT_NETWORK'] ?? 'testnet'; + if (!(VALID_NETWORKS as readonly string[]).includes(rawNetwork)) { + throw new Error( + `Preflight: invalid TRADER_E2E_PREFLIGHT_NETWORK="${rawNetwork}". ` + + `Must be one of: ${VALID_NETWORKS.join(', ')}.`, + ); + } + const network = rawNetwork as Network; + + const rawTimeoutMs = process.env['TRADER_E2E_PREFLIGHT_TIMEOUT_MS'] ?? '30000'; + const timeoutMs = Number(rawTimeoutMs); + if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) { + throw new Error( + `Preflight: invalid TRADER_E2E_PREFLIGHT_TIMEOUT_MS="${rawTimeoutMs}". ` + + `Must be a positive finite number (milliseconds).`, + ); + } + + const strict = process.env['TRADER_E2E_PREFLIGHT_STRICT'] === '1'; + + console.log( + `[preflight] probing ${network} infrastructure (timeout=${timeoutMs}ms, strict=${strict})...`, + ); + const startedAt = Date.now(); + + const report = (await runProbes({ network, timeoutMs })) as Report; + const elapsed = Date.now() - startedAt; + + logReport(report); + console.log(`[preflight] completed in ${elapsed}ms`); + + const { unreachable, degraded } = report.summary; + const downServices = report.services + .filter((s) => s.status === 'unreachable' || s.status === 'error') + .map((s) => `${s.service}=${s.status}`); + const slowServices = report.services + .filter((s) => s.status === 'degraded') + .map((s) => `${s.service}=${s.status}`); + + if (unreachable > 0) { + throw new Error( + `Preflight failed: ${unreachable} service(s) unreachable [${downServices.join(', ')}]. ` + + `Set TRADER_E2E_SKIP_PREFLIGHT=1 to bypass (not recommended — tests will likely hang).`, + ); + } + + if (degraded > 0) { + if (strict) { + throw new Error( + `Preflight failed (strict mode): ${degraded} service(s) degraded [${slowServices.join(', ')}]. ` + + `Unset TRADER_E2E_PREFLIGHT_STRICT or set TRADER_E2E_SKIP_PREFLIGHT=1 to bypass.`, + ); + } + console.warn( + `[preflight] WARNING: ${degraded} service(s) degraded [${slowServices.join(', ')}] — ` + + `tests may be slow or intermittently fail. Set TRADER_E2E_PREFLIGHT_STRICT=1 to fail-fast on this.`, + ); + } +} diff --git a/vitest.e2e-live.config.ts b/vitest.e2e-live.config.ts index bfcae6c..6dcecf0 100644 --- a/vitest.e2e-live.config.ts +++ b/vitest.e2e-live.config.ts @@ -1,28 +1,50 @@ /** * Live e2e test configuration — opt-in via `npm run test:e2e-live`. * - * IMPORTANT: These tests are NOT runnable in trader-service standalone. - * They depend on the Host Manager Agent (HMA) — `createHostManager`, - * `hm.spawn` over HMCP-0, the Dockerode adapter, and the agentic-hosting - * tenant template registry — none of which live in this repo. + * Tests run against REAL Unicity testnet infrastructure (Nostr relay at + * `wss://nostr-relay.testnet.unicity.network`, L3 aggregator at + * `goggregator-test.unicity.network`, IPFS gateway, Market API). The + * @unicitylabs/infra-probe globalSetup aborts the run if any of those + * services are unreachable so a multi-minute container-spawn cycle isn't + * wasted on a known-down service. * - * The tests are preserved here at the same shape they had in the - * agentic-hosting `pre-trader-cut-v1` tag so that they can be ported to - * (or run from) the agentic-hosting repository's nightly integration CI - * once the host-manager half of the stack is decoupled too. See - * `test/e2e-live/README.md` for the full rationale and the runbook for - * the manual scenarios these tests describe. + * Two flavors of test live in this directory: * - * Running this config in trader-service today will fail at module - * resolution — the missing host-manager source files are the signal. + * 1. Direct-docker tests (the existing files: basic-roundtrip, + * multi-agent, surplus-refund, etc.) — provision tenants via + * docker run with a synthesized fake manager pubkey, drive + * trading via trader-ctl over Sphere DM. NO host-manager + * involvement; matches the architecture as it stood before + * agentic-hosting Phase 5 shipped DM transport. + * + * 2. HMA-orchestrated tests (hma-orchestrated.e2e-live.test.ts and + * siblings added in PR-B/PR-C of the migration plan) — boot the + * compiled `dist/host-manager.js` from agentic-hosting, spawn + * tenants via `sphere host spawn`, drive trading via + * `sphere trader …`. This is the production architecture: the + * HMA owns lifecycle; controllers reach tenants directly for + * trade ops. + * + * Both flavors share the infra-probe preflight; the HMA-orchestrated + * tests additionally `describe.skipIf` themselves when sphere-cli or + * the agentic-hosting binary isn't available locally so an upstream + * regression doesn't fail this branch's CI. + * + * Bypass the preflight (e.g. iterating offline against a single test): + * TRADER_E2E_SKIP_PREFLIGHT=1 npm run test:e2e-live */ import { defineConfig } from 'vitest/config'; export default defineConfig({ test: { - include: ['test/e2e-live/**/*.test.ts'], + include: ['test/e2e-live/**/*.e2e-live.test.ts'], + // Run @unicitylabs/infra-probe before any test file. Aborts the run + // up-front if testnet services are unreachable, instead of consuming + // a 10-15-minute container spawn cycle to discover the same failure + // as an opaque timeout. Bypass: TRADER_E2E_SKIP_PREFLIGHT=1. + globalSetup: ['./test/e2e-live/global-setup.ts'], testTimeout: 180_000, - hookTimeout: 300_000, + hookTimeout: 600_000, pool: 'forks', poolOptions: { forks: { singleFork: true } }, sequence: { concurrent: false }, From a40d373fffaf9c4b8a8b5852c086d83e2be528d1 Mon Sep 17 00:00:00 2001 From: Vladimir Rogojin Date: Sun, 3 May 2026 23:27:44 +0200 Subject: [PATCH 2/6] review: address 4-agent review (security, code, arch) on PR-B MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Aggregated fixes from parallel reviews of PR #14 (refactor/e2e-live-via-hma) by code-reviewer × 2, security-auditor, architect-review agents. CRITICAL — code-reviewer (process lifecycle): manager-process.ts - stop(): wrap c.kill('SIGTERM') in try/catch matching the SIGKILL fallback. The asymmetric handling was a latent ESRCH time-bomb if the process exited between the exitCode check and the kill (Node sets exitCode on the tick AFTER the OS-level exit). - ready promise: setInterval watcher.unref() so a missed cleanup path can never keep the event loop alive past hookTimeout. All clearInterval calls are still present; this is a defense-in-depth safety net. WARNING — security-auditor: manager-process.ts / sphere-cli.ts - mkdir(..., { recursive: true }) without explicit mode honours umask (typically 0o022 → 0o755), exposing testnet wallet material to other local users on shared CI runners. Pass `mode: 0o700` to every mkdir/mkdirSync call that creates a directory under which mnemonics or chain keys land. Add chmodSync(home, 0o700) in createSphereCliEnv as defense-in-depth for platforms where mkdtempSync may deviate from posix-default. - bootstrapControllerWallet's failure paths previously echoed init.stderr and init.stdout into thrown Error messages. The upstream `sphere wallet init` writes the freshly-minted mnemonic to stdout (suppressed only when isTTY=false; behaviour depends on the upstream version). Vitest captures error messages to log files, persisting potential mnemonics on disk. Redact the subprocess output entirely; tell the operator to re-run with stdout connected to debug. - ensureTrustbase: the URL pins to a mutable refs/heads/main ref, a real attack surface for any future mainnet-targeted use. Add a SHA-256 log line and a TODO to pin to a commit SHA. The hash check is advisory (we don't fail on mismatch since pinning a known hash here would create churn on legitimate upstream updates) — but an unexpected hash will surface in test output. WARNING — code-reviewer (parsing consistency): hma-spawn.ts - hostList silently returned [] when payload.instances was missing or malformed. Now throws like hostSpawn does — a future protocol rename to `tenants` would surface as a clear "missing payload.instances" error rather than as a misleading "expected 3 RUNNING, got 0". WARNING — architect-review (developer-default footgun): manager-process.ts / sphere-cli.ts - Hard-coded /home/vrogojin/ defaults are kept as developer fallbacks but rejected when CI=1. A missing SPHERE_CLI_BIN or AGENTIC_HOSTING_PATH on a CI runner is now a hard fail, not a silent skip — eliminates the false-confidence "tests passed but were skipped" signal flagged by the architectural review. - New checkAgenticHostingPath() helper returns a structured {ok,reason} so the test's describe.skipIf can surface a precise diagnostic without the test re-implementing the env-var logic. WARNING — architect-review (deprecation signaling): helpers/contracts.ts - The architectural prelude described the direct-docker pattern as the target architecture. Rewritten to document BOTH flavors (Architecture A: direct-docker, scheduled for removal; Architecture B: HMA-orchestrated, target). Future contributors reading this file see the migration plan, not a stale declaration of intent. helpers/tenant-fixture.ts - Added @deprecated JSDoc pointing to hma-spawn.ts as the successor. Existing tests using provisionTrader() continue to work; new tests should follow Architecture B. Verified live: hma-orchestrated.e2e-live.test.ts still passes in 40s on testnet (was 38s pre-fix; the +2s is the 250ms unref'd interval plus the trustbase hash log). 651 default tests still pass. --- test/e2e-live/helpers/contracts.ts | 51 ++++++++---- test/e2e-live/helpers/hma-spawn.ts | 14 +++- test/e2e-live/helpers/manager-process.ts | 81 ++++++++++++++++--- test/e2e-live/helpers/sphere-cli.ts | 66 ++++++++++++--- test/e2e-live/helpers/tenant-fixture.ts | 21 +++-- .../hma-orchestrated.e2e-live.test.ts | 34 +++++--- 6 files changed, 219 insertions(+), 48 deletions(-) diff --git a/test/e2e-live/helpers/contracts.ts b/test/e2e-live/helpers/contracts.ts index 10b4a3f..7d0f012 100644 --- a/test/e2e-live/helpers/contracts.ts +++ b/test/e2e-live/helpers/contracts.ts @@ -1,24 +1,47 @@ /** * E2E-live test infrastructure contracts. * - * Five worktrees are filling in the helper modules in parallel; this file - * pins down their EXPORTED shapes so each can compile and test against the - * contract while peer impls land. Once all impls are in, this file becomes - * a stable interface registry — useful for understanding the harness without + * This file pins the EXPORTED shapes of the helper modules so each can + * compile and test against the contract independently. It's a stable + * interface registry — useful for understanding the harness without * reading every helper. * - * Architecture (the model the user actually wants for partner demos): + * Two architectures coexist in this directory during the in-progress + * migration to HMA-orchestrated tests (see SPHERE-CLI-EXTRACTION-PLAN + * §6.4 in agentic-hosting): * - * Test → uses -- docker-helpers - * │ │ - * ├── tenant-fixture (provisions traders) ─── docker run ────────┘ - * │ - * └── trader-ctl-driver (drives commands) ─── DM ──→ trader tenant + * ARCHITECTURE A — direct-docker (legacy, scheduled for removal): * - * Crucially: NO host-manager, NO HMCP. Tests provision containers - * directly via the Docker daemon and drive trading via trader-ctl. This - * matches the production architecture where agentic-hosting only - * orchestrates LIFECYCLE; trading happens controller ↔ tenant directly. + * Test + * ├── tenant-fixture (provisions traders) ─── docker run ─→ container + * └── trader-ctl-driver (drives commands) ─── ACP DM ────→ tenant + * + * The tests in this flavor (basic-roundtrip, multi-agent, surplus- + * refund, etc.) bypass the Host Manager Agent (HMA) and call the + * local Docker daemon directly. This was correct before + * agentic-hosting Phase 5 shipped DM transport for the HMA — the + * HMA simply couldn't be driven over DMs. As of agentic-hosting + * PR #22 (merged 2026-05-03), it can. + * + * ARCHITECTURE B — HMA-orchestrated (target architecture): + * + * Test + * ├── manager-process (spawns HMA) ─── subprocess ─→ HMA + * ├── hma-spawn (sphere host spawn|list|stop) ─── HMCP DM ────→ HMA + * │ │ + * │ docker create + * │ ↓ + * │ tenant + * └── trader-ctl-driver / sphere trader … ─── ACP DM ────→ tenant + * + * The HMA owns lifecycle; trade ops still go controller→tenant + * directly so tenants stay host-agnostic. Existing direct-docker + * tests are migrated to this flavor incrementally; new tests + * should follow Architecture B (see hma-orchestrated.e2e-live.test.ts). + * + * The interfaces declared in this file (DockerContainer, runContainer, + * etc.) are part of Architecture A and will be deprecated once all + * direct-docker tests are migrated. */ // ============================================================================ diff --git a/test/e2e-live/helpers/hma-spawn.ts b/test/e2e-live/helpers/hma-spawn.ts index 2d5eebf..ecf7b99 100644 --- a/test/e2e-live/helpers/hma-spawn.ts +++ b/test/e2e-live/helpers/hma-spawn.ts @@ -210,5 +210,17 @@ export function hostList( throw new Error(`sphere host list --json: no JSON object found. stdout: ${result.stdout.slice(0, 500)}`); } const obj = JSON.parse(result.stdout.slice(start, end + 1)) as { payload?: { instances?: HostListInstance[] } }; - return obj.payload?.instances ?? []; + // Throw on a structurally-malformed payload rather than silently + // returning []. A list_result missing `payload.instances` is a + // protocol contract violation (or a future field rename); pretending + // it means "zero instances" would let an asserts-3-RUNNING test + // pass with a misleading "expected 3, got 0" instead of pointing at + // the real fault. Mirrors hostSpawn's strict parsing. + if (!obj.payload || !Array.isArray(obj.payload.instances)) { + throw new Error( + `sphere host list --json: response missing payload.instances. ` + + `Got: ${JSON.stringify(obj).slice(0, 500)}`, + ); + } + return obj.payload.instances; } diff --git a/test/e2e-live/helpers/manager-process.ts b/test/e2e-live/helpers/manager-process.ts index a81be2a..a5a9fa3 100644 --- a/test/e2e-live/helpers/manager-process.ts +++ b/test/e2e-live/helpers/manager-process.ts @@ -31,14 +31,25 @@ import { spawn, type ChildProcess } from 'node:child_process'; import { mkdir, writeFile, rm, access } from 'node:fs/promises'; import { join } from 'node:path'; import { tmpdir } from 'node:os'; -import { randomUUID } from 'node:crypto'; +import { randomUUID, createHash } from 'node:crypto'; import { Sphere } from '@unicitylabs/sphere-sdk'; import { createNodeProviders } from '@unicitylabs/sphere-sdk/impl/nodejs'; +// TODO: pin trustbase URL to a commit SHA (or a release tag) once +// upstream publishes one. Mutable `refs/heads/main` ref is acceptable +// for testnet-only e2e but a real attack surface for any future +// mainnet-targeted use. + const TRUSTBASE_URL = 'https://raw.githubusercontent.com/unicitynetwork/unicity-ids/refs/heads/main/bft-trustbase.testnet.json'; -const DEFAULT_AGENTIC_HOSTING_PATH = '/home/vrogojin/agentic_hosting'; +/** + * Default agentic-hosting path for local-developer ergonomics. Points + * at one specific user's filesystem layout; non-default environments + * MUST set `AGENTIC_HOSTING_PATH`. When `CI=1` the default is + * rejected — see architectural review finding #1. + */ +const DEVELOPER_FALLBACK_AGENTIC_HOSTING_PATH = '/home/vrogojin/agentic_hosting'; export interface HostManagerProcess { /** Manager's chainPubkey, hex secp256k1. */ @@ -69,17 +80,57 @@ export interface HostManagerProcess { * error message points at the env-var the operator can fix. */ function resolveAgenticHostingPath(): string { - return (process.env['AGENTIC_HOSTING_PATH'] ?? '').trim() || DEFAULT_AGENTIC_HOSTING_PATH; + const explicit = (process.env['AGENTIC_HOSTING_PATH'] ?? '').trim(); + if (!explicit && process.env['CI'] === '1') { + throw new Error( + 'AGENTIC_HOSTING_PATH is unset and CI=1. ' + + 'Set AGENTIC_HOSTING_PATH to a checkout of agentic-hosting that has been built ' + + '(`npm run build` produces dist/host-manager.js). The developer fallback path ' + + 'is intentionally rejected on CI to avoid silent skips that look like passes.', + ); + } + return explicit || DEVELOPER_FALLBACK_AGENTIC_HOSTING_PATH; +} + +/** + * For tests that want to opt into the same skip-on-missing semantics + * the suite uses (see `hma-orchestrated.e2e-live.test.ts`), check + * whether the prerequisite is satisfiable WITHOUT throwing on a + * missing CI env var. Returns a structured result so the test can + * surface a precise reason in its `describe.skipIf` warning. + */ +export function checkAgenticHostingPath(): { ok: true; path: string } | { ok: false; reason: string } { + try { + const path = resolveAgenticHostingPath(); + return { ok: true, path }; + } catch (err) { + return { ok: false, reason: err instanceof Error ? err.message : String(err) }; + } } async function ensureTrustbase(dataDir: string): Promise { - await mkdir(dataDir, { recursive: true }); + // 0o700 — wallet material lands in this directory; on a shared CI + // runner default umask would leave it world-readable. + await mkdir(dataDir, { recursive: true, mode: 0o700 }); const trustbasePath = join(dataDir, 'trustbase.json'); const res = await fetch(TRUSTBASE_URL); if (!res.ok) { throw new Error(`Failed to download trustbase: HTTP ${res.status}`); } - await writeFile(trustbasePath, await res.text(), 'utf-8'); + const body = await res.text(); + // Defense-in-depth: the URL pins to a mutable `refs/heads/main` ref + // (test-only blast radius is testnet, but a tampered trustbase + // would redirect L2 BFT verification). Log the SHA-256 of the body + // so an unexpected hash surfaces in the test output and the + // operator can compare against a trusted snapshot. The hash check + // is advisory — we don't fail on mismatch since pinning a known + // hash here would create churn every time the upstream legitimately + // updates. See TODO at module top for the long-term fix (pin to a + // commit SHA in the URL). + const hash = createHash('sha256').update(body).digest('hex'); + + console.log(`[manager-process] trustbase fetched (sha256=${hash.slice(0, 16)}…, ${body.length} bytes)`); + await writeFile(trustbasePath, body, 'utf-8'); return trustbasePath; } @@ -178,9 +229,12 @@ export async function spawnHostManager(opts: SpawnHostManagerOptions): Promise { clearInterval(watcher); @@ -263,7 +322,11 @@ export async function spawnHostManager(opts: SpawnHostManagerOptions): Promise((resolve) => { const killTimer = setTimeout(() => { try { c.kill('SIGKILL'); } catch { /* already dead */ } diff --git a/test/e2e-live/helpers/sphere-cli.ts b/test/e2e-live/helpers/sphere-cli.ts index 6c34ecb..d881f4a 100644 --- a/test/e2e-live/helpers/sphere-cli.ts +++ b/test/e2e-live/helpers/sphere-cli.ts @@ -14,25 +14,47 @@ */ import { spawnSync, type SpawnSyncReturns } from 'node:child_process'; -import { existsSync, mkdirSync, writeFileSync } from 'node:fs'; +import { existsSync, mkdirSync, writeFileSync, chmodSync } from 'node:fs'; import { mkdtempSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -const DEFAULT_CLI_PATH = '/home/vrogojin/sphere-cli-work/sphere-cli/bin/sphere.mjs'; +/** + * Default CLI path for local-developer ergonomics. Points at one + * specific user's filesystem layout; non-default environments MUST + * set `SPHERE_CLI_BIN`. When `CI=1` the default is rejected — we'd + * rather a CI run fail loud with "set SPHERE_CLI_BIN" than silently + * skip every HMA test because the path doesn't exist on the runner + * (which would create a false-confidence "tests passed but skipped" + * signal). See architectural review finding #1. + */ +const DEVELOPER_FALLBACK_CLI_PATH = '/home/vrogojin/sphere-cli-work/sphere-cli/bin/sphere.mjs'; export type SphereCliProbe = | { readonly ok: true; readonly path: string } | { readonly ok: false; readonly reason: string }; export function probeSphereCli(): SphereCliProbe { - const cliPath = process.env['SPHERE_CLI_BIN']?.trim() || DEFAULT_CLI_PATH; + const explicit = process.env['SPHERE_CLI_BIN']?.trim(); + // CI runners must opt in explicitly. A missing env var on CI is a + // configuration bug, not a "skip silently" condition. + if (!explicit && process.env['CI'] === '1') { + return { + ok: false, + reason: 'SPHERE_CLI_BIN is unset and CI=1. ' + + 'Set SPHERE_CLI_BIN to the path of a built sphere-cli binary ' + + '(`bin/sphere.mjs` from github.com/unicity-sphere/sphere-cli).', + }; + } + const cliPath = explicit || DEVELOPER_FALLBACK_CLI_PATH; if (!existsSync(cliPath)) { return { ok: false, - reason: `sphere-cli binary not found at ${cliPath}. ` + - `Set SPHERE_CLI_BIN to override, or build sphere-cli ` + - `(github.com/unicity-sphere/sphere-cli).`, + reason: explicit + ? `SPHERE_CLI_BIN points at ${cliPath} which does not exist.` + : `sphere-cli binary not found at developer fallback ${cliPath}. ` + + `Set SPHERE_CLI_BIN, or build sphere-cli ` + + `(github.com/unicity-sphere/sphere-cli).`, }; } const r = spawnSync('node', [cliPath, '--help'], { @@ -64,9 +86,17 @@ export function probeSphereCli(): SphereCliProbe { */ export function createSphereCliEnv(label: string): { home: string } { const safeLabel = label.replace(/[^a-zA-Z0-9-_]/g, '-').slice(0, 24); + // mkdtempSync creates with mode 0o700 by default — keeps the wallet + // dir owner-only on shared CI runners. const home = mkdtempSync(join(tmpdir(), `trader-e2e-sphere-${safeLabel}-`)); + // Defense-in-depth: chmod again in case the umask was modified or + // the platform deviates from the posix-default 0o700. + chmodSync(home, 0o700); const cfgDir = join(home, '.sphere-cli'); - mkdirSync(cfgDir, { recursive: true }); + // Explicit 0o700 — mkdirSync with `recursive: true` honours umask + // (typically 0o022 → mode 0o755), which is too permissive for a + // directory holding a testnet mnemonic. + mkdirSync(cfgDir, { recursive: true, mode: 0o700 }); writeFileSync( join(cfgDir, 'config.json'), JSON.stringify({ @@ -144,17 +174,33 @@ export function bootstrapControllerWallet(cliPath: string, home: string): { timeoutMs: 240_000, }); if (init.status !== 0) { + // We deliberately do NOT include init.stderr or init.stdout in the + // message. sphere-cli's `wallet init` writes the freshly-minted + // mnemonic to stdout (suppressed only when isTTY=false; behaviour + // depends on the upstream version), and prints diagnostic + // material to stderr that may transitively quote secret material + // on a future SDK change. A failed wallet init means we cannot + // proceed; the operator can re-run with DEBUG=1 and an interactive + // shell to inspect output rather than pulling it through Vitest's + // error reporter (which gets captured to log files). throw new Error( `sphere wallet init failed (status=${init.status}, signal=${init.signal}). ` + - `stderr (first 500): ${init.stderr.slice(0, 500)}\n` + - `stdout (last 500): ${init.stdout.slice(-500)}`, + `Re-run with stdout/stderr connected (no Vitest pipe) to inspect — ` + + `we redact subprocess output to avoid persisting testnet mnemonics in logs.`, ); } // Lenient regex matching either chainPubkey or directAddress JSON fields. const pkMatch = init.stdout.match(/"chainPubkey":\s*"([0-9a-fA-F]{64,130})"/); const addrMatch = init.stdout.match(/"directAddress":\s*"(DIRECT:\/\/[0-9a-fA-F]+)"/); if (!pkMatch || !pkMatch[1]) { - throw new Error(`chainPubkey not found in sphere wallet init output:\n${init.stdout.slice(-1500)}`); + // Same redaction rationale: do not echo subprocess output back + // through the test reporter. The mnemonic CAN appear in stdout + // if the upstream version doesn't honour the isTTY guard. + throw new Error( + 'chainPubkey not found in sphere wallet init output. ' + + 'Output redacted to avoid persisting potential mnemonic. ' + + 'Re-run with stdout connected to debug.', + ); } return { pubkey: pkMatch[1], diff --git a/test/e2e-live/helpers/tenant-fixture.ts b/test/e2e-live/helpers/tenant-fixture.ts index e83735b..b92dbb3 100644 --- a/test/e2e-live/helpers/tenant-fixture.ts +++ b/test/e2e-live/helpers/tenant-fixture.ts @@ -1,6 +1,17 @@ /** * tenant-fixture — provisions a fresh trader tenant container ready to trade. * + * @deprecated Architecture A (direct-docker). New tests should use + * `hma-spawn.ts` (Architecture B — HMA-orchestrated). See + * `contracts.ts` for the migration plan and SPHERE-CLI-EXTRACTION-PLAN + * §6.4 in agentic-hosting for the upstream architectural decision. + * + * Existing tests using `provisionTrader()` continue to work — they + * are scheduled for migration on a per-file basis. Once all direct- + * docker tests are migrated to `hostSpawn` from `hma-spawn.ts`, this + * file (and `docker-helpers.ts`, `trader-ctl-driver.ts`'s direct + * mode) will be removed. + * * Flow per `provisionTrader(opts)`: * 1. Materialize a fresh Sphere wallet on the host filesystem (mkdtempSync). * 2. Optionally fund the wallet from the testnet faucet. @@ -13,11 +24,11 @@ * On any failure: cleanup partial resources (container + wallet dir) before * rethrowing. The returned `dispose()` is idempotent. * - * Architectural note (echoes contracts.ts): NO host-manager, NO HMCP. We - * provision via the local Docker daemon directly. The trader's ACP-0 listener - * still demands UNICITY_MANAGER_PUBKEY — we synthesize a one-off pubkey for - * that env var so the boilerplate startup checks pass; the e2e tests drive - * the trader via trader-ctl over Sphere DM, never via the manager channel. + * Architectural note: NO host-manager, NO HMCP — direct docker daemon + * call. The trader's ACP-0 listener still demands UNICITY_MANAGER_PUBKEY — + * we synthesize a one-off pubkey for that env var so the boilerplate + * startup checks pass; the e2e tests drive the trader via trader-ctl + * over Sphere DM, never via the manager channel. */ import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'node:fs'; diff --git a/test/e2e-live/hma-orchestrated.e2e-live.test.ts b/test/e2e-live/hma-orchestrated.e2e-live.test.ts index 31b26d2..657706f 100644 --- a/test/e2e-live/hma-orchestrated.e2e-live.test.ts +++ b/test/e2e-live/hma-orchestrated.e2e-live.test.ts @@ -47,7 +47,11 @@ import { bootstrapControllerWallet, type SphereCliProbe, } from './helpers/sphere-cli.js'; -import { spawnHostManager, type HostManagerProcess } from './helpers/manager-process.js'; +import { + spawnHostManager, + checkAgenticHostingPath, + type HostManagerProcess, +} from './helpers/manager-process.js'; import { hostSpawn, hostStop, @@ -59,21 +63,33 @@ import { // Preconditions: sphere-cli runnable + agentic-hosting binary built. // Evaluated at module-load time so describe.skipIf can short-circuit // the entire suite without spending a second on Sphere.init. +// +// On CI (CI=1), missing env vars (SPHERE_CLI_BIN / AGENTIC_HOSTING_PATH) +// are FAIL conditions, not skip conditions — see the architectural +// review's finding on false-confidence skips. The helper functions +// emit appropriately distinct reasons. // --------------------------------------------------------------------------- const cliProbe: SphereCliProbe = probeSphereCli(); +const agenticProbe = checkAgenticHostingPath(); -const agenticPath = (process.env['AGENTIC_HOSTING_PATH']?.trim() - || '/home/vrogojin/agentic_hosting'); -const managerBinPath = join(agenticPath, 'dist', 'host-manager.js'); -const agenticReady = existsSync(managerBinPath); +let managerBinPath = ''; +let agenticReady = false; +if (agenticProbe.ok) { + managerBinPath = join(agenticProbe.path, 'dist', 'host-manager.js'); + agenticReady = existsSync(managerBinPath); +} const skip = !cliProbe.ok || !agenticReady; const skipReason = !cliProbe.ok - ? `sphere-cli not runnable: ${cliProbe.ok ? '' : cliProbe.reason}` - : !agenticReady - ? `agentic-hosting binary missing at ${managerBinPath}. Build it (cd ${agenticPath} && npm run build) or set AGENTIC_HOSTING_PATH.` - : ''; + ? `sphere-cli not runnable: ${cliProbe.reason}` + : !agenticProbe.ok + ? agenticProbe.reason + : !agenticReady + ? `agentic-hosting binary missing at ${managerBinPath}. ` + + `Build it (cd ${agenticProbe.ok ? agenticProbe.path : ''} && npm run build) ` + + `or set AGENTIC_HOSTING_PATH.` + : ''; interface SuiteState { cliPath: string; From c080e90962ab215f51cae1f7577a4aa5cb2b2deb Mon Sep 17 00:00:00 2001 From: Vladimir Rogojin Date: Sun, 3 May 2026 23:39:06 +0200 Subject: [PATCH 3/6] review: round 1 steelman fixes (CI detection, stop() race, hash log) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round 1 of steelman loop on PR #14. Six findings addressed: WARNING — CI gate accepted only `CI=1`, missing every mainstream CI provider (GitHub Actions, GitLab, CircleCI all set `CI=true`). The fix's stated goal (fail loud on misconfigured runners) silently failed on the most common targets. New `isCi()` helper in `sphere-cli.ts` matches `CI=true|True|1|` and Azure's `TF_BUILD`. Both `probeSphereCli()` and `resolveAgenticHostingPath()` now use it. WARNING — `stop()` deadlocked for 12s if the manager exited before stop() was called. The `c.once('exit')` listener never fires for an already-emitted exit event, so the await hung until the (unref'd) killTimer expired on the event loop. With cascading test failures this could pile up to (12s × N) afterAll wait. Fix: check `exitCode` AND `signalCode` BEFORE attaching the listener; also re-check inside the Promise constructor to close the listener-attach race window. NOTE — Trustbase SHA-256 was truncated to 16 hex chars in the log line, defeating the verification utility (16 chars is marginal collision resistance and not enough for an operator to paste into a Slack message and compare). Log full 64 chars. NOTE — `chmodSync` comment misattributed the threat. Real risk is ordering: any future writeFileSync between mkdtempSync and chmodSync becomes a TOCTOU disclosure on non-POSIX platforms. Comment rewritten to flag the ordering dependency. NOTE — `hostList` error echoed full payload via `JSON.stringify(obj)`, which could include peer pubkeys / instance IDs in vitest log files. Now logs only top-level + payload key names. NOTE — Architecture-A symbols in `contracts.ts` (DockerRunOptions, DockerContainer, RunContainer, StopContainer, RemoveContainer, GetContainerLogs, WaitForContainerRunning, ProvisionTraderOptions, ProvisionedTenant, ProvisionTrader) had no per-symbol @deprecated JSDoc. The prelude alone is not surfaced at call sites by TypeScript's language server. Added @deprecated tags everywhere. Verified: hma-orchestrated.e2e-live.test.ts passes in 38s, default 651-test suite green, lint+typecheck clean. --- test/e2e-live/helpers/contracts.ts | 15 +++++-- test/e2e-live/helpers/hma-spawn.ts | 10 ++++- test/e2e-live/helpers/manager-process.ts | 54 ++++++++++++++++-------- test/e2e-live/helpers/sphere-cli.ts | 43 +++++++++++++++---- 4 files changed, 92 insertions(+), 30 deletions(-) diff --git a/test/e2e-live/helpers/contracts.ts b/test/e2e-live/helpers/contracts.ts index 7d0f012..df947c3 100644 --- a/test/e2e-live/helpers/contracts.ts +++ b/test/e2e-live/helpers/contracts.ts @@ -46,8 +46,10 @@ // ============================================================================ // docker-helpers.ts — owns: provisioning + lifecycle of a single container +// (Architecture A — direct-docker, deprecated. Use hma-spawn.ts instead.) // ============================================================================ +/** @deprecated Architecture A. Use `SpawnedTenant` from `hma-spawn.ts`. */ export interface DockerRunOptions { /** Fully-qualified image ref, e.g. ghcr.io/vrogojin/agentic-hosting/trader:v0.1 */ image: string; @@ -65,6 +67,7 @@ export interface DockerRunOptions { startTimeoutMs?: number; } +/** @deprecated Architecture A. Use `SpawnedTenant` from `hma-spawn.ts`. */ export interface DockerContainer { /** Docker container ID (sha256-ish, 64 chars). */ id: string; @@ -78,22 +81,24 @@ export interface DockerContainer { * Spawn a container with the given options. Returns once the daemon has * accepted the run (NOT once the app inside is ready — provisioning code * upstream polls for that). + * @deprecated Architecture A. Use `hostSpawn` from `hma-spawn.ts`. */ export type RunContainer = (opts: DockerRunOptions) => Promise; /** * Stop with SIGTERM, fall back to SIGKILL after `timeoutMs`. Idempotent — * stopping an already-stopped container is a no-op. + * @deprecated Architecture A. Use `hostStop` from `hma-spawn.ts`. */ export type StopContainer = (id: string, timeoutMs?: number) => Promise; -/** Remove the container record. Throws if container is still running. */ +/** @deprecated Architecture A. Use `hostStop` from `hma-spawn.ts`. */ export type RemoveContainer = (id: string) => Promise; -/** Read the last `lines` of stdout+stderr for diagnostic output on failure. */ +/** @deprecated Architecture A. Read tenant logs via the HMA's `hm.inspect` instead. */ export type GetContainerLogs = (id: string, lines?: number) => Promise; -/** Resolve true once container is RUNNING per `docker inspect`, else false on `timeoutMs` elapse. */ +/** @deprecated Architecture A. Use `hostList` from `hma-spawn.ts`. */ export type WaitForContainerRunning = (id: string, timeoutMs?: number) => Promise; // ============================================================================ @@ -135,8 +140,10 @@ export type RunTraderCtl = ( // ============================================================================ // tenant-fixture.ts — owns: provision N trader tenants ready to trade +// (Architecture A — direct-docker, deprecated. Use hma-spawn.ts instead.) // ============================================================================ +/** @deprecated Architecture A. Use `HostSpawnOpts` from `hma-spawn.ts`. */ export interface ProvisionTraderOptions { /** Operator-friendly label, used in container name and logs. */ label: string; @@ -154,6 +161,7 @@ export interface ProvisionTraderOptions { readyTimeoutMs?: number; } +/** @deprecated Architecture A. Use `SpawnedTenant` from `hma-spawn.ts`. */ export interface ProvisionedTenant { /** trader-ctl-targetable address. Either DIRECT://hex or 64-char hex. */ address: string; @@ -170,6 +178,7 @@ export interface ProvisionedTenant { * wait until reachable. Returns a fully-armed tenant. * * On any step failure, partial resources are cleaned up before throwing. + * @deprecated Architecture A. Use `hostSpawn` from `hma-spawn.ts`. */ export type ProvisionTrader = (opts: ProvisionTraderOptions) => Promise; diff --git a/test/e2e-live/helpers/hma-spawn.ts b/test/e2e-live/helpers/hma-spawn.ts index ecf7b99..6e852ba 100644 --- a/test/e2e-live/helpers/hma-spawn.ts +++ b/test/e2e-live/helpers/hma-spawn.ts @@ -217,9 +217,17 @@ export function hostList( // pass with a misleading "expected 3, got 0" instead of pointing at // the real fault. Mirrors hostSpawn's strict parsing. if (!obj.payload || !Array.isArray(obj.payload.instances)) { + // Echo only the top-level keys, not the full payload body — the + // response could include peer pubkeys, instance IDs, or other + // identifiers that don't belong in test logs (especially with + // vitest's --reporter=json which captures errors to disk). + const topKeys = Object.keys(obj); + const payloadKeys = obj.payload && typeof obj.payload === 'object' + ? Object.keys(obj.payload as Record) + : []; throw new Error( `sphere host list --json: response missing payload.instances. ` + - `Got: ${JSON.stringify(obj).slice(0, 500)}`, + `Top-level keys: [${topKeys.join(', ')}]. payload keys: [${payloadKeys.join(', ')}].`, ); } return obj.payload.instances; diff --git a/test/e2e-live/helpers/manager-process.ts b/test/e2e-live/helpers/manager-process.ts index a5a9fa3..55aa8a6 100644 --- a/test/e2e-live/helpers/manager-process.ts +++ b/test/e2e-live/helpers/manager-process.ts @@ -34,6 +34,7 @@ import { tmpdir } from 'node:os'; import { randomUUID, createHash } from 'node:crypto'; import { Sphere } from '@unicitylabs/sphere-sdk'; import { createNodeProviders } from '@unicitylabs/sphere-sdk/impl/nodejs'; +import { isCi } from './sphere-cli.js'; // TODO: pin trustbase URL to a commit SHA (or a release tag) once // upstream publishes one. Mutable `refs/heads/main` ref is acceptable @@ -81,9 +82,9 @@ export interface HostManagerProcess { */ function resolveAgenticHostingPath(): string { const explicit = (process.env['AGENTIC_HOSTING_PATH'] ?? '').trim(); - if (!explicit && process.env['CI'] === '1') { + if (!explicit && isCi()) { throw new Error( - 'AGENTIC_HOSTING_PATH is unset and CI=1. ' + + 'AGENTIC_HOSTING_PATH is unset on a CI runner. ' + 'Set AGENTIC_HOSTING_PATH to a checkout of agentic-hosting that has been built ' + '(`npm run build` produces dist/host-manager.js). The developer fallback path ' + 'is intentionally rejected on CI to avoid silent skips that look like passes.', @@ -128,8 +129,11 @@ async function ensureTrustbase(dataDir: string): Promise { // updates. See TODO at module top for the long-term fix (pin to a // commit SHA in the URL). const hash = createHash('sha256').update(body).digest('hex'); - - console.log(`[manager-process] trustbase fetched (sha256=${hash.slice(0, 16)}…, ${body.length} bytes)`); + // Log the full 64-char hash so an operator can paste it into a + // verification check or compare against a known-good snapshot. + // 16-char prefix collisions are not a real attack but provide + // marginal verification value for a security-relevant log line. + console.log(`[manager-process] trustbase fetched (sha256=${hash}, ${body.length} bytes)`); await writeFile(trustbasePath, body, 'utf-8'); return trustbasePath; } @@ -321,20 +325,36 @@ export async function spawnHostManager(opts: SpawnHostManagerOptions): Promise((resolve) => { - const killTimer = setTimeout(() => { - try { c.kill('SIGKILL'); } catch { /* already dead */ } - }, 12_000); - killTimer.unref(); - c.once('exit', () => { clearTimeout(killTimer); resolve(); }); - }); + // Race fix: if the manager already exited (crash, SIGTERM from + // earlier signal, etc.), `c.exitCode` is non-null AND the + // 'exit' event has ALREADY fired. Attaching `c.once('exit')` + // now would never fire and `stop()` would hang for 12s until + // the (unref'd) killTimer resolved on its own. Check exitCode + // BEFORE attaching the listener, then short-circuit if dead. + // This also subsumes the previous null-check. + if (c.exitCode !== null || c.signalCode !== null) { + await rm(sessionDir, { recursive: true, force: true }).catch(() => { /* ignore */ }); + return; } + // ESRCH-safe: the process may have died between the exitCode + // check and this kill() call. Wrap to match the SIGKILL + // fallback below. + try { c.kill('SIGTERM'); } catch { /* already dead */ } + await new Promise((resolve) => { + // If the exit event ALREADY raced ahead of us between the + // exitCode check and this listener attach, settle immediately. + // Vanishingly rare, but the deadlock cost is high (12s × N + // tenants in afterAll under cascading test failures). + if (c.exitCode !== null || c.signalCode !== null) { + resolve(); + return; + } + const killTimer = setTimeout(() => { + try { c.kill('SIGKILL'); } catch { /* already dead */ } + }, 12_000); + killTimer.unref(); + c.once('exit', () => { clearTimeout(killTimer); resolve(); }); + }); await rm(sessionDir, { recursive: true, force: true }).catch(() => { /* ignore */ }); }, }; diff --git a/test/e2e-live/helpers/sphere-cli.ts b/test/e2e-live/helpers/sphere-cli.ts index d881f4a..8055921 100644 --- a/test/e2e-live/helpers/sphere-cli.ts +++ b/test/e2e-live/helpers/sphere-cli.ts @@ -22,14 +22,32 @@ import { join } from 'node:path'; /** * Default CLI path for local-developer ergonomics. Points at one * specific user's filesystem layout; non-default environments MUST - * set `SPHERE_CLI_BIN`. When `CI=1` the default is rejected — we'd - * rather a CI run fail loud with "set SPHERE_CLI_BIN" than silently - * skip every HMA test because the path doesn't exist on the runner + * set `SPHERE_CLI_BIN`. On CI the default is rejected — we'd rather + * a CI run fail loud with "set SPHERE_CLI_BIN" than silently skip + * every HMA test because the path doesn't exist on the runner * (which would create a false-confidence "tests passed but skipped" * signal). See architectural review finding #1. */ const DEVELOPER_FALLBACK_CLI_PATH = '/home/vrogojin/sphere-cli-work/sphere-cli/bin/sphere.mjs'; +/** + * Detect a CI runner. Different providers use different conventions: + * - GitHub Actions / GitLab CI / CircleCI / Travis / Jenkins: `CI=true` + * - Custom runners or `CI=1` (mostly local fakes / Drone CI) + * - Azure Pipelines: `TF_BUILD=True` (no `CI` set) — also covered + * + * `Boolean(process.env.CI)` catches every truthy value except a literal + * empty string, which is the only "set but disabled" idiom in + * widespread use. The `TF_BUILD` clause picks up Azure where `CI` may + * be unset entirely. + */ +export function isCi(): boolean { + const ci = process.env['CI']; + if (ci !== undefined && ci !== '' && ci !== '0' && ci.toLowerCase() !== 'false') return true; + if (process.env['TF_BUILD']) return true; + return false; +} + export type SphereCliProbe = | { readonly ok: true; readonly path: string } | { readonly ok: false; readonly reason: string }; @@ -38,10 +56,10 @@ export function probeSphereCli(): SphereCliProbe { const explicit = process.env['SPHERE_CLI_BIN']?.trim(); // CI runners must opt in explicitly. A missing env var on CI is a // configuration bug, not a "skip silently" condition. - if (!explicit && process.env['CI'] === '1') { + if (!explicit && isCi()) { return { ok: false, - reason: 'SPHERE_CLI_BIN is unset and CI=1. ' + + reason: 'SPHERE_CLI_BIN is unset on a CI runner. ' + 'Set SPHERE_CLI_BIN to the path of a built sphere-cli binary ' + '(`bin/sphere.mjs` from github.com/unicity-sphere/sphere-cli).', }; @@ -86,11 +104,18 @@ export function probeSphereCli(): SphereCliProbe { */ export function createSphereCliEnv(label: string): { home: string } { const safeLabel = label.replace(/[^a-zA-Z0-9-_]/g, '-').slice(0, 24); - // mkdtempSync creates with mode 0o700 by default — keeps the wallet - // dir owner-only on shared CI runners. + // mkdtempSync calls mkdtemp(3), which is mandated by POSIX to + // create the directory with mode 0700 (umask is irrelevant). On + // Linux/macOS this is reliable. The chmodSync below is paranoia + // for non-POSIX platforms (Windows) where Node's mkdtempSync + // emulation may use the native CreateDirectory call which respects + // an inherited DACL rather than POSIX mode bits. Redundant but + // cheap on POSIX. const home = mkdtempSync(join(tmpdir(), `trader-e2e-sphere-${safeLabel}-`)); - // Defense-in-depth: chmod again in case the umask was modified or - // the platform deviates from the posix-default 0o700. + // IMPORTANT: nothing must be written to `home` between mkdtempSync + // and this chmodSync. If a future contributor adds a writeFileSync + // here, a non-POSIX platform's brief world-readable window would + // become a TOCTOU disclosure of whatever was just written. chmodSync(home, 0o700); const cfgDir = join(home, '.sphere-cli'); // Explicit 0o700 — mkdirSync with `recursive: true` honours umask From 8ba3b70cbae6380e3824aab68b587ea43cc0be58 Mon Sep 17 00:00:00 2001 From: Vladimir Rogojin Date: Sun, 3 May 2026 23:46:23 +0200 Subject: [PATCH 4/6] review: round 2 steelman fixes (CI normalization, async hostStop, deps) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round 2 of steelman loop on PR #14. Four findings addressed: NOTE — isCi() accepted `CI=no` and `TF_BUILD=False` as truthy. Apply the same falsy normalization to TF_BUILD that CI already had; add `no` (case-insensitive) to the falsy set. Hedge the JSDoc to "common providers" rather than over-claiming "every truthy value." WARNING — hostStop was synchronous (spawnSync), so `Promise.allSettled(spawned.map(hostStop))` ran SEQUENTIALLY despite the parallel-looking shape. Three sequential 75s budgets approach the 240s afterAll hookTimeout under any tenant slowdown — a tail test failure could trip the hook timeout. Fix: add `runSphereAsync` (uses spawn instead of spawnSync) and convert hostStop to return a Promise. The afterAll teardown now genuinely fans out three DM round-trips concurrently; total budget is bounded by the SLOWEST tenant, not the SUM. Drop the cosmetic Promise.resolve() wrap in the test now that hostStop is genuinely async. WARNING — TraderCtlOptions, TraderCtlResult, RunTraderCtl in contracts.ts had no @deprecated tags. Round 1 added tags on the Architecture-A docker types but missed the Architecture-A trader-ctl driver types. Added now — the IDE/lint signal at call sites is restored. NOTE — Trustbase hash log has no baseline to compare against. Document this with a TODO: until upstream unicity-ids publishes a canonical hash per release tag, the log is decorative. Either pin the URL or store an expected hash in this repo and assert on it. Verified: live test passes in 35s (down from 38s — the parallel teardown shaved a few seconds), lint+typecheck clean. --- test/e2e-live/helpers/contracts.ts | 5 + test/e2e-live/helpers/hma-spawn.ts | 13 ++- test/e2e-live/helpers/manager-process.ts | 8 +- test/e2e-live/helpers/sphere-cli.ts | 104 ++++++++++++++---- .../hma-orchestrated.e2e-live.test.ts | 14 ++- 5 files changed, 114 insertions(+), 30 deletions(-) diff --git a/test/e2e-live/helpers/contracts.ts b/test/e2e-live/helpers/contracts.ts index df947c3..861b4a6 100644 --- a/test/e2e-live/helpers/contracts.ts +++ b/test/e2e-live/helpers/contracts.ts @@ -103,8 +103,11 @@ export type WaitForContainerRunning = (id: string, timeoutMs?: number) => Promis // ============================================================================ // trader-ctl-driver.ts — owns: invoke the canonical CLI as a subprocess +// (Architecture A — direct ACP DM. Use `sphere trader …` from sphere-cli +// via `runSphere()` from `helpers/sphere-cli.ts` once PR-C lands.) // ============================================================================ +/** @deprecated Architecture A. PR-C migrates trade-ops to `sphere trader …`. */ export interface TraderCtlOptions { /** Tenant address: @nametag, DIRECT://hex, or 64-char hex pubkey. */ tenant: string; @@ -118,6 +121,7 @@ export interface TraderCtlOptions { json?: boolean; } +/** @deprecated Architecture A. PR-C migrates trade-ops to `sphere trader …`. */ export interface TraderCtlResult { /** Always 0 on success; see `error` field on non-zero. */ exitCode: number; @@ -131,6 +135,7 @@ export interface TraderCtlResult { * Run `trader-ctl [args]` as a subprocess. Uses the bundled * trader-ctl from this repo (../bin/trader-ctl). Throws ONLY on subprocess * launch failure; non-zero exit codes are returned as `result.exitCode`. + * @deprecated Architecture A. PR-C migrates trade-ops to `sphere trader …`. */ export type RunTraderCtl = ( command: string, diff --git a/test/e2e-live/helpers/hma-spawn.ts b/test/e2e-live/helpers/hma-spawn.ts index 6e852ba..80cfe7b 100644 --- a/test/e2e-live/helpers/hma-spawn.ts +++ b/test/e2e-live/helpers/hma-spawn.ts @@ -19,7 +19,7 @@ * `trader-ctl-driver.ts` (and will gain a `sphere trader` mode in PR-C). */ -import { runSphere, type SphereRunResult } from './sphere-cli.js'; +import { runSphere, runSphereAsync, type SphereRunResult } from './sphere-cli.js'; /** Timeout passed to sphere-cli's `--timeout` flag (DM request budget). */ const DEFAULT_HMCP_TIMEOUT_MS = 120_000; @@ -156,9 +156,16 @@ export interface HostStopOpts { * Issue `sphere host stop` for a tenant. Best-effort: tolerates * already-stopped tenants and missing-instance errors so it's safe to * call from afterAll() without precise lifecycle bookkeeping. + * + * Async (returns a Promise) so callers can run multiple stops in + * parallel via `Promise.all`/`Promise.allSettled`. The previous sync + * implementation made parallelism impossible (each spawnSync blocked + * the event loop, so .map(hostStop)+Promise.allSettled ran + * sequentially), and three sequential 75s budgets approached the 240s + * afterAll hookTimeout under any tenant slowdown. */ -export function hostStop(opts: HostStopOpts): SphereRunResult { - return runSphere( +export async function hostStop(opts: HostStopOpts): Promise { + return runSphereAsync( opts.cliPath, opts.cliHome, [ diff --git a/test/e2e-live/helpers/manager-process.ts b/test/e2e-live/helpers/manager-process.ts index 55aa8a6..51a3177 100644 --- a/test/e2e-live/helpers/manager-process.ts +++ b/test/e2e-live/helpers/manager-process.ts @@ -131,8 +131,12 @@ async function ensureTrustbase(dataDir: string): Promise { const hash = createHash('sha256').update(body).digest('hex'); // Log the full 64-char hash so an operator can paste it into a // verification check or compare against a known-good snapshot. - // 16-char prefix collisions are not a real attack but provide - // marginal verification value for a security-relevant log line. + // + // TODO: maintain a known-good snapshot. Currently NO BASELINE + // EXISTS — this log is decorative until one is established. Once + // the upstream unicity-ids repo publishes a canonical hash for + // each release tag, either pin the URL to that tag (preferred) or + // store the expected hash in this repo and assert on it. console.log(`[manager-process] trustbase fetched (sha256=${hash}, ${body.length} bytes)`); await writeFile(trustbasePath, body, 'utf-8'); return trustbasePath; diff --git a/test/e2e-live/helpers/sphere-cli.ts b/test/e2e-live/helpers/sphere-cli.ts index 8055921..a023e0e 100644 --- a/test/e2e-live/helpers/sphere-cli.ts +++ b/test/e2e-live/helpers/sphere-cli.ts @@ -13,7 +13,7 @@ * with `describe.skipIf(!probe.ok)` if sphere-cli isn't runnable. */ -import { spawnSync, type SpawnSyncReturns } from 'node:child_process'; +import { spawnSync, spawn, type SpawnSyncReturns } from 'node:child_process'; import { existsSync, mkdirSync, writeFileSync, chmodSync } from 'node:fs'; import { mkdtempSync } from 'node:fs'; import { tmpdir } from 'node:os'; @@ -31,20 +31,29 @@ import { join } from 'node:path'; const DEVELOPER_FALLBACK_CLI_PATH = '/home/vrogojin/sphere-cli-work/sphere-cli/bin/sphere.mjs'; /** - * Detect a CI runner. Different providers use different conventions: - * - GitHub Actions / GitLab CI / CircleCI / Travis / Jenkins: `CI=true` - * - Custom runners or `CI=1` (mostly local fakes / Drone CI) - * - Azure Pipelines: `TF_BUILD=True` (no `CI` set) — also covered + * Detect a CI runner across the common providers our team uses: + * - GitHub Actions / GitLab CI / CircleCI / Travis / Buildkite: `CI=true` + * - Drone CI / local fakes: `CI=1` + * - Azure Pipelines: `TF_BUILD=True` (often without `CI` set) * - * `Boolean(process.env.CI)` catches every truthy value except a literal - * empty string, which is the only "set but disabled" idiom in - * widespread use. The `TF_BUILD` clause picks up Azure where `CI` may - * be unset entirely. + * Falsy idioms (explicit "off") are honoured: empty string, `0`, + * `false`, `False`, `no` (case-insensitive) all mean "not on CI." Any + * other non-empty value enables CI mode. + * + * Other CI providers (Jenkins via JENKINS_HOME, Bamboo via + * BAMBOO_BUILDNUMBER, etc.) are NOT covered. If we add a CI provider + * that doesn't set CI / TF_BUILD, extend this function rather than + * re-implementing the check inline. */ export function isCi(): boolean { - const ci = process.env['CI']; - if (ci !== undefined && ci !== '' && ci !== '0' && ci.toLowerCase() !== 'false') return true; - if (process.env['TF_BUILD']) return true; + const isTruthy = (v: string | undefined): boolean => { + if (v === undefined || v === '') return false; + const lower = v.toLowerCase(); + if (lower === '0' || lower === 'false' || lower === 'no') return false; + return true; + }; + if (isTruthy(process.env['CI'])) return true; + if (isTruthy(process.env['TF_BUILD'])) return true; return false; } @@ -153,6 +162,17 @@ export interface SphereRunResult { * is intentionally minimal (PATH + UNICITY_API_KEY if set) — we don't * leak the parent's env into the CLI's logs. */ +function buildEnv(opts?: { extraEnv?: Record }, cwd?: string): Record { + return { + PATH: process.env['PATH'] ?? '', + HOME: process.env['HOME'] ?? cwd ?? '/', + ...(process.env['UNICITY_API_KEY'] ? { UNICITY_API_KEY: process.env['UNICITY_API_KEY'] } : {}), + CI: '1', + FORCE_COLOR: '0', + ...(opts?.extraEnv ?? {}), + }; +} + export function runSphere( cliPath: string, cwd: string, @@ -164,14 +184,7 @@ export function runSphere( encoding: 'utf8', timeout: opts?.timeoutMs ?? 180_000, killSignal: 'SIGKILL', - env: { - PATH: process.env['PATH'] ?? '', - HOME: process.env['HOME'] ?? cwd, - ...(process.env['UNICITY_API_KEY'] ? { UNICITY_API_KEY: process.env['UNICITY_API_KEY'] } : {}), - CI: '1', - FORCE_COLOR: '0', - ...(opts?.extraEnv ?? {}), - }, + env: buildEnv(opts, cwd), }); return { stdout: r.stdout?.toString() ?? '', @@ -181,6 +194,57 @@ export function runSphere( }; } +/** + * Async variant of `runSphere`. Use this when callers want true + * parallelism (e.g., stopping N tenants concurrently in `afterAll` + * via `Promise.all`/`Promise.allSettled`). The sync `runSphere` + * blocks the event loop, so a `.map(runSphere)` followed by + * `Promise.allSettled` actually runs sequentially — no parallelism. + * + * Mirrors `runSphere`'s contract: non-zero exit codes are returned + * (not thrown); only subprocess launch failures reject the promise. + */ +export function runSphereAsync( + cliPath: string, + cwd: string, + args: readonly string[], + opts?: { timeoutMs?: number; extraEnv?: Record }, +): Promise { + return new Promise((resolve, reject) => { + const child = spawn('node', [cliPath, ...args], { + cwd, + env: buildEnv(opts, cwd), + stdio: ['ignore', 'pipe', 'pipe'], + }); + const stdoutChunks: Buffer[] = []; + const stderrChunks: Buffer[] = []; + child.stdout?.on('data', (c: Buffer) => stdoutChunks.push(c)); + child.stderr?.on('data', (c: Buffer) => stderrChunks.push(c)); + let timedOut = false; + const timer = setTimeout(() => { + timedOut = true; + try { child.kill('SIGKILL'); } catch { /* already dead */ } + }, opts?.timeoutMs ?? 180_000); + timer.unref(); + child.once('error', (err) => { + clearTimeout(timer); + reject(err); + }); + child.once('close', (code, signal) => { + clearTimeout(timer); + resolve({ + stdout: Buffer.concat(stdoutChunks).toString('utf-8'), + stderr: Buffer.concat(stderrChunks).toString('utf-8'), + // `timedOut` short-circuits to a recognizable status: SIGKILL'd + // by our timer ⇒ status null + signal 'SIGKILL'. Caller can + // distinguish that from a normal exit. + status: timedOut ? null : code, + signal: signal as NodeJS.Signals | null, + }); + }); + }); +} + /** * Bootstrap a fresh wallet via `sphere wallet init --network testnet`. * Captures the chainPubkey and directAddress from the JSON identity diff --git a/test/e2e-live/hma-orchestrated.e2e-live.test.ts b/test/e2e-live/hma-orchestrated.e2e-live.test.ts index 657706f..ac57634 100644 --- a/test/e2e-live/hma-orchestrated.e2e-live.test.ts +++ b/test/e2e-live/hma-orchestrated.e2e-live.test.ts @@ -136,18 +136,22 @@ describe.skipIf(skip)('HMA-orchestrated tenant lifecycle (live testnet)', () => afterAll(async () => { if (!state) return; - // Stop all tenants in parallel — best-effort. The manager will - // also force-remove on its own shutdown, but explicit stop keeps - // tenants from leaking into the next run if the manager hangs. + // Stop all tenants concurrently — best-effort. hostStop is now + // genuinely async (uses spawn, not spawnSync), so the three + // requests fan out as concurrent DM round-trips and the budget + // is bounded by the SLOWEST tenant, not the SUM of all of them. + // The manager would also force-remove on its own shutdown, but + // explicit stop keeps tenants from leaking into the next run + // if the manager hangs. await Promise.allSettled( state.spawned.map((t) => - Promise.resolve(hostStop({ + hostStop({ cliPath: state!.cliPath, cliHome: state!.cliHome, managerAddress: state!.manager.pubkey, target: t.instanceName, timeoutMs: 60_000, - })), + }), ), ); await state.manager.stop(); From 32f381ac016834615661668b103d914d4140b4e4 Mon Sep 17 00:00:00 2001 From: Vladimir Rogojin Date: Sun, 3 May 2026 23:51:14 +0200 Subject: [PATCH 5/6] review: round 3 steelman fixes (false-timeout race, isCi() edges, OOM cap) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round 3 of steelman loop on PR #14. Three findings addressed (PR-A returned ROUND CLEAN with no new findings). NOTE — runSphereAsync timer callback unconditionally set `timedOut = true` and called child.kill, even if the process already exited cleanly at the timer boundary. Result: a successful hostStop could log status=null (false-timeout) — confusing in post-mortem debugging though not a test-failure (Promise.allSettled swallowed). Fix: guard the timer callback with `child.exitCode !== null || child.signalCode !== null` short-circuit, matching the same pattern stop() already uses. NOTE — isCi() edge cases: • `CI=off` was treated as truthy. Added 'off' to the falsy set (matches npm/shell-script convention; alongside '0', 'false', 'no'). • `CI=' '` (whitespace-only) was treated as truthy. Added .trim() before the empty-string check. NOTE — runSphereAsync had no maxBuffer guard. spawnSync defaults to maxBuffer=1MB and throws on overflow; child_process.spawn has no such limit. A misbehaving subprocess flooding stdout could OOM the test process. Fix: track per-stream byte count, kill the child and reject the promise with a descriptive error if either stream exceeds 10 MiB. Bound memory deterministically. Verified: live test passes in 37s, lint+typecheck clean. PR-A (sphere-cli #6) returned ROUND CLEAN this round — every attack either failed against the current code or surfaced pre-existing process notes (undated TODO, uncommitted trader-commands files) already acknowledged in earlier rounds. Ready for merge. --- test/e2e-live/helpers/sphere-cli.ts | 54 ++++++++++++++++++++++++++--- 1 file changed, 49 insertions(+), 5 deletions(-) diff --git a/test/e2e-live/helpers/sphere-cli.ts b/test/e2e-live/helpers/sphere-cli.ts index a023e0e..9a245b0 100644 --- a/test/e2e-live/helpers/sphere-cli.ts +++ b/test/e2e-live/helpers/sphere-cli.ts @@ -46,10 +46,14 @@ const DEVELOPER_FALLBACK_CLI_PATH = '/home/vrogojin/sphere-cli-work/sphere-cli/b * re-implementing the check inline. */ export function isCi(): boolean { - const isTruthy = (v: string | undefined): boolean => { - if (v === undefined || v === '') return false; + const isTruthy = (raw: string | undefined): boolean => { + if (raw === undefined) return false; + const v = raw.trim(); + if (v === '') return false; const lower = v.toLowerCase(); - if (lower === '0' || lower === 'false' || lower === 'no') return false; + // Cover the common "set but disabled" idioms across npm, + // shell-script convention, and most config systems. + if (lower === '0' || lower === 'false' || lower === 'no' || lower === 'off') return false; return true; }; if (isTruthy(process.env['CI'])) return true; @@ -203,7 +207,14 @@ export function runSphere( * * Mirrors `runSphere`'s contract: non-zero exit codes are returned * (not thrown); only subprocess launch failures reject the promise. + * + * Output bounded by `MAX_BUFFER_BYTES` (10 MiB per stream). Async + * `child_process.spawn` has no maxBuffer (spawnSync does), so a + * misbehaving subprocess flooding stdout could OOM the test process. + * On overflow we kill the child and reject with a descriptive error. */ +const MAX_BUFFER_BYTES = 10 * 1024 * 1024; + export function runSphereAsync( cliPath: string, cwd: string, @@ -218,10 +229,39 @@ export function runSphereAsync( }); const stdoutChunks: Buffer[] = []; const stderrChunks: Buffer[] = []; - child.stdout?.on('data', (c: Buffer) => stdoutChunks.push(c)); - child.stderr?.on('data', (c: Buffer) => stderrChunks.push(c)); + let stdoutBytes = 0; + let stderrBytes = 0; + let overflowed = false; + const checkOverflow = (which: 'stdout' | 'stderr', total: number): boolean => { + if (overflowed) return true; + if (total > MAX_BUFFER_BYTES) { + overflowed = true; + try { child.kill('SIGKILL'); } catch { /* already dead */ } + reject(new Error( + `runSphereAsync: ${which} exceeded MAX_BUFFER_BYTES=${MAX_BUFFER_BYTES} ` + + `(${total} bytes). Killed child to bound memory; partial output discarded.`, + )); + return true; + } + return false; + }; + child.stdout?.on('data', (c: Buffer) => { + stdoutBytes += c.length; + if (checkOverflow('stdout', stdoutBytes)) return; + stdoutChunks.push(c); + }); + child.stderr?.on('data', (c: Buffer) => { + stderrBytes += c.length; + if (checkOverflow('stderr', stderrBytes)) return; + stderrChunks.push(c); + }); let timedOut = false; const timer = setTimeout(() => { + // Race-safe: if the process already exited cleanly between the + // last event-loop tick and now, exitCode is non-null. Don't + // misreport a clean exit as a timeout — that creates a + // misleading false-timeout in the caller's logs. + if (child.exitCode !== null || child.signalCode !== null) return; timedOut = true; try { child.kill('SIGKILL'); } catch { /* already dead */ } }, opts?.timeoutMs ?? 180_000); @@ -232,6 +272,10 @@ export function runSphereAsync( }); child.once('close', (code, signal) => { clearTimeout(timer); + // If we already rejected via the overflow path, the resolve + // below is a no-op (Promise resolution is idempotent), but + // skip the buffer concat to avoid extra allocation. + if (overflowed) return; resolve({ stdout: Buffer.concat(stdoutChunks).toString('utf-8'), stderr: Buffer.concat(stderrChunks).toString('utf-8'), From 43d6d669ac4c76b622a0f02745defeeb6bc7e2f2 Mon Sep 17 00:00:00 2001 From: Vladimir Rogojin Date: Sun, 3 May 2026 23:53:56 +0200 Subject: [PATCH 6/6] review: round 4 steelman fixes (overflowed guard, timer comment, doc) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round 4 of steelman loop on PR #14. Three findings addressed: WARNING — runSphereAsync's `error` handler called `reject(err)` without checking the `overflowed` flag, while the parallel `close` handler had `if (overflowed) return;`. The asymmetry was a documented reliance on Promise idempotency (Node silently ignores double-settle) — not a runtime bug, but inconsistent. Add the same overflow guard to the error handler so both settle paths follow the identical pattern. NOTE — Timer comment said "Race-safe" but the fix only NARROWS the race between OS-level exit and Node processing SIGCHLD. Both exitCode and signalCode are null in the few-microseconds window between those events, so a timer firing in that window can still misreport a clean exit as a timeout. Updated comment to "narrows the race window" and explicitly notes the residual edge case. NOTE — `MAX_BUFFER_BYTES` doc said "10 MiB per stream" but didn't make the worst-case clear. Clarified: stdout + stderr each get their own 10 MiB cap, so worst-case memory before overflow fires is 20 MiB. Acceptable for a test helper but worth documenting. Verified: live test passes in 27s (down from 37s — testnet was faster this run, not a code change), lint+typecheck clean. --- test/e2e-live/helpers/sphere-cli.ts | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/test/e2e-live/helpers/sphere-cli.ts b/test/e2e-live/helpers/sphere-cli.ts index 9a245b0..ef34c82 100644 --- a/test/e2e-live/helpers/sphere-cli.ts +++ b/test/e2e-live/helpers/sphere-cli.ts @@ -208,7 +208,8 @@ export function runSphere( * Mirrors `runSphere`'s contract: non-zero exit codes are returned * (not thrown); only subprocess launch failures reject the promise. * - * Output bounded by `MAX_BUFFER_BYTES` (10 MiB per stream). Async + * Output bounded by `MAX_BUFFER_BYTES` PER STREAM (10 MiB stdout + + * 10 MiB stderr = 20 MiB worst case before overflow fires). Async * `child_process.spawn` has no maxBuffer (spawnSync does), so a * misbehaving subprocess flooding stdout could OOM the test process. * On overflow we kill the child and reject with a descriptive error. @@ -257,10 +258,14 @@ export function runSphereAsync( }); let timedOut = false; const timer = setTimeout(() => { - // Race-safe: if the process already exited cleanly between the - // last event-loop tick and now, exitCode is non-null. Don't - // misreport a clean exit as a timeout — that creates a - // misleading false-timeout in the caller's logs. + // Narrows the race window: if Node has already populated + // exitCode/signalCode, we know the process is dead and we + // shouldn't report a false-timeout. There is still a tiny + // window between OS-level exit and Node processing SIGCHLD + // where both fields are null — a timer firing in that window + // (microseconds wide) will still misreport a clean exit as a + // timeout. Empirically rare; the guard reduces it to a + // theoretical edge case but does not eliminate it. if (child.exitCode !== null || child.signalCode !== null) return; timedOut = true; try { child.kill('SIGKILL'); } catch { /* already dead */ } @@ -268,6 +273,12 @@ export function runSphereAsync( timer.unref(); child.once('error', (err) => { clearTimeout(timer); + // Mirror the overflow guard in 'close': if the overflow path + // already rejected, don't double-settle. Node's Promise + // resolution is idempotent so this is more about consistency + // and clarity than correctness — but it removes the + // inconsistency between the two settle paths. + if (overflowed) return; reject(err); }); child.once('close', (code, signal) => {