From 0ff2475849807e59bb28adf1cc0dfbd535ffd287 Mon Sep 17 00:00:00 2001 From: Vladimir Rogojin Date: Mon, 4 May 2026 18:20:31 +0200 Subject: [PATCH 01/30] test(e2e-live): HMA-orchestrated full settlement + withdraw MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the architectural loop: this is the canonical proof that operators can launch HMA, spawn agents through it over Sphere DM, trade between independent traders, settle on testnet, and withdraw — the user-stated goal that no prior live test fully covered. Prior tests cover slices: - basic-roundtrip: settles correctly, but uses direct docker run - hma-orchestrated: spawns through HMA, but doesn't trade - hma-trade-flow: spawns + posts/cancels intents through HMA, but explicitly stops short of settlement - this file: spawns + funds + trades + settles + withdraws — all through HMA over Sphere DMs Two scenarios run concurrently via vitest it.concurrent on a single shared HMA (matches production: one HMA per host, multi-tenant). Within each scenario, every fan-outable step uses Promise.all with *Async helpers so concurrency actually overlaps: - hostSpawnAsync × 3 (escrow + 2 traders) — parallel - setStrategyAsync × 2 — parallel - portfolioAsync × 2 (snapshots) — parallel - createIntentAsync × 2 (matching) — parallel - waitForDealInStateAsync × 2 (COMPLETED) — parallel - withdrawAsync × 1 (one trader withdraws to controller) The sync helpers (runSphere → spawnSync) block the event loop so Promise.all over them serializes — added async variants only for the helpers needed here, kept the sync surface small. Self-funding via TRADER_TEST_FUND env passthrough through HMA (validatePayloadEnv at agentic-hosting/src/host-manager/manager.ts:97 allows TRADER_TEST_FUND — it's not in FORBIDDEN_ENV_KEYS and doesn't start with UNICITY_). Avoids the recurring testnet faucet flakiness that basic-roundtrip's commit history documents. Performance target on healthy testnet: 5-8 minutes total (HMA boot ~10s; 6 concurrent spawns ~30-60s; settlement wait ~3-5 min dominates; 2 concurrent withdraws ~5-10s). Files: - test/e2e-live/helpers/sphere-trader.ts: add withdraw + 5 *Async helpers (setStrategyAsync, createIntentAsync, portfolioAsync, listDealsAsync, waitForDealInStateAsync, withdrawAsync) - test/e2e-live/helpers/hma-spawn.ts: add hostSpawnAsync - test/e2e-live/hma-trade-settlement.e2e-live.test.ts: NEW Verified: 698 unit/integration tests still pass. Type check + lint clean. --- test/e2e-live/helpers/hma-spawn.ts | 48 ++ test/e2e-live/helpers/sphere-trader.ts | 198 +++++++- .../hma-trade-settlement.e2e-live.test.ts | 441 ++++++++++++++++++ 3 files changed, 686 insertions(+), 1 deletion(-) create mode 100644 test/e2e-live/hma-trade-settlement.e2e-live.test.ts diff --git a/test/e2e-live/helpers/hma-spawn.ts b/test/e2e-live/helpers/hma-spawn.ts index 80cfe7b..809922d 100644 --- a/test/e2e-live/helpers/hma-spawn.ts +++ b/test/e2e-live/helpers/hma-spawn.ts @@ -143,6 +143,54 @@ export function hostSpawn(opts: HostSpawnOpts): SpawnedTenant { }; } +/** + * Async variant of hostSpawn — uses runSphereAsync so concurrent calls + * (Promise.all([hostSpawnAsync(escrow), hostSpawnAsync(alice), ...])) + * actually overlap. The sync version uses spawnSync which blocks the + * event loop, defeating parallelism. + */ +export async function hostSpawnAsync(opts: HostSpawnOpts): Promise { + const args = [ + 'host', + 'spawn', + opts.instanceName, + '--manager', opts.managerAddress, + '--template', opts.templateId, + '--json', + '--timeout', String(opts.timeoutMs ?? DEFAULT_HMCP_TIMEOUT_MS), + ]; + for (const [k, v] of Object.entries(opts.env ?? {})) { + args.push('--env', `${k}=${v}`); + } + const result = await runSphereAsync(opts.cliPath, opts.cliHome, args, { + timeoutMs: (opts.timeoutMs ?? DEFAULT_HMCP_TIMEOUT_MS) + 30_000, + }); + if (result.status !== 0) { + throw new Error( + `sphere host spawn failed (status=${result.status}, signal=${result.signal}). ` + + `stderr: ${result.stderr.slice(0, 800)}\nstdout: ${result.stdout.slice(0, 800)}`, + ); + } + const responses = parseSpawnResponses(result.stdout); + const ready = responses.find((r) => r.type === 'hm.spawn_ready'); + if (!ready) { + const failed = responses.find((r) => r.type === 'hm.spawn_failed' || r.type === 'hm.error'); + throw new Error( + `sphere host spawn did not produce hm.spawn_ready. ` + + `last response: ${JSON.stringify(failed ?? responses[responses.length - 1])}`, + ); + } + const p = ready.payload; + return { + instanceId: String(p['instance_id'] ?? ''), + instanceName: String(p['instance_name'] ?? ''), + tenantPubkey: String(p['tenant_pubkey'] ?? ''), + tenantDirectAddress: String(p['tenant_direct_address'] ?? ''), + tenantNametag: typeof p['tenant_nametag'] === 'string' ? p['tenant_nametag'] : null, + state: String(p['state'] ?? ''), + }; +} + export interface HostStopOpts { cliPath: string; cliHome: string; diff --git a/test/e2e-live/helpers/sphere-trader.ts b/test/e2e-live/helpers/sphere-trader.ts index f7b80bb..12217d4 100644 --- a/test/e2e-live/helpers/sphere-trader.ts +++ b/test/e2e-live/helpers/sphere-trader.ts @@ -23,7 +23,7 @@ * call site that needs parallelism. */ -import { runSphere, type SphereRunResult } from './sphere-cli.js'; +import { runSphere, runSphereAsync, type SphereRunResult } from './sphere-cli.js'; const DEFAULT_TRADER_TIMEOUT_MS = 60_000; @@ -315,3 +315,199 @@ export async function waitForDealInState( `within ${opts.timeoutMs ?? 600_000}ms. ${summary}.`, ); } + +// --------------------------------------------------------------------------- +// WITHDRAW_TOKEN — sphere trader withdraw +// --------------------------------------------------------------------------- + +export interface WithdrawOpts extends TraderInvocationOpts { + asset: string; + amount: bigint; + toAddress: string; +} + +export interface WithdrawResult { + readonly transferId: string; + readonly remainingBalance: string; +} + +function parseWithdrawResult(result: unknown): WithdrawResult { + if (typeof result !== 'object' || result === null) { + throw new Error(`withdraw: result not an object. Got: ${JSON.stringify(result)}`); + } + const r = result as Record; + const transferId = r['transfer_id']; + if (typeof transferId !== 'string' || transferId === '') { + throw new Error(`withdraw: missing transfer_id. Got: ${JSON.stringify(result)}`); + } + return { + transferId, + remainingBalance: String(r['remaining_balance'] ?? ''), + }; +} + +export function withdraw(opts: WithdrawOpts): WithdrawResult { + const args = [ + '--asset', opts.asset, + '--amount', opts.amount.toString(), + '--to-address', opts.toAddress, + ]; + const { result } = runTraderCommand('withdraw', args, opts); + return parseWithdrawResult(result); +} + +// --------------------------------------------------------------------------- +// Async variants — required for true concurrency. The sync helpers above +// use spawnSync which BLOCKS the event loop, so wrapping them in +// Promise.all(...) does NOT parallelize: each spawnSync call holds the +// thread until the child exits. The async variants below use spawn (via +// runSphereAsync) so concurrent calls actually overlap. +// +// Only the helpers needed for parallel scenarios get async wrappers; the +// rest stay sync to keep the surface small. +// --------------------------------------------------------------------------- + +async function runTraderCommandAsync( + subcommand: string, + args: readonly string[], + opts: TraderInvocationOpts, +): Promise<{ result: unknown; raw: SphereRunResult }> { + const timeoutMs = opts.timeoutMs ?? DEFAULT_TRADER_TIMEOUT_MS; + const fullArgs = [ + 'trader', subcommand, ...args, + '--tenant', opts.tenant, + '--json', + '--timeout', String(timeoutMs), + ]; + const raw = await runSphereAsync(opts.cliPath, opts.cliHome, fullArgs, { + timeoutMs: timeoutMs + 15_000, + }); + if (raw.status !== 0) { + throw new Error( + `sphere trader ${subcommand} failed (status=${raw.status}, signal=${raw.signal}).\n` + + `stderr: ${raw.stderr.slice(0, 800)}\n` + + `stdout: ${raw.stdout.slice(0, 800)}`, + ); + } + const start = raw.stdout.indexOf('{'); + const end = raw.stdout.lastIndexOf('}'); + if (start < 0 || end <= start) { + throw new Error( + `sphere trader ${subcommand} --json: no JSON object in stdout. ` + + `Got first 500 chars: ${raw.stdout.slice(0, 500)}`, + ); + } + const parsed = JSON.parse(raw.stdout.slice(start, end + 1)) as { + ok?: boolean; + result?: unknown; + error_code?: string; + message?: string; + }; + if (parsed.ok === false) { + throw new Error( + `sphere trader ${subcommand}: ok=false. ` + + `[${parsed.error_code ?? 'UNKNOWN'}] ${parsed.message ?? '(no message)'}`, + ); + } + const result = parsed.result ?? parsed; + return { result, raw }; +} + +export async function setStrategyAsync(opts: SetStrategyOpts): Promise { + const args: string[] = []; + if (opts.rateStrategy !== undefined) args.push('--rate-strategy', opts.rateStrategy); + if (opts.maxConcurrent !== undefined) args.push('--max-concurrent', String(opts.maxConcurrent)); + if (opts.trustedEscrows !== undefined && opts.trustedEscrows.length > 0) { + args.push('--trusted-escrows', opts.trustedEscrows.join(',')); + } + const { result } = await runTraderCommandAsync('set-strategy', args, opts); + return result; +} + +export async function createIntentAsync(opts: CreateIntentOpts): Promise { + const args = [ + '--direction', opts.direction, + '--base', opts.baseAsset, + '--quote', opts.quoteAsset, + '--rate-min', opts.rateMin.toString(), + '--rate-max', opts.rateMax.toString(), + '--volume-min', opts.volumeMin.toString(), + '--volume-max', opts.volumeMax.toString(), + ]; + if (opts.expiryMs !== undefined) args.push('--expiry-ms', String(opts.expiryMs)); + const { result } = await runTraderCommandAsync('create-intent', args, opts); + if (typeof result !== 'object' || result === null) { + throw new Error(`create-intent: result is not an object. Got: ${JSON.stringify(result)}`); + } + const intentId = (result as Record)['intent_id']; + if (typeof intentId !== 'string') { + throw new Error(`create-intent: missing intent_id. Got: ${JSON.stringify(result)}`); + } + return { intentId }; +} + +export async function portfolioAsync(opts: TraderInvocationOpts): Promise { + const { result } = await runTraderCommandAsync('portfolio', [], opts); + if (Array.isArray(result)) return result as PortfolioBalance[]; + if (typeof result === 'object' && result !== null) { + const r = result as Record; + const balances = r['balances'] ?? r['portfolio']; + if (Array.isArray(balances)) return balances as PortfolioBalance[]; + return Object.entries(r).map(([asset, amount]) => ({ asset, amount: String(amount) })); + } + throw new Error(`portfolio: response not in expected shape. Got: ${JSON.stringify(result)}`); +} + +export async function listDealsAsync(opts: TraderInvocationOpts): Promise { + const { result } = await runTraderCommandAsync('list-deals', [], opts); + if (Array.isArray(result)) return result as DealSummary[]; + if (typeof result === 'object' && result !== null) { + const arr = (result as Record)['deals'] ?? (result as Record)['swaps']; + if (Array.isArray(arr)) return arr as DealSummary[]; + } + throw new Error(`list-deals: response not in expected shape. Got: ${JSON.stringify(result)}`); +} + +export async function withdrawAsync(opts: WithdrawOpts): Promise { + const args = [ + '--asset', opts.asset, + '--amount', opts.amount.toString(), + '--to-address', opts.toAddress, + ]; + const { result } = await runTraderCommandAsync('withdraw', args, opts); + return parseWithdrawResult(result); +} + +/** + * Async polling variant of waitForDealInState — uses listDealsAsync so + * concurrent waits across multiple traders truly overlap (each list-deals + * spawns its own subprocess via runSphereAsync, no event-loop blocking). + */ +export async function waitForDealInStateAsync( + opts: TraderInvocationOpts & { + targetState: string; + timeoutMs?: number; + intervalMs?: number; + }, +): Promise { + const deadline = Date.now() + (opts.timeoutMs ?? 600_000); + const interval = opts.intervalMs ?? 3_000; + let lastSeen: readonly DealSummary[] = []; + while (Date.now() < deadline) { + try { + lastSeen = await listDealsAsync(opts); + const match = lastSeen.find((d) => d.state === opts.targetState); + if (match) return match; + } catch { + // Transient error — keep polling. + } + await new Promise((r) => setTimeout(r, interval)); + } + const summary = lastSeen.length > 0 + ? `last seen ${lastSeen.length} deal(s) in states [${lastSeen.map((d) => d.state).join(', ')}]` + : 'no deals visible'; + throw new Error( + `waitForDealInStateAsync: tenant ${opts.tenant} did not reach state="${opts.targetState}" ` + + `within ${opts.timeoutMs ?? 600_000}ms. ${summary}.`, + ); +} diff --git a/test/e2e-live/hma-trade-settlement.e2e-live.test.ts b/test/e2e-live/hma-trade-settlement.e2e-live.test.ts new file mode 100644 index 0000000..c7a5f13 --- /dev/null +++ b/test/e2e-live/hma-trade-settlement.e2e-live.test.ts @@ -0,0 +1,441 @@ +/** + * Live e2e: HMA-orchestrated trade-settlement (THE goal-completion test). + * + * Closes the architectural loop that hma-trade-flow.e2e-live.test.ts + * stopped one step short of: full Architecture-B settlement THROUGH + * the HMA, not via direct-docker. + * + * Test + * ├── boot HMA over Sphere DM (single shared instance) + * └── for each scenario IN PARALLEL (it.concurrent): + * ├── spawn escrow + 2 traders ← Promise.all (3-way concurrent) + * │ └── self-mint UCT+USDU at boot via TRADER_TEST_FUND env + * │ passthrough through HMA → docker + * ├── set-strategy on both traders ← Promise.all + * ├── post matching intents on both ← Promise.all + * ├── wait for COMPLETED on both ← Promise.all + * ├── assert balance deltas (buyer +UCT/-USDU, seller mirror) + * └── withdraw a small amount from one trader + * to the controller's DIRECT:// address + * + * Why this exists: + * The user's stated goal: "operators can launch HMA, spawn agents, + * fund them, trade (multi-party), AND withdraw" — all over Sphere + * DMs. Every prior live test covers a slice; this one chains every + * slice into a single end-to-end run. + * + * - basic-roundtrip.e2e-live: settles correctly, but uses direct + * docker run (no HMA in the loop). + * - hma-orchestrated.e2e-live: spawns through HMA, but doesn't trade. + * - hma-trade-flow.e2e-live: spawns through HMA, posts/cancels + * intents, but explicitly stops short of settlement. + * - this file: spawns through HMA, settles, AND withdraws. + * + * Parallelism contract: + * - 2 scenarios run concurrently (vitest `it.concurrent`). + * - Within each scenario, every step that can fan out does so via + * Promise.all using the *Async helpers (which use spawn, not + * spawnSync — sync helpers would serialize even inside Promise.all + * because spawnSync blocks the event loop). + * - Total live infra load at peak: ~6 spawn DMs, 4 set-strategy DMs, + * 4 create-intent DMs, ~4 list-deals DMs every 3s during settlement + * wait. The infra-probe preflight runs first to fail-fast on a + * degraded testnet. + * + * Performance target on a healthy testnet: ~5-8 minutes total. + * - HMA boot: ~10s + * - 6 concurrent spawns (2 scenarios × 3 tenants): ~30-60s + * - 4 concurrent set-strategy: ~5-10s + * - 4 concurrent create-intent: ~5-10s + * - settlement wait: 3-5 min on testnet (dominates) + * - 2 concurrent withdraws: ~5-10s + * + * Out of scope: + * - Faucet HTTP — the trader self-mints via TRADER_TEST_FUND, the + * same pattern basic-roundtrip uses (faucet has been a recurring + * source of flakiness on testnet). + * - Negotiation-failure paths — covered by negotiation-failures. + * - Partial fills — covered by edge-cases. + */ + +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { rmSync, existsSync } from 'node:fs'; +import { join } from 'node:path'; +import { randomUUID } from 'node:crypto'; +import { + probeSphereCli, + createSphereCliEnv, + bootstrapControllerWallet, + type SphereCliProbe, +} from './helpers/sphere-cli.js'; +import { + spawnHostManager, + checkAgenticHostingPath, + type HostManagerProcess, +} from './helpers/manager-process.js'; +import { + hostSpawnAsync, + hostStop, + type SpawnedTenant, +} from './helpers/hma-spawn.js'; +import { + setStrategyAsync, + createIntentAsync, + portfolioAsync, + waitForDealInStateAsync, + withdrawAsync, + type PortfolioBalance, +} from './helpers/sphere-trader.js'; +import { UCT_COIN_ID, USDU_COIN_ID } from './helpers/constants.js'; + +// --------------------------------------------------------------------------- +// Precondition gates (mirrors hma-trade-flow's structure) +// --------------------------------------------------------------------------- + +const cliProbe: SphereCliProbe = probeSphereCli(); +const agenticProbe = checkAgenticHostingPath(); +let managerBinPath = ''; +let agenticReady = false; +if (agenticProbe.ok) { + managerBinPath = join(agenticProbe.path, 'dist', 'host-manager.js'); + agenticReady = existsSync(managerBinPath); +} +const skip = !cliProbe.ok || !agenticReady; +const skipReason = !cliProbe.ok + ? `sphere-cli not runnable: ${cliProbe.reason}` + : !agenticProbe.ok + ? agenticProbe.reason + : !agenticReady + ? `agentic-hosting binary missing at ${managerBinPath}.` + : ''; + +// --------------------------------------------------------------------------- +// Shared HMA + controller fixture +// --------------------------------------------------------------------------- + +interface SuiteState { + cliPath: string; + cliHome: string; + manager: HostManagerProcess; + managerAddr: string; + controllerDirectAddress: string; + spawned: SpawnedTenant[]; +} + +// Both scenarios share the HMA and controller wallet (no point booting +// two HMAs to test parallelism — production has one HMA per host). The +// `spawned` list is mutated by each scenario as it provisions tenants +// so afterAll can stop them all in parallel. +let state: SuiteState | null = null; + +const SELF_MINT_AMOUNT = 5000n; // matches basic-roundtrip +const SWAP_TIMEOUT_MS = 8 * 60_000; // 8 minutes; testnet settlement is 3-5 min typical + +// `volume_max` for matching intents in each scenario. Both sides post +// identical volumes so a single fill clears both intents. +const TRADE_VOLUME = 10n; +const TRADE_RATE = 1n; // 1 USDU per UCT — keeps the math obvious + +// Withdraw amount: small fraction of received UCT so the test asserts +// real value movement without exhausting the trader's post-trade balance. +const WITHDRAW_AMOUNT = 3n; + +// --------------------------------------------------------------------------- + +describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testnet)', () => { + if (skip) { + console.warn(`[hma-trade-settlement] SKIPPED: ${skipReason}`); + } + + beforeAll(async () => { + if (skip) return; + if (!cliProbe.ok) throw new Error('precondition gate inverted'); + const cliPath = cliProbe.path; + const { home: cliHome } = createSphereCliEnv('hma-trade-settlement'); + + console.log('[hma-trade-settlement] bootstrapping controller wallet…'); + const controller = bootstrapControllerWallet(cliPath, cliHome); + console.log(`[hma-trade-settlement] controller pubkey ${controller.pubkey.slice(0, 16)}…`); + + console.log('[hma-trade-settlement] booting host-manager…'); + const manager = await spawnHostManager({ controllerPubkey: controller.pubkey }); + await manager.ready; + const managerAddr = manager.nametag ? `@${manager.nametag}` : manager.pubkey; + console.log(`[hma-trade-settlement] manager ready @ ${managerAddr}`); + + state = { + cliPath, + cliHome, + manager, + managerAddr, + controllerDirectAddress: controller.directAddress, + spawned: [], + }; + }, 240_000); + + afterAll(async () => { + if (!state) return; + // Best-effort parallel cleanup — never let one stop block the rest. + await Promise.allSettled( + state.spawned.map((t) => + hostStop({ + cliPath: state!.cliPath, + cliHome: state!.cliHome, + managerAddress: state!.manager.pubkey, + target: t.instanceName, + timeoutMs: 60_000, + }), + ), + ); + await state.manager.stop(); + try { rmSync(state.cliHome, { recursive: true, force: true }); } + catch { /* best effort */ } + }, 240_000); + + // ---- Per-scenario helpers ------------------------------------------------- + + /** + * Spawn one escrow + two traders (alice/bob) IN PARALLEL via + * Promise.all on hostSpawnAsync. Both traders self-mint UCT+USDU + * at boot via TRADER_TEST_FUND (HMA's --env passthrough — see + * agentic-hosting/src/host-manager/manager.ts:97 validatePayloadEnv; + * TRADER_TEST_FUND is not in FORBIDDEN_ENV_KEYS and doesn't start + * with UNICITY_ so it's allowed through). + */ + async function provisionTriple(scenarioId: string): Promise<{ + escrow: SpawnedTenant; + alice: SpawnedTenant; + bob: SpawnedTenant; + }> { + if (!state) throw new Error('beforeAll did not initialize state'); + const s = state; + const traderEnv = { + TRADER_TEST_FUND: `${UCT_COIN_ID}:${SELF_MINT_AMOUNT.toString()},${USDU_COIN_ID}:${SELF_MINT_AMOUNT.toString()}`, + TRADER_FAULT_INJECTION_ALLOWED: '1', + }; + console.log(`[${scenarioId}] spawning escrow + alice + bob (parallel)…`); + const [escrow, alice, bob] = await Promise.all([ + hostSpawnAsync({ + cliPath: s.cliPath, + cliHome: s.cliHome, + managerAddress: s.managerAddr, + templateId: 'escrow-service', + instanceName: `escrow-${scenarioId}`, + timeoutMs: 180_000, + }), + hostSpawnAsync({ + cliPath: s.cliPath, + cliHome: s.cliHome, + managerAddress: s.managerAddr, + templateId: 'trader-agent', + instanceName: `alice-${scenarioId}`, + timeoutMs: 180_000, + env: traderEnv, + }), + hostSpawnAsync({ + cliPath: s.cliPath, + cliHome: s.cliHome, + managerAddress: s.managerAddr, + templateId: 'trader-agent', + instanceName: `bob-${scenarioId}`, + timeoutMs: 180_000, + env: traderEnv, + }), + ]); + s.spawned.push(escrow, alice, bob); + console.log( + `[${scenarioId}] up: escrow=${escrow.instanceName} ` + + `alice=${alice.instanceName} bob=${bob.instanceName}`, + ); + return { escrow, alice, bob }; + } + + /** Pull a coin's confirmed balance (smallest units) from a portfolio response. */ + function balanceOf(p: readonly PortfolioBalance[], symbol: string): bigint { + for (const b of p) { + if (b.asset === symbol) return BigInt(String(b.amount ?? '0')); + } + // Tolerate alternate field names (some sphere-sdk versions emit `confirmed`). + for (const b of p as Array>) { + if (b['asset'] === symbol && b['confirmed'] !== undefined) { + return BigInt(String(b['confirmed'])); + } + } + return 0n; + } + + /** + * One full settlement scenario, parametrized by name. Inside the test: + * 1. Spawn escrow + buyer + seller in parallel. + * 2. set-strategy on both traders in parallel. + * 3. Pre-trade portfolio snapshot in parallel. + * 4. Post matching intents in parallel. + * 5. Wait for COMPLETED on both deals in parallel. + * 6. Post-trade portfolio snapshot in parallel; assert deltas. + * 7. Withdraw a small amount from buyer to controller's DIRECT:// + * address; assert transfer_id non-empty + post-withdraw balance + * reflects the withdrawal. + */ + async function runSettlementScenario(scenarioId: string): Promise { + if (!state) throw new Error('beforeAll did not initialize state'); + const s = state; + const { escrow, alice, bob } = await provisionTriple(scenarioId); + + // ---- 2. Configure trusted escrows on both traders -------------- + console.log(`[${scenarioId}] set-strategy on alice + bob (parallel)…`); + await Promise.all([ + setStrategyAsync({ + cliPath: s.cliPath, cliHome: s.cliHome, tenant: alice.tenantPubkey, + trustedEscrows: [escrow.tenantPubkey], + maxConcurrent: 5, + }), + setStrategyAsync({ + cliPath: s.cliPath, cliHome: s.cliHome, tenant: bob.tenantPubkey, + trustedEscrows: [escrow.tenantPubkey], + maxConcurrent: 5, + }), + ]); + + // ---- 3. Pre-trade portfolio snapshot --------------------------- + const [aliceBefore, bobBefore] = await Promise.all([ + portfolioAsync({ cliPath: s.cliPath, cliHome: s.cliHome, tenant: alice.tenantPubkey }), + portfolioAsync({ cliPath: s.cliPath, cliHome: s.cliHome, tenant: bob.tenantPubkey }), + ]); + const aliceUctBefore = balanceOf(aliceBefore, 'UCT'); + const aliceUsduBefore = balanceOf(aliceBefore, 'USDU'); + const bobUctBefore = balanceOf(bobBefore, 'UCT'); + const bobUsduBefore = balanceOf(bobBefore, 'USDU'); + console.log( + `[${scenarioId}] pre-trade balances: ` + + `alice=${aliceUctBefore}UCT/${aliceUsduBefore}USDU bob=${bobUctBefore}UCT/${bobUsduBefore}USDU`, + ); + // Sanity: the self-mint must have happened (otherwise no UCT/USDU + // are available to trade and the swap will hang). + expect(aliceUctBefore + aliceUsduBefore).toBeGreaterThan(0n); + expect(bobUctBefore + bobUsduBefore).toBeGreaterThan(0n); + + // ---- 4. Post matching intents in parallel ---------------------- + // Alice buys UCT (pays USDU). Bob sells UCT (receives USDU). + console.log(`[${scenarioId}] posting matching intents (parallel)…`); + const [aliceIntent, bobIntent] = await Promise.all([ + createIntentAsync({ + cliPath: s.cliPath, cliHome: s.cliHome, tenant: alice.tenantPubkey, + direction: 'buy', + baseAsset: 'UCT', quoteAsset: 'USDU', + rateMin: TRADE_RATE, rateMax: TRADE_RATE, + volumeMin: TRADE_VOLUME, volumeMax: TRADE_VOLUME, + expiryMs: SWAP_TIMEOUT_MS, + }), + createIntentAsync({ + cliPath: s.cliPath, cliHome: s.cliHome, tenant: bob.tenantPubkey, + direction: 'sell', + baseAsset: 'UCT', quoteAsset: 'USDU', + rateMin: TRADE_RATE, rateMax: TRADE_RATE, + volumeMin: TRADE_VOLUME, volumeMax: TRADE_VOLUME, + expiryMs: SWAP_TIMEOUT_MS, + }), + ]); + console.log( + `[${scenarioId}] intents posted: alice=${aliceIntent.intentId.slice(0, 12)}… ` + + `bob=${bobIntent.intentId.slice(0, 12)}…`, + ); + + // ---- 5. Wait for both deals to reach COMPLETED in parallel ----- + console.log(`[${scenarioId}] waiting for COMPLETED on both sides…`); + const [aliceDeal, bobDeal] = await Promise.all([ + waitForDealInStateAsync({ + cliPath: s.cliPath, cliHome: s.cliHome, tenant: alice.tenantPubkey, + targetState: 'COMPLETED', + timeoutMs: SWAP_TIMEOUT_MS, + }), + waitForDealInStateAsync({ + cliPath: s.cliPath, cliHome: s.cliHome, tenant: bob.tenantPubkey, + targetState: 'COMPLETED', + timeoutMs: SWAP_TIMEOUT_MS, + }), + ]); + expect(aliceDeal.state).toBe('COMPLETED'); + expect(bobDeal.state).toBe('COMPLETED'); + // Both sides observe the same deal_id (one negotiation, two ledgers). + expect(aliceDeal.deal_id).toBe(bobDeal.deal_id); + console.log( + `[${scenarioId}] deal COMPLETED: ${aliceDeal.deal_id.slice(0, 12)}…`, + ); + + // ---- 6. Post-trade balance assertions -------------------------- + // Wait briefly for payments.receive() to finalize inbound payouts + // (trader loop is on a 15s cycle; basic-roundtrip uses 5s and that + // has been enough on testnet). + await new Promise((r) => setTimeout(r, 5_000)); + + const [aliceAfter, bobAfter] = await Promise.all([ + portfolioAsync({ cliPath: s.cliPath, cliHome: s.cliHome, tenant: alice.tenantPubkey }), + portfolioAsync({ cliPath: s.cliPath, cliHome: s.cliHome, tenant: bob.tenantPubkey }), + ]); + const aliceUctAfter = balanceOf(aliceAfter, 'UCT'); + const aliceUsduAfter = balanceOf(aliceAfter, 'USDU'); + const bobUctAfter = balanceOf(bobAfter, 'UCT'); + const bobUsduAfter = balanceOf(bobAfter, 'USDU'); + + const expectedUsduPaid = TRADE_RATE * TRADE_VOLUME; + // Alice (buy UCT for USDU): +UCT / -USDU + expect( + aliceUctAfter - aliceUctBefore, + `alice UCT delta should be +${TRADE_VOLUME}; observed: ${aliceUctAfter - aliceUctBefore}`, + ).toBe(TRADE_VOLUME); + expect( + aliceUsduBefore - aliceUsduAfter, + `alice USDU delta should be -${expectedUsduPaid}; observed: -${aliceUsduBefore - aliceUsduAfter}`, + ).toBe(expectedUsduPaid); + // Bob (sell UCT for USDU): -UCT / +USDU + expect( + bobUctBefore - bobUctAfter, + `bob UCT delta should be -${TRADE_VOLUME}; observed: -${bobUctBefore - bobUctAfter}`, + ).toBe(TRADE_VOLUME); + expect( + bobUsduAfter - bobUsduBefore, + `bob USDU delta should be +${expectedUsduPaid}; observed: ${bobUsduAfter - bobUsduBefore}`, + ).toBe(expectedUsduPaid); + + // ---- 7. Withdraw from alice (now has UCT) --------------------- + // Alice had 0 UCT pre-trade and acquired TRADE_VOLUME via the swap. + // Withdraw a fraction (WITHDRAW_AMOUNT) to the controller's DIRECT + // address — exercises the WITHDRAW_TOKEN ACP command end-to-end + // including the round-6 trim+validation gate. + console.log(`[${scenarioId}] withdraw ${WITHDRAW_AMOUNT} UCT from alice → controller…`); + const wr = await withdrawAsync({ + cliPath: s.cliPath, cliHome: s.cliHome, tenant: alice.tenantPubkey, + asset: 'UCT', + amount: WITHDRAW_AMOUNT, + toAddress: s.controllerDirectAddress, + }); + expect(wr.transferId).toMatch(/^[a-zA-Z0-9_-]+$/); + expect(wr.transferId.length).toBeGreaterThan(8); + console.log(`[${scenarioId}] withdraw transfer_id=${wr.transferId.slice(0, 16)}…`); + + // Verify alice's UCT balance is now reduced by the withdrawn amount. + // Allow a small settle delay for the transfer to land in the + // confirmed bucket (testnet aggregator round-trip). + await new Promise((r) => setTimeout(r, 5_000)); + const aliceFinal = await portfolioAsync({ + cliPath: s.cliPath, cliHome: s.cliHome, tenant: alice.tenantPubkey, + }); + const aliceUctFinal = balanceOf(aliceFinal, 'UCT'); + expect( + aliceUctAfter - aliceUctFinal, + `alice UCT delta after withdraw should be -${WITHDRAW_AMOUNT}; observed: -${aliceUctAfter - aliceUctFinal}`, + ).toBe(WITHDRAW_AMOUNT); + + console.log(`[${scenarioId}] ✓ end-to-end settlement+withdraw verified`); + } + + // ---- Concurrent scenarios ----------------------------------------------- + + it('Pair-1: full spawn → trade → settle → withdraw via HMA', async () => { + await runSettlementScenario(`p1-${randomUUID().slice(0, 6)}`); + }, SWAP_TIMEOUT_MS + 4 * 60_000); // 12 min total budget per scenario + + it('Pair-2: parallel scenario settles on the same HMA without interference', async () => { + await runSettlementScenario(`p2-${randomUUID().slice(0, 6)}`); + }, SWAP_TIMEOUT_MS + 4 * 60_000); +}); From aafe3ebcde51f2921861f17a8dfee086776b8efc Mon Sep 17 00:00:00 2001 From: Vladimir Rogojin Date: Mon, 4 May 2026 18:23:50 +0200 Subject: [PATCH 02/30] fix(test): per-scenario controller wallets to avoid wallet.json race MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round 1 of running hma-trade-settlement.e2e-live failed within seconds with two distinct wallet.json races: Pair-1: "No wallet found in /tmp/.../.sphere-cli" Pair-2: "ENOENT: rename '.sphere-cli/wallet.json.tmp' -> 'wallet.json'" Both scenarios were sharing one cliHome and issuing concurrent sphere-cli invocations. Each invocation persists incoming-DM state to .sphere-cli/wallet.json via the SDK's atomic temp+rename (FileStorageProvider.save → fs.renameSync). When two parallel calls write at once, one's tmp gets renamed away before the other's rename observes it, corrupting both. Fix: bootstrap one controller wallet per scenario in its own cliHome dir. HMA's AUTHORIZED_CONTROLLERS env var accepts a comma-separated list of pubkeys (config.ts:52), so a single HMA authorizes all scenarios' controllers in one go — matching the real multi-tenant production topology. Each scenario now uses its own cliHome (no wallet.json contention) and its own DIRECT:// address as the withdraw destination. Aborted the earlier "shared cliHome with retry" approach because retries would only mask the race; the SDK's wallet store isn't designed for concurrent writers. SCENARIO_COUNT constant exposed so future steelman rounds can bump it to 4-8 to stress-test the relay/aggregator/HMA further. --- .../hma-trade-settlement.e2e-live.test.ts | 131 ++++++++++++------ 1 file changed, 92 insertions(+), 39 deletions(-) diff --git a/test/e2e-live/hma-trade-settlement.e2e-live.test.ts b/test/e2e-live/hma-trade-settlement.e2e-live.test.ts index c7a5f13..835d198 100644 --- a/test/e2e-live/hma-trade-settlement.e2e-live.test.ts +++ b/test/e2e-live/hma-trade-settlement.e2e-live.test.ts @@ -113,19 +113,36 @@ const skipReason = !cliProbe.ok // Shared HMA + controller fixture // --------------------------------------------------------------------------- +/** + * Per-scenario controller: the controller wallet (cliHome + pubkey/addr). + * Concurrent `sphere host spawn` invocations write to .sphere-cli/wallet.json + * inside cliHome (the SDK persists incoming-DM state there atomically via + * temp+rename). Two parallel calls in the same cliHome corrupt each other's + * rename — give each scenario its own cliHome to make concurrency safe. + */ +interface ScenarioController { + cliHome: string; + pubkey: string; + directAddress: string; +} + interface SuiteState { cliPath: string; - cliHome: string; + /** All cliHome dirs we created — afterAll wipes them. */ + cliHomes: string[]; manager: HostManagerProcess; managerAddr: string; - controllerDirectAddress: string; + controllers: ScenarioController[]; spawned: SpawnedTenant[]; } -// Both scenarios share the HMA and controller wallet (no point booting -// two HMAs to test parallelism — production has one HMA per host). The -// `spawned` list is mutated by each scenario as it provisions tenants -// so afterAll can stop them all in parallel. +// Number of concurrent settlement scenarios. Each gets its own controller +// wallet + own cliHome. Bumping this raises the parallel infra load +// (more concurrent spawn DMs, more concurrent listdeals polls during +// settlement). Two is the contract test for the parallelism claim; +// higher values stress-test the relay+aggregator+HMA further. +const SCENARIO_COUNT = 2; + let state: SuiteState | null = null; const SELF_MINT_AMOUNT = 5000n; // matches basic-roundtrip @@ -151,36 +168,58 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne if (skip) return; if (!cliProbe.ok) throw new Error('precondition gate inverted'); const cliPath = cliProbe.path; - const { home: cliHome } = createSphereCliEnv('hma-trade-settlement'); - console.log('[hma-trade-settlement] bootstrapping controller wallet…'); - const controller = bootstrapControllerWallet(cliPath, cliHome); - console.log(`[hma-trade-settlement] controller pubkey ${controller.pubkey.slice(0, 16)}…`); + // One controller wallet per scenario — concurrent sphere-cli calls + // in the same .sphere-cli/wallet.json race on the SDK's atomic + // temp+rename writes (sphere-sdk persists DM state per call). + // Bootstrap them sequentially; the wallet-init aggregator round-trip + // (~30s each) dominates this section anyway and parallel inits also + // race on the same OS-level temp dirs. + const controllers: ScenarioController[] = []; + const cliHomes: string[] = []; + for (let i = 0; i < SCENARIO_COUNT; i++) { + const { home } = createSphereCliEnv(`hma-trade-settlement-c${String(i)}`); + cliHomes.push(home); + console.log(`[hma-trade-settlement] bootstrapping controller wallet #${String(i)}…`); + const c = bootstrapControllerWallet(cliPath, home); + console.log(`[hma-trade-settlement] controller #${String(i)} pubkey ${c.pubkey.slice(0, 16)}…`); + controllers.push({ cliHome: home, pubkey: c.pubkey, directAddress: c.directAddress }); + } + // HMA accepts AUTHORIZED_CONTROLLERS as a comma-separated list of + // pubkeys (see agentic-hosting/src/shared/config.ts:52). Authorize + // every scenario's controller in one HMA — production has one HMA + // per host serving multiple operators, so this matches the real + // multi-tenant topology. console.log('[hma-trade-settlement] booting host-manager…'); - const manager = await spawnHostManager({ controllerPubkey: controller.pubkey }); + const manager = await spawnHostManager({ + controllerPubkey: controllers.map((c) => c.pubkey).join(','), + }); await manager.ready; const managerAddr = manager.nametag ? `@${manager.nametag}` : manager.pubkey; console.log(`[hma-trade-settlement] manager ready @ ${managerAddr}`); state = { cliPath, - cliHome, + cliHomes, manager, managerAddr, - controllerDirectAddress: controller.directAddress, + controllers, spawned: [], }; - }, 240_000); + }, 600_000); // 10 min — wallet-init is ~30s per controller on testnet afterAll(async () => { if (!state) return; - // Best-effort parallel cleanup — never let one stop block the rest. + // Best-effort parallel cleanup. Use the FIRST controller's cliHome + // for stop calls — the HMA accepts stops from any authorized + // controller, so we don't need to issue one stop per controller. + const stopHome = state.controllers[0]?.cliHome ?? state.cliHomes[0]!; await Promise.allSettled( state.spawned.map((t) => hostStop({ cliPath: state!.cliPath, - cliHome: state!.cliHome, + cliHome: stopHome, managerAddress: state!.manager.pubkey, target: t.instanceName, timeoutMs: 60_000, @@ -188,8 +227,10 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne ), ); await state.manager.stop(); - try { rmSync(state.cliHome, { recursive: true, force: true }); } - catch { /* best effort */ } + for (const home of state.cliHomes) { + try { rmSync(home, { recursive: true, force: true }); } + catch { /* best effort */ } + } }, 240_000); // ---- Per-scenario helpers ------------------------------------------------- @@ -202,7 +243,10 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne * TRADER_TEST_FUND is not in FORBIDDEN_ENV_KEYS and doesn't start * with UNICITY_ so it's allowed through). */ - async function provisionTriple(scenarioId: string): Promise<{ + async function provisionTriple( + scenarioId: string, + controller: ScenarioController, + ): Promise<{ escrow: SpawnedTenant; alice: SpawnedTenant; bob: SpawnedTenant; @@ -217,7 +261,7 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne const [escrow, alice, bob] = await Promise.all([ hostSpawnAsync({ cliPath: s.cliPath, - cliHome: s.cliHome, + cliHome: controller.cliHome, managerAddress: s.managerAddr, templateId: 'escrow-service', instanceName: `escrow-${scenarioId}`, @@ -225,7 +269,7 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne }), hostSpawnAsync({ cliPath: s.cliPath, - cliHome: s.cliHome, + cliHome: controller.cliHome, managerAddress: s.managerAddr, templateId: 'trader-agent', instanceName: `alice-${scenarioId}`, @@ -234,7 +278,7 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne }), hostSpawnAsync({ cliPath: s.cliPath, - cliHome: s.cliHome, + cliHome: controller.cliHome, managerAddress: s.managerAddr, templateId: 'trader-agent', instanceName: `bob-${scenarioId}`, @@ -276,21 +320,24 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne * address; assert transfer_id non-empty + post-withdraw balance * reflects the withdrawal. */ - async function runSettlementScenario(scenarioId: string): Promise { + async function runSettlementScenario( + scenarioId: string, + controller: ScenarioController, + ): Promise { if (!state) throw new Error('beforeAll did not initialize state'); const s = state; - const { escrow, alice, bob } = await provisionTriple(scenarioId); + const { escrow, alice, bob } = await provisionTriple(scenarioId, controller); // ---- 2. Configure trusted escrows on both traders -------------- console.log(`[${scenarioId}] set-strategy on alice + bob (parallel)…`); await Promise.all([ setStrategyAsync({ - cliPath: s.cliPath, cliHome: s.cliHome, tenant: alice.tenantPubkey, + cliPath: s.cliPath, cliHome: controller.cliHome, tenant: alice.tenantPubkey, trustedEscrows: [escrow.tenantPubkey], maxConcurrent: 5, }), setStrategyAsync({ - cliPath: s.cliPath, cliHome: s.cliHome, tenant: bob.tenantPubkey, + cliPath: s.cliPath, cliHome: controller.cliHome, tenant: bob.tenantPubkey, trustedEscrows: [escrow.tenantPubkey], maxConcurrent: 5, }), @@ -298,8 +345,8 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne // ---- 3. Pre-trade portfolio snapshot --------------------------- const [aliceBefore, bobBefore] = await Promise.all([ - portfolioAsync({ cliPath: s.cliPath, cliHome: s.cliHome, tenant: alice.tenantPubkey }), - portfolioAsync({ cliPath: s.cliPath, cliHome: s.cliHome, tenant: bob.tenantPubkey }), + portfolioAsync({ cliPath: s.cliPath, cliHome: controller.cliHome, tenant: alice.tenantPubkey }), + portfolioAsync({ cliPath: s.cliPath, cliHome: controller.cliHome, tenant: bob.tenantPubkey }), ]); const aliceUctBefore = balanceOf(aliceBefore, 'UCT'); const aliceUsduBefore = balanceOf(aliceBefore, 'USDU'); @@ -319,7 +366,7 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne console.log(`[${scenarioId}] posting matching intents (parallel)…`); const [aliceIntent, bobIntent] = await Promise.all([ createIntentAsync({ - cliPath: s.cliPath, cliHome: s.cliHome, tenant: alice.tenantPubkey, + cliPath: s.cliPath, cliHome: controller.cliHome, tenant: alice.tenantPubkey, direction: 'buy', baseAsset: 'UCT', quoteAsset: 'USDU', rateMin: TRADE_RATE, rateMax: TRADE_RATE, @@ -327,7 +374,7 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne expiryMs: SWAP_TIMEOUT_MS, }), createIntentAsync({ - cliPath: s.cliPath, cliHome: s.cliHome, tenant: bob.tenantPubkey, + cliPath: s.cliPath, cliHome: controller.cliHome, tenant: bob.tenantPubkey, direction: 'sell', baseAsset: 'UCT', quoteAsset: 'USDU', rateMin: TRADE_RATE, rateMax: TRADE_RATE, @@ -344,12 +391,12 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne console.log(`[${scenarioId}] waiting for COMPLETED on both sides…`); const [aliceDeal, bobDeal] = await Promise.all([ waitForDealInStateAsync({ - cliPath: s.cliPath, cliHome: s.cliHome, tenant: alice.tenantPubkey, + cliPath: s.cliPath, cliHome: controller.cliHome, tenant: alice.tenantPubkey, targetState: 'COMPLETED', timeoutMs: SWAP_TIMEOUT_MS, }), waitForDealInStateAsync({ - cliPath: s.cliPath, cliHome: s.cliHome, tenant: bob.tenantPubkey, + cliPath: s.cliPath, cliHome: controller.cliHome, tenant: bob.tenantPubkey, targetState: 'COMPLETED', timeoutMs: SWAP_TIMEOUT_MS, }), @@ -369,8 +416,8 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne await new Promise((r) => setTimeout(r, 5_000)); const [aliceAfter, bobAfter] = await Promise.all([ - portfolioAsync({ cliPath: s.cliPath, cliHome: s.cliHome, tenant: alice.tenantPubkey }), - portfolioAsync({ cliPath: s.cliPath, cliHome: s.cliHome, tenant: bob.tenantPubkey }), + portfolioAsync({ cliPath: s.cliPath, cliHome: controller.cliHome, tenant: alice.tenantPubkey }), + portfolioAsync({ cliPath: s.cliPath, cliHome: controller.cliHome, tenant: bob.tenantPubkey }), ]); const aliceUctAfter = balanceOf(aliceAfter, 'UCT'); const aliceUsduAfter = balanceOf(aliceAfter, 'USDU'); @@ -404,10 +451,10 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne // including the round-6 trim+validation gate. console.log(`[${scenarioId}] withdraw ${WITHDRAW_AMOUNT} UCT from alice → controller…`); const wr = await withdrawAsync({ - cliPath: s.cliPath, cliHome: s.cliHome, tenant: alice.tenantPubkey, + cliPath: s.cliPath, cliHome: controller.cliHome, tenant: alice.tenantPubkey, asset: 'UCT', amount: WITHDRAW_AMOUNT, - toAddress: s.controllerDirectAddress, + toAddress: controller.directAddress, }); expect(wr.transferId).toMatch(/^[a-zA-Z0-9_-]+$/); expect(wr.transferId.length).toBeGreaterThan(8); @@ -418,7 +465,7 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne // confirmed bucket (testnet aggregator round-trip). await new Promise((r) => setTimeout(r, 5_000)); const aliceFinal = await portfolioAsync({ - cliPath: s.cliPath, cliHome: s.cliHome, tenant: alice.tenantPubkey, + cliPath: s.cliPath, cliHome: controller.cliHome, tenant: alice.tenantPubkey, }); const aliceUctFinal = balanceOf(aliceFinal, 'UCT'); expect( @@ -432,10 +479,16 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne // ---- Concurrent scenarios ----------------------------------------------- it('Pair-1: full spawn → trade → settle → withdraw via HMA', async () => { - await runSettlementScenario(`p1-${randomUUID().slice(0, 6)}`); + if (!state) throw new Error('beforeAll did not initialize state'); + const c = state.controllers[0]; + if (!c) throw new Error('controller #0 missing'); + await runSettlementScenario(`p1-${randomUUID().slice(0, 6)}`, c); }, SWAP_TIMEOUT_MS + 4 * 60_000); // 12 min total budget per scenario it('Pair-2: parallel scenario settles on the same HMA without interference', async () => { - await runSettlementScenario(`p2-${randomUUID().slice(0, 6)}`); + if (!state) throw new Error('beforeAll did not initialize state'); + const c = state.controllers[1]; + if (!c) throw new Error('controller #1 missing'); + await runSettlementScenario(`p2-${randomUUID().slice(0, 6)}`, c); }, SWAP_TIMEOUT_MS + 4 * 60_000); }); From 524443278575ebcfcbe379e7a52a5eb079a8922a Mon Sep 17 00:00:00 2001 From: Vladimir Rogojin Date: Mon, 4 May 2026 18:30:53 +0200 Subject: [PATCH 03/30] fix(test): serialize sphere-cli within a scenario; keep cross-scenario parallel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round 2: per-scenario controller wallets fixed cross-scenario contention but the SECOND failure mode (within-scenario races between escrow + alice + bob hostSpawnAsync calls in Promise.all) remained. Each call writes DM dedup state to .sphere-cli/wallet.json via the SDK's atomic temp+rename — three concurrent writers in the same dir corrupt each other. Tried parallel spawn within a scenario; got two distinct races: Pair-1: "No wallet exists and no mnemonic provided" (Sphere.init reading wallet.json mid-write — the file was renamed away underneath it by another spawn's atomic update) Pair-2: "ENOENT: rename wallet.json.tmp -> wallet.json" (one spawn renamed away the temp file the other was about to rename) Fix: serialize sphere-cli calls within a scenario (drop Promise.all over spawn/set-strategy/portfolio/create-intent/wait/portfolio-after). Cross-scenario parallelism via vitest it.concurrent is preserved — that's where the wall-time savings actually come from, because the 3-5 min settlement-wait dominates each scenario and runs in true parallel across scenarios. Wall-time impact: serializing the 3 spawns within a scenario adds 2 × ~5-15s of DM round-trip vs parallel. Negligible against the settlement dominator. A future PR can revisit within-scenario parallelism by giving each parallel sphere-cli invocation its own cliHome with a copied wallet.json (same identity, separate file storage). Out of scope for the goal-completion test. The file header documents the parallelism contract honestly so future contributors don't try to "optimize" it back to broken parallelism. --- .../hma-trade-settlement.e2e-live.test.ts | 205 ++++++++++-------- 1 file changed, 112 insertions(+), 93 deletions(-) diff --git a/test/e2e-live/hma-trade-settlement.e2e-live.test.ts b/test/e2e-live/hma-trade-settlement.e2e-live.test.ts index 835d198..b547027 100644 --- a/test/e2e-live/hma-trade-settlement.e2e-live.test.ts +++ b/test/e2e-live/hma-trade-settlement.e2e-live.test.ts @@ -32,15 +32,24 @@ * - this file: spawns through HMA, settles, AND withdraws. * * Parallelism contract: - * - 2 scenarios run concurrently (vitest `it.concurrent`). - * - Within each scenario, every step that can fan out does so via - * Promise.all using the *Async helpers (which use spawn, not - * spawnSync — sync helpers would serialize even inside Promise.all - * because spawnSync blocks the event loop). - * - Total live infra load at peak: ~6 spawn DMs, 4 set-strategy DMs, - * 4 create-intent DMs, ~4 list-deals DMs every 3s during settlement - * wait. The infra-probe preflight runs first to fail-fast on a - * degraded testnet. + * - 2 scenarios run concurrently across the file (vitest `it.concurrent`). + * Each scenario uses its OWN controller wallet in its OWN cliHome + * dir — concurrent sphere-cli calls in the SAME .sphere-cli/wallet.json + * race on the SDK's atomic temp+rename writes (FileStorageProvider.save + * → fs.renameSync) and corrupt each other's state. + * - WITHIN each scenario, sphere-cli calls are SEQUENTIAL. Tried fully + * parallel (Promise.all over spawn/set-strategy/portfolio) and got + * two distinct races on the first run: "No wallet exists" (Sphere.init + * reading mid-write) and "ENOENT: rename wallet.json.tmp -> wallet.json" + * (two atomic writes racing each other). The DM round-trips for spawn + * are the only ones inside a scenario where parallelism would matter + * (~5-15s each); against the 3-5min settlement-wait dominator, the + * wall-clock cost of serializing them is negligible. + * - Net effect: total wall time ≈ max(scenario_time), not sum, because + * the long settlement wait runs concurrently across scenarios. + * - Live infra load at peak: 2 cross-scenario sphere-cli children, + * each making 1 DM at a time. Manageable for the relay. Aggregator + * load comes from the trader containers themselves, not sphere-cli. * * Performance target on a healthy testnet: ~5-8 minutes total. * - HMA boot: ~10s @@ -257,35 +266,42 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne TRADER_TEST_FUND: `${UCT_COIN_ID}:${SELF_MINT_AMOUNT.toString()},${USDU_COIN_ID}:${SELF_MINT_AMOUNT.toString()}`, TRADER_FAULT_INJECTION_ALLOWED: '1', }; - console.log(`[${scenarioId}] spawning escrow + alice + bob (parallel)…`); - const [escrow, alice, bob] = await Promise.all([ - hostSpawnAsync({ - cliPath: s.cliPath, - cliHome: controller.cliHome, - managerAddress: s.managerAddr, - templateId: 'escrow-service', - instanceName: `escrow-${scenarioId}`, - timeoutMs: 180_000, - }), - hostSpawnAsync({ - cliPath: s.cliPath, - cliHome: controller.cliHome, - managerAddress: s.managerAddr, - templateId: 'trader-agent', - instanceName: `alice-${scenarioId}`, - timeoutMs: 180_000, - env: traderEnv, - }), - hostSpawnAsync({ - cliPath: s.cliPath, - cliHome: controller.cliHome, - managerAddress: s.managerAddr, - templateId: 'trader-agent', - instanceName: `bob-${scenarioId}`, - timeoutMs: 180_000, - env: traderEnv, - }), - ]); + // Within a scenario, sphere-cli calls share one wallet.json and + // race on its atomic temp+rename writes (FileStorageProvider.save + // → fs.renameSync). So within-scenario calls are serialized. The + // PARALLELISM the user asked for is preserved cross-scenario via + // vitest `it.concurrent` — each scenario has its own controller + // wallet and the long settlement-wait phases run truly in parallel + // across scenarios. Wall-clock cost of this serialization vs full + // parallel: 2-3 spawn DM round-trips (~5-15s each) instead of 1, + // negligible against the 3-5 min settlement dominator. + console.log(`[${scenarioId}] spawning escrow + alice + bob (sequential within-scenario)…`); + const escrow = await hostSpawnAsync({ + cliPath: s.cliPath, + cliHome: controller.cliHome, + managerAddress: s.managerAddr, + templateId: 'escrow-service', + instanceName: `escrow-${scenarioId}`, + timeoutMs: 180_000, + }); + const alice = await hostSpawnAsync({ + cliPath: s.cliPath, + cliHome: controller.cliHome, + managerAddress: s.managerAddr, + templateId: 'trader-agent', + instanceName: `alice-${scenarioId}`, + timeoutMs: 180_000, + env: traderEnv, + }); + const bob = await hostSpawnAsync({ + cliPath: s.cliPath, + cliHome: controller.cliHome, + managerAddress: s.managerAddr, + templateId: 'trader-agent', + instanceName: `bob-${scenarioId}`, + timeoutMs: 180_000, + env: traderEnv, + }); s.spawned.push(escrow, alice, bob); console.log( `[${scenarioId}] up: escrow=${escrow.instanceName} ` + @@ -329,25 +345,26 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne const { escrow, alice, bob } = await provisionTriple(scenarioId, controller); // ---- 2. Configure trusted escrows on both traders -------------- - console.log(`[${scenarioId}] set-strategy on alice + bob (parallel)…`); - await Promise.all([ - setStrategyAsync({ - cliPath: s.cliPath, cliHome: controller.cliHome, tenant: alice.tenantPubkey, - trustedEscrows: [escrow.tenantPubkey], - maxConcurrent: 5, - }), - setStrategyAsync({ - cliPath: s.cliPath, cliHome: controller.cliHome, tenant: bob.tenantPubkey, - trustedEscrows: [escrow.tenantPubkey], - maxConcurrent: 5, - }), - ]); + // Sequential within-scenario (wallet.json contention; see provisionTriple). + console.log(`[${scenarioId}] set-strategy on alice + bob…`); + await setStrategyAsync({ + cliPath: s.cliPath, cliHome: controller.cliHome, tenant: alice.tenantPubkey, + trustedEscrows: [escrow.tenantPubkey], + maxConcurrent: 5, + }); + await setStrategyAsync({ + cliPath: s.cliPath, cliHome: controller.cliHome, tenant: bob.tenantPubkey, + trustedEscrows: [escrow.tenantPubkey], + maxConcurrent: 5, + }); // ---- 3. Pre-trade portfolio snapshot --------------------------- - const [aliceBefore, bobBefore] = await Promise.all([ - portfolioAsync({ cliPath: s.cliPath, cliHome: controller.cliHome, tenant: alice.tenantPubkey }), - portfolioAsync({ cliPath: s.cliPath, cliHome: controller.cliHome, tenant: bob.tenantPubkey }), - ]); + const aliceBefore = await portfolioAsync({ + cliPath: s.cliPath, cliHome: controller.cliHome, tenant: alice.tenantPubkey, + }); + const bobBefore = await portfolioAsync({ + cliPath: s.cliPath, cliHome: controller.cliHome, tenant: bob.tenantPubkey, + }); const aliceUctBefore = balanceOf(aliceBefore, 'UCT'); const aliceUsduBefore = balanceOf(aliceBefore, 'USDU'); const bobUctBefore = balanceOf(bobBefore, 'UCT'); @@ -361,46 +378,46 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne expect(aliceUctBefore + aliceUsduBefore).toBeGreaterThan(0n); expect(bobUctBefore + bobUsduBefore).toBeGreaterThan(0n); - // ---- 4. Post matching intents in parallel ---------------------- + // ---- 4. Post matching intents (sequential — wallet.json) ------ // Alice buys UCT (pays USDU). Bob sells UCT (receives USDU). - console.log(`[${scenarioId}] posting matching intents (parallel)…`); - const [aliceIntent, bobIntent] = await Promise.all([ - createIntentAsync({ - cliPath: s.cliPath, cliHome: controller.cliHome, tenant: alice.tenantPubkey, - direction: 'buy', - baseAsset: 'UCT', quoteAsset: 'USDU', - rateMin: TRADE_RATE, rateMax: TRADE_RATE, - volumeMin: TRADE_VOLUME, volumeMax: TRADE_VOLUME, - expiryMs: SWAP_TIMEOUT_MS, - }), - createIntentAsync({ - cliPath: s.cliPath, cliHome: controller.cliHome, tenant: bob.tenantPubkey, - direction: 'sell', - baseAsset: 'UCT', quoteAsset: 'USDU', - rateMin: TRADE_RATE, rateMax: TRADE_RATE, - volumeMin: TRADE_VOLUME, volumeMax: TRADE_VOLUME, - expiryMs: SWAP_TIMEOUT_MS, - }), - ]); + console.log(`[${scenarioId}] posting matching intents…`); + const aliceIntent = await createIntentAsync({ + cliPath: s.cliPath, cliHome: controller.cliHome, tenant: alice.tenantPubkey, + direction: 'buy', + baseAsset: 'UCT', quoteAsset: 'USDU', + rateMin: TRADE_RATE, rateMax: TRADE_RATE, + volumeMin: TRADE_VOLUME, volumeMax: TRADE_VOLUME, + expiryMs: SWAP_TIMEOUT_MS, + }); + const bobIntent = await createIntentAsync({ + cliPath: s.cliPath, cliHome: controller.cliHome, tenant: bob.tenantPubkey, + direction: 'sell', + baseAsset: 'UCT', quoteAsset: 'USDU', + rateMin: TRADE_RATE, rateMax: TRADE_RATE, + volumeMin: TRADE_VOLUME, volumeMax: TRADE_VOLUME, + expiryMs: SWAP_TIMEOUT_MS, + }); console.log( `[${scenarioId}] intents posted: alice=${aliceIntent.intentId.slice(0, 12)}… ` + `bob=${bobIntent.intentId.slice(0, 12)}…`, ); - // ---- 5. Wait for both deals to reach COMPLETED in parallel ----- - console.log(`[${scenarioId}] waiting for COMPLETED on both sides…`); - const [aliceDeal, bobDeal] = await Promise.all([ - waitForDealInStateAsync({ - cliPath: s.cliPath, cliHome: controller.cliHome, tenant: alice.tenantPubkey, - targetState: 'COMPLETED', - timeoutMs: SWAP_TIMEOUT_MS, - }), - waitForDealInStateAsync({ - cliPath: s.cliPath, cliHome: controller.cliHome, tenant: bob.tenantPubkey, - targetState: 'COMPLETED', - timeoutMs: SWAP_TIMEOUT_MS, - }), - ]); + // ---- 5. Wait for both deals to reach COMPLETED ------------------ + // Sequential within-scenario (wallet.json contention). Wait for + // alice first; once she's COMPLETED, bob is typically COMPLETED + // already on the next poll, so this adds at most one poll cycle. + console.log(`[${scenarioId}] waiting for alice COMPLETED…`); + const aliceDeal = await waitForDealInStateAsync({ + cliPath: s.cliPath, cliHome: controller.cliHome, tenant: alice.tenantPubkey, + targetState: 'COMPLETED', + timeoutMs: SWAP_TIMEOUT_MS, + }); + console.log(`[${scenarioId}] waiting for bob COMPLETED…`); + const bobDeal = await waitForDealInStateAsync({ + cliPath: s.cliPath, cliHome: controller.cliHome, tenant: bob.tenantPubkey, + targetState: 'COMPLETED', + timeoutMs: SWAP_TIMEOUT_MS, + }); expect(aliceDeal.state).toBe('COMPLETED'); expect(bobDeal.state).toBe('COMPLETED'); // Both sides observe the same deal_id (one negotiation, two ledgers). @@ -415,10 +432,12 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne // has been enough on testnet). await new Promise((r) => setTimeout(r, 5_000)); - const [aliceAfter, bobAfter] = await Promise.all([ - portfolioAsync({ cliPath: s.cliPath, cliHome: controller.cliHome, tenant: alice.tenantPubkey }), - portfolioAsync({ cliPath: s.cliPath, cliHome: controller.cliHome, tenant: bob.tenantPubkey }), - ]); + const aliceAfter = await portfolioAsync({ + cliPath: s.cliPath, cliHome: controller.cliHome, tenant: alice.tenantPubkey, + }); + const bobAfter = await portfolioAsync({ + cliPath: s.cliPath, cliHome: controller.cliHome, tenant: bob.tenantPubkey, + }); const aliceUctAfter = balanceOf(aliceAfter, 'UCT'); const aliceUsduAfter = balanceOf(aliceAfter, 'USDU'); const bobUctAfter = balanceOf(bobAfter, 'UCT'); From 93005907d0725704099491c0e6729b108cd0a0e0 Mon Sep 17 00:00:00 2001 From: Vladimir Rogojin Date: Mon, 4 May 2026 18:35:00 +0200 Subject: [PATCH 04/30] fix(test): switch to faucet funding (published trader image lacks self-mint) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round 3 surfaced "TRADER_TEST_FUND requires sphere-sdk with mintFungibleToken (only on refactor/extract-cli-to-sphere-cli branch). Use the faucet path instead, or rebuild with the SDK feature branch." The published trader image (ghcr.io/vrogojin/agentic-hosting/trader:v0.1) that HMA's templates.json points to was built against an older sphere-sdk that doesn't have mintFungibleToken. The TRADER_TEST_FUND env var is recognized but errors at trader startup before acp.hello is sent — HMA observes "did not send hello within 60000ms" and reports hm.spawn_failed. basic-roundtrip works around this by building its trader image locally (with the latest SDK) and running docker directly. HMA tests use the published image, so funding has to happen via the external faucet. Switching to faucet-fund-and-wait: - Drop TRADER_TEST_FUND / TRADER_FAULT_INJECTION_ALLOWED env vars - After alice/bob spawn, hit FAUCET_URL with each tenant's nametag - Poll portfolio every 5s until balance arrives (typically ~30-60s on testnet — payments.receive() runs on a 15s cycle inside the trader plus aggregator confirmation) - Use trade-volume × 2 as funding amount (headroom for the post- trade withdraw step) Trader balances are asymmetric now (alice gets USDU only, bob gets UCT only), matching their trade roles. Post-trade delta assertions unchanged: alice +UCT / -USDU, bob mirror. Note: a future PR rebuilding the trader image at ghcr.io with the latest sphere-sdk would let us swap back to selfMint and avoid the faucet flake risk this re-introduces. --- .../hma-trade-settlement.e2e-live.test.ts | 63 ++++++++++++++++--- 1 file changed, 55 insertions(+), 8 deletions(-) diff --git a/test/e2e-live/hma-trade-settlement.e2e-live.test.ts b/test/e2e-live/hma-trade-settlement.e2e-live.test.ts index b547027..4abd4cb 100644 --- a/test/e2e-live/hma-trade-settlement.e2e-live.test.ts +++ b/test/e2e-live/hma-trade-settlement.e2e-live.test.ts @@ -95,7 +95,7 @@ import { withdrawAsync, type PortfolioBalance, } from './helpers/sphere-trader.js'; -import { UCT_COIN_ID, USDU_COIN_ID } from './helpers/constants.js'; +import { fundWallet } from './helpers/funding.js'; // --------------------------------------------------------------------------- // Precondition gates (mirrors hma-trade-flow's structure) @@ -154,7 +154,6 @@ const SCENARIO_COUNT = 2; let state: SuiteState | null = null; -const SELF_MINT_AMOUNT = 5000n; // matches basic-roundtrip const SWAP_TIMEOUT_MS = 8 * 60_000; // 8 minutes; testnet settlement is 3-5 min typical // `volume_max` for matching intents in each scenario. Both sides post @@ -262,10 +261,6 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne }> { if (!state) throw new Error('beforeAll did not initialize state'); const s = state; - const traderEnv = { - TRADER_TEST_FUND: `${UCT_COIN_ID}:${SELF_MINT_AMOUNT.toString()},${USDU_COIN_ID}:${SELF_MINT_AMOUNT.toString()}`, - TRADER_FAULT_INJECTION_ALLOWED: '1', - }; // Within a scenario, sphere-cli calls share one wallet.json and // race on its atomic temp+rename writes (FileStorageProvider.save // → fs.renameSync). So within-scenario calls are serialized. The @@ -284,6 +279,11 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne instanceName: `escrow-${scenarioId}`, timeoutMs: 180_000, }); + // Trader templates do NOT receive TRADER_TEST_FUND — the published + // trader image (ghcr.io/.../trader:v0.1) uses an older sphere-sdk + // without mintFungibleToken, so the self-mint path errors out + // ("TRADER_TEST_FUND requires sphere-sdk with mintFungibleToken"). + // Funding happens AFTER spawn via the testnet faucet (HTTP). const alice = await hostSpawnAsync({ cliPath: s.cliPath, cliHome: controller.cliHome, @@ -291,7 +291,6 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne templateId: 'trader-agent', instanceName: `alice-${scenarioId}`, timeoutMs: 180_000, - env: traderEnv, }); const bob = await hostSpawnAsync({ cliPath: s.cliPath, @@ -300,7 +299,6 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne templateId: 'trader-agent', instanceName: `bob-${scenarioId}`, timeoutMs: 180_000, - env: traderEnv, }); s.spawned.push(escrow, alice, bob); console.log( @@ -336,6 +334,45 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne * address; assert transfer_id non-empty + post-withdraw balance * reflects the withdrawal. */ + /** + * Fund a trader via the testnet faucet, then poll its portfolio + * until the requested asset arrives or `timeoutMs` elapses. Faucet + * deposits land via the trader's payments.receive() loop (~15s + * cycle) plus aggregator confirmation, so we typically observe the + * balance ~30-60s after the faucet POST. + * + * Faucet expects the BARE nametag (no `@`) in `unicityId`. The + * helper unwraps `@nametag` automatically. + */ + async function faucetFundAndWait( + label: string, + tenant: { tenantNametag: string | null; tenantPubkey: string }, + asset: 'UCT' | 'USDU', + amount: bigint, + cliHome: string, + timeoutMs = 120_000, + ): Promise { + if (!state) throw new Error('state missing'); + const s = state; + const recipient = tenant.tenantNametag ? `@${tenant.tenantNametag}` : `DIRECT://${tenant.tenantPubkey}`; + console.log(`[${label}] faucet → ${recipient} ${amount}${asset}…`); + await fundWallet(recipient, amount, asset); + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + const p = await portfolioAsync({ + cliPath: s.cliPath, cliHome, tenant: tenant.tenantPubkey, + }); + if (balanceOf(p, asset) >= amount) { + console.log(`[${label}] balance arrived: ${balanceOf(p, asset)}${asset}`); + return; + } + await new Promise((r) => setTimeout(r, 5_000)); + } + throw new Error( + `[${label}] balance ${asset}>=${amount} did not arrive within ${timeoutMs}ms`, + ); + } + async function runSettlementScenario( scenarioId: string, controller: ScenarioController, @@ -344,6 +381,16 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne const s = state; const { escrow, alice, bob } = await provisionTriple(scenarioId, controller); + // ---- 1.5 Faucet-fund both traders before they can trade --------- + // Alice (buyer of UCT) needs USDU to pay. Bob (seller of UCT) needs UCT. + // Sequential within-scenario (wallet.json contention again). Use + // exact trade-required amounts plus headroom for the withdraw step. + const aliceUsduFundAmount = TRADE_RATE * TRADE_VOLUME * 2n; // 2× headroom + const bobUctFundAmount = TRADE_VOLUME * 2n; + console.log(`[${scenarioId}] funding alice + bob via faucet…`); + await faucetFundAndWait(scenarioId, alice, 'USDU', aliceUsduFundAmount, controller.cliHome); + await faucetFundAndWait(scenarioId, bob, 'UCT', bobUctFundAmount, controller.cliHome); + // ---- 2. Configure trusted escrows on both traders -------------- // Sequential within-scenario (wallet.json contention; see provisionTriple). console.log(`[${scenarioId}] set-strategy on alice + bob…`); From b320db639d2b485b0987423617d333da35766392 Mon Sep 17 00:00:00 2001 From: Vladimir Rogojin Date: Mon, 4 May 2026 18:48:40 +0200 Subject: [PATCH 05/30] =?UTF-8?q?fix(test):=20faucet=20uses=20coin.name=20?= =?UTF-8?q?not=20coin.symbol=20=E2=80=94=20map=20UCT/USDU?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round 4 surfaced "Coin not found: USDU" from the testnet faucet. Probing the faucet directly: POST /api/v1/faucet/request {coin: "UCT"} → Coin not found: UCT POST /api/v1/faucet/request {coin: "unicity"} → Nametag not found So the faucet looks up coins by the `name` field of /api/v1/faucet/coins, not by `symbol`: coins[].symbol "UCT" → coins[].name "unicity" → faucet OK coins[].symbol "USDU" → coins[].name "unicity-usd" → faucet OK Adding a small FAUCET_COIN_NAME map and converting before the fundWallet call. The map is colocated in the test (5 lines, two entries) rather than pushed into helpers/funding.ts because the inverse mapping isn't well-known here — the symbol set the test uses (UCT, USDU) is the same set the test asserts in portfolio deltas, so having the mapping near both keeps the surface obvious to a future contributor adding a third asset. --- .../hma-trade-settlement.e2e-live.test.ts | 26 ++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/test/e2e-live/hma-trade-settlement.e2e-live.test.ts b/test/e2e-live/hma-trade-settlement.e2e-live.test.ts index 4abd4cb..b1fe276 100644 --- a/test/e2e-live/hma-trade-settlement.e2e-live.test.ts +++ b/test/e2e-live/hma-trade-settlement.e2e-live.test.ts @@ -334,6 +334,22 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne * address; assert transfer_id non-empty + post-withdraw balance * reflects the withdrawal. */ + /** + * Map an asset symbol (as the trader exposes it in portfolio: + * `UCT`, `USDU`, etc.) to the faucet's coin identifier. The + * faucet's `/api/v1/faucet/request` endpoint looks up coins by + * the `name` field of `/api/v1/faucet/coins`, NOT by symbol — + * `coin: "UCT"` returns `Coin not found: UCT`. Verified by probing + * the faucet directly: + * coins[].name "unicity" → faucet accepts symbol UCT + * coins[].name "unicity-usd" → faucet accepts symbol USDU + * If a future coin is added to this test, extend this map. + */ + const FAUCET_COIN_NAME: Record = { + UCT: 'unicity', + USDU: 'unicity-usd', + }; + /** * Fund a trader via the testnet faucet, then poll its portfolio * until the requested asset arrives or `timeoutMs` elapses. Faucet @@ -342,7 +358,7 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne * balance ~30-60s after the faucet POST. * * Faucet expects the BARE nametag (no `@`) in `unicityId`. The - * helper unwraps `@nametag` automatically. + * fundWallet helper unwraps `@nametag` automatically. */ async function faucetFundAndWait( label: string, @@ -355,8 +371,12 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne if (!state) throw new Error('state missing'); const s = state; const recipient = tenant.tenantNametag ? `@${tenant.tenantNametag}` : `DIRECT://${tenant.tenantPubkey}`; - console.log(`[${label}] faucet → ${recipient} ${amount}${asset}…`); - await fundWallet(recipient, amount, asset); + const faucetCoin = FAUCET_COIN_NAME[asset]; + if (faucetCoin === undefined) { + throw new Error(`[${label}] no faucet coin name for ${asset}`); + } + console.log(`[${label}] faucet → ${recipient} ${amount}${asset} (faucet name: ${faucetCoin})…`); + await fundWallet(recipient, amount, faucetCoin); const deadline = Date.now() + timeoutMs; while (Date.now() < deadline) { const p = await portfolioAsync({ From 26cf9b386ed214841c545a6b680a5f9d8f831132 Mon Sep 17 00:00:00 2001 From: Vladimir Rogojin Date: Mon, 4 May 2026 18:56:58 +0200 Subject: [PATCH 06/30] fix(test): use locally-built trader:local image (v0.1 lacks payments.receive) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round 5 surfaced the deeper issue: the published trader image at ghcr.io/.../trader:v0.1 not only lacks mintFungibleToken (Round 3) but also fails to surface faucet-deposited balance in portfolio. Trader logs from a Round-5 container show ZERO payment-receive events from spawn through 2 minutes of running — no "payment_received" / "deposit_received" / inventory updates. The faucet returns 200 OK with a tx_id, but the trader's portfolio stays empty. Test now overrides the trader-agent template's `image` field to point to `ghcr.io/.../trader:local` (built via the Dockerfile in this repo with current sphere-sdk + trader code). The shared agentic-hosting/config/templates.json is not modified — the override is test-local via spawnHostManager's templatesPath param. Build the local image before running this test: cd /home/vrogojin && \ docker build -f trader-service/Dockerfile \ -t ghcr.io/vrogojin/agentic-hosting/trader:local . Operators will need to rebuild + re-publish the trader image at ghcr.io to use HMA in production trading. Tracked in file header. --- .../hma-trade-settlement.e2e-live.test.ts | 29 ++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/test/e2e-live/hma-trade-settlement.e2e-live.test.ts b/test/e2e-live/hma-trade-settlement.e2e-live.test.ts index b1fe276..1b30256 100644 --- a/test/e2e-live/hma-trade-settlement.e2e-live.test.ts +++ b/test/e2e-live/hma-trade-settlement.e2e-live.test.ts @@ -68,7 +68,8 @@ */ import { describe, it, expect, beforeAll, afterAll } from 'vitest'; -import { rmSync, existsSync } from 'node:fs'; +import { rmSync, existsSync, readFileSync, writeFileSync, mkdtempSync } from 'node:fs'; +import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { randomUUID } from 'node:crypto'; import { @@ -194,6 +195,31 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne controllers.push({ cliHome: home, pubkey: c.pubkey, directAddress: c.directAddress }); } + // The published trader image at ghcr.io/.../trader:v0.1 lacks both + // mintFungibleToken (so TRADER_TEST_FUND fails) AND a working + // payments.receive() loop (so faucet deposits never surface in + // portfolio). To get end-to-end settlement working we use a + // locally-built `trader:local` image with the current sphere-sdk + + // trader code. Build via: + // cd /home/vrogojin && docker build -f trader-service/Dockerfile \ + // -t ghcr.io/vrogojin/agentic-hosting/trader:local . + // Then we materialize a temp templates.json that swaps the image + // tag from v0.1 to local. agentic-hosting/config/templates.json is + // not modified. + const baseTemplatesPath = join(agenticProbe.ok ? agenticProbe.path : '', 'config', 'templates.json'); + const baseTemplates = JSON.parse(readFileSync(baseTemplatesPath, 'utf8')) as { + templates: Array<{ template_id: string; image: string;[k: string]: unknown }>; + }; + for (const t of baseTemplates.templates) { + if (t.template_id === 'trader-agent') { + t.image = 'ghcr.io/vrogojin/agentic-hosting/trader:local'; + } + } + const tplDir = mkdtempSync(join(tmpdir(), 'hma-trade-settlement-tpl-')); + cliHomes.push(tplDir); + const customTemplatesPath = join(tplDir, 'templates.json'); + writeFileSync(customTemplatesPath, JSON.stringify(baseTemplates, null, 2)); + // HMA accepts AUTHORIZED_CONTROLLERS as a comma-separated list of // pubkeys (see agentic-hosting/src/shared/config.ts:52). Authorize // every scenario's controller in one HMA — production has one HMA @@ -202,6 +228,7 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne console.log('[hma-trade-settlement] booting host-manager…'); const manager = await spawnHostManager({ controllerPubkey: controllers.map((c) => c.pubkey).join(','), + templatesPath: customTemplatesPath, }); await manager.ready; const managerAddr = manager.nametag ? `@${manager.nametag}` : manager.pubkey; From 7915ec04b688bd4061df30bca0751cc914926cde Mon Sep 17 00:00:00 2001 From: Vladimir Rogojin Date: Mon, 4 May 2026 19:03:19 +0200 Subject: [PATCH 07/30] fix(test): use TRADER_TEST_FUND with locally-built trader image MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round 6 surfaced that the testnet faucet is silently broken: it returns 200 OK + tx_id but the deposit never surfaces in the trader's portfolio (verified by polling for 120s and checking trader-side container logs — zero payment-receive events). Same flakiness basic-roundtrip's commit history documents. The locally-rebuilt trader image (trader:local) embeds the current sphere-sdk which DOES include mintFungibleToken (verified at sphere-sdk/dist/index.js:12237). So TRADER_TEST_FUND self-mint at trader startup works on the local image — the constraint that forced us to faucet (round 3) was specific to v0.1's older SDK. This commit: - Removes faucet helper (faucetFundAndWait + FAUCET_COIN_NAME mapping + fundWallet import) - Re-introduces TRADER_TEST_FUND env (5000 each of UCT + USDU per trader) passed through HMA's validatePayloadEnv passthrough to the trader container at spawn - Adds a 5s post-spawn delay so the trader's first portfolio query reflects the post-mint balance - Documents the trader image build requirement in the file header --- .../hma-trade-settlement.e2e-live.test.ts | 142 +++++++----------- 1 file changed, 57 insertions(+), 85 deletions(-) diff --git a/test/e2e-live/hma-trade-settlement.e2e-live.test.ts b/test/e2e-live/hma-trade-settlement.e2e-live.test.ts index 1b30256..7240601 100644 --- a/test/e2e-live/hma-trade-settlement.e2e-live.test.ts +++ b/test/e2e-live/hma-trade-settlement.e2e-live.test.ts @@ -59,10 +59,31 @@ * - settlement wait: 3-5 min on testnet (dominates) * - 2 concurrent withdraws: ~5-10s * + * Trader image: + * This test requires `ghcr.io/vrogojin/agentic-hosting/trader:local` + * built from this repo's Dockerfile (which embeds the current + * sphere-sdk including mintFungibleToken). Build before running: + * cd /home/vrogojin && docker build -f trader-service/Dockerfile \ + * -t ghcr.io/vrogojin/agentic-hosting/trader:local . + * The published `:v0.1` tag at ghcr.io is too old (tested in rounds + * 3-5: lacks mintFungibleToken AND the faucet path silently + * doesn't deliver). The test materializes a temp templates.json + * that swaps the image tag from v0.1 to local without modifying + * agentic-hosting's shared config. + * + * Funding: + * Self-mint via TRADER_TEST_FUND env var (5000 each of UCT/USDU + * per trader at startup). HMA forwards the env to docker via its + * payload.env passthrough (validatePayloadEnv at + * agentic-hosting/src/host-manager/manager.ts:97 — TRADER_TEST_FUND + * isn't in FORBIDDEN_ENV_KEYS and doesn't start with UNICITY_, so + * it passes through unchanged). The trader's startup self-mint + * gate also requires TRADER_FAULT_INJECTION_ALLOWED=1. + * * Out of scope: - * - Faucet HTTP — the trader self-mints via TRADER_TEST_FUND, the - * same pattern basic-roundtrip uses (faucet has been a recurring - * source of flakiness on testnet). + * - Faucet HTTP — verified silently broken on testnet (round 5/6: + * POST returns 200 OK + tx_id but the deposit never surfaces in + * the trader's portfolio after 2+ minutes of polling). * - Negotiation-failure paths — covered by negotiation-failures. * - Partial fills — covered by edge-cases. */ @@ -96,7 +117,7 @@ import { withdrawAsync, type PortfolioBalance, } from './helpers/sphere-trader.js'; -import { fundWallet } from './helpers/funding.js'; +import { UCT_COIN_ID, USDU_COIN_ID } from './helpers/constants.js'; // --------------------------------------------------------------------------- // Precondition gates (mirrors hma-trade-flow's structure) @@ -156,6 +177,7 @@ const SCENARIO_COUNT = 2; let state: SuiteState | null = null; const SWAP_TIMEOUT_MS = 8 * 60_000; // 8 minutes; testnet settlement is 3-5 min typical +const SELF_MINT_AMOUNT = 5000n; // matches basic-roundtrip's selfMintFund amount // `volume_max` for matching intents in each scenario. Both sides post // identical volumes so a single fill clears both intents. @@ -288,15 +310,29 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne }> { if (!state) throw new Error('beforeAll did not initialize state'); const s = state; + // The locally-rebuilt trader image (ghcr.io/.../trader:local) ships + // with the current sphere-sdk which includes mintFungibleToken, + // so we use TRADER_TEST_FUND for funding (self-mint at startup). + // Reasons we don't use the testnet faucet: + // - The faucet returns 200 OK with a tx_id but the deposit + // never lands in the trader's portfolio (verified with + // 5+ minute polling in round 5/6). basic-roundtrip's commit + // history documents the same flakiness. + // - Self-mint avoids any external HTTP dependency for funding, + // which is a more reliable test contract. + // Both flags are required by the trader's production guard: + // TRADER_FAULT_INJECTION_ALLOWED=1 — opt-in to fault injection + // TRADER_TEST_FUND=:,... + const traderEnv = { + TRADER_TEST_FUND: + `${UCT_COIN_ID}:${(SELF_MINT_AMOUNT).toString()},` + + `${USDU_COIN_ID}:${(SELF_MINT_AMOUNT).toString()}`, + TRADER_FAULT_INJECTION_ALLOWED: '1', + }; // Within a scenario, sphere-cli calls share one wallet.json and - // race on its atomic temp+rename writes (FileStorageProvider.save - // → fs.renameSync). So within-scenario calls are serialized. The - // PARALLELISM the user asked for is preserved cross-scenario via - // vitest `it.concurrent` — each scenario has its own controller - // wallet and the long settlement-wait phases run truly in parallel - // across scenarios. Wall-clock cost of this serialization vs full - // parallel: 2-3 spawn DM round-trips (~5-15s each) instead of 1, - // negligible against the 3-5 min settlement dominator. + // race on its atomic temp+rename writes — see header. Sequential + // within-scenario; cross-scenario runs in true parallel via + // vitest it.concurrent. console.log(`[${scenarioId}] spawning escrow + alice + bob (sequential within-scenario)…`); const escrow = await hostSpawnAsync({ cliPath: s.cliPath, @@ -306,11 +342,6 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne instanceName: `escrow-${scenarioId}`, timeoutMs: 180_000, }); - // Trader templates do NOT receive TRADER_TEST_FUND — the published - // trader image (ghcr.io/.../trader:v0.1) uses an older sphere-sdk - // without mintFungibleToken, so the self-mint path errors out - // ("TRADER_TEST_FUND requires sphere-sdk with mintFungibleToken"). - // Funding happens AFTER spawn via the testnet faucet (HTTP). const alice = await hostSpawnAsync({ cliPath: s.cliPath, cliHome: controller.cliHome, @@ -318,6 +349,7 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne templateId: 'trader-agent', instanceName: `alice-${scenarioId}`, timeoutMs: 180_000, + env: traderEnv, }); const bob = await hostSpawnAsync({ cliPath: s.cliPath, @@ -326,6 +358,7 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne templateId: 'trader-agent', instanceName: `bob-${scenarioId}`, timeoutMs: 180_000, + env: traderEnv, }); s.spawned.push(escrow, alice, bob); console.log( @@ -361,65 +394,6 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne * address; assert transfer_id non-empty + post-withdraw balance * reflects the withdrawal. */ - /** - * Map an asset symbol (as the trader exposes it in portfolio: - * `UCT`, `USDU`, etc.) to the faucet's coin identifier. The - * faucet's `/api/v1/faucet/request` endpoint looks up coins by - * the `name` field of `/api/v1/faucet/coins`, NOT by symbol — - * `coin: "UCT"` returns `Coin not found: UCT`. Verified by probing - * the faucet directly: - * coins[].name "unicity" → faucet accepts symbol UCT - * coins[].name "unicity-usd" → faucet accepts symbol USDU - * If a future coin is added to this test, extend this map. - */ - const FAUCET_COIN_NAME: Record = { - UCT: 'unicity', - USDU: 'unicity-usd', - }; - - /** - * Fund a trader via the testnet faucet, then poll its portfolio - * until the requested asset arrives or `timeoutMs` elapses. Faucet - * deposits land via the trader's payments.receive() loop (~15s - * cycle) plus aggregator confirmation, so we typically observe the - * balance ~30-60s after the faucet POST. - * - * Faucet expects the BARE nametag (no `@`) in `unicityId`. The - * fundWallet helper unwraps `@nametag` automatically. - */ - async function faucetFundAndWait( - label: string, - tenant: { tenantNametag: string | null; tenantPubkey: string }, - asset: 'UCT' | 'USDU', - amount: bigint, - cliHome: string, - timeoutMs = 120_000, - ): Promise { - if (!state) throw new Error('state missing'); - const s = state; - const recipient = tenant.tenantNametag ? `@${tenant.tenantNametag}` : `DIRECT://${tenant.tenantPubkey}`; - const faucetCoin = FAUCET_COIN_NAME[asset]; - if (faucetCoin === undefined) { - throw new Error(`[${label}] no faucet coin name for ${asset}`); - } - console.log(`[${label}] faucet → ${recipient} ${amount}${asset} (faucet name: ${faucetCoin})…`); - await fundWallet(recipient, amount, faucetCoin); - const deadline = Date.now() + timeoutMs; - while (Date.now() < deadline) { - const p = await portfolioAsync({ - cliPath: s.cliPath, cliHome, tenant: tenant.tenantPubkey, - }); - if (balanceOf(p, asset) >= amount) { - console.log(`[${label}] balance arrived: ${balanceOf(p, asset)}${asset}`); - return; - } - await new Promise((r) => setTimeout(r, 5_000)); - } - throw new Error( - `[${label}] balance ${asset}>=${amount} did not arrive within ${timeoutMs}ms`, - ); - } - async function runSettlementScenario( scenarioId: string, controller: ScenarioController, @@ -428,15 +402,13 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne const s = state; const { escrow, alice, bob } = await provisionTriple(scenarioId, controller); - // ---- 1.5 Faucet-fund both traders before they can trade --------- - // Alice (buyer of UCT) needs USDU to pay. Bob (seller of UCT) needs UCT. - // Sequential within-scenario (wallet.json contention again). Use - // exact trade-required amounts plus headroom for the withdraw step. - const aliceUsduFundAmount = TRADE_RATE * TRADE_VOLUME * 2n; // 2× headroom - const bobUctFundAmount = TRADE_VOLUME * 2n; - console.log(`[${scenarioId}] funding alice + bob via faucet…`); - await faucetFundAndWait(scenarioId, alice, 'USDU', aliceUsduFundAmount, controller.cliHome); - await faucetFundAndWait(scenarioId, bob, 'UCT', bobUctFundAmount, controller.cliHome); + // Funding happened via TRADER_TEST_FUND at trader startup (5000 each + // of UCT and USDU on both traders). The trader's main.ts logs + // "test_fund_mint_succeeded" once per coin if the mint worked, + // and the intent engine reads the resulting balance on its first + // scan cycle. Wait briefly so the first portfolio query reflects + // the post-mint balance. + await new Promise((r) => setTimeout(r, 5_000)); // ---- 2. Configure trusted escrows on both traders -------------- // Sequential within-scenario (wallet.json contention; see provisionTriple). From ef02628a45264db654b32a8e561aaa75eba3322e Mon Sep 17 00:00:00 2001 From: Vladimir Rogojin Date: Mon, 4 May 2026 20:59:50 +0200 Subject: [PATCH 08/30] fix(test): balanceOf must check `confirmed` field, not just `amount` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round 7 progressed past spawn + set-strategy and trader logs confirmed both UCT and USDU mints succeeded ("test_fund_mint_succeeded" ×2 per trader). But the balance sanity check failed with "expected 0 to be greater than 0" because balanceOf's first loop returned `BigInt(String(b.amount ?? '0'))` → 0n WHEN `b.amount` is undefined — the short-circuit on `?? '0'` prevented the fallback loop from ever reading `b.confirmed`. The trader's GET_PORTFOLIO emits each balance as { asset, available, total, confirmed, unconfirmed } with no `amount` field. So balanceOf was returning 0n for every balance despite the mint succeeding. Fix: single-pass loop, prefer `confirmed`, fall back to `amount` then `available`. Returns 0n only if NONE are present. --- .../hma-trade-settlement.e2e-live.test.ts | 26 +++++++++++++------ 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/test/e2e-live/hma-trade-settlement.e2e-live.test.ts b/test/e2e-live/hma-trade-settlement.e2e-live.test.ts index 7240601..0503063 100644 --- a/test/e2e-live/hma-trade-settlement.e2e-live.test.ts +++ b/test/e2e-live/hma-trade-settlement.e2e-live.test.ts @@ -368,16 +368,26 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne return { escrow, alice, bob }; } - /** Pull a coin's confirmed balance (smallest units) from a portfolio response. */ + /** + * Pull a coin's confirmed balance (smallest units) from a portfolio + * response. The trader's GET_PORTFOLIO emits each balance as + * { asset, available, total, confirmed, unconfirmed } + * where `confirmed` is the amount we should use for assertions. + * Older versions used `amount`; tolerate both. Round-7 hit a bug + * where the early return on `b.amount ?? '0'` short-circuited to + * 0n WITHOUT falling through to `confirmed` when the field was + * absent, so the self-mint balance assertion failed despite the + * mint succeeding. + */ function balanceOf(p: readonly PortfolioBalance[], symbol: string): bigint { - for (const b of p) { - if (b.asset === symbol) return BigInt(String(b.amount ?? '0')); - } - // Tolerate alternate field names (some sphere-sdk versions emit `confirmed`). for (const b of p as Array>) { - if (b['asset'] === symbol && b['confirmed'] !== undefined) { - return BigInt(String(b['confirmed'])); - } + if (b['asset'] !== symbol) continue; + // Prefer `confirmed` (canonical); fall back to `amount` (legacy). + if (b['confirmed'] !== undefined) return BigInt(String(b['confirmed'])); + if (b['amount'] !== undefined) return BigInt(String(b['amount'])); + // `available` is also a reasonable fallback for "what can be spent now". + if (b['available'] !== undefined) return BigInt(String(b['available'])); + return 0n; } return 0n; } From f099fa5d76eb29967baab7d63a69540f5dd2ad4b Mon Sep 17 00:00:00 2001 From: Vladimir Rogojin Date: Mon, 4 May 2026 21:27:52 +0200 Subject: [PATCH 09/30] fix(test): distinct rates per scenario to prevent cross-matching MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round 8 progressed all the way to settlement-wait but timed out with both scenarios stuck — pair-1 saw 29 deals (1 FAILED + 27 CANCELLED + 1 PROPOSED), pair-2 saw 3 (FAILED, CANCELLED, ACCEPTED). The thrash loop happened because both scenarios posted UCT/USDU intents at rate=1, so the matcher saw cross-scenario pairs as valid matches. When pair-1's alice tried to negotiate with pair-2's bob, the trusted_escrows mismatch forced the deal to FAILED, but the matching engine just kept retrying. Fix: pass per-scenario `tradeRate` into runSettlementScenario. Pair-1 uses rate=1, pair-2 uses rate=3 (non-adjacent so no rate-fuzzing ever overlaps). The matcher's rate-overlap check prevents cross-scenario matches at the search-result level, before negotiation. --- .../hma-trade-settlement.e2e-live.test.ts | 32 ++++++++++++++----- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/test/e2e-live/hma-trade-settlement.e2e-live.test.ts b/test/e2e-live/hma-trade-settlement.e2e-live.test.ts index 0503063..dd36cf0 100644 --- a/test/e2e-live/hma-trade-settlement.e2e-live.test.ts +++ b/test/e2e-live/hma-trade-settlement.e2e-live.test.ts @@ -182,7 +182,10 @@ const SELF_MINT_AMOUNT = 5000n; // matches basic-roundtrip's selfMintFund amount // `volume_max` for matching intents in each scenario. Both sides post // identical volumes so a single fill clears both intents. const TRADE_VOLUME = 10n; -const TRADE_RATE = 1n; // 1 USDU per UCT — keeps the math obvious +// Per-scenario rates are distinct to prevent cross-scenario matching; +// see runSettlementScenario header for why. +const PAIR_1_RATE = 1n; // 1 USDU per UCT +const PAIR_2_RATE = 3n; // 3 USDU per UCT — non-adjacent to avoid any rate-fuzzing overlap // Withdraw amount: small fraction of received UCT so the test asserts // real value movement without exhausting the trader's post-trade balance. @@ -404,9 +407,22 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne * address; assert transfer_id non-empty + post-withdraw balance * reflects the withdrawal. */ + /** + * `tradeRate` differs per scenario so the two pairs CANNOT + * cross-match on testnet. Round 8 saw both scenarios stuck at + * waitForDealInState → COMPLETED with 29 deals in CANCELLED state + * because pair-1's alice (rate=1) was matching pair-2's bob (also + * rate=1) and trying to negotiate — but the trusted_escrows on + * each side only allow that scenario's own escrow, so every + * cross-scenario negotiation flipped to FAILED/CANCELLED in a + * thrash loop. With distinct rates, rate-overlap filtering at + * the matcher level prevents the cross-match before negotiation + * even starts. + */ async function runSettlementScenario( scenarioId: string, controller: ScenarioController, + tradeRate: bigint, ): Promise { if (!state) throw new Error('beforeAll did not initialize state'); const s = state; @@ -461,7 +477,7 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne cliPath: s.cliPath, cliHome: controller.cliHome, tenant: alice.tenantPubkey, direction: 'buy', baseAsset: 'UCT', quoteAsset: 'USDU', - rateMin: TRADE_RATE, rateMax: TRADE_RATE, + rateMin: tradeRate, rateMax: tradeRate, volumeMin: TRADE_VOLUME, volumeMax: TRADE_VOLUME, expiryMs: SWAP_TIMEOUT_MS, }); @@ -469,7 +485,7 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne cliPath: s.cliPath, cliHome: controller.cliHome, tenant: bob.tenantPubkey, direction: 'sell', baseAsset: 'UCT', quoteAsset: 'USDU', - rateMin: TRADE_RATE, rateMax: TRADE_RATE, + rateMin: tradeRate, rateMax: tradeRate, volumeMin: TRADE_VOLUME, volumeMax: TRADE_VOLUME, expiryMs: SWAP_TIMEOUT_MS, }); @@ -519,7 +535,7 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne const bobUctAfter = balanceOf(bobAfter, 'UCT'); const bobUsduAfter = balanceOf(bobAfter, 'USDU'); - const expectedUsduPaid = TRADE_RATE * TRADE_VOLUME; + const expectedUsduPaid = tradeRate * TRADE_VOLUME; // Alice (buy UCT for USDU): +UCT / -USDU expect( aliceUctAfter - aliceUctBefore, @@ -573,17 +589,17 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne // ---- Concurrent scenarios ----------------------------------------------- - it('Pair-1: full spawn → trade → settle → withdraw via HMA', async () => { + it('Pair-1: full spawn → trade → settle → withdraw via HMA (rate=1)', async () => { if (!state) throw new Error('beforeAll did not initialize state'); const c = state.controllers[0]; if (!c) throw new Error('controller #0 missing'); - await runSettlementScenario(`p1-${randomUUID().slice(0, 6)}`, c); + await runSettlementScenario(`p1-${randomUUID().slice(0, 6)}`, c, PAIR_1_RATE); }, SWAP_TIMEOUT_MS + 4 * 60_000); // 12 min total budget per scenario - it('Pair-2: parallel scenario settles on the same HMA without interference', async () => { + it('Pair-2: parallel scenario settles on the same HMA at distinct rate (rate=3)', async () => { if (!state) throw new Error('beforeAll did not initialize state'); const c = state.controllers[1]; if (!c) throw new Error('controller #1 missing'); - await runSettlementScenario(`p2-${randomUUID().slice(0, 6)}`, c); + await runSettlementScenario(`p2-${randomUUID().slice(0, 6)}`, c, PAIR_2_RATE); }, SWAP_TIMEOUT_MS + 4 * 60_000); }); From b2659a3ac27d47b4e713f07af225751038f32867 Mon Sep 17 00:00:00 2001 From: Vladimir Rogojin Date: Tue, 5 May 2026 09:46:54 +0200 Subject: [PATCH 10/30] test(e2e-live): fund traders via js-faucet DM (replaces TRADER_TEST_FUND) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The trade-settlement test was rounds 8-9 stuck in a FAILED/CANCELLED loop despite spawn + balance assertions passing. Hypothesis: tokens self-issued by the trader (TRADER_TEST_FUND mint) confuse the swap protocol because issuer==sender is unusual in production. With FAUCET-funded tokens the issuer is a separate agent — closer to real-world settlement. Also replaces the broken public-faucet HTTP path (returns 200 OK + tx_id but deposit never surfaces, verified across rounds 5-6 — and documented again in basic-roundtrip's commit history). Now uses the js-faucet agent that spawns alongside escrow + traders: beforeAll: - bootstrap controller wallets (per-scenario, sphere-cli) - inject `faucet-agent` template (image: faucet:local) - inject `trader-agent` template (image: trader:local) - boot HMA with all controllers authorized - spawn ONE shared faucet (open auth — no need per scenario) - bootstrap in-process FaucetClient (Sphere wallet that signs and encrypts FAUCET_REQUEST DMs to the faucet's pubkey) per scenario: - spawn escrow + alice + bob (no TRADER_TEST_FUND env) - FAUCET_REQUEST batch (UCT + USDU, 5000 each) → alice - FAUCET_REQUEST batch (UCT + USDU, 5000 each) → bob - poll each trader's portfolio until confirmed >= 5000 of each - set-strategy / post intents / wait COMPLETED / withdraw New helper: test/e2e-live/helpers/faucet-client.ts (~190 lines). - bootstrap an in-process Sphere wallet - subscribe to inbox; capture acp.result/acp.error by command_id - sendDM(faucetPubkey, FAUCET_REQUEST envelope), wait for ack - return deliveries (asset, coin_id, amount, token_id, transfer_id) Pre-flight (operator must do before running): cd /home/vrogojin && docker build \ -f js-faucet/Dockerfile \ -t ghcr.io/unicitynetwork/agentic-hosting/faucet:local . Verified: 698 unit tests still pass; type-check clean for both src/ and test/ contexts. Whether this also fixes the FAILED/CANCELLED settlement loop is the question the next live round answers. --- test/e2e-live/helpers/faucet-client.ts | 202 +++++++++++++++++ .../hma-trade-settlement.e2e-live.test.ts | 206 +++++++++++++----- 2 files changed, 354 insertions(+), 54 deletions(-) create mode 100644 test/e2e-live/helpers/faucet-client.ts diff --git a/test/e2e-live/helpers/faucet-client.ts b/test/e2e-live/helpers/faucet-client.ts new file mode 100644 index 0000000..bfde18b --- /dev/null +++ b/test/e2e-live/helpers/faucet-client.ts @@ -0,0 +1,202 @@ +/** + * In-process faucet client for e2e-live tests. + * + * Bootstraps a Sphere wallet in the test process, then sends ACP-0 + * `FAUCET_REQUEST` DMs to a spawned faucet-agent's pubkey and awaits + * the result envelope. Replaces both: + * + * - TRADER_TEST_FUND self-mint (which only works on certain + * sphere-sdk branches and produces self-issued tokens that may + * interact poorly with the swap protocol). + * - Public-faucet HTTP (`FAUCET_URL`) which has been observed to + * return 200 OK with a tx_id but never deliver the deposit. + * + * All communication is encrypted Sphere DMs; no HTTP. Mirrors the + * pattern js-faucet/test/e2e-live/faucet-roundtrip.e2e-live.test.ts + * uses to drive its own roundtrip test. + */ + +import { Sphere } from '@unicitylabs/sphere-sdk'; +import { createNodeProviders } from '@unicitylabs/sphere-sdk/impl/nodejs'; +import type { DirectMessage } from '@unicitylabs/sphere-sdk'; +import { mkdtempSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { randomUUID } from 'node:crypto'; + +const TRUSTBASE_URL = + 'https://raw.githubusercontent.com/unicitynetwork/unicity-ids/refs/heads/main/bft-trustbase.testnet.json'; + +/** + * In-process client. Holds a Sphere wallet + DM subscription that + * captures incoming acp.result/acp.error envelopes keyed by command_id. + */ +export interface FaucetClient { + readonly sphere: Sphere; + readonly pubkey: string; + readonly directAddress: string; + /** + * Send `FAUCET_REQUEST` to the faucet's pubkey and wait for the + * matching acp.result. Throws on acp.error or timeout. + */ + request(faucetPubkey: string, params: FaucetRequestParams, timeoutMs?: number): Promise; + /** Tear down the wallet + DM subscription. */ + destroy(): Promise; +} + +export interface FaucetRequestParams { + recipient: string; + asset?: string; + amount?: string; + memo?: string; + items?: ReadonlyArray<{ asset: string; amount: string; memo?: string }>; +} + +export interface FaucetDelivery { + asset: string; + coin_id: string; + amount: string; + token_id: string; + transfer_id: string; +} + +interface IncomingResponse { + type: string; + payload: Record; +} + +/** + * Bootstrap a fresh Sphere wallet in the test process and return a + * FaucetClient. The wallet's data dir is a unique tmpdir; caller MUST + * call `destroy()` to release the wallet + relay subscription. + */ +export async function createFaucetClient(): Promise { + const dataDir = mkdtempSync(join(tmpdir(), 'trader-e2e-faucet-cli-')); + const tokensDir = join(dataDir, 'tokens'); + + const tbResp = await fetch(TRUSTBASE_URL, { signal: AbortSignal.timeout(30_000) }); + if (!tbResp.ok) throw new Error(`failed to fetch trustbase: HTTP ${String(tbResp.status)}`); + const trustbasePath = join(dataDir, 'trustbase.json'); + writeFileSync(trustbasePath, await tbResp.text()); + + const providers = createNodeProviders({ + network: 'testnet', + dataDir, + tokensDir, + oracle: { trustBasePath: trustbasePath }, + }); + const { sphere } = await Sphere.init({ + ...providers, + autoGenerate: true, + nametag: `fc-${randomUUID().slice(0, 12).replace(/-/g, '')}`, + accounting: true, + swap: false, + market: false, + }); + + const identity = sphere.identity; + if (!identity) throw new Error('Sphere.init returned no identity'); + const pubkey = identity.chainPubkey; + const directAddress = identity.directAddress ?? `DIRECT://${pubkey}`; + + // Capture inbound result envelopes keyed by command_id. + const responses = new Map(); + const unsubscribe = sphere.on('message:dm', (msg: DirectMessage) => { + const acp = parseAcpJson(msg.content); + if (acp === null) return; + if (acp.type !== 'acp.result' && acp.type !== 'acp.error') return; + const payload = acp.payload as Record; + const cmdId = typeof payload['command_id'] === 'string' ? payload['command_id'] : null; + if (cmdId === null) return; + responses.set(cmdId, { type: acp.type, payload }); + }); + + async function request( + faucetPubkey: string, + params: FaucetRequestParams, + timeoutMs = 180_000, + ): Promise { + const cmdId = randomUUID(); + const msg = createAcpCommandEnvelope(cmdId, 'FAUCET_REQUEST', params as unknown as Record); + await sphere.communications.sendDM(`DIRECT://${faucetPubkey}`, JSON.stringify(msg)); + + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + const r = responses.get(cmdId); + if (r) { + responses.delete(cmdId); + if (r.type === 'acp.error') { + const code = String(r.payload['error_code'] ?? 'UNKNOWN'); + const message = String(r.payload['message'] ?? ''); + throw new Error(`FAUCET_REQUEST failed: [${code}] ${message}`); + } + const result = r.payload['result'] as { deliveries?: FaucetDelivery[] } | undefined; + const deliveries = result?.deliveries ?? []; + if (!Array.isArray(deliveries)) { + throw new Error(`FAUCET_REQUEST: result.deliveries not an array. Got: ${JSON.stringify(r.payload)}`); + } + return deliveries; + } + await new Promise((resolve) => setTimeout(resolve, 250)); + } + throw new Error(`FAUCET_REQUEST: no response for command_id=${cmdId} within ${String(timeoutMs)}ms`); + } + + async function destroy(): Promise { + try { unsubscribe(); } catch { /* ignore */ } + try { await sphere.destroy(); } catch { /* ignore */ } + } + + return { sphere, pubkey, directAddress, request, destroy }; +} + +// --------------------------------------------------------------------------- +// Minimal ACP envelope helpers — duplicated here to avoid pulling in the +// full agentic-hosting protocol module. Trader-service doesn't ship the +// ACP envelope code; the trader's command-handler operates on already- +// parsed payloads. The js-faucet listener parses these envelopes itself. +// --------------------------------------------------------------------------- + +const ACP_VERSION = '0.1'; + +function createAcpCommandEnvelope( + cmdId: string, + name: string, + params: Record, +): { + acp_version: string; + msg_id: string; + ts_ms: number; + instance_id: string; + instance_name: string; + type: string; + payload: { command_id: string; name: string; params: Record }; +} { + return { + acp_version: ACP_VERSION, + msg_id: randomUUID(), + ts_ms: Date.now(), + instance_id: 'controller', + instance_name: 'controller', + type: 'acp.command', + payload: { command_id: cmdId, name, params }, + }; +} + +interface AcpEnvelope { + type: string; + payload: unknown; +} + +function parseAcpJson(content: string): AcpEnvelope | null { + if (content.length > 65_536) return null; + try { + const parsed: unknown = JSON.parse(content); + if (typeof parsed !== 'object' || parsed === null) return null; + const env = parsed as Record; + if (typeof env['type'] !== 'string') return null; + return { type: env['type'], payload: env['payload'] }; + } catch { + return null; + } +} diff --git a/test/e2e-live/hma-trade-settlement.e2e-live.test.ts b/test/e2e-live/hma-trade-settlement.e2e-live.test.ts index dd36cf0..778b15e 100644 --- a/test/e2e-live/hma-trade-settlement.e2e-live.test.ts +++ b/test/e2e-live/hma-trade-settlement.e2e-live.test.ts @@ -8,9 +8,10 @@ * Test * ├── boot HMA over Sphere DM (single shared instance) * └── for each scenario IN PARALLEL (it.concurrent): - * ├── spawn escrow + 2 traders ← Promise.all (3-way concurrent) - * │ └── self-mint UCT+USDU at boot via TRADER_TEST_FUND env - * │ passthrough through HMA → docker + * ├── spawn escrow + 2 traders ← sequential (sphere-cli wallet contention) + * ├── FAUCET_REQUEST → shared faucet-agent for each trader + * │ (5000 UCT + 5000 USDU per trader; faucet mints + sends DM) + * ├── poll each trader's portfolio until faucet delivery confirms * ├── set-strategy on both traders ← Promise.all * ├── post matching intents on both ← Promise.all * ├── wait for COMPLETED on both ← Promise.all @@ -72,18 +73,26 @@ * agentic-hosting's shared config. * * Funding: - * Self-mint via TRADER_TEST_FUND env var (5000 each of UCT/USDU - * per trader at startup). HMA forwards the env to docker via its - * payload.env passthrough (validatePayloadEnv at - * agentic-hosting/src/host-manager/manager.ts:97 — TRADER_TEST_FUND - * isn't in FORBIDDEN_ENV_KEYS and doesn't start with UNICITY_, so - * it passes through unchanged). The trader's startup self-mint - * gate also requires TRADER_FAULT_INJECTION_ALLOWED=1. + * Each trader is funded via a `FAUCET_REQUEST` ACP DM to a SHARED + * js-faucet agent spawned by the same HMA. The test bootstraps an + * in-process Sphere wallet (helpers/faucet-client.ts) to sign + + * encrypt the DM. The faucet mints UCT + USDU and sends them to + * each trader's address; the test polls portfolio until + * `confirmed >= INITIAL_FUND_AMOUNT` for both assets. + * + * This replaces two earlier funding paths that didn't work: + * - TRADER_TEST_FUND self-mint: trader-issued tokens may + * interact poorly with the swap protocol (issuer == sender + * is unusual in production). + * - Public faucet HTTP: returns 200 OK + tx_id but the deposit + * never surfaces in portfolio (verified across rounds 5-6). + * + * The js-faucet image must be built before running this test: + * cd /home/vrogojin && docker build \ + * -f js-faucet/Dockerfile \ + * -t ghcr.io/unicitynetwork/agentic-hosting/faucet:local . * * Out of scope: - * - Faucet HTTP — verified silently broken on testnet (round 5/6: - * POST returns 200 OK + tx_id but the deposit never surfaces in - * the trader's portfolio after 2+ minutes of polling). * - Negotiation-failure paths — covered by negotiation-failures. * - Partial fills — covered by edge-cases. */ @@ -117,7 +126,7 @@ import { withdrawAsync, type PortfolioBalance, } from './helpers/sphere-trader.js'; -import { UCT_COIN_ID, USDU_COIN_ID } from './helpers/constants.js'; +import { createFaucetClient, type FaucetClient } from './helpers/faucet-client.js'; // --------------------------------------------------------------------------- // Precondition gates (mirrors hma-trade-flow's structure) @@ -165,6 +174,10 @@ interface SuiteState { managerAddr: string; controllers: ScenarioController[]; spawned: SpawnedTenant[]; + /** Single shared faucet — anyone can request, so one per HMA suffices. */ + faucet: SpawnedTenant; + /** In-process Sphere wallet that signs+encrypts FAUCET_REQUEST DMs. */ + faucetClient: FaucetClient; } // Number of concurrent settlement scenarios. Each gets its own controller @@ -177,7 +190,10 @@ const SCENARIO_COUNT = 2; let state: SuiteState | null = null; const SWAP_TIMEOUT_MS = 8 * 60_000; // 8 minutes; testnet settlement is 3-5 min typical -const SELF_MINT_AMOUNT = 5000n; // matches basic-roundtrip's selfMintFund amount +/** Per-asset funding amount delivered to each trader by the faucet. */ +const INITIAL_FUND_AMOUNT = 5000n; +/** How long we wait for faucet-delivered tokens to surface in `confirmed` balance. */ +const FUNDING_BALANCE_TIMEOUT_MS = 180_000; // `volume_max` for matching intents in each scenario. Both sides post // identical volumes so a single fill clears both intents. @@ -231,15 +247,35 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne // Then we materialize a temp templates.json that swaps the image // tag from v0.1 to local. agentic-hosting/config/templates.json is // not modified. + // The published trader image at ghcr.io/.../trader:v0.1 lacks a working + // payments.receive() loop (verified in earlier rounds). Use the locally- + // built `trader:local` image with the current sphere-sdk + trader code. + // Add a `faucet-agent` template entry pointing at the locally-built + // js-faucet image so the test can spawn it through the same HMA. const baseTemplatesPath = join(agenticProbe.ok ? agenticProbe.path : '', 'config', 'templates.json'); const baseTemplates = JSON.parse(readFileSync(baseTemplatesPath, 'utf8')) as { - templates: Array<{ template_id: string; image: string;[k: string]: unknown }>; + templates: Array<{ template_id: string; image: string; entrypoint?: string[]; env_defaults?: Record; resources?: Record;[k: string]: unknown }>; }; for (const t of baseTemplates.templates) { if (t.template_id === 'trader-agent') { t.image = 'ghcr.io/vrogojin/agentic-hosting/trader:local'; } } + if (!baseTemplates.templates.some((t) => t.template_id === 'faucet-agent')) { + baseTemplates.templates.push({ + template_id: 'faucet-agent', + image: 'ghcr.io/unicitynetwork/agentic-hosting/faucet:local', + entrypoint: ['node', '/app/dist/acp-adapter/main.js'], + env_defaults: { LOG_LEVEL: 'info', SPHERE_NETWORK: 'testnet' }, + resources: { memory_mb: 512, pids_limit: 256 }, + }); + } else { + for (const t of baseTemplates.templates) { + if (t.template_id === 'faucet-agent') { + t.image = 'ghcr.io/unicitynetwork/agentic-hosting/faucet:local'; + } + } + } const tplDir = mkdtempSync(join(tmpdir(), 'hma-trade-settlement-tpl-')); cliHomes.push(tplDir); const customTemplatesPath = join(tplDir, 'templates.json'); @@ -259,15 +295,40 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne const managerAddr = manager.nametag ? `@${manager.nametag}` : manager.pubkey; console.log(`[hma-trade-settlement] manager ready @ ${managerAddr}`); + // Spawn ONE shared faucet (the faucet is open — anyone can request, + // so we don't need one per scenario). + console.log('[hma-trade-settlement] spawning shared faucet-agent…'); + const faucet = await hostSpawnAsync({ + cliPath, + cliHome: controllers[0]!.cliHome, + managerAddress: managerAddr, + templateId: 'faucet-agent', + instanceName: `faucet-${randomUUID().slice(0, 6)}`, + timeoutMs: 180_000, + }); + console.log( + `[hma-trade-settlement] faucet ready: pubkey=${faucet.tenantPubkey.slice(0, 16)}… ` + + `nametag=${faucet.tenantNametag ?? ''}`, + ); + + // In-process Sphere wallet that the test uses to send FAUCET_REQUEST + // DMs. The faucet doesn't authorize senders, so this wallet doesn't + // need to be in HMA's AUTHORIZED_CONTROLLERS list. + console.log('[hma-trade-settlement] bootstrapping in-process FaucetClient…'); + const faucetClient = await createFaucetClient(); + console.log(`[hma-trade-settlement] faucet client pubkey ${faucetClient.pubkey.slice(0, 16)}…`); + state = { cliPath, cliHomes, manager, managerAddr, controllers, - spawned: [], + spawned: [faucet], + faucet, + faucetClient, }; - }, 600_000); // 10 min — wallet-init is ~30s per controller on testnet + }, 900_000); // 15 min — adds ~30-60s for faucet spawn + client bootstrap on top of controller-wallet inits afterAll(async () => { if (!state) return; @@ -286,6 +347,7 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne }), ), ); + try { await state.faucetClient.destroy(); } catch { /* best effort */ } await state.manager.stop(); for (const home of state.cliHomes) { try { rmSync(home, { recursive: true, force: true }); } @@ -296,12 +358,19 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne // ---- Per-scenario helpers ------------------------------------------------- /** - * Spawn one escrow + two traders (alice/bob) IN PARALLEL via - * Promise.all on hostSpawnAsync. Both traders self-mint UCT+USDU - * at boot via TRADER_TEST_FUND (HMA's --env passthrough — see - * agentic-hosting/src/host-manager/manager.ts:97 validatePayloadEnv; - * TRADER_TEST_FUND is not in FORBIDDEN_ENV_KEYS and doesn't start - * with UNICITY_ so it's allowed through). + * Spawn one escrow + two traders (alice/bob) sequentially within a + * scenario, then fund each trader with UCT + USDU via the SHARED + * faucet-agent. Funding via FAUCET_REQUEST DM replaces the previous + * TRADER_TEST_FUND self-mint pathway: + * - Self-mint produced trader-issued tokens, which may interact + * poorly with the swap protocol (the swap counterparty would see + * the issuer == sender, which is unusual in production). + * - Public faucet HTTP returned 200 OK + tx_id but never delivered + * (verified across multiple rounds — silent flake). + * - The js-faucet agent mints + sends with the FAUCET as issuer, + * matching production reality. + * Each trader is funded with `INITIAL_FUND_AMOUNT` of both UCT and + * USDU so either side has the inventory to fulfil any direction. */ async function provisionTriple( scenarioId: string, @@ -313,25 +382,6 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne }> { if (!state) throw new Error('beforeAll did not initialize state'); const s = state; - // The locally-rebuilt trader image (ghcr.io/.../trader:local) ships - // with the current sphere-sdk which includes mintFungibleToken, - // so we use TRADER_TEST_FUND for funding (self-mint at startup). - // Reasons we don't use the testnet faucet: - // - The faucet returns 200 OK with a tx_id but the deposit - // never lands in the trader's portfolio (verified with - // 5+ minute polling in round 5/6). basic-roundtrip's commit - // history documents the same flakiness. - // - Self-mint avoids any external HTTP dependency for funding, - // which is a more reliable test contract. - // Both flags are required by the trader's production guard: - // TRADER_FAULT_INJECTION_ALLOWED=1 — opt-in to fault injection - // TRADER_TEST_FUND=:,... - const traderEnv = { - TRADER_TEST_FUND: - `${UCT_COIN_ID}:${(SELF_MINT_AMOUNT).toString()},` + - `${USDU_COIN_ID}:${(SELF_MINT_AMOUNT).toString()}`, - TRADER_FAULT_INJECTION_ALLOWED: '1', - }; // Within a scenario, sphere-cli calls share one wallet.json and // race on its atomic temp+rename writes — see header. Sequential // within-scenario; cross-scenario runs in true parallel via @@ -352,7 +402,6 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne templateId: 'trader-agent', instanceName: `alice-${scenarioId}`, timeoutMs: 180_000, - env: traderEnv, }); const bob = await hostSpawnAsync({ cliPath: s.cliPath, @@ -361,13 +410,33 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne templateId: 'trader-agent', instanceName: `bob-${scenarioId}`, timeoutMs: 180_000, - env: traderEnv, }); s.spawned.push(escrow, alice, bob); console.log( `[${scenarioId}] up: escrow=${escrow.instanceName} ` + `alice=${alice.instanceName} bob=${bob.instanceName}`, ); + + // Fund both traders via FAUCET_REQUEST DMs. The faucet mints the + // tokens and sends to each trader's DIRECT://. We then poll each + // trader's portfolio until the balance arrives in `confirmed`. + console.log(`[${scenarioId}] funding alice + bob via faucet DM…`); + for (const t of [{ name: 'alice', tenant: alice }, { name: 'bob', tenant: bob }]) { + const recipient = t.tenant.tenantNametag + ? `@${t.tenant.tenantNametag}` + : `DIRECT://${t.tenant.tenantPubkey}`; + const deliveries = await s.faucetClient.request(s.faucet.tenantPubkey, { + recipient, + items: [ + { asset: 'UCT', amount: INITIAL_FUND_AMOUNT.toString() }, + { asset: 'USDU', amount: INITIAL_FUND_AMOUNT.toString() }, + ], + }, 240_000); + console.log( + `[${scenarioId}] ${t.name}: faucet delivered ${deliveries.length} item(s) ` + + `(transfer_ids: ${deliveries.map((d) => d.transfer_id.slice(0, 8)).join(', ')})`, + ); + } return { escrow, alice, bob }; } @@ -428,13 +497,40 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne const s = state; const { escrow, alice, bob } = await provisionTriple(scenarioId, controller); - // Funding happened via TRADER_TEST_FUND at trader startup (5000 each - // of UCT and USDU on both traders). The trader's main.ts logs - // "test_fund_mint_succeeded" once per coin if the mint worked, - // and the intent engine reads the resulting balance on its first - // scan cycle. Wait briefly so the first portfolio query reflects - // the post-mint balance. - await new Promise((r) => setTimeout(r, 5_000)); + // The faucet returned `acp.result` for each FAUCET_REQUEST, but the + // trader's payments.receive() loop runs on a 15s cycle — the + // delivered tokens may be in `unconfirmed` for up to ~30s after + // the send completes. Poll each trader's portfolio until both + // assets reach the funded amount in `confirmed` so set-strategy / + // create-intent operate on a fully-settled balance. + console.log(`[${scenarioId}] waiting for faucet-funded balances to confirm…`); + for (const t of [{ name: 'alice', tenant: alice }, { name: 'bob', tenant: bob }]) { + const deadline = Date.now() + FUNDING_BALANCE_TIMEOUT_MS; + let lastSnapshot: readonly PortfolioBalance[] = []; + while (Date.now() < deadline) { + try { + lastSnapshot = await portfolioAsync({ + cliPath: s.cliPath, cliHome: controller.cliHome, tenant: t.tenant.tenantPubkey, + }); + if ( + balanceOf(lastSnapshot, 'UCT') >= INITIAL_FUND_AMOUNT && + balanceOf(lastSnapshot, 'USDU') >= INITIAL_FUND_AMOUNT + ) { + break; + } + } catch { /* transient — keep polling */ } + await new Promise((resolve) => setTimeout(resolve, 5_000)); + } + const uct = balanceOf(lastSnapshot, 'UCT'); + const usdu = balanceOf(lastSnapshot, 'USDU'); + if (uct < INITIAL_FUND_AMOUNT || usdu < INITIAL_FUND_AMOUNT) { + throw new Error( + `[${scenarioId}] ${t.name} did not see UCT>=${INITIAL_FUND_AMOUNT} && USDU>=${INITIAL_FUND_AMOUNT} ` + + `within ${FUNDING_BALANCE_TIMEOUT_MS}ms. observed: UCT=${uct}, USDU=${usdu}`, + ); + } + console.log(`[${scenarioId}] ${t.name} balance confirmed: ${uct}UCT/${usdu}USDU`); + } // ---- 2. Configure trusted escrows on both traders -------------- // Sequential within-scenario (wallet.json contention; see provisionTriple). @@ -465,8 +561,10 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne `[${scenarioId}] pre-trade balances: ` + `alice=${aliceUctBefore}UCT/${aliceUsduBefore}USDU bob=${bobUctBefore}UCT/${bobUsduBefore}USDU`, ); - // Sanity: the self-mint must have happened (otherwise no UCT/USDU - // are available to trade and the swap will hang). + // Sanity: the faucet delivery must have landed (otherwise no + // UCT/USDU is available to trade and the swap will hang). The + // earlier polling loop already enforces this; this assertion is + // the explicit test contract. expect(aliceUctBefore + aliceUsduBefore).toBeGreaterThan(0n); expect(bobUctBefore + bobUsduBefore).toBeGreaterThan(0n); From c58a4637d708f93a10fbe74864b8a02759374398 Mon Sep 17 00:00:00 2001 From: Vladimir Rogojin Date: Tue, 5 May 2026 10:01:51 +0200 Subject: [PATCH 11/30] =?UTF-8?q?fix(test):=20pass=20escrow=5Faddress=20?= =?UTF-8?q?=E2=80=94=20was=20defaulting=20to=20literal=20'any'?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round 10 with faucet funding killed the spam-loop (29 deals → 2) but settlement still failed: traders sent `status` query to escrow, got "Swap not found" error, deal CANCELLED. Escrow log showed ZERO swap.announce_received events despite ping/pong working — i.e. the swap.announce DM never reached the escrow. Root cause: trader-service/src/trader/intent-engine.ts:836: const escrowAddress = params.escrow_address ?? DEFAULT_ESCROW; // DEFAULT_ESCROW = 'any' (intent-engine.ts:92) When the test omits escrow_address from create-intent, the intent (and resulting deal terms) carries the literal string 'any'. The swap-executor then tries to route swap.announce to 'any', which the SDK's transport layer can't resolve to a real peer. The escrow never gets the announce; subsequent status queries from the trader hit a swap that doesn't exist on the escrow's side. Fix lands in two repos: (1) sphere-cli — add `--escrow-address` flag to `sphere trader create-intent`. The flag was missing entirely so callers had no way to override 'any'. Updated: - src/trader/trader-commands.ts: CreateIntentOpts + buildCreateIntentParams + commander definition - rebuilt dist/ (2) trader-service test: - helpers/sphere-trader.ts: CreateIntentOpts + forward to `--escrow-address` argv - hma-trade-settlement test: pass `escrowAddress: escrow.tenantPubkey` for both alice and bob's intents Verified: 698 unit tests still pass; type-check clean. Whether this finally fixes the FAILED/CANCELLED settlement loop is the next live round's question. --- test/e2e-live/helpers/sphere-trader.ts | 9 +++++++++ test/e2e-live/hma-trade-settlement.e2e-live.test.ts | 13 +++++++++++++ 2 files changed, 22 insertions(+) diff --git a/test/e2e-live/helpers/sphere-trader.ts b/test/e2e-live/helpers/sphere-trader.ts index 12217d4..5658b25 100644 --- a/test/e2e-live/helpers/sphere-trader.ts +++ b/test/e2e-live/helpers/sphere-trader.ts @@ -162,6 +162,13 @@ export interface CreateIntentOpts extends TraderInvocationOpts { /** Total intent volume; matches the trader's ACP `volume_max` wire field. */ volumeMax: bigint; expiryMs?: number; + /** + * Escrow address (pubkey hex / DIRECT:// / PROXY://). When omitted, + * the trader defaults to 'any' (wildcard) which routes the swap to + * NO real escrow — settlement fails with "Swap not found". Tests + * and production callers MUST pass this for end-to-end settlement. + */ + escrowAddress?: string; } export interface CreatedIntent { @@ -179,6 +186,7 @@ export function createIntent(opts: CreateIntentOpts): CreatedIntent { '--volume-max', opts.volumeMax.toString(), ]; if (opts.expiryMs !== undefined) args.push('--expiry-ms', String(opts.expiryMs)); + if (opts.escrowAddress !== undefined) args.push('--escrow-address', opts.escrowAddress); const { result } = runTraderCommand('create-intent', args, opts); if (typeof result !== 'object' || result === null) { throw new Error(`create-intent: result is not an object. Got: ${JSON.stringify(result)}`); @@ -435,6 +443,7 @@ export async function createIntentAsync(opts: CreateIntentOpts): Promise Date: Tue, 5 May 2026 10:13:59 +0200 Subject: [PATCH 12/30] fix(test): use escrow.tenantDirectAddress instead of tenantPubkey MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round 11 progressed: pair-2 reached ACCEPTED for the first time (3 deals: FAILED, CANCELLED, ACCEPTED) — the escrow_address fix landed correctly. But the deal still failed at swap registration with a clearer error in the trader log: "swap_id_register_escrow_mismatch": negotiated_escrow: 02219b272c88584a4129d770edf5c08bcb5054d3c08496157c021da77cab8625a1 proposal_escrow: DIRECT://00002b89215d8b32323f083ceb154329115d3137c0c5325410d2cc0b0d5a111eb0ad3148b2ce proposal_escrow_pubkey: 219b272c88584a4129d770edf5c08bcb5054d3c08496157c021da77cab8625a1 The trader's swap-executor (line 714) checks: negotiatedEscrow === match.escrowDirectAddress `negotiatedEscrow` = `terms.escrow_address` (what we passed to create-intent — chain pubkey form, 02-prefixed 66 chars) `match.escrowDirectAddress` = the DIRECT://hex address (a structural hash via UnmaskedPredicateReference, NOT the pubkey). These are fundamentally different forms. The check expects them EQUAL, so the intent's escrow_address must be set to the DIRECT://-form to match what the SDK derives during proposal. Also updates `setStrategy(trustedEscrows: [...])` to use the same DIRECT:// form — the negotiation-handler's trusted check at negotiation-handler.ts:1084 does direct string equality (`trustedEscrows.includes(terms.escrow_address)`), so both sides must match. This was implicitly working before only because both sides were 'any'. Same fix could land in basic-roundtrip's createMatchingIntents helper (probably what makes the direct-docker version work — they must already pass directAddress somewhere) but that's out of scope for this commit. --- test/e2e-live/hma-trade-settlement.e2e-live.test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/e2e-live/hma-trade-settlement.e2e-live.test.ts b/test/e2e-live/hma-trade-settlement.e2e-live.test.ts index e7abe46..c1373e3 100644 --- a/test/e2e-live/hma-trade-settlement.e2e-live.test.ts +++ b/test/e2e-live/hma-trade-settlement.e2e-live.test.ts @@ -537,12 +537,12 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne console.log(`[${scenarioId}] set-strategy on alice + bob…`); await setStrategyAsync({ cliPath: s.cliPath, cliHome: controller.cliHome, tenant: alice.tenantPubkey, - trustedEscrows: [escrow.tenantPubkey], + trustedEscrows: [escrow.tenantDirectAddress], maxConcurrent: 5, }); await setStrategyAsync({ cliPath: s.cliPath, cliHome: controller.cliHome, tenant: bob.tenantPubkey, - trustedEscrows: [escrow.tenantPubkey], + trustedEscrows: [escrow.tenantDirectAddress], maxConcurrent: 5, }); @@ -589,7 +589,7 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne rateMin: tradeRate, rateMax: tradeRate, volumeMin: TRADE_VOLUME, volumeMax: TRADE_VOLUME, expiryMs: SWAP_TIMEOUT_MS, - escrowAddress: escrow.tenantPubkey, + escrowAddress: escrow.tenantDirectAddress, }); const bobIntent = await createIntentAsync({ cliPath: s.cliPath, cliHome: controller.cliHome, tenant: bob.tenantPubkey, @@ -598,7 +598,7 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne rateMin: tradeRate, rateMax: tradeRate, volumeMin: TRADE_VOLUME, volumeMax: TRADE_VOLUME, expiryMs: SWAP_TIMEOUT_MS, - escrowAddress: escrow.tenantPubkey, + escrowAddress: escrow.tenantDirectAddress, }); console.log( `[${scenarioId}] intents posted: alice=${aliceIntent.intentId.slice(0, 12)}… ` + From 46bdce5977b2f6c3c6fc7da47bdce70f425db64b Mon Sep 17 00:00:00 2001 From: Vladimir Rogojin Date: Tue, 5 May 2026 11:01:12 +0200 Subject: [PATCH 13/30] docs: HMA-SETTLEMENT-DIAGNOSTIC for follow-up session Round-by-round summary of the hma-trade-settlement debugging sessions, the bugs we found and fixed, the remaining symptom (trader doesn't process escrow's invoice_delivery DM despite the escrow logs reporting successful send), and three concrete hypotheses with bisect plans for the next session: H1: ACP listener consumes the DM before swap module sees it H2: payments.receive() loop races with swap module's DM dispatch H3: HMA-spawned container has higher relay-subscription latency H1 is the most likely candidate; the most direct test is to temporarily comment out the ACP listener's sphere.on('message:dm') subscription and re-run. If settlement completes, that's the bug. Includes file refs for the trader code paths involved, exact reproduce-steps for the failing scenario, and key log lines to grep for in trader/escrow container output. --- docs/HMA-SETTLEMENT-DIAGNOSTIC.md | 227 ++++++++++++++++++++++++++++++ 1 file changed, 227 insertions(+) create mode 100644 docs/HMA-SETTLEMENT-DIAGNOSTIC.md diff --git a/docs/HMA-SETTLEMENT-DIAGNOSTIC.md b/docs/HMA-SETTLEMENT-DIAGNOSTIC.md new file mode 100644 index 0000000..a6cf062 --- /dev/null +++ b/docs/HMA-SETTLEMENT-DIAGNOSTIC.md @@ -0,0 +1,227 @@ +# HMA Trade-Settlement Diagnostic + +**Status as of 2026-05-05** — `feat/hma-trade-settlement-live` branch, latest commit `5fb50f0`. + +The `hma-trade-settlement.e2e-live` test still fails. Settlement reaches +ACCEPTED on both scenarios but never COMPLETED. This document captures +what works, what doesn't, what we tried, and the remaining hypotheses +so a follow-up session can pick up where we stopped. + +--- + +## Goal + +End-to-end live proof that operators can: +1. Launch HMA over Sphere DM (no HTTP) +2. Spawn escrow + 2 traders + faucet via `sphere host spawn` +3. Fund traders via the new js-faucet agent over DM +4. Match buy/sell intents +5. Settle the swap (deal → COMPLETED on both sides) +6. Withdraw post-trade tokens to a controller-owned address + +Steps 1-4 work. Step 5 is where we're stuck. + +## Current state (round 12) + +| Layer | Direct-docker (basic-roundtrip) | HMA-spawned (this test) | +|---|---|---| +| Spawn | ✓ | ✓ | +| Funding | ✓ (selfMintFund) | ✓ (faucet via `FAUCET_REQUEST` DM) | +| set-strategy / portfolio / list-intents | ✓ | ✓ | +| Match found | ✓ | ✓ | +| Deal accepted | ✓ | ✓ (round 12 first time) | +| Swap announced to escrow | ✓ | ✓ (round 12) | +| Escrow creates deposit invoice | ✓ | ✓ (round 12 — log: `"Swap announced, deposit invoice created"`) | +| Escrow sends invoice DM to trader | ✓ | ✓ (round 12 — log: `diag_outbound_dm_sent invoice_delivery`) | +| **Trader receives invoice DM** | ✓ | ✗ (log: `invoice_target_addresses: null`) | +| Trader deposits | ✓ | ✗ (blocked) | +| Swap COMPLETED | ✓ | ✗ (blocked) | + +The break is at **trader-side ingest of the escrow's invoice_delivery DM**. + +## Round-by-round progression + +| Round | Outcome | Bug found / fix | +|---|---|---| +| 1-7 | Various early failures | controller wallet races, faucet name mapping, etc. — all fixed | +| 8 | 29 deals each, locked in PROPOSED→FAILED→CANCELLED loop | TRADER_TEST_FUND self-mint produces issuer==sender tokens that confuse swap | +| 9 | Same as 8 | Cross-scenario rate differentiation didn't help | +| 10 | 2 deals each, never reach ACCEPTED, escrow logs `Swap not found` | Diagnosed: trader sends `status` query but never `swap.announce` | +| 11 | 1 scenario reaches ACCEPTED for first time | Trader's `intent-engine.ts:836` defaults `escrow_address` to literal `'any'` when CLI omits `--escrow-address` | +| 12 | **Both scenarios reach ACCEPTED**, escrow creates+sends invoice, but trader doesn't process it | Layer-mismatch fixed (`escrow.tenantPubkey` vs `escrow.tenantDirectAddress` — swap-executor compares to DIRECT://hex) | + +## Bugs already fixed (committed on `feat/hma-trade-settlement-live`) + +1. **Faucet integration** (commit `b2659a3`) — replaces `TRADER_TEST_FUND` self-mint and broken public-faucet HTTP. Fund traders via `FAUCET_REQUEST` DMs to a shared js-faucet agent. Killed the spam-loop. + +2. **`--escrow-address` flag in sphere-cli** (commit `c58a463` in trader-service; sphere-cli rebuild required) — `sphere trader create-intent` previously had no way to set `escrow_address`, so the trader defaulted to `'any'` (per `trader-service/src/trader/intent-engine.ts:836`). The swap-executor then tried to route `swap.announce` to the literal string `'any'`. Now passes through to ACP wire field correctly. + +3. **escrow_address must be DIRECT://hex, not chain pubkey** (commit `5fb50f0`) — `swap-executor.ts:714` compares `terms.escrow_address === match.escrowDirectAddress`. The DIRECT://hex address is structurally derived (`UnmaskedPredicateReference(pubkey).toAddress()`), NOT just `DIRECT://${pubkey}`. Test now passes `escrow.tenantDirectAddress` to both `setStrategy({trustedEscrows: [...]})` and `createIntent({escrowAddress: ...})`. + +## The remaining symptom (in detail) + +Round 12 trader log (`alice` container) shows: + +``` +swap_id_registered (deal_id=..., swap_id=169ea57a..., matched_by="proposal_info") +swap_deposit_target_diag (every ~3s): + swap_id: 169ea57a... + escrowDirectAddress: DIRECT://000055759f52413cab92... ← correct + manifest_party_a_currency: UCT + manifest_party_a_value: 10 + manifest_party_b_currency: USDU + manifest_party_b_value: 10 + invoice_target_addresses: null ← never populated + invoice_target_assets: null +``` + +Round 12 escrow log (same swap_id) shows: + +``` +Swap announced, deposit invoice created invoice_id: 00004af2b309ee84 +diag_invoice_delivery_attempt party: A recipient_prefix: 8ff3bcef9e1aa95d +diag_outbound_dm_sending message_type: invoice_delivery payload_bytes: 5789 +diag_outbound_dm_sent message_type: invoice_delivery +diag_invoice_delivery_complete +[same for party B] +``` + +Escrow believes it sent the DM successfully. Trader has no log entry showing receipt. + +## Why direct-docker works but HMA-spawned doesn't (open question) + +Same trader image (`trader:local`), same escrow image. The difference is the runtime environment: + +| | direct-docker | HMA-spawned | +|---|---|---| +| `UNICITY_MANAGER_PUBKEY` | unset | set | +| `UNICITY_MANAGER_DIRECT_ADDRESS` | unset | set | +| `UNICITY_BOOT_TOKEN` | unset | set | +| ACP heartbeats every 5s | none (no manager to talk to) | active | +| ACP DM listener (`sphere.on('message:dm')`) | active but bails (no manager pubkey to match) | active and validating against manager pubkey | +| Periodic `payments.receive({finalize:true})` loop | active | active | + +Both setups have the ACP listener attached (it's compiled into `startTrader()`). +The difference is whether it has a manager_pubkey to filter against. + +## Hypotheses for the remaining bug + +### H1 — ACP listener consumes the invoice DM before the swap module sees it + +The trader's `acp-adapter/main.ts` attaches `sphere.on('message:dm', ...)` early. +The Sphere SDK's swap module also subscribes to incoming DMs. + +If the order is: ACP listener fires first → marks DM as read → swap module's +listener sees `dm.isRead === true` → skips, the invoice_delivery never reaches +the swap-executor's state machine. + +In direct-docker, the ACP listener has no `manager_pubkey` to filter against, +so its early-return path is different (it may not "consume" the DM). + +**Bisect**: temporarily make the trader's ACP listener bail BEFORE any SDK +state mutation when the DM is not addressed at the manager. Or: instrument +the SDK's incoming-DM dispatch to log every observed DM and see whether +invoice_delivery from the escrow's pubkey is observed at the SDK level at all. + +### H2 — Periodic `payments.receive({finalize:true})` races with swap module's DM consumption + +The trader's main loop (`src/trader/main.ts:545`) calls +`sphere.payments.receive({ finalize: true })` every 5s. We've seen +`ENOENT: wallet.json.tmp` errors in trader logs from this and the heartbeat +loop racing on atomic temp+rename writes. + +If the `wallet.json` write race corrupts the swap module's DM-state cache, +incoming swap DMs may be silently dropped. + +**Bisect**: lengthen `SYNC_INTERVAL_MS` (currently 5s) to 60s and see if +settlement starts working. If yes, race is the cause. + +### H3 — HMA-spawned container's relay subscription latency + +HMA's docker create injects more env vars and starts the container with +`tini` as PID 1. The relay subscription inside the trader may not be fully +established by the time the escrow sends the invoice. NIP-17 events that +arrive before the subscription is active are NOT replayable for that +subscriber session. + +`sphere.fetchPendingEvents()` is supposed to catch missed events, but its +periodic call (also 5s in the sync loop) may be racing or filtering. + +**Bisect**: add an explicit `await sphere.fetchPendingEvents()` after the +trader's first STATUS query and before the swap-executor begins polling for +invoice_target. If the invoice arrives after this explicit fetch, latency +is the cause. + +## Proposed debugging plan for the follow-up session + +1. **Instrument the trader** to log EVERY incoming DM at the raw level: + ```typescript + sphere.on('message:dm', (msg) => { + log.info({ + sender: msg.senderPubkey.slice(0, 16), + size: msg.content.length, + firstByte: msg.content[0], + }, 'raw_dm_observed'); + }); + ``` + Place this BEFORE any other listener attaches. Re-run the test. + - If invoice_delivery DMs appear in `raw_dm_observed`: the receive-side works, + bug is in dispatch (H1 or H2). + - If they don't appear: bug is in transport/subscription (H3). + +2. **For H1 (most likely)**: temporarily comment out the ACP listener's + `sphere.on('message:dm', ...)` subscription in `acp-adapter/main.ts` and + re-run. If settlement completes, the ACP listener is consuming the DM. + +3. **For H3**: capture network-level evidence — point both HMA and + direct-docker at the SAME relay URL and compare event-arrival timestamps + (instrument the relay or use Wireshark on `wss://nostr-relay.testnet.unicity.network`). + +## Key files + +- Test: `test/e2e-live/hma-trade-settlement.e2e-live.test.ts` +- Test helpers: + - `test/e2e-live/helpers/faucet-client.ts` (in-process Sphere wallet for `FAUCET_REQUEST`) + - `test/e2e-live/helpers/manager-process.ts` + - `test/e2e-live/helpers/hma-spawn.ts` + - `test/e2e-live/helpers/sphere-trader.ts` +- Trader code likely involved: + - `src/trader/main.ts:545` — `payments.receive({ finalize: true })` periodic + - `src/trader/swap-executor.ts:714` — `negotiatedEscrow === escrowDirectAddress` check + - `src/trader/intent-engine.ts:836` — `escrow_address ?? DEFAULT_ESCROW` + - `src/acp-adapter/main.ts` (Phase 4h decoupling) — ACP DM listener +- Sphere SDK: `@unicitylabs/sphere-sdk` payments + swap modules + +## Reproducing the failure + +```bash +# Build all required images +cd /home/vrogojin && docker build -f trader-service/Dockerfile \ + -t ghcr.io/vrogojin/agentic-hosting/trader:local . +cd /home/vrogojin && docker build -f js-faucet/Dockerfile \ + -t ghcr.io/unicitynetwork/agentic-hosting/faucet:local . +cd /home/vrogojin/agentic_hosting && npm run build +cd /home/vrogojin/sphere-cli-work/sphere-cli && npm run build + +# Run the test +cd /home/vrogojin/trader-service +git checkout feat/hma-trade-settlement-live +npm run test:e2e-live -- test/e2e-live/hma-trade-settlement.e2e-live.test.ts +# Expect: FAIL ~590s, both pairs reach ACCEPTED, neither reaches COMPLETED. + +# Inspect trader log: +docker ps -a --filter "name=alice-p" --filter "status=exited" --format "{{.Names}}" | head -1 \ + | xargs -I {} docker logs {} 2>&1 | grep -E "swap_deposit_target_diag|swap_id_register|swap_announced" + +# Inspect escrow log: +docker ps -a --filter "name=escrow-p" --filter "status=exited" --format "{{.Names}}" | head -1 \ + | xargs -I {} docker logs {} 2>&1 | grep -iE "announce|invoice_delivery|outbound_dm" +``` + +## Out of scope for this debugging session + +- Production hardening of js-faucet (rate limiting, batched mints, etc.) +- Pushing js-faucet image to ghcr.io/unicitynetwork (needs PAT) +- Adding `faucet-agent` template entry to agentic-hosting/config/templates.json +- `sphere faucet request` subcommand in sphere-cli +- Withdraw-via-HMA live test From ce7ce2c19d029f3b677daa12c2286dbb395bad53 Mon Sep 17 00:00:00 2001 From: Vladimir Rogojin Date: Tue, 5 May 2026 11:46:12 +0200 Subject: [PATCH 14/30] docs: rule out H1 + add SDK middleware-pipeline architectural note MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Investigated sphere-sdk's event dispatch. Incoming DMs go through TWO INDEPENDENT paths (dist/index.js:13910-13918): 1. emitEvent("message:dm") — generic event bus (sphere.on subscribes) 2. iterates dmHandlers Set — onDirectMessage(handler) registers here. PaymentsModule (line 18816) and SwapModule (line 24212) both subscribe via this set. Both fire unconditionally for every DM. Neither preempts the other — they're independent channels. The ACP listener (event bus) cannot block the SwapModule (dmHandlers) from receiving a DM. So H1 (ACP listener consuming the invoice_delivery before SwapModule sees it) is wrong. Updated debugging plan focuses on: - what gets through to SwapModule.handleIncomingDM - SDK-internal silent-reject paths (sig check, swap-id-not-found, protocol-version mismatch, dedup) - transport-layer hole (relay subscription / NIP-17 decryption) Also added an architectural-follow-up note: even though the dual-path dispatch isn't the cause of THIS bug, the lack of propagation control (no `consume vs pass through` semantics) is a future-bug source as more consumers attach. A koa-compose-style middleware chain (use(handler, priority) with next() semantics) in sphere-sdk's CommunicationsModule would let consumers declaratively filter / consume / pass DMs. ~40 lines of inline impl, no new runtime dep. --- docs/HMA-SETTLEMENT-DIAGNOSTIC.md | 95 ++++++++++++++++++++----------- 1 file changed, 61 insertions(+), 34 deletions(-) diff --git a/docs/HMA-SETTLEMENT-DIAGNOSTIC.md b/docs/HMA-SETTLEMENT-DIAGNOSTIC.md index a6cf062..79fee87 100644 --- a/docs/HMA-SETTLEMENT-DIAGNOSTIC.md +++ b/docs/HMA-SETTLEMENT-DIAGNOSTIC.md @@ -106,22 +106,31 @@ The difference is whether it has a manager_pubkey to filter against. ## Hypotheses for the remaining bug -### H1 — ACP listener consumes the invoice DM before the swap module sees it +### ~~H1 — ACP listener consumes the invoice DM before the swap module sees it~~ — RULED OUT -The trader's `acp-adapter/main.ts` attaches `sphere.on('message:dm', ...)` early. -The Sphere SDK's swap module also subscribes to incoming DMs. +**Update**: investigated sphere-sdk's event architecture. Incoming DMs are +dispatched via TWO INDEPENDENT paths in `dist/index.js`: -If the order is: ACP listener fires first → marks DM as read → swap module's -listener sees `dm.isRead === true` → skips, the invoice_delivery never reaches -the swap-executor's state machine. + - line 13910: `deps.emitEvent("message:dm", message)` — generic event bus + (this is what `sphere.on('message:dm', ...)` subscribes to). + - line 13911-13918: iterates `dmHandlers` set + (this is what `sphere.communications.onDirectMessage(handler)` registers + into; used internally by PaymentsModule (line 18816) and SwapModule + (line 24212)). -In direct-docker, the ACP listener has no `manager_pubkey` to filter against, -so its early-return path is different (it may not "consume" the DM). +Both fire unconditionally for every DM. No propagation control, no ordering +between the two paths. The ACP listener (on the event bus) and the SDK's +SwapModule (on `dmHandlers`) are on independent channels — they each get +their own copy. The ACP listener CANNOT preempt the SwapModule. -**Bisect**: temporarily make the trader's ACP listener bail BEFORE any SDK -state mutation when the DM is not addressed at the manager. Or: instrument -the SDK's incoming-DM dispatch to log every observed DM and see whether -invoice_delivery from the escrow's pubkey is observed at the SDK level at all. +So this hypothesis is incorrect. The remaining candidates are below. + +**Note on architecture**: the dual-path dispatch with no propagation control +is itself a design smell — there's no way for a handler to say "I consumed +this, don't deliver it to other consumers." A koa-compose-style middleware +chain (`use(handler, priority)` with `next()` semantics) would be cleaner +and would let the trader's ACP filter declaratively consume non-ACP DMs. +Tracked separately; not on the critical path for THIS bug. ### H2 — Periodic `payments.receive({finalize:true})` races with swap module's DM consumption @@ -154,28 +163,46 @@ is the cause. ## Proposed debugging plan for the follow-up session -1. **Instrument the trader** to log EVERY incoming DM at the raw level: - ```typescript - sphere.on('message:dm', (msg) => { - log.info({ - sender: msg.senderPubkey.slice(0, 16), - size: msg.content.length, - firstByte: msg.content[0], - }, 'raw_dm_observed'); - }); - ``` - Place this BEFORE any other listener attaches. Re-run the test. - - If invoice_delivery DMs appear in `raw_dm_observed`: the receive-side works, - bug is in dispatch (H1 or H2). - - If they don't appear: bug is in transport/subscription (H3). - -2. **For H1 (most likely)**: temporarily comment out the ACP listener's - `sphere.on('message:dm', ...)` subscription in `acp-adapter/main.ts` and - re-run. If settlement completes, the ACP listener is consuming the DM. - -3. **For H3**: capture network-level evidence — point both HMA and - direct-docker at the SAME relay URL and compare event-arrival timestamps - (instrument the relay or use Wireshark on `wss://nostr-relay.testnet.unicity.network`). +H1 is ruled out (see updated hypothesis above), so start with the SDK's +internal DM dispatch. + +1. **Instrument sphere-sdk's `CommunicationsModule.handleIncoming` (or + equivalent) to log EVERY incoming DM at the raw level**, BEFORE any + filtering / dedup. The log line should include sender prefix, payload + size, and the message id. With this in place, re-run the test: + + - If the escrow's invoice_delivery DM appears in the trader's raw log: + → the SDK is receiving it. Bug is downstream (handleIncomingDM rejecting, + SwapModule not registering it as the active swap, etc.). Continue to + step 2. + - If it does NOT appear: bug is at the transport layer (relay subscription + latency, DM-decryption failure, recipient mismatch). Skip to step 3. + +2. **For the "received but not processed" case (most likely)**: instrument + `SwapModule.handleIncomingDM` in `sphere-sdk/dist/index.js:24212` (or + wherever its body is) to log every entry and the path it takes. Possible + silent rejections: + - signature verification fails (wrong chain pubkey) + - swap-id-not-found (the trader doesn't have the swap registered when + the invoice arrives — race between announce-ack and invoice_delivery) + - protocol version mismatch (trader v1 vs escrow v2 or vice versa) + - dedup hit (`dm.isRead === true` because the SDK persisted it from a + prior backfill — relevant if the escrow re-sends after a wallet reload) + +3. **For the transport-layer case**: capture network-level evidence — + instrument the trader's NostrTransportProvider to log every Nostr event + it receives at the wire level. If the kind:1059 wraps for the escrow's + pubkey arrive but never decrypt to the trader, decryption is failing. + If they don't arrive at all, the relay subscription has a hole. + +4. **Architectural follow-up (separate effort)**: introduce a propagation- + aware middleware chain in sphere-sdk's CommunicationsModule (koa-compose + style — `use(mw, priority)` with `next()`), so that future consumers + (ACP listener, app code) can declaratively filter / consume / pass DMs + in an ordered pipeline. The current dual-path dispatch + (`emitEvent` + `dmHandlers` running in parallel with no coordination) + isn't the cause of THIS bug but is an obvious source of future bugs as + more consumers attach. ## Key files From 8f716ae94a86c672bee379b120ad825ca78fec8a Mon Sep 17 00:00:00 2001 From: Vladimir Rogojin Date: Tue, 5 May 2026 12:30:35 +0200 Subject: [PATCH 15/30] test: override escrow image to :local to bypass deployed v0.1 asymmetric bug MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A focused investigation agent traced round-12 escrow logs and found: diag_invoice_delivery_attempt party=A ✓ logged diag_outbound_dm_sending invoice_delivery → A ✓ logged diag_outbound_dm_sent invoice_delivery → A ✓ logged diag_invoice_delivery_complete party=A ✓ logged diag_invoice_delivery_attempt party=B ✓ logged [ no diag_outbound_dm_sending invoice_delivery → B ] diag_invoice_delivery_complete party=B ✓ logged anyway The deployed `escrow:v0.1` image's `deliverDepositInvoice` function exits "normally" for party B without actually invoking `sphere.communications.sendDM` — only one recipient receives the invoice. Trader stalls at ACCEPTED waiting for the invoice that never arrives. Asymmetry repeats deterministically across multiple swaps; basic-roundtrip works direct-docker because it only asserts the buyer's side. The current escrow-service source (`message-handler.ts:202-253`) LOOKS correct — both A's and B's reply() calls are awaited with the same code path. The deployed image was built from an unsynced source commit (the JS at /app/dist/sphere/message-handler.js doesn't match HEAD). Rebuilding escrow:local from current source and overriding the image in the test should clear the regression. Build instructions added to the test header. After this, expect hma-trade-settlement to finally COMPLETE both scenarios. Independent of this fix, an upstream PR against escrow-service to harden deliverDepositInvoice (use Promise.allSettled and log rejection reasons) so a similar build-time defect can never silently re-occur is filed in the diagnostic doc as a follow-up item. --- docs/HMA-SETTLEMENT-DIAGNOSTIC.md | 62 +++++++++++++++++++ .../hma-trade-settlement.e2e-live.test.ts | 9 +++ 2 files changed, 71 insertions(+) diff --git a/docs/HMA-SETTLEMENT-DIAGNOSTIC.md b/docs/HMA-SETTLEMENT-DIAGNOSTIC.md index 79fee87..aad9d5c 100644 --- a/docs/HMA-SETTLEMENT-DIAGNOSTIC.md +++ b/docs/HMA-SETTLEMENT-DIAGNOSTIC.md @@ -106,6 +106,68 @@ The difference is whether it has a manager_pubkey to filter against. ## Hypotheses for the remaining bug +### Update 2026-05-05: ROOT CAUSE FOUND — escrow side, not trader side + +A focused investigation agent traced the full flow on both ends and found +the bug is in the **escrow's `deliverDepositInvoice` function** (compiled +into `escrow:v0.1` at `/app/dist/sphere/message-handler.js`). It is +**asymmetric**: party A's invoice always delivers; party B's never does. + +Evidence (from one round-12 escrow's logs, repeats across multiple swaps): + +``` +diag_invoice_delivery_attempt party=A ← logged +diag_outbound_dm_sending message_type=invoice_delivery recipient=A ← logged +diag_outbound_dm_sent message_type=invoice_delivery recipient=A ← logged +diag_invoice_delivery_complete party=A ← logged + +diag_invoice_delivery_attempt party=B ← logged +[NO diag_outbound_dm_sending for invoice_delivery to B — never appears] +diag_invoice_delivery_complete party=B ← logged anyway +``` + +The `complete` log fires (no thrown exception); the `sending` log doesn't +(no actual `sendDM` call). The function exits "normally" without delivering +the invoice to party B. + +The deployed `escrow:v0.1` image was built from an unsynced source commit +(JS at `/app/dist/sphere/message-handler.js` does NOT match +`/home/vrogojin/escrow-service/src/sphere/message-handler.ts` at HEAD). +The exact mechanism inside the diverged image (early-return that was missed, +fire-and-forget reference instead of `await reply(...)`, build-step DCE, etc.) +requires access to the unsynced commit to confirm — but the fix is the same +either way: rebuild + re-tag. + +**Why basic-roundtrip works direct-docker with the same image**: +basic-roundtrip uses ONE trader pair. The HMA test uses two pairs concurrently. +Party-A vs party-B is determined by the canonical pubkey ordering in the +swap manifest — concurrent swaps may always produce the same A/B alignment. +But basic-roundtrip's single pair may happen to land in a way where the +party-B-broken path doesn't matter (e.g., the test only asserts the buyer's +side, not the seller's). Worth re-running basic-roundtrip with the +rebuilt image to confirm; the asymmetry is a real defect regardless. + +**Remediation (in priority order)**: + +1. Rebuild `escrow:local` from `/home/vrogojin/escrow-service` source and + re-tag as the test's image. Re-run hma-trade-settlement and expect + COMPLETED. +2. After settlement works: file an upstream issue + PR against escrow-service + to harden `deliverDepositInvoice` — use `Promise.allSettled([reply(B,...), + reply(A,...)])` and log the rejection reasons explicitly so silent failures + are impossible. +3. Sphere-sdk secondary defect (independent): `SwapModule.handleIncomingDM` + walks `accepted → announced` via `status_result.state` (SwapModule.ts:3119–3171) + even when `swap.depositInvoiceId` is unset. This is what made the trader's + diag log say "registered, polling for invoice" while accounting actually + has no invoice record. Constrain the walk to require both + `swap.depositInvoiceId !== undefined` AND `accounting.getInvoice(id) !== null` + before transitioning. Medium priority — only relevant once the escrow + regression is fixed. + +The hypotheses below (H2/H3) are now **superseded** by the escrow-side root +cause. Kept for historical context. + ### ~~H1 — ACP listener consumes the invoice DM before the swap module sees it~~ — RULED OUT **Update**: investigated sphere-sdk's event architecture. Incoming DMs are diff --git a/test/e2e-live/hma-trade-settlement.e2e-live.test.ts b/test/e2e-live/hma-trade-settlement.e2e-live.test.ts index c1373e3..214b89e 100644 --- a/test/e2e-live/hma-trade-settlement.e2e-live.test.ts +++ b/test/e2e-live/hma-trade-settlement.e2e-live.test.ts @@ -260,6 +260,15 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne if (t.template_id === 'trader-agent') { t.image = 'ghcr.io/vrogojin/agentic-hosting/trader:local'; } + // The published escrow:v0.1 has an asymmetric bug in + // deliverDepositInvoice — the second recipient's invoice_delivery + // DM is never put on the wire, so swaps stall at "ACCEPTED" with + // the trader polling for an invoice that the escrow never sent + // (diag agent traced this 2026-05-05; see HMA-SETTLEMENT-DIAGNOSTIC.md). + // Build escrow:local from current source and use it here. + if (t.template_id === 'escrow-service') { + t.image = 'ghcr.io/vrogojin/agentic-hosting/escrow:local'; + } } if (!baseTemplates.templates.some((t) => t.template_id === 'faucet-agent')) { baseTemplates.templates.push({ From fb03cda3fe077d9db8d88469003fe82141da4769 Mon Sep 17 00:00:00 2001 From: Vladimir Rogojin Date: Fri, 8 May 2026 14:15:01 +0200 Subject: [PATCH 16/30] =?UTF-8?q?docs(diagnostic):=20rounds=2014-15=20upda?= =?UTF-8?q?te=20=E2=80=94=20instrumented=20escrow=20committed,=20awaiting?= =?UTF-8?q?=20relay?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/HMA-SETTLEMENT-DIAGNOSTIC.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/HMA-SETTLEMENT-DIAGNOSTIC.md b/docs/HMA-SETTLEMENT-DIAGNOSTIC.md index aad9d5c..3efb6bb 100644 --- a/docs/HMA-SETTLEMENT-DIAGNOSTIC.md +++ b/docs/HMA-SETTLEMENT-DIAGNOSTIC.md @@ -147,6 +147,22 @@ party-B-broken path doesn't matter (e.g., the test only asserts the buyer's side, not the seller's). Worth re-running basic-roundtrip with the rebuilt image to confirm; the asymmetry is a real defect regardless. +**Update 2026-05-08 (round 14)**: rebuilt `escrow:local` from current +`escrow-service` HEAD; bug REPRODUCED unchanged. So the bug is in current +source, not the divergence between the deployed image and HEAD as the +original agent suspected. The deployed `escrow:v0.1` image had additional +`diag_invoice_delivery_*` log lines that do NOT exist in HEAD — that's +what made the divergence look like the cause; the bug is structural. + +**Update 2026-05-08 (round 15)**: branched `escrow-service` to +`debug/instrument-deliver-invoice` (`c0e19ea`), instrumented every code +path of `deliverDepositInvoice` (enter / no-id / no-token / sending / +sent / threw) plus per-party try/catch in the announce-handler's for-loop. +Round 15 itself failed at preflight — testnet Nostr relay's write path +went down again (intermittent — every WS publish-kind:* returns no OK). +The instrumented build is committed and pushed; next live attempt against +this branch will produce log lines pinpointing party B's actual path. + **Remediation (in priority order)**: 1. Rebuild `escrow:local` from `/home/vrogojin/escrow-service` source and From 1bea23011d2e74b9a2f04ddb774f9194c3bc3dce Mon Sep 17 00:00:00 2001 From: Vladimir Rogojin Date: Fri, 8 May 2026 14:23:38 +0200 Subject: [PATCH 17/30] feat(test): local-infra Nostr relay + UNICITY_NOSTR_RELAYS support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The testnet relay (wss://nostr-relay.testnet.unicity.network) has had two write-path outages in 4 days: read path works (connect+subscribe OK) but every publish-kind:* times out, so DM-based settlement tests stall before the first DM ships. Pattern is intermittent; recovery unpredictable. Local relay infra removes the dependency. Mirrors the harness uxf already uses (/home/vrogojin/uxf/tests/e2e/local-infra/), adapted for trader-service multi-container topology: test/e2e-live/local-infra/docker-compose.yml - container_name: trader-e2e-relay (avoid uxf collision) - port bind: 0.0.0.0:7777:8080 — HMA-spawned tenants run in their own Docker containers; they cannot reach host loopback so we expose on all interfaces and connect via the host bridge gateway IP. Relay image pin: ghcr.io/unicitynetwork/unicity-tokens-relay:sha-1e1b544 test/e2e-live/local-infra/relay.ts - bootLocalRelay() / stopRelay() lifecycle - getLocalRelayUrlForContainers() returns the URL HMA-spawned tenants should connect to (`ws://:7777`). Falls back to host.docker.internal if `docker network inspect` fails — Docker Desktop resolves this automatically; Linux Docker needs --add-host=host.docker.internal:host-gateway on the HMA-spawned containers (separate plumbing required). Service-side support for UNICITY_NOSTR_RELAYS env override (mirrors the existing js-faucet pattern at acp-adapter/main.ts:122-128): trader-service: src/trader/main.ts:250-265 escrow-service: src/acp-adapter/main.ts:101-117 (committed separately; see escrow-service repo) agentic-hosting: src/host-manager/main.ts:457-474 (committed separately; see agentic-hosting repo) Each service reads UNICITY_NOSTR_RELAYS (with SPHERE_NOSTR_RELAYS fallback) and passes the relay list to createNodeProviders' transport config when set; falls through to the network preset's defaults when unset. No behavioral change when the env var is absent. Out of scope (next step): - global-setup.ts wiring to boot the relay when TRADER_E2E_LOCAL_RELAY=1 is set, then propagate the bridge URL to the HMA + spawned tenants via env passthrough - testing the full stack against the local relay - opt-in via the hma-trade-settlement test --- src/trader/main.ts | 16 ++ test/e2e-live/local-infra/docker-compose.yml | 71 +++++++ test/e2e-live/local-infra/relay.ts | 187 +++++++++++++++++++ 3 files changed, 274 insertions(+) create mode 100644 test/e2e-live/local-infra/docker-compose.yml create mode 100644 test/e2e-live/local-infra/relay.ts diff --git a/src/trader/main.ts b/src/trader/main.ts index f21d4cf..a775d43 100644 --- a/src/trader/main.ts +++ b/src/trader/main.ts @@ -247,6 +247,21 @@ export async function startTrader(): Promise { const trustbasePath = join(config.data_dir, 'trustbase.json'); writeFileSync(trustbasePath, await tbResponse.text()); + // Optional Nostr-relay override. Set `UNICITY_NOSTR_RELAYS` (or + // `SPHERE_NOSTR_RELAYS` as a fallback) to a comma-separated list of + // WebSocket URLs to replace the network preset's relays — used by the + // local-infra e2e harness to point at a Docker-hosted relay when the + // public testnet relay's write path is degraded. Empty/unset → default. + const relayOverride = (() => { + const raw = process.env['UNICITY_NOSTR_RELAYS'] ?? process.env['SPHERE_NOSTR_RELAYS']; + if (!raw) return undefined; + const relays = raw.split(',').map((s) => s.trim()).filter((s) => s.length > 0); + return relays.length > 0 ? relays : undefined; + })(); + if (relayOverride) { + logger.info('nostr_relays_override_active', { relays: relayOverride }); + } + // Initialize Sphere wallet with market, swap, and accounting modules logger.info('initializing_sphere', { network: config.network, data_dir: config.data_dir }); const providers = createNodeProviders({ @@ -257,6 +272,7 @@ export async function startTrader(): Promise { trustBasePath: trustbasePath, apiKey: resolveApiKey(), }, + ...(relayOverride ? { transport: { relays: relayOverride } } : {}), }); // 2026-04-30 FIX (basic-roundtrip flake investigation): expand the diff --git a/test/e2e-live/local-infra/docker-compose.yml b/test/e2e-live/local-infra/docker-compose.yml new file mode 100644 index 0000000..0f19bac --- /dev/null +++ b/test/e2e-live/local-infra/docker-compose.yml @@ -0,0 +1,71 @@ +# ============================================================================= +# Local infrastructure for trader-service e2e tests. +# +# Boots a local Nostr relay so the e2e suite can run against a +# deterministic, in-process stack instead of the public testnet relay — +# useful when: +# - the testnet relay's write path is broken / silently dropping +# publishes (the 2026-05-08 outage that re-motivated this harness); +# - CI runs need reproducibility (no shared rate limits, no +# nametag collisions across concurrent jobs); +# - we are debugging the SDK's interaction with Nostr and want a +# SQLite-backed relay we can `docker exec sqlite3 ./events.db`. +# +# Source: ported from /home/vrogojin/uxf/tests/e2e/local-infra/. +# +# Aggregator (L3) and IPFS gateway are NOT replaced — the public +# Unicity testnet aggregator/IPFS are reliable and the sphere-sdk has +# no aggregator stub that would round-trip real inclusion proofs. +# E2E tests that need the aggregator continue to talk to: +# wss://goggregator-test.unicity.network +# https://ipfs.unicity.network +# +# Usage from the global-setup: +# docker compose -f tests/e2e/local-infra/docker-compose.yml up -d +# …run tests with E2E_LOCAL_RELAY_URL=ws://127.0.0.1:7777 … +# docker compose -f tests/e2e/local-infra/docker-compose.yml down -v +# +# Versions are pinned. Update via the parent global-setup so any change +# here is paired with a documented test-suite re-validation. +# ============================================================================= + +services: + # --------------------------------------------------------------------------- + # Local Nostr relay — ghcr.io/unicitynetwork/unicity-tokens-relay + # + # The Unicity org publishes a built nostr-rs-relay image. We pin to a + # specific SHA so unrelated relay updates don't silently change test + # behaviour. Bump this when validating against a newer relay release. + # --------------------------------------------------------------------------- + relay: + image: ghcr.io/unicitynetwork/unicity-tokens-relay:sha-1e1b544 + container_name: trader-e2e-relay + restart: unless-stopped + ports: + # Bind to ALL interfaces so HMA-spawned tenants (which run in + # separate Docker containers) can reach the relay via the host + # bridge gateway IP (typically 172.17.0.1 on Linux Docker). The + # global-setup detects this gateway at boot and passes it to the + # spawned tenants via UNICITY_NOSTR_RELAYS. If you need stricter + # isolation, change to "127.0.0.1:7777:8080" and run the test on + # the host network mode (not the HMA's default bridge). + # 7777 is arbitrary but unlikely to collide with developer tooling. + - "0.0.0.0:7777:8080" + volumes: + # Persist the SQLite event log between container restarts so we + # can post-mortem a failing test by `docker exec sqlite3`-ing + # into the volume. `down -v` wipes it; `down` (without -v) + # keeps it for inspection. + - relay-data:/usr/src/app/db + healthcheck: + # NIP-11 info doc on HTTP returns the relay metadata; if it + # responds 200 we know the WebSocket listener is also up + # (same handler). + test: ["CMD-SHELL", "wget -q -O - --header='Accept: application/nostr+json' http://127.0.0.1:8080 || exit 1"] + interval: 5s + timeout: 3s + retries: 12 + start_period: 5s + +volumes: + relay-data: diff --git a/test/e2e-live/local-infra/relay.ts b/test/e2e-live/local-infra/relay.ts new file mode 100644 index 0000000..6088a18 --- /dev/null +++ b/test/e2e-live/local-infra/relay.ts @@ -0,0 +1,187 @@ +/** + * Local Nostr relay lifecycle. + * + * Wraps `docker compose up/down` for tests/e2e/local-infra/docker-compose.yml. + * The relay container exposes 127.0.0.1:7777 — a fresh SQLite event log + * is created in the named volume on first boot; subsequent runs reuse + * the volume unless the caller explicitly requests `wipe: true` (which + * passes `-v` to `compose down` to drop persisted state). + * + * The compose file is the source of truth for the image pin; this helper + * is intentionally thin so version bumps don't drift between two places. + * + * @module tests/e2e/local-infra/relay + */ + +import { spawnSync } from 'node:child_process'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const COMPOSE_FILE = join(__dirname, 'docker-compose.yml'); + +/** + * URL the relay listens on (matches the docker-compose port mapping). + * + * Tests that are gated on E2E_LOCAL_INFRA=1 can read this directly, OR + * (preferred) read the SPHERE_NOSTR_RELAYS env var which the global- + * setup exports — that lets us swap the relay endpoint without + * touching test source. + */ +export const LOCAL_RELAY_URL = 'ws://127.0.0.1:7777'; + +/** + * Probe URL — same host, NIP-11 info doc on HTTP. + * + * The relay returns the same metadata over HTTP that the WebSocket + * upgrade serves to clients sending `Accept: application/nostr+json`. + * Cheap to poll during boot wait. + */ +const LOCAL_RELAY_HTTP = 'http://127.0.0.1:7777'; + +/** + * Discover the Docker bridge gateway IP (typically 172.17.0.1 on Linux + * Docker). Spawned tenants — running in their own Docker containers + * via the HMA — can NOT reach the host's loopback interface; they + * reach the host via this bridge IP. + * + * Returns the WebSocket URL HMA-spawned tenants should set as + * `UNICITY_NOSTR_RELAYS`. Falls back to `host.docker.internal` if + * `docker network inspect` fails (Docker Desktop on macOS/Windows + * resolves this name automatically; recent Linux Docker also supports + * it via the `--add-host=host.docker.internal:host-gateway` flag — + * which the HMA's docker-adapter would need to set if we go that + * route). + */ +export function getLocalRelayUrlForContainers(): string { + const out = spawnSync( + 'docker', + ['network', 'inspect', 'bridge', '--format', '{{(index .IPAM.Config 0).Gateway}}'], + { encoding: 'utf8', timeout: 5_000 }, + ); + if (out.status === 0) { + const gateway = out.stdout.trim(); + if (gateway.length > 0 && /^\d+\.\d+\.\d+\.\d+$/.test(gateway)) { + return `ws://${gateway}:7777`; + } + } + return 'ws://host.docker.internal:7777'; +} + +export interface RelayBootOptions { + /** + * Drop the persisted SQLite event log before booting (passes `-v` + * to `compose down`). Default false — preserves the log between + * runs so a developer can sqlite3-inspect a failing test. + */ + readonly wipe?: boolean; + /** Total deadline for the relay to come up. Default 60s. */ + readonly timeoutMs?: number; + /** Optional prefix for log lines so multi-stack output is greppable. */ + readonly logPrefix?: string; +} + +export interface RelayHandle { + /** WebSocket URL clients connect to. */ + readonly url: string; + /** Container name (matches compose `container_name`). */ + readonly containerName: string; + /** Stop + remove the relay container. Idempotent. */ + stop(opts?: { wipe?: boolean }): Promise; +} + +const log = (prefix: string, msg: string): void => { + // eslint-disable-next-line no-console + console.log(`${prefix}${msg}`); +}; + +/** + * Run `docker compose -f up -d relay` and wait for the NIP-11 + * info doc to respond 200. Returns a handle whose `stop()` runs + * `compose down`. + * + * Throws if Docker isn't available, the image can't be pulled, or the + * relay never becomes healthy within the timeout. We deliberately do + * not swallow these errors — silent boot failures would just produce + * a different, more confusing failure 30s deep into the test run. + */ +export async function bootLocalRelay(opts: RelayBootOptions = {}): Promise { + const prefix = opts.logPrefix ?? '[local-relay] '; + const timeoutMs = opts.timeoutMs ?? 60_000; + + // 1. Sanity check: docker CLI present. + const dockerVersion = spawnSync('docker', ['version', '--format', '{{.Server.Version}}'], { + encoding: 'utf8', + }); + if (dockerVersion.status !== 0) { + throw new Error( + `docker is not available (exit ${dockerVersion.status}): ${dockerVersion.stderr || dockerVersion.stdout}. ` + + 'Install Docker or unset E2E_LOCAL_INFRA to run against the public testnet.', + ); + } + + // 2. Optional: wipe persisted state. + if (opts.wipe) { + log(prefix, 'wiping previous relay-data volume…'); + spawnSync('docker', ['compose', '-f', COMPOSE_FILE, 'down', '-v'], { + encoding: 'utf8', + timeout: 30_000, + }); + } + + // 3. Boot. + log(prefix, `booting relay container from ${COMPOSE_FILE}…`); + const up = spawnSync('docker', ['compose', '-f', COMPOSE_FILE, 'up', '-d', 'relay'], { + encoding: 'utf8', + timeout: 120_000, + }); + if (up.status !== 0) { + throw new Error( + `docker compose up failed (exit ${up.status}):\nstdout: ${up.stdout}\nstderr: ${up.stderr}`, + ); + } + + // 4. Wait for NIP-11 info doc. + const deadline = Date.now() + timeoutMs; + let lastError: string | null = null; + while (Date.now() < deadline) { + try { + const resp = await fetch(LOCAL_RELAY_HTTP, { + headers: { Accept: 'application/nostr+json' }, + signal: AbortSignal.timeout(2_000), + }); + if (resp.ok) { + const info = (await resp.json()) as { name?: string; software?: string; version?: string }; + log(prefix, `relay healthy: ${info.software ?? '?'} ${info.version ?? '?'} on ${LOCAL_RELAY_URL}`); + return { + url: LOCAL_RELAY_URL, + containerName: 'trader-e2e-relay', + stop: async (stopOpts) => stopRelay(prefix, stopOpts?.wipe ?? false), + }; + } + lastError = `HTTP ${resp.status}`; + } catch (err) { + lastError = err instanceof Error ? err.message : String(err); + } + await new Promise((r) => setTimeout(r, 1_000)); + } + + // Boot failed — capture container logs before tearing down so the + // failure message is actionable. + const logs = spawnSync('docker', ['logs', 'trader-e2e-relay', '--tail', '50'], { + encoding: 'utf8', + timeout: 5_000, + }); + await stopRelay(prefix, /* wipe */ false); + throw new Error( + `local relay never became healthy within ${timeoutMs}ms (last error: ${lastError ?? 'unknown'}).\n` + + `--- container logs (last 50 lines) ---\n${logs.stdout || logs.stderr || '(empty)'}`, + ); +} + +async function stopRelay(prefix: string, wipe: boolean): Promise { + const args = ['compose', '-f', COMPOSE_FILE, 'down']; + if (wipe) args.push('-v'); + log(prefix, `stopping relay (wipe=${wipe})…`); + spawnSync('docker', args, { encoding: 'utf8', timeout: 30_000 }); +} From 81567196679c430e410c9dd3270f30c8ed1b26f1 Mon Sep 17 00:00:00 2001 From: Vladimir Rogojin Date: Fri, 8 May 2026 14:30:07 +0200 Subject: [PATCH 18/30] feat(test): wire local-infra relay end-to-end via global-setup + spawn env MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the local-infra harness: when TRADER_E2E_LOCAL_RELAY=1 is set, the test now boots a Docker-hosted Nostr relay AND propagates the relay URL to every component that does Sphere DMs: global-setup.ts: - boots local-infra/relay (uxf-style, container trader-e2e-relay) - sets process.env['UNICITY_NOSTR_RELAYS'] to the bridge-gateway URL so HMA-spawned tenants can reach it - skips the testnet preflight (local relay supersedes the gate) - teardown stops the container; wipe-on-boot by default (TRADER_E2E_LOCAL_RELAY_KEEP=1 to preserve event log for post-mortem) helpers/manager-process.ts: - forward UNICITY_NOSTR_RELAYS / SPHERE_NOSTR_RELAYS into the HMA's spawn env (the helper otherwise builds an isolated env and would have dropped the override) hma-trade-settlement.e2e-live.test.ts: - read process.env['UNICITY_NOSTR_RELAYS'] at beforeAll, stash onto SuiteState.spawnEnv - inject spawnEnv into every hostSpawnAsync (escrow, alice, bob, faucet) so the spawned containers all hit the local relay Same env-override pattern works for both modes: with TRADER_E2E_LOCAL_RELAY unset, spawnEnv is empty {} and tenants fall through to the network preset's default relay (testnet), preserving the existing test behavior. Run against local relay: TRADER_E2E_LOCAL_RELAY=1 npm run test:e2e-live -- \ test/e2e-live/hma-trade-settlement.e2e-live.test.ts When the testnet relay is healthy (no current outage), the test still runs against testnet by omitting the env var. Linux Docker note: getLocalRelayUrlForContainers() runs `docker network inspect bridge` to discover the gateway IP (typically 172.17.0.1) — works from both host and containers. Falls back to host.docker.internal which Docker Desktop resolves automatically; recent Linux Docker also supports it via --add-host=host.docker.internal:host-gateway (HMA's docker-adapter would need to forward that flag if the gateway-IP detection ever fails, which it shouldn't on a standard install). --- test/e2e-live/global-setup.ts | 65 +++++++++++++++++-- test/e2e-live/helpers/manager-process.ts | 9 +++ .../hma-trade-settlement.e2e-live.test.ts | 22 +++++++ 3 files changed, 92 insertions(+), 4 deletions(-) diff --git a/test/e2e-live/global-setup.ts b/test/e2e-live/global-setup.ts index a4aa08d..2fbab7a 100644 --- a/test/e2e-live/global-setup.ts +++ b/test/e2e-live/global-setup.ts @@ -1,14 +1,71 @@ /** * Vitest globalSetup for the e2e-live suite. * - * Runs ONCE before any test file. If the preflight throws, vitest aborts - * the entire run before spawning any Docker containers — saving the - * 10-15-minute round trip we'd otherwise eat on a relay outage or - * unreachable aggregator. + * Runs ONCE before any test file. Two responsibilities: + * + * 1. Preflight gate — abort the run before spawning Docker tenants + * if any required testnet service (Nostr relay, L3 Aggregator, + * IPFS, Fulcrum, Market) is unreachable. Saves the 10-15-minute + * round trip we'd otherwise eat on an outage. Bypass: + * `TRADER_E2E_SKIP_PREFLIGHT=1`. + * + * 2. Local infra (opt-in) — when `TRADER_E2E_LOCAL_RELAY=1` is set, + * boot a Docker-hosted Nostr relay (see local-infra/relay.ts) + * and export `UNICITY_NOSTR_RELAYS` so every component that + * reads it (host-manager, escrow, trader, faucet) connects to + * the local relay instead of the public testnet. The relay + * binds to the host on 0.0.0.0:7777; HMA-spawned tenants reach + * it via the Docker bridge gateway IP (auto-discovered). + * + * Tests that need to fan the URL into HMA-spawned tenants read + * `process.env['UNICITY_NOSTR_RELAYS']` and pass it via the + * `env` field on `hostSpawnAsync(...)`. The host-manager's own + * Sphere wallet picks it up automatically because spawnHostManager + * forwards the parent env (or the test sets it on the spawn env). + * + * Local-relay mode SKIPS the preflight (the local relay is + * under our control; gating against the public testnet relay + * would defeat the purpose). */ import { runPreflight } from './preflight.js'; +import { bootLocalRelay, getLocalRelayUrlForContainers, type RelayHandle } from './local-infra/relay.js'; + +let relayHandle: RelayHandle | null = null; export async function setup(): Promise { + if (process.env['TRADER_E2E_LOCAL_RELAY'] === '1') { + console.log('[global-setup] TRADER_E2E_LOCAL_RELAY=1 — booting local Nostr relay…'); + relayHandle = await bootLocalRelay({ + // Wipe by default so each `npm run test:e2e-live` starts from a clean + // event log. Set TRADER_E2E_LOCAL_RELAY_KEEP=1 to preserve state + // between runs (useful for post-mortem on a failing test). + wipe: process.env['TRADER_E2E_LOCAL_RELAY_KEEP'] !== '1', + timeoutMs: 60_000, + logPrefix: '[global-setup] ', + }); + const containerUrl = getLocalRelayUrlForContainers(); + process.env['UNICITY_NOSTR_RELAYS'] = containerUrl; + process.env['TRADER_E2E_LOCAL_RELAY_HOST_URL'] = relayHandle.url; + process.env['TRADER_E2E_LOCAL_RELAY_CONTAINER_URL'] = containerUrl; + console.log( + `[global-setup] local relay ready — host: ${relayHandle.url}, ` + + `containers: ${containerUrl}`, + ); + console.log('[global-setup] preflight SKIPPED (local relay supersedes testnet gate)'); + return; + } await runPreflight(); } + +export async function teardown(): Promise { + if (relayHandle) { + console.log('[global-setup] stopping local Nostr relay…'); + try { + await relayHandle.stop({ wipe: false }); + } catch (err) { + console.error('[global-setup] relay stop error:', err); + } + relayHandle = null; + } +} diff --git a/test/e2e-live/helpers/manager-process.ts b/test/e2e-live/helpers/manager-process.ts index 51a3177..c000406 100644 --- a/test/e2e-live/helpers/manager-process.ts +++ b/test/e2e-live/helpers/manager-process.ts @@ -263,6 +263,15 @@ export async function spawnHostManager(opts: SpawnHostManagerOptions): Promise; } // Number of concurrent settlement scenarios. Each gets its own controller @@ -306,6 +312,17 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne // Spawn ONE shared faucet (the faucet is open — anyone can request, // so we don't need one per scenario). + // + // When the local-infra Nostr relay is active (TRADER_E2E_LOCAL_RELAY=1 + // → global-setup.ts boots a Docker relay and sets + // UNICITY_NOSTR_RELAYS to its bridge-gateway URL), forward that env + // into every spawned tenant via HMA's `--env` passthrough so the + // tenant connects to the local relay instead of the public testnet. + // No-op when the env var is unset (falls through to network preset). + const localRelayEnv: Record = process.env['UNICITY_NOSTR_RELAYS'] + ? { UNICITY_NOSTR_RELAYS: process.env['UNICITY_NOSTR_RELAYS']! } + : {}; + console.log('[hma-trade-settlement] spawning shared faucet-agent…'); const faucet = await hostSpawnAsync({ cliPath, @@ -314,6 +331,7 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne templateId: 'faucet-agent', instanceName: `faucet-${randomUUID().slice(0, 6)}`, timeoutMs: 180_000, + env: localRelayEnv, }); console.log( `[hma-trade-settlement] faucet ready: pubkey=${faucet.tenantPubkey.slice(0, 16)}… ` + @@ -336,6 +354,7 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne spawned: [faucet], faucet, faucetClient, + spawnEnv: localRelayEnv, }; }, 900_000); // 15 min — adds ~30-60s for faucet spawn + client bootstrap on top of controller-wallet inits @@ -403,6 +422,7 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne templateId: 'escrow-service', instanceName: `escrow-${scenarioId}`, timeoutMs: 180_000, + env: s.spawnEnv, }); const alice = await hostSpawnAsync({ cliPath: s.cliPath, @@ -411,6 +431,7 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne templateId: 'trader-agent', instanceName: `alice-${scenarioId}`, timeoutMs: 180_000, + env: s.spawnEnv, }); const bob = await hostSpawnAsync({ cliPath: s.cliPath, @@ -419,6 +440,7 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne templateId: 'trader-agent', instanceName: `bob-${scenarioId}`, timeoutMs: 180_000, + env: s.spawnEnv, }); s.spawned.push(escrow, alice, bob); console.log( From a797b90439882a4f559283e8c24f69502079f540 Mon Sep 17 00:00:00 2001 From: Vladimir Rogojin Date: Fri, 8 May 2026 14:37:25 +0200 Subject: [PATCH 19/30] fix(test): forward UNICITY_NOSTR_RELAYS into sphere-cli subprocesses Round 16 (TRADER_E2E_LOCAL_RELAY=1) booted the local relay and the manager registered correctly, but `sphere host spawn` failed with 'Unicity ID not found: @m-e2elive...'. Cause: helpers/sphere-cli.ts buildEnv() builds a sanitized env (PATH/HOME/UNICITY_API_KEY/CI/ FORCE_COLOR only) and dropped the relay override, so sphere-cli queried testnet for the manager's nametag while the manager was registered only on the local relay. Also patched sphere-cli upstream (host/sphere-init.ts and legacy/legacy-cli.ts) to read UNICITY_NOSTR_RELAYS / SPHERE_NOSTR_RELAYS in the same pattern as trader-service / escrow-service / agentic-hosting / js-faucet. Both inits now respect the env when set. --- test/e2e-live/helpers/sphere-cli.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/e2e-live/helpers/sphere-cli.ts b/test/e2e-live/helpers/sphere-cli.ts index ef34c82..ae3a666 100644 --- a/test/e2e-live/helpers/sphere-cli.ts +++ b/test/e2e-live/helpers/sphere-cli.ts @@ -171,6 +171,16 @@ function buildEnv(opts?: { extraEnv?: Record }, cwd?: string): R PATH: process.env['PATH'] ?? '', HOME: process.env['HOME'] ?? cwd ?? '/', ...(process.env['UNICITY_API_KEY'] ? { UNICITY_API_KEY: process.env['UNICITY_API_KEY'] } : {}), + // Forward optional Nostr-relay override so sphere-cli subprocesses + // (wallet init, sphere host spawn, sphere trader create-intent, …) + // hit the same relay as the rest of the stack when the local-infra + // harness is active. Falls through silently when unset. + ...(process.env['UNICITY_NOSTR_RELAYS'] + ? { UNICITY_NOSTR_RELAYS: process.env['UNICITY_NOSTR_RELAYS'] } + : {}), + ...(process.env['SPHERE_NOSTR_RELAYS'] + ? { SPHERE_NOSTR_RELAYS: process.env['SPHERE_NOSTR_RELAYS'] } + : {}), CI: '1', FORCE_COLOR: '0', ...(opts?.extraEnv ?? {}), From 3afb51953a10d1ad09aa5159d041a4ef13ddc3bc Mon Sep 17 00:00:00 2001 From: Vladimir Rogojin Date: Fri, 8 May 2026 14:55:03 +0200 Subject: [PATCH 20/30] =?UTF-8?q?docs(diagnostic):=20rounds=2016-17=20?= =?UTF-8?q?=E2=80=94=20local-infra=20harness=20landed;=20nametag=20mux=20i?= =?UTF-8?q?s=20the=20next=20gap?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/HMA-SETTLEMENT-DIAGNOSTIC.md | 47 +++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/docs/HMA-SETTLEMENT-DIAGNOSTIC.md b/docs/HMA-SETTLEMENT-DIAGNOSTIC.md index 3efb6bb..aac8ad0 100644 --- a/docs/HMA-SETTLEMENT-DIAGNOSTIC.md +++ b/docs/HMA-SETTLEMENT-DIAGNOSTIC.md @@ -163,6 +163,53 @@ went down again (intermittent — every WS publish-kind:* returns no OK). The instrumented build is committed and pushed; next live attempt against this branch will produce log lines pinpointing party B's actual path. +**Update 2026-05-08 (rounds 16-17, local-infra harness)**: +ported uxf's local-infra Nostr relay setup to trader-service to +escape the testnet write-path outages. Added +`UNICITY_NOSTR_RELAYS` env override across every component +(trader-service, escrow-service, agentic-hosting host-manager, +js-faucet, sphere-cli host/legacy inits) plus `helpers/sphere-cli.ts +buildEnv()` forwards the env into sphere-cli subprocesses. +Global-setup boots the relay when `TRADER_E2E_LOCAL_RELAY=1`. + +What works: + - Local relay container boots, tests skip preflight, env propagates + to host-manager + spawned tenants + - Wallet events (kind:30078) and DM gift-wraps (kind:1059) ARE + published to the local relay (verified by tailing relay logs) + - HMA dist needed a rebuild (was 4 days stale on disk) + +What does NOT yet work — SDK-level gap: + - Nametag binding events (kind:31113/31115/31116) bypass the + `transport.relays` override. They route through + `MultiAddressTransportMux` (sphere-sdk/transport/MultiAddressTransportMux.ts:9) + which has its OWN relay list independent of the per-provider + transport config. Because of this, the manager registers a + nametag against the (default/testnet) mux-relay but sphere-cli's + `queryPubkeyByNametag` (also via the mux) doesn't find it on + the local relay → "Unicity ID not found: @m-…". + - `nostr-js-sdk`'s `publishNametagBinding` calls + `queryPubkeyByNametag` first to detect conflicts; both + operations target the mux's hard-coded relay list, not the + SDK consumer's override. + +Fix path: + 1. **SDK change** — extend `MultiAddressTransportMux` to accept a + relay override so it picks up `transport.relays` (or a sibling + `transport.muxRelays`) from the createNodeProviders config. + Default behavior unchanged. + 2. **Tactical workaround** — for tests that use the local relay, + identify peers by raw `DIRECT://hex` (which the SDK resolves + transport-side, not via the nametag mux) and avoid `@nametag`. + The hma-trade-settlement test already passes + `escrow.tenantDirectAddress` for the swap routing; the only + remaining `@nametag` usage is sphere-cli's manager-address + resolution. The test could read `manager.directAddress` and + pass that instead of `@${manager.nametag}` — quick fix. + +Local-infra commits already pushed; the workaround in (2) is the +fastest path to a green run. + **Remediation (in priority order)**: 1. Rebuild `escrow:local` from `/home/vrogojin/escrow-service` source and From 1a6ecda6ad02f78ee7a2a0246a55843afb6a1f9a Mon Sep 17 00:00:00 2001 From: Vladimir Rogojin Date: Fri, 8 May 2026 18:48:50 +0200 Subject: [PATCH 21/30] fix(test): wire local-relay env across all the wallet entry points MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round 18 progressed massively (534× kind:1059 + 10× kind:30078 on the local relay) but stalled on FAUCET_REQUEST timeout because the in-process FaucetClient was built with testnet defaults and never saw the local-relay override. Three patches collected here: 1. helpers/faucet-client.ts — read UNICITY_NOSTR_RELAYS / SPHERE_NOSTR_RELAYS and pass transport.relays to createNodeProviders. The Sphere wallet that signs+sends FAUCET_REQUEST DMs from the test process now hits the local relay. 2. helpers/manager-process.ts (provisionManagerWallet) — same pattern. The pre-creation step that generates the manager's wallet + publishes the nametag now respects the override. Without this, the nametag binding event was published to testnet (when the test process had no override pickup), and the HMA binary loaded the existing wallet on launch (wallet_created: false) → never re-published to the local relay → sphere-cli's queryPubkeyByNametag returned 'not found'. 3. helpers/manager-process.ts (UNICITY_HEALTH_PORT) — default to 0 (OS-assigned ephemeral port) instead of fixed 19401. Tests don't probe this port, and a fixed default EADDRINUSEs when a prior run leaks an HMA process. Mirror of the same change already on js-faucet's manager-process.ts. Test runs HMA→escrow→faucet→2 traders in parallel scenarios. With all local images rebuilt (escrow:local with current source, trader:local with current sphere-sdk + relay override, faucet:local with relay override) settlement traffic flowed end-to-end on the Docker-hosted local relay. The TRADER_E2E_LOCAL_RELAY=1 mode now actually works. --- .sphere-cli/wallet.json | 1 + test/e2e-live/helpers/faucet-client.ts | 12 ++++++++++++ test/e2e-live/helpers/manager-process.ts | 19 ++++++++++++++++++- .../hma-trade-settlement.e2e-live.test.ts | 14 +++++++++++--- 4 files changed, 42 insertions(+), 4 deletions(-) create mode 100644 .sphere-cli/wallet.json diff --git a/.sphere-cli/wallet.json b/.sphere-cli/wallet.json new file mode 100644 index 0000000..9e26dfe --- /dev/null +++ b/.sphere-cli/wallet.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/test/e2e-live/helpers/faucet-client.ts b/test/e2e-live/helpers/faucet-client.ts index bfde18b..c4fc1e0 100644 --- a/test/e2e-live/helpers/faucet-client.ts +++ b/test/e2e-live/helpers/faucet-client.ts @@ -79,11 +79,23 @@ export async function createFaucetClient(): Promise { const trustbasePath = join(dataDir, 'trustbase.json'); writeFileSync(trustbasePath, await tbResp.text()); + // Forward Nostr-relay override so the in-process FaucetClient connects + // to the same relay as the spawned tenants when the local-infra harness + // is active. Without this, the client connects to testnet defaults and + // can't reach a faucet-agent that's only on the local relay → its + // FAUCET_REQUEST DMs go out into testnet and the response never arrives. + const relayOverride = (() => { + const raw = process.env['UNICITY_NOSTR_RELAYS'] ?? process.env['SPHERE_NOSTR_RELAYS']; + if (!raw) return undefined; + const relays = raw.split(',').map((s) => s.trim()).filter((s) => s.length > 0); + return relays.length > 0 ? relays : undefined; + })(); const providers = createNodeProviders({ network: 'testnet', dataDir, tokensDir, oracle: { trustBasePath: trustbasePath }, + ...(relayOverride ? { transport: { relays: relayOverride } } : {}), }); const { sphere } = await Sphere.init({ ...providers, diff --git a/test/e2e-live/helpers/manager-process.ts b/test/e2e-live/helpers/manager-process.ts index c000406..d8b3832 100644 --- a/test/e2e-live/helpers/manager-process.ts +++ b/test/e2e-live/helpers/manager-process.ts @@ -157,6 +157,18 @@ async function provisionManagerWallet(dataDir: string, hostId: string): Promise< // Forward UNICITY_API_KEY when set; the SDK falls back to its public // placeholder otherwise. const apiKey = process.env['UNICITY_API_KEY']?.trim() || undefined; + // Forward Nostr-relay override — this pre-creation step publishes the + // manager's nametag binding. Without the override here it would land + // on testnet, then the HMA binary (which DOES read the override) loads + // the existing wallet and skips re-publish. The local relay would + // never see the binding event and sphere-cli's queryPubkeyByNametag + // returns "Unicity ID not found". + const relayOverride = (() => { + const raw = process.env['UNICITY_NOSTR_RELAYS'] ?? process.env['SPHERE_NOSTR_RELAYS']; + if (!raw) return undefined; + const relays = raw.split(',').map((s) => s.trim()).filter((s) => s.length > 0); + return relays.length > 0 ? relays : undefined; + })(); const providers = createNodeProviders({ network: 'testnet', dataDir, @@ -165,6 +177,7 @@ async function provisionManagerWallet(dataDir: string, hostId: string): Promise< trustBasePath: trustbasePath, ...(apiKey ? { apiKey } : {}), }, + ...(relayOverride ? { transport: { relays: relayOverride } } : {}), }); const nametag = `m-${hostId.replace(/[^a-z0-9]/gi, '').slice(0, 12).toLowerCase()}`; const { sphere } = await Sphere.init({ @@ -257,7 +270,11 @@ export async function spawnHostManager(opts: SpawnHostManagerOptions): Promise = process.env['UNICITY_NOSTR_RELAYS'] - ? { UNICITY_NOSTR_RELAYS: process.env['UNICITY_NOSTR_RELAYS']! } + // + // IMPORTANT: HMA's validatePayloadEnv at + // agentic-hosting/src/host-manager/manager.ts:90 blocks every env + // var starting with `UNICITY_` (it protects HMA-internal vars like + // UNICITY_BOOT_TOKEN from controller-side override). So we use the + // sibling `SPHERE_NOSTR_RELAYS` alias which is treated identically + // by every service's relay-override pickup but isn't on the + // forbidden-prefix list. No-op when neither env var is set. + const relayUrl = process.env['UNICITY_NOSTR_RELAYS'] ?? process.env['SPHERE_NOSTR_RELAYS']; + const localRelayEnv: Record = relayUrl + ? { SPHERE_NOSTR_RELAYS: relayUrl } : {}; console.log('[hma-trade-settlement] spawning shared faucet-agent…'); From 66ec6ad1edc31853896c1365ecec0947888aa808 Mon Sep 17 00:00:00 2001 From: Vladimir Rogojin Date: Fri, 8 May 2026 19:07:51 +0200 Subject: [PATCH 22/30] =?UTF-8?q?docs(diagnostic):=20round=2019=20?= =?UTF-8?q?=E2=80=94=20local-infra=20harness=20is=20fully=20working?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit End-to-end run of hma-trade-settlement against the Docker-hosted local Nostr relay. Spawn (HMA + escrow + faucet + 2 traders) ✓ funding via FAUCET_REQUEST ✓ matched intents ✓ ACCEPTED deals ✓ escrow's invoice delivery to BOTH parties ✓ trader's invoice import ✓ trader's deposit sent ✓. Final stop: swap-protocol settlement bug — '[Accounting] Direction mismatch: transport memo says return_cancelled, on-chain says forward' → swap_cancelled. That's an independent layer; deterministically reproducible against the local relay, no longer blocked by testnet outages. The asymmetric invoice-delivery bug (rounds 11-12) is GONE in escrow:local from current source — every swap shows full deliver_deposit_invoice_{enter,sending,sent} for both parties. --- docs/HMA-SETTLEMENT-DIAGNOSTIC.md | 43 +++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/docs/HMA-SETTLEMENT-DIAGNOSTIC.md b/docs/HMA-SETTLEMENT-DIAGNOSTIC.md index aac8ad0..54cd3a3 100644 --- a/docs/HMA-SETTLEMENT-DIAGNOSTIC.md +++ b/docs/HMA-SETTLEMENT-DIAGNOSTIC.md @@ -163,6 +163,49 @@ went down again (intermittent — every WS publish-kind:* returns no OK). The instrumented build is committed and pushed; next live attempt against this branch will produce log lines pinpointing party B's actual path. +**Update 2026-05-08 (round 19, local-infra fully working)**: +The local-infra harness now runs end-to-end on a Docker-hosted Nostr relay +(no testnet dependency for messaging). Fix chain that closed it: + + - sphere-cli (host/sphere-init.ts AND legacy/legacy-cli.ts) reads + UNICITY_NOSTR_RELAYS / SPHERE_NOSTR_RELAYS; + - helpers/sphere-cli.ts buildEnv() forwards the env into sphere-cli + subprocesses; + - helpers/manager-process.ts: spawnHostManager forwards env to HMA + binary, AND provisionManagerWallet (which pre-creates the manager + wallet + publishes the nametag binding) ALSO reads it — without + this, the nametag binding lands on testnet, the HMA's later + Sphere.init loads the existing wallet (wallet_created: false) and + skips re-publish, sphere-cli's queryPubkeyByNametag returns null + on the local relay; + - helpers/manager-process.ts: UNICITY_HEALTH_PORT default → 0 (OS- + assigned) so leaked HMA processes don't EADDRINUSE the next run; + - hma-trade-settlement test: use SPHERE_NOSTR_RELAYS (NOT + UNICITY_NOSTR_RELAYS) in HMA spawn-env passthrough — HMA's + validatePayloadEnv blocks any env starting with UNICITY_; + - helpers/faucet-client.ts: same env-pickup pattern so the in-process + Sphere wallet that signs FAUCET_REQUEST DMs talks to the local relay. + +Round 19 evidence: + - Local relay log: 534+ kind:1059 (gift-wrap DMs) + 10+ kind:30078 + (wallet/nametag bindings) — full settlement traffic on local infra. + - Escrow's instrumentation: every swap shows `deliver_deposit_invoice_enter + → _sending → _sent` for BOTH parties (the asymmetric bug from + rounds 11-12 IS GONE in escrow:local from current source). + - Trader log: `diag_invoice_delivery_received` → `_imported` → + `swap_deposit_target_diag` populated → `swap_deposit_sent`. The + deposit IS sent. The trader DOES process the invoice. + - Final failure: `[Accounting] Direction mismatch: transport memo says + return_cancelled, on-chain says forward for invoice — using + on-chain` → `swap_cancelled`. + +**The local-infra goal is met.** What remains is a swap-protocol +settlement-layer issue (transport memo vs on-chain direction +mismatch) that's independent of the relay infra. This is the next +real bug to chase, and it now reproduces deterministically against +a controlled local relay — debug iterations no longer wait on +testnet propagation or burn through testnet rate limits. + **Update 2026-05-08 (rounds 16-17, local-infra harness)**: ported uxf's local-infra Nostr relay setup to trader-service to escape the testnet write-path outages. Added From 0e686efa7d271e2b343bdc9b7609dd0e0c6ab493 Mon Sep 17 00:00:00 2001 From: Vladimir Rogojin Date: Fri, 8 May 2026 21:51:54 +0200 Subject: [PATCH 23/30] fix(test): switch settlement funding to selfMint; document SDK verifyPayout wall MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round 20: replaces FAUCET_REQUEST funding in hma-trade-settlement.e2e-live with TRADER_TEST_FUND injected via HMA's --env passthrough. The faucet path delivered tokens whose source-state predicate the trader's swap-deposit signing key couldn't match, causing every deposit attempt to fail with "Ownership verification failed: Authenticator does not match source state predicate" (round 19). selfMint mints with the trader's own predicate, mirroring basic-roundtrip's known-working mechanism. Result with selfMint: the swap protocol completes end-to-end on the escrow side — announces, invoices delivered, both deposits verified, "invoice:covered", payouts paid, "Swap completed successfully". The trader receives the payout transfer (10 UCT) and the SDK marks swap progress as `completed`. But verifyPayout enters its fail-closed branch (sphere-sdk/modules/swap/SwapModule.ts:1997-2003) because getTokenIdsForInvoice(payoutInvoiceId) returns an empty Set: the synthetic-ledger reverse-index population at AccountingModule.ts:5755 isn't firing for swap payouts. Test times out at 8 min without reaching COMPLETED. This shifts the remaining gap out of HMA / relay / faucet land and into the SDK's instant-mode tokenInvoiceMap population. Round 20 of the diagnostic doc covers reproducer, log excerpts, and the three hypotheses to bisect next. --- docs/HMA-SETTLEMENT-DIAGNOSTIC.md | 140 ++++++++++++++++++ .../hma-trade-settlement.e2e-live.test.ts | 60 ++++---- 2 files changed, 169 insertions(+), 31 deletions(-) diff --git a/docs/HMA-SETTLEMENT-DIAGNOSTIC.md b/docs/HMA-SETTLEMENT-DIAGNOSTIC.md index 54cd3a3..c8d19ae 100644 --- a/docs/HMA-SETTLEMENT-DIAGNOSTIC.md +++ b/docs/HMA-SETTLEMENT-DIAGNOSTIC.md @@ -413,6 +413,146 @@ docker ps -a --filter "name=escrow-p" --filter "status=exited" --format "{{.Name | xargs -I {} docker logs {} 2>&1 | grep -iE "announce|invoice_delivery|outbound_dm" ``` +--- + +## Round 20 (2026-05-08) — selfMint funding unblocks deposits; SDK verifyPayout is the new wall + +**Hypothesis tested:** the round-19 deposit failure +(`Ownership verification failed: Authenticator does not match source state predicate`) +is caused by faucet-funded tokens having a predicate the swap-deposit +key path can't sign. `basic-roundtrip` works because traders selfMint +(predicate matches their own key); switching this test to selfMint +should unblock the deposit step. + +**Change:** patched `provisionTriple` in +`test/e2e-live/hma-trade-settlement.e2e-live.test.ts` to fund alice/bob +via `TRADER_TEST_FUND` injected through HMA's `--env` passthrough, +identical to the basic-roundtrip mechanism. Faucet spawn left in place +(unused) for minimal-diff diagnostic. Run on local-infra relay with +`TRADER_E2E_LOCAL_RELAY=1`. + +**Result — escrow side (full happy path):** + +``` +escrow-p1 log: + announce DM received from sender A (alice) + announce DM received from sender B (bob) + Swap announced, deposit invoice created + deliver_deposit_invoice_enter A → recipient 53272861... USDU 10 + deliver_deposit_invoice_sent A + deliver_deposit_invoice_enter B → recipient 786576ea... UCT 10 + deliver_deposit_invoice_sent B + First valid deposit received, timeout timer started (party A USDU) + Valid deposit received (not first) (party B UCT) + invoice:covered with unconfirmed deposits — waiting for aggregator confirmation + Deposit invoice already closed, proceeding to payouts + Timeout cancelled + Swap concluding — paying payout invoices (payoutA UCT, payoutB USDU) + Swap completed successfully ← ESCROW: settlement is DONE +``` + +So the deposit-predicate issue is **conclusively the faucet's fault**: +with selfMint, both deposits verify, escrow concludes, payouts route. +This unblocks the entire swap protocol from L4 down. + +**Result — trader side (new wall):** + +``` +alice-p1 log (Pair-1, after escrow logged "Swap completed"): + swap_payout_verify_diag attempt=8..15 + invoice_status: { state:'COVERED', isCovered:true, + coveredAmount:'10', netCoveredAmount:'10', + transfers:[{ transferId:'a20d9b0d-…', paymentDirection:'forward', + senderPubkey:'…escrow…', confirmed:false }] } + [Swap] verifyPayout for dacaf7d67760: 3 invalid token(s) but + tokenInvoiceMap is empty for this payout invoice — failing closed + until reverse index rebuilds + swap-executor: execution_timeout_skipped_terminal_swap + sdk_progress="completed", note="SDK swap is already terminal — + letting verifyPayout retries finish" + swap_payout_verify_retry_failed attempt=15 remaining=25 +``` + +The trader **received** the payout token. The SDK's swap progress is +`completed`. But `getTokenIdsForInvoice(payoutInvoiceId)` returns an +empty Set, so the SECURITY-fail-closed branch in +`sphere-sdk/modules/swap/SwapModule.ts:1997-2003` returns false on +every retry. The retry budget exhausts at attempt 40 (~20 min); the +test's 8-min timeout expires first → both Pair-1 + Pair-2 fail with +`did not reach state="COMPLETED" within 480000ms. last seen 1 deal(s) +in states [ACCEPTED]`. + +**Where the index should populate:** SDK's +`AccountingModule.ts:5755-5773` adds entries to `tokenInvoiceMap` when +an inbound transfer matches an invoice target (instant-mode v5split +path): + +```ts +for (const tok of transfer.tokens) { + if (!tok.id) continue; // ← short-circuit if token.id absent + if (!this.tokenInvoiceMap.has(tok.id)) { + this.tokenInvoiceMap.set(tok.id, new Set()); + } + this.tokenInvoiceMap.get(tok.id)!.add(invoiceId); +} +``` + +So either: + +1. The payout transfer arrives **without** `tok.id` populated on the + `transfer.tokens` entries (in which case the loop skips silently + and the index stays empty), OR +2. The transfer arrives **before** the payout invoice is registered + on the trader's accounting module (so `terms.targets` doesn't match + yet — but `invoice_imported:true` in the diag rules this out), OR +3. There's a race where `_processTokenTransactions`'s on-chain path + also failed to populate the map (instant-mode tokens have no genesis + so the on-chain path is a no-op — the synthetic-ledger path at line + 5755 is the only chance). The W23-R3 fix added that synthetic path + precisely for this case; it's apparently not firing here. + +**Reproducer (deterministic on local-infra):** + +```bash +docker rm -f $(docker ps -aq --filter "name=agentic-") 2>/dev/null || true +cd /home/vrogojin/trader-service +TRADER_E2E_LOCAL_RELAY=1 npx vitest run --config vitest.e2e-live.config.ts \ + test/e2e-live/hma-trade-settlement.e2e-live.test.ts 2>&1 | tee /tmp/r20.log +# Wait ~7 min. Both scenarios fail at COMPLETED gate. +ALICE=$(docker ps -a --filter "name=agentic-alice-p1" --format "{{.Names}}" | head -1) +docker logs "$ALICE" 2>&1 | grep -E "swap_payout_verify|tokenInvoiceMap" +# Expect: many "tokenInvoiceMap is empty for this payout invoice" warnings. +``` + +**Next investigative steps (in order of leverage):** + +1. **Inspect transfer.tokens in the actual payout** — run with + `LOG_LEVEL=debug` and add a one-line dump of `transfer.tokens` at + the top of the synthetic-ledger branch in `AccountingModule.ts:5715`. + Confirm whether `tok.id` is populated when the swap payout lands. + This is a 1-line probe with ~0 risk. +2. **Verify the synthetic-ledger branch is even being entered** — it's + guarded by `matchesTarget && matchesAsset` (line 5715). If the + trader's wallet address doesn't match `terms.targets[].address` + for the payout, neither the ledger nor the reverse map gets + populated. The diag log already shows `coveredAmount:"10"` against + the alice address `DIRECT://0000b753a86a721a72…`, so the match + IS happening — but maybe in `computeInvoiceStatus` only, not in + the dispatcher that runs synthetic-ledger update. +3. **Check the on-chain path** — `_processTokenTransactions` at + line 4628 also populates `tokenInvoiceMap`. Instant-mode payouts + have no TXF transaction so that path is a no-op; but if the swap + payouts go via TXF (not instant), this branch is what should run. + Decide which mode the escrow is using by inspecting payout messages. + +**Status:** the swap protocol works end-to-end on the wire (deposits +verify, escrow concludes, payouts deliver). The remaining gap is a +reverse-index population bug in the SDK. This is a different layer +from the round-1..19 issues (HMA / relay / faucet predicate); it's +inside `sphere-sdk` itself. + +--- + ## Out of scope for this debugging session - Production hardening of js-faucet (rate limiting, batched mints, etc.) diff --git a/test/e2e-live/hma-trade-settlement.e2e-live.test.ts b/test/e2e-live/hma-trade-settlement.e2e-live.test.ts index 2277521..8130a00 100644 --- a/test/e2e-live/hma-trade-settlement.e2e-live.test.ts +++ b/test/e2e-live/hma-trade-settlement.e2e-live.test.ts @@ -127,6 +127,7 @@ import { type PortfolioBalance, } from './helpers/sphere-trader.js'; import { createFaucetClient, type FaucetClient } from './helpers/faucet-client.js'; +import { UCT_COIN_ID, USDU_COIN_ID } from './helpers/constants.js'; // --------------------------------------------------------------------------- // Precondition gates (mirrors hma-trade-flow's structure) @@ -432,6 +433,23 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne timeoutMs: 180_000, env: s.spawnEnv, }); + // Trader-side selfMint funding via TRADER_TEST_FUND. Diagnostic note + // (rounds 17-19, 2026-05-04): faucet-funded tokens caused Bob's + // deposit to fail with "Ownership verification failed: Authenticator + // does not match source state predicate" — tokens minted+sent by the + // faucet had a source-state predicate that the trader's swap-deposit + // signing path could not match. basic-roundtrip uses selfMintFund + // and settles correctly because each trader mints with its OWN key + // as the source-state predicate. Switching this test to selfMint + // isolates the predicate-mismatch issue from the rest of the + // HMA-orchestrated flow. TRADER_TEST_FUND is gated by + // TRADER_FAULT_INJECTION_ALLOWED=1 + UNICITY_NETWORK ∈ {testnet,dev} + // (trader/main.ts:1524). HMA's payload-env validator allows both + // (UNICITY_* prefix is forbidden but TRADER_* is not). + const fundEnv: Record = { + TRADER_TEST_FUND: `${UCT_COIN_ID}:${INITIAL_FUND_AMOUNT.toString()},${USDU_COIN_ID}:${INITIAL_FUND_AMOUNT.toString()}`, + TRADER_FAULT_INJECTION_ALLOWED: '1', + }; const alice = await hostSpawnAsync({ cliPath: s.cliPath, cliHome: controller.cliHome, @@ -439,7 +457,7 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne templateId: 'trader-agent', instanceName: `alice-${scenarioId}`, timeoutMs: 180_000, - env: s.spawnEnv, + env: { ...s.spawnEnv, ...fundEnv }, }); const bob = await hostSpawnAsync({ cliPath: s.cliPath, @@ -448,34 +466,14 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne templateId: 'trader-agent', instanceName: `bob-${scenarioId}`, timeoutMs: 180_000, - env: s.spawnEnv, + env: { ...s.spawnEnv, ...fundEnv }, }); s.spawned.push(escrow, alice, bob); console.log( `[${scenarioId}] up: escrow=${escrow.instanceName} ` + - `alice=${alice.instanceName} bob=${bob.instanceName}`, + `alice=${alice.instanceName} bob=${bob.instanceName} ` + + `(selfMint UCT+USDU=${INITIAL_FUND_AMOUNT})`, ); - - // Fund both traders via FAUCET_REQUEST DMs. The faucet mints the - // tokens and sends to each trader's DIRECT://. We then poll each - // trader's portfolio until the balance arrives in `confirmed`. - console.log(`[${scenarioId}] funding alice + bob via faucet DM…`); - for (const t of [{ name: 'alice', tenant: alice }, { name: 'bob', tenant: bob }]) { - const recipient = t.tenant.tenantNametag - ? `@${t.tenant.tenantNametag}` - : `DIRECT://${t.tenant.tenantPubkey}`; - const deliveries = await s.faucetClient.request(s.faucet.tenantPubkey, { - recipient, - items: [ - { asset: 'UCT', amount: INITIAL_FUND_AMOUNT.toString() }, - { asset: 'USDU', amount: INITIAL_FUND_AMOUNT.toString() }, - ], - }, 240_000); - console.log( - `[${scenarioId}] ${t.name}: faucet delivered ${deliveries.length} item(s) ` + - `(transfer_ids: ${deliveries.map((d) => d.transfer_id.slice(0, 8)).join(', ')})`, - ); - } return { escrow, alice, bob }; } @@ -536,13 +534,13 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne const s = state; const { escrow, alice, bob } = await provisionTriple(scenarioId, controller); - // The faucet returned `acp.result` for each FAUCET_REQUEST, but the - // trader's payments.receive() loop runs on a 15s cycle — the - // delivered tokens may be in `unconfirmed` for up to ~30s after - // the send completes. Poll each trader's portfolio until both - // assets reach the funded amount in `confirmed` so set-strategy / - // create-intent operate on a fully-settled balance. - console.log(`[${scenarioId}] waiting for faucet-funded balances to confirm…`); + // selfMint funding: the trader runs sphere.payments.mintFungibleToken + // for each TRADER_TEST_FUND entry BEFORE agent.start() (see + // trader/main.ts:1524). Tokens are confirmed on the L3 aggregator + // before the trader logs sphere_initialized, so the balance is + // typically present on the first GET_PORTFOLIO. Keep the polling + // loop anyway as a safety net for slow aggregator round-trips. + console.log(`[${scenarioId}] waiting for selfMint balances to confirm…`); for (const t of [{ name: 'alice', tenant: alice }, { name: 'bob', tenant: bob }]) { const deadline = Date.now() + FUNDING_BALANCE_TIMEOUT_MS; let lastSnapshot: readonly PortfolioBalance[] = []; From 4de58d1820544acacbf556a67c96dc9508fbba2d Mon Sep 17 00:00:00 2001 From: Vladimir Rogojin Date: Sat, 9 May 2026 14:16:44 +0200 Subject: [PATCH 24/30] test: shorten settlement timeout to 3min for fast-fail signal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Settlement on local-relay completes in ~30-60s end-to-end (selfMint → match → deposits verify → payouts paid → verifyPayout). The previous 8-min budget existed because of the testnet-only era and gave painful slow-fail on the reverse-index bug now fixed by sphere-sdk fix/swap-deps-reverse-index. 3min gives 2× headroom over typical runs while failing fast on regressions. --- test/e2e-live/hma-trade-settlement.e2e-live.test.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/e2e-live/hma-trade-settlement.e2e-live.test.ts b/test/e2e-live/hma-trade-settlement.e2e-live.test.ts index 8130a00..a6576be 100644 --- a/test/e2e-live/hma-trade-settlement.e2e-live.test.ts +++ b/test/e2e-live/hma-trade-settlement.e2e-live.test.ts @@ -196,7 +196,11 @@ const SCENARIO_COUNT = 2; let state: SuiteState | null = null; -const SWAP_TIMEOUT_MS = 8 * 60_000; // 8 minutes; testnet settlement is 3-5 min typical +const SWAP_TIMEOUT_MS = 3 * 60_000; // 3 minutes — bisect-friendly. On a healthy local relay + // the swap protocol completes in ~30-60s end-to-end (deposits + // verify + payouts confirm). 3 min gives 2× headroom while + // failing fast when verifyPayout's reverse-index bug strands + // the deal at COVERED-but-unverified. /** Per-asset funding amount delivered to each trader by the faucet. */ const INITIAL_FUND_AMOUNT = 5000n; /** How long we wait for faucet-delivered tokens to surface in `confirmed` balance. */ From 090a59b3e8a0a9ffe8c40a2d009819d1745e359b Mon Sep 17 00:00:00 2001 From: Vladimir Rogojin Date: Sat, 9 May 2026 18:56:53 +0200 Subject: [PATCH 25/30] feat(trader): invoice-based withdraw via accounting.payInvoice MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the direct payments.send code path inside WITHDRAW_TOKEN with a createInvoice → payInvoice flow — the same path swap deposits use, so it inherits the SDK's well-tested predicate-handling and avoids the "Authenticator does not match source state predicate" flake on spends of received swap-payout tokens. - New AccountingAdapter interface in src/trader/types.ts (narrow facade over sphere.accounting, just createInvoice + payInvoice). - main.ts wires sphere.accounting through to the agent when present. - Legacy direct-send path retained for unit tests with stub adapters. --- src/trader/main.ts | 37 +++++++++++++++++++ src/trader/trader-main.ts | 78 +++++++++++++++++++++++++++++++++++++++ src/trader/types.ts | 50 +++++++++++++++++++++++++ 3 files changed, 165 insertions(+) diff --git a/src/trader/main.ts b/src/trader/main.ts index a775d43..022a893 100644 --- a/src/trader/main.ts +++ b/src/trader/main.ts @@ -1036,6 +1036,43 @@ export async function startTrader(): Promise { market, swap, comms: { sendDm: sender.sendDm.bind(sender) }, + // Invoice-based withdraw path. When the SDK's accounting module is + // available, expose a narrow facade so the trader's WITHDRAW_TOKEN + // handler can use createInvoice + payInvoice instead of payments.send + // directly. Mirrors the swap-deposit flow and avoids the "Authenticator + // does not match source state predicate" flake on spends of received + // swap-payout tokens. + ...(sphere.accounting + ? { + accounting: { + createInvoice: async (req: import('./types.js').AccountingCreateInvoiceRequest) => { + const result = await sphere.accounting!.createInvoice({ + targets: req.targets.map((t) => ({ + address: t.address, + assets: t.assets.map((a) => ({ coin: a.coin as [string, string] })), + })), + ...(req.memo !== undefined ? { memo: req.memo } : {}), + }); + return { + success: result.success, + ...(result.invoiceId !== undefined ? { invoiceId: result.invoiceId } : {}), + ...(result.error !== undefined ? { error: result.error } : {}), + }; + }, + payInvoice: async ( + invoiceId: string, + params: import('./types.js').AccountingPayInvoiceParams, + ) => { + const result = await sphere.accounting!.payInvoice(invoiceId, params); + return { + id: result.id, + status: String(result.status), + ...(result.error !== undefined ? { error: result.error } : {}), + }; + }, + }, + } + : {}), // subscribeEvent is retained for interface compatibility but swap events // are ALL handled by direct sphere.on() listeners below (lines 418+). // This bridge is only needed for non-swap events in the future. diff --git a/src/trader/trader-main.ts b/src/trader/trader-main.ts index dc776f2..142a2d0 100644 --- a/src/trader/trader-main.ts +++ b/src/trader/trader-main.ts @@ -23,6 +23,7 @@ import { createCommandHandler } from '../tenant/command-handler.js'; import type { CommandHandler } from '../tenant/command-handler.js'; import type { + AccountingAdapter, PaymentsAdapter, MarketAdapter, MarketSearchResult, @@ -60,6 +61,14 @@ export interface TraderMainDeps { readonly market: MarketAdapter; readonly swap: SwapAdapter; readonly comms: { sendDm: (to: string, content: string) => Promise }; + /** + * Optional invoice-based withdraw path. When present, WITHDRAW_TOKEN + * routes through accounting.createInvoice + payInvoice instead of + * payments.send directly — the same code path swap deposits use, so + * predicate-handling is well-tested. Optional so unit tests with + * stub adapters don't need to wire this layer. + */ + readonly accounting?: AccountingAdapter; // Sphere instance controls readonly subscribeEvent: (eventType: string, handler: (...args: unknown[]) => void) => () => void; @@ -145,6 +154,7 @@ export function createTraderAgent(deps: TraderMainDeps): TraderAgent { market, swap, comms, + accounting, // subscribeEvent is available but unused — all swap events are handled // by direct sphere.on() listeners in main.ts, not via this bridge. signMessage, @@ -196,6 +206,74 @@ export function createTraderAgent(deps: TraderMainDeps): TraderAgent { async function withdraw( params: WithdrawTokenParams, ): Promise<{ transfer_id: string; remaining_balance: bigint }> { + // Invoice-based path (preferred). The trader creates a local invoice + // with a single target = `params.to_address`, then pays it via + // `accounting.payInvoice`. This is the same code path swap deposits + // use, so it inherits the SDK's well-tested handling for invoice-target + // predicates. The recipient sees the inbound transfer with an invoice + // memo and (if their wallet has accounting enabled) auto-imports the + // invoice — matching how swap-payouts are received. + // + // The direct `payments.send` path used to be the implementation but + // can flake with "Authenticator does not match source state predicate" + // when the spend queue picks a token whose source-state predicate + // doesn't match the wallet's main key (typical of swap-payout tokens + // received with a per-transfer salted predicate). + if (accounting !== undefined) { + // The accounting module's createInvoice validates `coin[0]` as a + // SYMBOL (≤20 chars, alphanumeric), not a 64-hex coinId. We resolve + // the symbol to a coinIdHex separately for the post-pay balance + // computation, but pass the symbol to createInvoice itself. + const balances = payments.getAllBalances(); + const matched = balances.find((b) => b.symbol === params.asset || b.coinId === params.asset); + if (matched === undefined) { + throw new Error(`withdraw: unknown asset "${params.asset}" — no token in wallet matches`); + } + const symbol = matched.symbol ?? params.asset; + const coinIdHexForBalance = matched.coinId; + const invoice = await accounting.createInvoice({ + targets: [{ + address: params.to_address, + assets: [{ coin: [symbol, params.amount] }], + }], + memo: `withdraw ${params.amount} ${symbol} → ${params.to_address}`, + }); + if (!invoice.success || invoice.invoiceId === undefined) { + throw new Error(`accounting.createInvoice failed: ${invoice.error ?? 'unknown'}`); + } + const payResult = await accounting.payInvoice(invoice.invoiceId, { + targetIndex: 0, + amount: params.amount, + }); + if (payResult.error !== undefined && payResult.error !== '') { + logger.warn('withdraw_pay_invoice_returned_error', { + asset: params.asset, + amount: params.amount, + to_address: params.to_address, + invoice_id: invoice.invoiceId, + transfer_id: payResult.id, + status: payResult.status, + error: payResult.error, + }); + throw new Error(`accounting.payInvoice failed: ${payResult.error}`); + } + logger.info('withdraw_sent_via_invoice', { + asset: params.asset, + amount: params.amount, + to_address: params.to_address, + invoice_id: invoice.invoiceId, + transfer_id: payResult.id, + status: payResult.status, + }); + const remaining = payments.getConfirmedBalance(coinIdHexForBalance) - BigInt(params.amount); + return { + transfer_id: payResult.id, + remaining_balance: remaining < 0n ? 0n : remaining, + }; + } + + // Legacy direct-send path. Kept for unit tests with stub adapters and + // for environments where the accounting module is unavailable. const sendResult = await payments.send({ coinId: params.asset, amount: params.amount, diff --git a/src/trader/types.ts b/src/trader/types.ts index 9473fac..61c45e1 100644 --- a/src/trader/types.ts +++ b/src/trader/types.ts @@ -260,6 +260,56 @@ export interface PaymentsAdapter { send(request: SendTokenRequest): Promise; } +// --------------------------------------------------------------------------- +// AccountingAdapter — narrow abstraction over Sphere SDK AccountingModule +// --------------------------------------------------------------------------- + +/** + * Subset of `accounting.createInvoice` request needed by the trader's + * invoice-based withdraw path. Mirrors the SDK's CreateInvoiceRequest + * but stays narrow so tests can inject a recording stub. + */ +export interface AccountingInvoiceTarget { + readonly address: string; + /** Each entry's `coin` is `[coinIdHex, amountStr]`. */ + readonly assets: ReadonlyArray<{ readonly coin: readonly [string, string] }>; +} + +export interface AccountingCreateInvoiceRequest { + readonly targets: readonly AccountingInvoiceTarget[]; + readonly memo?: string; +} + +export interface AccountingCreateInvoiceResult { + readonly success: boolean; + readonly invoiceId?: string; + readonly error?: string; +} + +export interface AccountingPayInvoiceParams { + readonly targetIndex: number; + readonly assetIndex?: number; + readonly amount?: string; +} + +export interface AccountingPayInvoiceResult { + readonly id: string; + readonly status: string; + readonly error?: string; +} + +/** + * Narrow facade over `sphere.accounting`. Used by the trader's withdraw + * flow to route value via the invoicing system (create invoice locally + * → pay it via payInvoice) instead of `payments.send` directly. The + * invoicing path is the same one swap deposits use, so it inherits the + * SDK's well-tested predicate-handling for invoice-target predicates. + */ +export interface AccountingAdapter { + createInvoice(request: AccountingCreateInvoiceRequest): Promise; + payInvoice(invoiceId: string, params: AccountingPayInvoiceParams): Promise; +} + // --------------------------------------------------------------------------- // MarketAdapter — narrow abstraction over Sphere SDK MarketModule // --------------------------------------------------------------------------- From 5193e6fac9a77b8c8a4065c0a3bdf523cbb50226 Mon Sep 17 00:00:00 2001 From: Vladimir Rogojin Date: Sat, 9 May 2026 23:15:10 +0200 Subject: [PATCH 26/30] test(hma-settlement): revert funding from TRADER_TEST_FUND to FAUCET_REQUEST MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Switches the funding path back to FAUCET_REQUEST DMs against the shared js-faucet agent (the production-realistic path) now that js-faucet's conservative-transferMode fix and escrow's matching fix let recipients spend the delivered tokens without hitting the "Authenticator does not match source state predicate" race. Recipient address resolution: send to the trader's @nametag (canonical identity per project guidelines). The previous attempt at DIRECT:// hit "No binding event found" because the SDK's resolveAddressInfo queries by hashed address and the trader's binding publishes the L3-predicate-derived directAddress, not the bare DIRECT:// string. @nametag goes through queryPubkeyByNametag which is the path the trader registers via Sphere.init(nametag=…). Verified end-to-end: both Pair-1 (rate=1) and Pair-2 (rate=3) reach deal COMPLETED + withdraw verified in ~140-175s each. --- .../hma-trade-settlement.e2e-live.test.ts | 90 ++++++++++++------- 1 file changed, 57 insertions(+), 33 deletions(-) diff --git a/test/e2e-live/hma-trade-settlement.e2e-live.test.ts b/test/e2e-live/hma-trade-settlement.e2e-live.test.ts index a6576be..049676e 100644 --- a/test/e2e-live/hma-trade-settlement.e2e-live.test.ts +++ b/test/e2e-live/hma-trade-settlement.e2e-live.test.ts @@ -127,7 +127,6 @@ import { type PortfolioBalance, } from './helpers/sphere-trader.js'; import { createFaucetClient, type FaucetClient } from './helpers/faucet-client.js'; -import { UCT_COIN_ID, USDU_COIN_ID } from './helpers/constants.js'; // --------------------------------------------------------------------------- // Precondition gates (mirrors hma-trade-flow's structure) @@ -437,23 +436,12 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne timeoutMs: 180_000, env: s.spawnEnv, }); - // Trader-side selfMint funding via TRADER_TEST_FUND. Diagnostic note - // (rounds 17-19, 2026-05-04): faucet-funded tokens caused Bob's - // deposit to fail with "Ownership verification failed: Authenticator - // does not match source state predicate" — tokens minted+sent by the - // faucet had a source-state predicate that the trader's swap-deposit - // signing path could not match. basic-roundtrip uses selfMintFund - // and settles correctly because each trader mints with its OWN key - // as the source-state predicate. Switching this test to selfMint - // isolates the predicate-mismatch issue from the rest of the - // HMA-orchestrated flow. TRADER_TEST_FUND is gated by - // TRADER_FAULT_INJECTION_ALLOWED=1 + UNICITY_NETWORK ∈ {testnet,dev} - // (trader/main.ts:1524). HMA's payload-env validator allows both - // (UNICITY_* prefix is forbidden but TRADER_* is not). - const fundEnv: Record = { - TRADER_TEST_FUND: `${UCT_COIN_ID}:${INITIAL_FUND_AMOUNT.toString()},${USDU_COIN_ID}:${INITIAL_FUND_AMOUNT.toString()}`, - TRADER_FAULT_INJECTION_ALLOWED: '1', - }; + // Funding is delivered separately via FAUCET_REQUEST DM (see below) + // — traders boot with no balance, then the test sends FAUCET_REQUEST + // ACP DMs to the shared faucet-agent which mints + transfers tokens. + // This matches production reality (faucet as token issuer) more + // closely than TRADER_TEST_FUND self-mint and exercises the SDK's + // payments.receive({finalize:true}) ingestion path. const alice = await hostSpawnAsync({ cliPath: s.cliPath, cliHome: controller.cliHome, @@ -461,7 +449,7 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne templateId: 'trader-agent', instanceName: `alice-${scenarioId}`, timeoutMs: 180_000, - env: { ...s.spawnEnv, ...fundEnv }, + env: s.spawnEnv, }); const bob = await hostSpawnAsync({ cliPath: s.cliPath, @@ -470,14 +458,50 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne templateId: 'trader-agent', instanceName: `bob-${scenarioId}`, timeoutMs: 180_000, - env: { ...s.spawnEnv, ...fundEnv }, + env: s.spawnEnv, }); s.spawned.push(escrow, alice, bob); console.log( `[${scenarioId}] up: escrow=${escrow.instanceName} ` + - `alice=${alice.instanceName} bob=${bob.instanceName} ` + - `(selfMint UCT+USDU=${INITIAL_FUND_AMOUNT})`, + `alice=${alice.instanceName} bob=${bob.instanceName}`, ); + + // Fund each trader via FAUCET_REQUEST — sends an ACP-0 DM to the + // shared faucet-agent which mints UCT+USDU and sends them to the + // trader. Sequential within scenario to avoid relay-side contention + // (the in-process FaucetClient holds a single Sphere wallet whose + // DM-send path serializes anyway). The faucet handles mint + send; + // the trader's periodic payments.receive({finalize:true}) loop + // ingests the inbound transfer and surfaces it in portfolio. + // + // Recipient address: use the trader's @nametag (canonical identity + // per project guidelines). The trader publishes its nametag binding + // event during Sphere.init and verifies the binding is queryable on + // the relay before announcing sphere_initialized — so by the time + // the trader is "spawned" (HMA acp.hello received), the relay has + // the binding. Sending to DIRECT:// would also work IF the + // SDK's resolveAddressInfo could find the binding, but the binding + // event publishes the L3-predicate-derived directAddress (not the + // bare pubkey-prefixed shape), so that lookup misses. @nametag goes + // through queryPubkeyByNametag which is what the trader registered. + for (const t of [{ name: 'alice', tenant: alice }, { name: 'bob', tenant: bob }]) { + const recipient = t.tenant.tenantNametag !== null + ? `@${t.tenant.tenantNametag}` + : t.tenant.tenantDirectAddress; + console.log(`[${scenarioId}] FAUCET_REQUEST → ${t.name} (recipient=${recipient}, UCT+USDU=${INITIAL_FUND_AMOUNT})…`); + const deliveries = await s.faucetClient.request(s.faucet.tenantPubkey, { + recipient, + items: [ + { asset: 'UCT', amount: INITIAL_FUND_AMOUNT.toString() }, + { asset: 'USDU', amount: INITIAL_FUND_AMOUNT.toString() }, + ], + }); + console.log( + `[${scenarioId}] ${t.name} faucet deliveries: ` + + deliveries.map((d) => `${d.asset}=${d.amount} (transfer=${d.transfer_id.slice(0, 12)}…)`).join(', '), + ); + } + return { escrow, alice, bob }; } @@ -538,13 +562,13 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne const s = state; const { escrow, alice, bob } = await provisionTriple(scenarioId, controller); - // selfMint funding: the trader runs sphere.payments.mintFungibleToken - // for each TRADER_TEST_FUND entry BEFORE agent.start() (see - // trader/main.ts:1524). Tokens are confirmed on the L3 aggregator - // before the trader logs sphere_initialized, so the balance is - // typically present on the first GET_PORTFOLIO. Keep the polling - // loop anyway as a safety net for slow aggregator round-trips. - console.log(`[${scenarioId}] waiting for selfMint balances to confirm…`); + // FAUCET_REQUEST funding: the test already issued FAUCET_REQUEST DMs + // in provisionTriple and the faucet returned acp.result with + // delivery records. The trader's periodic + // payments.receive({finalize:true}) loop must ingest the inbound + // transfer before the balance surfaces in portfolio — poll until + // confirmed >= INITIAL_FUND_AMOUNT for both UCT and USDU. + console.log(`[${scenarioId}] waiting for faucet-funded balances to confirm…`); for (const t of [{ name: 'alice', tenant: alice }, { name: 'bob', tenant: bob }]) { const deadline = Date.now() + FUNDING_BALANCE_TIMEOUT_MS; let lastSnapshot: readonly PortfolioBalance[] = []; @@ -708,10 +732,10 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne ).toBe(expectedUsduPaid); // ---- 7. Withdraw from alice (now has UCT) --------------------- - // Alice had 0 UCT pre-trade and acquired TRADE_VOLUME via the swap. - // Withdraw a fraction (WITHDRAW_AMOUNT) to the controller's DIRECT - // address — exercises the WITHDRAW_TOKEN ACP command end-to-end - // including the round-6 trim+validation gate. + // Alice had INITIAL_FUND_AMOUNT UCT pre-trade and acquired + // TRADE_VOLUME via the swap. Withdraw a fraction (WITHDRAW_AMOUNT) + // to the controller's DIRECT address — exercises the WITHDRAW_TOKEN + // ACP command end-to-end including the round-6 trim+validation gate. console.log(`[${scenarioId}] withdraw ${WITHDRAW_AMOUNT} UCT from alice → controller…`); const wr = await withdrawAsync({ cliPath: s.cliPath, cliHome: controller.cliHome, tenant: alice.tenantPubkey, From 38be0b5c2fab33b10ce5c07972884c22a5ee4112 Mon Sep 17 00:00:00 2001 From: Vladimir Rogojin Date: Sat, 9 May 2026 23:25:14 +0200 Subject: [PATCH 27/30] fix(trader): bounded retry in swap:proposal_received to handle DM arrival-order race MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pair-2 of HMA-trade-settlement.e2e-live was failing with deals stuck at ACCEPTED → CANCELLED while Pair-1 (same code, different rate) reliably passed. Investigation traced the failure to a two-DM race in the proposer-side swap initiation: When a counterparty accepts a match, negotiation-handler sends np.propose_deal to us, then swap-executor.executeDeal() calls swap.proposeSwap() (sphere-sdk SwapModule) which sends an independent swap_proposal DM. These are two separate Nostr events with no causal ordering. On the receiver side: - sphere-sdk fires swap:proposal_received synchronously from the NIP-17 receive path → main.ts immediately calls agent.registerSwapId(). - registerSwapId looks up activeByDealId for an entry with swapId=null. That entry is created by swap-executor.executeDeal()'s registerActive(), which is reached only AFTER negotiation-handler.handleProposeDeal finishes its long async pipeline (validate → intent lookup → terms check → transitionDeal('ACCEPTED') → reply DM → onDealAccepted → executeDeal). In Pair-1, np.propose_deal arrived ~106ms before swap_proposal — the heavier handler completed in time and registerSwapId saw the deal. In Pair-2, swap_proposal arrived just 21ms before NP-0 finished registering, so registerSwapId returned false → swap rejected with NO_LIVE_NP0_DEAL → deal stuck ACCEPTED with no path forward. Fix: bounded retry in main.ts swap:proposal_received handler. registerSwapId is called up to 40 times with 50ms backoff (~2s total wait) before falling through to rejectSwap. Applied symmetrically to both the status-based path and the legacy fallback path. Safe because registerSwapId still cross-checks counterparty pubkey, currencies, amounts, escrow address, and timeout against negotiated DealTerms; retrying only papers over the microsecond-scale ordering hazard. A genuinely hostile peer still gets rejected after the bounded wait. Plus docs/HMA-SETTLEMENT-DIAGNOSTIC.md updated with rounds 21-22 + final SDK changes summary. Verified: HMA-trade-settlement.e2e-live both Pair-1 and Pair-2 reach deal COMPLETED + withdraw verified concurrently. --- docs/HMA-SETTLEMENT-DIAGNOSTIC.md | 160 +++++++++++++++++++++++++++++- src/trader/main.ts | 41 +++++++- 2 files changed, 197 insertions(+), 4 deletions(-) diff --git a/docs/HMA-SETTLEMENT-DIAGNOSTIC.md b/docs/HMA-SETTLEMENT-DIAGNOSTIC.md index c8d19ae..6b0986e 100644 --- a/docs/HMA-SETTLEMENT-DIAGNOSTIC.md +++ b/docs/HMA-SETTLEMENT-DIAGNOSTIC.md @@ -553,10 +553,168 @@ inside `sphere-sdk` itself. --- +## Round 21 (2026-05-08) — swap deps facade missing `getTokenIdsForInvoice` → `verifyPayout` permanently fail-closed + +**Context:** after the round-20 selfMint switch, escrow logged +`Swap completed successfully` and the trader received the payout. But +`swap_payout_verify_diag` retried 40 times with +`tokenInvoiceMap is empty for this payout invoice — failing closed until +reverse index rebuilds`. The 8-min test timeout expired; both pairs +failed at `did not reach state="COMPLETED"`. + +**Investigation:** added 4 `logger.warn` probes inside +`sphere-sdk/modules/accounting/AccountingModule.ts` to trace which path +was populating `tokenInvoiceMap`: + +- `_handleTransferConfirmed` (on-chain confirmed event handler) +- `_processInvoiceTransferEvent` synthetic-ledger branch +- `_processInvoiceHistoryEvent` (history-update path) +- per-token populate inside the synthetic-ledger loop + +Three runs showed only ONE `transfer:confirmed` line per trader (the +deposit-send confirmation, sender side). NONE of the receive-side +populate paths fired for the swap payout. Yet `swap_payout_verify_diag` +showed `coveredAmount: 10` from `getInvoiceStatus`. So the invoice +ledger HAD an entry — but `getTokenIdsForInvoice(payoutInvoiceId)` +returned an empty Set. + +Added a 5th probe directly inside `verifyPayout`'s fail-closed branch +to dump `tokenInvoiceMap` state. Result: + +``` +R20-DIAG verifyPayout-fail-closed swap=… (no tokenInvoiceMap accessor) +``` + +The optional-chain `acct.tokenInvoiceMap?.` fell through. The accessor +did not exist on whatever object `acct` was. + +**Root cause:** `Sphere.ts` constructs the `accounting` dep facade for +`SwapModule` at lines 2537-2543 and 4361-4367 as a hand-written narrow +object exposing only 5 methods (`importInvoice`, `getInvoice`, +`getInvoiceStatus`, `payInvoice`, `on`). `getTokenIdsForInvoice` was +NOT on the facade. The `verifyPayout` call site used a defensive +optional-chain type cast which silently returned `undefined` → empty +`Set` → fail-closed forever. There was no way for the index to "rebuild" +because the SDK was checking a method that didn't exist on the object +the swap module had been handed at construction. + +**Fix:** added `getTokenIdsForInvoice` to the +`SwapModuleDependencies.accounting` interface and wired it through both +facade construction sites in `Sphere.ts`. Defensive `tokenInvoiceMap` +migration also added in `importInvoice` (defense-in-depth for the +orphan-buffer race where token-receive lands before invoice-import). + +**Outcome:** `hma-trade-settlement.e2e-live`'s settlement phase now +completes end-to-end. Both pairs reach `deal COMPLETED` in ~140s, down +from the 600s+ timeout. Pair-2 settled successfully in this round; +Pair-1 failed only on a transient testnet Market API HTTP 502 +(unrelated to the SDK fix). + +--- + +## Round 22 (2026-05-09) — `finalizeReceivedToken` error paths flipped `status='confirmed'` on un-finalized tokens → withdraw flake + +**Context:** with round-21's SDK fix in place, settlement reaches +COMPLETED reliably. But the next step — withdraw of 3 UCT from alice → +controller — FLAKED. Sometimes the withdraw succeeded with +`transfer_id=…`; sometimes it failed with the SAME error pattern as the +round-19 faucet-funded predicate issue: + +``` +Ownership verification failed: Authenticator does not match source +state predicate. +``` + +**Earlier mis-diagnosis:** at first I thought the bug must be +`payments.send`'s direct-spend path vs the invoicing path. The user +correctly pushed back: *"We apparently using the SDK wrong. SDK itself +supposedly have no issue."* Switched the trader's `WITHDRAW_TOKEN` +handler to use `accounting.payInvoice` (create local invoice with +`target=to_address`, then pay it) — same code path swap deposits use. +Test still flaked with the same error. Confirmed by code trace that +`accounting.payInvoice` ultimately calls `payments.send`, so the bug +couldn't be in the entry-point choice. + +**Investigation:** added a probe in `payments.send`'s commitment-submit +path to log +`commitment.transactionData.sourceState.predicate.publicKey` vs +`commitment.authenticator.publicKey`. The probe didn't fire on most +runs (different code path), but careful inspection of the receive flow +led to the right place. + +**Root cause:** `sphere-sdk/modules/payments/PaymentsModule.ts:5362-5388, +5423-5431` — `finalizeReceivedToken`. Three error paths set +`token.status = 'confirmed'` WITHOUT updating `sdkData`: + +1. Missing `waitForProofSdk` (line 5362-5367) +2. Missing `stClient` / `trustBase` (line 5382-5388) +3. Caught exception during `finalizeTransferToken` (line 5424-5430, + with comment *"Mark as confirmed anyway (user has the token)"*) + +The original intent was "user has the token, mark confirmed for UI." +The flaw: `sdkData` was never updated, so the token kept the SENDER's +source-state predicate. The spend queue's filter +(`SpendQueue.ts:91 status !== 'confirmed' continue`) let these +mislabeled tokens through. When picked, the resulting commitment built +`sourceState.predicate` from the SENDER's stored state and +`authenticator.publicKey` from the RECEIVER's signing service. +`predicate.isOwner(...)` is a hex compare → false → state-transition-sdk +threw the predicate-mismatch error. + +The flake explanation: alice's wallet had selfMint UCT 5000 (truly +finalized, predicate=alice's key) AND swap-payout UCT 10 (status flipped +to 'confirmed' by the buggy error paths even though finalization didn't +fully complete). The spend queue's pick varied: selfMint → success; +swap-payout → fail. Same wallet, same withdraw call, two distinct +outcomes depending on which token the queue iterator yielded first. + +**Fix:** all three error paths now leave `status='submitted'`. The +next periodic `resolveUnconfirmed()` / `receive({finalize:true})` will +retry the finalize properly. Until then, the spend queue correctly +skips the un-finalized token and picks a truly-finalized one. + +**Outcome:** `hma-trade-settlement.e2e-live` Pair-1 reaches: + +``` +deal COMPLETED +withdraw transfer_id=22bc307f-d865-41c5-… +✓ end-to-end settlement+withdraw verified +``` + +in 128s. **The user's stated goal — operators launch HMA → spawn → fund +→ trade → settle → withdraw, all over Sphere DMs — is reached.** + +Pair-2 still fails on a separate concurrent-settlement race (deals go +ACCEPTED → CANCELLED). That race is being investigated as a follow-up +and is independent of the three SDK fixes landed in this session. + +--- + +## Final SDK changes summary + +The three SDK fixes are split into three focused PRs against +`sphere-sdk`: + +| Branch | Fix | +|---|---| +| `fix/swap-getTokenIdsForInvoice` | Wire `getTokenIdsForInvoice` through the `SwapModuleDependencies.accounting` facade in `Sphere.ts` (2 construction sites). Stops `verifyPayout` from permanently fail-closing on an empty Set returned by an absent accessor. | +| `fix/accounting-importInvoice-token-map-migration` | Defense-in-depth: when `importInvoice` runs after the inbound transfer already landed in the orphan buffer, migrate any orphaned token entries into `tokenInvoiceMap` for the freshly-imported invoice. | +| `fix/payments-finalize-error-status` | `finalizeReceivedToken` no longer sets `status='confirmed'` on the three error paths (missing `waitForProofSdk`, missing `stClient`/`trustBase`, caught finalize exception). Leaves `status='submitted'` so the spend queue skips and the next `resolveUnconfirmed()` retries. | + +The trader + CLI work that made the settlement+withdraw test usable +ships separately: + +| Branch | Repo | Contents | +|---|---|---| +| `feat/hma-trade-settlement-live` | trader-service | `hma-trade-settlement.e2e-live.test.ts` + invoicing-based `WITHDRAW_TOKEN` handler in trader. | +| `feat/trader-withdraw-cli` | sphere-cli | `sphere trader withdraw` subcommand exposing the WITHDRAW_TOKEN ACP message over DM. | + +--- + ## Out of scope for this debugging session - Production hardening of js-faucet (rate limiting, batched mints, etc.) - Pushing js-faucet image to ghcr.io/unicitynetwork (needs PAT) - Adding `faucet-agent` template entry to agentic-hosting/config/templates.json - `sphere faucet request` subcommand in sphere-cli -- Withdraw-via-HMA live test +- Concurrent-settlement race on Pair-2 (ACCEPTED → CANCELLED) — follow-up diff --git a/src/trader/main.ts b/src/trader/main.ts index 022a893..3f95ce4 100644 --- a/src/trader/main.ts +++ b/src/trader/main.ts @@ -1292,7 +1292,23 @@ export async function startTrader(): Promise { return; } - registered = agent.registerSwapId(data.swapId, { + // R23 fix (Pair-2 race): the swap_proposal DM and np.propose_deal + // DM can arrive in either order. When swap_proposal arrives FIRST, + // the np.propose_deal handler is still running (validation → + // transitionDeal('ACCEPTED') → onDealAccepted → executeDeal → + // registerActive); `activeByDealId` is empty so the very first + // registerSwapId call returns false and we'd reject the swap + // even though we're about to accept the deal. + // + // Fix: bounded retry — registerSwapId still cross-checks + // counterparty pubkey, currencies, amounts, escrow address, and + // timeout against negotiated DealTerms, so retrying is safe; we + // only paper over the microsecond-scale ordering hazard. If the + // deal really wasn't accepted (hostile peer, stale state), we + // still reject after the bounded wait. + const REGISTER_MAX_ATTEMPTS = 40; + const REGISTER_BACKOFF_MS = 50; + const registerArgs = { partyACurrency: s.deal?.partyACurrency, partyAAmount: s.deal?.partyAAmount, partyBCurrency: s.deal?.partyBCurrency, @@ -1307,13 +1323,32 @@ export async function startTrader(): Promise { escrowPubkey: (s as unknown as { escrowPubkey?: string }).escrowPubkey, depositTimeoutSec: (s.deal as unknown as { timeout?: number; depositTimeoutSec?: number })?.depositTimeoutSec ?? (s.deal as unknown as { timeout?: number })?.timeout, - }); + }; + for (let attempt = 0; attempt < REGISTER_MAX_ATTEMPTS; attempt++) { + registered = agent.registerSwapId(data.swapId, registerArgs); + if (registered) break; + if (attempt + 1 < REGISTER_MAX_ATTEMPTS) { + await new Promise((r) => setTimeout(r, REGISTER_BACKOFF_MS)); + } + } } catch (err: unknown) { logger.warn('swap_proposal_status_fetch_failed', { swap_id: data.swapId, error: err instanceof Error ? err.message : String(err), }); - registered = agent.registerSwapId(data.swapId); + // R23 fix (legacy fallback): same bounded retry as the + // status-based path above — without it, a transient + // getSwapStatus failure compounds the np.propose_deal / + // swap_proposal arrival-order race. + const REGISTER_MAX_ATTEMPTS = 40; + const REGISTER_BACKOFF_MS = 50; + for (let attempt = 0; attempt < REGISTER_MAX_ATTEMPTS; attempt++) { + registered = agent.registerSwapId(data.swapId); + if (registered) break; + if (attempt + 1 < REGISTER_MAX_ATTEMPTS) { + await new Promise((r) => setTimeout(r, REGISTER_BACKOFF_MS)); + } + } } if (!registered) { From de9f0b7d9fd24c7a32d0f12360879501391a5cd4 Mon Sep 17 00:00:00 2001 From: Vladimir Rogojin Date: Sun, 10 May 2026 11:38:29 +0200 Subject: [PATCH 28/30] fix(withdraw): use conservative transferMode through accounting.payInvoice Withdraw was racing recipient-side proof-poll on the sender's swap-payout tokens. The default 'instant' transferMode delivers an unconfirmed {sourceToken, transferTx} bundle whose finalization completes asynchronously after receipt. When the trader's spend queue picked a not-yet-finalized token (e.g., the swap payout that just arrived) for the withdraw transfer, it produced "Authenticator does not match source state predicate" because the recipient's confirmed Token wasn't yet bound to its predicate. Switch withdraw's accounting.payInvoice call to transferMode: 'conservative'. The SDK now collects the inclusion proof on the SENDER (controller) side before delivery, so the controller's wallet receives a fully-finalized bundle and produces a 'confirmed' Token immediately bound to its own predicate. This mirrors the faucet's funding flow and the escrow's swap-payout flow, which both already use 'conservative' on their direct payments.send paths. The trader's invoiced withdraw was the only forwarding flow still using the default. Requires the corresponding sphere-sdk change that exposes transferMode on PayInvoiceParams (test/all-fixes-r23 branch, commit 4e77b2f). --- src/trader/trader-main.ts | 11 +++++++++++ src/trader/types.ts | 9 +++++++++ 2 files changed, 20 insertions(+) diff --git a/src/trader/trader-main.ts b/src/trader/trader-main.ts index 142a2d0..cf9f7fb 100644 --- a/src/trader/trader-main.ts +++ b/src/trader/trader-main.ts @@ -244,6 +244,17 @@ export function createTraderAgent(deps: TraderMainDeps): TraderAgent { const payResult = await accounting.payInvoice(invoice.invoiceId, { targetIndex: 0, amount: params.amount, + // Conservative mode: the SDK collects the inclusion proof on the + // SENDER's side before delivery, so the recipient receives a + // fully-finalized {sourceToken, transferTx} bundle and can produce + // a 'confirmed' Token immediately bound to its own predicate. This + // mirrors the faucet and escrow's payout flows. The default + // 'instant' mode ships an unconfirmed bundle whose recipient-side + // proof-poll races with any chained spend (e.g. another withdraw or + // a swap deposit) and intermittently surfaces "Authenticator does + // not match source state predicate" errors when the spend queue + // picks a not-yet-finalized token. + transferMode: 'conservative', }); if (payResult.error !== undefined && payResult.error !== '') { logger.warn('withdraw_pay_invoice_returned_error', { diff --git a/src/trader/types.ts b/src/trader/types.ts index 61c45e1..71496c1 100644 --- a/src/trader/types.ts +++ b/src/trader/types.ts @@ -290,6 +290,15 @@ export interface AccountingPayInvoiceParams { readonly targetIndex: number; readonly assetIndex?: number; readonly amount?: string; + /** + * Transfer-delivery mode. `'conservative'` collects the inclusion + * proof on the sender's side before delivery; the recipient gets a + * fully-finalized bundle and can spend it immediately. `'instant'` + * (default) ships an unconfirmed bundle that the recipient finalizes + * via background proof-poll. Use `'conservative'` for withdraw flows + * so the recipient's spend doesn't race the proof-poll. + */ + readonly transferMode?: 'instant' | 'conservative'; } export interface AccountingPayInvoiceResult { From 924e70000ed7dcee5e8d51a092ee56d18a2ef38a Mon Sep 17 00:00:00 2001 From: Vladimir Rogojin Date: Sun, 10 May 2026 12:34:34 +0200 Subject: [PATCH 29/30] =?UTF-8?q?docs(diagnostic):=20Round=2023=20?= =?UTF-8?q?=E2=80=94=20conservative=20transferMode=20resolves=20predicate-?= =?UTF-8?q?mismatch=20flake?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/HMA-SETTLEMENT-DIAGNOSTIC.md | 84 +++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/docs/HMA-SETTLEMENT-DIAGNOSTIC.md b/docs/HMA-SETTLEMENT-DIAGNOSTIC.md index 6b0986e..2509b16 100644 --- a/docs/HMA-SETTLEMENT-DIAGNOSTIC.md +++ b/docs/HMA-SETTLEMENT-DIAGNOSTIC.md @@ -718,3 +718,87 @@ ships separately: - Adding `faucet-agent` template entry to agentic-hosting/config/templates.json - `sphere faucet request` subcommand in sphere-cli - Concurrent-settlement race on Pair-2 (ACCEPTED → CANCELLED) — follow-up + +--- + +## Round 23 — RESOLUTION (2026-05-10) + +### TL;DR + +**Both Pair-1 and Pair-2 ✓ PASS end-to-end** in the latest live e2e +run after baking `transferMode: 'conservative'` into the trader's +withdraw path. The intermittent **"Authenticator does not match +source state predicate"** error is gone. + +``` +✓ test/e2e-live/hma-trade-settlement.e2e-live.test.ts (2 tests) 169s + ✓ Pair-1: full spawn → trade → settle → withdraw via HMA (rate=1) 153456ms + ✓ Pair-2: parallel scenario settles on the same HMA at distinct rate (rate=3) 154783ms + +[p1-d54068] withdraw transfer_id=5a151db3-ce88-41… ✓ end-to-end settlement+withdraw verified +[p2-a64a0b] withdraw transfer_id=67eb1c73-2645-46… ✓ end-to-end settlement+withdraw verified +``` + +### Root cause (final) + +The trader's invoiced withdraw was the **only** forwarding flow in the +chain still using the default `transferMode: 'instant'`. Faucet and +escrow already used `'conservative'` on their direct `payments.send` +paths. + +Under `'instant'` mode, `PaymentsModule.send` ships a V6 +combined-transfer bundle whose recipient saves the token at +`status='submitted'` with the **sender's** `sdkData` and finalizes via +background proof-poll. If the trader's spend queue (or, for withdraw, +the controller's spend queue) picks a not-yet-finalized incoming token +— such as a swap-payout that arrived seconds earlier — it produces: + +> Authenticator does not match source state predicate + +because the recipient-side `Token` isn't yet bound to the recipient's +predicate. + +`'conservative'` mode collects the inclusion proof on the **sender's** +side before delivery, so the recipient receives a fully-finalized +`{sourceToken, transferTx}` bundle and produces a `'confirmed'` Token +immediately bound to its own predicate. Chained spends are then +race-free. + +### Fix + +A single new field on `PayInvoiceParams` plus three call-site +opt-ins: + +| Repo | Branch / Commit | Change | +|---|---|---| +| sphere-sdk | `feat/accounting-payinvoice-transfermode` (#131) | Add `transferMode?: 'instant' \| 'conservative'` to `PayInvoiceParams`; forward to `PaymentsModule.send`. Default unchanged. | +| trader-service | `feat/hma-trade-settlement-live` (`de9f0b7`) | Withdraw path uses `transferMode: 'conservative'` through `accounting.payInvoice`. | +| escrow-service | `fix/conservative-payout-mode` (#18) | Swap-payout `payments.send` uses `'conservative'`. | +| js-faucet | `fix/faucet-funded-predicate` (#2) | `FAUCET_REQUEST` sends use `'conservative'`. | + +### Why earlier rounds appeared to fix it sometimes + +Previous Pair-2 success was misleading — when the trader image was +rebuilt during Round 22, Pair-2 happened to win the +finalization-vs-spend race purely on timing. The conservative-mode +opt-in eliminates the race architecturally rather than narrowing the +window. + +### Side fixes that landed alongside + +- `feat/market-tolerance` (#132): retry + circuit breaker for transient + Market API 502/503/504/408 errors, so a single load-balancer hiccup + no longer kills an in-progress e2e run. +- `feat/hma-trade-settlement-live` (`38be0b5`): bounded retry in + `swap:proposal_received` to handle the two-DM arrival-order race + that caused Pair-2 deals to flake ACCEPTED→CANCELLED. + +### Verification command + +```bash +cd /home/vrogojin/trader-service +TRADER_E2E_LOCAL_RELAY=1 \ + npx vitest run --config vitest.e2e-live.config.ts \ + test/e2e-live/hma-trade-settlement.e2e-live.test.ts +``` + From 21f6397cf3c469b399f8b8c89a4d35d36991bf2f Mon Sep 17 00:00:00 2001 From: Vladimir Rogojin Date: Sat, 16 May 2026 00:21:54 +0200 Subject: [PATCH 30/30] chore(e2e): adopt escrow v0.2 in HMA-trade-settlement override MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two coordinated changes: 1. constants.ts ESCROW_IMAGE: v0.1 → v0.2 (mirrors master bump from PR #20). Adds composition note in the docstring. 2. hma-trade-settlement.e2e-live.test.ts templates override: was forcing `ghcr.io/vrogojin/agentic-hosting/escrow:local` with a comment blaming the published v0.1 asymmetric deliverDepositInvoice bug. Switched to ESCROW_IMAGE (= v0.2) so the HMA-spawned escrow runs the same code as the direct-Docker-spawned escrow in basic-roundtrip — and devs/CI no longer need to docker-build `escrow:local` before running this test. Why the override remains (vs deleting it entirely): agentic-hosting's config/templates.json still pins escrow:v0.1 on its own release cadence. Until that templates.json bumps, we override here so the HMA-spawned escrow picks up v0.2's: - deliverDepositInvoice fix (round-19 evidence confirms gone) - Conservative transferMode for swap payouts - sphere-sdk UXF protocol PRs (#105, #115, #119, #128, #146/147/149/152) + all payments/* faucet-flow regression fixes v0.2 digest: sha256:311903b6f98b33a63791bf79db6522a66d118588ba56fcf6e56654ed6670ebac Typecheck: clean against tsconfig.json + tsconfig.test.json. --- test/e2e-live/helpers/constants.ts | 25 +++++++++++++++-- .../hma-trade-settlement.e2e-live.test.ts | 28 ++++++++++++++----- 2 files changed, 44 insertions(+), 9 deletions(-) diff --git a/test/e2e-live/helpers/constants.ts b/test/e2e-live/helpers/constants.ts index b39f7ed..dedf422 100644 --- a/test/e2e-live/helpers/constants.ts +++ b/test/e2e-live/helpers/constants.ts @@ -56,8 +56,29 @@ export const USDU_COIN_ID = '8f0f3d7a5e7297be0ee98c63b81bcebb2740f43f616566fc290 /** Default trader image (matches templates.json shortcut). */ export const TRADER_IMAGE = 'ghcr.io/vrogojin/agentic-hosting/trader:v0.2'; -/** Default escrow image. */ -export const ESCROW_IMAGE = 'ghcr.io/vrogojin/agentic-hosting/escrow:v0.1'; +/** + * Default escrow image. + * + * Bumped 2026-05-16 from v0.1 (2026-04-25, predates UXF protocol + + * the deliverDepositInvoice asymmetric-delivery fix surfaced in + * round 19 of the HMA settlement diagnostic) to v0.2 (2026-05-16, + * published from escrow-service@d427e5d + uxf sphere-sdk@3a575cd — + * integration/all-fixes HEAD). Digest: + * + * sha256:311903b6f98b33a63791bf79db6522a66d118588ba56fcf6e56654ed6670ebac + * + * What v0.2 adds vs v0.1: + * - Conservative transferMode for swap payouts (fix/conservative- + * payout-mode HEAD) + * - UNICITY_NOSTR_RELAYS env override (matches the local-infra + * plumbing this harness already uses) + * - deliverDepositInvoice instrumentation + per-party try/catch + * (round-19 evidence confirms the asymmetric bug is GONE in + * current source vs the v0.1 deployed image) + * - sphere-sdk UXF protocol PRs (#105, #115, #119, #128, + * #146/147/149/152) + all payments/* faucet-flow regression fixes + */ +export const ESCROW_IMAGE = 'ghcr.io/vrogojin/agentic-hosting/escrow:v0.2'; /** Per-test default timeout (slow because real-network). */ export const DEFAULT_TIMEOUT_MS = 30_000; diff --git a/test/e2e-live/hma-trade-settlement.e2e-live.test.ts b/test/e2e-live/hma-trade-settlement.e2e-live.test.ts index 049676e..116b912 100644 --- a/test/e2e-live/hma-trade-settlement.e2e-live.test.ts +++ b/test/e2e-live/hma-trade-settlement.e2e-live.test.ts @@ -127,6 +127,7 @@ import { type PortfolioBalance, } from './helpers/sphere-trader.js'; import { createFaucetClient, type FaucetClient } from './helpers/faucet-client.js'; +import { ESCROW_IMAGE } from './helpers/constants.js'; // --------------------------------------------------------------------------- // Precondition gates (mirrors hma-trade-flow's structure) @@ -270,14 +271,27 @@ describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testne if (t.template_id === 'trader-agent') { t.image = 'ghcr.io/vrogojin/agentic-hosting/trader:local'; } - // The published escrow:v0.1 has an asymmetric bug in - // deliverDepositInvoice — the second recipient's invoice_delivery - // DM is never put on the wire, so swaps stall at "ACCEPTED" with - // the trader polling for an invoice that the escrow never sent - // (diag agent traced this 2026-05-05; see HMA-SETTLEMENT-DIAGNOSTIC.md). - // Build escrow:local from current source and use it here. + // Use the same v0.2 image pin as the rest of the e2e-live suite + // (constants.ts ESCROW_IMAGE) so the HMA-spawned escrow runs the + // same code as the direct-Docker-spawned escrow in basic-roundtrip. + // + // Why this override exists at all: agentic-hosting's + // config/templates.json still pins escrow:v0.1 (its own release + // cadence is independent). Until that templates.json bumps, we + // override here so HMA-spawned escrows pick up v0.2's + // deliverDepositInvoice fix + conservative-payout + UXF protocol + // (PR #105, #115, #119, #128, #146/147/149/152). + // + // History: previously this overrode to `escrow:local` because the + // published v0.1 had an asymmetric deliverDepositInvoice bug + // (every other party's invoice_delivery DM was dropped) — see + // HMA-SETTLEMENT-DIAGNOSTIC.md rounds 11-19. Round 19 evidence + // confirmed the bug is GONE in current source (= what we shipped + // as v0.2 on 2026-05-16). The `escrow:local` build dependency + // is now removed — devs/CI no longer need to docker-build the + // escrow image before running this test. if (t.template_id === 'escrow-service') { - t.image = 'ghcr.io/vrogojin/agentic-hosting/escrow:local'; + t.image = ESCROW_IMAGE; } } if (!baseTemplates.templates.some((t) => t.template_id === 'faucet-agent')) {