diff --git a/.sphere-cli/wallet.json b/.sphere-cli/wallet.json new file mode 100644 index 0000000..9e26dfe --- /dev/null +++ b/.sphere-cli/wallet.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/docs/HMA-SETTLEMENT-DIAGNOSTIC.md b/docs/HMA-SETTLEMENT-DIAGNOSTIC.md new file mode 100644 index 0000000..2509b16 --- /dev/null +++ b/docs/HMA-SETTLEMENT-DIAGNOSTIC.md @@ -0,0 +1,804 @@ +# HMA Trade-Settlement Diagnostic + +**Status as of 2026-05-05** — `feat/hma-trade-settlement-live` branch, latest commit `5fb50f0`. + +The `hma-trade-settlement.e2e-live` test still fails. Settlement reaches +ACCEPTED on both scenarios but never COMPLETED. This document captures +what works, what doesn't, what we tried, and the remaining hypotheses +so a follow-up session can pick up where we stopped. + +--- + +## Goal + +End-to-end live proof that operators can: +1. Launch HMA over Sphere DM (no HTTP) +2. Spawn escrow + 2 traders + faucet via `sphere host spawn` +3. Fund traders via the new js-faucet agent over DM +4. Match buy/sell intents +5. Settle the swap (deal → COMPLETED on both sides) +6. Withdraw post-trade tokens to a controller-owned address + +Steps 1-4 work. Step 5 is where we're stuck. + +## Current state (round 12) + +| Layer | Direct-docker (basic-roundtrip) | HMA-spawned (this test) | +|---|---|---| +| Spawn | ✓ | ✓ | +| Funding | ✓ (selfMintFund) | ✓ (faucet via `FAUCET_REQUEST` DM) | +| set-strategy / portfolio / list-intents | ✓ | ✓ | +| Match found | ✓ | ✓ | +| Deal accepted | ✓ | ✓ (round 12 first time) | +| Swap announced to escrow | ✓ | ✓ (round 12) | +| Escrow creates deposit invoice | ✓ | ✓ (round 12 — log: `"Swap announced, deposit invoice created"`) | +| Escrow sends invoice DM to trader | ✓ | ✓ (round 12 — log: `diag_outbound_dm_sent invoice_delivery`) | +| **Trader receives invoice DM** | ✓ | ✗ (log: `invoice_target_addresses: null`) | +| Trader deposits | ✓ | ✗ (blocked) | +| Swap COMPLETED | ✓ | ✗ (blocked) | + +The break is at **trader-side ingest of the escrow's invoice_delivery DM**. + +## Round-by-round progression + +| Round | Outcome | Bug found / fix | +|---|---|---| +| 1-7 | Various early failures | controller wallet races, faucet name mapping, etc. — all fixed | +| 8 | 29 deals each, locked in PROPOSED→FAILED→CANCELLED loop | TRADER_TEST_FUND self-mint produces issuer==sender tokens that confuse swap | +| 9 | Same as 8 | Cross-scenario rate differentiation didn't help | +| 10 | 2 deals each, never reach ACCEPTED, escrow logs `Swap not found` | Diagnosed: trader sends `status` query but never `swap.announce` | +| 11 | 1 scenario reaches ACCEPTED for first time | Trader's `intent-engine.ts:836` defaults `escrow_address` to literal `'any'` when CLI omits `--escrow-address` | +| 12 | **Both scenarios reach ACCEPTED**, escrow creates+sends invoice, but trader doesn't process it | Layer-mismatch fixed (`escrow.tenantPubkey` vs `escrow.tenantDirectAddress` — swap-executor compares to DIRECT://hex) | + +## Bugs already fixed (committed on `feat/hma-trade-settlement-live`) + +1. **Faucet integration** (commit `b2659a3`) — replaces `TRADER_TEST_FUND` self-mint and broken public-faucet HTTP. Fund traders via `FAUCET_REQUEST` DMs to a shared js-faucet agent. Killed the spam-loop. + +2. **`--escrow-address` flag in sphere-cli** (commit `c58a463` in trader-service; sphere-cli rebuild required) — `sphere trader create-intent` previously had no way to set `escrow_address`, so the trader defaulted to `'any'` (per `trader-service/src/trader/intent-engine.ts:836`). The swap-executor then tried to route `swap.announce` to the literal string `'any'`. Now passes through to ACP wire field correctly. + +3. **escrow_address must be DIRECT://hex, not chain pubkey** (commit `5fb50f0`) — `swap-executor.ts:714` compares `terms.escrow_address === match.escrowDirectAddress`. The DIRECT://hex address is structurally derived (`UnmaskedPredicateReference(pubkey).toAddress()`), NOT just `DIRECT://${pubkey}`. Test now passes `escrow.tenantDirectAddress` to both `setStrategy({trustedEscrows: [...]})` and `createIntent({escrowAddress: ...})`. + +## The remaining symptom (in detail) + +Round 12 trader log (`alice` container) shows: + +``` +swap_id_registered (deal_id=..., swap_id=169ea57a..., matched_by="proposal_info") +swap_deposit_target_diag (every ~3s): + swap_id: 169ea57a... + escrowDirectAddress: DIRECT://000055759f52413cab92... ← correct + manifest_party_a_currency: UCT + manifest_party_a_value: 10 + manifest_party_b_currency: USDU + manifest_party_b_value: 10 + invoice_target_addresses: null ← never populated + invoice_target_assets: null +``` + +Round 12 escrow log (same swap_id) shows: + +``` +Swap announced, deposit invoice created invoice_id: 00004af2b309ee84 +diag_invoice_delivery_attempt party: A recipient_prefix: 8ff3bcef9e1aa95d +diag_outbound_dm_sending message_type: invoice_delivery payload_bytes: 5789 +diag_outbound_dm_sent message_type: invoice_delivery +diag_invoice_delivery_complete +[same for party B] +``` + +Escrow believes it sent the DM successfully. Trader has no log entry showing receipt. + +## Why direct-docker works but HMA-spawned doesn't (open question) + +Same trader image (`trader:local`), same escrow image. The difference is the runtime environment: + +| | direct-docker | HMA-spawned | +|---|---|---| +| `UNICITY_MANAGER_PUBKEY` | unset | set | +| `UNICITY_MANAGER_DIRECT_ADDRESS` | unset | set | +| `UNICITY_BOOT_TOKEN` | unset | set | +| ACP heartbeats every 5s | none (no manager to talk to) | active | +| ACP DM listener (`sphere.on('message:dm')`) | active but bails (no manager pubkey to match) | active and validating against manager pubkey | +| Periodic `payments.receive({finalize:true})` loop | active | active | + +Both setups have the ACP listener attached (it's compiled into `startTrader()`). +The difference is whether it has a manager_pubkey to filter against. + +## Hypotheses for the remaining bug + +### Update 2026-05-05: ROOT CAUSE FOUND — escrow side, not trader side + +A focused investigation agent traced the full flow on both ends and found +the bug is in the **escrow's `deliverDepositInvoice` function** (compiled +into `escrow:v0.1` at `/app/dist/sphere/message-handler.js`). It is +**asymmetric**: party A's invoice always delivers; party B's never does. + +Evidence (from one round-12 escrow's logs, repeats across multiple swaps): + +``` +diag_invoice_delivery_attempt party=A ← logged +diag_outbound_dm_sending message_type=invoice_delivery recipient=A ← logged +diag_outbound_dm_sent message_type=invoice_delivery recipient=A ← logged +diag_invoice_delivery_complete party=A ← logged + +diag_invoice_delivery_attempt party=B ← logged +[NO diag_outbound_dm_sending for invoice_delivery to B — never appears] +diag_invoice_delivery_complete party=B ← logged anyway +``` + +The `complete` log fires (no thrown exception); the `sending` log doesn't +(no actual `sendDM` call). The function exits "normally" without delivering +the invoice to party B. + +The deployed `escrow:v0.1` image was built from an unsynced source commit +(JS at `/app/dist/sphere/message-handler.js` does NOT match +`/home/vrogojin/escrow-service/src/sphere/message-handler.ts` at HEAD). +The exact mechanism inside the diverged image (early-return that was missed, +fire-and-forget reference instead of `await reply(...)`, build-step DCE, etc.) +requires access to the unsynced commit to confirm — but the fix is the same +either way: rebuild + re-tag. + +**Why basic-roundtrip works direct-docker with the same image**: +basic-roundtrip uses ONE trader pair. The HMA test uses two pairs concurrently. +Party-A vs party-B is determined by the canonical pubkey ordering in the +swap manifest — concurrent swaps may always produce the same A/B alignment. +But basic-roundtrip's single pair may happen to land in a way where the +party-B-broken path doesn't matter (e.g., the test only asserts the buyer's +side, not the seller's). Worth re-running basic-roundtrip with the +rebuilt image to confirm; the asymmetry is a real defect regardless. + +**Update 2026-05-08 (round 14)**: rebuilt `escrow:local` from current +`escrow-service` HEAD; bug REPRODUCED unchanged. So the bug is in current +source, not the divergence between the deployed image and HEAD as the +original agent suspected. The deployed `escrow:v0.1` image had additional +`diag_invoice_delivery_*` log lines that do NOT exist in HEAD — that's +what made the divergence look like the cause; the bug is structural. + +**Update 2026-05-08 (round 15)**: branched `escrow-service` to +`debug/instrument-deliver-invoice` (`c0e19ea`), instrumented every code +path of `deliverDepositInvoice` (enter / no-id / no-token / sending / +sent / threw) plus per-party try/catch in the announce-handler's for-loop. +Round 15 itself failed at preflight — testnet Nostr relay's write path +went down again (intermittent — every WS publish-kind:* returns no OK). +The instrumented build is committed and pushed; next live attempt against +this branch will produce log lines pinpointing party B's actual path. + +**Update 2026-05-08 (round 19, local-infra fully working)**: +The local-infra harness now runs end-to-end on a Docker-hosted Nostr relay +(no testnet dependency for messaging). Fix chain that closed it: + + - sphere-cli (host/sphere-init.ts AND legacy/legacy-cli.ts) reads + UNICITY_NOSTR_RELAYS / SPHERE_NOSTR_RELAYS; + - helpers/sphere-cli.ts buildEnv() forwards the env into sphere-cli + subprocesses; + - helpers/manager-process.ts: spawnHostManager forwards env to HMA + binary, AND provisionManagerWallet (which pre-creates the manager + wallet + publishes the nametag binding) ALSO reads it — without + this, the nametag binding lands on testnet, the HMA's later + Sphere.init loads the existing wallet (wallet_created: false) and + skips re-publish, sphere-cli's queryPubkeyByNametag returns null + on the local relay; + - helpers/manager-process.ts: UNICITY_HEALTH_PORT default → 0 (OS- + assigned) so leaked HMA processes don't EADDRINUSE the next run; + - hma-trade-settlement test: use SPHERE_NOSTR_RELAYS (NOT + UNICITY_NOSTR_RELAYS) in HMA spawn-env passthrough — HMA's + validatePayloadEnv blocks any env starting with UNICITY_; + - helpers/faucet-client.ts: same env-pickup pattern so the in-process + Sphere wallet that signs FAUCET_REQUEST DMs talks to the local relay. + +Round 19 evidence: + - Local relay log: 534+ kind:1059 (gift-wrap DMs) + 10+ kind:30078 + (wallet/nametag bindings) — full settlement traffic on local infra. + - Escrow's instrumentation: every swap shows `deliver_deposit_invoice_enter + → _sending → _sent` for BOTH parties (the asymmetric bug from + rounds 11-12 IS GONE in escrow:local from current source). + - Trader log: `diag_invoice_delivery_received` → `_imported` → + `swap_deposit_target_diag` populated → `swap_deposit_sent`. The + deposit IS sent. The trader DOES process the invoice. + - Final failure: `[Accounting] Direction mismatch: transport memo says + return_cancelled, on-chain says forward for invoice — using + on-chain` → `swap_cancelled`. + +**The local-infra goal is met.** What remains is a swap-protocol +settlement-layer issue (transport memo vs on-chain direction +mismatch) that's independent of the relay infra. This is the next +real bug to chase, and it now reproduces deterministically against +a controlled local relay — debug iterations no longer wait on +testnet propagation or burn through testnet rate limits. + +**Update 2026-05-08 (rounds 16-17, local-infra harness)**: +ported uxf's local-infra Nostr relay setup to trader-service to +escape the testnet write-path outages. Added +`UNICITY_NOSTR_RELAYS` env override across every component +(trader-service, escrow-service, agentic-hosting host-manager, +js-faucet, sphere-cli host/legacy inits) plus `helpers/sphere-cli.ts +buildEnv()` forwards the env into sphere-cli subprocesses. +Global-setup boots the relay when `TRADER_E2E_LOCAL_RELAY=1`. + +What works: + - Local relay container boots, tests skip preflight, env propagates + to host-manager + spawned tenants + - Wallet events (kind:30078) and DM gift-wraps (kind:1059) ARE + published to the local relay (verified by tailing relay logs) + - HMA dist needed a rebuild (was 4 days stale on disk) + +What does NOT yet work — SDK-level gap: + - Nametag binding events (kind:31113/31115/31116) bypass the + `transport.relays` override. They route through + `MultiAddressTransportMux` (sphere-sdk/transport/MultiAddressTransportMux.ts:9) + which has its OWN relay list independent of the per-provider + transport config. Because of this, the manager registers a + nametag against the (default/testnet) mux-relay but sphere-cli's + `queryPubkeyByNametag` (also via the mux) doesn't find it on + the local relay → "Unicity ID not found: @m-…". + - `nostr-js-sdk`'s `publishNametagBinding` calls + `queryPubkeyByNametag` first to detect conflicts; both + operations target the mux's hard-coded relay list, not the + SDK consumer's override. + +Fix path: + 1. **SDK change** — extend `MultiAddressTransportMux` to accept a + relay override so it picks up `transport.relays` (or a sibling + `transport.muxRelays`) from the createNodeProviders config. + Default behavior unchanged. + 2. **Tactical workaround** — for tests that use the local relay, + identify peers by raw `DIRECT://hex` (which the SDK resolves + transport-side, not via the nametag mux) and avoid `@nametag`. + The hma-trade-settlement test already passes + `escrow.tenantDirectAddress` for the swap routing; the only + remaining `@nametag` usage is sphere-cli's manager-address + resolution. The test could read `manager.directAddress` and + pass that instead of `@${manager.nametag}` — quick fix. + +Local-infra commits already pushed; the workaround in (2) is the +fastest path to a green run. + +**Remediation (in priority order)**: + +1. Rebuild `escrow:local` from `/home/vrogojin/escrow-service` source and + re-tag as the test's image. Re-run hma-trade-settlement and expect + COMPLETED. +2. After settlement works: file an upstream issue + PR against escrow-service + to harden `deliverDepositInvoice` — use `Promise.allSettled([reply(B,...), + reply(A,...)])` and log the rejection reasons explicitly so silent failures + are impossible. +3. Sphere-sdk secondary defect (independent): `SwapModule.handleIncomingDM` + walks `accepted → announced` via `status_result.state` (SwapModule.ts:3119–3171) + even when `swap.depositInvoiceId` is unset. This is what made the trader's + diag log say "registered, polling for invoice" while accounting actually + has no invoice record. Constrain the walk to require both + `swap.depositInvoiceId !== undefined` AND `accounting.getInvoice(id) !== null` + before transitioning. Medium priority — only relevant once the escrow + regression is fixed. + +The hypotheses below (H2/H3) are now **superseded** by the escrow-side root +cause. Kept for historical context. + +### ~~H1 — ACP listener consumes the invoice DM before the swap module sees it~~ — RULED OUT + +**Update**: investigated sphere-sdk's event architecture. Incoming DMs are +dispatched via TWO INDEPENDENT paths in `dist/index.js`: + + - line 13910: `deps.emitEvent("message:dm", message)` — generic event bus + (this is what `sphere.on('message:dm', ...)` subscribes to). + - line 13911-13918: iterates `dmHandlers` set + (this is what `sphere.communications.onDirectMessage(handler)` registers + into; used internally by PaymentsModule (line 18816) and SwapModule + (line 24212)). + +Both fire unconditionally for every DM. No propagation control, no ordering +between the two paths. The ACP listener (on the event bus) and the SDK's +SwapModule (on `dmHandlers`) are on independent channels — they each get +their own copy. The ACP listener CANNOT preempt the SwapModule. + +So this hypothesis is incorrect. The remaining candidates are below. + +**Note on architecture**: the dual-path dispatch with no propagation control +is itself a design smell — there's no way for a handler to say "I consumed +this, don't deliver it to other consumers." A koa-compose-style middleware +chain (`use(handler, priority)` with `next()` semantics) would be cleaner +and would let the trader's ACP filter declaratively consume non-ACP DMs. +Tracked separately; not on the critical path for THIS bug. + +### H2 — Periodic `payments.receive({finalize:true})` races with swap module's DM consumption + +The trader's main loop (`src/trader/main.ts:545`) calls +`sphere.payments.receive({ finalize: true })` every 5s. We've seen +`ENOENT: wallet.json.tmp` errors in trader logs from this and the heartbeat +loop racing on atomic temp+rename writes. + +If the `wallet.json` write race corrupts the swap module's DM-state cache, +incoming swap DMs may be silently dropped. + +**Bisect**: lengthen `SYNC_INTERVAL_MS` (currently 5s) to 60s and see if +settlement starts working. If yes, race is the cause. + +### H3 — HMA-spawned container's relay subscription latency + +HMA's docker create injects more env vars and starts the container with +`tini` as PID 1. The relay subscription inside the trader may not be fully +established by the time the escrow sends the invoice. NIP-17 events that +arrive before the subscription is active are NOT replayable for that +subscriber session. + +`sphere.fetchPendingEvents()` is supposed to catch missed events, but its +periodic call (also 5s in the sync loop) may be racing or filtering. + +**Bisect**: add an explicit `await sphere.fetchPendingEvents()` after the +trader's first STATUS query and before the swap-executor begins polling for +invoice_target. If the invoice arrives after this explicit fetch, latency +is the cause. + +## Proposed debugging plan for the follow-up session + +H1 is ruled out (see updated hypothesis above), so start with the SDK's +internal DM dispatch. + +1. **Instrument sphere-sdk's `CommunicationsModule.handleIncoming` (or + equivalent) to log EVERY incoming DM at the raw level**, BEFORE any + filtering / dedup. The log line should include sender prefix, payload + size, and the message id. With this in place, re-run the test: + + - If the escrow's invoice_delivery DM appears in the trader's raw log: + → the SDK is receiving it. Bug is downstream (handleIncomingDM rejecting, + SwapModule not registering it as the active swap, etc.). Continue to + step 2. + - If it does NOT appear: bug is at the transport layer (relay subscription + latency, DM-decryption failure, recipient mismatch). Skip to step 3. + +2. **For the "received but not processed" case (most likely)**: instrument + `SwapModule.handleIncomingDM` in `sphere-sdk/dist/index.js:24212` (or + wherever its body is) to log every entry and the path it takes. Possible + silent rejections: + - signature verification fails (wrong chain pubkey) + - swap-id-not-found (the trader doesn't have the swap registered when + the invoice arrives — race between announce-ack and invoice_delivery) + - protocol version mismatch (trader v1 vs escrow v2 or vice versa) + - dedup hit (`dm.isRead === true` because the SDK persisted it from a + prior backfill — relevant if the escrow re-sends after a wallet reload) + +3. **For the transport-layer case**: capture network-level evidence — + instrument the trader's NostrTransportProvider to log every Nostr event + it receives at the wire level. If the kind:1059 wraps for the escrow's + pubkey arrive but never decrypt to the trader, decryption is failing. + If they don't arrive at all, the relay subscription has a hole. + +4. **Architectural follow-up (separate effort)**: introduce a propagation- + aware middleware chain in sphere-sdk's CommunicationsModule (koa-compose + style — `use(mw, priority)` with `next()`), so that future consumers + (ACP listener, app code) can declaratively filter / consume / pass DMs + in an ordered pipeline. The current dual-path dispatch + (`emitEvent` + `dmHandlers` running in parallel with no coordination) + isn't the cause of THIS bug but is an obvious source of future bugs as + more consumers attach. + +## Key files + +- Test: `test/e2e-live/hma-trade-settlement.e2e-live.test.ts` +- Test helpers: + - `test/e2e-live/helpers/faucet-client.ts` (in-process Sphere wallet for `FAUCET_REQUEST`) + - `test/e2e-live/helpers/manager-process.ts` + - `test/e2e-live/helpers/hma-spawn.ts` + - `test/e2e-live/helpers/sphere-trader.ts` +- Trader code likely involved: + - `src/trader/main.ts:545` — `payments.receive({ finalize: true })` periodic + - `src/trader/swap-executor.ts:714` — `negotiatedEscrow === escrowDirectAddress` check + - `src/trader/intent-engine.ts:836` — `escrow_address ?? DEFAULT_ESCROW` + - `src/acp-adapter/main.ts` (Phase 4h decoupling) — ACP DM listener +- Sphere SDK: `@unicitylabs/sphere-sdk` payments + swap modules + +## Reproducing the failure + +```bash +# Build all required images +cd /home/vrogojin && docker build -f trader-service/Dockerfile \ + -t ghcr.io/vrogojin/agentic-hosting/trader:local . +cd /home/vrogojin && docker build -f js-faucet/Dockerfile \ + -t ghcr.io/unicitynetwork/agentic-hosting/faucet:local . +cd /home/vrogojin/agentic_hosting && npm run build +cd /home/vrogojin/sphere-cli-work/sphere-cli && npm run build + +# Run the test +cd /home/vrogojin/trader-service +git checkout feat/hma-trade-settlement-live +npm run test:e2e-live -- test/e2e-live/hma-trade-settlement.e2e-live.test.ts +# Expect: FAIL ~590s, both pairs reach ACCEPTED, neither reaches COMPLETED. + +# Inspect trader log: +docker ps -a --filter "name=alice-p" --filter "status=exited" --format "{{.Names}}" | head -1 \ + | xargs -I {} docker logs {} 2>&1 | grep -E "swap_deposit_target_diag|swap_id_register|swap_announced" + +# Inspect escrow log: +docker ps -a --filter "name=escrow-p" --filter "status=exited" --format "{{.Names}}" | head -1 \ + | xargs -I {} docker logs {} 2>&1 | grep -iE "announce|invoice_delivery|outbound_dm" +``` + +--- + +## Round 20 (2026-05-08) — selfMint funding unblocks deposits; SDK verifyPayout is the new wall + +**Hypothesis tested:** the round-19 deposit failure +(`Ownership verification failed: Authenticator does not match source state predicate`) +is caused by faucet-funded tokens having a predicate the swap-deposit +key path can't sign. `basic-roundtrip` works because traders selfMint +(predicate matches their own key); switching this test to selfMint +should unblock the deposit step. + +**Change:** patched `provisionTriple` in +`test/e2e-live/hma-trade-settlement.e2e-live.test.ts` to fund alice/bob +via `TRADER_TEST_FUND` injected through HMA's `--env` passthrough, +identical to the basic-roundtrip mechanism. Faucet spawn left in place +(unused) for minimal-diff diagnostic. Run on local-infra relay with +`TRADER_E2E_LOCAL_RELAY=1`. + +**Result — escrow side (full happy path):** + +``` +escrow-p1 log: + announce DM received from sender A (alice) + announce DM received from sender B (bob) + Swap announced, deposit invoice created + deliver_deposit_invoice_enter A → recipient 53272861... USDU 10 + deliver_deposit_invoice_sent A + deliver_deposit_invoice_enter B → recipient 786576ea... UCT 10 + deliver_deposit_invoice_sent B + First valid deposit received, timeout timer started (party A USDU) + Valid deposit received (not first) (party B UCT) + invoice:covered with unconfirmed deposits — waiting for aggregator confirmation + Deposit invoice already closed, proceeding to payouts + Timeout cancelled + Swap concluding — paying payout invoices (payoutA UCT, payoutB USDU) + Swap completed successfully ← ESCROW: settlement is DONE +``` + +So the deposit-predicate issue is **conclusively the faucet's fault**: +with selfMint, both deposits verify, escrow concludes, payouts route. +This unblocks the entire swap protocol from L4 down. + +**Result — trader side (new wall):** + +``` +alice-p1 log (Pair-1, after escrow logged "Swap completed"): + swap_payout_verify_diag attempt=8..15 + invoice_status: { state:'COVERED', isCovered:true, + coveredAmount:'10', netCoveredAmount:'10', + transfers:[{ transferId:'a20d9b0d-…', paymentDirection:'forward', + senderPubkey:'…escrow…', confirmed:false }] } + [Swap] verifyPayout for dacaf7d67760: 3 invalid token(s) but + tokenInvoiceMap is empty for this payout invoice — failing closed + until reverse index rebuilds + swap-executor: execution_timeout_skipped_terminal_swap + sdk_progress="completed", note="SDK swap is already terminal — + letting verifyPayout retries finish" + swap_payout_verify_retry_failed attempt=15 remaining=25 +``` + +The trader **received** the payout token. The SDK's swap progress is +`completed`. But `getTokenIdsForInvoice(payoutInvoiceId)` returns an +empty Set, so the SECURITY-fail-closed branch in +`sphere-sdk/modules/swap/SwapModule.ts:1997-2003` returns false on +every retry. The retry budget exhausts at attempt 40 (~20 min); the +test's 8-min timeout expires first → both Pair-1 + Pair-2 fail with +`did not reach state="COMPLETED" within 480000ms. last seen 1 deal(s) +in states [ACCEPTED]`. + +**Where the index should populate:** SDK's +`AccountingModule.ts:5755-5773` adds entries to `tokenInvoiceMap` when +an inbound transfer matches an invoice target (instant-mode v5split +path): + +```ts +for (const tok of transfer.tokens) { + if (!tok.id) continue; // ← short-circuit if token.id absent + if (!this.tokenInvoiceMap.has(tok.id)) { + this.tokenInvoiceMap.set(tok.id, new Set()); + } + this.tokenInvoiceMap.get(tok.id)!.add(invoiceId); +} +``` + +So either: + +1. The payout transfer arrives **without** `tok.id` populated on the + `transfer.tokens` entries (in which case the loop skips silently + and the index stays empty), OR +2. The transfer arrives **before** the payout invoice is registered + on the trader's accounting module (so `terms.targets` doesn't match + yet — but `invoice_imported:true` in the diag rules this out), OR +3. There's a race where `_processTokenTransactions`'s on-chain path + also failed to populate the map (instant-mode tokens have no genesis + so the on-chain path is a no-op — the synthetic-ledger path at line + 5755 is the only chance). The W23-R3 fix added that synthetic path + precisely for this case; it's apparently not firing here. + +**Reproducer (deterministic on local-infra):** + +```bash +docker rm -f $(docker ps -aq --filter "name=agentic-") 2>/dev/null || true +cd /home/vrogojin/trader-service +TRADER_E2E_LOCAL_RELAY=1 npx vitest run --config vitest.e2e-live.config.ts \ + test/e2e-live/hma-trade-settlement.e2e-live.test.ts 2>&1 | tee /tmp/r20.log +# Wait ~7 min. Both scenarios fail at COMPLETED gate. +ALICE=$(docker ps -a --filter "name=agentic-alice-p1" --format "{{.Names}}" | head -1) +docker logs "$ALICE" 2>&1 | grep -E "swap_payout_verify|tokenInvoiceMap" +# Expect: many "tokenInvoiceMap is empty for this payout invoice" warnings. +``` + +**Next investigative steps (in order of leverage):** + +1. **Inspect transfer.tokens in the actual payout** — run with + `LOG_LEVEL=debug` and add a one-line dump of `transfer.tokens` at + the top of the synthetic-ledger branch in `AccountingModule.ts:5715`. + Confirm whether `tok.id` is populated when the swap payout lands. + This is a 1-line probe with ~0 risk. +2. **Verify the synthetic-ledger branch is even being entered** — it's + guarded by `matchesTarget && matchesAsset` (line 5715). If the + trader's wallet address doesn't match `terms.targets[].address` + for the payout, neither the ledger nor the reverse map gets + populated. The diag log already shows `coveredAmount:"10"` against + the alice address `DIRECT://0000b753a86a721a72…`, so the match + IS happening — but maybe in `computeInvoiceStatus` only, not in + the dispatcher that runs synthetic-ledger update. +3. **Check the on-chain path** — `_processTokenTransactions` at + line 4628 also populates `tokenInvoiceMap`. Instant-mode payouts + have no TXF transaction so that path is a no-op; but if the swap + payouts go via TXF (not instant), this branch is what should run. + Decide which mode the escrow is using by inspecting payout messages. + +**Status:** the swap protocol works end-to-end on the wire (deposits +verify, escrow concludes, payouts deliver). The remaining gap is a +reverse-index population bug in the SDK. This is a different layer +from the round-1..19 issues (HMA / relay / faucet predicate); it's +inside `sphere-sdk` itself. + +--- + +## Round 21 (2026-05-08) — swap deps facade missing `getTokenIdsForInvoice` → `verifyPayout` permanently fail-closed + +**Context:** after the round-20 selfMint switch, escrow logged +`Swap completed successfully` and the trader received the payout. But +`swap_payout_verify_diag` retried 40 times with +`tokenInvoiceMap is empty for this payout invoice — failing closed until +reverse index rebuilds`. The 8-min test timeout expired; both pairs +failed at `did not reach state="COMPLETED"`. + +**Investigation:** added 4 `logger.warn` probes inside +`sphere-sdk/modules/accounting/AccountingModule.ts` to trace which path +was populating `tokenInvoiceMap`: + +- `_handleTransferConfirmed` (on-chain confirmed event handler) +- `_processInvoiceTransferEvent` synthetic-ledger branch +- `_processInvoiceHistoryEvent` (history-update path) +- per-token populate inside the synthetic-ledger loop + +Three runs showed only ONE `transfer:confirmed` line per trader (the +deposit-send confirmation, sender side). NONE of the receive-side +populate paths fired for the swap payout. Yet `swap_payout_verify_diag` +showed `coveredAmount: 10` from `getInvoiceStatus`. So the invoice +ledger HAD an entry — but `getTokenIdsForInvoice(payoutInvoiceId)` +returned an empty Set. + +Added a 5th probe directly inside `verifyPayout`'s fail-closed branch +to dump `tokenInvoiceMap` state. Result: + +``` +R20-DIAG verifyPayout-fail-closed swap=… (no tokenInvoiceMap accessor) +``` + +The optional-chain `acct.tokenInvoiceMap?.` fell through. The accessor +did not exist on whatever object `acct` was. + +**Root cause:** `Sphere.ts` constructs the `accounting` dep facade for +`SwapModule` at lines 2537-2543 and 4361-4367 as a hand-written narrow +object exposing only 5 methods (`importInvoice`, `getInvoice`, +`getInvoiceStatus`, `payInvoice`, `on`). `getTokenIdsForInvoice` was +NOT on the facade. The `verifyPayout` call site used a defensive +optional-chain type cast which silently returned `undefined` → empty +`Set` → fail-closed forever. There was no way for the index to "rebuild" +because the SDK was checking a method that didn't exist on the object +the swap module had been handed at construction. + +**Fix:** added `getTokenIdsForInvoice` to the +`SwapModuleDependencies.accounting` interface and wired it through both +facade construction sites in `Sphere.ts`. Defensive `tokenInvoiceMap` +migration also added in `importInvoice` (defense-in-depth for the +orphan-buffer race where token-receive lands before invoice-import). + +**Outcome:** `hma-trade-settlement.e2e-live`'s settlement phase now +completes end-to-end. Both pairs reach `deal COMPLETED` in ~140s, down +from the 600s+ timeout. Pair-2 settled successfully in this round; +Pair-1 failed only on a transient testnet Market API HTTP 502 +(unrelated to the SDK fix). + +--- + +## Round 22 (2026-05-09) — `finalizeReceivedToken` error paths flipped `status='confirmed'` on un-finalized tokens → withdraw flake + +**Context:** with round-21's SDK fix in place, settlement reaches +COMPLETED reliably. But the next step — withdraw of 3 UCT from alice → +controller — FLAKED. Sometimes the withdraw succeeded with +`transfer_id=…`; sometimes it failed with the SAME error pattern as the +round-19 faucet-funded predicate issue: + +``` +Ownership verification failed: Authenticator does not match source +state predicate. +``` + +**Earlier mis-diagnosis:** at first I thought the bug must be +`payments.send`'s direct-spend path vs the invoicing path. The user +correctly pushed back: *"We apparently using the SDK wrong. SDK itself +supposedly have no issue."* Switched the trader's `WITHDRAW_TOKEN` +handler to use `accounting.payInvoice` (create local invoice with +`target=to_address`, then pay it) — same code path swap deposits use. +Test still flaked with the same error. Confirmed by code trace that +`accounting.payInvoice` ultimately calls `payments.send`, so the bug +couldn't be in the entry-point choice. + +**Investigation:** added a probe in `payments.send`'s commitment-submit +path to log +`commitment.transactionData.sourceState.predicate.publicKey` vs +`commitment.authenticator.publicKey`. The probe didn't fire on most +runs (different code path), but careful inspection of the receive flow +led to the right place. + +**Root cause:** `sphere-sdk/modules/payments/PaymentsModule.ts:5362-5388, +5423-5431` — `finalizeReceivedToken`. Three error paths set +`token.status = 'confirmed'` WITHOUT updating `sdkData`: + +1. Missing `waitForProofSdk` (line 5362-5367) +2. Missing `stClient` / `trustBase` (line 5382-5388) +3. Caught exception during `finalizeTransferToken` (line 5424-5430, + with comment *"Mark as confirmed anyway (user has the token)"*) + +The original intent was "user has the token, mark confirmed for UI." +The flaw: `sdkData` was never updated, so the token kept the SENDER's +source-state predicate. The spend queue's filter +(`SpendQueue.ts:91 status !== 'confirmed' continue`) let these +mislabeled tokens through. When picked, the resulting commitment built +`sourceState.predicate` from the SENDER's stored state and +`authenticator.publicKey` from the RECEIVER's signing service. +`predicate.isOwner(...)` is a hex compare → false → state-transition-sdk +threw the predicate-mismatch error. + +The flake explanation: alice's wallet had selfMint UCT 5000 (truly +finalized, predicate=alice's key) AND swap-payout UCT 10 (status flipped +to 'confirmed' by the buggy error paths even though finalization didn't +fully complete). The spend queue's pick varied: selfMint → success; +swap-payout → fail. Same wallet, same withdraw call, two distinct +outcomes depending on which token the queue iterator yielded first. + +**Fix:** all three error paths now leave `status='submitted'`. The +next periodic `resolveUnconfirmed()` / `receive({finalize:true})` will +retry the finalize properly. Until then, the spend queue correctly +skips the un-finalized token and picks a truly-finalized one. + +**Outcome:** `hma-trade-settlement.e2e-live` Pair-1 reaches: + +``` +deal COMPLETED +withdraw transfer_id=22bc307f-d865-41c5-… +✓ end-to-end settlement+withdraw verified +``` + +in 128s. **The user's stated goal — operators launch HMA → spawn → fund +→ trade → settle → withdraw, all over Sphere DMs — is reached.** + +Pair-2 still fails on a separate concurrent-settlement race (deals go +ACCEPTED → CANCELLED). That race is being investigated as a follow-up +and is independent of the three SDK fixes landed in this session. + +--- + +## Final SDK changes summary + +The three SDK fixes are split into three focused PRs against +`sphere-sdk`: + +| Branch | Fix | +|---|---| +| `fix/swap-getTokenIdsForInvoice` | Wire `getTokenIdsForInvoice` through the `SwapModuleDependencies.accounting` facade in `Sphere.ts` (2 construction sites). Stops `verifyPayout` from permanently fail-closing on an empty Set returned by an absent accessor. | +| `fix/accounting-importInvoice-token-map-migration` | Defense-in-depth: when `importInvoice` runs after the inbound transfer already landed in the orphan buffer, migrate any orphaned token entries into `tokenInvoiceMap` for the freshly-imported invoice. | +| `fix/payments-finalize-error-status` | `finalizeReceivedToken` no longer sets `status='confirmed'` on the three error paths (missing `waitForProofSdk`, missing `stClient`/`trustBase`, caught finalize exception). Leaves `status='submitted'` so the spend queue skips and the next `resolveUnconfirmed()` retries. | + +The trader + CLI work that made the settlement+withdraw test usable +ships separately: + +| Branch | Repo | Contents | +|---|---|---| +| `feat/hma-trade-settlement-live` | trader-service | `hma-trade-settlement.e2e-live.test.ts` + invoicing-based `WITHDRAW_TOKEN` handler in trader. | +| `feat/trader-withdraw-cli` | sphere-cli | `sphere trader withdraw` subcommand exposing the WITHDRAW_TOKEN ACP message over DM. | + +--- + +## Out of scope for this debugging session + +- Production hardening of js-faucet (rate limiting, batched mints, etc.) +- Pushing js-faucet image to ghcr.io/unicitynetwork (needs PAT) +- Adding `faucet-agent` template entry to agentic-hosting/config/templates.json +- `sphere faucet request` subcommand in sphere-cli +- Concurrent-settlement race on Pair-2 (ACCEPTED → CANCELLED) — follow-up + +--- + +## Round 23 — RESOLUTION (2026-05-10) + +### TL;DR + +**Both Pair-1 and Pair-2 ✓ PASS end-to-end** in the latest live e2e +run after baking `transferMode: 'conservative'` into the trader's +withdraw path. The intermittent **"Authenticator does not match +source state predicate"** error is gone. + +``` +✓ test/e2e-live/hma-trade-settlement.e2e-live.test.ts (2 tests) 169s + ✓ Pair-1: full spawn → trade → settle → withdraw via HMA (rate=1) 153456ms + ✓ Pair-2: parallel scenario settles on the same HMA at distinct rate (rate=3) 154783ms + +[p1-d54068] withdraw transfer_id=5a151db3-ce88-41… ✓ end-to-end settlement+withdraw verified +[p2-a64a0b] withdraw transfer_id=67eb1c73-2645-46… ✓ end-to-end settlement+withdraw verified +``` + +### Root cause (final) + +The trader's invoiced withdraw was the **only** forwarding flow in the +chain still using the default `transferMode: 'instant'`. Faucet and +escrow already used `'conservative'` on their direct `payments.send` +paths. + +Under `'instant'` mode, `PaymentsModule.send` ships a V6 +combined-transfer bundle whose recipient saves the token at +`status='submitted'` with the **sender's** `sdkData` and finalizes via +background proof-poll. If the trader's spend queue (or, for withdraw, +the controller's spend queue) picks a not-yet-finalized incoming token +— such as a swap-payout that arrived seconds earlier — it produces: + +> Authenticator does not match source state predicate + +because the recipient-side `Token` isn't yet bound to the recipient's +predicate. + +`'conservative'` mode collects the inclusion proof on the **sender's** +side before delivery, so the recipient receives a fully-finalized +`{sourceToken, transferTx}` bundle and produces a `'confirmed'` Token +immediately bound to its own predicate. Chained spends are then +race-free. + +### Fix + +A single new field on `PayInvoiceParams` plus three call-site +opt-ins: + +| Repo | Branch / Commit | Change | +|---|---|---| +| sphere-sdk | `feat/accounting-payinvoice-transfermode` (#131) | Add `transferMode?: 'instant' \| 'conservative'` to `PayInvoiceParams`; forward to `PaymentsModule.send`. Default unchanged. | +| trader-service | `feat/hma-trade-settlement-live` (`de9f0b7`) | Withdraw path uses `transferMode: 'conservative'` through `accounting.payInvoice`. | +| escrow-service | `fix/conservative-payout-mode` (#18) | Swap-payout `payments.send` uses `'conservative'`. | +| js-faucet | `fix/faucet-funded-predicate` (#2) | `FAUCET_REQUEST` sends use `'conservative'`. | + +### Why earlier rounds appeared to fix it sometimes + +Previous Pair-2 success was misleading — when the trader image was +rebuilt during Round 22, Pair-2 happened to win the +finalization-vs-spend race purely on timing. The conservative-mode +opt-in eliminates the race architecturally rather than narrowing the +window. + +### Side fixes that landed alongside + +- `feat/market-tolerance` (#132): retry + circuit breaker for transient + Market API 502/503/504/408 errors, so a single load-balancer hiccup + no longer kills an in-progress e2e run. +- `feat/hma-trade-settlement-live` (`38be0b5`): bounded retry in + `swap:proposal_received` to handle the two-DM arrival-order race + that caused Pair-2 deals to flake ACCEPTED→CANCELLED. + +### Verification command + +```bash +cd /home/vrogojin/trader-service +TRADER_E2E_LOCAL_RELAY=1 \ + npx vitest run --config vitest.e2e-live.config.ts \ + test/e2e-live/hma-trade-settlement.e2e-live.test.ts +``` + diff --git a/src/trader/main.ts b/src/trader/main.ts index f21d4cf..3f95ce4 100644 --- a/src/trader/main.ts +++ b/src/trader/main.ts @@ -247,6 +247,21 @@ export async function startTrader(): Promise { const trustbasePath = join(config.data_dir, 'trustbase.json'); writeFileSync(trustbasePath, await tbResponse.text()); + // Optional Nostr-relay override. Set `UNICITY_NOSTR_RELAYS` (or + // `SPHERE_NOSTR_RELAYS` as a fallback) to a comma-separated list of + // WebSocket URLs to replace the network preset's relays — used by the + // local-infra e2e harness to point at a Docker-hosted relay when the + // public testnet relay's write path is degraded. Empty/unset → default. + const relayOverride = (() => { + const raw = process.env['UNICITY_NOSTR_RELAYS'] ?? process.env['SPHERE_NOSTR_RELAYS']; + if (!raw) return undefined; + const relays = raw.split(',').map((s) => s.trim()).filter((s) => s.length > 0); + return relays.length > 0 ? relays : undefined; + })(); + if (relayOverride) { + logger.info('nostr_relays_override_active', { relays: relayOverride }); + } + // Initialize Sphere wallet with market, swap, and accounting modules logger.info('initializing_sphere', { network: config.network, data_dir: config.data_dir }); const providers = createNodeProviders({ @@ -257,6 +272,7 @@ export async function startTrader(): Promise { trustBasePath: trustbasePath, apiKey: resolveApiKey(), }, + ...(relayOverride ? { transport: { relays: relayOverride } } : {}), }); // 2026-04-30 FIX (basic-roundtrip flake investigation): expand the @@ -1020,6 +1036,43 @@ export async function startTrader(): Promise { market, swap, comms: { sendDm: sender.sendDm.bind(sender) }, + // Invoice-based withdraw path. When the SDK's accounting module is + // available, expose a narrow facade so the trader's WITHDRAW_TOKEN + // handler can use createInvoice + payInvoice instead of payments.send + // directly. Mirrors the swap-deposit flow and avoids the "Authenticator + // does not match source state predicate" flake on spends of received + // swap-payout tokens. + ...(sphere.accounting + ? { + accounting: { + createInvoice: async (req: import('./types.js').AccountingCreateInvoiceRequest) => { + const result = await sphere.accounting!.createInvoice({ + targets: req.targets.map((t) => ({ + address: t.address, + assets: t.assets.map((a) => ({ coin: a.coin as [string, string] })), + })), + ...(req.memo !== undefined ? { memo: req.memo } : {}), + }); + return { + success: result.success, + ...(result.invoiceId !== undefined ? { invoiceId: result.invoiceId } : {}), + ...(result.error !== undefined ? { error: result.error } : {}), + }; + }, + payInvoice: async ( + invoiceId: string, + params: import('./types.js').AccountingPayInvoiceParams, + ) => { + const result = await sphere.accounting!.payInvoice(invoiceId, params); + return { + id: result.id, + status: String(result.status), + ...(result.error !== undefined ? { error: result.error } : {}), + }; + }, + }, + } + : {}), // subscribeEvent is retained for interface compatibility but swap events // are ALL handled by direct sphere.on() listeners below (lines 418+). // This bridge is only needed for non-swap events in the future. @@ -1239,7 +1292,23 @@ export async function startTrader(): Promise { return; } - registered = agent.registerSwapId(data.swapId, { + // R23 fix (Pair-2 race): the swap_proposal DM and np.propose_deal + // DM can arrive in either order. When swap_proposal arrives FIRST, + // the np.propose_deal handler is still running (validation → + // transitionDeal('ACCEPTED') → onDealAccepted → executeDeal → + // registerActive); `activeByDealId` is empty so the very first + // registerSwapId call returns false and we'd reject the swap + // even though we're about to accept the deal. + // + // Fix: bounded retry — registerSwapId still cross-checks + // counterparty pubkey, currencies, amounts, escrow address, and + // timeout against negotiated DealTerms, so retrying is safe; we + // only paper over the microsecond-scale ordering hazard. If the + // deal really wasn't accepted (hostile peer, stale state), we + // still reject after the bounded wait. + const REGISTER_MAX_ATTEMPTS = 40; + const REGISTER_BACKOFF_MS = 50; + const registerArgs = { partyACurrency: s.deal?.partyACurrency, partyAAmount: s.deal?.partyAAmount, partyBCurrency: s.deal?.partyBCurrency, @@ -1254,13 +1323,32 @@ export async function startTrader(): Promise { escrowPubkey: (s as unknown as { escrowPubkey?: string }).escrowPubkey, depositTimeoutSec: (s.deal as unknown as { timeout?: number; depositTimeoutSec?: number })?.depositTimeoutSec ?? (s.deal as unknown as { timeout?: number })?.timeout, - }); + }; + for (let attempt = 0; attempt < REGISTER_MAX_ATTEMPTS; attempt++) { + registered = agent.registerSwapId(data.swapId, registerArgs); + if (registered) break; + if (attempt + 1 < REGISTER_MAX_ATTEMPTS) { + await new Promise((r) => setTimeout(r, REGISTER_BACKOFF_MS)); + } + } } catch (err: unknown) { logger.warn('swap_proposal_status_fetch_failed', { swap_id: data.swapId, error: err instanceof Error ? err.message : String(err), }); - registered = agent.registerSwapId(data.swapId); + // R23 fix (legacy fallback): same bounded retry as the + // status-based path above — without it, a transient + // getSwapStatus failure compounds the np.propose_deal / + // swap_proposal arrival-order race. + const REGISTER_MAX_ATTEMPTS = 40; + const REGISTER_BACKOFF_MS = 50; + for (let attempt = 0; attempt < REGISTER_MAX_ATTEMPTS; attempt++) { + registered = agent.registerSwapId(data.swapId); + if (registered) break; + if (attempt + 1 < REGISTER_MAX_ATTEMPTS) { + await new Promise((r) => setTimeout(r, REGISTER_BACKOFF_MS)); + } + } } if (!registered) { diff --git a/src/trader/trader-main.ts b/src/trader/trader-main.ts index dc776f2..cf9f7fb 100644 --- a/src/trader/trader-main.ts +++ b/src/trader/trader-main.ts @@ -23,6 +23,7 @@ import { createCommandHandler } from '../tenant/command-handler.js'; import type { CommandHandler } from '../tenant/command-handler.js'; import type { + AccountingAdapter, PaymentsAdapter, MarketAdapter, MarketSearchResult, @@ -60,6 +61,14 @@ export interface TraderMainDeps { readonly market: MarketAdapter; readonly swap: SwapAdapter; readonly comms: { sendDm: (to: string, content: string) => Promise }; + /** + * Optional invoice-based withdraw path. When present, WITHDRAW_TOKEN + * routes through accounting.createInvoice + payInvoice instead of + * payments.send directly — the same code path swap deposits use, so + * predicate-handling is well-tested. Optional so unit tests with + * stub adapters don't need to wire this layer. + */ + readonly accounting?: AccountingAdapter; // Sphere instance controls readonly subscribeEvent: (eventType: string, handler: (...args: unknown[]) => void) => () => void; @@ -145,6 +154,7 @@ export function createTraderAgent(deps: TraderMainDeps): TraderAgent { market, swap, comms, + accounting, // subscribeEvent is available but unused — all swap events are handled // by direct sphere.on() listeners in main.ts, not via this bridge. signMessage, @@ -196,6 +206,85 @@ export function createTraderAgent(deps: TraderMainDeps): TraderAgent { async function withdraw( params: WithdrawTokenParams, ): Promise<{ transfer_id: string; remaining_balance: bigint }> { + // Invoice-based path (preferred). The trader creates a local invoice + // with a single target = `params.to_address`, then pays it via + // `accounting.payInvoice`. This is the same code path swap deposits + // use, so it inherits the SDK's well-tested handling for invoice-target + // predicates. The recipient sees the inbound transfer with an invoice + // memo and (if their wallet has accounting enabled) auto-imports the + // invoice — matching how swap-payouts are received. + // + // The direct `payments.send` path used to be the implementation but + // can flake with "Authenticator does not match source state predicate" + // when the spend queue picks a token whose source-state predicate + // doesn't match the wallet's main key (typical of swap-payout tokens + // received with a per-transfer salted predicate). + if (accounting !== undefined) { + // The accounting module's createInvoice validates `coin[0]` as a + // SYMBOL (≤20 chars, alphanumeric), not a 64-hex coinId. We resolve + // the symbol to a coinIdHex separately for the post-pay balance + // computation, but pass the symbol to createInvoice itself. + const balances = payments.getAllBalances(); + const matched = balances.find((b) => b.symbol === params.asset || b.coinId === params.asset); + if (matched === undefined) { + throw new Error(`withdraw: unknown asset "${params.asset}" — no token in wallet matches`); + } + const symbol = matched.symbol ?? params.asset; + const coinIdHexForBalance = matched.coinId; + const invoice = await accounting.createInvoice({ + targets: [{ + address: params.to_address, + assets: [{ coin: [symbol, params.amount] }], + }], + memo: `withdraw ${params.amount} ${symbol} → ${params.to_address}`, + }); + if (!invoice.success || invoice.invoiceId === undefined) { + throw new Error(`accounting.createInvoice failed: ${invoice.error ?? 'unknown'}`); + } + const payResult = await accounting.payInvoice(invoice.invoiceId, { + targetIndex: 0, + amount: params.amount, + // Conservative mode: the SDK collects the inclusion proof on the + // SENDER's side before delivery, so the recipient receives a + // fully-finalized {sourceToken, transferTx} bundle and can produce + // a 'confirmed' Token immediately bound to its own predicate. This + // mirrors the faucet and escrow's payout flows. The default + // 'instant' mode ships an unconfirmed bundle whose recipient-side + // proof-poll races with any chained spend (e.g. another withdraw or + // a swap deposit) and intermittently surfaces "Authenticator does + // not match source state predicate" errors when the spend queue + // picks a not-yet-finalized token. + transferMode: 'conservative', + }); + if (payResult.error !== undefined && payResult.error !== '') { + logger.warn('withdraw_pay_invoice_returned_error', { + asset: params.asset, + amount: params.amount, + to_address: params.to_address, + invoice_id: invoice.invoiceId, + transfer_id: payResult.id, + status: payResult.status, + error: payResult.error, + }); + throw new Error(`accounting.payInvoice failed: ${payResult.error}`); + } + logger.info('withdraw_sent_via_invoice', { + asset: params.asset, + amount: params.amount, + to_address: params.to_address, + invoice_id: invoice.invoiceId, + transfer_id: payResult.id, + status: payResult.status, + }); + const remaining = payments.getConfirmedBalance(coinIdHexForBalance) - BigInt(params.amount); + return { + transfer_id: payResult.id, + remaining_balance: remaining < 0n ? 0n : remaining, + }; + } + + // Legacy direct-send path. Kept for unit tests with stub adapters and + // for environments where the accounting module is unavailable. const sendResult = await payments.send({ coinId: params.asset, amount: params.amount, diff --git a/src/trader/types.ts b/src/trader/types.ts index 9473fac..71496c1 100644 --- a/src/trader/types.ts +++ b/src/trader/types.ts @@ -260,6 +260,65 @@ export interface PaymentsAdapter { send(request: SendTokenRequest): Promise; } +// --------------------------------------------------------------------------- +// AccountingAdapter — narrow abstraction over Sphere SDK AccountingModule +// --------------------------------------------------------------------------- + +/** + * Subset of `accounting.createInvoice` request needed by the trader's + * invoice-based withdraw path. Mirrors the SDK's CreateInvoiceRequest + * but stays narrow so tests can inject a recording stub. + */ +export interface AccountingInvoiceTarget { + readonly address: string; + /** Each entry's `coin` is `[coinIdHex, amountStr]`. */ + readonly assets: ReadonlyArray<{ readonly coin: readonly [string, string] }>; +} + +export interface AccountingCreateInvoiceRequest { + readonly targets: readonly AccountingInvoiceTarget[]; + readonly memo?: string; +} + +export interface AccountingCreateInvoiceResult { + readonly success: boolean; + readonly invoiceId?: string; + readonly error?: string; +} + +export interface AccountingPayInvoiceParams { + readonly targetIndex: number; + readonly assetIndex?: number; + readonly amount?: string; + /** + * Transfer-delivery mode. `'conservative'` collects the inclusion + * proof on the sender's side before delivery; the recipient gets a + * fully-finalized bundle and can spend it immediately. `'instant'` + * (default) ships an unconfirmed bundle that the recipient finalizes + * via background proof-poll. Use `'conservative'` for withdraw flows + * so the recipient's spend doesn't race the proof-poll. + */ + readonly transferMode?: 'instant' | 'conservative'; +} + +export interface AccountingPayInvoiceResult { + readonly id: string; + readonly status: string; + readonly error?: string; +} + +/** + * Narrow facade over `sphere.accounting`. Used by the trader's withdraw + * flow to route value via the invoicing system (create invoice locally + * → pay it via payInvoice) instead of `payments.send` directly. The + * invoicing path is the same one swap deposits use, so it inherits the + * SDK's well-tested predicate-handling for invoice-target predicates. + */ +export interface AccountingAdapter { + createInvoice(request: AccountingCreateInvoiceRequest): Promise; + payInvoice(invoiceId: string, params: AccountingPayInvoiceParams): Promise; +} + // --------------------------------------------------------------------------- // MarketAdapter — narrow abstraction over Sphere SDK MarketModule // --------------------------------------------------------------------------- diff --git a/test/e2e-live/global-setup.ts b/test/e2e-live/global-setup.ts index a4aa08d..2fbab7a 100644 --- a/test/e2e-live/global-setup.ts +++ b/test/e2e-live/global-setup.ts @@ -1,14 +1,71 @@ /** * Vitest globalSetup for the e2e-live suite. * - * Runs ONCE before any test file. If the preflight throws, vitest aborts - * the entire run before spawning any Docker containers — saving the - * 10-15-minute round trip we'd otherwise eat on a relay outage or - * unreachable aggregator. + * Runs ONCE before any test file. Two responsibilities: + * + * 1. Preflight gate — abort the run before spawning Docker tenants + * if any required testnet service (Nostr relay, L3 Aggregator, + * IPFS, Fulcrum, Market) is unreachable. Saves the 10-15-minute + * round trip we'd otherwise eat on an outage. Bypass: + * `TRADER_E2E_SKIP_PREFLIGHT=1`. + * + * 2. Local infra (opt-in) — when `TRADER_E2E_LOCAL_RELAY=1` is set, + * boot a Docker-hosted Nostr relay (see local-infra/relay.ts) + * and export `UNICITY_NOSTR_RELAYS` so every component that + * reads it (host-manager, escrow, trader, faucet) connects to + * the local relay instead of the public testnet. The relay + * binds to the host on 0.0.0.0:7777; HMA-spawned tenants reach + * it via the Docker bridge gateway IP (auto-discovered). + * + * Tests that need to fan the URL into HMA-spawned tenants read + * `process.env['UNICITY_NOSTR_RELAYS']` and pass it via the + * `env` field on `hostSpawnAsync(...)`. The host-manager's own + * Sphere wallet picks it up automatically because spawnHostManager + * forwards the parent env (or the test sets it on the spawn env). + * + * Local-relay mode SKIPS the preflight (the local relay is + * under our control; gating against the public testnet relay + * would defeat the purpose). */ import { runPreflight } from './preflight.js'; +import { bootLocalRelay, getLocalRelayUrlForContainers, type RelayHandle } from './local-infra/relay.js'; + +let relayHandle: RelayHandle | null = null; export async function setup(): Promise { + if (process.env['TRADER_E2E_LOCAL_RELAY'] === '1') { + console.log('[global-setup] TRADER_E2E_LOCAL_RELAY=1 — booting local Nostr relay…'); + relayHandle = await bootLocalRelay({ + // Wipe by default so each `npm run test:e2e-live` starts from a clean + // event log. Set TRADER_E2E_LOCAL_RELAY_KEEP=1 to preserve state + // between runs (useful for post-mortem on a failing test). + wipe: process.env['TRADER_E2E_LOCAL_RELAY_KEEP'] !== '1', + timeoutMs: 60_000, + logPrefix: '[global-setup] ', + }); + const containerUrl = getLocalRelayUrlForContainers(); + process.env['UNICITY_NOSTR_RELAYS'] = containerUrl; + process.env['TRADER_E2E_LOCAL_RELAY_HOST_URL'] = relayHandle.url; + process.env['TRADER_E2E_LOCAL_RELAY_CONTAINER_URL'] = containerUrl; + console.log( + `[global-setup] local relay ready — host: ${relayHandle.url}, ` + + `containers: ${containerUrl}`, + ); + console.log('[global-setup] preflight SKIPPED (local relay supersedes testnet gate)'); + return; + } await runPreflight(); } + +export async function teardown(): Promise { + if (relayHandle) { + console.log('[global-setup] stopping local Nostr relay…'); + try { + await relayHandle.stop({ wipe: false }); + } catch (err) { + console.error('[global-setup] relay stop error:', err); + } + relayHandle = null; + } +} diff --git a/test/e2e-live/helpers/constants.ts b/test/e2e-live/helpers/constants.ts index b39f7ed..dedf422 100644 --- a/test/e2e-live/helpers/constants.ts +++ b/test/e2e-live/helpers/constants.ts @@ -56,8 +56,29 @@ export const USDU_COIN_ID = '8f0f3d7a5e7297be0ee98c63b81bcebb2740f43f616566fc290 /** Default trader image (matches templates.json shortcut). */ export const TRADER_IMAGE = 'ghcr.io/vrogojin/agentic-hosting/trader:v0.2'; -/** Default escrow image. */ -export const ESCROW_IMAGE = 'ghcr.io/vrogojin/agentic-hosting/escrow:v0.1'; +/** + * Default escrow image. + * + * Bumped 2026-05-16 from v0.1 (2026-04-25, predates UXF protocol + + * the deliverDepositInvoice asymmetric-delivery fix surfaced in + * round 19 of the HMA settlement diagnostic) to v0.2 (2026-05-16, + * published from escrow-service@d427e5d + uxf sphere-sdk@3a575cd — + * integration/all-fixes HEAD). Digest: + * + * sha256:311903b6f98b33a63791bf79db6522a66d118588ba56fcf6e56654ed6670ebac + * + * What v0.2 adds vs v0.1: + * - Conservative transferMode for swap payouts (fix/conservative- + * payout-mode HEAD) + * - UNICITY_NOSTR_RELAYS env override (matches the local-infra + * plumbing this harness already uses) + * - deliverDepositInvoice instrumentation + per-party try/catch + * (round-19 evidence confirms the asymmetric bug is GONE in + * current source vs the v0.1 deployed image) + * - sphere-sdk UXF protocol PRs (#105, #115, #119, #128, + * #146/147/149/152) + all payments/* faucet-flow regression fixes + */ +export const ESCROW_IMAGE = 'ghcr.io/vrogojin/agentic-hosting/escrow:v0.2'; /** Per-test default timeout (slow because real-network). */ export const DEFAULT_TIMEOUT_MS = 30_000; diff --git a/test/e2e-live/helpers/faucet-client.ts b/test/e2e-live/helpers/faucet-client.ts new file mode 100644 index 0000000..c4fc1e0 --- /dev/null +++ b/test/e2e-live/helpers/faucet-client.ts @@ -0,0 +1,214 @@ +/** + * In-process faucet client for e2e-live tests. + * + * Bootstraps a Sphere wallet in the test process, then sends ACP-0 + * `FAUCET_REQUEST` DMs to a spawned faucet-agent's pubkey and awaits + * the result envelope. Replaces both: + * + * - TRADER_TEST_FUND self-mint (which only works on certain + * sphere-sdk branches and produces self-issued tokens that may + * interact poorly with the swap protocol). + * - Public-faucet HTTP (`FAUCET_URL`) which has been observed to + * return 200 OK with a tx_id but never deliver the deposit. + * + * All communication is encrypted Sphere DMs; no HTTP. Mirrors the + * pattern js-faucet/test/e2e-live/faucet-roundtrip.e2e-live.test.ts + * uses to drive its own roundtrip test. + */ + +import { Sphere } from '@unicitylabs/sphere-sdk'; +import { createNodeProviders } from '@unicitylabs/sphere-sdk/impl/nodejs'; +import type { DirectMessage } from '@unicitylabs/sphere-sdk'; +import { mkdtempSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { randomUUID } from 'node:crypto'; + +const TRUSTBASE_URL = + 'https://raw.githubusercontent.com/unicitynetwork/unicity-ids/refs/heads/main/bft-trustbase.testnet.json'; + +/** + * In-process client. Holds a Sphere wallet + DM subscription that + * captures incoming acp.result/acp.error envelopes keyed by command_id. + */ +export interface FaucetClient { + readonly sphere: Sphere; + readonly pubkey: string; + readonly directAddress: string; + /** + * Send `FAUCET_REQUEST` to the faucet's pubkey and wait for the + * matching acp.result. Throws on acp.error or timeout. + */ + request(faucetPubkey: string, params: FaucetRequestParams, timeoutMs?: number): Promise; + /** Tear down the wallet + DM subscription. */ + destroy(): Promise; +} + +export interface FaucetRequestParams { + recipient: string; + asset?: string; + amount?: string; + memo?: string; + items?: ReadonlyArray<{ asset: string; amount: string; memo?: string }>; +} + +export interface FaucetDelivery { + asset: string; + coin_id: string; + amount: string; + token_id: string; + transfer_id: string; +} + +interface IncomingResponse { + type: string; + payload: Record; +} + +/** + * Bootstrap a fresh Sphere wallet in the test process and return a + * FaucetClient. The wallet's data dir is a unique tmpdir; caller MUST + * call `destroy()` to release the wallet + relay subscription. + */ +export async function createFaucetClient(): Promise { + const dataDir = mkdtempSync(join(tmpdir(), 'trader-e2e-faucet-cli-')); + const tokensDir = join(dataDir, 'tokens'); + + const tbResp = await fetch(TRUSTBASE_URL, { signal: AbortSignal.timeout(30_000) }); + if (!tbResp.ok) throw new Error(`failed to fetch trustbase: HTTP ${String(tbResp.status)}`); + const trustbasePath = join(dataDir, 'trustbase.json'); + writeFileSync(trustbasePath, await tbResp.text()); + + // Forward Nostr-relay override so the in-process FaucetClient connects + // to the same relay as the spawned tenants when the local-infra harness + // is active. Without this, the client connects to testnet defaults and + // can't reach a faucet-agent that's only on the local relay → its + // FAUCET_REQUEST DMs go out into testnet and the response never arrives. + const relayOverride = (() => { + const raw = process.env['UNICITY_NOSTR_RELAYS'] ?? process.env['SPHERE_NOSTR_RELAYS']; + if (!raw) return undefined; + const relays = raw.split(',').map((s) => s.trim()).filter((s) => s.length > 0); + return relays.length > 0 ? relays : undefined; + })(); + const providers = createNodeProviders({ + network: 'testnet', + dataDir, + tokensDir, + oracle: { trustBasePath: trustbasePath }, + ...(relayOverride ? { transport: { relays: relayOverride } } : {}), + }); + const { sphere } = await Sphere.init({ + ...providers, + autoGenerate: true, + nametag: `fc-${randomUUID().slice(0, 12).replace(/-/g, '')}`, + accounting: true, + swap: false, + market: false, + }); + + const identity = sphere.identity; + if (!identity) throw new Error('Sphere.init returned no identity'); + const pubkey = identity.chainPubkey; + const directAddress = identity.directAddress ?? `DIRECT://${pubkey}`; + + // Capture inbound result envelopes keyed by command_id. + const responses = new Map(); + const unsubscribe = sphere.on('message:dm', (msg: DirectMessage) => { + const acp = parseAcpJson(msg.content); + if (acp === null) return; + if (acp.type !== 'acp.result' && acp.type !== 'acp.error') return; + const payload = acp.payload as Record; + const cmdId = typeof payload['command_id'] === 'string' ? payload['command_id'] : null; + if (cmdId === null) return; + responses.set(cmdId, { type: acp.type, payload }); + }); + + async function request( + faucetPubkey: string, + params: FaucetRequestParams, + timeoutMs = 180_000, + ): Promise { + const cmdId = randomUUID(); + const msg = createAcpCommandEnvelope(cmdId, 'FAUCET_REQUEST', params as unknown as Record); + await sphere.communications.sendDM(`DIRECT://${faucetPubkey}`, JSON.stringify(msg)); + + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + const r = responses.get(cmdId); + if (r) { + responses.delete(cmdId); + if (r.type === 'acp.error') { + const code = String(r.payload['error_code'] ?? 'UNKNOWN'); + const message = String(r.payload['message'] ?? ''); + throw new Error(`FAUCET_REQUEST failed: [${code}] ${message}`); + } + const result = r.payload['result'] as { deliveries?: FaucetDelivery[] } | undefined; + const deliveries = result?.deliveries ?? []; + if (!Array.isArray(deliveries)) { + throw new Error(`FAUCET_REQUEST: result.deliveries not an array. Got: ${JSON.stringify(r.payload)}`); + } + return deliveries; + } + await new Promise((resolve) => setTimeout(resolve, 250)); + } + throw new Error(`FAUCET_REQUEST: no response for command_id=${cmdId} within ${String(timeoutMs)}ms`); + } + + async function destroy(): Promise { + try { unsubscribe(); } catch { /* ignore */ } + try { await sphere.destroy(); } catch { /* ignore */ } + } + + return { sphere, pubkey, directAddress, request, destroy }; +} + +// --------------------------------------------------------------------------- +// Minimal ACP envelope helpers — duplicated here to avoid pulling in the +// full agentic-hosting protocol module. Trader-service doesn't ship the +// ACP envelope code; the trader's command-handler operates on already- +// parsed payloads. The js-faucet listener parses these envelopes itself. +// --------------------------------------------------------------------------- + +const ACP_VERSION = '0.1'; + +function createAcpCommandEnvelope( + cmdId: string, + name: string, + params: Record, +): { + acp_version: string; + msg_id: string; + ts_ms: number; + instance_id: string; + instance_name: string; + type: string; + payload: { command_id: string; name: string; params: Record }; +} { + return { + acp_version: ACP_VERSION, + msg_id: randomUUID(), + ts_ms: Date.now(), + instance_id: 'controller', + instance_name: 'controller', + type: 'acp.command', + payload: { command_id: cmdId, name, params }, + }; +} + +interface AcpEnvelope { + type: string; + payload: unknown; +} + +function parseAcpJson(content: string): AcpEnvelope | null { + if (content.length > 65_536) return null; + try { + const parsed: unknown = JSON.parse(content); + if (typeof parsed !== 'object' || parsed === null) return null; + const env = parsed as Record; + if (typeof env['type'] !== 'string') return null; + return { type: env['type'], payload: env['payload'] }; + } catch { + return null; + } +} diff --git a/test/e2e-live/helpers/hma-spawn.ts b/test/e2e-live/helpers/hma-spawn.ts index 80cfe7b..809922d 100644 --- a/test/e2e-live/helpers/hma-spawn.ts +++ b/test/e2e-live/helpers/hma-spawn.ts @@ -143,6 +143,54 @@ export function hostSpawn(opts: HostSpawnOpts): SpawnedTenant { }; } +/** + * Async variant of hostSpawn — uses runSphereAsync so concurrent calls + * (Promise.all([hostSpawnAsync(escrow), hostSpawnAsync(alice), ...])) + * actually overlap. The sync version uses spawnSync which blocks the + * event loop, defeating parallelism. + */ +export async function hostSpawnAsync(opts: HostSpawnOpts): Promise { + const args = [ + 'host', + 'spawn', + opts.instanceName, + '--manager', opts.managerAddress, + '--template', opts.templateId, + '--json', + '--timeout', String(opts.timeoutMs ?? DEFAULT_HMCP_TIMEOUT_MS), + ]; + for (const [k, v] of Object.entries(opts.env ?? {})) { + args.push('--env', `${k}=${v}`); + } + const result = await runSphereAsync(opts.cliPath, opts.cliHome, args, { + timeoutMs: (opts.timeoutMs ?? DEFAULT_HMCP_TIMEOUT_MS) + 30_000, + }); + if (result.status !== 0) { + throw new Error( + `sphere host spawn failed (status=${result.status}, signal=${result.signal}). ` + + `stderr: ${result.stderr.slice(0, 800)}\nstdout: ${result.stdout.slice(0, 800)}`, + ); + } + const responses = parseSpawnResponses(result.stdout); + const ready = responses.find((r) => r.type === 'hm.spawn_ready'); + if (!ready) { + const failed = responses.find((r) => r.type === 'hm.spawn_failed' || r.type === 'hm.error'); + throw new Error( + `sphere host spawn did not produce hm.spawn_ready. ` + + `last response: ${JSON.stringify(failed ?? responses[responses.length - 1])}`, + ); + } + const p = ready.payload; + return { + instanceId: String(p['instance_id'] ?? ''), + instanceName: String(p['instance_name'] ?? ''), + tenantPubkey: String(p['tenant_pubkey'] ?? ''), + tenantDirectAddress: String(p['tenant_direct_address'] ?? ''), + tenantNametag: typeof p['tenant_nametag'] === 'string' ? p['tenant_nametag'] : null, + state: String(p['state'] ?? ''), + }; +} + export interface HostStopOpts { cliPath: string; cliHome: string; diff --git a/test/e2e-live/helpers/manager-process.ts b/test/e2e-live/helpers/manager-process.ts index 51a3177..d8b3832 100644 --- a/test/e2e-live/helpers/manager-process.ts +++ b/test/e2e-live/helpers/manager-process.ts @@ -157,6 +157,18 @@ async function provisionManagerWallet(dataDir: string, hostId: string): Promise< // Forward UNICITY_API_KEY when set; the SDK falls back to its public // placeholder otherwise. const apiKey = process.env['UNICITY_API_KEY']?.trim() || undefined; + // Forward Nostr-relay override — this pre-creation step publishes the + // manager's nametag binding. Without the override here it would land + // on testnet, then the HMA binary (which DOES read the override) loads + // the existing wallet and skips re-publish. The local relay would + // never see the binding event and sphere-cli's queryPubkeyByNametag + // returns "Unicity ID not found". + const relayOverride = (() => { + const raw = process.env['UNICITY_NOSTR_RELAYS'] ?? process.env['SPHERE_NOSTR_RELAYS']; + if (!raw) return undefined; + const relays = raw.split(',').map((s) => s.trim()).filter((s) => s.length > 0); + return relays.length > 0 ? relays : undefined; + })(); const providers = createNodeProviders({ network: 'testnet', dataDir, @@ -165,6 +177,7 @@ async function provisionManagerWallet(dataDir: string, hostId: string): Promise< trustBasePath: trustbasePath, ...(apiKey ? { apiKey } : {}), }, + ...(relayOverride ? { transport: { relays: relayOverride } } : {}), }); const nametag = `m-${hostId.replace(/[^a-z0-9]/gi, '').slice(0, 12).toLowerCase()}`; const { sphere } = await Sphere.init({ @@ -257,12 +270,25 @@ export async function spawnHostManager(opts: SpawnHostManagerOptions): Promise }, cwd?: string): R PATH: process.env['PATH'] ?? '', HOME: process.env['HOME'] ?? cwd ?? '/', ...(process.env['UNICITY_API_KEY'] ? { UNICITY_API_KEY: process.env['UNICITY_API_KEY'] } : {}), + // Forward optional Nostr-relay override so sphere-cli subprocesses + // (wallet init, sphere host spawn, sphere trader create-intent, …) + // hit the same relay as the rest of the stack when the local-infra + // harness is active. Falls through silently when unset. + ...(process.env['UNICITY_NOSTR_RELAYS'] + ? { UNICITY_NOSTR_RELAYS: process.env['UNICITY_NOSTR_RELAYS'] } + : {}), + ...(process.env['SPHERE_NOSTR_RELAYS'] + ? { SPHERE_NOSTR_RELAYS: process.env['SPHERE_NOSTR_RELAYS'] } + : {}), CI: '1', FORCE_COLOR: '0', ...(opts?.extraEnv ?? {}), diff --git a/test/e2e-live/helpers/sphere-trader.ts b/test/e2e-live/helpers/sphere-trader.ts index f7b80bb..5658b25 100644 --- a/test/e2e-live/helpers/sphere-trader.ts +++ b/test/e2e-live/helpers/sphere-trader.ts @@ -23,7 +23,7 @@ * call site that needs parallelism. */ -import { runSphere, type SphereRunResult } from './sphere-cli.js'; +import { runSphere, runSphereAsync, type SphereRunResult } from './sphere-cli.js'; const DEFAULT_TRADER_TIMEOUT_MS = 60_000; @@ -162,6 +162,13 @@ export interface CreateIntentOpts extends TraderInvocationOpts { /** Total intent volume; matches the trader's ACP `volume_max` wire field. */ volumeMax: bigint; expiryMs?: number; + /** + * Escrow address (pubkey hex / DIRECT:// / PROXY://). When omitted, + * the trader defaults to 'any' (wildcard) which routes the swap to + * NO real escrow — settlement fails with "Swap not found". Tests + * and production callers MUST pass this for end-to-end settlement. + */ + escrowAddress?: string; } export interface CreatedIntent { @@ -179,6 +186,7 @@ export function createIntent(opts: CreateIntentOpts): CreatedIntent { '--volume-max', opts.volumeMax.toString(), ]; if (opts.expiryMs !== undefined) args.push('--expiry-ms', String(opts.expiryMs)); + if (opts.escrowAddress !== undefined) args.push('--escrow-address', opts.escrowAddress); const { result } = runTraderCommand('create-intent', args, opts); if (typeof result !== 'object' || result === null) { throw new Error(`create-intent: result is not an object. Got: ${JSON.stringify(result)}`); @@ -315,3 +323,200 @@ export async function waitForDealInState( `within ${opts.timeoutMs ?? 600_000}ms. ${summary}.`, ); } + +// --------------------------------------------------------------------------- +// WITHDRAW_TOKEN — sphere trader withdraw +// --------------------------------------------------------------------------- + +export interface WithdrawOpts extends TraderInvocationOpts { + asset: string; + amount: bigint; + toAddress: string; +} + +export interface WithdrawResult { + readonly transferId: string; + readonly remainingBalance: string; +} + +function parseWithdrawResult(result: unknown): WithdrawResult { + if (typeof result !== 'object' || result === null) { + throw new Error(`withdraw: result not an object. Got: ${JSON.stringify(result)}`); + } + const r = result as Record; + const transferId = r['transfer_id']; + if (typeof transferId !== 'string' || transferId === '') { + throw new Error(`withdraw: missing transfer_id. Got: ${JSON.stringify(result)}`); + } + return { + transferId, + remainingBalance: String(r['remaining_balance'] ?? ''), + }; +} + +export function withdraw(opts: WithdrawOpts): WithdrawResult { + const args = [ + '--asset', opts.asset, + '--amount', opts.amount.toString(), + '--to-address', opts.toAddress, + ]; + const { result } = runTraderCommand('withdraw', args, opts); + return parseWithdrawResult(result); +} + +// --------------------------------------------------------------------------- +// Async variants — required for true concurrency. The sync helpers above +// use spawnSync which BLOCKS the event loop, so wrapping them in +// Promise.all(...) does NOT parallelize: each spawnSync call holds the +// thread until the child exits. The async variants below use spawn (via +// runSphereAsync) so concurrent calls actually overlap. +// +// Only the helpers needed for parallel scenarios get async wrappers; the +// rest stay sync to keep the surface small. +// --------------------------------------------------------------------------- + +async function runTraderCommandAsync( + subcommand: string, + args: readonly string[], + opts: TraderInvocationOpts, +): Promise<{ result: unknown; raw: SphereRunResult }> { + const timeoutMs = opts.timeoutMs ?? DEFAULT_TRADER_TIMEOUT_MS; + const fullArgs = [ + 'trader', subcommand, ...args, + '--tenant', opts.tenant, + '--json', + '--timeout', String(timeoutMs), + ]; + const raw = await runSphereAsync(opts.cliPath, opts.cliHome, fullArgs, { + timeoutMs: timeoutMs + 15_000, + }); + if (raw.status !== 0) { + throw new Error( + `sphere trader ${subcommand} failed (status=${raw.status}, signal=${raw.signal}).\n` + + `stderr: ${raw.stderr.slice(0, 800)}\n` + + `stdout: ${raw.stdout.slice(0, 800)}`, + ); + } + const start = raw.stdout.indexOf('{'); + const end = raw.stdout.lastIndexOf('}'); + if (start < 0 || end <= start) { + throw new Error( + `sphere trader ${subcommand} --json: no JSON object in stdout. ` + + `Got first 500 chars: ${raw.stdout.slice(0, 500)}`, + ); + } + const parsed = JSON.parse(raw.stdout.slice(start, end + 1)) as { + ok?: boolean; + result?: unknown; + error_code?: string; + message?: string; + }; + if (parsed.ok === false) { + throw new Error( + `sphere trader ${subcommand}: ok=false. ` + + `[${parsed.error_code ?? 'UNKNOWN'}] ${parsed.message ?? '(no message)'}`, + ); + } + const result = parsed.result ?? parsed; + return { result, raw }; +} + +export async function setStrategyAsync(opts: SetStrategyOpts): Promise { + const args: string[] = []; + if (opts.rateStrategy !== undefined) args.push('--rate-strategy', opts.rateStrategy); + if (opts.maxConcurrent !== undefined) args.push('--max-concurrent', String(opts.maxConcurrent)); + if (opts.trustedEscrows !== undefined && opts.trustedEscrows.length > 0) { + args.push('--trusted-escrows', opts.trustedEscrows.join(',')); + } + const { result } = await runTraderCommandAsync('set-strategy', args, opts); + return result; +} + +export async function createIntentAsync(opts: CreateIntentOpts): Promise { + const args = [ + '--direction', opts.direction, + '--base', opts.baseAsset, + '--quote', opts.quoteAsset, + '--rate-min', opts.rateMin.toString(), + '--rate-max', opts.rateMax.toString(), + '--volume-min', opts.volumeMin.toString(), + '--volume-max', opts.volumeMax.toString(), + ]; + if (opts.expiryMs !== undefined) args.push('--expiry-ms', String(opts.expiryMs)); + if (opts.escrowAddress !== undefined) args.push('--escrow-address', opts.escrowAddress); + const { result } = await runTraderCommandAsync('create-intent', args, opts); + if (typeof result !== 'object' || result === null) { + throw new Error(`create-intent: result is not an object. Got: ${JSON.stringify(result)}`); + } + const intentId = (result as Record)['intent_id']; + if (typeof intentId !== 'string') { + throw new Error(`create-intent: missing intent_id. Got: ${JSON.stringify(result)}`); + } + return { intentId }; +} + +export async function portfolioAsync(opts: TraderInvocationOpts): Promise { + const { result } = await runTraderCommandAsync('portfolio', [], opts); + if (Array.isArray(result)) return result as PortfolioBalance[]; + if (typeof result === 'object' && result !== null) { + const r = result as Record; + const balances = r['balances'] ?? r['portfolio']; + if (Array.isArray(balances)) return balances as PortfolioBalance[]; + return Object.entries(r).map(([asset, amount]) => ({ asset, amount: String(amount) })); + } + throw new Error(`portfolio: response not in expected shape. Got: ${JSON.stringify(result)}`); +} + +export async function listDealsAsync(opts: TraderInvocationOpts): Promise { + const { result } = await runTraderCommandAsync('list-deals', [], opts); + if (Array.isArray(result)) return result as DealSummary[]; + if (typeof result === 'object' && result !== null) { + const arr = (result as Record)['deals'] ?? (result as Record)['swaps']; + if (Array.isArray(arr)) return arr as DealSummary[]; + } + throw new Error(`list-deals: response not in expected shape. Got: ${JSON.stringify(result)}`); +} + +export async function withdrawAsync(opts: WithdrawOpts): Promise { + const args = [ + '--asset', opts.asset, + '--amount', opts.amount.toString(), + '--to-address', opts.toAddress, + ]; + const { result } = await runTraderCommandAsync('withdraw', args, opts); + return parseWithdrawResult(result); +} + +/** + * Async polling variant of waitForDealInState — uses listDealsAsync so + * concurrent waits across multiple traders truly overlap (each list-deals + * spawns its own subprocess via runSphereAsync, no event-loop blocking). + */ +export async function waitForDealInStateAsync( + opts: TraderInvocationOpts & { + targetState: string; + timeoutMs?: number; + intervalMs?: number; + }, +): Promise { + const deadline = Date.now() + (opts.timeoutMs ?? 600_000); + const interval = opts.intervalMs ?? 3_000; + let lastSeen: readonly DealSummary[] = []; + while (Date.now() < deadline) { + try { + lastSeen = await listDealsAsync(opts); + const match = lastSeen.find((d) => d.state === opts.targetState); + if (match) return match; + } catch { + // Transient error — keep polling. + } + await new Promise((r) => setTimeout(r, interval)); + } + const summary = lastSeen.length > 0 + ? `last seen ${lastSeen.length} deal(s) in states [${lastSeen.map((d) => d.state).join(', ')}]` + : 'no deals visible'; + throw new Error( + `waitForDealInStateAsync: tenant ${opts.tenant} did not reach state="${opts.targetState}" ` + + `within ${opts.timeoutMs ?? 600_000}ms. ${summary}.`, + ); +} diff --git a/test/e2e-live/hma-trade-settlement.e2e-live.test.ts b/test/e2e-live/hma-trade-settlement.e2e-live.test.ts new file mode 100644 index 0000000..116b912 --- /dev/null +++ b/test/e2e-live/hma-trade-settlement.e2e-live.test.ts @@ -0,0 +1,795 @@ +/** + * Live e2e: HMA-orchestrated trade-settlement (THE goal-completion test). + * + * Closes the architectural loop that hma-trade-flow.e2e-live.test.ts + * stopped one step short of: full Architecture-B settlement THROUGH + * the HMA, not via direct-docker. + * + * Test + * ├── boot HMA over Sphere DM (single shared instance) + * └── for each scenario IN PARALLEL (it.concurrent): + * ├── spawn escrow + 2 traders ← sequential (sphere-cli wallet contention) + * ├── FAUCET_REQUEST → shared faucet-agent for each trader + * │ (5000 UCT + 5000 USDU per trader; faucet mints + sends DM) + * ├── poll each trader's portfolio until faucet delivery confirms + * ├── set-strategy on both traders ← Promise.all + * ├── post matching intents on both ← Promise.all + * ├── wait for COMPLETED on both ← Promise.all + * ├── assert balance deltas (buyer +UCT/-USDU, seller mirror) + * └── withdraw a small amount from one trader + * to the controller's DIRECT:// address + * + * Why this exists: + * The user's stated goal: "operators can launch HMA, spawn agents, + * fund them, trade (multi-party), AND withdraw" — all over Sphere + * DMs. Every prior live test covers a slice; this one chains every + * slice into a single end-to-end run. + * + * - basic-roundtrip.e2e-live: settles correctly, but uses direct + * docker run (no HMA in the loop). + * - hma-orchestrated.e2e-live: spawns through HMA, but doesn't trade. + * - hma-trade-flow.e2e-live: spawns through HMA, posts/cancels + * intents, but explicitly stops short of settlement. + * - this file: spawns through HMA, settles, AND withdraws. + * + * Parallelism contract: + * - 2 scenarios run concurrently across the file (vitest `it.concurrent`). + * Each scenario uses its OWN controller wallet in its OWN cliHome + * dir — concurrent sphere-cli calls in the SAME .sphere-cli/wallet.json + * race on the SDK's atomic temp+rename writes (FileStorageProvider.save + * → fs.renameSync) and corrupt each other's state. + * - WITHIN each scenario, sphere-cli calls are SEQUENTIAL. Tried fully + * parallel (Promise.all over spawn/set-strategy/portfolio) and got + * two distinct races on the first run: "No wallet exists" (Sphere.init + * reading mid-write) and "ENOENT: rename wallet.json.tmp -> wallet.json" + * (two atomic writes racing each other). The DM round-trips for spawn + * are the only ones inside a scenario where parallelism would matter + * (~5-15s each); against the 3-5min settlement-wait dominator, the + * wall-clock cost of serializing them is negligible. + * - Net effect: total wall time ≈ max(scenario_time), not sum, because + * the long settlement wait runs concurrently across scenarios. + * - Live infra load at peak: 2 cross-scenario sphere-cli children, + * each making 1 DM at a time. Manageable for the relay. Aggregator + * load comes from the trader containers themselves, not sphere-cli. + * + * Performance target on a healthy testnet: ~5-8 minutes total. + * - HMA boot: ~10s + * - 6 concurrent spawns (2 scenarios × 3 tenants): ~30-60s + * - 4 concurrent set-strategy: ~5-10s + * - 4 concurrent create-intent: ~5-10s + * - settlement wait: 3-5 min on testnet (dominates) + * - 2 concurrent withdraws: ~5-10s + * + * Trader image: + * This test requires `ghcr.io/vrogojin/agentic-hosting/trader:local` + * built from this repo's Dockerfile (which embeds the current + * sphere-sdk including mintFungibleToken). Build before running: + * cd /home/vrogojin && docker build -f trader-service/Dockerfile \ + * -t ghcr.io/vrogojin/agentic-hosting/trader:local . + * The published `:v0.1` tag at ghcr.io is too old (tested in rounds + * 3-5: lacks mintFungibleToken AND the faucet path silently + * doesn't deliver). The test materializes a temp templates.json + * that swaps the image tag from v0.1 to local without modifying + * agentic-hosting's shared config. + * + * Funding: + * Each trader is funded via a `FAUCET_REQUEST` ACP DM to a SHARED + * js-faucet agent spawned by the same HMA. The test bootstraps an + * in-process Sphere wallet (helpers/faucet-client.ts) to sign + + * encrypt the DM. The faucet mints UCT + USDU and sends them to + * each trader's address; the test polls portfolio until + * `confirmed >= INITIAL_FUND_AMOUNT` for both assets. + * + * This replaces two earlier funding paths that didn't work: + * - TRADER_TEST_FUND self-mint: trader-issued tokens may + * interact poorly with the swap protocol (issuer == sender + * is unusual in production). + * - Public faucet HTTP: returns 200 OK + tx_id but the deposit + * never surfaces in portfolio (verified across rounds 5-6). + * + * The js-faucet image must be built before running this test: + * cd /home/vrogojin && docker build \ + * -f js-faucet/Dockerfile \ + * -t ghcr.io/unicitynetwork/agentic-hosting/faucet:local . + * + * Out of scope: + * - Negotiation-failure paths — covered by negotiation-failures. + * - Partial fills — covered by edge-cases. + */ + +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { rmSync, existsSync, readFileSync, writeFileSync, mkdtempSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { randomUUID } from 'node:crypto'; +import { + probeSphereCli, + createSphereCliEnv, + bootstrapControllerWallet, + type SphereCliProbe, +} from './helpers/sphere-cli.js'; +import { + spawnHostManager, + checkAgenticHostingPath, + type HostManagerProcess, +} from './helpers/manager-process.js'; +import { + hostSpawnAsync, + hostStop, + type SpawnedTenant, +} from './helpers/hma-spawn.js'; +import { + setStrategyAsync, + createIntentAsync, + portfolioAsync, + waitForDealInStateAsync, + withdrawAsync, + type PortfolioBalance, +} from './helpers/sphere-trader.js'; +import { createFaucetClient, type FaucetClient } from './helpers/faucet-client.js'; +import { ESCROW_IMAGE } from './helpers/constants.js'; + +// --------------------------------------------------------------------------- +// Precondition gates (mirrors hma-trade-flow's structure) +// --------------------------------------------------------------------------- + +const cliProbe: SphereCliProbe = probeSphereCli(); +const agenticProbe = checkAgenticHostingPath(); +let managerBinPath = ''; +let agenticReady = false; +if (agenticProbe.ok) { + managerBinPath = join(agenticProbe.path, 'dist', 'host-manager.js'); + agenticReady = existsSync(managerBinPath); +} +const skip = !cliProbe.ok || !agenticReady; +const skipReason = !cliProbe.ok + ? `sphere-cli not runnable: ${cliProbe.reason}` + : !agenticProbe.ok + ? agenticProbe.reason + : !agenticReady + ? `agentic-hosting binary missing at ${managerBinPath}.` + : ''; + +// --------------------------------------------------------------------------- +// Shared HMA + controller fixture +// --------------------------------------------------------------------------- + +/** + * Per-scenario controller: the controller wallet (cliHome + pubkey/addr). + * Concurrent `sphere host spawn` invocations write to .sphere-cli/wallet.json + * inside cliHome (the SDK persists incoming-DM state there atomically via + * temp+rename). Two parallel calls in the same cliHome corrupt each other's + * rename — give each scenario its own cliHome to make concurrency safe. + */ +interface ScenarioController { + cliHome: string; + pubkey: string; + directAddress: string; +} + +interface SuiteState { + cliPath: string; + /** All cliHome dirs we created — afterAll wipes them. */ + cliHomes: string[]; + manager: HostManagerProcess; + managerAddr: string; + controllers: ScenarioController[]; + spawned: SpawnedTenant[]; + /** Single shared faucet — anyone can request, so one per HMA suffices. */ + faucet: SpawnedTenant; + /** In-process Sphere wallet that signs+encrypts FAUCET_REQUEST DMs. */ + faucetClient: FaucetClient; + /** + * Env to inject into every HMA-spawned tenant. Carries + * `UNICITY_NOSTR_RELAYS` when the local-infra relay is active; + * empty object otherwise. Forwarded via hostSpawnAsync's `env` field. + */ + spawnEnv: Record; +} + +// Number of concurrent settlement scenarios. Each gets its own controller +// wallet + own cliHome. Bumping this raises the parallel infra load +// (more concurrent spawn DMs, more concurrent listdeals polls during +// settlement). Two is the contract test for the parallelism claim; +// higher values stress-test the relay+aggregator+HMA further. +const SCENARIO_COUNT = 2; + +let state: SuiteState | null = null; + +const SWAP_TIMEOUT_MS = 3 * 60_000; // 3 minutes — bisect-friendly. On a healthy local relay + // the swap protocol completes in ~30-60s end-to-end (deposits + // verify + payouts confirm). 3 min gives 2× headroom while + // failing fast when verifyPayout's reverse-index bug strands + // the deal at COVERED-but-unverified. +/** Per-asset funding amount delivered to each trader by the faucet. */ +const INITIAL_FUND_AMOUNT = 5000n; +/** How long we wait for faucet-delivered tokens to surface in `confirmed` balance. */ +const FUNDING_BALANCE_TIMEOUT_MS = 180_000; + +// `volume_max` for matching intents in each scenario. Both sides post +// identical volumes so a single fill clears both intents. +const TRADE_VOLUME = 10n; +// Per-scenario rates are distinct to prevent cross-scenario matching; +// see runSettlementScenario header for why. +const PAIR_1_RATE = 1n; // 1 USDU per UCT +const PAIR_2_RATE = 3n; // 3 USDU per UCT — non-adjacent to avoid any rate-fuzzing overlap + +// Withdraw amount: small fraction of received UCT so the test asserts +// real value movement without exhausting the trader's post-trade balance. +const WITHDRAW_AMOUNT = 3n; + +// --------------------------------------------------------------------------- + +describe.skipIf(skip).concurrent('HMA-orchestrated trade settlement (live testnet)', () => { + if (skip) { + console.warn(`[hma-trade-settlement] SKIPPED: ${skipReason}`); + } + + beforeAll(async () => { + if (skip) return; + if (!cliProbe.ok) throw new Error('precondition gate inverted'); + const cliPath = cliProbe.path; + + // One controller wallet per scenario — concurrent sphere-cli calls + // in the same .sphere-cli/wallet.json race on the SDK's atomic + // temp+rename writes (sphere-sdk persists DM state per call). + // Bootstrap them sequentially; the wallet-init aggregator round-trip + // (~30s each) dominates this section anyway and parallel inits also + // race on the same OS-level temp dirs. + const controllers: ScenarioController[] = []; + const cliHomes: string[] = []; + for (let i = 0; i < SCENARIO_COUNT; i++) { + const { home } = createSphereCliEnv(`hma-trade-settlement-c${String(i)}`); + cliHomes.push(home); + console.log(`[hma-trade-settlement] bootstrapping controller wallet #${String(i)}…`); + const c = bootstrapControllerWallet(cliPath, home); + console.log(`[hma-trade-settlement] controller #${String(i)} pubkey ${c.pubkey.slice(0, 16)}…`); + controllers.push({ cliHome: home, pubkey: c.pubkey, directAddress: c.directAddress }); + } + + // The published trader image at ghcr.io/.../trader:v0.1 lacks both + // mintFungibleToken (so TRADER_TEST_FUND fails) AND a working + // payments.receive() loop (so faucet deposits never surface in + // portfolio). To get end-to-end settlement working we use a + // locally-built `trader:local` image with the current sphere-sdk + + // trader code. Build via: + // cd /home/vrogojin && docker build -f trader-service/Dockerfile \ + // -t ghcr.io/vrogojin/agentic-hosting/trader:local . + // Then we materialize a temp templates.json that swaps the image + // tag from v0.1 to local. agentic-hosting/config/templates.json is + // not modified. + // The published trader image at ghcr.io/.../trader:v0.1 lacks a working + // payments.receive() loop (verified in earlier rounds). Use the locally- + // built `trader:local` image with the current sphere-sdk + trader code. + // Add a `faucet-agent` template entry pointing at the locally-built + // js-faucet image so the test can spawn it through the same HMA. + const baseTemplatesPath = join(agenticProbe.ok ? agenticProbe.path : '', 'config', 'templates.json'); + const baseTemplates = JSON.parse(readFileSync(baseTemplatesPath, 'utf8')) as { + templates: Array<{ template_id: string; image: string; entrypoint?: string[]; env_defaults?: Record; resources?: Record;[k: string]: unknown }>; + }; + for (const t of baseTemplates.templates) { + if (t.template_id === 'trader-agent') { + t.image = 'ghcr.io/vrogojin/agentic-hosting/trader:local'; + } + // Use the same v0.2 image pin as the rest of the e2e-live suite + // (constants.ts ESCROW_IMAGE) so the HMA-spawned escrow runs the + // same code as the direct-Docker-spawned escrow in basic-roundtrip. + // + // Why this override exists at all: agentic-hosting's + // config/templates.json still pins escrow:v0.1 (its own release + // cadence is independent). Until that templates.json bumps, we + // override here so HMA-spawned escrows pick up v0.2's + // deliverDepositInvoice fix + conservative-payout + UXF protocol + // (PR #105, #115, #119, #128, #146/147/149/152). + // + // History: previously this overrode to `escrow:local` because the + // published v0.1 had an asymmetric deliverDepositInvoice bug + // (every other party's invoice_delivery DM was dropped) — see + // HMA-SETTLEMENT-DIAGNOSTIC.md rounds 11-19. Round 19 evidence + // confirmed the bug is GONE in current source (= what we shipped + // as v0.2 on 2026-05-16). The `escrow:local` build dependency + // is now removed — devs/CI no longer need to docker-build the + // escrow image before running this test. + if (t.template_id === 'escrow-service') { + t.image = ESCROW_IMAGE; + } + } + if (!baseTemplates.templates.some((t) => t.template_id === 'faucet-agent')) { + baseTemplates.templates.push({ + template_id: 'faucet-agent', + image: 'ghcr.io/unicitynetwork/agentic-hosting/faucet:local', + entrypoint: ['node', '/app/dist/acp-adapter/main.js'], + env_defaults: { LOG_LEVEL: 'info', SPHERE_NETWORK: 'testnet' }, + resources: { memory_mb: 512, pids_limit: 256 }, + }); + } else { + for (const t of baseTemplates.templates) { + if (t.template_id === 'faucet-agent') { + t.image = 'ghcr.io/unicitynetwork/agentic-hosting/faucet:local'; + } + } + } + const tplDir = mkdtempSync(join(tmpdir(), 'hma-trade-settlement-tpl-')); + cliHomes.push(tplDir); + const customTemplatesPath = join(tplDir, 'templates.json'); + writeFileSync(customTemplatesPath, JSON.stringify(baseTemplates, null, 2)); + + // HMA accepts AUTHORIZED_CONTROLLERS as a comma-separated list of + // pubkeys (see agentic-hosting/src/shared/config.ts:52). Authorize + // every scenario's controller in one HMA — production has one HMA + // per host serving multiple operators, so this matches the real + // multi-tenant topology. + console.log('[hma-trade-settlement] booting host-manager…'); + const manager = await spawnHostManager({ + controllerPubkey: controllers.map((c) => c.pubkey).join(','), + templatesPath: customTemplatesPath, + }); + await manager.ready; + const managerAddr = manager.nametag ? `@${manager.nametag}` : manager.pubkey; + console.log(`[hma-trade-settlement] manager ready @ ${managerAddr}`); + + // Spawn ONE shared faucet (the faucet is open — anyone can request, + // so we don't need one per scenario). + // + // When the local-infra Nostr relay is active (TRADER_E2E_LOCAL_RELAY=1 + // → global-setup.ts boots a Docker relay and sets + // UNICITY_NOSTR_RELAYS to its bridge-gateway URL), forward that env + // into every spawned tenant via HMA's `--env` passthrough so the + // tenant connects to the local relay instead of the public testnet. + // + // IMPORTANT: HMA's validatePayloadEnv at + // agentic-hosting/src/host-manager/manager.ts:90 blocks every env + // var starting with `UNICITY_` (it protects HMA-internal vars like + // UNICITY_BOOT_TOKEN from controller-side override). So we use the + // sibling `SPHERE_NOSTR_RELAYS` alias which is treated identically + // by every service's relay-override pickup but isn't on the + // forbidden-prefix list. No-op when neither env var is set. + const relayUrl = process.env['UNICITY_NOSTR_RELAYS'] ?? process.env['SPHERE_NOSTR_RELAYS']; + const localRelayEnv: Record = relayUrl + ? { SPHERE_NOSTR_RELAYS: relayUrl } + : {}; + + console.log('[hma-trade-settlement] spawning shared faucet-agent…'); + const faucet = await hostSpawnAsync({ + cliPath, + cliHome: controllers[0]!.cliHome, + managerAddress: managerAddr, + templateId: 'faucet-agent', + instanceName: `faucet-${randomUUID().slice(0, 6)}`, + timeoutMs: 180_000, + env: localRelayEnv, + }); + console.log( + `[hma-trade-settlement] faucet ready: pubkey=${faucet.tenantPubkey.slice(0, 16)}… ` + + `nametag=${faucet.tenantNametag ?? ''}`, + ); + + // In-process Sphere wallet that the test uses to send FAUCET_REQUEST + // DMs. The faucet doesn't authorize senders, so this wallet doesn't + // need to be in HMA's AUTHORIZED_CONTROLLERS list. + console.log('[hma-trade-settlement] bootstrapping in-process FaucetClient…'); + const faucetClient = await createFaucetClient(); + console.log(`[hma-trade-settlement] faucet client pubkey ${faucetClient.pubkey.slice(0, 16)}…`); + + state = { + cliPath, + cliHomes, + manager, + managerAddr, + controllers, + spawned: [faucet], + faucet, + faucetClient, + spawnEnv: localRelayEnv, + }; + }, 900_000); // 15 min — adds ~30-60s for faucet spawn + client bootstrap on top of controller-wallet inits + + afterAll(async () => { + if (!state) return; + // Best-effort parallel cleanup. Use the FIRST controller's cliHome + // for stop calls — the HMA accepts stops from any authorized + // controller, so we don't need to issue one stop per controller. + const stopHome = state.controllers[0]?.cliHome ?? state.cliHomes[0]!; + await Promise.allSettled( + state.spawned.map((t) => + hostStop({ + cliPath: state!.cliPath, + cliHome: stopHome, + managerAddress: state!.manager.pubkey, + target: t.instanceName, + timeoutMs: 60_000, + }), + ), + ); + try { await state.faucetClient.destroy(); } catch { /* best effort */ } + await state.manager.stop(); + for (const home of state.cliHomes) { + try { rmSync(home, { recursive: true, force: true }); } + catch { /* best effort */ } + } + }, 240_000); + + // ---- Per-scenario helpers ------------------------------------------------- + + /** + * Spawn one escrow + two traders (alice/bob) sequentially within a + * scenario, then fund each trader with UCT + USDU via the SHARED + * faucet-agent. Funding via FAUCET_REQUEST DM replaces the previous + * TRADER_TEST_FUND self-mint pathway: + * - Self-mint produced trader-issued tokens, which may interact + * poorly with the swap protocol (the swap counterparty would see + * the issuer == sender, which is unusual in production). + * - Public faucet HTTP returned 200 OK + tx_id but never delivered + * (verified across multiple rounds — silent flake). + * - The js-faucet agent mints + sends with the FAUCET as issuer, + * matching production reality. + * Each trader is funded with `INITIAL_FUND_AMOUNT` of both UCT and + * USDU so either side has the inventory to fulfil any direction. + */ + async function provisionTriple( + scenarioId: string, + controller: ScenarioController, + ): Promise<{ + escrow: SpawnedTenant; + alice: SpawnedTenant; + bob: SpawnedTenant; + }> { + if (!state) throw new Error('beforeAll did not initialize state'); + const s = state; + // Within a scenario, sphere-cli calls share one wallet.json and + // race on its atomic temp+rename writes — see header. Sequential + // within-scenario; cross-scenario runs in true parallel via + // vitest it.concurrent. + console.log(`[${scenarioId}] spawning escrow + alice + bob (sequential within-scenario)…`); + const escrow = await hostSpawnAsync({ + cliPath: s.cliPath, + cliHome: controller.cliHome, + managerAddress: s.managerAddr, + templateId: 'escrow-service', + instanceName: `escrow-${scenarioId}`, + timeoutMs: 180_000, + env: s.spawnEnv, + }); + // Funding is delivered separately via FAUCET_REQUEST DM (see below) + // — traders boot with no balance, then the test sends FAUCET_REQUEST + // ACP DMs to the shared faucet-agent which mints + transfers tokens. + // This matches production reality (faucet as token issuer) more + // closely than TRADER_TEST_FUND self-mint and exercises the SDK's + // payments.receive({finalize:true}) ingestion path. + const alice = await hostSpawnAsync({ + cliPath: s.cliPath, + cliHome: controller.cliHome, + managerAddress: s.managerAddr, + templateId: 'trader-agent', + instanceName: `alice-${scenarioId}`, + timeoutMs: 180_000, + env: s.spawnEnv, + }); + const bob = await hostSpawnAsync({ + cliPath: s.cliPath, + cliHome: controller.cliHome, + managerAddress: s.managerAddr, + templateId: 'trader-agent', + instanceName: `bob-${scenarioId}`, + timeoutMs: 180_000, + env: s.spawnEnv, + }); + s.spawned.push(escrow, alice, bob); + console.log( + `[${scenarioId}] up: escrow=${escrow.instanceName} ` + + `alice=${alice.instanceName} bob=${bob.instanceName}`, + ); + + // Fund each trader via FAUCET_REQUEST — sends an ACP-0 DM to the + // shared faucet-agent which mints UCT+USDU and sends them to the + // trader. Sequential within scenario to avoid relay-side contention + // (the in-process FaucetClient holds a single Sphere wallet whose + // DM-send path serializes anyway). The faucet handles mint + send; + // the trader's periodic payments.receive({finalize:true}) loop + // ingests the inbound transfer and surfaces it in portfolio. + // + // Recipient address: use the trader's @nametag (canonical identity + // per project guidelines). The trader publishes its nametag binding + // event during Sphere.init and verifies the binding is queryable on + // the relay before announcing sphere_initialized — so by the time + // the trader is "spawned" (HMA acp.hello received), the relay has + // the binding. Sending to DIRECT:// would also work IF the + // SDK's resolveAddressInfo could find the binding, but the binding + // event publishes the L3-predicate-derived directAddress (not the + // bare pubkey-prefixed shape), so that lookup misses. @nametag goes + // through queryPubkeyByNametag which is what the trader registered. + for (const t of [{ name: 'alice', tenant: alice }, { name: 'bob', tenant: bob }]) { + const recipient = t.tenant.tenantNametag !== null + ? `@${t.tenant.tenantNametag}` + : t.tenant.tenantDirectAddress; + console.log(`[${scenarioId}] FAUCET_REQUEST → ${t.name} (recipient=${recipient}, UCT+USDU=${INITIAL_FUND_AMOUNT})…`); + const deliveries = await s.faucetClient.request(s.faucet.tenantPubkey, { + recipient, + items: [ + { asset: 'UCT', amount: INITIAL_FUND_AMOUNT.toString() }, + { asset: 'USDU', amount: INITIAL_FUND_AMOUNT.toString() }, + ], + }); + console.log( + `[${scenarioId}] ${t.name} faucet deliveries: ` + + deliveries.map((d) => `${d.asset}=${d.amount} (transfer=${d.transfer_id.slice(0, 12)}…)`).join(', '), + ); + } + + return { escrow, alice, bob }; + } + + /** + * Pull a coin's confirmed balance (smallest units) from a portfolio + * response. The trader's GET_PORTFOLIO emits each balance as + * { asset, available, total, confirmed, unconfirmed } + * where `confirmed` is the amount we should use for assertions. + * Older versions used `amount`; tolerate both. Round-7 hit a bug + * where the early return on `b.amount ?? '0'` short-circuited to + * 0n WITHOUT falling through to `confirmed` when the field was + * absent, so the self-mint balance assertion failed despite the + * mint succeeding. + */ + function balanceOf(p: readonly PortfolioBalance[], symbol: string): bigint { + for (const b of p as Array>) { + if (b['asset'] !== symbol) continue; + // Prefer `confirmed` (canonical); fall back to `amount` (legacy). + if (b['confirmed'] !== undefined) return BigInt(String(b['confirmed'])); + if (b['amount'] !== undefined) return BigInt(String(b['amount'])); + // `available` is also a reasonable fallback for "what can be spent now". + if (b['available'] !== undefined) return BigInt(String(b['available'])); + return 0n; + } + return 0n; + } + + /** + * One full settlement scenario, parametrized by name. Inside the test: + * 1. Spawn escrow + buyer + seller in parallel. + * 2. set-strategy on both traders in parallel. + * 3. Pre-trade portfolio snapshot in parallel. + * 4. Post matching intents in parallel. + * 5. Wait for COMPLETED on both deals in parallel. + * 6. Post-trade portfolio snapshot in parallel; assert deltas. + * 7. Withdraw a small amount from buyer to controller's DIRECT:// + * address; assert transfer_id non-empty + post-withdraw balance + * reflects the withdrawal. + */ + /** + * `tradeRate` differs per scenario so the two pairs CANNOT + * cross-match on testnet. Round 8 saw both scenarios stuck at + * waitForDealInState → COMPLETED with 29 deals in CANCELLED state + * because pair-1's alice (rate=1) was matching pair-2's bob (also + * rate=1) and trying to negotiate — but the trusted_escrows on + * each side only allow that scenario's own escrow, so every + * cross-scenario negotiation flipped to FAILED/CANCELLED in a + * thrash loop. With distinct rates, rate-overlap filtering at + * the matcher level prevents the cross-match before negotiation + * even starts. + */ + async function runSettlementScenario( + scenarioId: string, + controller: ScenarioController, + tradeRate: bigint, + ): Promise { + if (!state) throw new Error('beforeAll did not initialize state'); + const s = state; + const { escrow, alice, bob } = await provisionTriple(scenarioId, controller); + + // FAUCET_REQUEST funding: the test already issued FAUCET_REQUEST DMs + // in provisionTriple and the faucet returned acp.result with + // delivery records. The trader's periodic + // payments.receive({finalize:true}) loop must ingest the inbound + // transfer before the balance surfaces in portfolio — poll until + // confirmed >= INITIAL_FUND_AMOUNT for both UCT and USDU. + console.log(`[${scenarioId}] waiting for faucet-funded balances to confirm…`); + for (const t of [{ name: 'alice', tenant: alice }, { name: 'bob', tenant: bob }]) { + const deadline = Date.now() + FUNDING_BALANCE_TIMEOUT_MS; + let lastSnapshot: readonly PortfolioBalance[] = []; + while (Date.now() < deadline) { + try { + lastSnapshot = await portfolioAsync({ + cliPath: s.cliPath, cliHome: controller.cliHome, tenant: t.tenant.tenantPubkey, + }); + if ( + balanceOf(lastSnapshot, 'UCT') >= INITIAL_FUND_AMOUNT && + balanceOf(lastSnapshot, 'USDU') >= INITIAL_FUND_AMOUNT + ) { + break; + } + } catch { /* transient — keep polling */ } + await new Promise((resolve) => setTimeout(resolve, 5_000)); + } + const uct = balanceOf(lastSnapshot, 'UCT'); + const usdu = balanceOf(lastSnapshot, 'USDU'); + if (uct < INITIAL_FUND_AMOUNT || usdu < INITIAL_FUND_AMOUNT) { + throw new Error( + `[${scenarioId}] ${t.name} did not see UCT>=${INITIAL_FUND_AMOUNT} && USDU>=${INITIAL_FUND_AMOUNT} ` + + `within ${FUNDING_BALANCE_TIMEOUT_MS}ms. observed: UCT=${uct}, USDU=${usdu}`, + ); + } + console.log(`[${scenarioId}] ${t.name} balance confirmed: ${uct}UCT/${usdu}USDU`); + } + + // ---- 2. Configure trusted escrows on both traders -------------- + // Sequential within-scenario (wallet.json contention; see provisionTriple). + console.log(`[${scenarioId}] set-strategy on alice + bob…`); + await setStrategyAsync({ + cliPath: s.cliPath, cliHome: controller.cliHome, tenant: alice.tenantPubkey, + trustedEscrows: [escrow.tenantDirectAddress], + maxConcurrent: 5, + }); + await setStrategyAsync({ + cliPath: s.cliPath, cliHome: controller.cliHome, tenant: bob.tenantPubkey, + trustedEscrows: [escrow.tenantDirectAddress], + maxConcurrent: 5, + }); + + // ---- 3. Pre-trade portfolio snapshot --------------------------- + const aliceBefore = await portfolioAsync({ + cliPath: s.cliPath, cliHome: controller.cliHome, tenant: alice.tenantPubkey, + }); + const bobBefore = await portfolioAsync({ + cliPath: s.cliPath, cliHome: controller.cliHome, tenant: bob.tenantPubkey, + }); + const aliceUctBefore = balanceOf(aliceBefore, 'UCT'); + const aliceUsduBefore = balanceOf(aliceBefore, 'USDU'); + const bobUctBefore = balanceOf(bobBefore, 'UCT'); + const bobUsduBefore = balanceOf(bobBefore, 'USDU'); + console.log( + `[${scenarioId}] pre-trade balances: ` + + `alice=${aliceUctBefore}UCT/${aliceUsduBefore}USDU bob=${bobUctBefore}UCT/${bobUsduBefore}USDU`, + ); + // Sanity: the faucet delivery must have landed (otherwise no + // UCT/USDU is available to trade and the swap will hang). The + // earlier polling loop already enforces this; this assertion is + // the explicit test contract. + expect(aliceUctBefore + aliceUsduBefore).toBeGreaterThan(0n); + expect(bobUctBefore + bobUsduBefore).toBeGreaterThan(0n); + + // ---- 4. Post matching intents (sequential — wallet.json) ------ + // Alice buys UCT (pays USDU). Bob sells UCT (receives USDU). + console.log(`[${scenarioId}] posting matching intents…`); + // CRITICAL: pass escrow_address. Trader's intent-engine defaults + // it to the literal string 'any' when omitted (intent-engine.ts:836) + // — that's a wildcard that means "any escrow", but the swap-executor + // uses terms.escrow_address as a routing target and tries to send + // swap.announce to 'any', which doesn't resolve. The escrow then + // never sees the announce and rejects subsequent status queries + // with "Swap not found", which trips deal CANCELLED. Round 10 + // diagnosed this from escrow logs (zero announce_received events + // despite ping/pong round-trips working). Fix: route via the + // actual HMA-spawned escrow's pubkey (must match a value in + // trustedEscrows from the earlier set-strategy call). + const aliceIntent = await createIntentAsync({ + cliPath: s.cliPath, cliHome: controller.cliHome, tenant: alice.tenantPubkey, + direction: 'buy', + baseAsset: 'UCT', quoteAsset: 'USDU', + rateMin: tradeRate, rateMax: tradeRate, + volumeMin: TRADE_VOLUME, volumeMax: TRADE_VOLUME, + expiryMs: SWAP_TIMEOUT_MS, + escrowAddress: escrow.tenantDirectAddress, + }); + const bobIntent = await createIntentAsync({ + cliPath: s.cliPath, cliHome: controller.cliHome, tenant: bob.tenantPubkey, + direction: 'sell', + baseAsset: 'UCT', quoteAsset: 'USDU', + rateMin: tradeRate, rateMax: tradeRate, + volumeMin: TRADE_VOLUME, volumeMax: TRADE_VOLUME, + expiryMs: SWAP_TIMEOUT_MS, + escrowAddress: escrow.tenantDirectAddress, + }); + console.log( + `[${scenarioId}] intents posted: alice=${aliceIntent.intentId.slice(0, 12)}… ` + + `bob=${bobIntent.intentId.slice(0, 12)}…`, + ); + + // ---- 5. Wait for both deals to reach COMPLETED ------------------ + // Sequential within-scenario (wallet.json contention). Wait for + // alice first; once she's COMPLETED, bob is typically COMPLETED + // already on the next poll, so this adds at most one poll cycle. + console.log(`[${scenarioId}] waiting for alice COMPLETED…`); + const aliceDeal = await waitForDealInStateAsync({ + cliPath: s.cliPath, cliHome: controller.cliHome, tenant: alice.tenantPubkey, + targetState: 'COMPLETED', + timeoutMs: SWAP_TIMEOUT_MS, + }); + console.log(`[${scenarioId}] waiting for bob COMPLETED…`); + const bobDeal = await waitForDealInStateAsync({ + cliPath: s.cliPath, cliHome: controller.cliHome, tenant: bob.tenantPubkey, + targetState: 'COMPLETED', + timeoutMs: SWAP_TIMEOUT_MS, + }); + expect(aliceDeal.state).toBe('COMPLETED'); + expect(bobDeal.state).toBe('COMPLETED'); + // Both sides observe the same deal_id (one negotiation, two ledgers). + expect(aliceDeal.deal_id).toBe(bobDeal.deal_id); + console.log( + `[${scenarioId}] deal COMPLETED: ${aliceDeal.deal_id.slice(0, 12)}…`, + ); + + // ---- 6. Post-trade balance assertions -------------------------- + // Wait briefly for payments.receive() to finalize inbound payouts + // (trader loop is on a 15s cycle; basic-roundtrip uses 5s and that + // has been enough on testnet). + await new Promise((r) => setTimeout(r, 5_000)); + + const aliceAfter = await portfolioAsync({ + cliPath: s.cliPath, cliHome: controller.cliHome, tenant: alice.tenantPubkey, + }); + const bobAfter = await portfolioAsync({ + cliPath: s.cliPath, cliHome: controller.cliHome, tenant: bob.tenantPubkey, + }); + const aliceUctAfter = balanceOf(aliceAfter, 'UCT'); + const aliceUsduAfter = balanceOf(aliceAfter, 'USDU'); + const bobUctAfter = balanceOf(bobAfter, 'UCT'); + const bobUsduAfter = balanceOf(bobAfter, 'USDU'); + + const expectedUsduPaid = tradeRate * TRADE_VOLUME; + // Alice (buy UCT for USDU): +UCT / -USDU + expect( + aliceUctAfter - aliceUctBefore, + `alice UCT delta should be +${TRADE_VOLUME}; observed: ${aliceUctAfter - aliceUctBefore}`, + ).toBe(TRADE_VOLUME); + expect( + aliceUsduBefore - aliceUsduAfter, + `alice USDU delta should be -${expectedUsduPaid}; observed: -${aliceUsduBefore - aliceUsduAfter}`, + ).toBe(expectedUsduPaid); + // Bob (sell UCT for USDU): -UCT / +USDU + expect( + bobUctBefore - bobUctAfter, + `bob UCT delta should be -${TRADE_VOLUME}; observed: -${bobUctBefore - bobUctAfter}`, + ).toBe(TRADE_VOLUME); + expect( + bobUsduAfter - bobUsduBefore, + `bob USDU delta should be +${expectedUsduPaid}; observed: ${bobUsduAfter - bobUsduBefore}`, + ).toBe(expectedUsduPaid); + + // ---- 7. Withdraw from alice (now has UCT) --------------------- + // Alice had INITIAL_FUND_AMOUNT UCT pre-trade and acquired + // TRADE_VOLUME via the swap. Withdraw a fraction (WITHDRAW_AMOUNT) + // to the controller's DIRECT address — exercises the WITHDRAW_TOKEN + // ACP command end-to-end including the round-6 trim+validation gate. + console.log(`[${scenarioId}] withdraw ${WITHDRAW_AMOUNT} UCT from alice → controller…`); + const wr = await withdrawAsync({ + cliPath: s.cliPath, cliHome: controller.cliHome, tenant: alice.tenantPubkey, + asset: 'UCT', + amount: WITHDRAW_AMOUNT, + toAddress: controller.directAddress, + }); + expect(wr.transferId).toMatch(/^[a-zA-Z0-9_-]+$/); + expect(wr.transferId.length).toBeGreaterThan(8); + console.log(`[${scenarioId}] withdraw transfer_id=${wr.transferId.slice(0, 16)}…`); + + // Verify alice's UCT balance is now reduced by the withdrawn amount. + // Allow a small settle delay for the transfer to land in the + // confirmed bucket (testnet aggregator round-trip). + await new Promise((r) => setTimeout(r, 5_000)); + const aliceFinal = await portfolioAsync({ + cliPath: s.cliPath, cliHome: controller.cliHome, tenant: alice.tenantPubkey, + }); + const aliceUctFinal = balanceOf(aliceFinal, 'UCT'); + expect( + aliceUctAfter - aliceUctFinal, + `alice UCT delta after withdraw should be -${WITHDRAW_AMOUNT}; observed: -${aliceUctAfter - aliceUctFinal}`, + ).toBe(WITHDRAW_AMOUNT); + + console.log(`[${scenarioId}] ✓ end-to-end settlement+withdraw verified`); + } + + // ---- Concurrent scenarios ----------------------------------------------- + + it('Pair-1: full spawn → trade → settle → withdraw via HMA (rate=1)', async () => { + if (!state) throw new Error('beforeAll did not initialize state'); + const c = state.controllers[0]; + if (!c) throw new Error('controller #0 missing'); + await runSettlementScenario(`p1-${randomUUID().slice(0, 6)}`, c, PAIR_1_RATE); + }, SWAP_TIMEOUT_MS + 4 * 60_000); // 12 min total budget per scenario + + it('Pair-2: parallel scenario settles on the same HMA at distinct rate (rate=3)', async () => { + if (!state) throw new Error('beforeAll did not initialize state'); + const c = state.controllers[1]; + if (!c) throw new Error('controller #1 missing'); + await runSettlementScenario(`p2-${randomUUID().slice(0, 6)}`, c, PAIR_2_RATE); + }, SWAP_TIMEOUT_MS + 4 * 60_000); +}); diff --git a/test/e2e-live/local-infra/docker-compose.yml b/test/e2e-live/local-infra/docker-compose.yml new file mode 100644 index 0000000..0f19bac --- /dev/null +++ b/test/e2e-live/local-infra/docker-compose.yml @@ -0,0 +1,71 @@ +# ============================================================================= +# Local infrastructure for trader-service e2e tests. +# +# Boots a local Nostr relay so the e2e suite can run against a +# deterministic, in-process stack instead of the public testnet relay — +# useful when: +# - the testnet relay's write path is broken / silently dropping +# publishes (the 2026-05-08 outage that re-motivated this harness); +# - CI runs need reproducibility (no shared rate limits, no +# nametag collisions across concurrent jobs); +# - we are debugging the SDK's interaction with Nostr and want a +# SQLite-backed relay we can `docker exec sqlite3 ./events.db`. +# +# Source: ported from /home/vrogojin/uxf/tests/e2e/local-infra/. +# +# Aggregator (L3) and IPFS gateway are NOT replaced — the public +# Unicity testnet aggregator/IPFS are reliable and the sphere-sdk has +# no aggregator stub that would round-trip real inclusion proofs. +# E2E tests that need the aggregator continue to talk to: +# wss://goggregator-test.unicity.network +# https://ipfs.unicity.network +# +# Usage from the global-setup: +# docker compose -f tests/e2e/local-infra/docker-compose.yml up -d +# …run tests with E2E_LOCAL_RELAY_URL=ws://127.0.0.1:7777 … +# docker compose -f tests/e2e/local-infra/docker-compose.yml down -v +# +# Versions are pinned. Update via the parent global-setup so any change +# here is paired with a documented test-suite re-validation. +# ============================================================================= + +services: + # --------------------------------------------------------------------------- + # Local Nostr relay — ghcr.io/unicitynetwork/unicity-tokens-relay + # + # The Unicity org publishes a built nostr-rs-relay image. We pin to a + # specific SHA so unrelated relay updates don't silently change test + # behaviour. Bump this when validating against a newer relay release. + # --------------------------------------------------------------------------- + relay: + image: ghcr.io/unicitynetwork/unicity-tokens-relay:sha-1e1b544 + container_name: trader-e2e-relay + restart: unless-stopped + ports: + # Bind to ALL interfaces so HMA-spawned tenants (which run in + # separate Docker containers) can reach the relay via the host + # bridge gateway IP (typically 172.17.0.1 on Linux Docker). The + # global-setup detects this gateway at boot and passes it to the + # spawned tenants via UNICITY_NOSTR_RELAYS. If you need stricter + # isolation, change to "127.0.0.1:7777:8080" and run the test on + # the host network mode (not the HMA's default bridge). + # 7777 is arbitrary but unlikely to collide with developer tooling. + - "0.0.0.0:7777:8080" + volumes: + # Persist the SQLite event log between container restarts so we + # can post-mortem a failing test by `docker exec sqlite3`-ing + # into the volume. `down -v` wipes it; `down` (without -v) + # keeps it for inspection. + - relay-data:/usr/src/app/db + healthcheck: + # NIP-11 info doc on HTTP returns the relay metadata; if it + # responds 200 we know the WebSocket listener is also up + # (same handler). + test: ["CMD-SHELL", "wget -q -O - --header='Accept: application/nostr+json' http://127.0.0.1:8080 || exit 1"] + interval: 5s + timeout: 3s + retries: 12 + start_period: 5s + +volumes: + relay-data: diff --git a/test/e2e-live/local-infra/relay.ts b/test/e2e-live/local-infra/relay.ts new file mode 100644 index 0000000..6088a18 --- /dev/null +++ b/test/e2e-live/local-infra/relay.ts @@ -0,0 +1,187 @@ +/** + * Local Nostr relay lifecycle. + * + * Wraps `docker compose up/down` for tests/e2e/local-infra/docker-compose.yml. + * The relay container exposes 127.0.0.1:7777 — a fresh SQLite event log + * is created in the named volume on first boot; subsequent runs reuse + * the volume unless the caller explicitly requests `wipe: true` (which + * passes `-v` to `compose down` to drop persisted state). + * + * The compose file is the source of truth for the image pin; this helper + * is intentionally thin so version bumps don't drift between two places. + * + * @module tests/e2e/local-infra/relay + */ + +import { spawnSync } from 'node:child_process'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const COMPOSE_FILE = join(__dirname, 'docker-compose.yml'); + +/** + * URL the relay listens on (matches the docker-compose port mapping). + * + * Tests that are gated on E2E_LOCAL_INFRA=1 can read this directly, OR + * (preferred) read the SPHERE_NOSTR_RELAYS env var which the global- + * setup exports — that lets us swap the relay endpoint without + * touching test source. + */ +export const LOCAL_RELAY_URL = 'ws://127.0.0.1:7777'; + +/** + * Probe URL — same host, NIP-11 info doc on HTTP. + * + * The relay returns the same metadata over HTTP that the WebSocket + * upgrade serves to clients sending `Accept: application/nostr+json`. + * Cheap to poll during boot wait. + */ +const LOCAL_RELAY_HTTP = 'http://127.0.0.1:7777'; + +/** + * Discover the Docker bridge gateway IP (typically 172.17.0.1 on Linux + * Docker). Spawned tenants — running in their own Docker containers + * via the HMA — can NOT reach the host's loopback interface; they + * reach the host via this bridge IP. + * + * Returns the WebSocket URL HMA-spawned tenants should set as + * `UNICITY_NOSTR_RELAYS`. Falls back to `host.docker.internal` if + * `docker network inspect` fails (Docker Desktop on macOS/Windows + * resolves this name automatically; recent Linux Docker also supports + * it via the `--add-host=host.docker.internal:host-gateway` flag — + * which the HMA's docker-adapter would need to set if we go that + * route). + */ +export function getLocalRelayUrlForContainers(): string { + const out = spawnSync( + 'docker', + ['network', 'inspect', 'bridge', '--format', '{{(index .IPAM.Config 0).Gateway}}'], + { encoding: 'utf8', timeout: 5_000 }, + ); + if (out.status === 0) { + const gateway = out.stdout.trim(); + if (gateway.length > 0 && /^\d+\.\d+\.\d+\.\d+$/.test(gateway)) { + return `ws://${gateway}:7777`; + } + } + return 'ws://host.docker.internal:7777'; +} + +export interface RelayBootOptions { + /** + * Drop the persisted SQLite event log before booting (passes `-v` + * to `compose down`). Default false — preserves the log between + * runs so a developer can sqlite3-inspect a failing test. + */ + readonly wipe?: boolean; + /** Total deadline for the relay to come up. Default 60s. */ + readonly timeoutMs?: number; + /** Optional prefix for log lines so multi-stack output is greppable. */ + readonly logPrefix?: string; +} + +export interface RelayHandle { + /** WebSocket URL clients connect to. */ + readonly url: string; + /** Container name (matches compose `container_name`). */ + readonly containerName: string; + /** Stop + remove the relay container. Idempotent. */ + stop(opts?: { wipe?: boolean }): Promise; +} + +const log = (prefix: string, msg: string): void => { + // eslint-disable-next-line no-console + console.log(`${prefix}${msg}`); +}; + +/** + * Run `docker compose -f up -d relay` and wait for the NIP-11 + * info doc to respond 200. Returns a handle whose `stop()` runs + * `compose down`. + * + * Throws if Docker isn't available, the image can't be pulled, or the + * relay never becomes healthy within the timeout. We deliberately do + * not swallow these errors — silent boot failures would just produce + * a different, more confusing failure 30s deep into the test run. + */ +export async function bootLocalRelay(opts: RelayBootOptions = {}): Promise { + const prefix = opts.logPrefix ?? '[local-relay] '; + const timeoutMs = opts.timeoutMs ?? 60_000; + + // 1. Sanity check: docker CLI present. + const dockerVersion = spawnSync('docker', ['version', '--format', '{{.Server.Version}}'], { + encoding: 'utf8', + }); + if (dockerVersion.status !== 0) { + throw new Error( + `docker is not available (exit ${dockerVersion.status}): ${dockerVersion.stderr || dockerVersion.stdout}. ` + + 'Install Docker or unset E2E_LOCAL_INFRA to run against the public testnet.', + ); + } + + // 2. Optional: wipe persisted state. + if (opts.wipe) { + log(prefix, 'wiping previous relay-data volume…'); + spawnSync('docker', ['compose', '-f', COMPOSE_FILE, 'down', '-v'], { + encoding: 'utf8', + timeout: 30_000, + }); + } + + // 3. Boot. + log(prefix, `booting relay container from ${COMPOSE_FILE}…`); + const up = spawnSync('docker', ['compose', '-f', COMPOSE_FILE, 'up', '-d', 'relay'], { + encoding: 'utf8', + timeout: 120_000, + }); + if (up.status !== 0) { + throw new Error( + `docker compose up failed (exit ${up.status}):\nstdout: ${up.stdout}\nstderr: ${up.stderr}`, + ); + } + + // 4. Wait for NIP-11 info doc. + const deadline = Date.now() + timeoutMs; + let lastError: string | null = null; + while (Date.now() < deadline) { + try { + const resp = await fetch(LOCAL_RELAY_HTTP, { + headers: { Accept: 'application/nostr+json' }, + signal: AbortSignal.timeout(2_000), + }); + if (resp.ok) { + const info = (await resp.json()) as { name?: string; software?: string; version?: string }; + log(prefix, `relay healthy: ${info.software ?? '?'} ${info.version ?? '?'} on ${LOCAL_RELAY_URL}`); + return { + url: LOCAL_RELAY_URL, + containerName: 'trader-e2e-relay', + stop: async (stopOpts) => stopRelay(prefix, stopOpts?.wipe ?? false), + }; + } + lastError = `HTTP ${resp.status}`; + } catch (err) { + lastError = err instanceof Error ? err.message : String(err); + } + await new Promise((r) => setTimeout(r, 1_000)); + } + + // Boot failed — capture container logs before tearing down so the + // failure message is actionable. + const logs = spawnSync('docker', ['logs', 'trader-e2e-relay', '--tail', '50'], { + encoding: 'utf8', + timeout: 5_000, + }); + await stopRelay(prefix, /* wipe */ false); + throw new Error( + `local relay never became healthy within ${timeoutMs}ms (last error: ${lastError ?? 'unknown'}).\n` + + `--- container logs (last 50 lines) ---\n${logs.stdout || logs.stderr || '(empty)'}`, + ); +} + +async function stopRelay(prefix: string, wipe: boolean): Promise { + const args = ['compose', '-f', COMPOSE_FILE, 'down']; + if (wipe) args.push('-v'); + log(prefix, `stopping relay (wipe=${wipe})…`); + spawnSync('docker', args, { encoding: 'utf8', timeout: 30_000 }); +}