From b75c0476030955a0ac59d1265d6bce9f52da9755 Mon Sep 17 00:00:00 2001 From: ducnmm <165614309+ducnmm@users.noreply.github.com> Date: Thu, 2 Jul 2026 19:14:53 +0700 Subject: [PATCH 1/2] fix(relayer): recover Walrus register destroy_zero from stale WAL price (#351) Mainnet writes were failing 100% at the register_sponsor phase with an Enoki dry_run_failed -> MoveAbort in 0x2::coin::destroy_zero (ENonZero). Root cause is not Enoki: @mysten/walrus '#withWal' pre-funds an *exact* WAL payment computed from the client's cached systemState price, then asserts the coin is empty via coin::destroy_zero. When mainnet storage/write price drifts down between the cached read and execution, the contract deducts less WAL than we split off and the leftover trips destroy_zero. Verified on prod: on-chain price moved 71464->70922 within ~15 min, and a fresh relayer boot served 6 writes before the same client flipped to 100% destroy_zero at the next price move. Two gaps let this fail hard instead of self-healing: - The sidecar's auto-refresh only fired on isMoveAbortBalanceSplit ('balance' + 'split'); this message says destroy_zero, so refreshWalrusClient never ran. - The Rust worker swept it into the MoveAbort -> Permanent catch, so Apalis Dead-marked every write with no retry. Fix: - enoki.ts: add isMoveAbortWalDestroyZero detector. - walrus-upload.ts: refreshWalrusClient() on the destroy_zero abort so the retry rebuilds against the live price. - jobs.rs: classify the register destroy_zero abort as Transient (retryable), disjoint from the balance::split gas-budget path. - config.ts: drop WALRUS_CLIENT_MAX_AGE_MS default 30m -> 60s so the cached price tracks a live-drifting mainnet price (still env-overridable). Tests: new TS detector suite (7) + Rust classification test using the verbatim prod error. Full suites green (85 TS, 40 jobs). --- .../__tests__/walrus-wal-destroy-zero.test.ts | 62 +++++++++++++++++++ services/server/scripts/sidecar/config.ts | 9 ++- services/server/scripts/sidecar/enoki.ts | 16 +++++ .../scripts/sidecar/routes/walrus-upload.ts | 11 +++- services/server/src/jobs.rs | 44 +++++++++++++ 5 files changed, 140 insertions(+), 2 deletions(-) create mode 100644 services/server/scripts/__tests__/walrus-wal-destroy-zero.test.ts diff --git a/services/server/scripts/__tests__/walrus-wal-destroy-zero.test.ts b/services/server/scripts/__tests__/walrus-wal-destroy-zero.test.ts new file mode 100644 index 00000000..94f6e76b --- /dev/null +++ b/services/server/scripts/__tests__/walrus-wal-destroy-zero.test.ts @@ -0,0 +1,62 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { isMoveAbortBalanceSplit, isMoveAbortWalDestroyZero } from "../sidecar/enoki.js"; + +// Production format reference (issue #351): the Walrus register PTB pre-funds an +// exact WAL payment from the client's cached storage price, then asserts the +// coin is empty via `0x2::coin::destroy_zero`. When the on-chain price drops +// between the cached read and execution, the contract deducts less WAL and the +// leftover trips `destroy_zero` with ENonZero (abort code 0). Enoki surfaces it +// during its budget dry-run as a 400 dry_run_failed. + +const PROD_DESTROY_ZERO_ERROR = + 'Enoki API error (400): {"errors":[{"code":"dry_run_failed","message":"Dry run failed, ' + + "could not automatically determine a budget: MoveAbort(MoveLocation { module: ModuleId { " + + "address: 0000000000000000000000000000000000000000000000000000000000000002, name: " + + 'Identifier(\\"balance\\") }, function: 9, instruction: 8, function_name: ' + + 'Some(\\"destroy_zero\\") }, 0) in command 2"}]}'; + +test("matches the verbatim prod destroy_zero abort", () => { + assert.equal(isMoveAbortWalDestroyZero(PROD_DESTROY_ZERO_ERROR), true); +}); + +test("matches the compact MoveLocation shape", () => { + assert.equal( + isMoveAbortWalDestroyZero( + "MoveAbort(MoveLocation { module: coin, function_name: Some(\"destroy_zero\") }, 0) in command 9", + ), + true, + ); +}); + +test("is case-insensitive on both anchors", () => { + assert.equal(isMoveAbortWalDestroyZero("moveabort ... destroy_zero"), true); +}); + +test("bare destroy_zero without MoveAbort context is rejected", () => { + // Guards against unrelated log lines that merely mention the function. + assert.equal(isMoveAbortWalDestroyZero("calling coin::destroy_zero"), false); +}); + +test("balance::split abort does not match this detector", () => { + // The stale-price destroy_zero path must stay disjoint from the gas-budget + // balance::split path so the two handlers never double-fire. + const balanceSplit = + "MoveAbort(MoveLocation { module: balance, function_name: Some(\"split\") }, 2) in command 1"; + assert.equal(isMoveAbortWalDestroyZero(balanceSplit), false); + assert.equal(isMoveAbortBalanceSplit(balanceSplit), true); +}); + +test("the destroy_zero abort is NOT matched by the balance-split detector", () => { + // The prod message contains the `balance` module name but no `split`, so the + // existing isMoveAbortBalanceSplit stays false — this is exactly the gap the + // new detector closes. + assert.equal(isMoveAbortBalanceSplit(PROD_DESTROY_ZERO_ERROR), false); +}); + +test("unrelated errors do not match", () => { + assert.equal(isMoveAbortWalDestroyZero("connection refused"), false); + assert.equal(isMoveAbortWalDestroyZero("HTTP 500 from upload relay"), false); + assert.equal(isMoveAbortWalDestroyZero(""), false); +}); diff --git a/services/server/scripts/sidecar/config.ts b/services/server/scripts/sidecar/config.ts index 26e3d796..7a5a5c7c 100644 --- a/services/server/scripts/sidecar/config.ts +++ b/services/server/scripts/sidecar/config.ts @@ -123,9 +123,16 @@ export function clampWalrusEpochs(rawEpochs: unknown): number { return Math.min(Math.floor(parsed), MAX_WALRUS_EPOCHS); } +// The cached WalrusClient snapshots `systemState` (storage/write price, +// committee) at creation and reuses it for its whole lifetime. The register +// PTB pre-funds an *exact* WAL payment from that cached price, so when mainnet +// price drifts the split leaves change and `coin::destroy_zero` aborts. Keep +// the client young so the cached price tracks the live price closely; the +// register-path `destroy_zero` handler also recreates it on demand. Was 30 min, +// which let the cached price fall far behind a live-drifting mainnet price. export const WALRUS_CLIENT_MAX_AGE_MS = (() => { const parsed = Number.parseInt(process.env.WALRUS_CLIENT_MAX_AGE_MS || "", 10); - return Number.isFinite(parsed) && parsed > 0 ? parsed : 30 * 60 * 1000; + return Number.isFinite(parsed) && parsed > 0 ? parsed : 60 * 1000; })(); // Mirror of services/server/src/alerts.rs SIDECAR_WALRUS_DEP_VERSION. diff --git a/services/server/scripts/sidecar/enoki.ts b/services/server/scripts/sidecar/enoki.ts index 0d8380c0..0c44ea39 100644 --- a/services/server/scripts/sidecar/enoki.ts +++ b/services/server/scripts/sidecar/enoki.ts @@ -53,6 +53,22 @@ export function isMoveAbortBalanceSplit(message: string): boolean { return /moveabort/i.test(message) && /balance.*split|split.*balance/i.test(message); } +/** + * Detect the `0x2::coin::destroy_zero` abort (ENonZero, abort code 0) that the + * Walrus register PTB raises when the WAL payment coin still holds a non-zero + * remainder. The `@mysten/walrus` `#withWal` helper pre-funds an *exact* WAL + * amount (`storageUnits × price × epochs`) computed from the client's cached + * `systemState`, then asserts the coin is empty via `destroy_zero`. When the + * on-chain storage/write price drops between the cached read and execution, the + * contract deducts less WAL than we split off, leaving change that trips + * `destroy_zero`. It is not input-specific — refreshing the Walrus client so the + * next attempt re-reads the live price clears it, so callers treat it as + * transient rather than a permanent MoveAbort. + */ +export function isMoveAbortWalDestroyZero(message: string): boolean { + return /moveabort/i.test(message) && /destroy_zero/i.test(message); +} + export async function callEnoki(path: string, payload: unknown): Promise { if (!ENOKI_API_KEY) { throw new Error("ENOKI_API_KEY is not configured"); diff --git a/services/server/scripts/sidecar/routes/walrus-upload.ts b/services/server/scripts/sidecar/routes/walrus-upload.ts index 0c03be97..837dae77 100644 --- a/services/server/scripts/sidecar/routes/walrus-upload.ts +++ b/services/server/scripts/sidecar/routes/walrus-upload.ts @@ -45,7 +45,7 @@ import { sleep, truncateForLog, } from "../util.js"; -import { isMoveAbortBalanceSplit } from "../enoki.js"; +import { isMoveAbortBalanceSplit, isMoveAbortWalDestroyZero } from "../enoki.js"; import { patchGasCoinIntents, submitRebuildableWalletTransaction, @@ -362,6 +362,15 @@ export function registerWalrusUploadRoute(app: Express): void { if (phase === "register_sponsor" && isMoveAbortBalanceSplit(message)) { refreshWalrusClient("register_sponsor_balance_split"); } + // `coin::destroy_zero` (ENonZero) means the WAL payment coin the + // register PTB pre-funded from the cached storage price wasn't fully + // consumed on-chain — the live price dropped between the cached read + // and execution. Recreate the client so the retry re-reads the + // current price and splits the exact amount. The Rust worker + // classifies this abort Transient so Apalis actually retries. + if (isMoveAbortWalDestroyZero(message)) { + refreshWalrusClient("walrus_wal_payment_destroy_zero"); + } if (isWalrusPackageVersionMismatch(message)) { // EWrongVersion is phase-independent: can fire from register / upload / certify // any time the Walrus system package gets upgraded on-chain after this sidecar diff --git a/services/server/src/jobs.rs b/services/server/src/jobs.rs index 7bb2eab0..d7c42889 100644 --- a/services/server/src/jobs.rs +++ b/services/server/src/jobs.rs @@ -1511,6 +1511,9 @@ async fn maybe_alert_walrus_upload_exhausted( /// /// Mapping rules (enforced at the point of error origination): /// - `MoveAbort(_)` → `Permanent` (deterministic Move-level failure) +/// - Walrus register `0x2::coin::destroy_zero` ENonZero → `Transient` (WAL +/// payment over-funded from a stale cached price; the sidecar refreshes the +/// client so the retry re-reads the live price and splits the exact amount) /// - Enoki dry-run `0x2::balance::split` ENotEnough → `GasPoolExhausted` /// (abort: pool SUI gas coins are fragmented/insufficient; retrying rotates /// to the next starved wallet — needs ops gas consolidation/top-up) @@ -1602,6 +1605,20 @@ impl WalletJobError { && lower.contains("split") } + /// True if `msg` is the Walrus register PTB's `0x2::coin::destroy_zero` + /// abort (ENonZero). The `@mysten/walrus` `#withWal` helper splits an exact + /// WAL payment from the client's cached storage/write price and asserts the + /// coin is empty afterwards; when mainnet price drops between the cached read + /// and execution the contract deducts less WAL, leaving change that trips + /// `destroy_zero`. Recoverable: retrying against a client that re-read the + /// live price splits the correct amount, so this is Transient — never a + /// Permanent MoveAbort. + pub fn is_walrus_wal_payment_price_abort(msg: &str) -> bool { + let lower = msg.to_ascii_lowercase(); + (lower.contains("moveabort") || lower.contains("move abort")) + && lower.contains("destroy_zero") + } + /// Heuristic classification from the sidecar's error string. The sidecar /// surfaces Sui execution errors verbatim (Move abort codes, lock errors). /// Until the sidecar emits structured error codes, we match on substrings. @@ -1646,6 +1663,17 @@ impl WalletJobError { { return WalletJobError::Transient(msg.to_string()); } + // Walrus register PTB `0x2::coin::destroy_zero` abort (ENonZero). The + // `@mysten/walrus` `#withWal` helper pre-funds an exact WAL payment from + // the client's cached storage price, then asserts the coin is empty. When + // the on-chain price drops between the cached read and execution the + // contract deducts less WAL, leaving change that trips `destroy_zero`. + // Not input-specific: the sidecar recreates the client on this error, so + // classifying Transient lets Apalis retry against the refreshed (live) + // price instead of Dead-marking a job the next attempt will succeed on. + if Self::is_walrus_wal_payment_price_abort(msg) { + return WalletJobError::Transient(msg.to_string()); + } // Sui owned-object lock / equivocation. The referenced object+version // is locked to a competing transaction and stays locked until the lock // clears (typically the next epoch boundary), so retrying within this @@ -2395,6 +2423,22 @@ different transaction: TransactionDigest(8bjFgRyXRRYwrzQapgEjpHnGhdfNDY7d6xA82Bt )); } + #[test] + fn walrus_register_destroy_zero_is_transient() { + // Verbatim prod error (issue #351): the register PTB over-funds the WAL + // payment from a stale cached price and `coin::destroy_zero` aborts with + // ENonZero. Must be Transient (retry re-reads the live price), NOT swept + // into the MoveAbort→Permanent catch that would Dead-mark every write. + let destroy_zero = "walrus upload failed: Enoki API error (400): {\"errors\":[{\"code\":\"dry_run_failed\",\"message\":\"Dry run failed, could not automatically determine a budget: MoveAbort(MoveLocation { module: ModuleId { address: 0000000000000000000000000000000000000000000000000000000000000002, name: Identifier(\\\"balance\\\") }, function: 9, instruction: 8, function_name: Some(\\\"destroy_zero\\\") }, 0) in command 2\"}]}"; + assert!(WalletJobError::is_walrus_wal_payment_price_abort( + destroy_zero + )); + let classified = WalletJobError::classify_sidecar_error(destroy_zero); + assert!(matches!(classified, WalletJobError::Transient(_))); + assert!(!classified.is_permanent()); + assert!(!classified.aborts_retries()); + } + #[test] fn parse_locked_object_info_from_prod_error() { let info = parse_locked_object_info(PROD_OBJECT_LOCK_ERROR); From 542071f357497b46c268321c6121afcd77d4a7cd Mon Sep 17 00:00:00 2001 From: ducnmm <165614309+ducnmm@users.noreply.github.com> Date: Fri, 3 Jul 2026 09:06:15 +0700 Subject: [PATCH 2/2] fix(relayer): honor SUI_RPC_URL env to move off degraded public fullnode The sidecar hardcoded getJsonRpcFullnodeUrl(mainnet) and ignored the SUI_RPC_URL env, so it could not be pointed at a dedicated RPC when the public fullnode pool degrades. Observed today: the public pool served stale reads (a blob certified at 00:49:14 read back as 'does not exist' at 00:49:49, 35s later), failing uploads at get_blob/certify/metadata. Honor an explicit SUI_RPC_URL override so ops can switch to a healthy RPC via env; fall back to the network default. --- services/server/scripts/sidecar/config.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/services/server/scripts/sidecar/config.ts b/services/server/scripts/sidecar/config.ts index 7a5a5c7c..2fcea868 100644 --- a/services/server/scripts/sidecar/config.ts +++ b/services/server/scripts/sidecar/config.ts @@ -30,7 +30,11 @@ export function parsePositiveIntEnv( // ============================================================ export const SUI_NETWORK = (process.env.SUI_NETWORK || "mainnet") as "mainnet" | "testnet"; -export const SUI_RPC_URL = getJsonRpcFullnodeUrl(SUI_NETWORK); +// Honor an explicit SUI_RPC_URL override (e.g. a dedicated / premium RPC) so we +// can move off the public fullnode when its pool degrades — the public endpoint +// has served stale reads (a certified blob read back as "does not exist"), +// which fails uploads at get_blob / certify. Falls back to the network default. +export const SUI_RPC_URL = process.env.SUI_RPC_URL?.trim() || getJsonRpcFullnodeUrl(SUI_NETWORK); export const SUI_TYPE = "0x2::sui::SUI"; // ============================================================