fix(relayer): recover Walrus register destroy_zero from stale WAL price (#351)

services/server/scripts/__tests__/walrus-wal-destroy-zero.test.ts

@@ -0,0 +1,62 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { isMoveAbortBalanceSplit, isMoveAbortWalDestroyZero } from "../sidecar/enoki.js";
+
+// Production format reference (issue #351): the Walrus register PTB pre-funds an
+// exact WAL payment from the client's cached storage price, then asserts the
+// coin is empty via `0x2::coin::destroy_zero`. When the on-chain price drops
+// between the cached read and execution, the contract deducts less WAL and the
+// leftover trips `destroy_zero` with ENonZero (abort code 0). Enoki surfaces it
+// during its budget dry-run as a 400 dry_run_failed.
+
+const PROD_DESTROY_ZERO_ERROR =
+    'Enoki API error (400): {"errors":[{"code":"dry_run_failed","message":"Dry run failed, ' +
+    "could not automatically determine a budget: MoveAbort(MoveLocation { module: ModuleId { " +
+    "address: 0000000000000000000000000000000000000000000000000000000000000002, name: " +
+    'Identifier(\\"balance\\") }, function: 9, instruction: 8, function_name: ' +
+    'Some(\\"destroy_zero\\") }, 0) in command 2"}]}';
+
+test("matches the verbatim prod destroy_zero abort", () => {
+    assert.equal(isMoveAbortWalDestroyZero(PROD_DESTROY_ZERO_ERROR), true);
+});
+
+test("matches the compact MoveLocation shape", () => {
+    assert.equal(
+        isMoveAbortWalDestroyZero(
+            "MoveAbort(MoveLocation { module: coin, function_name: Some(\"destroy_zero\") }, 0) in command 9",
+        ),
+        true,
+    );
+});
+
+test("is case-insensitive on both anchors", () => {
+    assert.equal(isMoveAbortWalDestroyZero("moveabort ... destroy_zero"), true);
+});
+
+test("bare destroy_zero without MoveAbort context is rejected", () => {
+    // Guards against unrelated log lines that merely mention the function.
+    assert.equal(isMoveAbortWalDestroyZero("calling coin::destroy_zero"), false);
+});
+
+test("balance::split abort does not match this detector", () => {
+    // The stale-price destroy_zero path must stay disjoint from the gas-budget
+    // balance::split path so the two handlers never double-fire.
+    const balanceSplit =
+        "MoveAbort(MoveLocation { module: balance, function_name: Some(\"split\") }, 2) in command 1";
+    assert.equal(isMoveAbortWalDestroyZero(balanceSplit), false);
+    assert.equal(isMoveAbortBalanceSplit(balanceSplit), true);
+});
+
+test("the destroy_zero abort is NOT matched by the balance-split detector", () => {
+    // The prod message contains the `balance` module name but no `split`, so the
+    // existing isMoveAbortBalanceSplit stays false — this is exactly the gap the
+    // new detector closes.
+    assert.equal(isMoveAbortBalanceSplit(PROD_DESTROY_ZERO_ERROR), false);
+});
+
+test("unrelated errors do not match", () => {
+    assert.equal(isMoveAbortWalDestroyZero("connection refused"), false);
+    assert.equal(isMoveAbortWalDestroyZero("HTTP 500 from upload relay"), false);
+    assert.equal(isMoveAbortWalDestroyZero(""), false);
+});

services/server/scripts/sidecar/config.ts

@@ -30,7 +30,11 @@ export function parsePositiveIntEnv(
 // ============================================================
 
 export const SUI_NETWORK = (process.env.SUI_NETWORK || "mainnet") as "mainnet" | "testnet";
-export const SUI_RPC_URL = getJsonRpcFullnodeUrl(SUI_NETWORK);
+// Honor an explicit SUI_RPC_URL override (e.g. a dedicated / premium RPC) so we
+// can move off the public fullnode when its pool degrades — the public endpoint
+// has served stale reads (a certified blob read back as "does not exist"),
+// which fails uploads at get_blob / certify. Falls back to the network default.
+export const SUI_RPC_URL = process.env.SUI_RPC_URL?.trim() || getJsonRpcFullnodeUrl(SUI_NETWORK);
 export const SUI_TYPE = "0x2::sui::SUI";
 
 // ============================================================
@@ -123,9 +127,16 @@ export function clampWalrusEpochs(rawEpochs: unknown): number {
     return Math.min(Math.floor(parsed), MAX_WALRUS_EPOCHS);
 }
 
+// The cached WalrusClient snapshots `systemState` (storage/write price,
+// committee) at creation and reuses it for its whole lifetime. The register
+// PTB pre-funds an *exact* WAL payment from that cached price, so when mainnet
+// price drifts the split leaves change and `coin::destroy_zero` aborts. Keep
+// the client young so the cached price tracks the live price closely; the
+// register-path `destroy_zero` handler also recreates it on demand. Was 30 min,
+// which let the cached price fall far behind a live-drifting mainnet price.
 export const WALRUS_CLIENT_MAX_AGE_MS = (() => {
     const parsed = Number.parseInt(process.env.WALRUS_CLIENT_MAX_AGE_MS || "", 10);
-    return Number.isFinite(parsed) && parsed > 0 ? parsed : 30 * 60 * 1000;
+    return Number.isFinite(parsed) && parsed > 0 ? parsed : 60 * 1000;
 })();
 
 // Mirror of services/server/src/alerts.rs SIDECAR_WALRUS_DEP_VERSION.

services/server/scripts/sidecar/enoki.ts

@@ -53,6 +53,22 @@ export function isMoveAbortBalanceSplit(message: string): boolean {
     return /moveabort/i.test(message) && /balance.*split|split.*balance/i.test(message);
 }
 
+/**
+ * Detect the `0x2::coin::destroy_zero` abort (ENonZero, abort code 0) that the
+ * Walrus register PTB raises when the WAL payment coin still holds a non-zero
+ * remainder. The `@mysten/walrus` `#withWal` helper pre-funds an *exact* WAL
+ * amount (`storageUnits × price × epochs`) computed from the client's cached
+ * `systemState`, then asserts the coin is empty via `destroy_zero`. When the
+ * on-chain storage/write price drops between the cached read and execution, the
+ * contract deducts less WAL than we split off, leaving change that trips
+ * `destroy_zero`. It is not input-specific — refreshing the Walrus client so the
+ * next attempt re-reads the live price clears it, so callers treat it as
+ * transient rather than a permanent MoveAbort.
+ */
+export function isMoveAbortWalDestroyZero(message: string): boolean {
+    return /moveabort/i.test(message) && /destroy_zero/i.test(message);
+}
+
 export async function callEnoki<T>(path: string, payload: unknown): Promise<T> {
     if (!ENOKI_API_KEY) {
         throw new Error("ENOKI_API_KEY is not configured");

services/server/scripts/sidecar/routes/walrus-upload.ts

@@ -45,7 +45,7 @@ import {
     sleep,
     truncateForLog,
 } from "../util.js";
-import { isMoveAbortBalanceSplit } from "../enoki.js";
+import { isMoveAbortBalanceSplit, isMoveAbortWalDestroyZero } from "../enoki.js";
 import {
     patchGasCoinIntents,
     submitRebuildableWalletTransaction,
@@ -362,6 +362,15 @@ export function registerWalrusUploadRoute(app: Express): void {
             if (phase === "register_sponsor" && isMoveAbortBalanceSplit(message)) {
                 refreshWalrusClient("register_sponsor_balance_split");
             }
+            // `coin::destroy_zero` (ENonZero) means the WAL payment coin the
+            // register PTB pre-funded from the cached storage price wasn't fully
+            // consumed on-chain — the live price dropped between the cached read
+            // and execution. Recreate the client so the retry re-reads the
+            // current price and splits the exact amount. The Rust worker
+            // classifies this abort Transient so Apalis actually retries.
+            if (isMoveAbortWalDestroyZero(message)) {
+                refreshWalrusClient("walrus_wal_payment_destroy_zero");
+            }
             if (isWalrusPackageVersionMismatch(message)) {
                 // EWrongVersion is phase-independent: can fire from register / upload / certify
                 // any time the Walrus system package gets upgraded on-chain after this sidecar

services/server/src/jobs.rs

@@ -1511,6 +1511,9 @@ async fn maybe_alert_walrus_upload_exhausted(
 ///
 /// Mapping rules (enforced at the point of error origination):
 /// - `MoveAbort(_)` → `Permanent` (deterministic Move-level failure)
+/// - Walrus register `0x2::coin::destroy_zero` ENonZero → `Transient` (WAL
+///   payment over-funded from a stale cached price; the sidecar refreshes the
+///   client so the retry re-reads the live price and splits the exact amount)
 /// - Enoki dry-run `0x2::balance::split` ENotEnough → `GasPoolExhausted`
 ///   (abort: pool SUI gas coins are fragmented/insufficient; retrying rotates
 ///   to the next starved wallet — needs ops gas consolidation/top-up)
@@ -1602,6 +1605,20 @@ impl WalletJobError {
             && lower.contains("split")
     }
 
+    /// True if `msg` is the Walrus register PTB's `0x2::coin::destroy_zero`
+    /// abort (ENonZero). The `@mysten/walrus` `#withWal` helper splits an exact
+    /// WAL payment from the client's cached storage/write price and asserts the
+    /// coin is empty afterwards; when mainnet price drops between the cached read
+    /// and execution the contract deducts less WAL, leaving change that trips
+    /// `destroy_zero`. Recoverable: retrying against a client that re-read the
+    /// live price splits the correct amount, so this is Transient — never a
+    /// Permanent MoveAbort.
+    pub fn is_walrus_wal_payment_price_abort(msg: &str) -> bool {
+        let lower = msg.to_ascii_lowercase();
+        (lower.contains("moveabort") || lower.contains("move abort"))
+            && lower.contains("destroy_zero")
+    }
+
     /// Heuristic classification from the sidecar's error string. The sidecar
     /// surfaces Sui execution errors verbatim (Move abort codes, lock errors).
     /// Until the sidecar emits structured error codes, we match on substrings.
@@ -1646,6 +1663,17 @@ impl WalletJobError {
         {
             return WalletJobError::Transient(msg.to_string());
         }
+        // Walrus register PTB `0x2::coin::destroy_zero` abort (ENonZero). The
+        // `@mysten/walrus` `#withWal` helper pre-funds an exact WAL payment from
+        // the client's cached storage price, then asserts the coin is empty. When
+        // the on-chain price drops between the cached read and execution the
+        // contract deducts less WAL, leaving change that trips `destroy_zero`.
+        // Not input-specific: the sidecar recreates the client on this error, so
+        // classifying Transient lets Apalis retry against the refreshed (live)
+        // price instead of Dead-marking a job the next attempt will succeed on.
+        if Self::is_walrus_wal_payment_price_abort(msg) {
+            return WalletJobError::Transient(msg.to_string());
+        }
         // Sui owned-object lock / equivocation. The referenced object+version
         // is locked to a competing transaction and stays locked until the lock
         // clears (typically the next epoch boundary), so retrying within this
@@ -2395,6 +2423,22 @@ different transaction: TransactionDigest(8bjFgRyXRRYwrzQapgEjpHnGhdfNDY7d6xA82Bt
         ));
     }
 
+    #[test]
+    fn walrus_register_destroy_zero_is_transient() {
+        // Verbatim prod error (issue #351): the register PTB over-funds the WAL
+        // payment from a stale cached price and `coin::destroy_zero` aborts with
+        // ENonZero. Must be Transient (retry re-reads the live price), NOT swept
+        // into the MoveAbort→Permanent catch that would Dead-mark every write.
+        let destroy_zero = "walrus upload failed: Enoki API error (400): {\"errors\":[{\"code\":\"dry_run_failed\",\"message\":\"Dry run failed, could not automatically determine a budget: MoveAbort(MoveLocation { module: ModuleId { address: 0000000000000000000000000000000000000000000000000000000000000002, name: Identifier(\\\"balance\\\") }, function: 9, instruction: 8, function_name: Some(\\\"destroy_zero\\\") }, 0) in command 2\"}]}";
+        assert!(WalletJobError::is_walrus_wal_payment_price_abort(
+            destroy_zero
+        ));
+        let classified = WalletJobError::classify_sidecar_error(destroy_zero);
+        assert!(matches!(classified, WalletJobError::Transient(_)));
+        assert!(!classified.is_permanent());
+        assert!(!classified.aborts_retries());
+    }
+
     #[test]
     fn parse_locked_object_info_from_prod_error() {
         let info = parse_locked_object_info(PROD_OBJECT_LOCK_ERROR);