From 8e96f5c45cae18c86e079b3907f539c755b3ef19 Mon Sep 17 00:00:00 2001 From: jasong-03 Date: Thu, 4 Jun 2026 11:48:57 +0700 Subject: [PATCH 01/78] feat(observability): add OpenObserve self-hosted observability PoC Add a docker-compose stack (OpenObserve + OpenTelemetry Collector) and collector config that scrapes the relayer Prometheus /metrics, tails structured JSON container logs, and accepts OTLP for future traces, exporting all signals to OpenObserve. Includes a README with run instructions, an API-health dashboard query set, alert definitions, rollout notes, and the known gaps (no trace instrumentation yet, no job-queue metric). --- services/server/observability/README.md | 116 ++++++++++++++++++ .../docker-compose.observability.yml | 54 ++++++++ .../observability/otel-collector-config.yaml | 85 +++++++++++++ 3 files changed, 255 insertions(+) create mode 100644 services/server/observability/README.md create mode 100644 services/server/observability/docker-compose.observability.yml create mode 100644 services/server/observability/otel-collector-config.yaml diff --git a/services/server/observability/README.md b/services/server/observability/README.md new file mode 100644 index 00000000..38745441 --- /dev/null +++ b/services/server/observability/README.md @@ -0,0 +1,116 @@ +# MemWal observability PoC — OpenObserve (WALM-81) + +Self-hosted observability stack: **OpenObserve** + an **OpenTelemetry Collector** +that scrapes the relayer Prometheus `/metrics`, tails structured JSON container +logs, and accepts OTLP (ready for future traces). Follows the OpenObserve +recommendation from WALM-79. + +> Status: **PoC**. Verified locally end-to-end (collector → OpenObserve ingest +> + query). Designed to be pointed at a staging/self-hosted environment; the +> production rollout notes and known gaps are at the bottom. + +## Contents + +| File | Purpose | +|------|---------| +| `docker-compose.observability.yml` | OpenObserve + OTel Collector services | +| `otel-collector-config.yaml` | metrics scrape + logs tail + OTLP in → OpenObserve | + +## Run + +```bash +cd services/server/observability + +# Credentials for the OpenObserve root user (change these). +export O2_ROOT_EMAIL=root@memwal.local +export O2_ROOT_PASSWORD='Complexpass#123' + +# OTLP/HTTP Basic auth header value = base64("email:password"). +export O2_AUTH=$(printf '%s' "$O2_ROOT_EMAIL:$O2_ROOT_PASSWORD" | base64) + +# Where the relayer exposes /metrics (defaults to host.docker.internal:8000). +# Point this at your relayer or the main compose stack's published port. +export RELAYER_METRICS_TARGET=host.docker.internal:8000 + +docker compose -f docker-compose.observability.yml up -d +``` + +OpenObserve UI: (log in with `O2_ROOT_EMAIL` / `O2_ROOT_PASSWORD`). + +Tear down: `docker compose -f docker-compose.observability.yml down` (add `-v` to wipe data). + +## Ingestion + +| Signal | Source | Path | +|--------|--------|------| +| **Metrics** | relayer Prometheus `/metrics` (`memwal_*`) | collector `prometheus` receiver → OTLP → OpenObserve | +| **Logs** | relayer/sidecar stdout with `LOG_FORMAT=json` | collector `file_log` (Docker json-file) → OpenObserve | +| **Traces** | none yet (see Gaps) | collector `otlp` receiver wired and ready | + +Run the relayer and sidecar with `LOG_FORMAT=json` so the collector parses the +inner application JSON instead of opaque log lines. + +### Quick ingestion smoke test (no relayer needed) + +```bash +curl -X POST http://localhost:4318/v1/logs -H 'Content-Type: application/json' -d '{ + "resourceLogs":[{"scopeLogs":[{"logRecords":[{ + "timeUnixNano":"'$(date +%s)'000000000","severityText":"INFO", + "body":{"stringValue":"poc test"}}]}]}]}' +# then query it back: +curl -u "$O2_ROOT_EMAIL:$O2_ROOT_PASSWORD" -X POST \ + "http://localhost:5080/api/default/_search" -H 'Content-Type: application/json' \ + -d '{"query":{"sql":"SELECT * FROM \"default\" ORDER BY _timestamp DESC","start_time":'$(( ($(date +%s)-600)*1000000 ))',"end_time":'$(( ($(date +%s)+60)*1000000 ))',"size":5}}' +``` + +## Dashboard — API health + +OpenObserve runs PromQL over ingested Prometheus metrics. Create a dashboard +(Dashboards → New) with these panels. Metric labels are +`{method, route, status}` for HTTP metrics. + +| Panel | PromQL | +|-------|--------| +| Request rate (req/s) | `sum(rate(memwal_http_requests_total[5m]))` | +| Request rate by route | `sum by (route) (rate(memwal_http_requests_total[5m]))` | +| Error rate (5xx %) | `sum(rate(memwal_http_requests_total{status=~"5.."}[5m])) / sum(rate(memwal_http_requests_total[5m]))` | +| p95 latency (s) | `histogram_quantile(0.95, sum by (le) (rate(memwal_http_request_duration_seconds_bucket[5m])))` | +| In-flight requests | `memwal_http_requests_in_flight` | +| Dependency failures | `sum by (service) (rate(memwal_external_request_duration_seconds_count{status!="200"}[5m]))` | +| Sidecar failures | `sum by (operation, reason) (rate(memwal_sidecar_failures_total[5m]))` | +| DB query p95 (s) | `histogram_quantile(0.95, sum by (le, operation) (rate(memwal_db_query_duration_seconds_bucket[5m])))` | +| DB pool by state | `memwal_db_pool_connections` | + +## Alerts + +Create under Alerts. Suggested PoC thresholds (tune per environment): + +| Alert | Condition | +|-------|-----------| +| 5xx error-rate spike | `sum(rate(memwal_http_requests_total{status=~"5.."}[5m])) / sum(rate(memwal_http_requests_total[5m])) > 0.05` for 5m | +| p95 latency breach | `histogram_quantile(0.95, sum by (le) (rate(memwal_http_request_duration_seconds_bucket[5m]))) > 2` for 10m | +| Sidecar / Walrus failure | `sum(rate(memwal_sidecar_failures_total[5m])) > 0` | +| No telemetry received | `absent(memwal_http_requests_total)` for 5m | + +## Production / staging rollout notes + +- **Metrics**: keep the relayer `/metrics` endpoint reachable by the collector + (private network). Set `RELAYER_METRICS_TARGET` to the staging relayer. +- **Logs on Railway**: the `file_log` receiver tails Docker json-file logs, which + works for self-hosted / docker-compose. On Railway, forward logs via a Railway + log drain (HTTP) into OpenObserve's `/api///_json` ingest, or run + the collector as a sidecar with access to the log stream. +- Set `LOG_FORMAT=json` on the relayer and sidecar. +- Replace the root credentials and pin image tags (this PoC uses `:latest`). + +## Known gaps (follow-up) + +1. **Traces**: the Rust relayer has no OpenTelemetry instrumentation, so no + spans are emitted. The collector `traces` pipeline is wired; adding the + `tracing-opentelemetry` layer + OTLP exporter to the relayer is the next step. +2. **Job-queue health**: there is no apalis/job-queue metric exposed today, so a + queue-depth/in-flight dashboard isn't possible without adding one. +3. **External dependency status labels**: dependency failures are derived from + `memwal_external_request_duration_seconds{status}` and + `memwal_sidecar_failures_total`; per-dependency (Walrus vs OpenAI vs Sui) + breakdown depends on the `service` label values the relayer emits. diff --git a/services/server/observability/docker-compose.observability.yml b/services/server/observability/docker-compose.observability.yml new file mode 100644 index 00000000..13487f2f --- /dev/null +++ b/services/server/observability/docker-compose.observability.yml @@ -0,0 +1,54 @@ +# Self-hosted observability PoC for MemWal (WALM-81). +# +# Brings up OpenObserve + an OpenTelemetry Collector that scrapes the relayer +# /metrics endpoint, tails structured JSON container logs, and accepts OTLP. +# Runs independently of the main services/server/docker-compose.yml. +# +# Usage: +# export O2_AUTH=$(printf '%s' "$O2_ROOT_EMAIL:$O2_ROOT_PASSWORD" | base64) +# docker compose -f docker-compose.observability.yml up -d +# OpenObserve UI: http://localhost:5080 (login with O2_ROOT_EMAIL/PASSWORD) + +name: memwal-observability + +services: + openobserve: + image: public.ecr.aws/zinclabs/openobserve:latest + container_name: memwal-openobserve + restart: unless-stopped + environment: + ZO_ROOT_USER_EMAIL: ${O2_ROOT_EMAIL:-root@memwal.local} + ZO_ROOT_USER_PASSWORD: ${O2_ROOT_PASSWORD:-Complexpass#123} + ZO_DATA_DIR: /data + ports: + - "5080:5080" # HTTP UI + OTLP/HTTP ingest + - "5081:5081" # OTLP/gRPC ingest + volumes: + - openobserve_data:/data + + otel-collector: + image: otel/opentelemetry-collector-contrib:latest + container_name: memwal-otel-collector + restart: unless-stopped + depends_on: + - openobserve + command: ["--config=/etc/otel/config.yaml"] + environment: + O2_ORG: ${O2_ORG:-default} + # base64("email:password") — see README. Required. + O2_AUTH: ${O2_AUTH:?set O2_AUTH=base64(email:password) — see README} + # Where the relayer exposes /metrics. host.docker.internal reaches a + # relayer on the host or the main compose's published :8000. + RELAYER_METRICS_TARGET: ${RELAYER_METRICS_TARGET:-host.docker.internal:8000} + extra_hosts: + - "host.docker.internal:host-gateway" # Linux compatibility + volumes: + - ./otel-collector-config.yaml:/etc/otel/config.yaml:ro + # Read-only access to container logs for the filelog receiver. + - /var/lib/docker/containers:/var/lib/docker/containers:ro + ports: + - "4317:4317" # OTLP/gRPC in + - "4318:4318" # OTLP/HTTP in + +volumes: + openobserve_data: diff --git a/services/server/observability/otel-collector-config.yaml b/services/server/observability/otel-collector-config.yaml new file mode 100644 index 00000000..05a92ba1 --- /dev/null +++ b/services/server/observability/otel-collector-config.yaml @@ -0,0 +1,85 @@ +# OpenTelemetry Collector config for the MemWal observability PoC (WALM-81). +# +# Pipelines: +# metrics — scrape the relayer Prometheus /metrics endpoint (+ accept OTLP) +# logs — tail structured JSON container logs (relayer/sidecar run with +# LOG_FORMAT=json) (+ accept OTLP) +# traces — accept OTLP only. No service currently emits traces (the Rust +# relayer has no OpenTelemetry instrumentation yet); this pipeline +# is wired so traces flow the moment instrumentation is added. +# +# All signals are exported to OpenObserve over OTLP/HTTP. The exporter endpoint +# must NOT end in a slash — the collector appends /v1/{metrics,logs,traces}. + +receivers: + # Pull the relayer's Prometheus metrics. RELAYER_METRICS_TARGET defaults to + # the host gateway so it can scrape a relayer running on the host or the + # main docker-compose stack's published :8000. + prometheus: + config: + scrape_configs: + - job_name: memwal-relayer + scrape_interval: 15s + metrics_path: /metrics + static_configs: + - targets: ["${env:RELAYER_METRICS_TARGET}"] + + # OTLP in — future app traces/metrics/logs. + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + + # Tail Docker json-file container logs. Inner payload is parsed only when it + # looks like JSON (LOG_FORMAT=json), so plain-text containers don't error. + file_log: + include: ["/var/lib/docker/containers/*/*-json.log"] + include_file_path: true + operators: + - type: json_parser # docker json-file wrapper: {log, stream, time} + timestamp: + parse_from: attributes.time + layout: "%Y-%m-%dT%H:%M:%S.%LZ" + - type: json_parser # inner application JSON (tracing_subscriber .json()) + parse_from: attributes.log + if: 'attributes.log matches "^\\s*\\{"' + severity: + parse_from: attributes.level + +processors: + batch: + timeout: 5s + send_batch_size: 1024 + resource_detection: + detectors: [env, system] + timeout: 5s + override: false + +exporters: + otlp_http/openobserve: + endpoint: "http://openobserve:5080/api/${env:O2_ORG}" + headers: + Authorization: "Basic ${env:O2_AUTH}" + stream-name: default + retry_on_failure: + enabled: true + +service: + telemetry: + logs: + level: info + pipelines: + metrics: + receivers: [prometheus, otlp] + processors: [resource_detection, batch] + exporters: [otlp_http/openobserve] + logs: + receivers: [file_log, otlp] + processors: [resource_detection, batch] + exporters: [otlp_http/openobserve] + traces: + receivers: [otlp] + processors: [resource_detection, batch] + exporters: [otlp_http/openobserve] From 7145fa07ea966014898d2af21ccc870873a6e2d5 Mon Sep 17 00:00:00 2001 From: jasong-03 Date: Thu, 4 Jun 2026 13:26:48 +0700 Subject: [PATCH 02/78] fix(relayer): classify Enoki balance::split ENotEnough as gas-pool failure Enoki sponsored dry-run aborts in 0x2::balance::split with ENotEnough when a pool wallet's SUI gas coins are fragmented or too small to cover the budget. It was classified Transient, so it burned all 5 wallet retries rotating through equally-starved pool wallets and raised a misleading retries-exhausted alert. Add a distinct GasPoolExhausted classification that aborts retries (like the object-lock case) and fires a dedicated alert pointing ops at SUI gas coin consolidation/top-up. Add a gas-pool maintenance runbook. --- docs/relayer/runbook-gas-pool.md | 86 ++++++++++++++++++++++ services/server/src/alerts.rs | 118 +++++++++++++++++++++++++++++++ services/server/src/jobs.rs | 109 ++++++++++++++++++++++++---- 3 files changed, 298 insertions(+), 15 deletions(-) create mode 100644 docs/relayer/runbook-gas-pool.md diff --git a/docs/relayer/runbook-gas-pool.md b/docs/relayer/runbook-gas-pool.md new file mode 100644 index 00000000..12458a3c --- /dev/null +++ b/docs/relayer/runbook-gas-pool.md @@ -0,0 +1,86 @@ +# Runbook — relayer SUI gas pool maintenance + +When to use this: a **gas-pool alert** fires ("MemWal Walrus upload blocked — SUI +gas pool maintenance"), or relayer logs show wallet jobs aborting with +`classification=gas_pool_exhausted` / Enoki `dry_run_failed` + `0x2::balance::split` +(ENotEnough, abort code 2). + +## What it means + +The relayer sponsors Walrus register transactions through Enoki. Enoki's dry-run +splits a SUI gas coin on the selected pool wallet to cover the sponsored budget. +If that wallet has **no single SUI coin large enough** — i.e. its gas is +fragmented into many small coins, or it is low on SUI — Sui aborts in +`0x2::balance::split` with `ENotEnough`. + +The relayer now classifies this as `GasPoolExhausted` and **aborts the job +immediately** instead of retrying across the whole pool (which would just +re-fail on the next equally-starved wallet and burn the attempt budget). The +fix is operational: consolidate and/or top up SUI on the pool wallets. + +> WAL balance is unrelated — this is specifically about **SUI** gas coins. + +## Pool wallets + +Pool signing keys come from `SERVER_SUI_PRIVATE_KEYS` (comma-separated +`suiprivkey1...`), or the single `SERVER_SUI_PRIVATE_KEY`. Each key maps to one +Sui address. Derive the addresses (per key): + +```bash +sui keytool import "$KEY" ed25519 # then `sui keytool list`, or: +sui keytool list # shows addresses for imported keys +``` + +## Diagnose (per pool wallet) + +Check total SUI, the **largest single coin**, and fragmentation (coin count): + +```bash +# All SUI coins for an address, largest first: +sui client gas + +# Or via RPC (balance + coin count): +curl -s https://fullnode.mainnet.sui.io:443 -H 'content-type: application/json' \ + -d '{"jsonrpc":"2.0","id":1,"method":"suix_getBalance","params":["","0x2::sui::SUI"]}' +``` + +Red flags: +- **Largest coin** smaller than the sponsored budget (fragmentation) → consolidate. +- **Total SUI** low → top up. +- Many tiny coins → consolidate. + +## Fix + +### Consolidate (merge fragmented coins) + +Merge all SUI coins on a wallet into one (gas budgeting picks the largest coin): + +```bash +sui client switch --address +# Merge every other SUI coin into the primary one: +sui client merge-coin --primary-coin --coin-to-merge [--coin-to-merge ...] +``` + +For many coins, a PTB / `sui client pay-all-sui` to self consolidates in one tx: + +```bash +sui client pay-all-sui --input-coins ... --recipient --gas-budget 5000000 +``` + +### Top up + +Send SUI to the starved pool wallet(s) so the largest coin comfortably exceeds +the sponsored budget with headroom. + +## Verify + +Re-run the diagnose step — each pool wallet should have one (or few) large SUI +coins. Re-trigger a failed upload (or wait for the next job); the gas-pool alert +should not re-fire (alert dedup is per network, default 600s window — tunable via +`WALRUS_GAS_POOL_ALERT_DEDUP_SECS`). + +## Prevention (follow-up) + +A preflight pool-health check (min total SUI, min largest-coin size, max coin +count) before Walrus upload is tracked as follow-up in WALM-88 — it would catch +fragmentation before a job fails rather than after. diff --git a/services/server/src/alerts.rs b/services/server/src/alerts.rs index 4c8aad28..69ab136b 100644 --- a/services/server/src/alerts.rs +++ b/services/server/src/alerts.rs @@ -10,6 +10,8 @@ const WALRUS_UPGRADE_ALERT_DEDUP_SECS_ENV: &str = "WALRUS_PACKAGE_UPGRADE_ALERT_ const WALRUS_UPGRADE_ALERT_DEDUP_DEFAULT: Duration = Duration::from_secs(600); const WALRUS_OBJECT_LOCK_ALERT_DEDUP_SECS_ENV: &str = "WALRUS_OBJECT_LOCK_ALERT_DEDUP_SECS"; const WALRUS_OBJECT_LOCK_ALERT_DEDUP_DEFAULT: Duration = Duration::from_secs(600); +const WALRUS_GAS_POOL_ALERT_DEDUP_SECS_ENV: &str = "WALRUS_GAS_POOL_ALERT_DEDUP_SECS"; +const WALRUS_GAS_POOL_ALERT_DEDUP_DEFAULT: Duration = Duration::from_secs(600); /// Mirrors the `@mysten/walrus` dep version in /// `services/server/scripts/package.json`. Bump this constant in lockstep @@ -73,6 +75,11 @@ pub struct AlertManager { /// every concurrent job touching it raises the same error, so one /// notification per (network, object) is enough until the window elapses. walrus_object_lock_dedup: AlertDedup, + /// Suppresses Walrus gas-pool alert spam. A gas-pool starvation is + /// pool-wide, so every concurrent upload job raises the same + /// `balance::split` ENotEnough — one notification per network is enough + /// until ops tops up gas or the window elapses. + walrus_gas_pool_dedup: AlertDedup, } impl AlertManager { @@ -91,6 +98,10 @@ impl AlertManager { WALRUS_OBJECT_LOCK_ALERT_DEDUP_SECS_ENV, WALRUS_OBJECT_LOCK_ALERT_DEDUP_DEFAULT, )), + walrus_gas_pool_dedup: AlertDedup::new(dedup_window_from_env( + WALRUS_GAS_POOL_ALERT_DEDUP_SECS_ENV, + WALRUS_GAS_POOL_ALERT_DEDUP_DEFAULT, + )), } } @@ -150,6 +161,23 @@ impl AlertManager { let payload = SlackPayload::for_walrus_object_locked(&alert); slack.send_payload(&payload).await } + + pub async fn notify_walrus_gas_pool_exhausted( + &self, + alert: WalrusGasPoolExhaustedAlert, + ) -> Result<(), AlertError> { + let Some(slack) = &self.slack else { + return Ok(()); + }; + // Gas-pool starvation is pool-wide: dedup per network so a burst of + // concurrent jobs collapses to one ops notification per window. + let key = (alert.sui_network.clone(), "gas_pool".to_string()); + if self.walrus_gas_pool_dedup.should_suppress(key) { + return Ok(()); + } + let payload = SlackPayload::for_walrus_gas_pool_exhausted(&alert); + slack.send_payload(&payload).await + } } /// Read a dedup window (seconds) from `env_var`, falling back to `default` @@ -258,6 +286,23 @@ pub struct WalrusObjectLockedAlert { pub error: String, } +/// Fired when a wallet job aborts because an Enoki sponsored dry-run failed in +/// `0x2::balance::split` with ENotEnough — the relayer pool wallets' SUI gas +/// coins are fragmented or too small to cover the sponsored budget. Distinct +/// from the "exhausted retries" alert: this aborts immediately rather than +/// burning the whole pool, and the on-call message must point ops at SUI gas +/// coin consolidation/top-up rather than implying an app bug. +#[derive(Debug)] +pub struct WalrusGasPoolExhaustedAlert { + pub remember_job_id: Option, + pub owner: Option, + pub namespace: Option, + pub sui_network: String, + pub wallet_index: usize, + pub configured_wallets: usize, + pub error: String, +} + #[derive(Debug)] pub enum AlertError { Transport(String), @@ -434,6 +479,55 @@ impl SlackPayload { ], } } + + fn for_walrus_gas_pool_exhausted(alert: &WalrusGasPoolExhaustedAlert) -> Self { + let title = "MemWal Walrus upload blocked — SUI gas pool maintenance".to_string(); + let summary = format!( + "Walrus upload aborted on {}: Enoki sponsored dry-run failed in \ + `0x2::balance::split` (ENotEnough). The relayer pool wallets' SUI \ + gas coins are fragmented or too small to cover the sponsored budget. \ + Not retried — retrying just rotates to the next starved pool wallet.", + alert.sui_network, + ); + let action = "*Action (ops):* consolidate/merge SUI gas coins on the \ + relayer pool wallets and top up SUI if low. See the gas-pool runbook." + .to_string(); + let job = alert.remember_job_id.as_deref().unwrap_or("-"); + let owner = alert + .owner + .as_deref() + .map(short_address) + .unwrap_or_else(|| "-".to_string()); + let namespace = alert.namespace.as_deref().unwrap_or("-"); + let details = format!( + "*Network:* `{}`\n*Pool wallet:* `{}` of `{}`\n*Job:* `{}`\n*Owner:* `{}`\n*Namespace:* `{}`\n*Error:* ```{}```", + alert.sui_network, + alert.wallet_index, + alert.configured_wallets, + job, + owner, + namespace, + truncate(&alert.error, MAX_SLACK_ERROR_LEN), + ); + + Self { + text: summary.clone(), + blocks: vec![ + SlackBlock::Header { + text: plain_text(title), + }, + SlackBlock::Section { + text: mrkdwn(summary), + }, + SlackBlock::Section { + text: mrkdwn(action), + }, + SlackBlock::Section { + text: mrkdwn(details), + }, + ], + } + } } fn plain_text(text: String) -> SlackText { @@ -679,4 +773,28 @@ mod tests { assert!(json.contains("unparsed")); assert!(json.contains("mainnet")); } + + #[test] + fn walrus_gas_pool_payload_points_ops_to_gas_consolidation_not_exhausted_copy() { + let payload = SlackPayload::for_walrus_gas_pool_exhausted(&WalrusGasPoolExhaustedAlert { + remember_job_id: Some("591c13bc".into()), + owner: Some( + "0x51667727aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa04bca2".into(), + ), + namespace: Some("people".into()), + sui_network: "mainnet".into(), + wallet_index: 4, + configured_wallets: 5, + error: "Dry run failed: MoveAbort(0x2::balance::split, 2)".into(), + }); + + let json = serde_json::to_string(&payload).unwrap(); + // Points ops at gas-coin maintenance, not the misleading exhausted copy. + assert!(json.to_lowercase().contains("gas")); + assert!(json.contains("consolidate") || json.contains("top up")); + assert!(!json.to_lowercase().contains("exhausted retries")); + assert!(json.contains("balance::split") || json.contains("balance")); + assert!(json.contains("mainnet")); + assert!(json.contains("people")); + } } diff --git a/services/server/src/jobs.rs b/services/server/src/jobs.rs index 4d0d8699..32b47d0c 100644 --- a/services/server/src/jobs.rs +++ b/services/server/src/jobs.rs @@ -20,8 +20,8 @@ use redis::AsyncCommands; use serde::{Deserialize, Serialize}; use crate::alerts::{ - SIDECAR_WALRUS_DEP_VERSION, WalrusObjectLockedAlert, WalrusPackageUpgradeDetectedAlert, - WalrusUploadExhaustedAlert, + SIDECAR_WALRUS_DEP_VERSION, WalrusGasPoolExhaustedAlert, WalrusObjectLockedAlert, + WalrusPackageUpgradeDetectedAlert, WalrusUploadExhaustedAlert, }; use crate::storage::walrus::{SetMetadataBatchEntry, UploadBlobError}; use crate::types::{configured_walrus_storage_epochs, AppState, BLOB_CACHE_KEY_PREFIX}; @@ -893,6 +893,16 @@ async fn execute_upload_and_transfer( &msg, ) .await; + maybe_alert_walrus_gas_pool_exhausted( + state, + &classified, + remember_job_id.as_deref(), + Some(&owner), + Some(&namespace), + wallet_index, + &msg, + ) + .await; maybe_alert_walrus_upload_exhausted( state, &classified, @@ -1086,6 +1096,42 @@ async fn maybe_alert_walrus_object_locked( } } +/// Fire the distinct gas-pool maintenance alert when a wallet job fails with +/// `GasPoolExhausted` (Enoki dry-run `balance::split` ENotEnough). Kept separate +/// from the "exhausted retries" alert: this case aborts immediately, so the +/// on-call message must name the real cause — fragmented/insufficient SUI gas on +/// the pool wallets — and point ops at gas-coin consolidation/top-up. +async fn maybe_alert_walrus_gas_pool_exhausted( + state: &AppState, + error: &WalletJobError, + remember_job_id: Option<&str>, + owner: Option<&str>, + namespace: Option<&str>, + wallet_index: usize, + msg: &str, +) { + if !matches!(error, WalletJobError::GasPoolExhausted(_)) { + return; + } + + let alert = WalrusGasPoolExhaustedAlert { + remember_job_id: remember_job_id.map(str::to_owned), + owner: owner.map(str::to_owned), + namespace: namespace.map(str::to_owned), + sui_network: state.config.sui_network.clone(), + wallet_index, + configured_wallets: state.key_pool.len(), + error: msg.to_string(), + }; + + if let Err(err) = state.alerts.notify_walrus_gas_pool_exhausted(alert).await { + tracing::warn!( + "[wallet-job:upload] failed to send Slack alert for Walrus gas-pool exhaustion: {}", + err + ); + } +} + async fn maybe_alert_walrus_upload_exhausted( state: &AppState, error: &WalletJobError, @@ -1131,8 +1177,10 @@ async fn maybe_alert_walrus_upload_exhausted( /// don't burn retry budget on inputs that can never succeed. /// /// Mapping rules (enforced at the point of error origination): -/// - `MoveAbort(_)` → `Permanent` (deterministic Move-level failure), except -/// `balance::split` stale-state failures that can recover after sidecar refresh +/// - `MoveAbort(_)` → `Permanent` (deterministic Move-level failure) +/// - Enoki dry-run `0x2::balance::split` ENotEnough → `GasPoolExhausted` +/// (abort: pool SUI gas coins are fragmented/insufficient; retrying rotates +/// to the next starved wallet — needs ops gas consolidation/top-up) /// - `ObjectLockedAtVersion(_)` → `Transient` (retry can rebuild with a fresh /// wallet assignment) /// - owned-object lock / equivocation ("already locked by a different @@ -1156,6 +1204,15 @@ pub enum WalletJobError { /// Apalis aborts so we surface a distinct object-lock alert rather than a /// misleading "wallet retries exhausted" one. ObjectLockedUntilEpoch(String), + /// An Enoki sponsored dry-run aborted in `0x2::balance::split` with + /// ENotEnough (abort code 2): the pool wallet's SUI gas coins are + /// fragmented or too small to cover the sponsored budget. Retrying rotates + /// to the next pool wallet and re-fails the same way, burning the attempt + /// budget without progress. NOT `Permanent`: it succeeds again once ops + /// consolidates / tops up SUI gas on the pool wallets. Apalis aborts so we + /// surface a distinct gas-pool maintenance alert rather than a misleading + /// "wallet retries exhausted" one. + GasPoolExhausted(String), } impl WalletJobError { @@ -1164,6 +1221,7 @@ impl WalletJobError { WalletJobError::Transient(_) => "transient", WalletJobError::Permanent(_) => "permanent", WalletJobError::ObjectLockedUntilEpoch(_) => "object_locked_until_epoch", + WalletJobError::GasPoolExhausted(_) => "gas_pool_exhausted", } } @@ -1180,7 +1238,9 @@ impl WalletJobError { pub fn aborts_retries(&self) -> bool { matches!( self, - WalletJobError::Permanent(_) | WalletJobError::ObjectLockedUntilEpoch(_) + WalletJobError::Permanent(_) + | WalletJobError::ObjectLockedUntilEpoch(_) + | WalletJobError::GasPoolExhausted(_) ) } @@ -1189,11 +1249,20 @@ impl WalletJobError { /// Until the sidecar emits structured error codes, we match on substrings. pub fn classify_sidecar_error(msg: &str) -> Self { let lower = msg.to_ascii_lowercase(); + // Enoki sponsored dry-run aborts in 0x2::balance::split with ENotEnough + // (abort code 2) when the pool wallet's selected SUI gas coin cannot be + // split to cover the sponsored budget — i.e. the relayer's SUI gas coins + // are fragmented or too small. This is a gas-pool maintenance issue, not + // a transient: retrying just rotates to the next equally-starved pool + // wallet and burns the whole attempt budget. Abort and surface a + // dedicated gas-pool alert pointing ops to consolidate/top-up SUI gas. + // (balance::split's only abort is ENotEnough, so the balance+split anchor + // is sufficient to identify it.) if (lower.contains("moveabort") || lower.contains("move abort")) && lower.contains("balance") && lower.contains("split") { - return WalletJobError::Transient(msg.to_string()); + return WalletJobError::GasPoolExhausted(msg.to_string()); } // Walrus on-chain package upgrade — the cached @mysten/walrus client // carries stale package metadata until refreshed. The sidecar already @@ -1250,9 +1319,9 @@ impl WalletJobError { let error = io::Error::other(self.to_string()); match self { WalletJobError::Transient(_) => Error::Failed(Arc::new(Box::new(error))), - WalletJobError::Permanent(_) | WalletJobError::ObjectLockedUntilEpoch(_) => { - Error::Abort(Arc::new(Box::new(error))) - } + WalletJobError::Permanent(_) + | WalletJobError::ObjectLockedUntilEpoch(_) + | WalletJobError::GasPoolExhausted(_) => Error::Abort(Arc::new(Box::new(error))), } } } @@ -1265,6 +1334,9 @@ impl std::fmt::Display for WalletJobError { WalletJobError::ObjectLockedUntilEpoch(msg) => { write!(f, "wallet job error (object locked until epoch): {}", msg) } + WalletJobError::GasPoolExhausted(msg) => { + write!(f, "wallet job error (gas pool exhausted): {}", msg) + } } } } @@ -1793,13 +1865,14 @@ different transaction: TransactionDigest(8bjFgRyXRRYwrzQapgEjpHnGhdfNDY7d6xA82Bt #[test] fn equivocation_does_not_regress_recoverable_classes() { - // balance::split and EWrongVersion must still be Transient (recoverable), - // not swept into the new abort path. + // EWrongVersion must still be Transient (recoverable). balance::split + // ENotEnough is a gas-pool maintenance abort, not an object lock — it + // must classify as GasPoolExhausted, not swept into the object-lock path. let balance = "Enoki dry run failed: MoveAbort(0x2::balance, split, 2)"; let ewrong = "MoveAbort in 1st command, abort code: 1, in '0xabc::system::inner_mut'"; assert!(matches!( WalletJobError::classify_sidecar_error(balance), - WalletJobError::Transient(_) + WalletJobError::GasPoolExhausted(_) )); assert!(matches!( WalletJobError::classify_sidecar_error(ewrong), @@ -1844,16 +1917,22 @@ different transaction: TransactionDigest(8bjFgRyXRRYwrzQapgEjpHnGhdfNDY7d6xA82Bt } #[test] - fn classify_balance_split_move_abort_as_transient() { + fn classify_balance_split_move_abort_as_gas_pool_exhausted() { + // Enoki dry-run balance::split ENotEnough → gas-pool maintenance abort: + // classified GasPoolExhausted, aborts retries (so it does not burn the + // whole wallet pool), and is not Permanent (succeeds after gas top-up). for msg in [ "walrus upload failed: Enoki API error (400): {\"errors\":[{\"code\":\"dry_run_failed\",\"message\":\"Dry run failed: MoveAbort(MoveLocation { module: 0x2::balance, function_name: Some(\\\"split\\\") }, 2)\"}]}", "move abort during balance split", ] { + let classified = WalletJobError::classify_sidecar_error(msg); assert!( - !WalletJobError::classify_sidecar_error(msg).is_permanent(), - "expected transient for: {}", + matches!(classified, WalletJobError::GasPoolExhausted(_)), + "expected gas_pool_exhausted for: {}", msg ); + assert!(classified.aborts_retries(), "must abort retries: {}", msg); + assert!(!classified.is_permanent(), "must not be permanent: {}", msg); } } From 1d7c5d0c3433598243586a629bfb1dc2667b6e6a Mon Sep 17 00:00:00 2001 From: jasong-03 Date: Thu, 4 Jun 2026 23:23:13 +0700 Subject: [PATCH 03/78] docs(relayer): fix gas-pool runbook for Sui style guide Move title to frontmatter, remove em dashes, replace 'i.e.' and prose 'via' to satisfy the Sui documentation style guide audit. --- docs/relayer/runbook-gas-pool.md | 38 +++++++++++++++++--------------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/docs/relayer/runbook-gas-pool.md b/docs/relayer/runbook-gas-pool.md index 12458a3c..2bc58ee2 100644 --- a/docs/relayer/runbook-gas-pool.md +++ b/docs/relayer/runbook-gas-pool.md @@ -1,24 +1,26 @@ -# Runbook — relayer SUI gas pool maintenance +--- +title: "Gas Pool Maintenance Runbook" +--- -When to use this: a **gas-pool alert** fires ("MemWal Walrus upload blocked — SUI -gas pool maintenance"), or relayer logs show wallet jobs aborting with -`classification=gas_pool_exhausted` / Enoki `dry_run_failed` + `0x2::balance::split` -(ENotEnough, abort code 2). +When to use this: a **gas-pool alert** fires (the "SUI gas pool maintenance" +alert), or relayer logs show wallet jobs aborting with +`classification=gas_pool_exhausted` or Enoki `dry_run_failed` plus +`0x2::balance::split` (ENotEnough, abort code 2). ## What it means The relayer sponsors Walrus register transactions through Enoki. Enoki's dry-run splits a SUI gas coin on the selected pool wallet to cover the sponsored budget. -If that wallet has **no single SUI coin large enough** — i.e. its gas is -fragmented into many small coins, or it is low on SUI — Sui aborts in -`0x2::balance::split` with `ENotEnough`. +When that wallet has **no single SUI coin large enough** (its gas is fragmented +into many small coins, or it is low on SUI), Sui aborts in `0x2::balance::split` +with `ENotEnough`. The relayer now classifies this as `GasPoolExhausted` and **aborts the job immediately** instead of retrying across the whole pool (which would just re-fail on the next equally-starved wallet and burn the attempt budget). The fix is operational: consolidate and/or top up SUI on the pool wallets. -> WAL balance is unrelated — this is specifically about **SUI** gas coins. +> WAL balance is unrelated. This is specifically about **SUI** gas coins. ## Pool wallets @@ -39,15 +41,15 @@ Check total SUI, the **largest single coin**, and fragmentation (coin count): # All SUI coins for an address, largest first: sui client gas -# Or via RPC (balance + coin count): +# Or through RPC (balance + coin count): curl -s https://fullnode.mainnet.sui.io:443 -H 'content-type: application/json' \ -d '{"jsonrpc":"2.0","id":1,"method":"suix_getBalance","params":["","0x2::sui::SUI"]}' ``` Red flags: -- **Largest coin** smaller than the sponsored budget (fragmentation) → consolidate. -- **Total SUI** low → top up. -- Many tiny coins → consolidate. +- **Largest coin** smaller than the sponsored budget (fragmentation): consolidate. +- **Total SUI** low: top up. +- Many tiny coins: consolidate. ## Fix @@ -61,7 +63,7 @@ sui client switch --address sui client merge-coin --primary-coin --coin-to-merge [--coin-to-merge ...] ``` -For many coins, a PTB / `sui client pay-all-sui` to self consolidates in one tx: +For many coins, a PTB or `sui client pay-all-sui` to self consolidates in one tx: ```bash sui client pay-all-sui --input-coins ... --recipient --gas-budget 5000000 @@ -74,13 +76,13 @@ the sponsored budget with headroom. ## Verify -Re-run the diagnose step — each pool wallet should have one (or few) large SUI +Re-run the diagnose step. Each pool wallet should have one (or few) large SUI coins. Re-trigger a failed upload (or wait for the next job); the gas-pool alert -should not re-fire (alert dedup is per network, default 600s window — tunable via -`WALRUS_GAS_POOL_ALERT_DEDUP_SECS`). +should not re-fire (alert dedup is per network, default 600s window, tunable +through `WALRUS_GAS_POOL_ALERT_DEDUP_SECS`). ## Prevention (follow-up) A preflight pool-health check (min total SUI, min largest-coin size, max coin -count) before Walrus upload is tracked as follow-up in WALM-88 — it would catch +count) before Walrus upload is tracked as a follow-up in WALM-88. It would catch fragmentation before a job fails rather than after. From 8b480f9e67b6b8dc9beddc0ce8277e02f8600fb2 Mon Sep 17 00:00:00 2001 From: jasong-03 Date: Thu, 4 Jun 2026 23:39:07 +0700 Subject: [PATCH 04/78] fix(relayer): retry gas-pool failures across the pool before aborting Address review on #231: - P1: a balance::split ENotEnough now stays Transient so Apalis rotates onto another pool wallet, and only escalates to GasPoolExhausted once every candidate wallet (min(pool_size, max_attempts)) has hit the same gas-budget failure. A single starved wallet no longer fails an upload a healthy wallet could serve. - P2: the metadata-transfer recovery path applies the same escalation and dispatches the gas-pool ops alert (previously only the upload arm did). Tests: single bad wallet stays retriable, full-pool exhaustion escalates and aborts, threshold computation, non-gas-budget passthrough. --- services/server/src/jobs.rs | 176 +++++++++++++++++++++++++++++------- 1 file changed, 145 insertions(+), 31 deletions(-) diff --git a/services/server/src/jobs.rs b/services/server/src/jobs.rs index 32b47d0c..fa1e75e8 100644 --- a/services/server/src/jobs.rs +++ b/services/server/src/jobs.rs @@ -535,7 +535,28 @@ pub(crate) async fn execute_wallet_job( )), }, Err(err) => { + // Same gas-pool escalation as the upload path: a + // balance::split failure stays retriable until the pool is + // exhausted, then aborts. Dispatch the dedicated ops alert + // here too so a gas-pool failure on the metadata-transfer + // recovery path isn't silently marked failed. + let err = escalate_if_gas_pool_exhausted( + err, + attempt_info.current, + attempt_info.max, + state.key_pool.len(), + ); let msg = err.to_string(); + maybe_alert_walrus_gas_pool_exhausted( + state, + &err, + remember_job_id.as_deref(), + Some(&owner), + Some(&namespace), + enqueued_wallet_index, + &msg, + ) + .await; update_remember_job_after_wallet_error( state, remember_job_id.as_deref(), @@ -875,7 +896,15 @@ async fn execute_upload_and_transfer( } Err(UploadBlobError::App(e)) => { let msg = format!("walrus upload failed: {}", e); - let classified = WalletJobError::classify_sidecar_error(&msg); + // A balance::split gas-budget failure stays retriable (rotates onto + // another pool wallet) until every candidate wallet has failed it, + // then escalates to an aborting GasPoolExhausted (+ ops alert below). + let classified = escalate_if_gas_pool_exhausted( + WalletJobError::classify_sidecar_error(&msg), + attempt_info.current, + attempt_info.max, + state.key_pool.len(), + ); maybe_alert_walrus_package_upgrade_detected( state, remember_job_id.as_deref(), @@ -1096,6 +1125,37 @@ async fn maybe_alert_walrus_object_locked( } } +/// Attempts after which repeated gas-budget (`balance::split` ENotEnough) +/// failures are treated as pool-level exhaustion rather than a single starved +/// wallet. Upload retries rotate round-robin across the pool, so once the +/// attempt count reaches the pool size (capped by the max attempt budget) every +/// candidate wallet has been tried. At least 1, so a single-wallet pool +/// escalates immediately (nothing else to rotate onto). +fn gas_pool_exhaustion_threshold(pool_size: usize, max_attempts: usize) -> usize { + pool_size.min(max_attempts).max(1) +} + +/// Escalate a retriable gas-budget failure to an aborting `GasPoolExhausted` +/// once every candidate pool wallet has failed the same way. Below the +/// threshold the error stays `Transient` so Apalis rotates onto another wallet — +/// a single starved wallet must not fail an upload a healthy wallet could serve. +/// Non-gas-budget errors pass through unchanged. +fn escalate_if_gas_pool_exhausted( + classified: WalletJobError, + attempt: usize, + max_attempts: usize, + pool_size: usize, +) -> WalletJobError { + if let WalletJobError::Transient(ref msg) = classified { + if WalletJobError::is_gas_pool_budget_error(msg) + && attempt >= gas_pool_exhaustion_threshold(pool_size, max_attempts) + { + return WalletJobError::GasPoolExhausted(msg.clone()); + } + } + classified +} + /// Fire the distinct gas-pool maintenance alert when a wallet job fails with /// `GasPoolExhausted` (Enoki dry-run `balance::split` ENotEnough). Kept separate /// from the "exhausted retries" alert: this case aborts immediately, so the @@ -1244,25 +1304,33 @@ impl WalletJobError { ) } + /// True if `msg` is an Enoki sponsored dry-run gas-budget failure: an abort + /// in `0x2::balance::split` (ENotEnough). A single occurrence only proves the + /// *selected* pool wallet is gas-starved, not the whole pool — escalation to + /// `GasPoolExhausted` is gated on pool-level confirmation by the caller. + pub fn is_gas_pool_budget_error(msg: &str) -> bool { + let lower = msg.to_ascii_lowercase(); + (lower.contains("moveabort") || lower.contains("move abort")) + && lower.contains("balance") + && lower.contains("split") + } + /// Heuristic classification from the sidecar's error string. The sidecar /// surfaces Sui execution errors verbatim (Move abort codes, lock errors). /// Until the sidecar emits structured error codes, we match on substrings. pub fn classify_sidecar_error(msg: &str) -> Self { let lower = msg.to_ascii_lowercase(); // Enoki sponsored dry-run aborts in 0x2::balance::split with ENotEnough - // (abort code 2) when the pool wallet's selected SUI gas coin cannot be - // split to cover the sponsored budget — i.e. the relayer's SUI gas coins - // are fragmented or too small. This is a gas-pool maintenance issue, not - // a transient: retrying just rotates to the next equally-starved pool - // wallet and burns the whole attempt budget. Abort and surface a - // dedicated gas-pool alert pointing ops to consolidate/top-up SUI gas. - // (balance::split's only abort is ENotEnough, so the balance+split anchor - // is sufficient to identify it.) - if (lower.contains("moveabort") || lower.contains("move abort")) - && lower.contains("balance") - && lower.contains("split") - { - return WalletJobError::GasPoolExhausted(msg.to_string()); + // (abort code 2) when the selected pool wallet's SUI gas coin cannot be + // split to cover the sponsored budget (its SUI is fragmented or too low). + // A single failure only proves THAT wallet is gas-starved, not the whole + // pool, so classify Transient: Apalis rotates onto another pool wallet on + // retry rather than failing an upload a healthy wallet could serve. The + // wallet-job error arm escalates to an aborting GasPoolExhausted (+ ops + // alert) only once every candidate wallet has failed the same way — see + // escalate_if_gas_pool_exhausted. + if Self::is_gas_pool_budget_error(msg) { + return WalletJobError::Transient(msg.to_string()); } // Walrus on-chain package upgrade — the cached @mysten/walrus client // carries stale package metadata until refreshed. The sidecar already @@ -1727,7 +1795,8 @@ mod tests { use sqlx::postgres::PgPoolOptions; use super::{ - classify_wallet_remember_handoff_failure, is_walrus_package_version_mismatch, + classify_wallet_remember_handoff_failure, escalate_if_gas_pool_exhausted, + gas_pool_exhaustion_threshold, is_walrus_package_version_mismatch, mark_remember_job_failed, parse_locked_object_info, wallet_job_request, WalletJob, WalletJobAttemptInfo, WalletJobError, WalletOperation, MAX_ATTEMPTS, }; @@ -1865,14 +1934,15 @@ different transaction: TransactionDigest(8bjFgRyXRRYwrzQapgEjpHnGhdfNDY7d6xA82Bt #[test] fn equivocation_does_not_regress_recoverable_classes() { - // EWrongVersion must still be Transient (recoverable). balance::split - // ENotEnough is a gas-pool maintenance abort, not an object lock — it - // must classify as GasPoolExhausted, not swept into the object-lock path. + // EWrongVersion and a lone balance::split ENotEnough must both stay + // Transient (recoverable) — neither is swept into the object-lock abort + // path. balance::split only escalates to GasPoolExhausted at the pool + // level (see gas_pool_* tests below), not on a single occurrence. let balance = "Enoki dry run failed: MoveAbort(0x2::balance, split, 2)"; let ewrong = "MoveAbort in 1st command, abort code: 1, in '0xabc::system::inner_mut'"; assert!(matches!( WalletJobError::classify_sidecar_error(balance), - WalletJobError::GasPoolExhausted(_) + WalletJobError::Transient(_) )); assert!(matches!( WalletJobError::classify_sidecar_error(ewrong), @@ -1916,26 +1986,70 @@ different transaction: TransactionDigest(8bjFgRyXRRYwrzQapgEjpHnGhdfNDY7d6xA82Bt } } + const BALANCE_SPLIT_ERR: &str = "walrus upload failed: Enoki API error (400): {\"errors\":[{\"code\":\"dry_run_failed\",\"message\":\"Dry run failed: MoveAbort(MoveLocation { module: 0x2::balance, function_name: Some(\\\"split\\\") }, 2)\"}]}"; + #[test] - fn classify_balance_split_move_abort_as_gas_pool_exhausted() { - // Enoki dry-run balance::split ENotEnough → gas-pool maintenance abort: - // classified GasPoolExhausted, aborts retries (so it does not burn the - // whole wallet pool), and is not Permanent (succeeds after gas top-up). - for msg in [ - "walrus upload failed: Enoki API error (400): {\"errors\":[{\"code\":\"dry_run_failed\",\"message\":\"Dry run failed: MoveAbort(MoveLocation { module: 0x2::balance, function_name: Some(\\\"split\\\") }, 2)\"}]}", - "move abort during balance split", - ] { + fn classify_balance_split_is_retriable_transient_by_default() { + // A single balance::split ENotEnough is Transient (retriable) so Apalis + // rotates onto another pool wallet — it must NOT abort on its own. + for msg in [BALANCE_SPLIT_ERR, "move abort during balance split"] { let classified = WalletJobError::classify_sidecar_error(msg); assert!( - matches!(classified, WalletJobError::GasPoolExhausted(_)), - "expected gas_pool_exhausted for: {}", + matches!(classified, WalletJobError::Transient(_)), + "expected transient for: {}", msg ); - assert!(classified.aborts_retries(), "must abort retries: {}", msg); - assert!(!classified.is_permanent(), "must not be permanent: {}", msg); + assert!(!classified.aborts_retries(), "must retry: {}", msg); + assert!(WalletJobError::is_gas_pool_budget_error(msg)); } } + #[test] + fn gas_pool_threshold_tracks_pool_then_caps_at_max_attempts() { + assert_eq!(gas_pool_exhaustion_threshold(1, 5), 1); // single wallet → escalate immediately + assert_eq!(gas_pool_exhaustion_threshold(2, 5), 2); // try both wallets first + assert_eq!(gas_pool_exhaustion_threshold(10, 5), 5); // capped by max attempts + assert_eq!(gas_pool_exhaustion_threshold(0, 5), 1); // never 0 + } + + #[test] + fn gas_pool_one_bad_wallet_keeps_retrying_onto_healthy_wallet() { + // pool of 2: the first wallet's balance::split must stay retriable so the + // job rotates onto the second (healthy) wallet instead of failing. + let classified = WalletJobError::classify_sidecar_error(BALANCE_SPLIT_ERR); + let after = escalate_if_gas_pool_exhausted(classified, /*attempt*/ 1, /*max*/ 5, /*pool*/ 2); + assert!( + matches!(after, WalletJobError::Transient(_)) && !after.aborts_retries(), + "single bad wallet must keep retrying, got {:?}", + after + ); + } + + #[test] + fn gas_pool_all_wallets_failed_escalates_and_aborts() { + // pool of 2, both wallets hit balance::split → at attempt 2 we have tried + // every candidate wallet → escalate to an aborting GasPoolExhausted. + let classified = WalletJobError::classify_sidecar_error(BALANCE_SPLIT_ERR); + let after = escalate_if_gas_pool_exhausted(classified, /*attempt*/ 2, /*max*/ 5, /*pool*/ 2); + assert!( + matches!(after, WalletJobError::GasPoolExhausted(_)), + "exhausted pool must escalate, got {:?}", + after + ); + assert!(after.aborts_retries()); + assert!(!after.is_permanent()); + assert_eq!(after.kind(), "gas_pool_exhausted"); + } + + #[test] + fn escalation_ignores_non_gas_budget_transient() { + // A generic transient (e.g. timeout) must never be escalated to the + // gas-pool abort path, even past the threshold. + let other = WalletJobError::Transient("network timeout".into()); + let after = escalate_if_gas_pool_exhausted(other, 5, 5, 2); + assert!(matches!(after, WalletJobError::Transient(_))); + } + #[test] fn classify_walrus_version_mismatch_as_transient() { // The sidecar refreshes the cached @mysten/walrus client on EWrongVersion; From 9b2f8334be2a53c2b280ca140e18e8efc8ecbc8d Mon Sep 17 00:00:00 2001 From: jasong-03 Date: Thu, 4 Jun 2026 23:44:07 +0700 Subject: [PATCH 05/78] docs(relayer): full Sui style-guide compliance for gas-pool runbook Beyond the regex audit: remove quotation marks, replace the 'and/or' slash, add body text between stacked headings, and write out word abbreviations (tx, min, max) and the (s) plural per the Sui documentation style guide. --- docs/relayer/runbook-gas-pool.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/docs/relayer/runbook-gas-pool.md b/docs/relayer/runbook-gas-pool.md index 2bc58ee2..2860459a 100644 --- a/docs/relayer/runbook-gas-pool.md +++ b/docs/relayer/runbook-gas-pool.md @@ -2,7 +2,7 @@ title: "Gas Pool Maintenance Runbook" --- -When to use this: a **gas-pool alert** fires (the "SUI gas pool maintenance" +When to use this: a **gas-pool alert** fires (the SUI gas pool maintenance alert), or relayer logs show wallet jobs aborting with `classification=gas_pool_exhausted` or Enoki `dry_run_failed` plus `0x2::balance::split` (ENotEnough, abort code 2). @@ -18,7 +18,7 @@ with `ENotEnough`. The relayer now classifies this as `GasPoolExhausted` and **aborts the job immediately** instead of retrying across the whole pool (which would just re-fail on the next equally-starved wallet and burn the attempt budget). The -fix is operational: consolidate and/or top up SUI on the pool wallets. +fix is operational: consolidate or top up SUI on the pool wallets, or both. > WAL balance is unrelated. This is specifically about **SUI** gas coins. @@ -53,6 +53,8 @@ Red flags: ## Fix +There are two operational fixes, depending on the diagnosis. + ### Consolidate (merge fragmented coins) Merge all SUI coins on a wallet into one (gas budgeting picks the largest coin): @@ -63,7 +65,7 @@ sui client switch --address sui client merge-coin --primary-coin --coin-to-merge [--coin-to-merge ...] ``` -For many coins, a PTB or `sui client pay-all-sui` to self consolidates in one tx: +For many coins, a PTB or `sui client pay-all-sui` to self consolidates in one transaction: ```bash sui client pay-all-sui --input-coins ... --recipient --gas-budget 5000000 @@ -71,7 +73,7 @@ sui client pay-all-sui --input-coins ... --recipient Date: Fri, 5 Jun 2026 09:20:32 +0700 Subject: [PATCH 06/78] docs(relayer): add frontmatter description/keywords, unquote title Satisfy the Sui style-guide audit: remove quotation marks from the frontmatter title and add the required description and keywords fields. --- docs/relayer/runbook-gas-pool.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/relayer/runbook-gas-pool.md b/docs/relayer/runbook-gas-pool.md index 2860459a..0b698b29 100644 --- a/docs/relayer/runbook-gas-pool.md +++ b/docs/relayer/runbook-gas-pool.md @@ -1,5 +1,7 @@ --- -title: "Gas Pool Maintenance Runbook" +title: Gas Pool Maintenance Runbook +description: How to diagnose and fix Enoki gas-pool failures on relayer pool wallets by consolidating or topping up SUI gas coins. +keywords: [relayer, gas pool, Enoki, SUI, sponsored transaction, runbook] --- When to use this: a **gas-pool alert** fires (the SUI gas pool maintenance From 73915b4dc6bdf2a5691aa4b41920c9f7dc515ace Mon Sep 17 00:00:00 2001 From: jasong-03 Date: Fri, 5 Jun 2026 11:15:07 +0700 Subject: [PATCH 07/78] feat(relayer): push Prometheus metrics to OpenObserve via remote_write Add an opt-in background task (ZO_REMOTE_WRITE_URL) that gathers the relayer's Prometheus registry and pushes it to OpenObserve's /prometheus/api/v1/write endpoint as snappy-compressed protobuf (counters, gauges, histograms expanded to _bucket/_sum/_count, summaries). No-op when the env var is unset, so a single OpenObserve service can ingest the existing memwal_* metrics without a collector and production is unchanged until an environment opts in. --- services/server/Cargo.lock | 31 ++ services/server/Cargo.toml | 4 + services/server/src/main.rs | 5 + services/server/src/metrics_remote_write.rs | 316 ++++++++++++++++++++ 4 files changed, 356 insertions(+) create mode 100644 services/server/src/metrics_remote_write.rs diff --git a/services/server/Cargo.lock b/services/server/Cargo.lock index 4ca5876c..3a2723a6 100644 --- a/services/server/Cargo.lock +++ b/services/server/Cargo.lock @@ -1286,11 +1286,13 @@ dependencies = [ "percent-encoding", "pgvector", "prometheus", + "prost", "redis", "reqwest", "serde", "serde_json", "sha2", + "snap", "sqlx", "tokio", "tower", @@ -1599,6 +1601,29 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "prost" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-derive" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "protobuf" version = "2.28.0" @@ -2076,6 +2101,12 @@ dependencies = [ "serde", ] +[[package]] +name = "snap" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" + [[package]] name = "socket2" version = "0.5.10" diff --git a/services/server/Cargo.toml b/services/server/Cargo.toml index 07fd122b..c3599fab 100644 --- a/services/server/Cargo.toml +++ b/services/server/Cargo.toml @@ -41,6 +41,10 @@ prometheus = "0.13" # HTTP client (for Sui RPC onchain verification) reqwest = { version = "0.12", features = ["json", "stream"] } +# Prometheus remote_write push to OpenObserve (snappy-compressed protobuf) +prost = "0.13" +snap = "1" + # Async utilities futures = "0.3" async-trait = "0.1" diff --git a/services/server/src/main.rs b/services/server/src/main.rs index 3530de5e..ed9823bf 100644 --- a/services/server/src/main.rs +++ b/services/server/src/main.rs @@ -4,6 +4,7 @@ mod compatibility; mod engine; mod jobs; mod mcp_proxy; +mod metrics_remote_write; mod observability; mod rate_limit; mod routes; @@ -378,6 +379,10 @@ async fn main() { let alerts = Arc::new(AlertManager::from_env(http_client.clone())); + // Push Prometheus metrics to OpenObserve via remote_write (opt-in via + // ZO_REMOTE_WRITE_URL; no-op when unset). + metrics_remote_write::spawn(http_client.clone()); + // Shared application state let state = Arc::new(AppState { db, diff --git a/services/server/src/metrics_remote_write.rs b/services/server/src/metrics_remote_write.rs new file mode 100644 index 00000000..1649ec89 --- /dev/null +++ b/services/server/src/metrics_remote_write.rs @@ -0,0 +1,316 @@ +//! Prometheus remote_write push to OpenObserve (WALM-81). +//! +//! The relayer exposes `/metrics` (Prometheus pull). OpenObserve cannot scrape +//! a pull endpoint, so to keep a single OpenObserve service (no collector) we +//! periodically gather the registry and push it via the Prometheus +//! remote_write protocol — snappy-compressed protobuf — to OpenObserve's +//! `/prometheus/api/v1/write` endpoint. +//! +//! Opt-in: disabled (no-op) unless `ZO_REMOTE_WRITE_URL` is set, so production +//! behaviour is unchanged until an environment explicitly enables it. + +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +const REMOTE_WRITE_URL_ENV: &str = "ZO_REMOTE_WRITE_URL"; +/// Full `Authorization` header value, e.g. `Basic `. +const REMOTE_WRITE_AUTH_ENV: &str = "ZO_REMOTE_WRITE_AUTH"; +const REMOTE_WRITE_INTERVAL_ENV: &str = "ZO_REMOTE_WRITE_INTERVAL_SECS"; +const DEFAULT_INTERVAL_SECS: u64 = 30; + +// ── Prometheus remote_write protobuf (minimal subset) ────────────────────── +// Matches prometheus/prompb `WriteRequest`. Only the fields OpenObserve needs. + +#[derive(Clone, PartialEq, prost::Message)] +struct WriteRequest { + #[prost(message, repeated, tag = "1")] + timeseries: Vec, +} + +#[derive(Clone, PartialEq, prost::Message)] +struct TimeSeries { + #[prost(message, repeated, tag = "1")] + labels: Vec