From 8227663fb99b44a59d016f883edb77d6d43151fd Mon Sep 17 00:00:00 2001 From: Ant Somers Date: Thu, 15 Oct 2026 01:15:45 +0300 Subject: [PATCH 1/2] fix(monitoring): follow decdn v0.1.0 buyer metrics, quiet the registry-resync alert (#125) decdn v0.1.0 (decdn/decdn#2406) removed decdn_voucher_nonce_gaps_total and widened the pool-open and lane-seed counters: - chain dashboard: drop the nonce-gap series (the panel is now "Lane and receipt-log writes"); name the pool-open aggregate's real causes (store, open-task, capability signing, lane seed) and the four lane-seed causes. - delivery dashboard: pool open failures are gas, RPC, buyer store or lane seed, not an expired pool. - DecdnBuyerLaneSeedFailing: "Buyer lane seeds refused", all four causes. Alerts and panels that misled on the live fleet: - DecdnRpcErrorRatioHigh and "RPC failures by outcome": rpc_error on eth_getLogs is mostly a provider's transient error the poller retries in place, not a range cap (that has its own counter). Text only. - DecdnCapacityBondRegistryResyncFailing fires on two failed attempts in the hour (>= 2), not one transient eth_call error; promtool tests added. - Node drilldown "Peers known" gates on max_over_time of pull attempts, so a restart no longer blanks it. Closes #125 Co-Authored-By: Claude Opus 5.5 --- .claude/skills/grafana-dashboards/SKILL.md | 5 ++- monitoring/README.md | 2 +- monitoring/decdn-node/dashboard-chain.json | 19 +++------- monitoring/decdn-node/dashboard-delivery.json | 2 +- monitoring/decdn-node/dashboard-node.json | 4 +- monitoring/decdn-node/prometheus-alerts.yml | 38 ++++++++++++++----- .../decdn-node/prometheus-alerts_test.yml | 32 +++++++++++++++- 7 files changed, 70 insertions(+), 32 deletions(-) diff --git a/.claude/skills/grafana-dashboards/SKILL.md b/.claude/skills/grafana-dashboards/SKILL.md index 458267f..a159126 100644 --- a/.claude/skills/grafana-dashboards/SKILL.md +++ b/.claude/skills/grafana-dashboards/SKILL.md @@ -40,8 +40,9 @@ the iroh relay's in `monitoring/iroh-relay/`): - `prometheus-alerts.yml` — three groups: `decdn-slash-safety`, `decdn-liveness`, `decdn-delivery`. Every rule carries a `component` label. Only rules with a matching section in upstream's `docs/runbook.md` (20 of 51) also carry a `runbook_url`. - `DecdnNodeDown` has promtool unit tests in `prometheus-alerts_test.yml` (the - chart's `.helmignore` keeps that file out of the package): keep them in step. + `DecdnNodeDown` and `DecdnCapacityBondRegistryResyncFailing` have promtool unit + tests in `prometheus-alerts_test.yml` (the chart's `.helmignore` keeps that file + out of the package): keep them in step. - `monitoring/sponsord/` — the onboarding sponsor's own pair: `dashboard-sponsord.json` (uid `decdn-sponsord`) and `prometheus-alerts.yml` (group `sponsord`, `job="sponsord"`). diff --git a/monitoring/README.md b/monitoring/README.md index 0aa78b6..a4dc2e1 100644 --- a/monitoring/README.md +++ b/monitoring/README.md @@ -15,7 +15,7 @@ this repo's MIT license. sponsord's were written here. | `decdn-node/dashboard-chain.json` | Chain, payments and slash safety (`uid: decdn-chain`): watcher liveness, chain RPC, registries, payments. | | `decdn-node/dashboard-node.json` | Single-node drilldown (`uid: decdn-node`): host, process, iroh transport, DHT and probe, logs, traces. | | `decdn-node/prometheus-alerts.yml` | Rule groups `decdn-slash-safety`, `decdn-liveness`, `decdn-delivery`. A rule with a matching runbook section carries a `runbook_url` into upstream's `docs/runbook.md`. | -| `decdn-node/prometheus-alerts_test.yml` | promtool unit tests for `DecdnNodeDown` (run by `make lint-helm`; the chart's `.helmignore` keeps them out of the package). | +| `decdn-node/prometheus-alerts_test.yml` | promtool unit tests for `DecdnNodeDown` and `DecdnCapacityBondRegistryResyncFailing` (run by `make lint-helm`; the chart's `.helmignore` keeps them out of the package). | | `sponsord/` | The onboarding sponsor's dashboard and alert rules ([below](#sponsord)). Not rendered by the chart. | | `iroh-relay/` | The iroh relay's dashboard and alert rules ([below](#iroh-relay)). Not rendered by the chart. | diff --git a/monitoring/decdn-node/dashboard-chain.json b/monitoring/decdn-node/dashboard-chain.json index 8cc7986..b2b29d0 100644 --- a/monitoring/decdn-node/dashboard-chain.json +++ b/monitoring/decdn-node/dashboard-chain.json @@ -2224,19 +2224,10 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "title": "Voucher and ledger integrity", + "title": "Lane and receipt-log writes", "type": "timeseries", - "description": "A nonce gap means a voucher in the sequence was never seen. Dropped or failed receipt writes are lost audit-log rows, not lost revenue.", + "description": "Lane flush failures are background lane-store flushes that failed. Each retries on the next tick, but the frontier a crash would replay keeps growing (DecdnLaneFlushFailures). Dropped or failed receipt writes are lost audit-log rows, not lost revenue.", "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(decdn_voucher_nonce_gaps_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", - "refId": "A", - "legendFormat": "voucher nonce gaps" - }, { "datasource": { "type": "prometheus", @@ -2831,7 +2822,7 @@ }, "title": "Pool-open failures by cause", "type": "timeseries", - "description": "Sibling counters for the openPool tx. The aggregate decdn_node_pull_pool_open_failures_total on the delivery dashboard is larger: it also counts store and expired-reclaim causes that never reach a transaction.", + "description": "Sibling counters for the openPool tx. The aggregate decdn_node_pull_pool_open_failures_total on the delivery dashboard is larger: it also counts store, open-task, capability-signing and lane-seed causes that never reach a transaction. Buyer ledger integrity's lane seed failures are the lane-seed share.", "targets": [ { "datasource": { @@ -2943,7 +2934,7 @@ }, "title": "Buyer ledger integrity", "type": "timeseries", - "description": "A skipped undecodable record is a buyer-pool row silently omitted from an otherwise successful store hydration, counted once per row per load. A lane seed failure is a pull refused because the lane's already-paid watermark could not be read; a sustained rate means this node buys nothing from the affected providers. A pool adoption failure means bootstrap could not tell whether this node already owns a pool, so the first miss may escrow a second deposit; it runs once per process, so read it as a value since start.", + "description": "A skipped undecodable record is a buyer-pool row silently omitted from an otherwise successful store hydration, counted once per row per load. A lane seed failure is a pull refused because the lane's already-paid watermark could not be established: getWatermark failed or timed out, the buyer store faulted on a read under the seed lock, the buyer row vanished before or during the seed, or the store could not persist the seed. Each also counts as a pool open failure. A sustained rate means this node buys nothing from the affected providers. A pool adoption failure means bootstrap could not tell whether this node already owns a pool, so the first miss may escrow a second deposit; it runs once per process, so read it as a value since start.", "targets": [ { "datasource": { @@ -3357,7 +3348,7 @@ }, "title": "RPC failures by outcome", "type": "timeseries", - "description": "Failed JSON-RPC requests per second, by outcome. rate_limited: the provider throttled the node (HTTP 429, Retry-After, or a rate-limit error code); raise the plan or lower the poll cadence. transport_error: no usable reply (refused or reset connection, HTTP 5xx, unreadable body). timeout: the request did not resolve before its deadline. rpc_error: any other JSON-RPC error response, such as an eth_getLogs range cap. reverted: the call reverted; this is not a provider fault and is shown for context only.", + "description": "Failed JSON-RPC requests per second, by outcome. rate_limited: the provider throttled the node (HTTP 429, Retry-After, or a rate-limit error code); raise the plan or lower the poll cadence. transport_error: no usable reply (refused or reset connection, HTTP 5xx, unreadable body). timeout: the request did not resolve before its deadline. rpc_error: any other JSON-RPC error response. On eth_getLogs that is a provider's transient error (such as \"temporary internal error\"), which the poller retries in place and defers to the next tick after the last retry, or a range cap, which shrinks the window; the eth_getLogs window span panel's retries/s and rejections/s tell them apart. While the watchers keep ticking, a steady rpc_error rate is an unreliable provider the node is absorbing. reverted: the call reverted; this is not a provider fault and is shown for context only.", "targets": [ { "datasource": { diff --git a/monitoring/decdn-node/dashboard-delivery.json b/monitoring/decdn-node/dashboard-delivery.json index f181072..8c459c0 100644 --- a/monitoring/decdn-node/dashboard-delivery.json +++ b/monitoring/decdn-node/dashboard-delivery.json @@ -1700,7 +1700,7 @@ }, "title": "Pool plumbing on the pull leg", "type": "timeseries", - "description": "Buyer-side pool trouble on node-to-node miss fills. pool open failures: this node's own pool open or reuse failed (gas, RPC, an expired pool); a sustained rate means node-to-node buying is wedged. pool open pending: an open was still in flight when the candidate budget ran out, so the pull moved on; not a failure. pool wedged: an upstream rejected this node's voucher on terms the lane cannot recover from; that provider is suppressed for an hour and the deposit stays escrowed for the others. recovery step: every candidate refused this node's funding, so a funding recovery step (ADR 003 § Funding recovery) topped up, replaced or settled the buyer pool; rare once buyer_working_deposit_micro_usdc fits the blobs this node pulls. recovery step refused: the step could not fund the next pass (funding off, nothing landed, or the transaction failed; a failed transaction also ticks decdn_buyer_topup_failure_total).", + "description": "Buyer-side pool trouble on node-to-node miss fills. pool open failures: this node's own pool open or reuse failed (gas, RPC, the buyer store, or a lane seed refused: decdn_buyer_lane_seed_failures_total on the chain dashboard is that share); a sustained rate means node-to-node buying is wedged. pool open pending: an open was still in flight when the candidate budget ran out, so the pull moved on; not a failure. pool wedged: an upstream rejected this node's voucher on terms the lane cannot recover from; that provider is suppressed for an hour and the deposit stays escrowed for the others. recovery step: every candidate refused this node's funding, so a funding recovery step (ADR 003 § Funding recovery) topped up, replaced or settled the buyer pool; rare once buyer_working_deposit_micro_usdc fits the blobs this node pulls. recovery step refused: the step could not fund the next pass (funding off, nothing landed, or the transaction failed; a failed transaction also ticks decdn_buyer_topup_failure_total).", "targets": [ { "datasource": { diff --git a/monitoring/decdn-node/dashboard-node.json b/monitoring/decdn-node/dashboard-node.json index 190d1e2..2cae1d9 100644 --- a/monitoring/decdn-node/dashboard-node.json +++ b/monitoring/decdn-node/dashboard-node.json @@ -1260,14 +1260,14 @@ }, "title": "Peers known", "type": "stat", - "description": "Entries in this node's address directory: the NodeId → operator address bindings the node-to-node pull leg resolves against. Scoped to nodes that have attempted a pull — every metric is registered unconditionally, so a cache-only node exports a permanent 0 that is not a collapsed directory, and only having attempted a pull distinguishes the two (#1231, #2072).", + "description": "Entries in this node's address directory: the NodeId → operator address bindings the node-to-node pull leg resolves against. Scoped to nodes that have attempted a pull in the last day (max_over_time, so a restart, which resets the counter, does not blank it) — every metric is registered unconditionally, so a cache-only node exports a permanent 0 that is not a collapsed directory, and only having attempted a pull distinguishes the two (#1231, #2072).", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "decdn_node_address_directory_size{instance=~\"$node\"}\n and on(instance) (decdn_node_pull_attempts_total > 0)", + "expr": "decdn_node_address_directory_size{instance=~\"$node\"}\n and on(instance) (max_over_time(decdn_node_pull_attempts_total[1d]) > 0)", "refId": "A" } ], diff --git a/monitoring/decdn-node/prometheus-alerts.yml b/monitoring/decdn-node/prometheus-alerts.yml index 67e7dc8..23d4589 100644 --- a/monitoring/decdn-node/prometheus-alerts.yml +++ b/monitoring/decdn-node/prometheus-alerts.yml @@ -396,8 +396,13 @@ groups: # pushes two increments further apart than that — which resets `for` and # means a permanently failing resync never fires. `increase` over 1h always # spans several attempts. See adr/appendix-observability.md. + # + # `>= 2` asks for two failed attempts out of the hour's four: one + # transient eth_call error is retried 15m later and is not a failing + # repair. `increase` only scales the raw delta up, so one failure reads + # about 1.0 and never reaches 2. - alert: DecdnCapacityBondRegistryResyncFailing - expr: increase(decdn_capacity_bond_registry_resync_failures_total[1h]) > 0 + expr: increase(decdn_capacity_bond_registry_resync_failures_total[1h]) >= 2 for: 15m labels: component: registry @@ -406,10 +411,10 @@ groups: runbook_url: "https://github.com/decdn/decdn/blob/main/docs/runbook.md#active-staker-set-repair-not-running" summary: "Capacity-bond registry re-enumeration failing on {{ $labels.instance }}" description: | - The repair for a drifted active-staker set is itself failing, so a - membership change dropped by a failed nodeIdOf stays dropped. Check - the RPC provider's eth_call path; correlate with - decdn_staker_set_watcher_resolve_failures_total. + The repair for a drifted active-staker set failed at least twice in + the last hour, so a membership change dropped by a failed nodeIdOf + stays dropped. Check the RPC provider's eth_call path; correlate + with decdn_staker_set_watcher_resolve_failures_total. - alert: DecdnCapacityBondRegistryResyncStale # The `> 0` guard covers only the window before bootstrap completes: @@ -570,7 +575,16 @@ groups: "RPC failures by outcome" panel names the cause: rate_limited means the provider plan is too small for the node's poll cadence; transport_error and timeout mean the provider is unreachable or - overloaded; rpc_error usually means an eth_getLogs range cap. + overloaded; rpc_error is any other JSON-RPC error response. On + eth_getLogs that is a provider's transient error (such as "temporary + internal error"), which the poller retries in place + (decdn_chain_get_logs_retries_total) and defers to the next tick + after the last retry (decdn_chain_get_logs_deferred_total), or a + range cap, which shrinks the window + (decdn_chain_get_logs_range_rejections_total). Firing while the + watchers keep ticking means the node is absorbing an unreliable + provider: move to a better provider or plan. Lag that reaches the + watchers fires DecdnChainWatcherFlapping or a *WatcherStalled rule. - alert: DecdnConfigReloadFailed expr: increase(decdn_config_reload_failures_total[15m]) > 0 @@ -1019,11 +1033,15 @@ groups: component: payments severity: warning annotations: - summary: "Buyer lane watermark reads failing on {{ $labels.instance }}" + summary: "Buyer lane seeds refused on {{ $labels.instance }}" description: | - The node keeps refusing node-to-node pulls because it cannot read - a lane's already-paid watermark. It refuses rather than resume - the lane from zero, which would strand the lane for good. While + The node keeps refusing node-to-node pulls because it cannot + establish a lane's already-paid watermark: getWatermark failed or + timed out, the buyer store faulted on a read under the seed lock, + the buyer row vanished before or during the seed, or the store + could not persist the seed. It refuses rather than resume the lane + from zero, which would strand the lane for good. Each refusal also + counts on decdn_node_pull_pool_open_failures_total. While this holds the node buys nothing from the affected providers and their misses fall through to origin. Check the RPC provider and the buyer store (buyer.redb). diff --git a/monitoring/decdn-node/prometheus-alerts_test.yml b/monitoring/decdn-node/prometheus-alerts_test.yml index af1a8cc..11375ec 100644 --- a/monitoring/decdn-node/prometheus-alerts_test.yml +++ b/monitoring/decdn-node/prometheus-alerts_test.yml @@ -1,7 +1,9 @@ # promtool unit tests for prometheus-alerts.yml beside it, run by the chart's # tests/render-test.sh (`make lint-helm`). DecdnNodeDown's absence arm only shows -# itself on a series that stops, which promtool check cannot catch. The test clock -# starts at 0. The chart's .helmignore keeps this file out of the package. +# itself on a series that stops, and DecdnCapacityBondRegistryResyncFailing's +# threshold only on `increase` extrapolation, neither of which promtool check can +# catch. The test clock starts at 0. The chart's .helmignore keeps this file out of +# the package. # # It does not run in place: render-test.sh copies it into a scratch directory as # decdn-node-alerts_test.yml, beside a copy of prometheus-alerts.yml with the @@ -92,3 +94,29 @@ tests: - eval_time: 25h alertname: DecdnNodeDown exp_alerts: [] + + # Capacity-bond registry resync: attempts are 15m apart. a fails once (15m) and + # never fires: one failure reads about 1.0. b fails twice (15m, 30m), so the + # expression turns true at 30m and fires after the 15m for. It resolves once the + # first failure leaves the 1h window (checked at 1h20m). + - interval: 1m + input_series: + - series: 'decdn_capacity_bond_registry_resync_failures_total{job="decdn-node", instance="a"}' + values: '0x14 1x100' + - series: 'decdn_capacity_bond_registry_resync_failures_total{job="decdn-node", instance="b"}' + values: '0x14 1x14 2x100' + alert_rule_test: + - eval_time: 44m + alertname: DecdnCapacityBondRegistryResyncFailing + exp_alerts: [] + - eval_time: 45m + alertname: DecdnCapacityBondRegistryResyncFailing + exp_alerts: + - exp_labels: {job: decdn-node, instance: b, component: registry, severity: warning} + - eval_time: 1h + alertname: DecdnCapacityBondRegistryResyncFailing + exp_alerts: + - exp_labels: {job: decdn-node, instance: b, component: registry, severity: warning} + - eval_time: 1h20m + alertname: DecdnCapacityBondRegistryResyncFailing + exp_alerts: [] From 93ea508874cc4884d2541348e54c247a5d5a3d66 Mon Sep 17 00:00:00 2001 From: Ant Somers Date: Thu, 15 Oct 2026 01:19:56 +0300 Subject: [PATCH 2/2] fix(monitoring): describe the Peers known gate as a nonzero counter, not a recent pull Co-Authored-By: Claude Opus 5.5 --- monitoring/decdn-node/dashboard-node.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monitoring/decdn-node/dashboard-node.json b/monitoring/decdn-node/dashboard-node.json index 2cae1d9..7d67533 100644 --- a/monitoring/decdn-node/dashboard-node.json +++ b/monitoring/decdn-node/dashboard-node.json @@ -1260,7 +1260,7 @@ }, "title": "Peers known", "type": "stat", - "description": "Entries in this node's address directory: the NodeId → operator address bindings the node-to-node pull leg resolves against. Scoped to nodes that have attempted a pull in the last day (max_over_time, so a restart, which resets the counter, does not blank it) — every metric is registered unconditionally, so a cache-only node exports a permanent 0 that is not a collapsed directory, and only having attempted a pull distinguishes the two (#1231, #2072).", + "description": "Entries in this node's address directory: the NodeId → operator address bindings the node-to-node pull leg resolves against. Scoped to nodes whose pull-attempt counter was nonzero at some point in the last day; max_over_time keeps a restart, which resets the counter, from blanking it — every metric is registered unconditionally, so a cache-only node exports a permanent 0 that is not a collapsed directory, and only having attempted a pull distinguishes the two (#1231, #2072).", "targets": [ { "datasource": {