From 86ca4b0bf96b78f0d8c339b323a751458774016a Mon Sep 17 00:00:00 2001 From: Ant Somers Date: Fri, 25 Sep 2026 05:48:17 +0300 Subject: [PATCH] chore: sync upstream decdn/decdn @ 3ebf5f17 Re-sync the upstream mirrors and wire the one new config key. - charts/decdn-node/files/monitoring: re-vendored (new DecdnChainWatcherFlapping alert, probe latency/TTFB and eth_getLogs span panels). - schema-keys.txt: regenerated (155 -> 156 paths); the tracked ref in gen-schema-keys.py and the role README moves d3bc7da7 -> 3ebf5f17. - decdn_node: new decdn_get_logs_max_block_span knob, rendered as blockchain.get_logs_max_block_span (#2152), >= 1 when set, following decdn_fee_shares_poll_interval_sec through defaults, template, the shape and strictly-positive asserts, README, schema converge and the chart's ci-values. - molecule/validation: range-min1 case, the first negative test of the ">= 1 when set" assert. - molecule/schema/verify.yml: the key-count comment now matches the render (130, slack 5). networks.yml was already current. IPv6 bind (#2144) and the watchdog heartbeat (#2145) were already covered; the other upstream commits need no devops change. Co-Authored-By: Claude Opus 5.5 --- ansible/galaxy/CHANGELOG.md | 12 + ansible/molecule/schema/converge.yml | 1 + .../molecule/schema/files/gen-schema-keys.py | 2 +- ansible/molecule/schema/files/schema-keys.txt | 3 +- ansible/molecule/schema/verify.yml | 4 +- ansible/molecule/validation/converge.yml | 19 +- ansible/roles/decdn_node/README.md | 9 +- ansible/roles/decdn_node/defaults/main.yml | 4 + ansible/roles/decdn_node/tasks/main.yml | 6 +- .../roles/decdn_node/templates/node.toml.j2 | 3 + charts/decdn-node/ci/ci-values.yaml | 1 + charts/decdn-node/files/monitoring/SOURCE | 12 +- .../files/monitoring/dashboard-chain.json | 179 +++++++++-- .../files/monitoring/dashboard-delivery.json | 299 ++++++++++++++++-- .../files/monitoring/dashboard-node.json | 137 +++++++- .../files/monitoring/grafana-dashboard.json | 4 +- .../files/monitoring/prometheus-alerts.yml | 118 +++++-- 17 files changed, 696 insertions(+), 117 deletions(-) diff --git a/ansible/galaxy/CHANGELOG.md b/ansible/galaxy/CHANGELOG.md index 9cfc9aa..66f5a45 100644 --- a/ansible/galaxy/CHANGELOG.md +++ b/ansible/galaxy/CHANGELOG.md @@ -8,6 +8,15 @@ collection adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) ### Added +- `decdn_get_logs_max_block_span` (default `""`, daemon default 10000, `>= 1`): the + ceiling on one chain-watcher `eth_getLogs` block span, rendered as + `blockchain.get_logs_max_block_span` (decdn/decdn @ 3ebf5f17). The poller halves + its window on a provider range rejection and grows it back toward this ceiling; + if rejections keep coming, set it to the lowest span reached while they do, not + the provider's quoted limit. A binary built without that commit rejects the key + and the role's `decdn config validate` gate fails the deploy, so leave it unset + there. + - `decdn_network` (default `""`): set it to `arbitrum-sepolia` and `decdn_chain_id` plus every contract address default to upstream's deployment manifest, mirrored into `roles/decdn_node/vars/main/networks.yml` by `scripts/sync-network-profiles.py`. @@ -69,6 +78,9 @@ collection adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) ### Changed +- The role now tracks the config schema of decdn/decdn @ 3ebf5f17 (was + d3bc7da7). + - `grafana_alloy` fails loud on an architecture Alloy has no package for, instead of a 404 at download time. diff --git a/ansible/molecule/schema/converge.yml b/ansible/molecule/schema/converge.yml index e5b3c6e..0bcf51d 100644 --- a/ansible/molecule/schema/converge.yml +++ b/ansible/molecule/schema/converge.yml @@ -51,6 +51,7 @@ decdn_rpc_watchdog_interval_sec: 15 decdn_event_poll_interval_ms: 5000 decdn_fee_shares_poll_interval_sec: 1800 + decdn_get_logs_max_block_span: 2000 decdn_redeem_threshold_micro_usdc: 2000000 decdn_redeem_max_vouchers_per_tx: 250 decdn_redeem_interval_secs: 120 diff --git a/ansible/molecule/schema/files/gen-schema-keys.py b/ansible/molecule/schema/files/gen-schema-keys.py index 9aef2c3..f1eb9ed 100755 --- a/ansible/molecule/schema/files/gen-schema-keys.py +++ b/ansible/molecule/schema/files/gen-schema-keys.py @@ -135,7 +135,7 @@ def main(): "# a leaf-name inventory cannot tell the two apart. See gen-schema-keys.py for\n" "# the struct -> path mapping and molecule/schema/README.md for why this exists.\n" "#\n" - f"# Synced from decdn/decdn @ d3bc7da7 (crate version 0.0.0): {len(paths)} paths." + f"# Synced from decdn/decdn @ 3ebf5f17 (crate version 0.0.0): {len(paths)} paths." ) for path in sorted(paths): print(path) diff --git a/ansible/molecule/schema/files/schema-keys.txt b/ansible/molecule/schema/files/schema-keys.txt index ee84184..1f5a86f 100644 --- a/ansible/molecule/schema/files/schema-keys.txt +++ b/ansible/molecule/schema/files/schema-keys.txt @@ -9,7 +9,7 @@ # a leaf-name inventory cannot tell the two apart. See gen-schema-keys.py for # the struct -> path mapping and molecule/schema/README.md for why this exists. # -# Synced from decdn/decdn @ d3bc7da7 (crate version 0.0.0): 155 paths. +# Synced from decdn/decdn @ 3ebf5f17 (crate version 0.0.0): 156 paths. blockchain.buyer_max_approve blockchain.buyer_working_deposit_micro_usdc blockchain.capacity_bond_address @@ -20,6 +20,7 @@ blockchain.content_blacklist_poll_interval_sec blockchain.eth_keystore blockchain.event_poll_interval_ms blockchain.fee_shares_poll_interval_sec +blockchain.get_logs_max_block_span blockchain.origin_assignment_address blockchain.origin_directory_cache_capacity blockchain.origin_directory_negative_ttl_sec diff --git a/ansible/molecule/schema/verify.yml b/ansible/molecule/schema/verify.yml index 047b8b2..f826e61 100644 --- a/ansible/molecule/schema/verify.yml +++ b/ansible/molecule/schema/verify.yml @@ -131,8 +131,8 @@ exit 1 fi # Key floor stays a count: knobs are added upstream routinely, so an exact - # number would need editing on every sync. Today's render is 128 (this sync - # removed two knobs), so the slack is 3 — the next upstream removal trips + # number would need editing on every sync. Today's render is 130 (decdn/decdn + # @ 3ebf5f17), so the slack is 5 — the next upstream removals trip # this, and the fix is to re-count and lower the floor, not to widen it blindly. if [ "$keys" -lt 125 ]; then echo "only $keys scalar keys rendered (expected >= 125) — the converge" >&2 diff --git a/ansible/molecule/validation/converge.yml b/ansible/molecule/validation/converge.yml index ee302f7..26f1230 100644 --- a/ansible/molecule/validation/converge.yml +++ b/ansible/molecule/validation/converge.yml @@ -421,6 +421,21 @@ decdn_rejected: "{{ decdn_rejected + ['range'] }}" when: ansible_failed_task.name is match('^Validate optional') + # --- Case: range-min1 (0 for a knob the daemon requires strictly positive) ---- + # 0 passes the non-negative shape assert, so only the ranges task can reject it. + - name: "Case range-min1 — get_logs_max_block_span of 0" + block: + - name: Run decdn_node with a zero getLogs span ceiling + ansible.builtin.include_role: + name: decdn_node + vars: + decdn_get_logs_max_block_span: 0 + rescue: + - name: Record range-min1 rejection (only if the ranges assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['range-min1'] }}" + when: ansible_failed_task.name is match('^Validate optional knob ranges the daemon rejects') + # --- Case: shape (non-numeric string for an integer knob) --------------------- - name: "Case shape — non-numeric integer knob" block: @@ -1623,8 +1638,8 @@ _decdn_expected: ["cross-field", "binary-format", "binary-check-mode", "arch-mismatch", "arch-unset", "network-unknown", "network-unknown-noaddr", "network-chain", "backup-secret-key", "backup-bare-string", - "backup-empty", "path-cache-home", "path-backup-home", "path-decommission-cache", "range", "bounded-range", - "shape", "bool", + "backup-empty", "path-cache-home", "path-backup-home", "path-decommission-cache", "range", + "range-min1", "bounded-range", "shape", "bool", "enum", "dict", "float", "list", "string", "string-newline", "otlp-https", "otlp-noport", "otlp-path", "otlp-userinfo", "otlp-colon", "otlp-port", "otlp-newline", "env-control-rpc", "env-control-extra", "env-gate", diff --git a/ansible/roles/decdn_node/README.md b/ansible/roles/decdn_node/README.md index 1705c4a..2f31113 100644 --- a/ansible/roles/decdn_node/README.md +++ b/ansible/roles/decdn_node/README.md @@ -12,7 +12,7 @@ machine. This is the repo's deployment (`playbooks/site.yml`). > download. Build the two binaries from a checkout until that changes. **Schema tracking.** This role renders `node.toml` against the config schema of -`decdn/decdn` main @ `d3bc7da7` (crate version 0.0.0 — unreleased). Upstream marks every config +`decdn/decdn` main @ `3ebf5f17` (crate version 0.0.0 — unreleased). Upstream marks every config section `#[serde(deny_unknown_fields)]` and defines **no** serde aliases, so a key this role emits that your binary does not know is a startup crash-loop, not a warning. The role runs `decdn config validate` against the installed binary after @@ -261,7 +261,12 @@ asserts miss. See `defaults/main.yml` for every knob's upstream default, unit an `decdn_event_poll_interval_ms` (`>= 250`), `decdn_content_blacklist_poll_interval_sec` (`>= 1`), `decdn_chain_staleness_grace_sec` (seconds the node may go without a successful chain read before it stops serving — ADR 011; `> 0`, daemon default - `1800`), `decdn_fee_shares_poll_interval_sec`, + `1800`), `decdn_fee_shares_poll_interval_sec`, `decdn_get_logs_max_block_span` + (`>= 1`, daemon default `10000`: the ceiling on one `eth_getLogs` block span. The + poller halves its window on a range rejection and grows it back, so leave this + unset unless `decdn_chain_get_logs_range_rejections_total` keeps rising; then set + it to the lowest `decdn_chain_get_logs_span` reached while it does, not to the + limit your provider quotes), and the `decdn_origin_directory_*` cache knobs. - **Network** — `decdn_relay_urls` (list; the singular `relay_url` config key no longer exists). Operator-run address discovery (#818) via diff --git a/ansible/roles/decdn_node/defaults/main.yml b/ansible/roles/decdn_node/defaults/main.yml index 1c480a4..0f5cd45 100644 --- a/ansible/roles/decdn_node/defaults/main.yml +++ b/ansible/roles/decdn_node/defaults/main.yml @@ -154,6 +154,10 @@ decdn_pool_floor_signer_live_windows: "" # daemon dflt 8; lower-clamped to 1 wi # --- Blockchain watcher tuning ("" => daemon default) ------------------------- decdn_rpc_watchdog_interval_sec: "" # daemon dflt 30; 0 disables, else >= 10 — RPC watchdog decdn_event_poll_interval_ms: "" # daemon dflt 7000; >= 250 — multiplexed getLogs poll +# Ceiling on one eth_getLogs block span. The poller halves it on a provider range +# rejection and grows back toward it. If rejections keep coming, set it to the +# lowest span reached while they do, not the provider's quoted limit. +decdn_get_logs_max_block_span: "" # daemon dflt 10_000; >= 1 — getLogs span ceiling decdn_content_blacklist_poll_interval_sec: "" # daemon dflt 600; >= 1 — blacklist re-scope cadence # Seconds the node may go without a successful chain read before it stops serving # (ADR 011 §Serving while chain-stale). Always evaluated (no on/off flag) and only diff --git a/ansible/roles/decdn_node/tasks/main.yml b/ansible/roles/decdn_node/tasks/main.yml index 6007021..5517a3d 100644 --- a/ansible/roles/decdn_node/tasks/main.yml +++ b/ansible/roles/decdn_node/tasks/main.yml @@ -729,7 +729,7 @@ decdn_pool_min_remaining_deposit_micro_usdc, decdn_pool_floor_signer_live_windows, decdn_chain_staleness_grace_sec, decdn_rpc_watchdog_interval_sec, decdn_event_poll_interval_ms, decdn_content_blacklist_poll_interval_sec, - decdn_fee_shares_poll_interval_sec, + decdn_fee_shares_poll_interval_sec, decdn_get_logs_max_block_span, decdn_origin_directory_positive_ttl_sec, decdn_origin_directory_negative_ttl_sec, decdn_origin_directory_cache_capacity, decdn_max_blob_size_mb, decdn_disk_headroom_mb, decdn_max_rate_per_mb, decdn_max_probe_holds, @@ -868,12 +868,12 @@ decdn_chain_staleness_grace_sec (>= 1); and each of decdn_redeem_threshold_micro_usdc, decdn_redeem_max_vouchers_per_tx, decdn_redeem_interval_secs, decdn_buyer_working_deposit_micro_usdc, - decdn_fee_shares_poll_interval_sec, + decdn_fee_shares_poll_interval_sec, decdn_get_logs_max_block_span, decdn_voucher_commit_interval_ms must be >= 1 when set ("{{ item }}" is not). loop: >- {{ [decdn_redeem_threshold_micro_usdc, decdn_redeem_max_vouchers_per_tx, decdn_redeem_interval_secs, decdn_buyer_working_deposit_micro_usdc, - decdn_fee_shares_poll_interval_sec, + decdn_fee_shares_poll_interval_sec, decdn_get_logs_max_block_span, decdn_voucher_commit_interval_ms] }} # Bounded ranges. Each is a closed interval the daemon range-checks at load, so an diff --git a/ansible/roles/decdn_node/templates/node.toml.j2 b/ansible/roles/decdn_node/templates/node.toml.j2 index b58c85e..1162fff 100644 --- a/ansible/roles/decdn_node/templates/node.toml.j2 +++ b/ansible/roles/decdn_node/templates/node.toml.j2 @@ -82,6 +82,9 @@ rpc_watchdog_interval_sec = {{ decdn_rpc_watchdog_interval_sec | int }} {% if decdn_event_poll_interval_ms not in ["", none] %} event_poll_interval_ms = {{ decdn_event_poll_interval_ms | int }} {% endif %} +{% if decdn_get_logs_max_block_span not in ["", none] %} +get_logs_max_block_span = {{ decdn_get_logs_max_block_span | int }} +{% endif %} {% if decdn_fee_shares_poll_interval_sec not in ["", none] %} fee_shares_poll_interval_sec = {{ decdn_fee_shares_poll_interval_sec | int }} {% endif %} diff --git a/charts/decdn-node/ci/ci-values.yaml b/charts/decdn-node/ci/ci-values.yaml index e107595..cb68a91 100644 --- a/charts/decdn-node/ci/ci-values.yaml +++ b/charts/decdn-node/ci/ci-values.yaml @@ -66,6 +66,7 @@ config: rpc_watchdog_interval_sec: 15 event_poll_interval_ms: 5000 fee_shares_poll_interval_sec: 1800 + get_logs_max_block_span: 2000 redeem_threshold_micro_usdc: 2000000 redeem_max_vouchers_per_tx: 250 redeem_interval_secs: 120 diff --git a/charts/decdn-node/files/monitoring/SOURCE b/charts/decdn-node/files/monitoring/SOURCE index f615b0a..c8e26ce 100644 --- a/charts/decdn-node/files/monitoring/SOURCE +++ b/charts/decdn-node/files/monitoring/SOURCE @@ -1,9 +1,9 @@ # GENERATED by scripts/sync-monitoring.sh -- do not hand-edit. upstream: decdn/decdn -commit: bdba9e60c0d51becc614bc8c23b7e4508f6418cf +commit: 3ebf5f17eec0399d43a744e6efbb997675e371d7 files: - dashboard-chain.json: sha256:ff74aadb34f42930f11ddbdbed263a097a31f91524a7b13da1d71b1b5c325b1f - dashboard-delivery.json: sha256:2f5faa7e91da9ce5b5136399894cfee2ff87ae2ed290297f5406bf5e2371f9ec - dashboard-node.json: sha256:9f58cb3672abf0d9b63f7fb72bbde402644d7f0df4e1b6509e4bbefed89006a4 - grafana-dashboard.json: sha256:c000677c88eeb20f15a4480ae06dacca47199a249000d18f79668a9b0c00a616 - prometheus-alerts.yml: sha256:69de9f66074ecefafc1962ac225a42927ac5513c2cbfc0719ef593a5e9f2c858 + dashboard-chain.json: sha256:f547f355f93fd17ffbcedb53c2e90c53a78e6f2f3d62f92dc5c84a27aea5000b + dashboard-delivery.json: sha256:bd933f571cb866102bd7b594045906e29d9b30b50cddf4aa4b16dc3543415d1f + dashboard-node.json: sha256:ec73ce192e0cc3823aa42f0b157515fffe58ddf6c6f88d1bc45b7cf1bf6c3ecc + grafana-dashboard.json: sha256:4b3d4512352ae33761152df9ad6bda8bc1b5d20ef709eb0b478113ccc20ab53a + prometheus-alerts.yml: sha256:e1296916f92151afb7d8dc72e5f75e545fe5e306a95addfcb4d178d13bc93be9 diff --git a/charts/decdn-node/files/monitoring/dashboard-chain.json b/charts/decdn-node/files/monitoring/dashboard-chain.json index 4d981eb..df98e61 100644 --- a/charts/decdn-node/files/monitoring/dashboard-chain.json +++ b/charts/decdn-node/files/monitoring/dashboard-chain.json @@ -103,14 +103,14 @@ }, "title": "Watcher tick age", "type": "table", - "description": "Age of each watcher's last successful poll tick, stamped every tick including idle ones, so this is a positive liveness signal rather than an absence of errors. Built with label_replace on the raw selector because `time() - x` drops __name__. Red at 180s matches the stalled-watcher alerts. A watcher that has never ticked reads 0 and is filtered out by the `> 0` guard rather than showing as infinitely stale.", + "description": "Age of each watcher's last successful poll tick, stamped every tick including idle ones, so this is a positive liveness signal rather than an absence of errors. Built with label_replace on the raw selector because `time() - x` drops __name__. Red at 180s matches the stalled-watcher alerts. A watcher that has never ticked reads 0, so the `> 0` guard drops its tick age; if its poll keeps failing, the row shows its down-seconds (time since the first failed tick) instead, matching the stalled-watcher alerts' second arm. A watcher that is simply not wired on this node shows no row.", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "time() - label_replace({__name__=~\"decdn_.+_watcher_last_tick_timestamp_seconds\",deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"} > 0, \"watcher\", \"$1\", \"__name__\", \"decdn_(.+)_watcher_last_tick_timestamp_seconds\")", + "expr": "(time() - label_replace({__name__=~\"decdn_.+_watcher_last_tick_timestamp_seconds\",deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"} > 0, \"watcher\", \"$1\", \"__name__\", \"decdn_(.+)_watcher_last_tick_timestamp_seconds\")) or (label_replace({__name__=~\"decdn_.+_watcher_down_seconds\",deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"} > 0, \"watcher\", \"$1\", \"__name__\", \"decdn_(.+)_watcher_down_seconds\") * 1)", "refId": "A", "instant": true, "range": false, @@ -736,13 +736,135 @@ }, "id": 6 }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "eth_getLogs window span", + "type": "timeseries", + "description": "Block span of the shared chain-event poller's next eth_getLogs window, per node, and the rate of windows the RPC provider rejected for their range or result count. The span starts at blockchain.get_logs_max_block_span, drops to half a rejected window, and doubles back after 32 accepted windows, so a caught-up node climbs back to the ceiling after a transient rejection. Rejections that keep coming mean the provider caps eth_getLogs: set the ceiling to the lowest span reached while they come.", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "decdn_chain_get_logs_span{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}", + "refId": "A", + "legendFormat": "{{instance}} span" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(decdn_chain_get_logs_range_rejections_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval])", + "refId": "B", + "legendFormat": "{{instance}} rejections/s" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit", + "value": "reqps" + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "min", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 32 + }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 17 + "y": 24 }, "id": 7, "panels": [], @@ -837,7 +959,7 @@ "h": 7, "w": 8, "x": 0, - "y": 18 + "y": 25 }, "id": 8 }, @@ -937,7 +1059,7 @@ "h": 7, "w": 8, "x": 8, - "y": 18 + "y": 25 }, "id": 9 }, @@ -1040,7 +1162,7 @@ "h": 7, "w": 8, "x": 16, - "y": 18 + "y": 25 }, "id": 10 }, @@ -1154,7 +1276,7 @@ "h": 12, "w": 14, "x": 0, - "y": 25 + "y": 32 }, "id": 27 }, @@ -1238,7 +1360,7 @@ "h": 12, "w": 5, "x": 14, - "y": 25 + "y": 32 }, "id": 29 }, @@ -1304,7 +1426,7 @@ "h": 12, "w": 5, "x": 19, - "y": 25 + "y": 32 }, "id": 28 }, @@ -1314,7 +1436,7 @@ "h": 1, "w": 24, "x": 0, - "y": 37 + "y": 44 }, "id": 11, "panels": [], @@ -1408,7 +1530,7 @@ "h": 7, "w": 8, "x": 0, - "y": 38 + "y": 45 }, "id": 12 }, @@ -1500,7 +1622,7 @@ "h": 7, "w": 8, "x": 8, - "y": 38 + "y": 45 }, "id": 13 }, @@ -1597,7 +1719,7 @@ "h": 7, "w": 8, "x": 16, - "y": 38 + "y": 45 }, "id": 14 }, @@ -1607,7 +1729,7 @@ "h": 1, "w": 24, "x": 0, - "y": 45 + "y": 52 }, "id": 15, "panels": [], @@ -1702,7 +1824,7 @@ "h": 7, "w": 8, "x": 0, - "y": 46 + "y": 53 }, "id": 16 }, @@ -1794,7 +1916,7 @@ "h": 7, "w": 8, "x": 8, - "y": 46 + "y": 53 }, "id": 17 }, @@ -1885,7 +2007,7 @@ "h": 7, "w": 8, "x": 16, - "y": 46 + "y": 53 }, "id": 18 }, @@ -2015,7 +2137,7 @@ "h": 7, "w": 8, "x": 0, - "y": 53 + "y": 60 }, "id": 19 }, @@ -2136,7 +2258,7 @@ "h": 7, "w": 8, "x": 8, - "y": 53 + "y": 60 }, "id": 20 }, @@ -2147,13 +2269,14 @@ }, "title": "Settlement watcher freshness", "type": "timeseries", + "description": "Age of the settlement watcher's last successful poll tick. A watcher that has never ticked reads 0, so the `> 0` guard drops its tick age; if its poll keeps failing, the series shows its down-seconds (time since the first failed tick) instead, matching DecdnSettlementWatcherStalled's second arm.", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "time() - (decdn_settlement_watcher_last_tick_timestamp_seconds{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"} > 0)", + "expr": "(time() - (decdn_settlement_watcher_last_tick_timestamp_seconds{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"} > 0)) or ((decdn_settlement_watcher_down_seconds{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"} > 0) * 1)", "refId": "A", "legendFormat": "{{instance}}" } @@ -2235,7 +2358,7 @@ "h": 7, "w": 8, "x": 16, - "y": 53 + "y": 60 }, "id": 21 }, @@ -2365,7 +2488,7 @@ "h": 8, "w": 12, "x": 0, - "y": 60 + "y": 67 }, "id": 30 }, @@ -2477,7 +2600,7 @@ "h": 8, "w": 12, "x": 12, - "y": 60 + "y": 67 }, "id": 31 }, @@ -2487,7 +2610,7 @@ "h": 1, "w": 24, "x": 0, - "y": 68 + "y": 75 }, "id": 22, "panels": [], @@ -2592,7 +2715,7 @@ "h": 7, "w": 8, "x": 0, - "y": 69 + "y": 76 }, "id": 23 }, @@ -2704,7 +2827,7 @@ "h": 7, "w": 8, "x": 8, - "y": 69 + "y": 76 }, "id": 24 }, @@ -2807,7 +2930,7 @@ "h": 7, "w": 8, "x": 16, - "y": 69 + "y": 76 }, "id": 25 }, @@ -2849,7 +2972,7 @@ "h": 9, "w": 24, "x": 0, - "y": 76 + "y": 83 }, "id": 26 } diff --git a/charts/decdn-node/files/monitoring/dashboard-delivery.json b/charts/decdn-node/files/monitoring/dashboard-delivery.json index ed44b8c..4ac76db 100644 --- a/charts/decdn-node/files/monitoring/dashboard-delivery.json +++ b/charts/decdn-node/files/monitoring/dashboard-delivery.json @@ -1035,13 +1035,151 @@ }, "id": 7 }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Serve time to first byte", + "type": "timeseries", + "description": "Fleet p50, p95 and p99 time to first byte of a paid serve, from the decoded request to the first ChunkData frame, split by the blob-availability gate class. A hit is served from held bytes. A miss also waits for discovery and the fill that produces its first frame. The first frame rides the opening credit window, so no client payment round trip is in the window. Refused and reset streams write no frame and are not in these series. The hit series and window-paced misses do not grow with blob size, so an SLO can use them. A buffered fill completes the whole blob before its first frame, so on that tier the miss series grows with blob size.", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.5, sum by (le) (rate(decdn_serve_first_byte_hit_seconds_bucket{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval])))", + "refId": "A", + "legendFormat": "hit p50" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(decdn_serve_first_byte_hit_seconds_bucket{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval])))", + "refId": "B", + "legendFormat": "hit p95" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(decdn_serve_first_byte_hit_seconds_bucket{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval])))", + "refId": "C", + "legendFormat": "hit p99" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.5, sum by (le) (rate(decdn_serve_first_byte_miss_seconds_bucket{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval])))", + "refId": "D", + "legendFormat": "miss p50" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(decdn_serve_first_byte_miss_seconds_bucket{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval])))", + "refId": "E", + "legendFormat": "miss p95" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(decdn_serve_first_byte_miss_seconds_bucket{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval])))", + "refId": "F", + "legendFormat": "miss p99" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 33 + }, + "id": 9009 + }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 33 + "y": 41 }, "id": 8, "panels": [], @@ -1146,7 +1284,7 @@ "h": 8, "w": 12, "x": 0, - "y": 34 + "y": 42 }, "id": 9 }, @@ -1348,7 +1486,7 @@ "h": 8, "w": 12, "x": 12, - "y": 34 + "y": 42 }, "id": 10 }, @@ -1477,7 +1615,7 @@ "h": 7, "w": 8, "x": 0, - "y": 42 + "y": 50 }, "id": 11 }, @@ -1606,7 +1744,7 @@ "h": 7, "w": 8, "x": 8, - "y": 42 + "y": 50 }, "id": 12 }, @@ -1735,17 +1873,128 @@ "h": 7, "w": 8, "x": 16, - "y": 42 + "y": 50 }, "id": 13 }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Pull time to first byte", + "type": "timeseries", + "description": "Fleet p50, p95 and p99 time to first byte of one paid node-to-node pull leg, from the start of its open to the first bao bytes read. The window includes a dial when the node holds no warm connection to the peer. An adopted header-handshake pull is timed from the start of the handshake open. A rising p95 with a flat serve-hit p95 points at the upstream peers or the path to them, not at the serving nodes\u2019 own stores.", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.5, sum by (le) (rate(decdn_node_pull_first_byte_seconds_bucket{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval])))", + "refId": "A", + "legendFormat": "pull leg p50" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(decdn_node_pull_first_byte_seconds_bucket{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval])))", + "refId": "B", + "legendFormat": "pull leg p95" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(decdn_node_pull_first_byte_seconds_bucket{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval])))", + "refId": "C", + "legendFormat": "pull leg p99" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 57 + }, + "id": 9010 + }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 49 + "y": 65 }, "id": 14, "panels": [], @@ -1828,7 +2077,7 @@ "h": 7, "w": 8, "x": 0, - "y": 50 + "y": 66 }, "id": 15 }, @@ -1930,7 +2179,7 @@ "h": 7, "w": 8, "x": 8, - "y": 50 + "y": 66 }, "id": 16 }, @@ -2041,7 +2290,7 @@ "h": 7, "w": 8, "x": 16, - "y": 50 + "y": 66 }, "id": 17 }, @@ -2161,7 +2410,7 @@ "h": 7, "w": 8, "x": 0, - "y": 57 + "y": 73 }, "id": 18 }, @@ -2261,7 +2510,7 @@ "h": 7, "w": 8, "x": 8, - "y": 57 + "y": 73 }, "id": 19 }, @@ -2391,7 +2640,7 @@ "h": 7, "w": 8, "x": 16, - "y": 57 + "y": 73 }, "id": 20 }, @@ -2401,7 +2650,7 @@ "h": 1, "w": 24, "x": 0, - "y": 64 + "y": 80 }, "id": 21, "panels": [], @@ -2533,7 +2782,7 @@ "h": 7, "w": 8, "x": 0, - "y": 65 + "y": 81 }, "id": 22 }, @@ -2644,7 +2893,7 @@ "h": 7, "w": 8, "x": 8, - "y": 65 + "y": 81 }, "id": 23 }, @@ -2756,7 +3005,7 @@ "h": 7, "w": 8, "x": 16, - "y": 65 + "y": 81 }, "id": 24 }, @@ -2867,7 +3116,7 @@ "h": 7, "w": 12, "x": 0, - "y": 72 + "y": 88 }, "id": 25 }, @@ -2909,7 +3158,7 @@ "h": 7, "w": 12, "x": 12, - "y": 72 + "y": 88 }, "id": 26 }, @@ -3001,7 +3250,7 @@ "h": 8, "w": 24, "x": 0, - "y": 79 + "y": 95 }, "id": 9003 }, @@ -3011,7 +3260,7 @@ "h": 1, "w": 24, "x": 0, - "y": 87 + "y": 103 }, "id": 9004, "panels": [], @@ -3115,7 +3364,7 @@ "h": 7, "w": 8, "x": 0, - "y": 88 + "y": 104 }, "id": 9005 }, @@ -3216,7 +3465,7 @@ "h": 7, "w": 8, "x": 8, - "y": 88 + "y": 104 }, "id": 9006 }, @@ -3326,7 +3575,7 @@ "h": 7, "w": 8, "x": 16, - "y": 88 + "y": 104 }, "id": 9007 }, @@ -3427,7 +3676,7 @@ "h": 7, "w": 8, "x": 0, - "y": 95 + "y": 111 }, "id": 9008 } diff --git a/charts/decdn-node/files/monitoring/dashboard-node.json b/charts/decdn-node/files/monitoring/dashboard-node.json index abe9a1b..46d7e5b 100644 --- a/charts/decdn-node/files/monitoring/dashboard-node.json +++ b/charts/decdn-node/files/monitoring/dashboard-node.json @@ -3618,13 +3618,126 @@ "id": 46, "description": "Distinct peers in the local Kademlia routing table, set after bootstrap and on every bucket-refresh tick." }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Probe collection latency", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.5, sum by (le) (rate(decdn_probe_collection_latency_seconds_bucket{instance=~\"$node\"}[$__rate_interval])))", + "refId": "A", + "legendFormat": "collection p50" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(decdn_probe_collection_latency_seconds_bucket{instance=~\"$node\"}[$__rate_interval])))", + "refId": "B", + "legendFormat": "collection p95" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(decdn_probe_collection_latency_seconds_bucket{instance=~\"$node\"}[$__rate_interval])))", + "refId": "C", + "legendFormat": "collection p99" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 92 + }, + "id": 48, + "description": "p50, p95 and p99 duration of one probe collection window on the cache-miss path, from the start of the concurrent probes (dials included) to the end of collection (ADR 001 §Probe response collection). Collection stops early once enough holders answer, or when every probe resolves; the 500 ms probe timeout bounds each probe. A p95 near 0.5 s means rounds wait on a slow or unreachable peer before enough holders answer. A probe-cache hit sends no probe and is not in these series." + }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 92 + "y": 99 }, "id": 34, "panels": [], @@ -3722,7 +3835,7 @@ "h": 7, "w": 8, "x": 0, - "y": 93 + "y": 100 }, "id": 35 }, @@ -3816,7 +3929,7 @@ "h": 7, "w": 8, "x": 8, - "y": 93 + "y": 100 }, "id": 36 }, @@ -3908,7 +4021,7 @@ "h": 7, "w": 8, "x": 16, - "y": 93 + "y": 100 }, "id": 37 }, @@ -3949,7 +4062,7 @@ "h": 11, "w": 24, "x": 0, - "y": 100 + "y": 107 }, "id": 38 }, @@ -3959,7 +4072,7 @@ "h": 1, "w": 24, "x": 0, - "y": 111 + "y": 118 }, "id": 39, "panels": [], @@ -3973,7 +4086,7 @@ }, "title": "Span rate by operation", "type": "timeseries", - "description": "Spans per second by name. Only deCDN spans are exported; dependency crates' spans are filtered at the node. serve_stream is one inbound client stream. A buffered miss nests pull_through, origin_pull, node_pull, upstream_stream and open_progressive_pull under it; a streaming miss nests serve_miss_pull and open_progressive_pull. origin_range_pull is a ranged origin fetch; dht_lookup, redeem_cycle and onchain_tx cover discovery and settlement.", + "description": "Spans per second by name. Only deCDN spans are exported; dependency crates' spans are filtered at the node. serve_stream is one inbound client stream. A buffered miss nests pull_through, origin_pull, node_pull, upstream_stream and open_progressive_pull under it; a streaming miss from a peer nests serve_miss_pull, upstream_stream and open_progressive_pull. origin_range_pull is a ranged origin fetch; dht_lookup, redeem_cycle and onchain_tx cover discovery and settlement.", "targets": [ { "datasource": { @@ -4057,7 +4170,7 @@ "h": 8, "w": 24, "x": 0, - "y": 112 + "y": 119 }, "id": 40 }, @@ -4152,7 +4265,7 @@ "h": 8, "w": 12, "x": 0, - "y": 120 + "y": 127 }, "id": 42 }, @@ -4185,7 +4298,7 @@ "h": 8, "w": 12, "x": 12, - "y": 120 + "y": 127 }, "id": 43 }, @@ -4196,7 +4309,7 @@ }, "title": "deCDN operation latency", "type": "timeseries", - "description": "p50 and p95 duration of each deCDN span. serve_stream covers a whole client stream, so its duration tracks blob size; upstream_stream and open_progressive_pull are the paid miss pull and its time to first byte; onchain_tx is one redeemMany from send to receipt. Filter a trace by outcome to split completed from refused or failed streams.", + "description": "p50 and p95 duration of each deCDN span, for drill-down. The latency SLOs read the histograms instead: probe collection latency above, and time to first byte on the delivery dashboard. serve_stream covers a whole client stream, so its duration tracks blob size; upstream_stream and open_progressive_pull are the paid miss pull and its time to first byte; onchain_tx is one redeemMany from send to receipt. Filter a trace by outcome to split completed from refused or failed streams.", "targets": [ { "datasource": { @@ -4280,7 +4393,7 @@ "h": 8, "w": 24, "x": 0, - "y": 128 + "y": 135 }, "id": 44 } diff --git a/charts/decdn-node/files/monitoring/grafana-dashboard.json b/charts/decdn-node/files/monitoring/grafana-dashboard.json index 368296a..cbd00d9 100644 --- a/charts/decdn-node/files/monitoring/grafana-dashboard.json +++ b/charts/decdn-node/files/monitoring/grafana-dashboard.json @@ -251,14 +251,14 @@ }, "title": "Oldest watcher tick", "type": "stat", - "description": "Age of the least recently ticked chain-event watcher, across all five watchers and every node. Red at 180s matches DecdnBlacklistWatcherStalled \u2014 a stalled blacklist watcher is a slash risk, not a cosmetic lag.", + "description": "Age of the least recently ticked chain-event watcher, across all five watchers and every node. A watcher that has never ticked because its poll keeps failing counts by its down-seconds (time since the first failed tick) instead of dropping out. Red at 180s matches DecdnBlacklistWatcherStalled \u2014 a stalled blacklist watcher is a slash risk, not a cosmetic lag.", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "max(time() - label_replace({__name__=~\"decdn_.+_watcher_last_tick_timestamp_seconds\",deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"} > 0, \"watcher\", \"$1\", \"__name__\", \"decdn_(.+)_watcher_last_tick_timestamp_seconds\"))", + "expr": "max((time() - label_replace({__name__=~\"decdn_.+_watcher_last_tick_timestamp_seconds\",deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"} > 0, \"watcher\", \"$1\", \"__name__\", \"decdn_(.+)_watcher_last_tick_timestamp_seconds\")) or (label_replace({__name__=~\"decdn_.+_watcher_down_seconds\",deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"} > 0, \"watcher\", \"$1\", \"__name__\", \"decdn_(.+)_watcher_down_seconds\") * 1))", "refId": "A" } ], diff --git a/charts/decdn-node/files/monitoring/prometheus-alerts.yml b/charts/decdn-node/files/monitoring/prometheus-alerts.yml index 51c73d2..ade29c3 100644 --- a/charts/decdn-node/files/monitoring/prometheus-alerts.yml +++ b/charts/decdn-node/files/monitoring/prometheus-alerts.yml @@ -57,13 +57,17 @@ groups: # alert on. DecdnBlacklistWatcherStalled below is the coverage: it fires on # staleness of the watcher's last successful tick, which IS exported. - alert: DecdnBlacklistWatcherStalled - # Positive liveness: the gauge is stamped every successful poll tick, so - # staleness catches a panicked, wedged, or exited watcher that the - # error-triggered down-seconds gauge reads as a healthy 0 (#1316/#1320). - # 180s ≈ several poll intervals; tune to content_blacklist_poll_interval_sec. - # The `> 0` guard skips the pre-first-tick sentinel (the gauge reads 0 - # until the first successful tick), so a fresh boot doesn't self-fire. - expr: (decdn_blacklist_watcher_last_tick_timestamp_seconds > 0) and (time() - decdn_blacklist_watcher_last_tick_timestamp_seconds > 180) + # Two arms. Positive liveness: the tick gauge is stamped every + # successful poll tick, so its staleness catches a panicked, wedged, or + # exited watcher that the error-triggered down-seconds gauge reads as a + # healthy 0 (#1316/#1320). The `> 0` guard skips the pre-first-tick + # sentinel (the gauge reads 0 until the first successful tick), so a + # fresh boot doesn't self-fire. Failing ticks: down-seconds counts from + # the first failed poll tick, so it also catches a watcher that never + # ticked because every poll tick fails (a dead or rate-limiting RPC, or one + # that rejects even a one-block eth_getLogs window), + # which the guarded tick arm cannot see. 180s ≈ several poll intervals. + expr: ((decdn_blacklist_watcher_last_tick_timestamp_seconds > 0) and (time() - decdn_blacklist_watcher_last_tick_timestamp_seconds > 180)) or (decdn_blacklist_watcher_down_seconds > 180) for: 5m labels: component: blacklist @@ -74,9 +78,11 @@ groups: summary: "Blacklist watcher not ticking on {{ $labels.instance }}" description: | No successful blacklist poll tick for several intervals — the - watcher has panicked, wedged, or exited. The node may be serving - content blacklisted after the failure, which is slashable. Restart - the daemon and check the RPC provider. See adr/011-content-takedown.md. + watcher has panicked, wedged, or exited, or every poll tick is + failing. The node may be serving content blacklisted after the + failure, which is slashable. Check the RPC provider first (the + `multiplexed poller tick error` log lines name the cause), then + restart the daemon. See adr/011-content-takedown.md. - alert: DecdnBlacklistEnforcementFailing expr: rate(decdn_blacklist_enforcement_failures_total[10m]) > 0 @@ -130,6 +136,46 @@ groups: expected under the anti-panic policy. Capture the error! log line naming the watcher and file a bug. + - alert: DecdnChainWatcherFlapping + # Catches a node whose chain watchers fail on and off. Every successful + # tick clears down-seconds and stamps the tick gauge, so a watcher that + # recovers every few minutes never holds either *WatcherStalled arm for + # 5m, yet it lags head and misses events for most of the time. + # *_watcher_restarts_total counts once per drift window (the edge into + # failure), so its increase measures the on/off pattern directly; a + # single sustained outage is one window and stays with *WatcherStalled. + # + # increase() handles a daemon restart (the counter resets to 0), which a + # plain `x - x offset 30m` does not: that form read negative after a + # restart and cleared a pending alert. It runs as a subquery over the + # label_replace so the per-watcher names survive (increase() on the raw + # regex selector drops __name__ and collides the five series), and + # `max without (watcher)` folds them back: the five watchers share one + # poller, so they flap together and this raises one alert per node. + # + # A burst steps the 30m increase up for exactly 30m, so `for: 45m` + # outlasts any single burst and fires only on repeated failure. Checked + # against 3 days of fleet data: 4+ windows held for 45m only on the one + # node that was flapping; a fleet-wide 4-minute RPC blip stayed silent. + expr: >- + max without (watcher) (increase(label_replace({__name__=~"decdn_.+_watcher_restarts_total"}, + "watcher", "$1", "__name__", "decdn_(.+)_watcher_restarts_total")[30m:1m])) >= 4 + for: 45m + labels: + component: watcher + severity: warning + annotations: + runbook_url: "https://github.com/decdn/decdn/blob/main/docs/runbook.md#rpc-unreachable" + summary: "Chain watchers keep failing and recovering on {{ $labels.instance }}" + description: | + The chain watchers entered about {{ $value | printf "%.0f" }} failure + windows in the last 30 minutes, and have kept doing so for 45 + minutes. Their poll ticks fail on and off, so they lag chain head + and events reach the node late. Check the RPC provider: the + `multiplexed poller tick error` log lines name the cause, and a + rising decdn_chain_get_logs_range_rejections_total means the + provider caps eth_getLogs. + # A latent-bug report, not a degradation (#1517). The engine recovers the # guard and clears the poison, so coalescing never stops and no restart is # needed — but the workspace anti-panic policy means no production path is @@ -234,17 +280,20 @@ groups: accepting connections. The sum-without(direction) aggregation avoids false alerts when only one direction is idle. - # Chain-event watcher liveness (#1316): the *_last_tick_timestamp_seconds - # gauge advances only while the watcher actually completes ticks, so - # staleness catches a panicked/wedged/exited task the error-triggered - # down-seconds gauge reads as a healthy 0. 180s ≈ several poll intervals; - # tune to your event_poll_interval. The `> 0` guard skips the pre-first-tick - # sentinel (gauge reads 0 until the first success), so a fresh boot — and a - # node that never enables a conditionally-spawned watcher — doesn't - # self-fire. Blacklist has its own (slash-risk) rule in decdn-slash-safety; - # settlement is in decdn-delivery. + # Chain-event watcher liveness (#1316), two arms. The + # *_last_tick_timestamp_seconds gauge advances only while the watcher + # actually completes ticks, so staleness catches a panicked/wedged/exited + # task the error-triggered down-seconds gauge reads as a healthy 0. The + # `> 0` guard skips the pre-first-tick sentinel (gauge reads 0 until the + # first success), so a fresh boot — and a node that never enables a + # conditionally-spawned watcher — doesn't self-fire. The *_down_seconds arm + # counts from the first failed poll tick, so it catches a watcher that + # never ticked because every eth_getLogs fails; an unregistered watcher + # never fails a tick, so it stays 0. 180s ≈ several poll intervals; tune to + # your event_poll_interval. Blacklist has its own (slash-risk) rule in + # decdn-slash-safety; settlement is in decdn-delivery. - alert: DecdnSlashWatcherStalled - expr: (decdn_slash_watcher_last_tick_timestamp_seconds > 0) and (time() - decdn_slash_watcher_last_tick_timestamp_seconds > 180) + expr: ((decdn_slash_watcher_last_tick_timestamp_seconds > 0) and (time() - decdn_slash_watcher_last_tick_timestamp_seconds > 180)) or (decdn_slash_watcher_down_seconds > 180) for: 5m labels: component: watcher @@ -254,11 +303,12 @@ groups: summary: "Slash watcher not ticking on {{ $labels.instance }}" description: | No successful slash-detection poll tick for several intervals — the - watcher may have panicked, wedged, or exited, so a slash against this - operator could go undetected. Restart and check the RPC provider. + watcher may have panicked, wedged, or exited, or every poll tick is + failing, so a slash against this operator could go undetected. + Check the RPC provider, then restart. - alert: DecdnStakerSetWatcherStalled - expr: (decdn_staker_set_watcher_last_tick_timestamp_seconds > 0) and (time() - decdn_staker_set_watcher_last_tick_timestamp_seconds > 180) + expr: ((decdn_staker_set_watcher_last_tick_timestamp_seconds > 0) and (time() - decdn_staker_set_watcher_last_tick_timestamp_seconds > 180)) or (decdn_staker_set_watcher_down_seconds > 180) for: 5m labels: component: watcher @@ -357,7 +407,7 @@ groups: BatchSpanProcessor.SpanDroppingStarted warning. - alert: DecdnFeeSharesWatcherStalled - expr: (decdn_fee_shares_watcher_last_tick_timestamp_seconds > 0) and (time() - decdn_fee_shares_watcher_last_tick_timestamp_seconds > 180) + expr: ((decdn_fee_shares_watcher_last_tick_timestamp_seconds > 0) and (time() - decdn_fee_shares_watcher_last_tick_timestamp_seconds > 180)) or (decdn_fee_shares_watcher_down_seconds > 180) for: 5m labels: component: watcher @@ -366,9 +416,10 @@ groups: summary: "Fee-shares watcher not ticking on {{ $labels.instance }}" description: | No successful fee-shares poll tick for several intervals — the - watcher may have panicked, wedged, or exited. The node keeps - pricing its settlement share off the last split it read, which the - chain may have moved. Restart and check the RPC provider. + watcher may have panicked, wedged, or exited, or every poll tick is + failing. The node keeps pricing its settlement share off the last + split it read, which the chain may have moved. Check the RPC + provider, then restart. # getShares() is the safety-net re-read behind the SharesUpdated event # tail. Its failure keeps the current share and the tick still succeeds, @@ -561,11 +612,12 @@ groups: default up instead. max_probe_holds is restart-required. # Chain-event watcher liveness (#1316) for the payment-settlement watcher - # — staleness of its last successful poll tick. 180s ≈ several poll - # intervals; tune to your event_poll_interval. The `> 0` guard skips the - # pre-first-tick sentinel (gauge reads 0 until the first success). + # — staleness of its last successful poll tick, or a failing poll that + # never ticked (down-seconds). 180s ≈ several poll intervals; tune to your + # event_poll_interval. The `> 0` guard skips the pre-first-tick sentinel + # (gauge reads 0 until the first success). - alert: DecdnSettlementWatcherStalled - expr: (decdn_settlement_watcher_last_tick_timestamp_seconds > 0) and (time() - decdn_settlement_watcher_last_tick_timestamp_seconds > 180) + expr: ((decdn_settlement_watcher_last_tick_timestamp_seconds > 0) and (time() - decdn_settlement_watcher_last_tick_timestamp_seconds > 180)) or (decdn_settlement_watcher_down_seconds > 180) for: 5m labels: component: settlement @@ -576,8 +628,8 @@ groups: description: | No successful payment-settlement poll tick for several intervals — channel open/settle/close events are not being ingested, so - voucher redemption and reconciliation stall. Restart and check the - RPC provider. + voucher redemption and reconciliation stall. Check the RPC + provider, then restart. # #1520. Three deposit guards route through this one counter (the miss floor, # the window ceiling, the direct-serve ceiling), so it cannot say WHICH — the