diff --git a/.claude/settings.json b/.claude/settings.json deleted file mode 100644 index dfecf2d..0000000 --- a/.claude/settings.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "enabledPlugins": { - "cicd-automation@claude-code-workflows": true - } -} diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000..6d93b09 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,68 @@ +--- +name: Bug report +description: A deploy path (Ansible, Helm, Compose) does something wrong. +labels: [bug] +body: + - type: markdown + attributes: + value: >- + Thanks for reporting. **Redact before you paste**: RPC URLs (they often embed an + API key), Grafana Cloud tokens, keystore passwords and real host addresses must + not appear in a public issue. + - type: dropdown + id: path + attributes: + label: Deploy path + options: + - Ansible (this repo's playbooks) + - Ansible (decdn.node Galaxy collection) + - Helm chart + - Docker Compose + validations: + required: true + - type: dropdown + id: install + attributes: + label: Install method + description: Ansible only. + options: + - release (signed tarball) + - manual (locally built binaries) + - not applicable + validations: + required: true + - type: input + id: version + attributes: + label: decdn version + description: "`decdn-node --version`, or the image digest." + validations: + required: true + - type: input + id: revision + attributes: + label: devops revision + description: Commit, tag or collection/chart version of this repo. + validations: + required: true + - type: input + id: platform + attributes: + label: Target OS and architecture + placeholder: Ubuntu 26.04, aarch64 + validations: + required: true + - type: textarea + id: what + attributes: + label: What happened, and what did you expect? + validations: + required: true + - type: textarea + id: repro + attributes: + label: How to reproduce + description: Command run, relevant (redacted) inventory or values, and the failing task or error. + render: shell + validations: + required: true diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..0289a38 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,9 @@ +--- +blank_issues_enabled: false +contact_links: + - name: Security vulnerability + url: https://github.com/decdn/devops/blob/main/SECURITY.md + about: Report privately by email (see SECURITY.md), never in a public issue. That includes a secret or real host address you find committed here. + - name: The node daemon or the protocol + url: https://github.com/decdn/decdn/issues + about: Bugs in decdn-node itself, the CLI or the contracts belong in decdn/decdn. diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 0000000..b776b25 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,26 @@ +--- +name: Feature request +description: Something operators need from the deploy tooling. +labels: [enhancement] +body: + - type: textarea + id: problem + attributes: + label: What are you trying to do? + description: The operator task, and where the current tooling gets in the way. + validations: + required: true + - type: textarea + id: proposal + attributes: + label: What would help? + - type: dropdown + id: path + attributes: + label: Deploy path + multiple: true + options: + - Ansible + - Helm chart + - Docker Compose + - All / not specific diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 50270ec..c4b1195 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,15 +1,24 @@ --- -# Dependabot — keeps the SHA-pinned GitHub Actions current (it rewrites -# `uses: owner/repo@ # vX.Y.Z` to the new sha + version comment). +# Dependabot — keeps the SHA-pinned GitHub Actions and the pre-commit hook revs +# current. # -# github-actions is the only applicable ecosystem: the repo has no pip/npm/etc. -# manifests, and `ansible/requirements.yml` is Ansible Galaxy, which Dependabot -# does not support. +# github-actions rewrites `uses: owner/repo@ # vX.Y.Z` to the new sha + comment +# pre-commit bumps the `rev:` of each repo in .pre-commit-config.yaml # -# NOT covered here, bump manually: -# - the KICS engine image digest in the Makefile — see CONTRIBUTING.md +# Considered and not applicable here, so not listed: +# docker-compose compose/compose.yaml builds its image from ${DECDN_IMAGE_REPO} and +# ${DECDN_IMAGE_DIGEST}, which the operator pins to a signed +# release digest; nothing to bump +# helm the chart has no dependencies, and its image is operator-set +# pip the repo has no pip manifest (CI installs are unpinned) +# Ansible Galaxy (ansible/requirements.yml) is not a Dependabot ecosystem. +# +# NOT covered, bump manually (CONTRIBUTING.md § Supply-chain / pinning rules): +# - the KICS and kubeconform image digests in the Makefile +# - the molecule image digests in ansible/molecule/*/molecule.yml +# - the four setup-helm `version:` inputs (two in ci.yml, two in release.yml) # - the Galaxy collections in ansible/requirements.yml -# - pre-commit hook revs — run `pre-commit autoupdate` +# - the local yamllint hook's `additional_dependencies` pin version: 2 updates: - package-ecosystem: github-actions @@ -23,3 +32,14 @@ updates: groups: actions: patterns: ["*"] + - package-ecosystem: pre-commit + directory: / + schedule: + interval: weekly + open-pull-requests-limit: 5 + commit-message: + prefix: ci + include: scope + groups: + pre-commit-hooks: + patterns: ["*"] diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b647df2..43e919a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,7 +18,8 @@ concurrency: # malicious code — cf. the March 2026 KICS action compromise). The trailing # comment records the human-readable version; .github/dependabot.yml bumps them. jobs: - # Detect whether ansible/ or the Helm chart changed so heavy jobs skip unrelated PRs. + # Detect whether ansible/, the Helm chart or compose/ changed so heavy jobs skip + # unrelated PRs. changes: runs-on: ubuntu-latest timeout-minutes: 5 # ~5s in practice @@ -28,6 +29,7 @@ jobs: outputs: ansible: ${{ steps.filter.outputs.ansible }} helm: ${{ steps.filter.outputs.helm }} + compose: ${{ steps.filter.outputs.compose }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v4.0.3 @@ -45,6 +47,11 @@ jobs: - 'ansible/molecule/schema/files/**' - 'Makefile' - '.github/workflows/ci.yml' + # lint-compose and security-compose live in the root Makefile. + compose: + - 'compose/**' + - 'Makefile' + - '.github/workflows/ci.yml' # Ansible style + best-practice + the production-profile SECURITY rules, # plus a syntax-check of every playbook. Runs only when ansible/ changed. @@ -108,10 +115,14 @@ jobs: key: galaxy-${{ runner.os }}-${{ hashFiles('ansible/requirements.yml') }} - name: Lint (yamllint + ansible-lint) run: make lint - - name: Syntax-check playbook + - name: Syntax-check playbooks # Dummy inventory: no real hosts.yml is committed upstream (only hosts.yml.example), - # and --syntax-check only parses, it never connects. - run: ansible-playbook playbooks/site.yml --syntax-check -i localhost, + # and --syntax-check only parses, it never connects. Every playbook, so a new + # one (backup, decommission, ...) is covered without editing this step. + run: | + for pb in playbooks/*.yml; do + ansible-playbook "$pb" --syntax-check -i localhost, + done # Build the public `decdn.node` collection and run galaxy-importer's checks — # the same validation Galaxy runs on upload (metadata, license, README, embedded @@ -175,7 +186,8 @@ jobs: timeout-minutes: 20 # pulls the kubeconform image; ~1m in practice steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - # Helm version is pinned here AND in the kics job below; bump both. + # Helm version is pinned here, in the kics job below and twice in release.yml; + # bump all four. - uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 with: version: v4.3.0 @@ -188,6 +200,20 @@ jobs: # DECDN_CLI=... before bumping the decdn version. run: make lint-helm + # Docker Compose path: render compose/compose.yaml with its example env files and + # assert the invariants its README promises (host network, no published ports, + # digest-pinned image, read-only rootfs, no capabilities, graceful stop). Runs + # only when compose/ or the root Makefile changed. + compose: + needs: changes + if: needs.changes.outputs.compose == 'true' + runs-on: ubuntu-latest + timeout-minutes: 5 # docker compose + jq ship on ubuntu-latest; seconds in practice + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Compose invariants + run: make lint-compose + # Render roles/grafana_alloy's templates and validate them with the REAL pinned # Grafana Alloy binary. The molecule `grafana-cloud` scenario deliberately runs # against a stub that exits 0 for every subcommand, so it proves plumbing but @@ -218,8 +244,9 @@ jobs: - name: Validate the rendered Alloy configuration run: make lint-alloy - # Dedicated IaC security scan of the Ansible tree and the rendered Helm chart, driven straight from the - # digest-pinned KICS *engine* image by `make security` — the exact command + # Dedicated IaC security scan of the Ansible tree, the rendered Helm chart and + # compose/, driven straight from the digest-pinned KICS *engine* image by + # `make security` — the exact command # developers run locally, so CI and local results cannot drift. KICS severities # are CRITICAL/HIGH/MEDIUM/LOW/INFO; the engine's own `--fail-on high` exit # code is the gate. @@ -237,7 +264,9 @@ jobs: # hijacked action). See CONTRIBUTING.md. kics: needs: changes - if: needs.changes.outputs.ansible == 'true' || needs.changes.outputs.helm == 'true' + if: >- + needs.changes.outputs.ansible == 'true' || needs.changes.outputs.helm == 'true' + || needs.changes.outputs.compose == 'true' runs-on: ubuntu-latest timeout-minutes: 20 # pulls the KICS engine image; ~30s in practice steps: @@ -245,15 +274,15 @@ jobs: - uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 with: version: v4.3.0 - - name: KICS security scan of ansible/ + the rendered chart (fail on HIGH) + - name: KICS security scan of ansible/, the rendered chart and compose/ (fail on HIGH) run: make security # Replaces the action's `enable_jobs_summary`. - name: Summarise KICS findings if: always() run: | - for scan in ansible helm; do + for scan in ansible helm compose; do results=kics-results/results.json - [ "$scan" = helm ] && results=kics-results/helm/results.json + [ "$scan" != ansible ] && results="kics-results/$scan/results.json" { echo "### KICS IaC scan ($scan)" echo @@ -289,6 +318,41 @@ jobs: # with: # sarif_file: kics-results/results.sarif + # Every pre-commit hook on every file: hygiene, shellcheck, markdownlint and + # yamllint. The same gate `make lint` runs locally, so contributors who never + # installed the hook still meet it. The manual-stage ansible-lint hook is skipped + # (the ansible-lint job above covers it). Always runs: it is cheap and covers + # every path. + pre-commit: + runs-on: ubuntu-latest + timeout-minutes: 10 # hook environments are cached; ~1m cold + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.12' + - name: Install pre-commit + run: python -m pip install --upgrade pre-commit + # Hook environments (node for markdownlint, the shellcheck image, yamllint's + # virtualenv) keyed on the config that pins them. + - uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 + with: + path: ~/.cache/pre-commit + key: pre-commit-${{ runner.os }}-${{ hashFiles('.pre-commit-config.yaml') }} + - name: Run every hook on every file + run: pre-commit run --all-files --show-diff-on-failure + + # The repo's own guard rails: ansible/Makefile's scoping guards (dry runs), the + # release gate, and lint-compose's negative cases (tests/scripts-test.sh). Always + # runs: seconds, and it covers files every path filter above would miss. + scripts: + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Script and guard tests + run: make test-scripts + # Lint the workflow files themselves. actionlint: runs-on: ubuntu-latest diff --git a/.github/workflows/molecule.yml b/.github/workflows/molecule.yml index dcd8b0d..b462892 100644 --- a/.github/workflows/molecule.yml +++ b/.github/workflows/molecule.yml @@ -23,10 +23,11 @@ concurrency: jobs: molecule: runs-on: ubuntu-latest # Docker is preinstalled - # ~4m41s at JOBS=3. A bound, not a target: without one a wedged privileged - # systemd container burns the 360-minute default, and with cancel-in-progress - # above, some branch-protection setups read the resulting cancelled check as - # "not failed" rather than as a failure. + # ~4m41s at JOBS=3 with 8 scenarios. os-matrix (3 containers) and lifecycle took the + # suite to ~10m at JOBS=3 on a local 16-thread box; re-measure from CI's first run. + # A bound, not a target: without one a wedged privileged systemd container burns + # the 360-minute default, and with cancel-in-progress above, some branch-protection + # setups read the resulting cancelled check as "not failed" rather than as a failure. timeout-minutes: 45 defaults: run: @@ -66,8 +67,8 @@ jobs: # flaky galaxy.ansible.com, for no added coverage. The cache above wraps that # Make-owned invocation instead. # - # JOBS is capped at 3 rather than the default (one job per scenario, currently - # 7): every scenario is a privileged systemd container, and they share this + # JOBS is capped at 3 rather than the default (one job per scenario): every + # scenario is a privileged systemd container, and they share this # runner's cores and cgroup hierarchy. If this job turns flaky, drop to JOBS=1 # or swap in `make molecule-serial` — the latter also serialises the output. run: make molecule JOBS=3 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..a183175 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,137 @@ +--- +name: Release + +# One repo version, two artifacts: a vX.Y.Z tag releases the `decdn.node` Ansible +# collection and the `decdn-node` Helm chart together. See RELEASING.md. +# +# `build` always runs: it gates the tag (collection, chart and both changelogs must +# agree on X.Y.Z), re-runs the chart and collection checks, packages both, and +# uploads them with SHA256SUMS as a workflow artifact. `publish` makes them public +# (Galaxy, oci://ghcr.io/decdn/charts with a keyless cosign signature, and a GitHub +# Release) ONLY when the repository variable PUBLISH_ENABLED is 'true'. Until then +# a tag push is a dry run. A manual dispatch is always a dry run. +on: + push: + tags: ['v[0-9]+.[0-9]+.[0-9]+'] + workflow_dispatch: + inputs: + tag: + description: Version to dry-run, e.g. v0.1.0 (nothing is published) + required: true + +permissions: + contents: read + +concurrency: + group: release-${{ github.ref }} + cancel-in-progress: false # never cut a half-published release short + +env: + TAG: ${{ github.event_name == 'push' && github.ref_name || inputs.tag }} + CHART_REGISTRY: oci://ghcr.io/decdn/charts + +jobs: + build: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Gate the tag against the collection, the chart and both changelogs + run: scripts/check-release-version.sh "$TAG" --notes release-notes.md + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.12' + # Helm is pinned four times: ci.yml's helm and kics jobs, and both jobs here. + # Bump all four together. + - uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 + with: + version: v4.3.0 + - name: Install collection build + import tooling + run: python -m pip install --upgrade ansible-core ansible-lint galaxy-importer + - name: Build and validate the decdn.node collection + working-directory: ansible + env: + ANSIBLE_COLLECTIONS_PATH: collections + run: | + make deps + make galaxy-check + - name: Chart lint and render tests + run: make lint-helm + - name: Package the chart and write checksums + run: | + mkdir -p dist + cp ansible/build/decdn-node-*.tar.gz dist/ + helm package charts/decdn-node --destination dist + (cd dist && sha256sum -- * > SHA256SUMS) + cp release-notes.md dist/ + cat dist/SHA256SUMS + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: release-${{ env.TAG }} + path: dist/ + if-no-files-found: error + + publish: + needs: build + if: vars.PUBLISH_ENABLED == 'true' && github.event_name == 'push' + runs-on: ubuntu-latest + timeout-minutes: 20 + # Protect this environment with required reviewers in the repository settings. + environment: release + permissions: + contents: write # the GitHub Release + packages: write # the chart on ghcr.io + id-token: write # keyless cosign signature (Sigstore) + steps: + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: release-${{ env.TAG }} + path: dist + - name: Verify the artifacts are the ones build checksummed + working-directory: dist + run: sha256sum --check SHA256SUMS + - name: Require the Galaxy API key + env: + GALAXY_API_KEY: ${{ secrets.GALAXY_API_KEY }} + run: | + [ -n "$GALAXY_API_KEY" ] || { echo "::error::secret GALAXY_API_KEY is not set"; exit 1; } + - uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 + with: + version: v4.3.0 + - uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 + - name: Push and sign the chart + env: + GH_TOKEN: ${{ github.token }} + run: | + # Helm and cosign keep separate credential stores (Helm's registry config + # vs Docker's config.json), so log both in: cosign pushes the signature. + echo "$GH_TOKEN" | helm registry login ghcr.io --username "$GITHUB_ACTOR" --password-stdin + echo "$GH_TOKEN" | cosign login ghcr.io --username "$GITHUB_ACTOR" --password-stdin + # Capture and print on both paths: under `bash -e` a bare out="$(…)" would + # exit on failure before the error text was shown. + if ! out="$(helm push dist/decdn-node-"${TAG#v}".tgz "$CHART_REGISTRY" 2>&1)"; then + echo "$out" + echo "::error::helm push failed" + exit 1 + fi + echo "$out" + digest="$(sed -nE 's/^Digest: (sha256:[0-9a-f]{64})$/\1/p' <<<"$out")" + [ -n "$digest" ] || { echo "::error::no digest in helm push output"; exit 1; } + cosign sign --yes "ghcr.io/decdn/charts/decdn-node@$digest" + echo "chart: ${CHART_REGISTRY}/decdn-node:${TAG#v} ($digest)" >> "$GITHUB_STEP_SUMMARY" + - name: Install ansible-core + run: python3 -m pip install --upgrade ansible-core + # Not idempotent: Galaxy refuses a version that already exists, so a re-run + # after this step succeeded fails here. RELEASING.md covers recovery. + - name: Publish the collection to Galaxy + env: + GALAXY_API_KEY: ${{ secrets.GALAXY_API_KEY }} + run: ansible-galaxy collection publish dist/decdn-node-"${TAG#v}".tar.gz --api-key "$GALAXY_API_KEY" + # Last, so the Release page only appears once both artifacts are live. + - name: Create the GitHub Release + env: + GH_TOKEN: ${{ github.token }} + run: | + gh release create "$TAG" --repo "$GITHUB_REPOSITORY" --title "$TAG" \ + --notes-file dist/release-notes.md \ + dist/decdn-node-*.tar.gz dist/decdn-node-*.tgz dist/SHA256SUMS diff --git a/.github/workflows/upstream-drift.yml b/.github/workflows/upstream-drift.yml new file mode 100644 index 0000000..866de31 --- /dev/null +++ b/.github/workflows/upstream-drift.yml @@ -0,0 +1,73 @@ +--- +name: Upstream drift + +# Three things in this repo mirror decdn/decdn and go stale when upstream moves: +# - roles/decdn_node/vars/main/networks.yml (contract addresses; upstream +# redeploys change every one of them at once) +# - charts/decdn-node/files/monitoring/ (dashboards + alert rules) +# - ansible/molecule/schema/files/schema-keys.txt (the node.toml key inventory) +# This job regenerates each from upstream main and fails when a committed copy +# differs. It is a scheduled early warning, not a PR gate: an upstream change must +# not turn unrelated PRs red. Fix drift by re-running the generator it names. +on: + schedule: + - cron: '17 6 * * 1' # Mondays 06:17 UTC + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: upstream-drift + cancel-in-progress: true + +jobs: + drift: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + # Public repo, default branch. The sync scripts read origin/main through git, + # which this checkout provides as a remote-tracking ref. + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: decdn/decdn + ref: main + path: upstream + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.12' + # Each check exits 0 (current), 1 (stale) or anything else (could not run), and + # the two failure kinds are reported differently: a broken generator is not an + # upstream change. Full output goes to the job summary; annotations stay one line. + - name: Compare the mirrors with upstream main + run: | + rc=0 + check() { + local name="$1" out status; shift + out="$("$@" 2>&1)" && status=0 || status=$? + case "$status" in + 0) echo "- ✅ **$name**: current" >> "$GITHUB_STEP_SUMMARY" ;; + 1) rc=1 + echo "::error title=$name is stale::re-run its generator (see the job summary)" + { echo "- ❌ **$name**: stale"; echo '```'; echo "$out"; echo '```'; } >> "$GITHUB_STEP_SUMMARY" ;; + *) rc=1 + echo "::error title=$name check could not run::exit $status (see the job summary)" + { echo "- ⚠️ **$name**: check failed to run (exit $status)"; echo '```'; echo "$out"; echo '```'; } >> "$GITHUB_STEP_SUMMARY" ;; + esac + } + # The schema inventory has no --check of its own: generate, then diff, so a + # crashing generator (exit 2) is not mistaken for "every key removed" (exit 1). + # shellcheck disable=SC2329 # invoked indirectly, through check() + schema_check() { + local fresh + fresh="$(mktemp)" + python3 ansible/molecule/schema/files/gen-schema-keys.py upstream > "$fresh" || return 2 + diff -u ansible/molecule/schema/files/schema-keys.txt "$fresh" + } + echo "### Upstream drift (decdn/decdn@$(git -C upstream rev-parse --short HEAD))" >> "$GITHUB_STEP_SUMMARY" + check "network profiles" python3 scripts/sync-network-profiles.py upstream --check + check "monitoring assets" scripts/sync-monitoring.sh upstream --check + check "config schema keys" schema_check + exit "$rc" diff --git a/.gitignore b/.gitignore index 5d8888d..9edff2f 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,9 @@ **/cert.pem **/*.pem +# Encrypted node backups (`make backup`); they belong with the operator, not in git +*.tar.age + # Local, per-developer Claude Code settings (not shared) .claude/settings.local.json @@ -40,3 +43,7 @@ __pycache__/ # Pinned third-party binaries downloaded by the test harnesses (`make lint-alloy` # caches the verified Grafana Alloy release here so repeat runs skip the fetch). ansible/.cache/ + +# Release packaging output (the release workflow builds into dist/) +dist/ +release-notes.md diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9ef69e5..8e16276 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,7 +15,8 @@ default_install_hook_types: [pre-commit] exclude: >- (?x)^( ansible/collections/| - ansible/\.ansible/ + ansible/\.ansible/| + charts/[^/]+/files/monitoring/[^/]+\.(json|ya?ml)$ ) repos: diff --git a/AGENTS.md b/AGENTS.md index 7398d6d..1da8253 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -6,12 +6,16 @@ DevOps repo. ## What this repo is The official DevOps project for deploying a **deCDN node**: infrastructure, deployment, -and operational tooling. There are two deploy paths: **Ansible** (`ansible/`, VMs/bare -metal, the primary path) and a **Helm chart** (`charts/decdn-node/`, Kubernetes). +and operational tooling, for node operators anywhere. There are three deploy paths: +**Ansible** (`ansible/`, VMs/bare metal, the primary path, also the `decdn.node` Galaxy +collection), **Docker Compose** (`compose/`, a single Docker host) and a **Helm chart** +(`charts/decdn-node/`, Kubernetes). This repo is **infrastructure only**. It is *not* a source of truth for protocol or economic claims — those trace to the deCDN ADRs. If something here states a protocol fact -(chain-id, token address, fee split), it must trace back to an ADR, not invent one. +(chain-id, token address, fee split), it must trace back to an ADR or upstream's +deployment manifest, not invent one; the contract addresses live only in the generated +mirror (`vars/main/networks.yml`), never hand-copied into inventory or docs. ## Hard rules @@ -49,15 +53,26 @@ economic claims — those trace to the deCDN ADRs. If something here states a pr ``` ansible/ # the deployment project (DevSec-hardened, lean roles) - playbooks/ # site.yml (decdn node) + playbooks/ # site.yml (decdn node), backup.yml, decommission.yml roles/ # baseline, decdn_node, grafana_alloy inventory/ galaxy/ molecule/ # see ansible/README.md +compose/ # Docker Compose deploy path for a single host (see its README.md) charts/ decdn-node/ # Helm chart for the node on Kubernetes (see its README.md) ci/ # CI values files (mirror molecule/schema's three plays) + files/monitoring/ # GENERATED: upstream dashboards + alert rules (scripts/sync-monitoring.sh) tests/render-test.sh # positive/negative render tests (`make lint-helm`) +docs/ # cross-path operator docs: requirements.md, lifecycle.md +scripts/ # upstream-mirror generators + the release gate ``` +**Generated mirrors of upstream — regenerate, never hand-edit:** +`ansible/roles/decdn_node/vars/main/networks.yml` (`scripts/sync-network-profiles.py`), +`charts/decdn-node/files/monitoring/` (`scripts/sync-monitoring.sh`) and +`ansible/molecule/schema/files/schema-keys.txt` (`gen-schema-keys.py`). The weekly +`upstream-drift` workflow flags staleness. These are the only protocol facts (contract +addresses) the repo carries, and they carry their upstream commit. + ## Current services - **`ansible/`** — the declarative deployment project. **The public deCDN node** @@ -65,8 +80,14 @@ charts/ tarball — verified against the release's GPG-signed `SHA256SUMS` — or, while upstream has no release tag cut (the current default), from locally-built binaries; under a hardened systemd unit; public QUIC udp/4433, loopback metrics/admin, operator-provisioned eth - keystore, operator-provisionable secret env file, and required chain knobs (no baked - protocol facts — sourced from ADRs), over a shared DevSec-hardened `baseline`. + keystore, operator-provisionable secret env file, over a shared DevSec-hardened + `baseline`. The release target triple is derived from the host's architecture + (x86_64/aarch64). The chain comes from `decdn_network` (the generated manifest mirror + above; explicit inventory addresses still win) or from explicit variables. + `playbooks/backup.yml` / `decommission.yml` (`make backup` / `make decommission`) are + role entry points (`tasks_from: backup|decommission`): backups are encrypted on the host + to operator public keys; decommission needs `LIMIT` + typed confirmation, keeps the + identity and never touches the chain. Leaving `decdn_rpc_url` empty means the operator wrote `0600 /etc/decdn/decdn.env` on the host and the role only gates on it, so no secret transits the control machine. See `ansible/README.md`. (On-chain node stake/registration, ADR 019 Phase 2, is a manual operator step, driven by `decdn setup`.) @@ -102,6 +123,14 @@ charts/ everything, so only the real pinned binary proves the rendered config loads. See `ansible/roles/grafana_alloy/README.md`. +- **`compose/`** — the same node under Docker Compose on one host: the upstream image, + always by digest (`compose.yaml` builds `DECDN_IMAGE_REPO@DECDN_IMAGE_DIGEST`), the + role's host layout (`/etc/decdn` read-only, `/var/lib/decdn`), host + networking (so loopback metrics/admin stay loopback and Docker publishes no ports), + read-only rootfs, no capabilities, 300 s SIGTERM grace. `make lint-compose` asserts + those invariants (and `make test-scripts` that it rejects broken variants); + `make security` scans it. + - **`charts/decdn-node/`** — the same node on Kubernetes: a one-replica StatefulSet (one release = one identity) on the upstream daemon-only image (`ghcr.io/decdn/decdn-node`; unpublished, so `image.tag`/`image.digest` is required), PVC data dir, a `prepare` init @@ -114,41 +143,37 @@ charts/ above applies here too:** `make lint-helm` runs the same `check-schema-keys.py` + `schema-keys.txt` on the rendered ConfigMap, so a re-sync covers both paths. Unlike the role, CI has no real-binary `decdn config validate` for the chart — run - `DECDN_CLI=… make lint-helm` locally when bumping the decdn version. + `DECDN_CLI=… make lint-helm` locally when bumping the decdn version. Optional + `PrometheusRule` + dashboard ConfigMaps render the vendored `files/monitoring/` (never + through `tpl`: the alert annotations carry Prometheus templates). ## Commands -Two Makefiles: the **root** is the hygiene/security/CI mirror; **`ansible/`** drives -deploys (its targets must run from `ansible/`). `make help` lists root targets. +Two Makefiles: the **root** is the check driver CI calls (`make help`); **`ansible/`** +drives deploys (its targets must run from `ansible/`). The full target list is in +[CONTRIBUTING.md](CONTRIBUTING.md#make-targets). ```bash -# Root — lint & security (mirror CI) -make hooks # one-time: install pre-commit git hook (pip install pre-commit first) -make lint # all pre-commit hooks on all files (hygiene, shellcheck, yamllint, markdown) -make lint-ansible # vendor collections + full ansible-lint (production profile) -make molecule # containerised converge/verify of the decdn_node + grafana_alloy - # roles — every molecule/*/ scenario in parallel (the target - # discovers them by glob, so adding one needs no edit here); - # needs Docker, cap with JOBS= -make molecule-serial # the same suite one scenario at a time (readable failure output) -make lint-helm # chart: helm lint + render tests + kubeconform + schema keys (needs helm, yq, Docker) -make lint-alloy # grafana_alloy: render its templates + `alloy validate` them with the real - # pinned binary (the molecule stub exits 0 for everything and cannot) -make security # = security-ansible + security-helm (KICS over the rendered chart; needs helm) - -# Ansible deploys — run from ansible/ (see ansible/README.md for the full flow) -cd ansible -make deps # vendor pinned Galaxy collections into ./collections -make check / deploy # deCDN node (site.yml): dry-run / provision - # fleet-wide by default; LIMIT= scopes, ANSIBLE_ARGS='…' passes through; - # INVENTORY= targets a private fleet overlay (default inventory/hosts.yml) -make build / galaxy-check # stage + build the decdn.node collection, then validate it +# Root — the gates (CI runs the same) +make lint # every pre-commit hook, every file (also the CI `pre-commit` job) +make lint-ansible # vendor collections + ansible-lint (production profile) +make molecule # every ansible/molecule/*/ scenario in parallel (Docker; JOBS=) +make lint-helm # chart: lint + render tests + kubeconform + schema keys +make lint-alloy # grafana_alloy config against the real pinned Alloy binary +make lint-compose # compose/ invariants +make test-scripts # Makefile guards, release gate, lint-compose negatives +make security # KICS over ansible/, the rendered chart and compose/ + +# Ansible — run from ansible/ +make deps # vendor pinned Galaxy collections +make check / deploy # site.yml; fleet-wide unless LIMIT=; INVENTORY= +make backup / decommission LIMIT=… # lifecycle playbooks (decommission requires LIMIT) +make build / galaxy-check # the decdn.node collection ``` **Inventory is private; the firewall hole is not.** This repo is public, so `ansible/inventory/hosts.yml` is git-ignored and a real fleet lives in a private overlay -(template: `ansible/inventory/fleet.example/`, launch sequence in -`ansible/docs/launch-runbook.md`). Inventory-adjacent group_vars do not load for an +(template: `ansible/inventory/fleet.example/`). Inventory-adjacent group_vars do not load for an overlay, so anything every node needs regardless of inventory (today only the udp/4433 `baseline_extra_inbound` hole) lives in `ansible/playbooks/group_vars/decdn_nodes.yml`. Don't move it back under `inventory/`. @@ -158,14 +183,12 @@ Don't move it back under `inventory/`. distributable collection. The overlay lives in `ansible/galaxy/` and is staged into a clean collection tree by `galaxy/build.sh` — there is **no** `galaxy.yml` at the `ansible/` root (that would make ansible-lint treat the deploy project as a collection). Build/validate with -`make build` / `make galaxy-check`; **publishing is a manual step** -(`ansible-galaxy collection publish`), not automated. - -**Gotcha — pre-commit is local-only.** Hygiene/shellcheck/yamllint/markdown run via -`make hooks`/`make lint` on your machine, **not** in CI. CI (`.github/workflows/`) is the -blocking gate and runs `ansible-lint` + `galaxy-build` (on `ansible/**`), `helm` (`make lint-helm`, -on `charts/**`, the shared schema checker/inventory, `Makefile` or `ci.yml`), KICS (on either) + -`actionlint`. `ansible-lint` -is **not** a per-commit hook (it needs collections vendored) — run `make lint-ansible`. -A separate `molecule.yml` workflow runs the containerised converge/verify in CI too, so -`make molecule` is not purely local. +`make build` / `make galaxy-check`. **Publishing** is `release.yml` on a `vX.Y.Z` tag, +together with the chart at the same version, and only while the `PUBLISH_ENABLED` +repository variable is `true` (RELEASING.md). Log changes under `[Unreleased]` in +`ansible/galaxy/CHANGELOG.md` and `charts/decdn-node/CHANGELOG.md`. + +**CI.** `ci.yml` is the blocking gate: `pre-commit`, `scripts` and `actionlint` on every PR, the +Ansible, chart and compose jobs path-filtered, KICS on any of them; `molecule.yml` runs the +molecule suite on `ansible/**`. `ansible-lint` is **not** a per-commit hook (it needs +collections vendored): run `make lint-ansible`. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 459912b..f3f7be5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,11 +1,12 @@ # Contributing -Two checks run on every change: **pre-commit** locally and **GitHub Actions** on -push / PR. The repo's #1 rule still stands — **never commit secrets** (see -[`AGENTS.md`](AGENTS.md)); secrets are generated on the target host and the repo -ships `*.example` templates for them (never the real thing; non-secret config is -committed directly). There is no dedicated secret-scanner in the -pipeline — keep secrets out by design (and rely on GitHub's push protection). +The repo's first rule: **never commit secrets** (see [`AGENTS.md`](AGENTS.md)). Secrets +are generated on, or operator-provisioned to, the target host; the repo ships `*.example` +templates for them and commits non-secret config directly. There is no dedicated +secret scanner in the pipeline: keep secrets out by design (and rely on GitHub's push +protection). + +Commits follow [Conventional Commits](https://www.conventionalcommits.org). ## One-time setup @@ -14,45 +15,73 @@ pip install pre-commit # or: pipx install pre-commit make hooks # installs the git pre-commit hook ``` -After this, every commit runs hygiene checks, `shellcheck`, `yamllint` (Ansible -tree), and `markdownlint`. +After this, every commit runs hygiene checks, `shellcheck`, `yamllint` (Ansible tree) +and `markdownlint`. CI runs the same hooks on every file, so skipping this only moves +the failure to the PR. + +## Make targets -## Useful targets (`make help`) +The root `Makefile` is the check driver (`make help` lists it); CI calls the same +targets, so a local pass means a CI pass. Deploy targets live in +[`ansible/Makefile`](ansible/README.md) and run from `ansible/`. | Target | What it does | |--------|--------------| -| `make lint` | run all pre-commit hooks on every file (the full local hygiene gate) | -| `make lint-ansible` | install Galaxy collections + run `ansible-lint` (its production profile includes the Ansible security rules) | -| `make lint-helm` | Helm chart: `helm lint --strict`, positive/negative render tests, kubeconform (digest-pinned image), shared schema-key check and its fixtures (needs `helm`, `yq`, `python3` ≥ 3.11, Docker). Set `DECDN_CLI=` to also run the real `decdn config validate` (CI can't). | -| `make lint-alloy` | Renders `roles/grafana_alloy`'s templates and validates them with the **real** digest-pinned Grafana Alloy binary (`alloy validate` + an `ExecStart` flag check). The `grafana-cloud` molecule scenario uses a stub that exits 0 for every subcommand, so this is the only gate that proves the config loads. Set `ALLOY_BIN=` to skip the download. | -| `make security` | KICS IaC security scan of `ansible/` and the rendered Helm chart (digest-pinned engine image — CI runs this same target) | +| `make lint` | every pre-commit hook on every file (CI job `pre-commit`) | +| `make lint-ansible` | install Galaxy collections + `ansible-lint` (production profile, which includes the Ansible security rules) | +| `make molecule` | every `ansible/molecule/*/` scenario in parallel, in privileged systemd containers (needs Docker; cap with `JOBS=`). `make molecule-serial` runs them one at a time for readable failures. | +| `make lint-helm` | chart: `helm lint --strict`, positive/negative render tests, kubeconform (digest-pinned image), the shared schema-key check (needs `helm`, `yq`, `python3` ≥ 3.11, Docker). Set `DECDN_CLI=` to also run the real `decdn config validate` (CI can't). | +| `make lint-alloy` | renders `roles/grafana_alloy`'s templates and validates them with the **real** digest-pinned Alloy binary. The molecule stub exits 0 for everything, so this is the only gate that proves the config loads. `ALLOY_BIN=` skips the download. | +| `make lint-compose` | renders `compose/compose.yaml` with its example env and asserts its security invariants | +| `make test-scripts` | `tests/scripts-test.sh`: the `ansible/Makefile` scoping guards (dry runs), the release gate, and `lint-compose`'s negative cases. `UPSTREAM=` adds the sync generators' exit codes. | +| `make security` | KICS IaC scan of `ansible/`, the rendered chart and `compose/` (digest-pinned engine, fail on HIGH) | +| `make galaxy-check` | build the `decdn.node` collection and run galaxy-importer's checks | + +`ansible-lint` is **not** a per-commit hook (it needs the collections installed). Run it +with `make lint-ansible`, or `pre-commit run ansible-lint --hook-stage manual`. + +### Upstream mirrors -`ansible-lint` is **not** a per-commit hook (it needs the collections installed). -Run it on demand with `make lint-ansible`, or `pre-commit run ansible-lint --hook-stage manual`. +Three things here are generated from `decdn/decdn`; regenerate, never hand-edit: + +| Mirror | Regenerate with | +|--------|-----------------| +| `ansible/roles/decdn_node/vars/main/networks.yml` (contract addresses per network) | `scripts/sync-network-profiles.py ` | +| `charts/decdn-node/files/monitoring/` (dashboards, alert rules) | `scripts/sync-monitoring.sh ` | +| `ansible/molecule/schema/files/schema-keys.txt` (node.toml keys) | `ansible/molecule/schema/files/gen-schema-keys.py > …` | + +The two `scripts/sync-*` generators read `origin/main` through git (override with +`--ref`), so the checkout's own branch doesn't matter for them; `gen-schema-keys.py` +reads the checkout's working tree, so check out the ref you mean first. The generators +exit 1 for "stale" and 2 for "could not run"; the weekly `upstream-drift` workflow +reports the two differently. ## CI overview -- **`ci.yml`** — `ansible-lint` + `galaxy-build` (on `ansible/**`), `helm` (on - `charts/**`, the shared schema inventory/checker, the root `Makefile` or `ci.yml` - itself), `kics` (on either) and `actionlint`. Bash-only PRs skip the Ansible jobs. Hygiene/shellcheck/markdownlint - run via **pre-commit locally only** (`make hooks` / `make lint`), not in CI. -- **`molecule.yml`** — containerised converge + idempotence + verify of the `decdn_node` - role (privileged systemd Docker container; scoped to `ansible/**`). Run locally with - `make molecule` (needs Docker) — it runs all six scenarios in parallel, so reach for - `make molecule-serial` when you need to read a failure in order. +- **`ci.yml`**, path-filtered so heavy jobs skip unrelated PRs: + - always: `pre-commit` (every hook, every file), `scripts` (`make test-scripts`) and + `actionlint`; + - on `ansible/**`: `ansible-lint` (plus a syntax-check of every playbook), + `galaxy-build` and `alloy-config` (`make lint-alloy`); + - on `charts/**` (or the shared schema files, the root `Makefile`, `ci.yml`): `helm` + (`make lint-helm`); + - on `compose/**` (or the root `Makefile`, `ci.yml`): `compose` (`make lint-compose`); + - on any of those: `kics` (`make security`). +- **`molecule.yml`**: `make molecule JOBS=3` on `ansible/**` changes. +- **`release.yml`**: on `vX.Y.Z` tags; see [RELEASING.md](RELEASING.md). +- **`upstream-drift.yml`**: weekly, non-blocking; see "Upstream mirrors" above. - **Every job is bounded** by `timeout-minutes`. The values are bounds sized off - observed runtimes, not targets — without one a hung job burns the 360-minute - default, and combined with `cancel-in-progress` some branch-protection setups read - the resulting *cancelled* check as "not failed" rather than as a failure. -- **Two caches.** `ansible/collections` is cached across `ansible-lint`, - `galaxy-build` and `molecule` under one shared key; a hit makes `make deps` a no-op - that never contacts `galaxy.ansible.com`, which is what keeps a transient Galaxy - error from failing an unrelated PR. pip is cached via `setup-python`, keyed on the - workflow file (the repo has no pip manifest, so the workflow *is* the package list); - the installs stay unpinned, so that saves the download but not the PyPI round trip. - Only the Galaxy cache is wired by hand — it uses `actions/cache`'s split - `restore`/`save` with `save` gated on success, so a part-way Galaxy failure can't - poison it. The pip cache is `setup-python`'s built-in one and manages itself. + observed runtimes, not targets. Without one a hung job burns the 360-minute default, + and combined with `cancel-in-progress` some branch-protection setups read the + resulting *cancelled* check as "not failed". +- **Caches.** `ansible/collections` is cached across `ansible-lint`, `galaxy-build` and + `molecule` under one shared key; a hit makes `make deps` a no-op that never contacts + `galaxy.ansible.com`, which keeps a transient Galaxy error from failing an unrelated + PR. It uses `actions/cache`'s split `restore`/`save` with `save` gated on success, so a + part-way Galaxy failure can't poison it. pip is cached by `setup-python`, keyed on the + workflow file (the repo has no pip manifest; installs stay unpinned, so that saves the + download, not the PyPI round trip). pre-commit's hook environments are cached on + `.pre-commit-config.yaml`. ## Supply-chain / pinning rules @@ -68,11 +97,14 @@ Run it on demand with `make lint-ansible`, or `pre-commit run ansible-lint --hoo drives the **Docker Hub** KICS engine image — a different artifact from the hijacked action — pinned by a digest verified against Docker Hub, currently `v2.1.20`. The engine's `--fail-on high` exit code is the gate. -- **Dependabot** (`.github/dependabot.yml`) bumps the other action SHAs weekly. -- **Bump manually** (Dependabot can't): the `KICS_IMAGE` and `KUBECONFORM_IMAGE` digests - in the `Makefile`, both `setup-helm` `version:` inputs in `ci.yml` (`helm` and `kics` - jobs), the collection versions in `ansible/requirements.yml`, and the pre-commit hook - revs via `pre-commit autoupdate`. +- **Dependabot** (`.github/dependabot.yml`) bumps the action SHAs and the pre-commit + hook revs weekly. +- **Bump manually** (Dependabot can't parse them): the `KICS_IMAGE` and + `KUBECONFORM_IMAGE` digests in the `Makefile`; the molecule image digests in + `ansible/molecule/*/molecule.yml` (all together, `docker buildx imagetools inspect`); + the four `setup-helm` `version:` inputs (`ci.yml`'s `helm` and `kics` jobs, both + jobs in `release.yml`); the collection versions in `ansible/requirements.yml`; and the local + yamllint hook's `additional_dependencies` pin. - **Bump the `cache-epoch:` counter in `ansible/requirements.yml` to make CI re-resolve the collections.** Those are `>=` ranges, so a warm cache pins the resolved set — transitive collections like `community.crypto` included — until the @@ -80,9 +112,3 @@ Run it on demand with `make lint-ansible`, or `pre-commit run ansible-lint --hoo themselves. It is a comment, but a load-bearing one: the cache key is that file's hash. Keeping it *in* the hashed file is deliberate — an epoch duplicated across both workflows could drift, since neither workflow runs on a change to the other. - -## Solidity - -There is no Foundry project in the repo yet. The `forge fmt` pre-commit hook and the -CI `solidity` job are present but commented out; they self-activate once real sources -land. diff --git a/Makefile b/Makefile index a4d6ad6..ba33f91 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ # Convenience targets for the deCDN DevOps monorepo. # Run from the repo root. Ansible-specific work is delegated to ansible/Makefile. -.PHONY: help hooks lint lint-ansible lint-helm lint-alloy security security-ansible security-helm molecule molecule-serial galaxy-build galaxy-check +.PHONY: help hooks lint lint-ansible lint-helm lint-alloy lint-compose test-scripts security security-ansible security-helm security-compose molecule molecule-serial galaxy-build galaxy-check SHELL := /bin/bash # KICS runs straight from the engine image, pinned by digest. This target IS the @@ -30,9 +30,9 @@ lint-ansible: ## full ansible-lint locally (installs collections first) $(MAKE) -C ansible deps $(MAKE) -C ansible lint -# Both scans always run, so a finding in one never hides the other's results. -security: ## KICS IaC security scan of ansible/ and the Helm chart (CI runs this) - @rc=0; $(MAKE) security-ansible || rc=1; $(MAKE) security-helm || rc=1; exit $$rc +# All three scans always run, so a finding in one never hides the others' results. +security: ## KICS IaC security scan of ansible/, the Helm chart and compose/ (CI runs this) + @rc=0; $(MAKE) security-ansible || rc=1; $(MAKE) security-helm || rc=1; $(MAKE) security-compose || rc=1; exit $$rc # -w /repo so findings carry repo-relative paths (not ../../repo/...), which is # what the CI job summary prints and what SARIF code-scanning uploads need. @@ -56,6 +56,47 @@ security-helm: ## KICS scan of the decdn-node chart's rendered manifests ( --report-formats json,sarif --output-path /repo/kics-results/helm \ --no-progress --fail-on high +# One query is excluded, deliberately: "Volume Has Sensitive Host Directory" +# (1c1325ff-…) fires on the read-only /etc/decdn mount. That directory is the node's +# own config dir, mounted :ro, and keeping the Ansible host layout is what lets the +# host CLI, backups and restores (docs/lifecycle.md) work unchanged. The two MEDIUMs +# (host network, no healthcheck) are the documented design; see compose/README.md. +security-compose: ## KICS scan of compose/ (pinned engine image) + mkdir -p kics-results + docker run --rm --user $(shell id -u):$(shell id -g) -w /repo -v "$(CURDIR):/repo" $(KICS_IMAGE) \ + scan --path /repo/compose --type DockerCompose \ + --exclude-queries 1c1325ff-831d-43a1-973e-839ae57dfcc0 \ + --report-formats json,sarif --output-path /repo/kics-results/compose \ + --no-progress --fail-on high + +# Renders compose/compose.yaml with the example env files and asserts the invariants +# the README promises: host networking and no published ports (so metrics and the +# admin RPC stay loopback), a digest-pinned image, read-only rootfs, every capability +# dropped, no-new-privileges, and a stop grace long enough for the daemon's drain. +COMPOSE_INVARIANTS := .services["decdn-node"] as $$s | \ + ($$s.network_mode == "host") and ($$s.ports == null) and \ + ($$s.image | test("@sha256:[0-9a-f]{64}$$")) and ($$s.read_only == true) and \ + ($$s.cap_drop == ["ALL"]) and ($$s.security_opt | index("no-new-privileges:true") != null) and \ + ($$s.stop_signal == "SIGTERM") and ($$s.stop_grace_period == "5m0s") and \ + ($$s.user | test("^[0-9]+:[0-9]+$$")) + +# COMPOSE_FILE is overridable so tests/scripts-test.sh can feed it broken variants. +COMPOSE_FILE ?= compose/compose.yaml +lint-compose: ## render compose/ with its examples and check its security invariants (needs docker, jq) + @set -o pipefail; \ + rendered="$$(DECDN_ENV_FILE=$(CURDIR)/compose/decdn.env.example docker compose -f '$(COMPOSE_FILE)' \ + --env-file compose/.env.example config --format json)" \ + || { echo "lint-compose: docker compose could not render $(COMPOSE_FILE) (see above)" >&2; exit 2; }; \ + jq -e '$(COMPOSE_INVARIANTS)' <<<"$$rendered" >/dev/null \ + || { echo "$(COMPOSE_FILE) violates an invariant (see the lint-compose comment in Makefile)" >&2; exit 1; } + @echo "compose invariants hold" + +# The guard rails nothing else exercises: ansible/Makefile's scoping guards, the +# release gate, lint-compose's negative cases, and (with UPSTREAM=) +# the upstream-mirror generators' exit codes. CI job `scripts`. +test-scripts: ## test the Makefile guards, release gate and lint-compose negatives (needs docker, jq) + tests/scripts-test.sh + lint-helm: ## helm lint + render tests + kubeconform + shared schema-key check (needs helm, yq, python3>=3.11, docker) KUBECONFORM="docker run --rm -i $(KUBECONFORM_IMAGE)" $(CHART)/tests/render-test.sh diff --git a/README.md b/README.md index 89d4118..bc9c4ae 100644 --- a/README.md +++ b/README.md @@ -8,145 +8,110 @@ [![shellcheck](https://img.shields.io/badge/shellcheck-passing-brightgreen)](https://www.shellcheck.net/) [![Conventional Commits](https://img.shields.io/badge/Conventional%20Commits-1.0.0-yellow.svg)](https://www.conventionalcommits.org) -The official **DevOps repo** for deploying a deCDN node — infrastructure, deployment, and -operational tooling: a declarative [Ansible](ansible/README.md) project for VMs and bare -metal, and a [Helm chart](charts/decdn-node/README.md) for Kubernetes. +The official **DevOps repo** for running a deCDN node: infrastructure, deployment and +day-2 tooling for operators anywhere. Three ways to deploy the same node: -This repo is **infrastructure only**. It is *not* a source of truth for protocol or -economic facts (chain-id, token addresses, fee splits) — those trace to the deCDN ADRs. -Anything here that states a protocol fact traces back to an ADR; nothing is invented in -this repo. - -## What it deploys - -The same node, two ways: - -| Path | Deploys | Exposure | -|------|---------|----------| -| **`ansible/playbooks/site.yml`** | A public **deCDN node** (`decdn-node`) — the product. Installed from a pinned GitHub release tarball under a hardened systemd unit, over a hardened host baseline. | Public QUIC **udp/4433** | -| **`charts/decdn-node`** | The same node on Kubernetes: one-replica StatefulSet, PVC data dir, operator-provisioned Secrets, restricted pod security. | Public QUIC **udp/4433** (LoadBalancer/NodePort/hostPort); metrics ClusterIP + NetworkPolicy | - -## Architecture - -``` -baseline host hardening — DevSec os/ssh, nftables default-deny inbound, - fail2ban, unattended-upgrades, chrony, an admin sudo user - │ - └─ site.yml → decdn-node public QUIC udp/4433; metrics+admin loopback; - signed-tarball or local-build install; - hardened systemd unit -``` - -On-chain node stake + registration (ADR 019 Phase 2) is an **operator step**, not -automated here — the node serves paid traffic only after it is staked and registered. -Upstream's `decdn setup` walks that phase end to end (with `--dry-run`); this repo -stops at host prep and startup. +| Path | For | Start here | +|------|-----|------------| +| **Ansible** (`ansible/`) | VMs and bare metal, one node or a fleet. Hardens the host too (firewall, SSH, patching). Also published as the `decdn.node` Galaxy collection. | [`ansible/README.md`](ansible/README.md) | +| **Docker Compose** (`compose/`) | One host that already runs Docker. | [`compose/README.md`](compose/README.md) | +| **Helm** (`charts/decdn-node/`) | Kubernetes, one release per node. | [`charts/decdn-node/README.md`](charts/decdn-node/README.md) | -## Repository layout - -| Path | What it is | -|------|------------| -| [`ansible/`](ansible/README.md) | The **declarative deployment project** — `inventory/`, `playbooks/`, `roles/` (baseline, decdn_node). The whole deploy surface lives here. | -| [`charts/decdn-node/`](charts/decdn-node/README.md) | The **Helm chart** for running the node on Kubernetes. | -| `Makefile` | Root hygiene/security/CI mirror — runs the same lint + IaC scans CI does. | -| `ansible/Makefile` | The deploy driver — `make deps/check/deploy`. | -| `.github/workflows/` | The blocking CI gate (`ansible-lint` + `helm` + KICS + `galaxy-build` + `molecule` + `actionlint`). | +Supported hosts: Debian 12/13 and Ubuntu 24.04/26.04 on x86_64 or aarch64. Network, +disk and platform requirements, and how to choose a path: +[`docs/requirements.md`](docs/requirements.md). -## Quickstart +This repo is **infrastructure only**. It is *not* a source of truth for protocol or +economic facts (chain-id, token addresses, fee splits): those trace to the deCDN ADRs +and upstream's deployment manifests. Where this repo carries one (the contract addresses +behind `decdn_network`), it is a generated mirror with its upstream commit recorded. + +## What every path gives you + +- **The node, hardened.** `decdn-node` as a non-root service with a minimal privilege + set: a hardened systemd unit, a locked-down container, or a restricted pod. +- **One public port.** QUIC on **udp/4433**. Metrics (9090) and the admin RPC (9191) + stay on loopback (on Kubernetes, behind a ClusterIP Service and a NetworkPolicy). +- **The right chain config.** Contract addresses come from upstream's deployment + manifest: `decdn_network: arbitrum-sepolia` on Ansible, `decdn config init --chain` + for Compose and Helm. Nothing is hand-copied. +- **Signed installs.** Ansible verifies release tarballs against the GPG-signed + `SHA256SUMS`; Compose only takes the image by digest; Helm takes a digest + (recommended) or a tag. +- **Monitoring.** Upstream's Grafana dashboards and alert rules, with the labels they + expect: opt-in Grafana Cloud shipping via `grafana_alloy` on Ansible, a + `ServiceMonitor` + `PrometheusRule` + dashboard ConfigMaps on Helm + ([`charts/decdn-node/files/monitoring/`](charts/decdn-node/files/monitoring/README.md)). +- **Day 2.** Encrypted backups, restore and host migration, and a guarded + decommission: [`docs/lifecycle.md`](docs/lifecycle.md). + +On-chain stake and registration (ADR 019 Phase 2) is an **operator step** on every +path: the node serves paid traffic only after it. Upstream's `decdn setup` walks it +end to end (with `--dry-run`); this repo stops at host prep and startup. + +## Quickstart (Ansible) ```bash cd ansible -make deps # vendor pinned Galaxy collections into ./collections +make deps # vendor pinned Galaxy collections cp inventory/hosts.yml.example inventory/hosts.yml -$EDITOR inventory/hosts.yml # set hosts for decdn_nodes -$EDITOR inventory/group_vars/all.yml # optional: add admins to baseline_sudo_users (runner is auto-detected) -cp inventory/host_vars/decdn-node-1/secret.yml.example inventory/host_vars/decdn-node-1/secret.yml -$EDITOR inventory/host_vars/decdn-node-1/secret.yml # set decdn_rpc_url (per-node config is in main.yml) -make check # dry run (--check --diff) -make deploy # provision the deCDN node +$EDITOR inventory/hosts.yml # your hosts in decdn_nodes +$EDITOR inventory/host_vars/decdn-node-1/main.yml # binaries, region, origin; chain via decdn_network +# RPC URL: provision 0600 /etc/decdn/decdn.env on the host (preferred), or secret.yml +make check LIMIT=decdn-node-1 ANSIBLE_ARGS='-u root' # dry run; -u root only until the first converge +make deploy LIMIT=decdn-node-1 ANSIBLE_ARGS='-u root' # first converge creates your admin account +make deploy LIMIT=decdn-node-1 # every run after that ``` -See [`ansible/README.md`](ansible/README.md) for the full setup and the deCDN-node -prerequisites (release tarball, per-node `host_vars`, operator-provisioned eth keystore). - -On Kubernetes, create the keystore and RPC Secrets out of band, then install the chart -(see [`charts/decdn-node/README.md`](charts/decdn-node/README.md)): - -```bash -helm install decdn-node-1 charts/decdn-node -n decdn -f values-node-1.yaml -``` +The full flow (bootstrap user, keystore, secrets, fleets in a private inventory) is in +[`ansible/README.md`](ansible/README.md). Compose and Helm have their own quickstarts. ## Security model -- **Nothing secret is committed.** The eth keystore and `rpc_url` (which may embed an API - key) are **generated on — or operator-provisioned to — the target host**, never the repo. - Ansible roles render them on the host (`no_log`, `0600`); the repo ships a `*.example` - template for the one node secret (`rpc_url`) plus a `hosts.yml.example` starter, and commits - non-secret per-node config directly (`host_vars//main.yml`), with the root `.gitignore` - as a backstop. -- **Localhost-only by default.** Backends bind `127.0.0.1`; a service that must accept - public traffic declares its port explicitly. The node host opens one extra hole - (udp/4433 QUIC); everything else (node metrics 9090, admin RPC 9191) stays loopback. -- **Default-deny inbound (nftables).** SSH is the only universally-open port; extra public - ports are declared explicitly via `baseline_extra_inbound`. -- **DevSec host hardening.** `os_hardening` + `ssh_hardening` (key-only SSH, no root login, - kernel/sysctl/PAM hardening) — applied last, after the admin key is in place, so you - can't lock yourself out. - -## Commands - -Two Makefiles, two jobs. The **root** Makefile mirrors CI's hygiene/security gates; the -**`ansible/`** Makefile drives deploys (run its targets from `ansible/`). +This is the canonical statement; the per-path READMEs add only what is specific to them. + +- **Nothing secret is committed.** The eth keystore and the RPC URL (which may embed an + API key) are **generated on, or operator-provisioned to, the target** and live in + `0600` files readable only by whoever must read them: the service account for the + keystore and its password; the service account (Ansible) or root (Compose, where + Docker reads it before starting the container) for the RPC env file. On Kubernetes + they are operator-created Secrets the chart only references. The repo ships `*.example` templates for secret files only; + non-secret config such as `host_vars//main.yml` is committed. The `.gitignore` + is a backstop, not the mechanism. Backups are encrypted on the host to public keys + you choose. +- **Localhost-only by default.** Backends bind `127.0.0.1`. A service that must accept + public traffic declares its port explicitly, and the node declares exactly one: + udp/4433. On Kubernetes, metrics bind `0.0.0.0` in the pod only behind a ClusterIP + Service and a NetworkPolicy. +- **Default-deny inbound** (Ansible's `baseline`, nftables). SSH is the only + universally open port; extra public ports are declared via `baseline_extra_inbound`. +- **DevSec host hardening** (`os_hardening` + `ssh_hardening`: key-only SSH, no root + login, kernel/sysctl/PAM hardening), applied last, after the admin key is in place, so + you can't lock yourself out. +- **Pinned supply chain.** Release tarballs are GPG-verified; images, CI actions and + scanners are pinned by digest or commit SHA ([`SECURITY.md`](SECURITY.md), + [`CONTRIBUTING.md`](CONTRIBUTING.md#supply-chain--pinning-rules)). -```bash -# Root — lint & security (mirror CI) -make hooks # one-time: install the pre-commit git hook (pip install pre-commit first) -make lint # all pre-commit hooks on all files (hygiene, shellcheck, yamllint, markdown) -make lint-ansible # vendor collections + full ansible-lint (production profile) -make lint-helm # chart: helm lint + render tests + kubeconform + schema keys -make lint-alloy # grafana_alloy: render its templates, validate with the real Alloy binary -make security # KICS IaC scan of ansible/ + the rendered chart (pinned engine image) - -# Ansible deploys — run from ansible/ -cd ansible -make deps # vendor pinned Galaxy collections into ./collections -make check / deploy # deCDN node (site.yml): dry-run / provision - # fleet-wide by default; LIMIT= scopes, ANSIBLE_ARGS='…' passes through -``` +## Repository layout -**Gotcha — pre-commit is local-only.** Hygiene/shellcheck/yamllint/markdown run via -`make hooks`/`make lint` on your machine, **not** in CI. The blocking gate is -`.github/workflows/` (`ansible-lint` + KICS + `galaxy-build` + `molecule` on `ansible/**`, -`helm` + KICS on `charts/**` and the shared schema checker, plus `actionlint`). `ansible-lint` -is not a per-commit hook (it needs collections vendored) — run `make lint-ansible`. - -## CI & quality gates - -- **`ci.yml`** — path-filtered so heavy jobs skip unrelated PRs: `ansible-lint` (production - profile + playbook syntax-check), a `galaxy-build` readiness gate (builds the `decdn.node` - collection and runs galaxy-importer's checks), an `alloy-config` job (`make lint-alloy`: renders - the `grafana_alloy` templates and runs the real digest-pinned Alloy binary's `alloy validate` - over them — the molecule stub cannot), a `helm` job (`make lint-helm`: strict lint, - positive/negative render tests, kubeconform, and the upstream schema-key check shared with - molecule), **KICS** IaC scan of `ansible/` and the rendered chart (fail on HIGH), and - `actionlint` on the workflows themselves. The KICS engine is pinned by digest and every - third-party action by full commit SHA (a re-pointed tag can ship malicious code). -- **`molecule.yml`** — a containerised converge + idempotence + verify of the `decdn_node` - role in a privileged systemd Docker container (scoped to `ansible/**` changes). - -## Conventions & source of truth - -- **ADRs are the only source of truth for protocol facts.** The deCDN ADRs cover, e.g., - payments (ADR 003), node onboarding (ADR 019), and tokenomics (ADR 026). If a doc here - contradicts an ADR, fix the doc. -- **Role templates render to their target paths.** Ansible roles template config directly - onto the host (e.g. `roles/decdn_node/templates/decdn-node.service.j2` → - `/etc/systemd/system/`), with secrets generated on the host at `0600`. - -## Further reading - -- [`ansible/README.md`](ansible/README.md) — full setup, security model, and deploy steps -- [`ansible/roles/decdn_node/README.md`](ansible/roles/decdn_node/README.md) — the deCDN node role -- [`charts/decdn-node/README.md`](charts/decdn-node/README.md) — the Helm chart -- [`CONTRIBUTING.md`](CONTRIBUTING.md) — the local + CI check workflow -- [`AGENTS.md`](AGENTS.md) — repo hard rules and conventions (for humans and AI agents) +| Path | What it is | +|------|------------| +| [`ansible/`](ansible/README.md) | The Ansible project: `inventory/`, `playbooks/` (`site.yml`, `backup.yml`, `decommission.yml`), `roles/` (`baseline`, `decdn_node`, `grafana_alloy`), `galaxy/` (the `decdn.node` collection), `molecule/`. | +| [`compose/`](compose/README.md) | The Docker Compose deploy path. | +| [`charts/decdn-node/`](charts/decdn-node/README.md) | The Helm chart, with vendored dashboards and alert rules in `files/monitoring/`. | +| [`docs/`](docs/requirements.md) | Cross-path operator docs: requirements, lifecycle. | +| `scripts/` | Generators for the upstream mirrors (network profiles, monitoring) and the release gate. | +| `Makefile` | Lint, test and security targets; CI runs the same ones. `make help` lists them. | +| `.github/workflows/` | CI (`ci.yml`, `molecule.yml`), releases (`release.yml`), the weekly upstream drift check. | + +## Contributing and releases + +- [`CONTRIBUTING.md`](CONTRIBUTING.md): local checks, every `make` target, what CI runs, + and the pinning rules. +- [`RELEASING.md`](RELEASING.md): how a `vX.Y.Z` tag publishes the collection and the chart. +- [`AGENTS.md`](AGENTS.md): the repo's hard rules (for humans and AI agents). +- [`SECURITY.md`](SECURITY.md): reporting a vulnerability, verifying releases. + +Conventions: commits follow Conventional Commits. The deCDN ADRs are the only source +of truth for protocol facts (payments ADR 003, node onboarding ADR 019, tokenomics +ADR 026); if a doc here contradicts an ADR, fix the doc. diff --git a/RELEASING.md b/RELEASING.md new file mode 100644 index 0000000..13ea2b7 --- /dev/null +++ b/RELEASING.md @@ -0,0 +1,79 @@ +# Releasing + +A `vX.Y.Z` tag releases two artifacts at one version: + +| Artifact | Published to | Install | +|----------|--------------|---------| +| `decdn.node` Ansible collection | [Ansible Galaxy](https://galaxy.ansible.com/ui/repo/published/decdn/node/) | `ansible-galaxy collection install decdn.node:==X.Y.Z` | +| `decdn-node` Helm chart | `oci://ghcr.io/decdn/charts`, signed with cosign (keyless) | `helm install oci://ghcr.io/decdn/charts/decdn-node --version X.Y.Z` | + +[`.github/workflows/release.yml`](.github/workflows/release.yml) does the work. Its +`build` job runs on every matching tag push. It gates the tag, re-runs +`make galaxy-check` and `make lint-helm`, packages both, and uploads them with a +`SHA256SUMS` as a workflow artifact. The `publish` job pushes to Galaxy and ghcr.io and +creates the GitHub Release, but **only** when the repository variable +`PUBLISH_ENABLED` is `true`. Until then, a tag push is a dry run. So is a manual +*Run workflow* (`workflow_dispatch`) with a tag, which is the way to rehearse. + +## Cutting a release + +1. **Versions.** Set the same `X.Y.Z` in `ansible/galaxy/galaxy.yml` (`version:`) and + `charts/decdn-node/Chart.yaml` (`version:`). Bump the chart's `appVersion` if it now + targets a newer decdn release. +2. **Changelogs.** Both `ansible/galaxy/CHANGELOG.md` and `charts/decdn-node/CHANGELOG.md` + collect changes under `## [Unreleased]`. At release time, move those entries under a + dated `## [X.Y.Z] — YYYY-MM-DD` heading and leave an empty `[Unreleased]` above it. + **First release only:** both files already hold a `## [0.1.0] — unreleased` section + describing the initial state; fold `[Unreleased]` into it and replace "unreleased" + with the date. The gate rejects a missing section and one still marked + "unreleased". These two sections become the GitHub Release notes. +3. **Check locally:** `scripts/check-release-version.sh vX.Y.Z`, then + `make -C ansible galaxy-check` and `make lint-helm`. +4. **Merge** that as a PR, then tag the merge commit on `main` and push the tag: + + ```bash + git tag -s vX.Y.Z -m "vX.Y.Z" && git push origin vX.Y.Z + ``` + +5. **Watch** the Release workflow. With publishing enabled, the `publish` job waits for + approval on the `release` environment. + +## First release only + +Do these before setting `PUBLISH_ENABLED`: + +- **Galaxy namespace.** The `decdn` namespace must exist on galaxy.ansible.com and the + account behind `GALAXY_API_KEY` must be allowed to publish to it. Add the key as the + repository secret `GALAXY_API_KEY`. +- **`release` environment.** Create it under *Settings → Environments* with required + reviewers, and put `GALAXY_API_KEY` there rather than as a repository secret if you + want the reviewer gate to guard it too. +- **GHCR visibility.** The first `helm push` creates the `decdn/charts/decdn-node` + package as **private**. Make it public under the org's *Packages* settings, or + nobody outside the org can pull it. +- **Enable:** set the repository variable `PUBLISH_ENABLED` to `true`. + +## When a publish fails half-way + +The steps run chart, then Galaxy, then the GitHub Release. The Release is last, so it +only appears once both artifacts are live. + +- **Chart push failed:** nothing is public yet. Fix the cause and re-run the job. +- **Galaxy publish failed after the chart was pushed:** fix the cause and re-run. The + chart push is repeatable for the same version (the tag just moves to an identical + digest), but **Galaxy refuses a version that already exists**. If Galaxy actually + accepted the upload before the job failed, finish by hand: create the Release with + `gh release create vX.Y.Z` and the files from the `release-vX.Y.Z` workflow artifact. +- **Never re-use a version** for different content. Cut `vX.Y.Z+1`. + +## Verifying a release + +```bash +# The chart's signature: keyless, tied to this repo's release workflow. +cosign verify ghcr.io/decdn/charts/decdn-node:X.Y.Z \ + --certificate-identity-regexp '^https://github.com/decdn/devops/.github/workflows/release.yml@refs/tags/v' \ + --certificate-oidc-issuer https://token.actions.githubusercontent.com + +# The files attached to the GitHub Release. +sha256sum --check SHA256SUMS +``` diff --git a/SECURITY.md b/SECURITY.md index fd46ed8..89e46ab 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -8,7 +8,7 @@ reports. That includes a secret or real host address you find committed here. ## Scope This repo holds the deployment tooling for a deCDN node: the Ansible roles, the -`decdn.node` Galaxy collection, and the Helm chart. Reports about the node daemon or +`decdn.node` Galaxy collection, the Docker Compose file, and the Helm chart. Reports about the node daemon or protocol belong to [decdn/decdn](https://github.com/decdn/decdn), but the same address reaches both. @@ -37,3 +37,7 @@ Check a new copy of the vendored key against it before trusting that copy, with Setting `decdn_verify_release_signature: false` (meant for an air-gapped mirror that strips signatures) drops that guarantee, and the role prints a warning when it's off. `manual` mode verifies nothing: it installs whatever binaries you point it at. + +Published Helm charts (`oci://ghcr.io/decdn/charts/decdn-node`) are signed with a +keyless cosign signature from this repo's release workflow; see +[RELEASING.md § Verifying a release](RELEASING.md#verifying-a-release). diff --git a/ansible/.gitignore b/ansible/.gitignore index fb6807b..37fa9a9 100644 --- a/ansible/.gitignore +++ b/ansible/.gitignore @@ -12,12 +12,16 @@ inventory/host_vars/*/secret.* inventory/*/host_vars/*/secret.* !inventory/*/host_vars/*/secret.*.example -# The real inventory. This repo is PUBLIC, so real host IPs must never land here -# (Launch/release-checklist.md §5). Copy hosts.yml.example to hosts.yml for a local -# run, or keep a fleet's inventory in a private overlay and pass INVENTORY= -# (see README.md "Private fleet inventory"). The .example templates stay tracked. +# The real inventory. This repo is PUBLIC, so real host IPs must never land here. +# Copy hosts.yml.example to hosts.yml for a local run, or keep a fleet's inventory +# in a private overlay and pass INVENTORY= (see README.md "Private fleet +# inventory"). The .example templates stay tracked. inventory/hosts.yml inventory/hosts.yaml # Ansible scratch *.retry + +# Encrypted node backups fetched by `make backup` (decdn_backup_local_dir). Encrypted, +# but still per-operator key material: never commit them. +backups/ diff --git a/ansible/Makefile b/ansible/Makefile index be7522c..8a8d6ba 100644 --- a/ansible/Makefile +++ b/ansible/Makefile @@ -1,6 +1,6 @@ # Convenience targets for the deCDN Ansible project. # Always run from the ansible/ directory. -.PHONY: deps lint check deploy molecule molecule-serial build galaxy-check +.PHONY: deps lint check deploy backup decommission molecule molecule-serial build galaxy-check SHELL := /bin/bash # Install the required Galaxy collections (>= constraints in requirements.yml) @@ -26,7 +26,7 @@ lint: # (history expansion) or 'decdn-node-*' (globbing) before make ever starts. # - ANSIBLE_ARGS is substituted unquoted on purpose: the recipe's shell splits it # back into separate flags (so inner quoting works — -# ANSIBLE_ARGS='--start-at-task="Install the decdn binaries"' survives). Unquoted +# ANSIBLE_ARGS='--start-at-task="Install decdn-node + decdn CLI"' survives). Unquoted # at your prompt, make takes the extra words as goals and flags of its own — a # bare -vv is make's own -v. A literal $ must be written $$. LIMIT ?= @@ -42,19 +42,21 @@ limit_arg = $(if $(LIMIT),--limit '$(LIMIT)') INVENTORY ?= inventory/hosts.yml # A missing or unparseable inventory fails the run. Ansible's default only warns, # skips the `hosts: decdn_nodes` play and exits 0, so a mistyped INVENTORY= would -# report a successful deploy that did nothing. This applies to check/deploy only: +# report a successful deploy that did nothing. This applies to the playbook targets +# (check/deploy/backup/decommission) only: # ansible.cfg can't set it globally, because ansible-lint's own syntax checks run # with no inventory present. strict_inventory = ANSIBLE_INVENTORY_UNPARSED_FAILED=True # Fail loud on the two ways a run can look scoped but silently be fleet-wide. Scoped to -# check/deploy via MAKECMDGOALS so an unrelated `make lint` is never blocked. NOTE: this +# the playbook targets (check/deploy/backup/decommission) via MAKECMDGOALS so an +# unrelated `make lint` is never blocked. NOTE: this # does not catch an argument that is well-formed but selects nothing — `--tags` with a # typo (decdn-node vs decdn_node) runs zero tasks and still exits 0. See ansible/README.md. -ifneq ($(filter check deploy,$(MAKECMDGOALS)),) +ifneq ($(filter check deploy backup decommission,$(MAKECMDGOALS)),) # Echo back the goal the operator actually typed, so the hint never nudges a # `make check` dry-run toward the mutating `make deploy`. - scoped_goal := $(firstword $(filter check deploy,$(MAKECMDGOALS))) + scoped_goal := $(firstword $(filter check deploy backup decommission,$(MAKECMDGOALS))) ifeq ($(origin LIMIT),environment) $(error LIMIT is set in your environment ($(LIMIT)) and would silently scope this run. Unset it or pass it on the command line: make $(scoped_goal) LIMIT=) endif @@ -72,6 +74,14 @@ ifneq ($(filter check deploy,$(MAKECMDGOALS)),) $(error LIMIT was supplied but expanded empty (unset shell variable?) - refusing to run fleet-wide by accident. Omit LIMIT entirely to mean "all hosts") endif endif + # Decommission is never fleet-wide by default: it needs an explicit LIMIT on the + # command line (the role also caps it at decdn_decommission_max_hosts and asks for + # typed confirmation). + ifneq ($(filter decommission,$(MAKECMDGOALS)),) + ifneq ($(origin LIMIT),command line) + $(error make decommission needs LIMIT= on the command line - it never runs fleet-wide by default) + endif + endif endif # Dry run against $(INVENTORY) — shows diffs, mutates nothing. @@ -82,6 +92,17 @@ check: deploy: $(strict_inventory) ansible-playbook -i '$(INVENTORY)' playbooks/site.yml $(limit_arg) $(ANSIBLE_ARGS) +# Encrypted backup of each node (identity by default; -e decdn_backup_scope=full for +# the whole data dir, which stops the node for the copy). One host at a time. +# Needs decdn_backup_age_recipients. See docs/lifecycle.md. +backup: + $(strict_inventory) ansible-playbook -i '$(INVENTORY)' playbooks/backup.yml $(limit_arg) $(ANSIBLE_ARGS) + +# Stop the node and remove its unit on LIMIT (required). Keeps the identity; prints +# the on-chain exit steps. Asks for typed confirmation. See docs/lifecycle.md. +decommission: + $(strict_inventory) ansible-playbook -i '$(INVENTORY)' playbooks/decommission.yml $(limit_arg) $(ANSIBLE_ARGS) + # --- Tests ------------------------------------------------------------------- # Containerised converge + idempotence + verify of the decdn_node role against a # stub daemon (needs Docker; a privileged systemd container). See molecule/. @@ -89,7 +110,10 @@ deploy: # against the upstream field list), validation (bad knobs, both roles, must be # rejected by their own asserts), generate-keystore (opt-in host-side wallet), # host-env (host-provisioned /etc/decdn/decdn.env), slow-readiness (advisory -# /metrics probe timeout), grafana-cloud (opt-in Grafana Cloud observability). +# /metrics probe timeout), grafana-cloud and grafana-cloud-token (opt-in Grafana +# Cloud observability, host-provisioned vs inventory token), os-matrix (default's +# plays on Debian 13, Ubuntu 24.04 and 26.04) and lifecycle (network profile, +# backup + decommission). The glob below is the authority; this list is a guide. # # The scenarios run concurrently: distinct container names, no published host ports, # per-scenario ephemeral dirs, and what they share on the control machine (the @@ -141,9 +165,9 @@ molecule-serial: deps molecule test --all # --- Galaxy collection (decdn.node) ------------------------------------------ -# Stage baseline + decdn_node into a clean collection tree and build the artifact -# under build/. Only the three deployment roles ship; see galaxy/README.md. Publishing stays -# a manual step (ansible-galaxy collection publish build/decdn-node-*.tar.gz). +# Stage baseline + decdn_node + grafana_alloy into a clean collection tree and build the +# artifact under build/. Only the three roles ship; see galaxy/README.md. Publishing +# runs from a vX.Y.Z tag in .github/workflows/release.yml (see RELEASING.md). build: ./galaxy/build.sh diff --git a/ansible/README.md b/ansible/README.md index 5b3b9ba..91ee7a3 100644 --- a/ansible/README.md +++ b/ansible/README.md @@ -1,38 +1,38 @@ # ansible — deCDN deployment -Declarative Ansible project for deploying a deCDN node. One deployment over a shared host -baseline: +Declarative Ansible project for deploying deCDN nodes, one or a fleet, over a +hardened host baseline. The roles also ship as the `decdn.node` Galaxy collection. -| Playbook | Purpose | Exposure | -|----------|---------|----------| -| **`site.yml`** | A public **deCDN node** (`decdn-node`) — the product. | Public QUIC udp/4433 | +| Playbook | Purpose | Make target | +|----------|---------|-------------| +| **`site.yml`** | Harden the host and deploy the public **deCDN node** (`decdn-node`). | `make check` / `make deploy` | +| `backup.yml` | Encrypted backup of a node's identity or full state. | `make backup` | +| `decommission.yml` | Stop a node and remove its service (keeps the identity; no on-chain steps). | `make decommission` | ``` -baseline host hardening — DevSec os/ssh, nftables default-deny inbound, - fail2ban, unattended-upgrades, chrony, an admin sudo user - │ - └─ site.yml → decdn-node public QUIC udp/4433; metrics+admin loopback; - release-tarball install; hardened systemd unit +baseline host hardening: DevSec os/ssh, nftables default-deny inbound, + │ fail2ban, unattended-upgrades, chrony, an admin sudo user + ├─ grafana_alloy opt-in Grafana Cloud agent (metrics, journald, traces), loopback-only + └─ decdn_node public QUIC udp/4433; metrics + admin loopback; signed-tarball or + local-build install; hardened systemd unit; backup/decommission ``` ## Security model -- **Default-deny inbound (nftables).** SSH is the only universally-open port. The node host - additionally opens **udp/4433** (QUIC) via `baseline_extra_inbound`. Everything else - (node metrics 9090, admin RPC 9191) stays **loopback** with no hole. **nftables is the - firewall** — baseline turns off `os_hardening`'s ufw config template (`ufw_manage_defaults: - false`) so a misleading DROP-policy `/etc/default/ufw` is never written; and, separately, - operators must not install or enable ufw, which would replace the nftables ruleset and - drop QUIC/SSH. -- **No secrets in the repo.** The node's eth keystore is **operator-provisioned** and never - generated here; its `rpc_url` (which may embed an API key) is provisioned the same way — - a `0600 /etc/decdn/decdn.env` written on the target host, which the role gates on but - never reads back — or, if you prefer, carried in a git-ignored - `host_vars//secret.yml` (the rest of `host_vars` is committed, non-secret config) - and rendered to that same `0600` file. -- **Host hardening via DevSec** (`os_hardening` + `ssh_hardening`): key-only SSH, no root - login, kernel/sysctl/PAM hardening — applied last, after the admin key is in place. - baseline overrides a few `os_hardening` sysctls so hardening can't sever node +The repo-wide model is in the [root README](../README.md#security-model). What is +specific to this path: + +- **nftables is the firewall.** SSH plus the node's **udp/4433** (via + `baseline_extra_inbound`) are the only holes; metrics 9090 and the admin RPC 9191 stay + loopback. Baseline turns off `os_hardening`'s ufw config template + (`ufw_manage_defaults: false`) so a misleading DROP-policy `/etc/default/ufw` is never + written. Do not install or enable ufw: it would replace the nftables ruleset and drop + QUIC and SSH. +- **Two homes for the RPC URL**: a `0600 /etc/decdn/decdn.env` written on the host + (preferred; the role gates on it but never reads it back), or a git-ignored + `host_vars//secret.yml` the role renders into that same file. Details: + [`roles/decdn_node/README.md` § Secrets](roles/decdn_node/README.md#secrets). +- **DevSec hardening can't sever the node.** Baseline overrides a few `os_hardening` sysctls so hardening can't sever node connectivity: it preserves IPv6 RA/autoconf (`baseline_preserve_ipv6_autoconf`, so SLAAC-assigned addresses survive) and can loosen reverse-path filtering for multi-homed hosts (`baseline_rp_filter_loose`). @@ -40,7 +40,11 @@ baseline host hardening — DevSec os/ssh, nftables default-deny inbound, ## Requirements - Control machine: **Ansible ≥ 2.15**, `ansible-lint`, `yamllint`. -- Target: **Debian (bookworm)** host(s) reachable over SSH with a sudo-capable user. +- Target: **Debian 12 (bookworm) / 13 (trixie)** or **Ubuntu 24.04 (noble) / 26.04 + (resolute)** host(s), x86_64 or aarch64, reachable over SSH with a sudo-capable user. + `make molecule` converges `decdn_node` on all four (`grafana_alloy`'s install path on + Debian 12 only); `baseline` is verified on real + hosts (see `roles/baseline/README.md` § Platforms). - **Ubuntu sudo-rs note:** 25.10+ (and 26.04) ship `sudo-rs` as the default `sudo`, which doesn't honor the custom `-p` become prompt Ansible relies on — so `--ask-become-pass` hangs with "Timeout waiting for privilege escalation prompt". On @@ -101,11 +105,10 @@ inventory, currently only the udp/4433 QUIC firewall hole, live in `playbooks/group_vars/decdn_nodes.yml`, so an overlay can't drop them. Override that per node in `host_vars` if you have to. Playbook group_vars beat inventory group_vars. -[`inventory/fleet.example/`](inventory/fleet.example/hosts.yml) is the launch-fleet -template. Hosts are grouped by role (`decdn_seed` holds the catalogue in an fs origin, -`decdn_edge` pulls through) and by billing (`decdn_metered` gets an egress budget, -`decdn_unmetered` does not). The group_vars size the cache for large model blobs. The -launch sequence is in [`docs/launch-runbook.md`](docs/launch-runbook.md). +[`inventory/fleet.example/`](inventory/fleet.example/hosts.yml) is the overlay +template: every host in `decdn_nodes`, the chain from `decdn_network`, and the cache +sizing knobs to fill per disk. Add child groups of your own when hosts differ by group +(an origin, an egress cap on metered bandwidth); the template's header shows how. By default baseline **deploys you as yourself**: the runner (your control-machine `$USER` + its autodetected `~/.ssh` key, `id_ed25519` > `ecdsa` > `rsa`) is prepended as the head of @@ -138,22 +141,22 @@ ignored on the first converge. **Prerequisites** (see `roles/decdn_node/README.md` for the full flow): -1. A published **`v` release** exists and is **publicly downloadable** — the role - fetches the release tarball from `decdn_node_release_base` - (default `https://github.com/decdn/decdn/releases/download`). To install from a mirror, - override `decdn_node_release_base`; to deploy a locally-built binary with no release at - all, use the **`manual`** install method (`decdn_node_install_method: manual` + - `decdn_node_manual_bin_src` / `decdn_cli_manual_bin_src`). -2. Per-node config in `inventory/host_vars//main.yml` (committed) — - the install method + binary sources, the **four** required contract addresses, - `decdn_region`, cache origin, … — plus the one secret, `decdn_rpc_url`, either - provisioned as `/etc/decdn/decdn.env` on the host (preferred — see - [Setup](#setup)) or in a sibling git-ignored `secret.yml` (copy the shipped - `host_vars/decdn-node-1/secret.yml.example`). - The committed `main.yml` already carries the current Arbitrum Sepolia addresses; edit - `decdn_region`, the binary paths and the cache origin for your node. Contract addresses - are protocol facts — copy them from the upstream deployment manifest - `decdn/contracts/deployments/.json`, never guess. +1. **Binaries.** Upstream has not tagged a release yet, so the default install method + is **`manual`**: build `decdn-node` and `decdn` from a `decdn/decdn` checkout and point + the role at them (`decdn_release_target_dir`, or both `decdn_node_manual_bin_src` and + `decdn_cli_manual_bin_src`). Cross-compile for aarch64 hosts; the role derives each + host's target from its architecture and checks the ELF before shipping. Once a + `v` release exists, switch to `decdn_node_install_method: release` + + `decdn_node_version`; the role then downloads the tarballs from + `decdn_node_release_base` and verifies them against the GPG-signed `SHA256SUMS`. +2. **Per-node config** in `inventory/host_vars//main.yml` (committed): the binary + sources, `decdn_region`, the cache origin, and the chain, as + `decdn_network: arbitrum-sepolia`. The network profile supplies `chain_id` and every + contract address from the role's mirror of upstream's deployment manifest + ([`roles/decdn_node/README.md` § Network profiles](roles/decdn_node/README.md#network-profiles-decdn_network)), + so nothing is hand-copied. Plus the one secret, the RPC URL: provisioned as + `/etc/decdn/decdn.env` on the host (preferred, see [Setup](#setup)) or in a sibling + git-ignored `secret.yml` (copy the shipped `host_vars/decdn-node-1/secret.yml.example`). 3. The **eth keystore + password file** provisioned on the host (operator step — the wallet must be funded + staked per the deCDN node-onboarding ADR, 019). As the `decdn` user, create the password file FIRST (`key-gen` reads it, never creates it), then generate the @@ -188,9 +191,10 @@ connects as your `$USER` with no flag. You do not want that bootstrap play — o — reaching nodes already serving paid traffic. Same when re-running a single node after a config change, a failed play, or a binary bump. -`make check`/`make deploy` refuse to run if `LIMIT` or `ANSIBLE_ARGS` reaches them from an -exported shell variable, or if `LIMIT` expands empty — both are ways a run looks scoped but -is silently fleet-wide. +`make check`, `deploy`, `backup` and `decommission` refuse to run if `LIMIT`, +`ANSIBLE_ARGS` or `INVENTORY` reaches them from an exported shell variable, or if `LIMIT` +or `INVENTORY` expands empty: all ways a run looks scoped but silently targets something +else. `make decommission` additionally requires `LIMIT` on the command line. `ANSIBLE_ARGS` passes anything else straight through. Quote the whole value at your prompt, or make will read the extra words as its own goals and flags (a bare `-vv` is make's `-v`); @@ -199,7 +203,7 @@ literal `$` as `$$`: ```bash make deploy LIMIT=decdn-node-1 ANSIBLE_ARGS='--tags decdn_node -vv' -make deploy LIMIT=decdn-node-1 ANSIBLE_ARGS='--start-at-task="Install the decdn binaries"' +make deploy LIMIT=decdn-node-1 ANSIBLE_ARGS='--start-at-task="Install decdn-node + decdn CLI"' ``` > **Watch the PLAY RECAP.** A *well-formed* argument that selects nothing is not an error: @@ -226,66 +230,32 @@ primitives underneath it. All take `--dry-run`. See ### Grafana Cloud observability (opt-in) -The play ships a third role, `grafana_alloy`, that installs a loopback-only -[Grafana Alloy](https://grafana.com/docs/alloy/) agent shipping five signals to your -Grafana Cloud org — the node's `/metrics`, **the machine itself** (CPU/memory/disk/ -filesystem/network/load plus per-unit state, via Alloy's in-process `node_exporter`), -**journald**, the agent's own health, and the daemon's OTLP spans (`127.0.0.1:4317`) — -off by default, driven by ONE mirrored inventory flag: +A third role, `grafana_alloy`, installs a loopback-only +[Grafana Alloy](https://grafana.com/docs/alloy/) agent that ships the node's metrics, the +machine's own metrics, journald, the agent's health and the daemon's OTLP traces to your +Grafana Cloud org. One mirrored inventory flag drives both roles: ```yaml -# group_vars/host_vars — drives BOTH roles from one knob -decdn_grafana_cloud_enabled: true +decdn_grafana_cloud_enabled: true # group_vars or host_vars ``` -Machine metrics, journald and Alloy's own health come with it (each has a -`grafana_alloy_*_enabled` sub-knob, all defaulting to `true`). Host metrics and logs -carry `job="integrations/node_exporter"`, so Grafana Cloud's prebuilt **Linux Server** -dashboards and alert rules work unmodified. +Only the API token is secret. It lives on each host in `0600 /etc/grafana-alloy.env`, or +in the git-ignored `secret.yml` as `grafana_alloy_api_token`. The endpoints and instance +IDs are ordinary inventory variables. Setup, token handling, upgrade notes, cost +guardrails and rollback: [`roles/grafana_alloy/README.md`](roles/grafana_alloy/README.md). +To import upstream's decdn dashboards and alert rules into that stack, see +[`charts/decdn-node/files/monitoring/`](../charts/decdn-node/files/monitoring/README.md). -Only the API token is a secret. Put the non-secret connection settings in inventory -once for the fleet (`grafana_alloy_prom_url`, `_prom_username`, `_otlp_endpoint`, -`_otlp_username`, `_loki_url`, `_loki_username` — Prometheus, OTLP and Loki each have -their **own** instance ID, so copy each from the portal page that names it), and -provision just the token **on each target host** (on this path it never transits this -repo or the control machine; the inventory alternative below trades that for a -git-ignored `secret.yml`): +### Backup and decommission ```bash -umask 077 -printf 'GC_API_TOKEN=glc_…\n' > grafana-alloy.env -sudo install -m 600 -o root -g root grafana-alloy.env /etc/grafana-alloy.env +make backup LIMIT= # encrypted identity backup +make backup LIMIT= ANSIBLE_ARGS='-e decdn_backup_scope=full' # full state (stops the node briefly; stays on the host) +make decommission LIMIT= # typed confirmation; keeps the identity ``` -Each of those variables is optional: left empty, the value is read from the matching -`GC_…` key in that file instead (`roles/grafana_alloy/files/grafana-alloy.env.example` -lists them all; URLs must be https). - -Alternatively the token itself can ride git-ignored inventory — set -`grafana_alloy_api_token: "glc_…"` in `host_vars//secret.yml`, same channel as -`decdn_rpc_url`, and the role authors `/etc/grafana-alloy.env` for you (token-only, -root 0600). The authoring rewrites the file wholesale, so any hand-added `GC_*` keys -must move to their inventory variables first; provenance guards fail loud on silent -adoption or clobbering. Migrating a host that already has a hand-provisioned file -needs `grafana_alloy_overwrite_host_file: true` for exactly one converge — set it -back to `false` afterwards, or the guard stays off on that host. See "The API token -has two homes now" in the role README for the ordered procedure in both directions. - -**Upgrading a host deployed before machine monitoring existed:** journald shipping is on -by default and needs a Loki endpoint + instance ID the old four-key file does not carry, -so preflight fails until you either set `grafana_alloy_loki_url` / `_loki_username` in -inventory, add `GC_LOKI_URL` / `GC_LOKI_USERNAME` to the env file, or set -`grafana_alloy_logs_enabled: false`. The daemon's own metrics also gain an explicit -`job="decdn-node"` label (previously the implicit `prometheus.scrape.decdn_node`) — -`grafana_alloy_node_job: ""` restores the old identity. If traces start returning 401 -while metrics still flow, the stack's OTLP instance ID is not its Prometheus one: set -`grafana_alloy_otlp_username` (or `GC_OTLP_USERNAME`), which earlier versions had no way -to express. - -Setup, rotation, cost/cardinality guardrails, the two systemd-hardening relaxations -machine monitoring requires, and rollback: see -[`roles/grafana_alloy/README.md`](roles/grafana_alloy/README.md). The Helm chart path is -separate and deliberately untouched. +Backups need `decdn_backup_age_recipients` (public keys). Restore, host migration and +the on-chain exit: [`docs/lifecycle.md`](../docs/lifecycle.md). --- @@ -293,8 +263,8 @@ separate and deliberately untouched. ```bash make lint # yamllint + ansible-lint (production profile) -ansible-playbook playbooks/site.yml --syntax-check -make molecule # all seven molecule scenarios, in parallel (needs Docker) +for pb in playbooks/*.yml; do ansible-playbook "$pb" --syntax-check -i localhost,; done +make molecule # every molecule scenario, in parallel (needs Docker) make molecule JOBS=2 # …capped to two at a time on a small machine make molecule-serial # …one at a time, when a failure needs readable output @@ -308,10 +278,12 @@ for both roles, must be rejected by their own asserts), `generate-keystore` (opt host-side wallet), `host-env` (host-provisioned `/etc/decdn/decdn.env`), `slow-readiness` (advisory `/metrics` probe timeout), `grafana-cloud` (the opt-in observability wiring — see [Grafana Cloud observability](#grafana-cloud-observability-opt-in)) -and `grafana-cloud-token` (the same wiring with the API token carried through -git-ignored inventory instead: role-authored env file + provenance record). -They are independent, so they run concurrently — -~151s instead of ~595s — and each line of output is prefixed with its scenario name +`grafana-cloud-token` (the same wiring with the API token carried through +git-ignored inventory instead: role-authored env file + provenance record), +`os-matrix` (default's plays on Debian 13, Ubuntu 24.04 and Ubuntu 26.04; `default` +itself is Debian 12) and `lifecycle` (a `decdn_network` profile with an override, both +backup scopes decrypted and checked, a rejected and a real decommission). +They are independent, so they run concurrently, and each line of output is prefixed with its scenario name because the runs interleave. `make molecule-serial` is the escape hatch when that interleaving gets in the way of reading a failure. @@ -350,37 +322,35 @@ the RPC URL when it is not provisioned on the host instead). Highlights: | `baseline_extra_inbound` | `[]` | public inbound ports; `decdn_nodes` opens udp/4433. | | `baseline_preserve_ipv6_autoconf` | `true` | Keep IPv6 RA/autoconf under hardening; set `false` for static-IPv6 hosts. | | `baseline_rp_filter_loose` | `false` | `true` loosens reverse-path filtering (`rp_filter=2`) for multi-homed nodes. | -| `decdn_node_version` | `""` | **required**; a `v` release must exist. | -| `decdn_rpc_url` + 3 contract addresses | `""` | **required** per node — `rpc_url` from a host-provisioned `0600 /etc/decdn/decdn.env` (preferred) *or* `host_vars//secret.yml`, addresses in `main.yml`; sourced from an ADR/deployment. | +| `decdn_node_install_method` | `manual` | `manual` (local build) until upstream tags a release, then `release` with `decdn_node_version`. | +| `decdn_node_target` | from the host | The release target triple, derived from the host architecture (x86_64 or aarch64). | +| `decdn_network` | `""` | `arbitrum-sepolia` sets `chain_id` and every contract address from the role's manifest mirror; inventory values still win. `""` = set them yourself. | +| `decdn_rpc_url` | `""` | **required** per node, from a host-provisioned `0600 /etc/decdn/decdn.env` (preferred) *or* `host_vars//secret.yml`. | | `decdn_region` / `decdn_bind_port` / `decdn_rate_per_mb` | `""` / `4433` / `10` | node identity, QUIC port, USDC base units/MB. | | `decdn_env_checksum_file` / `decdn_env_overwrite_host_file` | `/etc/decdn/.decdn.env.sha256` / `false` | Provenance record for the secret env file (`0600 root`), and the opt-in that lets an inventory `decdn_rpc_url` overwrite a host-edited one. | | `decdn_grafana_cloud_enabled` | `false` | ONE mirrored knob (identical default in both roles) wiring on Grafana Cloud observability: installs + configures `grafana_alloy` — node metrics, machine metrics, journald, agent health — AND injects `otlp_endpoint` into `node.toml`. Only the API token is provisioned per host *or* carried by `grafana_alloy_api_token` in git-ignored inventory; the rest are inventory variables. Label/cost guardrails in the role README. | | `grafana_alloy_api_token` / `_env_checksum_file` / `_overwrite_host_file` | `""` / `/etc/grafana-alloy.env.sha256` / `false` | The dual-homed Grafana Cloud token and its provenance machinery (`#39` parity with the row above); the record path is fixed to `.sha256` and survives disable with the secret. See [`roles/grafana_alloy/README.md`](roles/grafana_alloy/README.md). | +| `decdn_backup_age_recipients` / `decdn_backup_scope` | `[]` / `identity` | Public keys backups are encrypted to (required for `make backup`), and `identity` or `full`. See [`docs/lifecycle.md`](../docs/lifecycle.md). | --- ## Packaging as a Galaxy collection (`decdn.node`) -The two roles (`baseline` + `decdn_node`) are also packaged as the distributable -**`decdn.node`** collection — deployment options for external node operators. +The three roles (`baseline`, `decdn_node`, `grafana_alloy`) are also packaged as the +distributable **`decdn.node`** collection, for operators who bring their own playbooks. The collection overlay lives in [`galaxy/`](galaxy/) (`galaxy.yml`, the collection `README.md`/`CHANGELOG.md`, `meta/runtime.yml`, `build.sh`). It is deliberately **not** -a `galaxy.yml` at the project root: `galaxy/build.sh` stages only the two roles into a +a `galaxy.yml` at the project root: `galaxy/build.sh` stages only the three roles into a clean `ansible_collections/decdn/node/` tree and builds the artifact, so this project -stays a plain Ansible project (the internal `make deploy`/`lint` flow is unchanged). +stays a plain Ansible project (the `make deploy`/`lint` flow is unchanged). ```bash make build # stage + build -> build/decdn-node-.tar.gz make galaxy-check # build + validate with galaxy-importer (the checks Galaxy runs) ``` -Publishing is a **manual** step (no auto-publish workflow, no token in CI yet): - -```bash -ansible-galaxy collection publish build/decdn-node-*.tar.gz --api-key "$GALAXY_TOKEN" -``` - -Bump `version:` in `galaxy/galaxy.yml` and add a `galaxy/CHANGELOG.md` entry per release. -CI's `galaxy-build` job builds + validates the collection on every `ansible/**` change but -never publishes. +CI's `galaxy-build` job builds and validates the collection on every `ansible/**` change. +Publishing happens from a `vX.Y.Z` tag through `.github/workflows/release.yml`, together +with the Helm chart at the same version: see [RELEASING.md](../RELEASING.md). Record every +change under `[Unreleased]` in `galaxy/CHANGELOG.md`. diff --git a/ansible/docs/launch-runbook.md b/ansible/docs/launch-runbook.md deleted file mode 100644 index 1ea5732..0000000 --- a/ansible/docs/launch-runbook.md +++ /dev/null @@ -1,110 +0,0 @@ -# Launch fleet runbook - -The ops sequence for putting the launch fleet up with this project. The plan it -serves, with targets, owners and dates, is `Launch/fleet-plan.md` in the private -`decdn/internal` repo. This file records only the mechanics. Protocol and economic -facts (bond sizing, fees) come from the deCDN ADRs; nothing here restates them. - -## 0. Inventory - -1. Copy [`inventory/fleet.example/`](../inventory/fleet.example/hosts.yml) into a - **private** repo or dir (see [README § Private fleet inventory](../README.md#private-fleet-inventory)). - Never commit real IPs to this public repo. -2. For each host set `ansible_host` and `decdn_region`, then put it in exactly one - **role** group (`decdn_seed` / `decdn_edge`) and one **billing** group - (`decdn_metered` / `decdn_unmetered`). Hero and wow-factor traffic belongs on - unmetered nodes. -3. In `group_vars/decdn_nodes.yml`, fill the MUST-EDIT lines: binary source and - contract addresses copied verbatim from `decdn/contracts/deployments/421614.json`. - Size `decdn_cache_size_mb` to each host's disk. -4. Provision each node's secret on the host, the preferred way (`0600 /etc/decdn/decdn.env`, - [roles/decdn_node/README.md § Secrets](../roles/decdn_node/README.md#secrets)). - Provision the keystore too, or set `decdn_node_generate_keystore: true`. -5. For every metered node, set a **billing alert** in the provider console (Hetzner / - OVHcloud). `decdn_load_shed_egress_budget_mbps` caps throughput, and the alert makes - the spend visible. - -## 1. Deploy - -```bash -cd ansible -make deps -INV=../../decdn-fleet/hosts.yml # your private overlay -make check INVENTORY=$INV LIMIT= ANSIBLE_ARGS='-u root' # first converge: bootstrap user -make deploy INVENTORY=$INV LIMIT= ANSIBLE_ARGS='-u root' -make deploy INVENTORY=$INV # every run after bootstrap, fleet-wide -``` - -The role runs `decdn config validate` against the installed binary after templating, -so a key the binary doesn't know fails the deploy instead of crash-looping the node. - -## 2. Stake and register each node (manual) - -On-chain onboarding ([ADR 019](https://github.com/decdn/decdn/blob/main/adr/019-node-onboarding.md) -Phase 2) is an operator step, not automated here. Fund the node's eth address first -(`key-gen` printed it). Then run `decdn setup` on the node. It reads the rendered -config plus `DECDN_RPC_URL` from `decdn.env`, and `systemd-run` loads that file with -the same parser the node's unit uses, so the RPC key never lands in argv: - -```bash -# Preview first: pre-flight checks and the bond it would post, no transactions. -sudo systemd-run --pty --wait --collect -p User=decdn \ - -p EnvironmentFile=/etc/decdn/decdn.env \ - /usr/local/bin/decdn --config /etc/decdn/node.toml setup \ - --mbps --region \ - --multiaddr /ip4//udp/4433/quic-v1 \ - --keystore-password-file /etc/decdn/keystore.password --dry-run -# Then the same command without --dry-run. It shows the operator terms and the bond to confirm. -``` - -- `--region` is the on-chain `regionHint` ([ADR 030](https://github.com/decdn/decdn/blob/main/adr/030-node-region-self-attestation.md)). - It must match the host's `decdn_region`, since the stats map and takedown scope read it. -- The bond is read from the on-chain `bondRequired(mbps)` curve - ([ADR 026](https://github.com/decdn/decdn/blob/main/adr/026-tokenomics.md)). Declare - `--mbps` for what the node can really serve; there's no flat minimum to quote. -- The flags, the `node bond` / `node register` primitives and the exit path are in - [roles/decdn_node/README.md § On-chain onboarding](../roles/decdn_node/README.md#on-chain-onboarding). - -A node counts toward launch metrics only once it is registered and serving. - -## 3. Seed the catalogue (seed nodes) - -On each `decdn_seed` host, run `Launch/seed-model.sh` (from `decdn/internal`) as the -service user, writing into the fs origin that group_vars configure: - -```bash -sudo -u decdn OUT_DIR=/var/lib/decdn/origin ./seed-model.sh -``` - -It needs the `hf` CLI and disk for **both** the origin copy and the cache, on the same -`/var/lib/decdn` volume. With the template values that's about 352 GiB of origin (the -current whole-repo set, measured 2026-09-22), plus `decdn_cache_size_mb` (400 GiB), plus -`decdn_disk_headroom_mb` (8 GiB): **about 760 GiB per seed**. `hf download` also stages -a full copy before import, so allow for the largest model (~177 GiB, Mixtral) on top -while seeding, or lower the seeds' `decdn_cache_size_mb` in `group_vars/decdn_seed.yml`. -Record every BLAKE3 hash it prints: the bundle manifests plus the large blobs. - -## 4. Pin and fill the edges - -1. Add the recorded hashes (lowercase 64-hex, no `b3:` prefix) to - `decdn_pinned_hashes` in the overlay's `group_vars/decdn_nodes.yml`, then - `make deploy INVENTORY=$INV`. -2. A pin only stops eviction. Each edge still fetches a blob on its first miss, and - it's launch-ready once the blob is resident on enough nodes per region. - -## 5. Verify: the fleet part of the go/no-go - -- From a cold machine near each region, pull the **largest** blob (currently the - 13.5 GiB Mistral v0.3 `consolidated.safetensors`) by hash. A `BlobTooLarge` error - means that node's `decdn_max_blob_size_mb` is too small. -- `curl -s 127.0.0.1:9090/metrics` on a node (over SSH, since metrics are loopback) - shows cache hits and misses. With `decdn_grafana_cloud_enabled`, watch egress and - errors in Grafana Cloud. -- Every node shows up registered, with its region, on the explorer / stats map. - -## 6. When `v0.1.0` is tagged - -Switch the overlay to the signed tarballs (`decdn_node_install_method: release`, -`decdn_node_version: "0.1.0"`) and redeploy. The role verifies them against the -release's GPG-signed `SHA256SUMS`, using the vendored maintainer key. Re-check the config -schema coupling (`molecule/schema`, `make lint-helm`) against the tagged `types.rs`. diff --git a/ansible/galaxy/CHANGELOG.md b/ansible/galaxy/CHANGELOG.md index d8344b3..ed81c55 100644 --- a/ansible/galaxy/CHANGELOG.md +++ b/ansible/galaxy/CHANGELOG.md @@ -8,6 +8,30 @@ collection adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) ### Added +- `decdn_network` (default `""`): set it to `arbitrum-sepolia` and `decdn_chain_id` + plus every contract address default to upstream's deployment manifest, mirrored + into `roles/decdn_node/vars/main/networks.yml` by `scripts/sync-network-profiles.py`. + Inventory addresses still win (the role reports them); an unknown network or a + `decdn_chain_id` that disagrees with the profile fails the play. +- arm64: `decdn_node_target` now derives from the host's architecture + (`x86_64-unknown-linux-gnu` or `aarch64-unknown-linux-gnu`), so aarch64 hosts get + the right release tarball and ELF check with no inventory change. An unsupported or + mismatching triple fails loud. +- `tasks_from: backup` (`decdn_backup_*`): tars the node's identity (hot) or its full + state minus the cache plus the keystore wherever it lives (stopping the node for the + copy and restarting it if it was up, even when the archive step fails), encrypts it + on the host to `decdn_backup_age_recipients` (age or SSH public keys, required), and + fetches the ciphertext of identity archives only (a full one would be read into + memory; the run prints a streaming copy command instead). +- `tasks_from: decommission` (`decdn_decommission_*`): typed confirmation that fails + closed after `decdn_decommission_prompt_seconds`, one host by default, stops the + node with systemctl and removes its unit, optionally purges the cache, keeps the + identity, prints the on-chain exit steps. +- Both lifecycle entry points refuse a `decdn_cache_dir` that equals or encloses the + identity and a `decdn_backup_dir` inside or around the node's directories. +- `decdn_node` is tested on Debian 13 and Ubuntu 24.04/26.04 (molecule `os-matrix`) + as well as Debian 12; `grafana_alloy`'s install path still runs on Debian 12 only. + - `grafana_alloy_api_token`: the Grafana Cloud API token may now come from a git-ignored `host_vars//secret.yml` instead of only being operator- provisioned on the host, the same dual-home pattern `decdn_rpc_url` uses. Set, the @@ -34,6 +58,8 @@ collection adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) ### Removed +- Ubuntu jammy from the roles' supported platforms: it was never tested. + - `decdn_delivery_floor` and `decdn_rate_bounds_poll_interval_sec`: upstream (decdn/decdn @ d3bc7da7) removed `payment.delivery_floor` and `blockchain.rate_bounds_poll_interval_sec`, so emitting either is a startup @@ -43,6 +69,9 @@ collection adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) ### Changed +- `grafana_alloy` fails loud on an architecture Alloy has no package for, instead of + a 404 at download time. + - `decdn_otlp_endpoint` must be `http://host:port`, matching upstream: `https://`, a missing port, a path/query/fragment and userinfo are rejected at deploy time. OTLP export is always compiled in; no `--features otlp` build is needed. @@ -78,9 +107,10 @@ Not yet published to Galaxy (pre-1.0; the published shape may still change). - `decdn.node.baseline` — Debian/Ubuntu host baseline: nftables default-deny inbound, fail2ban, unattended-upgrades, chrony, an admin sudo account, and DevSec OS + SSH hardening applied last. -- `decdn.node.decdn_node` — the `decdn-node` daemon, installed from a pinned GitHub - Release tarball under a hardened systemd unit; public QUIC udp/4433, loopback - metrics + admin RPC. +- `decdn.node.decdn_node` — the `decdn-node` daemon under a hardened systemd unit, + from locally built binaries (`manual`, the default until upstream tags a release) + or a GPG-verified GitHub Release tarball (`release`); public QUIC udp/4433, + loopback metrics + admin RPC. - Release-integrity verification: `release` mode fetches the release's `SHA256SUMS` and `SHA256SUMS.asc`, verifies the detached signature against the maintainer keyring vendored at `roles/decdn_node/files/decdn-release-KEYS.asc`, then checks diff --git a/ansible/galaxy/README.md b/ansible/galaxy/README.md index d64a1cd..06c1f86 100644 --- a/ansible/galaxy/README.md +++ b/ansible/galaxy/README.md @@ -6,19 +6,33 @@ repository — three roles and nothing else: | Role | Purpose | |------|---------| -| `decdn.node.baseline` | Debian host baseline — nftables default-deny inbound, fail2ban, unattended-upgrades, chrony, an admin sudo user, then DevSec OS + SSH hardening (applied last). | -| `decdn.node.decdn_node` | The `decdn-node` daemon — installed from a pinned GitHub Release tarball under a hardened systemd unit; public QUIC udp/4433, loopback metrics + admin RPC. | +| `decdn.node.baseline` | Debian/Ubuntu host baseline — nftables default-deny inbound, fail2ban, unattended-upgrades, chrony, an admin sudo user, then DevSec OS + SSH hardening (applied last). | +| `decdn.node.decdn_node` | The `decdn-node` daemon under a hardened systemd unit — from locally built binaries (the default until upstream tags a release) or a GPG-verified release tarball; public QUIC udp/4433, loopback metrics + admin RPC. `decdn_network` sets the chain from upstream's manifest; `tasks_from: backup` / `decommission` for day 2. | | `decdn.node.grafana_alloy` | Opt-in Grafana Cloud observability agent — loopback-only Alloy receiver and hardened telemetry export. | ## Requirements - **ansible-core ≥ 2.15** on the control machine. -- Target: **Debian (bookworm)** or **Ubuntu (jammy/noble)** over SSH with a sudo user. +- Target: **Debian 12/13** or **Ubuntu 24.04/26.04**, x86_64 or aarch64, over SSH with a + sudo user. Facts must be gathered (the node role derives the release target from + the host architecture), or set `decdn_node_target` explicitly. - Collection dependencies (installed automatically with this collection): `devsec.hardening (>=10.0.0)`, `ansible.posix (>=1.5.0)`. ## Install +> **Not on Galaxy yet.** The first release (`v0.1.0`) has not been cut; see +> [RELEASING.md](https://github.com/decdn/devops/blob/main/RELEASING.md). Until then, +> build and install it from a checkout: +> +> ```bash +> git clone https://github.com/decdn/devops && cd devops/ansible +> make build +> ansible-galaxy collection install build/decdn-node-*.tar.gz +> ``` + +Once published: + ```bash ansible-galaxy collection install decdn.node ``` @@ -50,9 +64,12 @@ hardening), then the node: baseline_extra_inbound: - { proto: udp, port: 4433, comment: "deCDN QUIC" } - role: decdn.node.decdn_node - # decdn_node_version + rpc_url + the three contract addresses + region are - # REQUIRED — set them per host (host_vars). Contract addresses/chain-id are - # protocol facts: source them from the deployment / an ADR, never guess. + vars: + decdn_network: arbitrum-sepolia # chain_id + every contract address, from upstream's manifest + # Also REQUIRED per host (host_vars): decdn_region, the binaries + # (decdn_release_target_dir, or decdn_node_install_method: release + + # decdn_node_version), and the RPC URL (host-provisioned /etc/decdn/decdn.env, + # or decdn_rpc_url in a git-ignored secret.yml). ``` The node serves paid traffic only **after** on-chain stake + registration — an diff --git a/ansible/galaxy/galaxy.yml b/ansible/galaxy/galaxy.yml index e015c6f..9d9f4b1 100644 --- a/ansible/galaxy/galaxy.yml +++ b/ansible/galaxy/galaxy.yml @@ -3,7 +3,7 @@ # slice of this DevOps repo: deploy + harden a deCDN node. # # This file is an OVERLAY, not the project root manifest. It is assembled into a -# clean collection tree (with the baseline + decdn_node roles) by galaxy/build.sh; +# clean collection tree (with the baseline, decdn_node and grafana_alloy roles) by galaxy/build.sh; # it is deliberately NOT placed at ansible/ root, so the deploy project stays a # plain Ansible project (bare role names, roles_path, production-profile # ansible-lint) rather than being reinterpreted as a collection. See galaxy/README.md. @@ -13,7 +13,9 @@ version: 0.1.0 # pre-1.0: published shape not yet stable readme: README.md authors: - deCDN Contributors -description: Deploy and harden a public deCDN node — DevSec host baseline + decdn_node. +description: >- + Deploy and harden a public deCDN node: DevSec host baseline, the decdn_node + daemon (with backup and decommission), and opt-in Grafana Cloud observability. license: - MIT # Galaxy tags must match ^[a-z0-9]+$ (no hyphens/underscores). @@ -24,6 +26,7 @@ tags: - systemd - hardening - debian + - ubuntu - devsec - web3 # Collection-level dependencies — only what the SHIPPED roles use: diff --git a/ansible/inventory/fleet.example/group_vars/decdn_edge.yml b/ansible/inventory/fleet.example/group_vars/decdn_edge.yml deleted file mode 100644 index 9932f44..0000000 --- a/ansible/inventory/fleet.example/group_vars/decdn_edge.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- -# Edge nodes have no origin of their own. With no decdn_cache_origin_kind set, the -# role defaults node-to-node pull-through ON, so an edge fills a cache miss from -# a seed or peer that holds the blob. diff --git a/ansible/inventory/fleet.example/group_vars/decdn_metered.yml b/ansible/inventory/fleet.example/group_vars/decdn_metered.yml deleted file mode 100644 index c9cf405..0000000 --- a/ansible/inventory/fleet.example/group_vars/decdn_metered.yml +++ /dev/null @@ -1,9 +0,0 @@ ---- -# Metered nodes bill per TB of egress, so a model that trends on r/LocalLLaMA -# must not run up the bill (Launch/fleet-plan.md). Cap their egress. -# This is OPERATOR COST CONFIG, not a protocol fact. Size it from the provider's -# per-TB price and the monthly spend you accept, e.g. 250 Mbps sustained is about -# 81 TB/month. -decdn_load_shed_egress_budget_mbps: 250 -# Also set a billing alert in the provider console (Hetzner / OVHcloud) for every -# metered node. The budget above caps throughput, the alert makes the spend visible. diff --git a/ansible/inventory/fleet.example/group_vars/decdn_nodes.yml b/ansible/inventory/fleet.example/group_vars/decdn_nodes.yml index 1796ef2..500c7b9 100644 --- a/ansible/inventory/fleet.example/group_vars/decdn_nodes.yml +++ b/ansible/inventory/fleet.example/group_vars/decdn_nodes.yml @@ -1,59 +1,34 @@ --- -# Settings shared by every launch node. Copied with the overlay; edit the private copy. +# Settings shared by every node in the fleet. Copied with the overlay; edit the +# private copy. Per-host differences go in host_vars//main.yml. # --- Binaries ----------------------------------------------------------------- -# Until upstream tags v0.1.0, install locally built binaries. Build them from a -# decdn/decdn checkout on the release commit: +# Until upstream tags a release, install locally built binaries. Build them from a +# decdn/decdn checkout (cross-compile for aarch64 hosts; the role derives each +# host's target from its architecture and checks the ELF before shipping): # cargo build --release -p decdn-node -p decdn-cli decdn_node_install_method: manual decdn_release_target_dir: "" # MUST-EDIT - control-machine path to decdn's target/release -# Once the v0.1.0 release exists (Launch/release-checklist.md §1), switch to the -# GPG-verified tarballs and drop the two lines above: +# Once a release exists, switch to the GPG-verified tarballs and drop the two +# lines above: # decdn_node_install_method: release -# decdn_node_version: "0.1.0" +# decdn_node_version: "X.Y.Z" -# --- Contracts (Arbitrum Sepolia, 421614) ---------------------------------------- -# Protocol facts. Copy them VERBATIM from the upstream deployment manifest -# decdn/contracts/deployments/421614.json. The public repo's -# inventory/host_vars/decdn-node-1/main.yml carries the current copy with its sync -# note. Never guess. Upstream redeploys change every address at once. -decdn_chain_id: 421614 -decdn_payment_pool_address: "" # MUST-EDIT -decdn_capacity_bond_address: "" # MUST-EDIT -decdn_slash_judge_address: "" # MUST-EDIT -decdn_content_blacklist_address: "" # MUST-EDIT -decdn_slash_appeal_address: "" -decdn_origin_assignment_address: "" -decdn_publisher_registry_address: "" -decdn_usdc_address: "" +# --- Chain -------------------------------------------------------------------- +# Chain id and every contract address come from the role's mirror of upstream's +# deployment manifest (roles/decdn_node/README.md § Network profiles). Nothing to +# copy by hand, and an upstream redeploy is a role update, not an inventory edit. +decdn_network: arbitrum-sepolia -# --- Cache sizing for the launch model catalogue --------------------------------- -# `decdn origin import` stores every file as ONE whole-file blob (--optimize only -# adds chunk hints to the manifest), and a node rejects any blob over -# max_blob_size_mb with BlobTooLarge on pull-through. The role default (1024) is far -# too small for model weights. Measured from the Hugging Face file trees on -# 2026-09-22 for the seed-model.sh set (whole repos, as that script downloads them): -# largest single file: 13.5 GiB Mistral-7B-Instruct-v0.3 consolidated.safetensors -# 12.8 GiB gpt-oss-20b original/model.safetensors -# 11.3 GiB Mixtral-8x7B consolidated.02.pt -# 9.6 GiB SDXL unet (fp32) -# whole set: ~352 GiB (Mixtral alone ~177 GiB) -# Re-measure if the seed list or its --exclude patterns change. -decdn_max_blob_size_mb: 16384 # 16 GiB: largest file 13.5 GiB, plus headroom -# Must hold the catalogue you expect a node to serve, plus room for LRU churn, and -# must be >= max_blob_size_mb. The daemon also keeps decdn_disk_headroom_mb (default -# 8 GiB) free on the volume. Size it to each host's disk: 400 GiB covers the whole -# set above. Override per host in host_vars if disks differ. -decdn_cache_size_mb: 409600 # 400 GiB - -# --- Pinning the catalogue -------------------------------------------------------- -# Eviction-exempt pins. Fill these after seeding: the lowercase 64-hex BLAKE3 hashes -# `decdn origin import` prints for each model's bundle manifest AND its large blobs. -# A pin only stops eviction. The node still has to fetch the blob once before it -# holds it. -decdn_pinned_hashes: [] +# --- Cache sizing --------------------------------------------------------------- +# Size to each host's disk and to the largest object you expect to serve; see +# docs/requirements.md for the disk formula. A node rejects any blob larger than +# decdn_max_blob_size_mb with BlobTooLarge, and decdn_cache_size_mb must be at +# least that. Override per host in host_vars when disks differ. +# decdn_max_blob_size_mb: 1024 # role default +# decdn_cache_size_mb: 10240 # role default (10 GiB) # --- Observability ---------------------------------------------------------------- -# Recommended for launch: live metrics + journald per node (egress, errors). -# Provision the token first (roles/grafana_alloy/README.md). +# Opt-in Grafana Cloud metrics + journald per node. Provision the token first +# (roles/grafana_alloy/README.md). # decdn_grafana_cloud_enabled: true diff --git a/ansible/inventory/fleet.example/group_vars/decdn_seed.yml b/ansible/inventory/fleet.example/group_vars/decdn_seed.yml deleted file mode 100644 index 7eeed0c..0000000 --- a/ansible/inventory/fleet.example/group_vars/decdn_seed.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- -# Seed nodes hold the model catalogue in a local filesystem origin, which is the -# directory Launch/seed-model.sh writes into: -# sudo -u decdn OUT_DIR=/var/lib/decdn/origin ./seed-model.sh -# /var/lib/decdn is the unit's StateDirectory, so the hardened daemon can read it -# under ProtectSystem=strict. The role creates the directory on first converge. -decdn_cache_origin_kind: fs -decdn_cache_origin_path: /var/lib/decdn/origin -# Seeds need disk for the origin copy (~352 GiB for the full set) AND the cache -# (decdn_cache_size_mb) on the same volume, about 760 GiB with the template values. -# See docs/launch-runbook.md §3. Lower decdn_cache_size_mb here for smaller seed disks. diff --git a/ansible/inventory/fleet.example/group_vars/decdn_unmetered.yml b/ansible/inventory/fleet.example/group_vars/decdn_unmetered.yml deleted file mode 100644 index fc0a3ba..0000000 --- a/ansible/inventory/fleet.example/group_vars/decdn_unmetered.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- -# Unmetered nodes: flat-rate bandwidth, capped by port speed rather than billed per -# TB. They take the hero and wow-factor traffic, so leave egress uncapped -# (decdn_load_shed_egress_budget_mbps unset, the daemon default of no budget). diff --git a/ansible/inventory/fleet.example/host_vars/fsn1-a/secret.yml.example b/ansible/inventory/fleet.example/host_vars/node-a/secret.yml.example similarity index 100% rename from ansible/inventory/fleet.example/host_vars/fsn1-a/secret.yml.example rename to ansible/inventory/fleet.example/host_vars/node-a/secret.yml.example diff --git a/ansible/inventory/fleet.example/hosts.yml b/ansible/inventory/fleet.example/hosts.yml index 87f8a4c..9054671 100644 --- a/ansible/inventory/fleet.example/hosts.yml +++ b/ansible/inventory/fleet.example/hosts.yml @@ -1,68 +1,28 @@ --- -# TEMPLATE for a private fleet overlay: the launch fleet from -# decdn/internal Launch/fleet-plan.md (10–20 nodes, Hetzner + OVHcloud, >= 4–5 -# countries on 3 continents). Copy this whole directory OUT of this public repo: -# cp -r ansible/inventory/fleet.example ../../decdn-fleet # a PRIVATE repo/dir -# make deploy INVENTORY=../../decdn-fleet/hosts.yml LIMIT= +# TEMPLATE for a private fleet overlay. Copy this whole directory OUT of this +# public repo, into a private repo or directory, and fill it in there: +# cp -r ansible/inventory/fleet.example ../../my-fleet # a PRIVATE repo/dir +# make deploy INVENTORY=../../my-fleet/hosts.yml LIMIT= # Ansible loads group_vars/ and host_vars/ from beside the hosts.yml you pass, so the # overlay carries its own. The QUIC firewall hole is NOT in here: it lives in # playbooks/group_vars/decdn_nodes.yml, which applies to every inventory. # -# Host names and addresses below are placeholders. Real IPs go in the private copy only. -# Bootstrap, ansible_user and the sudo-rs note: see ../hosts.yml.example. +# Host names and addresses below are placeholders. Real IPs go in the private copy +# only. Bootstrap, ansible_user and the Ubuntu sudo-rs note: see ../hosts.yml.example. # -# Every host belongs to decdn_nodes, the group site.yml targets, and to one group -# from each of two axes: -# role: decdn_seed - holds the catalogue in a local fs origin (seed-model.sh) -# decdn_edge - no origin; fills its cache from seeds/peers on demand -# billing: decdn_unmetered - flat-rate bandwidth: hero/wow-factor traffic goes here -# decdn_metered - pay-per-TB egress: map coverage, egress-capped -# Record the billing choice here, before provisioning (fleet-plan.md). The group is -# the record. +# Every host belongs to decdn_nodes, the group site.yml targets. Add child groups of +# your own when hosts differ in a way that is easier to express per group than per +# host (disk size, an origin, an egress cap on pay-per-TB bandwidth), and give each +# one a group_vars/.yml. Ansible merges a child group's vars over decdn_nodes'. # # decdn_region is REQUIRED per host (ISO 3166-1 alpha-2 of the physical location). -# It drives the stats map and the ADR 030 / ADR 011 regional scope. The role -# refuses to deploy without it. +# It is the on-chain region hint (ADR 030) and scopes ContentBlacklist compliance +# (ADR 011). The role refuses to deploy without it. decdn_nodes: - children: - decdn_seed: - hosts: - fsn1-a: # Hetzner Falkenstein - ansible_host: REPLACE_WITH_IP_OR_DNS - decdn_region: DE - ash-a: # Hetzner Ashburn - ansible_host: REPLACE_WITH_IP_OR_DNS - decdn_region: US - decdn_edge: - hosts: - hel1-a: # Hetzner Helsinki - ansible_host: REPLACE_WITH_IP_OR_DNS - decdn_region: FI - hil-a: # Hetzner Hillsboro - ansible_host: REPLACE_WITH_IP_OR_DNS - decdn_region: US - gra-a: # OVHcloud Gravelines - ansible_host: REPLACE_WITH_IP_OR_DNS - decdn_region: FR - waw-a: # OVHcloud Warsaw - ansible_host: REPLACE_WITH_IP_OR_DNS - decdn_region: PL - bhs-a: # OVHcloud Beauharnois - ansible_host: REPLACE_WITH_IP_OR_DNS - decdn_region: CA - sgp-a: # OVHcloud Singapore - ansible_host: REPLACE_WITH_IP_OR_DNS - decdn_region: SG - # Billing axis: the same hosts, listed again by name only. - decdn_unmetered: - hosts: - fsn1-a: - ash-a: - hel1-a: - hil-a: - decdn_metered: - hosts: - gra-a: - waw-a: - bhs-a: - sgp-a: + hosts: + node-a: + ansible_host: REPLACE_WITH_IP_OR_DNS + decdn_region: DE + node-b: + ansible_host: REPLACE_WITH_IP_OR_DNS + decdn_region: US diff --git a/ansible/inventory/host_vars/decdn-node-1/main.yml b/ansible/inventory/host_vars/decdn-node-1/main.yml index 12e4dc1..dcf02b4 100644 --- a/ansible/inventory/host_vars/decdn-node-1/main.yml +++ b/ansible/inventory/host_vars/decdn-node-1/main.yml @@ -7,19 +7,16 @@ # secret.yml (copy secret.yml.example). The role refuses to start the node until # this file is set and ONE of those two secret sources exists. # -# CONTRACT ADDRESSES ARE NOT INVENTED HERE. They are a verbatim copy of the -# upstream deployment manifest decdn/contracts/deployments/421614.json (mirrored -# byte-identically at decdn/crates/cli/deployments/421614.json — a -# deployment-manifest-mirror pre-commit hook and a CI `cmp` keep the two in step). -# That manifest is what `decdn config init --chain arbitrum-sepolia` bakes in, so -# diffing this file against a fresh `config init` is the cross-check after any -# redeploy. If you target a different deployment, replace these (and chain_id) with -# that deployment's manifest values. -# -# Synced from deployBlock 11613778 (decdn/decdn @ d306cc5c, 2026-09-01). Upstream -# has redeployed three times; addresses change wholesale each time, so re-sync -# rather than patching individual lines. -# MUST-EDIT lines are yours to fill; the rest match the manifest. +# CONTRACT ADDRESSES ARE NOT INVENTED HERE, and not copied here either: +# decdn_network below selects the role's mirror of upstream's deployment manifest +# (roles/decdn_node/vars/main/networks.yml, regenerated by +# scripts/sync-network-profiles.py from contracts/deployments/.json in +# decdn/decdn). +# It is the same manifest `decdn config init --chain arbitrum-sepolia` bakes in, and +# upstream redeploys change every address at once, so one mirror beats a hand copy +# per inventory. To target a different deployment, clear decdn_network and set +# decdn_chain_id plus the decdn_*_address variables from that deployment's manifest. +# MUST-EDIT lines are yours to fill. # --- Which binaries to install ------------------------------------------------ # Upstream has cut NO release tag yet (`git ls-remote --tags decdn/decdn` is empty @@ -38,32 +35,10 @@ decdn_cli_manual_bin_src: "" # MUST-EDIT — control-machine path to decdn # --- Chain (Arbitrum Sepolia, chain 421614) ----------------------------------- # The RPC endpoint is the one sensitive value — provision it as DECDN_RPC_URL in # /etc/decdn/decdn.env on the host, or set decdn_rpc_url in secret.yml. -decdn_chain_id: 421614 # Arbitrum Sepolia -# Required contracts. All four are mandatory upstream — a node will not resolve its -# config without them. -decdn_payment_pool_address: "0x64140155a931C01c5ff008c0667F0C3657a6d357" # PaymentPool -decdn_capacity_bond_address: "0xb6262F55bf20935d22F95A1aEc5b6101A5D98B5e" # CapacityBond -decdn_slash_judge_address: "0x63Bc91195baDfc1c65a9A2cf0556Bbfd18E94A55" # SlashJudge -# ContentBlacklist became REQUIRED (ADR 011/031): an absent or zero address is a -# fail-open compliance trap, and serving a blacklisted hash past its compliance -# window is slashable. -decdn_content_blacklist_address: "0x1971CeD7Ac2c37e87a1b8Cf25f4611B0D3F2b9B0" # ContentBlacklist - -# --- Slash appeals (optional; ADR 028) ---------------------------------------- -# Only `decdn appeal slash` uses this; the daemon validates but ignores it. -decdn_slash_appeal_address: "0x2C56029cD21E5eaa325C01f3Cd22c46C6E34040D" # SlashAppeal - -# --- Origin directory (optional; ADR 022) ------------------------------------- -# Independently optional since #1737 dropped the whole-directory mirror. -# OriginAssignment feeds the daemon's cache-miss pull-through fallback; -# PublisherRegistry is validate-only on the daemon and consumed by `decdn publish`. -decdn_origin_assignment_address: "0x25C872456E8BC7c1a8c698CEB3FF7Ea6667d9cd2" # OriginAssignment -decdn_publisher_registry_address: "0x5C99fAF051c1684f1b481982c7D730883b60e339" # PublisherRegistry - -# --- CLI-only (optional) ------------------------------------------------------ -# externalDeps.usdc from the same manifest. The daemon validates but never reads -# it; `decdn setup` uses it to bond with USDC instead of the native token. -decdn_usdc_address: "0x75faf114eafb1BDbe2F0316DF893fd58CE46AA4d" +# The profile supplies decdn_chain_id and all eight contract addresses (the four +# required ones, SlashAppeal, OriginAssignment, PublisherRegistry and USDC). Set any +# decdn_*_address here only to override it deliberately; the role reports overrides. +decdn_network: arbitrum-sepolia # --- Cache pull-through origin (what the node fetches on a cache miss) --------- # MUST-EDIT — operator-specific backing origin (NOT a chain fact). A serving node diff --git a/ansible/molecule/default/converge.yml b/ansible/molecule/default/converge.yml index b7eb69b..568f194 100644 --- a/ansible/molecule/default/converge.yml +++ b/ansible/molecule/default/converge.yml @@ -1,13 +1,14 @@ --- # Exercise the decdn_node role only (baseline is real-host-only — see molecule.yml). -# `manual` install copies the stub daemon from the scenario's files/ dir, so no +# `manual` install copies the stub daemon from molecule/default/files/, so no # published release is needed. The required chain/contract knobs use placeholder # (but well-formed) values — the role's fail-loud asserts still run against them. - name: Converge hosts: all become: true vars: - stub_bin: "{{ lookup('ansible.builtin.env', 'MOLECULE_SCENARIO_DIRECTORY') }}/files/decdn-node-stub" + # Anchored on the project, not the scenario dir: os-matrix reuses this play. + stub_bin: "{{ lookup('ansible.builtin.env', 'MOLECULE_PROJECT_DIRECTORY') }}/molecule/default/files/decdn-node-stub" decdn_node_install_method: manual decdn_node_manual_bin_src: "{{ stub_bin }}" decdn_cli_manual_bin_src: "{{ stub_bin }}" diff --git a/ansible/molecule/default/molecule.yml b/ansible/molecule/default/molecule.yml index 6261c2e..1fb90c9 100644 --- a/ansible/molecule/default/molecule.yml +++ b/ansible/molecule/default/molecule.yml @@ -22,9 +22,9 @@ driver: platforms: - name: decdn-node-molecule # Pinned by digest for reproducible CI (repo convention — cf. the SHA-pinned - # actions/images in .github/workflows/ci.yml). Tag: :latest as of 2026-07-11 + # actions/images in .github/workflows/ci.yml). Tag: :latest as of 2026-09-24 # (this image publishes only :latest). Re-resolve the digest to bump. - image: geerlingguy/docker-debian12-ansible@sha256:4553092be2c00b1ffe580927b9ff03f3c3a0df32b7dd693a3eb02efb6c2b77b7 + image: geerlingguy/docker-debian12-ansible@sha256:a131e3b7a9a38dda40e98b49b39d6200ffab0ad853c714a513f5eea41bab9a25 pre_build_image: true command: /usr/lib/systemd/systemd privileged: true diff --git a/ansible/molecule/generate-keystore/molecule.yml b/ansible/molecule/generate-keystore/molecule.yml index aafb9d9..b48aa03 100644 --- a/ansible/molecule/generate-keystore/molecule.yml +++ b/ansible/molecule/generate-keystore/molecule.yml @@ -20,7 +20,7 @@ platforms: - name: decdn-node-genks-molecule # Same image as the default scenario, pinned by digest for reproducible CI. # Re-resolve the digest to bump (keep in sync with ../default/molecule.yml). - image: geerlingguy/docker-debian12-ansible@sha256:4553092be2c00b1ffe580927b9ff03f3c3a0df32b7dd693a3eb02efb6c2b77b7 + image: geerlingguy/docker-debian12-ansible@sha256:a131e3b7a9a38dda40e98b49b39d6200ffab0ad853c714a513f5eea41bab9a25 pre_build_image: true command: /usr/lib/systemd/systemd privileged: true diff --git a/ansible/molecule/grafana-cloud-token/molecule.yml b/ansible/molecule/grafana-cloud-token/molecule.yml index 2cb78f1..ac75a1f 100644 --- a/ansible/molecule/grafana-cloud-token/molecule.yml +++ b/ansible/molecule/grafana-cloud-token/molecule.yml @@ -15,7 +15,7 @@ platforms: - name: decdn-node-grafana-cloud-token # Same digest-pinned image as the other scenarios (repo convention). Re-resolve # all of them together to bump. - image: geerlingguy/docker-debian12-ansible@sha256:4553092be2c00b1ffe580927b9ff03f3c3a0df32b7dd693a3eb02efb6c2b77b7 + image: geerlingguy/docker-debian12-ansible@sha256:a131e3b7a9a38dda40e98b49b39d6200ffab0ad853c714a513f5eea41bab9a25 pre_build_image: true command: /usr/lib/systemd/systemd privileged: true diff --git a/ansible/molecule/grafana-cloud/molecule.yml b/ansible/molecule/grafana-cloud/molecule.yml index 50da26b..b437baa 100644 --- a/ansible/molecule/grafana-cloud/molecule.yml +++ b/ansible/molecule/grafana-cloud/molecule.yml @@ -24,7 +24,7 @@ platforms: - name: decdn-node-grafana-cloud # Same digest-pinned image as the other scenarios (repo convention). Re-resolve # all of them together to bump. - image: geerlingguy/docker-debian12-ansible@sha256:4553092be2c00b1ffe580927b9ff03f3c3a0df32b7dd693a3eb02efb6c2b77b7 + image: geerlingguy/docker-debian12-ansible@sha256:a131e3b7a9a38dda40e98b49b39d6200ffab0ad853c714a513f5eea41bab9a25 pre_build_image: true command: /usr/lib/systemd/systemd privileged: true diff --git a/ansible/molecule/host-env/molecule.yml b/ansible/molecule/host-env/molecule.yml index 26c19d9..1a65736 100644 --- a/ansible/molecule/host-env/molecule.yml +++ b/ansible/molecule/host-env/molecule.yml @@ -22,7 +22,7 @@ platforms: - name: decdn-node-hostenv-molecule # Same image as the default scenario, pinned by digest for reproducible CI. # Re-resolve the digest to bump (keep in sync with ../default/molecule.yml). - image: geerlingguy/docker-debian12-ansible@sha256:4553092be2c00b1ffe580927b9ff03f3c3a0df32b7dd693a3eb02efb6c2b77b7 + image: geerlingguy/docker-debian12-ansible@sha256:a131e3b7a9a38dda40e98b49b39d6200ffab0ad853c714a513f5eea41bab9a25 pre_build_image: true command: /usr/lib/systemd/systemd privileged: true diff --git a/ansible/molecule/lifecycle/converge.yml b/ansible/molecule/lifecycle/converge.yml new file mode 100644 index 0000000..d86c871 --- /dev/null +++ b/ansible/molecule/lifecycle/converge.yml @@ -0,0 +1,193 @@ +--- +- name: Converge decdn_node from a network profile + hosts: all + become: true + vars: + stub_bin: "{{ lookup('ansible.builtin.env', 'MOLECULE_PROJECT_DIRECTORY') }}/molecule/default/files/decdn-node-stub" + decdn_node_install_method: manual + decdn_node_manual_bin_src: "{{ stub_bin }}" + decdn_cli_manual_bin_src: "{{ stub_bin }}" + decdn_rpc_url: "https://rpc.example.invalid/" + decdn_region: "US" + # The profile supplies chain_id and all eight addresses ... + decdn_network: arbitrum-sepolia + # ... and an explicit inventory value must still win over it. + decdn_slash_appeal_address: "0x5555555555555555555555555555555555555555" + roles: + - role: decdn_node + post_tasks: + - name: Read the backup recipient's public key + ansible.builtin.command: + cmd: age-keygen -y /root/molecule-age.key + register: _molecule_age_pub + changed_when: false + + - name: Encrypt the backups to it + ansible.builtin.set_fact: + decdn_backup_age_recipients: ["{{ _molecule_age_pub.stdout | trim }}"] + +- name: Identity backup + ansible.builtin.import_playbook: ../../playbooks/backup.yml + +- name: Switch to a full backup + hosts: all + gather_facts: false + tasks: + - name: Record the node's invocation before the full backup + ansible.builtin.systemd_service: + name: decdn-node + register: _molecule_inv_before + + - name: Select the full scope + ansible.builtin.set_fact: + decdn_backup_scope: full + +- name: Full backup + ansible.builtin.import_playbook: ../../playbooks/backup.yml + +- name: Full-backup stop/restart, failure recovery and leave_stopped + hosts: all + become: true + tasks: + - name: Read the node's state after the full backup + ansible.builtin.systemd_service: + name: decdn-node + register: _molecule_inv_after + + - name: Require the node to have been stopped for the copy and started again + ansible.builtin.assert: + that: + - _molecule_inv_after.status.ActiveState == 'active' + - _molecule_inv_after.status.InvocationID != _molecule_inv_before.status.InvocationID + fail_msg: The full backup did not stop and restart decdn-node. + + # A recipient that passes the role's shape check but that age rejects: the + # archive step fails after the node was stopped, and the always: block must + # restart it and remove the .partial. + - name: Full backup that fails at the archive step + block: + - name: Run a full backup with a recipient age rejects + ansible.builtin.include_role: + name: decdn_node + tasks_from: backup + vars: + decdn_backup_age_recipients: ["age1{{ 'q' * 58 }}"] + - name: Fail if the broken backup succeeded + ansible.builtin.fail: + msg: A backup with an unusable recipient reported success. + rescue: + - name: Require the archive step to be what failed + ansible.builtin.assert: + that: + - ansible_failed_task.name is match('^Write the encrypted archive') + fail_msg: "Failed at '{{ ansible_failed_task.name }}', not the archive step." + + - name: Read the node's state after the failed backup + ansible.builtin.systemd_service: + name: decdn-node + register: _molecule_inv_failed + + - name: Require the node to be back up after the failed backup + ansible.builtin.assert: + that: + - _molecule_inv_failed.status.ActiveState == 'active' + - _molecule_inv_failed.status.InvocationID != _molecule_inv_after.status.InvocationID + fail_msg: A failed full backup left decdn-node stopped (or never stopped it). + + - name: Full backup that leaves the node stopped (host migration) + ansible.builtin.include_role: + name: decdn_node + tasks_from: backup + vars: + decdn_backup_leave_stopped: true + + - name: Read the node's state after the leave_stopped backup + ansible.builtin.systemd_service: + name: decdn-node + register: _molecule_stopped + + - name: Require the node to be stopped + ansible.builtin.assert: + that: + - _molecule_stopped.status.ActiveState != 'active' + fail_msg: decdn_backup_leave_stopped did not leave decdn-node stopped. + +- name: Decommission refuses more hosts than decdn_decommission_max_hosts + # localhost joins the play only to make it two hosts; the cap fails first. + hosts: decdn_nodes:localhost + gather_facts: false + become: false + tasks: + - name: Decommission two hosts at once + block: + - name: Run decommission against two hosts + ansible.builtin.include_role: + name: decdn_node + tasks_from: decommission + vars: + decdn_decommission_confirm: "{{ ansible_play_hosts_all | sort | join(',') }}" + - name: Fail if decommission accepted two hosts + ansible.builtin.fail: + msg: decommission ran against more hosts than decdn_decommission_max_hosts. + rescue: + - name: Require the host cap to be what failed + ansible.builtin.assert: + that: + - ansible_failed_task.name is match('^Refuse to decommission more hosts') + fail_msg: "Failed at '{{ ansible_failed_task.name }}', not the host cap." + +- name: Decommission refuses a missing or wrong confirmation + hosts: all + become: true + tasks: + # Molecule has no TTY, and the 1 s timeout covers an interactive run: either way + # the prompt yields empty input, which must not count as confirmation. + - name: Decommission with no confirmation given + block: + - name: Run decommission without decdn_decommission_confirm + ansible.builtin.include_role: + name: decdn_node + tasks_from: decommission + vars: + decdn_decommission_prompt_seconds: 1 + - name: Fail if decommission proceeded unconfirmed + ansible.builtin.fail: + msg: decommission proceeded without a typed confirmation. + rescue: + - name: Require the confirmation assert to be what failed (unconfirmed) + ansible.builtin.assert: + that: + - ansible_failed_task.name is match('^Require the typed confirmation') + fail_msg: "Failed at '{{ ansible_failed_task.name }}', not the confirmation check." + + - name: Decommission with the wrong confirmation + block: + - name: Run decommission with a mistyped host list + ansible.builtin.include_role: + name: decdn_node + tasks_from: decommission + vars: + decdn_decommission_confirm: not-this-host + - name: Fail if decommission accepted a wrong confirmation + ansible.builtin.fail: + msg: decommission proceeded with a confirmation that did not match. + rescue: + - name: Require the confirmation assert to be what failed (mistyped) + ansible.builtin.assert: + that: + - ansible_failed_task.name is match('^Require the typed confirmation') + fail_msg: "Failed at '{{ ansible_failed_task.name }}', not the confirmation check." + + - name: Require the unit to survive the rejected runs + ansible.builtin.stat: + path: /etc/systemd/system/decdn-node.service + register: _molecule_unit + failed_when: not _molecule_unit.stat.exists + + - name: Confirm the real decommission non-interactively, with a cache purge + ansible.builtin.set_fact: + decdn_decommission_confirm: "{{ ansible_play_hosts_all | sort | join(',') }}" + decdn_decommission_purge_cache: true + +- name: Decommission + ansible.builtin.import_playbook: ../../playbooks/decommission.yml diff --git a/ansible/molecule/lifecycle/molecule.yml b/ansible/molecule/lifecycle/molecule.yml new file mode 100644 index 0000000..166f369 --- /dev/null +++ b/ansible/molecule/lifecycle/molecule.yml @@ -0,0 +1,58 @@ +--- +# Day-2 lifecycle, end to end against the stub daemon: +# 1. converge decdn_node from a NETWORK PROFILE (decdn_network) plus one explicit +# address override, and prove the override wins and chain_id renders as an int; +# 2. playbooks/backup.yml: an identity backup (hot, fetched) and a full one (the +# node is stopped for the copy and restarted: its InvocationID changes; not +# fetched by default), encrypted on the host to a throwaway age key; +# 3. a full backup that fails at the archive step must restart the node and leave +# no .partial; one with decdn_backup_leave_stopped must leave it stopped; +# 4. decommission must refuse two hosts, a missing confirmation and a wrong one, +# touching nothing; then playbooks/decommission.yml runs for real with a cache +# purge. +# verify.yml decrypts every archive inside the container and checks each scope's +# members (daemon state only in full, the cache in neither), the fetched copy's +# checksum, and that decommission removed the unit and the cache but kept the +# identity. +# +# No idempotence step: every backup run writes a new archive by design, and +# decommission is one-shot. +driver: + name: docker +platforms: + - name: decdn-node-lifecycle + # Same digest-pinned image as default (repo convention). Re-resolve together. + image: geerlingguy/docker-debian12-ansible@sha256:a131e3b7a9a38dda40e98b49b39d6200ffab0ad853c714a513f5eea41bab9a25 + pre_build_image: true + command: /usr/lib/systemd/systemd + privileged: true + cgroupns_mode: host + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw + # The lifecycle playbooks target decdn_nodes, like site.yml. + groups: + - decdn_nodes +provisioner: + name: ansible + env: + ANSIBLE_ROLES_PATH: "${MOLECULE_PROJECT_DIRECTORY}/roles" + ANSIBLE_COLLECTIONS_PATH: "${MOLECULE_PROJECT_DIRECTORY}/collections" + # See default/molecule.yml for why pipelining is on. + ANSIBLE_PIPELINING: "true" + inventory: + host_vars: + decdn-node-lifecycle: + # Fetched archives land in the scenario's ephemeral dir, never the repo, so + # parallel scenarios and repeat runs cannot collide. + decdn_backup_local_dir: "${MOLECULE_EPHEMERAL_DIRECTORY}/backups" +verifier: + name: ansible +scenario: + test_sequence: + # Leading destroy: see default/molecule.yml. + - destroy + - create + - prepare + - converge + - verify + - destroy diff --git a/ansible/molecule/lifecycle/prepare.yml b/ansible/molecule/lifecycle/prepare.yml new file mode 100644 index 0000000..9cf4e38 --- /dev/null +++ b/ansible/molecule/lifecycle/prepare.yml @@ -0,0 +1,37 @@ +--- +# Placeholder identity (as in default), plus age and a throwaway age key pair the +# backups are encrypted to. The private key never leaves the container. Two marker +# files let verify.yml tell the scopes apart: daemon state that only a full backup +# may carry, and a cached blob that no backup may carry. +- name: Stage the placeholder identity + ansible.builtin.import_playbook: ../default/prepare.yml + +- name: Prepare an age key pair and state markers for the backup test + hosts: all + become: true + tasks: + - name: Install age + ansible.builtin.apt: + name: age + state: present + update_cache: true + + - name: Generate a throwaway age identity + ansible.builtin.command: + cmd: age-keygen -o /root/molecule-age.key + creates: /root/molecule-age.key + + - name: Create the cache directory + ansible.builtin.file: + path: /var/lib/decdn/cache + state: directory + mode: "0700" + + - name: Stage daemon state and a cached blob + ansible.builtin.copy: + dest: "{{ item }}" + content: "molecule marker\n" + mode: "0600" + loop: + - /var/lib/decdn/molecule-state.redb + - /var/lib/decdn/cache/molecule-blob diff --git a/ansible/molecule/lifecycle/verify.yml b/ansible/molecule/lifecycle/verify.yml new file mode 100644 index 0000000..78e9b54 --- /dev/null +++ b/ansible/molecule/lifecycle/verify.yml @@ -0,0 +1,154 @@ +--- +- name: Verify the lifecycle + hosts: all + become: true + vars: + _profile: >- + {{ (lookup('ansible.builtin.file', + lookup('ansible.builtin.env', 'MOLECULE_PROJECT_DIRECTORY') ~ '/roles/decdn_node/vars/main/networks.yml') + | from_yaml)._decdn_network_profiles['arbitrum-sepolia'] }} + _local_dir: "{{ lookup('ansible.builtin.env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/backups/{{ inventory_hostname }}" + tasks: + # --- Network profile ---------------------------------------------------------- + - name: Read the rendered node.toml + ansible.builtin.slurp: + src: /etc/decdn/node.toml + register: _toml + + - name: Require profile values, the inventory override and an integer chain_id + vars: + _t: "{{ _toml.content | b64decode }}" + ansible.builtin.assert: + that: + - _t is search('(?m)^chain_id = ' ~ _profile.chain_id ~ '$') + - _t is search('(?m)^payment_pool_address = "' ~ _profile.payment_pool_address ~ '"$') + - _t is search('(?m)^content_blacklist_address = "' ~ _profile.content_blacklist_address ~ '"$') + - _t is search('(?m)^usdc_address = "' ~ _profile.usdc_address ~ '"$') + - _t is search('(?m)^slash_appeal_address = "0x5555555555555555555555555555555555555555"$') + fail_msg: node.toml does not carry the profile values with the inventory override on top. + + # --- Backups -------------------------------------------------------------------- + - name: Find the archives on the host + ansible.builtin.find: + paths: /var/backups/decdn + patterns: ["*.tar.age", "*.partial"] + register: _archives + + # One identity backup, two full ones (the plain one and leave_stopped); the + # failed run must have left nothing, not even a .partial. + - name: Require one identity and two full archives, root-only, no partials + ansible.builtin.assert: + that: + - _archives.files | map(attribute='path') | select('search', '\.partial$') | list | length == 0 + - _archives.files | map(attribute='path') | select('search', '-identity-') | list | length == 1 + - _archives.files | map(attribute='path') | select('search', '-full-') | list | length == 2 + - _archives.files | map(attribute='mode') | unique == ['0600'] + fail_msg: "Unexpected archives: {{ _archives.files | map(attribute='path') | list }}" + + - name: List each archive's members (decrypting with the throwaway key) + ansible.builtin.shell: + cmd: set -o pipefail; age -d -i /root/molecule-age.key {{ item.path | quote }} | tar -tzf - + executable: /bin/bash + loop: "{{ _archives.files }}" + loop_control: + label: "{{ item.path | basename }}" + register: _members + changed_when: false + + - name: Require each scope's members + vars: + _list: "{{ item.stdout_lines }}" + _full: "{{ '-full-' in item.item.path }}" + ansible.builtin.assert: + that: + - "'var/lib/decdn/node.secret' in _list" + - "'var/lib/decdn/keystore.json' in _list" + - "'etc/decdn/keystore.password' in _list" + - "'etc/decdn/decdn.env' not in _list" + # The cache is in no backup ... + - _list | select('match', '^var/lib/decdn/cache/') | list | length == 0 + # ... daemon state only in a full one, which is more than the identity. + - ('var/lib/decdn/molecule-state.redb' in _list) if _full | bool else (_list | length == 3) + - (_list | select('match', '^var/lib/decdn/') | list | length > 2) if _full | bool else true + fail_msg: "{{ item.item.path | basename }} holds {{ _list }}" + loop: "{{ _members.results }}" + loop_control: + label: "{{ item.item.path | basename }}" + + - name: Checksum the archives on the host + ansible.builtin.stat: + path: "{{ item.path }}" + checksum_algorithm: sha256 + loop: "{{ _archives.files }}" + loop_control: + label: "{{ item.path | basename }}" + register: _remote + + - name: Checksum the copies on the control machine + ansible.builtin.stat: + path: "{{ _local_dir }}/{{ item.path | basename }}" + checksum_algorithm: sha256 + loop: "{{ _archives.files }}" + loop_control: + label: "{{ item.path | basename }}" + register: _local + delegate_to: localhost + become: false + + # Identity archives are fetched by default; full ones are not (fetch would read + # them into memory on both ends). + - name: Require the identity archive fetched intact and full archives left on the host + vars: + _pairs: "{{ _remote.results | zip(_local.results) | list }}" + ansible.builtin.assert: + that: + - >- + (item.1.stat.exists and item.1.stat.checksum == item.0.stat.checksum) + if '-identity-' in item.0.item.path else not item.1.stat.exists + fail_msg: "{{ item.0.item.path | basename }}: fetched copy wrong or unexpected." + loop: "{{ _pairs }}" + loop_control: + label: "{{ item.0.item.path | basename }}" + + # --- Decommission ----------------------------------------------------------------- + - name: Stat the unit and the identity + ansible.builtin.stat: + path: "{{ item }}" + loop: + - /etc/systemd/system/decdn-node.service + - /var/lib/decdn/node.secret + - /var/lib/decdn/keystore.json + - /etc/decdn/keystore.password + register: _after + + - name: Require the unit gone and the identity kept + ansible.builtin.assert: + that: + - not _after.results[0].stat.exists + - _after.results[1:] | map(attribute='stat.exists') | unique == [true] + fail_msg: Decommission must remove the unit and keep node.secret, keystore.json and the password file. + + - name: Stat the cache and the daemon state + ansible.builtin.stat: + path: "{{ item }}" + loop: + - /var/lib/decdn/cache + - /var/lib/decdn/molecule-state.redb + register: _purge + + - name: Require the cache purged and the rest of the data dir kept + ansible.builtin.assert: + that: + - not _purge.results[0].stat.exists + - _purge.results[1].stat.exists + fail_msg: decdn_decommission_purge_cache must delete the cache dir and nothing else. + + - name: Read service state + ansible.builtin.service_facts: + + - name: Require decdn-node to be gone + ansible.builtin.assert: + that: + - ansible_facts.services['decdn-node.service'] is not defined + or ansible_facts.services['decdn-node.service'].state != 'running' + fail_msg: decdn-node is still running after decommission. diff --git a/ansible/molecule/os-matrix/molecule.yml b/ansible/molecule/os-matrix/molecule.yml new file mode 100644 index 0000000..7a08fc9 --- /dev/null +++ b/ansible/molecule/os-matrix/molecule.yml @@ -0,0 +1,72 @@ +--- +# The `default` scenario's converge + idempotence + verify, run on every other +# distribution the roles' meta/main.yml claims: Debian 13 (trixie), Ubuntu 24.04 +# (noble) and Ubuntu 26.04 (resolute). `default` itself stays on Debian 12 +# (bookworm), so the four platforms together are exactly the supported set. +# Nothing here is scenario-specific: the playbooks are default's, by path, so a +# change to what `default` proves is proved on every distribution at once. +# +# A separate scenario rather than extra platforms in `default`: a failure names the +# distribution in the scenario's own output, and `default` stays a fast single-host +# loop for development. `make molecule` finds it by glob. +# +# Same boundary as `default`: decdn_node end to end, plus grafana_alloy's disabled +# path and baseline's admin-account tasks, which default's converge includes. +# grafana_alloy's install path (grafana-cloud*) and baseline's hardening are not +# covered here: the former runs on Debian 12 only, the latter on real hosts. +# +# Images: digest-pinned multi-arch indexes (amd64 + arm64), :latest as of +# 2026-09-24 (these images publish only :latest). Re-resolve with +# `docker buildx imagetools inspect geerlingguy/docker--ansible:latest` +# and bump them together with the debian12 pin in the other scenarios. +driver: + name: docker +platforms: + - name: decdn-node-deb13 + image: geerlingguy/docker-debian13-ansible@sha256:ba6fd5df5088d353e27c3e5f63b2631d2655d130bebf8c144e319213d3424098 + pre_build_image: true + command: /usr/lib/systemd/systemd + privileged: true + cgroupns_mode: host + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw + - name: decdn-node-u2404 + image: geerlingguy/docker-ubuntu2404-ansible@sha256:3394e335df58c961c00b78ae601ef30cdd7463b112f21b0c5f78674a70526302 + pre_build_image: true + command: /usr/lib/systemd/systemd + privileged: true + cgroupns_mode: host + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw + - name: decdn-node-u2604 + image: geerlingguy/docker-ubuntu2604-ansible@sha256:a81f2af83a18b609eabdc926d56bc4813d3d7558d60dc891db6056f868fe6054 + pre_build_image: true + command: /usr/lib/systemd/systemd + privileged: true + cgroupns_mode: host + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw +provisioner: + name: ansible + env: + ANSIBLE_ROLES_PATH: "${MOLECULE_PROJECT_DIRECTORY}/roles" + ANSIBLE_COLLECTIONS_PATH: "${MOLECULE_PROJECT_DIRECTORY}/collections" + # See default/molecule.yml for why pipelining is on. + ANSIBLE_PIPELINING: "true" + playbooks: + prepare: ../default/prepare.yml + converge: ../default/converge.yml + verify: ../default/verify.yml +verifier: + name: ansible +scenario: + test_sequence: + # Leading destroy: see default/molecule.yml (a reused orphan would make + # converge + idempotence pass trivially). + - destroy + - create + - prepare + - converge + - idempotence + - verify + - destroy diff --git a/ansible/molecule/schema/molecule.yml b/ansible/molecule/schema/molecule.yml index 8434e5d..4bbd38a 100644 --- a/ansible/molecule/schema/molecule.yml +++ b/ansible/molecule/schema/molecule.yml @@ -23,9 +23,9 @@ driver: platforms: - name: decdn-node-molecule-schema # Pinned by digest for reproducible CI (repo convention — cf. the SHA-pinned - # actions/images in .github/workflows/ci.yml). Tag: :latest as of 2026-07-11 + # actions/images in .github/workflows/ci.yml). Tag: :latest as of 2026-09-24 # (this image publishes only :latest). Re-resolve the digest to bump. - image: geerlingguy/docker-debian12-ansible@sha256:4553092be2c00b1ffe580927b9ff03f3c3a0df32b7dd693a3eb02efb6c2b77b7 + image: geerlingguy/docker-debian12-ansible@sha256:a131e3b7a9a38dda40e98b49b39d6200ffab0ad853c714a513f5eea41bab9a25 pre_build_image: true command: /usr/lib/systemd/systemd privileged: true diff --git a/ansible/molecule/slow-readiness/molecule.yml b/ansible/molecule/slow-readiness/molecule.yml index 1a2cc52..5f8200e 100644 --- a/ansible/molecule/slow-readiness/molecule.yml +++ b/ansible/molecule/slow-readiness/molecule.yml @@ -21,7 +21,7 @@ platforms: - name: decdn-node-slow-readiness # Same digest-pinned image as the default scenario (repo convention). Re-resolve # both together to bump. - image: geerlingguy/docker-debian12-ansible@sha256:4553092be2c00b1ffe580927b9ff03f3c3a0df32b7dd693a3eb02efb6c2b77b7 + image: geerlingguy/docker-debian12-ansible@sha256:a131e3b7a9a38dda40e98b49b39d6200ffab0ad853c714a513f5eea41bab9a25 pre_build_image: true command: /usr/lib/systemd/systemd privileged: true diff --git a/ansible/molecule/validation/converge.yml b/ansible/molecule/validation/converge.yml index f1ad9a5..3f19283 100644 --- a/ansible/molecule/validation/converge.yml +++ b/ansible/molecule/validation/converge.yml @@ -150,10 +150,6 @@ decdn_node_manual_bin_src: "" decdn_cli_manual_bin_src: "" decdn_release_target_dir: "{{ _decdn_check_dir }}" - decdn_node_target: >- - {{ 'aarch64-unknown-linux-gnu' - if ansible_facts.architecture in ['aarch64', 'arm64'] - else 'x86_64-unknown-linux-gnu' }} decdn_rpc_url: "" - name: Dry-run decdn_node in manual dir-mode @@ -179,9 +175,186 @@ decdn_node_manual_bin_src: "{{ stub_bin }}" decdn_cli_manual_bin_src: "{{ stub_bin }}" decdn_release_target_dir: "" - decdn_node_target: x86_64-unknown-linux-gnu decdn_rpc_url: "https://rpc.example.invalid/" + # --- Case: arch-mismatch (explicit triple for the wrong architecture) -------- + # decdn_node_target is derived from the host's facts; an explicit value naming + # the OTHER architecture would ship binaries the node cannot exec. Release mode + # so the triple is consulted; the assert fires before any download. + - name: "Case arch-mismatch — explicit target triple for the other architecture" + block: + - name: Run decdn_node with a target triple that does not match the host + ansible.builtin.include_role: + name: decdn_node + vars: + decdn_node_install_method: release + decdn_node_version: "0.0.0" + decdn_node_target: >- + {{ 'x86_64-unknown-linux-gnu' + if ansible_facts.architecture == 'aarch64' + else 'aarch64-unknown-linux-gnu' }} + rescue: + - name: Record arch-mismatch rejection (only if the arch assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['arch-mismatch'] }}" + when: ansible_failed_task.name is match('^Require the target triple to match') + + # --- Case: network-unknown / network-chain (decdn_network profile guards) ----- + - name: "Case network-unknown — decdn_network names no known deployment" + block: + - name: Run decdn_node with an unknown network profile + ansible.builtin.include_role: + name: decdn_node + vars: + decdn_network: arbitrum-mainnet-typo + rescue: + - name: Record network-unknown rejection (only if the network assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['network-unknown'] }}" + when: ansible_failed_task.name is match('^Require a known decdn_network') + + # An inventory chain_id for another chain would pair that chain with this + # profile's addresses; the profile guard refuses it. + - name: "Case network-chain — chain_id disagrees with the network profile" + block: + - name: Run decdn_node with a profile and a foreign chain_id + ansible.builtin.include_role: + name: decdn_node + vars: + decdn_network: arbitrum-sepolia + decdn_chain_id: 1 + rescue: + - name: Record network-chain rejection (only if the chain_id assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['network-chain'] }}" + when: ansible_failed_task.name is match('^Require chain_id to match the network profile') + + # --- Lifecycle entry points (tasks_from: backup / decommission) -------------- + # Each fails at its settings or path asserts, before anything on the host is + # touched. A pasted private key must never reach the host's recipients file. + - name: "Case backup-secret-key — an age SECRET key pasted as a recipient" + block: + - name: Run decdn_node for case backup-secret-key + ansible.builtin.include_role: + name: decdn_node + tasks_from: backup + vars: + decdn_backup_age_recipients: ["AGE-SECRET-KEY-1QQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQ"] + rescue: + - name: Record backup-secret-key rejection + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['backup-secret-key'] }}" + when: ansible_failed_task.name is match('^Validate the backup settings') + + - name: "Case backup-bare-string — recipients given as a bare string, not a list" + block: + - name: Run decdn_node for case backup-bare-string + ansible.builtin.include_role: + name: decdn_node + tasks_from: backup + vars: + decdn_backup_age_recipients: "age1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq" + rescue: + - name: Record backup-bare-string rejection + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['backup-bare-string'] }}" + when: ansible_failed_task.name is match('^Validate the backup settings') + + - name: "Case backup-empty — no recipients" + block: + - name: Run decdn_node for case backup-empty + ansible.builtin.include_role: + name: decdn_node + tasks_from: backup + vars: + decdn_backup_age_recipients: [] + rescue: + - name: Record backup-empty rejection + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['backup-empty'] }}" + when: ansible_failed_task.name is match('^Validate the backup settings') + + - name: "Case path-cache-home — cache dir is the data dir (backup)" + block: + - name: Run decdn_node for case path-cache-home + ansible.builtin.include_role: + name: decdn_node + tasks_from: backup + vars: + decdn_backup_age_recipients: ["age1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq"] + decdn_cache_dir: /var/lib/decdn + rescue: + - name: Record path-cache-home rejection + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['path-cache-home'] }}" + when: ansible_failed_task.name is match('^Require the cache dir to hold no identity') + + - name: "Case path-backup-home — backup dir is the data dir" + block: + - name: Run decdn_node for case path-backup-home + ansible.builtin.include_role: + name: decdn_node + tasks_from: backup + vars: + decdn_backup_age_recipients: ["age1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq"] + decdn_backup_dir: /var/lib/decdn + rescue: + - name: Record path-backup-home rejection + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['path-backup-home'] }}" + when: ansible_failed_task.name is match('^Require the backup dir to stay clear') + + - name: "Case path-decommission-cache — cache dir encloses the keystore (decommission purge)" + block: + - name: Run decdn_node for case path-decommission-cache + ansible.builtin.include_role: + name: decdn_node + tasks_from: decommission + vars: + decdn_decommission_confirm: "{{ ansible_play_hosts_all | sort | join(',') }}" + decdn_decommission_purge_cache: true + decdn_cache_dir: /var/lib + rescue: + - name: Record path-decommission-cache rejection + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['path-decommission-cache'] }}" + when: ansible_failed_task.name is match('^Require the cache dir to hold no identity') + + # --- Case: arch-unset (no facts / explicit empty triple) -------------------- + - name: "Case arch-unset — release mode with no resolvable target triple" + block: + - name: Run decdn_node for case arch-unset + ansible.builtin.include_role: + name: decdn_node + vars: + decdn_node_install_method: release + decdn_node_version: "0.0.0" + decdn_node_target: "" + rescue: + - name: Record arch-unset rejection + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['arch-unset'] }}" + when: ansible_failed_task.name is match('^Require a supported target triple') + + # The network assert must run before the address asserts: with a typo'd + # network and no explicit addresses, the error has to name the network. + - name: "Case network-unknown-noaddr — typo'd network with no explicit addresses" + block: + - name: Run decdn_node for case network-unknown-noaddr + ansible.builtin.include_role: + name: decdn_node + vars: + decdn_network: arbitrum-sepoila + decdn_payment_pool_address: "" + decdn_capacity_bond_address: "" + decdn_slash_judge_address: "" + decdn_content_blacklist_address: "" + rescue: + - name: Record network-unknown-noaddr rejection + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['network-unknown-noaddr'] }}" + when: ansible_failed_task.name is match('^Require a known decdn_network') + # --- Case: range (numeric but below the daemon's minimum) --------------------- - name: "Case range — event_poll_interval_ms below 250" block: @@ -1396,7 +1569,10 @@ the daemon/crash-loop it at config load, or ship credentials insecurely). vars: _decdn_expected: - ["cross-field", "binary-format", "binary-check-mode", "range", "bounded-range", "shape", "bool", + ["cross-field", "binary-format", "binary-check-mode", "arch-mismatch", "arch-unset", "network-unknown", + "network-unknown-noaddr", "network-chain", "backup-secret-key", "backup-bare-string", + "backup-empty", "path-cache-home", "path-backup-home", "path-decommission-cache", "range", "bounded-range", + "shape", "bool", "enum", "dict", "float", "list", "string", "string-newline", "otlp-https", "otlp-noport", "otlp-path", "otlp-userinfo", "otlp-colon", "otlp-port", "otlp-newline", "env-control-rpc", "env-control-extra", "env-gate", diff --git a/ansible/molecule/validation/molecule.yml b/ansible/molecule/validation/molecule.yml index ba36a4a..983c162 100644 --- a/ansible/molecule/validation/molecule.yml +++ b/ansible/molecule/validation/molecule.yml @@ -19,7 +19,7 @@ platforms: - name: decdn-node-validation # Same digest-pinned image as the default scenario (repo convention). Re-resolve # both together to bump. - image: geerlingguy/docker-debian12-ansible@sha256:4553092be2c00b1ffe580927b9ff03f3c3a0df32b7dd693a3eb02efb6c2b77b7 + image: geerlingguy/docker-debian12-ansible@sha256:a131e3b7a9a38dda40e98b49b39d6200ffab0ad853c714a513f5eea41bab9a25 pre_build_image: true command: /usr/lib/systemd/systemd privileged: true diff --git a/ansible/playbooks/backup.yml b/ansible/playbooks/backup.yml new file mode 100644 index 0000000..52fc85b --- /dev/null +++ b/ansible/playbooks/backup.yml @@ -0,0 +1,18 @@ +--- +# Encrypted backup of each node's identity (default) or full state. +# make backup LIMIT= ANSIBLE_ARGS='-e decdn_backup_scope=full' +# decdn_backup_age_recipients (public keys) is required; set it in group_vars. +# See roles/decdn_node/tasks/backup.yml for what is archived and docs/lifecycle.md +# for restore and host migration. +- name: Back up deCDN nodes + hosts: decdn_nodes + become: true + # One host at a time: a "full" backup stops the node for the copy, and a fleet + # should never lose every node at once. + serial: 1 + gather_facts: false + tasks: + - name: Back up this node + ansible.builtin.include_role: + name: decdn_node + tasks_from: backup diff --git a/ansible/playbooks/decommission.yml b/ansible/playbooks/decommission.yml new file mode 100644 index 0000000..ca1285b --- /dev/null +++ b/ansible/playbooks/decommission.yml @@ -0,0 +1,23 @@ +--- +# Take a node out of service: stop decdn-node, remove its unit, tear down the +# opt-in Grafana Alloy agent. Keeps the identity (needed to withdraw the bond) and +# does NOT touch the chain; the last task prints the deregister/unbond steps. +# make decommission LIMIT= # LIMIT is required; asks for typed confirmation +# Take a backup first (make backup). See docs/lifecycle.md. +- name: Decommission deCDN nodes + hosts: decdn_nodes + become: true + tasks: + - name: Stop and remove the node service + ansible.builtin.include_role: + name: decdn_node + tasks_from: decommission + + # grafana_alloy's disabled path removes only an installation this role wrote + # (it checks the managed-by marker in the unit), so this is a no-op on a host + # that never enabled Grafana Cloud or runs a foreign Alloy. + - name: Tear down the Grafana Alloy agent this repo installed + ansible.builtin.include_role: + name: grafana_alloy + vars: + decdn_grafana_cloud_enabled: false diff --git a/ansible/playbooks/site.yml b/ansible/playbooks/site.yml index 4cd2e92..8340098 100644 --- a/ansible/playbooks/site.yml +++ b/ansible/playbooks/site.yml @@ -1,6 +1,6 @@ --- # PRIMARY deployment: a hardened, public deCDN node. -# baseline -> decdn-node +# baseline -> grafana_alloy (opt-in, off by default) -> decdn-node # The node listens on public QUIC udp/4433; metrics + admin RPC stay loopback. # On-chain stake + registration (ADR 019 Phase 2) is an operator action, not # automated here — see roles/decdn_node/README.md. diff --git a/ansible/roles/baseline/README.md b/ansible/roles/baseline/README.md index fa346c9..efc1f32 100644 --- a/ansible/roles/baseline/README.md +++ b/ansible/roles/baseline/README.md @@ -82,4 +82,12 @@ set `baseline_sudo_autodetect_runner: false` and list admins explicitly in that ## Platforms -Debian (bookworm), Ubuntu (jammy, noble). +Debian 12 (bookworm) and 13 (trixie), Ubuntu 24.04 (noble) and 26.04 (resolute), on +x86_64 or aarch64. Unlike `decdn_node`, which molecule converges on all four in +containers (`grafana_alloy`'s install path runs on Debian 12 only), this role's +hardening is **not container-testable** (only its admin-account tasks run there): +nftables, +DevSec `os_hardening`/`ssh_hardening` and fail2ban need a real machine. It is +verified on real hosts (the deCDN team's own fleet runs Ubuntu 26.04) and with +`make check` as a dry run. On Ubuntu 25.10+ note the sudo-rs become workaround in +`inventory/hosts.yml.example`. diff --git a/ansible/roles/baseline/meta/main.yml b/ansible/roles/baseline/meta/main.yml index 2e52ef9..d220b38 100644 --- a/ansible/roles/baseline/meta/main.yml +++ b/ansible/roles/baseline/meta/main.yml @@ -8,7 +8,7 @@ galaxy_info: galaxy_tags: [decdn, hardening, firewall, fail2ban, devsec, debian] platforms: - name: Debian - versions: [bookworm] + versions: [bookworm, trixie] - name: Ubuntu - versions: [jammy, noble] + versions: [noble, resolute] dependencies: [] diff --git a/ansible/roles/decdn_node/README.md b/ansible/roles/decdn_node/README.md index 6bf4017..a608a2c 100644 --- a/ansible/roles/decdn_node/README.md +++ b/ansible/roles/decdn_node/README.md @@ -71,11 +71,18 @@ Per the deCDN node-onboarding ADR (019), a node only serves paid traffic after copied **verbatim**. This is the escape hatch: the role trusts them and does *not* consult `decdn_node_target` (so a test-harness stub that is legitimately not an ELF for the node's arch still works). You own building for the host's - architecture (default target `x86_64-unknown-linux-gnu`). Set **both together + architecture. Set **both together or neither** (a partial pair is rejected); when both are set they **take precedence** over `decdn_release_target_dir`, so a host can override a fleet-wide target dir without having to blank it. + **Architecture.** `decdn_node_target` is derived from the host's gathered + architecture: `x86_64-unknown-linux-gnu` on x86_64, + `aarch64-unknown-linux-gnu` on aarch64 (Graviton, Ampere, Hetzner CAX). It + picks the release tarball and the ELF check above. A play with + `gather_facts: false` must set it explicitly (or gather `min`); an explicit + value that disagrees with the host fails the deploy. + Build both `decdn-node` and `decdn` from the upstream `decdn` repo. `decdn_node_version` is **not** required in this mode — but if it is set (e.g. left over from a `release` deploy) the `--version` backstop still enforces it, @@ -123,7 +130,29 @@ Per the deCDN node-onboarding ADR (019), a node only serves paid traffic after (see [Prerequisites](#prerequisites) above), an RPC endpoint (sensitive — may embed an API key; provision it on the host or set `decdn_rpc_url` in the git-ignored `secret.yml` — see [Secrets](#secrets)), `decdn_region` (ISO 3166-1 alpha-2), and -**four** contract addresses (all `0x`+40-hex, none the zero address): +the chain: **either** `decdn_network` **or** the **four** contract addresses below +(all `0x`+40-hex, none the zero address). + +### Network profiles (`decdn_network`) + +```yaml +decdn_network: arbitrum-sepolia # the only chain upstream's `config init --chain` knows today +``` + +sets `decdn_chain_id` and **all eight** contract addresses (the four below plus +the optional SlashAppeal, OriginAssignment, PublisherRegistry and USDC) from the +role's mirror of upstream's deployment manifest, +[`vars/main/networks.yml`](vars/main/networks.yml). That file is generated by +`scripts/sync-network-profiles.py` from `contracts/deployments/.json` in `decdn/decdn` +and the `KNOWN_CHAINS` registry in `crates/cli/src/known_chains.rs`, which is exactly +what `decdn config init --chain` bakes in. A weekly CI job flags when upstream +redeploys and the mirror falls behind. + +The profile only supplies defaults: an address you also set in inventory wins, and +the role prints which ones differ from the profile. It refuses an unknown network +name and an inventory `decdn_chain_id` that disagrees with the profile. Leave +`decdn_network: ""` (the default) to set the chain yourself, e.g. for a private +deployment: | Variable | Contract | Why required | |---|---|---| @@ -132,10 +161,10 @@ API key; provision it on the host or set `decdn_rpc_url` in the git-ignored | `decdn_slash_judge_address` | SlashJudge | EIP-712 `verifyingContract` (ADR 014). | | `decdn_content_blacklist_address` | ContentBlacklist | **Newly required.** Upstream refuses to resolve a config without it: an absent or zero address is a fail-open compliance trap, and serving a blacklisted hash past its window is slashable (ADR 011/031). | -Contract addresses and chain-id are protocol facts. Copy them from the upstream -deployment manifest `decdn/contracts/deployments/.json` — the same file -`decdn config init --chain arbitrum-sepolia` bakes in — and never guess. Upstream -redeploys wholesale, so re-sync the whole set rather than patching lines. +Contract addresses and chain-id are protocol facts. Without a profile, copy them +from that deployment's manifest (`contracts/deployments/.json`) and never +guess. Upstream redeploys wholesale, so re-sync the whole set rather than patching +lines, or use `decdn_network` and let the mirror carry it. Optional (omitted from `node.toml` unless set): @@ -433,8 +462,14 @@ and re-bonding (resetting `firstBondedAt`). **Running these from a playbook:** `decdn setup` and `decdn node register` prompt for terms acceptance and abort in any non-TTY context, so both need `--accept-terms`; `setup` additionally needs `--yes` to skip its bond-amount -confirmation. All of them read the same `node.toml` this role renders, so the -contract addresses only have to be right once. +confirmation. All of them read the contract addresses from the `node.toml` this role +renders (`--config /etc/decdn/node.toml`), so those only have to be right once. + +**The RPC URL is the exception.** These commands do not read `DECDN_RPC_URL` from the +environment (only the daemon does), and this role keeps `rpc_url` out of `node.toml` +because it may embed an API key. Pass it with `--rpc-url`. The `decdn_chain` helper in +[`docs/lifecycle.md` § Running on-chain commands](../../../docs/lifecycle.md#running-on-chain-commands) +does that from `/etc/decdn/decdn.env`, as the `decdn` user. ## Network @@ -507,6 +542,10 @@ decdn node reload # re-read node.toml, apply the five hot-reloadable sect decdn node drain --wait # graceful shutdown. Without --wait it is fire-and-forget # (the response means "asked to stop", not "stopped"); # --wait polls to completion (--wait-timeout-secs 30). + # Under this role's unit (Restart=always) systemd starts + # a drained daemon again 5 s later: to take the node + # down, use `systemctl stop decdn-node` (same graceful + # path, via SIGTERM) or `make decommission`. # Read-only chain query, not admin RPC: lists active registered nodes and maps # node-ids/regions to operator addresses. Loads no keystore and spends nothing. @@ -518,6 +557,9 @@ decdn node lookup --region US --probe decdn appeal slash ``` +Backups, restores, moving a node to a new host and decommissioning: `make backup`, +`make decommission` and the procedures in [`docs/lifecycle.md`](../../../docs/lifecycle.md). + Upgrades (`release` mode): bump `decdn_node_version` and re-deploy — the version stamp triggers re-install + restart; the persistent `node.secret` and `keystore.json` are untouched. diff --git a/ansible/roles/decdn_node/defaults/main.yml b/ansible/roles/decdn_node/defaults/main.yml index 9a0f3a1..f90f854 100644 --- a/ansible/roles/decdn_node/defaults/main.yml +++ b/ansible/roles/decdn_node/defaults/main.yml @@ -29,7 +29,12 @@ decdn_release_target_dir: "" # --- Release to install (REQUIRED in "release" mode — a v GitHub Release must exist) --- decdn_node_version: "" # e.g. "0.1.1" (asserted non-empty in "release" mode) -decdn_node_target: x86_64-unknown-linux-gnu +# Release target triple. Derived from the host's gathered architecture (x86_64 or +# aarch64, see vars/main/arch.yml), so arm64 hosts (Graviton, Ampere, Hetzner CAX) +# get the aarch64 tarball with no inventory change. Needs facts: a play with +# `gather_facts: false` must set this explicitly or gather `min`. The role fails +# loud when it resolves to an unsupported value or disagrees with the host. +decdn_node_target: "{{ _decdn_arch_targets.get(ansible_facts.get('architecture', ''), '') }}" decdn_node_release_base: "https://github.com/decdn/decdn/releases/download" # Release integrity. Upstream publishes a SHA256SUMS manifest covering every # archive, GPG-signed by a maintainer key from decdn/decdn's root KEYS file @@ -42,10 +47,21 @@ decdn_verify_release_signature: true decdn_release_keyring: decdn-release-KEYS.asc # --- Identity / on-chain ----------------------------------------------------- -# The FOUR contract addresses and region are REQUIRED with no usable default -# (empty → the asserts in tasks/main.yml fail loud). Supply via host_vars. -# chain_id is the one exception: it defaults to the upstream binary's own default -# (Arbitrum Sepolia) rather than being a deployment-specific fact to source. +# Network profile. Set decdn_network to a chain upstream's `decdn config init +# --chain ` knows (today only "arbitrum-sepolia") and chain_id plus every +# contract address below default to that deployment's manifest values, mirrored +# into vars/main/networks.yml by scripts/sync-network-profiles.py. Any address you +# also set in inventory still wins (the role prints which ones differ). Leave it "" +# to supply chain_id and the addresses yourself, e.g. for a private deployment. +decdn_network: "" +# The selected profile ({} when decdn_network is "" or unknown — the role asserts +# the name before anything reads it). Internal: set decdn_network, not this. +_decdn_profile: "{{ _decdn_network_profiles.get(decdn_network, {}) }}" +# +# Without a profile, the FOUR required contract addresses and region have no usable +# default (empty → the asserts in tasks/main.yml fail loud). Supply via host_vars. +# region is never in a profile: it is where the node is, not a chain fact. +# chain_id falls back to the upstream binary's own default (Arbitrum Sepolia). # # rpc_url is required too, but has TWO sources. Leave it empty here and provision # a 0600 /etc/decdn/decdn.env on the target host instead (the preferred posture — @@ -54,30 +70,32 @@ decdn_release_keyring: decdn-release-KEYS.asc # one. The role fails loud when NEITHER is present. See roles/decdn_node/README.md # ("Secrets") and roles/decdn_node/files/decdn.env.example. decdn_rpc_url: "" # SENSITIVE; may embed an API key — or provision decdn.env on the host -decdn_payment_pool_address: "" -decdn_capacity_bond_address: "" -decdn_slash_judge_address: "" +decdn_payment_pool_address: "{{ _decdn_profile.get('payment_pool_address', '') }}" +decdn_capacity_bond_address: "{{ _decdn_profile.get('capacity_bond_address', '') }}" +decdn_slash_judge_address: "{{ _decdn_profile.get('slash_judge_address', '') }}" # REQUIRED (ADR 011/031). ContentBlacklist watcher — evicts blacklisted blobs in # your region scope. Upstream now refuses to resolve a config without it: an # absent or zero address is a fail-open compliance trap, and serving a # blacklisted hash past its compliance window is slashable. -decdn_content_blacklist_address: "" +decdn_content_blacklist_address: "{{ _decdn_profile.get('content_blacklist_address', '') }}" # Optional (ADR 028). SlashAppeal contract — only needed to file appeals with -# `decdn appeal slash`; the daemon ignores it. Empty => key omitted from node.toml. -decdn_slash_appeal_address: "" # accepted but never read by the daemon; `decdn appeal slash` uses it +# `decdn appeal slash`; the daemon accepts but never reads it. Empty => key +# omitted from node.toml. +decdn_slash_appeal_address: "{{ _decdn_profile.get('slash_appeal_address', '') }}" # Optional (ADR 022). Chain-backed origin directory. These are INDEPENDENTLY # optional upstream — the old "both or neither" rule is gone. The daemon reads # origin_assignment_address for cache-miss pull-through fallback; it only # validates publisher_registry_address, which `decdn publish` consumes. # Empty origin_assignment_address => the directory is empty (no authorized origins). -decdn_origin_assignment_address: "" -decdn_publisher_registry_address: "" +decdn_origin_assignment_address: "{{ _decdn_profile.get('origin_assignment_address', '') }}" +decdn_publisher_registry_address: "{{ _decdn_profile.get('publisher_registry_address', '') }}" # Origin-directory lookup cache ("" => daemon default). decdn_origin_directory_positive_ttl_sec: "" # daemon dflt 300 decdn_origin_directory_negative_ttl_sec: "" # daemon dflt 30 decdn_origin_directory_cache_capacity: "" # daemon dflt 4096 decdn_region: "" # ISO 3166-1 alpha-2 -decdn_chain_id: 421614 # Arbitrum Sepolia (matches decdn-node's --chain-id default) +# Arbitrum Sepolia (matches decdn-node's --chain-id default) unless a profile says otherwise. +decdn_chain_id: "{{ _decdn_profile.get('chain_id', 421614) }}" # --- Network ------------------------------------------------------------------ decdn_bind_port: 4433 # public QUIC (udp); opened in the baseline firewall @@ -149,7 +167,7 @@ decdn_fee_shares_poll_interval_sec: "" # daemon dflt 3600; > 0 — FeeRouter s # only `decdn setup` / `decdn pool` do. So nothing downstream catches a malformed # value, which is exactly why the role asserts its shape itself. (The former # decdn_swap_* knobs were removed upstream along with the on-chain swap path.) -decdn_usdc_address: "" +decdn_usdc_address: "{{ _decdn_profile.get('usdc_address', '') }}" # --- Cache -------------------------------------------------------------------- decdn_cache_size_mb: 10240 # 10 GB @@ -387,3 +405,29 @@ decdn_cli_bin: /usr/local/bin/decdn # deliveries (a money-losing cut), so prefer waiting over cutting. See # templates/decdn-node.service.j2. decdn_node_stop_timeout_sec: 300 + +# --- Lifecycle: backup + decommission (tasks_from: backup / decommission) ------- +# Only read by those entry points (playbooks/backup.yml, playbooks/decommission.yml); +# a normal deploy ignores them. See docs/lifecycle.md. +# +# Backups are encrypted on the host to these PUBLIC keys, so plaintext key material +# never leaves the node: age recipients (age1…, `age-keygen -y key.txt`) and/or SSH +# public keys (ssh-ed25519 / ssh-rsa lines). REQUIRED for a backup. Keep the matching +# private keys somewhere other than the node and the control machine's repo. +decdn_backup_age_recipients: [] +decdn_backup_scope: identity # "identity" (hot) | "full" (stops the node for the copy) +decdn_backup_include_env: false # also archive decdn.env (holds the RPC URL, maybe an API key) +decdn_backup_leave_stopped: false # "full": leave the node stopped afterwards (host migration) +decdn_backup_dir: /var/backups/decdn # on the host, root 0700 +# Copy the ENCRYPTED archive to the control machine. On for identity archives +# (kilobytes); off for full ones, because fetch under become reads the whole file +# into memory on both ends. The report prints a streaming `ssh … sudo cat` instead. +decdn_backup_fetch: "{{ decdn_backup_scope == 'identity' }}" +decdn_backup_local_dir: "{{ playbook_dir }}/../backups" # control machine; git-ignored in this repo +# +# Decommission refuses to touch more hosts than this in one run. Its typed +# confirmation is interactive; a non-interactive run sets decdn_decommission_confirm +# to the comma-joined, sorted host list (deliberately NOT defaulted here). +decdn_decommission_max_hosts: 1 +decdn_decommission_purge_cache: false +decdn_decommission_prompt_seconds: 300 # an unanswered confirmation prompt fails closed after this diff --git a/ansible/roles/decdn_node/meta/main.yml b/ansible/roles/decdn_node/meta/main.yml index 617a982..e6a6974 100644 --- a/ansible/roles/decdn_node/meta/main.yml +++ b/ansible/roles/decdn_node/meta/main.yml @@ -10,7 +10,7 @@ galaxy_info: galaxy_tags: [decdn, cdn, node, systemd, web3, debian] platforms: - name: Debian - versions: [bookworm] + versions: [bookworm, trixie] - name: Ubuntu - versions: [jammy, noble] + versions: [noble, resolute] dependencies: [] diff --git a/ansible/roles/decdn_node/tasks/backup.yml b/ansible/roles/decdn_node/tasks/backup.yml new file mode 100644 index 0000000..1f5d2f7 --- /dev/null +++ b/ansible/roles/decdn_node/tasks/backup.yml @@ -0,0 +1,223 @@ +--- +# Encrypted backup of a node: `include_role: {name: decdn_node, tasks_from: backup}` +# (playbooks/backup.yml, `make backup`). See docs/lifecycle.md for restore and host +# migration. +# +# The archive is encrypted ON THE HOST to the operator's age recipients (public +# keys), so plaintext key material never leaves the node and nothing secret is +# needed on the control machine: only the encrypted file is fetched. Decrypting it +# needs the matching private key, which never comes near this role. +# +# Two scopes (decdn_backup_scope): +# identity node.secret + keystore.json + the keystore password file. Taken hot: +# the daemon never rewrites them while running. Enough to bring the +# SAME on-chain identity up elsewhere. +# full the whole data dir minus the cache, plus the keystore (wherever it +# lives) and the password file. The daemon's redb stores (vouchers not +# yet redeemed live in them) are only consistent at rest, so the node is +# stopped for the copy and restarted afterwards if it was up. Use it +# before a host migration. + +- name: Refuse to take a backup in check mode + ansible.builtin.assert: + that: + - not ansible_check_mode + fail_msg: >- + A backup writes an archive on the host and (scope "full") stops the node, so + it has no meaningful dry run. Run it without --check. + +- name: Validate the backup settings + ansible.builtin.assert: + that: + - decdn_backup_scope in ["identity", "full"] + - decdn_backup_age_recipients is sequence + - decdn_backup_age_recipients is not string + - decdn_backup_age_recipients is not mapping + - decdn_backup_age_recipients | length > 0 + - >- + decdn_backup_age_recipients + | reject('match', '^(age1[0-9a-z]{58}|ssh-ed25519 [A-Za-z0-9+/=]+|ssh-rsa [A-Za-z0-9+/=]+)( .*)?$') + | list | length == 0 + - decdn_backup_include_env is boolean + - decdn_backup_leave_stopped is boolean + - decdn_backup_fetch is boolean + fail_msg: >- + decdn_backup_scope must be "identity" or "full". decdn_backup_age_recipients + must be a non-empty LIST of public keys to encrypt to: age recipients + (age1…, from `age-keygen -y`) or SSH public keys (ssh-ed25519 / ssh-rsa lines). + Never a private key. decdn_backup_include_env, decdn_backup_leave_stopped and + decdn_backup_fetch must be booleans. + +- name: Validate the lifecycle paths + ansible.builtin.import_tasks: validate-lifecycle-paths.yml + +# The keystore is listed explicitly in both scopes: decdn_keystore_file may live +# outside decdn_home, and a migration backup without the wallet strands the bond. +# In the full scope it is only added when the data dir does not already cover it, +# so tar does not store it twice. +- name: Build the list of paths to archive + vars: + _keystore_in_home: "{{ decdn_keystore_file.startswith(decdn_home.rstrip('/') ~ '/') }}" + ansible.builtin.set_fact: + _decdn_backup_paths: >- + {{ ([decdn_home ~ '/node.secret', decdn_keystore_file] + if decdn_backup_scope == 'identity' + else [decdn_home] + ([] if _keystore_in_home | bool else [decdn_keystore_file])) + + [decdn_keystore_password_file] + + ([decdn_env_file] if decdn_backup_include_env | bool else []) }} + _decdn_backup_file: >- + {{ decdn_backup_dir }}/{{ inventory_hostname }}-{{ decdn_backup_scope }}-{{ + now(utc=true, fmt='%Y%m%dT%H%M%SZ') }}.tar.age + _decdn_backup_written: false + +- name: Check that every path to archive exists + ansible.builtin.stat: + path: "{{ item }}" + register: _decdn_backup_stat + loop: "{{ _decdn_backup_paths + [decdn_home ~ '/node.secret', decdn_keystore_file] }}" + +- name: Require every path to archive + ansible.builtin.assert: + that: + - _decdn_backup_stat.results | rejectattr('stat.exists') | list | length == 0 + fail_msg: >- + Missing on the host: {{ _decdn_backup_stat.results | rejectattr('stat.exists') + | map(attribute='item') | unique | join(', ') }}. There is no identity to back up + yet (deploy and provision the keystore first), or a path variable does not match + this host. + +- name: Install age + ansible.builtin.apt: + name: age + state: present + update_cache: true + cache_valid_time: 3600 + +- name: Create the backup directory + ansible.builtin.file: + path: "{{ decdn_backup_dir }}" + state: directory + owner: root + group: root + mode: "0700" + +# Recipients go in a file (`age -R`) rather than argv: SSH public keys carry spaces +# and comments, and a file needs no shell quoting at all. +- name: Write the recipients file + ansible.builtin.copy: + dest: "{{ decdn_backup_dir }}/.recipients" + content: "{{ decdn_backup_age_recipients | join('\n') }}\n" + owner: root + group: root + mode: "0600" + +# The unit's ActiveState rather than service_facts: service_facts reports only the +# "running" sub-state as running, so a Restart=always unit caught between restarts +# (activating / auto-restart) would read as stopped and never be restarted. +- name: Record whether the node is up + ansible.builtin.systemd_service: + name: decdn-node + register: _decdn_backup_active + failed_when: false + +- name: Decide whether to restart the node afterwards + ansible.builtin.set_fact: + _decdn_was_up: >- + {{ (_decdn_backup_active.status | default({})).get('ActiveState', '') + in ['active', 'activating', 'reloading'] }} + +- name: Check for the decdn-node unit + ansible.builtin.stat: + path: /etc/systemd/system/decdn-node.service + register: _decdn_backup_unit + +- name: Take the backup + block: + # Stopped whenever the unit exists, not only when it reads as up: a crash-looping + # node would otherwise be restarted by systemd mid-copy. + - name: Stop the node for a consistent full backup + ansible.builtin.systemd_service: + name: decdn-node + state: stopped + when: + - decdn_backup_scope == "full" + - _decdn_backup_unit.stat.exists + + # tar reads the absolute paths relative to / (so a restore is `tar -C /`) and + # stores numeric owners. The cache is excluded from a full backup: it is + # re-fetchable content and usually most of the disk. Written to a .partial name + # and renamed, so an interrupted run never leaves a truncated archive behind + # that looks complete. + - name: Write the encrypted archive + ansible.builtin.shell: + cmd: >- + set -euo pipefail; + umask 077; + tar -C / --numeric-owner + --exclude={{ (decdn_cache_dir | regex_replace('^/', '')) | quote }} + -czf - -- {{ _decdn_backup_paths | map('regex_replace', '^/', '') | map('quote') | join(' ') }} + | age -R {{ (decdn_backup_dir ~ '/.recipients') | quote }} -o {{ (_decdn_backup_file ~ '.partial') | quote }}; + mv -- {{ (_decdn_backup_file ~ '.partial') | quote }} {{ _decdn_backup_file | quote }} + executable: /bin/bash + changed_when: true + + - name: Mark the archive as written + ansible.builtin.set_fact: + _decdn_backup_written: true + always: + - name: Remove a partial archive left by a failed run + ansible.builtin.file: + path: "{{ _decdn_backup_file }}.partial" + state: absent + + # leave_stopped is honoured only when the archive was written: a failed backup + # must never leave the node down with nothing to show for it. + - name: Restart the node after the full backup + ansible.builtin.systemd_service: + name: decdn-node + state: started + when: + - decdn_backup_scope == "full" + - _decdn_backup_unit.stat.exists + - _decdn_was_up | bool + - not (decdn_backup_leave_stopped | bool and _decdn_backup_written | bool) + +- name: Checksum the archive + ansible.builtin.stat: + path: "{{ _decdn_backup_file }}" + checksum_algorithm: sha256 + register: _decdn_backup_result + +# fetch is fine for identity archives (kilobytes). Under become it reads the whole +# file into memory on both ends (base64, via slurp), so a full archive is not +# fetched by default; the report prints a streaming copy command instead. +- name: Create the local backup directory for this host + ansible.builtin.file: + path: "{{ decdn_backup_local_dir }}/{{ inventory_hostname }}" + state: directory + mode: "0700" + delegate_to: localhost + become: false + when: decdn_backup_fetch | bool + +- name: Fetch the encrypted archive to the control machine + ansible.builtin.fetch: + src: "{{ _decdn_backup_file }}" + dest: "{{ decdn_backup_local_dir }}/{{ inventory_hostname }}/" + flat: true + validate_checksum: true + when: decdn_backup_fetch | bool + +- name: Report the backup + ansible.builtin.debug: + msg: >- + {{ decdn_backup_scope }} backup written to {{ _decdn_backup_file }} + (sha256 {{ _decdn_backup_result.stat.checksum }}, {{ _decdn_backup_result.stat.size }} bytes) + {{- (', fetched to ' ~ decdn_backup_local_dir ~ '/' ~ inventory_hostname ~ '/') + if decdn_backup_fetch | bool + else ('. Not fetched; copy it off the host with: ssh ' ~ inventory_hostname + ~ ' sudo cat ' ~ _decdn_backup_file ~ ' > ' ~ (_decdn_backup_file | basename)) }}. + Prove it restores before you rely on it: + `age -d -i | tar -tz`. + {{- ' The node was left STOPPED (decdn_backup_leave_stopped).' if decdn_backup_scope == 'full' + and decdn_backup_leave_stopped | bool and _decdn_was_up | bool else '' }} diff --git a/ansible/roles/decdn_node/tasks/decommission.yml b/ansible/roles/decdn_node/tasks/decommission.yml new file mode 100644 index 0000000..45249bb --- /dev/null +++ b/ansible/roles/decdn_node/tasks/decommission.yml @@ -0,0 +1,116 @@ +--- +# Take a node out of service: `include_role: {name: decdn_node, tasks_from: +# decommission}` (playbooks/decommission.yml, `make decommission LIMIT=`). +# +# What it does: stop the daemon, disable and remove its systemd unit, and +# (opt-in) purge the cache. What it deliberately KEEPS: the identity (node.secret, +# keystore.json, the password file), /etc/decdn and the binaries. The eth keystore +# is what withdraws the bond after the unbonding window, so deleting it here would +# strand the stake; remove it by hand once the withdrawal has landed. +# +# It does NOT touch the chain. Deregistering and unbonding are operator steps with +# the decdn CLI; the final task prints them. Stop with systemctl, never `decdn node +# drain`: the unit is Restart=always, so a drained daemon is simply restarted. + +- name: Refuse to decommission in check mode + ansible.builtin.assert: + that: + - not ansible_check_mode + fail_msg: Decommissioning has no dry run; run it without --check once you mean it. + +- name: Refuse to decommission more hosts than allowed at once + ansible.builtin.assert: + that: + - decdn_decommission_max_hosts | int > 0 + - ansible_play_hosts_all | length <= decdn_decommission_max_hosts | int + - decdn_decommission_purge_cache is boolean + - decdn_decommission_prompt_seconds | int > 0 + fail_msg: >- + This run targets {{ ansible_play_hosts_all | length }} hosts + ({{ ansible_play_hosts_all | sort | join(', ') }}), more than + decdn_decommission_max_hosts ({{ decdn_decommission_max_hosts }}). Scope it + with LIMIT, or raise decdn_decommission_max_hosts deliberately. + decdn_decommission_purge_cache must be a boolean and + decdn_decommission_prompt_seconds a positive number of seconds. + +- name: Validate the lifecycle paths + ansible.builtin.import_tasks: validate-lifecycle-paths.yml + +# Typed confirmation (AGENTS.md hard rule 4). Interactive by default; a +# non-interactive caller passes decdn_decommission_confirm explicitly. With no TTY +# the prompt returns empty input, and an unanswered prompt times out after +# decdn_decommission_prompt_seconds with empty input too; both fail the check below +# rather than proceeding. +- name: Ask for typed confirmation + ansible.builtin.pause: + prompt: >- + About to STOP decdn-node and REMOVE its systemd unit on: + {{ ansible_play_hosts_all | sort | join(',') }}. + On-chain deregistration and unbonding are NOT done by this playbook. + Type the host list above exactly to continue + (times out after {{ decdn_decommission_prompt_seconds }} s) + seconds: "{{ decdn_decommission_prompt_seconds | int }}" + register: _decdn_decommission_typed + run_once: true + when: decdn_decommission_confirm is not defined + +- name: Require the typed confirmation to match the targeted hosts + ansible.builtin.assert: + that: + - (decdn_decommission_confirm | default(_decdn_decommission_typed.user_input | default('')) | trim) + == (ansible_play_hosts_all | sort | join(',')) + fail_msg: >- + Confirmation did not match "{{ ansible_play_hosts_all | sort | join(',') }}". + Nothing was changed. + +- name: Check for the decdn-node unit + ansible.builtin.stat: + path: /etc/systemd/system/decdn-node.service + register: _decdn_decommission_unit + +# SIGTERM runs the daemon's graceful shutdown (the same path `decdn node drain` +# takes); systemd waits up to decdn_node_stop_timeout_sec before escalating. +- name: Stop and disable decdn-node + ansible.builtin.systemd_service: + name: decdn-node + state: stopped + enabled: false + when: _decdn_decommission_unit.stat.exists + +- name: Remove the decdn-node unit + ansible.builtin.file: + path: /etc/systemd/system/decdn-node.service + state: absent + notify: Reload systemd + +- name: Apply the systemd reload now + ansible.builtin.meta: flush_handlers + +- name: Purge the cache + ansible.builtin.file: + path: "{{ decdn_cache_dir }}" + state: absent + when: decdn_decommission_purge_cache | bool + +- name: Report what is left to do + ansible.builtin.debug: + msg: + - "decdn-node is stopped and its unit removed on {{ inventory_hostname }}." + - >- + A Grafana Alloy agent installed by this repo is removed separately: + playbooks/decommission.yml does it next; with the collection, include + grafana_alloy with decdn_grafana_cloud_enabled false. + - >- + Kept: {{ decdn_home }} (node.secret, keystore.json{{ '' if decdn_decommission_purge_cache | bool + else ', cache' }}), {{ decdn_etc }} and the binaries. Keep the keystore until + the bond is withdrawn. + - >- + On-chain, with the decdn CLI on this host (each takes --dry-run first): + `decdn node deregister` removes the node from the active set but does NOT + return the bond, which stays deposited and slashable; then + `decdn node unbond --all` starts the unbonding window; re-run it after the + window to withdraw. The CLI does not read decdn.env: pass the RPC URL with + --rpc-url, as in docs/lifecycle.md § Running on-chain commands. + - >- + The public udp/{{ decdn_bind_port }} firewall rule stays until you re-run + baseline without it (e.g. after removing the host from decdn_nodes). diff --git a/ansible/roles/decdn_node/tasks/main.yml b/ansible/roles/decdn_node/tasks/main.yml index 7da0765..6007021 100644 --- a/ansible/roles/decdn_node/tasks/main.yml +++ b/ansible/roles/decdn_node/tasks/main.yml @@ -40,6 +40,39 @@ it off means the tarballs are taken on trust. when: decdn_node_install_method == "release" +# The target triple picks the release tarball and, in manual dir-mode, the ELF +# architecture the binaries must match. It is derived from gathered facts by +# default; an explicit value that disagrees with the host would install binaries +# the node cannot exec, so both cases are checked before anything is fetched. +# Explicit *_manual_bin_src paths never consult the triple, so they skip this. +- name: Resolve the node target triple + when: >- + decdn_node_install_method == "release" + or (decdn_release_target_dir | length > 0 + and not (decdn_node_manual_bin_src | length > 0 and decdn_cli_manual_bin_src | length > 0)) + block: + - name: Require a supported target triple + ansible.builtin.assert: + that: + - decdn_node_target in _decdn_arch_targets.values() + fail_msg: >- + decdn_node_target resolved to "{{ decdn_node_target }}", which is not one of + the Linux targets upstream releases ({{ _decdn_arch_targets.values() | join(', ') }}). + It is derived from the host's gathered architecture + ("{{ ansible_facts.get('architecture', 'not gathered') }}"): gather facts + (gather_subset: [min] is enough) or set decdn_node_target explicitly. + + - name: Require the target triple to match the host architecture + ansible.builtin.assert: + that: + - _decdn_arch_targets[ansible_facts['architecture']] == decdn_node_target + fail_msg: >- + decdn_node_target is {{ decdn_node_target }}, but this host is + {{ ansible_facts['architecture'] }} and needs + {{ _decdn_arch_targets[ansible_facts['architecture']] }}. Drop the explicit + decdn_node_target (it is derived from the host) or fix it for this host. + when: ansible_facts.get('architecture', '') in _decdn_arch_targets + # Resolve + validate the manual binary sources on the CONTROL MACHINE before they # are copied to the node. Two ways to point at them, in precedence order: # 1. both explicit *_manual_bin_src paths — the escape hatch, copied verbatim and @@ -191,10 +224,9 @@ loop_control: label: "{{ item.item }}" vars: - _decdn_arch_file_tokens: - x86_64-unknown-linux-gnu: x86-64 - aarch64-unknown-linux-gnu: aarch64 - _decdn_expect_arch: "{{ _decdn_arch_file_tokens[decdn_node_target] | default('x86-64') }}" + # Keyed lookup with no fallback: "Resolve the node target triple" above has + # already proved decdn_node_target is one of the map's keys. + _decdn_expect_arch: "{{ _decdn_arch_file_tokens[decdn_node_target] }}" when: _decdn_use_dir | bool - name: Validate decdn_node_generate_keystore is a real boolean @@ -254,6 +286,49 @@ {{ ((decdn_env_recorded.content | default('') | b64decode | trim).split() | length == 2) | ternary((decdn_env_recorded.content | default('') | b64decode | trim).split()[1], '') }} +# Network profile (decdn_network). The profile only supplies defaults, so these +# checks run before the address asserts below: an unknown name would otherwise +# surface as "addresses missing", and an inventory chain_id pointing at another +# chain would silently pair that chain with this profile's addresses. +- name: Require a known decdn_network + ansible.builtin.assert: + that: + - decdn_network == "" or decdn_network in _decdn_network_profiles + fail_msg: >- + decdn_network "{{ decdn_network }}" is not a known network. Known: + {{ _decdn_network_profiles.keys() | join(', ') }} (the chains upstream's + `decdn config init --chain` supports). Leave it "" to set chain_id and the + contract addresses yourself. + +- name: Check inventory values against the network profile + when: decdn_network | length > 0 + block: + - name: Require chain_id to match the network profile + ansible.builtin.assert: + that: + - decdn_chain_id | int == _decdn_profile.chain_id | int + fail_msg: >- + decdn_chain_id is {{ decdn_chain_id }}, but decdn_network "{{ decdn_network }}" + is chain {{ _decdn_profile.chain_id }}. Drop decdn_chain_id from inventory (the + profile supplies it) or clear decdn_network. + + - name: Report contract addresses overridden in inventory + ansible.builtin.debug: + msg: >- + decdn_network "{{ decdn_network }}" is in use, but inventory overrides + {{ _decdn_diff | join(', ') }}. Those values win over the profile; make sure + that is deliberate (a stale hand-copied address is the usual cause). + vars: + _decdn_diff: >- + {%- set ns = namespace(out=[]) -%} + {%- for k, v in _decdn_profile.items() if k is match('.*_address$') -%} + {%- if (lookup('ansible.builtin.vars', 'decdn_' ~ k) | lower) != (v | lower) -%} + {%- set ns.out = ns.out + ['decdn_' ~ k] -%} + {%- endif -%} + {%- endfor -%} + {{ ns.out }} + when: _decdn_diff | length > 0 + - name: Require chain + contract configuration ansible.builtin.assert: that: @@ -275,7 +350,9 @@ ContentBlacklist, because an absent or zero address is a fail-open compliance trap — serving a blacklisted hash past its window is slashable. Source chain-id + contract addresses from the deCDN contract deployment manifest - (decdn/contracts/deployments/.json) — never guess. Set them in host_vars. + (contracts/deployments/.json in decdn/decdn) — never guess — or set + decdn_network (e.g. "arbitrum-sepolia") to take them from the role's mirror of + that manifest. vars: decdn_zero_address: "0x0000000000000000000000000000000000000000" diff --git a/ansible/roles/decdn_node/tasks/validate-lifecycle-paths.yml b/ansible/roles/decdn_node/tasks/validate-lifecycle-paths.yml new file mode 100644 index 0000000..e69754e --- /dev/null +++ b/ansible/roles/decdn_node/tasks/validate-lifecycle-paths.yml @@ -0,0 +1,67 @@ +--- +# Path guards for the destructive lifecycle entry points (backup, decommission). +# They run through tasks_from, so none of main.yml's asserts precede them, and they +# act on paths an operator can override: backup excludes decdn_cache_dir from the +# archive and makes decdn_backup_dir root-only; decommission can delete +# decdn_cache_dir. A cache dir that equals or encloses the identity would empty +# every backup and let the purge delete the keys that withdraw the bond, so refuse +# those layouts before anything is touched. Same shape as grafana_alloy's +# validate-paths.yml: segment checks, then `startswith` on '/'-terminated paths +# (`a` encloses `b` when (b ~ '/').startswith(a ~ '/')). + +- name: Constrain the lifecycle paths (absolute, no traversal, no empty segments) + ansible.builtin.assert: + that: + - item.value is match('^/[A-Za-z0-9._/-]+$') + - "'.' not in item.value.split('/')" + - "'..' not in item.value.split('/')" + - "'' not in item.value.rstrip('/').split('/')[1:]" + quiet: true + fail_msg: >- + {{ item.name }} must be an absolute path whose segments use only + [A-Za-z0-9._-], with no '.', '..' or empty segments (got "{{ item.value }}"). + Backup and decommission compare and delete these paths, so their shape is + enforced first. + loop: + - {name: decdn_home, value: "{{ decdn_home }}"} + - {name: decdn_etc, value: "{{ decdn_etc }}"} + - {name: decdn_cache_dir, value: "{{ decdn_cache_dir }}"} + - {name: decdn_keystore_file, value: "{{ decdn_keystore_file }}"} + - {name: decdn_keystore_password_file, value: "{{ decdn_keystore_password_file }}"} + - {name: decdn_backup_dir, value: "{{ decdn_backup_dir }}"} + loop_control: + label: "{{ item.name }}" + +- name: Require the cache dir to hold no identity + ansible.builtin.assert: + that: + - not (item.rstrip('/') ~ '/').startswith(decdn_cache_dir.rstrip('/') ~ '/') + quiet: true + fail_msg: >- + decdn_cache_dir ({{ decdn_cache_dir }}) equals or encloses {{ item }}. The cache + is excluded from backups and deleted by decommission's cache purge, so it must + be a directory of its own, e.g. {{ decdn_home }}/cache. + loop: + - "{{ decdn_home }}" + - "{{ decdn_etc }}" + - "{{ decdn_home }}/node.secret" + - "{{ decdn_keystore_file }}" + - "{{ decdn_keystore_password_file }}" + +- name: Require the backup dir to stay clear of the node and system directories + vars: + _dir: "{{ decdn_backup_dir.rstrip('/') ~ '/' }}" + ansible.builtin.assert: + that: + - _dir not in ['/etc/', '/var/', '/var/lib/', '/usr/', '/root/', '/home/', '/opt/', '/srv/', '/boot/', '/tmp/'] + # Not the data or config dir, nor anything enclosing them (the role makes it + # root 0700) ... + - not (decdn_home.rstrip('/') ~ '/').startswith(_dir) + - not (decdn_etc.rstrip('/') ~ '/').startswith(_dir) + # ... and not inside the data dir, or a full backup would archive itself. + - not _dir.startswith(decdn_home.rstrip('/') ~ '/') + quiet: true + fail_msg: >- + decdn_backup_dir ({{ decdn_backup_dir }}) must be a dedicated directory outside + {{ decdn_home }} and {{ decdn_etc }}, and not a system directory: the role makes + it root-only (0700) and writes archives into it. The default is /var/backups/decdn. diff --git a/ansible/roles/decdn_node/vars/main/arch.yml b/ansible/roles/decdn_node/vars/main/arch.yml new file mode 100644 index 0000000..ad27b3f --- /dev/null +++ b/ansible/roles/decdn_node/vars/main/arch.yml @@ -0,0 +1,15 @@ +--- +# Host architecture (ansible_facts['architecture']) -> the upstream release target +# triple. These are the two Linux targets decdn/decdn's release.yml builds for; the +# defaults derive decdn_node_target from this map, and tasks/main.yml asserts the +# result is one of its values. Role vars, not defaults: the supported set is a fact +# about upstream's release matrix, not an operator knob. +_decdn_arch_targets: + x86_64: x86_64-unknown-linux-gnu + aarch64: aarch64-unknown-linux-gnu + +# decdn_node_target -> the architecture token `file -b` prints for a matching ELF, +# used by the manual dir-mode binary check. +_decdn_arch_file_tokens: + x86_64-unknown-linux-gnu: x86-64 + aarch64-unknown-linux-gnu: aarch64 diff --git a/ansible/roles/decdn_node/vars/main/networks.yml b/ansible/roles/decdn_node/vars/main/networks.yml new file mode 100644 index 0000000..661f467 --- /dev/null +++ b/ansible/roles/decdn_node/vars/main/networks.yml @@ -0,0 +1,22 @@ +--- +# GENERATED by scripts/sync-network-profiles.py -- do not hand-edit. +# Source: decdn/decdn @ 5ca3e7f1c5ac84890fb5b666b29b6f384e3797ec +# crates/cli/src/known_chains.rs (KNOWN_CHAINS, KnownChain::addresses) +# + contracts/deployments/.json +# +# The addresses `decdn config init --chain ` bakes into node.toml. The role's +# address defaults read this map lazily when decdn_network is set, so an explicit +# inventory value still wins. Protocol facts: upstream's deployment manifests are +# the source, this file only mirrors them. +_decdn_network_profiles: + arbitrum-sepolia: + chain_id: 421614 + payment_pool_address: "0x351E5544471e11638b89DF8E3A3299c28E00cEc3" + capacity_bond_address: "0x65a9e38F41B9bE2fcB97950a0E3c33e506E6E353" + slash_judge_address: "0x8e4d5945615bDF00106ff429da07bd08bCdEdB94" + content_blacklist_address: "0x36487Eca107F85aD361Eb5802C47a25D0CB330A9" + slash_appeal_address: "0x64456CBa97613a128dd9C4ac33949Ac3DcD1A07F" + origin_assignment_address: "0xdD2546E68dE6bae829Fe19b6faF4F2DB5b432FEb" + publisher_registry_address: "0x5db5079e2C8A60ecc9297bE16e1a167794E382a6" + usdc_address: "0x75faf114eafb1BDbe2F0316DF893fd58CE46AA4d" + deploy_block: 11744904 diff --git a/ansible/roles/grafana_alloy/meta/main.yml b/ansible/roles/grafana_alloy/meta/main.yml index 7d8eb63..2d8c83f 100644 --- a/ansible/roles/grafana_alloy/meta/main.yml +++ b/ansible/roles/grafana_alloy/meta/main.yml @@ -10,7 +10,7 @@ galaxy_info: galaxy_tags: [decdn, cdn, observability, grafana, otlp, debian] platforms: - name: Debian - versions: [bookworm] + versions: [bookworm, trixie] - name: Ubuntu - versions: [jammy, noble] + versions: [noble, resolute] dependencies: [] diff --git a/ansible/roles/grafana_alloy/tasks/install.yml b/ansible/roles/grafana_alloy/tasks/install.yml index 3745005..ba901c3 100644 --- a/ansible/roles/grafana_alloy/tasks/install.yml +++ b/ansible/roles/grafana_alloy/tasks/install.yml @@ -8,11 +8,23 @@ # testing) straight to /usr/local/bin/alloy. No package is created. # Map Ansible's machine facts onto Debian arch names used by the asset set -# (alloy--1..deb). Unsupported architectures fail loud here so an -# operator gets a clear message instead of a 404 mid-play. +# (alloy--1..deb). Unsupported architectures (or a play that did +# not gather facts) fail loud below, so an operator gets a clear message instead +# of a 404 mid-play. - name: Resolve the Debian package architecture ansible.builtin.set_fact: - _ga_deb_arch: "{{ ({'x86_64': 'amd64', 'aarch64': 'arm64'}[ansible_architecture] | default('unsupported')) }}" + _ga_deb_arch: >- + {{ {'x86_64': 'amd64', 'aarch64': 'arm64'}.get(ansible_facts.get('architecture', ''), 'unsupported') }} + +- name: Require a supported host architecture for the Alloy package + ansible.builtin.assert: + that: + - _ga_deb_arch != "unsupported" + fail_msg: >- + Grafana Alloy ships .deb packages for x86_64 and aarch64 only; this host reports + "{{ ansible_facts.get('architecture', 'not gathered') }}". Gather facts + (gather_subset: [min] is enough), or use grafana_alloy_install_method "manual". + when: grafana_alloy_install_method == "release" - name: Name the .deb asset for this architecture # Reusable across the download + install tasks below. diff --git a/charts/decdn-node/CHANGELOG.md b/charts/decdn-node/CHANGELOG.md new file mode 100644 index 0000000..f1e9d11 --- /dev/null +++ b/charts/decdn-node/CHANGELOG.md @@ -0,0 +1,42 @@ +# Changelog — `decdn-node` Helm chart + +All notable changes to the chart are documented here. The format follows +[Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and the chart adheres to +[Semantic Versioning](https://semver.org/spec/v2.0.0.html). The chart and the +`decdn.node` Ansible collection share one repo version: a `vX.Y.Z` tag releases +both (see [RELEASING.md](../../RELEASING.md)). + +## [Unreleased] + +### Added + +- `ServiceMonitor`: the `job`, `region`, `deployment_environment` and `instance` + target labels the upstream dashboards and alerts select on, configurable through + `metrics.serviceMonitor.jobLabel`, `.deploymentEnvironment` and `.relabelings`. +- Optional `PrometheusRule` with upstream's reference alert rules + (`metrics.prometheusRule`), and the upstream Grafana dashboards as sidecar-labelled + ConfigMaps (`metrics.grafanaDashboards`). Both are vendored in `files/monitoring/` + by `scripts/sync-monitoring.sh`, with the upstream commit recorded in `SOURCE`. + +## [0.1.0] — unreleased + +Initial chart. Not yet published (pre-1.0; the values shape may still change). + +### Added + +- One-replica `StatefulSet` running the upstream daemon-only image + (`ghcr.io/decdn/decdn-node`, `image.tag` or `image.digest` required), non-root, + read-only root filesystem, every capability dropped, `RuntimeDefault` seccomp. +- `PersistentVolumeClaim` data dir. A `prepare` init container installs the node + identity from an operator-provisioned `existingSecret` at `0600`, and the keystore + password into an in-memory volume, never onto the PVC. +- `values.config` mirrors `node.toml`; the chart injects the path and port keys, + refuses secret-bearing keys, and checks the rendered key set against the upstream + config schema (`make lint-helm`). +- `DECDN_RPC_URL` and named passthrough keys via `secretKeyRef` only, never + `envFrom`. +- Public QUIC over a `LoadBalancer`, `NodePort` or `ClusterIP` Service, or `hostPort`. +- Metrics bound `0.0.0.0` in the pod behind a ClusterIP-only Service and a + `NetworkPolicy` that admits only `metrics.networkPolicy.from`; disabling the policy + needs `networkPolicy.allowUnrestrictedMetrics: true`. +- Optional `ServiceMonitor` (`metrics.serviceMonitor`). diff --git a/charts/decdn-node/README.md b/charts/decdn-node/README.md index 3363b8e..9303e24 100644 --- a/charts/decdn-node/README.md +++ b/charts/decdn-node/README.md @@ -33,7 +33,9 @@ image has no CLI. | `Service` (quic) | Public UDP. The type is `LoadBalancer`, `NodePort` or `ClusterIP`, with `externalTrafficPolicy: Local`. | | `Service` (metrics) | ClusterIP only, never public. | | `NetworkPolicy` | Ingress allows QUIC from anywhere and metrics only from `metrics.networkPolicy.from`. Egress is open unless `networkPolicy.egress` is set. | -| `ServiceMonitor` | Optional (`metrics.serviceMonitor.enabled`). | +| `ServiceMonitor` | Optional (`metrics.serviceMonitor.enabled`). Adds the `job`, `region`, `deployment_environment` and `instance` target labels the upstream dashboards and alerts select on. | +| `PrometheusRule` | Optional (`metrics.prometheusRule.enabled`). Upstream's reference alert rules. | +| `ConfigMap` × 4 (dashboards) | Optional (`metrics.grafanaDashboards.enabled`). Upstream's Grafana dashboards, labelled for the Grafana sidecar. | What it does **not** do: @@ -118,8 +120,12 @@ in the daemon. | `config.blockchain.chain_id` | For example, `421614` (Arbitrum Sepolia). | | `config.blockchain.{payment_pool,capacity_bond,slash_judge,content_blacklist}_address` | Non-zero `0x` addresses. | -Take chain IDs and contract addresses from the deCDN ADRs and the deployment manifest for -your chain. Never make them up. Values are validated by `values.schema.json` plus +Take chain IDs and contract addresses from the deployment manifest for your chain. Never +make them up. The quickest correct source is the CLI itself: `decdn config init --chain +arbitrum-sepolia --output /tmp/node.toml` writes the manifest's `[blockchain]` section, +and the Ansible role's mirror of the same manifest is +[`networks.yml`](../../ansible/roles/decdn_node/vars/main/networks.yml). (The chart has +no `network` shortcut yet; the addresses stay explicit in values.) Values are validated by `values.schema.json` plus template guards, so a missing or malformed value fails `helm install` with a message that names it. @@ -141,9 +147,17 @@ config: ``` ```bash +# From a checkout of this repo: helm install decdn-node-1 charts/decdn-node -n decdn -f values-node-1.yaml + +# From the OCI registry, once the first release is published (RELEASING.md): +helm install decdn-node-1 oci://ghcr.io/decdn/charts/decdn-node --version 0.1.0 \ + -n decdn -f values-node-1.yaml ``` +Published charts are signed with cosign; [RELEASING.md](../../RELEASING.md#verifying-a-release) +shows how to verify one. + ## Configuration (`config`) `config` mirrors `node.toml` section for section. The role's variable names map onto it @@ -231,6 +245,39 @@ their own token volume, so they should work with the chart's DECDN_ADMIN_URL=http://127.0.0.1:9191 decdn node health ``` +## Monitoring + +The chart vendors upstream's Grafana dashboards and Prometheus alert rules in +[`files/monitoring/`](files/monitoring/README.md) (re-vendored by +`scripts/sync-monitoring.sh`; a weekly CI job flags drift). With the Prometheus +Operator and a Grafana sidecar (kube-prometheus-stack), turn them on next to the +ServiceMonitor: + +```yaml +metrics: + networkPolicy: + from: + - namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: monitoring}} + serviceMonitor: + enabled: true + deploymentEnvironment: prod # feeds the dashboards' $env variable + labels: {release: kube-prometheus-stack} # whatever your serviceMonitorSelector needs + prometheusRule: + enabled: true + labels: {release: kube-prometheus-stack} # ruleSelector + grafanaDashboards: + enabled: true # label/labelValue must match the sidecar's +``` + +- The alert rules and dashboards select across **every** node Prometheus scrapes, and + the dashboards have fixed uids. Enable `prometheusRule` and `grafanaDashboards` on + **one** release per Prometheus/Grafana, or alerts fire once per release and + dashboards collide. +- `metrics.serviceMonitor.jobLabel` (default `decdn-node`) is the `job` the dashboards + and the `DecdnNodeDown` alert expect. `region` comes from `config.identity.region`. +- Loki and Tempo panels stay empty unless the node's logs and traces reach those + backends; the chart ships neither. + ## Probes and lifecycle - `/metrics` is the only HTTP route suitable for probes; there is no `/health` endpoint. diff --git a/charts/decdn-node/ci/ci-values.yaml b/charts/decdn-node/ci/ci-values.yaml index c1ccc74..e107595 100644 --- a/charts/decdn-node/ci/ci-values.yaml +++ b/charts/decdn-node/ci/ci-values.yaml @@ -26,6 +26,15 @@ metrics: kubernetes.io/metadata.name: monitoring serviceMonitor: enabled: true + deploymentEnvironment: ci + prometheusRule: + enabled: true + ruleLabels: + team: node-ops + # Every vendored rule sets its own severity; it must win over this. + severity: overridden + grafanaDashboards: + enabled: true config: identity: region: DE diff --git a/charts/decdn-node/files/monitoring/README.md b/charts/decdn-node/files/monitoring/README.md new file mode 100644 index 0000000..0860fe7 --- /dev/null +++ b/charts/decdn-node/files/monitoring/README.md @@ -0,0 +1,34 @@ +# Vendored monitoring assets + +Copies of upstream [`decdn/decdn`](https://github.com/decdn/decdn)'s +`monitoring/` directory: four Grafana dashboards and the reference Prometheus alert +rules (companion to upstream's `adr/appendix-observability.md`). `SOURCE` records the +upstream commit and a sha256 per file. Upstream licenses them MIT OR Apache-2.0. + +**Do not edit these files here.** Change them upstream, then re-vendor: + +```bash +scripts/sync-monitoring.sh # default ref origin/main +scripts/sync-monitoring.sh --check # what CI's weekly drift job runs +``` + +## On Kubernetes + +The chart renders them when asked (see the chart README, "Monitoring"): +`metrics.prometheusRule.enabled` creates a `PrometheusRule`, and +`metrics.grafanaDashboards.enabled` creates one sidecar-labelled ConfigMap per +dashboard. `metrics.serviceMonitor` adds the target labels they select on (`job`, +`region`, `deployment_environment`, `instance`). + +## On the Ansible path (Grafana Cloud) + +The `grafana_alloy` role already stamps the labels these assets expect: the node's +metrics carry `job="decdn-node"`, `region` and `deployment_environment`; machine +metrics and logs carry `job="integrations/node_exporter"`. So: + +- **Dashboards:** in Grafana, *Dashboards → New → Import*, upload each `*.json`, and + pick your Prometheus, Loki and Tempo datasources for the `DS_*` variables. Loki and + Tempo panels stay empty unless logs and traces are shipped. +- **Alerts:** load `prometheus-alerts.yml` as a rule group, e.g. with + `mimirtool rules load prometheus-alerts.yml` against your Grafana Cloud Prometheus + endpoint, or through *Alerting → Alert rules → Import*. diff --git a/charts/decdn-node/files/monitoring/SOURCE b/charts/decdn-node/files/monitoring/SOURCE new file mode 100644 index 0000000..b54d95c --- /dev/null +++ b/charts/decdn-node/files/monitoring/SOURCE @@ -0,0 +1,9 @@ +# GENERATED by scripts/sync-monitoring.sh -- do not hand-edit. +upstream: decdn/decdn +commit: 2b3d8e471103f8436d6fd955149a5ae4689dea1a +files: + dashboard-chain.json: sha256:ff74aadb34f42930f11ddbdbed263a097a31f91524a7b13da1d71b1b5c325b1f + dashboard-delivery.json: sha256:d678dd5628e34b0a4eb139deb1329aae8adc3d12324a31a5c2383b769178fae7 + dashboard-node.json: sha256:9f58cb3672abf0d9b63f7fb72bbde402644d7f0df4e1b6509e4bbefed89006a4 + grafana-dashboard.json: sha256:f4af3682f06f67412c986d343868691bd27b460262cb21c61a71628bc6030821 + prometheus-alerts.yml: sha256:69de9f66074ecefafc1962ac225a42927ac5513c2cbfc0719ef593a5e9f2c858 diff --git a/charts/decdn-node/files/monitoring/dashboard-chain.json b/charts/decdn-node/files/monitoring/dashboard-chain.json new file mode 100644 index 0000000..4d981eb --- /dev/null +++ b/charts/decdn-node/files/monitoring/dashboard-chain.json @@ -0,0 +1,2992 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "red", + "name": "deCDN alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [ + "decdn" + ], + "type": "tags" + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "enable": true, + "hide": false, + "iconColor": "purple", + "name": "Node restarts", + "expr": "resets(decdn_node_uptime_seconds{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]) > 0", + "titleFormat": "decdn-node restarted", + "textFormat": "{{instance}}", + "step": "60s", + "useValueForTime": false + } + ] + }, + "description": "Everything that touches the chain: watcher liveness for all five chain-event watchers, the capacity-bond and active-staker registries, seller and buyer payment flow, and the slash-risk signals. Thresholds here match monitoring/prometheus-alerts.yml rather than sitting looser than the pages.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "decdn" + ], + "targetBlank": false, + "title": "deCDN dashboards", + "tooltip": "", + "type": "dashboards", + "url": "" + }, + { + "asDropdown": false, + "icon": "doc", + "includeVars": false, + "keepTime": false, + "tags": [], + "targetBlank": true, + "title": "Operator runbook", + "tooltip": "", + "type": "link", + "url": "https://github.com/decdn/decdn/blob/main/docs/runbook.md" + } + ], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "Chain-event watcher liveness", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Watcher tick age", + "type": "table", + "description": "Age of each watcher's last successful poll tick, stamped every tick including idle ones, so this is a positive liveness signal rather than an absence of errors. Built with label_replace on the raw selector because `time() - x` drops __name__. Red at 180s matches the stalled-watcher alerts. A watcher that has never ticked reads 0 and is filtered out by the `> 0` guard rather than showing as infinitely stale.", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "time() - label_replace({__name__=~\"decdn_.+_watcher_last_tick_timestamp_seconds\",deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"} > 0, \"watcher\", \"$1\", \"__name__\", \"decdn_(.+)_watcher_last_tick_timestamp_seconds\")", + "refId": "A", + "instant": true, + "range": false, + "format": "table" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Tick age" + }, + "properties": [ + { + "id": "unit", + "value": "s" + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-background" + } + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 120 + }, + { + "color": "red", + "value": 180 + } + ] + } + } + ] + } + ] + }, + "options": { + "cellHeight": "sm", + "showHeader": true, + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + } + }, + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "__name__": true, + "job": true, + "service_name": true, + "service_namespace": true, + "service_instance_id": true, + "deployment_environment": true + }, + "indexByName": { + "watcher": 0, + "instance": 1, + "region": 2, + "Value": 3 + }, + "renameByName": { + "Value": "Tick age", + "watcher": "Watcher", + "instance": "Node", + "region": "Region" + } + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "field": "Tick age", + "desc": true + } + ] + } + } + ], + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 2 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Watcher downtime", + "type": "timeseries", + "description": "True time spent in error/backoff. Reads 0 for the whole life of an established cycle, and i64::MAX on a poisoned internal lock — the safe direction. Prefer this over the restart-rate below: the restart counter is edge-triggered through a lock whose update is skipped on poisoning, so it can freeze silently.", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "decdn_slash_watcher_down_seconds{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}", + "refId": "A", + "legendFormat": "slash — {{instance}}" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "decdn_staker_set_watcher_down_seconds{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}", + "refId": "B", + "legendFormat": "staker_set — {{instance}}" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "decdn_blacklist_watcher_down_seconds{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}", + "refId": "C", + "legendFormat": "blacklist — {{instance}}" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "decdn_settlement_watcher_down_seconds{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}", + "refId": "D", + "legendFormat": "settlement — {{instance}}" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "decdn_fee_shares_watcher_down_seconds{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}", + "refId": "E", + "legendFormat": "fee_shares — {{instance}}" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 3 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Watcher restarts", + "type": "timeseries", + "description": "Edge-triggered: one increment per healthy-to-error transition, not per backoff iteration.", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_slash_watcher_restarts_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "slash" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_staker_set_watcher_restarts_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "B", + "legendFormat": "staker_set" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_blacklist_watcher_restarts_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "C", + "legendFormat": "blacklist" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_settlement_watcher_restarts_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "D", + "legendFormat": "settlement" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_fee_shares_watcher_restarts_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "E", + "legendFormat": "fee_shares" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 10 + }, + "id": 4 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Watcher task panics", + "type": "timeseries", + "description": "Any non-zero value is a bug, raised by a Drop guard in resumable_watcher::run.", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_slash_watcher_task_panicked_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "slash" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_staker_set_watcher_task_panicked_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "B", + "legendFormat": "staker_set" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_blacklist_watcher_task_panicked_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "C", + "legendFormat": "blacklist" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_settlement_watcher_task_panicked_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "D", + "legendFormat": "settlement" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_fee_shares_watcher_task_panicked_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "E", + "legendFormat": "fee_shares" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 10 + }, + "id": 5 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Watcher persistence and chain RPC", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_watcher_persist_failures_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "watcher persist failures" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "min(decdn_rpc_healthy{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"})", + "refId": "B", + "legendFormat": "min rpc healthy" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "sum", + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 10 + }, + "id": 6 + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 7, + "panels": [], + "title": "Registry and staking", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Active staker set", + "type": "timeseries", + "description": "Each node's cached view of the active-staker set. Divergence between nodes means one of them is working from stale chain state.", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "decdn_staker_set_active_count{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}", + "refId": "A", + "legendFormat": "{{instance}}" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 18 + }, + "id": 8 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Capacity-bond registry resync age", + "type": "timeseries", + "description": "Time since the last successful getRegisteredNodes enumeration. The `> 0` guard drops a node that has not bootstrapped yet rather than reporting it as infinitely stale.", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "time() - (decdn_capacity_bond_registry_last_resync_timestamp_seconds{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"} > 0)", + "refId": "A", + "legendFormat": "{{instance}}" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1800 + }, + { + "color": "red", + "value": 3600 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 18 + }, + "id": 9 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Registry failures", + "type": "timeseries", + "description": "A dropped nodeIdOf lookup means a Reinstated or UnbondingRequested event never landed, so the cached active set has drifted from chain state for that operator until the next full resync.", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_capacity_bond_registry_resync_failures_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "resync failures" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_staker_set_watcher_resolve_failures_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "B", + "legendFormat": "nodeIdOf resolve failures" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "yellow", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 18 + }, + "id": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Active nodes by country", + "type": "geomap", + "description": "Active stakers per declared region (ADR 030), from each selected node's CapacityBond registry. Every node holds the whole network, so the query takes the max across nodes, not the sum. A node with a frozen watcher keeps its last view, and max can hold a departed region on the map: check watcher downtime above. A code absent from Grafana's country gazetteer has no location and is not drawn. In current Grafana these are the user-assigned codes (AA, QM–QZ, XA–XZ, ZZ) plus XK and SS. The table to the right lists every region, including these.", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "max by (node_region) (decdn_staker_set_active_by_region{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"})", + "refId": "A", + "instant": true, + "range": false, + "format": "table", + "legendFormat": "{{node_region}}" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-BlYlRd" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short", + "mappings": [] + }, + "overrides": [] + }, + "options": { + "view": { + "id": "zero", + "lat": 20, + "lon": 0, + "zoom": 1, + "allLayers": true + }, + "controls": { + "showZoom": true, + "mouseWheelZoom": false, + "showAttribution": true + }, + "basemap": { + "type": "default", + "name": "Basemap", + "config": {} + }, + "layers": [ + { + "type": "markers", + "name": "Active nodes", + "tooltip": true, + "location": { + "mode": "lookup", + "lookup": "node_region", + "gazetteer": "public/gazetteer/countries.json" + }, + "config": { + "showLegend": true, + "style": { + "size": { + "field": "Value", + "fixed": 5, + "min": 6, + "max": 30 + }, + "color": { + "field": "Value" + }, + "opacity": 0.6, + "symbol": { + "mode": "fixed", + "fixed": "img/icons/marker/circle.svg" + }, + "text": { + "mode": "field", + "field": "Value", + "fixed": "" + }, + "textConfig": { + "fontSize": 11, + "offsetX": 0, + "offsetY": 0, + "textAlign": "center", + "textBaseline": "middle" + } + } + } + } + ], + "tooltip": { + "mode": "details" + } + }, + "gridPos": { + "h": 12, + "w": 14, + "x": 0, + "y": 25 + }, + "id": 27 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Active nodes per region", + "type": "table", + "description": "Every region the map reads, including codes the map cannot place.", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "max by (node_region) (decdn_staker_set_active_by_region{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"})", + "refId": "A", + "instant": true, + "range": false, + "format": "table" + } + ], + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true + }, + "renameByName": { + "node_region": "Region", + "Value": "Active nodes" + } + } + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "cellHeight": "sm", + "footer": { + "show": false, + "reducer": [ + "sum" + ], + "countRows": false, + "fields": "" + }, + "showHeader": true, + "sortBy": [ + { + "displayName": "Active nodes", + "desc": true + } + ] + }, + "gridPos": { + "h": 12, + "w": 5, + "x": 14, + "y": 25 + }, + "id": 29 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Active nodes without a region", + "type": "stat", + "description": "Active stakers whose on-chain regionHint is empty or not an accepted region code. These nodes are absent from the map and the table. Non-zero means an operator should run `decdn node update-region`; the panel follows on the next watcher tick.", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "max(decdn_staker_set_active_unknown_region{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"})", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 1 + } + ] + }, + "unit": "short", + "mappings": [] + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "auto", + "wideLayout": true, + "percentChangeColorMode": "standard", + "showPercentChange": false, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "gridPos": { + "h": 12, + "w": 5, + "x": 19, + "y": 25 + }, + "id": 28 + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 37 + }, + "id": 11, + "panels": [], + "title": "Slash safety", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Slashes detected", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_slashes_detected_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "{{instance}}" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 38 + }, + "id": 12 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Blacklist enforcement failures", + "type": "timeseries", + "description": "Distinct hashes a batched re-scope could not re-verify or evict this pass. While non-zero the deny-set is not fully enforced and a blacklisted blob may still be servable.", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_blacklist_enforcement_failures_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "{{instance}}" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 38 + }, + "id": 13 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Probe-hold budget", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "decdn_probe_hold_slots_used{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"} / clamp_min(decdn_probe_hold_slots_max{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}, 1)", + "refId": "A", + "legendFormat": "{{instance}}" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "percentunit", + "min": 0, + "max": 1 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 38 + }, + "id": 14 + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 45 + }, + "id": 15, + "panels": [], + "title": "Payments — seller side", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Lanes open", + "type": "timeseries", + "description": "Distinct (pool, signer, provider) keys carrying unredeemed vouchers.", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "decdn_lanes_open{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}", + "refId": "A", + "legendFormat": "{{instance}}" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 46 + }, + "id": 16 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Unredeemed value", + "type": "timeseries", + "description": "Raw 6-decimal USDC base units scaled to dollars. Sums owed minus paid across accepted vouchers, refreshed once per redeemer self-tick.", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "decdn_unredeemed_usdc{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"} / 1e6", + "refId": "A", + "legendFormat": "{{instance}}" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "currencyUSD" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 46 + }, + "id": 17 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Pool deposits backing this node", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "decdn_pool_deposit_usdc{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"} / 1e6", + "refId": "A", + "legendFormat": "{{instance}}" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "currencyUSD" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 46 + }, + "id": 18 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Redemption failures", + "type": "timeseries", + "description": "Earned USDC that did not make it on chain. The On-chain transaction outcomes panel splits the revert, send, receipt and timeout cases.", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_redemption_failures_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "redemption failures" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_redemption_skipped_insolvent_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "B", + "legendFormat": "skipped — pool insolvent" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_redeem_hints_dropped_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "C", + "legendFormat": "redeem hints dropped" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_redemption_reconciled_skip_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "D", + "legendFormat": "skipped — already reconciled" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_redemption_reconcile_failures_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "E", + "legendFormat": "reconcile read failed (fail-open)" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 53 + }, + "id": 19 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Voucher and ledger integrity", + "type": "timeseries", + "description": "A nonce gap means a voucher in the sequence was never seen. Dropped or failed receipt writes are lost audit-log rows, not lost revenue.", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_voucher_nonce_gaps_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "voucher nonce gaps" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_lane_flush_failures_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "B", + "legendFormat": "lane flush failures" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_receipt_writes_dropped_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "C", + "legendFormat": "receipt writes dropped" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_receipt_write_failures_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "D", + "legendFormat": "receipt writes failed" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 53 + }, + "id": 20 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Settlement watcher freshness", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "time() - (decdn_settlement_watcher_last_tick_timestamp_seconds{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"} > 0)", + "refId": "A", + "legendFormat": "{{instance}}" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 120 + }, + { + "color": "red", + "value": 180 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 53 + }, + "id": 21 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "On-chain transaction outcomes", + "type": "timeseries", + "description": "Every node transaction counts once by outcome. receipt failed and timeout were issued and can still mine — their tx hash is in the warn! line.", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_onchain_tx_landed_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "landed" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_onchain_tx_reverted_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "B", + "legendFormat": "reverted" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_onchain_tx_send_failed_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "C", + "legendFormat": "send failed" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_onchain_tx_receipt_failed_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "D", + "legendFormat": "receipt failed" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_onchain_tx_timeout_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "E", + "legendFormat": "timeout" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 60 + }, + "id": 30 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Vouchers, redemptions and grace closes", + "type": "timeseries", + "description": "vouchers accepted counts signed vouchers on inbound streams (PayWord reveals are not vouchers). vouchers redeemed counts lane claims that landed on chain — one cumulative voucher per lane. A grace close is a pool its owner is closing while this node still holds unredeemed vouchers on it.", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_vouchers_received_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "vouchers accepted" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_pool_redemptions_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "B", + "legendFormat": "vouchers redeemed" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_pool_grace_closes_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "C", + "legendFormat": "grace closes with unredeemed vouchers" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 60 + }, + "id": 31 + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 68 + }, + "id": 22, + "panels": [], + "title": "Payments — buyer side", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Top-ups", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_buyer_topup_ok_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "top-up ok" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_buyer_topup_failure_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "B", + "legendFormat": "top-up failed" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 69 + }, + "id": 23 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Pool-open failures by cause", + "type": "timeseries", + "description": "Sibling counters for the openChannel tx. The aggregate decdn_node_pull_pool_open_failures_total on the delivery dashboard is larger: it also counts store and expired-reclaim causes that never reach a transaction.", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_pool_open_failures_insufficient_deposit_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "insufficient deposit" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_pool_open_failures_contract_revert_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "B", + "legendFormat": "contract revert" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_pool_open_failures_rpc_error_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "C", + "legendFormat": "rpc error" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 60, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 69 + }, + "id": 24 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Buyer ledger integrity", + "type": "timeseries", + "description": "A skipped undecodable record is a buyer-pool row silently omitted from an otherwise successful store hydration, counted once per row per load.", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_buyer_reclaim_failures_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "reclaim failures" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_buyer_pool_store_skipped_undecodable_records_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "B", + "legendFormat": "skipped undecodable records" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 69 + }, + "id": 25 + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "title": "Settlement and chain-RPC log lines", + "type": "logs", + "description": "The redeemMany revert / send-failed / receipt-failed / receipt-timeout cluster, plus the admit-path getPool and getAuthorization failures that make the node refuse revenue.", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "expr": "{unit=\"decdn-node.service\", instance=~\"$instance\"} | json lvl=\"level\", tgt=\"target\", msg=\"fields.message\" |~ \"(?i)redeemmany|redeem|topup|top-up|getpool|getauthorization|slash\" | lvl=~\"WARN|ERROR\" | line_format \"{{.instance}} {{.lvl}} {{.tgt}} :: {{.msg}}\"", + "refId": "A", + "queryType": "range" + } + ], + "options": { + "dedupStrategy": "none", + "enableInfiniteScrolling": true, + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 76 + }, + "id": 26 + } + ], + "preload": false, + "refresh": "30s", + "schemaVersion": 39, + "tags": [ + "decdn", + "chain", + "payments" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "grafanacloud-decdn-prom", + "value": "grafanacloud-prom" + }, + "hide": 0, + "includeAll": false, + "label": "Metrics", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": "grafanacloud-decdn-logs", + "value": "grafanacloud-logs" + }, + "hide": 0, + "includeAll": false, + "label": "Logs", + "multi": false, + "name": "DS_LOKI", + "options": [], + "query": "loki", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": null, + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(decdn_node_uptime_seconds, deployment_environment)", + "hide": 0, + "includeAll": false, + "label": "Environment", + "multi": false, + "name": "env", + "options": [], + "refresh": 2, + "query": { + "qryType": 1, + "query": "label_values(decdn_node_uptime_seconds, deployment_environment)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "allValue": null, + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(decdn_node_uptime_seconds{deployment_environment=\"$env\"}, region)", + "hide": 0, + "includeAll": true, + "label": "Region", + "multi": true, + "name": "region", + "options": [], + "refresh": 2, + "query": { + "qryType": 1, + "query": "label_values(decdn_node_uptime_seconds{deployment_environment=\"$env\"}, region)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "allValue": null, + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(decdn_node_uptime_seconds{deployment_environment=\"$env\",region=~\"$region\"}, instance)", + "hide": 0, + "includeAll": true, + "label": "Node", + "multi": true, + "name": "instance", + "options": [], + "refresh": 2, + "query": { + "qryType": 1, + "query": "label_values(decdn_node_uptime_seconds{deployment_environment=\"$env\",region=~\"$region\"}, instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "deCDN — chain, payments and slash safety", + "uid": "decdn-chain", + "version": 1, + "weekStart": "" +} diff --git a/charts/decdn-node/files/monitoring/dashboard-delivery.json b/charts/decdn-node/files/monitoring/dashboard-delivery.json new file mode 100644 index 0000000..5365cd3 --- /dev/null +++ b/charts/decdn-node/files/monitoring/dashboard-delivery.json @@ -0,0 +1,3560 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "red", + "name": "deCDN alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [ + "decdn" + ], + "type": "tags" + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "enable": true, + "hide": false, + "iconColor": "purple", + "name": "Node restarts", + "expr": "resets(decdn_node_uptime_seconds{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]) > 0", + "titleFormat": "decdn-node restarted", + "textFormat": "{{instance}}", + "step": "60s", + "useValueForTime": false + } + ] + }, + "description": "The bytes path end to end: the serve leg where this fleet is paid, the pull leg where it pays, and the cache and origin behind both. Every refusal and failure reason the exporter records, one series per reason.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "decdn" + ], + "targetBlank": false, + "title": "deCDN dashboards", + "tooltip": "", + "type": "dashboards", + "url": "" + }, + { + "asDropdown": false, + "icon": "doc", + "includeVars": false, + "keepTime": false, + "tags": [], + "targetBlank": true, + "title": "Operator runbook", + "tooltip": "", + "type": "link", + "url": "https://github.com/decdn/decdn/blob/main/docs/runbook.md" + } + ], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "Serve leg \u2014 this node as the seller", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Active streams by direction", + "type": "timeseries", + "description": "inbound is content this node is serving out; outbound is its own cache-miss pull leg. The only decdn_ series with a domain label.", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by (direction) (decdn_streams_active{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"})", + "refId": "A", + "legendFormat": "{{direction}}" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 1 + }, + "id": 2 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Bytes served, received and pulled through", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_cache_bytes_returned_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "served to callers" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_cache_pull_through_bytes_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "B", + "legendFormat": "pulled from origin" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_bytes_served_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "C", + "legendFormat": "served on the wire" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_bytes_received_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "D", + "legendFormat": "received from peers" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 1 + }, + "id": 3, + "description": "served on the wire and received from peers count cdn/client/v1 payload bytes; served to callers and pulled from origin are the cache engine's own view." + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Load shedding", + "type": "timeseries", + "description": "Paired with the load-shed refusal counters below. Pressure active with no refusals means the ceiling is biting before it has to turn work away.", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "max(decdn_load_shed_egress_bps{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"})", + "refId": "A", + "legendFormat": "egress ceiling" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 1 + }, + "id": 4 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Serve refusals by reason", + "type": "timeseries", + "description": "All of these collapse to one NotFound wire code on the client side. insufficient deposit is fed by three separate guards (the pre-spend miss floor, the window tier's ceiling and the direct-serve ceiling) and cannot attribute a spike to one \u2014 the node's throttled warn! line does. Sustained insufficient deposit means either clients running dry, or this node's chain watcher lagging an on-chain top-up and turning funded clients away.", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_serve_stream_rejected_cache_miss_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "cache miss" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_serve_stream_rejected_insufficient_deposit_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "B", + "legendFormat": "insufficient deposit" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_serve_stream_rejected_unknown_lane_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "C", + "legendFormat": "unknown lane" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_serve_stream_rejected_owner_mismatch_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "D", + "legendFormat": "owner mismatch" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_serve_stream_rejected_pool_unconfirmed_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "E", + "legendFormat": "pool unconfirmed" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_serve_stream_rejected_signer_cap_exhausted_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "F", + "legendFormat": "signer cap exhausted" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_serve_stream_rejected_signer_floor_at_cap_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "G", + "legendFormat": "signer floor at cap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_serve_stream_rejected_evicted_since_probe_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "H", + "legendFormat": "evicted since probe" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_serve_stream_rejected_range_not_satisfiable_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "I", + "legendFormat": "range not satisfiable" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_serve_stream_rejected_hash_denied_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "J", + "legendFormat": "hash denied" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_serve_stream_rejected_chain_hash_denied_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "K", + "legendFormat": "chain hash denied" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_serve_stream_rejected_chain_stale_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "L", + "legendFormat": "chain stale" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_serve_stream_rejected_origin_denied_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "M", + "legendFormat": "origin denied" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_serve_stream_rejected_foreign_declined_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "N", + "legendFormat": "foreign declined" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_serve_stream_rejected_load_shed_hit_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "O", + "legendFormat": "load shed (hit)" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_serve_stream_rejected_load_shed_miss_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "P", + "legendFormat": "load shed (miss)" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_serve_stream_rejected_internal_error_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "Q", + "legendFormat": "internal error" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_serve_stream_rejected_bad_binding_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "R", + "legendFormat": "bad binding" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 60, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 5 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Stream outcomes by direction", + "type": "timeseries", + "description": "Every ended paid stream counts once as completed or failed. inbound is this node serving; outbound is a stream this node opened to an upstream candidate, or one assembled range on the streaming miss path. failed includes routine outcomes \u2014 a cache-miss refusal, a client that closes early, and the free header handshake a downstream node opens and closes \u2014 so read the refusal and pull-outcome panels for health. Mid-stream stops name the funding cause.", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by (direction) (rate(decdn_streams_completed_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "completed {{direction}}" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by (direction) (rate(decdn_streams_failed_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "B", + "legendFormat": "failed {{direction}}" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_serve_stream_midstream_pool_exhausted_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "C", + "legendFormat": "stopped mid-stream: pool exhausted" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_serve_stream_midstream_signer_cap_exhausted_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "D", + "legendFormat": "stopped mid-stream: signer cap exhausted" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 27 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Node-side faults \u2014 every one of these costs a delivery", + "type": "timeseries", + "description": "The first two are latent-bug reports, not degradation: the node's own byte accounting or encoder refused to proceed. Alert on > 0 and file it. Takedown terminations are correct behaviour, shown here so they are not mistaken for faults.", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_serve_stream_node_fault_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "stream node fault" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_serve_frame_accounting_fault_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "B", + "legendFormat": "frame accounting fault" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_serve_stream_midstream_signer_cap_exhausted_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "C", + "legendFormat": "midstream signer cap exhausted" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_serve_stream_terminated_takedown_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "D", + "legendFormat": "terminated by takedown" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_client_idle_close_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "E", + "legendFormat": "idle close" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_probe_read_faults_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "F", + "legendFormat": "probe read faults" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 25 + }, + "id": 6 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Connection dispatch", + "type": "timeseries", + "description": "Sibling counters, not a labelled reason split: a global-ceiling rejection and a per-source one have unrelated remedies.", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(decdn_dispatch_in_flight{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"})", + "refId": "A", + "legendFormat": "in flight" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_dispatch_rejected_global_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "B", + "legendFormat": "rejected (global ceiling)" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_dispatch_rejected_per_source_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "C", + "legendFormat": "rejected (per source)" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_dispatch_per_source_skipped_no_addr_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "D", + "legendFormat": "per-source skipped (no addr)" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "last", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 25 + }, + "id": 7 + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 33 + }, + "id": 8, + "panels": [], + "title": "Pull leg \u2014 this node as the paying client", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Attempts vs success", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_node_pull_attempts_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "attempts" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_node_pull_success_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "B", + "legendFormat": "success" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 34 + }, + "id": 9 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Pull outcomes by failure reason", + "type": "timeseries", + "description": "corruption is the one that costs money for nothing: USDC was spent on bytes that failed bao verification. The peer it came from is only recoverable from the decdn::reputation log target.", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_node_pull_no_providers_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "no providers" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_node_pull_unreachable_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "B", + "legendFormat": "unreachable" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_node_pull_timeout_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "C", + "legendFormat": "timeout" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_node_pull_corruption_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "D", + "legendFormat": "corruption" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_node_pull_stalled_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "E", + "legendFormat": "stalled" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_node_pull_refused_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "F", + "legendFormat": "refused" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_node_pull_refused_unattributable_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "G", + "legendFormat": "refused (unattributable)" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_node_pull_too_large_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "H", + "legendFormat": "too large" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_node_pull_rate_above_ceiling_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "I", + "legendFormat": "rate above ceiling" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_node_pull_voucher_rejected_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "J", + "legendFormat": "voucher rejected" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_node_pull_local_fault_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "K", + "legendFormat": "local fault" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_node_pull_abandon_drain_timeout_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "L", + "legendFormat": "abandon drain timeout" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_node_pull_pool_open_failures_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "M", + "legendFormat": "pool open failure" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 60, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 34 + }, + "id": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Pull-through health", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_node_pull_through_timeouts_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "timeouts" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_node_pull_through_errors_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "B", + "legendFormat": "errors" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_node_pull_through_window_paused_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "C", + "legendFormat": "window paused" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_node_pull_through_client_abandoned_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "D", + "legendFormat": "client abandoned" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_node_pull_through_upstream_verify_failed_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "E", + "legendFormat": "upstream verify failed" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 42 + }, + "id": 11 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Pool plumbing on the pull leg", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_node_pull_pool_open_failures_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "pool open failures" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_node_pull_pool_open_pending_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "B", + "legendFormat": "pool open pending" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_node_pull_pool_wedged_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "C", + "legendFormat": "pool wedged" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_node_pull_reactive_topup_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "D", + "legendFormat": "reactive top-up" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_node_pull_reactive_topup_refused_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "E", + "legendFormat": "reactive top-up refused" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 42 + }, + "id": 12 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Progress persistence and upstream pressure", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_node_pull_progress_dropped_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "progress dropped" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_node_pull_progress_persist_failures_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "B", + "legendFormat": "progress persist failures" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_node_pull_progress_superseded_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "C", + "legendFormat": "progress superseded" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_node_upstream_rate_limited_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "D", + "legendFormat": "upstream rate limited" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_node_region_latency_penalty_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "E", + "legendFormat": "region latency penalty" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 42 + }, + "id": 13 + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 49 + }, + "id": 14, + "panels": [], + "title": "Cache", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Cache utilisation", + "type": "bargauge", + "description": "Configured cache_size_mb is the denominator, not the disk. Check the node drilldown for the filesystem behind it.", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "decdn_cache_bytes{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"} / decdn_cache_size_limit_bytes{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}", + "refId": "A", + "legendFormat": "{{instance}}" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.85 + }, + { + "color": "red", + "value": 0.95 + } + ] + }, + "unit": "percentunit", + "min": 0, + "max": 1 + }, + "overrides": [] + }, + "options": { + "displayMode": "gradient", + "orientation": "horizontal", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "showUnfilled": true, + "sizing": "auto", + "valueMode": "auto", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 50 + }, + "id": 15 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Cache size", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "decdn_cache_bytes{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}", + "refId": "A", + "legendFormat": "{{instance}} used" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "decdn_cache_size_limit_bytes{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}", + "refId": "B", + "legendFormat": "{{instance}} limit" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 50 + }, + "id": 16 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Serve hit ratio", + "type": "timeseries", + "description": "Serve-path hit rate: (hit + partial hit) / (hit + partial hit + miss), decided at the blob-availability gate before any fill tier or load-shed admission. This is NOT decdn_cache_hits_total \u2014 that one meters CacheEngine::get, the whole-blob buffered read the paid serve path never calls, so it sits at zero on a node that only serves paying clients however full its store is. Deliberately unguarded division: an idle node renders a gap rather than a fabricated 0%.", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "(\n sum by (instance) (rate(decdn_serve_cache_hit_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n + sum by (instance) (rate(decdn_serve_cache_partial_hit_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n)\n/\n(\n sum by (instance) (rate(decdn_serve_cache_hit_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n + sum by (instance) (rate(decdn_serve_cache_partial_hit_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n + sum by (instance) (rate(decdn_serve_cache_miss_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n)", + "refId": "A", + "legendFormat": "{{instance}}" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by (instance) (rate(decdn_serve_cache_partial_hit_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n/\n(\n sum by (instance) (rate(decdn_serve_cache_hit_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n + sum by (instance) (rate(decdn_serve_cache_partial_hit_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n + sum by (instance) (rate(decdn_serve_cache_miss_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n)", + "refId": "B", + "legendFormat": "{{instance}} \u2014 partial share" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 0.5 + }, + { + "color": "green", + "value": 0.8 + } + ] + }, + "unit": "percentunit", + "min": 0, + "max": 1 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 50 + }, + "id": 17 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Evictions and GC", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_cache_evictions_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "evictions" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_cache_evicted_operator_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "B", + "legendFormat": "operator evictions" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_cache_evictions_starved_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "C", + "legendFormat": "evictions starved" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_cache_gc_runs_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "D", + "legendFormat": "gc runs" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 57 + }, + "id": 18 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Bytes reclaimed", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_cache_evictions_bytes_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "evicted" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_cache_gc_bytes_reclaimed_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "B", + "legendFormat": "gc reclaimed" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 57 + }, + "id": 19 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Cache integrity \u2014 non-zero needs a human", + "type": "timeseries", + "description": "A quarantined hash means disk rot or tampering. A tag-drop failure leaves bytes GC-protected and is not retried. A poisoned mutex is process-local and does not clear on its own \u2014 see the runbook before restarting.", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_cache_held_corruption_quarantined_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "corruption quarantined" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_cache_inflight_mutex_poisoned_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "B", + "legendFormat": "inflight mutex poisoned" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_cache_tag_drop_failures_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "C", + "legendFormat": "tag drop failures" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_cache_size_measure_failures_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "D", + "legendFormat": "size measure failures" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_probe_post_eviction_failures_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "E", + "legendFormat": "post-eviction failures" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 57 + }, + "id": 20 + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 64 + }, + "id": 21, + "panels": [], + "title": "Origin and warming", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Origin fetches and fallbacks", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_cache_origin_fetches_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "fetches" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_cache_origin_fallback_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "B", + "legendFormat": "fallback" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_cache_origin_retry_exhausted_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "C", + "legendFormat": "retry budget exhausted" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_cache_origin_probe_failures_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "D", + "legendFormat": "probe failures" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_cache_origin_enumerate_failures_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "E", + "legendFormat": "enumerate failures" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 65 + }, + "id": 22 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Origin circuit breaker", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_cache_circuit_breaker_trips_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "trips" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_cache_circuit_breaker_short_circuits_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "B", + "legendFormat": "short circuits" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_cache_circuit_breaker_recoveries_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "C", + "legendFormat": "recoveries" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 65 + }, + "id": 23 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Stream-while-store and warming credits", + "type": "timeseries", + "description": "Dropped credits never reached the allowance ledger \u2014 the queue was full or the aggregator was gone. Always conservative, never over-credits.", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_local_outboard_serves_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "local outboard serves" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_warming_credits_applied_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "B", + "legendFormat": "warming credits applied" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_warming_credits_dropped_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "C", + "legendFormat": "warming credits dropped" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 65 + }, + "id": 24 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Origin directory", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(decdn_origin_directory_cache_size{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"})", + "refId": "A", + "legendFormat": "cached entries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_origin_directory_get_origins_failures_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "B", + "legendFormat": "getOrigins failures" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(decdn_cache_pinned_count{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"})", + "refId": "C", + "legendFormat": "pinned hashes" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 72 + }, + "id": 25 + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "title": "Delivery money-path log lines", + "type": "logs", + "description": "The mid-stream PoolExhausted and SignerCapExhausted warnings are the only place the pool-level and signer-level distinction is observable; neither has its own counter on the terminating path.", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "expr": "{unit=\"decdn-node.service\", instance=~\"$instance\"} | json lvl=\"level\", tgt=\"target\", msg=\"fields.message\" |~ \"(?i)poolexhausted|signercapexhausted|bao|quarantin|zero-length frame\" | line_format \"{{.instance}} {{.lvl}} {{.tgt}} :: {{.msg}}\"", + "refId": "A", + "queryType": "range" + } + ], + "options": { + "dedupStrategy": "none", + "enableInfiniteScrolling": true, + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 72 + }, + "id": 26 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Unattributed stream failures", + "type": "timeseries", + "description": "Seller-leg stream failures that no reason counter claims: rate(decdn_streams_failed_total{direction=\"inbound\"}) minus the 22 siblings that each end exactly one inbound stream \u2014 every serve_stream_rejected_* reason, the two midstream exhaustion stops, the node fault and the takedown termination. decdn_serve_economics_refused_total is deliberately NOT subtracted: it is a buyer-leg gate that declines to buy from a candidate upstream, and ends no inbound stream. A standing positive floor is expected and healthy \u2014 a client that closes early, a peer-attributable stream error, and the free header handshake a downstream node's miss pull opens and closes after the response all count as failures by design and have no reason counter. Read the SHAPE, not the level: a step change means a failure mode arrived that no counter names, and the refusal breakdown will not show it. Goes negative only if a reason counter fires without ending a stream.", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_streams_failed_total{direction=\"inbound\",deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_bad_binding_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_cache_miss_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_chain_hash_denied_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_chain_stale_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_evicted_since_probe_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_foreign_declined_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_hash_denied_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_insufficient_deposit_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_internal_error_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_load_shed_hit_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_load_shed_miss_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_origin_denied_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_owner_mismatch_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_pool_unconfirmed_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_range_not_satisfiable_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_signer_cap_exhausted_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_signer_floor_at_cap_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_unknown_lane_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_midstream_pool_exhausted_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_midstream_signer_cap_exhausted_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_node_fault_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_terminated_takedown_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "no reason counter" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 79 + }, + "id": 9003 + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 87 + }, + "id": 9004, + "panels": [], + "title": "Refuse-to-serve & load-shedding (ADR 041)", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Serve-economics buy refusals", + "type": "timeseries", + "description": "ADR 041 buy-ceiling refusals on the cache-miss leg, split by regime. `warming` = the source still had allowance so the ceiling was the market price max(sell, amortized) and the market is above it (no local fix). `amortized` = the source's warming allowance is spent (or warming is off) so the ceiling is the grief-proof floor and this node's sell rate / margin config is too tight. The refusal signs the client NotFound, so it never reaches the wire and is visible only here.", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_serve_economics_refused_warming_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "warming (market too hot)" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_serve_economics_refused_amortized_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "B", + "legendFormat": "amortized (economics too tight)" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 88 + }, + "id": 9005 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Warming allowance blocking", + "type": "timeseries", + "description": "ADR 041 per-source warming allowance being griefed down. `sources blocked` is how many upstream sources currently hold no allowance (gauge). `buys downgraded /s` is the rate of above-floor speculative buys forced to the amortized floor because a source's allowance was spent. Both stay at zero when warming is unconstrained; a sustained rate with a rising blocked count is real per-source loss, not a bug.", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "max(decdn_warming_sources_blocked{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"})", + "refId": "A", + "legendFormat": "sources blocked" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_warming_speculative_blocked_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "B", + "legendFormat": "buys downgraded /s" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 88 + }, + "id": 9006 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Load-shed refusals by reason", + "type": "timeseries", + "description": "New serves shed under overload, split by cause \u2014 each has a different remedy. `node at capacity` = concurrency above the high-water mark (add capacity / raise max_concurrent_serves_high). `egress saturated` = measured egress hit the budget (raise egress_budget_mbps / add bandwidth). `client at capacity` = one client held its fair share while pressured (fairness working as designed). Orthogonal to the hit/miss split in the Serve refusals panel.", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_load_shed_refused_node_at_capacity_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "node at capacity" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_load_shed_refused_egress_saturated_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "B", + "legendFormat": "egress saturated" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_load_shed_refused_client_at_capacity_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "C", + "legendFormat": "client at capacity" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 88 + }, + "id": 9007 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Load-shed concurrency & pressure", + "type": "timeseries", + "description": "`streams in flight` is node-wide serves in flight, sampled from the shed controller \u2014 read against max_concurrent_serves_high for headroom. `pressure active` is 1 while the policy considers the node pressured. The refusal counters above only fire once a request is actually shed; this panel shows the approach.", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "max(decdn_load_shed_streams_in_flight{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"})", + "refId": "A", + "legendFormat": "streams in flight" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "max(decdn_load_shed_pressure_active{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"})", + "refId": "B", + "legendFormat": "pressure active (0/1)" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 95 + }, + "id": 9008 + } + ], + "preload": false, + "refresh": "30s", + "schemaVersion": 39, + "tags": [ + "decdn", + "delivery" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "grafanacloud-decdn-prom", + "value": "grafanacloud-prom" + }, + "hide": 0, + "includeAll": false, + "label": "Metrics", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": "grafanacloud-decdn-logs", + "value": "grafanacloud-logs" + }, + "hide": 0, + "includeAll": false, + "label": "Logs", + "multi": false, + "name": "DS_LOKI", + "options": [], + "query": "loki", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": null, + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(decdn_node_uptime_seconds, deployment_environment)", + "hide": 0, + "includeAll": false, + "label": "Environment", + "multi": false, + "name": "env", + "options": [], + "refresh": 2, + "query": { + "qryType": 1, + "query": "label_values(decdn_node_uptime_seconds, deployment_environment)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "allValue": null, + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(decdn_node_uptime_seconds{deployment_environment=\"$env\"}, region)", + "hide": 0, + "includeAll": true, + "label": "Region", + "multi": true, + "name": "region", + "options": [], + "refresh": 2, + "query": { + "qryType": 1, + "query": "label_values(decdn_node_uptime_seconds{deployment_environment=\"$env\"}, region)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "allValue": null, + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(decdn_node_uptime_seconds{deployment_environment=\"$env\",region=~\"$region\"}, instance)", + "hide": 0, + "includeAll": true, + "label": "Node", + "multi": true, + "name": "instance", + "options": [], + "refresh": 2, + "query": { + "qryType": 1, + "query": "label_values(decdn_node_uptime_seconds{deployment_environment=\"$env\",region=~\"$region\"}, instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "deCDN \u2014 delivery and cache", + "uid": "decdn-delivery", + "version": 1, + "weekStart": "" +} diff --git a/charts/decdn-node/files/monitoring/dashboard-node.json b/charts/decdn-node/files/monitoring/dashboard-node.json new file mode 100644 index 0000000..abe9a1b --- /dev/null +++ b/charts/decdn-node/files/monitoring/dashboard-node.json @@ -0,0 +1,4412 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "red", + "name": "deCDN alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [ + "decdn" + ], + "type": "tags" + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "enable": true, + "hide": false, + "iconColor": "purple", + "name": "Node restarts", + "expr": "resets(decdn_node_uptime_seconds{instance=~\"$node\"}[$__rate_interval]) > 0", + "titleFormat": "decdn-node restarted", + "textFormat": "{{instance}}", + "step": "60s", + "useValueForTime": false + } + ] + }, + "description": "One node at a time: host resources, process state, the iroh transport underneath every byte, DHT and probe behaviour, and this node's logs and traces. Host and decdn series share the same instance label, so the node selector drives both.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "decdn" + ], + "targetBlank": false, + "title": "deCDN dashboards", + "tooltip": "", + "type": "dashboards", + "url": "" + }, + { + "asDropdown": false, + "icon": "doc", + "includeVars": false, + "keepTime": false, + "tags": [], + "targetBlank": true, + "title": "Operator runbook", + "tooltip": "", + "type": "link", + "url": "https://github.com/decdn/decdn/blob/main/docs/runbook.md" + } + ], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "Host", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "CPU busy", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "100 - (avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\",job=\"integrations/node_exporter\",instance=~\"$node\"}[$__rate_interval])) * 100)", + "refId": "A", + "legendFormat": "{{instance}}" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 75 + }, + { + "color": "red", + "value": 90 + } + ] + }, + "unit": "percent", + "min": 0, + "max": 100 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 1 + }, + "id": 2 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Load average", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "node_load1{job=\"integrations/node_exporter\",instance=~\"$node\"}", + "refId": "A", + "legendFormat": "1m" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "node_load5{job=\"integrations/node_exporter\",instance=~\"$node\"}", + "refId": "B", + "legendFormat": "5m" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "node_load15{job=\"integrations/node_exporter\",instance=~\"$node\"}", + "refId": "C", + "legendFormat": "15m" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 1 + }, + "id": 3 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Memory available", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "node_memory_MemAvailable_bytes{job=\"integrations/node_exporter\",instance=~\"$node\"}", + "refId": "A", + "legendFormat": "available" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "node_memory_MemTotal_bytes{job=\"integrations/node_exporter\",instance=~\"$node\"}", + "refId": "B", + "legendFormat": "total" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 1 + }, + "id": 4 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Root filesystem used", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "1 - (node_filesystem_avail_bytes{mountpoint=\"/\",job=\"integrations/node_exporter\",instance=~\"$node\"} / node_filesystem_size_bytes{mountpoint=\"/\",job=\"integrations/node_exporter\",instance=~\"$node\"})", + "refId": "A", + "legendFormat": "{{instance}}" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.8 + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "percentunit", + "min": 0, + "max": 1 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 8 + }, + "id": 5 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Cache limit against disk", + "type": "timeseries", + "description": "cache_size_mb is configured per node and is not checked against the disk behind it. If the limit line ever approaches the filesystem line, the eviction driver is not what will stop the node filling the disk.", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "decdn_cache_size_limit_bytes{instance=~\"$node\"}", + "refId": "A", + "legendFormat": "configured cache limit" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "decdn_cache_bytes{instance=~\"$node\"}", + "refId": "B", + "legendFormat": "cache in use" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "node_filesystem_size_bytes{mountpoint=\"/\",job=\"integrations/node_exporter\",instance=~\"$node\"}", + "refId": "C", + "legendFormat": "root filesystem" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 8 + }, + "id": 6 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Disk throughput", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(node_disk_read_bytes_total{job=\"integrations/node_exporter\",instance=~\"$node\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "read" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(node_disk_written_bytes_total{job=\"integrations/node_exporter\",instance=~\"$node\"}[$__rate_interval]))", + "refId": "B", + "legendFormat": "written" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 8 + }, + "id": 7 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Network throughput", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(node_network_receive_bytes_total{device!=\"lo\",job=\"integrations/node_exporter\",instance=~\"$node\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "receive" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(node_network_transmit_bytes_total{device!=\"lo\",job=\"integrations/node_exporter\",instance=~\"$node\"}[$__rate_interval]))", + "refId": "B", + "legendFormat": "transmit" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 15 + }, + "id": 8 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "File descriptors", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "node_filefd_allocated{job=\"integrations/node_exporter\",instance=~\"$node\"}", + "refId": "A", + "legendFormat": "allocated" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "node_filefd_maximum{job=\"integrations/node_exporter\",instance=~\"$node\"}", + "refId": "B", + "legendFormat": "maximum" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 15 + }, + "id": 9 + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 10, + "panels": [], + "title": "Process", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Uptime", + "type": "stat", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "decdn_node_uptime_seconds{instance=~\"$node\"}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 900 + } + ] + }, + "unit": "s", + "mappings": [] + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "auto", + "wideLayout": true, + "percentChangeColorMode": "standard", + "showPercentChange": false, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 23 + }, + "id": 11 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Chain RPC", + "type": "stat", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "decdn_rpc_healthy{instance=~\"$node\"}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, + "unit": "short", + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "unreachable", + "index": 0, + "color": "red" + }, + "1": { + "text": "healthy", + "index": 1, + "color": "green" + } + } + } + ] + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "auto", + "wideLayout": true, + "percentChangeColorMode": "standard", + "showPercentChange": false, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 23 + }, + "id": 12 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Connections", + "type": "stat", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "decdn_active_connections{instance=~\"$node\"}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "short", + "mappings": [] + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "auto", + "wideLayout": true, + "percentChangeColorMode": "standard", + "showPercentChange": false, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 23 + }, + "id": 13 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Dispatch in flight", + "type": "stat", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "decdn_dispatch_in_flight{instance=~\"$node\"}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "short", + "mappings": [] + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "auto", + "wideLayout": true, + "percentChangeColorMode": "standard", + "showPercentChange": false, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 23 + }, + "id": 14 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Lanes open", + "type": "stat", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "decdn_lanes_open{instance=~\"$node\"}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "short", + "mappings": [] + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "auto", + "wideLayout": true, + "percentChangeColorMode": "standard", + "showPercentChange": false, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 23 + }, + "id": 15 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Peers known", + "type": "stat", + "description": "Entries in this node's address directory: the NodeId → operator address bindings the node-to-node pull leg resolves against. Scoped to nodes that have attempted a pull — every metric is registered unconditionally, so a cache-only node exports a permanent 0 that is not a collapsed directory, and only having attempted a pull distinguishes the two (#1231, #2072).", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "decdn_node_address_directory_size{instance=~\"$node\"}\n and on(instance) (decdn_node_pull_attempts_total > 0)", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "short", + "mappings": [] + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "auto", + "wideLayout": true, + "percentChangeColorMode": "standard", + "showPercentChange": false, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 23 + }, + "id": 16 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Load shedding", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "decdn_load_shed_pressure_active{instance=~\"$node\"}", + "refId": "A", + "legendFormat": "pressure active" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "decdn_load_shed_egress_bps{instance=~\"$node\"}", + "refId": "B", + "legendFormat": "egress ceiling" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "last", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 27 + }, + "id": 17 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Telemetry export health", + "type": "timeseries", + "description": "A zero here does not prove no traces were lost: the counter wraps the exporter's own export call and does not see batch-queue drops or collector-side partial-success rejections. Those surface as SpanDroppingStarted / SpansDropped at error level under the opentelemetry_sdk target in the log panel below.", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_otlp_export_failures_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "otlp export failures" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_client_idle_close_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "B", + "legendFormat": "client idle closes" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "yellow", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 27 + }, + "id": 18 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Operator-path failures", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_config_reload_failures_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "config reload failures" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_receipt_write_failures_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "B", + "legendFormat": "receipt write failures" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 75 + }, + { + "color": "red", + "value": 90 + } + ] + }, + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 47, + "description": "A failed config reload kept the previous values; a failed receipt write is a lost audit-log row. Neither affects settlement." + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 41 + }, + "id": 19, + "panels": [], + "title": "iroh transport", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Bytes sent by path", + "type": "timeseries", + "description": "Relay traffic is the expensive path — it means hole punching did not produce a direct route to that peer.", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_iroh_socket_send_ipv4_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "ipv4" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_iroh_socket_send_ipv6_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "B", + "legendFormat": "ipv6" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_iroh_socket_send_relay_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "C", + "legendFormat": "relay" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 42 + }, + "id": 20 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Bytes received by path", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_iroh_socket_recv_data_ipv4_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "ipv4" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_iroh_socket_recv_data_ipv6_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "B", + "legendFormat": "ipv6" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_iroh_socket_recv_data_relay_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "C", + "legendFormat": "relay" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_iroh_socket_recv_data_custom_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "D", + "legendFormat": "custom" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 42 + }, + "id": 21 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Datagrams", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_iroh_socket_recv_datagrams_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "received" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_iroh_socket_recv_gro_datagrams_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "B", + "legendFormat": "received via GRO" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 49 + }, + "id": 22 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Connections and hole punching", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_iroh_socket_num_conns_opened_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "opened" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_iroh_socket_num_conns_closed_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "B", + "legendFormat": "closed" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_iroh_socket_num_conns_direct_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "C", + "legendFormat": "direct" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_iroh_socket_holepunch_attempts_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "D", + "legendFormat": "holepunch attempts" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 49 + }, + "id": 23 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Path churn", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_iroh_socket_transport_ip_paths_added_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "ip added" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_iroh_socket_transport_ip_paths_removed_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "B", + "legendFormat": "ip removed" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_iroh_socket_transport_relay_paths_added_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "C", + "legendFormat": "relay added" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_iroh_socket_transport_relay_paths_removed_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "D", + "legendFormat": "relay removed" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_iroh_socket_relay_home_change_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "E", + "legendFormat": "home relay changed" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_iroh_socket_paths_direct_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "F", + "legendFormat": "direct paths" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_iroh_socket_paths_relay_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "G", + "legendFormat": "relay paths" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_iroh_socket_paths_custom_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "H", + "legendFormat": "custom paths" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_iroh_socket_transport_custom_paths_added_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "I", + "legendFormat": "custom added" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_iroh_socket_transport_custom_paths_removed_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "J", + "legendFormat": "custom removed" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_iroh_socket_update_direct_addrs_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "K", + "legendFormat": "direct addrs updated" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 49 + }, + "id": 24 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Net reports and port mapping", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_iroh_net_report_reports_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "reports" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_iroh_net_report_reports_full_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "B", + "legendFormat": "full reports" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_iroh_net_report_portmap_attempts_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "C", + "legendFormat": "portmap attempts" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_iroh_net_report_portmap_external_address_updated_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "D", + "legendFormat": "external address updated" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 56 + }, + "id": 25 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Socket actor ticks", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_iroh_socket_actor_tick_main_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "main" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_iroh_socket_actor_tick_msg_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "B", + "legendFormat": "msg" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_iroh_socket_actor_tick_re_stun_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "C", + "legendFormat": "re-stun" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_iroh_socket_actor_tick_direct_addr_heartbeat_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "D", + "legendFormat": "direct addr heartbeat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_iroh_socket_actor_tick_portmap_changed_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "E", + "legendFormat": "portmap changed" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_iroh_socket_actor_tick_other_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "F", + "legendFormat": "other" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_iroh_socket_actor_link_change_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "G", + "legendFormat": "link change" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 56 + }, + "id": 26 + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 63 + }, + "id": 27, + "panels": [], + "title": "DHT and probe", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "DHT store outcomes", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_dht_store_accepted_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "accepted" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_dht_store_rejected_holder_mismatch_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "B", + "legendFormat": "holder mismatch" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_dht_store_rejected_non_staked_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "C", + "legendFormat": "non-staked" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_dht_store_rejected_quota_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "D", + "legendFormat": "over quota" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_dht_batch_store_received_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "E", + "legendFormat": "batch stores received" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_dht_batch_store_hashes_deferred_rate_limit_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "F", + "legendFormat": "batch hashes deferred" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 64 + }, + "id": 28 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "DHT republish", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_dht_republish_lag_sweeps_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "lag sweeps" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_dht_republish_lag_sweeps_coalesced_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "B", + "legendFormat": "sweeps coalesced" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_dht_republish_sweep_reseeded_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "C", + "legendFormat": "hashes reseeded" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_dht_republish_seed_store_walk_failures_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "D", + "legendFormat": "seed store walk failures" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_dht_republish_seed_origin_probe_failures_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "E", + "legendFormat": "seed origin probe failures" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_dht_republish_lag_sweep_panics_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "F", + "legendFormat": "sweep panics" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_dht_requests_failed_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "G", + "legendFormat": "requests failed" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_dht_lookup_round_ceiling_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "H", + "legendFormat": "lookup round ceiling" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 64 + }, + "id": 29 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Rate-limit rejections", + "type": "timeseries", + "description": "Sibling counters rather than one counter with a reason label: a per-IP shed catches one host cycling node IDs, a global shed means the node itself is at its ceiling, and the remedies differ.", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_probe_rate_limit_rejected_per_peer_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "probe — per peer" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_probe_rate_limit_rejected_per_ip_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "B", + "legendFormat": "probe — per ip" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_probe_rate_limit_rejected_global_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "C", + "legendFormat": "probe — global" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_dht_rate_limit_rejected_per_peer_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "D", + "legendFormat": "dht — per peer" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_dht_rate_limit_rejected_per_ip_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "E", + "legendFormat": "dht — per ip" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_dht_rate_limit_rejected_global_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "F", + "legendFormat": "dht — global" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 71 + }, + "id": 30 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Rate-limit buckets tracked", + "type": "timeseries", + "description": "Bucket counts climbing while the sweep rate stays flat is the memory-exhaustion shape.", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "decdn_probe_rate_limit_tracked_per_ip{instance=~\"$node\"}", + "refId": "A", + "legendFormat": "probe — per ip" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "decdn_probe_rate_limit_tracked_per_peer{instance=~\"$node\"}", + "refId": "B", + "legendFormat": "probe — per peer" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "decdn_dht_rate_limit_tracked_per_ip{instance=~\"$node\"}", + "refId": "C", + "legendFormat": "dht — per ip" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "decdn_dht_rate_limit_tracked_per_peer{instance=~\"$node\"}", + "refId": "D", + "legendFormat": "dht — per peer" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_probe_rate_limit_prune_sweeps_per_ip_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "E", + "legendFormat": "probe ip sweeps" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_dht_rate_limit_prune_sweeps_per_ip_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "F", + "legendFormat": "dht ip sweeps" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_probe_rate_limit_prune_sweeps_per_peer_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "G", + "legendFormat": "probe peer sweeps" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_dht_rate_limit_prune_sweeps_per_peer_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "H", + "legendFormat": "dht peer sweeps" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "last", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 71 + }, + "id": 31 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Probe serving and probe cache", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_probe_requests_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "probe requests served" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_probe_cache_hits_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "B", + "legendFormat": "probe-cache hits" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_probe_cache_misses_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "C", + "legendFormat": "probe-cache misses" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_probe_read_faults_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "D", + "legendFormat": "probe read faults" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 78 + }, + "id": 32 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Probe-hold slots", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "decdn_probe_hold_slots_used{instance=~\"$node\"}", + "refId": "A", + "legendFormat": "used" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "decdn_probe_hold_slots_max{instance=~\"$node\"}", + "refId": "B", + "legendFormat": "configured max" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 78 + }, + "id": 33 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "DHT discovery and publishing", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_dht_findvalue_queries_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "find_value lookups" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_dht_store_published_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "B", + "legendFormat": "stores accepted by peers" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_dht_lookup_round_timeouts_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "C", + "legendFormat": "lookup round timeouts" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_dht_bucket_refresh_failures_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "D", + "legendFormat": "bucket refresh failures" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_dht_bootstrap_find_node_failures_total{instance=~\"$node\"}[$__rate_interval]))", + "refId": "E", + "legendFormat": "bootstrap failures" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 75 + }, + { + "color": "red", + "value": 90 + } + ] + }, + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 85 + }, + "id": 45, + "description": "Lookups this node ran and STORE records peers accepted from it. A round timeout is a lookup round whose peers did not all answer in time. A refresh failure is a FindNode error, an invalid peer key or a panicked refresh task; a bootstrap failure is a FindNode against a seed that failed." + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Routing table size", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "max(decdn_dht_routing_table_size{instance=~\"$node\"})", + "refId": "A", + "legendFormat": "entries" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 75 + }, + { + "color": "red", + "value": 90 + } + ] + }, + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 85 + }, + "id": 46, + "description": "Distinct peers in the local Kademlia routing table, set after bootstrap and on every bucket-refresh tick." + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 92 + }, + "id": 34, + "panels": [], + "title": "Logs", + "type": "row" + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "title": "Events by level", + "type": "timeseries", + "description": "From the JSON body. The Loki level stream label is journald priority and reads info for every line this daemon writes, whatever its real severity.", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "expr": "sum by (lvl) (count_over_time({unit=\"decdn-node.service\", instance=~\"$node\"} | json lvl=\"level\" [$__auto]))", + "refId": "A", + "legendFormat": "{{lvl}}" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "cps" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 93 + }, + "id": 35 + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "title": "Noisiest targets", + "type": "timeseries", + "description": "Which Rust module is producing the volume. Dependency crates dominating this list is a log-filter problem, not an application problem.", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "expr": "topk(10, sum by (tgt) (count_over_time({unit=\"decdn-node.service\", instance=~\"$node\"} | json tgt=\"target\" [$__auto])))", + "refId": "A", + "legendFormat": "{{tgt}}" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "cps" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 93 + }, + "id": 36 + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "title": "Log bytes shipped", + "type": "timeseries", + "description": "What this node costs in Loki ingest.", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "expr": "sum by (instance) (bytes_over_time({unit=\"decdn-node.service\", instance=~\"$node\"}[$__auto]))", + "refId": "A", + "legendFormat": "{{instance}}" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 93 + }, + "id": 37 + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "title": "Log tail", + "type": "logs", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "expr": "{unit=\"decdn-node.service\", instance=~\"$node\"} | json lvl=\"level\", tgt=\"target\", msg=\"fields.message\" | lvl=~\"$level\" | line_format \"{{.lvl}} {{.tgt}} :: {{.msg}}\"", + "refId": "A", + "queryType": "range" + } + ], + "options": { + "dedupStrategy": "none", + "enableInfiniteScrolling": true, + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 24, + "x": 0, + "y": 100 + }, + "id": 38 + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 111 + }, + "id": 39, + "panels": [], + "title": "Traces", + "type": "row" + }, + { + "datasource": { + "type": "tempo", + "uid": "${DS_TEMPO}" + }, + "title": "Span rate by operation", + "type": "timeseries", + "description": "Spans per second by name. Only deCDN spans are exported; dependency crates' spans are filtered at the node. serve_stream is one inbound client stream. A buffered miss nests pull_through, origin_pull, node_pull, upstream_stream and open_progressive_pull under it; a streaming miss nests serve_miss_pull and open_progressive_pull. origin_range_pull is a ranged origin fetch; dht_lookup, redeem_cycle and onchain_tx cover discovery and settlement.", + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "${DS_TEMPO}" + }, + "query": "{resource.service.name=\"decdn-node\" && resource.service.instance.id=\"$node\"} | rate() by (name)", + "refId": "A", + "legendFormat": "{{name}}", + "queryType": "traceql", + "limit": 50, + "tableType": "spans" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "cps" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 112 + }, + "id": 40 + }, + { + "datasource": { + "type": "tempo", + "uid": "${DS_TEMPO}" + }, + "title": "Error spans", + "type": "timeseries", + "description": "status=error has to sit inside the selector braces; TraceQL has no top-level && between a selector and a condition.", + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "${DS_TEMPO}" + }, + "query": "{resource.service.name=\"decdn-node\" && resource.service.instance.id=\"$node\" && status=error} | rate()", + "refId": "A", + "legendFormat": "{{name}}", + "queryType": "traceql", + "limit": 50, + "tableType": "spans" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "cps" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 120 + }, + "id": 42 + }, + { + "datasource": { + "type": "tempo", + "uid": "${DS_TEMPO}" + }, + "title": "Recent traces", + "type": "table", + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "${DS_TEMPO}" + }, + "query": "{resource.service.name=\"decdn-node\" && resource.service.instance.id=\"$node\"}", + "refId": "A", + "queryType": "traceql", + "limit": 50, + "tableType": "spans" + } + ], + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "options": {}, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 120 + }, + "id": 43 + }, + { + "datasource": { + "type": "tempo", + "uid": "${DS_TEMPO}" + }, + "title": "deCDN operation latency", + "type": "timeseries", + "description": "p50 and p95 duration of each deCDN span. serve_stream covers a whole client stream, so its duration tracks blob size; upstream_stream and open_progressive_pull are the paid miss pull and its time to first byte; onchain_tx is one redeemMany from send to receipt. Filter a trace by outcome to split completed from refused or failed streams.", + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "${DS_TEMPO}" + }, + "query": "{resource.service.name=\"decdn-node\" && resource.service.instance.id=\"$node\" && name=~\"serve_stream|pull_through|serve_miss_pull|origin_pull|origin_range_pull|node_pull|upstream_stream|open_progressive_pull|dht_lookup|redeem_cycle|onchain_tx\"} | quantile_over_time(duration, .5, .95) by (name)", + "refId": "A", + "legendFormat": "{{name}} p{{p}}", + "queryType": "traceql", + "limit": 50, + "tableType": "spans" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 128 + }, + "id": 44 + } + ], + "preload": false, + "refresh": "30s", + "schemaVersion": 39, + "tags": [ + "decdn", + "node" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "grafanacloud-decdn-prom", + "value": "grafanacloud-prom" + }, + "hide": 0, + "includeAll": false, + "label": "Metrics", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": "grafanacloud-decdn-logs", + "value": "grafanacloud-logs" + }, + "hide": 0, + "includeAll": false, + "label": "Logs", + "multi": false, + "name": "DS_LOKI", + "options": [], + "query": "loki", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": "grafanacloud-decdn-traces", + "value": "grafanacloud-traces" + }, + "hide": 0, + "includeAll": false, + "label": "Traces", + "multi": false, + "name": "DS_TEMPO", + "options": [], + "query": "tempo", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": null, + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(decdn_node_uptime_seconds, instance)", + "hide": 0, + "includeAll": false, + "label": "Node", + "multi": false, + "name": "node", + "options": [], + "refresh": 2, + "query": { + "qryType": 1, + "query": "label_values(decdn_node_uptime_seconds, instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "allValue": null, + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "hide": 0, + "includeAll": true, + "label": "Log level", + "multi": true, + "name": "level", + "options": [], + "query": "TRACE,DEBUG,INFO,WARN,ERROR", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "deCDN — node drilldown", + "uid": "decdn-node", + "version": 1, + "weekStart": "" +} diff --git a/charts/decdn-node/files/monitoring/grafana-dashboard.json b/charts/decdn-node/files/monitoring/grafana-dashboard.json new file mode 100644 index 0000000..b9da4d7 --- /dev/null +++ b/charts/decdn-node/files/monitoring/grafana-dashboard.json @@ -0,0 +1,2873 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "red", + "name": "deCDN alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [ + "decdn" + ], + "type": "tags" + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "enable": true, + "hide": false, + "iconColor": "purple", + "name": "Node restarts", + "expr": "resets(decdn_node_uptime_seconds{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]) > 0", + "titleFormat": "decdn-node restarted", + "textFormat": "{{instance}}", + "step": "60s", + "useValueForTime": false + } + ] + }, + "description": "Fleet-wide health for every decdn-node: status, delivery, slash safety, and the logs and traces that explain them. Deep dives live in the deCDN dashboard dropdown. Companion to adr/appendix-observability.md, monitoring/prometheus-alerts.yml and docs/runbook.md.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "decdn" + ], + "targetBlank": false, + "title": "deCDN dashboards", + "tooltip": "", + "type": "dashboards", + "url": "" + }, + { + "asDropdown": false, + "icon": "doc", + "includeVars": false, + "keepTime": false, + "tags": [], + "targetBlank": true, + "title": "Operator runbook", + "tooltip": "", + "type": "link", + "url": "https://github.com/decdn/decdn/blob/main/docs/runbook.md" + } + ], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "Fleet status", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Nodes up", + "type": "stat", + "description": "Prometheus scrape health for job=decdn-node. Falls to zero before any decdn_ series can tell you anything.", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(up{job=\"decdn-node\",deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"})", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, + "unit": "short", + "mappings": [] + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "auto", + "wideLayout": true, + "percentChangeColorMode": "standard", + "showPercentChange": false, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 1 + }, + "id": 2 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Chain RPC", + "type": "stat", + "description": "Worst node's view of the Arbitrum RPC. Every chain watcher and every settlement stalls behind this.", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "min(decdn_rpc_healthy{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"})", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, + "unit": "short", + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "unreachable", + "index": 0, + "color": "red" + }, + "1": { + "text": "healthy", + "index": 1, + "color": "green" + } + } + } + ] + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "auto", + "wideLayout": true, + "percentChangeColorMode": "standard", + "showPercentChange": false, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 3, + "y": 1 + }, + "id": 3 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Oldest watcher tick", + "type": "stat", + "description": "Age of the least recently ticked chain-event watcher, across all five watchers and every node. Red at 180s matches DecdnBlacklistWatcherStalled \u2014 a stalled blacklist watcher is a slash risk, not a cosmetic lag.", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "max(time() - label_replace({__name__=~\"decdn_.+_watcher_last_tick_timestamp_seconds\",deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"} > 0, \"watcher\", \"$1\", \"__name__\", \"decdn_(.+)_watcher_last_tick_timestamp_seconds\"))", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 120 + }, + { + "color": "red", + "value": 180 + } + ] + }, + "unit": "s", + "mappings": [] + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "auto", + "wideLayout": true, + "percentChangeColorMode": "standard", + "showPercentChange": false, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 6, + "y": 1 + }, + "id": 4 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Shortest uptime", + "type": "stat", + "description": "A value under 5 minutes means something restarted.", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "min(decdn_node_uptime_seconds{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"})", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 900 + } + ] + }, + "unit": "s", + "mappings": [] + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "auto", + "wideLayout": true, + "percentChangeColorMode": "standard", + "showPercentChange": false, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 9, + "y": 1 + }, + "id": 5 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Cache fullest node", + "type": "stat", + "description": "Highest cache utilisation in the fleet. Past the limit the eviction driver starts discarding content the network may still want.", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "max(decdn_cache_bytes{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"} / decdn_cache_size_limit_bytes{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"})", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.85 + }, + { + "color": "red", + "value": 0.95 + } + ] + }, + "unit": "percentunit", + "mappings": [], + "decimals": 1 + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "auto", + "wideLayout": true, + "percentChangeColorMode": "standard", + "showPercentChange": false, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 12, + "y": 1 + }, + "id": 6 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Active streams", + "type": "stat", + "description": "Open cdn/client/v1 deliveries, both directions.", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(decdn_streams_active{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"})", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "short", + "mappings": [] + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "auto", + "wideLayout": true, + "percentChangeColorMode": "standard", + "showPercentChange": false, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 15, + "y": 1 + }, + "id": 7 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Lanes open", + "type": "stat", + "description": "Distinct (pool, signer, provider) keys holding unredeemed vouchers.", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(decdn_lanes_open{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"})", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "short", + "mappings": [] + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "auto", + "wideLayout": true, + "percentChangeColorMode": "standard", + "showPercentChange": false, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 18, + "y": 1 + }, + "id": 8 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Unredeemed", + "type": "stat", + "description": "Accepted-but-unredeemed voucher value. Raw USDC base units scaled to dollars. Refreshed once per redeemer tick, so it lags by up to one redeem_interval_secs.", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(decdn_unredeemed_usdc{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}) / 1e6", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "currencyUSD", + "mappings": [], + "decimals": 2 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "auto", + "wideLayout": true, + "percentChangeColorMode": "standard", + "showPercentChange": false, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 21, + "y": 1 + }, + "id": 9 + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 10, + "panels": [], + "title": "Delivery", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Throughput \u2014 served vs origin pull-through", + "type": "timeseries", + "description": "Payload bytes this fleet wrote to paying clients, against the origin bytes it had to buy to do it. Their ratio is the origin-egress amplification factor: 1.0 while the fleet is a pure pass-through, trending to 0 as content gets re-served from cache. Served reads decdn_bytes_served_total (the wire write on the serve leg), NOT decdn_cache_bytes_returned_total \u2014 that counter is scoped to CacheEngine::get, which the paid serve path never calls, so it holds at zero under full production load.", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_bytes_served_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "served on the wire" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_cache_pull_through_bytes_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "B", + "legendFormat": "pulled from origin" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 6 + }, + "id": 11 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Serve hit ratio", + "type": "timeseries", + "description": "Serve-path hit rate: (hit + partial hit) / (hit + partial hit + miss), decided at the blob-availability gate before any fill tier or load-shed admission. This is NOT decdn_cache_hits_total \u2014 that one meters CacheEngine::get, the whole-blob buffered read the paid serve path never calls, so it sits at zero on a node that only serves paying clients however full its store is. Deliberately unguarded division: an idle node renders a gap rather than a fabricated 0%.", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "(\n sum by (instance) (rate(decdn_serve_cache_hit_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n + sum by (instance) (rate(decdn_serve_cache_partial_hit_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n)\n/\n(\n sum by (instance) (rate(decdn_serve_cache_hit_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n + sum by (instance) (rate(decdn_serve_cache_partial_hit_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n + sum by (instance) (rate(decdn_serve_cache_miss_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n)", + "refId": "A", + "legendFormat": "{{instance}}" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 0.5 + }, + { + "color": "green", + "value": 0.8 + } + ] + }, + "unit": "percentunit", + "min": 0, + "max": 1 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 6 + }, + "id": 12 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Serve refusals by reason", + "type": "timeseries", + "description": "Every refusal reason the serve leg records. The payment-class reasons and cache miss all collapse to the same NotFound wire code, so a client cannot tell them apart \u2014 these counters are the only place the cause survives.", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_serve_stream_rejected_cache_miss_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "cache miss" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_serve_stream_rejected_insufficient_deposit_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "B", + "legendFormat": "insufficient deposit" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_serve_stream_rejected_unknown_lane_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "C", + "legendFormat": "unknown lane" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_serve_stream_rejected_owner_mismatch_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "D", + "legendFormat": "owner mismatch" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_serve_stream_rejected_pool_unconfirmed_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "E", + "legendFormat": "pool unconfirmed" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_serve_stream_rejected_signer_cap_exhausted_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "F", + "legendFormat": "signer cap exhausted" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_serve_stream_rejected_signer_floor_at_cap_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "G", + "legendFormat": "signer floor at cap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_serve_stream_rejected_evicted_since_probe_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "H", + "legendFormat": "evicted since probe" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_serve_stream_rejected_range_not_satisfiable_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "I", + "legendFormat": "range not satisfiable" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_serve_stream_rejected_hash_denied_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "J", + "legendFormat": "hash denied" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_serve_stream_rejected_chain_hash_denied_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "K", + "legendFormat": "chain hash denied" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_serve_stream_rejected_chain_stale_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "L", + "legendFormat": "chain stale" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_serve_stream_rejected_origin_denied_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "M", + "legendFormat": "origin denied" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_serve_stream_rejected_foreign_declined_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "N", + "legendFormat": "foreign declined" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_serve_stream_rejected_load_shed_hit_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "O", + "legendFormat": "load shed (hit)" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_serve_stream_rejected_load_shed_miss_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "P", + "legendFormat": "load shed (miss)" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_serve_stream_rejected_internal_error_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "Q", + "legendFormat": "internal error" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_serve_economics_refused_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "R", + "legendFormat": "economics \u2014 upstream buy refused (buyer leg)" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_serve_stream_rejected_bad_binding_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "S", + "legendFormat": "bad binding" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 60, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 13 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Pull funnel", + "type": "timeseries", + "description": "Node-to-node cache-miss pulls, where this fleet is the paying client. The gap between the two lines is the failure budget broken out beside.", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_node_pull_attempts_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "attempts" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_node_pull_success_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "B", + "legendFormat": "success" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 23 + }, + "id": 14 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Pull failures by reason", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_node_pull_no_providers_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "no providers" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_node_pull_unreachable_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "B", + "legendFormat": "unreachable" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_node_pull_timeout_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "C", + "legendFormat": "timeout" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_node_pull_corruption_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "D", + "legendFormat": "corruption" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_node_pull_stalled_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "E", + "legendFormat": "stalled" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_node_pull_refused_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "F", + "legendFormat": "refused" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_node_pull_refused_unattributable_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "G", + "legendFormat": "refused (unattributable)" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_node_pull_too_large_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "H", + "legendFormat": "too large" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_node_pull_rate_above_ceiling_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "I", + "legendFormat": "rate above ceiling" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_node_pull_voucher_rejected_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "J", + "legendFormat": "voucher rejected" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_node_pull_pool_wedged_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "K", + "legendFormat": "pool wedged" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_node_pull_local_fault_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "L", + "legendFormat": "local fault" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_node_pull_abandon_drain_timeout_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "M", + "legendFormat": "abandon drain timeout" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_node_pull_pool_open_failures_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "N", + "legendFormat": "pool open failure" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 60, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 23 + }, + "id": 15 + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 39 + }, + "id": 16, + "panels": [], + "title": "Slash safety", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Probe-hold exhausted", + "type": "timeseries", + "description": "Probes for a present blob that got no eviction hold because every slot was live. Filtered to reason=exhausted on purpose: disabled is an operator choice and stake_lane_reserved has its own knob, and neither should trip the alert.", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(decdn_probe_hold_unavailable_total{reason=\"exhausted\",deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval])", + "refId": "A", + "legendFormat": "{{instance}}" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 17 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Probe-hold saturation", + "type": "timeseries", + "description": "clamp_min is correct here \u2014 the denominator is an integer capacity gauge, not a rate.", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "decdn_probe_hold_slots_used{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"} / clamp_min(decdn_probe_hold_slots_max{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}, 1)", + "refId": "A", + "legendFormat": "{{instance}}" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "percentunit", + "min": 0, + "max": 1 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 18 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Blacklist enforcement failures", + "type": "timeseries", + "description": "Hashes a batched re-scope could not re-verify or evict. Non-zero means the deny-set is not fully enforced and a blacklisted blob may still be servable \u2014 directly slashable.", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_blacklist_enforcement_failures_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "hashes not enforced" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 47 + }, + "id": 19 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Watcher task panics", + "type": "timeseries", + "description": "Any non-zero value is a bug. Capture the error! line and file it.", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_slash_watcher_task_panicked_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "slash" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_staker_set_watcher_task_panicked_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "B", + "legendFormat": "staker_set" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_blacklist_watcher_task_panicked_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "C", + "legendFormat": "blacklist" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_settlement_watcher_task_panicked_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "D", + "legendFormat": "settlement" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_fee_shares_watcher_task_panicked_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "E", + "legendFormat": "fee_shares" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 47 + }, + "id": 20 + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 54 + }, + "id": 21, + "panels": [], + "title": "Logs and traces", + "type": "row" + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "title": "Log events by level", + "type": "timeseries", + "description": "Parsed out of the JSON body. The Loki level stream label is journald's syslog priority, which is info for every line the daemon writes to stdout regardless of its real severity \u2014 do not group by it.", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "expr": "sum by (lvl) (count_over_time({unit=\"decdn-node.service\", instance=~\"$instance\"} | json lvl=\"level\" [$__auto]))", + "refId": "A", + "legendFormat": "{{lvl}}" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "cps" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 55 + }, + "id": 22 + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "title": "WARN and ERROR per node", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "expr": "sum by (instance) (count_over_time({unit=\"decdn-node.service\", instance=~\"$instance\"} | json lvl=\"level\" | lvl=~\"WARN|ERROR\" [$__auto]))", + "refId": "A", + "legendFormat": "{{instance}}" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "yellow", + "value": null + } + ] + }, + "unit": "cps" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 55 + }, + "id": 23 + }, + { + "datasource": { + "type": "tempo", + "uid": "${DS_TEMPO}" + }, + "title": "Span rate by operation", + "type": "timeseries", + "description": "Spans per second by name across the fleet. Only deCDN spans are exported; dependency crates' spans are filtered at the node. serve_stream counts inbound client streams; upstream_stream and open_progressive_pull count paid miss pulls from other nodes; onchain_tx counts redeemMany submissions.", + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "${DS_TEMPO}" + }, + "query": "{resource.service.name=\"decdn-node\"} | rate() by (name)", + "refId": "A", + "legendFormat": "{{name}}", + "queryType": "traceql", + "limit": 50, + "tableType": "spans" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "cps" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 62 + }, + "id": 24 + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "title": "Recent WARN and ERROR", + "type": "logs", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "expr": "{unit=\"decdn-node.service\", instance=~\"$instance\"} | json lvl=\"level\", tgt=\"target\", msg=\"fields.message\" | lvl=~\"WARN|ERROR\" | line_format \"{{.instance}} {{.lvl}} {{.tgt}} :: {{.msg}}\"", + "refId": "A", + "queryType": "range" + } + ], + "options": { + "dedupStrategy": "none", + "enableInfiniteScrolling": true, + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 62 + }, + "id": 25 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Pull success rate", + "type": "timeseries", + "description": "Node-to-node cache-miss pulls that completed, over pulls attempted. This fleet is the paying client on this leg, so a collapse here means misses are not being backfilled and every subsequent request for the same blob pays origin again. A ratio carries that where the raw funnel counts do not: zero successes against a low attempt rate reads as light traffic in the counts and as a floored ratio here. Unguarded division: no attempts renders a gap.", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_node_pull_success_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n/ sum(rate(decdn_node_pull_attempts_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "success rate" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "orange", + "value": 0.5 + }, + { + "color": "green", + "value": 0.9 + } + ] + }, + "unit": "percentunit", + "min": 0, + "max": 1 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 31 + }, + "id": 9001 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Unattributed stream failures", + "type": "timeseries", + "description": "Seller-leg stream failures that no reason counter claims: rate(decdn_streams_failed_total{direction=\"inbound\"}) minus the 22 siblings that each end exactly one inbound stream \u2014 every serve_stream_rejected_* reason, the two midstream exhaustion stops, the node fault and the takedown termination. decdn_serve_economics_refused_total is deliberately NOT subtracted: it is a buyer-leg gate that declines to buy from a candidate upstream, and ends no inbound stream. A standing positive floor is expected and healthy \u2014 a client that closes early, a peer-attributable stream error, and the free header handshake a downstream node's miss pull opens and closes after the response all count as failures by design and have no reason counter. Read the SHAPE, not the level: a step change means a failure mode arrived that no counter names, and the refusal breakdown will not show it. Goes negative only if a reason counter fires without ending a stream.", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_streams_failed_total{direction=\"inbound\",deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_bad_binding_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_cache_miss_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_chain_hash_denied_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_chain_stale_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_evicted_since_probe_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_foreign_declined_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_hash_denied_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_insufficient_deposit_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_internal_error_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_load_shed_hit_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_load_shed_miss_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_origin_denied_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_owner_mismatch_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_pool_unconfirmed_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_range_not_satisfiable_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_signer_cap_exhausted_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_signer_floor_at_cap_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_rejected_unknown_lane_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_midstream_pool_exhausted_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_midstream_signer_cap_exhausted_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_node_fault_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))\n - sum(rate(decdn_serve_stream_terminated_takedown_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "no reason counter" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 31 + }, + "id": 9002 + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 70 + }, + "id": 9003, + "panels": [], + "title": "Refuse-to-serve & load-shedding (ADR 041)", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Serve-economics buy refusals", + "type": "timeseries", + "description": "ADR 041 cache-miss buy-ceiling refusals, split by regime (see the delivery dashboard for detail). A rising `amortized` line is the operator-actionable one: this node's sell rate or margin config is too tight.", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_serve_economics_refused_warming_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "warming (market too hot)" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_serve_economics_refused_amortized_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "B", + "legendFormat": "amortized (economics too tight)" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 71 + }, + "id": 9004 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Load-shed refusals by reason", + "type": "timeseries", + "description": "Fleet-wide new serves shed under overload, split by cause. See the delivery dashboard for per-reason remedies.", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_load_shed_refused_node_at_capacity_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "node at capacity" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_load_shed_refused_egress_saturated_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "B", + "legendFormat": "egress saturated" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_load_shed_refused_client_at_capacity_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "C", + "legendFormat": "client at capacity" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 71 + }, + "id": 9005 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Warming sources blocked", + "type": "timeseries", + "description": "Upstream sources whose ADR 041 warming allowance is currently spent, fleet-wide, beside the rate of speculative buys downgraded to the amortized floor.", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(decdn_warming_sources_blocked{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"})", + "refId": "A", + "legendFormat": "sources blocked" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(decdn_warming_speculative_blocked_total{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"}[$__rate_interval]))", + "refId": "B", + "legendFormat": "buys downgraded /s" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 71 + }, + "id": 9006 + } + ], + "preload": false, + "refresh": "30s", + "schemaVersion": 39, + "tags": [ + "decdn", + "overview" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "grafanacloud-decdn-prom", + "value": "grafanacloud-prom" + }, + "hide": 0, + "includeAll": false, + "label": "Metrics", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": "grafanacloud-decdn-logs", + "value": "grafanacloud-logs" + }, + "hide": 0, + "includeAll": false, + "label": "Logs", + "multi": false, + "name": "DS_LOKI", + "options": [], + "query": "loki", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": "grafanacloud-decdn-traces", + "value": "grafanacloud-traces" + }, + "hide": 0, + "includeAll": false, + "label": "Traces", + "multi": false, + "name": "DS_TEMPO", + "options": [], + "query": "tempo", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": null, + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(decdn_node_uptime_seconds, deployment_environment)", + "hide": 0, + "includeAll": false, + "label": "Environment", + "multi": false, + "name": "env", + "options": [], + "refresh": 2, + "query": { + "qryType": 1, + "query": "label_values(decdn_node_uptime_seconds, deployment_environment)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "allValue": null, + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(decdn_node_uptime_seconds{deployment_environment=\"$env\"}, region)", + "hide": 0, + "includeAll": true, + "label": "Region", + "multi": true, + "name": "region", + "options": [], + "refresh": 2, + "query": { + "qryType": 1, + "query": "label_values(decdn_node_uptime_seconds{deployment_environment=\"$env\"}, region)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "allValue": null, + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(decdn_node_uptime_seconds{deployment_environment=\"$env\",region=~\"$region\"}, instance)", + "hide": 0, + "includeAll": true, + "label": "Node", + "multi": true, + "name": "instance", + "options": [], + "refresh": 2, + "query": { + "qryType": 1, + "query": "label_values(decdn_node_uptime_seconds{deployment_environment=\"$env\",region=~\"$region\"}, instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "deCDN \u2014 fleet overview", + "uid": "decdn-poc-overview", + "version": 1, + "weekStart": "" +} diff --git a/charts/decdn-node/files/monitoring/prometheus-alerts.yml b/charts/decdn-node/files/monitoring/prometheus-alerts.yml new file mode 100644 index 0000000..51c73d2 --- /dev/null +++ b/charts/decdn-node/files/monitoring/prometheus-alerts.yml @@ -0,0 +1,819 @@ +# deCDN reference Prometheus alerting rules +# +# Companion to adr/appendix-observability.md. Thresholds in the +# `decdn-slash-safety` group are copied verbatim from §2.1 +# "Recommended alert thresholds"; the others are operator-onboarding +# defaults, not normative. +# +# Validate locally: +# promtool check rules monitoring/prometheus-alerts.yml +# +# Wire into Prometheus via: +# rule_files: +# - /etc/prometheus/prometheus-alerts.yml +# (or copy the file under whatever filename your Prometheus deployment expects). + +groups: + # --------------------------------------------------------------- + # Slash-safety — copied from appendix-observability §2.1. + # A sustained alert here means the operator is at slash risk, with one + # deliberate exception: DecdnProbeHoldViolations is grouped here because it + # is a probe-hold capacity metric, but it signals lost revenue rather than + # slash exposure (see its annotation and appendix-observability §Slash-Safety + # Metrics). Carries no slash_risk label for that reason. + # --------------------------------------------------------------- + - name: decdn-slash-safety + interval: 30s + rules: + - alert: DecdnProbeHoldViolations + expr: rate(decdn_probe_hold_unavailable_total{reason="exhausted"}[5m]) > 0 + for: 5m + labels: + component: probe + severity: critical + annotations: + runbook_url: "https://github.com/decdn/decdn/blob/main/docs/runbook.md#slashing-risk" + summary: "Probe-hold budget exhausted on {{ $labels.instance }}" + description: | + decdn_probe_hold_unavailable_total{reason="exhausted"} has been + incrementing for 5m on {{ $labels.instance }}. Each increment is a + blob that IS present but could not be held because every probe-hold + slot was live. Holds are best-effort, so the node still advertised + has_blob: true and forwent only the hold — the blob is now visible + to the LRU driver and may be evicted before the requester's pull + arrives, costing a wasted round trip and a lost delivery. Not slash + evidence: no offense pairs a probe with a later miss. Reduce load or + raise max_probe_holds; investigate OOM. The reason filter is + load-bearing: reason="disabled" (max_probe_holds=0) and + reason="stake_lane_reserved" are deliberate operator decisions and + must not fire this alert. Note max_probe_holds is + restart-required — editing it and reloading logs "requires restart" + and changes nothing. See + adr/005-protocol.md#probe-triggered-eviction-hold. + + # There is no blacklist sync-lag or version-delta alert: the watcher + # rebuilds the deny-set by full enumeration and keeps no version cursor + # (adr/011-content-takedown.md § Node Behavior), so no such gauge exists to + # alert on. DecdnBlacklistWatcherStalled below is the coverage: it fires on + # staleness of the watcher's last successful tick, which IS exported. + - alert: DecdnBlacklistWatcherStalled + # Positive liveness: the gauge is stamped every successful poll tick, so + # staleness catches a panicked, wedged, or exited watcher that the + # error-triggered down-seconds gauge reads as a healthy 0 (#1316/#1320). + # 180s ≈ several poll intervals; tune to content_blacklist_poll_interval_sec. + # The `> 0` guard skips the pre-first-tick sentinel (the gauge reads 0 + # until the first successful tick), so a fresh boot doesn't self-fire. + expr: (decdn_blacklist_watcher_last_tick_timestamp_seconds > 0) and (time() - decdn_blacklist_watcher_last_tick_timestamp_seconds > 180) + for: 5m + labels: + component: blacklist + severity: critical + slash_risk: blacklist_violation + annotations: + runbook_url: "https://github.com/decdn/decdn/blob/main/docs/runbook.md#contentblacklist-compliance" + summary: "Blacklist watcher not ticking on {{ $labels.instance }}" + description: | + No successful blacklist poll tick for several intervals — the + watcher has panicked, wedged, or exited. The node may be serving + content blacklisted after the failure, which is slashable. Restart + the daemon and check the RPC provider. See adr/011-content-takedown.md. + + - alert: DecdnBlacklistEnforcementFailing + expr: rate(decdn_blacklist_enforcement_failures_total[10m]) > 0 + for: 2m + labels: + component: blacklist + severity: critical + slash_risk: blacklist_violation + annotations: + runbook_url: "https://github.com/decdn/decdn/blob/main/docs/runbook.md#contentblacklist-compliance" + summary: "Blacklist deny-set not fully enforced on {{ $labels.instance }}" + description: | + A re-scope could not re-verify or evict every known entry (disk or + scope eth_call failure). A blacklisted blob may still be servable and + slashable even while blacklist_watcher_down_seconds reads 0. Check + disk health and the ContentBlacklist RPC. See adr/011-content-takedown.md. + + - alert: DecdnWatcherTaskPanicked + # One rule covers all five chain-event watchers — slash, staker_set, + # blacklist, settlement and fee_shares — via the metric-name match. + # + # The obvious spelling, `rate({__name__=~"..."}[10m]) > 0`, does not + # evaluate: rate() drops __name__, so the five per-watcher series on a + # node collapse to five identical label sets and Prometheus refuses the + # vector. label_replace lifts the watcher out of the metric name while + # the raw selector still has it, and the offset subtraction takes the + # place of rate() so nothing ever strips the name mid-expression. A + # counter reset reads negative, not positive, so a restart cannot fire + # this. + # + # The extraction regex ends `_watcher_task_panicked_total` so `watcher` + # reads `slash`, not `slash_watcher`, and the description does not say + # "slash_watcher watcher". The SELECTOR stays broad on purpose: every + # panic counter today is a watcher's, but a future one that is not would + # still fire this rule, just with an empty `watcher` label. Firing with a + # thin message beats not firing at all for a counter whose every non-zero + # value is a bug. + expr: >- + (label_replace({__name__=~"decdn_.+_task_panicked_total"}, "watcher", "$1", + "__name__", "decdn_(.+)_watcher_task_panicked_total") - label_replace({__name__=~"decdn_.+_task_panicked_total"} + offset 10m, "watcher", "$1", "__name__", "decdn_(.+)_watcher_task_panicked_total")) > 0 + for: 0m + labels: + component: watcher + severity: critical + annotations: + runbook_url: "https://github.com/decdn/decdn/blob/main/docs/runbook.md#slashing-risk" + summary: "A chain-event watcher task panicked on {{ $labels.instance }}" + description: | + The {{ $labels.watcher }} watcher task unwound on a panic. Never + expected under the anti-panic policy. Capture the error! log line + naming the watcher and file a bug. + + # A latent-bug report, not a degradation (#1517). The engine recovers the + # guard and clears the poison, so coalescing never stops and no restart is + # needed — but the workspace anti-panic policy means no production path is + # supposed to be able to poison this at all, so any nonzero value is a bug + # worth filing. + # + # `increase(...[1h]) > 0`, not the bare `> 0` this used to carry: the + # counter is monotonic and process-local, and the annotation tells the + # operator not to restart, so a bare comparison latches for the life of + # the process and can never resolve. A single occurrence still fires — + # the window only decides when the alert clears. + - alert: DecdnCacheInflightMutexPoisoned + expr: increase(decdn_cache_inflight_mutex_poisoned_total[1h]) > 0 + for: 0m + labels: + component: cache + severity: warning + annotations: + runbook_url: "https://github.com/decdn/decdn/blob/main/docs/runbook.md#cache-coalescing-mutex-poisoned" + summary: "Cache coalescing mutex poisoned on {{ $labels.instance }}" + description: | + A task panicked while holding the in-flight fill-coalescing map. + Coalescing is preserved and no operator action is needed beyond + reporting it — do NOT restart to "clear" it. File the panic with + the surrounding ERROR log line (emitted once per process). + See docs/runbook.md § Cache coalescing mutex poisoned. + + # The value, not increase(): the slash watcher's bring-up backfill counts + # every still-appealable slash before the first scrape, so increase() + # would miss a slash found at startup. A non-zero value means this process + # has seen a slash; after a restart it re-counts only the slashes whose + # appeal window is still open, so the alert clears once the last window + # closes and the node restarts. + - alert: DecdnSlashDetected + expr: decdn_slashes_detected_total > 0 + for: 0m + labels: + component: slashing + severity: critical + slash_risk: "true" + annotations: + runbook_url: "https://github.com/decdn/decdn/blob/main/docs/runbook.md#slashing-risk" + summary: "Operator slashed on {{ $labels.instance }}" + description: | + The slash watcher saw a SlashJudge.Slashed event against this + operator. The bond lost stake. Read the slash reason from the + watcher's log line and consider `decdn appeal slash` inside the + 30-day appeal window. + + # --------------------------------------------------------------- + # Liveness / capacity — operator-onboarding defaults. + # Tune `for:` and thresholds for your fleet size before paging. + # --------------------------------------------------------------- + - name: decdn-liveness + interval: 30s + rules: + - alert: DecdnNodeDown + expr: up{job=~".*decdn.*"} == 0 + for: 1m + labels: + component: node + severity: critical + annotations: + runbook_url: "https://github.com/decdn/decdn/blob/main/docs/runbook.md#rpc-unreachable" + summary: "Node {{ $labels.instance }} unreachable for 1m" + description: | + Prometheus has been unable to scrape this node for at least + one minute (the standard `up` metric is 0). This catches the + crash-and-stay-down case that decdn_node_uptime_seconds + cannot — once the process is gone, no decdn_* metric is + exported. Adjust the `job=~".*decdn.*"` selector to match + your scrape-config job name(s). + + # `resets(...)`, not `decdn_node_uptime_seconds < 300`: the comparison + # holds true for a full five minutes after every deploy, and a + # crash-looping node re-enters it continuously instead of escalating. + # `resets` counts the gauge going backwards, so one restart is one event. + - alert: DecdnNodeRestarted + expr: resets(decdn_node_uptime_seconds[15m]) > 0 + for: 0m + labels: + component: node + severity: info + annotations: + summary: "{{ $labels.instance }} restarted recently" + description: | + decdn_node_uptime_seconds went backwards — the node process + restarted. Correlate with deploy events. + + - alert: DecdnNoActiveStreams + expr: sum without (direction) (decdn_streams_active) == 0 + for: 15m + labels: + component: delivery + severity: warning + annotations: + summary: "No active streams on {{ $labels.instance }} for 15m" + description: | + decdn_streams_active has been zero across all directions + (inbound + outbound) for 15 minutes. Either traffic dried up, + the node was de-prioritised by client selection, or it stopped + accepting connections. The sum-without(direction) aggregation + avoids false alerts when only one direction is idle. + + # Chain-event watcher liveness (#1316): the *_last_tick_timestamp_seconds + # gauge advances only while the watcher actually completes ticks, so + # staleness catches a panicked/wedged/exited task the error-triggered + # down-seconds gauge reads as a healthy 0. 180s ≈ several poll intervals; + # tune to your event_poll_interval. The `> 0` guard skips the pre-first-tick + # sentinel (gauge reads 0 until the first success), so a fresh boot — and a + # node that never enables a conditionally-spawned watcher — doesn't + # self-fire. Blacklist has its own (slash-risk) rule in decdn-slash-safety; + # settlement is in decdn-delivery. + - alert: DecdnSlashWatcherStalled + expr: (decdn_slash_watcher_last_tick_timestamp_seconds > 0) and (time() - decdn_slash_watcher_last_tick_timestamp_seconds > 180) + for: 5m + labels: + component: watcher + severity: warning + annotations: + runbook_url: "https://github.com/decdn/decdn/blob/main/docs/runbook.md#slashing-risk" + summary: "Slash watcher not ticking on {{ $labels.instance }}" + description: | + No successful slash-detection poll tick for several intervals — the + watcher may have panicked, wedged, or exited, so a slash against this + operator could go undetected. Restart and check the RPC provider. + + - alert: DecdnStakerSetWatcherStalled + expr: (decdn_staker_set_watcher_last_tick_timestamp_seconds > 0) and (time() - decdn_staker_set_watcher_last_tick_timestamp_seconds > 180) + for: 5m + labels: + component: watcher + severity: warning + annotations: + runbook_url: "https://github.com/decdn/decdn/blob/main/docs/runbook.md#active-staker-set-repair-not-running" + summary: "Staker-set watcher not ticking on {{ $labels.instance }}" + description: | + The cached active-staker set / node-address bindings projection is no + longer being refreshed and will drift from chain state. Correlate with + decdn_staker_set_watcher_down_seconds and check the RPC provider. + + # The registry re-enumeration is the only systematic repair for a drifted + # active-staker set. It reports success upward on a failed read (an error + # would stall event pickup), so the counter is the only thing that moves + # when it fails — and nothing at all moves when it is *skipped*, which is + # what the staleness rule below catches. + # + # The window must be longer than the attempt spacing. Attempts are 15m + # apart, so a 15m window sees a zero-rate gap whenever a poll or RPC delay + # pushes two increments further apart than that — which resets `for` and + # means a permanently failing resync never fires. `increase` over 1h always + # spans several attempts. See adr/appendix-observability.md. + - alert: DecdnCapacityBondRegistryResyncFailing + expr: increase(decdn_capacity_bond_registry_resync_failures_total[1h]) > 0 + for: 15m + labels: + component: registry + severity: warning + annotations: + runbook_url: "https://github.com/decdn/decdn/blob/main/docs/runbook.md#active-staker-set-repair-not-running" + summary: "Capacity-bond registry re-enumeration failing on {{ $labels.instance }}" + description: | + The repair for a drifted active-staker set is itself failing, so a + membership change dropped by a failed nodeIdOf stays dropped. Check + the RPC provider's eth_call path; correlate with + decdn_staker_set_watcher_resolve_failures_total. + + - alert: DecdnCapacityBondRegistryResyncStale + # The `> 0` guard covers only the window before bootstrap completes: + # bootstrap is itself a successful enumeration and stamps the gauge, so a + # node whose every later resync fails or is skipped still trips this. + expr: (decdn_capacity_bond_registry_last_resync_timestamp_seconds > 0) and (time() - decdn_capacity_bond_registry_last_resync_timestamp_seconds > 1800) + for: 15m + labels: + component: registry + severity: warning + annotations: + runbook_url: "https://github.com/decdn/decdn/blob/main/docs/runbook.md#active-staker-set-repair-not-running" + summary: "Capacity-bond registry not re-enumerating on {{ $labels.instance }}" + description: | + No successful getRegisteredNodes re-enumeration in two intervals. The + repair is skipped entirely while the watcher route is errored, which + emits nothing — not even the failure counter. Correlate with + decdn_staker_set_watcher_down_seconds and check the RPC provider. + + # The origin directory is a lazy on-demand cache, not a watcher, so it has + # no last-tick gauge to stale out; alert on its getOrigins failure counter + # instead. Each failure means a DHT provider-lookup miss fell back to + # resolving no origins (fail-closed), so sustained failures degrade origin + # discovery silently. Warning, not critical: normal DHT provider lookup is + # the primary path and is unaffected. See adr/appendix-observability.md §3. + - alert: DecdnOriginDirectoryLookupFailing + expr: rate(decdn_origin_directory_get_origins_failures_total[5m]) > 0 + for: 15m + labels: + component: origin + severity: warning + annotations: + runbook_url: "https://github.com/decdn/decdn/blob/main/docs/runbook.md#origin-rescan-probes-faulting" + summary: "Origin directory getOrigins lookups failing on {{ $labels.instance }}" + description: | + The lazy origin directory cannot read getOrigins from chain, so a DHT + provider-lookup miss resolves no origins (fail-closed). Sustained + failures degrade origin discovery silently. Check the RPC provider and + the OriginAssignment contract address. + + # Only meaningful when observability.otlp_endpoint is set; the counter + # stays 0 otherwise. Warning: lost traces do not affect delivery. + - alert: DecdnOtlpExportFailing + expr: increase(decdn_otlp_export_failures_total[15m]) > 0 + for: 15m + labels: + component: observability + severity: warning + annotations: + summary: "OTLP span export failing on {{ $labels.instance }}" + description: | + The node's OTLP exporter kept failing span-export batches through + the last 15 minutes, so traces are being lost. Check that the + collector at observability.otlp_endpoint is up and speaks OTLP/gRPC + (usually port 4317). The SDK also logs each failed batch at error + level under the opentelemetry_sdk target. An idle node exports no + batches, so a dead collector does not fire this until spans flow. + Queue-full drops are not counted; watch for the SDK's + BatchSpanProcessor.SpanDroppingStarted warning. + + - alert: DecdnFeeSharesWatcherStalled + expr: (decdn_fee_shares_watcher_last_tick_timestamp_seconds > 0) and (time() - decdn_fee_shares_watcher_last_tick_timestamp_seconds > 180) + for: 5m + labels: + component: watcher + severity: warning + annotations: + summary: "Fee-shares watcher not ticking on {{ $labels.instance }}" + description: | + No successful fee-shares poll tick for several intervals — the + watcher may have panicked, wedged, or exited. The node keeps + pricing its settlement share off the last split it read, which the + chain may have moved. Restart and check the RPC provider. + + # getShares() is the safety-net re-read behind the SharesUpdated event + # tail. Its failure keeps the current share and the tick still succeeds, + # so the stall alert above cannot see it. The re-read runs on a slow + # cadence (hours), hence the long window. + - alert: DecdnFeeSharesPollFailing + expr: increase(decdn_fee_shares_watcher_poll_failures_total[6h]) > 0 + for: 0m + labels: + component: watcher + severity: warning + annotations: + summary: "Fee-shares getShares() re-read failing on {{ $labels.instance }}" + description: | + The authoritative FeeRouter.getShares() re-read failed. The node + keeps the last share it read and relies on the event tail alone + until a re-read lands. Check the RPC provider. + + - alert: DecdnRpcUnhealthy + expr: decdn_rpc_healthy == 0 + for: 2m + labels: + component: chain + severity: critical + annotations: + runbook_url: "https://github.com/decdn/decdn/blob/main/docs/runbook.md#rpc-unreachable" + summary: "Chain RPC unreachable from {{ $labels.instance }}" + description: | + The RPC watchdog cannot reach blockchain.rpc_url. While it is down + the node cannot read blacklist updates (slash exposure), follow + pool events, or redeem vouchers. See docs/runbook.md § RPC + unreachable. + + - alert: DecdnConfigReloadFailed + expr: increase(decdn_config_reload_failures_total[15m]) > 0 + for: 0m + labels: + component: config + severity: warning + annotations: + summary: "Config reload failed on {{ $labels.instance }}" + description: | + A SIGHUP or admin reload failed. Sections committed before the + failure stay applied; the rest keep their previous values. The + warn! line beside it names the cause. Fix the config file and + reload again. + + - alert: DecdnReceiptLogWriteFailing + expr: increase(decdn_receipt_write_failures_total[1h]) > 0 + for: 0m + labels: + component: receipts + severity: warning + annotations: + summary: "Download-receipt log writes failing on {{ $labels.instance }}" + description: | + The receipt writer could not persist audit records. Settlement is + unaffected, but the audit log has gaps. Check disk space and + permissions under identity.data_dir. + + # NOTE: /health 503 detection requires an external prober (e.g. + # blackbox_exporter) — it is not surfaced as a decdn_* metric. Add + # an alert in your blackbox rules file along the lines of + # probe_http_status_code{job="decdn_health"} == 503 + # if you want to page on not_ready. See adr/appendix-observability.md §3. + + # --------------------------------------------------------------- + # Delivery health — operator-tunable. + # --------------------------------------------------------------- + - name: decdn-delivery + interval: 30s + rules: + # An absolute rate on the one refusal reason that means "the node is at + # fault", not a ratio over decdn_streams_{failed,completed}_total: the + # failed family counts every refusal, and every other + # decdn_serve_stream_rejected_* value is a client-side or policy outcome + # (unfunded channel, unknown channel, denied hash) that a healthy node + # produces on purpose, so a failure ratio would page on normal traffic. + - alert: DecdnServeInternalErrorRate + expr: sum by (instance) (rate(decdn_serve_stream_rejected_internal_error_total[5m])) > 0 + for: 10m + labels: + component: delivery + severity: warning + annotations: + summary: "Serve path returning internal errors on {{ $labels.instance }}" + description: | + The node is refusing paid stream requests with an internal error + rather than a policy reason. Sustained non-zero means a node-side + fault — a degraded blob store, a wedged origin, or a signing + failure — not a client problem. Check the node's error logs; the + throttled `error!` names the specific failure. + + # ADR 041 serve-economics buy refusals. Alert only on the `amortized` + # regime: it means the source's warming allowance is spent (or warming is + # off) so the ceiling is the grief-proof floor and every candidate is + # above it — this node's own sell rate or margin config is too tight, + # which is the operator-actionable half. The `warming` regime is "the + # market is above the market price", a condition no local change fixes, so + # it is dashboard-only — alerting it would page for someone else's prices. + # A raw rate, not a ratio: the refusal signs the client NotFound and never + # reaches the wire, so there is no serve-attempt counter to divide by. + - alert: DecdnServeEconomicsTooTight + expr: sum by (instance) (rate(decdn_serve_economics_refused_amortized_total[10m])) > 0.1 + for: 15m + labels: + component: delivery + severity: warning + annotations: + summary: "Serve economics too tight to relay on {{ $labels.instance }}" + description: | + This node keeps declining cache-miss relays because every upstream + quote is above its amortized buy floor — the sell rate or margin + config cannot resell the bytes for a profit. The node serves its own + content fine but relays nothing, silently (the refusal signs the + client NotFound). Raise sell_rate_base or loosen serve-economics + margin config, or confirm this node is meant to be origin-only. + + # ADR 041 / load-shed overload refusals. Alert on the two reasons a node + # can act on — node concurrency and egress saturation — summed. The third + # reason, client_at_capacity, is per-client fairness working as designed on + # a healthy node, so it is dashboard- and log-only (same treatment as + # decdn_serve_stream_rejected_signer_floor_at_cap_total); alerting it would + # page for one client's own burst. + - alert: DecdnLoadSheddingUnderPressure + expr: | + sum by (instance) ( + rate(decdn_load_shed_refused_node_at_capacity_total[5m]) + + rate(decdn_load_shed_refused_egress_saturated_total[5m]) + ) > 0.1 + for: 10m + labels: + component: delivery + severity: warning + annotations: + summary: "Shedding new serves under load on {{ $labels.instance }}" + description: | + The node is refusing new paid serves because it is at its concurrency + high-water mark or its egress budget. In-flight streams stay fast, but + new clients are turned away (NotFound, so they re-route). Check the + load-shed panels: raise max_concurrent_serves_high for concurrency, or + egress_budget_mbps / bandwidth for egress. Sustained shedding means + this node needs more capacity or a lighter share of traffic. + + # ADR 041 per-source warming grief cap. A sustained rate of speculative + # buys downgraded to the amortized floor means one or more upstream sources + # are draining their allowance faster than serves refund it — either a + # genuine dud source or a grief attempt. The cap is doing its job (the loss + # is bounded per source), so this is informational, not urgent. + - alert: DecdnWarmingGriefedDown + expr: sum by (instance) (rate(decdn_warming_speculative_blocked_total[15m])) > 0.05 + for: 30m + labels: + component: delivery + severity: info + annotations: + summary: "Warming allowance being griefed down on {{ $labels.instance }}" + description: | + Above-floor warming buys are being downgraded to the amortized floor + because upstream sources have spent their per-source allowance. The + grief cap bounds the loss, so this is not urgent, but a sustained rate + with a rising decdn_warming_sources_blocked means warming is largely + disabled in practice — investigate which sources are serving duds. + + # Removed in #1513: DecdnHashMismatchAppearing, which read + # decdn_streams_failed_total{reason="hash_mismatch"}. That family carries + # `direction` only, never a `reason`, so such a rule is permanently + # silent. Delivery hash mismatch is genuinely + # unmetered today — the cache returns CacheError::HashMismatch and the + # node logs it, but nothing counts it. This is a real coverage gap, not a + # naming fix; it is recorded in docs/runbook.md rather than papered over + # with another rule that cannot fire. + + - alert: DecdnProbeHoldsSaturated + expr: | + (decdn_probe_hold_slots_used / clamp_min(decdn_probe_hold_slots_max, 1)) > 0.9 + for: 5m + labels: + component: probe + severity: warning + annotations: + runbook_url: "https://github.com/decdn/decdn/blob/main/docs/runbook.md#slashing-risk" + summary: "Probe-hold slots >90% used on {{ $labels.instance }}" + description: | + Saturation means cached content is advertised without an eviction + hold, so it can be evicted before the pull arrives — wasted round + trips and lost deliveries. Raise max_probe_holds — on a small cache keep it + proportional to cache size (recommended: no more than 25% of cache + capacity); a production node serving many peers should scale the + default up instead. max_probe_holds is restart-required. + + # Chain-event watcher liveness (#1316) for the payment-settlement watcher + # — staleness of its last successful poll tick. 180s ≈ several poll + # intervals; tune to your event_poll_interval. The `> 0` guard skips the + # pre-first-tick sentinel (gauge reads 0 until the first success). + - alert: DecdnSettlementWatcherStalled + expr: (decdn_settlement_watcher_last_tick_timestamp_seconds > 0) and (time() - decdn_settlement_watcher_last_tick_timestamp_seconds > 180) + for: 5m + labels: + component: settlement + severity: warning + annotations: + runbook_url: "https://github.com/decdn/decdn/blob/main/docs/runbook.md#refusing-paying-clients-insufficient-deposit" + summary: "Settlement watcher not ticking on {{ $labels.instance }}" + description: | + No successful payment-settlement poll tick for several intervals — + channel open/settle/close events are not being ingested, so + voucher redemption and reconciliation stall. Restart and check the + RPC provider. + + # #1520. Three deposit guards route through this one counter (the miss floor, + # the window ceiling, the direct-serve ceiling), so it cannot say WHICH — the + # node's throttled `warn!` names that. The per-signer floor sub-cap does NOT + # route here: it has its own + # `decdn_serve_stream_rejected_signer_floor_at_cap_total`, so this rule stays + # a pool-solvency signal and its two causes below remain the whole story. + # That sibling counter is deliberately NOT alerted: a signer at its share is + # a per-counterparty condition on a solvent node, not a node-health fault, so + # it is dashboard- and log-only (see docs/runbook.md cause 3). Alerting it + # would page an operator for one client's abandoned streams. + # + # A raw rate, not a ratio, because the node exports no serve-ATTEMPT counter: + # `decdn_serve_stream_rejected_*` is a per-reason family with no success + # sibling, so there is nothing to divide by. (#1513 deleted the one rule + # that pretended otherwise — see `DecdnServeInternalErrorRate` above.) + # The threshold is therefore + # absolute and deliberately loose: any sustained refusal rate is worth a look, + # because before #1520 this counter had no alert, panel, or runbook entry at + # all and a node could refuse every paying client in silence. + - alert: DecdnRefusingPayingClients + expr: sum by (instance) (rate(decdn_serve_stream_rejected_insufficient_deposit_total[10m])) > 0.1 + for: 15m + labels: + component: payments + severity: warning + annotations: + runbook_url: "https://github.com/decdn/decdn/blob/main/docs/runbook.md#refusing-paying-clients-insufficient-deposit" + summary: "Sustained insufficient-deposit refusals on {{ $labels.instance }}" + description: | + This node is refusing delivery because the requesting channel's + remaining deposit cannot cover the reserved cost, and has been for 15 + minutes. Two causes with opposite remedies, and this series cannot tell + them apart — read the node's throttled `warn!` line, which carries the + channel and the shortfall: + (a) clients genuinely running dry, which needs no action; or + (b) this node's chain watcher lagging an on-chain `ChannelToppedUp`, in + which case funded clients are being turned away — check RPC health and + `decdn_settlement_watcher_last_tick_timestamp_seconds`. + Long `for:` because a burst of dry clients is normal. + See adr/003-payments.md and docs/runbook.md. + + - alert: DecdnRedemptionFailing + expr: increase(decdn_redemption_failures_total[30m]) > 2 + for: 30m + labels: + component: payments + severity: warning + annotations: + summary: "Voucher redemptions failing on {{ $labels.instance }}" + description: | + redeemMany keeps failing: reverted, refused at send, no receipt, a + timed-out receipt, or a lane-store read failure. + The claims stay and retry, but unredeemed revenue accrues and a + closing pool's grace window can run out. Read the redeemMany warn! + lines (each carries the tx hash when one was issued) and check the + RPC provider and the operator key's gas balance. + + - alert: DecdnWatermarkReconcileFailing + expr: increase(decdn_redemption_reconcile_failures_total[30m]) > 2 + for: 30m + labels: + component: payments + severity: warning + annotations: + summary: "Pre-redeem watermark reconciliation failing on {{ $labels.instance }}" + description: | + The getWatermarks batch read before each redeemMany keeps failing: + an RPC error, a timeout, or a return whose length did not match the + batch. + The read is fail-open, so claims are still submitted and the + contract's own no-op guard keeps them correct — the cost is gas + spent on lanes the chain already settled, not lost revenue. + Check the RPC endpoint's eth_call response-size and gas ceilings + against the read batch size, and read the stage="reconcile" warn! + lines for the underlying error. + + - alert: DecdnOnchainTxTimeouts + expr: increase(decdn_onchain_tx_timeout_total[1h]) > 0 + for: 0m + labels: + component: payments + severity: warning + annotations: + summary: "On-chain transaction receipts timing out on {{ $labels.instance }}" + description: | + A node transaction was issued but its receipt did not arrive in + time. It can still mine; look it up by the tx hash in the warn! + line. A repeat points at a congested chain or a low gas price. + + # The next four are latent-bug reports, not degradations: each non-zero + # increase is a node-side fault worth filing. increase(...[1h]) rather + # than a bare > 0, so the alert resolves an hour after the last event. + - alert: DecdnServeNodeFault + expr: increase(decdn_serve_stream_node_fault_total[1h]) > 0 + for: 0m + labels: + component: delivery + severity: warning + annotations: + summary: "Delivery abandoned on a node-side fault on {{ $labels.instance }}" + description: | + A cdn/client/v1 delivery ended on a fault this node caused (encode, + alignment, store or framing). The client saw only a short stream. + File the error! line beside it. + + - alert: DecdnFrameAccountingFault + expr: increase(decdn_serve_frame_accounting_fault_total[1h]) > 0 + for: 0m + labels: + component: delivery + severity: warning + annotations: + summary: "Serve frame accounting fault on {{ $labels.instance }}" + description: | + A serve leg refused to frame a chunk because its own byte + accounting did not add up. The delivery aborted cleanly; file it. + + - alert: DecdnWatcherPersistFailures + expr: increase(decdn_watcher_persist_failures_total[1h]) > 0 + for: 0m + labels: + component: watcher + severity: warning + annotations: + summary: "Chain watcher could not persist state on {{ $labels.instance }}" + description: | + A chain watcher failed to write its projection or lane state. Check + disk health under identity.data_dir. + + - alert: DecdnLaneFlushFailures + expr: increase(decdn_lane_flush_failures_total[1h]) > 0 + for: 15m + labels: + component: payments + severity: warning + annotations: + summary: "Lane store cannot flush on {{ $labels.instance }}" + description: | + Background lane-store flushes keep failing; the dirty set retries + each tick, but the frontier a crash would replay keeps growing. + Check disk health and space under identity.data_dir. + + - alert: DecdnNodePullCorruption + expr: increase(decdn_node_pull_corruption_total[1h]) > 0 + for: 0m + labels: + component: delivery + severity: warning + annotations: + summary: "Upstream node served corrupt bytes to {{ $labels.instance }}" + description: | + A node-to-node pull received bytes that failed bao verification. + The node rejected them and scored the upstream; a repeat from one + peer points at a faulty or dishonest node. + + - alert: DecdnPoolGraceCloseWithUnredeemed + expr: increase(decdn_pool_grace_closes_total[1h]) > 0 + for: 0m + labels: + component: payments + severity: warning + annotations: + summary: "A pool closing on {{ $labels.instance }} holds unredeemed vouchers" + description: | + A pool owner started a close while this node held unredeemed + vouchers on it. The redeemer must land them before the grace window + ends or the revenue is lost. Check that redeemMany is landing + (decdn_onchain_tx_landed_total, DecdnRedemptionFailing) and that the + operator key has gas. + + - alert: DecdnBuyerWalletUnfunded + # Scoped on `> 0`, not on the series existing: every metric is registered + # unconditionally, so a cache-only node exports + # decdn_node_pull_attempts_total at 0 and an `>= 0` guard would match it + # too — firing on exactly the nodes whose empty wallet is correct. + # Having ATTEMPTED a pull is the observable marker that this node buys. + # + # The cost is a warm-up gap: a freshly started node does not alert until + # its first miss. That is the safe direction, and an unfunded wallet is + # reached through an attempt anyway — the pull is attempted, then the + # openPool reverts — so the real condition still fires. + # + # Threshold is the 10 USDC default for + # blockchain.buyer_working_deposit_micro_usdc; raise it alongside that + # knob, which PromQL cannot read. + expr: | + decdn_buyer_wallet_usdc < 10000000 + and on(instance) (decdn_node_pull_attempts_total > 0) + for: 30m + labels: + component: payments + severity: warning + annotations: + summary: "Buyer wallet below one working deposit on {{ $labels.instance }}" + description: | + The node's own USDC wallet cannot cover + blockchain.buyer_working_deposit_micro_usdc, so the next openPool + reverts on the ERC-20 transfer and every cache miss goes to origin + instead of to a peer. The node keeps serving normally, which is why + nothing else here fires. Fund the operator address, or lower the + working deposit. Pair with + decdn_pool_open_failures_insufficient_deposit_total, which starts + counting once the wallet is actually empty. + + - alert: DecdnNodePullNeverSucceeds + # attempts without a single success over a long window. Not a rate + # threshold: one success clears it, so this only fires on a leg that is + # wholly broken rather than merely unlucky. + expr: | + increase(decdn_node_pull_attempts_total[6h]) > 10 + and increase(decdn_node_pull_success_total[6h]) == 0 + for: 30m + labels: + component: delivery + severity: warning + annotations: + summary: "No node-to-node pull has succeeded on {{ $labels.instance }} in 6h" + description: | + The node keeps trying to backfill cache misses from peers and never + completes one, so every miss falls through to origin and the fleet + pays origin egress repeatedly for blobs it already holds. Check + decdn_pool_open_failures_{insufficient_deposit,contract_revert,rpc_error}_total + for the buyer leg, decdn_node_pull_no_providers_total for discovery, + and decdn_buyer_wallet_usdc for funding. + + - alert: DecdnDhtRoutingTableEmpty + expr: decdn_dht_routing_table_size == 0 + for: 30m + labels: + component: dht + severity: warning + annotations: + summary: "DHT routing table empty on {{ $labels.instance }}" + description: | + The node knows no DHT peers after bootstrap and refresh, so it + cannot discover providers or publish its content. Check network + reachability and that the staker set lists live seeds. The gauge is + set after bootstrap and on each bucket-refresh tick. diff --git a/charts/decdn-node/templates/dashboards-configmap.yaml b/charts/decdn-node/templates/dashboards-configmap.yaml new file mode 100644 index 0000000..f277fc2 --- /dev/null +++ b/charts/decdn-node/templates/dashboards-configmap.yaml @@ -0,0 +1,39 @@ +{{- /* +Upstream's Grafana dashboards (files/monitoring/*.json, vendored by +scripts/sync-monitoring.sh), one ConfigMap each, labelled for the Grafana sidecar +(kube-prometheus-stack's grafana.sidecar.dashboards). The dashboards pick their +datasources through DS_PROMETHEUS / DS_LOKI / DS_TEMPO template variables, so +nothing is rewritten here. Dashboard uids are fixed upstream: enable this on ONE +release per Grafana. +*/}} +{{- if .Values.metrics.grafanaDashboards.enabled }} +{{- $cfg := .Values.metrics.grafanaDashboards }} +{{- $dashboards := .Files.Glob "files/monitoring/*.json" }} +{{- if not $dashboards }} +{{- fail "metrics.grafanaDashboards.enabled, but files/monitoring/ holds no *.json dashboards; re-run scripts/sync-monitoring.sh" }} +{{- end }} +{{- range $path, $_ := $dashboards }} +{{- $file := base $path }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + {{- /* No trunc 63: ConfigMap names may be 253 characters, and truncating the + suffix made dashboards collide once the fullname passed ~52 characters. */}} + name: {{ printf "%s-%s" (include "decdn-node.fullname" $) (trimSuffix ".json" $file) }} + {{- with $cfg.namespace }} + namespace: {{ . }} + {{- end }} + labels: + {{- include "decdn-node.labels" $ | nindent 4 }} + app.kubernetes.io/component: dashboard + {{ $cfg.label }}: {{ $cfg.labelValue | quote }} + {{- with $cfg.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +data: + {{ $file }}: |- + {{- $.Files.Get $path | nindent 4 }} +{{- end }} +{{- end }} diff --git a/charts/decdn-node/templates/prometheusrule.yaml b/charts/decdn-node/templates/prometheusrule.yaml new file mode 100644 index 0000000..60488ad --- /dev/null +++ b/charts/decdn-node/templates/prometheusrule.yaml @@ -0,0 +1,40 @@ +{{- /* +Upstream's reference alert rules (files/monitoring/prometheus-alerts.yml, vendored +by scripts/sync-monitoring.sh) as a PrometheusRule. The file is read raw and parsed +as YAML: never pass it through `tpl`, its annotations carry Prometheus templates +({{ $labels.instance }}) that are Prometheus's to expand, not Helm's. +The rules match on metric names across every scraped node, so enable this on ONE +release per Prometheus, or each alert fires once per release. +*/}} +{{- if .Values.metrics.prometheusRule.enabled }} +{{- $rules := .Files.Get "files/monitoring/prometheus-alerts.yml" | fromYaml }} +{{- if hasKey $rules "Error" }} +{{- fail (printf "files/monitoring/prometheus-alerts.yml does not parse: %s" $rules.Error) }} +{{- end }} +{{- if not $rules.groups }} +{{- fail "files/monitoring/prometheus-alerts.yml has no rule groups; re-run scripts/sync-monitoring.sh" }} +{{- end }} +{{- $extra := .Values.metrics.prometheusRule.ruleLabels }} +{{- if $extra }} +{{- range $group := $rules.groups }} +{{- range $rule := $group.rules }} +{{- /* A rule's own labels (severity, component, ...) win over the extras. */}} +{{- $_ := set $rule "labels" (merge (deepCopy ($rule.labels | default dict)) $extra) }} +{{- end }} +{{- end }} +{{- end }} +--- +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ include "decdn-node.fullname" . }} + labels: + {{- include "decdn-node.labels" . | nindent 4 }} + app.kubernetes.io/component: alerts + {{- with .Values.metrics.prometheusRule.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + groups: + {{- toYaml $rules.groups | nindent 4 }} +{{- end }} diff --git a/charts/decdn-node/templates/servicemonitor.yaml b/charts/decdn-node/templates/servicemonitor.yaml index 34380fc..efc169a 100644 --- a/charts/decdn-node/templates/servicemonitor.yaml +++ b/charts/decdn-node/templates/servicemonitor.yaml @@ -24,4 +24,24 @@ spec: path: /metrics interval: {{ .Values.metrics.serviceMonitor.interval }} scrapeTimeout: {{ .Values.metrics.serviceMonitor.scrapeTimeout }} + # Target labels the upstream dashboards and alerts key on (the node does not + # emit them itself): a stable job, the node's region and an optional + # deployment_environment, plus instance = pod name so $instance reads as a + # node, not an IP:port. grafana_alloy stamps the same set on the Ansible path. + relabelings: + {{- with .Values.metrics.serviceMonitor.jobLabel }} + - targetLabel: job + replacement: {{ . | quote }} + {{- end }} + - targetLabel: region + replacement: {{ .Values.config.identity.region | quote }} + {{- with .Values.metrics.serviceMonitor.deploymentEnvironment }} + - targetLabel: deployment_environment + replacement: {{ . | quote }} + {{- end }} + - sourceLabels: [__meta_kubernetes_pod_name] + targetLabel: instance + {{- with .Values.metrics.serviceMonitor.relabelings }} + {{- toYaml . | nindent 8 }} + {{- end }} {{- end }} diff --git a/charts/decdn-node/tests/render-test.sh b/charts/decdn-node/tests/render-test.sh index 1c38bfe..ec6e17d 100755 --- a/charts/decdn-node/tests/render-test.sh +++ b/charts/decdn-node/tests/render-test.sh @@ -53,14 +53,18 @@ for values in "$chart"/ci/*.yaml; do helm lint --strict --quiet "$chart" -f "$values" >/dev/null \ || { helm lint --strict "$chart" -f "$values" >&2 || true; fail "helm lint ($name)"; } helm template t "$chart" -f "$values" > "$work/$name.yaml" || fail "helm template ($name)" - yq 'select(.kind == "ConfigMap") | .data["node.toml"]' "$work/$name.yaml" > "$work/$name.toml" + # Filter on the key, not just the kind: the dashboard ConfigMaps would otherwise + # contribute `null` documents here. + yq 'select(.kind == "ConfigMap" and .data["node.toml"] != null) | .data["node.toml"]' \ + "$work/$name.yaml" > "$work/$name.toml" [ -s "$work/$name.toml" ] || fail "no node.toml in the rendered ConfigMap ($name)" python3 "$checker" "$work/$name.toml" >/dev/null 2>"$work/$name.err" \ || { cat "$work/$name.err" >&2; fail "schema keys ($name)"; } if [ -n "$kubeconform" ]; then - # Only the ServiceMonitor CRD lacks a bundled schema; skip it by kind, so any - # other unrecognised resource (e.g. a typo'd kind) still fails. - $kubeconform -strict -summary -skip ServiceMonitor -kubernetes-version 1.30.0 \ + # Only the Prometheus Operator CRDs (ServiceMonitor, PrometheusRule) lack a + # bundled schema; skip them by kind, so any other unrecognised resource (e.g. a + # typo'd kind) still fails. Their shape is checked in the invariants below. + $kubeconform -strict -summary -skip ServiceMonitor,PrometheusRule -kubernetes-version 1.30.0 \ < "$work/$name.yaml" > "$work/$name.kc" 2>&1 \ || { cat "$work/$name.kc" >&2; fail "kubeconform ($name)"; } fi @@ -114,7 +118,7 @@ grep -qx 'node_to_node_pull_through_enabled = false' "$work/ci-origins.toml" || grep -qx 'node_to_node_pull_through_enabled = true' "$work/ci-resolve-only.toml" || fail "pull-through without origin should derive true" helm template t "$chart" -f "$chart/ci/ci-resolve-only.yaml" \ --set config.cache.node_to_node_pull_through_enabled=false \ - | yq 'select(.kind == "ConfigMap") | .data["node.toml"]' \ + | yq 'select(.kind == "ConfigMap" and .data["node.toml"] != null) | .data["node.toml"]' \ | grep -qx 'node_to_node_pull_through_enabled = false' || fail "explicit pull-through=false should win" pass "pull-through derivation" @@ -126,9 +130,12 @@ pass "resolve-only discovery" # --- workload and exposure invariants, per render -------------------------------- # Explicit checks rather than `assert`, which PYTHONOPTIMIZE would turn into no-ops. +# The vendored monitoring files the ci-values render must carry in full. +yq -o=json '.' "$chart/files/monitoring/prometheus-alerts.yml" > "$work/alerts.json" for name in ci-values ci-origins ci-resolve-only; do yq ea -o=json '[.]' "$work/$name.yaml" > "$work/$name.json" - python3 - "$work/$name.json" "$work/$name.toml" "$chart/ci/$name.yaml" <<'PY' || fail "workload/exposure invariants ($name)" + python3 - "$work/$name.json" "$work/$name.toml" "$chart/ci/$name.yaml" \ + "$work/alerts.json" "$chart/files/monitoring" <<'PY' || fail "workload/exposure invariants ($name)" import json, sys, tomllib docs = [d for d in json.load(open(sys.argv[1])) if d] cfg = tomllib.load(open(sys.argv[2], "rb")) @@ -205,6 +212,41 @@ if name == "ci-values.yaml": check("hostPort" not in ports["quic"], "hostPort open by default") check(len(tcp_rules) == 1, "metrics rule missing although metrics.networkPolicy.from is set") check(one("ServiceMonitor") is not None, "ServiceMonitor missing") + # Target labels the upstream dashboards/alerts select on. + relabel = {r.get("targetLabel"): r for r in one("ServiceMonitor")["spec"]["endpoints"][0].get("relabelings", [])} + check(relabel.get("job", {}).get("replacement") == "decdn-node", "ServiceMonitor job relabel") + check(relabel.get("region", {}).get("replacement") == cfg["identity"]["region"], "ServiceMonitor region relabel") + check(relabel.get("deployment_environment", {}).get("replacement") == "ci", "ServiceMonitor deployment_environment relabel") + check(relabel.get("instance", {}).get("sourceLabels") == ["__meta_kubernetes_pod_name"], "ServiceMonitor instance relabel") + # PrometheusRule: every vendored rule, extras merged in, the rule's own labels kept. + vendored = [r for g in json.load(open(sys.argv[4]))["groups"] for r in g["rules"]] + pr = one("PrometheusRule", "alerts") + check(pr is not None, "PrometheusRule missing") + if pr: + rules = [r for g in pr["spec"]["groups"] for r in g["rules"]] + check(len(rules) == len(vendored) > 0, f"PrometheusRule has {len(rules)} rules, vendored file {len(vendored)}") + check(all(r["labels"].get("team") == "node-ops" for r in rules), "ruleLabels not merged into every rule") + # ci-values sets ruleLabels.severity: a rule's own severity must win, and + # only a rule without one may take the extra. + check([r["labels"].get("severity") for r in rules] + == [r.get("labels", {}).get("severity", "overridden") for r in vendored], + "ruleLabels overrode a rule's own labels") + # One sidecar-labelled ConfigMap per vendored dashboard, each valid JSON with a uid. + import pathlib + want = sorted(p.name for p in pathlib.Path(sys.argv[5]).glob("*.json")) + check(want, "no vendored dashboards in files/monitoring/") + cms = [d for d in docs if d["kind"] == "ConfigMap" + and d["metadata"]["labels"].get("app.kubernetes.io/component") == "dashboard"] + check(sorted(k for d in cms for k in d["data"]) == want, "dashboard ConfigMaps vs vendored files") + for d in cms: + check(d["metadata"]["labels"].get("grafana_dashboard") == "1", "dashboard sidecar label") + for v in d["data"].values(): + check("uid" in json.loads(v), "dashboard without a uid") +else: + check(not [d for d in docs if d["kind"] == "PrometheusRule"], "PrometheusRule rendered while disabled") + check(not [d for d in docs if d["kind"] == "ConfigMap" + and d["metadata"]["labels"].get("app.kubernetes.io/component") == "dashboard"], + "dashboard ConfigMaps rendered while disabled") if name == "ci-origins.yaml": check(ports["quic"].get("hostPort") == 5000, "hostPort") check(qsvc["spec"]["ports"][0].get("nodePort") == 30443, "nodePort") @@ -283,6 +325,27 @@ expect_fail "writable root filesystem" template 'readOnlyRootFilesystem must expect_fail "capabilities added" template 'capabilities.add must be empty' --set 'securityContext.capabilities.add[0]=NET_ADMIN' expect_fail "capabilities not dropped" template 'capabilities.drop must include ALL' --set-json 'securityContext.capabilities.drop=["NET_RAW"]' expect_fail "ServiceMonitor, policy blocks" template 'metrics.networkPolicy.from' --set metrics.serviceMonitor.enabled=true --set networkPolicy.enabled=true +expect_fail "non-string rule label" schema '/metrics/prometheusRule/ruleLabels' --set-json 'metrics.prometheusRule.ruleLabels={"team":1}' +expect_fail "invalid dashboard label key" schema '/metrics/grafanaDashboards/label' --set 'metrics.grafanaDashboards.label=not a label' +expect_fail "unknown monitoring knob" schema "additional propert(y|ies) 'rules'" --set metrics.prometheusRule.rules=x + +# Long release names: every rendered object keeps a unique name. The dashboard +# ConfigMaps used to truncate to 63 characters and collide. +long="$(printf 'n%.0s' $(seq 1 63))" +helm template t "$chart" -f "$chart/ci/ci-values.yaml" --set fullnameOverride="$long" \ + | yq -N '.kind + "/" + .metadata.name' | grep -v '^null' | sort | uniq -d > "$work/dupes" +[ ! -s "$work/dupes" ] || { cat "$work/dupes" >&2; fail "duplicate object names with a 63-character fullname"; } +pass "unique object names with a 63-character fullname" + +# Dashboards enabled with none vendored must fail the render, not render nothing. +nodash="$work/nodash" +cp -r "$chart" "$nodash" +rm -f "$nodash"/files/monitoring/*.json +if out="$(helm template t "$nodash" -f "$chart/ci/ci-values.yaml" 2>&1)"; then + fail "render should fail: dashboards enabled, none vendored" +fi +grep -q 'holds no \*.json dashboards' <<<"$out" || { echo "$out" >&2; fail "wrong failure for missing dashboards"; } +pass "rejects: dashboards enabled, none vendored" # --- optional: the real binary -------------------------------------------------- if [ -n "${DECDN_CLI:-}" ]; then diff --git a/charts/decdn-node/values.schema.json b/charts/decdn-node/values.schema.json index 283be15..3b75d64 100644 --- a/charts/decdn-node/values.schema.json +++ b/charts/decdn-node/values.schema.json @@ -286,6 +286,57 @@ }, "labels": { "$ref": "#/definitions/stringMap" + }, + "jobLabel": { + "type": "string" + }, + "deploymentEnvironment": { + "type": "string" + }, + "relabelings": { + "type": "array", + "items": { + "type": "object" + } + } + } + }, + "prometheusRule": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean" + }, + "labels": { + "$ref": "#/definitions/stringMap" + }, + "ruleLabels": { + "$ref": "#/definitions/stringMap" + } + } + }, + "grafanaDashboards": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean" + }, + "label": { + "type": "string", + "pattern": "^([a-z0-9]([-a-z0-9.]*[a-z0-9])?/)?[A-Za-z0-9]([-A-Za-z0-9_.]*[A-Za-z0-9])?$", + "maxLength": 316 + }, + "labelValue": { + "type": "string", + "maxLength": 63 + }, + "namespace": { + "type": "string" + }, + "annotations": { + "$ref": "#/definitions/stringMap" } } } diff --git a/charts/decdn-node/values.yaml b/charts/decdn-node/values.yaml index 194586c..a5b4c5f 100644 --- a/charts/decdn-node/values.yaml +++ b/charts/decdn-node/values.yaml @@ -91,6 +91,30 @@ metrics: interval: 30s scrapeTimeout: 10s labels: {} + # Target labels for the upstream dashboards/alerts (files/monitoring/): job is + # what they select on ("" keeps the operator's default, the Service name); + # region comes from config.identity.region; deployment_environment is omitted + # when empty. Extra relabelings are appended after these. + jobLabel: decdn-node + deploymentEnvironment: "" + relabelings: [] + # Upstream's reference alert rules as a PrometheusRule (needs the Prometheus + # Operator CRDs). They select across every scraped node: enable on ONE release + # per Prometheus. ruleLabels are added to every rule (a rule's own labels win), + # e.g. to route them; labels go on the PrometheusRule object (ruleSelector). + prometheusRule: + enabled: false + labels: {} + ruleLabels: {} + # Upstream's Grafana dashboards, one ConfigMap each, labelled for the Grafana + # sidecar (label/labelValue must match its dashboards.label[Value]). Fixed uids: + # enable on ONE release per Grafana. namespace "" = the release namespace. + grafanaDashboards: + enabled: false + label: grafana_dashboard + labelValue: "1" + namespace: "" + annotations: {} networkPolicy: enabled: true diff --git a/compose/.env.example b/compose/.env.example new file mode 100644 index 0000000..88849b0 --- /dev/null +++ b/compose/.env.example @@ -0,0 +1,17 @@ +# Copy to .env beside compose.yaml. Nothing here is secret. + +# The node image, always pinned by digest (compose.yaml builds REPO@DIGEST). Take the +# digest from the release's signed image-digest.txt (verify its .asc against the +# maintainer key in decdn/decdn's KEYS), not from whatever a tag points at today. +# DECDN_IMAGE_REPO defaults to ghcr.io/decdn/decdn-node; override it for a mirror or a +# local registry (compose/README.md, "before a release"). +DECDN_IMAGE_DIGEST=sha256:0000000000000000000000000000000000000000000000000000000000000000 +# DECDN_IMAGE_REPO=ghcr.io/decdn/decdn-node + +# uid/gid of the host's dedicated `decdn` system account, which owns /var/lib/decdn +# and can read /etc/decdn/keystore.password: `id -u decdn`, `id -g decdn`. +DECDN_UID=999 +DECDN_GID=999 + +# Host path of the 0600 environment file holding DECDN_RPC_URL. +DECDN_ENV_FILE=/etc/decdn/decdn.env diff --git a/compose/README.md b/compose/README.md new file mode 100644 index 0000000..27cada9 --- /dev/null +++ b/compose/README.md @@ -0,0 +1,162 @@ +# deCDN node with Docker Compose + +The same node the Ansible role deploys, on one host, without Ansible: the upstream +image (`ghcr.io/decdn/decdn-node`, amd64 and arm64) under Compose, with the same host +paths, the same start command and the same hardening as the role's systemd unit. + +Pick this path for a single machine you already run Docker on. For a fleet, or a host +you want hardened from scratch (firewall, SSH, auto-patching), use the +[Ansible project](../ansible/README.md); on Kubernetes, the +[Helm chart](../charts/decdn-node/README.md). [`docs/requirements.md`](../docs/requirements.md) +compares the three. + +> **Upstream has not published a release yet**, so there is no signed image to pin. +> `compose.yaml` only takes an image by digest, and a locally built image has no +> digest until it is pushed somewhere. Until a release exists, see +> [Before a release](#before-a-release-a-local-image). + +## How it is laid out + +| Host path | In the container | Holds | +|-----------|------------------|-------| +| `/etc/decdn/node.toml` | same, read-only | node config, no secrets | +| `/etc/decdn/keystore.password` | same, read-only | keystore password (`decdn`, `0600`) | +| `/etc/decdn/decdn.env` | read by Docker, injected as env | `DECDN_RPC_URL` (`root`, `0600`) | +| `/var/lib/decdn/` | same, read-write | `node.secret`, `keystore.json`, state, cache | + +The layout matches the Ansible role, so the host `decdn` CLI, backups and restores +([`docs/lifecycle.md`](../docs/lifecycle.md)) work the same way on both. + +**Networking.** The container uses the host network. The daemon's metrics +(`127.0.0.1:9090`) and admin RPC (`127.0.0.1:9191`, hard-wired upstream) stay on the +host's loopback, and Docker publishes no ports, so its iptables rules never open +anything past your firewall. The only public port is QUIC **udp/4433**. + +## Set up + +1. **A system account and directories:** + + ```bash + sudo useradd --system --home-dir /var/lib/decdn --shell /usr/sbin/nologin decdn + sudo install -d -m 0700 -o decdn -g decdn /var/lib/decdn + sudo install -d -m 0750 -o decdn -g decdn /etc/decdn + ``` + +2. **The `decdn` CLI on the host.** The image is daemon-only. Install the CLI from the + release tarball, checking it against the GPG-signed `SHA256SUMS` first (the + maintainer key is in `decdn/decdn`'s `KEYS`; its fingerprint is in + [SECURITY.md](../SECURITY.md#release-verification)). Until a release exists, build it: + `cargo build --release -p decdn-cli`. + +3. **Keys.** Create the password file first (`key-gen` reads it, never creates it), + then generate the node key and eth keystore into the data dir: + + ```bash + umask 077 + openssl rand -base64 32 | sudo -u decdn tee /etc/decdn/keystore.password >/dev/null + sudo -u decdn decdn key-gen --output-dir /var/lib/decdn \ + --keystore-password-file /etc/decdn/keystore.password + ``` + + Back them up now ([`docs/lifecycle.md`](../docs/lifecycle.md#backup-make-backup)). + +4. **Config.** Let the CLI write it with the chain's contract addresses already filled + in, then point it at `/var/lib/decdn` and set the node's region (ISO 3166-1 + alpha-2). Add `--origin ` to `config init` if this node has a backing origin. + + ```bash + sudo -u decdn decdn config init --chain arbitrum-sepolia --output /etc/decdn/node.toml + sudo -u decdn sed -i \ + -e 's|^# data_dir = .*|data_dir = "/var/lib/decdn"|' \ + -e '0,/^# region = /s|^# region = .*|region = "DE"|' /etc/decdn/node.toml + ``` + +5. **The RPC endpoint**, which may embed an API key, goes in a root-only env file: + + ```bash + sudo install -m 0600 -o root -g root compose/decdn.env.example /etc/decdn/decdn.env + sudoedit /etc/decdn/decdn.env # DECDN_RPC_URL=https://… + ``` + +6. **Check the config** against the CLI, with the same environment the daemon gets: + + ```bash + sudo systemd-run --pty --wait --collect -p User=decdn -p EnvironmentFile=/etc/decdn/decdn.env \ + /usr/local/bin/decdn config validate --config /etc/decdn/node.toml \ + --keystore-password-file /etc/decdn/keystore.password + ``` + +7. **Firewall.** Allow inbound **udp/4433** on the host firewall *and* in your cloud + provider's security group. Nothing else needs to be open. + +8. **Start it:** + + ```bash + cp compose/.env.example compose/.env + $EDITOR compose/.env # DECDN_IMAGE_DIGEST; DECDN_UID/GID = `id -u decdn` / `id -g decdn` + sudo docker compose -f compose/compose.yaml up -d + curl -s 127.0.0.1:9090/metrics | head # once "node runtime ready" is in the logs + decdn node health # admin RPC, from the host + ``` + +9. **Stake and register** (on-chain onboarding, ADR 019 Phase 2) with `decdn setup`, + described in + [`roles/decdn_node/README.md` § On-chain onboarding](../ansible/roles/decdn_node/README.md#on-chain-onboarding). + The CLI does not read `DECDN_RPC_URL`, and `node.toml` still names the public RPC + `config init` wrote, so run it through the `decdn_chain` helper in + [`docs/lifecycle.md` § Running on-chain commands](../docs/lifecycle.md#running-on-chain-commands), + which passes your endpoint as `--rpc-url`. The node serves paid traffic only after + that. + +## Operate + +- **Logs:** `sudo docker compose -f compose/compose.yaml logs -f`. To send them to the + journal like the systemd unit, switch the `logging` driver (commented in + `compose.yaml`). +- **Stop:** `docker compose stop`. It sends SIGTERM, the daemon's graceful drain, and + waits up to 300 s. Do not use `decdn node drain` here: `restart: unless-stopped` + starts the drained container again. +- **Upgrade:** set the new release's digest as `DECDN_IMAGE_DIGEST` in `.env`, then + `docker compose up -d`. +- **Config change:** edit `/etc/decdn/node.toml`, then `docker compose restart` + (or `decdn node reload` for the hot-reloadable sections). +- **Health:** there is no container healthcheck. The image has no HTTP client, and the + metrics listener serves only `/metrics`. Probe `http://127.0.0.1:9090/metrics` from + the host, or ship metrics with Grafana Alloy / Prometheus. Dashboards and alert rules + are in [`charts/decdn-node/files/monitoring/`](../charts/decdn-node/files/monitoring/README.md). + +## Before a release: a local image + +Build the daemon image from a `decdn/decdn` checkout (its `Dockerfile` header shows +how) and push it to a registry on the host's loopback, which gives it a digest: + +```bash +sudo docker run -d --name registry --restart unless-stopped -p 127.0.0.1:5000:5000 registry:2 +sudo docker build -t 127.0.0.1:5000/decdn-node:dev # after staging dist// +sudo docker push 127.0.0.1:5000/decdn-node:dev # prints "dev: digest: sha256:… size: …" +``` + +Then in `.env`: + +```bash +DECDN_IMAGE_REPO=127.0.0.1:5000/decdn-node +DECDN_IMAGE_DIGEST=sha256:… # from the push output +``` + +The upstream image is based on Debian bookworm (glibc 2.36). A `decdn-node` built +natively on a newer distribution will not start in it; build with `cross`, as the +upstream release does, or on bookworm. + +## Security notes + +- The container runs as the host's `decdn` account with a read-only root filesystem, + every capability dropped and `no-new-privileges`. +- The two remaining KICS findings are the design, not an oversight: host networking + (loopback-only metrics and admin, no Docker-published ports) and no healthcheck (see + above). The read-only `/etc/decdn` mount is excluded from the scan for the reason + given in the root `Makefile`. +- The image is always referenced by digest: `compose.yaml` builds + `DECDN_IMAGE_REPO@DECDN_IMAGE_DIGEST` itself, so no `.env` value can turn it into a + mutable tag. +- `make lint-compose` (CI job `compose`) renders this file and fails if any of those + properties regress; `make test-scripts` checks it rejects broken variants. diff --git a/compose/compose.yaml b/compose/compose.yaml new file mode 100644 index 0000000..ed11e4a --- /dev/null +++ b/compose/compose.yaml @@ -0,0 +1,61 @@ +# deCDN node under Docker Compose: the same daemon, paths and hardening as the +# Ansible role's systemd unit, for a single host without Ansible. See README.md. +# +# cp .env.example .env && $EDITOR .env # image digest + the decdn user's uid/gid +# sudo docker compose up -d +# +# Host networking, deliberately. The daemon binds metrics to 127.0.0.1 by default +# and its admin RPC is hard-wired to 127.0.0.1, so on the host network both stay +# loopback-only, exactly as on a systemd host, and the host's `decdn` CLI reaches +# the admin RPC directly. There are no `ports:` for Docker to publish, which also +# means Docker's own iptables rules never punch a hole past the host firewall. The +# one public port is the node's QUIC udp/4433: open it in the host firewall. +name: decdn + +services: + decdn-node: + # Always by digest, structurally: the reference is built here as repo@digest, so + # no .env can swap in a mutable tag. The release's signed image-digest.txt names + # the digest; upstream only tags an image after a maintainer signs the release. + image: ${DECDN_IMAGE_REPO:-ghcr.io/decdn/decdn-node}@${DECDN_IMAGE_DIGEST:?set DECDN_IMAGE_DIGEST in .env to sha256:} + network_mode: host + # A dedicated host account (see README.md), not the image's uid 1000, which on + # most hosts is the first human login. + user: "${DECDN_UID:?set DECDN_UID in .env}:${DECDN_GID:?set DECDN_GID in .env}" + # Mirrors the systemd unit's ExecStart. + command: + - run + - --config + - /etc/decdn/node.toml + - --keystore-password-file + - /etc/decdn/keystore.password + # The RPC URL (it may embed an API key) stays in a root-owned 0600 file on the + # host. DECDN_* variables override node.toml, so only put DECDN_RPC_URL (and + # e.g. AWS_* for an S3 origin) here. + env_file: + - path: ${DECDN_ENV_FILE:-/etc/decdn/decdn.env} + required: true + volumes: + - /etc/decdn:/etc/decdn:ro + - /var/lib/decdn:/var/lib/decdn + read_only: true + tmpfs: + - /tmp + cap_drop: [ALL] + security_opt: + - no-new-privileges:true + # SIGTERM is the daemon's graceful drain. Docker's default 10 s grace would + # SIGKILL it mid-drain; this matches the unit's TimeoutStopSec. + stop_signal: SIGTERM + stop_grace_period: 300s + restart: unless-stopped + logging: + driver: json-file + options: + max-size: 50m + max-file: "5" + # Or send the daemon's JSON logs to the host journal, like the systemd unit: + # driver: journald + # options: {tag: decdn-node} + # No healthcheck: the image has no HTTP client, and the metrics listener only + # serves /metrics. Probe http://127.0.0.1:9090/metrics from the host instead. diff --git a/compose/decdn.env.example b/compose/decdn.env.example new file mode 100644 index 0000000..a5cec2a --- /dev/null +++ b/compose/decdn.env.example @@ -0,0 +1,10 @@ +# Template for /etc/decdn/decdn.env (root:root 0600). Docker reads it as root before +# starting the container, so it never needs to be readable by the decdn account. +# One KEY=value per line, no quotes, no `export`. +# +# The RPC endpoint. It may embed a provider API key, which is why it lives here and +# not in node.toml. Overrides blockchain.rpc_url. +DECDN_RPC_URL= +# For an S3 cache origin with credentials from the environment: +# AWS_ACCESS_KEY_ID= +# AWS_SECRET_ACCESS_KEY= diff --git a/docs/lifecycle.md b/docs/lifecycle.md new file mode 100644 index 0000000..fccdfc4 --- /dev/null +++ b/docs/lifecycle.md @@ -0,0 +1,200 @@ +# Node lifecycle: backup, restore, migration, decommission + +Day-2 procedures for a running deCDN node. The Ansible targets run from `ansible/`; +the manual commands work for any deploy path. + +## What makes a node *that* node + +| File | What it is | Lose it and… | +|------|-----------|--------------| +| `/var/lib/decdn/node.secret` | iroh node key (the wire identity) | peers see a new node; the on-chain binding points at a key you no longer hold | +| `/var/lib/decdn/keystore.json` | operator Ethereum key, encrypted | you cannot unbond, withdraw or appeal a slash: **the bond is stranded** | +| `/etc/decdn/keystore.password` | password for the keystore | same as losing the keystore | +| `/var/lib/decdn/*.redb`, receipts | daemon state, including vouchers not yet redeemed | unredeemed revenue and settlement progress | +| `/var/lib/decdn/cache/` | cached blobs | nothing lasting: it refills from origins and peers | + +Back up the first three the day a node is created, and again before any host change. + +## Backup (`make backup`) + +Backups are encrypted **on the node** to public keys you choose, so plaintext key +material never leaves the host and nothing secret is needed on the control machine. +Generate a key pair once, somewhere that is neither the node nor this repo: + +```bash +age-keygen -o ~/.config/decdn/backup-age.key # keep this file safe and offline-capable +age-keygen -y ~/.config/decdn/backup-age.key # prints the public recipient, age1… +``` + +Put the public key(s) in inventory (SSH public keys work too): + +```yaml +# group_vars/decdn_nodes.yml +decdn_backup_age_recipients: + - age1… # yours + - ssh-ed25519 AAAA… # a second custodian, optional +``` + +```bash +cd ansible +make backup LIMIT= # identity, hot +make backup LIMIT= ANSIBLE_ARGS='-e decdn_backup_scope=full' # stops the node for the copy +``` + +| Scope | Archives | Node downtime | +|-------|----------|---------------| +| `identity` (default) | `node.secret`, `keystore.json`, `keystore.password` | none | +| `full` | the whole data dir except the cache, plus the keystore (wherever it lives) and the password file | the copy (the redb stores are only consistent at rest); restarted afterwards if it was up | + +Add `-e decdn_backup_include_env=true` to include `decdn.env` (your RPC URL, which +may embed an API key). The archive is written to `/var/backups/decdn/` on the host +(`root` `0600`). `make backup` runs one host at a time. + +- **Identity** archives (kilobytes) are fetched to `ansible/backups//`, which is + git-ignored. +- **Full** archives are **not** fetched by default: Ansible's `fetch` would read the + whole file into memory on both ends. The run prints a streaming copy command instead + (`ssh sudo cat > `). Force a fetch with + `-e decdn_backup_fetch=true` if you know the archive is small. + +If the archive step fails (a bad recipient, a full disk), the node is restarted, no +partial file is left behind and the run fails. `decdn_backup_leave_stopped` only +applies to a backup that succeeded. + +**Test the restore path, not just the backup:** + +```bash +age -d -i ~/.config/decdn/backup-age.key ansible/backups//.tar.age | tar -tz +``` + +Without Ansible (Compose or a hand-built host), the same archive is one command on +the node: + +```bash +sudo tar -C / --numeric-owner -czf - var/lib/decdn/node.secret var/lib/decdn/keystore.json \ + etc/decdn/keystore.password | age -r age1… -o node-identity.tar.age +``` + +## Restore and host migration + +Run two hosts with the same `node.secret` and keystore and they will fight over one +identity on the network and on-chain. **Stop the old host first.** + +1. **Freeze the old host** and take a full backup in the same step: + + ```bash + make backup LIMIT=old-host ANSIBLE_ARGS='-e decdn_backup_scope=full -e decdn_backup_leave_stopped=true' + ``` + + Then make sure it cannot come back on a reboot. Either run + `make decommission LIMIT=old-host` (it removes the node's service and this repo's + Alloy agent and keeps the data; ignore the on-chain exit steps it prints, since the + identity is moving, not leaving) or run `sudo systemctl disable --now decdn-node` + on it. + +2. **Prepare the new host** with a normal deploy. Do *not* set + `decdn_node_generate_keystore: true` for it. The deploy creates the `decdn` user + and directories, then stops at the identity gate because the keys are not there + yet. That is expected. + + ```bash + make deploy LIMIT=new-host ANSIBLE_ARGS='-u root' # first converge of a fresh box + ``` + +3. **Restore**, streaming the archive from the old host through your workstation's + `age` straight into the new host, so the plaintext never touches a disk off the + target (the full archive stays on the old host; step 1 printed its path): + + ```bash + ssh old-host sudo cat /var/backups/decdn/.tar.age \ + | age -d -i ~/.config/decdn/backup-age.key \ + | ssh new-host 'sudo tar -xzf - -C / --no-same-owner \ + && sudo chown -R decdn:decdn /var/lib/decdn /etc/decdn/keystore.password' + ``` + + `--no-same-owner` plus the `chown` matter: the `decdn` user's uid on the new host + need not match the old one. The next deploy re-applies `0600` to the key files. + +4. **Converge** the new host: `make deploy LIMIT=new-host`. + +5. **Update what is on-chain**, if it changed. The node is registered with its + multiaddr and region. A new public IP needs `decdn node update-multiaddrs`; a new + country needs `decdn node update-region`. With `decdn_chain` from + [Running on-chain commands](#running-on-chain-commands), preview first: + + ```bash + decdn_chain node update-multiaddrs --multiaddr /ip4//udp/4433/quic-v1 --dry-run + ``` + + Then the same command without `--dry-run`. The address set you pass replaces the + one on-chain. + +## Decommission (`make decommission`) + +```bash +make backup LIMIT= # first, always +make decommission LIMIT= # LIMIT is required; you type the host name to confirm +``` + +It stops `decdn-node` with `systemctl` (SIGTERM, which is the daemon's graceful drain +path), disables it and removes the unit, and tears down the Grafana Alloy agent this +repo installed, if any. `-e decdn_decommission_purge_cache=true` also deletes the +cache. It refuses to run on more than one host unless you raise +`decdn_decommission_max_hosts`, and an unanswered confirmation prompt fails after +`decdn_decommission_prompt_seconds` (300). Both entry points refuse a +`decdn_cache_dir` that equals or encloses the identity, so a purge can never take the +keys with it. + +It **keeps** the identity, `/etc/decdn` and the binaries, because the keystore is +what withdraws the bond. It does **not** touch the chain. The exit is two operator +steps with the `decdn` CLI, both with `--dry-run` first (use `decdn_chain` from +[Running on-chain commands](#running-on-chain-commands)): + +1. `decdn_chain node deregister` leaves the active node set. The bond is **not** + returned: it stays deposited and slashable. +2. `decdn_chain node unbond --all` starts the unbonding window. Run it again after the + window to withdraw. + +Delete the keystore only after the withdrawal has landed. The public `udp/4433` +firewall rule stays until baseline is re-run without it. + +Do not use `decdn node drain` to take a systemd-managed node down: the unit is +`Restart=always`, so systemd starts the drained daemon again five seconds later. + +## Running on-chain commands + +`decdn setup`, `node bond`, `node register`, `node update-multiaddrs`, +`node update-region`, `node deregister` and `node unbond` sign with the operator key +and need the RPC endpoint. **They do not read `DECDN_RPC_URL`** from the environment, +unlike the daemon: they take `--rpc-url`, or `blockchain.rpc_url` from `node.toml`. +The Ansible role leaves `rpc_url` out of `node.toml` on purpose (it may embed an API +key), and on Compose `config init` wrote the public endpoint there. So pass it +explicitly. This helper runs the CLI as `decdn` with the unit's own environment file +and hands the URL over as `--rpc-url`: + +```bash +decdn_chain() { + sudo systemd-run --pty --wait --collect -p User=decdn \ + -p EnvironmentFile=/etc/decdn/decdn.env \ + /bin/sh -c 'exec /usr/local/bin/decdn "$@" --config /etc/decdn/node.toml \ + --rpc-url "$DECDN_RPC_URL" --keystore-password-file /etc/decdn/keystore.password' \ + decdn "$@" +} + +decdn_chain setup --mbps 100 --region DE \ + --multiaddr /ip4//udp/4433/quic-v1 --dry-run +``` + +The URL is in the `decdn` process's arguments while the command runs, so other local +users could read it with `ps`. On a shared host, write a `0600` copy of `node.toml` +with `rpc_url` set under `[blockchain]` and pass that as `--config` instead. + +## Compose and Kubernetes + +- **Compose** ([`compose/`](../compose/README.md)) uses the same host paths + (`/var/lib/decdn`, `/etc/decdn`), so the manual backup command and the restore steps + above apply unchanged; stop the node with `docker compose stop`. +- **Helm**: the identity lives in the operator-provisioned `existingSecret`, which you + created off-cluster and should already hold elsewhere. The daemon's state is on the + PVC; snapshot it with your storage's `VolumeSnapshot` support after scaling the + StatefulSet to zero. Never run two releases with the same identity Secret. diff --git a/docs/requirements.md b/docs/requirements.md new file mode 100644 index 0000000..f32d4f1 --- /dev/null +++ b/docs/requirements.md @@ -0,0 +1,91 @@ +# Requirements and choosing a deploy path + +What a deCDN node needs from its host and network, and which of this repo's three +deploy paths fits. Protocol facts (bond sizing, fees) are not here: they come from the +deCDN ADRs. + +## Choosing a path + +| | Ansible | Docker Compose | Helm | +|---|---|---|---| +| Target | VMs, bare metal | one host running Docker | Kubernetes | +| Host hardening (firewall, SSH, patching) | yes, `baseline` | no, yours | no, the cluster's | +| Fleets | yes, one inventory | one host per compose project | one release per node | +| Secrets | host file or git-ignored inventory | host file | operator-created Secrets | +| Install source | signed release tarball, or local build | image by digest (enforced) | image by digest (recommended) or tag | +| Chain config | `decdn_network` profile | `decdn config init --chain` | explicit values | +| Monitoring | opt-in Grafana Cloud agent | bring your own | ServiceMonitor, PrometheusRule, dashboards | +| Backup / decommission | `make backup` / `make decommission` | manual commands | PVC snapshot | +| Guide | [ansible/README.md](../ansible/README.md) | [compose/README.md](../compose/README.md) | [charts/decdn-node/README.md](../charts/decdn-node/README.md) | + +If you are unsure: a VPS or dedicated server you control end to end is the Ansible +path. It is the only one that hardens the host as well as installing the node. + +## Platforms + +| | Supported | How it is tested | +|---|---|---| +| OS | Debian 12 (bookworm), Debian 13 (trixie), Ubuntu 24.04 (noble), Ubuntu 26.04 (resolute) | molecule converges `decdn_node` on all four in systemd containers; `grafana_alloy`'s install path on Debian 12 only (its disabled path on all four); `baseline` on real hosts | +| Architecture | x86_64, aarch64 | upstream builds both; the Ansible role derives the target from the host | +| Ansible (control machine) | ansible-core ≥ 2.15 | CI runs the current release | +| Kubernetes | ≥ 1.25 | rendered and validated with kubeconform against 1.30 | +| Docker Compose | v2 with `env_file.required` support (2.24+) | rendered in CI | + +On Ubuntu 25.10 and later, `sudo` is sudo-rs; see the `ansible_become_exe` note in +`ansible/inventory/hosts.yml.example`. + +## Network + +- **Inbound: udp/4433 (QUIC)** from anywhere. This is the node's only public port. Open + it in the host firewall (the Ansible `baseline` does this) **and** in your cloud + provider's security group or firewall, which the host cannot see. +- **Outbound:** HTTPS to your RPC provider and to any cache origin; QUIC/UDP to peers; + HTTPS to the iroh relays (upstream's defaults unless you set your own). +- **NAT:** a node without direct inbound reachability still works through iroh relays, + at a latency cost. A node that serves paid traffic should be directly reachable, and + the multiaddr it registers on-chain must be its public address + (`/ip4//udp/4433/quic-v1`). +- **IPv6:** supported. On Ansible, `baseline_preserve_ipv6_autoconf` (default `true`) + keeps SLAAC addresses alive under DevSec hardening; set it `false` on static-IPv6 + hosts. Multi-homed hosts may need `baseline_rp_filter_loose: true`. +- **Loopback only, never exposed:** metrics `127.0.0.1:9090`, admin RPC + `127.0.0.1:9191`. Reach them over SSH, or ship metrics with the monitoring options. +- **Clock:** keep NTP running (`baseline` installs chrony). `decdn setup`'s pre-flight + checks clock skew. +- **Bandwidth:** the node serves what it is asked for. On metered bandwidth, cap egress + with `[load_shed] egress_budget_mbps` in `node.toml` + (`decdn_load_shed_egress_budget_mbps` on Ansible, `config.load_shed.egress_budget_mbps` + on Helm), and set a billing alert with your provider. + +## Disk + +Everything lives on the data volume (`/var/lib/decdn`, or the PVC): + +``` +cache_size_mb the cache (role default 10 GiB) ++ disk_headroom_mb free space the daemon keeps on the volume (default 8 GiB) ++ receipts rotated download-receipt log, bounded by its [receipts] settings ++ daemon state (redb stores) small, grows with activity; no published bound ++ an fs origin, if the node holds one +``` + +`max_blob_size_mb` must be ≤ `cache_size_mb`: the node rejects any blob larger than it +(`BlobTooLarge`), so size it to the largest single object you expect to serve (role +default 1 GiB). Put `/var/lib/decdn` on the fast disk. + +## CPU and memory + +Upstream publishes no minimum figures yet, so this repo quotes none. Watch the node's +own metrics (`decdn node top`, the upstream dashboards) under real load and size from +that. + +## Accounts and keys + +- An **RPC endpoint** for the chain (Arbitrum Sepolia today). The public endpoint that + `config init` writes works for light use; production nodes should use their own + provider. Keep the URL secret if it embeds an API key. +- An **operator wallet**: the eth keystore `decdn key-gen` creates, funded for gas and + the bond before `decdn setup`. The bond comes from the on-chain + `bondRequired(mbps)` curve (ADR 026). +- Optional: a **Grafana Cloud** stack (Ansible `grafana_alloy`), and an **age key pair** + for encrypted backups ([docs/lifecycle.md](lifecycle.md)). diff --git a/scripts/check-release-version.sh b/scripts/check-release-version.sh new file mode 100755 index 0000000..d284b0a --- /dev/null +++ b/scripts/check-release-version.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# Gate a release tag, and optionally write its notes. +# +# scripts/check-release-version.sh v0.1.0 [--notes ] +# +# One repo version covers both published artifacts: the tag vX.Y.Z must equal the +# decdn.node collection version (ansible/galaxy/galaxy.yml) AND the chart version +# (charts/decdn-node/Chart.yaml), and both changelogs must carry a released +# `## [X.Y.Z]` heading (not "unreleased"). With --notes, the two changelog sections +# are written to as the GitHub Release body. +set -euo pipefail + +usage() { echo "usage: $0 vX.Y.Z [--notes ]" >&2; exit 2; } +tag="" notes="" +case $# in + 1) tag="$1" ;; + 3) [[ "$2" == "--notes" && -n "$3" ]] || usage; tag="$1" notes="$3" ;; + *) usage ;; +esac +[[ "$tag" =~ ^v([0-9]+\.[0-9]+\.[0-9]+)$ ]] || { echo "tag '$tag' is not vX.Y.Z" >&2; exit 1; } +version="${BASH_REMATCH[1]}" + +repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +galaxy_changelog="$repo/ansible/galaxy/CHANGELOG.md" +chart_changelog="$repo/charts/decdn-node/CHANGELOG.md" +collection="$(sed -nE 's/^version:[[:space:]]*"?([^"#[:space:]]+)"?.*/\1/p' "$repo/ansible/galaxy/galaxy.yml")" +chart="$(sed -nE 's/^version:[[:space:]]*"?([^"#[:space:]]+)"?.*/\1/p' "$repo/charts/decdn-node/Chart.yaml")" + +fail=0 +[[ "$collection" == "$version" ]] || { echo "ansible/galaxy/galaxy.yml version is $collection, tag is $tag" >&2; fail=1; } +[[ "$chart" == "$version" ]] || { echo "charts/decdn-node/Chart.yaml version is $chart, tag is $tag" >&2; fail=1; } + +# Print the body of the `## [X.Y.Z]` section, or fail if it is missing or unreleased. +section() { + local file="$1" heading + heading="$(grep -E "^## \[$version\]" "$file" || true)" + if [[ -z "$heading" ]]; then + echo "$file has no '## [$version]' section" >&2; return 1 + fi + if grep -qi 'unreleased' <<<"$heading"; then + echo "$file still marks $version as unreleased: $heading" >&2; return 1 + fi + awk -v v="## [$version]" 'index($0, v) == 1 {on=1; next} on && /^## \[/ {exit} on' "$file" +} +galaxy_notes="$(section "$galaxy_changelog")" || fail=1 +chart_notes="$(section "$chart_changelog")" || fail=1 +((fail == 0)) || exit 1 + +if [[ -n "$notes" ]]; then + { + echo "## Ansible collection \`decdn.node\` $version" + echo "$galaxy_notes" + echo + echo "## Helm chart \`decdn-node\` $version" + echo "$chart_notes" + } > "$notes" +fi +echo "release $tag: collection and chart are both $version" diff --git a/scripts/sync-monitoring.sh b/scripts/sync-monitoring.sh new file mode 100755 index 0000000..d573508 --- /dev/null +++ b/scripts/sync-monitoring.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash +# Vendor upstream's Grafana dashboards and Prometheus alert rules into the chart. +# +# scripts/sync-monitoring.sh [ref] [--check] +# +# Copies decdn/decdn's monitoring/*.json and monitoring/prometheus-alerts.yml at +# (default origin/main, read with `git show`, never from the working tree) +# into charts/decdn-node/files/monitoring/, and records the upstream commit plus a +# sha256 per file in SOURCE. The chart renders them as a PrometheusRule and Grafana +# dashboard ConfigMaps; Ansible/Grafana Cloud users import the same files by hand +# (see that directory's README.md). +# +# --check copies into a scratch dir instead and exits 1 when any vendored file's +# bytes, or SOURCE's file list and hashes, differ from (the commit line in +# SOURCE is ignored, so an unrelated upstream commit is not drift). +# +# Exit status: 0 current/written, 1 stale (--check), 2 could not run, so the drift +# job can tell drift from breakage. +set -euo pipefail + +usage() { echo "usage: $0 [ref] [--check]" >&2; exit 2; } +die() { echo "sync-monitoring: $*" >&2; exit 2; } +# Any other failure (a git show under set -e, a full disk) is "could not run" too. +scratch="" +on_exit() { + local rc=$? + [[ -z "$scratch" ]] || rm -rf "$scratch" + ((rc == 0 || rc == 1 || rc == 2)) || { echo "sync-monitoring: failed (exit $rc)" >&2; exit 2; } +} +trap on_exit EXIT + +upstream="" ref="origin/main" check=false +for arg in "$@"; do + case "$arg" in + --check) check=true ;; + -*) usage ;; + *) if [[ -z "$upstream" ]]; then upstream="$arg"; else ref="$arg"; fi ;; + esac +done +[[ -n "$upstream" ]] || usage + +repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +dest="$repo/charts/decdn-node/files/monitoring" + +commit="$(git -C "$upstream" rev-parse --verify "$ref^{commit}")" || die "no commit '$ref' in $upstream" +listing="$(git -C "$upstream" ls-tree --name-only "$ref" monitoring/)" || die "cannot list monitoring/ at $ref" +mapfile -t files < <(grep -E '^monitoring/[^/]+\.(json|ya?ml)$' <<<"$listing" | sort) +# The chart needs both kinds: an empty dashboard set would render nothing and an +# absent rule file would fail every render, so a changed upstream layout stops here +# instead of silently vendoring half of it. +printf '%s\n' "${files[@]}" | grep -q '\.json$' || die "no dashboards (monitoring/*.json) at $ref" +printf '%s\n' "${files[@]}" | grep -qx 'monitoring/prometheus-alerts.yml' || die "no monitoring/prometheus-alerts.yml at $ref" + +out="$dest" +if $check; then + out="$(mktemp -d)" + scratch="$out" +fi +mkdir -p "$out" + +{ + echo "# GENERATED by scripts/sync-monitoring.sh -- do not hand-edit." + echo "upstream: decdn/decdn" + echo "commit: $commit" + echo "files:" +} > "$out/SOURCE" +for f in "${files[@]}"; do + name="$(basename "$f")" + git -C "$upstream" show "$ref:$f" > "$out/$name" + echo " $name: sha256:$(sha256sum "$out/$name" | cut -d' ' -f1)" >> "$out/SOURCE" +done + +if $check; then + stale=0 + # Byte-compare every vendored file with upstream, so a local hand-edit is drift + # even when SOURCE still matches. + for f in "${files[@]}"; do + name="$(basename "$f")" + cmp -s "$out/$name" "$dest/$name" || { echo "differs from upstream: $name"; stale=1; } + done + # Vendored files present locally but gone upstream count as drift too. + for f in "$dest"/*.json "$dest"/*.yml "$dest"/*.yaml; do + [[ -e "$f" ]] || continue + [[ -e "$out/$(basename "$f")" ]] || { echo "removed upstream: $(basename "$f")"; stale=1; } + done + if ! diff -q <(grep -v '^commit:' "$dest/SOURCE") <(grep -v '^commit:' "$out/SOURCE") >/dev/null; then + diff <(grep -v '^commit:' "$dest/SOURCE") <(grep -v '^commit:' "$out/SOURCE") || true + stale=1 + fi + if ((stale)); then + echo "charts/decdn-node/files/monitoring is stale against $ref; re-run without --check" >&2 + exit 1 + fi + echo "charts/decdn-node/files/monitoring is current against $ref" + exit 0 +fi + +# Drop vendored files that upstream removed. +for f in "$dest"/*.json "$dest"/*.yml "$dest"/*.yaml; do + [[ -e "$f" ]] || continue + printf '%s\n' "${files[@]}" | grep -qx "monitoring/$(basename "$f")" || rm -f -- "$f" +done +echo "vendored ${#files[@]} files from decdn/decdn@${commit:0:12} into charts/decdn-node/files/monitoring" diff --git a/scripts/sync-network-profiles.py b/scripts/sync-network-profiles.py new file mode 100755 index 0000000..6c6232e --- /dev/null +++ b/scripts/sync-network-profiles.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +"""Regenerate the decdn_node role's network profiles from upstream deployment manifests. + + scripts/sync-network-profiles.py [--ref origin/main] [--check] + +Writes ansible/roles/decdn_node/vars/main/networks.yml: one entry per chain upstream's +`decdn config init --chain ` knows, holding the contract addresses that command +would bake into node.toml. Operators then set `decdn_network: ` instead of +hand-copying eight addresses that change wholesale on every upstream redeploy. + +Everything is read from git at --ref (default origin/main), never from the working +tree, so a checkout sitting on a feature branch still yields main's deployment. Two +upstream files are the source, and both the chain list AND the manifest-key mapping +are parsed from the first, so this script holds no hand-written copy of either: + + crates/cli/src/known_chains.rs KNOWN_CHAINS (name + chain_id) and + KnownChain::addresses() (field <- manifest key) + contracts/deployments/.json that chain's manifest (the CLI embeds a + byte-identical copy under crates/cli/deployments/) + +What it cannot derive is which role variable each upstream field feeds; that is +FIELDS below. If upstream adds, drops or renames an address field, the script stops +rather than guess, because the role and its node.toml template need the same change. + +Exit status: 0 current/written, 1 stale (--check), 2 could not run (git, parse or +manifest errors), so the drift job can tell drift from breakage. +""" + +import argparse +import json +import re +import subprocess +import sys +from pathlib import Path + +REPO = Path(__file__).resolve().parent.parent +OUT = REPO / "ansible/roles/decdn_node/vars/main/networks.yml" + +# Upstream ChainAddresses field -> decdn_node variable suffix (decdn_). +# Order is the output order. +FIELDS = { + "payment_pool": "payment_pool_address", + "capacity_bond": "capacity_bond_address", + "slash_judge": "slash_judge_address", + "content_blacklist": "content_blacklist_address", + "slash_appeal": "slash_appeal_address", + "origin_assignment": "origin_assignment_address", + "publisher_registry": "publisher_registry_address", + "usdc": "usdc_address", +} + +ADDRESS = re.compile(r"^0x[0-9a-fA-F]{40}$") +REGISTRY = re.compile(r"KNOWN_CHAINS\s*:\s*&\[\s*KnownChain\s*\]\s*=\s*&\[(?P.*?)\];", re.S) +ENTRY = re.compile(r"KnownChain\s*\{(?P.*?)\}", re.S) +NAME = re.compile(r"\bname:\s*\"(?P[^\"]+)\"") +CHAIN_ID = re.compile(r"\bchain_id:\s*(?P[0-9_]+)\s*,") +ADDRESSES_FN = re.compile(r"fn addresses\(.*?Ok\(ChainAddresses\s*\{(?P.*?)\}\)", re.S) +FROM_CONTRACT = re.compile(r"(?P\w+):\s*contract\(\"(?P\w+)\"\)\?") +FROM_EXTERNAL = re.compile(r"(?P\w+):\s*manifest\.external_deps\.(?P\w+)") + + +def die(msg: str) -> None: + print(f"sync-network-profiles: {msg}", file=sys.stderr) + sys.exit(2) + + +def git(upstream: Path, *args: str) -> str: + try: + return subprocess.run( + ["git", "-C", str(upstream), *args], check=True, capture_output=True, text=True + ).stdout + except subprocess.CalledProcessError as e: + die(f"git {' '.join(args)} failed: {e.stderr.strip() or e}") + except FileNotFoundError: + die("git not found") + + +def parse_registry(source: str) -> list[tuple[str, int]]: + registry = REGISTRY.search(source) + if not registry: + die("KNOWN_CHAINS not found in known_chains.rs") + entries = ENTRY.findall(registry.group("body")) + if not entries: + die("KNOWN_CHAINS is empty") + chains = [] + for body in entries: + name, chain_id = NAME.search(body), CHAIN_ID.search(body) + # Every entry must parse: a skipped chain would make --check pass vacuously. + if not (name and chain_id): + die(f"could not read name/chain_id from a KNOWN_CHAINS entry:\n{body.strip()}") + chains.append((name.group("v"), int(chain_id.group("v").replace("_", "")))) + return chains + + +def parse_mapping(source: str) -> dict[str, tuple[str, str]]: + """Upstream field -> ("contracts" | "externalDeps", manifest key).""" + fn = ADDRESSES_FN.search(source) + if not fn: + die("KnownChain::addresses() not found in known_chains.rs") + mapping = {m["field"]: ("contracts", m["key"]) for m in FROM_CONTRACT.finditer(fn.group("body"))} + mapping |= {m["field"]: ("externalDeps", m["key"]) for m in FROM_EXTERNAL.finditer(fn.group("body"))} + if set(mapping) != set(FIELDS): + die( + "upstream's address fields changed: " + f"added {sorted(set(mapping) - set(FIELDS))}, removed {sorted(set(FIELDS) - set(mapping))}. " + "Update FIELDS here, the role's defaults and node.toml.j2 together." + ) + return mapping + + +def render(upstream: Path, ref: str) -> str: + commit = git(upstream, "rev-parse", "--verify", f"{ref}^{{commit}}").strip() + source = git(upstream, "show", f"{ref}:crates/cli/src/known_chains.rs") + chains = parse_registry(source) + mapping = parse_mapping(source) + + lines = [ + "---", + "# GENERATED by scripts/sync-network-profiles.py -- do not hand-edit.", + f"# Source: decdn/decdn @ {commit}", + "# crates/cli/src/known_chains.rs (KNOWN_CHAINS, KnownChain::addresses)", + "# + contracts/deployments/.json", + "#", + "# The addresses `decdn config init --chain ` bakes into node.toml. The role's", + "# address defaults read this map lazily when decdn_network is set, so an explicit", + "# inventory value still wins. Protocol facts: upstream's deployment manifests are", + "# the source, this file only mirrors them.", + "_decdn_network_profiles:", + ] + for name, chain_id in chains: + raw = git(upstream, "show", f"{ref}:contracts/deployments/{chain_id}.json") + try: + manifest = json.loads(raw) + except json.JSONDecodeError as e: + die(f"{chain_id}.json is not valid JSON: {e}") + if manifest.get("chainId") != chain_id: + die(f"{chain_id}.json chainId {manifest.get('chainId')} != registry {chain_id} for {name}") + lines += [f" {name}:", f" chain_id: {chain_id}"] + for field, var in FIELDS.items(): + section, key = mapping[field] + value = manifest.get(section, {}).get(key) + if not isinstance(value, str) or not ADDRESS.match(value): + die(f"{name}: {section}.{key} is {value!r} in {chain_id}.json, not a 0x address") + lines.append(f' {var}: "{value}"') + lines.append(f" deploy_block: {int(manifest['deployBlock'])}") + return "\n".join(lines) + "\n" + + +def strip_commit(text: str) -> str: + return re.sub(r"^# Source: .*$", "", text, flags=re.M) + + +def main() -> None: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("upstream", type=Path, help="path to a decdn/decdn git checkout") + ap.add_argument("--ref", default="origin/main", help="git ref to read (default: origin/main)") + ap.add_argument("--check", action="store_true", help="exit 1 if the committed file is stale") + args = ap.parse_args() + + text = render(args.upstream, args.ref) + if args.check: + current = OUT.read_text() if OUT.exists() else "" + if strip_commit(current) != strip_commit(text): + print(f"{OUT.relative_to(REPO)} is stale against {args.ref}; re-run without --check", file=sys.stderr) + sys.exit(1) + print(f"{OUT.relative_to(REPO)} is current against {args.ref}") + return + OUT.write_text(text) + print(f"wrote {OUT.relative_to(REPO)}") + + +if __name__ == "__main__": + main() diff --git a/tests/scripts-test.sh b/tests/scripts-test.sh new file mode 100755 index 0000000..10884b3 --- /dev/null +++ b/tests/scripts-test.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +# Tests for the repo's own guard rails that no molecule scenario or chart render +# exercises: the ansible/ Makefile's scoping guards, the release gate, the +# lint-compose invariants (negative cases), and — with UPSTREAM= — +# the upstream-mirror generators' exit codes. `make test-scripts` runs it; CI's +# `scripts` job does too. Needs make, docker (compose v2), jq. +set -euo pipefail + +repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT +skipped=() +pass() { echo "ok $*"; } +fail() { echo "FAIL: $*" >&2; exit 1; } + +# expect : the command must exit with exactly that code. +expect() { + local want="$1" desc="$2" got=0; shift 2 + "$@" >"$work/out" 2>&1 || got=$? + [[ "$got" == "$want" ]] || { cat "$work/out" >&2; fail "$desc: exit $got, want $want"; } + pass "$desc" +} + +# --- ansible/Makefile scoping guards (dry runs: make -n never runs a playbook) ------ +mk() { make -s -C "$repo/ansible" -n "$@"; } +expect 2 "decommission refuses a missing LIMIT" mk decommission +expect 2 "decommission refuses an empty LIMIT" mk decommission LIMIT= +expect 2 "decommission refuses LIMIT from the environment" env LIMIT=h make -s -C "$repo/ansible" -n decommission +expect 2 "deploy refuses INVENTORY from the environment" env INVENTORY=x make -s -C "$repo/ansible" -n deploy +expect 2 "backup refuses ANSIBLE_ARGS from the environment" env ANSIBLE_ARGS=-v make -s -C "$repo/ansible" -n backup +mk decommission LIMIT=h | grep -q -- "playbooks/decommission.yml --limit 'h'" \ + || fail "decommission LIMIT=h does not scope the playbook" +pass "decommission LIMIT=h scopes playbooks/decommission.yml" +mk backup | grep -q -- "playbooks/backup.yml" || fail "backup does not run playbooks/backup.yml" +pass "backup runs playbooks/backup.yml fleet-wide by default" + +# --- release gate ---------------------------------------------------------------------- +gate="$repo/scripts/check-release-version.sh" +expect 2 "release gate rejects a stray argument" "$gate" v0.1.0 notes.md +expect 2 "release gate rejects no arguments" "$gate" +expect 1 "release gate rejects a malformed tag" "$gate" 0.1.0 +expect 1 "release gate rejects a version mismatch" "$gate" v9.9.9 +# A released copy of the tree: both changelogs dated, versions equal to the tag. +rel="$work/rel" +mkdir -p "$rel/scripts" "$rel/ansible/galaxy" "$rel/charts/decdn-node" +cp "$gate" "$rel/scripts/" +cp "$repo/ansible/galaxy/galaxy.yml" "$repo/ansible/galaxy/CHANGELOG.md" "$rel/ansible/galaxy/" +cp "$repo/charts/decdn-node/Chart.yaml" "$repo/charts/decdn-node/CHANGELOG.md" "$rel/charts/decdn-node/" +version="$(sed -nE 's/^version:[[:space:]]*"?([^"#[:space:]]+)"?.*/\1/p' "$rel/charts/decdn-node/Chart.yaml")" +expect 1 "release gate rejects an 'unreleased' changelog heading" "$rel/scripts/check-release-version.sh" "v$version" +sed -i -E "s/^## \[$version\] — unreleased$/## [$version] — 2099-01-01/" \ + "$rel/ansible/galaxy/CHANGELOG.md" "$rel/charts/decdn-node/CHANGELOG.md" +expect 0 "release gate accepts a released tree" "$rel/scripts/check-release-version.sh" "v$version" --notes "$work/notes.md" +for section in '^## Ansible collection' '^## Helm chart'; do + grep -q "$section" "$work/notes.md" || fail "release notes miss '$section'" +done +pass "release notes carry both changelog sections" + +# --- lint-compose negatives: each broken variant must be rejected ---------------------- +compose="$repo/compose/compose.yaml" +variant() { # + sed -E "$2" "$compose" > "$work/$1.yaml" + cmp -s "$compose" "$work/$1.yaml" && fail "variant $1 did not change compose.yaml" + # make exits 2 for any failing recipe, so tell "invariant violated" from "could + # not render" by the message, not the exit code. + if make -s -C "$repo" lint-compose COMPOSE_FILE="$work/$1.yaml" >"$work/out" 2>&1; then + fail "lint-compose accepted: $1" + fi + grep -q 'violates an invariant' "$work/out" || { cat "$work/out" >&2; fail "lint-compose failed for another reason: $1"; } + pass "lint-compose rejects: $1" +} +variant "bridge network" 's/^(\s*)network_mode: host$/\1network_mode: bridge/' +variant "writable rootfs" 's/^(\s*)read_only: true$/\1read_only: false/' +variant "short stop grace" 's/^(\s*)stop_grace_period: 300s$/\1stop_grace_period: 10s/' +variant "capabilities kept" 's/^(\s*)cap_drop: \[ALL\]$/\1cap_drop: [NET_RAW]/' +variant "published port" 's/^(\s*)network_mode: host$/\1ports: ["127.0.0.1:9090:9090"]/' +variant "tag instead of digest" 's#^(\s*)image: .*#\1image: ghcr.io/decdn/decdn-node:latest#' + +# --- upstream-mirror generators (optional: needs a decdn/decdn checkout) -------------- +if [[ -n "${UPSTREAM:-}" ]]; then + expect 0 "network profiles current" "$repo/scripts/sync-network-profiles.py" "$UPSTREAM" --check + expect 0 "monitoring assets current" "$repo/scripts/sync-monitoring.sh" "$UPSTREAM" --check + expect 2 "network profiles: bad ref is an error, not drift" "$repo/scripts/sync-network-profiles.py" "$UPSTREAM" --ref no/such/ref --check + expect 2 "monitoring assets: bad ref is an error, not drift" "$repo/scripts/sync-monitoring.sh" "$UPSTREAM" no/such/ref --check +else + skipped+=("upstream-mirror generators (set UPSTREAM=)") +fi + +for s in "${skipped[@]}"; do echo "SKIPPED: $s"; done +echo "all script tests passed"