diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d801240..b08a194 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,7 +18,7 @@ concurrency: # malicious code — cf. the March 2026 KICS action compromise). The trailing # comment records the human-readable version; .github/dependabot.yml bumps them. jobs: - # Detect whether ansible/ changed so the heavy Ansible jobs skip unrelated PRs. + # Detect whether ansible/ or the Helm chart changed so heavy jobs skip unrelated PRs. changes: runs-on: ubuntu-latest permissions: @@ -26,6 +26,7 @@ jobs: pull-requests: read outputs: ansible: ${{ steps.filter.outputs.ansible }} + helm: ${{ steps.filter.outputs.helm }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v4.0.3 @@ -34,6 +35,12 @@ jobs: filters: | ansible: - 'ansible/**' + # The chart shares the schema-key inventory and checker with molecule. + helm: + - 'charts/**' + - 'ansible/molecule/schema/files/**' + - 'Makefile' + - '.github/workflows/ci.yml' # Ansible style + best-practice + the production-profile SECURITY rules, # plus a syntax-check of every playbook. Runs only when ansible/ changed. @@ -96,7 +103,29 @@ jobs: path: ansible/build/decdn-node-*.tar.gz if-no-files-found: ignore - # Dedicated IaC security scan of the Ansible tree, driven straight from the + # Helm chart: `helm lint --strict`, positive + negative render tests, kubeconform + # (digest-pinned image) and the upstream schema-key check shared with molecule — + # all via `make lint-helm`, the same command developers run locally. + helm: + needs: changes + if: needs.changes.outputs.helm == 'true' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + # Helm version is pinned here AND in the kics job below; bump both. + - uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 + with: + version: v4.3.0 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.12' + - name: Chart lint + render tests + # yq (mikefarah) and Docker ship on ubuntu-latest. No decdn binary here, so + # the real `decdn config validate` step reports SKIPPED; run it locally with + # DECDN_CLI=... before bumping the decdn version. + run: make lint-helm + + # Dedicated IaC security scan of the Ansible tree and the rendered Helm chart, driven straight from the # digest-pinned KICS *engine* image by `make security` — the exact command # developers run locally, so CI and local results cannot drift. KICS severities # are CRITICAL/HIGH/MEDIUM/LOW/INFO; the engine's own `--fail-on high` exit @@ -115,19 +144,24 @@ jobs: # hijacked action). See CONTRIBUTING.md. kics: needs: changes - if: needs.changes.outputs.ansible == 'true' + if: needs.changes.outputs.ansible == 'true' || needs.changes.outputs.helm == 'true' runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - name: KICS Ansible security scan (fail on HIGH) + - uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 + with: + version: v4.3.0 + - name: KICS security scan of ansible/ + the rendered chart (fail on HIGH) run: make security # Replaces the action's `enable_jobs_summary`. - name: Summarise KICS findings if: always() run: | + for scan in ansible helm; do results=kics-results/results.json + [ "$scan" = helm ] && results=kics-results/helm/results.json { - echo "### KICS IaC scan" + echo "### KICS IaC scan ($scan)" echo if [ -f "$results" ]; then jq -r '.severity_counters @@ -142,7 +176,9 @@ jobs: else echo "No results file — the scan did not complete." fi + echo } >> "$GITHUB_STEP_SUMMARY" + done - name: Upload KICS results if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index aa33938..9ef69e5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,8 +4,8 @@ # pip install pre-commit && make hooks # one-time install # make lint # run on all files # -# Heavier Ansible checks (ansible-lint, syntax-check, KICS security scan) run in -# CI only — see .github/workflows/. ansible-lint's production profile already +# Heavier checks (ansible-lint, syntax-check, KICS security scan, and the Helm +# chart's `make lint-helm`) run in CI — see .github/workflows/. ansible-lint's production profile already # carries the Ansible security rules; this file is the fast local gate (hygiene, # shellcheck, yamllint, markdown). minimum_pre_commit_version: "3.5.0" @@ -38,6 +38,9 @@ repos: args: [--fix=lf] - id: check-yaml args: [--unsafe] # tolerate custom/!vault tags; syntax check only + # Helm templates are Go templates, not YAML; `make lint-helm` renders and + # validates them instead. + exclude: ^charts/[^/]+/templates/ - id: check-json - id: check-toml diff --git a/AGENTS.md b/AGENTS.md index a90c3d1..0c37784 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -5,9 +5,9 @@ DevOps repo. ## What this repo is -The official Ansible project for deploying a **deCDN node**: infrastructure, deployment, -and operational tooling. The whole repo is **Ansible-driven** — `ansible/` is the -deployment project. +The official DevOps project for deploying a **deCDN node**: infrastructure, deployment, +and operational tooling. There are two deploy paths: **Ansible** (`ansible/`, VMs/bare +metal, the primary path) and a **Helm chart** (`charts/decdn-node/`, Kubernetes). This repo is **infrastructure only**. It is *not* a source of truth for protocol or economic claims — those trace to the deCDN ADRs. If something here states a protocol fact @@ -20,12 +20,22 @@ economic claims — those trace to the deCDN ADRs. If something here states a pr (e.g. the node's eth keystore, or `rpc_url` which may embed an API key) and stored under `/etc//` with `chmod 600` and a dedicated owner. The repo ships `*.example` templates for secret files only (non-secret config may be committed directly). The root - `.gitignore` is a backstop — do not rely on it; keep secrets out by design. + `.gitignore` is a backstop — do not rely on it; keep secrets out by design. The Helm + chart never creates a Secret: it references operator-provisioned ones + (`existingSecret`), injects only named env keys (never `envFrom` — `DECDN_*` env + overrides `node.toml`), keeps the keystore password off the PVC, and refuses + secret-bearing keys in `config`. 2. **Localhost-only by default.** Service daemons bind `127.0.0.1` (e.g. the node's metrics and admin RPC). A service that must accept public traffic declares exactly one hole (the node's QUIC udp/4433) via `baseline_extra_inbound`; if a service ever needs an HTTP-facing public path, front it with an explicit reverse proxy that terminates auth + TLS. Never bind a *backend* to `0.0.0.0` or expose its raw port. + **Kubernetes exception (chart only):** the node's metrics bind `0.0.0.0` inside the pod + so kubelet probes and Prometheus can reach them. That is allowed only behind a + ClusterIP-only Service and the chart's NetworkPolicy (metrics ingress limited to + `metrics.networkPolicy.from`); disabling the policy fails the render unless + `networkPolicy.allowUnrestrictedMetrics=true` acknowledges it. Never front metrics with + a LoadBalancer/NodePort/Ingress. 3. **Role templates render to their target paths.** Ansible roles template config directly onto the host (e.g. `roles/decdn_node/templates/decdn-node.service.j2` → `/etc/systemd/system/`), with secrets generated on the host at `0600`. @@ -42,6 +52,10 @@ ansible/ # the deployment project (DevSec-hardened, lean roles) playbooks/ # site.yml (decdn node) roles/ # baseline, decdn_node inventory/ galaxy/ molecule/ # see ansible/README.md +charts/ + decdn-node/ # Helm chart for the node on Kubernetes (see its README.md) + ci/ # CI values files (mirror molecule/schema's three plays) + tests/render-test.sh # positive/negative render tests (`make lint-helm`) ``` ## Current services @@ -65,6 +79,20 @@ ansible/ # the deployment project (DevSec-hardened, lean roles) `molecule/schema` scenario checks the rendered key set against a committed inventory of upstream field names. Re-sync both when bumping the pinned decdn version. +- **`charts/decdn-node/`** — the same node on Kubernetes: a one-replica StatefulSet (one + release = one identity) on the upstream daemon-only image (`ghcr.io/decdn/decdn-node`; + unpublished, so `image.tag`/`image.digest` is required), PVC data dir, a `prepare` init + container that installs the identity files from an `existingSecret` onto the PVC at + `0600` (upstream rejects symlinked or group/world-readable key files) and the password + into an in-memory volume, `DECDN_RPC_URL` via `secretKeyRef` (named keys only), public + UDP Service (LoadBalancer/NodePort/ClusterIP) or `hostPort`, and metrics bound `0.0.0.0` + behind a ClusterIP Service + NetworkPolicy. `values.config` mirrors `node.toml`; the + chart injects the path/port keys and fails on collisions. **The config-schema coupling + above applies here too:** `make lint-helm` runs the same `check-schema-keys.py` + + `schema-keys.txt` on the rendered ConfigMap, so a re-sync covers both paths. Unlike the + role, CI has no real-binary `decdn config validate` for the chart — run + `DECDN_CLI=… make lint-helm` locally when bumping the decdn version. + ## Commands Two Makefiles: the **root** is the hygiene/security/CI mirror; **`ansible/`** drives @@ -75,8 +103,9 @@ deploys (its targets must run from `ansible/`). `make help` lists root targets. make hooks # one-time: install pre-commit git hook (pip install pre-commit first) make lint # all pre-commit hooks on all files (hygiene, shellcheck, yamllint, markdown) make lint-ansible # vendor collections + full ansible-lint (production profile) -make security # KICS IaC scan of ansible/ (pinned engine image) make molecule # containerised converge/verify of the decdn_node role (needs Docker) +make lint-helm # chart: helm lint + render tests + kubeconform + schema keys (needs helm, yq, Docker) +make security # = security-ansible + security-helm (KICS over the rendered chart; needs helm) # Ansible deploys — run from ansible/ (see ansible/README.md for the full flow) cd ansible @@ -95,7 +124,9 @@ collection tree by `galaxy/build.sh` — there is **no** `galaxy.yml` at the `an **Gotcha — pre-commit is local-only.** Hygiene/shellcheck/yamllint/markdown run via `make hooks`/`make lint` on your machine, **not** in CI. CI (`.github/workflows/`) is the -blocking gate and runs `ansible-lint` + KICS + `galaxy-build` (on `ansible/**`) + `actionlint`. `ansible-lint` +blocking gate and runs `ansible-lint` + `galaxy-build` (on `ansible/**`), `helm` (`make lint-helm`, +on `charts/**`, the shared schema checker/inventory, `Makefile` or `ci.yml`), KICS (on either) + +`actionlint`. `ansible-lint` is **not** a per-commit hook (it needs collections vendored) — run `make lint-ansible`. A separate `molecule.yml` workflow runs the containerised converge/verify in CI too, so `make molecule` is not purely local. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c6000d2..1adfc79 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -23,15 +23,17 @@ tree), and `markdownlint`. |--------|--------------| | `make lint` | run all pre-commit hooks on every file (the full local hygiene gate) | | `make lint-ansible` | install Galaxy collections + run `ansible-lint` (its production profile includes the Ansible security rules) | -| `make security` | KICS IaC security scan of `ansible/` (digest-pinned engine image — CI runs this same target) | +| `make lint-helm` | Helm chart: `helm lint --strict`, positive/negative render tests, kubeconform (digest-pinned image), shared schema-key check and its fixtures (needs `helm`, `yq`, `python3` ≥ 3.11, Docker). Set `DECDN_CLI=` to also run the real `decdn config validate` (CI can't). | +| `make security` | KICS IaC security scan of `ansible/` and the rendered Helm chart (digest-pinned engine image — CI runs this same target) | `ansible-lint` is **not** a per-commit hook (it needs the collections installed). Run it on demand with `make lint-ansible`, or `pre-commit run ansible-lint --hook-stage manual`. ## CI overview -- **`ci.yml`** — `ansible-lint` + `galaxy-build` + `kics` (on `ansible/**`) and - `actionlint`. Bash-only PRs skip the Ansible jobs. Hygiene/shellcheck/markdownlint +- **`ci.yml`** — `ansible-lint` + `galaxy-build` (on `ansible/**`), `helm` (on + `charts/**`, the shared schema inventory/checker, the root `Makefile` or `ci.yml` + itself), `kics` (on either) and `actionlint`. Bash-only PRs skip the Ansible jobs. Hygiene/shellcheck/markdownlint run via **pre-commit locally only** (`make hooks` / `make lint`), not in CI. - **`molecule.yml`** — containerised converge + idempotence + verify of the `decdn_node` role (privileged systemd Docker container; scoped to `ansible/**`). Run locally with @@ -52,7 +54,9 @@ Run it on demand with `make lint-ansible`, or `pre-commit run ansible-lint --hoo hijacked action — pinned by a digest verified against Docker Hub, currently `v2.1.20`. The engine's `--fail-on high` exit code is the gate. - **Dependabot** (`.github/dependabot.yml`) bumps the other action SHAs weekly. -- **Bump manually** (Dependabot can't): the `KICS_IMAGE` digest in the `Makefile`, +- **Bump manually** (Dependabot can't): the `KICS_IMAGE` and `KUBECONFORM_IMAGE` digests + in the `Makefile`, both `setup-helm` `version:` inputs in `ci.yml` (`helm` and `kics` + jobs), and the pre-commit hook revs via `pre-commit autoupdate`. ## Solidity diff --git a/Makefile b/Makefile index 1466e09..98e9d5e 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ # Convenience targets for the deCDN DevOps monorepo. # Run from the repo root. Ansible-specific work is delegated to ansible/Makefile. -.PHONY: help hooks lint lint-ansible security molecule galaxy-build galaxy-check +.PHONY: help hooks lint lint-ansible lint-helm security security-ansible security-helm molecule galaxy-build galaxy-check SHELL := /bin/bash # KICS runs straight from the engine image, pinned by digest. This target IS the @@ -10,6 +10,12 @@ SHELL := /bin/bash # against Docker Hub on each bump. v2.1.20 (March 2026). KICS_IMAGE := checkmarx/kics:v2.1.20-alpine@sha256:990ae994fbbe59760c8e4f7e89b1193a39a0c2968909058ec29335cb6d80efc1 +# kubeconform validates rendered chart manifests against the Kubernetes schemas. +# Digest-pinned for the same reason as KICS. v0.7.0. +KUBECONFORM_IMAGE := ghcr.io/yannh/kubeconform:v0.7.0@sha256:85dbef6b4b312b99133decc9c6fc9495e9fc5f92293d4ff3b7e1b30f5611823c + +CHART := charts/decdn-node + help: ## list targets @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort \ | awk 'BEGIN{FS=":.*?## "}{printf " \033[36m%-14s\033[0m %s\n", $$1, $$2}' @@ -24,9 +30,13 @@ lint-ansible: ## full ansible-lint locally (installs collections first) $(MAKE) -C ansible deps $(MAKE) -C ansible lint +# Both scans always run, so a finding in one never hides the other's results. +security: ## KICS IaC security scan of ansible/ and the Helm chart (CI runs this) + @rc=0; $(MAKE) security-ansible || rc=1; $(MAKE) security-helm || rc=1; exit $$rc + # -w /repo so findings carry repo-relative paths (not ../../repo/...), which is # what the CI job summary prints and what SARIF code-scanning uploads need. -security: ## KICS IaC security scan of ansible/ (pinned engine image; CI runs this) +security-ansible: ## KICS scan of ansible/ (pinned engine image) mkdir -p kics-results docker run --rm --user $(shell id -u):$(shell id -g) -w /repo -v "$(CURDIR):/repo" $(KICS_IMAGE) \ scan --path /repo/ansible --type Ansible \ @@ -34,6 +44,21 @@ security: ## KICS IaC security scan of ansible/ (pinned engine image; --report-formats json,sarif --output-path /repo/kics-results \ --no-progress --fail-on high +# The chart cannot render with its defaults (required values fail loud), so KICS +# scans the manifests rendered from the widest CI values file instead of the chart +# directory (which it would try, and fail, to render itself). Findings therefore +# point at kics-results/helm-render/decdn-node.yaml, not at the chart sources. +security-helm: ## KICS scan of the decdn-node chart's rendered manifests (needs helm) + mkdir -p kics-results/helm-render + helm template decdn-node $(CHART) -f $(CHART)/ci/ci-values.yaml > kics-results/helm-render/decdn-node.yaml + docker run --rm --user $(shell id -u):$(shell id -g) -w /repo -v "$(CURDIR):/repo" $(KICS_IMAGE) \ + scan --path /repo/kics-results/helm-render --type Kubernetes \ + --report-formats json,sarif --output-path /repo/kics-results/helm \ + --no-progress --fail-on high + +lint-helm: ## helm lint + render tests + kubeconform + shared schema-key check (needs helm, yq, python3>=3.11, docker) + KUBECONFORM="docker run --rm -i $(KUBECONFORM_IMAGE)" $(CHART)/tests/render-test.sh + molecule: ## containerised converge/verify of the decdn_node role (needs Docker) $(MAKE) -C ansible molecule diff --git a/README.md b/README.md index db442de..b379c97 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,8 @@ [![Conventional Commits](https://img.shields.io/badge/Conventional%20Commits-1.0.0-yellow.svg)](https://www.conventionalcommits.org) The official **DevOps repo** for deploying a deCDN node — infrastructure, deployment, and -operational tooling, driven by a single declarative [Ansible](ansible/README.md) project. +operational tooling: a declarative [Ansible](ansible/README.md) project for VMs and bare +metal, and a [Helm chart](charts/decdn-node/README.md) for Kubernetes. This repo is **infrastructure only**. It is *not* a source of truth for protocol or economic facts (chain-id, token addresses, fee splits) — those trace to the deCDN ADRs. @@ -18,11 +19,12 @@ this repo. ## What it deploys -A single deployment over a hardened host baseline: +The same node, two ways: -| Playbook | Deploys | Exposure | -|----------|---------|----------| -| **`site.yml`** | A public **deCDN node** (`decdn-node`) — the product. Installed from a pinned GitHub release tarball under a hardened systemd unit. | Public QUIC **udp/4433** | +| Path | Deploys | Exposure | +|------|---------|----------| +| **`ansible/playbooks/site.yml`** | A public **deCDN node** (`decdn-node`) — the product. Installed from a pinned GitHub release tarball under a hardened systemd unit, over a hardened host baseline. | Public QUIC **udp/4433** | +| **`charts/decdn-node`** | The same node on Kubernetes: one-replica StatefulSet, PVC data dir, operator-provisioned Secrets, restricted pod security. | Public QUIC **udp/4433** (LoadBalancer/NodePort/hostPort); metrics ClusterIP + NetworkPolicy | ## Architecture @@ -45,9 +47,10 @@ stops at host prep and startup. | Path | What it is | |------|------------| | [`ansible/`](ansible/README.md) | The **declarative deployment project** — `inventory/`, `playbooks/`, `roles/` (baseline, decdn_node). The whole deploy surface lives here. | +| [`charts/decdn-node/`](charts/decdn-node/README.md) | The **Helm chart** for running the node on Kubernetes. | | `Makefile` | Root hygiene/security/CI mirror — runs the same lint + IaC scans CI does. | | `ansible/Makefile` | The deploy driver — `make deps/check/deploy`. | -| `.github/workflows/` | The blocking CI gate (`ansible-lint` + KICS + `galaxy-build` + `molecule` + `actionlint`). | +| `.github/workflows/` | The blocking CI gate (`ansible-lint` + `helm` + KICS + `galaxy-build` + `molecule` + `actionlint`). | ## Quickstart @@ -66,6 +69,13 @@ make deploy # provision the deCDN node See [`ansible/README.md`](ansible/README.md) for the full setup and the deCDN-node prerequisites (release tarball, per-node `host_vars`, operator-provisioned eth keystore). +On Kubernetes, create the keystore and RPC Secrets out of band, then install the chart +(see [`charts/decdn-node/README.md`](charts/decdn-node/README.md)): + +```bash +helm install decdn-node-1 charts/decdn-node -n decdn -f values-node-1.yaml +``` + ## Security model - **Nothing secret is committed.** The eth keystore and `rpc_url` (which may embed an API @@ -93,7 +103,8 @@ Two Makefiles, two jobs. The **root** Makefile mirrors CI's hygiene/security gat make hooks # one-time: install the pre-commit git hook (pip install pre-commit first) make lint # all pre-commit hooks on all files (hygiene, shellcheck, yamllint, markdown) make lint-ansible # vendor collections + full ansible-lint (production profile) -make security # KICS IaC scan of ansible/ (pinned engine image) +make lint-helm # chart: helm lint + render tests + kubeconform + schema keys +make security # KICS IaC scan of ansible/ + the rendered chart (pinned engine image) # Ansible deploys — run from ansible/ cd ansible @@ -105,14 +116,16 @@ make check / deploy # deCDN node (site.yml): dry-run / provision **Gotcha — pre-commit is local-only.** Hygiene/shellcheck/yamllint/markdown run via `make hooks`/`make lint` on your machine, **not** in CI. The blocking gate is `.github/workflows/` (`ansible-lint` + KICS + `galaxy-build` + `molecule` on `ansible/**`, -plus `actionlint`). `ansible-lint` +`helm` + KICS on `charts/**` and the shared schema checker, plus `actionlint`). `ansible-lint` is not a per-commit hook (it needs collections vendored) — run `make lint-ansible`. ## CI & quality gates - **`ci.yml`** — path-filtered so heavy jobs skip unrelated PRs: `ansible-lint` (production profile + playbook syntax-check), a `galaxy-build` readiness gate (builds the `decdn.node` - collection and runs galaxy-importer's checks), **KICS** IaC scan (fail on HIGH), and + collection and runs galaxy-importer's checks), a `helm` job (`make lint-helm`: strict lint, + positive/negative render tests, kubeconform, and the upstream schema-key check shared with + molecule), **KICS** IaC scan of `ansible/` and the rendered chart (fail on HIGH), and `actionlint` on the workflows themselves. The KICS engine is pinned by digest and every third-party action by full commit SHA (a re-pointed tag can ship malicious code). - **`molecule.yml`** — a containerised converge + idempotence + verify of the `decdn_node` @@ -131,5 +144,6 @@ is not a per-commit hook (it needs collections vendored) — run `make lint-ansi - [`ansible/README.md`](ansible/README.md) — full setup, security model, and deploy steps - [`ansible/roles/decdn_node/README.md`](ansible/roles/decdn_node/README.md) — the deCDN node role +- [`charts/decdn-node/README.md`](charts/decdn-node/README.md) — the Helm chart - [`CONTRIBUTING.md`](CONTRIBUTING.md) — the local + CI check workflow - [`AGENTS.md`](AGENTS.md) — repo hard rules and conventions (for humans and AI agents) diff --git a/ansible/molecule/schema/files/check-schema-keys.py b/ansible/molecule/schema/files/check-schema-keys.py new file mode 100755 index 0000000..780f1c0 --- /dev/null +++ b/ansible/molecule/schema/files/check-schema-keys.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +"""Flag any config PATH in a rendered node.toml that upstream rejects. + + check-schema-keys.py [] [] + +Defaults: /etc/decdn/node.toml, and schema-keys.txt next to this file. + +Shared by both deploy paths: the Ansible `schema` molecule scenario runs it on the +role's render, and `make lint-helm` runs it on the Helm chart's renders. One key +inventory guards both. +""" +import os +import sys +import tomllib + +config_path = sys.argv[1] if len(sys.argv) > 1 else "/etc/decdn/node.toml" +keys_path = ( + sys.argv[2] if len(sys.argv) > 2 + else os.path.join(os.path.dirname(os.path.abspath(__file__)), "schema-keys.txt") +) +with open(config_path, "rb") as fh: + config = tomllib.load(fh) +with open(keys_path, encoding="utf-8") as fh: + known = { + line.strip() for line in fh + if line.strip() and not line.startswith("#") + } + + +def _lookup(root, dotted): + """Resolve a dotted path back to its value, for leaf/table triage.""" + node = root + for part in dotted.split("."): + if isinstance(node, list): + node = node[0] if node else {} + if not isinstance(node, dict) or part not in node: + return None + node = node[part] + return node + + +def walk(node, path=""): + """Yield the dotted path of every key in every table.""" + if isinstance(node, dict): + for key, value in node.items(): + here = f"{path}.{key}" if path else key + yield here + yield from walk(value, here) + elif isinstance(node, list): + # [[cache.origins]] is an array of tables. Every element shares one + # schema, so they collapse onto the same path -- an index would make + # the inventory depend on how many origins an operator configured. + for item in node: + if isinstance(item, dict): + yield from walk(item, path) + + +def normalize(dotted): + """Collapse the one path hop that is data rather than a schema field. + + [network.discovery.peers.] is keyed by a 64-char NodeId, so the + hop itself is never in the inventory; the DiscoveryPeer fields BELOW it + are (as network.discovery.peers.*). Only that single hop is rewritten, + so a bogus key beside relay_url/addrs is still caught. + """ + prefix = "network.discovery.peers." + if dotted.startswith(prefix): + rest = dotted[len(prefix):].split(".", 1) + return prefix + "*" + ("." + rest[1] if len(rest) > 1 else "") + return dotted + + +def is_table(value): + """A table or an array of tables: a section, not a field. + + Arrays of SCALARS (relay_urls, pinned_hashes, ...) and empty arrays are + fields like any other and must be checked -- treating every list as a table + once let a misspelled list key through. + """ + if isinstance(value, dict): + return True + return isinstance(value, list) and bool(value) and all(isinstance(i, dict) for i in value) + + +emitted = list(walk(config)) +if not emitted: + print(f"{config_path} is empty: nothing to check, which proves nothing", + file=sys.stderr) + sys.exit(1) + +# A table header is itself a path (`cache.tinylfu`), and intermediate tables +# are not fields of anything -- they are the sections the inventory is keyed +# BY. Only leaf paths are checked; a bogus TABLE surfaces as its children +# being unknown, or (if empty) as the section-set assertion in the caller. +unknown = sorted({ + normalize(dotted) for dotted in emitted + if not is_table(_lookup(config, dotted)) + and normalize(dotted) not in known +}) +# An empty table has no leaves, so it would otherwise pass unseen. +unknown += sorted({ + normalize(dotted) for dotted in emitted + if _lookup(config, dotted) == {} and normalize(dotted) not in known + and not any(k.startswith(normalize(dotted) + ".") for k in known) +}) + +if unknown: + print(f"{config_path} emits paths absent from the upstream config schema:", + file=sys.stderr) + for path in unknown: + print(f" {path}", file=sys.stderr) + print( + "\nEvery config section upstream is deny_unknown_fields with no " + "serde aliases, so each of these is a daemon startup failure. Note " + "a path can be wrong because the KEY is unknown or because a known " + "key landed in the wrong TABLE -- a scalar [cache] key emitted below " + "a [cache.*] header nests into it silently. Re-sync the renderer " + "(ansible/roles/decdn_node/templates/node.toml.j2 or charts/decdn-node), then " + "regenerate ansible/molecule/schema/files/schema-keys.txt (see gen-schema-keys.py).", + file=sys.stderr, + ) + sys.exit(1) + +print(f"schema OK ({config_path}): {len(known)} known paths, " + f"{len(emitted)} emitted, all recognised") diff --git a/ansible/molecule/schema/files/checker-fixtures/bad.expected b/ansible/molecule/schema/files/checker-fixtures/bad.expected new file mode 100644 index 0000000..31b8cf5 --- /dev/null +++ b/ansible/molecule/schema/files/checker-fixtures/bad.expected @@ -0,0 +1,7 @@ +identity.bogus_scalar +network.relay_urlz +network.discovery.peers.*.bogus_peer_field +content.denied_hashez +cache.tinylfu.cache_size_mb +cache.origins.bogus_origin_field +bogus_empty_table diff --git a/ansible/molecule/schema/files/checker-fixtures/bad.toml b/ansible/molecule/schema/files/checker-fixtures/bad.toml new file mode 100644 index 0000000..a832524 --- /dev/null +++ b/ansible/molecule/schema/files/checker-fixtures/bad.toml @@ -0,0 +1,26 @@ +# Must FAIL check-schema-keys.py, naming every path listed in bad.expected. +[identity] +region = "DE" +bogus_scalar = 1 + +[network] +relay_urlz = ["https://relay.example.invalid/"] + +[network.discovery.peers.253bad481e6371866c9f6276b2a7b3a10ca16255668e740e6fc01da1cacc4350] +addrs = ["203.0.113.4:4433"] +bogus_peer_field = "x" + +[content] +denied_hashez = [] + +[cache.tinylfu] +cache_size_mb = 10240 + +[[cache.origins]] +kind = "http" + +[[cache.origins]] +kind = "fs" +bogus_origin_field = "x" + +[bogus_empty_table] diff --git a/ansible/molecule/schema/files/checker-fixtures/good.toml b/ansible/molecule/schema/files/checker-fixtures/good.toml new file mode 100644 index 0000000..6eb7902 --- /dev/null +++ b/ansible/molecule/schema/files/checker-fixtures/good.toml @@ -0,0 +1,26 @@ +# Must PASS check-schema-keys.py: one of each shape the walker handles. +[identity] +region = "DE" + +[network] +bind_port = 4433 +relay_urls = ["https://relay.example.invalid/"] + +[network.discovery.peers.253bad481e6371866c9f6276b2a7b3a10ca16255668e740e6fc01da1cacc4350] +addrs = ["203.0.113.4:4433"] + +[cache] +pinned_hashes = [] + +[[cache.origins]] +kind = "http" +url = "https://a.example.invalid/" + +[[cache.origins]] +kind = "s3" +bucket = "b" + +[cache.origins.credentials] +source = "default-chain" + +[security] diff --git a/ansible/molecule/schema/verify.yml b/ansible/molecule/schema/verify.yml index e57b4d5..fc77dbd 100644 --- a/ansible/molecule/schema/verify.yml +++ b/ansible/molecule/schema/verify.yml @@ -16,96 +16,17 @@ dest: /root/schema-keys.txt mode: "0644" + # The checker is shared with the Helm chart (`make lint-helm`), so one key + # inventory guards both deploy paths. - name: Stage the schema-drift checker ansible.builtin.copy: + src: check-schema-keys.py dest: /root/molecule-assert-schema.py mode: "0755" - content: | - """Flag any config PATH in the rendered node.toml that upstream rejects.""" - import sys, tomllib - - config_path = sys.argv[1] if len(sys.argv) > 1 else "/etc/decdn/node.toml" - with open(config_path, "rb") as fh: - config = tomllib.load(fh) - with open("/root/schema-keys.txt", encoding="utf-8") as fh: - known = { - line.strip() for line in fh - if line.strip() and not line.startswith("#") - } - - def _lookup(root, dotted): - """Resolve a dotted path back to its value, for leaf/table triage.""" - node = root - for part in dotted.split("."): - if isinstance(node, list): - node = node[0] if node else {} - if not isinstance(node, dict) or part not in node: - return None - node = node[part] - return node - - def walk(node, path=""): - """Yield the dotted path of every key in every table.""" - if isinstance(node, dict): - for key, value in node.items(): - here = f"{path}.{key}" if path else key - yield here - yield from walk(value, here) - elif isinstance(node, list): - # [[cache.origins]] is an array of tables. Every element shares one - # schema, so they collapse onto the same path -- an index would make - # the inventory depend on how many origins an operator configured. - for item in node: - if isinstance(item, dict): - yield from walk(item, path) - - def normalize(dotted): - """Collapse the one path hop that is data rather than a schema field. - - [network.discovery.peers.] is keyed by a 64-char NodeId, so the - hop itself is never in the inventory; the DiscoveryPeer fields BELOW it - are (as network.discovery.peers.*). Only that single hop is rewritten, - so a bogus key beside relay_url/addrs is still caught. - """ - prefix = "network.discovery.peers." - if dotted.startswith(prefix): - rest = dotted[len(prefix):].split(".", 1) - return prefix + "*" + ("." + rest[1] if len(rest) > 1 else "") - return dotted - - # A table header is itself a path (`cache.tinylfu`), and intermediate tables - # are not fields of anything -- they are the sections the inventory is keyed - # BY. Only leaf paths are checked; a bogus TABLE surfaces as its children - # being unknown, or (if empty) as the section-set assertion below. - unknown = sorted({ - normalize(dotted) for dotted in walk(config) - if not isinstance(_lookup(config, dotted), (dict, list)) - and normalize(dotted) not in known - }) - - if unknown: - print("node.toml emits paths absent from the upstream config schema:", - file=sys.stderr) - for path in unknown: - print(f" {path}", file=sys.stderr) - print( - "\nEvery config section upstream is deny_unknown_fields with no " - "serde aliases, so each of these is a daemon startup failure. Note " - "a path can be wrong because the KEY is unknown or because a known " - "key landed in the wrong TABLE -- a scalar [cache] key emitted below " - "a [cache.*] header nests into it silently. Re-sync " - "roles/decdn_node/templates/node.toml.j2, then regenerate " - "molecule/schema/files/schema-keys.txt (see gen-schema-keys.py).", - file=sys.stderr, - ) - sys.exit(1) - - print(f"schema OK ({config_path}): {len(known)} known paths, " - f"{sum(1 for _ in walk(config))} emitted, all recognised") - name: Assert node.toml emits no key outside the upstream schema ansible.builtin.command: - cmd: python3 /root/molecule-assert-schema.py + cmd: python3 /root/molecule-assert-schema.py /etc/decdn/node.toml /root/schema-keys.txt changed_when: false # The second converge play renders the mutually-exclusive [[cache.origins]] @@ -113,12 +34,12 @@ # array-of-tables branch and the http/fs origin variants. - name: Assert the multi-origin render emits no key outside the schema ansible.builtin.command: - cmd: python3 /root/molecule-assert-schema.py /etc/decdn/node-origins.toml + cmd: python3 /root/molecule-assert-schema.py /etc/decdn/node-origins.toml /root/schema-keys.txt changed_when: false - name: Assert the resolve-only render emits no key outside the schema ansible.builtin.command: - cmd: python3 /root/molecule-assert-schema.py /etc/decdn/node-resolve-only.toml + cmd: python3 /root/molecule-assert-schema.py /etc/decdn/node-resolve-only.toml /root/schema-keys.txt changed_when: false # The schema checker above only proves every emitted key is LEGAL. It cannot diff --git a/charts/decdn-node/.helmignore b/charts/decdn-node/.helmignore new file mode 100644 index 0000000..e684858 --- /dev/null +++ b/charts/decdn-node/.helmignore @@ -0,0 +1,8 @@ +# Not part of the packaged chart. +.DS_Store +.git/ +*.swp +*.bak +*.tmp +*.orig +tests/ diff --git a/charts/decdn-node/Chart.yaml b/charts/decdn-node/Chart.yaml new file mode 100644 index 0000000..7f21330 --- /dev/null +++ b/charts/decdn-node/Chart.yaml @@ -0,0 +1,27 @@ +--- +apiVersion: v2 +name: decdn-node +description: >- + A deCDN node (decdn-node daemon) on Kubernetes: public QUIC over UDP, a + persistent data dir holding the operator-provisioned node identity, and a + ClusterIP-only metrics endpoint behind a NetworkPolicy. +type: application +# Chart version: bump on any chart change (SemVer). +version: 0.1.0 +# The decdn release the chart's config rendering is synced against. Upstream has +# no `v*` tag (or published image) yet, so 0.0.0 is a placeholder and the chart +# refuses to render until `image.tag` or `image.digest` is set explicitly. +appVersion: "0.0.0" +kubeVersion: ">=1.25.0-0" +home: https://decdn.org +sources: + - https://github.com/decdn/devops + - https://github.com/decdn/decdn +keywords: + - decdn + - cdn + - iroh + - quic +maintainers: + - name: deCDN + url: https://decdn.org diff --git a/charts/decdn-node/README.md b/charts/decdn-node/README.md new file mode 100644 index 0000000..3363b8e --- /dev/null +++ b/charts/decdn-node/README.md @@ -0,0 +1,277 @@ +# decdn-node Helm chart + +Deploys one **deCDN node** (`decdn-node`) on Kubernetes. It is the Kubernetes +counterpart of the Ansible [`decdn_node` role](../../ansible/roles/decdn_node/README.md) +and keeps the same guarantees: + +- no secrets in git or in values +- missing required values, secret-bearing keys and chart-managed keys fail at render time +- one public hole (QUIC udp/4433) +- no baked-in protocol facts + +One release is one node identity: one keystore and one data dir. To run a fleet, install +one release per node. The StatefulSet is fixed at one replica, because two pods sharing a +keystore would double-sign. + +`node.toml` renders against upstream `crates/common/src/config/types.rs`. Every section +there is `deny_unknown_fields`, so an unknown key crash-loops the pod. CI renders the +chart's own fixtures (`ci/*.yaml`) and checks every emitted key against the same +committed inventory the Ansible `schema` molecule scenario uses +([`schema-keys.txt`](../../ansible/molecule/schema/files/schema-keys.txt)). Re-sync +it when bumping the decdn version. Keys and value types **you** add to `config` are +validated only by the daemon at startup (a typo is a crash-loop, not a render error), +and unlike the Ansible role there is no in-cluster `decdn config validate`, because the +image has no CLI. + +## What it deploys + +| Object | Purpose | +|--------|---------| +| `StatefulSet` (1 replica) | `decdn-node run` under a non-root, read-only-rootfs, all-caps-dropped pod. It includes a `prepare` init container. | +| `PersistentVolumeClaim` (`data`) | The data dir: node identity, voucher state, receipts, and the cache. | +| `ConfigMap` | The rendered `node.toml`. Its checksum rolls the pod on any change. | +| `Service` (quic) | Public UDP. The type is `LoadBalancer`, `NodePort` or `ClusterIP`, with `externalTrafficPolicy: Local`. | +| `Service` (metrics) | ClusterIP only, never public. | +| `NetworkPolicy` | Ingress allows QUIC from anywhere and metrics only from `metrics.networkPolicy.from`. Egress is open unless `networkPolicy.egress` is set. | +| `ServiceMonitor` | Optional (`metrics.serviceMonitor.enabled`). | + +What it does **not** do: + +- **Create Secrets.** The operator provisions them. +- **Generate keys.** The image is daemon-only, with no `decdn` CLI. +- **Stake or register on-chain** (ADR 019 Phase 2). Run `decdn setup` off-cluster. + +## Prerequisites + +- **Kubernetes ≥ 1.25**, with a CNI that enforces NetworkPolicy (the chart relies on it + to keep metrics private), and a StorageClass that can provision a volume larger than + `config.cache.cache_size_mb` + `config.cache.disk_headroom_mb` (daemon default 8192 MiB). + With the defaults (10240 + 8192 MiB), the 20Gi `persistence.size` is tight. +- **UDP reachability.** Either a load balancer that supports UDP Services, or + `quic.hostPort.enabled` on nodes with a public IP. +- **An image.** Upstream has not published `ghcr.io/decdn/decdn-node` yet. The chart + refuses to render until `image.tag` or `image.digest` is set. To build one yourself, + follow the header of the upstream `decdn/Dockerfile`: `cargo build --release -p + decdn-node`, copy the binary to `dist//decdn-node`, then `docker build`. The + Dockerfile only packages that binary, on `debian:bookworm-slim` (glibc 2.36), so build + on a system with glibc ≤ 2.36 or the binary will not start. +- **The `decdn` CLI on your workstation**, for `key-gen`, `setup`, and `node health` + through a port-forward. + +## Secrets + +The chart only **references** Secrets. Create them out of band and keep the source files +off shared disks. + +```bash +# 1. Node identity, generated off-cluster. +umask 077 +mkdir -p ./node-1 && openssl rand -base64 32 > ./node-1/keystore.password +decdn key-gen --output-dir ./node-1 --keystore-password-file ./node-1/keystore.password + +kubectl -n decdn create secret generic decdn-node-1-keys \ + --from-file=keystore.json=./node-1/keystore.json \ + --from-file=node.secret=./node-1/node.secret \ + --from-file=keystore.password=./node-1/keystore.password + +# 2. The RPC endpoint (it may embed an API key): the k8s equivalent of +# /etc/decdn/decdn.env. Use one key per variable (--from-literal), not +# --from-file=decdn.env. +kubectl -n decdn create secret generic decdn-node-1-env \ + --from-literal=DECDN_RPC_URL='https://…' +``` + +Only **named** keys from the env Secret reach the node: `secrets.env.rpcUrlKey` (default +`DECDN_RPC_URL`) and anything in `secrets.env.passthroughKeys` (for example +`AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` for an S3 origin). The chart deliberately +does not use `envFrom`: upstream lets every `DECDN_*` env var override `node.toml`, so a +stray `DECDN_BIND_PORT`, `DECDN_METRICS_BIND`, `DECDN_ETH_KEYSTORE`, `DECDN_CHAIN_ID` or +address variable would silently bypass the managed keys and the schema checks. For the +same reason `DECDN_*` names in `passthroughKeys` are refused. + +If a Secret or a named key is missing, the pod does not start: it stays in +`ContainerCreating` (a `FailedMount` event, for the keystore Secret) or +`CreateContainerConfigError` (for an env key). Check `kubectl describe pod`. + +Back up `./node-1` offline. It is the node's on-chain identity. The data dir is not a +backup, because the init container overwrites the key files from the Secret on every +pod start. + +**Why the init container?** Upstream refuses `keystore.json` and `node.secret` if they are +symlinks or carry any group or world permission bit, and a data dir with group or world +bits. Secret volume files are symlinks (and `fsGroup` adjusts their modes), so they can't +be used in place. On every pod start, `prepare` creates `data_dir` as a subdirectory of +the PVC (the PVC root carries `fsGroup` bits), installs those two files into it at `0600`, +and installs `keystore.password` into an in-memory volume instead. The password is never +written to the PVC, so a volume snapshot or backup does not hold both the keystore and the +password that unlocks it. The Secret itself is mounted only in the init container, never +in the daemon. + +## Required values + +| Value | Notes | +|-------|-------| +| `image.tag` or `image.digest` | Required until upstream publishes a release. Prefer a digest. | +| `secrets.keystore.existingSecret` | Holds the keys `keystore.json`, `node.secret` and `keystore.password`. Rename them with `secrets.keystore.keys.*`. | +| `secrets.env.existingSecret` | Holds `DECDN_RPC_URL` (key name: `secrets.env.rpcUrlKey`). | +| `config.identity.region` | ISO 3166-1 alpha-2, uppercase. | +| `config.blockchain.chain_id` | For example, `421614` (Arbitrum Sepolia). | +| `config.blockchain.{payment_pool,capacity_bond,slash_judge,content_blacklist}_address` | Non-zero `0x` addresses. | + +Take chain IDs and contract addresses from the deCDN ADRs and the deployment manifest for +your chain. Never make them up. Values are validated by `values.schema.json` plus +template guards, so a missing or malformed value fails `helm install` with a message that +names it. + +```yaml +# values-node-1.yaml +image: + digest: sha256:… +secrets: + keystore: {existingSecret: decdn-node-1-keys} + env: {existingSecret: decdn-node-1-env} +config: + identity: {region: DE} + blockchain: + chain_id: 421614 + payment_pool_address: "0x…" + capacity_bond_address: "0x…" + slash_judge_address: "0x…" + content_blacklist_address: "0x…" +``` + +```bash +helm install decdn-node-1 charts/decdn-node -n decdn -f values-node-1.yaml +``` + +## Configuration (`config`) + +`config` mirrors `node.toml` section for section. The role's variable names map onto it +directly: `decdn_cache_size_mb` becomes `config.cache.cache_size_mb`, and +`decdn_tinylfu_sketch_bytes` becomes `config.cache.tinylfu.sketch_bytes`. The role's +[README](../../ansible/roles/decdn_node/README.md) documents each knob. Any key you do not +set takes the daemon default, and a `null` value removes a chart default. + +Chart defaults that differ from the daemon's are the same as the role's: + +- `cache.max_blob_size_mb: 1024`: a deliberate cap at one tenth of the default cache (the + daemon default is `min(51200, cache_size_mb)`). It must be at least 1: the cache engine's + admit gate has no zero special-case, despite the CLI help, so `0` rejects every blob. +- `observability.log_format: json` (the daemon default is `pretty`). +- `cache.cache_size_mb: 10240` and `payment.rate_per_mb: 10` are pinned explicitly, at the + daemon's own values. +- `cache.node_to_node_pull_through_enabled`: derived when unset. It is `true` with no + `cache.origin` or `cache.origins`, and `false` when an origin is configured. + +**Managed keys.** The chart sets these to match the pod spec. Setting any of them in +`config` fails the render. (Their `DECDN_*` env equivalents cannot be injected either; +see [Secrets](#secrets).) + +| Key | Set from | +|-----|----------| +| `identity.data_dir`, `cache.cache_dir`, `blockchain.eth_keystore` | Fixed PVC paths (`/var/lib/decdn/node`, `/var/lib/decdn/node/cache`, and `keystore.json` inside the data dir) | +| `network.bind_port` | `quic.port` | +| `observability.metrics_port` | `metrics.port` | +| `observability.metrics_bind` | Always `0.0.0.0`; see [Network](#network) | + +**Forbidden keys.** `rpc_url`, `access_key_id`, `session_token`, and any key containing +`password` or `secret` (dashes count as underscores) are refused anywhere in `config`. +`config` renders to a ConfigMap, which is readable by anyone with configmap read access in +the namespace. The guard checks key names only: never embed a credential in a value +either (such as `https://user:pass@…` in an origin URL). Put secrets in the env Secret. + +**Pod overrides.** `podLabels` cannot re-set a chart label (that would detach the pod from +its StatefulSet selector and Services), `podAnnotations` cannot re-set `checksum/config`, +and `podSecurityContext` / `securityContext` can be changed (for example, a different +non-root uid) but the render fails if the result runs as root, allows privilege escalation +or privileged mode, makes the root filesystem writable, adds capabilities, drops fewer than +`ALL`, or disables seccomp. + +**Other render-time checks.** Every top-level `config` entry must be a table; a `null` +inside a list element is refused (omit the key instead); and whole numbers of 2^53 or more +are refused, because values files decode numbers as float64 and they would render rounded. + +**Origins.** Use either `config.cache.origin` (one origin) or `config.cache.origins` (an +ordered, non-empty list), not both. For S3 credentials, use +`credentials: {source: default-chain}` and either pass the AWS variables through +`secrets.env.passthroughKeys`, or use IRSA (the role-ARN annotation in +`serviceAccount.annotations`) or an EKS Pod Identity association. Both webhooks inject +their own token volume, so they should work with the chart's +`automountServiceAccountToken: false` (not yet tested in a cluster). If you set +`networkPolicy.egress`, allow STS (or the Pod Identity agent). Never put credentials in +`config`. + +## Network + +- **QUIC** (`quic.port`, default 4433/udp) is the only public port. + - `service.type: LoadBalancer` (the default) gives the node a stable public UDP + address, and `externalTrafficPolicy: Local` preserves client source IPs for + per-source rate limiting. + - For bare-metal-style nodes, set `quic.hostPort.enabled: true` and + `service.enabled: false`. + - Without direct reachability, the node still works through iroh relays, at a latency + cost. +- **Metrics** bind `0.0.0.0` inside the pod rather than loopback, which the Ansible role + requires. That is how kubelet probes and Prometheus reach `/metrics`. Metrics are + exposed only by a ClusterIP Service, and the NetworkPolicy admits TCP to that port only + from `metrics.networkPolicy.from`. That list is empty by default, so no scraper is + allowed. Kubelet probes are unaffected: the NetworkPolicy spec always allows traffic + between a pod and the node it runs on. + - The NetworkPolicy is what keeps metrics private, so `networkPolicy.enabled: false` + fails the render unless `networkPolicy.allowUnrestrictedMetrics: true` accepts that + every pod in the cluster can reach them. + - With the policy enabled, `metrics.serviceMonitor.enabled` without any + `metrics.networkPolicy.from` entry fails the render. The chart cannot check that your + entries actually select Prometheus. +- **The admin RPC** is hardcoded to `127.0.0.1` upstream and cannot be reached from the + pod network. Use a port-forward: + + ```bash + kubectl -n decdn port-forward pod/decdn-node-1-0 9191 + DECDN_ADMIN_URL=http://127.0.0.1:9191 decdn node health + ``` + +## Probes and lifecycle + +- `/metrics` is the only HTTP route suitable for probes; there is no `/health` endpoint. + It binds only after startup completes: RPC preflight, identity and keystore load, cache + and endpoint build, then the chain bring-up (CapacityBond registry and slash-watcher + enumerations, blacklist and DHT bootstrap). + - The default `startupProbe` allows 5 minutes (5s × 60). A slow RPC delays readiness. + - A failed bring-up, such as an unreachable RPC, exits the container and Kubernetes + restarts it. + - **Ready is not the same as serving.** The paid-delivery QUIC listeners open only after + the first ContentBlacklist sync, which comes after `/metrics` binds. Check the logs + before counting on traffic. +- **SIGTERM runs a graceful drain.** `terminationGracePeriodSeconds` defaults to 300, the + role's `TimeoutStopSec`. Killing a pod mid-drain loses paid deliveries, so do not + force-delete it. +- **Config changes roll the pod** through the ConfigMap checksum. Secret changes do not: + after rotating the keystore or RPC URL, run + `kubectl rollout restart statefulset/`. The daemon's partial + SIGHUP hot reload (`log_level`, `pinned_hashes`, `[security]`, `[content]`, + `[load_shed]`) is not used, because a ConfigMap update reaches the mounted file with a + delay and a restart is deterministic. + +## On-chain onboarding + +The node serves paid traffic only after it is staked and registered (ADR 019 Phase 2). +That is a manual operator step, as it is for the Ansible path. Run `decdn setup`, which +supports `--dry-run`, from your workstation against the same keystore you put in the +Secret. + +## Development + +```bash +make lint-helm # helm lint --strict, positive/negative render tests, kubeconform, schema keys +make security-helm # KICS over the rendered manifests (fail on HIGH) +DECDN_CLI=../decdn/target/release/decdn make lint-helm + # also run the real `decdn config validate` on every CI render +``` + +CI has no decdn binary, so it reports `SKIPPED: decdn config validate`. Run the +`DECDN_CLI` form locally whenever you bump the decdn version or change the renderer. + +The `ci/*.yaml` values files mirror the three plays of the molecule `schema` scenario: +every knob with an S3 origin, the multi-origin list, and resolve-only discovery. When you +add a knob to one, add it to the other. diff --git a/charts/decdn-node/ci/ci-origins.yaml b/charts/decdn-node/ci/ci-origins.yaml new file mode 100644 index 0000000..8a4e09a --- /dev/null +++ b/charts/decdn-node/ci/ci-origins.yaml @@ -0,0 +1,67 @@ +--- +# The [[cache.origins]] array-of-tables form (http, fs, s3 with a nested +# credentials table), with pull-through left to the chart's derivation (must render +# false). Mirrors the multi-origin play of ansible/molecule/schema/converge.yml. +# Also the non-default wiring branches: custom ports (so port agreement between +# node.toml, the pod, the Services and the NetworkPolicy is actually tested), +# NodePort + nodePort, hostPort, egress rules, custom Secret key names, +# passthrough env, and an externally managed ServiceAccount. +image: + tag: 0.0.0-ci +fullnameOverride: node-origins +# Allowed pod overrides: extra labels/annotations and a different non-root uid. +podLabels: + team: edge +podAnnotations: + example.invalid/owner: edge +podSecurityContext: + runAsUser: 2000 + runAsGroup: 2000 + fsGroup: 2000 +imagePullSecrets: + - name: regcred +serviceAccount: + create: false + name: decdn-sa +service: + type: NodePort + nodePort: 30443 +quic: + port: 5000 + hostPort: + enabled: true + port: 5000 +metrics: + port: 9100 +networkPolicy: + egress: + - ports: + - protocol: TCP + port: 443 +secrets: + keystore: + existingSecret: decdn-node-keys + keys: + keystore: ks + nodeSecret: ns + password: pw + env: + existingSecret: decdn-node-env + rpcUrlKey: rpc + passthroughKeys: [AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY] +config: + identity: + region: DE + blockchain: + chain_id: 421614 + payment_pool_address: "0x1111111111111111111111111111111111111111" + capacity_bond_address: "0x2222222222222222222222222222222222222222" + slash_judge_address: "0x3333333333333333333333333333333333333333" + content_blacklist_address: "0x4444444444444444444444444444444444444444" + cache: + origins: + - {kind: http, url: "https://primary.example.invalid/", decompress: strict} + - {kind: fs, path: /var/lib/decdn/origin} + - {kind: s3, bucket: mirror-blobs, region: eu-west-1, path_style: false, + endpoint_url: "https://acct.b2.example.invalid", prefix: "blobs/", + credentials: {source: default-chain, profile: mirror}} diff --git a/charts/decdn-node/ci/ci-resolve-only.yaml b/charts/decdn-node/ci/ci-resolve-only.yaml new file mode 100644 index 0000000..204d295 --- /dev/null +++ b/charts/decdn-node/ci/ci-resolve-only.yaml @@ -0,0 +1,29 @@ +--- +# Resolve-only discovery: dns_origin WITHOUT pkarr_url, no origin (pull-through +# must derive true). Also exercises ClusterIP, a digest-pinned image, and the +# explicitly acknowledged NetworkPolicy-off path. Mirrors the resolve-only play of +# ansible/molecule/schema/converge.yml. +image: + digest: "sha256:0000000000000000000000000000000000000000000000000000000000000000" +service: + type: ClusterIP +networkPolicy: + enabled: false + allowUnrestrictedMetrics: true +secrets: + keystore: + existingSecret: decdn-node-keys + env: + existingSecret: decdn-node-env +config: + identity: + region: DE + network: + discovery: + dns_origin: resolve-only.example.invalid + blockchain: + chain_id: 421614 + payment_pool_address: "0x1111111111111111111111111111111111111111" + capacity_bond_address: "0x2222222222222222222222222222222222222222" + slash_judge_address: "0x3333333333333333333333333333333333333333" + content_blacklist_address: "0x4444444444444444444444444444444444444444" diff --git a/charts/decdn-node/ci/ci-values.yaml b/charts/decdn-node/ci/ci-values.yaml new file mode 100644 index 0000000..b11b0f6 --- /dev/null +++ b/charts/decdn-node/ci/ci-values.yaml @@ -0,0 +1,176 @@ +--- +# Every operator-facing knob set to a non-default value, so the schema-key check +# (make lint-helm) sees the maximal key set the chart can emit. Also the render +# `make security-helm` scans. Mirrors the first +# play of ansible/molecule/schema/converge.yml; keep the two in step. +# Placeholder data only — no real addresses, endpoints or secrets. +image: + tag: 0.0.0-ci +service: + loadBalancerIP: 203.0.113.10 + loadBalancerSourceRanges: ["0.0.0.0/0"] +persistence: + storageClass: fast-ssd + annotations: + example.invalid/backup: "exclude" +secrets: + keystore: + existingSecret: decdn-node-keys + env: + existingSecret: decdn-node-env +metrics: + networkPolicy: + from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: monitoring + serviceMonitor: + enabled: true +config: + identity: + region: DE + network: + relay_urls: ["https://relay1.example.invalid:443", "https://relay2.example.invalid:443"] + discovery: + pkarr_url: "https://pkarr.example.invalid/" + dns_origin: "discovery.example.invalid" + # A real ed25519 public key: upstream parses it as an iroh NodeId. + peers: + "253bad481e6371866c9f6276b2a7b3a10ca16255668e740e6fc01da1cacc4350": + relay_url: "https://relay.example.invalid/" + addrs: ["203.0.113.4:4433", "203.0.113.5:4433"] + blockchain: + chain_id: 421614 + payment_pool_address: "0x1111111111111111111111111111111111111111" + capacity_bond_address: "0x2222222222222222222222222222222222222222" + slash_judge_address: "0x3333333333333333333333333333333333333333" + content_blacklist_address: "0x4444444444444444444444444444444444444444" + slash_appeal_address: "0x5555555555555555555555555555555555555555" + origin_assignment_address: "0x6666666666666666666666666666666666666666" + publisher_registry_address: "0x7777777777777777777777777777777777777777" + usdc_address: "0x8888888888888888888888888888888888888888" + origin_directory_positive_ttl_sec: 600 + origin_directory_negative_ttl_sec: 60 + origin_directory_cache_capacity: 8192 + content_blacklist_poll_interval_sec: 300 + chain_staleness_grace_sec: 900 + rpc_watchdog_interval_sec: 15 + event_poll_interval_ms: 5000 + rate_bounds_poll_interval_sec: 1800 + fee_shares_poll_interval_sec: 1800 + redeem_threshold_micro_usdc: 2000000 + redeem_max_vouchers_per_tx: 250 + redeem_interval_secs: 120 + buyer_working_deposit_micro_usdc: 20000000 + buyer_max_approve: false + pool_min_remaining_deposit_micro_usdc: 500000 + pool_floor_signer_live_windows: 4 + payment: + rate_per_mb: 12 + delivery_floor: 0 + credit_max: 33554432 + credit_ramp_divisor: 4 + frame_target_bytes: 262144 + voucher_commit_interval_ms: 2500 + cache: + cache_size_mb: 20480 + disk_headroom_mb: 4096 + max_blob_size_mb: 2048 + max_rate_per_mb: 50 + pinned_hashes: ["aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"] + user_agent: "decdn-node/ci" + gc_interval_sec: 0 + fs_rescan_interval_sec: 30 + origin_probe_ttl_sec: 30 + origin_probe_negative_ttl_sec: 4 + origin_probe_fault_ttl_sec: 8 + origin_probe_timeout_ms: 1500 + origin_probe_memo_capacity: 2048 + eviction_high_water_pct: 85 + eviction_target_pct: 70 + eviction_per_sweep_budget: 32 + eviction_tick_secs: 2 + max_probe_holds: 512 + stake_lane_reserved_holds: 16 + node_to_node_pull_through_enabled: true + relay_foreign_namespaces: false + node_pull_probe_fanout: 8 + node_pull_timeout_sec: 30 + node_pull_stall_window_sec: 15 + node_pull_min_throughput_bps: 8192 + eviction_policy: tinylfu + admission_policy: tinylfu + tinylfu: + sketch_bytes: 524288 + promotion_threshold: 3 + probation_target_pct: 20 + aging_halflife_sec: 900 + serve_economics: + policy: margin + discount: 0.25 + n_max: 32 + warming_budget: 1000000 + warming_refill: 100 + origin_retry: + max_retries: 5 + initial_backoff_ms: 200 + max_backoff_ms: 20000 + jitter_ratio: 0.2 + buffered_max_bytes: 8388608 + circuit_breaker: + enabled: false + failure_threshold: 10 + cooldown_ms: 60000 + half_open_max_calls: 2 + origin: + kind: s3 + bucket: decdn-blobs + region: us-east-1 + endpoint_url: "https://acct.r2.cloudflarestorage.invalid" + path_style: true + prefix: "blobs/" + decompress: strict + credentials: + source: default-chain + profile: decdn + observability: + otlp_endpoint: "http://localhost:4317" + security: + max_concurrent_handlers: 128 + per_source_rate_per_sec: 50.5 + per_source_burst: 100 + max_tracked_sources: 2048 + load_shed: + policy: always-admit + egress_budget_mbps: 500 + max_concurrent_serves_high: 128 + max_concurrent_serves_low: 96 + per_client_serve_cap: 16 + dht: + rate_limit: + per_peer_rate_per_sec: 10.0 + per_peer_burst: 20 + per_ip_rate_per_sec: 50.0 + per_ip_burst: 100 + global_rate_per_sec: 500.0 + global_burst: 1000 + max_tracked_per_ip: 2048 + max_tracked_per_peer: 2048 + probe: + rate_limit: + per_peer_rate_per_sec: 2.5 + per_peer_burst: 5 + per_ip_rate_per_sec: 25.0 + per_ip_burst: 100 + global_rate_per_sec: 500.0 + global_burst: 1000 + max_tracked_per_ip: 1024 + max_tracked_per_peer: 1024 + receipts: + max_file_bytes: 67108864 + retained_files: 8 + content: + denied_hashes: ["bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"] + denied_origins: ["0x000000000000000000000000000000000000dEaD"] + client: + region_allowlist: ["US", "DE"] diff --git a/charts/decdn-node/templates/NOTES.txt b/charts/decdn-node/templates/NOTES.txt new file mode 100644 index 0000000..18c6007 --- /dev/null +++ b/charts/decdn-node/templates/NOTES.txt @@ -0,0 +1,30 @@ +{{ $admin := dig "observability" "admin_port" 9191 (.Values.config | default dict) -}} +deCDN node {{ include "decdn-node.fullname" . }} (namespace {{ .Release.Namespace }}). + +- Wait for the node to become Ready. Ready means /metrics answers (startup and + chain bring-up done); paid QUIC listeners open only after the first + ContentBlacklist sync, so check the logs before routing traffic: + + kubectl -n {{ .Release.Namespace }} rollout status statefulset/{{ include "decdn-node.fullname" . }} +{{- if and .Values.service.enabled (eq .Values.service.type "LoadBalancer") }} + +- Public QUIC address (udp/{{ .Values.quic.port }}): + + kubectl -n {{ .Release.Namespace }} get svc {{ include "decdn-node.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0]}' +{{- end }} +{{- if eq (int $admin) 0 }} + +- The admin RPC is disabled (observability.admin_port = 0). +{{- else }} + +- Health and status go through the loopback-only admin RPC. From your machine, + with the `decdn` CLI installed: + + kubectl -n {{ .Release.Namespace }} port-forward pod/{{ include "decdn-node.fullname" . }}-0 {{ $admin }} + DECDN_ADMIN_URL=http://127.0.0.1:{{ $admin }} decdn node health +{{- end }} + +- On-chain stake/registration (ADR 019 Phase 2) is a manual operator step: + run `decdn setup` off-cluster against the same keystore. + +Graceful drain window: {{ .Values.terminationGracePeriodSeconds }}s. Do not force-delete the pod. diff --git a/charts/decdn-node/templates/_helpers.tpl b/charts/decdn-node/templates/_helpers.tpl new file mode 100644 index 0000000..331a650 --- /dev/null +++ b/charts/decdn-node/templates/_helpers.tpl @@ -0,0 +1,329 @@ +{{/* --------------------------------------------------------------------------- +Names and labels +--------------------------------------------------------------------------- */}} +{{- define "decdn-node.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{- define "decdn-node.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{- define "decdn-node.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{- define "decdn-node.selectorLabels" -}} +app.kubernetes.io/name: {{ include "decdn-node.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{- define "decdn-node.labels" -}} +helm.sh/chart: {{ include "decdn-node.chart" . }} +{{ include "decdn-node.selectorLabels" . }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{- define "decdn-node.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "decdn-node.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* --------------------------------------------------------------------------- +Image. Upstream has no release (appVersion 0.0.0 is a placeholder), so an +unpinned render would reference an image that does not exist: fail loud. +--------------------------------------------------------------------------- */}} +{{- define "decdn-node.image" -}} +{{- $img := .Values.image }} +{{- if $img.digest }} +{{- printf "%s@%s" $img.repository $img.digest }} +{{- else }} +{{- $tag := default .Chart.AppVersion $img.tag }} +{{- if eq $tag "0.0.0" }} +{{- fail "image: upstream decdn has published no release image yet; set image.tag or image.digest (e.g. a locally built image of decdn/Dockerfile)" }} +{{- end }} +{{- printf "%s:%s" $img.repository $tag }} +{{- end }} +{{- end }} + +{{/* --------------------------------------------------------------------------- +In-pod paths. data_dir is a SUBDIRECTORY of the PVC mount: the mount root carries +fsGroup bits, and upstream refuses a data_dir with any group/world permission. +--------------------------------------------------------------------------- */}} +{{- define "decdn-node.dataMount" -}}/var/lib/decdn{{- end }} +{{- define "decdn-node.dataDir" -}}/var/lib/decdn/node{{- end }} +{{- define "decdn-node.cacheDir" -}}/var/lib/decdn/node/cache{{- end }} +{{- define "decdn-node.configFile" -}}/etc/decdn/node.toml{{- end }} +{{- /* In-memory (emptyDir medium: Memory): the keystore password, off the PVC. */}} +{{- define "decdn-node.secretsDir" -}}/run/decdn{{- end }} + +{{/* --------------------------------------------------------------------------- +Required Secret references (the chart never creates a Secret). +--------------------------------------------------------------------------- */}} +{{- define "decdn-node.validateSecrets" -}} +{{- if not .Values.secrets.keystore.existingSecret }} +{{- fail "secrets.keystore.existingSecret is required: a Secret holding keystore.json, node.secret and keystore.password from `decdn key-gen` (see the chart README)" }} +{{- end }} +{{- if not .Values.secrets.env.existingSecret }} +{{- fail "secrets.env.existingSecret is required: a Secret holding DECDN_RPC_URL (see the chart README)" }} +{{- end }} +{{- range .Values.secrets.env.passthroughKeys }} +{{- if hasPrefix "DECDN_" (upper .) }} +{{- fail (printf "secrets.env.passthroughKeys: %s is refused: DECDN_* env overrides node.toml and would bypass the chart's managed keys and checks; set it in config instead" .) }} +{{- end }} +{{- end }} +{{- end }} + +{{/* --------------------------------------------------------------------------- +Pod-level overrides that would silently break a chart invariant. Fail loud rather +than let a values override win: +- podLabels re-setting a chart label (the selector labels would detach the pod + from the StatefulSet and Services); +- podAnnotations re-setting checksum/config (config changes would stop rolling); +- a (pod)securityContext that drops the non-root / no-privilege / + read-only-rootfs / all-capabilities-dropped hardening (compared as strings: `get` + yields "" for a missing key, which `eq` cannot compare with a bool). Helm merges overrides + into the defaults, so these must hold on the merged values, and a `null` that + removes one of them fails too. +--------------------------------------------------------------------------- */}} +{{- define "decdn-node.validatePod" -}} +{{- $chartLabels := include "decdn-node.labels" . | fromYaml }} +{{- range $k, $_ := default (dict) .Values.podLabels }} +{{- if hasKey $chartLabels $k }} +{{- fail (printf "podLabels.%s is set by the chart and must not be overridden" $k) }} +{{- end }} +{{- end }} +{{- if hasKey (default (dict) .Values.podAnnotations) "checksum/config" }} +{{- fail "podAnnotations.checksum/config is set by the chart (it rolls the pod on config changes) and must not be overridden" }} +{{- end }} +{{- $psc := default (dict) .Values.podSecurityContext }} +{{- if not (eq (toString (get $psc "runAsNonRoot")) "true") }} +{{- fail "podSecurityContext.runAsNonRoot must be true" }} +{{- end }} +{{- range $f := list "runAsUser" "runAsGroup" }} +{{- if and (hasKey $psc $f) (eq (int (get $psc $f)) 0) }} +{{- fail (printf "podSecurityContext.%s must not be 0 (root)" $f) }} +{{- end }} +{{- end }} +{{- if eq (dig "seccompProfile" "type" "" $psc) "Unconfined" }} +{{- fail "podSecurityContext.seccompProfile.type must not be Unconfined" }} +{{- end }} +{{- $sc := default (dict) .Values.securityContext }} +{{- if not (eq (toString (get $sc "allowPrivilegeEscalation")) "false") }} +{{- fail "securityContext.allowPrivilegeEscalation must be false" }} +{{- end }} +{{- if not (eq (toString (get $sc "readOnlyRootFilesystem")) "true") }} +{{- fail "securityContext.readOnlyRootFilesystem must be true" }} +{{- end }} +{{- if eq (toString (get $sc "privileged")) "true" }} +{{- fail "securityContext.privileged must not be true" }} +{{- end }} +{{- if or (eq (toString (get $sc "runAsNonRoot")) "false") (and (hasKey $sc "runAsUser") (eq (int (get $sc "runAsUser")) 0)) }} +{{- fail "securityContext must not run as root (runAsNonRoot=false or runAsUser=0)" }} +{{- end }} +{{- $caps := default (dict) (get $sc "capabilities") }} +{{- if not (has "ALL" (default (list) (get $caps "drop"))) }} +{{- fail "securityContext.capabilities.drop must include ALL" }} +{{- end }} +{{- if get $caps "add" }} +{{- fail "securityContext.capabilities.add must be empty: the node needs no Linux capabilities" }} +{{- end }} +{{- end }} + +{{/* --------------------------------------------------------------------------- +Refuse secret-bearing keys anywhere in `config`. node.toml lands in a ConfigMap, +which is readable by anyone with get on configmaps in the namespace. +Arg: dict "node" "path" +--------------------------------------------------------------------------- */}} +{{- define "decdn-node.forbidSecretKeys" -}} +{{- $node := .node }} +{{- if kindIs "map" $node }} +{{- range $k, $v := $node }} +{{- $here := ternary $k (printf "%s.%s" $.path $k) (eq $.path "") }} +{{- $lk := lower $k | replace "-" "_" }} +{{- if or (has $lk (list "rpc_url" "access_key_id" "session_token")) (contains "password" $lk) (contains "secret" $lk) }} +{{- fail (printf "config.%s: secret-bearing keys must not be set in config (it renders to a ConfigMap); put DECDN_RPC_URL and other secrets in secrets.env.existingSecret" $here) }} +{{- end }} +{{- include "decdn-node.forbidSecretKeys" (dict "node" $v "path" $here) }} +{{- end }} +{{- else if kindIs "slice" $node }} +{{- range $node }} +{{- include "decdn-node.forbidSecretKeys" (dict "node" . "path" $.path) }} +{{- end }} +{{- end }} +{{- end }} + +{{/* --------------------------------------------------------------------------- +Build the effective node.toml tree IN PLACE on .cfg (a deep copy of +.Values.config): chart-managed keys + derived defaults. Fails on non-table +sections, managed-key collisions and cross-field errors. (Mutating in place just +avoids a JSON round-trip; numbers from values files are float64 regardless, which +decdn-node.toml.value handles.) +Arg: dict "cfg" "root" <$> +--------------------------------------------------------------------------- */}} +{{- define "decdn-node.config" -}} +{{- $cfg := .cfg }} +{{- $root := .root }} +{{- include "decdn-node.forbidSecretKeys" (dict "node" $cfg "path" "") }} +{{- /* Every top-level node.toml entry is a table. A scalar here would either be + replaced by the managed-key injection below or rendered as a bare top-level + key the daemon rejects — both silently wrong at render time. */}} +{{- range $section, $v := $cfg }} +{{- if not (kindIs "map" $v) }} +{{- fail (printf "config.%s must be a table (map), got %s" $section (kindOf $v)) }} +{{- end }} +{{- end }} + +{{- /* [section, key, managed value, what to set instead (hint)] */}} +{{- $managed := list + (list "identity" "data_dir" (include "decdn-node.dataDir" $root) "(fixed by the chart)") + (list "blockchain" "eth_keystore" (printf "%s/keystore.json" (include "decdn-node.dataDir" $root)) "secrets.keystore") + (list "cache" "cache_dir" (include "decdn-node.cacheDir" $root) "(fixed by the chart)") + (list "network" "bind_port" (int $root.Values.quic.port) "quic.port") + (list "observability" "metrics_port" (int $root.Values.metrics.port) "metrics.port") + (list "observability" "metrics_bind" "0.0.0.0" "(fixed by the chart; reach is limited by the NetworkPolicy)") +}} +{{- range $managed }} +{{- $section := index . 0 }} +{{- $key := index . 1 }} +{{- $table := default (dict) (get $cfg $section) }} +{{- if hasKey $table $key }} +{{- fail (printf "config.%s.%s is managed by the chart and must not be set; use %s" $section $key (index . 3)) }} +{{- end }} +{{- $_ := set $table $key (index . 2) }} +{{- $_ := set $cfg $section $table }} +{{- end }} + +{{- $cache := $cfg.cache }} +{{- /* cache always exists: the managed cache_dir was just set on it */}} +{{- $hasOrigin := not (empty (get $cache "origin")) }} +{{- $hasOrigins := not (empty (get $cache "origins")) }} +{{- if and (hasKey $cache "origins") (not $hasOrigins) }} +{{- fail "config.cache.origins must not be empty: omit it for a node with no origin" }} +{{- end }} +{{- if and $hasOrigin $hasOrigins }} +{{- fail "config.cache.origin and config.cache.origins are mutually exclusive" }} +{{- end }} +{{- /* Ported from the decdn_node role (0f58d03): a node with no origin can only + fill a miss from other nodes, so pull-through defaults on; with an origin, off. */}} +{{- if not (hasKey $cache "node_to_node_pull_through_enabled") }} +{{- $_ := set $cache "node_to_node_pull_through_enabled" (not (or $hasOrigin $hasOrigins)) }} +{{- end }} +{{- if and (hasKey $cache "max_blob_size_mb") (hasKey $cache "cache_size_mb") }} +{{- if gt (float64 $cache.max_blob_size_mb) (float64 $cache.cache_size_mb) }} +{{- fail "config.cache.max_blob_size_mb must be <= config.cache.cache_size_mb" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* --------------------------------------------------------------------------- +TOML rendering. Helm's toToml is not used: values decode YAML integers as +float64, which it would emit as `10240.0` and serde integer fields reject. +--------------------------------------------------------------------------- */}} + +{{/* A TOML key: bare when it can be, JSON-quoted otherwise. */}} +{{- define "decdn-node.toml.key" -}} +{{- if regexMatch "^[A-Za-z0-9_-]+$" . }}{{ . }}{{ else }}{{ toJson . }}{{ end }} +{{- end }} + +{{/* A scalar or array-of-scalars value. Whole numbers render as integers. +Values files decode every number as float64, which is exact only below 2^53: +anything larger would render rounded (or wrapped past int64) with no error. */}} +{{- define "decdn-node.toml.value" -}} +{{- $v := . }} +{{- if kindIs "slice" $v }} +{{- $items := list }} +{{- range $v }} +{{- $items = append $items (include "decdn-node.toml.value" .) }} +{{- end }} +{{- printf "[%s]" (join ", " $items) }} +{{- else if kindIs "bool" $v }} +{{- ternary "true" "false" $v }} +{{- else if or (kindIs "float64" $v) (kindIs "float32" $v) }} +{{- if or (ge (float64 $v) 9007199254740992.0) (le (float64 $v) -9007199254740992.0) }} +{{- fail (printf "config: %v is too large to render exactly (values are float64; limit 2^53)" $v) }} +{{- end }} +{{- if eq (floor $v) (float64 $v) }}{{ int64 $v }}{{ else }}{{ $v }}{{ end }} +{{- else if or (kindIs "int" $v) (kindIs "int64" $v) (kindIs "int32" $v) (kindIs "uint64" $v) }} +{{- $v }} +{{- else if kindIs "string" $v }} +{{- toJson $v }} +{{- else }} +{{- fail (printf "config: cannot render value %v (%s) as TOML" $v (kindOf $v)) }} +{{- end }} +{{- end }} + +{{/* True ("true") when v is a non-empty list whose elements are maps. */}} +{{- define "decdn-node.toml.isTableArray" -}} +{{- if and (kindIs "slice" .) (gt (len .) 0) (kindIs "map" (first .)) }}true{{ end }} +{{- end }} + +{{/* +One table body and everything beneath it. Scalars first (a scalar emitted below +a sub-table header would silently nest into that sub-table), then sub-tables, +then arrays of tables. A `[x]` header is emitted only for a table that carries +scalars or is empty, so pure intermediates (`[dht]` above `[dht.rate_limit]`) stay +implicit; a `[[x]]` header is always emitted (each one starts a new element). +Arg: dict "table" "path" "header" <"" | "[x]" | "[[x]]"> +*/}} +{{- define "decdn-node.toml.table" -}} +{{- $t := .table }} +{{- $scalars := list }} +{{- $tables := list }} +{{- $arrays := list }} +{{- range $k := keys $t | sortAlpha }} +{{- $v := get $t $k }} +{{- if kindIs "invalid" $v }} +{{- /* null: Helm already strips top-level nulls; one inside a list element would + otherwise vanish silently */}} +{{- fail (printf "config.%s: null inside a list element; omit the key instead" (ternary $k (printf "%s.%s" $.path $k) (eq $.path ""))) }} +{{- else if kindIs "map" $v }} +{{- $tables = append $tables $k }} +{{- else if include "decdn-node.toml.isTableArray" $v }} +{{- $arrays = append $arrays $k }} +{{- else }} +{{- $scalars = append $scalars $k }} +{{- end }} +{{- end }} +{{- if and .header (or $scalars (and (not $tables) (not $arrays)) (hasPrefix "[[" .header)) }} + +{{ .header }} +{{- end }} +{{- range $scalars }} +{{ include "decdn-node.toml.key" . }} = {{ include "decdn-node.toml.value" (get $t .) }} +{{- end }} +{{- range $tables }} +{{- $p := ternary (include "decdn-node.toml.key" .) (printf "%s.%s" $.path (include "decdn-node.toml.key" .)) (eq $.path "") }} +{{- $body := include "decdn-node.toml.table" (dict "table" (get $t .) "path" $p "header" (printf "[%s]" $p)) }} +{{- $body }} +{{- end }} +{{- range $arrays }} +{{- $p := ternary (include "decdn-node.toml.key" .) (printf "%s.%s" $.path (include "decdn-node.toml.key" .)) (eq $.path "") }} +{{- range (get $t .) }} +{{- if not (kindIs "map" .) }} +{{- fail (printf "config.%s: every element of an array of tables must be a map" $p) }} +{{- end }} +{{- include "decdn-node.toml.table" (dict "table" . "path" $p "header" (printf "[[%s]]" $p)) }} +{{- end }} +{{- end }} +{{- end }} + +{{- define "decdn-node.nodeToml" -}} +# Rendered by the decdn-node Helm chart. Do not edit in-cluster; change values. +{{- $cfg := deepCopy (default (dict) .Values.config) }} +{{- include "decdn-node.config" (dict "cfg" $cfg "root" .) }} +{{- include "decdn-node.toml.table" (dict "table" $cfg "path" "" "header" "") }} +{{- end }} diff --git a/charts/decdn-node/templates/configmap.yaml b/charts/decdn-node/templates/configmap.yaml new file mode 100644 index 0000000..8c59b88 --- /dev/null +++ b/charts/decdn-node/templates/configmap.yaml @@ -0,0 +1,10 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "decdn-node.fullname" . }} + labels: + {{- include "decdn-node.labels" . | nindent 4 }} +data: + node.toml: | + {{- include "decdn-node.nodeToml" . | nindent 4 }} diff --git a/charts/decdn-node/templates/networkpolicy.yaml b/charts/decdn-node/templates/networkpolicy.yaml new file mode 100644 index 0000000..b03d663 --- /dev/null +++ b/charts/decdn-node/templates/networkpolicy.yaml @@ -0,0 +1,39 @@ +{{- if not .Values.networkPolicy.enabled }} +{{- if not .Values.networkPolicy.allowUnrestrictedMetrics }} +{{- fail "networkPolicy.enabled=false leaves metrics (bound 0.0.0.0 in the pod) reachable from every pod in the cluster; set networkPolicy.allowUnrestrictedMetrics=true to accept that (AGENTS.md hard rule 2)" }} +{{- end }} +{{- else }} +--- +# Ingress: public QUIC from anywhere; metrics only from metrics.networkPolicy.from. +# Egress: open unless networkPolicy.egress is given. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "decdn-node.fullname" . }} + labels: + {{- include "decdn-node.labels" . | nindent 4 }} +spec: + podSelector: + matchLabels: + {{- include "decdn-node.selectorLabels" . | nindent 6 }} + policyTypes: + - Ingress + {{- if .Values.networkPolicy.egress }} + - Egress + {{- end }} + ingress: + - ports: + - protocol: UDP + port: {{ int .Values.quic.port }} + {{- with .Values.metrics.networkPolicy.from }} + - from: + {{- toYaml . | nindent 8 }} + ports: + - protocol: TCP + port: {{ int $.Values.metrics.port }} + {{- end }} + {{- with .Values.networkPolicy.egress }} + egress: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/decdn-node/templates/service-metrics.yaml b/charts/decdn-node/templates/service-metrics.yaml new file mode 100644 index 0000000..671b719 --- /dev/null +++ b/charts/decdn-node/templates/service-metrics.yaml @@ -0,0 +1,23 @@ +--- +# Metrics: ClusterIP only, never public. Reach is further limited by the +# NetworkPolicy to metrics.networkPolicy.from. +apiVersion: v1 +kind: Service +metadata: + name: {{ include "decdn-node.fullname" . }}-metrics + labels: + {{- include "decdn-node.labels" . | nindent 4 }} + app.kubernetes.io/component: metrics + {{- with .Values.metrics.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: ClusterIP + selector: + {{- include "decdn-node.selectorLabels" . | nindent 4 }} + ports: + - name: metrics + port: {{ int .Values.metrics.port }} + targetPort: metrics + protocol: TCP diff --git a/charts/decdn-node/templates/service.yaml b/charts/decdn-node/templates/service.yaml new file mode 100644 index 0000000..395468b --- /dev/null +++ b/charts/decdn-node/templates/service.yaml @@ -0,0 +1,43 @@ +{{- if .Values.service.enabled }} +{{- $type := .Values.service.type }} +{{- if not (has $type (list "LoadBalancer" "NodePort" "ClusterIP")) }} +{{- fail (printf "service.type must be LoadBalancer, NodePort or ClusterIP (got %q)" $type) }} +{{- end }} +--- +# Public QUIC (udp) — the node's one public hole. +apiVersion: v1 +kind: Service +metadata: + name: {{ include "decdn-node.fullname" . }} + labels: + {{- include "decdn-node.labels" . | nindent 4 }} + app.kubernetes.io/component: quic + {{- with .Values.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: {{ $type }} + {{- if ne $type "ClusterIP" }} + externalTrafficPolicy: {{ .Values.service.externalTrafficPolicy }} + {{- end }} + {{- if eq $type "LoadBalancer" }} + {{- with .Values.service.loadBalancerIP }} + loadBalancerIP: {{ . }} + {{- end }} + {{- with .Values.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} + selector: + {{- include "decdn-node.selectorLabels" . | nindent 4 }} + ports: + - name: quic + port: {{ int .Values.quic.port }} + targetPort: quic + protocol: UDP + {{- if and (ne $type "ClusterIP") .Values.service.nodePort }} + nodePort: {{ int .Values.service.nodePort }} + {{- end }} +{{- end }} diff --git a/charts/decdn-node/templates/serviceaccount.yaml b/charts/decdn-node/templates/serviceaccount.yaml new file mode 100644 index 0000000..ce72c4e --- /dev/null +++ b/charts/decdn-node/templates/serviceaccount.yaml @@ -0,0 +1,15 @@ +{{- if .Values.serviceAccount.create }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "decdn-node.serviceAccountName" . }} + labels: + {{- include "decdn-node.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +# The node never talks to the Kubernetes API. +automountServiceAccountToken: false +{{- end }} diff --git a/charts/decdn-node/templates/servicemonitor.yaml b/charts/decdn-node/templates/servicemonitor.yaml new file mode 100644 index 0000000..34380fc --- /dev/null +++ b/charts/decdn-node/templates/servicemonitor.yaml @@ -0,0 +1,27 @@ +{{- if .Values.metrics.serviceMonitor.enabled }} +{{- if not .Values.metrics.networkPolicy.from }} +{{- if .Values.networkPolicy.enabled }} +{{- fail "metrics.serviceMonitor.enabled needs metrics.networkPolicy.from to admit the Prometheus pods, or the NetworkPolicy blocks every scrape" }} +{{- end }} +{{- end }} +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "decdn-node.fullname" . }} + labels: + {{- include "decdn-node.labels" . | nindent 4 }} + {{- with .Values.metrics.serviceMonitor.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + {{- include "decdn-node.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: metrics + endpoints: + - port: metrics + path: /metrics + interval: {{ .Values.metrics.serviceMonitor.interval }} + scrapeTimeout: {{ .Values.metrics.serviceMonitor.scrapeTimeout }} +{{- end }} diff --git a/charts/decdn-node/templates/statefulset.yaml b/charts/decdn-node/templates/statefulset.yaml new file mode 100644 index 0000000..44b0d95 --- /dev/null +++ b/charts/decdn-node/templates/statefulset.yaml @@ -0,0 +1,212 @@ +{{- include "decdn-node.validateSecrets" . }} +{{- include "decdn-node.validatePod" . }} +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "decdn-node.fullname" . }} + labels: + {{- include "decdn-node.labels" . | nindent 4 }} +spec: + # One release = one node identity. Never scale this: two pods would share one + # keystore and node.secret (a double-signing / slashing hazard). + replicas: 1 + serviceName: {{ include "decdn-node.fullname" . }}-metrics + podManagementPolicy: OrderedReady + updateStrategy: + type: RollingUpdate + selector: + matchLabels: + {{- include "decdn-node.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + checksum/config: {{ include "decdn-node.nodeToml" . | sha256sum }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "decdn-node.labels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ include "decdn-node.serviceAccountName" . }} + automountServiceAccountToken: false + enableServiceLinks: false + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.priorityClassName }} + priorityClassName: {{ . }} + {{- end }} + terminationGracePeriodSeconds: {{ int .Values.terminationGracePeriodSeconds }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + initContainers: + # Upstream refuses key files that are symlinks or carry any group/world bit, + # and a data_dir with group/world bits. Secret volume files are symlinks + # (and fsGroup-adjusted), so on every pod start the identity files are + # installed into the PVC at 0600 — re-asserting the data/cache dir and key + # modes after any fsGroup change. The password goes to an in-memory volume, + # never the PVC: a volume snapshot must not hold the keystore and its password. + - name: prepare + image: {{ include "decdn-node.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: ["/bin/sh", "-euc"] + args: + - | + umask 077 + mkdir -p "$DATA_DIR" "$CACHE_DIR" + chmod 0700 "$DATA_DIR" "$CACHE_DIR" + for f in keystore.json node.secret keystore.password; do + if [ ! -s "/run/decdn-keys/$f" ]; then + echo "prepare: $f is empty in Secret {{ .Values.secrets.keystore.existingSecret }}" >&2 + exit 1 + fi + done + install -m 0600 /run/decdn-keys/keystore.json "$DATA_DIR/keystore.json" + install -m 0600 /run/decdn-keys/node.secret "$DATA_DIR/node.secret" + install -m 0600 /run/decdn-keys/keystore.password "$SECRETS_DIR/keystore.password" + # Earlier chart versions kept the password on the PVC. + rm -f "$DATA_DIR/keystore.password" + echo "prepare: data dir and key material ready" + env: + - name: DATA_DIR + value: {{ include "decdn-node.dataDir" . }} + - name: CACHE_DIR + value: {{ include "decdn-node.cacheDir" . }} + - name: SECRETS_DIR + value: {{ include "decdn-node.secretsDir" . }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + resources: + requests: + cpu: 10m + memory: 16Mi + limits: + memory: 64Mi + volumeMounts: + - name: data + mountPath: {{ include "decdn-node.dataMount" . }} + - name: keys + mountPath: /run/decdn-keys + readOnly: true + - name: run-secrets + mountPath: {{ include "decdn-node.secretsDir" . }} + containers: + - name: decdn-node + image: {{ include "decdn-node.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - --config + - {{ include "decdn-node.configFile" . }} + - run + - --keystore-password-file + - {{ include "decdn-node.secretsDir" . }}/keystore.password + # Named keys only (see values.yaml secrets.env): envFrom would let any + # DECDN_* in the Secret override node.toml. A missing key fails pod start. + env: + - name: DECDN_RPC_URL + valueFrom: + secretKeyRef: + name: {{ .Values.secrets.env.existingSecret }} + key: {{ .Values.secrets.env.rpcUrlKey }} + {{- range .Values.secrets.env.passthroughKeys }} + - name: {{ . }} + valueFrom: + secretKeyRef: + name: {{ $.Values.secrets.env.existingSecret }} + key: {{ . }} + {{- end }} + ports: + - name: quic + containerPort: {{ int .Values.quic.port }} + protocol: UDP + {{- if .Values.quic.hostPort.enabled }} + hostPort: {{ int .Values.quic.hostPort.port }} + {{- end }} + - name: metrics + containerPort: {{ int .Values.metrics.port }} + protocol: TCP + {{- with .Values.startupProbe }} + startupProbe: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 12 }} + {{- end }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + {{- with .Values.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - name: data + mountPath: {{ include "decdn-node.dataMount" . }} + - name: config + mountPath: /etc/decdn + readOnly: true + - name: run-secrets + mountPath: {{ include "decdn-node.secretsDir" . }} + readOnly: true + - name: tmp + mountPath: /tmp + volumes: + - name: config + configMap: + name: {{ include "decdn-node.fullname" . }} + - name: keys + secret: + secretName: {{ .Values.secrets.keystore.existingSecret }} + defaultMode: 0400 + items: + - key: {{ .Values.secrets.keystore.keys.keystore }} + path: keystore.json + - key: {{ .Values.secrets.keystore.keys.nodeSecret }} + path: node.secret + - key: {{ .Values.secrets.keystore.keys.password }} + path: keystore.password + - name: run-secrets + emptyDir: + medium: Memory + sizeLimit: 1Mi + - name: tmp + emptyDir: + medium: Memory + sizeLimit: 64Mi + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumeClaimTemplates: + - metadata: + name: data + {{- with .Values.persistence.annotations }} + annotations: + {{- toYaml . | nindent 10 }} + {{- end }} + spec: + accessModes: + {{- toYaml .Values.persistence.accessModes | nindent 10 }} + {{- with .Values.persistence.storageClass }} + storageClassName: {{ . | quote }} + {{- end }} + resources: + requests: + storage: {{ .Values.persistence.size }} diff --git a/charts/decdn-node/tests/render-test.sh b/charts/decdn-node/tests/render-test.sh new file mode 100755 index 0000000..1c38bfe --- /dev/null +++ b/charts/decdn-node/tests/render-test.sh @@ -0,0 +1,305 @@ +#!/usr/bin/env bash +# Render tests for the decdn-node chart. Run via `make lint-helm` from the repo root. +# +# charts/decdn-node/tests/render-test.sh +# +# Needs: helm, yq (mikefarah v4), python3 >= 3.11 (tomllib), and kubeconform on +# PATH unless KUBECONFORM is overridden. +# Optional env: +# KUBECONFORM command to run kubeconform (default: `kubeconform`; the Makefile +# passes a digest-pinned container). Set to "" to skip, loudly. +# DECDN_CLI path to a real `decdn` binary: also run `decdn config validate` +# on every positive render (catches value/type errors the key check +# can't). Skipped, loudly, when unset. +set -euo pipefail + +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +chart="$(dirname "$here")" +repo="$(cd "$chart/../.." && pwd)" +schema_files="$repo/ansible/molecule/schema/files" +checker="$schema_files/check-schema-keys.py" +kubeconform="${KUBECONFORM-kubeconform}" +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT +skipped=() + +fail() { echo "FAIL: $*" >&2; exit 1; } +pass() { echo "ok $*"; } + +# grep that distinguishes "no match" (1) from "error, e.g. missing file" (2), so a +# negated check cannot pass because the file under test does not exist. +absent() { # + local rc=0 + grep -qE -- "$1" "$2" || rc=$? + [ "$rc" -eq 1 ] || { [ "$rc" -eq 0 ] && return 1; fail "grep error ($rc) on $2"; } +} + +# --- the shared schema-key checker itself -------------------------------------- +fixtures="$schema_files/checker-fixtures" +python3 "$checker" "$fixtures/good.toml" >/dev/null || fail "checker rejects good.toml" +if python3 "$checker" "$fixtures/bad.toml" >/dev/null 2>"$work/bad.err"; then + fail "checker accepts bad.toml" +fi +while IFS= read -r path; do + grep -qxF " $path" "$work/bad.err" || { cat "$work/bad.err" >&2; fail "checker did not flag $path"; } +done < "$fixtures/bad.expected" +: > "$work/empty.toml" +if python3 "$checker" "$work/empty.toml" >/dev/null 2>&1; then fail "checker accepts an empty file"; fi +pass "schema-key checker: good/bad/empty fixtures" + +# --- positive renders ----------------------------------------------------------- +for values in "$chart"/ci/*.yaml; do + name="$(basename "$values" .yaml)" + helm lint --strict --quiet "$chart" -f "$values" >/dev/null \ + || { helm lint --strict "$chart" -f "$values" >&2 || true; fail "helm lint ($name)"; } + helm template t "$chart" -f "$values" > "$work/$name.yaml" || fail "helm template ($name)" + yq 'select(.kind == "ConfigMap") | .data["node.toml"]' "$work/$name.yaml" > "$work/$name.toml" + [ -s "$work/$name.toml" ] || fail "no node.toml in the rendered ConfigMap ($name)" + python3 "$checker" "$work/$name.toml" >/dev/null 2>"$work/$name.err" \ + || { cat "$work/$name.err" >&2; fail "schema keys ($name)"; } + if [ -n "$kubeconform" ]; then + # Only the ServiceMonitor CRD lacks a bundled schema; skip it by kind, so any + # other unrecognised resource (e.g. a typo'd kind) still fails. + $kubeconform -strict -summary -skip ServiceMonitor -kubernetes-version 1.30.0 \ + < "$work/$name.yaml" > "$work/$name.kc" 2>&1 \ + || { cat "$work/$name.kc" >&2; fail "kubeconform ($name)"; } + fi + pass "render + lint + schema keys${kubeconform:+ + kubeconform}: $name" +done +[ -n "$kubeconform" ] || skipped+=("kubeconform (KUBECONFORM is empty)") + +toml="$work/ci-values.toml" + +# The key check only proves emitted keys are legal. A table that silently vanished +# would pass it, so pin the widest render's table set exactly (same list as the +# molecule schema scenario). +tables="$(grep -oE '^\[+[a-z_0-9.]+' "$toml" | tr -d '[' | sort -u)" || fail "no tables in ci-values render" +expected="blockchain +cache +cache.circuit_breaker +cache.origin +cache.origin.credentials +cache.origin_retry +cache.serve_economics +cache.tinylfu +client +content +dht.rate_limit +identity +load_shed +network +network.discovery +network.discovery.peers.253bad481e6371866c9f6276b2a7b3a10ca16255668e740e6fc01da1cacc4350 +observability +payment +probe.rate_limit +receipts +security" +[ "$tables" = "$expected" ] || { diff <(echo "$expected") <(echo "$tables") >&2 || true; fail "table set of ci-values render"; } +keys="$(grep -cE '^[a-z_0-9]+ = ' "$toml" || true)" +[ "$keys" -ge 125 ] || fail "only $keys scalar keys in ci-values render (expected >= 125)" +pass "breadth: $keys keys, $(echo "$tables" | grep -c '') tables" + +for name in ci-values ci-origins ci-resolve-only; do + f="$work/$name.toml" + absent '^(rpc_url|[a-z_]*password|[a-z_]*secret|access_key_id|secret_access_key|session_token) =' "$f" \ + || fail "secret-bearing key in $name" + # Integers stay integers (Helm decodes YAML numbers as float64), in arrays too. + absent '(= |\[|, )-?[0-9]+\.0(\]|,|$)' "$f" || fail "whole number rendered as float in $name" +done +pass "no secret-bearing keys, no float-rendered integers" + +# Pull-through derivation (ported from the role): origins => false, none => true. +grep -qx 'node_to_node_pull_through_enabled = false' "$work/ci-origins.toml" || fail "pull-through with origins should derive false" +grep -qx 'node_to_node_pull_through_enabled = true' "$work/ci-resolve-only.toml" || fail "pull-through without origin should derive true" +helm template t "$chart" -f "$chart/ci/ci-resolve-only.yaml" \ + --set config.cache.node_to_node_pull_through_enabled=false \ + | yq 'select(.kind == "ConfigMap") | .data["node.toml"]' \ + | grep -qx 'node_to_node_pull_through_enabled = false' || fail "explicit pull-through=false should win" +pass "pull-through derivation" + +# Resolve-only discovery: dns_origin alone must still produce the table. +grep -qx '\[network.discovery\]' "$work/ci-resolve-only.toml" || fail "resolve-only: no [network.discovery]" +grep -qx 'dns_origin = "resolve-only.example.invalid"' "$work/ci-resolve-only.toml" || fail "resolve-only: dns_origin" +absent pkarr_url "$work/ci-resolve-only.toml" || fail "resolve-only: pkarr_url leaked" +pass "resolve-only discovery" + +# --- workload and exposure invariants, per render -------------------------------- +# Explicit checks rather than `assert`, which PYTHONOPTIMIZE would turn into no-ops. +for name in ci-values ci-origins ci-resolve-only; do + yq ea -o=json '[.]' "$work/$name.yaml" > "$work/$name.json" + python3 - "$work/$name.json" "$work/$name.toml" "$chart/ci/$name.yaml" <<'PY' || fail "workload/exposure invariants ($name)" +import json, sys, tomllib +docs = [d for d in json.load(open(sys.argv[1])) if d] +cfg = tomllib.load(open(sys.argv[2], "rb")) +name = sys.argv[3].rsplit("/", 1)[-1] +errors = [] +def check(cond, msg): + if not cond: + errors.append(msg) +def one(kind, component=None): + found = [d for d in docs if d["kind"] == kind + and (component is None or d["metadata"]["labels"].get("app.kubernetes.io/component") == component)] + check(len(found) <= 1, f"more than one {kind}/{component}") + return found[0] if found else None + +sts = one("StatefulSet") +spec = sts["spec"]; pod = spec["template"]["spec"] +main = pod["containers"][0]; init = pod["initContainers"][0] +ports = {p["name"]: p for p in main["ports"]} +quic_port, metrics_port = cfg["network"]["bind_port"], cfg["observability"]["metrics_port"] + +check(spec["replicas"] == 1, "replicas != 1") +check(pod["automountServiceAccountToken"] is False, "SA token automounted") +check(pod["terminationGracePeriodSeconds"] >= 300, "drain window < 300s") +check(pod["securityContext"]["runAsNonRoot"] is True, "runAsNonRoot") +for ctr in pod["containers"] + pod["initContainers"]: + sc = ctr["securityContext"] + check(sc["readOnlyRootFilesystem"] is True and sc["allowPrivilegeEscalation"] is False, f"{ctr['name']} securityContext") + check(sc["capabilities"]["drop"] == ["ALL"], f"{ctr['name']} caps") + +# Secrets: no envFrom, env limited to the RPC URL + declared passthrough keys, the +# key Secret never in the daemon, the password never on the PVC. +check("envFrom" not in main, "envFrom present: DECDN_* in the Secret would override node.toml") +env_names = [e["name"] for e in main.get("env", [])] +check(env_names[0] == "DECDN_RPC_URL", "DECDN_RPC_URL not injected") +check(not any(n.startswith("DECDN_") for n in env_names[1:]), "extra DECDN_* env injected") +check(not any(m["name"] == "keys" for m in main["volumeMounts"]), "key Secret mounted in daemon") +pwfile = main["args"][main["args"].index("--keystore-password-file") + 1] +check(pwfile.startswith("/run/decdn/"), f"password file {pwfile} not on the in-memory volume") +vols = {v["name"]: v for v in pod["volumes"]} +check(vols["run-secrets"]["emptyDir"].get("medium") == "Memory", "run-secrets not in memory") +check("keystore.password\" \"$DATA_DIR" not in init["args"][0] and "$SECRETS_DIR/keystore.password" in init["args"][0], + "init installs the password onto the PVC") +check(cfg["blockchain"]["eth_keystore"] == cfg["identity"]["data_dir"] + "/keystore.json", "eth_keystore path") + +# Ports agree across node.toml, the pod, the Services and the NetworkPolicy. +check(cfg["observability"]["metrics_bind"] == "0.0.0.0", "metrics_bind") +check(ports["quic"]["containerPort"] == quic_port and ports["quic"]["protocol"] == "UDP", "quic containerPort") +check(ports["metrics"]["containerPort"] == metrics_port, "metrics containerPort") +for probe in ("startupProbe", "readinessProbe", "livenessProbe"): + check(main[probe]["httpGet"]["port"] == "metrics", f"{probe} port") + +msvc = one("Service", "metrics") +check(msvc["spec"]["type"] == "ClusterIP", "metrics Service not ClusterIP") +check([(p["port"], p["protocol"]) for p in msvc["spec"]["ports"]] == [(metrics_port, "TCP")], "metrics Service ports") +qsvc = one("Service", "quic") +if qsvc: + check([(p["port"], p["protocol"], p["targetPort"]) for p in qsvc["spec"]["ports"]] == [(quic_port, "UDP", "quic")], + "public Service must carry only UDP quic") + +np = one("NetworkPolicy") +if name == "ci-resolve-only.yaml": + check(np is None, "NetworkPolicy rendered although disabled") +else: + ingress = np["spec"]["ingress"] + check(ingress[0] == {"ports": [{"protocol": "UDP", "port": quic_port}]}, "QUIC ingress rule") + tcp_rules = [r for r in ingress if any(p["protocol"] == "TCP" for p in r["ports"])] + for r in tcp_rules: + check(r.get("from"), "metrics ingress rule without `from` admits everyone") + check(r["ports"] == [{"protocol": "TCP", "port": metrics_port}], "metrics ingress port") + check(len(ingress) == 1 + len(tcp_rules), "unexpected ingress rules") + check(("Egress" in np["spec"]["policyTypes"]) == bool(np["spec"].get("egress")), "Egress policyType vs rules") + +if name == "ci-values.yaml": + check("hostPort" not in ports["quic"], "hostPort open by default") + check(len(tcp_rules) == 1, "metrics rule missing although metrics.networkPolicy.from is set") + check(one("ServiceMonitor") is not None, "ServiceMonitor missing") +if name == "ci-origins.yaml": + check(ports["quic"].get("hostPort") == 5000, "hostPort") + check(qsvc["spec"]["ports"][0].get("nodePort") == 30443, "nodePort") + check(len(tcp_rules) == 0, "metrics ingress rule rendered with empty from") + check([o["kind"] for o in cfg["cache"]["origins"]] == ["http", "fs", "s3"], "origins count/order") + check(cfg["cache"]["origins"][2]["credentials"]["profile"] == "mirror", "nested credentials under the wrong origin") + check(env_names == ["DECDN_RPC_URL", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"], "passthrough env") + check(main["env"][0]["valueFrom"]["secretKeyRef"]["key"] == "rpc", "rpcUrlKey") + check([i["key"] for i in vols["keys"]["secret"]["items"]] == ["ks", "ns", "pw"], "custom key names") + check(pod["serviceAccountName"] == "decdn-sa" and one("ServiceAccount") is None, "external ServiceAccount") + +if errors: + print("\n".join(f" {e}" for e in errors), file=sys.stderr) + sys.exit(1) +PY + pass "workload + exposure invariants: $name" +done + +# --- negative renders: each must FAIL, with a message naming the problem ---------- +base="$chart/ci/ci-resolve-only.yaml" +schema_err="values don't meet the specifications" +tmpl_err="execution error at" +expect_fail() { # + local desc="$1" layer="$2" msg="$3"; shift 3 + local out marker + if out="$(helm template t "$chart" -f "$base" "$@" 2>&1)"; then + fail "render should have failed: $desc" + fi + if [ "$layer" = schema ]; then marker="$schema_err"; else marker="$tmpl_err"; fi + if ! grep -qF -- "$marker" <<<"$out" || ! grep -qE -- "$msg" <<<"$out"; then + echo "$out" >&2 + fail "wrong failure ($layer: $msg) for: $desc" + fi + pass "rejects: $desc" +} +expect_fail "missing keystore Secret" template 'secrets.keystore.existingSecret is required' --set secrets.keystore.existingSecret= +expect_fail "missing env Secret" template 'secrets.env.existingSecret is required' --set secrets.env.existingSecret= +expect_fail "DECDN_* passthrough key" template 'passthroughKeys: DECDN_BIND_PORT is refused' --set 'secrets.env.passthroughKeys[0]=DECDN_BIND_PORT' +# A --set null is dropped by some Helm versions (missing property) and kept by others +# (got null); both are correct rejections. +expect_fail "missing required address" schema "missing propert(y|ies) 'slash_judge_address'|/config/blockchain/slash_judge_address': got null" --set config.blockchain.slash_judge_address=null +expect_fail "malformed address" schema "/config/blockchain/slash_judge_address" --set config.blockchain.slash_judge_address=0x12 +expect_fail "zero address" schema "/config/blockchain/payment_pool_address.*'not' failed|payment_pool_address.*not" --set config.blockchain.payment_pool_address=0x0000000000000000000000000000000000000000 +expect_fail "lowercase region" schema "/config/identity/region.*does not match" --set config.identity.region=de +expect_fail "missing chain_id" schema "missing propert(y|ies) 'chain_id'|/config/blockchain/chain_id': got null" --set config.blockchain.chain_id=null +expect_fail "empty origins list" schema "/config/cache/origins" --set-json 'config.cache.origins=[]' +expect_fail "managed metrics_port" template 'observability.metrics_port is managed' --set config.observability.metrics_port=9999 +expect_fail "managed metrics_bind" template 'observability.metrics_bind is managed' --set config.observability.metrics_bind=127.0.0.1 +expect_fail "managed bind_port" template 'network.bind_port is managed' --set config.network.bind_port=4000 +expect_fail "managed data_dir" template 'identity.data_dir is managed' --set config.identity.data_dir=/data +expect_fail "managed eth_keystore" template 'blockchain.eth_keystore is managed' --set config.blockchain.eth_keystore=/k.json +expect_fail "managed cache_dir" template 'cache.cache_dir is managed' --set config.cache.cache_dir=/c +expect_fail "non-table section" template 'config.network must be a table' --set config.network=foo +expect_fail "rpc_url in config" template 'config.blockchain.rpc_url: secret-bearing' --set config.blockchain.rpc_url=https://x.invalid +expect_fail "rpc-url (dashed) in config" template 'config.blockchain.rpc-url: secret-bearing' --set config.blockchain.rpc-url=https://x.invalid +expect_fail "secret key in origins list" template 'secret_access_key: secret-bearing' --set-json 'config.cache.origins=[{"kind":"s3","bucket":"b","credentials":{"secret_access_key":"x"}}]' +expect_fail "password key in config" template 'aws_password: secret-bearing' --set config.cache.origin.credentials.aws_password=x --set config.cache.origin.kind=s3 +expect_fail "replicas knob" schema "additional propert(y|ies) 'replicas'" --set replicas=2 +expect_fail "unpinned image" template 'published no release image' --set image.digest= --set image.tag= +expect_fail "bad service type" schema '/service/type' --set service.type=ExternalName +expect_fail "origin + origins" template 'mutually exclusive' --set config.cache.origin.kind=fs --set config.cache.origin.path=/o --set 'config.cache.origins[0].kind=fs' +expect_fail "max_blob > cache_size" template 'max_blob_size_mb must be <=' --set config.cache.max_blob_size_mb=20000 +expect_fail "integer beyond 2^53" template 'too large to render exactly' --set-json 'config.payment.credit_max=18446744073709551615' +expect_fail "null inside a list element" template 'null inside a list element' --set-json 'config.cache.origins=[{"kind":"fs","path":null}]' +expect_fail "policy off, not acknowledged" template 'allowUnrestrictedMetrics' --set networkPolicy.allowUnrestrictedMetrics=false +expect_fail "podLabels overrides selector" template 'podLabels.app.kubernetes.io/instance is set by the chart' --set-json 'podLabels={"app.kubernetes.io/instance":"x"}' +expect_fail "podAnnotations checksum" template 'podAnnotations.checksum/config is set by the chart' --set-json 'podAnnotations={"checksum/config":"pinned"}' +expect_fail "runAsNonRoot false" template 'runAsNonRoot must be true' --set podSecurityContext.runAsNonRoot=false +expect_fail "runAsNonRoot removed" template 'runAsNonRoot must be true' --set podSecurityContext.runAsNonRoot=null +expect_fail "runAsUser 0" template 'runAsUser must not be 0' --set podSecurityContext.runAsUser=0 +expect_fail "container runAsUser 0" template 'must not run as root' --set securityContext.runAsUser=0 +expect_fail "seccomp Unconfined" template 'must not be Unconfined' --set podSecurityContext.seccompProfile.type=Unconfined +expect_fail "privilege escalation" template 'allowPrivilegeEscalation must be false' --set securityContext.allowPrivilegeEscalation=true +expect_fail "privileged" template 'privileged must not be true' --set securityContext.privileged=true +expect_fail "writable root filesystem" template 'readOnlyRootFilesystem must be true' --set securityContext.readOnlyRootFilesystem=false +expect_fail "capabilities added" template 'capabilities.add must be empty' --set 'securityContext.capabilities.add[0]=NET_ADMIN' +expect_fail "capabilities not dropped" template 'capabilities.drop must include ALL' --set-json 'securityContext.capabilities.drop=["NET_RAW"]' +expect_fail "ServiceMonitor, policy blocks" template 'metrics.networkPolicy.from' --set metrics.serviceMonitor.enabled=true --set networkPolicy.enabled=true + +# --- optional: the real binary -------------------------------------------------- +if [ -n "${DECDN_CLI:-}" ]; then + data="$work/data"; mkdir -m 0700 "$data" + printf 'render-test-password\n' > "$work/pw"; chmod 0600 "$work/pw" + "$DECDN_CLI" key-gen --output-dir "$data" --keystore-password-file "$work/pw" >/dev/null + for values in "$chart"/ci/*.yaml; do + name="$(basename "$values" .yaml)" + sed "s#/var/lib/decdn/node#$data#g" "$work/$name.toml" > "$work/$name.local.toml" + DECDN_RPC_URL=https://rpc.example.invalid/ "$DECDN_CLI" config validate \ + --config "$work/$name.local.toml" --keystore-password-file "$work/pw" >"$work/$name.validate" 2>&1 \ + || { cat "$work/$name.validate" >&2; fail "decdn config validate ($name)"; } + pass "decdn config validate: $name" + done +else + skipped+=("decdn config validate (DECDN_CLI unset): value types are NOT checked") +fi + +for s in "${skipped[@]}"; do echo "SKIPPED: $s"; done +echo "all chart render tests passed" diff --git a/charts/decdn-node/values.schema.json b/charts/decdn-node/values.schema.json new file mode 100644 index 0000000..283be15 --- /dev/null +++ b/charts/decdn-node/values.schema.json @@ -0,0 +1,457 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "title": "decdn-node values", + "type": "object", + "additionalProperties": false, + "required": [ + "image", + "secrets", + "quic", + "service", + "metrics", + "networkPolicy", + "persistence", + "config" + ], + "definitions": { + "port": { + "type": "integer", + "minimum": 1, + "maximum": 65535 + }, + "address": { + "type": "string", + "pattern": "^0x[0-9a-fA-F]{40}$" + }, + "requiredAddress": { + "allOf": [ + { + "$ref": "#/definitions/address" + }, + { + "not": { + "pattern": "^0x0{40}$" + } + } + ] + }, + "stringMap": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "properties": { + "global": { + "type": "object" + }, + "image": { + "type": "object", + "additionalProperties": false, + "required": [ + "repository" + ], + "properties": { + "repository": { + "type": "string", + "minLength": 1 + }, + "tag": { + "type": "string" + }, + "digest": { + "type": "string", + "pattern": "^(sha256:[0-9a-f]{64})?$" + }, + "pullPolicy": { + "enum": [ + "Always", + "IfNotPresent", + "Never" + ] + } + } + }, + "imagePullSecrets": { + "type": "array" + }, + "nameOverride": { + "type": "string" + }, + "fullnameOverride": { + "type": "string" + }, + "serviceAccount": { + "type": "object", + "additionalProperties": false, + "properties": { + "create": { + "type": "boolean" + }, + "name": { + "type": "string" + }, + "annotations": { + "$ref": "#/definitions/stringMap" + } + } + }, + "podAnnotations": { + "$ref": "#/definitions/stringMap" + }, + "podLabels": { + "$ref": "#/definitions/stringMap" + }, + "secrets": { + "type": "object", + "additionalProperties": false, + "required": [ + "keystore", + "env" + ], + "properties": { + "keystore": { + "type": "object", + "additionalProperties": false, + "required": [ + "existingSecret", + "keys" + ], + "properties": { + "existingSecret": { + "type": "string" + }, + "keys": { + "type": "object", + "additionalProperties": false, + "required": [ + "keystore", + "nodeSecret", + "password" + ], + "properties": { + "keystore": { + "type": "string", + "minLength": 1 + }, + "nodeSecret": { + "type": "string", + "minLength": 1 + }, + "password": { + "type": "string", + "minLength": 1 + } + } + } + } + }, + "env": { + "type": "object", + "additionalProperties": false, + "required": [ + "existingSecret", + "rpcUrlKey" + ], + "properties": { + "existingSecret": { + "type": "string" + }, + "rpcUrlKey": { + "type": "string", + "minLength": 1 + }, + "passthroughKeys": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string", + "pattern": "^[A-Za-z_][A-Za-z0-9_]*$" + } + } + } + } + } + }, + "quic": { + "type": "object", + "additionalProperties": false, + "required": [ + "port" + ], + "properties": { + "port": { + "$ref": "#/definitions/port" + }, + "hostPort": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean" + }, + "port": { + "$ref": "#/definitions/port" + } + } + } + } + }, + "service": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean" + }, + "type": { + "enum": [ + "LoadBalancer", + "NodePort", + "ClusterIP" + ] + }, + "externalTrafficPolicy": { + "enum": [ + "Local", + "Cluster" + ] + }, + "nodePort": { + "type": [ + "integer", + "null" + ], + "minimum": 30000, + "maximum": 32767 + }, + "loadBalancerIP": { + "type": "string" + }, + "loadBalancerSourceRanges": { + "type": "array", + "items": { + "type": "string" + } + }, + "annotations": { + "$ref": "#/definitions/stringMap" + } + } + }, + "metrics": { + "type": "object", + "additionalProperties": false, + "required": [ + "port" + ], + "properties": { + "port": { + "$ref": "#/definitions/port" + }, + "service": { + "type": "object", + "additionalProperties": false, + "properties": { + "annotations": { + "$ref": "#/definitions/stringMap" + } + } + }, + "networkPolicy": { + "type": "object", + "additionalProperties": false, + "properties": { + "from": { + "type": "array", + "items": { + "type": "object" + } + } + } + }, + "serviceMonitor": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean" + }, + "interval": { + "type": "string" + }, + "scrapeTimeout": { + "type": "string" + }, + "labels": { + "$ref": "#/definitions/stringMap" + } + } + } + } + }, + "networkPolicy": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean" + }, + "egress": { + "type": "array", + "items": { + "type": "object" + } + }, + "allowUnrestrictedMetrics": { + "type": "boolean" + } + } + }, + "persistence": { + "type": "object", + "additionalProperties": false, + "required": [ + "size" + ], + "properties": { + "size": { + "type": "string", + "pattern": "^[0-9]+(\\.[0-9]+)?(Ki|Mi|Gi|Ti|Pi|K|M|G|T|P)?$" + }, + "storageClass": { + "type": "string" + }, + "accessModes": { + "type": "array", + "minItems": 1, + "items": { + "type": "string" + } + }, + "annotations": { + "$ref": "#/definitions/stringMap" + } + } + }, + "config": { + "description": "node.toml, section for section. Only the keys the chart must enforce are typed here; the chart's own CI fixtures are checked against the committed upstream key inventory, but operator-supplied keys and value types are validated only by the daemon at startup.", + "type": "object", + "required": [ + "identity", + "blockchain" + ], + "properties": { + "identity": { + "type": "object", + "required": [ + "region" + ], + "properties": { + "region": { + "type": "string", + "pattern": "^[A-Z]{2}$" + } + } + }, + "blockchain": { + "type": "object", + "required": [ + "chain_id", + "payment_pool_address", + "capacity_bond_address", + "slash_judge_address", + "content_blacklist_address" + ], + "properties": { + "chain_id": { + "type": "integer", + "minimum": 1 + }, + "payment_pool_address": { + "$ref": "#/definitions/requiredAddress" + }, + "capacity_bond_address": { + "$ref": "#/definitions/requiredAddress" + }, + "slash_judge_address": { + "$ref": "#/definitions/requiredAddress" + }, + "content_blacklist_address": { + "$ref": "#/definitions/requiredAddress" + } + }, + "patternProperties": { + "_address$": { + "$ref": "#/definitions/address" + } + } + }, + "cache": { + "type": "object", + "properties": { + "cache_size_mb": { + "type": "integer", + "minimum": 1 + }, + "max_blob_size_mb": { + "type": "integer", + "minimum": 1 + }, + "origins": { + "type": "array", + "items": { + "type": "object", + "required": [ + "kind" + ] + }, + "minItems": 1 + }, + "origin": { + "type": "object", + "required": [ + "kind" + ] + } + } + } + } + }, + "terminationGracePeriodSeconds": { + "type": "integer", + "minimum": 0 + }, + "startupProbe": { + "type": "object" + }, + "readinessProbe": { + "type": "object" + }, + "livenessProbe": { + "type": "object" + }, + "podSecurityContext": { + "type": "object" + }, + "securityContext": { + "type": "object" + }, + "resources": { + "type": "object" + }, + "nodeSelector": { + "$ref": "#/definitions/stringMap" + }, + "tolerations": { + "type": "array" + }, + "affinity": { + "type": "object" + }, + "priorityClassName": { + "type": "string" + } + } +} diff --git a/charts/decdn-node/values.yaml b/charts/decdn-node/values.yaml new file mode 100644 index 0000000..194586c --- /dev/null +++ b/charts/decdn-node/values.yaml @@ -0,0 +1,206 @@ +--- +# Default values for decdn-node. One release = one node identity (one keystore, +# one data dir); run a fleet by installing one release per node. +# +# NO SECRET VALUES BELONG IN THIS FILE OR ANY OVERRIDE OF IT (AGENTS.md hard +# rule 1). The chart never creates a Secret: key material and the RPC URL are +# referenced from Secrets the operator provisions out of band (see README.md). + +image: + repository: ghcr.io/decdn/decdn-node + # Defaults to the chart appVersion. Upstream has published no release yet, so + # rendering fails until tag or digest is set (e.g. a locally built image). + tag: "" + # sha256:... — takes precedence over tag when set. + digest: "" + pullPolicy: IfNotPresent + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + create: true + name: "" + annotations: {} + +# Chart-owned keys (the app.kubernetes.io/* and helm.sh/chart labels, the +# checksum/config annotation) cannot be overridden; setting one fails the render. +podAnnotations: {} +podLabels: {} + +# --- Secrets (referenced, never created) ------------------------------------ +secrets: + # Output of an off-cluster `decdn key-gen`. On every pod start the `prepare` + # init container installs keystore.json + node.secret into the data dir at 0600 + # (upstream rejects symlinked key files and any group/world bit, and Secret + # volume files are symlinks), and keystore.password into an in-memory volume — + # never onto the PVC, where a snapshot would hold the keystore AND its password. + keystore: + existingSecret: "" # required + keys: + keystore: keystore.json + nodeSecret: node.secret + password: keystore.password + # The k8s equivalent of /etc/decdn/decdn.env. Only named keys are injected — + # never the whole Secret: upstream lets DECDN_* env override node.toml, so a + # stray DECDN_BIND_PORT / DECDN_METRICS_BIND / DECDN_ETH_KEYSTORE / address + # would silently bypass the chart's managed keys and schema checks. + env: + existingSecret: "" # required + rpcUrlKey: DECDN_RPC_URL # the key holding the RPC URL (may embed an API key) + # Extra keys to inject verbatim, e.g. [AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY] + # for an S3 origin using the default credential chain. DECDN_* names are refused. + passthroughKeys: [] + +# --- Public QUIC (the one public hole) -------------------------------------- +quic: + port: 4433 + # Bind udp/ directly on the k8s node as well. Pair with + # service.enabled=false for a bare-metal-style deployment. + hostPort: + enabled: false + port: 4433 + +service: + enabled: true + # LoadBalancer | NodePort | ClusterIP + type: LoadBalancer + # Local preserves the client source IP (per-source rate limits need it). + # Only applied to LoadBalancer/NodePort. + externalTrafficPolicy: Local + nodePort: null + loadBalancerIP: "" + loadBalancerSourceRanges: [] + annotations: {} + +# --- Metrics (0.0.0.0 in the pod, ClusterIP-only, NetworkPolicy-gated) ------ +metrics: + port: 9090 + service: + annotations: {} + # Who may reach tcp/. Empty = no pod. Kubelet probes still work: + # the NetworkPolicy spec always allows traffic between a pod and its own node. + # Entries are NetworkPolicyPeer objects, e.g.: + # - namespaceSelector: + # matchLabels: {kubernetes.io/metadata.name: monitoring} + networkPolicy: + from: [] + serviceMonitor: + enabled: false + interval: 30s + scrapeTimeout: 10s + labels: {} + +networkPolicy: + enabled: true + # Egress is unrestricted unless rules are given here (RPC, relays, origins and + # peers are arbitrary endpoints). A non-empty list switches Egress enforcement on. + egress: [] + # Metrics bind 0.0.0.0 in the pod; this policy is what keeps them private. + # Disabling it fails the render unless this is set to accept cluster-wide reach. + allowUnrestrictedMetrics: false + +# --- Storage ---------------------------------------------------------------- +# Holds the node identity and the cache. Size it above +# config.cache.cache_size_mb + cache.disk_headroom_mb (daemon default 8192 MiB). +persistence: + size: 20Gi + storageClass: "" + accessModes: [ReadWriteOnce] + annotations: {} + +# --- node.toml ---------------------------------------------------------------- +# Mirrors node.toml section-for-section (upstream decdn +# crates/common/src/config/types.rs; every section is deny_unknown_fields, so an +# unknown key crash-loops the pod). Unset keys take the daemon default. +# +# Managed by the chart — setting them here fails the render: +# identity.data_dir, blockchain.eth_keystore, cache.cache_dir, +# network.bind_port (quic.port), observability.metrics_port (metrics.port), +# observability.metrics_bind. +# Refused here (the ConfigMap is readable by anyone who can read configmaps): +# rpc_url, access_key_id, session_token, and any key containing "password" or +# "secret" — use secrets.env (rpcUrlKey / passthroughKeys). +# +# cache.node_to_node_pull_through_enabled, when unset, is derived: true when no +# cache.origin / cache.origins is configured, false when one is (as the Ansible +# role does). +config: + identity: + region: "" # required, ISO 3166-1 alpha-2, uppercase + blockchain: + # Required. Protocol facts: take them from the deCDN ADRs / deployment + # manifest for your chain, never invent them. + chain_id: null # e.g. 421614 (Arbitrum Sepolia) + payment_pool_address: "" + capacity_bond_address: "" + slash_judge_address: "" + content_blacklist_address: "" + payment: + rate_per_mb: 10 # USDC base units (6 decimals) + cache: + cache_size_mb: 10240 # 10 GB + # Deliberate override (daemon default = min(51200, cache_size_mb)): caps one + # blob at a tenth of the default cache. Must be >= 1 — the cache engine's admit + # gate has no zero special-case (despite the CLI help), so 0 rejects every blob. + max_blob_size_mb: 1024 + observability: + log_level: info + log_format: json + +# --- Pod -------------------------------------------------------------------- +# SIGTERM runs a graceful drain; killing mid-drain loses paid deliveries. +terminationGracePeriodSeconds: 300 + +# /metrics is the only probe target: there is no /health route and the admin RPC +# is loopback-only. It binds only after startup completes — RPC preflight, key +# load, cache/endpoint build, then the chain bring-up (CapacityBond + slash-watcher +# enumerations, blacklist and DHT bootstrap) — so startup must cover a slow RPC; a +# failed bring-up exits the container and restarts it. Ready != serving: paid QUIC +# listeners open only after the first ContentBlacklist sync. +startupProbe: + httpGet: + path: /metrics + port: metrics + periodSeconds: 5 + failureThreshold: 60 +readinessProbe: + httpGet: + path: /metrics + port: metrics + periodSeconds: 10 + failureThreshold: 3 +livenessProbe: + httpGet: + path: /metrics + port: metrics + periodSeconds: 20 + timeoutSeconds: 5 + failureThreshold: 6 + +# Overridable (e.g. a different non-root uid), but the render fails if the merged +# result runs as root, allows privilege escalation or privileged mode, has a +# writable root filesystem, adds capabilities, drops fewer than ALL, or uses an +# Unconfined seccomp profile. +podSecurityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch + seccompProfile: + type: RuntimeDefault + +securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + privileged: false + capabilities: + drop: [ALL] + +resources: {} +nodeSelector: {} +tolerations: [] +affinity: {} +priorityClassName: ""