thiras · Sep 16, 2026 · Sep 16, 2026 · Sep 16, 2026 · Sep 16, 2026
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -188,6 +188,36 @@ jobs:
         # DECDN_CLI=... before bumping the decdn version.
         run: make lint-helm
 
+  # Render roles/grafana_alloy's templates and validate them with the REAL pinned
+  # Grafana Alloy binary. The molecule `grafana-cloud` scenario deliberately runs
+  # against a stub that exits 0 for every subcommand, so it proves plumbing but
+  # would happily ship a config Alloy cannot load (an unknown component, a block
+  # in the wrong parent, a CLI flag that does not exist). This job is that gate.
+  # Runs only when ansible/ changed.
+  alloy-config:
+    needs: changes
+    if: needs.changes.outputs.ansible == 'true'
+    runs-on: ubuntu-latest
+    timeout-minutes: 15         # ~1m in practice, most of it the first .deb fetch
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+        with:
+          python-version: '3.12'
+          cache: pip
+          cache-dependency-path: .github/workflows/ci.yml
+      - name: Install Ansible
+        run: python -m pip install --upgrade ansible
+      # Keyed on the role defaults, which is where the version + digest pin
+      # lives: bumping grafana_alloy_version invalidates the cache by
+      # construction, so a bump is always validated against the new binary.
+      - uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0
+        with:
+          path: ansible/.cache/alloy
+          key: alloy-${{ runner.os }}-${{ hashFiles('ansible/roles/grafana_alloy/defaults/main.yml') }}
+      - name: Validate the rendered Alloy configuration
+        run: make lint-alloy
+
   # Dedicated IaC security scan of the Ansible tree and the rendered Helm chart, driven straight from the
   # digest-pinned KICS *engine* image by `make security` — the exact command
   # developers run locally, so CI and local results cannot drift. KICS severities

diff --git a/.github/workflows/molecule.yml b/.github/workflows/molecule.yml
@@ -67,7 +67,7 @@ jobs:
         # Make-owned invocation instead.
         #
         # JOBS is capped at 3 rather than the default (one job per scenario, currently
-        # 6): every scenario is a privileged systemd container, and they share this
+        # 7): every scenario is a privileged systemd container, and they share this
         # runner's cores and cgroup hierarchy. If this job turns flaky, drop to JOBS=1
         # or swap in `make molecule-serial` — the latter also serialises the output.
         run: make molecule JOBS=3

diff --git a/.gitignore b/.gitignore
@@ -36,3 +36,7 @@ ansible/importer_result.json
 # Python bytecode (e.g. from the molecule stub daemon or any local tooling)
 __pycache__/
 *.pyc
+
+# Pinned third-party binaries downloaded by the test harnesses (`make lint-alloy`
+# caches the verified Grafana Alloy release here so repeat runs skip the fetch).
+ansible/.cache/
diff --git a/AGENTS.md b/AGENTS.md
@@ -107,6 +107,8 @@ make molecule         # containerised converge/verify of the decdn_node role —
                       # scenarios in parallel (needs Docker); cap with JOBS=<n>
 make molecule-serial  # the same suite one scenario at a time (readable failure output)
 make lint-helm        # chart: helm lint + render tests + kubeconform + schema keys (needs helm, yq, Docker)
+make lint-alloy       # grafana_alloy: render its templates + `alloy validate` them with the real
+                      # pinned binary (the molecule stub exits 0 for everything and cannot)
 make security         # = security-ansible + security-helm (KICS over the rendered chart; needs helm)
 
 # Ansible deploys — run from ansible/ (see ansible/README.md for the full flow)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -24,6 +24,7 @@ tree), and `markdownlint`.
 | `make lint` | run all pre-commit hooks on every file (the full local hygiene gate) |
 | `make lint-ansible` | install Galaxy collections + run `ansible-lint` (its production profile includes the Ansible security rules) |
 | `make lint-helm` | Helm chart: `helm lint --strict`, positive/negative render tests, kubeconform (digest-pinned image), shared schema-key check and its fixtures (needs `helm`, `yq`, `python3` ≥ 3.11, Docker). Set `DECDN_CLI=<path to decdn>` to also run the real `decdn config validate` (CI can't). |
+| `make lint-alloy` | Renders `roles/grafana_alloy`'s templates and validates them with the **real** digest-pinned Grafana Alloy binary (`alloy validate` + an `ExecStart` flag check). The `grafana-cloud` molecule scenario uses a stub that exits 0 for every subcommand, so this is the only gate that proves the config loads. Set `ALLOY_BIN=<path>` to skip the download. |
 | `make security` | KICS IaC security scan of `ansible/` and the rendered Helm chart (digest-pinned engine image — CI runs this same target) |
 
 `ansible-lint` is **not** a per-commit hook (it needs the collections installed).

diff --git a/Makefile b/Makefile
@@ -1,6 +1,6 @@
 # Convenience targets for the deCDN DevOps monorepo.
 # Run from the repo root. Ansible-specific work is delegated to ansible/Makefile.
-.PHONY: help hooks lint lint-ansible lint-helm security security-ansible security-helm molecule molecule-serial galaxy-build galaxy-check
+.PHONY: help hooks lint lint-ansible lint-helm lint-alloy security security-ansible security-helm molecule molecule-serial galaxy-build galaxy-check
 SHELL := /bin/bash
 
 # KICS runs straight from the engine image, pinned by digest. This target IS the
@@ -59,6 +59,16 @@ security-helm:       ## KICS scan of the decdn-node chart's rendered manifests (
 lint-helm:           ## helm lint + render tests + kubeconform + shared schema-key check (needs helm, yq, python3>=3.11, docker)
 	KUBECONFORM="docker run --rm -i $(KUBECONFORM_IMAGE)" $(CHART)/tests/render-test.sh
 
+# The molecule grafana-cloud scenario runs the role against a stub that exits 0
+# for every subcommand, so it can only prove plumbing. This target renders the
+# grafana_alloy templates and feeds them to the REAL pinned Alloy binary
+# (`alloy validate` + an ExecStart flag check) — the only thing that catches an
+# unknown component, a misplaced block or a non-existent CLI flag before a host
+# crash-loops. Downloads the role's pinned .deb once, then caches it under
+# ansible/.cache (git-ignored); ALLOY_BIN=<path> skips the download.
+lint-alloy:          ## validate grafana_alloy's rendered config against the real pinned Alloy binary
+	ansible/tests/alloy-config/validate.sh
+
 molecule:            ## containerised converge/verify of the decdn_node role, scenarios in parallel (needs Docker)
 	$(MAKE) -C ansible molecule
 

diff --git a/README.md b/README.md
@@ -104,6 +104,7 @@ make hooks            # one-time: install the pre-commit git hook (pip install p
 make lint             # all pre-commit hooks on all files (hygiene, shellcheck, yamllint, markdown)
 make lint-ansible     # vendor collections + full ansible-lint (production profile)
 make lint-helm        # chart: helm lint + render tests + kubeconform + schema keys
+make lint-alloy       # grafana_alloy: render its templates, validate with the real Alloy binary
 make security         # KICS IaC scan of ansible/ + the rendered chart (pinned engine image)
 
 # Ansible deploys — run from ansible/
@@ -123,7 +124,9 @@ is not a per-commit hook (it needs collections vendored) — run `make lint-ansi
 
 - **`ci.yml`** — path-filtered so heavy jobs skip unrelated PRs: `ansible-lint` (production
   profile + playbook syntax-check), a `galaxy-build` readiness gate (builds the `decdn.node`
-  collection and runs galaxy-importer's checks), a `helm` job (`make lint-helm`: strict lint,
+  collection and runs galaxy-importer's checks), an `alloy-config` job (`make lint-alloy`: renders
+  the `grafana_alloy` templates and runs the real digest-pinned Alloy binary's `alloy validate`
+  over them — the molecule stub cannot), a `helm` job (`make lint-helm`: strict lint,
   positive/negative render tests, kubeconform, and the upstream schema-key check shared with
   molecule), **KICS** IaC scan of `ansible/` and the rendered chart (fail on HIGH), and
   `actionlint` on the workflows themselves. The KICS engine is pinned by digest and every

diff --git a/ansible/Makefile b/ansible/Makefile
@@ -66,10 +66,10 @@ deploy:
 # Containerised converge + idempotence + verify of the decdn_node role against a
 # stub daemon (needs Docker; a privileged systemd container). See molecule/.
 # Scenarios: default (rendered values + idempotence), schema (config key-set drift
-# against the upstream field list), validation (bad knobs must be rejected by the
-# role's own asserts), generate-keystore (opt-in host-side wallet), host-env
-# (host-provisioned /etc/decdn/decdn.env), slow-readiness (advisory /metrics probe
-# timeout).
+# against the upstream field list), validation (bad knobs, both roles, must be
+# rejected by their own asserts), generate-keystore (opt-in host-side wallet),
+# host-env (host-provisioned /etc/decdn/decdn.env), slow-readiness (advisory
+# /metrics probe timeout), grafana-cloud (opt-in Grafana Cloud observability).
 #
 # The scenarios run concurrently: distinct container names, no published host ports,
 # per-scenario ephemeral dirs, and what they share on the control machine (the
@@ -122,7 +122,7 @@ molecule-serial: deps
 
 # --- Galaxy collection (decdn.node) ------------------------------------------
 # Stage baseline + decdn_node into a clean collection tree and build the artifact
-# under build/. Only those two roles ship; see galaxy/README.md. Publishing stays
+# under build/. Only the three deployment roles ship; see galaxy/README.md. Publishing stays
 # a manual step (ansible-galaxy collection publish build/decdn-node-*.tar.gz).
 build:
 	./galaxy/build.sh

diff --git a/ansible/README.md b/ansible/README.md
@@ -184,27 +184,66 @@ The node serves paid traffic only **after** on-chain stake + registration (ADR 0
 primitives underneath it. All take `--dry-run`. See
 [`roles/decdn_node/README.md`](roles/decdn_node/README.md#on-chain-onboarding).
 
+### Grafana Cloud observability (opt-in)
+
+The play ships a third role, `grafana_alloy`, that installs a loopback-only
+[Grafana Alloy](https://grafana.com/docs/alloy/) agent scraping the node's `/metrics`
+and receiving its OTLP span exports (`127.0.0.1:4317`), shipping both to your Grafana
+Cloud org — off by default, driven by ONE mirrored inventory flag:
+
+```yaml
+# group_vars/host_vars — drives BOTH roles from one knob
+decdn_grafana_cloud_enabled: true
+```
+
+Provision the credentials **on the target host** (they never transit this repo or the
+control machine):
+
+```bash
+umask 077
+sudo install -m 600 -o root -g root grafana-alloy.env /etc/decdn/grafana-alloy.env
+```
+
+with `roles/grafana_alloy/files/grafana-alloy.env.example` as the template (remote-write
+URL, OTLP endpoint, instance id, API token — all four keys required, https only). Setup,
+rotation, cost/cardinality guardrails and rollback: see
+[`roles/grafana_alloy/README.md`](roles/grafana_alloy/README.md). The Helm chart path is
+separate and deliberately untouched.
+
 ---
 
 ## Testing
 
 ```bash
 make lint           # yamllint + ansible-lint (production profile)
 ansible-playbook playbooks/site.yml --syntax-check
-make molecule       # all six molecule scenarios, in parallel (needs Docker)
+make molecule       # all seven molecule scenarios, in parallel (needs Docker)
 make molecule JOBS=2   # …capped to two at a time on a small machine
 make molecule-serial   # …one at a time, when a failure needs readable output
+
+# from the repo ROOT — the only test that uses a real Grafana Alloy binary
+make lint-alloy     # render grafana_alloy's templates, then `alloy validate` them
 ```
 
 `make molecule` runs every scenario under `molecule/`: **`default`** (described below),
-`schema` (config key-set drift against the upstream field list), `validation` (bad knobs
-must be rejected by the role's own asserts), `generate-keystore` (opt-in host-side
-wallet), `host-env` (host-provisioned `/etc/decdn/decdn.env`) and `slow-readiness`
-(advisory `/metrics` probe timeout). They are independent, so they run concurrently —
+`schema` (config key-set drift against the upstream field list), `validation` (bad knobs,
+for both roles, must be rejected by their own asserts), `generate-keystore` (opt-in
+host-side wallet), `host-env` (host-provisioned `/etc/decdn/decdn.env`) and
+`slow-readiness` (advisory `/metrics` probe timeout), `grafana-cloud` (the opt-in
+observability wiring — see [Grafana Cloud observability](#grafana-cloud-observability-opt-in)).
+They are independent, so they run concurrently —
 ~151s instead of ~595s — and each line of output is prefixed with its scenario name
 because the runs interleave. `make molecule-serial` is the escape hatch when that
 interleaving gets in the way of reading a failure.
 
+`grafana-cloud` runs against a *stub* Alloy that exits 0 for every subcommand, so it
+proves the role's plumbing but cannot prove the rendered `config.alloy` is loadable.
+That gap is closed by `make lint-alloy` (repo root; CI job `alloy-config`), which renders
+the templates in several variable combinations and runs the **real**, digest-pinned Alloy
+binary's `alloy validate` over them plus a check that every `ExecStart` flag actually
+exists in `alloy run --help`. Re-run it when bumping `grafana_alloy_version`. See
+[`tests/alloy-config/`](tests/alloy-config/).
+
 The `default` scenario converges the **`decdn_node`** role in a privileged systemd
 container against a stub daemon: it installs via the `manual` method (no
 published release needed), stages a placeholder keystore, renders `node.toml` + the
@@ -234,6 +273,7 @@ the RPC URL when it is not provisioned on the host instead). Highlights:
 | `decdn_rpc_url` + 3 contract addresses | `""` | **required** per node — `rpc_url` from a host-provisioned `0600 /etc/decdn/decdn.env` (preferred) *or* `host_vars/<node>/secret.yml`, addresses in `main.yml`; sourced from an ADR/deployment. |
 | `decdn_region` / `decdn_bind_port` / `decdn_rate_per_mb` | `""` / `4433` / `10` | node identity, QUIC port, USDC base units/MB. |
 | `decdn_env_checksum_file` / `decdn_env_overwrite_host_file` | `/etc/decdn/.decdn.env.sha256` / `false` | Provenance record for the secret env file (`0600 root`), and the opt-in that lets an inventory `decdn_rpc_url` overwrite a host-edited one. |
+| `decdn_grafana_cloud_enabled` | `false` | ONE mirrored knob (identical default in both roles) wiring on Grafana Cloud observability: installs + configures `grafana_alloy` AND injects `otlp_endpoint` into `node.toml`. Credentials are provisioned per role README; label/cost guardrails there too. |
 
 ---
 

diff --git a/ansible/galaxy/README.md b/ansible/galaxy/README.md
@@ -2,12 +2,13 @@
 
 Deploy and harden a **public [deCDN](https://decdn.org) node**. This collection is
 the public, reusable slice of the [`decdn/devops`](https://github.com/decdn/devops)
-repository — two roles and nothing else:
+repository — three roles and nothing else:
 
 | Role | Purpose |
 |------|---------|
 | `decdn.node.baseline` | Debian host baseline — nftables default-deny inbound, fail2ban, unattended-upgrades, chrony, an admin sudo user, then DevSec OS + SSH hardening (applied last). |
 | `decdn.node.decdn_node` | The `decdn-node` daemon — installed from a pinned GitHub Release tarball under a hardened systemd unit; public QUIC udp/4433, loopback metrics + admin RPC. |
+| `decdn.node.grafana_alloy` | Opt-in Grafana Cloud observability agent — loopback-only Alloy receiver and hardened telemetry export. |
 
 ## Requirements
 
@@ -60,6 +61,7 @@ full variable list, the eth-keystore prerequisite, and day-2 ops:
 
 - [`roles/baseline`](https://github.com/decdn/devops/tree/main/ansible/roles/baseline)
 - [`roles/decdn_node`](https://github.com/decdn/devops/tree/main/ansible/roles/decdn_node)
+- [`roles/grafana_alloy`](https://github.com/decdn/devops/tree/main/ansible/roles/grafana_alloy)
 
 ## Security model
 

diff --git a/ansible/galaxy/build.sh b/ansible/galaxy/build.sh
@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 # Stage and build the public `decdn.node` Galaxy collection.
 #
-# Only the roles/baseline + roles/decdn_node sources ship. All deploy machinery
+# Only the roles/baseline, roles/decdn_node, and roles/grafana_alloy sources ship. All deploy machinery
 # (inventory, Makefile, ansible.cfg) is excluded BY CONSTRUCTION — it is simply
 # never copied into the staging tree. This keeps the artifact clean and leaves the
 # internal project untouched (no galaxy.yml at the project root, so ansible-lint
@@ -16,7 +16,7 @@ repo_root="$(cd "$ansible_dir/.." && pwd)"             # repo root
 
 build_dir="$ansible_dir/build"
 stage="$build_dir/ansible_collections/decdn/node"
-roles=(baseline decdn_node)
+roles=(baseline decdn_node grafana_alloy)
 
 echo "staging decdn.node -> $stage"
 rm -rf "$stage"

diff --git a/ansible/galaxy/galaxy.yml b/ansible/galaxy/galaxy.yml
@@ -30,7 +30,8 @@ tags:
 #   baseline -> devsec.hardening (os_hardening + ssh_hardening), ansible.posix
 #               (authorized_key)
 #   decdn_node -> ansible.builtin only
-# community.general is NOT used by either role, so it is intentionally absent here.
+#   grafana_alloy -> ansible.builtin only
+# community.general is NOT used by any role, so it is intentionally absent here.
 dependencies:
   devsec.hardening: ">=10.0.0"
   ansible.posix: ">=1.5.0"

diff --git a/ansible/inventory/group_vars/decdn_nodes.yml b/ansible/inventory/group_vars/decdn_nodes.yml
@@ -10,3 +10,12 @@ baseline_extra_inbound:
   - proto: udp
     port: "{{ decdn_bind_port }}"
     comment: "decdn QUIC"
+
+# --- Grafana Cloud observability (opt-in) --------------------------------------
+# One mirrored flag drives BOTH roles (grafana_alloy install/config AND the
+# otlp_endpoint injection into node.toml). Uncomment after provisioning the
+# credential file on each host — see roles/grafana_alloy/files/grafana-alloy.env.example:
+#
+# decdn_grafana_cloud_enabled: true
+# grafana_alloy_region: "{{ decdn_region }}"  # override only when needed
+# grafana_alloy_deployment_environment: production
diff --git a/ansible/molecule/default/converge.yml b/ansible/molecule/default/converge.yml
@@ -118,4 +118,6 @@
         name: baseline
         tasks_from: sudo_users
   roles:
+    - role: grafana_alloy
+      tags: [observability, decdn_node, node]
     - role: decdn_node