From 8e98dfbc1eb5a13e9b1cf66975e09dd5afe0711c Mon Sep 17 00:00:00 2001 From: Ant Somers Date: Wed, 16 Sep 2026 10:43:10 +0300 Subject: [PATCH 1/3] feat(ansible): opt-in Grafana Cloud observability via new grafana_alloy role (#55) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a loopback-only Grafana Alloy agent that ships the node's /metrics and OTLP span exports to Grafana Cloud, gated by one mirrored inventory flag (decdn_grafana_cloud_enabled) defined identically in both roles: - roles/grafana_alloy: pinned v1.19.2 .deb install (sha256-verified) or manual stub mode; fail-loud preflight (secret file must be root:root 0600, four GC_* keys grepped on-host, https-only cloud URLs, ratio/duration/ label/port validators); hardened systemd unit with --config.expand-env so credentials never appear as literals in config.alloy; Alloy's own admin server pinned to loopback. - decdn_node: when the flag flips on, inject otlp_endpoint = "http://127.0.0.1:4317" unless explicitly set — the port is guarded by an assert + molecule tests on both sides. - site.yml runs grafana_alloy between baseline and decdn_node; the Galaxy collection stages the third role. - Tests: new grafana-cloud scenario (positive E2E vs a stub binary), disabled-path asserts in default, five negative cases in validation; suite is now seven scenarios (docs + CI comments synced). - Docs: ansible/README.md deploy subsection, group_vars commented opt-in, roles/grafana_alloy/README.md (rotation, cost/cardinality guardrails). Helm chart untouched. --- .github/workflows/molecule.yml | 2 +- ansible/Makefile | 8 +- ansible/README.md | 39 +++- ansible/galaxy/build.sh | 2 +- ansible/galaxy/galaxy.yml | 3 +- ansible/inventory/group_vars/decdn_nodes.yml | 9 + ansible/molecule/default/verify.yml | 41 ++++ ansible/molecule/grafana-cloud/converge.yml | 46 ++++ .../molecule/grafana-cloud/files/alloy-stub | 28 +++ ansible/molecule/grafana-cloud/molecule.yml | 47 +++++ ansible/molecule/grafana-cloud/prepare.yml | 53 +++++ ansible/molecule/grafana-cloud/verify.yml | 197 ++++++++++++++++++ ansible/molecule/validation/converge.yml | 149 ++++++++++++- ansible/playbooks/site.yml | 5 + ansible/roles/decdn_node/defaults/main.yml | 7 + .../roles/decdn_node/templates/node.toml.j2 | 5 + ansible/roles/grafana_alloy/README.md | 81 +++++++ ansible/roles/grafana_alloy/defaults/main.yml | 85 ++++++++ .../files/grafana-alloy.env.example | 18 ++ ansible/roles/grafana_alloy/handlers/main.yml | 9 + ansible/roles/grafana_alloy/meta/main.yml | 16 ++ ansible/roles/grafana_alloy/tasks/install.yml | 138 ++++++++++++ ansible/roles/grafana_alloy/tasks/main.yml | 166 +++++++++++++++ .../roles/grafana_alloy/tasks/preflight.yml | 166 +++++++++++++++ .../grafana_alloy/templates/alloy.service.j2 | 62 ++++++ .../grafana_alloy/templates/config.alloy.j2 | 138 ++++++++++++ 26 files changed, 1504 insertions(+), 16 deletions(-) create mode 100644 ansible/molecule/grafana-cloud/converge.yml create mode 100755 ansible/molecule/grafana-cloud/files/alloy-stub create mode 100644 ansible/molecule/grafana-cloud/molecule.yml create mode 100644 ansible/molecule/grafana-cloud/prepare.yml create mode 100644 ansible/molecule/grafana-cloud/verify.yml create mode 100644 ansible/roles/grafana_alloy/README.md create mode 100644 ansible/roles/grafana_alloy/defaults/main.yml create mode 100644 ansible/roles/grafana_alloy/files/grafana-alloy.env.example create mode 100644 ansible/roles/grafana_alloy/handlers/main.yml create mode 100644 ansible/roles/grafana_alloy/meta/main.yml create mode 100644 ansible/roles/grafana_alloy/tasks/install.yml create mode 100644 ansible/roles/grafana_alloy/tasks/main.yml create mode 100644 ansible/roles/grafana_alloy/tasks/preflight.yml create mode 100644 ansible/roles/grafana_alloy/templates/alloy.service.j2 create mode 100644 ansible/roles/grafana_alloy/templates/config.alloy.j2 diff --git a/.github/workflows/molecule.yml b/.github/workflows/molecule.yml index 471cf45..f2d9a0e 100644 --- a/.github/workflows/molecule.yml +++ b/.github/workflows/molecule.yml @@ -67,7 +67,7 @@ jobs: # Make-owned invocation instead. # # JOBS is capped at 3 rather than the default (one job per scenario, currently - # 6): every scenario is a privileged systemd container, and they share this + # 7): every scenario is a privileged systemd container, and they share this # runner's cores and cgroup hierarchy. If this job turns flaky, drop to JOBS=1 # or swap in `make molecule-serial` — the latter also serialises the output. run: make molecule JOBS=3 diff --git a/ansible/Makefile b/ansible/Makefile index 4cdcb7b..992ca11 100644 --- a/ansible/Makefile +++ b/ansible/Makefile @@ -66,10 +66,10 @@ deploy: # Containerised converge + idempotence + verify of the decdn_node role against a # stub daemon (needs Docker; a privileged systemd container). See molecule/. # Scenarios: default (rendered values + idempotence), schema (config key-set drift -# against the upstream field list), validation (bad knobs must be rejected by the -# role's own asserts), generate-keystore (opt-in host-side wallet), host-env -# (host-provisioned /etc/decdn/decdn.env), slow-readiness (advisory /metrics probe -# timeout). +# against the upstream field list), validation (bad knobs, both roles, must be +# rejected by their own asserts), generate-keystore (opt-in host-side wallet), +# host-env (host-provisioned /etc/decdn/decdn.env), slow-readiness (advisory +# /metrics probe timeout), grafana-cloud (opt-in Grafana Cloud observability). # # The scenarios run concurrently: distinct container names, no published host ports, # per-scenario ephemeral dirs, and what they share on the control machine (the diff --git a/ansible/README.md b/ansible/README.md index 0977195..c8715c9 100644 --- a/ansible/README.md +++ b/ansible/README.md @@ -184,6 +184,32 @@ The node serves paid traffic only **after** on-chain stake + registration (ADR 0 primitives underneath it. All take `--dry-run`. See [`roles/decdn_node/README.md`](roles/decdn_node/README.md#on-chain-onboarding). +### Grafana Cloud observability (opt-in) + +The play ships a third role, `grafana_alloy`, that installs a loopback-only +[Grafana Alloy](https://grafana.com/docs/alloy/) agent scraping the node's `/metrics` +and receiving its OTLP span exports (`127.0.0.1:4317`), shipping both to your Grafana +Cloud org — off by default, driven by ONE mirrored inventory flag: + +```yaml +# group_vars/host_vars — drives BOTH roles from one knob +decdn_grafana_cloud_enabled: true +``` + +Provision the credentials **on the target host** (they never transit this repo or the +control machine): + +```bash +umask 077 +sudo install -m 600 -o root -g root grafana-alloy.env /etc/decdn/grafana-alloy.env +``` + +with `roles/grafana_alloy/files/grafana-alloy.env.example` as the template (remote-write +URL, OTLP endpoint, instance id, API token — all four keys required, https only). Setup, +rotation, cost/cardinality guardrails and rollback: see +[`roles/grafana_alloy/README.md`](roles/grafana_alloy/README.md). The Helm chart path is +separate and deliberately untouched. + --- ## Testing @@ -191,16 +217,18 @@ primitives underneath it. All take `--dry-run`. See ```bash make lint # yamllint + ansible-lint (production profile) ansible-playbook playbooks/site.yml --syntax-check -make molecule # all six molecule scenarios, in parallel (needs Docker) +make molecule # all seven molecule scenarios, in parallel (needs Docker) make molecule JOBS=2 # …capped to two at a time on a small machine make molecule-serial # …one at a time, when a failure needs readable output ``` `make molecule` runs every scenario under `molecule/`: **`default`** (described below), -`schema` (config key-set drift against the upstream field list), `validation` (bad knobs -must be rejected by the role's own asserts), `generate-keystore` (opt-in host-side -wallet), `host-env` (host-provisioned `/etc/decdn/decdn.env`) and `slow-readiness` -(advisory `/metrics` probe timeout). They are independent, so they run concurrently — +`schema` (config key-set drift against the upstream field list), `validation` (bad knobs, +for both roles, must be rejected by their own asserts), `generate-keystore` (opt-in +host-side wallet), `host-env` (host-provisioned `/etc/decdn/decdn.env`) and +`slow-readiness` (advisory `/metrics` probe timeout), `grafana-cloud` (the opt-in +observability wiring — see [Grafana Cloud observability](#grafana-cloud-observability-opt-in)). +They are independent, so they run concurrently — ~151s instead of ~595s — and each line of output is prefixed with its scenario name because the runs interleave. `make molecule-serial` is the escape hatch when that interleaving gets in the way of reading a failure. @@ -234,6 +262,7 @@ the RPC URL when it is not provisioned on the host instead). Highlights: | `decdn_rpc_url` + 3 contract addresses | `""` | **required** per node — `rpc_url` from a host-provisioned `0600 /etc/decdn/decdn.env` (preferred) *or* `host_vars//secret.yml`, addresses in `main.yml`; sourced from an ADR/deployment. | | `decdn_region` / `decdn_bind_port` / `decdn_rate_per_mb` | `""` / `4433` / `10` | node identity, QUIC port, USDC base units/MB. | | `decdn_env_checksum_file` / `decdn_env_overwrite_host_file` | `/etc/decdn/.decdn.env.sha256` / `false` | Provenance record for the secret env file (`0600 root`), and the opt-in that lets an inventory `decdn_rpc_url` overwrite a host-edited one. | +| `decdn_grafana_cloud_enabled` | `false` | ONE mirrored knob (identical default in both roles) wiring on Grafana Cloud observability: installs + configures `grafana_alloy` AND injects `otlp_endpoint` into `node.toml`. Credentials are provisioned per role README; label/cost guardrails there too. | --- diff --git a/ansible/galaxy/build.sh b/ansible/galaxy/build.sh index 29ec93c..0c2545c 100755 --- a/ansible/galaxy/build.sh +++ b/ansible/galaxy/build.sh @@ -16,7 +16,7 @@ repo_root="$(cd "$ansible_dir/.." && pwd)" # repo root build_dir="$ansible_dir/build" stage="$build_dir/ansible_collections/decdn/node" -roles=(baseline decdn_node) +roles=(baseline decdn_node grafana_alloy) echo "staging decdn.node -> $stage" rm -rf "$stage" diff --git a/ansible/galaxy/galaxy.yml b/ansible/galaxy/galaxy.yml index be01a3b..e015c6f 100644 --- a/ansible/galaxy/galaxy.yml +++ b/ansible/galaxy/galaxy.yml @@ -30,7 +30,8 @@ tags: # baseline -> devsec.hardening (os_hardening + ssh_hardening), ansible.posix # (authorized_key) # decdn_node -> ansible.builtin only -# community.general is NOT used by either role, so it is intentionally absent here. +# grafana_alloy -> ansible.builtin only +# community.general is NOT used by any role, so it is intentionally absent here. dependencies: devsec.hardening: ">=10.0.0" ansible.posix: ">=1.5.0" diff --git a/ansible/inventory/group_vars/decdn_nodes.yml b/ansible/inventory/group_vars/decdn_nodes.yml index ea63981..e487f8d 100644 --- a/ansible/inventory/group_vars/decdn_nodes.yml +++ b/ansible/inventory/group_vars/decdn_nodes.yml @@ -10,3 +10,12 @@ baseline_extra_inbound: - proto: udp port: "{{ decdn_bind_port }}" comment: "decdn QUIC" + +# --- Grafana Cloud observability (opt-in) -------------------------------------- +# One mirrored flag drives BOTH roles (grafana_alloy install/config AND the +# otlp_endpoint injection into node.toml). Uncomment after provisioning the +# credential file on each host — see roles/grafana_alloy/files/grafana-alloy.env.example: +# +# decdn_grafana_cloud_enabled: true +# grafana_alloy_region: "" # e.g. "US" — same value as decdn_region +# grafana_alloy_deployment_environment: production diff --git a/ansible/molecule/default/verify.yml b/ansible/molecule/default/verify.yml index 750b419..3cecad3 100644 --- a/ansible/molecule/default/verify.yml +++ b/ansible/molecule/default/verify.yml @@ -3,6 +3,9 @@ # valid non-secret node.toml, a valid + hardened systemd unit, loopback-only # metrics binding, 0600 on the secret env file, and a running service. Real # paid-traffic readiness additionally needs on-chain registration (not tested). +# Plus DISABLED-PATH coverage for the opt-in Grafana Cloud integration: with +# decdn_grafana_cloud_enabled false (the default), nothing may appear on the host +# and node.toml must be byte-identical to the pre-integration rendering. - name: Verify hosts: all become: true @@ -336,6 +339,44 @@ and ansible_facts.services['decdn-node.service'].state == 'running' fail_msg: "decdn-node is not running — check: journalctl -u decdn-node -e" + # --- Grafana Cloud integration stays inert by default ------------------------ + # The converge playbook does NOT set decdn_grafana_cloud_enabled, so no part of + # grafana_alloy may have touched this container. These asserts fail if the + # opt-in ever leaks into the off-path (a stray unconditional task, a template + # side effect, or the role wired unconditionally into site.yml). + - name: Stat the paths the disabled Alloy path must never create + ansible.builtin.stat: + path: "{{ item }}" + register: ga_absent_paths + loop: + - /etc/alloy + - /var/lib/alloy + - /etc/systemd/system/alloy.service + - /etc/decdn/grafana-alloy.env + + - name: Assert no Alloy artifacts exist while observability is disabled + ansible.builtin.assert: + that: + # `| any` needs newer ansible-core than the suite pins; keep to filters. + - ga_absent_paths.results | map(attribute='stat.exists') | select | list | length == 0 + fail_msg: >- + decdn_grafana_cloud_enabled is false but an Alloy artifact exists: + {{ ga_absent_paths.results | selectattr('stat.exists') + | map(attribute='item') | list }} + + - name: Look up the alloy user (expected absent) + ansible.builtin.getent: + database: passwd + key: alloy + failed_when: false + changed_when: false + + - name: Assert the alloy system user was not created + ansible.builtin.assert: + that: + - getent_passwd['alloy'] is not defined + fail_msg: "grafana_alloy created its system user despite the flag being off" + # --- baseline_sudo_users: the two named sudo users created by converge --------- # molecule-operator (simple name, one key) and alice.smith (a '.' in the name, so # the sudoers.d filename-sanitize must map it to /etc/sudoers.d/alice_smith; two diff --git a/ansible/molecule/grafana-cloud/converge.yml b/ansible/molecule/grafana-cloud/converge.yml new file mode 100644 index 0000000..7cf302e --- /dev/null +++ b/ansible/molecule/grafana-cloud/converge.yml @@ -0,0 +1,46 @@ +--- +# Positive path only — see molecule.yml for scope and ../validation for the +# negative (block/rescue) matrix. Both roles run in sequence: Alloy first so its +# receivers exist before the daemon is started and exports spans at them. +# +# The stub does not serve /metrics, so decdn_node's readiness probe would sit out +# its full advisory window (~60s of retries); decdn_readiness_* below bounds that +# to ~3s without changing what the role asserts. +- name: Converge + hosts: all + become: true + vars: + stub_bin: "{{ lookup('ansible.builtin.env', 'MOLECULE_SCENARIO_DIRECTORY') }}/files/alloy-stub" + decdn_grafana_cloud_enabled: true + grafana_alloy_install_method: manual + grafana_alloy_manual_bin_src: "{{ stub_bin }}" + # The credential fixture (staged by prepare.yml) matches these shapes; region/ + # environment labels exercise the "set" branch of the attribute rendering. + grafana_alloy_deployment_environment: production + grafana_alloy_region: US + # Keep the advisory metrics probe short (stub serves nothing). + decdn_readiness_retries: 3 + decdn_readiness_delay: 1 + + # --- Required decdn_node knobs (mirrors ../default's core block) ------------- + _proj_dir: "{{ lookup('ansible.builtin.env', 'MOLECULE_PROJECT_DIRECTORY') }}" + decdn_node_install_method: manual + decdn_node_manual_bin_src: "{{ _proj_dir }}/molecule/default/files/decdn-node-stub" + decdn_cli_manual_bin_src: "{{ _proj_dir }}/molecule/default/files/decdn-node-stub" + decdn_node_version: "0.0.0-molecule-stub" + decdn_rpc_url: "https://rpc.example.invalid/" + decdn_chain_id: 421614 + decdn_region: "US" + decdn_payment_pool_address: "0x1111111111111111111111111111111111111111" + decdn_capacity_bond_address: "0x2222222222222222222222222222222222222222" + decdn_slash_judge_address: "0x3333333333333333333333333333333333333333" + decdn_content_blacklist_address: "0x4444444444444444444444444444444444444444" + + # Sanity-coupling override: verify.yml greps the RENDERED config for this value, + # proving template plumbing passes through rather than constants being checked. + grafana_alloy_batch_send_size: 512 + roles: + - role: grafana_alloy + tags: [observability] + - role: decdn_node + tags: [decdn_node, node] diff --git a/ansible/molecule/grafana-cloud/files/alloy-stub b/ansible/molecule/grafana-cloud/files/alloy-stub new file mode 100755 index 0000000..19dbd4e --- /dev/null +++ b/ansible/molecule/grafana-cloud/files/alloy-stub @@ -0,0 +1,28 @@ +#!/bin/sh +# Molecule stub standing in for the Grafana Alloy binary in CI. +# +# The grafana-cloud scenario runs the real role logic (user/group/dirs, secret +# gate, template rendering, hardened unit, systemd start) against this shim so no +# ~160 MB release download is needed per CI run. It intentionally does NOT listen +# on any socket: runtime socket assertions are out of scope for containerised CI +# (see ../verify.yml comments) and remain covered by real-host deploys + content- +# level checks here. +set -u + +case "${1-}" in + # Version backstop in tasks/install.yml (manual mode degrades to rc==0 anyway). + --version) + echo "alloy, version v${MOLECULE_STUB_VERSION:-0.0.0}-molecule-stub" + ;; + # fmt --test: the real command validates syntax + canonical formatting; the + # stub just exits 0 so the gate proves plumbing only — content correctness is + # asserted textually in verify.yml. + fmt) + : + ;; + # Long-running foreground process keeps the Type=simple unit 'running'. + run) + exec sleep infinity + ;; +esac +exit 0 diff --git a/ansible/molecule/grafana-cloud/molecule.yml b/ansible/molecule/grafana-cloud/molecule.yml new file mode 100644 index 0000000..8104164 --- /dev/null +++ b/ansible/molecule/grafana-cloud/molecule.yml @@ -0,0 +1,47 @@ +--- +# Positive-path exercise of the opt-in Grafana Cloud integration: the +# `grafana_alloy` role end-to-end against a stub binary (manual install mode, +# mirroring how ../default stubs the daemon) followed by `decdn_node`, proving +# the mirrored flag wires otlp_endpoint into node.toml pointing at Alloy's +# receiver. The broken-input matrix lives in ../validation (block/rescue cases +# cannot be idempotence-clean, which this scenario must stay). +# +# Known boundary (why no runtime socket assertions): the stub does NOT bind +# 4317/4318/12345, so `ss -ltn` proofs are meaningless here. Loopback-binding is +# asserted textually against the rendered config + unit, matches what real-binary +# deploys enforce via `alloy fmt --test` at deploy time. +# +# `baseline` is not exercised (real-host-only — see ../default/molecule.yml). +driver: + name: docker +platforms: + - name: decdn-node-grafana-cloud + # Same digest-pinned image as the other scenarios (repo convention). Re-resolve + # all of them together to bump. + image: geerlingguy/docker-debian12-ansible@sha256:4553092be2c00b1ffe580927b9ff03f3c3a0df32b7dd693a3eb02efb6c2b77b7 + pre_build_image: true + command: /usr/lib/systemd/systemd + privileged: true + cgroupns_mode: host + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw +provisioner: + name: ansible + env: + ANSIBLE_ROLES_PATH: "${MOLECULE_PROJECT_DIRECTORY}/roles" + ANSIBLE_COLLECTIONS_PATH: "${MOLECULE_PROJECT_DIRECTORY}/collections" + # Same pipelining note as ../default/molecule.yml (-22% measured there). + ANSIBLE_PIPELINING: "true" +verifier: + name: ansible +scenario: + test_sequence: + # Leading `destroy` so an orphaned container is never reused — see + # ../default/molecule.yml. + - destroy + - create + - prepare + - converge + - idempotence + - verify + - destroy diff --git a/ansible/molecule/grafana-cloud/prepare.yml b/ansible/molecule/grafana-cloud/prepare.yml new file mode 100644 index 0000000..ecc65ce --- /dev/null +++ b/ansible/molecule/grafana-cloud/prepare.yml @@ -0,0 +1,53 @@ +--- +# Stage the host-provisioned fixtures both roles' gates demand: +# 1. decdn_node's operator keystore + node identity + password (same trick as +# ../default/prepare.yml — placeholders at 0644 that the role locks to 0600). +# 2. The Grafana Cloud credential env file for grafana_alloy, which unlike those +# is asserted ALREADY-SECURED by preflight (the role never rewrites it), so +# stage it directly at root:root 0600 with all four required keys. +- name: Prepare + hosts: all + become: true + tasks: + - name: Ensure data + config directories exist + ansible.builtin.file: + path: "{{ item }}" + state: directory + mode: "0755" + loop: + - /var/lib/decdn + - /etc/decdn + + # Staged looser than target so verify.yml can attribute the 0600 lock-down + # to the role rather than to what prepare pre-set. + - name: Stage a placeholder eth keystore + ansible.builtin.copy: + dest: /var/lib/decdn/keystore.json + content: "{{ '{}' }}\n" + mode: "0644" + + - name: Stage a placeholder keystore password file + ansible.builtin.copy: + dest: /etc/decdn/keystore.password + content: "molecule-placeholder\n" + mode: "0644" + + - name: Stage a placeholder node identity + ansible.builtin.copy: + dest: /var/lib/decdn/node.secret + content: "molecule-placeholder-node-secret\n" + mode: "0644" + + # Values are throwaway; only the SHAPES matter to the role's greps (https://, + # non-empty). Written directly at 0600 because preflight refuses anything else. + - name: Stage the Grafana Cloud credential fixture on the host + ansible.builtin.copy: + dest: /etc/decdn/grafana-alloy.env + owner: root + group: root + mode: "0600" + content: | + GC_PROM_REMOTE_WRITE_URL=https://prometheus-prod-xx.molecule.invalid/api/prom/push + GC_OTLP_ENDPOINT=https://otlp-gateway-prod-xx.molecule.invalid/otlp + GC_PROM_USERNAME=999888777 + GC_API_TOKEN=molecule-test-token diff --git a/ansible/molecule/grafana-cloud/verify.yml b/ansible/molecule/grafana-cloud/verify.yml new file mode 100644 index 0000000..7f053a2 --- /dev/null +++ b/ansible/molecule/grafana-cloud/verify.yml @@ -0,0 +1,197 @@ +--- +# Verify the rendered Grafana Cloud wiring. Content-level assertions (parse + +# grep + TOML) replace runtime socket proofs, which are impossible against the +# stub binary — see molecule.yml's boundary note. +- name: Verify + hosts: all + become: true + tasks: + - name: Stat the artifacts the positive run must have produced + ansible.builtin.stat: + path: "{{ item.path }}" + register: ga_files + loop: + - {path: /etc/alloy/config.alloy, mode: "0644"} + - {path: /etc/decdn/grafana-alloy.env, mode: "0600"} + - {path: /etc/systemd/system/alloy.service, mode: "0644"} + + - name: Assert ownership/modes of the Alloy artifacts + ansible.builtin.assert: + that: + # exists-and-mode first so a missing file short-circuits to fail_msg + # instead of raising on absent stat fields. + - item.stat.exists and item.stat.mode == item.item.mode + - item.stat.pw_name == 'root' + fail_msg: "{{ item.item.path }} missing or wrong mode/owner" + loop: "{{ ga_files.results }}" + loop_control: + label: "{{ item.item.path }}" + + # --- Rendered pipeline configuration ----------------------------------------- + - name: Read the rendered config.alloy + ansible.builtin.slurp: + src: /etc/alloy/config.alloy + register: ga_config_b64 + + - name: Parse config.alloy into text + ansible.builtin.set_fact: + ga_config: "{{ ga_config_b64.content | b64decode }}" + + - name: Assert receivers + scrape target bind loopback only + ansible.builtin.assert: + that: + - >- + '127.0.0.1:4317' in ga_config + - >- + '127.0.0.1:4318' in ga_config + - >- + '127.0.0.1:9090' in ga_config + - >- + '0.0.0.0' not in ga_config + - >- + '[::]' not in ga_config + fail_msg: >- + Alloy listeners are not pinned to loopback as required — AGENTS.md hard + rule 2 forbids any backend binding to a wildcard address. + + - name: Assert identity labels, sampling and batch knobs render with converged values + ansible.builtin.assert: + that: + # ratio 0.25 -> percentage 25; batch size overrides pass through; labels + # dotted AND set-variable ones present; secrets remain ${...} placeholders. + - >- + 'sampling_percentage = 25' in ga_config + - >- + 'send_batch_size = 512' in ga_config + - >- + '"service.name"' in ga_config + - >- + '"service.namespace"' in ga_config + - >- + '"service.instance.id"' in ga_config + - >- + '"deployment.environment"' in ga_config + - >- + '"region"' in ga_config + - >- + '"decdn-node"' in ga_config + - >- + '${GC_PROM_REMOTE_WRITE_URL}' in ga_config + - >- + '${GC_OTLP_ENDPOINT}' in ga_config + - >- + '${GC_API_TOKEN}' in ga_config + fail_msg: >- + config.alloy is missing expected content (samplers, labels, ${GC_*} + expansion). Got: + {{ ga_config }} + + # --- Secret hygiene ------------------------------------------------------------- + - name: Read the credential fixture back + ansible.builtin.slurp: + src: /etc/decdn/grafana-alloy.env + register: ga_env_b64 + + - name: Assert the env file carries every required key verbatim + ansible.builtin.assert: + that: + - >- + 'GC_API_TOKEN=molecule-test-token' in ga_env_body + - >- + 'GC_PROM_USERNAME=999888777' in ga_env_body + - >- + 'GC_PROM_REMOTE_WRITE_URL=https://' in ga_env_body + - >- + 'GC_OTLP_ENDPOINT=https://' in ga_env_body + fail_msg: prepare.yml fixture diverged from what the greps require + vars: + ga_env_body: "{{ ga_env_b64.content | b64decode }}" + + # The token value must exist ONLY inside the 0600 env file. node.toml stays a + # non-secret diffable artifact, and the unit/config must hold placeholders only. + - name: Register the secret-token absence across non-secret files + ansible.builtin.command: + cmd: grep -Rq molecule-test-token {{ item }} + changed_when: false + failed_when: false + loop: + - /etc/decdn/node.toml + - /etc/systemd/system/alloy.service + - /etc/alloy/config.alloy + register: ga_leak_probe + + - name: Assert no secret literal leaked into non-secret artifacts + ansible.builtin.assert: + that: + - ga_leak_probe.results | selectattr('rc', 'equalto', 0) | list | length == 0 + fail_msg: >- + The GC_API_TOKEN literal appears outside the 0600 env file — config or + templates stopped using ${...} expansion. + + # --- Node-side wiring (the mirrored flag) ---------------------------------------- + - name: Stage the otlp_endpoint assertion script + ansible.builtin.copy: + dest: /root/molecule-assert-ga-toml.py + mode: "0755" + content: | + import sys, tomllib + d = tomllib.load(open("/etc/decdn/node.toml", "rb")) + want = "http://127.0.0.1:4317" + got = d.get("observability", {}).get("otlp_endpoint") + if got != want: + print(f"otlp_endpoint mismatch: got {got!r}, want {want!r}", file=sys.stderr) + sys.exit(1) + + - name: Assert node.toml exports spans to the local Alloy receiver + ansible.builtin.command: + cmd: python3 /root/molecule-assert-ga-toml.py + changed_when: false + + # --- systemd ---------------------------------------------------------------------- + - name: Validate the Alloy unit syntax + ansible.builtin.command: + cmd: systemd-analyze verify /etc/systemd/system/alloy.service + changed_when: false + + - name: Read the rendered systemd unit + ansible.builtin.slurp: + src: /etc/systemd/system/alloy.service + register: ga_unit_b64 + + - name: Assert the unit carries the environment-expansion + hardening directives + ansible.builtin.assert: + that: + - >- + 'EnvironmentFile=/etc/decdn/grafana-alloy.env' in ga_unit + - >- + '--config.expand-env' in ga_unit + - >- + 'User=alloy' in ga_unit + - >- + 'NoNewPrivileges=true' in ga_unit + - >- + 'ProtectSystem=strict' in ga_unit + - >- + 'SystemCallFilter=@system-service' in ga_unit + # Alloy's own admin server defaults to all interfaces; the role pins it. + - >- + '--server.http.listen-addr=127.0.0.1:' in ga_unit + - >- + '--server.grpc.listen-addr=127.0.0.1:' in ga_unit + fail_msg: hardened Alloy unit is missing an expected directive + vars: + ga_unit: "{{ ga_unit_b64.content | b64decode }}" + + - name: Gather service facts + ansible.builtin.service_facts: + + - name: Assert both services are running + ansible.builtin.assert: + that: + - >- + 'alloy.service' in ansible_facts.services + and ansible_facts.services['alloy.service'].state == 'running' + - >- + 'decdn-node.service' in ansible_facts.services + and ansible_facts.services['decdn-node.service'].state == 'running' + fail_msg: "alloy or decdn-node is not running — journalctl -u alloy -e / -u decdn-node -e" diff --git a/ansible/molecule/validation/converge.yml b/ansible/molecule/validation/converge.yml index ea9d35d..8a8a2d8 100644 --- a/ansible/molecule/validation/converge.yml +++ b/ansible/molecule/validation/converge.yml @@ -531,18 +531,159 @@ path: /etc/decdn/stub-bad-config state: absent + # --- Cases: grafana_alloy preflight gates (#55) -------------------------------- + # The opt-in Grafana Cloud role's OWN fail-loud gates get the same treatment. + # All five broken configs abort inside preflight-imported tasks BEFORE any + # host mutation (user/package/config), so they cannot interfere with the other + # cases or start services. Tag deduping at the bottom handles two cases per + # gate. Fixtures are staged/restored per case; none of this touches decdn.env. + - name: "Case ga-no-secret — enabled without /etc/decdn/grafana-alloy.env" + block: + - name: Stage only a placeholder credential path (absent file) + ansible.builtin.file: + path: /etc/decdn/grafana-alloy.env + state: absent + + - name: Run grafana_alloy with its credential file absent + ansible.builtin.include_role: + name: grafana_alloy + vars: + decdn_grafana_cloud_enabled: true + rescue: + - name: Record ga-no-secret rejection (only if the secret-file assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['secret-file'] }}" + when: ansible_failed_task.name is match('^Assert the secret file') + + - name: "Case ga-symlink-secret — symlinked credential file" + block: + - name: Stage the symlink target outside /etc/decdn + ansible.builtin.copy: + dest: /var/lib/grafana-alloy-fixture.env + owner: root + group: root + mode: "0600" + content: | + GC_PROM_REMOTE_WRITE_URL=https://prometheus-prod-xx.molecule.invalid/api/prom/push + GC_OTLP_ENDPOINT=https://otlp-gateway-prod-xx.molecule.invalid/otlp + GC_PROM_USERNAME=999888777 + GC_API_TOKEN=molecule-test-token + + - name: Point the credential path at the out-of-tree symlink target + ansible.builtin.file: + src: /var/lib/grafana-alloy-fixture.env + dest: /etc/decdn/grafana-alloy.env + state: link + force: true + + - name: Run grafana_alloy against a symlinked credential file + ansible.builtin.include_role: + name: grafana_alloy + vars: + decdn_grafana_cloud_enabled: true + rescue: + - name: Record ga-symlink-secret rejection (only if the secret-file assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['secret-file'] }}" + when: ansible_failed_task.name is match('^Assert the secret file') + always: + - name: Remove the symlink and target + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: + - /etc/decdn/grafana-alloy.env + - /var/lib/grafana-alloy-fixture.env + + - name: "Case ga-missing-key — GC_API_TOKEN line dropped" + block: + - name: Stage an otherwise-complete credential file without the token + ansible.builtin.copy: + dest: /etc/decdn/grafana-alloy.env + owner: root + group: root + mode: "0600" + content: | + GC_PROM_REMOTE_WRITE_URL=https://prometheus-prod-xx.molecule.invalid/api/prom/push + GC_OTLP_ENDPOINT=https://otlp-gateway-prod-xx.molecule.invalid/otlp + GC_PROM_USERNAME=999888777 + + - name: Run grafana_alloy with an incomplete credential file + ansible.builtin.include_role: + name: grafana_alloy + vars: + decdn_grafana_cloud_enabled: true + rescue: + - name: Record ga-missing-key rejection (only if the grep report gate failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['secret-content'] }}" + when: ansible_failed_task.name is match('^Report which credential key') + always: + - name: Remove the partial credential fixture + ansible.builtin.file: + path: /etc/decdn/grafana-alloy.env + state: absent + + - name: "Case ga-http-url — cleartext remote-write endpoint" + block: + - name: Stage a credential file with an http:// push URL + ansible.builtin.copy: + dest: /etc/decdn/grafana-alloy.env + owner: root + group: root + mode: "0600" + content: | + GC_PROM_REMOTE_WRITE_URL=http://prometheus-prod-xx.molecule.invalid/api/prom/push + GC_OTLP_ENDPOINT=https://otlp-gateway-prod-xx.molecule.invalid/otlp + GC_PROM_USERNAME=999888777 + GC_API_TOKEN=molecule-test-token + + - name: Run grafana_alloy with a non-https push URL + ansible.builtin.include_role: + name: grafana_alloy + vars: + decdn_grafana_cloud_enabled: true + rescue: + - name: Record ga-http-url rejection (only if the grep report gate failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['secret-content'] }}" + when: ansible_failed_task.name is match('^Report which credential key') + always: + - name: Remove the http-url credential fixture + ansible.builtin.file: + path: /etc/decdn/grafana-alloy.env + state: absent + + - name: "Case ga-ratio — trace sampler keep-ratio above 1" + block: + - name: Run grafana_alloy with a malformed sampling ratio + ansible.builtin.include_role: + name: grafana_alloy + vars: + decdn_grafana_cloud_enabled: true + grafana_alloy_trace_sampling_ratio: "0.5x" + rescue: + - name: Record ga-ratio rejection (only if the knob validator failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['pipeline-knobs'] }}" + when: ansible_failed_task.name is match('^Validate the pipeline knobs') + - name: Confirm every bad value was rejected by the role's own validation ansible.builtin.assert: that: - - decdn_rejected | sort == _decdn_expected | sort + # unique(): the grafana_alloy cases append the SAME tag for different + # inputs guarded by one shared gate (both bad file shapes -> 'secret-file'). + - decdn_rejected | unique | sort == _decdn_expected | sort fail_msg: >- - decdn_node did NOT reject every bad value at its validation asserts. + Neither decdn_node nor grafana_alloy rejected every bad value at their + validation asserts. Expected {{ _decdn_expected | sort }}, got {{ decdn_rejected | sort }} — a missing tag means that bad value slipped past validation (it would reach - the daemon and crash-loop it at config load). + the daemon/crash-loop it at config load, or ship credentials insecurely). vars: _decdn_expected: ["cross-field", "binary-format", "range", "bounded-range", "shape", "bool", "enum", "dict", "float", "list", "string", "string-newline", "otlp-https", "otlp-noport", "otlp-path", "otlp-userinfo", "otlp-colon", "otlp-port", - "otlp-newline", "env-gate", "env-extra", "env-content", "config-gate"] + "otlp-newline", "env-gate", "env-extra", "env-content", "config-gate", + "secret-file", "secret-content", "pipeline-knobs"] diff --git a/ansible/playbooks/site.yml b/ansible/playbooks/site.yml index 5f9ba31..6a79795 100644 --- a/ansible/playbooks/site.yml +++ b/ansible/playbooks/site.yml @@ -10,5 +10,10 @@ roles: - role: baseline tags: [baseline] + # Opt-in Grafana Cloud observability (off by default; one mirrored inventory + # flag drives this + the otlp_endpoint injection inside decdn_node). Runs + # BEFORE decdn_node so Alloy's receivers exist before the daemon exports. + - role: grafana_alloy + tags: [observability] - role: decdn_node tags: [decdn_node, node] diff --git a/ansible/roles/decdn_node/defaults/main.yml b/ansible/roles/decdn_node/defaults/main.yml index 24b9c7b..9a0f3a1 100644 --- a/ansible/roles/decdn_node/defaults/main.yml +++ b/ansible/roles/decdn_node/defaults/main.yml @@ -276,6 +276,13 @@ decdn_metrics_bind: "127.0.0.1" # keep loopback — Prometheus scrape is a fol decdn_admin_port: 9191 # Optional ("" => omit + use the daemon default). decdn_otlp_endpoint: "" # OTLP/gRPC span export; http://host:port only (e.g. http://localhost:4317) +# OPT-IN Grafana Cloud observability, defined IDENTICALLY in +# roles/grafana_alloy/defaults/main.yml so inventory overrides it ONCE and both +# roles react: enabling it makes THIS role inject +# otlp_endpoint = "http://127.0.0.1:4317" (the local Alloy gRPC receiver below), +# while the grafana_alloy role installs + configures that receiver. Keep the two +# definitions byte-identical by intent — do not fork the semantics. +decdn_grafana_cloud_enabled: false # --- Abuse limits, load shedding, receipts, local denylist -------------------- # [security] and [load_shed] and [content] are HOT-RELOADABLE: change them and run diff --git a/ansible/roles/decdn_node/templates/node.toml.j2 b/ansible/roles/decdn_node/templates/node.toml.j2 index b6dbf33..b58c85e 100644 --- a/ansible/roles/decdn_node/templates/node.toml.j2 +++ b/ansible/roles/decdn_node/templates/node.toml.j2 @@ -355,6 +355,11 @@ metrics_bind = "{{ decdn_metrics_bind }}" admin_port = {{ decdn_admin_port }} {% if decdn_otlp_endpoint not in ["", none] %} otlp_endpoint = "{{ decdn_otlp_endpoint }}" +{% elif decdn_grafana_cloud_enabled | bool %} +{#- Opt-in Grafana Cloud observability: span export flows to the loopback Alloy + receiver (roles/grafana_alloy). The port is a deliberate literal — it MUST + match grafana_alloy_otlp_grpc_port; preflight + molecule guard both sides. -#} +otlp_endpoint = "http://127.0.0.1:4317" {% endif %} {% if decdn_security_max_concurrent_handlers not in ["", none] or decdn_security_per_source_rate_per_sec not in ["", none] or decdn_security_per_source_burst not in ["", none] or decdn_security_max_tracked_sources not in ["", none] %} diff --git a/ansible/roles/grafana_alloy/README.md b/ansible/roles/grafana_alloy/README.md new file mode 100644 index 0000000..be7332f --- /dev/null +++ b/ansible/roles/grafana_alloy/README.md @@ -0,0 +1,81 @@ +# grafana_alloy + +Opt-in Grafana Cloud Application Observability for an Ansible-deployed deCDN +node ([decdn/devops#55](https://github.com/decdn/devops/issues/55)): a dedicated, +loopback-only [Grafana Alloy](https://grafana.com/docs/alloy/) agent that + +- scrapes the node's `/metrics` (Prometheus exposition on loopback port 9090) + and pushes it to your Grafana Cloud Prometheus, stamped with low-cardinality + identity labels, and +- receives the daemon's OTLP span exports (`otlp_endpoint`) on gRPC `127.0.0.1:4317` + and HTTP `127.0.0.1:4318`, parent-based samples traces (default keep-ratio 0.25), + batches them, and exports via OTLP/HTTP to your Grafana Cloud org. + +The Helm-chart path is separate and deliberately untouched by this role. + +## Opt in + +Set ONE variable — mirrored into both roles, so it must be overridden at group or +host level only: + +```yaml +# e.g. inventory/group_vars/.yml or host_vars/.yml +decdn_grafana_cloud_enabled: true +``` + +Then provision credentials **on the target host** (they never transit the control +machine): + +```bash +umask 077 +sudo install -m 600 -o root -g root grafana-alloy.env /etc/decdn/grafana-alloy.env +``` + +using `roles/grafana_alloy/files/grafana-alloy.env.example` as the template. The +four required keys are `GC_PROM_REMOTE_WRITE_URL`, `GC_OTLP_ENDPOINT`, +`GC_PROM_USERNAME` and `GC_API_TOKEN`; both URLs must be `https://`. Values are +expanded into `config.alloy` at load time via `--config.expand-env` — the file on +disk stays free of secret literals, and the role only ever greps shapes on the +host rather than reading values back. + +With the flag off, the role removes any prior unit + rendered config it manages +(disable/rollback); binary/package removal is manual. + +## Variables + +All prefixed `grafana_alloy_*` (see `defaults/main.yml` for comments and pinned +upstream version/sha256). Highlights: + +| Variable | Default | Notes | +| --- | --- | --- | +| `decdn_grafana_cloud_enabled` | `false` | THE opt-in switch (mirrored in `decdn_node`) | +| `grafana_alloy_install_method` | `release` | `manual` copies a control-machine binary (CI stubs) | +| `grafana_alloy_version` / `grafana_alloy_sha256` | pin | Bump together from upstream release digests | +| `grafana_alloy_trace_sampling_ratio` | `0.25` | Trace keep-ratio, validated to `[0,1]` | +| `grafana_alloy_scrape_target` | `127.0.0.1:9090` | Must mirror `decdn_metrics_port`'s default | +| `grafana_alloy_otlp_grpc_port` | `4317` | Hard-coupled to the literal emitted into node.toml | +| `grafana_alloy_region` | `""` | Set = your `decdn_region`; empty omits the attribute | + +Label variables (`service_name`, `service_namespace`, `instance_id`, +`deployment_environment`, `region`) ship on EVERY series/span/log line — keep +them low-cardinality; unique label sets are what your Grafana Cloud bill scales +with. Guardrails: one scrape interval per fleet (`30s`), sampling above zero for +spans, and no per-request/per-hash label values. + +## Couplings & guards + +Two couplings between roles are pinned by constants plus molecule assertions: + +1. This role's `prometheus.scrape` targets `127.0.0.1:9090` — the + `decdn_metrics_port` default. +2. `decdn_node` injects `otlp_endpoint = "http://127.0.0.1:4317"` when the flag + flips on — the port constant here. + +If you move either default, update BOTH sides and the molecule guard in +`ansible/molecule/grafana-cloud/`. + +## Testing + +Covered by the `grafana-cloud` molecule scenario (positive path with a stub +binary + negative preflight cases) and disabled-path assertions in the `default` +scenario. Real-binary validation happens at deploy time via `alloy fmt --test`. diff --git a/ansible/roles/grafana_alloy/defaults/main.yml b/ansible/roles/grafana_alloy/defaults/main.yml new file mode 100644 index 0000000..eabc0f7 --- /dev/null +++ b/ansible/roles/grafana_alloy/defaults/main.yml @@ -0,0 +1,85 @@ +--- +# grafana_alloy role defaults. This role ships with the decdn_node role and is +# OFF by default: the single opt-in point at group/host level is +# decdn_grafana_cloud_enabled, defined IDENTICALLY in roles/decdn_node/defaults +# (the mirroring is deliberate — it gives inventory one override knob that both +# roles read; see "Grafana Cloud observability" in ansible/README.md). + +decdn_grafana_cloud_enabled: false + +# --- Install ------------------------------------------------------------------ +# "release": download the pinned GitHub Release .deb and verify its sha256 before +# dpkg-install. "manual": copy a control-machine binary path (molecule stubs, +# pre-release testing) — no package, no version pin. +grafana_alloy_install_method: release # "release" | "manual" +grafana_alloy_manual_bin_src: "" +# Pinned upstream version + per-arch sha256 of the .deb asset. Bump BOTH together +# from https://github.com/grafana/alloy/releases/ (asset digests are shown on +# every release). A missing arch entry fails loud at preflight when enabled. +grafana_alloy_version: "1.19.2" +grafana_alloy_release_base: https://github.com/grafana/alloy/releases/download +grafana_alloy_sha256: + amd64: 9872732d43c6d14996e1ad5a075086a93381ea6375c8c756820da68b85422eea + arm64: 04dcc80ae4fe75832bfb9d3f376753210c635f1c0a9e851f3cded9ab20dc2b7d + +# --- Paths / identities ------------------------------------------------------- +grafana_alloy_user: alloy +grafana_alloy_group: alloy +grafana_alloy_bin: /usr/local/bin/alloy # manual mode; release deb installs /usr/bin/alloy +grafana_alloy_config_dir: /etc/alloy +grafana_alloy_config_file: /etc/alloy/config.alloy +grafana_alloy_state_dir: /var/lib/alloy # WAL/storage state, owned by the agent user +# SENSITIVE environment file holding the Grafana Cloud credentials as KEY=value +# pairs (template: files/grafana-alloy.env.example). Root-owned 0600 — systemd +# reads EnvironmentFile= as root BEFORE dropping to the agent user, so `alloy` +# never needs read access to it and no secret transits this repo. Provision it on +# the target host like /etc/decdn/decdn.env. +grafana_alloy_secret_file: /etc/decdn/grafana-alloy.env + +# --- Network (loopback-only; AGENTS.md hard rule 2) --------------------------- +# OTLP receivers bound to loopback ONLY — the daemon accepts telemetry pushed +# over localhost, never from the network; no firewall hole exists or is needed. +grafana_alloy_otlp_grpc_bind: 127.0.0.1 +grafana_alloy_otlp_grpc_port: 4317 # standard OTLP/gRPC port; decdn_node's +grafana_alloy_otlp_http_bind: 127.0.0.1 # injected otlp_endpoint depends on the +grafana_alloy_otlp_http_port: 4318 # gRPC value staying put (see node.toml.j2) +# Alloy's own admin/UI server defaults to :12345/:12347 on ALL interfaces — pin it +# to loopback explicitly via --server.*.listen-addr in the unit. +grafana_alloy_server_http_addr: 127.0.0.1:12345 +grafana_alloy_server_grpc_addr: 127.0.0.1:12347 + +# --- Metrics scrape (node /metrics -> Grafana Cloud Prometheus) --------------- +# COUPLING: mirrors decdn_metrics_port's DEFAULT (9090) in roles/decdn_node. +# Kept a plain constant here because cross-role vars are fragile; the molecule +# suite guards the drift — if you move decdn_metrics_port off 9090, set +# grafana_alloy_scrape_target on the same inventory level. +grafana_alloy_scrape_target: 127.0.0.1:9090 +grafana_alloy_scrape_interval: 30s + +# --- Resource identity labels (low-cardinality only!) -------------------------- +# These become metric labels and OTLP resource attributes shipped to your Grafana +# Cloud org for EVERY series/span/log line: never add high-cardinality values +# (hashes, per-request ids, timestamps) — the remote bill scales with unique +# label sets (see README cost guardrails). +grafana_alloy_service_name: decdn-node +grafana_alloy_service_namespace: decdn +grafana_alloy_instance_id: "" # "" => derive from inventory_hostname +grafana_alloy_deployment_environment: "" # "" => attribute omitted entirely +grafana_alloy_region: "" # operator sets it = decdn_region ("" => omitted) + +# --- Pipeline tuning ------------------------------------------------------------ +# Parent-based probabilistic sampler for TRACES, expressed as the 0..1 keep-ratio +# the deCDN issue settled on (0.25 == keep ~25% of spans); converted to the +# component's percentage internally. Validated at preflight. +grafana_alloy_trace_sampling_ratio: 0.25 +grafana_alloy_batch_timeout: 3s # max delay before a partial batch flushes +grafana_alloy_batch_send_size: 1024 # items per batch (soft target) +grafana_alloy_batch_max_size: 2048 # hard cap; larger batches split +grafana_alloy_queue_size: 5000 # exporter sending queue capacity +grafana_alloy_retry_initial_interval: 5s # first backoff after a failed push +grafana_alloy_retry_max_elapsed_time: 5m # total retry budget per request + +# systemd TimeoutStopSec for the unit (seconds): cap for flushing the exporter +# queue on shutdown. Truncating it drops buffered telemetry (not money), so the +# default stays modest. +grafana_alloy_stop_timeout_sec: 90 diff --git a/ansible/roles/grafana_alloy/files/grafana-alloy.env.example b/ansible/roles/grafana_alloy/files/grafana-alloy.env.example new file mode 100644 index 0000000..7c0d7cc --- /dev/null +++ b/ansible/roles/grafana_alloy/files/grafana-alloy.env.example @@ -0,0 +1,18 @@ +# Template for /etc/decdn/grafana-alloy.env (root-owned 0600). +# +# Provision it ON THE TARGET HOST — these values are your Grafana Cloud +# credentials and must never be committed or carried through inventory: +# +# umask 077 +# sudo install -m 600 -o root -g root grafana-alloy.env /etc/decdn/grafana-alloy.env +# +# Values come from your Grafana Cloud org: observe -> Application Observability -> +# "Instructions"/details page shows the exact remote-write URL, OTLP endpoint, +# instance id ("user") and API token. The role greps shapes only; it never reads +# values back onto the control machine. + +GC_PROM_REMOTE_WRITE_URL=https://prometheus-prod-XX.grafana.net/api/prom/push +GC_OTLP_ENDPOINT=https://otlp-gateway-prod-XX.grafana.net/otlp +GC_PROM_USERNAME=1234567 +# glc_... application token with write scope for this stack +GC_API_TOKEN=glc_example_token_replace_me diff --git a/ansible/roles/grafana_alloy/handlers/main.yml b/ansible/roles/grafana_alloy/handlers/main.yml new file mode 100644 index 0000000..73c5354 --- /dev/null +++ b/ansible/roles/grafana_alloy/handlers/main.yml @@ -0,0 +1,9 @@ +--- +- name: Reload systemd + ansible.builtin.systemd: + daemon_reload: true + +- name: Restart alloy + ansible.builtin.systemd: + name: alloy + state: restarted diff --git a/ansible/roles/grafana_alloy/meta/main.yml b/ansible/roles/grafana_alloy/meta/main.yml new file mode 100644 index 0000000..7d8eb63 --- /dev/null +++ b/ansible/roles/grafana_alloy/meta/main.yml @@ -0,0 +1,16 @@ +--- +galaxy_info: + role_name: grafana_alloy + author: deCDN Contributors + description: >- + Opt-in Grafana Cloud observability — a loopback-only Alloy agent shipping the + node's metrics + OTLP traces to your Grafana Cloud org. + license: MIT + min_ansible_version: "2.15" + galaxy_tags: [decdn, cdn, observability, grafana, otlp, debian] + platforms: + - name: Debian + versions: [bookworm] + - name: Ubuntu + versions: [jammy, noble] +dependencies: [] diff --git a/ansible/roles/grafana_alloy/tasks/install.yml b/ansible/roles/grafana_alloy/tasks/install.yml new file mode 100644 index 0000000..3aa7d8a --- /dev/null +++ b/ansible/roles/grafana_alloy/tasks/install.yml @@ -0,0 +1,138 @@ +--- +# Install the Alloy binary. Two methods, selected by grafana_alloy_install_method +# (the same shape roles/decdn_node uses): +# "release": download the pinned GitHub Release .deb for this architecture, +# verify its sha256 against the hand-pinned digest, then dpkg-install. +# Idempotent via a dpkg-query version gate — re-downloads only on upgrade. +# "manual": copy a control-machine binary path (molecule CI stubs, pre-release +# testing) straight to /usr/local/bin/alloy. No package is created. + +# Map Ansible's machine facts onto Debian arch names used by the asset set +# (alloy--1..deb). Unsupported architectures fail loud here so an +# operator gets a clear message instead of a 404 mid-play. +- name: Resolve the Debian package architecture + ansible.builtin.set_fact: + _ga_deb_arch: "{{ ({'x86_64': 'amd64', 'aarch64': 'arm64'}[ansible_architecture] | default('unsupported')) }}" + +- name: Name the .deb asset for this architecture + # Reusable across the download + install tasks below. + ansible.builtin.set_fact: + _ga_deb_file: "alloy-{{ grafana_alloy_version }}-1.{{ _ga_deb_arch }}.deb" + +- name: Report that the release download is not simulated in check mode + ansible.builtin.debug: + msg: >- + check mode: skipping the download + verification + install of Alloy + {{ grafana_alloy_version }}. The target host is NOT changed by this run's + install step; configuration templating below still diffs normally. + when: + - grafana_alloy_install_method == "release" + - not ansible_check_mode + +- name: Query the currently-installed package version + ansible.builtin.command: + cmd: dpkg-query -W -f='${Version}\n' alloy + register: _ga_dpkg_version + changed_when: false + failed_when: false # rc=1 just means "not installed yet" + when: + - grafana_alloy_install_method == "release" + - not ansible_check_mode + +- name: Decide whether (re)install is needed + ansible.builtin.set_fact: + _ga_need_install: "{{ (_ga_dpkg_version.rc | default(1)) != 0 + or (grafana_alloy_version not in (_ga_dpkg_version.stdout | default(''))) }}" + when: + - grafana_alloy_install_method == "release" + - not ansible_check_mode + +- name: Install the pinned upstream .deb package + when: + - grafana_alloy_install_method == "release" + - not ansible_check_mode + - _ga_need_install | bool + block: + - name: Look up the pinned sha256 for this architecture + ansible.builtin.assert: + that: + - (grafana_alloy_sha256[_ga_deb_arch] | default('', true)) | length == 64 + fail_msg: >- + No sha256 pin recorded for the {{ _ga_deb_arch }} flavor of Grafana Alloy + {{ grafana_alloy_version }}. Add one under grafana_alloy_sha256 in this + role's defaults (copy the digest published with the release), or bump the + pin together with grafana_alloy_version — never ship the .deb unverified. + + # A private staging directory, not a fixed /tmp path — these bytes are trusted + # on the strength of a pinned digest, and a predictable world-writable location + # lets any local user pre-seed between fetch and verification. get_url's native + # checksum parameter downloads-and-verifies atomically. + - name: Create a private download staging directory + ansible.builtin.tempfile: + state: directory + prefix: alloy-install- + register: _ga_stage + changed_when: false + + - name: Download the pinned Alloy .deb and verify its sha256 + ansible.builtin.get_url: + url: "{{ grafana_alloy_release_base }}/v{{ grafana_alloy_version }}/{{ _ga_deb_file }}" + dest: "{{ _ga_stage.path }}/{{ _ga_deb_file }}" + owner: root + group: root + mode: "0600" + checksum: "sha256:{{ grafana_alloy_sha256[_ga_deb_arch] }}" + + - name: Install the verified package + ansible.builtin.apt: + deb: "{{ _ga_stage.path }}/{{ _ga_deb_file }}" + notify: Restart alloy + + always: + - name: Remove the download staging directory + ansible.builtin.file: + path: "{{ _ga_stage.path }}" + state: absent + when: _ga_stage.path is defined + +# Binary path the unit will ExecStart. Package installs land at /usr/bin/alloy; +# manual copies go to grafana_alloy_bin (/usr/local/bin). Resolved into a fact so +# templates and checks share exactly one value. +- name: Resolve the effective binary path + ansible.builtin.set_fact: + grafana_alloy_bin_effective: >- + {{ '/usr/bin/alloy' if grafana_alloy_install_method == 'release' + else grafana_alloy_bin }} + +- name: Copy a local/stub binary from the control machine + when: grafana_alloy_install_method == "manual" + block: + - name: Require an explicit source path in manual mode + ansible.builtin.assert: + that: + - grafana_alloy_manual_bin_src | length > 0 + fail_msg: >- + grafana_alloy_install_method "manual" requires + grafana_alloy_manual_bin_src (a control-machine path to the alloy + binary). Set it in the play/inventory that invokes this role. + + - name: Copy the Alloy binary into place + ansible.builtin.copy: + src: "{{ grafana_alloy_manual_bin_src }}" + dest: "{{ grafana_alloy_bin }}" + owner: root + group: root + mode: "0755" + notify: Restart alloy + +# Integrity/liveness backstop: runs EVERY converge so a drifted/broken binary is +# caught loudly rather than silently. Strict version match only applies to the +# packaged flavor; manual mode degrades to a bare liveness check (rc == 0). +- name: Verify the installed Alloy runs (version backstop) + ansible.builtin.command: "{{ grafana_alloy_bin_effective }} --version" + changed_when: false + register: _ga_installed_version + failed_when: >- + _ga_installed_version.rc != 0 + or (grafana_alloy_install_method == "release" + and grafana_alloy_version not in _ga_installed_version.stdout) diff --git a/ansible/roles/grafana_alloy/tasks/main.yml b/ansible/roles/grafana_alloy/tasks/main.yml new file mode 100644 index 0000000..4421247 --- /dev/null +++ b/ansible/roles/grafana_alloy/tasks/main.yml @@ -0,0 +1,166 @@ +--- +# Opt-in Grafana Cloud Application Observability for an Ansible-deployed deCDN +# node: a loopback-only Grafana Alloy agent that scrapes the node's /metrics and +# receives the daemon's OTLP traces, then ships both to Grafana Cloud. +# +# Gated by decdn_grafana_cloud_enabled (mirrored in roles/decdn_node/defaults — +# ONE inventory knob drives both roles). When false this role does nothing except +# tear down any prior install (clean disable/rollback); when true it fail-louds on +# every misconfiguration BEFORE touching a running system (AGENTS.md hard rule 4). + +- name: Validate decdn_grafana_cloud_enabled is a real boolean + ansible.builtin.assert: + that: + - decdn_grafana_cloud_enabled is boolean + fail_msg: >- + decdn_grafana_cloud_enabled must be a boolean (true/false), not a quoted + string — it gates this entire role and the otlp_endpoint injection in + roles/decdn_node/templates/node.toml.j2. + +# --- Disabled path: stop + uninstall remnants so flipping the flag off is a ---- +# --- complete rollback (binary/package removal stays a manual step) ----------- +- name: Tear down any prior installation (flag off) + when: not decdn_grafana_cloud_enabled | bool + block: + - name: Check for a leftover Alloy systemd unit + ansible.builtin.stat: + path: /etc/systemd/system/alloy.service + register: _ga_leftover_unit + + - name: Stop and disable a leftover Alloy service + ansible.builtin.systemd: + name: alloy + state: stopped + enabled: false + when: _ga_leftover_unit.stat.exists + failed_when: false # best-effort teardown of an already-broken install + + - name: Remove the leftover unit and rendered configuration + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: + - /etc/systemd/system/alloy.service + - "{{ grafana_alloy_config_dir }}" + notify: Reload systemd + + - name: Apply pending teardown notifications + ansible.builtin.meta: flush_handlers + +# Everything below runs only when observability is explicitly enabled. +- name: Enable and configure the Grafana Cloud agent + when: decdn_grafana_cloud_enabled | bool + block: + # A derived id used by BOTH the preflight regex-free checks below and the two + # templates; resolved once here so labels never disagree. + - name: Resolve the low-cardinality instance identifier + ansible.builtin.set_fact: + _ga_instance_id: >- + {{ grafana_alloy_instance_id if grafana_alloy_instance_id | length > 0 + else inventory_hostname }} + + - name: Fail loud on preflight misconfiguration + ansible.builtin.import_tasks: preflight.yml + + # --- User + directories ---------------------------------------------------- + - name: Create the alloy system group + ansible.builtin.group: + name: "{{ grafana_alloy_group }}" + system: true + + - name: Create the alloy system user + ansible.builtin.user: + name: "{{ grafana_alloy_user }}" + group: "{{ grafana_alloy_group }}" + system: true + home: "{{ grafana_alloy_state_dir }}" + create_home: false + shell: /usr/sbin/nologin + + - name: Ensure config + state directories exist + ansible.builtin.file: + path: "{{ item.path }}" + state: directory + owner: "{{ item.owner }}" + group: "{{ grafana_alloy_group }}" + mode: "{{ item.mode }}" + loop: + - {path: "{{ grafana_alloy_config_dir }}", owner: root, mode: "0755"} + # 0750, not 0700: systemd's StateDirectory=alloy stamps this SAME mode on + # every unit start — a stricter here/looser-there pair would make the two + # flip-flop forever and break molecule's idempotence phase. + - {path: "{{ grafana_alloy_state_dir }}", owner: "{{ grafana_alloy_user }}", mode: "0750"} + loop_control: + label: "{{ item.path }}" + + # --- Install ---------------------------------------------------------------- + - name: Install the Alloy binary + ansible.builtin.import_tasks: install.yml + + # --- Configuration + hardened unit ------------------------------------------ + - name: Render the Alloy pipeline configuration + ansible.builtin.template: + src: config.alloy.j2 + dest: "{{ grafana_alloy_config_file }}" + owner: root + group: "{{ grafana_alloy_group }}" + mode: "0644" # non-secret by design: secrets live in ${GC_*} expansion + notify: Restart alloy + + - name: Install the hardened systemd unit + ansible.builtin.template: + src: alloy.service.j2 + dest: /etc/systemd/system/alloy.service + owner: root + group: root + mode: "0644" + notify: + - Reload systemd + - Restart alloy + + # --- Config validation gate (fail loud before any restart) ------------------ + # `alloy fmt --test` validates syntax AND canonical formatting (per upstream + # docs it fails on syntactically incorrect files). It cannot resolve component + # references — but a wrong ${GC_*} placeholder ONLY fails at runtime anyway, + # which the grafana-cloud molecule scenario covers textually. Under check mode + # the file has not been written yet, so skip like the decdn_node config gate. + - name: Check the rendered configuration with the installed binary + ansible.builtin.command: + cmd: "{{ grafana_alloy_bin }} fmt --test {{ grafana_alloy_config_file }}" + changed_when: false + register: _ga_fmt_check + failed_when: false + when: not ansible_check_mode + + - name: Report the configuration-validation failure + ansible.builtin.fail: + msg: >- + `alloy fmt --test` rejected {{ grafana_alloy_config_file }} + (rc={{ _ga_fmt_check.rc | default('?') }}): + {{ (_ga_fmt_check.stderr | default('', true)) + or (_ga_fmt_check.stdout | default('', true)) }} — fix the template + regression before deploying. + when: + - not ansible_check_mode + - _ga_fmt_check.rc | default(1) != 0 + + - name: Apply pending Alloy changes + ansible.builtin.meta: flush_handlers + + - name: Enable and start Alloy + ansible.builtin.systemd: + name: alloy + enabled: true + state: started + + # Backstop for a Type=simple unit that exited fast enough to have already + # flipped to failed by the time we got here (the stub in CI sleeps forever). + - name: Gather service facts + ansible.builtin.service_facts: + + - name: Confirm Alloy is running (not failed/crash-looping) + ansible.builtin.assert: + that: + - "'alloy.service' in ansible_facts.services" + - ansible_facts.services['alloy.service'].state == 'running' + fail_msg: "alloy is not running — check: journalctl -u alloy -e" diff --git a/ansible/roles/grafana_alloy/tasks/preflight.yml b/ansible/roles/grafana_alloy/tasks/preflight.yml new file mode 100644 index 0000000..9ea8269 --- /dev/null +++ b/ansible/roles/grafana_alloy/tasks/preflight.yml @@ -0,0 +1,166 @@ +--- +# Fail-loud preflight, run BEFORE any host mutation. Mirrors decdn_node's +# secret-file posture: this role never carries secrets in inventory and never +# reads the env file's values onto the control machine — it only greps shapes on +# the host itself. +# +# Ordering matters for testability (molecule/validation): PURE-INVENTORY gates +# run first so a bad knob rejects deterministically regardless of what fixtures +# exist on the host; the host-state credential gates follow. + +# --- Variable-shape gates ------------------------------------------------------- +# Env-var placeholders resolve AFTER these checks, so anything templatable from +# inventory must be validated here, on the spot. + +- name: Validate the pipeline knobs + ansible.builtin.assert: + that: + # ratio is the 0..1 keep-ratio documented for operators; the template renders + # sampling_percentage = ratio * 100, so out-of-range values are an operator + # surprise (0 = everything dropped, >1 upstream-rejected float32 semantics). + - grafana_alloy_trace_sampling_ratio is number + - grafana_alloy_trace_sampling_ratio >= 0 + - grafana_alloy_trace_sampling_ratio <= 1 + # Duration strings interpolate verbatim into Go duration fields — one wrong + # suffix is a config-load crash-loop. + - item is match('^[0-9]+(ms|s|m|h)$') + quiet: true + fail_msg: >- + grafana_alloy_trace_sampling_ratio must be a number in [0, 1] (got + "{{ grafana_alloy_trace_sampling_ratio }}"), and every *_interval / + *_timeout / max_elapsed_time knob must be a bare integer with an ms/s/m/h + suffix, e.g. 5s or 500ms. + loop: + - "{{ grafana_alloy_batch_timeout }}" + - "{{ grafana_alloy_retry_initial_interval }}" + - "{{ grafana_alloy_retry_max_elapsed_time }}" + loop_control: + label: "{{ item }}" + +- name: Validate integer pipeline knobs + ansible.builtin.assert: + that: + - item.value | int >= 1 + quiet: true + fail_msg: >- + {{ item.name }} must be a positive integer (got "{{ item.value }}"). + loop: + - {name: grafana_alloy_batch_send_size, value: "{{ grafana_alloy_batch_send_size }}"} + - {name: grafana_alloy_batch_max_size, value: "{{ grafana_alloy_batch_max_size }}"} + - {name: grafana_alloy_queue_size, value: "{{ grafana_alloy_queue_size }}"} + loop_control: + label: "{{ item.name }}" + +- name: Validate identity labels + ansible.builtin.assert: + that: + # "" is legal here (attribute omitted entirely) but a blank AFTER trimming + # whitespace must not sneak through. + - item in ["", none] or item is match('^[A-Za-z0-9._/-]{1,63}$') + quiet: true + fail_msg: >- + Identity label values are shipped to Grafana Cloud as metric labels and OTLP + resource attributes on EVERY series/span, so they are restricted to a + low-cardinality charset (letters, digits, dot, underscore, slash, ≤ 63 + chars): {{ item }} does not fit. Offending variables: + grafana_alloy_service_name / service_namespace / instance_id / + deployment_environment / region. + loop: + - "{{ grafana_alloy_service_name }}" + - "{{ grafana_alloy_service_namespace }}" + # _ga_instance_id is REQUIRED (never "") — it has a derived non-empty default. + - "{{ _ga_instance_id }}" + # These two may be "" (= attribute omitted entirely). + - "{{ grafana_alloy_deployment_environment | default('', true) }}" + - "{{ grafana_alloy_region | default('', true) }}" + loop_control: + label: "{{ item }}" + +# COUPLING GUARD: the node-side otlp_endpoint injected by +# roles/decdn_node/templates/node.toml.j2 hardcodes the gRPC receiver port +# constant. If this default ever moves, node.toml must move WITH it — the molecule +# suite asserts the pair stays coherent; update both plus their comments. +- name: Require the standard OTLP gRPC port the node's exporter expects + ansible.builtin.assert: + that: + - grafana_alloy_otlp_grpc_port | int == 4317 + fail_msg: >- + grafana_alloy_otlp_grpc_port was moved away from 4317, but + roles/decdn_node emits otlp_endpoint="http://127.0.0.1:4317" as a fixed + literal — spans would stop reaching Alloy while every deploy stays green. + Either restore 4317 or re-sync node.toml.j2 + the schema docs in the same + change. + +# --- Host-state credential gates ------------------------------------------------ + +- name: Require the Grafana Cloud secret environment file + ansible.builtin.stat: + path: "{{ grafana_alloy_secret_file }}" + follow: false + register: _ga_secret_stat + +# stat.exists alone cannot see the difference between a regular file and a +# symlink/directory; only `isreg` can. A symlink planted into the (root-owned) +# /etc/decdn dir that later passes a root chown/chmod would harden whatever TARGET +# it points at — exactly what decdn.env's gate refuses, so refuse here too. +- name: Assert the secret file is a real root-owned 0600 file + ansible.builtin.assert: + that: + - _ga_secret_stat.stat.exists + - _ga_secret_stat.stat.isreg | default(false) + - (_ga_secret_stat.stat.pw_name | default('')) == 'root' + - (_ga_secret_stat.stat.gr_name | default('')) == 'root' + - _ga_secret_stat.stat.mode | string == '0600' + fail_msg: >- + {{ grafana_alloy_secret_file }} is missing, not a regular file (symlink? + directory?), or not owned root:root 0600 (found owner + {{ _ga_secret_stat.stat.pw_name | default('?') }}, mode + {{ _ga_secret_stat.stat.mode | default('?') }}). Provision it on the target + host per roles/grafana_alloy/files/grafana-alloy.env.example: + umask 077 + sudo install -m 600 -o root -g root grafana-alloy.env {{ grafana_alloy_secret_file }} + systemd expands its KEY=value pairs into the unit before dropping + privileges, so root ownership is sufficient — do not relax it. + +# systemd skips an EnvironmentFile line it cannot parse SILENTLY, so each required +# key is grepped for presence + value shape directly on the host: grep keeps the +# values off stdout AND off the control machine (slurping the file back would put +# the API token there, defeating the point of the host-provisioned path). +- name: Confirm every required credential key is present and shaped correctly + ansible.builtin.command: + cmd: grep -qE -- '{{ item.regex }}' {{ grafana_alloy_secret_file | quote }} + changed_when: false + check_mode: false # read-only gate: must also run under make check + failed_when: false + loop: + # The two cloud URLs must be https:// (plain http would ship the API token in + # cleartext); the credential values must be non-empty, non-comment text. + # POSIX bracket classes only — grep -E does NOT interpret \r/\n escapes (they + # become literal 'r'/'n', silently poisoning the class), and each grep line is + # already newline-free by construction. + - key: GC_PROM_REMOTE_WRITE_URL + regex: ^GC_PROM_REMOTE_WRITE_URL=https://[^[:space:]]+$ + - key: GC_PROM_USERNAME + regex: ^GC_PROM_USERNAME=[^[:space:]#][^[:space:]]*$ + - key: GC_API_TOKEN + regex: ^GC_API_TOKEN=[^[:space:]#][^[:space:]]*$ + - key: GC_OTLP_ENDPOINT + regex: ^GC_OTLP_ENDPOINT=https://[^[:space:]]+$ + loop_control: + label: "{{ item.key }}" + register: _ga_secret_grep + +- name: Report which credential key was missing or malformed + ansible.builtin.fail: + msg: >- + {{ grafana_alloy_secret_file }} does not carry a usable line for + {{ _ga_missing_keys | join(', ') }}. The file must contain exactly the four + KEY=value pairs shown in roles/grafana_alloy/files/grafana-alloy.env.example; + both cloud URLs must be https:// and each value non-empty (systemd silently + drops unparseable lines, so a quoting typo looks identical to a missing key). + vars: + _ga_missing_keys: >- + {{ _ga_secret_grep.results + | rejectattr('rc', 'equalto', 0) + | map(attribute='item.key') | list }} + when: _ga_missing_keys | length > 0 diff --git a/ansible/roles/grafana_alloy/templates/alloy.service.j2 b/ansible/roles/grafana_alloy/templates/alloy.service.j2 new file mode 100644 index 0000000..88c540f --- /dev/null +++ b/ansible/roles/grafana_alloy/templates/alloy.service.j2 @@ -0,0 +1,62 @@ +# MANAGED BY the grafana_alloy role — do not edit by hand. +# +# Runs Grafana Alloy as the dedicated, unprivileged `{{ grafana_alloy_user }}` +# user. Both OTLP receivers and Alloy's own admin server are pinned to loopback +# (AGENTS.md hard rule 2): nothing here faces public traffic, so no firewall hole +# exists. The credential env file is read by systemd as ROOT before privileges +# drop, so the agent never needs (and must not get) direct read access. +[Unit] +Description=Grafana Alloy telemetry agent (deCDN node observability) +Documentation=https://grafana.com/docs/alloy/ +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User={{ grafana_alloy_user }} +Group={{ grafana_alloy_group }} + +# Sensitive: GC_PROM_REMOTE_WRITE_URL / GC_PROM_USERNAME / GC_API_TOKEN / +# GC_OTLP_ENDPOINT. Root-owned 0600; expanded into ${GC_*} placeholders inside +# config.alloy via --config.expand-env below. +EnvironmentFile={{ grafana_alloy_secret_file }} + +ExecStart={{ grafana_alloy_bin_effective }} run \ + --config.expand-env \ + --server.http.listen-addr={{ grafana_alloy_server_http_addr }} \ + --server.grpc.listen-addr={{ grafana_alloy_server_grpc_addr }} \ + {{ grafana_alloy_config_file }} + +# Graceful stop cap: flush the exporter queue on SIGTERM before SIGKILL would cut +# buffered telemetry loose. +KillSignal=SIGTERM +TimeoutStopSec={{ grafana_alloy_stop_timeout_sec }} + +Restart=on-failure +RestartSec=5 + +# --- Hardening --------------------------------------------------------------- +# Mirrors decdn-node.service: strict filesystem, privilege boundary, syscall +# filter. StateDirectory owns {{ grafana_alloy_state_dir }} (the only writable +# path under ProtectSystem=strict): WAL/queue state lives there. +StateDirectory=alloy +StateDirectoryMode=0750 +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +PrivateDevices=true +ProtectKernelTunables=true +ProtectKernelModules=true +ProtectControlGroups=true +ProtectClock=true +# AF_INET/AF_INET6 for the loopback OTLP/scrape/remotes; AF_UNIX for journald. +RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX +RestrictNamespaces=true +RestrictSUIDSGID=true +LockPersonality=true +SystemCallFilter=@system-service +SystemCallErrorNumber=EPERM + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/grafana_alloy/templates/config.alloy.j2 b/ansible/roles/grafana_alloy/templates/config.alloy.j2 new file mode 100644 index 0000000..7911460 --- /dev/null +++ b/ansible/roles/grafana_alloy/templates/config.alloy.j2 @@ -0,0 +1,138 @@ +# MANAGED BY the grafana_alloy Ansible role — do not edit by hand. +# +# Pipeline: scrape the node's /metrics and push it to Grafana Cloud Prometheus; +# receive OTLP traces (plus any logs/metrics pushed over localhost) on gRPC 4317 / +# HTTP 4318 bound to loopback only; stamp resource identity on everything, +# parent-based sample traces, batch, then export via OTLP/HTTP. +# +# Secrets are NOT literals here: every ${GC_*} placeholder expands at load time +# from the root-owned 0600 environment file systemd hands the unit +# (--config.expand-env in alloy.service; operator template: +# roles/grafana_alloy/files/grafana-alloy.env.example). + +prometheus.scrape "decdn_node" { + targets = [ + { "__address__" = "{{ grafana_alloy_scrape_target }}", + "instance" = "{{ _ga_instance_id }}" }, + ] + forward_to = [prometheus.relabel.decdn_identity.receiver] + scrape_interval = {{ grafana_alloy_scrape_interval | to_json }} +} + +{#- Low-cardinality identity labels shipped with every metric. Prom label NAMES + may not contain '.', so the dotted service.* naming flattens to underscores; + empty values omit their rule entirely. #} +{% set ga_metric_labels = [ + ["service_name", grafana_alloy_service_name], + ["service_namespace", grafana_alloy_service_namespace], + ["service_instance_id", _ga_instance_id], + ["deployment_environment", grafana_alloy_deployment_environment], + ["region", grafana_alloy_region], + ] %} +prometheus.relabel "decdn_identity" { + forward_to = [prometheus.remote_write.cloud.receiver] +{% for name, value in ga_metric_labels if value | length > 0 %} + + rule { + target_label = {{ name | tojson }} + replacement = {{ value | tojson }} + } +{% endfor %} +} + +prometheus.remote_write "cloud" { + endpoint { + url = "${GC_PROM_REMOTE_WRITE_URL}" + + basic_auth { + username = "${GC_PROM_USERNAME}" + password = "${GC_API_TOKEN}" + } + } +} + +otelcol.receiver.otlp "local" { + grpc { + endpoint = "{{ grafana_alloy_otlp_grpc_bind }}:{{ grafana_alloy_otlp_grpc_port }}" + } + + http { + endpoint = "{{ grafana_alloy_otlp_http_bind }}:{{ grafana_alloy_otlp_http_port }}" + } + + output { + traces = [otelcol.processor.probabilistic_sampler.parent_based_traces.input] + logs = [otelcol.processor.attributes.decdn_identity.input] + metrics = [otelcol.processor.attributes.decdn_identity.input] + } +} + +{#- Identity as OTLP resource attributes; unlike Prom labels these MAY use dotted + names. Sorted keys keep rendering byte-stable across converges (idempotence) + and empty values drop out of the action list entirely. #} +{% set ga_res_attrs = { + "deployment.environment": grafana_alloy_deployment_environment, + "region": grafana_alloy_region, + "service.instance.id": _ga_instance_id, + "service.name": grafana_alloy_service_name, + "service.namespace": grafana_alloy_service_namespace, + } %} +otelcol.processor.attributes "decdn_identity" { +{% for attr_key in ga_res_attrs.keys() | sort if ga_res_attrs[attr_key] | length > 0 %} + action { + key = {{ attr_key | tojson }} + action = "upsert" + value = {{ ga_res_attrs[attr_key] | tojson }} + } +{% endfor %} + + output { + traces = [otelcol.processor.batch.decdn.input] + logs = [otelcol.processor.batch.decdn.input] + metrics = [otelcol.processor.batch.decdn.input] + } +} + +{#- Parent-based sampling: spans without an explicit flag inherit their parent's + decision, so whole trace subtrees survive or drop together. Renders the + operator-facing 0..1 keep-ratio as the component's percentage. #} +otelcol.processor.probabilistic_sampler "parent_based_traces" { + sampling_percentage = {{ grafana_alloy_trace_sampling_ratio | float * 100 }} + + output { + traces = [otelcol.processor.attributes.decdn_identity.input] + } +} + +otelcol.processor.batch "decdn" { + send_batch_size = {{ grafana_alloy_batch_send_size }} + send_batch_max_size = {{ grafana_alloy_batch_max_size }} + timeout = {{ grafana_alloy_batch_timeout | to_json }} + + output { + traces = [otelcol.exporter.otlphttp.cloud.input] + logs = [otelcol.exporter.otlphttp.cloud.input] + metrics = [otelcol.exporter.otlphttp.cloud.input] + } +} + +otelcol.exporter.otlphttp "cloud" { + client { + endpoint = "${GC_OTLP_ENDPOINT}" + auth = otelcol.auth.basic.cloud.handler + + sending_queue { + queue_size = {{ grafana_alloy_queue_size }} + } + + retry_on_failure { + initial_interval = {{ grafana_alloy_retry_initial_interval | to_json }} + max_elapsed_time = {{ grafana_alloy_retry_max_elapsed_time | to_json }} + } + } +} + +otelcol.auth.basic "cloud" { + username = "${GC_PROM_USERNAME}" + password = "${GC_API_TOKEN}" +} From ebb2af4b500256c783093afa95e3344b4d06e805 Mon Sep 17 00:00:00 2001 From: Ant Somers Date: Wed, 16 Sep 2026 22:34:14 +0300 Subject: [PATCH 2/3] fix: address Grafana Alloy review findings Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- ansible/Makefile | 2 +- ansible/galaxy/README.md | 4 +- ansible/galaxy/build.sh | 2 +- ansible/inventory/group_vars/decdn_nodes.yml | 2 +- ansible/molecule/default/converge.yml | 2 + ansible/molecule/grafana-cloud/verify.yml | 2 +- ansible/playbooks/site.yml | 2 +- ansible/roles/grafana_alloy/README.md | 9 ++-- ansible/roles/grafana_alloy/defaults/main.yml | 11 ++--- ansible/roles/grafana_alloy/tasks/install.yml | 8 +++- ansible/roles/grafana_alloy/tasks/main.yml | 17 +++++++- .../roles/grafana_alloy/tasks/preflight.yml | 43 +++++++++++++++++++ .../grafana_alloy/templates/alloy.service.j2 | 3 +- .../grafana_alloy/templates/config.alloy.j2 | 21 +++++---- 14 files changed, 94 insertions(+), 34 deletions(-) diff --git a/ansible/Makefile b/ansible/Makefile index 992ca11..fcbd44d 100644 --- a/ansible/Makefile +++ b/ansible/Makefile @@ -122,7 +122,7 @@ molecule-serial: deps # --- Galaxy collection (decdn.node) ------------------------------------------ # Stage baseline + decdn_node into a clean collection tree and build the artifact -# under build/. Only those two roles ship; see galaxy/README.md. Publishing stays +# under build/. Only the three deployment roles ship; see galaxy/README.md. Publishing stays # a manual step (ansible-galaxy collection publish build/decdn-node-*.tar.gz). build: ./galaxy/build.sh diff --git a/ansible/galaxy/README.md b/ansible/galaxy/README.md index 9aec706..d64a1cd 100644 --- a/ansible/galaxy/README.md +++ b/ansible/galaxy/README.md @@ -2,12 +2,13 @@ Deploy and harden a **public [deCDN](https://decdn.org) node**. This collection is the public, reusable slice of the [`decdn/devops`](https://github.com/decdn/devops) -repository — two roles and nothing else: +repository — three roles and nothing else: | Role | Purpose | |------|---------| | `decdn.node.baseline` | Debian host baseline — nftables default-deny inbound, fail2ban, unattended-upgrades, chrony, an admin sudo user, then DevSec OS + SSH hardening (applied last). | | `decdn.node.decdn_node` | The `decdn-node` daemon — installed from a pinned GitHub Release tarball under a hardened systemd unit; public QUIC udp/4433, loopback metrics + admin RPC. | +| `decdn.node.grafana_alloy` | Opt-in Grafana Cloud observability agent — loopback-only Alloy receiver and hardened telemetry export. | ## Requirements @@ -60,6 +61,7 @@ full variable list, the eth-keystore prerequisite, and day-2 ops: - [`roles/baseline`](https://github.com/decdn/devops/tree/main/ansible/roles/baseline) - [`roles/decdn_node`](https://github.com/decdn/devops/tree/main/ansible/roles/decdn_node) +- [`roles/grafana_alloy`](https://github.com/decdn/devops/tree/main/ansible/roles/grafana_alloy) ## Security model diff --git a/ansible/galaxy/build.sh b/ansible/galaxy/build.sh index 0c2545c..029b23a 100755 --- a/ansible/galaxy/build.sh +++ b/ansible/galaxy/build.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # Stage and build the public `decdn.node` Galaxy collection. # -# Only the roles/baseline + roles/decdn_node sources ship. All deploy machinery +# Only the roles/baseline, roles/decdn_node, and roles/grafana_alloy sources ship. All deploy machinery # (inventory, Makefile, ansible.cfg) is excluded BY CONSTRUCTION — it is simply # never copied into the staging tree. This keeps the artifact clean and leaves the # internal project untouched (no galaxy.yml at the project root, so ansible-lint diff --git a/ansible/inventory/group_vars/decdn_nodes.yml b/ansible/inventory/group_vars/decdn_nodes.yml index e487f8d..05379d9 100644 --- a/ansible/inventory/group_vars/decdn_nodes.yml +++ b/ansible/inventory/group_vars/decdn_nodes.yml @@ -17,5 +17,5 @@ baseline_extra_inbound: # credential file on each host — see roles/grafana_alloy/files/grafana-alloy.env.example: # # decdn_grafana_cloud_enabled: true -# grafana_alloy_region: "" # e.g. "US" — same value as decdn_region +# grafana_alloy_region: "{{ decdn_region }}" # override only when needed # grafana_alloy_deployment_environment: production diff --git a/ansible/molecule/default/converge.yml b/ansible/molecule/default/converge.yml index 52ef2f0..f40a916 100644 --- a/ansible/molecule/default/converge.yml +++ b/ansible/molecule/default/converge.yml @@ -118,4 +118,6 @@ name: baseline tasks_from: sudo_users roles: + - role: grafana_alloy + tags: [observability, decdn_node, node] - role: decdn_node diff --git a/ansible/molecule/grafana-cloud/verify.yml b/ansible/molecule/grafana-cloud/verify.yml index 7f053a2..aa82040 100644 --- a/ansible/molecule/grafana-cloud/verify.yml +++ b/ansible/molecule/grafana-cloud/verify.yml @@ -60,7 +60,7 @@ # ratio 0.25 -> percentage 25; batch size overrides pass through; labels # dotted AND set-variable ones present; secrets remain ${...} placeholders. - >- - 'sampling_percentage = 25' in ga_config + 'sampling_percentage = 25.0' in ga_config - >- 'send_batch_size = 512' in ga_config - >- diff --git a/ansible/playbooks/site.yml b/ansible/playbooks/site.yml index 6a79795..4cd2e92 100644 --- a/ansible/playbooks/site.yml +++ b/ansible/playbooks/site.yml @@ -14,6 +14,6 @@ # flag drives this + the otlp_endpoint injection inside decdn_node). Runs # BEFORE decdn_node so Alloy's receivers exist before the daemon exports. - role: grafana_alloy - tags: [observability] + tags: [observability, decdn_node, node] - role: decdn_node tags: [decdn_node, node] diff --git a/ansible/roles/grafana_alloy/README.md b/ansible/roles/grafana_alloy/README.md index be7332f..56a9301 100644 --- a/ansible/roles/grafana_alloy/README.md +++ b/ansible/roles/grafana_alloy/README.md @@ -8,7 +8,7 @@ loopback-only [Grafana Alloy](https://grafana.com/docs/alloy/) agent that and pushes it to your Grafana Cloud Prometheus, stamped with low-cardinality identity labels, and - receives the daemon's OTLP span exports (`otlp_endpoint`) on gRPC `127.0.0.1:4317` - and HTTP `127.0.0.1:4318`, parent-based samples traces (default keep-ratio 0.25), + and HTTP `127.0.0.1:4318`, probabilistically samples traces (default keep-ratio 0.25), batches them, and exports via OTLP/HTTP to your Grafana Cloud org. The Helm-chart path is separate and deliberately untouched by this role. @@ -52,9 +52,8 @@ upstream version/sha256). Highlights: | `grafana_alloy_install_method` | `release` | `manual` copies a control-machine binary (CI stubs) | | `grafana_alloy_version` / `grafana_alloy_sha256` | pin | Bump together from upstream release digests | | `grafana_alloy_trace_sampling_ratio` | `0.25` | Trace keep-ratio, validated to `[0,1]` | -| `grafana_alloy_scrape_target` | `127.0.0.1:9090` | Must mirror `decdn_metrics_port`'s default | | `grafana_alloy_otlp_grpc_port` | `4317` | Hard-coupled to the literal emitted into node.toml | -| `grafana_alloy_region` | `""` | Set = your `decdn_region`; empty omits the attribute | +| `grafana_alloy_region` | `decdn_region` | Required identity attribute; derived from the node region | Label variables (`service_name`, `service_namespace`, `instance_id`, `deployment_environment`, `region`) ship on EVERY series/span/log line — keep @@ -66,8 +65,8 @@ spans, and no per-request/per-hash label values. Two couplings between roles are pinned by constants plus molecule assertions: -1. This role's `prometheus.scrape` targets `127.0.0.1:9090` — the - `decdn_metrics_port` default. +1. This role's `prometheus.scrape` targets `127.0.0.1:`, using + the node role's configured metrics port directly. 2. `decdn_node` injects `otlp_endpoint = "http://127.0.0.1:4317"` when the flag flips on — the port constant here. diff --git a/ansible/roles/grafana_alloy/defaults/main.yml b/ansible/roles/grafana_alloy/defaults/main.yml index eabc0f7..c77136a 100644 --- a/ansible/roles/grafana_alloy/defaults/main.yml +++ b/ansible/roles/grafana_alloy/defaults/main.yml @@ -49,11 +49,6 @@ grafana_alloy_server_http_addr: 127.0.0.1:12345 grafana_alloy_server_grpc_addr: 127.0.0.1:12347 # --- Metrics scrape (node /metrics -> Grafana Cloud Prometheus) --------------- -# COUPLING: mirrors decdn_metrics_port's DEFAULT (9090) in roles/decdn_node. -# Kept a plain constant here because cross-role vars are fragile; the molecule -# suite guards the drift — if you move decdn_metrics_port off 9090, set -# grafana_alloy_scrape_target on the same inventory level. -grafana_alloy_scrape_target: 127.0.0.1:9090 grafana_alloy_scrape_interval: 30s # --- Resource identity labels (low-cardinality only!) -------------------------- @@ -64,11 +59,11 @@ grafana_alloy_scrape_interval: 30s grafana_alloy_service_name: decdn-node grafana_alloy_service_namespace: decdn grafana_alloy_instance_id: "" # "" => derive from inventory_hostname -grafana_alloy_deployment_environment: "" # "" => attribute omitted entirely -grafana_alloy_region: "" # operator sets it = decdn_region ("" => omitted) +grafana_alloy_deployment_environment: production +grafana_alloy_region: "{{ decdn_region | default('') }}" # --- Pipeline tuning ------------------------------------------------------------ -# Parent-based probabilistic sampler for TRACES, expressed as the 0..1 keep-ratio +# Probabilistic sampler for TRACES, expressed as the 0..1 keep-ratio # the deCDN issue settled on (0.25 == keep ~25% of spans); converted to the # component's percentage internally. Validated at preflight. grafana_alloy_trace_sampling_ratio: 0.25 diff --git a/ansible/roles/grafana_alloy/tasks/install.yml b/ansible/roles/grafana_alloy/tasks/install.yml index 3aa7d8a..8224ca0 100644 --- a/ansible/roles/grafana_alloy/tasks/install.yml +++ b/ansible/roles/grafana_alloy/tasks/install.yml @@ -42,7 +42,9 @@ - name: Decide whether (re)install is needed ansible.builtin.set_fact: _ga_need_install: "{{ (_ga_dpkg_version.rc | default(1)) != 0 - or (grafana_alloy_version not in (_ga_dpkg_version.stdout | default(''))) }}" + or (_ga_dpkg_version.stdout | trim) + not in [grafana_alloy_version, + grafana_alloy_version ~ '-1'] }}" when: - grafana_alloy_install_method == "release" - not ansible_check_mode @@ -132,7 +134,9 @@ ansible.builtin.command: "{{ grafana_alloy_bin_effective }} --version" changed_when: false register: _ga_installed_version + when: not ansible_check_mode failed_when: >- _ga_installed_version.rc != 0 or (grafana_alloy_install_method == "release" - and grafana_alloy_version not in _ga_installed_version.stdout) + and not (_ga_installed_version.stdout + | regex_search('(^|[^0-9])v?' ~ (grafana_alloy_version | regex_escape) ~ '([^0-9]|$)'))) diff --git a/ansible/roles/grafana_alloy/tasks/main.yml b/ansible/roles/grafana_alloy/tasks/main.yml index 4421247..c903072 100644 --- a/ansible/roles/grafana_alloy/tasks/main.yml +++ b/ansible/roles/grafana_alloy/tasks/main.yml @@ -42,11 +42,23 @@ loop: - /etc/systemd/system/alloy.service - "{{ grafana_alloy_config_dir }}" + - "{{ grafana_alloy_state_dir }}" notify: Reload systemd - name: Apply pending teardown notifications ansible.builtin.meta: flush_handlers + - name: Remove the leftover Alloy account + ansible.builtin.user: + name: "{{ grafana_alloy_user }}" + state: absent + remove: true + + - name: Remove the leftover Alloy group + ansible.builtin.group: + name: "{{ grafana_alloy_group }}" + state: absent + # Everything below runs only when observability is explicitly enabled. - name: Enable and configure the Grafana Cloud agent when: decdn_grafana_cloud_enabled | bool @@ -126,7 +138,7 @@ # the file has not been written yet, so skip like the decdn_node config gate. - name: Check the rendered configuration with the installed binary ansible.builtin.command: - cmd: "{{ grafana_alloy_bin }} fmt --test {{ grafana_alloy_config_file }}" + cmd: "{{ grafana_alloy_bin_effective }} fmt --test {{ grafana_alloy_config_file }}" changed_when: false register: _ga_fmt_check failed_when: false @@ -152,11 +164,13 @@ name: alloy enabled: true state: started + when: not ansible_check_mode # Backstop for a Type=simple unit that exited fast enough to have already # flipped to failed by the time we got here (the stub in CI sleeps forever). - name: Gather service facts ansible.builtin.service_facts: + when: not ansible_check_mode - name: Confirm Alloy is running (not failed/crash-looping) ansible.builtin.assert: @@ -164,3 +178,4 @@ - "'alloy.service' in ansible_facts.services" - ansible_facts.services['alloy.service'].state == 'running' fail_msg: "alloy is not running — check: journalctl -u alloy -e" + when: not ansible_check_mode diff --git a/ansible/roles/grafana_alloy/tasks/preflight.yml b/ansible/roles/grafana_alloy/tasks/preflight.yml index 9ea8269..24cd6d0 100644 --- a/ansible/roles/grafana_alloy/tasks/preflight.yml +++ b/ansible/roles/grafana_alloy/tasks/preflight.yml @@ -34,12 +34,14 @@ - "{{ grafana_alloy_batch_timeout }}" - "{{ grafana_alloy_retry_initial_interval }}" - "{{ grafana_alloy_retry_max_elapsed_time }}" + - "{{ grafana_alloy_scrape_interval }}" loop_control: label: "{{ item }}" - name: Validate integer pipeline knobs ansible.builtin.assert: that: + - (item.value | string) is match('^[0-9]+$') - item.value | int >= 1 quiet: true fail_msg: >- @@ -76,6 +78,46 @@ loop_control: label: "{{ item }}" +- name: Require deployment identity attributes + ansible.builtin.assert: + that: + - grafana_alloy_deployment_environment | length > 0 + - grafana_alloy_region | length > 0 + fail_msg: >- + grafana_alloy_deployment_environment and grafana_alloy_region are required + identity attributes when Grafana Cloud observability is enabled. + +- name: Validate loopback listener addresses + ansible.builtin.assert: + that: + - grafana_alloy_otlp_grpc_bind == '127.0.0.1' + - grafana_alloy_otlp_http_bind == '127.0.0.1' + - grafana_alloy_server_http_addr is match('^127[.]0[.]0[.]1:[0-9]+$') + - grafana_alloy_server_grpc_addr is match('^127[.]0[.]0[.]1:[0-9]+$') + fail_msg: >- + Grafana Alloy OTLP and admin listeners must bind to 127.0.0.1 only; + wildcard or public listener addresses are not permitted. + +- name: Validate listener ports + ansible.builtin.assert: + that: + - (item | string) is match('^[0-9]+$') + - item | int >= 1 + - item | int <= 65535 + quiet: true + fail_msg: "Grafana Alloy listener ports must be decimal integers in 1..65535 (got {{ item }})." + loop: + - "{{ grafana_alloy_otlp_grpc_port }}" + - "{{ grafana_alloy_otlp_http_port }}" + - "{{ grafana_alloy_server_http_addr.split(':')[-1] }}" + - "{{ grafana_alloy_server_grpc_addr.split(':')[-1] }}" + +- name: Constrain the Alloy state directory to systemd's writable area + ansible.builtin.assert: + that: + - grafana_alloy_state_dir is match('^/var/lib/[A-Za-z0-9._/-]+$') + fail_msg: "grafana_alloy_state_dir must be a path below /var/lib." + # COUPLING GUARD: the node-side otlp_endpoint injected by # roles/decdn_node/templates/node.toml.j2 hardcodes the gRPC receiver port # constant. If this default ever moves, node.toml must move WITH it — the molecule @@ -84,6 +126,7 @@ ansible.builtin.assert: that: - grafana_alloy_otlp_grpc_port | int == 4317 + - (grafana_alloy_otlp_grpc_port | string) is match('^[0-9]+$') fail_msg: >- grafana_alloy_otlp_grpc_port was moved away from 4317, but roles/decdn_node emits otlp_endpoint="http://127.0.0.1:4317" as a fixed diff --git a/ansible/roles/grafana_alloy/templates/alloy.service.j2 b/ansible/roles/grafana_alloy/templates/alloy.service.j2 index 88c540f..e198c6d 100644 --- a/ansible/roles/grafana_alloy/templates/alloy.service.j2 +++ b/ansible/roles/grafana_alloy/templates/alloy.service.j2 @@ -25,6 +25,7 @@ ExecStart={{ grafana_alloy_bin_effective }} run \ --config.expand-env \ --server.http.listen-addr={{ grafana_alloy_server_http_addr }} \ --server.grpc.listen-addr={{ grafana_alloy_server_grpc_addr }} \ + --storage.path={{ grafana_alloy_state_dir }}/data \ {{ grafana_alloy_config_file }} # Graceful stop cap: flush the exporter queue on SIGTERM before SIGKILL would cut @@ -39,7 +40,7 @@ RestartSec=5 # Mirrors decdn-node.service: strict filesystem, privilege boundary, syscall # filter. StateDirectory owns {{ grafana_alloy_state_dir }} (the only writable # path under ProtectSystem=strict): WAL/queue state lives there. -StateDirectory=alloy +StateDirectory={{ grafana_alloy_state_dir | regex_replace('^/var/lib/', '') }} StateDirectoryMode=0750 NoNewPrivileges=true ProtectSystem=strict diff --git a/ansible/roles/grafana_alloy/templates/config.alloy.j2 b/ansible/roles/grafana_alloy/templates/config.alloy.j2 index 7911460..ff42cd8 100644 --- a/ansible/roles/grafana_alloy/templates/config.alloy.j2 +++ b/ansible/roles/grafana_alloy/templates/config.alloy.j2 @@ -3,7 +3,7 @@ # Pipeline: scrape the node's /metrics and push it to Grafana Cloud Prometheus; # receive OTLP traces (plus any logs/metrics pushed over localhost) on gRPC 4317 / # HTTP 4318 bound to loopback only; stamp resource identity on everything, -# parent-based sample traces, batch, then export via OTLP/HTTP. +# probabilistically sample traces, batch, then export via OTLP/HTTP. # # Secrets are NOT literals here: every ${GC_*} placeholder expands at load time # from the root-owned 0600 environment file systemd hands the unit @@ -12,7 +12,7 @@ prometheus.scrape "decdn_node" { targets = [ - { "__address__" = "{{ grafana_alloy_scrape_target }}", + { "__address__" = "127.0.0.1:{{ decdn_metrics_port | default(9090) }}", "instance" = "{{ _ga_instance_id }}" }, ] forward_to = [prometheus.relabel.decdn_identity.receiver] @@ -61,9 +61,9 @@ otelcol.receiver.otlp "local" { } output { - traces = [otelcol.processor.probabilistic_sampler.parent_based_traces.input] - logs = [otelcol.processor.attributes.decdn_identity.input] - metrics = [otelcol.processor.attributes.decdn_identity.input] + traces = [otelcol.processor.probabilistic_sampler.traces.input] + logs = [otelcol.processor.resource.decdn_identity.input] + metrics = [otelcol.processor.resource.decdn_identity.input] } } @@ -77,7 +77,7 @@ otelcol.receiver.otlp "local" { "service.name": grafana_alloy_service_name, "service.namespace": grafana_alloy_service_namespace, } %} -otelcol.processor.attributes "decdn_identity" { +otelcol.processor.resource "decdn_identity" { {% for attr_key in ga_res_attrs.keys() | sort if ga_res_attrs[attr_key] | length > 0 %} action { key = {{ attr_key | tojson }} @@ -93,14 +93,13 @@ otelcol.processor.attributes "decdn_identity" { } } -{#- Parent-based sampling: spans without an explicit flag inherit their parent's - decision, so whole trace subtrees survive or drop together. Renders the - operator-facing 0..1 keep-ratio as the component's percentage. #} -otelcol.processor.probabilistic_sampler "parent_based_traces" { +{#- Renders the operator-facing 0..1 keep-ratio as the component's percentage. + Parent-based decisions must be made by the emitting SDK. #} +otelcol.processor.probabilistic_sampler "traces" { sampling_percentage = {{ grafana_alloy_trace_sampling_ratio | float * 100 }} output { - traces = [otelcol.processor.attributes.decdn_identity.input] + traces = [otelcol.processor.resource.decdn_identity.input] } } From b7ce735e7e5c60e48c1521cb957b2c33ebb4ca02 Mon Sep 17 00:00:00 2001 From: Ant Somers Date: Wed, 16 Sep 2026 23:41:48 +0300 Subject: [PATCH 3/3] fix(grafana_alloy): make the rendered config one Alloy actually loads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Verified against the pinned Grafana Alloy v1.19.2 binary; every claim below is reproducible with `make lint-alloy`. Config (config.alloy.j2) — the file did not parse: - Alloy's syntax rejects '#' outright (illegal character U+0023). All comments are '//' now, and the Jinja '{#-' trim markers are gone: they were also gluing '}' onto the next block. - otelcol.processor.resource does not exist in Alloy (it is an upstream OTel collector name). Resource identity is now OTTL `set(attributes[...])` statements in the `resource` context of otelcol.processor.transform. - sending_queue / retry_on_failure are exporter blocks, not client blocks. Unit (alloy.service.j2) — the service crash-looped on start: - Alloy defines no --config.expand-env, so '${GC_*}' was a literal string, not an expansion. Credentials are read with sys.env("GC_…") instead, which still validates when unset — exactly what the deploy-time gate needs. - Alloy also defines no --server.grpc.listen-addr (that was Grafana Agent). Dropped, with grafana_alloy_server_grpc_addr and its preflight gates. Teardown scope — the disable path was destructive by default: - With the flag false (the default on every host) the role deleted /etc/alloy, /var/lib/alloy and the alloy account unconditionally, destroying an Alloy installed by the upstream apt repo or any other tool. Teardown is now gated on the managed-by marker this role's own unit template emits; anything else is reported and left alone. Path validation — a traversing path could delete /etc: - '^/var/lib/[A-Za-z0-9._/-]+$' accepts /var/lib/../../etc. Path gates moved to tasks/validate-paths.yml (imported by preflight AND the disable path, which never ran preflight), with per-segment charset, explicit '..' rejection, and coverage for config_dir / config_file / secret_file plus a containment check. Tests — the suite was green on a deployment that could not start: - Deploy gate: `alloy fmt --test` -> `alloy validate` (component graph, not just syntax). - New `make lint-alloy` + CI job `alloy-config`: renders the templates in three variable combinations and validates them with the REAL digest-pinned binary, plus asserts every ExecStart flag exists in `alloy run --help`. Reuses the role's own version+sha256 pin, so a bump without a re-pin fails there instead of on a host. - Molecule: teardown-scope play (rollback completeness + foreign-install survival), negative cases for path traversal and config containment, and assertions that the config keeps sys.env() and the unit keeps neither dead flag. Stub comments now state plainly what a green stub run does not prove. Also fixes an inverted check-mode condition in the install debug message. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/ci.yml | 30 ++++ .gitignore | 4 + AGENTS.md | 2 + CONTRIBUTING.md | 1 + Makefile | 12 +- README.md | 5 +- ansible/README.md | 11 ++ .../molecule/grafana-cloud/files/alloy-stub | 11 +- ansible/molecule/grafana-cloud/molecule.yml | 10 +- ansible/molecule/grafana-cloud/verify.yml | 148 ++++++++++++++++-- ansible/molecule/validation/converge.yml | 35 ++++- ansible/roles/grafana_alloy/README.md | 40 +++-- ansible/roles/grafana_alloy/defaults/main.yml | 8 +- ansible/roles/grafana_alloy/tasks/install.yml | 2 +- ansible/roles/grafana_alloy/tasks/main.yml | 134 ++++++++++------ .../roles/grafana_alloy/tasks/preflight.yml | 9 +- .../grafana_alloy/tasks/validate-paths.yml | 57 +++++++ .../grafana_alloy/templates/alloy.service.j2 | 8 +- .../grafana_alloy/templates/config.alloy.j2 | 104 +++++++----- ansible/roles/grafana_alloy/vars/main.yml | 10 ++ ansible/tests/alloy-config/render.yml | 121 ++++++++++++++ ansible/tests/alloy-config/validate.sh | 126 +++++++++++++++ 22 files changed, 754 insertions(+), 134 deletions(-) create mode 100644 ansible/roles/grafana_alloy/tasks/validate-paths.yml create mode 100644 ansible/roles/grafana_alloy/vars/main.yml create mode 100644 ansible/tests/alloy-config/render.yml create mode 100755 ansible/tests/alloy-config/validate.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c28f320..c323a25 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -188,6 +188,36 @@ jobs: # DECDN_CLI=... before bumping the decdn version. run: make lint-helm + # Render roles/grafana_alloy's templates and validate them with the REAL pinned + # Grafana Alloy binary. The molecule `grafana-cloud` scenario deliberately runs + # against a stub that exits 0 for every subcommand, so it proves plumbing but + # would happily ship a config Alloy cannot load (an unknown component, a block + # in the wrong parent, a CLI flag that does not exist). This job is that gate. + # Runs only when ansible/ changed. + alloy-config: + needs: changes + if: needs.changes.outputs.ansible == 'true' + runs-on: ubuntu-latest + timeout-minutes: 15 # ~1m in practice, most of it the first .deb fetch + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.12' + cache: pip + cache-dependency-path: .github/workflows/ci.yml + - name: Install Ansible + run: python -m pip install --upgrade ansible + # Keyed on the role defaults, which is where the version + digest pin + # lives: bumping grafana_alloy_version invalidates the cache by + # construction, so a bump is always validated against the new binary. + - uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 + with: + path: ansible/.cache/alloy + key: alloy-${{ runner.os }}-${{ hashFiles('ansible/roles/grafana_alloy/defaults/main.yml') }} + - name: Validate the rendered Alloy configuration + run: make lint-alloy + # Dedicated IaC security scan of the Ansible tree and the rendered Helm chart, driven straight from the # digest-pinned KICS *engine* image by `make security` — the exact command # developers run locally, so CI and local results cannot drift. KICS severities diff --git a/.gitignore b/.gitignore index 9a7507c..5d8888d 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,7 @@ ansible/importer_result.json # Python bytecode (e.g. from the molecule stub daemon or any local tooling) __pycache__/ *.pyc + +# Pinned third-party binaries downloaded by the test harnesses (`make lint-alloy` +# caches the verified Grafana Alloy release here so repeat runs skip the fetch). +ansible/.cache/ diff --git a/AGENTS.md b/AGENTS.md index addec7a..a3a30e0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -107,6 +107,8 @@ make molecule # containerised converge/verify of the decdn_node role — # scenarios in parallel (needs Docker); cap with JOBS= make molecule-serial # the same suite one scenario at a time (readable failure output) make lint-helm # chart: helm lint + render tests + kubeconform + schema keys (needs helm, yq, Docker) +make lint-alloy # grafana_alloy: render its templates + `alloy validate` them with the real + # pinned binary (the molecule stub exits 0 for everything and cannot) make security # = security-ansible + security-helm (KICS over the rendered chart; needs helm) # Ansible deploys — run from ansible/ (see ansible/README.md for the full flow) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 36cf6be..459912b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -24,6 +24,7 @@ tree), and `markdownlint`. | `make lint` | run all pre-commit hooks on every file (the full local hygiene gate) | | `make lint-ansible` | install Galaxy collections + run `ansible-lint` (its production profile includes the Ansible security rules) | | `make lint-helm` | Helm chart: `helm lint --strict`, positive/negative render tests, kubeconform (digest-pinned image), shared schema-key check and its fixtures (needs `helm`, `yq`, `python3` ≥ 3.11, Docker). Set `DECDN_CLI=` to also run the real `decdn config validate` (CI can't). | +| `make lint-alloy` | Renders `roles/grafana_alloy`'s templates and validates them with the **real** digest-pinned Grafana Alloy binary (`alloy validate` + an `ExecStart` flag check). The `grafana-cloud` molecule scenario uses a stub that exits 0 for every subcommand, so this is the only gate that proves the config loads. Set `ALLOY_BIN=` to skip the download. | | `make security` | KICS IaC security scan of `ansible/` and the rendered Helm chart (digest-pinned engine image — CI runs this same target) | `ansible-lint` is **not** a per-commit hook (it needs the collections installed). diff --git a/Makefile b/Makefile index c283631..a4d6ad6 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ # Convenience targets for the deCDN DevOps monorepo. # Run from the repo root. Ansible-specific work is delegated to ansible/Makefile. -.PHONY: help hooks lint lint-ansible lint-helm security security-ansible security-helm molecule molecule-serial galaxy-build galaxy-check +.PHONY: help hooks lint lint-ansible lint-helm lint-alloy security security-ansible security-helm molecule molecule-serial galaxy-build galaxy-check SHELL := /bin/bash # KICS runs straight from the engine image, pinned by digest. This target IS the @@ -59,6 +59,16 @@ security-helm: ## KICS scan of the decdn-node chart's rendered manifests ( lint-helm: ## helm lint + render tests + kubeconform + shared schema-key check (needs helm, yq, python3>=3.11, docker) KUBECONFORM="docker run --rm -i $(KUBECONFORM_IMAGE)" $(CHART)/tests/render-test.sh +# The molecule grafana-cloud scenario runs the role against a stub that exits 0 +# for every subcommand, so it can only prove plumbing. This target renders the +# grafana_alloy templates and feeds them to the REAL pinned Alloy binary +# (`alloy validate` + an ExecStart flag check) — the only thing that catches an +# unknown component, a misplaced block or a non-existent CLI flag before a host +# crash-loops. Downloads the role's pinned .deb once, then caches it under +# ansible/.cache (git-ignored); ALLOY_BIN= skips the download. +lint-alloy: ## validate grafana_alloy's rendered config against the real pinned Alloy binary + ansible/tests/alloy-config/validate.sh + molecule: ## containerised converge/verify of the decdn_node role, scenarios in parallel (needs Docker) $(MAKE) -C ansible molecule diff --git a/README.md b/README.md index b379c97..89d4118 100644 --- a/README.md +++ b/README.md @@ -104,6 +104,7 @@ make hooks # one-time: install the pre-commit git hook (pip install p make lint # all pre-commit hooks on all files (hygiene, shellcheck, yamllint, markdown) make lint-ansible # vendor collections + full ansible-lint (production profile) make lint-helm # chart: helm lint + render tests + kubeconform + schema keys +make lint-alloy # grafana_alloy: render its templates, validate with the real Alloy binary make security # KICS IaC scan of ansible/ + the rendered chart (pinned engine image) # Ansible deploys — run from ansible/ @@ -123,7 +124,9 @@ is not a per-commit hook (it needs collections vendored) — run `make lint-ansi - **`ci.yml`** — path-filtered so heavy jobs skip unrelated PRs: `ansible-lint` (production profile + playbook syntax-check), a `galaxy-build` readiness gate (builds the `decdn.node` - collection and runs galaxy-importer's checks), a `helm` job (`make lint-helm`: strict lint, + collection and runs galaxy-importer's checks), an `alloy-config` job (`make lint-alloy`: renders + the `grafana_alloy` templates and runs the real digest-pinned Alloy binary's `alloy validate` + over them — the molecule stub cannot), a `helm` job (`make lint-helm`: strict lint, positive/negative render tests, kubeconform, and the upstream schema-key check shared with molecule), **KICS** IaC scan of `ansible/` and the rendered chart (fail on HIGH), and `actionlint` on the workflows themselves. The KICS engine is pinned by digest and every diff --git a/ansible/README.md b/ansible/README.md index c8715c9..d39d512 100644 --- a/ansible/README.md +++ b/ansible/README.md @@ -220,6 +220,9 @@ ansible-playbook playbooks/site.yml --syntax-check make molecule # all seven molecule scenarios, in parallel (needs Docker) make molecule JOBS=2 # …capped to two at a time on a small machine make molecule-serial # …one at a time, when a failure needs readable output + +# from the repo ROOT — the only test that uses a real Grafana Alloy binary +make lint-alloy # render grafana_alloy's templates, then `alloy validate` them ``` `make molecule` runs every scenario under `molecule/`: **`default`** (described below), @@ -233,6 +236,14 @@ They are independent, so they run concurrently — because the runs interleave. `make molecule-serial` is the escape hatch when that interleaving gets in the way of reading a failure. +`grafana-cloud` runs against a *stub* Alloy that exits 0 for every subcommand, so it +proves the role's plumbing but cannot prove the rendered `config.alloy` is loadable. +That gap is closed by `make lint-alloy` (repo root; CI job `alloy-config`), which renders +the templates in several variable combinations and runs the **real**, digest-pinned Alloy +binary's `alloy validate` over them plus a check that every `ExecStart` flag actually +exists in `alloy run --help`. Re-run it when bumping `grafana_alloy_version`. See +[`tests/alloy-config/`](tests/alloy-config/). + The `default` scenario converges the **`decdn_node`** role in a privileged systemd container against a stub daemon: it installs via the `manual` method (no published release needed), stages a placeholder keystore, renders `node.toml` + the diff --git a/ansible/molecule/grafana-cloud/files/alloy-stub b/ansible/molecule/grafana-cloud/files/alloy-stub index 19dbd4e..58629ec 100755 --- a/ansible/molecule/grafana-cloud/files/alloy-stub +++ b/ansible/molecule/grafana-cloud/files/alloy-stub @@ -14,10 +14,13 @@ case "${1-}" in --version) echo "alloy, version v${MOLECULE_STUB_VERSION:-0.0.0}-molecule-stub" ;; - # fmt --test: the real command validates syntax + canonical formatting; the - # stub just exits 0 so the gate proves plumbing only — content correctness is - # asserted textually in verify.yml. - fmt) + # validate: the real binary builds the component graph and rejects an + # unloadable config. A stub CANNOT do that, so this exits 0 and proves the + # role's gate is wired up — nothing more. Content correctness is asserted + # textually in verify.yml and, against the REAL pinned binary, by + # ../../tests/alloy-config/validate.sh (`make lint-alloy`, run in CI). Do not + # read a green run here as "the configuration loads". + validate | fmt) : ;; # Long-running foreground process keeps the Type=simple unit 'running'. diff --git a/ansible/molecule/grafana-cloud/molecule.yml b/ansible/molecule/grafana-cloud/molecule.yml index 8104164..50da26b 100644 --- a/ansible/molecule/grafana-cloud/molecule.yml +++ b/ansible/molecule/grafana-cloud/molecule.yml @@ -8,8 +8,14 @@ # # Known boundary (why no runtime socket assertions): the stub does NOT bind # 4317/4318/12345, so `ss -ltn` proofs are meaningless here. Loopback-binding is -# asserted textually against the rendered config + unit, matches what real-binary -# deploys enforce via `alloy fmt --test` at deploy time. +# asserted textually against the rendered config + unit, matching what +# real-binary deploys enforce via `alloy validate` at deploy time. +# +# Second boundary (READ THIS BEFORE TRUSTING A GREEN RUN): the stub exits 0 for +# every subcommand, so the role's `alloy validate` gate proves only that the gate +# is wired up. Whether Alloy can actually LOAD the rendered config is proven by +# ../../tests/alloy-config/validate.sh (`make lint-alloy`), which renders the same +# templates against the real pinned binary and runs in CI. # # `baseline` is not exercised (real-host-only — see ../default/molecule.yml). driver: diff --git a/ansible/molecule/grafana-cloud/verify.yml b/ansible/molecule/grafana-cloud/verify.yml index aa82040..3b21cb3 100644 --- a/ansible/molecule/grafana-cloud/verify.yml +++ b/ansible/molecule/grafana-cloud/verify.yml @@ -58,7 +58,9 @@ ansible.builtin.assert: that: # ratio 0.25 -> percentage 25; batch size overrides pass through; labels - # dotted AND set-variable ones present; secrets remain ${...} placeholders. + # dotted AND set-variable ones present; credentials stay sys.env() + # indirections (Alloy has no ${VAR} expansion — a literal ${GC_*} here + # would ship a broken URL to the cloud endpoint). - >- 'sampling_percentage = 25.0' in ga_config - >- @@ -76,14 +78,24 @@ - >- '"decdn-node"' in ga_config - >- - '${GC_PROM_REMOTE_WRITE_URL}' in ga_config + 'sys.env("GC_PROM_REMOTE_WRITE_URL")' in ga_config - >- - '${GC_OTLP_ENDPOINT}' in ga_config + 'sys.env("GC_OTLP_ENDPOINT")' in ga_config - >- - '${GC_API_TOKEN}' in ga_config + 'sys.env("GC_API_TOKEN")' in ga_config + - >- + '${GC_' not in ga_config + # Alloy's syntax rejects '#' outright, and the template's own comments + # plus everything the Jinja layer emits must respect that. + - >- + ga_config.splitlines() | select('match', '\\s*#') | list | length == 0 + # Components that do not exist in Alloy (they are OTel collector names, + # not Alloy ones) — a rename upstream must not sneak back in. + - >- + 'otelcol.processor.resource ' not in ga_config fail_msg: >- - config.alloy is missing expected content (samplers, labels, ${GC_*} - expansion). Got: + config.alloy is missing expected content (samplers, labels, sys.env() + credential indirection) or carries syntax Alloy rejects. Got: {{ ga_config }} # --- Secret hygiene ------------------------------------------------------------- @@ -126,7 +138,7 @@ - ga_leak_probe.results | selectattr('rc', 'equalto', 0) | list | length == 0 fail_msg: >- The GC_API_TOKEN literal appears outside the 0600 env file — config or - templates stopped using ${...} expansion. + templates stopped using sys.env() indirection. # --- Node-side wiring (the mirrored flag) ---------------------------------------- - name: Stage the otlp_endpoint assertion script @@ -158,13 +170,19 @@ src: /etc/systemd/system/alloy.service register: ga_unit_b64 - - name: Assert the unit carries the environment-expansion + hardening directives + - name: Assert the unit carries the credential + hardening directives ansible.builtin.assert: that: - >- 'EnvironmentFile=/etc/decdn/grafana-alloy.env' in ga_unit + # Alloy defines no --config.expand-env and no gRPC admin listener; + # either flag makes `alloy run` exit non-zero, i.e. a crash-loop. + # Checked against the ExecStart flag lines ONLY — the unit's comments + # name both flags precisely to explain why they must not be passed. - >- - '--config.expand-env' in ga_unit + '--config.expand-env' not in ga_unit_flags + - >- + '--server.grpc.listen-addr' not in ga_unit_flags - >- 'User=alloy' in ga_unit - >- @@ -173,14 +191,15 @@ 'ProtectSystem=strict' in ga_unit - >- 'SystemCallFilter=@system-service' in ga_unit - # Alloy's own admin server defaults to all interfaces; the role pins it. + # Alloy's admin/UI server is pinned explicitly rather than left to the + # upstream default. - >- '--server.http.listen-addr=127.0.0.1:' in ga_unit - - >- - '--server.grpc.listen-addr=127.0.0.1:' in ga_unit fail_msg: hardened Alloy unit is missing an expected directive vars: ga_unit: "{{ ga_unit_b64.content | b64decode }}" + ga_unit_flags: >- + {{ ga_unit.splitlines() | select('match', '\\s*--') | join(' ') }} - name: Gather service facts ansible.builtin.service_facts: @@ -195,3 +214,108 @@ 'decdn-node.service' in ansible_facts.services and ansible_facts.services['decdn-node.service'].state == 'running' fail_msg: "alloy or decdn-node is not running — journalctl -u alloy -e / -u decdn-node -e" + +# Runs LAST, after the idempotence phase and the assertions above, because it +# deliberately mutates the converged host: it exercises the disable path, whose +# whole job is to remove things. +- name: Verify the disable path removes this role's install and nothing else + hosts: all + become: true + tasks: + # --- Part 1: flipping the flag off is a complete rollback ------------------- + - name: Run the role with observability disabled + ansible.builtin.include_role: + name: grafana_alloy + vars: + decdn_grafana_cloud_enabled: false + + - name: Stat what the teardown must have removed + ansible.builtin.stat: + path: "{{ item }}" + register: ga_removed + loop: + - /etc/systemd/system/alloy.service + - /etc/alloy + - /var/lib/alloy + + - name: Read the account database back + ansible.builtin.getent: + database: passwd + + - name: Assert the role removed its own installation + ansible.builtin.assert: + that: + - ga_removed.results | selectattr('stat.exists') | list | length == 0 + - "'alloy' not in ansible_facts.getent_passwd" + fail_msg: >- + Disabling decdn_grafana_cloud_enabled left this role's artifacts behind + ({{ ga_removed.results | selectattr('stat.exists') + | map(attribute='item') | list }}). + + # --- Part 2: it must NOT touch an Alloy installed by anything else ---------- + # THE REGRESSION THIS GUARDS: the teardown used to delete /etc/alloy, + # /var/lib/alloy and the alloy account unconditionally whenever the flag was + # false — which is the DEFAULT — so every converge of an unrelated host would + # destroy an Alloy installed from the upstream apt repo or another role. + - name: Stage the foreign installation's directories + ansible.builtin.file: + path: "{{ item }}" + state: directory + mode: "0755" + loop: + - /etc/alloy + - /var/lib/alloy + + - name: Stage a foreign Alloy installation (no managed-by marker) + ansible.builtin.copy: + dest: "{{ item.path }}" + content: "{{ item.content }}" + mode: "0644" + loop: + - path: /etc/systemd/system/alloy.service + content: | + [Unit] + Description=Somebody else's Alloy + [Service] + ExecStart=/bin/true + - path: /etc/alloy/config.alloy + content: "// installed by someone else\n" + loop_control: + label: "{{ item.path }}" + + - name: Stage the foreign install's account + ansible.builtin.user: + name: alloy + system: true + create_home: false + shell: /usr/sbin/nologin + + - name: Run the role with observability disabled again + ansible.builtin.include_role: + name: grafana_alloy + vars: + decdn_grafana_cloud_enabled: false + + - name: Stat what the teardown must have left alone + ansible.builtin.stat: + path: "{{ item }}" + register: ga_foreign + loop: + - /etc/systemd/system/alloy.service + - /etc/alloy/config.alloy + - /var/lib/alloy + + - name: Re-read the account database + ansible.builtin.getent: + database: passwd + + - name: Assert the foreign installation survived untouched + ansible.builtin.assert: + that: + - ga_foreign.results | rejectattr('stat.exists') | list | length == 0 + - "'alloy' in ansible_facts.getent_passwd" + fail_msg: >- + The disable path destroyed an Alloy installation this role never made + ({{ ga_foreign.results | rejectattr('stat.exists') + | map(attribute='item') | list }}). Teardown must be gated on the + managed-by marker in the unit file. diff --git a/ansible/molecule/validation/converge.yml b/ansible/molecule/validation/converge.yml index 8a8a2d8..629fd22 100644 --- a/ansible/molecule/validation/converge.yml +++ b/ansible/molecule/validation/converge.yml @@ -533,7 +533,7 @@ # --- Cases: grafana_alloy preflight gates (#55) -------------------------------- # The opt-in Grafana Cloud role's OWN fail-loud gates get the same treatment. - # All five broken configs abort inside preflight-imported tasks BEFORE any + # All seven broken configs abort inside preflight-imported tasks BEFORE any # host mutation (user/package/config), so they cannot interfere with the other # cases or start services. Tag deduping at the bottom handles two cases per # gate. Fixtures are staged/restored per case; none of this touches decdn.env. @@ -668,6 +668,37 @@ decdn_rejected: "{{ decdn_rejected + ['pipeline-knobs'] }}" when: ansible_failed_task.name is match('^Validate the pipeline knobs') + - name: "Case ga-path-traversal — state directory escaping /var/lib" + block: + - name: Run grafana_alloy with a traversing state directory + ansible.builtin.include_role: + name: grafana_alloy + vars: + decdn_grafana_cloud_enabled: true + # Passes a naive "^/var/lib/[A-Za-z0-9._/-]+$" prefix check, but + # resolves to /etc — which the disable path would delete outright. + grafana_alloy_state_dir: /var/lib/../../etc + rescue: + - name: Record ga-path-traversal rejection (only if the path gate failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['managed-paths'] }}" + when: ansible_failed_task.name is match('^Constrain the managed Alloy paths') + + - name: "Case ga-config-outside-dir — rendered config outside the managed directory" + block: + - name: Run grafana_alloy with a config file outside its config directory + ansible.builtin.include_role: + name: grafana_alloy + vars: + decdn_grafana_cloud_enabled: true + grafana_alloy_config_dir: /etc/alloy + grafana_alloy_config_file: /etc/somewhere-else/config.alloy + rescue: + - name: Record ga-config-outside-dir rejection (only if the containment gate failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['managed-paths'] }}" + when: ansible_failed_task.name is match('^Require the rendered config to live inside') + - name: Confirm every bad value was rejected by the role's own validation ansible.builtin.assert: that: @@ -686,4 +717,4 @@ "enum", "dict", "float", "list", "string", "string-newline", "otlp-https", "otlp-noport", "otlp-path", "otlp-userinfo", "otlp-colon", "otlp-port", "otlp-newline", "env-gate", "env-extra", "env-content", "config-gate", - "secret-file", "secret-content", "pipeline-knobs"] + "secret-file", "secret-content", "pipeline-knobs", "managed-paths"] diff --git a/ansible/roles/grafana_alloy/README.md b/ansible/roles/grafana_alloy/README.md index 56a9301..9d718d4 100644 --- a/ansible/roles/grafana_alloy/README.md +++ b/ansible/roles/grafana_alloy/README.md @@ -33,13 +33,18 @@ sudo install -m 600 -o root -g root grafana-alloy.env /etc/decdn/grafana-alloy.e using `roles/grafana_alloy/files/grafana-alloy.env.example` as the template. The four required keys are `GC_PROM_REMOTE_WRITE_URL`, `GC_OTLP_ENDPOINT`, -`GC_PROM_USERNAME` and `GC_API_TOKEN`; both URLs must be `https://`. Values are -expanded into `config.alloy` at load time via `--config.expand-env` — the file on -disk stays free of secret literals, and the role only ever greps shapes on the -host rather than reading values back. - -With the flag off, the role removes any prior unit + rendered config it manages -(disable/rollback); binary/package removal is manual. +`GC_PROM_USERNAME` and `GC_API_TOKEN`; both URLs must be `https://`. +`config.alloy` reads each of them at load time with `sys.env("GC_…")` — Alloy has +**no** `--config.expand-env` flag, so a `${GC_…}` placeholder would be a literal +string, not an expansion. The file on disk stays free of secret literals, and the +role only ever greps shapes on the host rather than reading values back. + +With the flag off, the role removes the unit, rendered config, state directory +and service account **it installed** — identified by the managed-by marker in +`/etc/systemd/system/alloy.service`. An Alloy installed by anything else (the +upstream apt repo, another role, by hand) carries no marker and is left strictly +alone, which matters because the flag is `false` by default on every host. +Binary/package removal is manual. ## Variables @@ -75,6 +80,21 @@ If you move either default, update BOTH sides and the molecule guard in ## Testing -Covered by the `grafana-cloud` molecule scenario (positive path with a stub -binary + negative preflight cases) and disabled-path assertions in the `default` -scenario. Real-binary validation happens at deploy time via `alloy fmt --test`. +Three layers, because the first one cannot prove correctness on its own: + +1. **`grafana-cloud` molecule scenario** — the role end-to-end against a *stub* + binary (plumbing, rendered content, hardened unit, teardown scope), plus + negative preflight cases in `validation` and disabled-path assertions in + `default`. The stub exits 0 for every subcommand, so a green run says nothing + about whether Alloy can load the config. +2. **`make lint-alloy`** (CI job `alloy-config`) — renders these templates in + several variable combinations and feeds them to the REAL pinned Alloy binary: + `alloy validate` (component graph, not just syntax) and a check that every + `ExecStart` flag exists in `alloy run --help`. See + `ansible/tests/alloy-config/`. +3. **Deploy time** — the role runs `alloy validate` against the just-rendered + file with the installed binary before any restart. + +Re-run `make lint-alloy` whenever `grafana_alloy_version` is bumped: the harness +downloads the same digest-pinned `.deb` the role installs, so a version bumped +without its sha256 fails there rather than on a host. diff --git a/ansible/roles/grafana_alloy/defaults/main.yml b/ansible/roles/grafana_alloy/defaults/main.yml index c77136a..2d8fd71 100644 --- a/ansible/roles/grafana_alloy/defaults/main.yml +++ b/ansible/roles/grafana_alloy/defaults/main.yml @@ -43,10 +43,12 @@ grafana_alloy_otlp_grpc_bind: 127.0.0.1 grafana_alloy_otlp_grpc_port: 4317 # standard OTLP/gRPC port; decdn_node's grafana_alloy_otlp_http_bind: 127.0.0.1 # injected otlp_endpoint depends on the grafana_alloy_otlp_http_port: 4318 # gRPC value staying put (see node.toml.j2) -# Alloy's own admin/UI server defaults to :12345/:12347 on ALL interfaces — pin it -# to loopback explicitly via --server.*.listen-addr in the unit. +# Alloy's own admin/UI server. Upstream already defaults it to 127.0.0.1:12345, +# but the unit passes it explicitly so the binding is reviewable in the unit and +# a future upstream default change cannot quietly widen it. There is NO gRPC +# admin listener in Alloy (that was Grafana Agent) — `alloy run` has no +# --server.grpc.listen-addr flag, and passing one is an immediate crash-loop. grafana_alloy_server_http_addr: 127.0.0.1:12345 -grafana_alloy_server_grpc_addr: 127.0.0.1:12347 # --- Metrics scrape (node /metrics -> Grafana Cloud Prometheus) --------------- grafana_alloy_scrape_interval: 30s diff --git a/ansible/roles/grafana_alloy/tasks/install.yml b/ansible/roles/grafana_alloy/tasks/install.yml index 8224ca0..3745005 100644 --- a/ansible/roles/grafana_alloy/tasks/install.yml +++ b/ansible/roles/grafana_alloy/tasks/install.yml @@ -27,7 +27,7 @@ install step; configuration templating below still diffs normally. when: - grafana_alloy_install_method == "release" - - not ansible_check_mode + - ansible_check_mode - name: Query the currently-installed package version ansible.builtin.command: diff --git a/ansible/roles/grafana_alloy/tasks/main.yml b/ansible/roles/grafana_alloy/tasks/main.yml index c903072..f6751f6 100644 --- a/ansible/roles/grafana_alloy/tasks/main.yml +++ b/ansible/roles/grafana_alloy/tasks/main.yml @@ -17,47 +17,84 @@ string — it gates this entire role and the otlp_endpoint injection in roles/decdn_node/templates/node.toml.j2. -# --- Disabled path: stop + uninstall remnants so flipping the flag off is a ---- -# --- complete rollback (binary/package removal stays a manual step) ----------- -- name: Tear down any prior installation (flag off) +# --- Disabled path: stop + uninstall ONLY what this role installed, so flipping - +# --- the flag off is a complete rollback and a no-op for anything else -------- +- name: Tear down a prior installation by this role (flag off) when: not decdn_grafana_cloud_enabled | bool block: + # Refuse to aim `state: absent` at an un-validated path even here — the + # disable path never runs preflight, and these variables are inventory- + # overridable (see validate-paths.yml). + - name: Validate the managed filesystem paths before removing them + ansible.builtin.import_tasks: validate-paths.yml + - name: Check for a leftover Alloy systemd unit ansible.builtin.stat: - path: /etc/systemd/system/alloy.service + path: "{{ _ga_unit_path }}" register: _ga_leftover_unit - - name: Stop and disable a leftover Alloy service - ansible.builtin.systemd: - name: alloy - state: stopped - enabled: false + # SCOPE GUARD: an Alloy installed by some OTHER mechanism (upstream apt repo, + # a different role, a hand-rolled unit) owns the very same conventional paths + # — /etc/alloy, /var/lib/alloy, the alloy user. Deleting those because a flag + # this repo defaults to FALSE is false would destroy an unrelated install on + # every single converge. So teardown is gated on the marker line this role's + # own unit template emits: no marker, no removal. + - name: Read the leftover unit to confirm this role wrote it + ansible.builtin.slurp: + src: "{{ _ga_unit_path }}" + register: _ga_leftover_unit_body when: _ga_leftover_unit.stat.exists - failed_when: false # best-effort teardown of an already-broken install - - - name: Remove the leftover unit and rendered configuration - ansible.builtin.file: - path: "{{ item }}" - state: absent - loop: - - /etc/systemd/system/alloy.service - - "{{ grafana_alloy_config_dir }}" - - "{{ grafana_alloy_state_dir }}" - notify: Reload systemd - - - name: Apply pending teardown notifications - ansible.builtin.meta: flush_handlers - - name: Remove the leftover Alloy account - ansible.builtin.user: - name: "{{ grafana_alloy_user }}" - state: absent - remove: true + - name: Decide whether the leftover install is ours to remove + ansible.builtin.set_fact: + _ga_ours: "{{ _ga_leftover_unit.stat.exists + and _ga_managed_marker in (_ga_leftover_unit_body.content | b64decode) }}" - - name: Remove the leftover Alloy group - ansible.builtin.group: - name: "{{ grafana_alloy_group }}" - state: absent + - name: Report that an unmanaged Alloy installation was left untouched + ansible.builtin.debug: + msg: >- + {{ _ga_unit_path }} exists but carries no "{{ _ga_managed_marker }}" + marker, so it was NOT installed by this role. Leaving it, its + configuration, its state directory and the + {{ grafana_alloy_user }} account alone — remove that installation with + whatever tool created it. + when: + - _ga_leftover_unit.stat.exists + - not _ga_ours | bool + + - name: Remove this role's installation + when: _ga_ours | bool + block: + - name: Stop and disable the Alloy service + ansible.builtin.systemd: + name: alloy + state: stopped + enabled: false + failed_when: false # best-effort teardown of an already-broken install + + - name: Remove the unit and rendered configuration + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: + - "{{ _ga_unit_path }}" + - "{{ grafana_alloy_config_dir }}" + - "{{ grafana_alloy_state_dir }}" + notify: Reload systemd + + - name: Apply pending teardown notifications + ansible.builtin.meta: flush_handlers + + - name: Remove the Alloy account + ansible.builtin.user: + name: "{{ grafana_alloy_user }}" + state: absent + remove: true + + - name: Remove the Alloy group + ansible.builtin.group: + name: "{{ grafana_alloy_group }}" + state: absent # Everything below runs only when observability is explicitly enabled. - name: Enable and configure the Grafana Cloud agent @@ -116,13 +153,13 @@ dest: "{{ grafana_alloy_config_file }}" owner: root group: "{{ grafana_alloy_group }}" - mode: "0644" # non-secret by design: secrets live in ${GC_*} expansion + mode: "0644" # non-secret by design: credentials are read via sys.env() notify: Restart alloy - name: Install the hardened systemd unit ansible.builtin.template: src: alloy.service.j2 - dest: /etc/systemd/system/alloy.service + dest: "{{ _ga_unit_path }}" owner: root group: root mode: "0644" @@ -131,30 +168,33 @@ - Restart alloy # --- Config validation gate (fail loud before any restart) ------------------ - # `alloy fmt --test` validates syntax AND canonical formatting (per upstream - # docs it fails on syntactically incorrect files). It cannot resolve component - # references — but a wrong ${GC_*} placeholder ONLY fails at runtime anyway, - # which the grafana-cloud molecule scenario covers textually. Under check mode - # the file has not been written yet, so skip like the decdn_node config gate. + # `alloy validate` (NOT `fmt --test`): fmt only parses and re-prints, so it + # accepts a syntactically valid file that names a component which does not + # exist or nests a block in the wrong place — exactly the class of regression + # that turns into a crash-loop. `validate` builds the component graph and + # rejects both. It does not need the credentials: `sys.env` on an unset + # variable yields "" and still validates, so no secret is required here. + # Under check mode the file has not been written yet, so skip like the + # decdn_node config gate. - name: Check the rendered configuration with the installed binary ansible.builtin.command: - cmd: "{{ grafana_alloy_bin_effective }} fmt --test {{ grafana_alloy_config_file }}" + cmd: "{{ grafana_alloy_bin_effective }} validate {{ grafana_alloy_config_file }}" changed_when: false - register: _ga_fmt_check + register: _ga_config_check failed_when: false when: not ansible_check_mode - name: Report the configuration-validation failure ansible.builtin.fail: msg: >- - `alloy fmt --test` rejected {{ grafana_alloy_config_file }} - (rc={{ _ga_fmt_check.rc | default('?') }}): - {{ (_ga_fmt_check.stderr | default('', true)) - or (_ga_fmt_check.stdout | default('', true)) }} — fix the template - regression before deploying. + `alloy validate` rejected {{ grafana_alloy_config_file }} + (rc={{ _ga_config_check.rc | default('?') }}): + {{ (_ga_config_check.stderr | default('', true)) + or (_ga_config_check.stdout | default('', true)) }} — fix the + template regression before deploying. when: - not ansible_check_mode - - _ga_fmt_check.rc | default(1) != 0 + - _ga_config_check.rc | default(1) != 0 - name: Apply pending Alloy changes ansible.builtin.meta: flush_handlers diff --git a/ansible/roles/grafana_alloy/tasks/preflight.yml b/ansible/roles/grafana_alloy/tasks/preflight.yml index 24cd6d0..4d6c661 100644 --- a/ansible/roles/grafana_alloy/tasks/preflight.yml +++ b/ansible/roles/grafana_alloy/tasks/preflight.yml @@ -93,7 +93,6 @@ - grafana_alloy_otlp_grpc_bind == '127.0.0.1' - grafana_alloy_otlp_http_bind == '127.0.0.1' - grafana_alloy_server_http_addr is match('^127[.]0[.]0[.]1:[0-9]+$') - - grafana_alloy_server_grpc_addr is match('^127[.]0[.]0[.]1:[0-9]+$') fail_msg: >- Grafana Alloy OTLP and admin listeners must bind to 127.0.0.1 only; wildcard or public listener addresses are not permitted. @@ -110,13 +109,9 @@ - "{{ grafana_alloy_otlp_grpc_port }}" - "{{ grafana_alloy_otlp_http_port }}" - "{{ grafana_alloy_server_http_addr.split(':')[-1] }}" - - "{{ grafana_alloy_server_grpc_addr.split(':')[-1] }}" -- name: Constrain the Alloy state directory to systemd's writable area - ansible.builtin.assert: - that: - - grafana_alloy_state_dir is match('^/var/lib/[A-Za-z0-9._/-]+$') - fail_msg: "grafana_alloy_state_dir must be a path below /var/lib." +- name: Validate the managed filesystem paths + ansible.builtin.import_tasks: validate-paths.yml # COUPLING GUARD: the node-side otlp_endpoint injected by # roles/decdn_node/templates/node.toml.j2 hardcodes the gRPC receiver port diff --git a/ansible/roles/grafana_alloy/tasks/validate-paths.yml b/ansible/roles/grafana_alloy/tasks/validate-paths.yml new file mode 100644 index 0000000..8935c9b --- /dev/null +++ b/ansible/roles/grafana_alloy/tasks/validate-paths.yml @@ -0,0 +1,57 @@ +--- +# Path-shape gates for every filesystem location this role manages. +# +# Imported by BOTH preflight.yml (enable path) and the teardown block in +# main.yml (disable path). The disable path deletes these directories outright, +# so it must not trust an un-validated variable either — that is the whole point +# of factoring them out here instead of leaving them inside preflight. + +- name: Constrain the managed Alloy paths (no traversal, no symlink-bait segments) + ansible.builtin.assert: + that: + # Anchored prefix AND a per-segment charset. A prefix match alone is not + # enough: "/var/lib/../../etc" satisfies "^/var/lib/[A-Za-z0-9._/-]+$" + # because '.' and '/' are in the class, which would let a hostile/typo'd + # inventory aim the disable-path `file: state=absent` at /etc. + - item.value is match(item.regex) + - "'..' not in item.value.split('/')" + - "'' not in item.value.split('/')[1:]" + quiet: true + fail_msg: >- + {{ item.name }} must be an absolute, traversal-free path under + {{ item.under }} whose segments use only [A-Za-z0-9._-] (got + "{{ item.value }}"). These paths are removed wholesale when + decdn_grafana_cloud_enabled is turned off, so '..' segments are refused + outright. + loop: + # Must stay under /var/lib: systemd's StateDirectory= is relative to it. + - name: grafana_alloy_state_dir + value: "{{ grafana_alloy_state_dir }}" + under: /var/lib + regex: '^/var/lib/[A-Za-z0-9._-]+(/[A-Za-z0-9._-]+)*$' + - name: grafana_alloy_config_dir + value: "{{ grafana_alloy_config_dir }}" + under: /etc + regex: '^/etc/[A-Za-z0-9._-]+(/[A-Za-z0-9._-]+)*$' + - name: grafana_alloy_config_file + value: "{{ grafana_alloy_config_file }}" + under: /etc + regex: '^/etc/[A-Za-z0-9._-]+(/[A-Za-z0-9._-]+)*$' + - name: grafana_alloy_secret_file + value: "{{ grafana_alloy_secret_file }}" + under: /etc + regex: '^/etc/[A-Za-z0-9._-]+(/[A-Za-z0-9._-]+)*$' + loop_control: + label: "{{ item.name }}" + +# The role templates the config INTO the directory it creates and chowns; a file +# outside it would be written to an unmanaged (possibly unsafe) location and +# would survive the disable path's directory removal. +- name: Require the rendered config to live inside the managed config directory + ansible.builtin.assert: + that: + - grafana_alloy_config_file.startswith(grafana_alloy_config_dir ~ '/') + - (grafana_alloy_config_file | dirname) == grafana_alloy_config_dir + fail_msg: >- + grafana_alloy_config_file ({{ grafana_alloy_config_file }}) must be a file + directly inside grafana_alloy_config_dir ({{ grafana_alloy_config_dir }}). diff --git a/ansible/roles/grafana_alloy/templates/alloy.service.j2 b/ansible/roles/grafana_alloy/templates/alloy.service.j2 index e198c6d..6aabc34 100644 --- a/ansible/roles/grafana_alloy/templates/alloy.service.j2 +++ b/ansible/roles/grafana_alloy/templates/alloy.service.j2 @@ -17,14 +17,14 @@ User={{ grafana_alloy_user }} Group={{ grafana_alloy_group }} # Sensitive: GC_PROM_REMOTE_WRITE_URL / GC_PROM_USERNAME / GC_API_TOKEN / -# GC_OTLP_ENDPOINT. Root-owned 0600; expanded into ${GC_*} placeholders inside -# config.alloy via --config.expand-env below. +# GC_OTLP_ENDPOINT. Root-owned 0600, read by systemd as root and handed to the +# process environment, where config.alloy picks each value up via sys.env(). +# (Alloy has NO --config.expand-env flag — passing one is an immediate +# crash-loop; `sys.env` is the supported indirection.) EnvironmentFile={{ grafana_alloy_secret_file }} ExecStart={{ grafana_alloy_bin_effective }} run \ - --config.expand-env \ --server.http.listen-addr={{ grafana_alloy_server_http_addr }} \ - --server.grpc.listen-addr={{ grafana_alloy_server_grpc_addr }} \ --storage.path={{ grafana_alloy_state_dir }}/data \ {{ grafana_alloy_config_file }} diff --git a/ansible/roles/grafana_alloy/templates/config.alloy.j2 b/ansible/roles/grafana_alloy/templates/config.alloy.j2 index ff42cd8..561458f 100644 --- a/ansible/roles/grafana_alloy/templates/config.alloy.j2 +++ b/ansible/roles/grafana_alloy/templates/config.alloy.j2 @@ -1,14 +1,20 @@ -# MANAGED BY the grafana_alloy Ansible role — do not edit by hand. -# -# Pipeline: scrape the node's /metrics and push it to Grafana Cloud Prometheus; -# receive OTLP traces (plus any logs/metrics pushed over localhost) on gRPC 4317 / -# HTTP 4318 bound to loopback only; stamp resource identity on everything, -# probabilistically sample traces, batch, then export via OTLP/HTTP. -# -# Secrets are NOT literals here: every ${GC_*} placeholder expands at load time -# from the root-owned 0600 environment file systemd hands the unit -# (--config.expand-env in alloy.service; operator template: -# roles/grafana_alloy/files/grafana-alloy.env.example). +// MANAGED BY the grafana_alloy Ansible role — do not edit by hand. +// +// Alloy's configuration syntax only understands `//` and `/* */` comments — a +// `#` is an illegal character that aborts the parse, so every comment in this +// file (and everything the Jinja layer emits) must stay `//`. +// +// Pipeline: scrape the node's /metrics and push it to Grafana Cloud Prometheus; +// receive OTLP traces (plus any logs/metrics pushed over localhost) on gRPC 4317 / +// HTTP 4318 bound to loopback only; stamp resource identity on everything, +// probabilistically sample traces, batch, then export via OTLP/HTTP. +// +// Secrets are NOT literals here: every credential is read at config-load time +// with the `sys.env` stdlib function from the root-owned 0600 environment file +// systemd hands the unit (EnvironmentFile= in alloy.service; operator template: +// roles/grafana_alloy/files/grafana-alloy.env.example). Alloy has no +// `--config.expand-env` flag — `${VAR}` inside a string is a literal, so +// `sys.env("VAR")` is the only working indirection. prometheus.scrape "decdn_node" { targets = [ @@ -19,9 +25,9 @@ prometheus.scrape "decdn_node" { scrape_interval = {{ grafana_alloy_scrape_interval | to_json }} } -{#- Low-cardinality identity labels shipped with every metric. Prom label NAMES - may not contain '.', so the dotted service.* naming flattens to underscores; - empty values omit their rule entirely. #} +{# Low-cardinality identity labels shipped with every metric. Prom label NAMES + may not contain '.', so the dotted service.* naming flattens to underscores; + empty values omit their rule entirely. #} {% set ga_metric_labels = [ ["service_name", grafana_alloy_service_name], ["service_namespace", grafana_alloy_service_namespace], @@ -42,11 +48,11 @@ prometheus.relabel "decdn_identity" { prometheus.remote_write "cloud" { endpoint { - url = "${GC_PROM_REMOTE_WRITE_URL}" + url = sys.env("GC_PROM_REMOTE_WRITE_URL") basic_auth { - username = "${GC_PROM_USERNAME}" - password = "${GC_API_TOKEN}" + username = sys.env("GC_PROM_USERNAME") + password = sys.env("GC_API_TOKEN") } } } @@ -62,14 +68,19 @@ otelcol.receiver.otlp "local" { output { traces = [otelcol.processor.probabilistic_sampler.traces.input] - logs = [otelcol.processor.resource.decdn_identity.input] - metrics = [otelcol.processor.resource.decdn_identity.input] + logs = [otelcol.processor.transform.decdn_identity.input] + metrics = [otelcol.processor.transform.decdn_identity.input] } } -{#- Identity as OTLP resource attributes; unlike Prom labels these MAY use dotted - names. Sorted keys keep rendering byte-stable across converges (idempotence) - and empty values drop out of the action list entirely. #} +{# Identity as OTLP resource attributes; unlike Prom labels these MAY use dotted + names. Alloy ships no `otelcol.processor.resource` component, so the upstream + collector's resource processor is expressed as OTTL `set()` statements in the + `resource` context of otelcol.processor.transform (the supported equivalent). + Sorted keys keep rendering byte-stable across converges (idempotence) and + empty values drop out of the statement list entirely. Values are constrained + to [A-Za-z0-9._/-] by preflight, so they cannot break out of the OTTL string + literal or the enclosing backtick-quoted Alloy string. #} {% set ga_res_attrs = { "deployment.environment": grafana_alloy_deployment_environment, "region": grafana_alloy_region, @@ -77,12 +88,23 @@ otelcol.receiver.otlp "local" { "service.name": grafana_alloy_service_name, "service.namespace": grafana_alloy_service_namespace, } %} -otelcol.processor.resource "decdn_identity" { +{% set ga_res_statements = [] %} {% for attr_key in ga_res_attrs.keys() | sort if ga_res_attrs[attr_key] | length > 0 %} - action { - key = {{ attr_key | tojson }} - action = "upsert" - value = {{ ga_res_attrs[attr_key] | tojson }} +{% set _ = ga_res_statements.append( + 'set(attributes[' ~ (attr_key | tojson) ~ '], ' ~ (ga_res_attrs[attr_key] | tojson) ~ ')') %} +{% endfor %} +otelcol.processor.transform "decdn_identity" { + // Fail loud rather than silently shipping unlabelled telemetry. + error_mode = "propagate" +{% for signal in ["trace", "metric", "log"] %} + + {{ signal }}_statements { + context = "resource" + statements = [ +{% for statement in ga_res_statements %} + `{{ statement }}`, +{% endfor %} + ] } {% endfor %} @@ -93,13 +115,13 @@ otelcol.processor.resource "decdn_identity" { } } -{#- Renders the operator-facing 0..1 keep-ratio as the component's percentage. - Parent-based decisions must be made by the emitting SDK. #} +{# Renders the operator-facing 0..1 keep-ratio as the component's percentage. + Parent-based decisions must be made by the emitting SDK. #} otelcol.processor.probabilistic_sampler "traces" { sampling_percentage = {{ grafana_alloy_trace_sampling_ratio | float * 100 }} output { - traces = [otelcol.processor.resource.decdn_identity.input] + traces = [otelcol.processor.transform.decdn_identity.input] } } @@ -115,23 +137,25 @@ otelcol.processor.batch "decdn" { } } +{# `sending_queue` and `retry_on_failure` are blocks of the EXPORTER, siblings of + `client` — nesting them inside `client` is rejected at config load. #} otelcol.exporter.otlphttp "cloud" { client { - endpoint = "${GC_OTLP_ENDPOINT}" + endpoint = sys.env("GC_OTLP_ENDPOINT") auth = otelcol.auth.basic.cloud.handler + } - sending_queue { - queue_size = {{ grafana_alloy_queue_size }} - } + sending_queue { + queue_size = {{ grafana_alloy_queue_size }} + } - retry_on_failure { - initial_interval = {{ grafana_alloy_retry_initial_interval | to_json }} - max_elapsed_time = {{ grafana_alloy_retry_max_elapsed_time | to_json }} - } + retry_on_failure { + initial_interval = {{ grafana_alloy_retry_initial_interval | to_json }} + max_elapsed_time = {{ grafana_alloy_retry_max_elapsed_time | to_json }} } } otelcol.auth.basic "cloud" { - username = "${GC_PROM_USERNAME}" - password = "${GC_API_TOKEN}" + username = sys.env("GC_PROM_USERNAME") + password = sys.env("GC_API_TOKEN") } diff --git a/ansible/roles/grafana_alloy/vars/main.yml b/ansible/roles/grafana_alloy/vars/main.yml new file mode 100644 index 0000000..ca7b108 --- /dev/null +++ b/ansible/roles/grafana_alloy/vars/main.yml @@ -0,0 +1,10 @@ +--- +# Role-internal constants (vars/, not defaults/: these are NOT operator knobs). +# +# _ga_managed_marker is the ownership stamp emitted as the first line of +# templates/alloy.service.j2. The disable path greps for it before removing +# anything, so the two must be edited together — losing the marker turns +# teardown into a silent no-op, changing it independently turns teardown into a +# destructive operation against an unrelated Alloy install. +_ga_unit_path: /etc/systemd/system/alloy.service +_ga_managed_marker: MANAGED BY the grafana_alloy role diff --git a/ansible/tests/alloy-config/render.yml b/ansible/tests/alloy-config/render.yml new file mode 100644 index 0000000..05a990d --- /dev/null +++ b/ansible/tests/alloy-config/render.yml @@ -0,0 +1,121 @@ +--- +# Render roles/grafana_alloy's two templates on the control machine so +# validate.sh can feed them to a REAL Grafana Alloy binary. +# +# Why this exists: the molecule `grafana-cloud` scenario runs against a stub that +# exits 0 for every subcommand, so it proves the role's plumbing but can never +# prove the rendered config is one Alloy accepts. Alloy's config syntax rejects +# '#' comments, has no `otelcol.processor.resource`, and puts `sending_queue` / +# `retry_on_failure` on the exporter rather than inside `client` — all three were +# shipped green by the stub. This harness closes that gap. +# +# Renders several variable combinations so the template's conditional branches +# (omitted identity attributes, 0 and 1 sampling ratios, overridden ports) are +# all parsed by the real binary, not just the default one. +# +# Not a deploy playbook: localhost only, writes only into $ALLOY_RENDER_DIR, and +# `become: false` overrides ansible.cfg's project-wide privilege escalation. + +- name: Render the default-values Alloy configuration + hosts: localhost + connection: local + gather_facts: false + become: false + vars_files: + - ../../roles/grafana_alloy/defaults/main.yml + vars: + _ga_render_dir: "{{ lookup('ansible.builtin.env', 'ALLOY_RENDER_DIR') }}" + _ga_instance_id: decdn-node-1 + grafana_alloy_bin_effective: /usr/bin/alloy + grafana_alloy_deployment_environment: production + grafana_alloy_region: US + tasks: + - name: Render config.alloy and the systemd unit with default knobs + ansible.builtin.template: + src: "{{ playbook_dir }}/../../roles/grafana_alloy/templates/{{ item.src }}" + dest: "{{ _ga_render_dir }}/defaults.{{ item.ext }}" + mode: "0644" + loop: + - {src: config.alloy.j2, ext: alloy} + - {src: alloy.service.j2, ext: service} + loop_control: + label: "defaults.{{ item.ext }}" + + # Hands validate.sh the release pin without making it parse YAML itself: the + # values come from the role's own defaults, so the download it verifies is + # byte-identical to the one the role installs. + - name: Export the pinned release for the validation script + ansible.builtin.copy: + dest: "{{ _ga_render_dir }}/pin.env" + mode: "0644" + content: | + GRAFANA_ALLOY_VERSION={{ grafana_alloy_version }} + GRAFANA_ALLOY_SHA256_AMD64={{ grafana_alloy_sha256.amd64 | default('') }} + GRAFANA_ALLOY_SHA256_ARM64={{ grafana_alloy_sha256.arm64 | default('') }} + +# Exercises the omit branches: an empty namespace/region must drop its relabel +# rule and its OTTL `set()` statement entirely rather than emit an empty value, +# and a 0 ratio must still render a valid float. +- name: Render the minimal-identity Alloy configuration + hosts: localhost + connection: local + gather_facts: false + become: false + vars_files: + - ../../roles/grafana_alloy/defaults/main.yml + vars: + _ga_render_dir: "{{ lookup('ansible.builtin.env', 'ALLOY_RENDER_DIR') }}" + _ga_instance_id: decdn-node-2 + grafana_alloy_bin_effective: /usr/local/bin/alloy + grafana_alloy_service_namespace: "" + grafana_alloy_deployment_environment: "" + grafana_alloy_region: "" + grafana_alloy_trace_sampling_ratio: 0 + tasks: + - name: Render config.alloy and the systemd unit with minimal identity + ansible.builtin.template: + src: "{{ playbook_dir }}/../../roles/grafana_alloy/templates/{{ item.src }}" + dest: "{{ _ga_render_dir }}/minimal.{{ item.ext }}" + mode: "0644" + loop: + - {src: config.alloy.j2, ext: alloy} + - {src: alloy.service.j2, ext: service} + loop_control: + label: "minimal.{{ item.ext }}" + +# Exercises the overridden branches: non-default ports, a keep-everything ratio, +# and pipeline knobs well away from their defaults. +- name: Render the fully-overridden Alloy configuration + hosts: localhost + connection: local + gather_facts: false + become: false + vars_files: + - ../../roles/grafana_alloy/defaults/main.yml + vars: + _ga_render_dir: "{{ lookup('ansible.builtin.env', 'ALLOY_RENDER_DIR') }}" + _ga_instance_id: decdn-node-3 + grafana_alloy_bin_effective: /usr/bin/alloy + grafana_alloy_deployment_environment: staging + grafana_alloy_region: EU + grafana_alloy_trace_sampling_ratio: 1 + grafana_alloy_scrape_interval: 15s + grafana_alloy_batch_timeout: 500ms + grafana_alloy_batch_send_size: 512 + grafana_alloy_batch_max_size: 4096 + grafana_alloy_queue_size: 100 + grafana_alloy_retry_initial_interval: 1s + grafana_alloy_retry_max_elapsed_time: 1h + grafana_alloy_otlp_http_port: 14318 + decdn_metrics_port: 19090 + tasks: + - name: Render config.alloy and the systemd unit with overridden knobs + ansible.builtin.template: + src: "{{ playbook_dir }}/../../roles/grafana_alloy/templates/{{ item.src }}" + dest: "{{ _ga_render_dir }}/overridden.{{ item.ext }}" + mode: "0644" + loop: + - {src: config.alloy.j2, ext: alloy} + - {src: alloy.service.j2, ext: service} + loop_control: + label: "overridden.{{ item.ext }}" diff --git a/ansible/tests/alloy-config/validate.sh b/ansible/tests/alloy-config/validate.sh new file mode 100755 index 0000000..10f462b --- /dev/null +++ b/ansible/tests/alloy-config/validate.sh @@ -0,0 +1,126 @@ +#!/usr/bin/env bash +# Validate roles/grafana_alloy's rendered output against a REAL Grafana Alloy +# binary — the pinned one operators actually install. +# +# The molecule `grafana-cloud` scenario runs the role against a stub that exits 0 +# for every subcommand: excellent for proving plumbing, worthless for proving the +# configuration is loadable. `alloy validate` builds the component graph, so it +# catches the whole class the stub waves through — illegal '#' comments, a +# component that does not exist, a block nested in the wrong parent. The unit +# check does the same for ExecStart: every flag must exist in `alloy run --help`. +# +# The binary comes from the SAME pinned .deb + sha256 the role installs +# (roles/grafana_alloy/defaults/main.yml, handed over by render.yml), so bumping +# grafana_alloy_version without re-pinning the digest fails here rather than on a +# production host. +# +# Usage: +# ansible/tests/alloy-config/validate.sh +# ALLOY_BIN=/path/to/alloy ansible/tests/alloy-config/validate.sh # skip the download +# ALLOY_CACHE_DIR=~/.cache/alloy ansible/tests/alloy-config/validate.sh +set -euo pipefail + +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ansible_dir="$(cd "$here/../.." && pwd)" + +work="" +cleanup() { + if [ -n "$work" ]; then rm -rf "$work"; fi +} +trap cleanup EXIT + +fail() { echo "FAIL: $*" >&2; exit 1; } + +work="$(mktemp -d)" +render_dir="$work/render" +mkdir -p "$render_dir" + +# --- Render every variable combination --------------------------------------- +# Also drops pin.env (the role's pinned version + digests) into render_dir. +# Run from ansible/ so the project's ansible.cfg applies (interpreter discovery +# stays quiet; `become` is overridden per-play). +# Run from ansible/ so the project's ansible.cfg applies (quiet interpreter +# discovery; its project-wide `become` is overridden per play in render.yml). +(cd "$ansible_dir" && ALLOY_RENDER_DIR="$render_dir" \ + ansible-playbook -i localhost, -c local "$here/render.yml" >/dev/null) \ + || fail "rendering roles/grafana_alloy templates failed (re-run: ansible-playbook $here/render.yml)" + +shopt -s nullglob +configs=("$render_dir"/*.alloy) +units=("$render_dir"/*.service) +[ "${#configs[@]}" -gt 0 ] || fail "render.yml produced no configuration files" +[ "${#units[@]}" -gt 0 ] || fail "render.yml produced no systemd units" + +# --- Resolve the pinned release ---------------------------------------------- +case "$(uname -m)" in + x86_64) arch=amd64 ;; + aarch64 | arm64) arch=arm64 ;; + *) fail "unsupported architecture $(uname -m) — the role pins amd64/arm64 only" ;; +esac + +# shellcheck source=/dev/null # generated above from the role's own defaults +. "$render_dir/pin.env" +version="$GRAFANA_ALLOY_VERSION" +case "$arch" in + amd64) sha256="$GRAFANA_ALLOY_SHA256_AMD64" ;; + arm64) sha256="$GRAFANA_ALLOY_SHA256_ARM64" ;; +esac +[ -n "$version" ] || fail "grafana_alloy_version is empty in the role defaults" + +cache_dir="${ALLOY_CACHE_DIR:-$ansible_dir/.cache/alloy}/$version-$arch" +alloy="${ALLOY_BIN:-$cache_dir/alloy}" + +if [ ! -x "$alloy" ]; then + if [ -n "${ALLOY_BIN:-}" ]; then + fail "ALLOY_BIN=$ALLOY_BIN is not an executable file" + fi + [ -n "$sha256" ] || fail "no grafana_alloy_sha256 pin for $arch in the role defaults" + deb="alloy-$version-1.$arch.deb" + url="https://github.com/grafana/alloy/releases/download/v$version/$deb" + echo "fetching pinned Grafana Alloy $version ($arch)" + curl -fsSL --retry 3 -o "$work/$deb" "$url" + echo "$sha256 $work/$deb" | sha256sum --check --status || fail \ + "sha256 mismatch for $deb — grafana_alloy_version and grafana_alloy_sha256 must be bumped together" + dpkg-deb -x "$work/$deb" "$work/unpacked" \ + || fail "dpkg-deb is required to unpack the pinned .deb (or point ALLOY_BIN at a binary)" + mkdir -p "$cache_dir" + install -m 0755 "$work/unpacked/usr/bin/alloy" "$cache_dir/alloy" +fi + +echo "using $("$alloy" --version | head -1)" + +# --- Gate 1: the configuration loads ----------------------------------------- +# No credentials are exported on purpose: sys.env() on an unset variable yields +# "" and still validates, which is exactly the property the role's deploy-time +# gate relies on (it runs before any secret is in scope). +for config in "${configs[@]}"; do + name="$(basename "$config")" + "$alloy" validate "$config" || fail "alloy validate rejected $name" + if grep -qE '^[[:space:]]*#' "$config"; then + fail "$name contains a '#' comment — Alloy's syntax only accepts // and /* */" + fi + # shellcheck disable=SC2016 # literal '${GC_' is the point: Alloy never expands it + if grep -qF '${GC_' "$config"; then + fail "$name still carries a \${GC_*} placeholder — Alloy never expands those; use sys.env(\"GC_…\")" + fi + if ! grep -qF 'sys.env("GC_API_TOKEN")' "$config"; then + fail "$name does not read the API token via sys.env — credentials must never be literals" + fi + echo "ok: $name" +done + +# --- Gate 2: every ExecStart flag exists ------------------------------------- +# `alloy run` ignores nothing: an unknown flag exits non-zero, i.e. a systemd +# crash-loop on the target host the moment the unit starts. +known_flags="$("$alloy" run --help 2>&1 | grep -oE '(^|[[:space:]])--[a-zA-Z0-9.-]+' | tr -d '[:blank:]')" +for unit in "${units[@]}"; do + name="$(basename "$unit")" + while read -r flag; do + [ -n "$flag" ] || continue + grep -qxF -- "$flag" <<<"$known_flags" \ + || fail "$name passes $flag to \`alloy run\`, which Alloy $version does not define" + done < <(grep -oE '^[[:space:]]*--[a-zA-Z0-9.-]+' "$unit" | tr -d '[:blank:]') + echo "ok: $name" +done + +echo "PASS: roles/grafana_alloy renders configuration Alloy $version accepts"