diff --git a/ansible/roles/grafana_alloy/README.md b/ansible/roles/grafana_alloy/README.md index 6c1fa67..966b6cb 100644 --- a/ansible/roles/grafana_alloy/README.md +++ b/ansible/roles/grafana_alloy/README.md @@ -197,7 +197,7 @@ Binary/package removal is manual. | --- | --- | --- | | Node `/metrics` | `decdn-node` | your own dashboards/queries | | Machine metrics | `integrations/node_exporter` | Grafana Cloud's **Linux Server** integration dashboards + alerts, unmodified | -| journald | `integrations/node_exporter` | the same integration's logs dashboards (correlated by `instance`) | +| journald | `integrations/node_exporter` | the same integration's logs dashboards (correlated by `instance`); the daemon's own unit (`decdn-node.service`) also carries `service_name=` (default `decdn-node`) | | Alloy self-metrics | `integrations/alloy` | agent health | | Traces | — | Application Observability | @@ -276,12 +276,30 @@ upstream version/sha256). Highlights: | `grafana_alloy_node_job` / `_host_job` / `_self_job` / `_logs_job` | see table above | Job labels; the `integrations/…` ones are what Grafana Cloud's dashboards match | Label variables (`service_name`, `service_namespace`, `instance_id`, -`deployment_environment`, `region`) ship on EVERY series/span/log line — keep -them low-cardinality; unique label sets are what your Grafana Cloud bill scales -with. Guardrails: one scrape interval per fleet (`30s` node / `60s` host), +`deployment_environment`, `region`) ship on most series, spans and log lines — +keep them low-cardinality; unique label sets are what your Grafana Cloud bill +scales with. Guardrails: one scrape interval per fleet (`30s` node / `60s` host), sampling above zero for spans, the curated collector set, the journald drops, and no per-request/per-hash label values. +Alloy sets `service_name` in exactly three places, and never on machine series, +its own self-metrics or host-unit log streams: + +- the node's `/metrics` series, as the `service_name` label; +- everything received on the local OTLP endpoint (spans, and any logs/metrics + pushed there), as the `service.name` resource attribute — overwriting whatever + the sender set; +- the `decdn-node.service` journald stream, as the `service_name` label. This + covers only lines the daemon process writes: systemd's own start/stop/crash + messages about it are logged by PID 1 under `unit="init.scope"`, so query that + stream too when chasing a crash loop. + +Loki in Grafana Cloud may derive a `service_name` for streams that arrive +without one (from labels such as `job`), so host-unit streams can still show one +in the UI — that value comes from Loki, not from this role. Setting +`grafana_alloy_service_name: ""` stops Alloy stamping it: the node-metric label +and the journald rule disappear, and OTLP data keeps its sender's `service.name`. + ## Couplings & guards Two couplings between roles are pinned by constants plus molecule assertions: diff --git a/ansible/roles/grafana_alloy/defaults/main.yml b/ansible/roles/grafana_alloy/defaults/main.yml index b4360fb..2ff8356 100644 --- a/ansible/roles/grafana_alloy/defaults/main.yml +++ b/ansible/roles/grafana_alloy/defaults/main.yml @@ -231,10 +231,14 @@ grafana_alloy_logs_drop_unit_regex: '(session-\d+\.scope|user@\d+\.service|systemd-timesyncd\.service|cron\.service|systemd-logind\.service)' # --- Resource identity labels (low-cardinality only!) -------------------------- -# These become metric labels and OTLP resource attributes shipped to your Grafana -# Cloud org for EVERY series/span/log line: never add high-cardinality values -# (hashes, per-request ids, timestamps) — the remote bill scales with unique -# label sets (see README cost guardrails). +# These become identity labels and OTLP resource attributes shipped to your +# Grafana Cloud org (which signal carries which: README "Label variables"): never +# add high-cardinality values (hashes, per-request ids, timestamps) — the remote +# bill scales with unique label sets (see README cost guardrails). +# service_name also names the daemon's own journald stream in Loki (unit +# "decdn-node.service" only; Alloy never sets it on host-unit streams). "" stops +# Alloy stamping it: the node-metric label and the journald rule disappear, and +# OTLP data keeps whatever service.name its sender set. grafana_alloy_service_name: decdn-node grafana_alloy_service_namespace: decdn grafana_alloy_instance_id: "" # "" => derive from inventory_hostname diff --git a/ansible/roles/grafana_alloy/tasks/preflight.yml b/ansible/roles/grafana_alloy/tasks/preflight.yml index 9cb6948..32b18fe 100644 --- a/ansible/roles/grafana_alloy/tasks/preflight.yml +++ b/ansible/roles/grafana_alloy/tasks/preflight.yml @@ -102,7 +102,7 @@ when: grafana_alloy_host_metrics_enabled | bool # job/instance are what Grafana Cloud's integration dashboards filter on, and -# like the identity labels they ride on EVERY series and log line. +# like the identity labels they ride on most series and log lines. - name: Validate the job labels ansible.builtin.assert: that: @@ -150,7 +150,7 @@ quiet: true fail_msg: >- Identity label values are shipped to Grafana Cloud as metric labels and OTLP - resource attributes on EVERY series/span, so they are restricted to a + resource attributes on most series/spans/log lines, so they are restricted to a low-cardinality charset (letters, digits, dot, underscore, slash, ≤ 63 chars): {{ item }} does not fit. Offending variables: grafana_alloy_service_name / service_namespace / instance_id / diff --git a/ansible/roles/grafana_alloy/templates/config.alloy.j2 b/ansible/roles/grafana_alloy/templates/config.alloy.j2 index c909189..c36ae2b 100644 --- a/ansible/roles/grafana_alloy/templates/config.alloy.j2 +++ b/ansible/roles/grafana_alloy/templates/config.alloy.j2 @@ -105,7 +105,7 @@ prometheus.remote_write "cloud" { every sample at scrape time) rather than prometheus.relabel (per SAMPLE, with a cardinality-sized cache). The scrape forwards STRAIGHT to remote_write, deliberately bypassing prometheus.relabel.decdn_identity: that component - stamps service_name=decdn-node, which is false for machine series. + stamps service_name (default decdn-node), which is false for machine series. `job` must stay integrations/node_exporter for Grafana Cloud's prebuilt Linux Server dashboards and alert rules to find these series. @@ -319,6 +319,25 @@ loki.relabel "journal_identity" { replacement = {{ value | tojson }} } {% endfor %} +{% if grafana_alloy_service_name | length > 0 %} +{# service_name describes the daemon, not the machine (the same reason host + metrics above omit it), so only the daemon's own unit gets it; every other + unit is unaffected by this rule. `job` stays whatever the identity rules above + set (by default the integration value), so the Linux Server log dashboards + still match this stream too. A replace rule whose regex does not match leaves + the stream unchanged. `unit` comes from journal_rules (__journal__systemd_unit), + so this catches only lines the daemon process writes: systemd's own + start/stop/crash messages about it come from PID 1 as unit="init.scope". The + name is the unit decdn_node installs, /etc/systemd/system/decdn-node.service + — a rename there must be mirrored here (validate.sh pins it). #} + + rule { + source_labels = ["unit"] + regex = "decdn-node\\.service" + target_label = "service_name" + replacement = {{ grafana_alloy_service_name | tojson }} + } +{% endif %} } loki.write "cloud" { diff --git a/ansible/tests/alloy-config/render.yml b/ansible/tests/alloy-config/render.yml index c0b884f..70e0e41 100644 --- a/ansible/tests/alloy-config/render.yml +++ b/ansible/tests/alloy-config/render.yml @@ -53,7 +53,7 @@ grafana_alloy_deployment_environment: production grafana_alloy_region: US - # Exercises the omit branches: an empty namespace/region must drop its relabel + # Exercises the omit branches: an empty service_name/namespace/region must drop its relabel # rule and its OTTL `set()` statement entirely rather than emit an empty value, # and a 0 ratio must still render a valid float. - name: Render the minimal-identity configuration @@ -67,6 +67,7 @@ vars: _ga_instance_id: decdn-node-2 grafana_alloy_bin_effective: /usr/local/bin/alloy + grafana_alloy_service_name: "" grafana_alloy_service_namespace: "" grafana_alloy_deployment_environment: "" grafana_alloy_region: "" diff --git a/ansible/tests/alloy-config/validate.sh b/ansible/tests/alloy-config/validate.sh index 603334d..bb2644e 100755 --- a/ansible/tests/alloy-config/validate.sh +++ b/ansible/tests/alloy-config/validate.sh @@ -137,6 +137,14 @@ assert_has defaults.alloy 'loki.write "cloud"' "the Loki writer" assert_has defaults.alloy '"job"' "an explicit job label" assert_has defaults.alloy 'integrations/node_exporter' "the Grafana Cloud integration job label" assert_has defaults.alloy '__journal__systemd_unit' "the journal field mapping" +# Daemon log-stream service_name rule. ALIGNMENT-SENSITIVE needles: the +# one-space `target_label = "service_name"` / `replacement = …` forms belong to +# the node-metric identity rule, so only these column-aligned spellings pin the +# journald rule. +assert_has defaults.alloy 'source_labels = ["unit"]' "the unit scoping of the daemon log-stream rule" +assert_has defaults.alloy 'regex = "decdn-node\\.service"' "the daemon unit match" +assert_has defaults.alloy 'target_label = "service_name"' "the daemon log-stream service_name rule" +assert_has defaults.alloy 'replacement = "decdn-node"' "the daemon log-stream service_name value" assert_has defaults.alloy 'systemd {' "the per-unit systemd collector" # Sub-knob isolation. @@ -149,6 +157,13 @@ assert_has logsonly.alloy 'path = "/var/log/journal"' "the explicit jo assert_lacks logsonly.alloy 'prometheus.exporter' "any exporter while only logs are enabled" assert_lacks logsonly.alloy 'action = "drop"' "a drop rule after both guardrail regexes were cleared" +# A blank service_name switches the whole identity off together: metric rule, +# journald rule and OTTL statement. The has-check keeps the lacks-checks honest +# (they must not pass merely because logs went missing). +assert_has minimal.alloy 'loki.relabel "journal_identity"' "the journal identity relabeller, which must survive a blank service_name" +assert_lacks minimal.alloy '"service_name"' "any service_name rule (metric or log) once it was blanked" +assert_lacks minimal.alloy 'service.name' "the OTLP service.name attribute once it was blanked" + # BACKWARDS COMPATIBILITY: sub-knobs off == the pre-machine-monitoring pipeline. assert_lacks legacy.alloy 'prometheus.exporter' "any exporter" assert_lacks legacy.alloy 'discovery.relabel' "any target relabeller"