From c043ce864a436050426a6975c14cbedb95b0a366 Mon Sep 17 00:00:00 2001 From: Ant Somers Date: Sat, 19 Sep 2026 19:27:54 +0300 Subject: [PATCH 1/2] fix(ansible): name the daemon's log stream decdn-node The journald pipeline shipped every unit with job=integrations/node_exporter and the fleet labels, but no service_name, so the daemon's logs could not be queried or correlated by service the way its metrics and spans are. loki.relabel "journal_identity" now stamps service_name (from grafana_alloy_service_name) on the decdn-node.service stream only. Host units keep their labels unchanged, and job stays on the integration value so the Linux Server log dashboards still match. The rule is skipped when grafana_alloy_service_name is empty. Co-Authored-By: Claude Opus 5 (1M context) --- ansible/roles/grafana_alloy/README.md | 6 ++++-- ansible/roles/grafana_alloy/defaults/main.yml | 2 ++ .../grafana_alloy/templates/config.alloy.j2 | 16 ++++++++++++++++ ansible/tests/alloy-config/validate.sh | 2 ++ 4 files changed, 24 insertions(+), 2 deletions(-) diff --git a/ansible/roles/grafana_alloy/README.md b/ansible/roles/grafana_alloy/README.md index 6c1fa673..e67ebbaf 100644 --- a/ansible/roles/grafana_alloy/README.md +++ b/ansible/roles/grafana_alloy/README.md @@ -197,7 +197,7 @@ Binary/package removal is manual. | --- | --- | --- | | Node `/metrics` | `decdn-node` | your own dashboards/queries | | Machine metrics | `integrations/node_exporter` | Grafana Cloud's **Linux Server** integration dashboards + alerts, unmodified | -| journald | `integrations/node_exporter` | the same integration's logs dashboards (correlated by `instance`) | +| journald | `integrations/node_exporter` | the same integration's logs dashboards (correlated by `instance`); the daemon's own unit (`decdn-node.service`) also carries `service_name=decdn-node` | | Alloy self-metrics | `integrations/alloy` | agent health | | Traces | — | Application Observability | @@ -280,7 +280,9 @@ Label variables (`service_name`, `service_namespace`, `instance_id`, them low-cardinality; unique label sets are what your Grafana Cloud bill scales with. Guardrails: one scrape interval per fleet (`30s` node / `60s` host), sampling above zero for spans, the curated collector set, the journald drops, and -no per-request/per-hash label values. +no per-request/per-hash label values. `service_name` never lands on machine +series or host-unit log streams — only on the daemon's metrics, its spans, and +its own `decdn-node.service` journald stream. ## Couplings & guards diff --git a/ansible/roles/grafana_alloy/defaults/main.yml b/ansible/roles/grafana_alloy/defaults/main.yml index b4360fbd..27af4f2e 100644 --- a/ansible/roles/grafana_alloy/defaults/main.yml +++ b/ansible/roles/grafana_alloy/defaults/main.yml @@ -235,6 +235,8 @@ grafana_alloy_logs_drop_unit_regex: # Cloud org for EVERY series/span/log line: never add high-cardinality values # (hashes, per-request ids, timestamps) — the remote bill scales with unique # label sets (see README cost guardrails). +# service_name also names the daemon's journald stream in Loki (unit +# "decdn-node.service" only; host units never get it). "" drops it everywhere. grafana_alloy_service_name: decdn-node grafana_alloy_service_namespace: decdn grafana_alloy_instance_id: "" # "" => derive from inventory_hostname diff --git a/ansible/roles/grafana_alloy/templates/config.alloy.j2 b/ansible/roles/grafana_alloy/templates/config.alloy.j2 index c9091898..b80b740c 100644 --- a/ansible/roles/grafana_alloy/templates/config.alloy.j2 +++ b/ansible/roles/grafana_alloy/templates/config.alloy.j2 @@ -319,6 +319,22 @@ loki.relabel "journal_identity" { replacement = {{ value | tojson }} } {% endfor %} +{% if grafana_alloy_service_name | length > 0 %} +{# service_name describes the daemon, not the machine (the same reason host + metrics above omit it), so only the daemon's own unit gets it: every other + unit keeps exactly the labels set above. `job` stays on the integration value + so the Linux Server log dashboards still match this stream too. A replace + rule whose regex does not match leaves the stream unchanged. `unit` comes from + journal_rules (__journal__systemd_unit); the name is the unit decdn_node + installs, /etc/systemd/system/decdn-node.service. #} + + rule { + source_labels = ["unit"] + regex = "decdn-node\\.service" + target_label = "service_name" + replacement = {{ grafana_alloy_service_name | tojson }} + } +{% endif %} } loki.write "cloud" { diff --git a/ansible/tests/alloy-config/validate.sh b/ansible/tests/alloy-config/validate.sh index 603334d0..a6d79ec4 100755 --- a/ansible/tests/alloy-config/validate.sh +++ b/ansible/tests/alloy-config/validate.sh @@ -137,6 +137,8 @@ assert_has defaults.alloy 'loki.write "cloud"' "the Loki writer" assert_has defaults.alloy '"job"' "an explicit job label" assert_has defaults.alloy 'integrations/node_exporter' "the Grafana Cloud integration job label" assert_has defaults.alloy '__journal__systemd_unit' "the journal field mapping" +assert_has defaults.alloy 'target_label = "service_name"' "the daemon log-stream service_name rule" +assert_has defaults.alloy 'regex = "decdn-node\\.service"' "the daemon unit match" assert_has defaults.alloy 'systemd {' "the per-unit systemd collector" # Sub-knob isolation. From 459b506cc9469bf9a90a3f20403a0f94f7269bc8 Mon Sep 17 00:00:00 2001 From: Ant Somers Date: Sat, 19 Sep 2026 19:32:02 +0300 Subject: [PATCH 2/2] fix(ansible): correct service_name docs and test the blank branch Review follow-ups for the daemon log-stream service_name rule: - Docs claimed service_name lands "only" on node metrics, spans and the daemon's journald stream, and that "" drops it everywhere. The OTLP transform also stamps service.name on logs/metrics pushed to the local endpoint, and "" there only stops the overwrite. README now lists the three places Alloy sets it, notes that systemd's crash-loop lines for the daemon arrive as unit="init.scope", and that Loki may derive a service_name for label-less streams. - Replace the "EVERY series/span/log line" wording (defaults, README, preflight) that the change made self-contradictory. - render.yml's minimal fixture now blanks grafana_alloy_service_name, and validate.sh asserts the metric rule, journald rule and OTTL statement all vanish while the journald relabeller survives. The defaults render also pins the rule's source label and replacement value. Co-Authored-By: Claude Opus 5 (1M context) --- ansible/roles/grafana_alloy/README.md | 30 ++++++++++++++----- ansible/roles/grafana_alloy/defaults/main.yml | 14 +++++---- .../roles/grafana_alloy/tasks/preflight.yml | 4 +-- .../grafana_alloy/templates/config.alloy.j2 | 17 ++++++----- ansible/tests/alloy-config/render.yml | 3 +- ansible/tests/alloy-config/validate.sh | 15 +++++++++- 6 files changed, 59 insertions(+), 24 deletions(-) diff --git a/ansible/roles/grafana_alloy/README.md b/ansible/roles/grafana_alloy/README.md index e67ebbaf..966b6cb4 100644 --- a/ansible/roles/grafana_alloy/README.md +++ b/ansible/roles/grafana_alloy/README.md @@ -197,7 +197,7 @@ Binary/package removal is manual. | --- | --- | --- | | Node `/metrics` | `decdn-node` | your own dashboards/queries | | Machine metrics | `integrations/node_exporter` | Grafana Cloud's **Linux Server** integration dashboards + alerts, unmodified | -| journald | `integrations/node_exporter` | the same integration's logs dashboards (correlated by `instance`); the daemon's own unit (`decdn-node.service`) also carries `service_name=decdn-node` | +| journald | `integrations/node_exporter` | the same integration's logs dashboards (correlated by `instance`); the daemon's own unit (`decdn-node.service`) also carries `service_name=` (default `decdn-node`) | | Alloy self-metrics | `integrations/alloy` | agent health | | Traces | — | Application Observability | @@ -276,13 +276,29 @@ upstream version/sha256). Highlights: | `grafana_alloy_node_job` / `_host_job` / `_self_job` / `_logs_job` | see table above | Job labels; the `integrations/…` ones are what Grafana Cloud's dashboards match | Label variables (`service_name`, `service_namespace`, `instance_id`, -`deployment_environment`, `region`) ship on EVERY series/span/log line — keep -them low-cardinality; unique label sets are what your Grafana Cloud bill scales -with. Guardrails: one scrape interval per fleet (`30s` node / `60s` host), +`deployment_environment`, `region`) ship on most series, spans and log lines — +keep them low-cardinality; unique label sets are what your Grafana Cloud bill +scales with. Guardrails: one scrape interval per fleet (`30s` node / `60s` host), sampling above zero for spans, the curated collector set, the journald drops, and -no per-request/per-hash label values. `service_name` never lands on machine -series or host-unit log streams — only on the daemon's metrics, its spans, and -its own `decdn-node.service` journald stream. +no per-request/per-hash label values. + +Alloy sets `service_name` in exactly three places, and never on machine series, +its own self-metrics or host-unit log streams: + +- the node's `/metrics` series, as the `service_name` label; +- everything received on the local OTLP endpoint (spans, and any logs/metrics + pushed there), as the `service.name` resource attribute — overwriting whatever + the sender set; +- the `decdn-node.service` journald stream, as the `service_name` label. This + covers only lines the daemon process writes: systemd's own start/stop/crash + messages about it are logged by PID 1 under `unit="init.scope"`, so query that + stream too when chasing a crash loop. + +Loki in Grafana Cloud may derive a `service_name` for streams that arrive +without one (from labels such as `job`), so host-unit streams can still show one +in the UI — that value comes from Loki, not from this role. Setting +`grafana_alloy_service_name: ""` stops Alloy stamping it: the node-metric label +and the journald rule disappear, and OTLP data keeps its sender's `service.name`. ## Couplings & guards diff --git a/ansible/roles/grafana_alloy/defaults/main.yml b/ansible/roles/grafana_alloy/defaults/main.yml index 27af4f2e..2ff83569 100644 --- a/ansible/roles/grafana_alloy/defaults/main.yml +++ b/ansible/roles/grafana_alloy/defaults/main.yml @@ -231,12 +231,14 @@ grafana_alloy_logs_drop_unit_regex: '(session-\d+\.scope|user@\d+\.service|systemd-timesyncd\.service|cron\.service|systemd-logind\.service)' # --- Resource identity labels (low-cardinality only!) -------------------------- -# These become metric labels and OTLP resource attributes shipped to your Grafana -# Cloud org for EVERY series/span/log line: never add high-cardinality values -# (hashes, per-request ids, timestamps) — the remote bill scales with unique -# label sets (see README cost guardrails). -# service_name also names the daemon's journald stream in Loki (unit -# "decdn-node.service" only; host units never get it). "" drops it everywhere. +# These become identity labels and OTLP resource attributes shipped to your +# Grafana Cloud org (which signal carries which: README "Label variables"): never +# add high-cardinality values (hashes, per-request ids, timestamps) — the remote +# bill scales with unique label sets (see README cost guardrails). +# service_name also names the daemon's own journald stream in Loki (unit +# "decdn-node.service" only; Alloy never sets it on host-unit streams). "" stops +# Alloy stamping it: the node-metric label and the journald rule disappear, and +# OTLP data keeps whatever service.name its sender set. grafana_alloy_service_name: decdn-node grafana_alloy_service_namespace: decdn grafana_alloy_instance_id: "" # "" => derive from inventory_hostname diff --git a/ansible/roles/grafana_alloy/tasks/preflight.yml b/ansible/roles/grafana_alloy/tasks/preflight.yml index 9cb6948a..32b18fe0 100644 --- a/ansible/roles/grafana_alloy/tasks/preflight.yml +++ b/ansible/roles/grafana_alloy/tasks/preflight.yml @@ -102,7 +102,7 @@ when: grafana_alloy_host_metrics_enabled | bool # job/instance are what Grafana Cloud's integration dashboards filter on, and -# like the identity labels they ride on EVERY series and log line. +# like the identity labels they ride on most series and log lines. - name: Validate the job labels ansible.builtin.assert: that: @@ -150,7 +150,7 @@ quiet: true fail_msg: >- Identity label values are shipped to Grafana Cloud as metric labels and OTLP - resource attributes on EVERY series/span, so they are restricted to a + resource attributes on most series/spans/log lines, so they are restricted to a low-cardinality charset (letters, digits, dot, underscore, slash, ≤ 63 chars): {{ item }} does not fit. Offending variables: grafana_alloy_service_name / service_namespace / instance_id / diff --git a/ansible/roles/grafana_alloy/templates/config.alloy.j2 b/ansible/roles/grafana_alloy/templates/config.alloy.j2 index b80b740c..c36ae2b6 100644 --- a/ansible/roles/grafana_alloy/templates/config.alloy.j2 +++ b/ansible/roles/grafana_alloy/templates/config.alloy.j2 @@ -105,7 +105,7 @@ prometheus.remote_write "cloud" { every sample at scrape time) rather than prometheus.relabel (per SAMPLE, with a cardinality-sized cache). The scrape forwards STRAIGHT to remote_write, deliberately bypassing prometheus.relabel.decdn_identity: that component - stamps service_name=decdn-node, which is false for machine series. + stamps service_name (default decdn-node), which is false for machine series. `job` must stay integrations/node_exporter for Grafana Cloud's prebuilt Linux Server dashboards and alert rules to find these series. @@ -321,12 +321,15 @@ loki.relabel "journal_identity" { {% endfor %} {% if grafana_alloy_service_name | length > 0 %} {# service_name describes the daemon, not the machine (the same reason host - metrics above omit it), so only the daemon's own unit gets it: every other - unit keeps exactly the labels set above. `job` stays on the integration value - so the Linux Server log dashboards still match this stream too. A replace - rule whose regex does not match leaves the stream unchanged. `unit` comes from - journal_rules (__journal__systemd_unit); the name is the unit decdn_node - installs, /etc/systemd/system/decdn-node.service. #} + metrics above omit it), so only the daemon's own unit gets it; every other + unit is unaffected by this rule. `job` stays whatever the identity rules above + set (by default the integration value), so the Linux Server log dashboards + still match this stream too. A replace rule whose regex does not match leaves + the stream unchanged. `unit` comes from journal_rules (__journal__systemd_unit), + so this catches only lines the daemon process writes: systemd's own + start/stop/crash messages about it come from PID 1 as unit="init.scope". The + name is the unit decdn_node installs, /etc/systemd/system/decdn-node.service + — a rename there must be mirrored here (validate.sh pins it). #} rule { source_labels = ["unit"] diff --git a/ansible/tests/alloy-config/render.yml b/ansible/tests/alloy-config/render.yml index c0b884f7..70e0e413 100644 --- a/ansible/tests/alloy-config/render.yml +++ b/ansible/tests/alloy-config/render.yml @@ -53,7 +53,7 @@ grafana_alloy_deployment_environment: production grafana_alloy_region: US - # Exercises the omit branches: an empty namespace/region must drop its relabel + # Exercises the omit branches: an empty service_name/namespace/region must drop its relabel # rule and its OTTL `set()` statement entirely rather than emit an empty value, # and a 0 ratio must still render a valid float. - name: Render the minimal-identity configuration @@ -67,6 +67,7 @@ vars: _ga_instance_id: decdn-node-2 grafana_alloy_bin_effective: /usr/local/bin/alloy + grafana_alloy_service_name: "" grafana_alloy_service_namespace: "" grafana_alloy_deployment_environment: "" grafana_alloy_region: "" diff --git a/ansible/tests/alloy-config/validate.sh b/ansible/tests/alloy-config/validate.sh index a6d79ec4..bb2644e2 100755 --- a/ansible/tests/alloy-config/validate.sh +++ b/ansible/tests/alloy-config/validate.sh @@ -137,8 +137,14 @@ assert_has defaults.alloy 'loki.write "cloud"' "the Loki writer" assert_has defaults.alloy '"job"' "an explicit job label" assert_has defaults.alloy 'integrations/node_exporter' "the Grafana Cloud integration job label" assert_has defaults.alloy '__journal__systemd_unit' "the journal field mapping" -assert_has defaults.alloy 'target_label = "service_name"' "the daemon log-stream service_name rule" +# Daemon log-stream service_name rule. ALIGNMENT-SENSITIVE needles: the +# one-space `target_label = "service_name"` / `replacement = …` forms belong to +# the node-metric identity rule, so only these column-aligned spellings pin the +# journald rule. +assert_has defaults.alloy 'source_labels = ["unit"]' "the unit scoping of the daemon log-stream rule" assert_has defaults.alloy 'regex = "decdn-node\\.service"' "the daemon unit match" +assert_has defaults.alloy 'target_label = "service_name"' "the daemon log-stream service_name rule" +assert_has defaults.alloy 'replacement = "decdn-node"' "the daemon log-stream service_name value" assert_has defaults.alloy 'systemd {' "the per-unit systemd collector" # Sub-knob isolation. @@ -151,6 +157,13 @@ assert_has logsonly.alloy 'path = "/var/log/journal"' "the explicit jo assert_lacks logsonly.alloy 'prometheus.exporter' "any exporter while only logs are enabled" assert_lacks logsonly.alloy 'action = "drop"' "a drop rule after both guardrail regexes were cleared" +# A blank service_name switches the whole identity off together: metric rule, +# journald rule and OTTL statement. The has-check keeps the lacks-checks honest +# (they must not pass merely because logs went missing). +assert_has minimal.alloy 'loki.relabel "journal_identity"' "the journal identity relabeller, which must survive a blank service_name" +assert_lacks minimal.alloy '"service_name"' "any service_name rule (metric or log) once it was blanked" +assert_lacks minimal.alloy 'service.name' "the OTLP service.name attribute once it was blanked" + # BACKWARDS COMPATIBILITY: sub-knobs off == the pre-machine-monitoring pipeline. assert_lacks legacy.alloy 'prometheus.exporter' "any exporter" assert_lacks legacy.alloy 'discovery.relabel' "any target relabeller"