fix(ansible): name the daemon's log stream decdn-node

ansible/roles/grafana_alloy/README.md

@@ -197,7 +197,7 @@ Binary/package removal is manual.
 | --- | --- | --- |
 | Node `/metrics` | `decdn-node` | your own dashboards/queries |
 | Machine metrics | `integrations/node_exporter` | Grafana Cloud's **Linux Server** integration dashboards + alerts, unmodified |
-| journald | `integrations/node_exporter` | the same integration's logs dashboards (correlated by `instance`) |
+| journald | `integrations/node_exporter` | the same integration's logs dashboards (correlated by `instance`); the daemon's own unit (`decdn-node.service`) also carries `service_name=<grafana_alloy_service_name>` (default `decdn-node`) |
 | Alloy self-metrics | `integrations/alloy` | agent health |
 | Traces | — | Application Observability |
 
@@ -276,12 +276,30 @@ upstream version/sha256). Highlights:
 | `grafana_alloy_node_job` / `_host_job` / `_self_job` / `_logs_job` | see table above | Job labels; the `integrations/…` ones are what Grafana Cloud's dashboards match |
 
 Label variables (`service_name`, `service_namespace`, `instance_id`,
-`deployment_environment`, `region`) ship on EVERY series/span/log line — keep
-them low-cardinality; unique label sets are what your Grafana Cloud bill scales
-with. Guardrails: one scrape interval per fleet (`30s` node / `60s` host),
+`deployment_environment`, `region`) ship on most series, spans and log lines —
+keep them low-cardinality; unique label sets are what your Grafana Cloud bill
+scales with. Guardrails: one scrape interval per fleet (`30s` node / `60s` host),
 sampling above zero for spans, the curated collector set, the journald drops, and
 no per-request/per-hash label values.
 
+Alloy sets `service_name` in exactly three places, and never on machine series,
+its own self-metrics or host-unit log streams:
+
+- the node's `/metrics` series, as the `service_name` label;
+- everything received on the local OTLP endpoint (spans, and any logs/metrics
+  pushed there), as the `service.name` resource attribute — overwriting whatever
+  the sender set;
+- the `decdn-node.service` journald stream, as the `service_name` label. This
+  covers only lines the daemon process writes: systemd's own start/stop/crash
+  messages about it are logged by PID 1 under `unit="init.scope"`, so query that
+  stream too when chasing a crash loop.
+
+Loki in Grafana Cloud may derive a `service_name` for streams that arrive
+without one (from labels such as `job`), so host-unit streams can still show one
+in the UI — that value comes from Loki, not from this role. Setting
+`grafana_alloy_service_name: ""` stops Alloy stamping it: the node-metric label
+and the journald rule disappear, and OTLP data keeps its sender's `service.name`.
+
 ## Couplings & guards
 
 Two couplings between roles are pinned by constants plus molecule assertions:

ansible/roles/grafana_alloy/defaults/main.yml

@@ -231,10 +231,14 @@ grafana_alloy_logs_drop_unit_regex:
   '(session-\d+\.scope|user@\d+\.service|systemd-timesyncd\.service|cron\.service|systemd-logind\.service)'
 
 # --- Resource identity labels (low-cardinality only!) --------------------------
-# These become metric labels and OTLP resource attributes shipped to your Grafana
-# Cloud org for EVERY series/span/log line: never add high-cardinality values
-# (hashes, per-request ids, timestamps) — the remote bill scales with unique
-# label sets (see README cost guardrails).
+# These become identity labels and OTLP resource attributes shipped to your
+# Grafana Cloud org (which signal carries which: README "Label variables"): never
+# add high-cardinality values (hashes, per-request ids, timestamps) — the remote
+# bill scales with unique label sets (see README cost guardrails).
+# service_name also names the daemon's own journald stream in Loki (unit
+# "decdn-node.service" only; Alloy never sets it on host-unit streams). "" stops
+# Alloy stamping it: the node-metric label and the journald rule disappear, and
+# OTLP data keeps whatever service.name its sender set.
 grafana_alloy_service_name: decdn-node
 grafana_alloy_service_namespace: decdn
 grafana_alloy_instance_id: ""            # "" => derive from inventory_hostname

ansible/roles/grafana_alloy/tasks/preflight.yml

@@ -102,7 +102,7 @@
   when: grafana_alloy_host_metrics_enabled | bool
 
 # job/instance are what Grafana Cloud's integration dashboards filter on, and
-# like the identity labels they ride on EVERY series and log line.
+# like the identity labels they ride on most series and log lines.
 - name: Validate the job labels
   ansible.builtin.assert:
     that:
@@ -150,7 +150,7 @@
     quiet: true
     fail_msg: >-
       Identity label values are shipped to Grafana Cloud as metric labels and OTLP
-      resource attributes on EVERY series/span, so they are restricted to a
+      resource attributes on most series/spans/log lines, so they are restricted to a
       low-cardinality charset (letters, digits, dot, underscore, slash, ≤ 63
       chars): {{ item }} does not fit. Offending variables:
       grafana_alloy_service_name / service_namespace / instance_id /

ansible/roles/grafana_alloy/templates/config.alloy.j2

@@ -105,7 +105,7 @@ prometheus.remote_write "cloud" {
    every sample at scrape time) rather than prometheus.relabel (per SAMPLE, with
    a cardinality-sized cache). The scrape forwards STRAIGHT to remote_write,
    deliberately bypassing prometheus.relabel.decdn_identity: that component
-   stamps service_name=decdn-node, which is false for machine series.
+   stamps service_name (default decdn-node), which is false for machine series.
 
    `job` must stay integrations/node_exporter for Grafana Cloud's prebuilt Linux
    Server dashboards and alert rules to find these series.
@@ -319,6 +319,25 @@ loki.relabel "journal_identity" {
     replacement  = {{ value | tojson }}
   }
 {% endfor %}
+{% if grafana_alloy_service_name | length > 0 %}
+{# service_name describes the daemon, not the machine (the same reason host
+   metrics above omit it), so only the daemon's own unit gets it; every other
+   unit is unaffected by this rule. `job` stays whatever the identity rules above
+   set (by default the integration value), so the Linux Server log dashboards
+   still match this stream too. A replace rule whose regex does not match leaves
+   the stream unchanged. `unit` comes from journal_rules (__journal__systemd_unit),
+   so this catches only lines the daemon process writes: systemd's own
+   start/stop/crash messages about it come from PID 1 as unit="init.scope". The
+   name is the unit decdn_node installs, /etc/systemd/system/decdn-node.service
+   — a rename there must be mirrored here (validate.sh pins it). #}
+
+  rule {
+    source_labels = ["unit"]
+    regex         = "decdn-node\\.service"
+    target_label  = "service_name"
+    replacement   = {{ grafana_alloy_service_name | tojson }}
+  }
+{% endif %}
 }
 
 loki.write "cloud" {

ansible/tests/alloy-config/render.yml

@@ -53,7 +53,7 @@
         grafana_alloy_deployment_environment: production
         grafana_alloy_region: US
 
-    # Exercises the omit branches: an empty namespace/region must drop its relabel
+    # Exercises the omit branches: an empty service_name/namespace/region must drop its relabel
     # rule and its OTTL `set()` statement entirely rather than emit an empty value,
     # and a 0 ratio must still render a valid float.
     - name: Render the minimal-identity configuration
@@ -67,6 +67,7 @@
       vars:
         _ga_instance_id: decdn-node-2
         grafana_alloy_bin_effective: /usr/local/bin/alloy
+        grafana_alloy_service_name: ""
         grafana_alloy_service_namespace: ""
         grafana_alloy_deployment_environment: ""
         grafana_alloy_region: ""

ansible/tests/alloy-config/validate.sh

@@ -137,6 +137,14 @@ assert_has defaults.alloy 'loki.write "cloud"' "the Loki writer"
 assert_has defaults.alloy '"job"' "an explicit job label"
 assert_has defaults.alloy 'integrations/node_exporter' "the Grafana Cloud integration job label"
 assert_has defaults.alloy '__journal__systemd_unit' "the journal field mapping"
+# Daemon log-stream service_name rule. ALIGNMENT-SENSITIVE needles: the
+# one-space `target_label = "service_name"` / `replacement = …` forms belong to
+# the node-metric identity rule, so only these column-aligned spellings pin the
+# journald rule.
+assert_has defaults.alloy 'source_labels = ["unit"]' "the unit scoping of the daemon log-stream rule"
+assert_has defaults.alloy 'regex         = "decdn-node\\.service"' "the daemon unit match"
+assert_has defaults.alloy 'target_label  = "service_name"' "the daemon log-stream service_name rule"
+assert_has defaults.alloy 'replacement   = "decdn-node"' "the daemon log-stream service_name value"
 assert_has defaults.alloy 'systemd {' "the per-unit systemd collector"
 
 # Sub-knob isolation.
@@ -149,6 +157,13 @@ assert_has logsonly.alloy 'path           = "/var/log/journal"' "the explicit jo
 assert_lacks logsonly.alloy 'prometheus.exporter' "any exporter while only logs are enabled"
 assert_lacks logsonly.alloy 'action        = "drop"' "a drop rule after both guardrail regexes were cleared"
 
+# A blank service_name switches the whole identity off together: metric rule,
+# journald rule and OTTL statement. The has-check keeps the lacks-checks honest
+# (they must not pass merely because logs went missing).
+assert_has minimal.alloy 'loki.relabel "journal_identity"' "the journal identity relabeller, which must survive a blank service_name"
+assert_lacks minimal.alloy '"service_name"' "any service_name rule (metric or log) once it was blanked"
+assert_lacks minimal.alloy 'service.name' "the OTLP service.name attribute once it was blanked"
+
 # BACKWARDS COMPATIBILITY: sub-knobs off == the pre-machine-monitoring pipeline.
 assert_lacks legacy.alloy 'prometheus.exporter' "any exporter"
 assert_lacks legacy.alloy 'discovery.relabel' "any target relabeller"