diff --git a/ansible/molecule/grafana-cloud/verify.yml b/ansible/molecule/grafana-cloud/verify.yml index b40bbde..d361e3a 100644 --- a/ansible/molecule/grafana-cloud/verify.yml +++ b/ansible/molecule/grafana-cloud/verify.yml @@ -127,6 +127,10 @@ 'loki.relabel "journal_rules"' in ga_config - >- '__journal__systemd_unit' in ga_config + # The daemon's level from its JSON body, not journald's priority + # (which is info for every stdout line). + - >- + 'loki.process "daemon_level"' in ga_config - >- 'loki.write "cloud"' in ga_config # The node's own series carry an explicit job rather than inheriting diff --git a/ansible/roles/grafana_alloy/README.md b/ansible/roles/grafana_alloy/README.md index 966b6cb..8ebfa8d 100644 --- a/ansible/roles/grafana_alloy/README.md +++ b/ansible/roles/grafana_alloy/README.md @@ -226,10 +226,15 @@ deliberately lean and every lever is a variable: - Host metrics scrape at `60s`, not the node's `30s`. - The `filesystem` / `netdev` / `disk` exclude regexes drop virtual filesystems and container/virtual interfaces. -- journald drops `debug` priority and a noisy-unit list +- journald drops `debug` (and the daemon's `trace`) and a noisy-unit list (`grafana_alloy_logs_drop_priority_regex` / `_drop_unit_regex`, `""` disables either), and `grafana_alloy_logs_max_age` bounds the catch-up burst after an outage — without it a restart can replay days of journal in one go. +- decdn-node's `level` label comes from its JSON `level` field, not the journald + priority (journald reports every stdout line as `info`), and the priority drop + regex is applied to that JSON level for this unit. Values follow journald's + keywords, so query `level="warning"`, not `warn`; TRACE becomes `trace`. Lines + that are not JSON keep the journald level and are never priority-dropped. ## Hardening: the two relaxations machine monitoring requires diff --git a/ansible/roles/grafana_alloy/defaults/main.yml b/ansible/roles/grafana_alloy/defaults/main.yml index 2ff8356..6e873e5 100644 --- a/ansible/roles/grafana_alloy/defaults/main.yml +++ b/ansible/roles/grafana_alloy/defaults/main.yml @@ -226,7 +226,12 @@ grafana_alloy_logs_journal_path: "" # anchored regexes matched against the whole value; "" disables that rule. # Priority keywords are emerg/alert/crit/error/warning/notice/info/debug # (journald's numeric priority 3 surfaces as "error", NOT "err"). -grafana_alloy_logs_drop_priority_regex: 'debug' +# decdn-node is judged by its JSON `level` instead of its journald priority +# (always info), anchored the same way (^(?:...)$) and normalised to these +# keywords: WARN -> warning, DEBUG/INFO/ERROR unchanged, TRACE -> trace. `trace` +# is not a journald keyword, which is why the default lists it: without it the +# daemon's most verbose level would bypass the guardrail. +grafana_alloy_logs_drop_priority_regex: 'debug|trace' grafana_alloy_logs_drop_unit_regex: '(session-\d+\.scope|user@\d+\.service|systemd-timesyncd\.service|cron\.service|systemd-logind\.service)' diff --git a/ansible/roles/grafana_alloy/templates/config.alloy.j2 b/ansible/roles/grafana_alloy/templates/config.alloy.j2 index c36ae2b..0d205fb 100644 --- a/ansible/roles/grafana_alloy/templates/config.alloy.j2 +++ b/ansible/roles/grafana_alloy/templates/config.alloy.j2 @@ -268,8 +268,27 @@ loki.relabel "journal_rules" { // Cost guardrail: the regex is anchored to the WHOLE value by Prometheus // relabel semantics, so "debug" drops debug and nothing else. + // + // decdn-node is exempt: journald reports every line it writes as info, so + // judging it by priority would be judging it by a level it never set (a + // regex containing "info" would drop its errors). loki.process + // "daemon_level" below applies the same regex to its real, JSON level. The + // exemption blanks a scratch copy of the keyword for that one unit; like + // every __-prefixed label it never leaves this ruleset. rule { source_labels = ["__journal_priority_keyword"] + target_label = "__priority_guardrail" + } + + rule { + source_labels = ["__journal__systemd_unit"] + regex = "decdn-node\\.service" + target_label = "__priority_guardrail" + replacement = "" + } + + rule { + source_labels = ["__priority_guardrail"] regex = {{ grafana_alloy_logs_drop_priority_regex | tojson }} action = "drop" } @@ -296,7 +315,57 @@ loki.source.journal "host" { max_age = {{ grafana_alloy_logs_max_age | to_json }} format_as_json = false relabel_rules = loki.relabel.journal_rules.rules - forward_to = [loki.relabel.journal_identity.receiver] + forward_to = [loki.process.daemon_level.receiver] +} + +{# The daemon's own severity. decdn-node logs JSON to stdout, and journald gives + every stdout line priority 6, so the journal_rules mapping above labels all of + them level="info" — WARN and ERROR included. The real level is the `level` + field of the tracing-subscriber JSON body, e.g. {"level":"WARN",...}. + + This runs after loki.source.journal, where the __journal* fields are already + gone, so it keys off the `unit` label journal_rules produced. The unit name + is the one decdn_node installs; it is also hard-coded in the journal_rules + priority exemption above and the service_name rule below — a rename must be + mirrored in all three (validate.sh pins them). + + Only the five tracing levels are accepted, trimmed and case-folded, and WARN + becomes journald's `warning`; DEBUG/INFO/ERROR already match journald's + keywords. TRACE stays `trace`, which journald has no keyword for, so the + priority guardrail default lists it explicitly. Anything else — a line that + is not JSON (log_format = "pretty", or a panic written raw to stderr), JSON + with no level, or an unexpected value — leaves decdn_level unset (the + template stage deletes a key it renders empty), so stage.labels and + stage.drop skip the line and its journald level stands. #} +loki.process "daemon_level" { + forward_to = [loki.relabel.journal_identity.receiver] + + stage.match { + selector = `{unit="decdn-node.service"}` + + stage.json { + expressions = { decdn_level = "level" } + } + + stage.template { + source = "decdn_level" + template = {% raw %}`{{ $l := ToLower (TrimSpace (print .Value)) }}{{ if eq $l "warn" "warning" }}warning{{ else if eq $l "trace" "debug" "info" "error" }}{{ $l }}{{ end }}`{% endraw +%} + } + + stage.labels { + values = { level = "decdn_level" } + } +{% if grafana_alloy_logs_drop_priority_regex | length > 0 %} + + // The priority guardrail for the daemon, which journal_rules exempts: the + // same regex, applied to the level the daemon actually logged. Unlike + // relabel regexes, stage.drop is NOT anchored, hence the explicit ^...$. + stage.drop { + source = "decdn_level" + expression = {{ ('^(?:' ~ grafana_alloy_logs_drop_priority_regex ~ ')$') | tojson }} + } +{% endif %} + } } {# Same job/instance pair as the host metrics above — Grafana Cloud's Linux @@ -329,7 +398,9 @@ loki.relabel "journal_identity" { so this catches only lines the daemon process writes: systemd's own start/stop/crash messages about it come from PID 1 as unit="init.scope". The name is the unit decdn_node installs, /etc/systemd/system/decdn-node.service - — a rename there must be mirrored here (validate.sh pins it). #} + — a rename there must be mirrored here AND in the journal_rules priority + exemption and the loki.process "daemon_level" selector above (validate.sh + pins all three), plus grafana_alloy_host_systemd_unit_include. #} rule { source_labels = ["unit"] diff --git a/ansible/tests/alloy-config/render.yml b/ansible/tests/alloy-config/render.yml index 70e0e41..dbb6960 100644 --- a/ansible/tests/alloy-config/render.yml +++ b/ansible/tests/alloy-config/render.yml @@ -257,6 +257,24 @@ # Hands validate.sh the release pin without making it parse YAML itself: the # values come from the role's own defaults, so the download it verifies is # byte-identical to the one the role installs. + # A priority guardrail that includes `info`: journald reports every + # decdn-node line as info, so this proves the daemon is judged by its JSON + # level instead (its warnings and errors must survive). validate.sh runs + # this file through the real pipeline. + - name: Render the info-dropping priority-guardrail configuration + ansible.builtin.template: + src: "{{ playbook_dir }}/../../roles/grafana_alloy/templates/{{ item.src }}" + dest: "{{ _ga_render_dir }}/priorityinfo.{{ item.ext }}" + mode: "0644" + loop: *ga_render_pair + loop_control: + label: "priorityinfo.{{ item.ext }}" + vars: + _ga_instance_id: decdn-node-1 + _ga_journal_groups: [systemd-journal, adm] + grafana_alloy_bin_effective: /usr/bin/alloy + grafana_alloy_logs_drop_priority_regex: 'info|debug|trace' + - name: Export the pinned release for the validation script ansible.builtin.copy: dest: "{{ _ga_render_dir }}/pin.env" diff --git a/ansible/tests/alloy-config/validate.sh b/ansible/tests/alloy-config/validate.sh index bb2644e..a9722af 100755 --- a/ansible/tests/alloy-config/validate.sh +++ b/ansible/tests/alloy-config/validate.sh @@ -146,6 +146,25 @@ assert_has defaults.alloy 'regex = "decdn-node\\.service"' "the daemon u assert_has defaults.alloy 'target_label = "service_name"' "the daemon log-stream service_name rule" assert_has defaults.alloy 'replacement = "decdn-node"' "the daemon log-stream service_name value" assert_has defaults.alloy 'systemd {' "the per-unit systemd collector" +# The daemon-level stage: present, WIRED IN (an orphaned loki.process still +# loads, so Gate 1c alone could not tell), scoped to the daemon's unit, and the +# guardrail anchored with its alternation grouped. Each unit-name needle is one +# of the three sites a decdn-node.service rename must touch. +assert_has defaults.alloy 'loki.process "daemon_level"' "the daemon JSON-level stage" +assert_has defaults.alloy 'forward_to = [loki.process.daemon_level.receiver]' "the journal source feeding the daemon-level stage" +assert_has defaults.alloy 'forward_to = [loki.relabel.journal_identity.receiver]' "the daemon-level stage feeding the identity relabeller" +# shellcheck disable=SC2016 # literal backticks: Alloy's raw-string syntax +assert_has defaults.alloy 'selector = `{unit="decdn-node.service"}`' "the daemon unit scoping of the level stage" +assert_has defaults.alloy 'target_label = "__priority_guardrail"' "the daemon's exemption from the journald priority drop" +assert_has defaults.alloy 'expression = "^(?:debug|trace)$"' "the anchored, grouped daemon level drop" +# trim_blocks eats the newline after a bare {% endraw %}; the stage's closing +# brace must stay on its own line. +# shellcheck disable=SC2016 # literal backtick +grep -qE '^ template = `.*`$' "$render_dir/defaults.alloy" \ + || fail "defaults.alloy: the stage.template line does not end at its closing backtick" +assert_has logsonly.alloy 'loki.process "daemon_level"' "the daemon JSON-level stage" +assert_lacks logsonly.alloy 'stage.drop' "a daemon level drop after the priority guardrail was cleared" +assert_lacks logsonly.alloy '__priority_guardrail' "a priority exemption after the priority guardrail was cleared" # Sub-knob isolation. assert_has hostonly.alloy 'prometheus.exporter.unix "host"' "the host metrics exporter" @@ -217,6 +236,160 @@ assert_lacks hostonly.service 'SupplementaryGroups' "journal groups while logs a assert_lacks legacy.service 'SupplementaryGroups' "journal groups while logs are disabled" echo "ok: machine-monitoring render matrix" +# --- Gate 1c: the daemon's level comes from its JSON body ---------------------- +# Loading proves syntax, not behaviour. Run the RENDERED journal_rules and +# loki.process "daemon_level" blocks in the real Alloy, fed sample lines through +# loki.source.api, and read what loki.echo prints. loki.source.api strips +# __-prefixed labels on push, so the lifted rules read journal_* instead of +# __journal_* (the only edit), and a final labeldrop removes those raw labels +# so the exact label set can be asserted. +harness="$work/level-harness" +mkdir -p "$harness" +harness_log="" +harness_fail() { # why — dumps the Alloy log, which the EXIT trap deletes + if [ -n "$harness_log" ] && [ -f "$harness_log" ]; then cat "$harness_log" >&2; fi + fail "level harness ($harness_log): $1" +} + +# Each stream shares one journald priority. Within a stream the lines that +# must be DROPPED come first, and the drop-only streams come first, so by the +# time the last kept line is echoed every dropped one has been processed. +samples_json() { # nanosecond timestamp + cat <"$config" + if ! grep -qF 'action = "labeldrop"' "$config" || ! grep -qF 'forward_to = [loki.echo.out.receiver]' "$config"; then + fail "could not lift journal_rules and daemon_level out of $1" + fi + + "$alloy" run "$config" --storage.path="$data" \ + --server.http.listen-addr="127.0.0.1:$http_port" >"$harness_log" 2>&1 & + alloy_pid=$! + trap 'kill "$alloy_pid" 2>/dev/null || true; cleanup' EXIT + local ready="" + for _ in $(seq 100); do + if curl -fs "http://127.0.0.1:$http_port/-/ready" >/dev/null; then ready=1; break; fi + kill -0 "$alloy_pid" 2>/dev/null || harness_fail "alloy run exited before becoming ready" + sleep 0.2 + done + [ -n "$ready" ] || harness_fail "alloy never became ready" + + # /-/ready means the graph is loaded; the API listener may bind a moment later. + samples_json "$(date +%s%N)" | curl -fsS --retry 10 --retry-connrefused --retry-delay 0 \ + -H 'Content-Type: application/json' --data @- "http://127.0.0.1:$api_port/loki/api/v1/push" \ + || harness_fail "pushing sample lines failed" + + local got=0 + for _ in $(seq 50); do + got="$(grep -c 'received log entry' "$harness_log" || true)" + [ "$got" -ge "$expect" ] && break + sleep 0.1 + done + sleep 0.5 # let a (wrongly) undropped line land too + kill "$alloy_pid" 2>/dev/null || true + local rc=0 + wait "$alloy_pid" || rc=$? + trap cleanup EXIT + # 143 = our SIGTERM; anything else means Alloy died on its own mid-run. + [ "$rc" -eq 143 ] || [ "$rc" -eq 0 ] || harness_fail "alloy run exited $rc" + got="$(grep -c 'received log entry' "$harness_log" || true)" + [ "$got" -eq "$expect" ] || harness_fail "expected $expect entries through the pipeline, got $got" +} + +# loki.echo prints labels logfmt-escaped (labels="{level=\"info\", ...}"); an +# Alloy bump that changes that format fails here loudly, never falsely passes. +kept() { # marker, level, unit + grep -F "$1" "$harness_log" | grep -qF "labels=\"{level=\\\"$2\\\", unit=\\\"$3\\\"}\"" \ + || harness_fail "$1 should be kept as level=\"$2\" unit=\"$3\"" +} +dropped() { # marker + ! grep -qF "$1" "$harness_log" || harness_fail "$1 should have been dropped" +} + +# Defaults (guardrail "debug|trace"). +run_level_harness defaults.alloy 9 +kept m-warn-uc warning decdn-node.service +kept m-warn-lc warning decdn-node.service +kept m-error error decdn-node.service +kept m-info info decdn-node.service +kept m-nolevel info decdn-node.service # JSON without a level: journald's stands +kept m-weird info decdn-node.service # a non-string level is not a label value +kept m-text info decdn-node.service # not JSON: journald's stands +kept m-other-info info other.service # another unit's JSON is never re-levelled +kept m-other-error error other.service +dropped m-trace # TRACE is not a journald keyword, so listed +dropped m-debug # padded + lower-cased before the drop +dropped m-other-debug # the journald-priority drop, other units +echo "ok: daemon log level follows the JSON body (defaults)" + +# Guardrail "info|debug|trace": the daemon's every line is journald-info, so it +# must be judged by its JSON level — errors and warnings survive, and lines with +# no JSON level are never dropped for a priority they did not choose. +run_level_harness priorityinfo.alloy 7 +kept m-warn-uc warning decdn-node.service +kept m-warn-lc warning decdn-node.service +kept m-error error decdn-node.service +kept m-nolevel info decdn-node.service +kept m-weird info decdn-node.service +kept m-text info decdn-node.service +kept m-other-error error other.service +dropped m-info +dropped m-trace +dropped m-debug +dropped m-other-info +dropped m-other-debug +echo "ok: daemon log level follows the JSON body (info in the guardrail)" + # --- Gate 2: every ExecStart flag exists ------------------------------------- # `alloy run` ignores nothing: an unknown flag exits non-zero, i.e. a systemd # crash-loop on the target host the moment the unit starts.