From 2aa739962640273e6185afc8b852a3d0b598aa59 Mon Sep 17 00:00:00 2001 From: Ant Somers Date: Sat, 19 Sep 2026 22:37:56 +0300 Subject: [PATCH 1/2] fix(ansible): take decdn-node's Loki level from its JSON log field MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit journald gives every stdout line priority 6, so the journal_rules mapping labelled all daemon lines level="info" — WARN and ERROR included — and level-filtered queries (error counts) returned zero. The debug guardrail never dropped the daemon's DEBUG lines for the same reason. Add loki.process "daemon_level", scoped to unit="decdn-node.service": parse the tracing JSON `level`, normalise it to journald's keywords (WARN -> warning), set the label, and apply the anchored priority-drop regex. Non-JSON lines and other units keep their journald level. make lint-alloy now runs the rendered stage in the real pinned Alloy and asserts the resulting labels, and fails against the old behaviour. Co-Authored-By: Claude Opus 5 (1M context) --- ansible/molecule/grafana-cloud/verify.yml | 4 + ansible/roles/grafana_alloy/README.md | 4 + ansible/roles/grafana_alloy/defaults/main.yml | 4 +- .../grafana_alloy/templates/config.alloy.j2 | 45 +++++++++- ansible/tests/alloy-config/validate.sh | 84 +++++++++++++++++++ 5 files changed, 139 insertions(+), 2 deletions(-) diff --git a/ansible/molecule/grafana-cloud/verify.yml b/ansible/molecule/grafana-cloud/verify.yml index b40bbde..d361e3a 100644 --- a/ansible/molecule/grafana-cloud/verify.yml +++ b/ansible/molecule/grafana-cloud/verify.yml @@ -127,6 +127,10 @@ 'loki.relabel "journal_rules"' in ga_config - >- '__journal__systemd_unit' in ga_config + # The daemon's level from its JSON body, not journald's priority + # (which is info for every stdout line). + - >- + 'loki.process "daemon_level"' in ga_config - >- 'loki.write "cloud"' in ga_config # The node's own series carry an explicit job rather than inheriting diff --git a/ansible/roles/grafana_alloy/README.md b/ansible/roles/grafana_alloy/README.md index 966b6cb..72a236d 100644 --- a/ansible/roles/grafana_alloy/README.md +++ b/ansible/roles/grafana_alloy/README.md @@ -230,6 +230,10 @@ deliberately lean and every lever is a variable: (`grafana_alloy_logs_drop_priority_regex` / `_drop_unit_regex`, `""` disables either), and `grafana_alloy_logs_max_age` bounds the catch-up burst after an outage — without it a restart can replay days of journal in one go. +- decdn-node's `level` label comes from its JSON `level` field, not the journald + priority (journald reports every stdout line as `info`). It uses journald's + keywords, so query `level="warning"`, not `warn`. Lines that are not JSON keep + the journald level. ## Hardening: the two relaxations machine monitoring requires diff --git a/ansible/roles/grafana_alloy/defaults/main.yml b/ansible/roles/grafana_alloy/defaults/main.yml index 2ff8356..d974906 100644 --- a/ansible/roles/grafana_alloy/defaults/main.yml +++ b/ansible/roles/grafana_alloy/defaults/main.yml @@ -225,7 +225,9 @@ grafana_alloy_logs_journal_path: "" # Cost guardrails, applied while the __journal* fields still exist. Both are # anchored regexes matched against the whole value; "" disables that rule. # Priority keywords are emerg/alert/crit/error/warning/notice/info/debug -# (journald's numeric priority 3 surfaces as "error", NOT "err"). +# (journald's numeric priority 3 surfaces as "error", NOT "err"). The priority +# regex also drops decdn-node lines by their JSON `level`, normalised to these +# same keywords (WARN -> warning, TRACE -> trace). grafana_alloy_logs_drop_priority_regex: 'debug' grafana_alloy_logs_drop_unit_regex: '(session-\d+\.scope|user@\d+\.service|systemd-timesyncd\.service|cron\.service|systemd-logind\.service)' diff --git a/ansible/roles/grafana_alloy/templates/config.alloy.j2 b/ansible/roles/grafana_alloy/templates/config.alloy.j2 index c36ae2b..57a2259 100644 --- a/ansible/roles/grafana_alloy/templates/config.alloy.j2 +++ b/ansible/roles/grafana_alloy/templates/config.alloy.j2 @@ -296,7 +296,50 @@ loki.source.journal "host" { max_age = {{ grafana_alloy_logs_max_age | to_json }} format_as_json = false relabel_rules = loki.relabel.journal_rules.rules - forward_to = [loki.relabel.journal_identity.receiver] + forward_to = [loki.process.daemon_level.receiver] +} + +{# The daemon's own severity. decdn-node logs JSON to stdout, and journald gives + every stdout line priority 6, so the journal_rules mapping above labels all of + them level="info" — WARN and ERROR included. The real level is the `level` + field of the tracing-subscriber JSON body, e.g. {"level":"WARN",...}. + + This runs after loki.source.journal, where the __journal* fields are already + gone, so it keys off the `unit` label journal_rules produced (the same unit + name the service_name rule below pins — a rename must be mirrored in both). + The value is normalised to journald's own keywords (WARN -> warning), so one + `level` vocabulary spans every unit. A line that is not JSON (log_format = + "text", or a panic written raw to stderr) extracts nothing; the template then + renders empty, which removes the key, and the journald level stands. #} +loki.process "daemon_level" { + forward_to = [loki.relabel.journal_identity.receiver] + + stage.match { + selector = `{unit="decdn-node.service"}` + + stage.json { + expressions = { decdn_level = "level" } + } + + stage.template { + source = "decdn_level" + template = {% raw %}`{{ with .Value }}{{ if eq (ToUpper .) "WARN" }}warning{{ else }}{{ ToLower . }}{{ end }}{{ end }}`{% endraw %} + } + + stage.labels { + values = { level = "decdn_level" } + } +{% if grafana_alloy_logs_drop_priority_regex | length > 0 %} + + // The priority guardrail again, for the daemon's DEBUG lines: journald + // reported them as info, so the journal_rules drop never saw them. Unlike + // relabel regexes, stage.drop is NOT anchored, hence the explicit ^...$. + stage.drop { + source = "decdn_level" + expression = {{ ('^(?:' ~ grafana_alloy_logs_drop_priority_regex ~ ')$') | tojson }} + } +{% endif %} + } } {# Same job/instance pair as the host metrics above — Grafana Cloud's Linux diff --git a/ansible/tests/alloy-config/validate.sh b/ansible/tests/alloy-config/validate.sh index bb2644e..a9a474b 100755 --- a/ansible/tests/alloy-config/validate.sh +++ b/ansible/tests/alloy-config/validate.sh @@ -146,6 +146,12 @@ assert_has defaults.alloy 'regex = "decdn-node\\.service"' "the daemon u assert_has defaults.alloy 'target_label = "service_name"' "the daemon log-stream service_name rule" assert_has defaults.alloy 'replacement = "decdn-node"' "the daemon log-stream service_name value" assert_has defaults.alloy 'systemd {' "the per-unit systemd collector" +assert_has defaults.alloy 'loki.process "daemon_level"' "the daemon JSON-level stage" +# shellcheck disable=SC2016 # literal backticks: Alloy's raw-string syntax +assert_has defaults.alloy 'selector = `{unit="decdn-node.service"}`' "the daemon unit scoping of the level stage" +assert_has defaults.alloy 'expression = "^(?:debug)$"' "the anchored daemon debug drop" +assert_has logsonly.alloy 'loki.process "daemon_level"' "the daemon JSON-level stage" +assert_lacks logsonly.alloy 'stage.drop' "a daemon level drop after the priority guardrail was cleared" # Sub-knob isolation. assert_has hostonly.alloy 'prometheus.exporter.unix "host"' "the host metrics exporter" @@ -217,6 +223,84 @@ assert_lacks hostonly.service 'SupplementaryGroups' "journal groups while logs a assert_lacks legacy.service 'SupplementaryGroups' "journal groups while logs are disabled" echo "ok: machine-monitoring render matrix" +# --- Gate 1c: the daemon's level comes from its JSON body ---------------------- +# Loading proves syntax, not behaviour. Lift the RENDERED +# loki.process "daemon_level" block out of defaults.alloy, feed it sample lines +# through loki.source.api, and read what loki.echo prints: the level label must +# follow the JSON field (in journald's vocabulary), debug must be dropped, and a +# text line or another unit must keep its journald level. +harness="$work/level-harness" +mkdir -p "$harness/data" +read -r api_port http_port < <(python3 -c ' +import socket +socks = [socket.socket() for _ in range(2)] +for s in socks: s.bind(("127.0.0.1", 0)) +print(*(s.getsockname()[1] for s in socks))') +{ + cat <"$harness/config.alloy" +grep -qF 'forward_to = [loki.echo.out.receiver]' "$harness/config.alloy" \ + || fail "could not lift loki.process \"daemon_level\" out of defaults.alloy" + +"$alloy" run "$harness/config.alloy" --storage.path="$harness/data" \ + --server.http.listen-addr="127.0.0.1:$http_port" >"$harness/alloy.log" 2>&1 & +alloy_pid=$! +trap 'kill "$alloy_pid" 2>/dev/null || true; cleanup' EXIT +for _ in $(seq 100); do + curl -fs "http://127.0.0.1:$http_port/-/ready" >/dev/null && break + kill -0 "$alloy_pid" 2>/dev/null || { cat "$harness/alloy.log" >&2; fail "alloy run exited on the level harness"; } + sleep 0.2 +done + +now="$(date +%s%N)" +curl -fsS -H 'Content-Type: application/json' "http://127.0.0.1:$api_port/loki/api/v1/push" --data @- </dev/null || true +wait "$alloy_pid" 2>/dev/null || true + +assert_entry() { # marker, expected labels, why + grep -F "$1" "$harness/alloy.log" | grep -qF "labels=\"$2\"" \ + || fail "level harness: $3 — got: $(grep -F "$1" "$harness/alloy.log" || echo '')" +} +assert_entry m-warn '{level=\"warning\", unit=\"decdn-node.service\"}' "a JSON WARN must be level=warning" +assert_entry m-error '{level=\"error\", unit=\"decdn-node.service\"}' "a JSON ERROR must be level=error" +assert_entry m-text '{level=\"info\", unit=\"decdn-node.service\"}' "a non-JSON line must keep its journald level" +assert_entry m-other '{level=\"info\", unit=\"other.service\"}' "another unit's JSON must not be re-levelled" +! grep -qF m-debug "$harness/alloy.log" \ + || fail "level harness: a JSON DEBUG line survived the priority guardrail" +echo "ok: daemon log level follows the JSON body" + # --- Gate 2: every ExecStart flag exists ------------------------------------- # `alloy run` ignores nothing: an unknown flag exits non-zero, i.e. a systemd # crash-loop on the target host the moment the unit starts. From 21274c19d9a88b281ebe2b8fcc5c1fac37a67d50 Mon Sep 17 00:00:00 2001 From: Ant Somers Date: Sat, 19 Sep 2026 22:48:56 +0300 Subject: [PATCH 2/2] fix(ansible): close review findings on the daemon log-level stage - Exempt decdn-node from the journald-priority drop in journal_rules and let daemon_level apply the same regex to its JSON level: every daemon line is journald-info, so a guardrail containing `info` dropped its warnings and errors before the JSON was ever read. - Default the priority guardrail to 'debug|trace'. TRACE normalises to `trace`, not a journald keyword, so it bypassed the old 'debug' default. - Accept only the five tracing levels (trimmed, case-folded); anything else keeps the journald level instead of becoming a label value. - `{% endraw +%}`: trim_blocks glued the stage's closing brace onto the template line in the rendered config. - Fix comments: upstream's formats are pretty/json (no "text"), the three places the unit name is hard-coded, and the empty-template path. validate.sh: assert the stage is wired in both directions, run the rendered journal_rules + daemon_level through the real Alloy for the defaults and a new info-dropping render variant, cover TRACE/lowercase/no-level/non-string cases, fail explicitly on readiness/entry-count timeouts and on an Alloy crash, and dump the Alloy log before cleanup deletes it. Co-Authored-By: Claude Opus 5 (1M context) --- ansible/roles/grafana_alloy/README.md | 9 +- ansible/roles/grafana_alloy/defaults/main.yml | 11 +- .../grafana_alloy/templates/config.alloy.j2 | 48 +++- ansible/tests/alloy-config/render.yml | 18 ++ ansible/tests/alloy-config/validate.sh | 207 +++++++++++++----- 5 files changed, 216 insertions(+), 77 deletions(-) diff --git a/ansible/roles/grafana_alloy/README.md b/ansible/roles/grafana_alloy/README.md index 72a236d..8ebfa8d 100644 --- a/ansible/roles/grafana_alloy/README.md +++ b/ansible/roles/grafana_alloy/README.md @@ -226,14 +226,15 @@ deliberately lean and every lever is a variable: - Host metrics scrape at `60s`, not the node's `30s`. - The `filesystem` / `netdev` / `disk` exclude regexes drop virtual filesystems and container/virtual interfaces. -- journald drops `debug` priority and a noisy-unit list +- journald drops `debug` (and the daemon's `trace`) and a noisy-unit list (`grafana_alloy_logs_drop_priority_regex` / `_drop_unit_regex`, `""` disables either), and `grafana_alloy_logs_max_age` bounds the catch-up burst after an outage — without it a restart can replay days of journal in one go. - decdn-node's `level` label comes from its JSON `level` field, not the journald - priority (journald reports every stdout line as `info`). It uses journald's - keywords, so query `level="warning"`, not `warn`. Lines that are not JSON keep - the journald level. + priority (journald reports every stdout line as `info`), and the priority drop + regex is applied to that JSON level for this unit. Values follow journald's + keywords, so query `level="warning"`, not `warn`; TRACE becomes `trace`. Lines + that are not JSON keep the journald level and are never priority-dropped. ## Hardening: the two relaxations machine monitoring requires diff --git a/ansible/roles/grafana_alloy/defaults/main.yml b/ansible/roles/grafana_alloy/defaults/main.yml index d974906..6e873e5 100644 --- a/ansible/roles/grafana_alloy/defaults/main.yml +++ b/ansible/roles/grafana_alloy/defaults/main.yml @@ -225,10 +225,13 @@ grafana_alloy_logs_journal_path: "" # Cost guardrails, applied while the __journal* fields still exist. Both are # anchored regexes matched against the whole value; "" disables that rule. # Priority keywords are emerg/alert/crit/error/warning/notice/info/debug -# (journald's numeric priority 3 surfaces as "error", NOT "err"). The priority -# regex also drops decdn-node lines by their JSON `level`, normalised to these -# same keywords (WARN -> warning, TRACE -> trace). -grafana_alloy_logs_drop_priority_regex: 'debug' +# (journald's numeric priority 3 surfaces as "error", NOT "err"). +# decdn-node is judged by its JSON `level` instead of its journald priority +# (always info), anchored the same way (^(?:...)$) and normalised to these +# keywords: WARN -> warning, DEBUG/INFO/ERROR unchanged, TRACE -> trace. `trace` +# is not a journald keyword, which is why the default lists it: without it the +# daemon's most verbose level would bypass the guardrail. +grafana_alloy_logs_drop_priority_regex: 'debug|trace' grafana_alloy_logs_drop_unit_regex: '(session-\d+\.scope|user@\d+\.service|systemd-timesyncd\.service|cron\.service|systemd-logind\.service)' diff --git a/ansible/roles/grafana_alloy/templates/config.alloy.j2 b/ansible/roles/grafana_alloy/templates/config.alloy.j2 index 57a2259..0d205fb 100644 --- a/ansible/roles/grafana_alloy/templates/config.alloy.j2 +++ b/ansible/roles/grafana_alloy/templates/config.alloy.j2 @@ -268,8 +268,27 @@ loki.relabel "journal_rules" { // Cost guardrail: the regex is anchored to the WHOLE value by Prometheus // relabel semantics, so "debug" drops debug and nothing else. + // + // decdn-node is exempt: journald reports every line it writes as info, so + // judging it by priority would be judging it by a level it never set (a + // regex containing "info" would drop its errors). loki.process + // "daemon_level" below applies the same regex to its real, JSON level. The + // exemption blanks a scratch copy of the keyword for that one unit; like + // every __-prefixed label it never leaves this ruleset. rule { source_labels = ["__journal_priority_keyword"] + target_label = "__priority_guardrail" + } + + rule { + source_labels = ["__journal__systemd_unit"] + regex = "decdn-node\\.service" + target_label = "__priority_guardrail" + replacement = "" + } + + rule { + source_labels = ["__priority_guardrail"] regex = {{ grafana_alloy_logs_drop_priority_regex | tojson }} action = "drop" } @@ -305,12 +324,19 @@ loki.source.journal "host" { field of the tracing-subscriber JSON body, e.g. {"level":"WARN",...}. This runs after loki.source.journal, where the __journal* fields are already - gone, so it keys off the `unit` label journal_rules produced (the same unit - name the service_name rule below pins — a rename must be mirrored in both). - The value is normalised to journald's own keywords (WARN -> warning), so one - `level` vocabulary spans every unit. A line that is not JSON (log_format = - "text", or a panic written raw to stderr) extracts nothing; the template then - renders empty, which removes the key, and the journald level stands. #} + gone, so it keys off the `unit` label journal_rules produced. The unit name + is the one decdn_node installs; it is also hard-coded in the journal_rules + priority exemption above and the service_name rule below — a rename must be + mirrored in all three (validate.sh pins them). + + Only the five tracing levels are accepted, trimmed and case-folded, and WARN + becomes journald's `warning`; DEBUG/INFO/ERROR already match journald's + keywords. TRACE stays `trace`, which journald has no keyword for, so the + priority guardrail default lists it explicitly. Anything else — a line that + is not JSON (log_format = "pretty", or a panic written raw to stderr), JSON + with no level, or an unexpected value — leaves decdn_level unset (the + template stage deletes a key it renders empty), so stage.labels and + stage.drop skip the line and its journald level stands. #} loki.process "daemon_level" { forward_to = [loki.relabel.journal_identity.receiver] @@ -323,7 +349,7 @@ loki.process "daemon_level" { stage.template { source = "decdn_level" - template = {% raw %}`{{ with .Value }}{{ if eq (ToUpper .) "WARN" }}warning{{ else }}{{ ToLower . }}{{ end }}{{ end }}`{% endraw %} + template = {% raw %}`{{ $l := ToLower (TrimSpace (print .Value)) }}{{ if eq $l "warn" "warning" }}warning{{ else if eq $l "trace" "debug" "info" "error" }}{{ $l }}{{ end }}`{% endraw +%} } stage.labels { @@ -331,8 +357,8 @@ loki.process "daemon_level" { } {% if grafana_alloy_logs_drop_priority_regex | length > 0 %} - // The priority guardrail again, for the daemon's DEBUG lines: journald - // reported them as info, so the journal_rules drop never saw them. Unlike + // The priority guardrail for the daemon, which journal_rules exempts: the + // same regex, applied to the level the daemon actually logged. Unlike // relabel regexes, stage.drop is NOT anchored, hence the explicit ^...$. stage.drop { source = "decdn_level" @@ -372,7 +398,9 @@ loki.relabel "journal_identity" { so this catches only lines the daemon process writes: systemd's own start/stop/crash messages about it come from PID 1 as unit="init.scope". The name is the unit decdn_node installs, /etc/systemd/system/decdn-node.service - — a rename there must be mirrored here (validate.sh pins it). #} + — a rename there must be mirrored here AND in the journal_rules priority + exemption and the loki.process "daemon_level" selector above (validate.sh + pins all three), plus grafana_alloy_host_systemd_unit_include. #} rule { source_labels = ["unit"] diff --git a/ansible/tests/alloy-config/render.yml b/ansible/tests/alloy-config/render.yml index 70e0e41..dbb6960 100644 --- a/ansible/tests/alloy-config/render.yml +++ b/ansible/tests/alloy-config/render.yml @@ -257,6 +257,24 @@ # Hands validate.sh the release pin without making it parse YAML itself: the # values come from the role's own defaults, so the download it verifies is # byte-identical to the one the role installs. + # A priority guardrail that includes `info`: journald reports every + # decdn-node line as info, so this proves the daemon is judged by its JSON + # level instead (its warnings and errors must survive). validate.sh runs + # this file through the real pipeline. + - name: Render the info-dropping priority-guardrail configuration + ansible.builtin.template: + src: "{{ playbook_dir }}/../../roles/grafana_alloy/templates/{{ item.src }}" + dest: "{{ _ga_render_dir }}/priorityinfo.{{ item.ext }}" + mode: "0644" + loop: *ga_render_pair + loop_control: + label: "priorityinfo.{{ item.ext }}" + vars: + _ga_instance_id: decdn-node-1 + _ga_journal_groups: [systemd-journal, adm] + grafana_alloy_bin_effective: /usr/bin/alloy + grafana_alloy_logs_drop_priority_regex: 'info|debug|trace' + - name: Export the pinned release for the validation script ansible.builtin.copy: dest: "{{ _ga_render_dir }}/pin.env" diff --git a/ansible/tests/alloy-config/validate.sh b/ansible/tests/alloy-config/validate.sh index a9a474b..a9722af 100755 --- a/ansible/tests/alloy-config/validate.sh +++ b/ansible/tests/alloy-config/validate.sh @@ -146,12 +146,25 @@ assert_has defaults.alloy 'regex = "decdn-node\\.service"' "the daemon u assert_has defaults.alloy 'target_label = "service_name"' "the daemon log-stream service_name rule" assert_has defaults.alloy 'replacement = "decdn-node"' "the daemon log-stream service_name value" assert_has defaults.alloy 'systemd {' "the per-unit systemd collector" +# The daemon-level stage: present, WIRED IN (an orphaned loki.process still +# loads, so Gate 1c alone could not tell), scoped to the daemon's unit, and the +# guardrail anchored with its alternation grouped. Each unit-name needle is one +# of the three sites a decdn-node.service rename must touch. assert_has defaults.alloy 'loki.process "daemon_level"' "the daemon JSON-level stage" +assert_has defaults.alloy 'forward_to = [loki.process.daemon_level.receiver]' "the journal source feeding the daemon-level stage" +assert_has defaults.alloy 'forward_to = [loki.relabel.journal_identity.receiver]' "the daemon-level stage feeding the identity relabeller" # shellcheck disable=SC2016 # literal backticks: Alloy's raw-string syntax assert_has defaults.alloy 'selector = `{unit="decdn-node.service"}`' "the daemon unit scoping of the level stage" -assert_has defaults.alloy 'expression = "^(?:debug)$"' "the anchored daemon debug drop" +assert_has defaults.alloy 'target_label = "__priority_guardrail"' "the daemon's exemption from the journald priority drop" +assert_has defaults.alloy 'expression = "^(?:debug|trace)$"' "the anchored, grouped daemon level drop" +# trim_blocks eats the newline after a bare {% endraw %}; the stage's closing +# brace must stay on its own line. +# shellcheck disable=SC2016 # literal backtick +grep -qE '^ template = `.*`$' "$render_dir/defaults.alloy" \ + || fail "defaults.alloy: the stage.template line does not end at its closing backtick" assert_has logsonly.alloy 'loki.process "daemon_level"' "the daemon JSON-level stage" assert_lacks logsonly.alloy 'stage.drop' "a daemon level drop after the priority guardrail was cleared" +assert_lacks logsonly.alloy '__priority_guardrail' "a priority exemption after the priority guardrail was cleared" # Sub-knob isolation. assert_has hostonly.alloy 'prometheus.exporter.unix "host"' "the host metrics exporter" @@ -224,82 +237,158 @@ assert_lacks legacy.service 'SupplementaryGroups' "journal groups while logs are echo "ok: machine-monitoring render matrix" # --- Gate 1c: the daemon's level comes from its JSON body ---------------------- -# Loading proves syntax, not behaviour. Lift the RENDERED -# loki.process "daemon_level" block out of defaults.alloy, feed it sample lines -# through loki.source.api, and read what loki.echo prints: the level label must -# follow the JSON field (in journald's vocabulary), debug must be dropped, and a -# text line or another unit must keep its journald level. +# Loading proves syntax, not behaviour. Run the RENDERED journal_rules and +# loki.process "daemon_level" blocks in the real Alloy, fed sample lines through +# loki.source.api, and read what loki.echo prints. loki.source.api strips +# __-prefixed labels on push, so the lifted rules read journal_* instead of +# __journal_* (the only edit), and a final labeldrop removes those raw labels +# so the exact label set can be asserted. harness="$work/level-harness" -mkdir -p "$harness/data" -read -r api_port http_port < <(python3 -c ' +mkdir -p "$harness" +harness_log="" +harness_fail() { # why — dumps the Alloy log, which the EXIT trap deletes + if [ -n "$harness_log" ] && [ -f "$harness_log" ]; then cat "$harness_log" >&2; fi + fail "level harness ($harness_log): $1" +} + +# Each stream shares one journald priority. Within a stream the lines that +# must be DROPPED come first, and the drop-only streams come first, so by the +# time the last kept line is echoed every dropped one has been processed. +samples_json() { # nanosecond timestamp + cat <"$harness/config.alloy" -grep -qF 'forward_to = [loki.echo.out.receiver]' "$harness/config.alloy" \ - || fail "could not lift loki.process \"daemon_level\" out of defaults.alloy" - -"$alloy" run "$harness/config.alloy" --storage.path="$harness/data" \ - --server.http.listen-addr="127.0.0.1:$http_port" >"$harness/alloy.log" 2>&1 & -alloy_pid=$! -trap 'kill "$alloy_pid" 2>/dev/null || true; cleanup' EXIT -for _ in $(seq 100); do - curl -fs "http://127.0.0.1:$http_port/-/ready" >/dev/null && break - kill -0 "$alloy_pid" 2>/dev/null || { cat "$harness/alloy.log" >&2; fail "alloy run exited on the level harness"; } - sleep 0.2 -done + awk '/^loki\.relabel "journal_rules" \{/ { on = 1 } + on && /^\}/ { print "\n rule {\n regex = \"journal_.*\"\n action = \"labeldrop\"\n }"; print; exit } + on { print }' "$src" | sed 's/"__journal_/"journal_/g' + awk '/^loki\.process "daemon_level" \{/ { on = 1 } on { print } on && /^\}/ { exit }' "$src" \ + | sed 's|forward_to = \[loki\.relabel\.journal_identity\.receiver\]|forward_to = [loki.echo.out.receiver]|' + } >"$config" + if ! grep -qF 'action = "labeldrop"' "$config" || ! grep -qF 'forward_to = [loki.echo.out.receiver]' "$config"; then + fail "could not lift journal_rules and daemon_level out of $1" + fi -now="$(date +%s%N)" -curl -fsS -H 'Content-Type: application/json' "http://127.0.0.1:$api_port/loki/api/v1/push" --data @- <"$harness_log" 2>&1 & + alloy_pid=$! + trap 'kill "$alloy_pid" 2>/dev/null || true; cleanup' EXIT + local ready="" + for _ in $(seq 100); do + if curl -fs "http://127.0.0.1:$http_port/-/ready" >/dev/null; then ready=1; break; fi + kill -0 "$alloy_pid" 2>/dev/null || harness_fail "alloy run exited before becoming ready" + sleep 0.2 + done + [ -n "$ready" ] || harness_fail "alloy never became ready" -# The pipeline is asynchronous: wait for the last expected entry, then assert. -for _ in $(seq 50); do - [ "$(grep -c 'received log entry' "$harness/alloy.log")" -ge 4 ] && break - sleep 0.1 -done -sleep 0.5 # let a (wrongly) undropped debug line land too -kill "$alloy_pid" 2>/dev/null || true -wait "$alloy_pid" 2>/dev/null || true + # /-/ready means the graph is loaded; the API listener may bind a moment later. + samples_json "$(date +%s%N)" | curl -fsS --retry 10 --retry-connrefused --retry-delay 0 \ + -H 'Content-Type: application/json' --data @- "http://127.0.0.1:$api_port/loki/api/v1/push" \ + || harness_fail "pushing sample lines failed" -assert_entry() { # marker, expected labels, why - grep -F "$1" "$harness/alloy.log" | grep -qF "labels=\"$2\"" \ - || fail "level harness: $3 — got: $(grep -F "$1" "$harness/alloy.log" || echo '')" + local got=0 + for _ in $(seq 50); do + got="$(grep -c 'received log entry' "$harness_log" || true)" + [ "$got" -ge "$expect" ] && break + sleep 0.1 + done + sleep 0.5 # let a (wrongly) undropped line land too + kill "$alloy_pid" 2>/dev/null || true + local rc=0 + wait "$alloy_pid" || rc=$? + trap cleanup EXIT + # 143 = our SIGTERM; anything else means Alloy died on its own mid-run. + [ "$rc" -eq 143 ] || [ "$rc" -eq 0 ] || harness_fail "alloy run exited $rc" + got="$(grep -c 'received log entry' "$harness_log" || true)" + [ "$got" -eq "$expect" ] || harness_fail "expected $expect entries through the pipeline, got $got" } -assert_entry m-warn '{level=\"warning\", unit=\"decdn-node.service\"}' "a JSON WARN must be level=warning" -assert_entry m-error '{level=\"error\", unit=\"decdn-node.service\"}' "a JSON ERROR must be level=error" -assert_entry m-text '{level=\"info\", unit=\"decdn-node.service\"}' "a non-JSON line must keep its journald level" -assert_entry m-other '{level=\"info\", unit=\"other.service\"}' "another unit's JSON must not be re-levelled" -! grep -qF m-debug "$harness/alloy.log" \ - || fail "level harness: a JSON DEBUG line survived the priority guardrail" -echo "ok: daemon log level follows the JSON body" + +# loki.echo prints labels logfmt-escaped (labels="{level=\"info\", ...}"); an +# Alloy bump that changes that format fails here loudly, never falsely passes. +kept() { # marker, level, unit + grep -F "$1" "$harness_log" | grep -qF "labels=\"{level=\\\"$2\\\", unit=\\\"$3\\\"}\"" \ + || harness_fail "$1 should be kept as level=\"$2\" unit=\"$3\"" +} +dropped() { # marker + ! grep -qF "$1" "$harness_log" || harness_fail "$1 should have been dropped" +} + +# Defaults (guardrail "debug|trace"). +run_level_harness defaults.alloy 9 +kept m-warn-uc warning decdn-node.service +kept m-warn-lc warning decdn-node.service +kept m-error error decdn-node.service +kept m-info info decdn-node.service +kept m-nolevel info decdn-node.service # JSON without a level: journald's stands +kept m-weird info decdn-node.service # a non-string level is not a label value +kept m-text info decdn-node.service # not JSON: journald's stands +kept m-other-info info other.service # another unit's JSON is never re-levelled +kept m-other-error error other.service +dropped m-trace # TRACE is not a journald keyword, so listed +dropped m-debug # padded + lower-cased before the drop +dropped m-other-debug # the journald-priority drop, other units +echo "ok: daemon log level follows the JSON body (defaults)" + +# Guardrail "info|debug|trace": the daemon's every line is journald-info, so it +# must be judged by its JSON level — errors and warnings survive, and lines with +# no JSON level are never dropped for a priority they did not choose. +run_level_harness priorityinfo.alloy 7 +kept m-warn-uc warning decdn-node.service +kept m-warn-lc warning decdn-node.service +kept m-error error decdn-node.service +kept m-nolevel info decdn-node.service +kept m-weird info decdn-node.service +kept m-text info decdn-node.service +kept m-other-error error other.service +dropped m-info +dropped m-trace +dropped m-debug +dropped m-other-info +dropped m-other-debug +echo "ok: daemon log level follows the JSON body (info in the guardrail)" # --- Gate 2: every ExecStart flag exists ------------------------------------- # `alloy run` ignores nothing: an unknown flag exits non-zero, i.e. a systemd