diff --git a/AGENTS.md b/AGENTS.md index 0392523..6efd051 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -87,9 +87,9 @@ charts/ the daemon's OTLP spans. Host metrics and logs carry `job="integrations/node_exporter"` so Grafana Cloud's prebuilt Linux Server dashboards work unmodified. Only the API token is host-provisioned (`0600 /etc/grafana-alloy.env`, read via `sys.env` — Alloy has no - `--config.expand-env`); the non-secret endpoints/instance IDs are inventory variables - that fall back to their `GC_…` env key when empty, and preflight refuses a token in any - of them. Two hardening relaxations are conditional on the signals being on + `--config.expand-env`); the non-secret endpoints and the three per-service instance IDs + are inventory variables that fall back to their `GC_…` env key when empty, and preflight + refuses a token in any of them. Two hardening relaxations are conditional on the signals being on (`ProtectHome=read-only` for correct filesystem metrics, `SupplementaryGroups= systemd-journal adm` for journal access — without which collection is silently empty); teardown is gated on the managed-by marker in the unit, so a foreign Alloy is never diff --git a/ansible/README.md b/ansible/README.md index 834e41b..278c905 100644 --- a/ansible/README.md +++ b/ansible/README.md @@ -205,9 +205,10 @@ dashboards and alert rules work unmodified. Only the API token is a secret. Put the non-secret connection settings in inventory once for the fleet (`grafana_alloy_prom_url`, `_prom_username`, `_otlp_endpoint`, -`_loki_url`, `_loki_username` — note the Loki instance ID differs from the Prometheus -one), and provision just the token **on each target host** (it never transits this repo -or the control machine): +`_otlp_username`, `_loki_url`, `_loki_username` — Prometheus, OTLP and Loki each have +their **own** instance ID, so copy each from the portal page that names it), and +provision just the token **on each target host** (it never transits this repo or the +control machine): ```bash umask 077 @@ -225,7 +226,10 @@ so preflight fails until you either set `grafana_alloy_loki_url` / `_loki_userna inventory, add `GC_LOKI_URL` / `GC_LOKI_USERNAME` to the env file, or set `grafana_alloy_logs_enabled: false`. The daemon's own metrics also gain an explicit `job="decdn-node"` label (previously the implicit `prometheus.scrape.decdn_node`) — -`grafana_alloy_node_job: ""` restores the old identity. +`grafana_alloy_node_job: ""` restores the old identity. If traces start returning 401 +while metrics still flow, the stack's OTLP instance ID is not its Prometheus one: set +`grafana_alloy_otlp_username` (or `GC_OTLP_USERNAME`), which earlier versions had no way +to express. Setup, rotation, cost/cardinality guardrails, the two systemd-hardening relaxations machine monitoring requires, and rollback: see diff --git a/ansible/galaxy/CHANGELOG.md b/ansible/galaxy/CHANGELOG.md index 0fa22bc..194bd33 100644 --- a/ansible/galaxy/CHANGELOG.md +++ b/ansible/galaxy/CHANGELOG.md @@ -31,6 +31,13 @@ collection adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) ### Fixed +- The OTLP gateway is authenticated with its own instance ID. The role reused + the Prometheus one, so a Grafana Cloud org whose stack ID differs 401s every + trace while metrics and logs keep flowing. New `grafana_alloy_otlp_username` / + `GC_OTLP_USERNAME`; when both are empty the rendered config still resolves the + Prometheus value, so an org where the IDs coincide is unaffected. The optional + host key is shape-checked whenever it is present, so a malformed value cannot + quietly outrank that fallback. - Grafana Alloy credentials now default to root-controlled `/etc/grafana-alloy.env`; preflight also rejects a non-root-owned or writable parent directory and a root service identity. Teardown requires the managed diff --git a/ansible/molecule/grafana-cloud/prepare.yml b/ansible/molecule/grafana-cloud/prepare.yml index 6d99a81..0447cb4 100644 --- a/ansible/molecule/grafana-cloud/prepare.yml +++ b/ansible/molecule/grafana-cloud/prepare.yml @@ -56,5 +56,6 @@ GC_PROM_REMOTE_WRITE_URL=https://prometheus-prod-xx.molecule.invalid/api/prom/push GC_OTLP_ENDPOINT=https://otlp-gateway-prod-xx.molecule.invalid/otlp GC_PROM_USERNAME=999888777 + GC_OTLP_USERNAME=999888779 GC_LOKI_USERNAME=999888778 GC_API_TOKEN=molecule-test-token diff --git a/ansible/molecule/grafana-cloud/verify.yml b/ansible/molecule/grafana-cloud/verify.yml index caae897..982ee2a 100644 --- a/ansible/molecule/grafana-cloud/verify.yml +++ b/ansible/molecule/grafana-cloud/verify.yml @@ -155,6 +155,11 @@ 'username = "1234567"' in ga_config - >- 'sys.env("GC_PROM_USERNAME")' not in ga_config + # The OTLP gateway takes the STACK instance ID, which converge leaves + # unset: the config must defer to the host's GC_OTLP_USERNAME and only + # then reuse the Prometheus ID, rather than binding the latter outright. + - >- + 'username = coalesce(sys.env("GC_OTLP_USERNAME"), "1234567")' in ga_config fail_msg: >- The inventory-vs-env credential split did not render as expected: grafana_alloy_loki_url and the numeric grafana_alloy_prom_username were @@ -181,6 +186,11 @@ 'GC_OTLP_ENDPOINT=https://' in ga_env_body - >- 'GC_LOKI_USERNAME=999888778' in ga_env_body + # Optional, and well-formed: proves preflight's present-then-shaped gate + # for the OTLP instance ID accepts a valid host value rather than only + # tolerating an absent key. + - >- + 'GC_OTLP_USERNAME=999888779' in ga_env_body # Deliberately absent: converge supplies the Loki URL from inventory, # which is what makes preflight's conditional requirement meaningful. - >- diff --git a/ansible/molecule/validation/converge.yml b/ansible/molecule/validation/converge.yml index b615e31..495c691 100644 --- a/ansible/molecule/validation/converge.yml +++ b/ansible/molecule/validation/converge.yml @@ -1026,6 +1026,23 @@ decdn_rejected: "{{ decdn_rejected + ['inventory-token'] }}" when: ansible_failed_task.name is match('^Refuse a Grafana Cloud token smuggled into inventory') + - name: "Case ga-otlp-username-shape — malformed OTLP instance ID in inventory" + block: + # The OTLP gateway's username is the stack instance ID, not the + # Prometheus one, so it is its own variable — and must be covered by the + # same charset gate rather than silently reaching basic_auth. + - name: Run grafana_alloy with a whitespace-bearing OTLP instance ID + ansible.builtin.include_role: + name: grafana_alloy + vars: + decdn_grafana_cloud_enabled: true + grafana_alloy_otlp_username: "2345678 # stack id" + rescue: + - name: Record ga-otlp-username-shape rejection (only if the ID validator failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['instance-id'] }}" + when: ansible_failed_task.name is match('^Validate any inventory-provided Grafana Cloud instance IDs') + - name: "Case ga-max-age — malformed journal catch-up window" block: - name: Run grafana_alloy with a bad max_age suffix @@ -1073,6 +1090,43 @@ path: /etc/grafana-alloy.env state: absent + - name: "Case ga-otlp-username-env — malformed GC_OTLP_USERNAME on the host" + block: + # The key is optional, so nothing above requires it — but a present, + # malformed value wins the rendered coalesce() over the valid Prometheus + # fallback and 401s every span, so it gets its own present-then-shaped + # gate. + - name: Stage a credential file whose OTLP instance ID carries a comment + ansible.builtin.copy: + dest: /etc/grafana-alloy.env + owner: root + group: root + mode: "0600" + content: | + GC_PROM_REMOTE_WRITE_URL=https://prometheus-prod-xx.molecule.invalid/api/prom/push + GC_OTLP_ENDPOINT=https://otlp-gateway-prod-xx.molecule.invalid/otlp + GC_PROM_USERNAME=999888777 + GC_OTLP_USERNAME=1830557 # stack id + GC_LOKI_URL=https://logs-prod-xx.molecule.invalid/loki/api/v1/push + GC_LOKI_USERNAME=999888778 + GC_API_TOKEN=molecule-test-token + + - name: Run grafana_alloy with a malformed host OTLP instance ID + ansible.builtin.include_role: + name: grafana_alloy + vars: + decdn_grafana_cloud_enabled: true + rescue: + - name: Record ga-otlp-username-env rejection (only if the OTLP ID gate failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['otlp-username-env'] }}" + when: ansible_failed_task.name is match('^Refuse a malformed host-provided OTLP instance ID') + always: + - name: Remove the malformed OTLP-username fixture + ansible.builtin.file: + path: /etc/grafana-alloy.env + state: absent + - name: Confirm every bad value was rejected by the role's own validation ansible.builtin.assert: that: @@ -1097,5 +1151,6 @@ "alloy-root-user", "alloy-root-group", "alloy-root-user-alias", "alloy-root-group-alias", "host-collectors", "host-collectors", "host-collectors", - "endpoint-scheme", "inventory-token", "inventory-token", + "endpoint-scheme", "inventory-token", "inventory-token", "instance-id", + "otlp-username-env", "managed-paths", "managed-paths", "managed-paths"] diff --git a/ansible/roles/grafana_alloy/README.md b/ansible/roles/grafana_alloy/README.md index 4878de5..479d6c5 100644 --- a/ansible/roles/grafana_alloy/README.md +++ b/ansible/roles/grafana_alloy/README.md @@ -42,7 +42,7 @@ Machine metrics, agent self-metrics and journald come with it (each has its own ### Credentials: one secret per host, the rest in inventory -Only the **API token** is a secret. The endpoint URLs and the two numeric +Only the **API token** is a secret. The endpoint URLs and the three numeric instance IDs are not, so they belong in inventory — written once for the fleet rather than typed on every host: @@ -51,10 +51,17 @@ rather than typed on every host: grafana_alloy_prom_url: https://prometheus-prod-13-prod-us-east-0.grafana.net/api/prom/push grafana_alloy_prom_username: "1234567" # Prometheus instance ID grafana_alloy_otlp_endpoint: https://otlp-gateway-prod-us-east-0.grafana.net/otlp +grafana_alloy_otlp_username: "2345678" # STACK instance ID, shown on the OTLP page grafana_alloy_loki_url: https://logs-prod-006.grafana.net/loki/api/v1/push grafana_alloy_loki_username: "7654321" # Loki instance ID — a DIFFERENT number ``` +Each of those three IDs is its own number in the portal. Copy each from the page +that names it (**Prometheus → Username / Instance ID**, **OTLP Endpoint → +Instance ID**, **Loki → User**) rather than assuming one value covers all three: +where the OTLP ID differs and is left unset, traces 401 while metrics and logs +keep flowing, which reads as "tracing is broken" rather than "auth is wrong". + Then the only thing to provision **on the target host** is the token (it never transits the control machine): @@ -83,10 +90,18 @@ key only then. Any mix of the two halves is valid. > `job="decdn-node"` instead of the implicit `job="prometheus.scrape.decdn_node"`. > Set `grafana_alloy_node_job: ""` to keep the old value if dashboards or alert > rules already hard-code it. +> +> **Traces 401 after an upgrade?** The OTLP gateway wants the stack instance ID, +> and earlier versions of this role reused the Prometheus one. Where your org's +> two IDs differ, set `grafana_alloy_otlp_username` (or add `GC_OTLP_USERNAME` to +> the env file); where they coincide, nothing changes and nothing is needed. +> `GC_OTLP_USERNAME` is the one credential key that may be absent — but if it is +> present, preflight checks its shape, because a malformed value outranks the +> Prometheus fallback at runtime and 401s traces just the same. `GC_API_TOKEN` has **no** inventory variable by design: the rendered `/etc/alloy/config.alloy` is world-readable, and preflight rejects a value that -looks like a token (or a URL with embedded credentials) in any of the five +looks like a token (or a URL with embedded credentials) in any of the six variables above. The credential path is intentionally restricted to a **direct child of `/etc`**. A @@ -184,7 +199,7 @@ upstream version/sha256). Highlights: | `grafana_alloy_logs_enabled` | `true` | journald → Grafana Cloud Loki | | `grafana_alloy_logs_max_age` | `12h` | Bounds the catch-up burst after an outage | | `grafana_alloy_self_metrics_enabled` | `true` | Alloy's own health | -| `grafana_alloy_prom_url` / `_prom_username` / `_otlp_endpoint` / `_loki_url` / `_loki_username` | `""` | Non-secret connection settings; empty ⇒ read the matching `GC_…` env key | +| `grafana_alloy_prom_url` / `_prom_username` / `_otlp_endpoint` / `_otlp_username` / `_loki_url` / `_loki_username` | `""` | Non-secret connection settings; empty ⇒ read the matching `GC_…` env key (`_otlp_username` then falls back to the Prometheus ID) | | `grafana_alloy_node_job` / `_host_job` / `_self_job` / `_logs_job` | see table above | Job labels; the `integrations/…` ones are what Grafana Cloud's dashboards match | Label variables (`service_name`, `service_namespace`, `instance_id`, diff --git a/ansible/roles/grafana_alloy/defaults/main.yml b/ansible/roles/grafana_alloy/defaults/main.yml index ae6db09..51b2ecd 100644 --- a/ansible/roles/grafana_alloy/defaults/main.yml +++ b/ansible/roles/grafana_alloy/defaults/main.yml @@ -67,6 +67,14 @@ grafana_alloy_server_http_addr: 127.0.0.1:12345 grafana_alloy_prom_url: "" # else sys.env("GC_PROM_REMOTE_WRITE_URL") grafana_alloy_prom_username: "" # else sys.env("GC_PROM_USERNAME") grafana_alloy_otlp_endpoint: "" # else sys.env("GC_OTLP_ENDPOINT") +# The OTLP gateway authenticates with the STACK instance ID, which on many +# Grafana Cloud orgs is a different number from the Prometheus one (portal: +# "OTLP Endpoint" -> "Instance ID"). This role used to reuse the Prometheus +# value, so a stack where the two differ 401s on traces while metrics keep +# flowing — a silent half-outage. Left empty, the rendered config falls back at +# RUNTIME to sys.env("GC_OTLP_USERNAME") and then to whatever the Prometheus +# side resolves to, so an org where the IDs coincide keeps working untouched. +grafana_alloy_otlp_username: "" # else sys.env("GC_OTLP_USERNAME"), else the Prometheus ID # Grafana Cloud's Loki instance ID is a DIFFERENT number from the Prometheus one # (the same access-policy token authenticates both). grafana_alloy_loki_url: "" # else sys.env("GC_LOKI_URL") diff --git a/ansible/roles/grafana_alloy/files/grafana-alloy.env.example b/ansible/roles/grafana_alloy/files/grafana-alloy.env.example index cf20bc8..24b0d81 100644 --- a/ansible/roles/grafana_alloy/files/grafana-alloy.env.example +++ b/ansible/roles/grafana_alloy/files/grafana-alloy.env.example @@ -19,8 +19,9 @@ # # Values come from your Grafana Cloud org: Home -> Stacks -> your stack, then the # "Details/Send Metrics|Logs|Traces" pages show each endpoint plus its numeric -# instance ID. The role greps shapes only; it never reads values back onto the -# control machine. +# instance ID. Each service has ITS OWN instance ID — copy each from the page that +# names it rather than reusing one value. The role greps shapes only; it never +# reads values back onto the control machine. # REQUIRED, always. An access-policy token with metrics:write + logs:write + # traces:write scope for this stack. This is the one value with no inventory @@ -36,6 +37,13 @@ GC_API_TOKEN=glc_example_token_replace_me # grafana_alloy_otlp_endpoint #GC_OTLP_ENDPOINT=https://otlp-gateway-prod-XX.grafana.net/otlp +# grafana_alloy_otlp_username — the numeric STACK instance ID shown on the OTLP +# page. It is NOT necessarily the Prometheus one; where your org's two differ and +# neither this key nor the variable is set, the config falls back to the +# Prometheus ID and the OTLP pipeline (traces) 401s while metrics keep flowing. +# Optional — but if you do set it, preflight checks its shape: a malformed value +# takes precedence over that fallback and 401s traces just the same. +#GC_OTLP_USERNAME=2345678 # Logs (grafana_alloy_logs_enabled, on by default). NOTE the Loki instance ID is # a DIFFERENT number from the Prometheus one above; the token is shared. diff --git a/ansible/roles/grafana_alloy/tasks/preflight.yml b/ansible/roles/grafana_alloy/tasks/preflight.yml index c459b4f..f852de5 100644 --- a/ansible/roles/grafana_alloy/tasks/preflight.yml +++ b/ansible/roles/grafana_alloy/tasks/preflight.yml @@ -207,11 +207,12 @@ instance ID on Grafana Cloud, a tenant name on a self-hosted Mimir/Loki — so it is restricted to letters, digits, dot, underscore and dash, and must not be a token or carry whitespace. Got "{{ item.value }}". - NOTE the Loki instance ID differs from the Prometheus one; the - access-policy token is shared and stays in + NOTE the Loki and OTLP instance IDs are each their own number, distinct + from the Prometheus one; the access-policy token is shared and stays in {{ grafana_alloy_secret_file }} as GC_API_TOKEN. loop: - {name: grafana_alloy_prom_username, value: "{{ grafana_alloy_prom_username | string }}"} + - {name: grafana_alloy_otlp_username, value: "{{ grafana_alloy_otlp_username | string }}"} - {name: grafana_alloy_loki_username, value: "{{ grafana_alloy_loki_username | string }}"} loop_control: label: "{{ item.name }}" @@ -233,6 +234,7 @@ - "{{ grafana_alloy_prom_url | string }}" - "{{ grafana_alloy_prom_username | string }}" - "{{ grafana_alloy_otlp_endpoint | string }}" + - "{{ grafana_alloy_otlp_username | string }}" - "{{ grafana_alloy_loki_url | string }}" - "{{ grafana_alloy_loki_username | string }}" loop_control: @@ -402,3 +404,53 @@ | rejectattr('rc', 'equalto', 0) | map(attribute='item') | list }} when: _ga_missing | length > 0 + +# GC_OTLP_USERNAME is the one credential key that is legitimately OPTIONAL: with +# neither inventory nor env-file value, the rendered config resolves the OTLP +# username at runtime to the Prometheus ID — exactly what every pre-existing +# deployment used — so ABSENT is correct and must not be required. But +# PRESENT-AND-MALFORMED is not harmless: coalesce() returns the first non-empty +# argument, so a value carrying whitespace or a stray comment wins over the valid +# Prometheus fallback and 401s the entire trace pipeline while metrics and logs +# stay healthy. The required-key gate above cannot cover it (it only greps keys it +# demands), hence this present-then-shaped pair. Skipped when inventory supplies +# the ID, because the template then inlines it and emits no sys.env lookup at all. +- name: Detect a host-provided OTLP instance ID + ansible.builtin.command: + cmd: grep -qE -- '^GC_OTLP_USERNAME=.*[^[:space:]]' {{ grafana_alloy_secret_file | quote }} + changed_when: false + check_mode: false # read-only gate: must also run under make check + failed_when: false + register: _ga_otlp_user_present + when: grafana_alloy_otlp_username | string | length == 0 + +# Same charset as the inventory instance-ID gate, plus the optional double quotes +# systemd strips off an EnvironmentFile value. +- name: Check the shape of a host-provided OTLP instance ID + ansible.builtin.command: + cmd: grep -qE -- '^GC_OTLP_USERNAME="?[A-Za-z0-9._-]{1,64}"?$' {{ grafana_alloy_secret_file | quote }} + changed_when: false + check_mode: false # read-only gate: must also run under make check + failed_when: false + register: _ga_otlp_user_shape + when: + - grafana_alloy_otlp_username | string | length == 0 + - (_ga_otlp_user_present.rc | default(1)) == 0 + +- name: Refuse a malformed host-provided OTLP instance ID + ansible.builtin.assert: + that: + - (_ga_otlp_user_shape.rc | default(0)) == 0 + fail_msg: >- + {{ grafana_alloy_secret_file }} sets GC_OTLP_USERNAME to a value that is + not an instance ID (letters, digits, dot, underscore, dash, ≤ 64 chars, no + whitespace and no trailing comment). The key is optional — drop the line + entirely and traces fall back to the Prometheus instance ID, which is what + this role used before the OTLP gateway got its own username — but a + malformed value is worse than none: it takes precedence over that fallback + and every span is rejected with a 401 while remote-write and Loki keep + flowing. Fix the line on the host, remove it, or set + grafana_alloy_otlp_username in inventory instead. + when: + - grafana_alloy_otlp_username | string | length == 0 + - (_ga_otlp_user_present.rc | default(1)) == 0 diff --git a/ansible/roles/grafana_alloy/templates/config.alloy.j2 b/ansible/roles/grafana_alloy/templates/config.alloy.j2 index 8083103..c909189 100644 --- a/ansible/roles/grafana_alloy/templates/config.alloy.j2 +++ b/ansible/roles/grafana_alloy/templates/config.alloy.j2 @@ -18,7 +18,7 @@ // `--config.expand-env` flag — `${VAR}` inside a string is a literal, so // `sys.env("VAR")` is the only working indirection. // -// The NON-secret half of the credentials (endpoint URLs, the two numeric +// The NON-secret half of the credentials (endpoint URLs, the three numeric // instance IDs) may instead come from inventory; see the ga_endpoint macro. {# Renders a Grafana Cloud connection value as either an inline string literal @@ -37,6 +37,19 @@ {% if ga_value | length > 0 %}{{ ga_value | tojson }}{% else %}sys.env({{ env_key | tojson }}){% endif %} {%- endmacro %} +{# The OTLP gateway's basic-auth username is the STACK instance ID, which is not + necessarily the Prometheus one — where they differ, reusing the Prometheus ID + 401s the whole OTLP pipeline while remote-write and Loki stay healthy. + Inventory wins; otherwise the decision is deferred to RUNTIME, because whether + the host's env file carries GC_OTLP_USERNAME is not knowable at render time. + `coalesce` returns its first non-empty argument and sys.env yields "" for an + unset variable, so an existing host with only the Prometheus key keeps + resolving exactly the value it resolved before. #} +{% macro ga_otlp_username() -%} +{% set ga_otlp_user = grafana_alloy_otlp_username | default('', true) | string %} +{% if ga_otlp_user | length > 0 %}{{ ga_otlp_user | tojson }}{% else %}coalesce(sys.env("GC_OTLP_USERNAME"), {{ ga_endpoint(grafana_alloy_prom_username, 'GC_PROM_USERNAME') }}){% endif %} +{%- endmacro %} + prometheus.scrape "decdn_node" { targets = [ { "__address__" = "127.0.0.1:{{ decdn_metrics_port | default(9090) }}", @@ -421,6 +434,6 @@ otelcol.exporter.otlphttp "cloud" { } otelcol.auth.basic "cloud" { - username = {{ ga_endpoint(grafana_alloy_prom_username, 'GC_PROM_USERNAME') }} + username = {{ ga_otlp_username() }} password = sys.env("GC_API_TOKEN") } diff --git a/ansible/tests/alloy-config/render.yml b/ansible/tests/alloy-config/render.yml index 4005fb8..c0b884f 100644 --- a/ansible/tests/alloy-config/render.yml +++ b/ansible/tests/alloy-config/render.yml @@ -182,7 +182,7 @@ grafana_alloy_self_metrics_enabled: false grafana_alloy_logs_enabled: false - # The inventory-driven connection settings: all five non-secret values set, + # The inventory-driven connection settings: all six non-secret values set, # so every sys.env lookup EXCEPT the token must disappear. - name: Render the inventory-endpoint configuration ansible.builtin.template: @@ -204,9 +204,34 @@ # must coerce rather than emit a bare number. grafana_alloy_prom_username: 1234567 grafana_alloy_otlp_endpoint: https://otlp-gateway-prod-99.render.invalid/otlp + # A THIRD distinct ID: the OTLP gateway authenticates with the stack + # instance ID, so an inventory value must win over the Prometheus one + # outright — no runtime fallback left in the rendered config. + grafana_alloy_otlp_username: 2345678 grafana_alloy_loki_url: https://logs-prod-99.render.invalid/loki/api/v1/push grafana_alloy_loki_username: 7654321 + # The upgrade shape: Prometheus settings in inventory, nothing for OTLP. The + # rendered config must defer to the host's GC_OTLP_USERNAME if it has one and + # only then reuse the Prometheus ID, which is what every pre-existing + # deployment resolved to. + - name: Render the OTLP-username-fallback configuration + ansible.builtin.template: + src: "{{ playbook_dir }}/../../roles/grafana_alloy/templates/{{ item.src }}" + dest: "{{ _ga_render_dir }}/otlpfallback.{{ item.ext }}" + mode: "0644" + loop: *ga_render_pair + loop_control: + label: "otlpfallback.{{ item.ext }}" + vars: + _ga_instance_id: decdn-node-9 + _ga_journal_groups: [systemd-journal, adm] + grafana_alloy_bin_effective: /usr/bin/alloy + grafana_alloy_deployment_environment: production + grafana_alloy_region: US + grafana_alloy_prom_url: https://prometheus-prod-99.render.invalid/api/prom/push + grafana_alloy_prom_username: 1234567 + # The two escape hatches an upgrading operator reaches for: vetoing a # collector must actually remove it (and its scoping block) rather than # leaving it enabled-but-unscoped, and blanking the node job must restore the diff --git a/ansible/tests/alloy-config/validate.sh b/ansible/tests/alloy-config/validate.sh index a88a189..603334d 100755 --- a/ansible/tests/alloy-config/validate.sh +++ b/ansible/tests/alloy-config/validate.sh @@ -156,21 +156,33 @@ assert_lacks legacy.alloy 'loki.' "any log component" assert_has legacy.alloy 'sys.env("GC_PROM_REMOTE_WRITE_URL")' "the env fallback for remote-write" assert_has legacy.alloy 'sys.env("GC_PROM_USERNAME")' "the env fallback for the instance ID" assert_has legacy.alloy 'sys.env("GC_OTLP_ENDPOINT")' "the env fallback for OTLP" +# With nothing in inventory the OTLP username is resolved entirely at runtime: +# the stack-specific key first, the Prometheus ID (what every pre-existing +# deployment used) only as the fallback. +assert_has legacy.alloy 'username = coalesce(sys.env("GC_OTLP_USERNAME"), sys.env("GC_PROM_USERNAME"))' \ + "the OTLP username fallback chain" assert_lacks legacy.alloy 'GC_LOKI' "a Loki credential lookup while logs are disabled" assert_lacks legacy.alloy '"job"' "an explicit job label, which did not exist before this change" # Inventory-driven connection settings: every non-secret sys.env lookup is gone, # the token's is not. -for key in GC_PROM_REMOTE_WRITE_URL GC_PROM_USERNAME GC_OTLP_ENDPOINT GC_LOKI_URL GC_LOKI_USERNAME; do +for key in GC_PROM_REMOTE_WRITE_URL GC_PROM_USERNAME GC_OTLP_ENDPOINT GC_OTLP_USERNAME GC_LOKI_URL GC_LOKI_USERNAME; do assert_lacks inventory.alloy "sys.env(\"$key\")" "an env lookup that inventory already supplies" done +assert_lacks inventory.alloy 'coalesce(' "a runtime fallback where inventory supplies every value" assert_has inventory.alloy 'https://prometheus-prod-99.render.invalid/api/prom/push' "the inventory remote-write URL" assert_has inventory.alloy 'https://logs-prod-99.render.invalid/loki/api/v1/push' "the inventory Loki URL" assert_has inventory.alloy 'sys.env("GC_API_TOKEN")' "the token env lookup, which must never move to inventory" # Numeric instance IDs arrive from YAML as ints; Alloy needs quoted strings. assert_has inventory.alloy 'username = "1234567"' "the Prometheus instance ID as a quoted string" +assert_has inventory.alloy 'username = "2345678"' "the OTLP stack instance ID as a quoted string" assert_has inventory.alloy 'username = "7654321"' "the Loki instance ID as a quoted string" +# The upgrade shape: an inventory Prometheus ID must NOT be silently reused as +# the OTLP username without giving the host's own key precedence. +assert_has otlpfallback.alloy 'username = coalesce(sys.env("GC_OTLP_USERNAME"), "1234567")' \ + "the OTLP username deferring to the host key before the Prometheus ID" + # Escape hatches: the collector veto must remove the collector AND its scoping # block from the rendered lists (upstream would otherwise re-enable it), and a # blank node job must restore the implicit component-ID identity.