diff --git a/charts/nudgebee-agent/values.yaml b/charts/nudgebee-agent/values.yaml index bbf14b10..e3beaa5d 100644 --- a/charts/nudgebee-agent/values.yaml +++ b/charts/nudgebee-agent/values.yaml @@ -65,7 +65,7 @@ runnerServiceAccount: runner: image: repository: ghcr.io/nudgebee/nudgebee-agent - tag: 2026-08-12T06-01-54_7ec58f7701909a3ce172ad2a9235f8b15255e363 + tag: 2026-08-12T08-10-04_e13ce92bc8c238e883d050b1276c6128b2c8fb44 # Image template the pod_profiler action launches debugger pods from. # The agent substitutes `{}` for the variant (bpf, jvm, python, perf, ruby). # Surfaces as PROFILER_IMAGE; leave empty to fall back to the binary default. diff --git a/runner/cmd/agent/main.go b/runner/cmd/agent/main.go index bafb2ec8..83e3fe67 100644 --- a/runner/cmd/agent/main.go +++ b/runner/cmd/agent/main.go @@ -256,7 +256,8 @@ func run(ctx context.Context, logger *slog.Logger, cfg *config.Config) error { var promClient *prometheus.Client if cfg.PrometheusURL != "" { promClient = prometheus.New(cfg.PrometheusURL, nil) - promClient.ExtraHeaders = config.ParseHeaders(cfg.PrometheusHeaders) + promClient.ExtraHeaders = promHeadersWithAuth(cfg) + promClient.URLQueryString = cfg.PrometheusURLQueryString // Managed-provider auth, same precedence as the legacy // generate_prometheus_config: AWS SigV4 → Coralogix token → Azure AD. // Plain header/basic auth stays in PROMETHEUS_HEADERS above. @@ -762,7 +763,8 @@ func run(ctx context.Context, logger *slog.Logger, cfg *config.Config) error { // Prometheus path the collector's `/prometheus-v2/*` route forwards // through (relay-server/pkg/utils/utils.go:77). gp := grafana.New(grafanaURL, grafanaUser, grafanaPass, extraHeaders, - cfg.PrometheusURL, config.ParseHeaders(cfg.PrometheusHeaders), nil) + cfg.PrometheusURL, promHeadersWithAuth(cfg), nil) + gp.PrometheusQueryString = cfg.PrometheusURLQueryString disp.SetGrafana(&grafanaAdapter{p: gp}) if grafanaURL != "" { logger.Info("grafana proxy enabled", "url", grafanaURL) @@ -1005,7 +1007,7 @@ func run(ctx context.Context, logger *slog.Logger, cfg *config.Config) error { LogProviderConfig: logCfg, PrometheusConnected: prometheusConnected(probeCtx, promClient, logger), NodeAgentCount: queryNodeAgentCount(probeCtx, promClient, logger), - PrometheusRetentionTime: telemetry.PrometheusRetention(probeCtx, promClient, logger), + PrometheusRetentionTime: telemetry.PrometheusRetention(probeCtx, promClient, cfg.PrometheusRetentionTime, logger), PrometheusAdditionalLabels: promExtraLabels, TraceTable: traceTable, JaegerEnabled: jaegerEnabled, @@ -1403,6 +1405,18 @@ func httpProbeErr(ctx context.Context, c *http.Client, url string, headers ...ma // sslVerify is false (the legacy default), it disables TLS certificate // verification to match the legacy client's verify_certs=False. For plain // http URLs the TLS config is inert. +// promHeadersWithAuth parses PROMETHEUS_HEADERS and, when PROMETHEUS_AUTH is +// set, overlays it as the Authorization header (legacy prometheus_auth). Set, +// not Add, so PROMETHEUS_AUTH takes precedence over any Authorization carried +// in PROMETHEUS_HEADERS instead of doubling it. +func promHeadersWithAuth(cfg *config.Config) http.Header { + h := config.ParseHeaders(cfg.PrometheusHeaders) + if cfg.PrometheusAuth != "" { + h.Set("Authorization", cfg.PrometheusAuth) + } + return h +} + func esHTTPClient(sslVerify bool) *http.Client { if sslVerify { return nil // nil → elasticsearch.New builds a default verifying client diff --git a/runner/docs/configuration.md b/runner/docs/configuration.md index 0ebf338b..02d2bd45 100644 --- a/runner/docs/configuration.md +++ b/runner/docs/configuration.md @@ -45,7 +45,10 @@ If a K8s subsystem is enabled but the agent fails to build a K8s client (no kube | Variable | Required | Description | |---|---|---| | `PROMETHEUS_URL` | recommended | Enables `prometheus_*` actions and `service_map` | -| `PROMETHEUS_HEADERS` | optional | Comma-separated `Header: value` pairs (e.g. `X-Scope-OrgID: tenant-1`); use for static basic/bearer auth | +| `PROMETHEUS_HEADERS` | optional | Semicolon-separated `Header: value` pairs (e.g. `X-Scope-OrgID: tenant-1`); use for static basic/bearer auth | +| `PROMETHEUS_AUTH` | optional | Static `Authorization` header value (e.g. `Basic dXNlcjpwYXNz` or `Bearer tok`); applied to every request, overriding any `Authorization` in `PROMETHEUS_HEADERS` | +| `PROMETHEUS_URL_QUERY_STRING` | optional | Query-string fragment appended to every Prometheus request (e.g. `extra_label=foo`) | +| `PROMETHEUS_RETENTION_TIME` | optional | Retention value reported to the UI when `/status/flags` is unavailable (e.g. VictoriaMetrics vmsingle) | | `AWS_ACCESS_KEY` / `AWS_SECRET_ACCESS_KEY` / `AWS_REGION` | optional | Managed Prometheus: sign requests with AWS SigV4. `AWS_SERVICE_NAME` defaults to `aps` | | `CORALOGIX_PROMETHEUS_TOKEN` | optional | Managed Prometheus: sent as `token` header | | `AZURE_USE_MANAGED_ID` / `AZURE_CLIENT_SECRET` (+ `AZURE_CLIENT_ID` / `AZURE_TENANT_ID`) | optional | Managed Prometheus: Azure AD Bearer token (managed identity or client-secret). Precedence: AWS → Coralogix → Azure | diff --git a/runner/pkg/config/config.go b/runner/pkg/config/config.go index 366eb4f5..f5fe4734 100644 --- a/runner/pkg/config/config.go +++ b/runner/pkg/config/config.go @@ -28,10 +28,23 @@ type Config struct { // Optional datasource URLs. Empty = subsystem disabled. PrometheusURL string PrometheusHeaders string // raw "Header: value" string, parsed into http.Header - LokiURL string - LokiHeaders string - LokiUsername string // optional Basic-Auth (LOKI_USERNAME) - LokiPassword string // optional Basic-Auth (LOKI_PASSWORD) + // PrometheusAuth (PROMETHEUS_AUTH) is a static Authorization header value + // (e.g. "Basic dXNlcjpwYXNz" or "Bearer tok"). Applied to every Prometheus + // request, matching the legacy prometheus_auth. Takes precedence over any + // Authorization set via PrometheusHeaders. + PrometheusAuth string + // PrometheusURLQueryString (PROMETHEUS_URL_QUERY_STRING) is appended to the + // query string of every Prometheus request (e.g. a tenant selector the + // upstream requires). Mirrors the legacy prometheus_url_query_string. + PrometheusURLQueryString string + // PrometheusRetentionTime (PROMETHEUS_RETENTION_TIME) is a fallback + // retention value reported to the UI when /status/flags is unavailable + // (e.g. VictoriaMetrics' vmsingle 404s that endpoint). + PrometheusRetentionTime string + LokiURL string + LokiHeaders string + LokiUsername string // optional Basic-Auth (LOKI_USERNAME) + LokiPassword string // optional Basic-Auth (LOKI_PASSWORD) // Prometheus managed-provider auth (mirrors the legacy prometrix configs). // Coralogix uses a `token` header; AWS signs each request with SigV4; @@ -200,18 +213,21 @@ type Config struct { // produce an error; missing optional fields are left blank. func FromEnv() (*Config, error) { c := &Config{ - AuthSecretKey: os.Getenv("NUDGEBEE_AUTH_SECRET_KEY"), - RelaySigningPublicKey: os.Getenv("RELAY_SIGNING_PUBLIC_KEY"), - RelayURL: os.Getenv("WEBSOCKET_RELAY_ADDRESS"), - BackendEndpoint: os.Getenv("NUDGEBEE_ENDPOINT"), - AccountID: os.Getenv("ACCOUNT_ID"), - ClusterName: os.Getenv("CLUSTER_NAME"), - PrometheusURL: os.Getenv("PROMETHEUS_URL"), - PrometheusHeaders: os.Getenv("PROMETHEUS_HEADERS"), // matches runner.yaml secret - LokiURL: os.Getenv("LOKI_URL"), - LokiHeaders: os.Getenv("LOKI_EXTRA_HEADER"), // matches runner.yaml secret - LokiUsername: os.Getenv("LOKI_USERNAME"), - LokiPassword: os.Getenv("LOKI_PASSWORD"), + AuthSecretKey: os.Getenv("NUDGEBEE_AUTH_SECRET_KEY"), + RelaySigningPublicKey: os.Getenv("RELAY_SIGNING_PUBLIC_KEY"), + RelayURL: os.Getenv("WEBSOCKET_RELAY_ADDRESS"), + BackendEndpoint: os.Getenv("NUDGEBEE_ENDPOINT"), + AccountID: os.Getenv("ACCOUNT_ID"), + ClusterName: os.Getenv("CLUSTER_NAME"), + PrometheusURL: os.Getenv("PROMETHEUS_URL"), + PrometheusHeaders: os.Getenv("PROMETHEUS_HEADERS"), // matches runner.yaml secret + PrometheusAuth: os.Getenv("PROMETHEUS_AUTH"), + PrometheusURLQueryString: os.Getenv("PROMETHEUS_URL_QUERY_STRING"), + PrometheusRetentionTime: os.Getenv("PROMETHEUS_RETENTION_TIME"), + LokiURL: os.Getenv("LOKI_URL"), + LokiHeaders: os.Getenv("LOKI_EXTRA_HEADER"), // matches runner.yaml secret + LokiUsername: os.Getenv("LOKI_USERNAME"), + LokiPassword: os.Getenv("LOKI_PASSWORD"), // Prometheus managed-provider auth. Defaults mirror the legacy // prometrix env handling (utils.py): AWS service defaults to "aps", // Azure resource/endpoints have the same fallbacks. diff --git a/runner/pkg/grafana/proxy.go b/runner/pkg/grafana/proxy.go index bb58355f..f4d6d836 100644 --- a/runner/pkg/grafana/proxy.go +++ b/runner/pkg/grafana/proxy.go @@ -62,11 +62,16 @@ type Proxy struct { PrometheusURL string // PrometheusHeaders is the parsed PROMETHEUS_HEADERS env (raw - // "Header: value, Header: value" string) — applied to every + // "Header: value; Header: value" string) — applied to every // Prometheus proxy request so X-Scope-OrgID / tenant headers reach - // the upstream. Same shape ExtraHeaders uses for Grafana. + // the upstream. Same shape ExtraHeaders uses for Grafana. When + // PROMETHEUS_AUTH is set the caller overlays it here as Authorization. PrometheusHeaders http.Header + // PrometheusQueryString (PROMETHEUS_URL_QUERY_STRING) is appended to the + // query string of every proxied Prometheus request. Empty is a no-op. + PrometheusQueryString string + // Username/Password for Grafana basic auth (GRAFANA_USERNAME / // GRAFANA_PASSWORD env). Empty disables. Username string @@ -134,7 +139,23 @@ func (p *Proxy) HandlePrometheus(ctx context.Context, req *Request) *Response { } enriched := *req enriched.Header = mergeHeaders(req.Header, p.PrometheusHeaders) - return p.do(ctx, p.PrometheusURL+req.URL, &enriched) + return p.do(ctx, appendQueryString(p.PrometheusURL+req.URL, p.PrometheusQueryString), &enriched) +} + +// appendQueryString appends extra (a "k=v&k2=v2" fragment, optional leading +// "?") to rawurl's query, using "?" or "&" depending on whether rawurl already +// has a query. Empty extra is a no-op. +func appendQueryString(rawurl, extra string) string { + extra = strings.TrimSpace(extra) + extra = strings.TrimPrefix(extra, "?") + if extra == "" { + return rawurl + } + sep := "?" + if strings.Contains(rawurl, "?") { + sep = "&" + } + return rawurl + sep + extra } // mergeHeaders returns a new header map containing every entry from diff --git a/runner/pkg/grafana/proxy_test.go b/runner/pkg/grafana/proxy_test.go index c4143f89..7bf7b6de 100644 --- a/runner/pkg/grafana/proxy_test.go +++ b/runner/pkg/grafana/proxy_test.go @@ -174,6 +174,37 @@ func TestProxy_HandlePrometheus_ForwardsToPrometheusURL(t *testing.T) { } } +// TestProxy_HandlePrometheus_QueryStringAndAuth verifies +// PROMETHEUS_URL_QUERY_STRING is appended and a PROMETHEUS_AUTH-derived +// Authorization header (carried in PrometheusHeaders) reaches upstream. +func TestProxy_HandlePrometheus_QueryStringAndAuth(t *testing.T) { + var gotQuery, gotAuth string + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotQuery = r.URL.RawQuery + gotAuth = r.Header.Get("Authorization") + _, _ = w.Write([]byte(`{"status":"success"}`)) + })) + defer upstream.Close() + + p := New("", "", "", nil, upstream.URL, + http.Header{"Authorization": {"Bearer tok-1"}}, nil) + p.PrometheusQueryString = "extra_label=foo" + + resp := p.HandlePrometheus(context.Background(), &Request{ + Method: "GET", + URL: "/api/v1/query?query=up", + }) + if resp.StatusCode != 200 { + t.Fatalf("status = %d; want 200", resp.StatusCode) + } + if !strings.Contains(gotQuery, "query=up") || !strings.Contains(gotQuery, "extra_label=foo") { + t.Errorf("upstream query = %q; want both query=up and extra_label=foo", gotQuery) + } + if gotAuth != "Bearer tok-1" { + t.Errorf("Authorization = %q; want Bearer tok-1", gotAuth) + } +} + // TestProxy_HandlePrometheus_UnconfiguredReturns503 — without // PROMETHEUS_URL the proxy must not panic. Same posture as // TestProxy_UnconfiguredGrafana for the Grafana path. diff --git a/runner/pkg/observability/prometheus/client.go b/runner/pkg/observability/prometheus/client.go index e00fd1f8..ba471909 100644 --- a/runner/pkg/observability/prometheus/client.go +++ b/runner/pkg/observability/prometheus/client.go @@ -37,8 +37,13 @@ type Client struct { // ExtraHeaders are sent on every request. Used for X-Scope-OrgID // (Cortex/Mimir multi-tenant) and any auth headers the operator // configures via the runner secret today (LOKI_EXTRA_HEADER pattern). + // PROMETHEUS_AUTH is delivered here as an Authorization header. ExtraHeaders http.Header + // URLQueryString (PROMETHEUS_URL_QUERY_STRING) is appended to the query + // string of every request. Empty is a no-op. Leading "?" is optional. + URLQueryString string + // Auth applies a managed-provider auth scheme (AWS SigV4 / Azure AD / // Coralogix) to each request. Nil when the backend uses plain headers // or no auth. See auth.go. @@ -171,6 +176,22 @@ func (c *Client) Flags(ctx context.Context) (json.RawMessage, error) { return c.get(ctx, "/api/v1/status/flags", nil) } +// appendQueryString appends extra (a "k=v&k2=v2" fragment, with an optional +// leading "?") to rawurl's query, choosing "?" or "&" based on whether +// rawurl already has a query. Empty extra is a no-op. +func appendQueryString(rawurl, extra string) string { + extra = strings.TrimSpace(extra) + extra = strings.TrimPrefix(extra, "?") + if extra == "" { + return rawurl + } + sep := "?" + if strings.Contains(rawurl, "?") { + sep = "&" + } + return rawurl + sep + extra +} + func (c *Client) get(ctx context.Context, path string, params url.Values) (json.RawMessage, error) { if c.BaseURL == "" { return nil, errors.New("prometheus: base URL not configured") @@ -179,6 +200,7 @@ func (c *Client) get(ctx context.Context, path string, params url.Values) (json. if len(params) > 0 { u += "?" + params.Encode() } + u = appendQueryString(u, c.URLQueryString) req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) if err != nil { return nil, err diff --git a/runner/pkg/observability/prometheus/prometheus_test.go b/runner/pkg/observability/prometheus/prometheus_test.go index 6d7cf278..6711240a 100644 --- a/runner/pkg/observability/prometheus/prometheus_test.go +++ b/runner/pkg/observability/prometheus/prometheus_test.go @@ -107,6 +107,43 @@ func TestExtraHeaders_Sent(t *testing.T) { } } +// TestURLQueryString_Appended verifies PROMETHEUS_URL_QUERY_STRING is merged +// into the request query string alongside the query params. +func TestURLQueryString_Appended(t *testing.T) { + var gotRegion, gotQuery string + c, srv := newTestClient(func(w http.ResponseWriter, r *http.Request) { + gotRegion = r.URL.Query().Get("region") + gotQuery = r.URL.Query().Get("query") + _, _ = w.Write([]byte(`{}`)) + }) + defer srv.Close() + c.URLQueryString = "?region=us-east" // leading ? tolerated + + if _, err := c.Query(context.Background(), "up", "", ""); err != nil { + t.Fatal(err) + } + if gotRegion != "us-east" { + t.Errorf("region = %q; want us-east", gotRegion) + } + if gotQuery != "up" { + t.Errorf("query = %q; want up (existing params preserved)", gotQuery) + } +} + +func TestAppendQueryString(t *testing.T) { + cases := []struct{ url, extra, want string }{ + {"http://h/api/v1/query?query=up", "region=us", "http://h/api/v1/query?query=up®ion=us"}, + {"http://h/api/v1/labels", "region=us", "http://h/api/v1/labels?region=us"}, + {"http://h/api/v1/labels", "?region=us", "http://h/api/v1/labels?region=us"}, + {"http://h/api/v1/labels", "", "http://h/api/v1/labels"}, + } + for _, tc := range cases { + if got := appendQueryString(tc.url, tc.extra); got != tc.want { + t.Errorf("appendQueryString(%q,%q) = %q; want %q", tc.url, tc.extra, got, tc.want) + } + } +} + func TestHandlers_DispatchesViaActionName(t *testing.T) { wantBody := `{"status":"success","data":[]}` c, srv := newTestClient(func(w http.ResponseWriter, r *http.Request) { diff --git a/runner/pkg/telemetry/prom_flags.go b/runner/pkg/telemetry/prom_flags.go index a373a6b8..dcf6f2c1 100644 --- a/runner/pkg/telemetry/prom_flags.go +++ b/runner/pkg/telemetry/prom_flags.go @@ -17,28 +17,30 @@ type prometheusFlagsResponse struct { Data map[string]string `json:"data"` } -// PrometheusRetention queries /api/v1/status/flags and returns the -// `storage.tsdb.retention.time` value. Empty string on any error. -// VictoriaMetrics' vmsingle returns 404 for this endpoint — caller must -// tolerate the empty return. -func PrometheusRetention(ctx context.Context, c *prometheus.Client, logger *slog.Logger) string { +// PrometheusRetention queries /api/v1/status/flags and returns the retention +// value. It reads Prometheus' `storage.tsdb.retention.time`, then the compat +// `retentionTime`, then VictoriaMetrics' `-retentionPeriod` (vmsingle exposes +// its flags under CLI-style keys). When the endpoint is unavailable (vmsingle +// 404s it) or none of those keys are present, it falls back to the caller- +// supplied value (PROMETHEUS_RETENTION_TIME). Empty string when nothing +// resolves. +func PrometheusRetention(ctx context.Context, c *prometheus.Client, fallback string, logger *slog.Logger) string { if c == nil || c.BaseURL == "" { - return "" + return fallback } raw, err := c.Flags(ctx) if err != nil { - logger.Debug("prometheus flags probe failed", "err", err) - return "" + logger.Debug("prometheus flags probe failed; using retention fallback", "err", err) + return fallback } var resp prometheusFlagsResponse if err := json.Unmarshal(raw, &resp); err != nil { - return "" + return fallback } - if v := resp.Data["storage.tsdb.retention.time"]; v != "" { - return v + for _, k := range []string{"storage.tsdb.retention.time", "retentionTime", "-retentionPeriod"} { + if v := resp.Data[k]; v != "" { + return v + } } - if v := resp.Data["retentionTime"]; v != "" { - return v - } - return "" + return fallback } diff --git a/runner/pkg/telemetry/prom_flags_test.go b/runner/pkg/telemetry/prom_flags_test.go index a3017a6a..6fc177ea 100644 --- a/runner/pkg/telemetry/prom_flags_test.go +++ b/runner/pkg/telemetry/prom_flags_test.go @@ -12,10 +12,11 @@ import ( func TestPrometheusRetention(t *testing.T) { cases := []struct { - name string - body string - code int - want string + name string + body string + code int + fallback string + want string }{ { name: "modern flag name", @@ -30,15 +31,28 @@ func TestPrometheusRetention(t *testing.T) { want: "10d", }, { - name: "endpoint missing (vmsingle)", + name: "victoriametrics retentionPeriod", + body: `{"status":"success","data":{"-retentionPeriod":"12"}}`, + code: 200, + want: "12", + }, + { + name: "endpoint missing falls back to env", + code: 404, + fallback: "30d", + want: "30d", + }, + { + name: "endpoint missing, no fallback", code: 404, want: "", }, { - name: "malformed body", - body: `not json`, - code: 200, - want: "", + name: "malformed body falls back to env", + body: `not json`, + code: 200, + fallback: "7d", + want: "7d", }, } for _, tc := range cases { @@ -56,14 +70,14 @@ func TestPrometheusRetention(t *testing.T) { })) defer srv.Close() c := prometheus.New(srv.URL, nil) - if got := PrometheusRetention(context.Background(), c, slog.Default()); got != tc.want { + if got := PrometheusRetention(context.Background(), c, tc.fallback, slog.Default()); got != tc.want { t.Errorf("got %q, want %q", got, tc.want) } }) } - // Nil client → empty without panic. - if got := PrometheusRetention(context.Background(), nil, slog.Default()); got != "" { - t.Errorf("nil client should return empty, got %q", got) + // Nil client → fallback without panic. + if got := PrometheusRetention(context.Background(), nil, "5d", slog.Default()); got != "5d" { + t.Errorf("nil client should return fallback, got %q", got) } }