diff --git a/charts/nudgebee-agent/values.yaml b/charts/nudgebee-agent/values.yaml index 08b473e7..5650743f 100644 --- a/charts/nudgebee-agent/values.yaml +++ b/charts/nudgebee-agent/values.yaml @@ -65,7 +65,7 @@ runnerServiceAccount: runner: image: repository: ghcr.io/nudgebee/nudgebee-agent - tag: 2026-07-31T05-49-10_de0915f92bf0a028846c7082445222810506ca9a + tag: 2026-07-31T06-50-14_8717b178359e9dce22453bf1de7e7d8e9eccfda3 # Image template the pod_profiler action launches debugger pods from. # The agent substitutes `{}` for the variant (bpf, jvm, python, perf, ruby). # Surfaces as PROFILER_IMAGE; leave empty to fall back to the binary default. diff --git a/runner/pkg/enrichers/slo.go b/runner/pkg/enrichers/slo.go index 46faaed1..b9152905 100644 --- a/runner/pkg/enrichers/slo.go +++ b/runner/pkg/enrichers/slo.go @@ -100,11 +100,18 @@ func (s *SLOEnricher) compute(ctx context.Context, params map[string]any) ([]map results := a.runQueries(ctx, queries, startTime, endTime, step, "", "", "") apps := extractMetricStats(results) + // Multi-window burn rates (coroot parity, see slo_burnrate.go). Best-effort: + // when this yields nothing the report keeps its legacy single-window + // `alert`/`alert_message`, so a Prometheus hiccup degrades rather than + // blanks the SLO. + burnRatesByApp := s.burnRatesByApp(ctx, cfg, endTime) + minValidEvents := envIntDefault("MIN_VALID_EVENTS", 1) reports := make([]map[string]any, 0, len(apps)) for _, app := range apps { stats := app.toResponse() report := buildSLOReport(cfg, stats, startTime, endTime, minValidEvents) + attachBurnRates(report, burnRatesByApp[app.Name+"/"+app.Namespace]) reports = append(reports, report) } return reports, nil @@ -260,7 +267,9 @@ func buildSLOReport(cfg sloConfig, stats map[string]any, startTime, endTime time ebRatio = ebValue * 100 / ebTarget ebBurnRate = math.Round(ebValue/ebTarget*10) / 10 // round(x, 1) } - hours := int(ebBurnRate / (60 * 60)) + // The window is what the burn rate was measured over — dividing the RATE by + // 3600 (as an earlier port of coroot's FormatSLOStatus did) is always 0. + hours := cfg.Window / (60 * 60) hourLabel := "hours" if hours == 1 { hourLabel = "hour" diff --git a/runner/pkg/enrichers/slo_burnrate.go b/runner/pkg/enrichers/slo_burnrate.go new file mode 100644 index 00000000..caaec234 --- /dev/null +++ b/runner/pkg/enrichers/slo_burnrate.go @@ -0,0 +1,333 @@ +package enrichers + +import ( + "context" + "fmt" + "math" + "time" +) + +// Multi-window multi-burn-rate SLO evaluation, ported from coroot +// (model/alert.go + watchers/incidents.go on coroot main). +// +// The single-window evaluation in slo.go answers "is the error budget being +// burned faster than 14.4x right now?" over one hour. That fires on any single +// bad hour, including one that has already recovered, and never fires at all on +// a slow burn that still exhausts a 30-day budget. +// +// The workbook algorithm instead pairs each burn-rate threshold with a long and +// a short window and requires BOTH to exceed the threshold: the long window +// establishes the burn is significant, the short window confirms it is still +// happening. See https://sre.google/workbook/alerting-on-slos/. + +const ( + severityOK = "OK" + severityWarning = "WARNING" + severityCritical = "CRITICAL" +) + +// alertRule mirrors coroot's model.AlertRule. Windows are in seconds. +// +// Kept in sync with coroot main model/alert.go: +// +// {Hour, 5*Minute, 14.4, CRITICAL} +// {6*Hour, 15*Minute, 6, CRITICAL} +type alertRule struct { + LongWindow int + ShortWindow int + Threshold float64 + Severity string +} + +var alertRules = []alertRule{ + {LongWindow: 3600, ShortWindow: 300, Threshold: 14.4, Severity: severityCritical}, + {LongWindow: 21600, ShortWindow: 900, Threshold: 6, Severity: severityCritical}, +} + +// coveragePoints is how many samples each window is split into when measuring +// data coverage, and minCoverage the fraction that must be present for the +// window to count. Coroot requires at least half the window to contain valid +// data before it will compute a burn rate at all — without this, a low-traffic +// service whose exporter goes quiet for a few minutes reads as a total outage. +const ( + coveragePoints = 12 + minCoverage = 0.5 + minStepSeconds = 15 +) + +// burnRate is the wire shape emitted per alert rule. Field names mirror +// coroot's model.BurnRate so the two stay comparable. +type burnRate struct { + LongWindow int `json:"long_window"` + ShortWindow int `json:"short_window"` + Threshold float64 `json:"threshold"` + LongWindowPercentage float64 `json:"long_window_percentage"` + ShortWindowPercentage float64 `json:"short_window_percentage"` + LongWindowBurnRate float64 `json:"long_window_burn_rate"` + ShortWindowBurnRate float64 `json:"short_window_burn_rate"` + Severity string `json:"severity"` +} + +// formatSLOStatus mirrors coroot's BurnRate.FormatSLOStatus. Note this reports +// the LONG WINDOW, not the burn rate divided by 3600 — the latter is always 0. +func (br burnRate) formatSLOStatus() string { + hours := br.LongWindow / 3600 + unit := "hours" + if hours == 1 { + unit = "hour" + } + if hours == 0 { + return fmt.Sprintf("error budget burn rate is %.1fx within %d minutes", br.LongWindowBurnRate, br.LongWindow/60) + } + return fmt.Sprintf("error budget burn rate is %.1fx within %d %s", br.LongWindowBurnRate, hours, unit) +} + +// windowStats is one workload's good/bad/valid counts over one window, plus how +// much of that window actually reported data. +type windowStats struct { + good float64 + bad float64 + valid float64 + hasValid bool + coverage float64 +} + +// total returns the denominator of the SLI. When the config supplies a valid +// series (distribution_cut always does; good_bad_ratio may) that series IS the +// total and "bad" is everything that is not good — matching coroot's +// slowF = total - fast. Otherwise the total is good+bad. +func (w windowStats) total() float64 { + if w.hasValid { + return w.valid + } + return w.good + w.bad +} + +func (w windowStats) badCount() float64 { + if w.hasValid { + if b := w.valid - w.good; b > 0 { + return b + } + return 0 + } + return w.bad +} + +// calcBurnRates evaluates every alert rule against the per-window stats and +// returns one burnRate per rule, ported from coroot's calcBurnRates. +// +// A rule is skipped (not reported) when either window lacks the data to judge +// it. A rule is reported with severity OK when it has data but is under +// threshold — the caller needs those to render "how close are we". +func calcBurnRates(byWindow map[int]windowStats, goal float64) []burnRate { + objective := 1 - goal + if objective <= 0 { + return nil + } + res := make([]burnRate, 0, len(alertRules)) + for _, r := range alertRules { + long, okLong := byWindow[r.LongWindow] + short, okShort := byWindow[r.ShortWindow] + if !okLong || !okShort { + continue + } + if long.coverage < minCoverage || short.coverage < minCoverage { + continue + } + longTotal, shortTotal := long.total(), short.total() + if longTotal <= 0 || shortTotal <= 0 { + continue + } + br := burnRate{ + LongWindow: r.LongWindow, + ShortWindow: r.ShortWindow, + Threshold: r.Threshold, + Severity: severityOK, + } + lr := long.badCount() / longTotal + sr := short.badCount() / shortTotal + if math.IsNaN(lr) || math.IsNaN(sr) { + continue + } + br.LongWindowPercentage = lr * 100 + br.ShortWindowPercentage = sr * 100 + br.LongWindowBurnRate = lr / objective + br.ShortWindowBurnRate = sr / objective + if br.LongWindowBurnRate > r.Threshold && br.ShortWindowBurnRate > r.Threshold { + br.Severity = r.Severity + } + res = append(res, br) + } + return res +} + +// worstSeverity returns the highest severity across the rules and the burn rate +// that produced it, so the caller can build the alert message from the window +// that actually fired. +func worstSeverity(rates []burnRate) (string, burnRate) { + rank := map[string]int{severityOK: 0, severityWarning: 1, severityCritical: 2} + worst := severityOK + var firing burnRate + for _, br := range rates { + if rank[br.Severity] > rank[worst] { + worst = br.Severity + firing = br + } + } + return worst, firing +} + +// requiredWindows is the deduplicated set of windows the alert rules need. +func requiredWindows() []int { + seen := map[int]bool{} + out := make([]int, 0, len(alertRules)*2) + for _, r := range alertRules { + for _, w := range []int{r.LongWindow, r.ShortWindow} { + if !seen[w] { + seen[w] = true + out = append(out, w) + } + } + } + return out +} + +// statsForWindow runs the config's SLI queries over one window and returns the +// per-workload counts keyed by "/". +// +// The query is evaluated at a step of window/coveragePoints rather than at the +// window itself. Every point is a full-window increase() so the LAST point is +// the value we want; the earlier points exist only so we can measure how much +// of the window reported data at all. +func (s *SLOEnricher) statsForWindow(ctx context.Context, cfg sloConfig, window int, endTime time.Time) (map[string]windowStats, error) { + wcfg := cfg + wcfg.Window = window + queries, err := buildSLOQueries(wcfg) + if err != nil { + return nil, err + } + step := int64(window / coveragePoints) + if step < minStepSeconds { + step = minStepSeconds + } + startTime := endTime.Add(-time.Duration(window) * time.Second) + + a := &AppStatsEnricher{q: s.q} + results := a.runQueries(ctx, queries, startTime, endTime, step, "", "", "") + + out := map[string]windowStats{} + for _, app := range extractMetricStats(results) { + ws := windowStats{ + good: lastOrZero(app.GoodData), + bad: lastOrZero(app.BadData), + coverage: definedFraction(app.ValidData, app.GoodData, app.BadData), + } + if app.ValidData != nil { + if v := app.ValidData.last(); !math.IsNaN(v) && !math.IsInf(v, 0) { + ws.valid = v + ws.hasValid = true + } + } + out[app.Name+"/"+app.Namespace] = ws + } + return out, nil +} + +// burnRatesByApp evaluates every window the alert rules need and returns the +// burn-rate vector per workload key. +// +// Best-effort by design: a window whose query fails is simply absent from +// byWindow, and calcBurnRates skips any rule missing either of its windows. +func (s *SLOEnricher) burnRatesByApp(ctx context.Context, cfg sloConfig, endTime time.Time) map[string][]burnRate { + byWindow := map[int]map[string]windowStats{} + for _, w := range requiredWindows() { + stats, err := s.statsForWindow(ctx, cfg, w, endTime) + if err != nil { + continue + } + byWindow[w] = stats + } + // Collect every workload seen in any window. + keys := map[string]bool{} + for _, stats := range byWindow { + for k := range stats { + keys[k] = true + } + } + out := make(map[string][]burnRate, len(keys)) + for k := range keys { + perWindow := map[int]windowStats{} + for w, stats := range byWindow { + if ws, ok := stats[k]; ok { + perWindow[w] = ws + } + } + if rates := calcBurnRates(perWindow, cfg.Goal); len(rates) > 0 { + out[k] = rates + } + } + return out +} + +// attachBurnRates writes the burn-rate vector onto a report and, when the +// vector is non-empty, lets it decide `alert`/`alert_message` instead of the +// legacy single-window comparison. +// +// The legacy fields are always left in place so a backend that predates +// `burn_rates` keeps working against a newer agent. +func attachBurnRates(report map[string]any, rates []burnRate) { + if len(rates) == 0 { + return + } + report["burn_rates"] = rates + severity, firing := worstSeverity(rates) + report["severity"] = severity + report["alert"] = severity != severityOK + if severity != severityOK { + report["alert_message"] = firing.formatSLOStatus() + } else { + report["alert_message"] = "" + } +} + +func lastOrZero(ts *timeSeries) float64 { + if ts == nil { + return 0 + } + v := ts.last() + if math.IsNaN(v) || math.IsInf(v, 0) { + return 0 + } + return v +} + +// definedFraction reports the share of grid points for which at least one of +// the series has a sample. A workload with zero errors legitimately has no +// filter_bad series at all, so coverage is measured across all of them rather +// than on any single one. +func definedFraction(series ...*timeSeries) float64 { + points, defined := 0, 0 + for _, ts := range series { + if ts == nil || len(ts.data) == 0 { + continue + } + if len(ts.data) > points { + points = len(ts.data) + } + } + if points == 0 { + return 0 + } + for i := 0; i < points; i++ { + for _, ts := range series { + if ts == nil || i >= len(ts.data) { + continue + } + if !math.IsNaN(ts.data[i]) { + defined++ + break + } + } + } + return float64(defined) / float64(points) +} diff --git a/runner/pkg/enrichers/slo_burnrate_test.go b/runner/pkg/enrichers/slo_burnrate_test.go new file mode 100644 index 00000000..6d8800a8 --- /dev/null +++ b/runner/pkg/enrichers/slo_burnrate_test.go @@ -0,0 +1,196 @@ +package enrichers + +import ( + "math" + "testing" +) + +// goal 0.99 => objective 0.01, so a bad-request ratio of 0.20 is a 20x burn. +const testGoal = 0.99 + +func ws(good, bad, coverage float64) windowStats { + return windowStats{good: good, bad: bad, coverage: coverage} +} + +// 1-0.99 is not exactly 0.01 in float64, so burn rates land a few ulps off. +func nearly(t *testing.T, label string, got, want float64) { + t.Helper() + if math.Abs(got-want) > 1e-9 { + t.Errorf("%s = %v; want %v", label, got, want) + } +} + +// Only the 1h/5m rule can be evaluated when byWindow holds just those two +// windows; the 6h/15m rule is skipped for lack of data. That keeps these tests +// pinned to one rule. +func oneRule(t *testing.T, rates []burnRate) burnRate { + t.Helper() + if len(rates) != 1 { + t.Fatalf("rates = %d; want 1: %+v", len(rates), rates) + } + return rates[0] +} + +func TestCalcBurnRates_BothWindowsOverThresholdFires(t *testing.T) { + br := oneRule(t, calcBurnRates(map[int]windowStats{ + 3600: ws(80, 20, 1), // 20% bad => 20x + 300: ws(70, 30, 1), // 30% bad => 30x + }, testGoal)) + + if br.Severity != severityCritical { + t.Errorf("severity = %q; want %q", br.Severity, severityCritical) + } + nearly(t, "long burn rate", br.LongWindowBurnRate, 20) + nearly(t, "short burn rate", br.ShortWindowBurnRate, 30) +} + +// The reason the port exists: a bad hour that has already recovered must not +// fire. Single-window evaluation alerts here; multi-window does not. +func TestCalcBurnRates_ShortWindowRecoveredDoesNotFire(t *testing.T) { + br := oneRule(t, calcBurnRates(map[int]windowStats{ + 3600: ws(80, 20, 1), // 20x over the hour + 300: ws(999, 1, 1), // 0.1x right now — already recovered + }, testGoal)) + + if br.Severity != severityOK { + t.Errorf("severity = %q; want %q (short window recovered)", br.Severity, severityOK) + } + nearly(t, "long burn rate (still reported)", br.LongWindowBurnRate, 20) +} + +// Coroot refuses to compute a burn rate unless at least half the window +// reported data — without this a low-traffic service whose exporter goes quiet +// reads as a total outage. +func TestCalcBurnRates_LowCoverageSkipsRule(t *testing.T) { + rates := calcBurnRates(map[int]windowStats{ + 3600: ws(80, 20, 0.4), + 300: ws(70, 30, 1), + }, testGoal) + if len(rates) != 0 { + t.Errorf("rates = %+v; want none (long window coverage 0.4 < 0.5)", rates) + } +} + +func TestCalcBurnRates_MissingWindowSkipsRule(t *testing.T) { + rates := calcBurnRates(map[int]windowStats{ + 3600: ws(80, 20, 1), + }, testGoal) + if len(rates) != 0 { + t.Errorf("rates = %+v; want none (no 5m window)", rates) + } +} + +// distribution_cut supplies valid (total) + good (fast); bad is the remainder, +// mirroring coroot's slowF = total - fast. +func TestCalcBurnRates_ValidSeriesDerivesBad(t *testing.T) { + withValid := func(valid, good, coverage float64) windowStats { + return windowStats{valid: valid, good: good, hasValid: true, coverage: coverage} + } + br := oneRule(t, calcBurnRates(map[int]windowStats{ + 3600: withValid(100, 80, 1), // 20 slow => 20x + 300: withValid(100, 70, 1), // 30 slow => 30x + }, testGoal)) + + if br.Severity != severityCritical { + t.Errorf("severity = %q; want %q", br.Severity, severityCritical) + } + nearly(t, "long burn rate", br.LongWindowBurnRate, 20) +} + +func TestCalcBurnRates_GoalOfOneYieldsNothing(t *testing.T) { + // objective = 0 would divide by zero. + if rates := calcBurnRates(map[int]windowStats{3600: ws(80, 20, 1), 300: ws(70, 30, 1)}, 1); rates != nil { + t.Errorf("rates = %+v; want nil for goal=1", rates) + } +} + +// Regression guard: an earlier port divided the burn RATE by 3600, so every +// message read "within 0 hours". The window is what gets formatted. +func TestFormatSLOStatus_UsesLongWindow(t *testing.T) { + for _, tc := range []struct { + long int + want string + }{ + {3600, "error budget burn rate is 20.0x within 1 hour"}, + {21600, "error budget burn rate is 20.0x within 6 hours"}, + {300, "error budget burn rate is 20.0x within 5 minutes"}, + } { + got := burnRate{LongWindow: tc.long, LongWindowBurnRate: 20}.formatSLOStatus() + if got != tc.want { + t.Errorf("long=%d: got %q; want %q", tc.long, got, tc.want) + } + } +} + +func TestDefinedFraction(t *testing.T) { + ts := newTimeSeries(0, 4, 1) + ts.set(1, 1) + ts.set(3, 2) + + if got := definedFraction(ts, nil); got != 0.5 { + t.Errorf("definedFraction = %v; want 0.5", got) + } + if got := definedFraction(nil, nil); got != 0 { + t.Errorf("definedFraction(nil) = %v; want 0", got) + } + + // A second series covering the gaps lifts coverage to full — a workload + // with zero errors has no filter_bad series and must not be penalised. + other := newTimeSeries(0, 4, 1) + other.set(0, 1) + other.set(2, 1) + if got := definedFraction(ts, other); got != 1 { + t.Errorf("definedFraction(both) = %v; want 1", got) + } +} + +func TestAttachBurnRates_OverridesLegacyAlert(t *testing.T) { + // Legacy single-window evaluation said "alert"; the burn-rate vector says + // the short window has recovered, so the report must not fire. + report := map[string]any{"alert": true, "alert_message": "stale"} + attachBurnRates(report, []burnRate{{LongWindow: 3600, ShortWindow: 300, Threshold: 14.4, Severity: severityOK}}) + + if report["alert"] != false { + t.Errorf("alert = %v; want false", report["alert"]) + } + if report["severity"] != severityOK { + t.Errorf("severity = %v; want %q", report["severity"], severityOK) + } + if report["alert_message"] != "" { + t.Errorf("alert_message = %q; want empty", report["alert_message"]) + } + if _, ok := report["burn_rates"]; !ok { + t.Error("burn_rates missing from report") + } +} + +func TestAttachBurnRates_EmptyKeepsLegacyFields(t *testing.T) { + // No burn rates (e.g. Prometheus hiccup) must leave the legacy decision + // alone rather than blanking the SLO. + report := map[string]any{"alert": true, "alert_message": "legacy"} + attachBurnRates(report, nil) + + if report["alert"] != true || report["alert_message"] != "legacy" { + t.Errorf("legacy fields modified: %+v", report) + } + if _, ok := report["burn_rates"]; ok { + t.Error("burn_rates should be absent") + } +} + +func TestRequiredWindows(t *testing.T) { + got := requiredWindows() + want := map[int]bool{} + for _, r := range alertRules { + want[r.LongWindow] = true + want[r.ShortWindow] = true + } + if len(got) != len(want) { + t.Fatalf("requiredWindows = %v; want %d distinct windows", got, len(want)) + } + for _, w := range got { + if !want[w] { + t.Errorf("unexpected window %d", w) + } + } +}