diff --git a/README.md b/README.md index ec2a696..6197762 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,9 @@ about it the moment it is not — without standing up an observability stack. - **Core system** — CPU (total, per-core, load average), RAM and swap, disk space per mount, disk I/O rates, network rates and lifetime totals per - interface, uptime, process count, open file descriptors. + interface, uptime, process count, open file descriptors. CPU and memory + alerts only fire once usage stays over the threshold for about a minute, so + a cron job or backup burst stays quiet. - **Thermal** — temperatures (thermal zones + hwmon), fan speeds, and throttling detection. Temperature alerts only fire once the threshold is held for several consecutive samples, so a CPU touching 90 °C for a moment diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 59f5f43..72306ce 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -134,7 +134,9 @@ stream of messages: - **Threshold rules** with **hysteresis** — fire once on crossing, clear on recovery past a margin — so a metric hovering at the line cannot spam. - **Sustain** — a rule can require the violation to hold for N consecutive - samples before firing (used so a brief thermal spike stays quiet). + samples before firing. Temperature needs 3 samples so a brief thermal spike + stays quiet; CPU and memory need about a minute (converted to samples from + `sample_interval`) so a cron job or backup burst does not alert. - **Cooldown** — a minimum gap between two fires of the same key. - **Snooze** — a key can be silenced until a time (the 🔕 button); a violation still present when the snooze lifts fires fresh. diff --git a/internal/agent/agent.go b/internal/agent/agent.go index 1ea40ee..d92281c 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -325,14 +325,32 @@ func (a *Agent) sampleOnce(ctx context.Context) { } // tempSustainSamples is how many consecutive over-threshold samples the -// hottest sensor must produce before Temperature alerts. Unlike CPU, which -// Compute averages over the whole sample interval, a temperature reading is -// the instant the sysfs file was read: a mobile CPU jumps from 40°C to 90°C -// within two seconds of any burst and falls back just as fast. Requiring the -// violation to hold across samples is what separates a boost from a box that -// is genuinely cooking. +// hottest sensor must produce before Temperature alerts. A temperature +// reading is the instant the sysfs file was read: a mobile CPU jumps from +// 40°C to 90°C within two seconds of any burst and falls back just as fast. +// Requiring the violation to hold across samples is what separates a boost +// from a box that is genuinely cooking. CPU and memory use systemSustain. const tempSustainSamples = 3 +// systemSustain is how long CPU or memory must stay over threshold before +// alerting. Compute averages CPU over the sample interval, but that only hides +// spikes shorter than one sample; a cron job, backup, package upgrade or build +// routinely fills a 15–30s window and is not an incident. Memory is an instant +// reading with the same bursty workloads behind it. It is a duration rather +// than a sample count so a shorter sample_interval does not shrink the window. +const systemSustain = time.Minute + +// sustainSamples converts a sustain duration into the number of consecutive +// samples at interval that cover it, rounding up and never below one. Because +// each CPU sample is an interval average, N violating samples mean at least +// N×interval of real saturation. +func sustainSamples(d, interval time.Duration) int { + if interval <= 0 { + return 1 + } + return max(1, int((d+interval-1)/interval)) +} + // rebootWindow is how fresh the host uptime must be at the agent's first // sample to call it a reboot. Agent restarts (self-update, crash recovery) // see hours of uptime and stay silent; only a fresh boot trips this. @@ -358,14 +376,17 @@ func (a *Agent) checkReboot(s collect.Snapshot, now time.Time) *alert.Alert { func (a *Agent) evalSystem(s collect.Snapshot, now time.Time) []*alert.Alert { t := a.thresholds() + sustain := sustainSamples(systemSustain, a.cfg.SampleInterval.Duration) out := []*alert.Alert{ a.engine.Threshold(alert.ThresholdOpts{ Key: "cpu", Title: "CPU usage", Severity: alert.Warning, Value: s.CPUTotal.Percent, Threshold: t.CPUPercent, ClearMargin: 10, Unit: "%", + Sustain: sustain, }, now), a.engine.Threshold(alert.ThresholdOpts{ Key: "mem", Title: "Memory usage", Severity: alert.Warning, Value: s.Mem.UsedPercent(), Threshold: t.MemPercent, ClearMargin: 10, Unit: "%", + Sustain: sustain, }, now), } for _, m := range s.Mounts { diff --git a/internal/agent/agent_test.go b/internal/agent/agent_test.go index b6004c4..31b19b4 100644 --- a/internal/agent/agent_test.go +++ b/internal/agent/agent_test.go @@ -128,7 +128,7 @@ func TestThresholdAlertPushed(t *testing.T) { now := time.Now() hot := collect.Snapshot{CPUTotal: collect.CPUUsage{Percent: 97}, Mem: procfs.MemInfo{Total: 100, Available: 60}} - for _, al := range a.evalSystem(hot, now) { + for _, al := range evalSustained(a, hot, now) { if al != nil { a.push(context.Background(), "ALERT:"+al.Title) } @@ -841,7 +841,7 @@ func TestSettingsButtons(t *testing.T) { } // Live evaluation uses the tuned threshold: CPU at 87% now alerts. - als := a.evalSystem(collect.Snapshot{CPUTotal: collect.CPUUsage{Percent: 87}, Mem: procfs.MemInfo{Total: 100, Available: 60}}, time.Now()) + als := evalSustained(a, collect.Snapshot{CPUTotal: collect.CPUUsage{Percent: 87}, Mem: procfs.MemInfo{Total: 100, Available: 60}}, time.Now()) if countNonNil(als) != 1 { t.Errorf("tuned threshold must fire at 87%%: %+v", als) } @@ -1442,6 +1442,162 @@ func TestValidTTY(t *testing.T) { } } +// systemSustainN is how many samples the CPU and memory rules need at a's +// configured interval. +func systemSustainN(a *Agent) int { + return sustainSamples(systemSustain, a.cfg.SampleInterval.Duration) +} + +// evalSustained feeds the same snapshot through evalSystem for a full sustain +// window, one sample interval apart, and returns the last evaluation. +func evalSustained(a *Agent, s collect.Snapshot, now time.Time) []*alert.Alert { + var out []*alert.Alert + for i := range systemSustainN(a) { + out = a.evalSystem(s, now.Add(time.Duration(i)*a.cfg.SampleInterval.Duration)) + } + return out +} + +// keyed returns the alert for key from one evaluation, or nil. +func keyed(als []*alert.Alert, key string) *alert.Alert { + for _, al := range als { + if al != nil && al.Key == key { + return al + } + } + return nil +} + +func cpuSnap(pct float64) collect.Snapshot { + return collect.Snapshot{CPUTotal: collect.CPUUsage{Percent: pct}, Mem: procfs.MemInfo{Total: 100, Available: 60}} +} + +func memSnap(usedPct float64) collect.Snapshot { + return collect.Snapshot{Mem: procfs.MemInfo{Total: 100, Available: uint64(100 - usedPct)}} +} + +func TestSustainSamples(t *testing.T) { + cases := []struct { + d, interval time.Duration + want int + }{ + {time.Minute, 15 * time.Second, 4}, + {time.Minute, 5 * time.Second, 12}, + {time.Minute, time.Second, 60}, + {time.Minute, 45 * time.Second, 2}, + {time.Minute, 2 * time.Minute, 1}, + {time.Minute, 0, 1}, + {0, 15 * time.Second, 1}, + } + for _, c := range cases { + if got := sustainSamples(c.d, c.interval); got != c.want { + t.Errorf("sustainSamples(%s, %s) = %d, want %d", c.d, c.interval, got, c.want) + } + } +} + +// TestCPUBurstIsSilent replays the #66 timeline: a backup pegs the CPU for +// one sample, then for one sample short of the window. Neither may push an +// alert or a recovery notice. +func TestCPUBurstIsSilent(t *testing.T) { + a := testAgent(&fakeSender{}) + step := a.cfg.SampleInterval.Duration + now := time.Now() + tick := func(pct float64) { + t.Helper() + if als := a.evalSystem(cpuSnap(pct), now); countNonNil(als) != 0 { + t.Fatalf("%.0f%% at %s pushed: %+v", pct, now.Format(time.TimeOnly), als) + } + now = now.Add(step) + } + + tick(12) + tick(97) + tick(18) + for range systemSustainN(a) - 1 { + tick(97) + } + tick(18) + if a.engine.Active("cpu") { + t.Fatal("a burst shorter than the sustain window must not leave cpu active") + } +} + +func TestCPUSustainedFiresOnce(t *testing.T) { + a := testAgent(&fakeSender{}) + step := a.cfg.SampleInterval.Duration + now := time.Now() + n := systemSustainN(a) + + for i := 1; i <= n; i++ { + al := keyed(a.evalSystem(cpuSnap(97), now), "cpu") + now = now.Add(step) + if i < n && al != nil { + t.Fatalf("sample %d/%d fired early: %+v", i, n, al) + } + if i == n && (al == nil || al.Resolved) { + t.Fatalf("sample %d/%d must fire: %+v", i, n, al) + } + } + // Still hot: hysteresis keeps it to one message. + for range 3 { + if al := keyed(a.evalSystem(cpuSnap(97), now), "cpu"); al != nil { + t.Fatalf("held violation re-fired: %+v", al) + } + now = now.Add(step) + } + // Recovery is immediate — no sustain on the way down. + rec := keyed(a.evalSystem(cpuSnap(20), now), "cpu") + if rec == nil || !rec.Resolved { + t.Fatalf("recovery expected on first cool sample: %+v", rec) + } +} + +func TestMemSustained(t *testing.T) { + a := testAgent(&fakeSender{}) + step := a.cfg.SampleInterval.Duration + now := time.Now() + n := systemSustainN(a) + + // A build that briefly eats memory stays quiet. + for range n - 1 { + if al := keyed(a.evalSystem(memSnap(95), now), "mem"); al != nil { + t.Fatalf("memory burst fired: %+v", al) + } + now = now.Add(step) + } + if al := keyed(a.evalSystem(memSnap(40), now), "mem"); al != nil { + t.Fatalf("memory recovery without a fire pushed: %+v", al) + } + now = now.Add(step) + + // Held for the full window, it fires on the last sample. + var al *alert.Alert + for range n { + al = keyed(a.evalSystem(memSnap(95), now), "mem") + now = now.Add(step) + } + if al == nil || al.Resolved { + t.Fatalf("sustained memory pressure must fire: %+v", al) + } +} + +func TestSustainScalesWithInterval(t *testing.T) { + a := testAgent(&fakeSender{}) + a.cfg.SampleInterval = config.Duration{Duration: 15 * time.Second} + now := time.Now() + + for i := 1; i <= 3; i++ { + if al := keyed(a.evalSystem(cpuSnap(97), now), "cpu"); al != nil { + t.Fatalf("sample %d at 15s fired before a minute: %+v", i, al) + } + now = now.Add(15 * time.Second) + } + if al := keyed(a.evalSystem(cpuSnap(97), now), "cpu"); al == nil { + t.Fatal("4th sample at 15s (a full minute) must fire") + } +} + func countNonNil(als []*alert.Alert) int { n := 0 for _, a := range als { diff --git a/internal/agent/snooze_test.go b/internal/agent/snooze_test.go index d3eceea..006fb9c 100644 --- a/internal/agent/snooze_test.go +++ b/internal/agent/snooze_test.go @@ -27,18 +27,18 @@ func TestSnoozeCallback(t *testing.T) { } // The snoozed key stays quiet; everything else still evaluates. - for _, al := range a.evalSystem(hot, time.Now()) { - if al != nil && al.Key == "cpu" { + // Run the full sustain window so the rule would otherwise fire. + now := time.Now() + step := a.cfg.SampleInterval.Duration + for i := range systemSustainN(a) { + if al := keyed(a.evalSystem(hot, now.Add(time.Duration(i)*step)), "cpu"); al != nil { t.Fatalf("cpu alerted while snoozed: %+v", al) } } // Foreign chats cannot snooze. a.handleCallback(ctx, &telegram.Callback{ID: "cb2", ChatID: 999, MessageID: 7, Data: "snz:mem"}) - for _, al := range a.evalSystem(collect.Snapshot{Mem: procfs.MemInfo{Total: 100, Available: 5}}, time.Now()) { - if al != nil && al.Key == "mem" { - return // mem still alerts — foreign snooze was ignored - } + if keyed(evalSustained(a, collect.Snapshot{Mem: procfs.MemInfo{Total: 100, Available: 5}}, time.Now()), "mem") == nil { + t.Fatal("mem did not alert; foreign-chat snooze was honored") } - t.Fatal("mem did not alert; foreign-chat snooze was honored") }