fix: require sustained threshold before alerting on CPU and memory (#66)

README.md

@@ -57,7 +57,9 @@ about it the moment it is not — without standing up an observability stack.
 
 - **Core system** — CPU (total, per-core, load average), RAM and swap, disk
   space per mount, disk I/O rates, network rates and lifetime totals per
-  interface, uptime, process count, open file descriptors.
+  interface, uptime, process count, open file descriptors. CPU and memory
+  alerts only fire once usage stays over the threshold for about a minute, so
+  a cron job or backup burst stays quiet.
 - **Thermal** — temperatures (thermal zones + hwmon), fan speeds, and
   throttling detection. Temperature alerts only fire once the threshold is held
   for several consecutive samples, so a CPU touching 90 °C for a moment

docs/ARCHITECTURE.md

@@ -134,7 +134,9 @@ stream of messages:
 - **Threshold rules** with **hysteresis** — fire once on crossing, clear on
   recovery past a margin — so a metric hovering at the line cannot spam.
 - **Sustain** — a rule can require the violation to hold for N consecutive
-  samples before firing (used so a brief thermal spike stays quiet).
+  samples before firing. Temperature needs 3 samples so a brief thermal spike
+  stays quiet; CPU and memory need about a minute (converted to samples from
+  `sample_interval`) so a cron job or backup burst does not alert.
 - **Cooldown** — a minimum gap between two fires of the same key.
 - **Snooze** — a key can be silenced until a time (the 🔕 button); a violation
   still present when the snooze lifts fires fresh.

internal/agent/agent.go

@@ -325,14 +325,32 @@ func (a *Agent) sampleOnce(ctx context.Context) {
 }
 
 // tempSustainSamples is how many consecutive over-threshold samples the
-// hottest sensor must produce before Temperature alerts. Unlike CPU, which
-// Compute averages over the whole sample interval, a temperature reading is
-// the instant the sysfs file was read: a mobile CPU jumps from 40°C to 90°C
-// within two seconds of any burst and falls back just as fast. Requiring the
-// violation to hold across samples is what separates a boost from a box that
-// is genuinely cooking.
+// hottest sensor must produce before Temperature alerts. A temperature
+// reading is the instant the sysfs file was read: a mobile CPU jumps from
+// 40°C to 90°C within two seconds of any burst and falls back just as fast.
+// Requiring the violation to hold across samples is what separates a boost
+// from a box that is genuinely cooking. CPU and memory use systemSustain.
 const tempSustainSamples = 3
 
+// systemSustain is how long CPU or memory must stay over threshold before
+// alerting. Compute averages CPU over the sample interval, but that only hides
+// spikes shorter than one sample; a cron job, backup, package upgrade or build
+// routinely fills a 15–30s window and is not an incident. Memory is an instant
+// reading with the same bursty workloads behind it. It is a duration rather
+// than a sample count so a shorter sample_interval does not shrink the window.
+const systemSustain = time.Minute
+
+// sustainSamples converts a sustain duration into the number of consecutive
+// samples at interval that cover it, rounding up and never below one. Because
+// each CPU sample is an interval average, N violating samples mean at least
+// N×interval of real saturation.
+func sustainSamples(d, interval time.Duration) int {
+	if interval <= 0 {
+		return 1
+	}
+	return max(1, int((d+interval-1)/interval))
+}
+
 // rebootWindow is how fresh the host uptime must be at the agent's first
 // sample to call it a reboot. Agent restarts (self-update, crash recovery)
 // see hours of uptime and stay silent; only a fresh boot trips this.
@@ -358,14 +376,17 @@ func (a *Agent) checkReboot(s collect.Snapshot, now time.Time) *alert.Alert {
 
 func (a *Agent) evalSystem(s collect.Snapshot, now time.Time) []*alert.Alert {
 	t := a.thresholds()
+	sustain := sustainSamples(systemSustain, a.cfg.SampleInterval.Duration)
 	out := []*alert.Alert{
 		a.engine.Threshold(alert.ThresholdOpts{
 			Key: "cpu", Title: "CPU usage", Severity: alert.Warning,
 			Value: s.CPUTotal.Percent, Threshold: t.CPUPercent, ClearMargin: 10, Unit: "%",
+			Sustain: sustain,
 		}, now),
 		a.engine.Threshold(alert.ThresholdOpts{
 			Key: "mem", Title: "Memory usage", Severity: alert.Warning,
 			Value: s.Mem.UsedPercent(), Threshold: t.MemPercent, ClearMargin: 10, Unit: "%",
+			Sustain: sustain,
 		}, now),
 	}
 	for _, m := range s.Mounts {

internal/agent/agent_test.go

@@ -128,7 +128,7 @@ func TestThresholdAlertPushed(t *testing.T) {
 	now := time.Now()
 
 	hot := collect.Snapshot{CPUTotal: collect.CPUUsage{Percent: 97}, Mem: procfs.MemInfo{Total: 100, Available: 60}}
-	for _, al := range a.evalSystem(hot, now) {
+	for _, al := range evalSustained(a, hot, now) {
 		if al != nil {
 			a.push(context.Background(), "ALERT:"+al.Title)
 		}
@@ -841,7 +841,7 @@ func TestSettingsButtons(t *testing.T) {
 	}
 
 	// Live evaluation uses the tuned threshold: CPU at 87% now alerts.
-	als := a.evalSystem(collect.Snapshot{CPUTotal: collect.CPUUsage{Percent: 87}, Mem: procfs.MemInfo{Total: 100, Available: 60}}, time.Now())
+	als := evalSustained(a, collect.Snapshot{CPUTotal: collect.CPUUsage{Percent: 87}, Mem: procfs.MemInfo{Total: 100, Available: 60}}, time.Now())
 	if countNonNil(als) != 1 {
 		t.Errorf("tuned threshold must fire at 87%%: %+v", als)
 	}
@@ -1442,6 +1442,162 @@ func TestValidTTY(t *testing.T) {
 	}
 }
 
+// systemSustainN is how many samples the CPU and memory rules need at a's
+// configured interval.
+func systemSustainN(a *Agent) int {
+	return sustainSamples(systemSustain, a.cfg.SampleInterval.Duration)
+}
+
+// evalSustained feeds the same snapshot through evalSystem for a full sustain
+// window, one sample interval apart, and returns the last evaluation.
+func evalSustained(a *Agent, s collect.Snapshot, now time.Time) []*alert.Alert {
+	var out []*alert.Alert
+	for i := range systemSustainN(a) {
+		out = a.evalSystem(s, now.Add(time.Duration(i)*a.cfg.SampleInterval.Duration))
+	}
+	return out
+}
+
+// keyed returns the alert for key from one evaluation, or nil.
+func keyed(als []*alert.Alert, key string) *alert.Alert {
+	for _, al := range als {
+		if al != nil && al.Key == key {
+			return al
+		}
+	}
+	return nil
+}
+
+func cpuSnap(pct float64) collect.Snapshot {
+	return collect.Snapshot{CPUTotal: collect.CPUUsage{Percent: pct}, Mem: procfs.MemInfo{Total: 100, Available: 60}}
+}
+
+func memSnap(usedPct float64) collect.Snapshot {
+	return collect.Snapshot{Mem: procfs.MemInfo{Total: 100, Available: uint64(100 - usedPct)}}
+}
+
+func TestSustainSamples(t *testing.T) {
+	cases := []struct {
+		d, interval time.Duration
+		want        int
+	}{
+		{time.Minute, 15 * time.Second, 4},
+		{time.Minute, 5 * time.Second, 12},
+		{time.Minute, time.Second, 60},
+		{time.Minute, 45 * time.Second, 2},
+		{time.Minute, 2 * time.Minute, 1},
+		{time.Minute, 0, 1},
+		{0, 15 * time.Second, 1},
+	}
+	for _, c := range cases {
+		if got := sustainSamples(c.d, c.interval); got != c.want {
+			t.Errorf("sustainSamples(%s, %s) = %d, want %d", c.d, c.interval, got, c.want)
+		}
+	}
+}
+
+// TestCPUBurstIsSilent replays the #66 timeline: a backup pegs the CPU for
+// one sample, then for one sample short of the window. Neither may push an
+// alert or a recovery notice.
+func TestCPUBurstIsSilent(t *testing.T) {
+	a := testAgent(&fakeSender{})
+	step := a.cfg.SampleInterval.Duration
+	now := time.Now()
+	tick := func(pct float64) {
+		t.Helper()
+		if als := a.evalSystem(cpuSnap(pct), now); countNonNil(als) != 0 {
+			t.Fatalf("%.0f%% at %s pushed: %+v", pct, now.Format(time.TimeOnly), als)
+		}
+		now = now.Add(step)
+	}
+
+	tick(12)
+	tick(97)
+	tick(18)
+	for range systemSustainN(a) - 1 {
+		tick(97)
+	}
+	tick(18)
+	if a.engine.Active("cpu") {
+		t.Fatal("a burst shorter than the sustain window must not leave cpu active")
+	}
+}
+
+func TestCPUSustainedFiresOnce(t *testing.T) {
+	a := testAgent(&fakeSender{})
+	step := a.cfg.SampleInterval.Duration
+	now := time.Now()
+	n := systemSustainN(a)
+
+	for i := 1; i <= n; i++ {
+		al := keyed(a.evalSystem(cpuSnap(97), now), "cpu")
+		now = now.Add(step)
+		if i < n && al != nil {
+			t.Fatalf("sample %d/%d fired early: %+v", i, n, al)
+		}
+		if i == n && (al == nil || al.Resolved) {
+			t.Fatalf("sample %d/%d must fire: %+v", i, n, al)
+		}
+	}
+	// Still hot: hysteresis keeps it to one message.
+	for range 3 {
+		if al := keyed(a.evalSystem(cpuSnap(97), now), "cpu"); al != nil {
+			t.Fatalf("held violation re-fired: %+v", al)
+		}
+		now = now.Add(step)
+	}
+	// Recovery is immediate — no sustain on the way down.
+	rec := keyed(a.evalSystem(cpuSnap(20), now), "cpu")
+	if rec == nil || !rec.Resolved {
+		t.Fatalf("recovery expected on first cool sample: %+v", rec)
+	}
+}
+
+func TestMemSustained(t *testing.T) {
+	a := testAgent(&fakeSender{})
+	step := a.cfg.SampleInterval.Duration
+	now := time.Now()
+	n := systemSustainN(a)
+
+	// A build that briefly eats memory stays quiet.
+	for range n - 1 {
+		if al := keyed(a.evalSystem(memSnap(95), now), "mem"); al != nil {
+			t.Fatalf("memory burst fired: %+v", al)
+		}
+		now = now.Add(step)
+	}
+	if al := keyed(a.evalSystem(memSnap(40), now), "mem"); al != nil {
+		t.Fatalf("memory recovery without a fire pushed: %+v", al)
+	}
+	now = now.Add(step)
+
+	// Held for the full window, it fires on the last sample.
+	var al *alert.Alert
+	for range n {
+		al = keyed(a.evalSystem(memSnap(95), now), "mem")
+		now = now.Add(step)
+	}
+	if al == nil || al.Resolved {
+		t.Fatalf("sustained memory pressure must fire: %+v", al)
+	}
+}
+
+func TestSustainScalesWithInterval(t *testing.T) {
+	a := testAgent(&fakeSender{})
+	a.cfg.SampleInterval = config.Duration{Duration: 15 * time.Second}
+	now := time.Now()
+
+	for i := 1; i <= 3; i++ {
+		if al := keyed(a.evalSystem(cpuSnap(97), now), "cpu"); al != nil {
+			t.Fatalf("sample %d at 15s fired before a minute: %+v", i, al)
+		}
+		now = now.Add(15 * time.Second)
+	}
+	if al := keyed(a.evalSystem(cpuSnap(97), now), "cpu"); al == nil {
+		t.Fatal("4th sample at 15s (a full minute) must fire")
+	}
+}
+
 func countNonNil(als []*alert.Alert) int {
 	n := 0
 	for _, a := range als {

internal/agent/snooze_test.go

@@ -27,18 +27,18 @@ func TestSnoozeCallback(t *testing.T) {
 	}
 
 	// The snoozed key stays quiet; everything else still evaluates.
-	for _, al := range a.evalSystem(hot, time.Now()) {
-		if al != nil && al.Key == "cpu" {
+	// Run the full sustain window so the rule would otherwise fire.
+	now := time.Now()
+	step := a.cfg.SampleInterval.Duration
+	for i := range systemSustainN(a) {
+		if al := keyed(a.evalSystem(hot, now.Add(time.Duration(i)*step)), "cpu"); al != nil {
 			t.Fatalf("cpu alerted while snoozed: %+v", al)
 		}
 	}
 
 	// Foreign chats cannot snooze.
 	a.handleCallback(ctx, &telegram.Callback{ID: "cb2", ChatID: 999, MessageID: 7, Data: "snz:mem"})
-	for _, al := range a.evalSystem(collect.Snapshot{Mem: procfs.MemInfo{Total: 100, Available: 5}}, time.Now()) {
-		if al != nil && al.Key == "mem" {
-			return // mem still alerts — foreign snooze was ignored
-		}
+	if keyed(evalSustained(a, collect.Snapshot{Mem: procfs.MemInfo{Total: 100, Available: 5}}, time.Now()), "mem") == nil {
+		t.Fatal("mem did not alert; foreign-chat snooze was honored")
 	}
-	t.Fatal("mem did not alert; foreign-chat snooze was honored")
 }